From 62279c34d97002d805c0619b4c818422a6f1fecf Mon Sep 17 00:00:00 2001 From: Denis Mazur Date: Mon, 1 Mar 2021 23:17:11 +0300 Subject: [PATCH 01/81] Compiling libp2p daemon on setup (#153) * add setup.py prototype * refactor --- setup.py | 56 +++++++++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 55 insertions(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 53cb6b77d..187d3ea43 100644 --- a/setup.py +++ b/setup.py @@ -2,6 +2,10 @@ import glob import os import re +import subprocess +import urllib.request +import tarfile +import tempfile from pkg_resources import parse_requirements from setuptools import setup, find_packages @@ -9,6 +13,19 @@ from setuptools.command.install import install +class cd: + """Context manager for changing the current working directory""" + def __init__(self, newPath): + self.newPath = os.path.expanduser(newPath) + + def __enter__(self): + self.savedPath = os.getcwd() + os.chdir(self.newPath) + + def __exit__(self, etype, value, traceback): + os.chdir(self.savedPath) + + def proto_compile(output_path): import grpc_tools.protoc @@ -28,6 +45,38 @@ def proto_compile(output_path): file.truncate() +def install_libp2p_daemon(): + # check go version: + try: + proc = subprocess.Popen(['go', 'version'], + stdout=subprocess.PIPE) + result, _ = proc.communicate() + result = result.decode('ascii', 'replace') + _, _, version, _ = result.split(' ') + version = version.lstrip('go') + + if version < "1.13": + raise EnvironmentError(f'newer version of go required: must be >= 1.13, found {version}') + + except FileNotFoundError: + raise FileNotFoundError('could not find golang installation') + + with tempfile.TemporaryDirectory() as tempdir: + url = 'https://github.com/libp2p/go-libp2p-daemon/archive/master.tar.gz' + dest = os.path.join(tempdir, 'libp2p-daemin.tar.gz') + urllib.request.urlretrieve(url, os.path.join(tempdir, dest)) + + tar = tarfile.open(dest, 'r:gz') + tar.extractall(tempdir) + tar.close() + + with cd(os.path.join(tempdir, 'go-libp2p-daemon-master')): + status = os.system('go install ./...') + if status: + raise RuntimeError('Failed to build or install libp2p-daemon:'\ + f' exited with status code :{status}') + + class ProtoCompileInstall(install): def run(self): proto_compile(os.path.join(self.build_lib, 'hivemind', 'proto')) @@ -40,6 +89,11 @@ def run(self): super().run() +class LibP2PInstall(install): + def run(self): + install_libp2p_daemon() + + here = os.path.abspath(os.path.dirname(__file__)) with open('requirements.txt') as requirements_file: @@ -63,7 +117,7 @@ def run(self): setup( name='hivemind', version=version_string, - cmdclass={'install': ProtoCompileInstall, 'develop': ProtoCompileDevelop}, + cmdclass={'install': ProtoCompileInstall, 'develop': ProtoCompileDevelop, 'libp2p': LibP2PInstall}, description='Decentralized deep learning in PyTorch', long_description='Decentralized deep learning in PyTorch. Built to train giant models on ' 'thousands of volunteers across the world.', From 97d7165681b99e24883d101ac7b5b50f106ab14b Mon Sep 17 00:00:00 2001 From: Ilya <37004806+skobellev@users.noreply.github.com> Date: Wed, 3 Mar 2021 12:25:17 +0300 Subject: [PATCH 02/81] feat: add p2p daemon (#164) * Add p2p daemon * Test p2p daemon exits correctly * Impose restriction on elapsed time Co-authored-by: Ilya Kobelev --- hivemind/__init__.py | 1 + hivemind/p2p/__init__.py | 1 + hivemind/p2p/p2p_daemon.py | 45 +++++++++++++++++++++++++++++++ tests/test_p2p_daemon.py | 55 ++++++++++++++++++++++++++++++++++++++ 4 files changed, 102 insertions(+) create mode 100644 hivemind/p2p/__init__.py create mode 100644 hivemind/p2p/p2p_daemon.py create mode 100644 tests/test_p2p_daemon.py diff --git a/hivemind/__init__.py b/hivemind/__init__.py index eedb69dcf..64d240ce7 100644 --- a/hivemind/__init__.py +++ b/hivemind/__init__.py @@ -1,5 +1,6 @@ from hivemind.client import * from hivemind.dht import * +from hivemind.p2p import * from hivemind.server import * from hivemind.utils import * diff --git a/hivemind/p2p/__init__.py b/hivemind/p2p/__init__.py new file mode 100644 index 000000000..6bae0b8bd --- /dev/null +++ b/hivemind/p2p/__init__.py @@ -0,0 +1 @@ +from hivemind.p2p.p2p_daemon import P2P diff --git a/hivemind/p2p/p2p_daemon.py b/hivemind/p2p/p2p_daemon.py new file mode 100644 index 000000000..3083c70e5 --- /dev/null +++ b/hivemind/p2p/p2p_daemon.py @@ -0,0 +1,45 @@ +import subprocess +import typing as tp + + +class P2P(object): + """ + Forks a child process and executes p2pd command with given arguments. + Sends SIGKILL to the child in destructor and on exit from contextmanager. + """ + + LIBP2P_CMD = 'p2pd' + + def __init__(self, *args, **kwargs): + self._child = subprocess.Popen(args=self._make_process_args(args, kwargs)) + try: + stdout, stderr = self._child.communicate(timeout=0.2) + except subprocess.TimeoutExpired: + pass + else: + raise RuntimeError(f'p2p daemon exited with stderr: {stderr}') + + def __enter__(self): + return self._child + + def __exit__(self, exc_type, exc_val, exc_tb): + self._kill_child() + + def __del__(self): + self._kill_child() + + def _kill_child(self): + if self._child.poll() is None: + self._child.kill() + self._child.wait() + + def _make_process_args(self, args: tp.Tuple[tp.Any], + kwargs: tp.Dict[str, tp.Any]) -> tp.List[str]: + proc_args = [self.LIBP2P_CMD] + proc_args.extend( + str(entry) for entry in args + ) + proc_args.extend( + f'-{key}={str(value)}' for key, value in kwargs.items() + ) + return proc_args diff --git a/tests/test_p2p_daemon.py b/tests/test_p2p_daemon.py new file mode 100644 index 000000000..ac57e9e2f --- /dev/null +++ b/tests/test_p2p_daemon.py @@ -0,0 +1,55 @@ +import subprocess +from time import perf_counter + +import pytest + +import hivemind.p2p +from hivemind.p2p import P2P + +RUNNING = 'running' +NOT_RUNNING = 'not running' +CHECK_PID_CMD = ''' +if ps -p {0} > /dev/null; +then + echo "{1}" +else + echo "{2}" +fi +''' + + +def is_process_running(pid: int) -> bool: + cmd = CHECK_PID_CMD.format(pid, RUNNING, NOT_RUNNING) + return subprocess.check_output(cmd, shell=True).decode('utf-8').strip() == RUNNING + + +@pytest.fixture() +def mock_p2p_class(): + P2P.LIBP2P_CMD = "sleep" + + +def test_daemon_killed_on_del(mock_p2p_class): + start = perf_counter() + p2p_daemon = P2P('10s') + + child_pid = p2p_daemon._child.pid + assert is_process_running(child_pid) + + del p2p_daemon + assert not is_process_running(child_pid) + assert perf_counter() - start < 1 + + +def test_daemon_killed_on_exit(mock_p2p_class): + start = perf_counter() + with P2P('10s') as daemon: + child_pid = daemon.pid + assert is_process_running(child_pid) + + assert not is_process_running(child_pid) + assert perf_counter() - start < 1 + + +def test_daemon_raises_on_faulty_args(): + with pytest.raises(RuntimeError): + P2P(faulty='argument') From 883fcf3c8a9e656cc987352171f5872a559113d8 Mon Sep 17 00:00:00 2001 From: Denis Mazur Date: Fri, 5 Mar 2021 05:19:39 +0300 Subject: [PATCH 03/81] compare golang versions using packaging.version --- setup.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/setup.py b/setup.py index 187d3ea43..f67235750 100644 --- a/setup.py +++ b/setup.py @@ -7,6 +7,7 @@ import tarfile import tempfile +from packaging import version from pkg_resources import parse_requirements from setuptools import setup, find_packages from setuptools.command.develop import develop @@ -52,10 +53,10 @@ def install_libp2p_daemon(): stdout=subprocess.PIPE) result, _ = proc.communicate() result = result.decode('ascii', 'replace') - _, _, version, _ = result.split(' ') - version = version.lstrip('go') + _, _, v, _ = result.split(' ') + v = v.lstrip('go') - if version < "1.13": + if version.parse(v) < version.parse("1.13"): raise EnvironmentError(f'newer version of go required: must be >= 1.13, found {version}') except FileNotFoundError: From 25b3f21c100f1fa61f7cbf6ec0b598ac4a12fd5f Mon Sep 17 00:00:00 2001 From: Denis Mazur Date: Tue, 9 Mar 2021 20:15:17 +0300 Subject: [PATCH 04/81] fix typo Co-authored-by: justheuristic --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index f67235750..d858d777c 100644 --- a/setup.py +++ b/setup.py @@ -64,7 +64,7 @@ def install_libp2p_daemon(): with tempfile.TemporaryDirectory() as tempdir: url = 'https://github.com/libp2p/go-libp2p-daemon/archive/master.tar.gz' - dest = os.path.join(tempdir, 'libp2p-daemin.tar.gz') + dest = os.path.join(tempdir, 'libp2p-daemon.tar.gz') urllib.request.urlretrieve(url, os.path.join(tempdir, dest)) tar = tarfile.open(dest, 'r:gz') From 843babcca19d507c017f343a58cde9418a5dea31 Mon Sep 17 00:00:00 2001 From: Denis Mazur Date: Wed, 10 Mar 2021 01:31:19 +0300 Subject: [PATCH 05/81] move p2pd executable to hivemind/hivemind_cli --- setup.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/setup.py b/setup.py index d858d777c..36630cd27 100644 --- a/setup.py +++ b/setup.py @@ -14,6 +14,9 @@ from setuptools.command.install import install +here = os.path.abspath(os.path.dirname(__file__)) + + class cd: """Context manager for changing the current working directory""" def __init__(self, newPath): @@ -71,8 +74,8 @@ def install_libp2p_daemon(): tar.extractall(tempdir) tar.close() - with cd(os.path.join(tempdir, 'go-libp2p-daemon-master')): - status = os.system('go install ./...') + with cd(os.path.join(tempdir, 'go-libp2p-daemon-master', 'p2pd')): + status = os.system(f'go build -o {os.path.join(here, "hivemind/hivemind_cli", "p2pd")}') if status: raise RuntimeError('Failed to build or install libp2p-daemon:'\ f' exited with status code :{status}') @@ -95,7 +98,6 @@ def run(self): install_libp2p_daemon() -here = os.path.abspath(os.path.dirname(__file__)) with open('requirements.txt') as requirements_file: install_requires = list(map(str, parse_requirements(requirements_file))) From 4409cd66b25c1ed1fde1823efbf6fc529adec46b Mon Sep 17 00:00:00 2001 From: Denis Mazur Date: Wed, 10 Mar 2021 20:58:26 +0300 Subject: [PATCH 06/81] Rebase master onto libp2p (#179) * copytree implementation for py37 compatibility (#162) * copytree implementation for py37 compatibility * Running tests for python3.7 * Increment version * Python3.7 notions * Remove pickle.loads in averager (#160) * Security update: remove pickle.loads in averager * add py37 to circleci config Co-authored-by: Alexander Borzunov Co-authored-by: Max Ryabinin * Support edge cases for DHT key/subkey/value, add tests, update .gitignore for pb2 (#167) * fix bug with subkey equals zero * add autogenerated protobuf files to .gitignore * test store and get "tricky" values in dht * Fix the remaining tests for py37 (#166) * DecentralizedAverager is now compatible with python37's acyncio exception * the problem was: grpc.aio with python37 raised concurrent.futures.CancelledError in some cases; * we relied on isinstance(asyncio.CancelledError, Exception) == False * but isinstance(concurrent.futures.CancelledError, Exception) == True * DecentralizedAverager now shuts down if dereferenced in the main process * though it won't shutdown if dereferenced in forks for obvious reasons * HIVEMIND_THREADS now actually works * test_averaging now shuts down dht and averager instances to avoid leaking processes Co-authored-by: Max Ryabinin Co-authored-by: Max Ryabinin * Move Averager metadata serialization out of user scope (#168) * move metadata serialization outside user scope * test_overcrowded: reduce the default number of peers * Handle edge cases in DecentralizedAverager (#171) * move metadata serialization outside user scope * retry averager.step on network errors * raise AllreduceException on partial tensor * test split/combine tensors, combine corrupted stream Co-authored-by: Max Ryabinin * Fix a typo in quickstart.md (#174) * Serialize DHTID source with msgpack (#172) * Change DHTID serializer * Remove unused serializers * Add msgpack tuple serialization * Move CLI server launch script to hivemind/hivemind_cli (#173) * Cast environment variables to correct types * Compiling libp2p daemon on setup (#153) * add setup.py prototype * refactor * feat: add p2p daemon (#164) * Add p2p daemon * Test p2p daemon exits correctly * Impose restriction on elapsed time Co-authored-by: Ilya Kobelev * compare golang versions using packaging.version * fix typo Co-authored-by: justheuristic * move p2pd executable to hivemind/hivemind_cli Co-authored-by: Alexey Bukhtiyarov Co-authored-by: justheuristic Co-authored-by: Alexander Borzunov Co-authored-by: Max Ryabinin Co-authored-by: Michael Diskin Co-authored-by: romakail <36082689+romakail@users.noreply.github.com> Co-authored-by: Ilya <37004806+skobellev@users.noreply.github.com> Co-authored-by: Ilya Kobelev --- hivemind/client/averaging/matchmaking.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/hivemind/client/averaging/matchmaking.py b/hivemind/client/averaging/matchmaking.py index de20ebc02..b06318389 100644 --- a/hivemind/client/averaging/matchmaking.py +++ b/hivemind/client/averaging/matchmaking.py @@ -462,5 +462,13 @@ async def _declare_averager_periodically(self, key_manager: GroupKeyManager): looking_for_group=False) +def compute_schema_hash(tensors: Sequence[torch.Tensor]) -> bytes: + """ A hash that describes follower's tensor shapes, dtypes, devices, but not the actual values """ + schema_dicts = [{field_name: str(field_value) + for field_name, field_value in asdict(TensorDescriptor.from_tensor(tensor)).items()} + for tensor in tensors] + return DHTID.generate(source=schema_dicts).to_bytes() + + class MatchmakingException(Exception): """ An internal exception that marks undesired edge cases during averaging """ From 0535efe51703a9a1d5afa5a55ce5a0218a509c18 Mon Sep 17 00:00:00 2001 From: Denis Mazur Date: Fri, 19 Mar 2021 22:13:57 +0300 Subject: [PATCH 07/81] Fix LibP2P-Daemon installation in setup.py (#186) * Rename ProtoCompileInstall and ProtoCompileDevelop to Install and Develop * Install LibP2P-Daemon on setup install and setup develop * Install Golang in Circle CI builds * Add P2PD binary to gitignore --- .circleci/config.yml | 18 ++++++++++++ .gitignore | 3 ++ setup.py | 67 +++++++++++++++++++++++--------------------- 3 files changed, 56 insertions(+), 32 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index b1ab5978e..daaac9b6a 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -1,5 +1,10 @@ version: 2.1 +parameters: + go-version: + type: string + default: 1.16.2 + jobs: build-and-test-py37: docker: @@ -9,6 +14,11 @@ jobs: - restore_cache: keys: - py37-v1-{{ checksum "requirements.txt" }}-{{ checksum "requirements-dev.txt" }} + - v1-{{ checksum "requirements.txt" }}-{{ checksum "requirements-dev.txt" }} + - run: | + wget https://golang.org/dl/go<< pipeline.parameters.go-version >>.linux-amd64.tar.gz -O go.tar.gz + tar -C ~/ -xzf go.tar.gz + echo "export PATH=~/go/bin:$PATH" >> $BASH_ENV - run: pip install -r requirements.txt - run: pip install -r requirements-dev.txt - save_cache: @@ -29,6 +39,10 @@ jobs: - restore_cache: keys: - py38-v1-{{ checksum "requirements.txt" }}-{{ checksum "requirements-dev.txt" }} + - run: | + wget https://golang.org/dl/go<< pipeline.parameters.go-version >>.linux-amd64.tar.gz -O go.tar.gz + tar -C ~/ -xzf go.tar.gz + echo "export PATH=~/go/bin:$PATH" >> $BASH_ENV - run: pip install -r requirements.txt - run: pip install -r requirements-dev.txt - save_cache: @@ -49,6 +63,10 @@ jobs: - restore_cache: keys: - py39-v1-{{ checksum "requirements.txt" }}-{{ checksum "requirements-dev.txt" }} + - run: | + wget https://golang.org/dl/go<< pipeline.parameters.go-version >>.linux-amd64.tar.gz -O go.tar.gz + tar -C ~/ -xzf go.tar.gz + echo "export PATH=~/go/bin:$PATH" >> $BASH_ENV - run: pip install -r requirements.txt - run: pip install -r requirements-dev.txt - save_cache: diff --git a/.gitignore b/.gitignore index 965aa8972..61e239d1c 100644 --- a/.gitignore +++ b/.gitignore @@ -78,3 +78,6 @@ debian/files # protobuf stuff hivemind/proto/*_pb2* + +# libp2p-daemon binary +hivemind/hivemind_cli/p2pd diff --git a/setup.py b/setup.py index 36630cd27..6135feda7 100644 --- a/setup.py +++ b/setup.py @@ -6,6 +6,7 @@ import urllib.request import tarfile import tempfile +import hashlib from packaging import version from pkg_resources import parse_requirements @@ -13,21 +14,18 @@ from setuptools.command.develop import develop from setuptools.command.install import install +P2PD_VERSION = 'v0.3.1' +P2PD_CHECKSUM = '5094d094740f4e375afe80a5683b1bb2' here = os.path.abspath(os.path.dirname(__file__)) -class cd: - """Context manager for changing the current working directory""" - def __init__(self, newPath): - self.newPath = os.path.expanduser(newPath) - - def __enter__(self): - self.savedPath = os.getcwd() - os.chdir(self.newPath) - - def __exit__(self, etype, value, traceback): - os.chdir(self.savedPath) +def md5(fname, chunk_size=4096): + hash_md5 = hashlib.md5() + with open(fname, "rb") as f: + for chunk in iter(lambda: f.read(chunk_size), b""): + hash_md5.update(chunk) + return hash_md5.hexdigest() def proto_compile(output_path): @@ -49,8 +47,7 @@ def proto_compile(output_path): file.truncate() -def install_libp2p_daemon(): - # check go version: +def libp2p_build_install(): try: proc = subprocess.Popen(['go', 'version'], stdout=subprocess.PIPE) @@ -58,7 +55,7 @@ def install_libp2p_daemon(): result = result.decode('ascii', 'replace') _, _, v, _ = result.split(' ') v = v.lstrip('go') - + if version.parse(v) < version.parse("1.13"): raise EnvironmentError(f'newer version of go required: must be >= 1.13, found {version}') @@ -66,39 +63,45 @@ def install_libp2p_daemon(): raise FileNotFoundError('could not find golang installation') with tempfile.TemporaryDirectory() as tempdir: - url = 'https://github.com/libp2p/go-libp2p-daemon/archive/master.tar.gz' - dest = os.path.join(tempdir, 'libp2p-daemon.tar.gz') + url = f'https://github.com/learning-at-home/go-libp2p-daemon/archive/refs/tags/{P2PD_VERSION}.tar.gz' + dest = os.path.join(tempdir, 'libp2p-daemon.tar.gz') urllib.request.urlretrieve(url, os.path.join(tempdir, dest)) - + tar = tarfile.open(dest, 'r:gz') tar.extractall(tempdir) tar.close() - - with cd(os.path.join(tempdir, 'go-libp2p-daemon-master', 'p2pd')): - status = os.system(f'go build -o {os.path.join(here, "hivemind/hivemind_cli", "p2pd")}') - if status: - raise RuntimeError('Failed to build or install libp2p-daemon:'\ - f' exited with status code :{status}') + result = subprocess.run(['go', 'build', '-o', os.path.join(here, "hivemind/hivemind_cli", "p2pd")], + cwd=os.path.join(tempdir, f'go-libp2p-daemon-{P2PD_VERSION[1:]}', 'p2pd')) + if result.returncode: + raise RuntimeError('Failed to build or install libp2p-daemon:' + f' exited with status code :{result.returncode}') + + +def libp2p_download_install(): + install_path = os.path.join(here, 'hivemind/hivemind_cli/') + binary_path = os.path.join(install_path, 'p2pd') + if 'p2pd' not in os.listdir(install_path) or md5(binary_path) != P2PD_CHECKSUM: + print('Downloading Peer to Peer Daemon') + url = f'https://github.com/learning-at-home/go-libp2p-daemon/releases/download/{P2PD_VERSION}/p2pd' + urllib.request.urlretrieve(url, binary_path) + os.chmod(binary_path, 777) -class ProtoCompileInstall(install): + +class Install(install): def run(self): + libp2p_download_install() proto_compile(os.path.join(self.build_lib, 'hivemind', 'proto')) super().run() -class ProtoCompileDevelop(develop): +class Develop(develop): def run(self): + libp2p_build_install() proto_compile(os.path.join('hivemind', 'proto')) super().run() -class LibP2PInstall(install): - def run(self): - install_libp2p_daemon() - - - with open('requirements.txt') as requirements_file: install_requires = list(map(str, parse_requirements(requirements_file))) @@ -120,7 +123,7 @@ def run(self): setup( name='hivemind', version=version_string, - cmdclass={'install': ProtoCompileInstall, 'develop': ProtoCompileDevelop, 'libp2p': LibP2PInstall}, + cmdclass={'install': Install, 'develop': Develop}, description='Decentralized deep learning in PyTorch', long_description='Decentralized deep learning in PyTorch. Built to train giant models on ' 'thousands of volunteers across the world.', From 3595c94e2dd1391f6b9976ddbb1ede2d3a0925f4 Mon Sep 17 00:00:00 2001 From: Ilya <37004806+skobellev@users.noreply.github.com> Date: Sun, 21 Mar 2021 22:23:03 +0300 Subject: [PATCH 08/81] feat p2p_daemon: add API to call peer handle (#181) * Extend P2P api * Add tests for new api * Add p2pclient dependencies * Test P2P from different processes * Fix typo in tests * Add default initialization * Fix daemon ports assignment * Replace del with __del__ in tests * Read from input stream with receive_exactly Co-authored-by: Ilya Kobelev --- hivemind/p2p/p2p_daemon.py | 190 +++++++++++++++++++++++++++++++++---- tests/test_p2p_daemon.py | 142 +++++++++++++++++++++++---- 2 files changed, 292 insertions(+), 40 deletions(-) diff --git a/hivemind/p2p/p2p_daemon.py b/hivemind/p2p/p2p_daemon.py index 3083c70e5..1f441c5d1 100644 --- a/hivemind/p2p/p2p_daemon.py +++ b/hivemind/p2p/p2p_daemon.py @@ -1,45 +1,197 @@ +import asyncio +import contextlib +import copy +from pathlib import Path +import pickle +import socket import subprocess import typing as tp +import warnings + +from multiaddr import Multiaddr +import p2pclient +from libp2p.peer.id import ID class P2P(object): """ Forks a child process and executes p2pd command with given arguments. - Sends SIGKILL to the child in destructor and on exit from contextmanager. + Can be used for peer to peer communication and procedure calls. + Sends SIGKILL to the child in destructor. """ - LIBP2P_CMD = 'p2pd' + P2PD_RELATIVE_PATH = 'hivemind_cli/p2pd' + NUM_RETRIES = 3 + RETRY_DELAY = 0.4 + HEADER_LEN = 8 + BYTEORDER = 'big' - def __init__(self, *args, **kwargs): - self._child = subprocess.Popen(args=self._make_process_args(args, kwargs)) - try: - stdout, stderr = self._child.communicate(timeout=0.2) - except subprocess.TimeoutExpired: - pass - else: - raise RuntimeError(f'p2p daemon exited with stderr: {stderr}') + def __init__(self): + self._child = None + self._listen_task = None + self._server_stopped = asyncio.Event() + self._buffer = bytearray() - def __enter__(self): - return self._child + @classmethod + async def create(cls, *args, quic=1, tls=1, conn_manager=1, dht_client=1, + nat_port_map=True, auto_nat=True, bootstrap=True, + host_port: int = None, daemon_listen_port: int = None, **kwargs): + self = cls() + p2pd_path = Path(__file__).resolve().parents[1] / P2P.P2PD_RELATIVE_PATH + proc_args = self._make_process_args( + str(p2pd_path), *args, + quic=quic, tls=tls, connManager=conn_manager, + dhtClient=dht_client, natPortMap=nat_port_map, + autonat=auto_nat, b=bootstrap, **kwargs) + self._assign_daemon_ports(host_port, daemon_listen_port) + for try_count in range(self.NUM_RETRIES): + try: + self._initialize(proc_args) + await self._identify_client(P2P.RETRY_DELAY * (2 ** try_count)) + except Exception as exc: + warnings.warn("Failed to initialize p2p daemon: " + str(exc), RuntimeWarning) + self._kill_child() + if try_count == P2P.NUM_RETRIES - 1: + raise + self._assign_daemon_ports() + continue + break + return self - def __exit__(self, exc_type, exc_val, exc_tb): - self._kill_child() + def _initialize(self, proc_args: tp.List[str]) -> None: + proc_args = copy.deepcopy(proc_args) + proc_args.extend(self._make_process_args( + hostAddrs=f'/ip4/0.0.0.0/tcp/{self._host_port},/ip4/0.0.0.0/udp/{self._host_port}/quic', + listen=f'/ip4/127.0.0.1/tcp/{self._daemon_listen_port}' + )) + self._child = subprocess.Popen( + args=proc_args, + stdin=subprocess.PIPE, stdout=subprocess.PIPE, + stderr=subprocess.PIPE, encoding="utf8" + ) + self._client_listen_port = find_open_port() + self._client = p2pclient.Client( + Multiaddr(f'/ip4/127.0.0.1/tcp/{self._daemon_listen_port}'), + Multiaddr(f'/ip4/127.0.0.1/tcp/{self._client_listen_port}')) + + async def _identify_client(self, delay): + await asyncio.sleep(delay) + encoded = await self._client.identify() + self.id = encoded[0].to_base58() + + def _assign_daemon_ports(self, host_port=None, daemon_listen_port=None): + self._host_port, self._daemon_listen_port = host_port, daemon_listen_port + if host_port is None: + self._host_port = find_open_port() + if daemon_listen_port is None: + self._daemon_listen_port = find_open_port() + while self._daemon_listen_port == self._host_port: + self._daemon_listen_port = find_open_port() + + @staticmethod + async def send_data(data, stream): + byte_str = pickle.dumps(data) + request = len(byte_str).to_bytes(P2P.HEADER_LEN, P2P.BYTEORDER) + byte_str + await stream.send_all(request) + + class IncompleteRead(Exception): + pass + + async def _receive_exactly(self, stream, n_bytes, max_bytes=1 << 16): + while len(self._buffer) < n_bytes: + data = await stream.receive_some(max_bytes) + if len(data) == 0: + raise P2P.IncompleteRead() + self._buffer.extend(data) + + result = self._buffer[:n_bytes] + self._buffer = self._buffer[n_bytes:] + return bytes(result) + + async def receive_data(self, stream, max_bytes=(1 < 16)): + header = await self._receive_exactly(stream, P2P.HEADER_LEN) + content_length = int.from_bytes(header, P2P.BYTEORDER) + data = await self._receive_exactly(stream, content_length) + return pickle.loads(data) + + def _handle_stream(self, handle): + async def do_handle_stream(stream_info, stream): + try: + request = await self.receive_data(stream) + except P2P.IncompleteRead: + warnings.warn("Incomplete read while receiving request from peer", RuntimeWarning) + return + finally: + stream.close() + try: + result = handle(request) + await self.send_data(result, stream) + except Exception as exc: + await self.send_data(exc, stream) + finally: + await stream.close() + + return do_handle_stream + + def start_listening(self): + async def listen(): + async with self._client.listen(): + await self._server_stopped.wait() + + self._listen_task = asyncio.create_task(listen()) + + async def stop_listening(self): + if self._listen_task is not None: + self._server_stopped.set() + self._listen_task.cancel() + try: + await self._listen_task + except asyncio.CancelledError: + self._listen_task = None + self._server_stopped.clear() + + async def add_stream_handler(self, name, handle): + if self._listen_task is None: + self.start_listening() + + await self._client.stream_handler(name, self._handle_stream(handle)) + + async def call_peer_handler(self, peer_id, handler_name, input_data): + libp2p_peer_id = ID.from_base58(peer_id) + stream_info, stream = await self._client.stream_open(libp2p_peer_id, (handler_name,)) + try: + await self.send_data(input_data, stream) + return await self.receive_data(stream) + finally: + await stream.close() def __del__(self): self._kill_child() def _kill_child(self): - if self._child.poll() is None: + if self._child is not None and self._child.poll() is None: self._child.kill() self._child.wait() - def _make_process_args(self, args: tp.Tuple[tp.Any], - kwargs: tp.Dict[str, tp.Any]) -> tp.List[str]: - proc_args = [self.LIBP2P_CMD] + def _make_process_args(self, *args, **kwargs) -> tp.List[str]: + proc_args = [] proc_args.extend( str(entry) for entry in args ) proc_args.extend( - f'-{key}={str(value)}' for key, value in kwargs.items() + f'-{key}={value}' if value is not None else f'-{key}' + for key, value in kwargs.items() ) return proc_args + + +def find_open_port(params=(socket.AF_INET, socket.SOCK_STREAM), + opt=(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)): + """ Finds a tcp port that can be occupied with a socket with *params and use *opt options """ + try: + with contextlib.closing(socket.socket(*params)) as sock: + sock.bind(('', 0)) + sock.setsockopt(*opt) + return sock.getsockname()[1] + except Exception: + raise diff --git a/tests/test_p2p_daemon.py b/tests/test_p2p_daemon.py index ac57e9e2f..75fd51cdc 100644 --- a/tests/test_p2p_daemon.py +++ b/tests/test_p2p_daemon.py @@ -1,6 +1,8 @@ +import asyncio +import multiprocessing as mp import subprocess -from time import perf_counter +import numpy as np import pytest import hivemind.p2p @@ -23,33 +25,131 @@ def is_process_running(pid: int) -> bool: return subprocess.check_output(cmd, shell=True).decode('utf-8').strip() == RUNNING -@pytest.fixture() -def mock_p2p_class(): - P2P.LIBP2P_CMD = "sleep" - - -def test_daemon_killed_on_del(mock_p2p_class): - start = perf_counter() - p2p_daemon = P2P('10s') +@pytest.mark.asyncio +async def test_daemon_killed_on_del(): + p2p_daemon = await P2P.create() child_pid = p2p_daemon._child.pid assert is_process_running(child_pid) - del p2p_daemon + p2p_daemon.__del__() assert not is_process_running(child_pid) - assert perf_counter() - start < 1 -def test_daemon_killed_on_exit(mock_p2p_class): - start = perf_counter() - with P2P('10s') as daemon: - child_pid = daemon.pid - assert is_process_running(child_pid) +def handle_square(x): + return x ** 2 - assert not is_process_running(child_pid) - assert perf_counter() - start < 1 + +def handle_add(args): + result = args[0] + for i in range(1, len(args)): + result = result + args[i] + return result + + +@pytest.mark.parametrize( + "test_input,handle", + [ + pytest.param(10, handle_square, id="square_integer"), + pytest.param((1, 2), handle_add, id="add_integers"), + pytest.param(([1, 2, 3], [12, 13]), handle_add, id="add_lists"), + pytest.param(2, lambda x: x ** 3, id="lambda") + ] +) +@pytest.mark.asyncio +async def test_call_peer_single_process(test_input, handle, handler_name="handle"): + server = await P2P.create() + server_pid = server._child.pid + await server.add_stream_handler(handler_name, handle) + assert is_process_running(server_pid) + + client = await P2P.create() + client_pid = client._child.pid + assert is_process_running(client_pid) + + await asyncio.sleep(1) + result = await client.call_peer_handler(server.id, handler_name, test_input) + assert result == handle(test_input) + + await server.stop_listening() + server.__del__() + assert not is_process_running(server_pid) + + client.__del__() + assert not is_process_running(client_pid) + + +@pytest.mark.asyncio +async def test_call_peer_different_processes(): + handler_name = "square" + test_input = np.random.randn(2, 3) + + server_side, client_side = mp.Pipe() + response_received = mp.Value(np.ctypeslib.as_ctypes_type(np.int32)) + response_received.value = 0 + + async def run_server(): + server = await P2P.create() + server_pid = server._child.pid + await server.add_stream_handler(handler_name, handle_square) + assert is_process_running(server_pid) + + server_side.send(server.id) + while response_received.value == 0: + await asyncio.sleep(0.5) + + await server.stop_listening() + server.__del__() + assert not is_process_running(server_pid) + + def server_target(): + asyncio.run(run_server()) + + proc = mp.Process(target=server_target) + proc.start() + + client = await P2P.create() + client_pid = client._child.pid + assert is_process_running(client_pid) + + await asyncio.sleep(1) + peer_id = client_side.recv() + + result = await client.call_peer_handler(peer_id, handler_name, test_input) + assert np.allclose(result, handle_square(test_input)) + response_received.value = 1 + + client.__del__() + assert not is_process_running(client_pid) + + proc.join() -def test_daemon_raises_on_faulty_args(): - with pytest.raises(RuntimeError): - P2P(faulty='argument') +@pytest.mark.parametrize( + "test_input,handle", + [ + pytest.param(np.random.randn(2, 3), handle_square, id="square"), + pytest.param([np.random.randn(2, 3), np.random.randn(2, 3)], handle_add, id="add"), + ] +) +@pytest.mark.asyncio +async def test_call_peer_numpy(test_input, handle, handler_name="handle"): + server = await P2P.create() + await server.add_stream_handler(handler_name, handle) + client = await P2P.create() + + await asyncio.sleep(1) + result = await client.call_peer_handler(server.id, handler_name, test_input) + assert np.allclose(result, handle(test_input)) + + +@pytest.mark.asyncio +async def test_call_peer_error(handler_name="handle"): + server = await P2P.create() + await server.add_stream_handler(handler_name, handle_add) + client = await P2P.create() + + await asyncio.sleep(1) + result = await client.call_peer_handler(server.id, handler_name, + [np.zeros((2, 3)), np.zeros((3, 2))]) + assert type(result) == ValueError From d2d849b147569a7241968a73dc98258433ef5169 Mon Sep 17 00:00:00 2001 From: Denis Mazur Date: Thu, 25 Mar 2021 00:18:52 +0300 Subject: [PATCH 09/81] fix chmod permissions (#194) --- setup.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 6135feda7..0bc15830e 100644 --- a/setup.py +++ b/setup.py @@ -73,6 +73,7 @@ def libp2p_build_install(): result = subprocess.run(['go', 'build', '-o', os.path.join(here, "hivemind/hivemind_cli", "p2pd")], cwd=os.path.join(tempdir, f'go-libp2p-daemon-{P2PD_VERSION[1:]}', 'p2pd')) + if result.returncode: raise RuntimeError('Failed to build or install libp2p-daemon:' f' exited with status code :{result.returncode}') @@ -85,7 +86,7 @@ def libp2p_download_install(): print('Downloading Peer to Peer Daemon') url = f'https://github.com/learning-at-home/go-libp2p-daemon/releases/download/{P2PD_VERSION}/p2pd' urllib.request.urlretrieve(url, binary_path) - os.chmod(binary_path, 777) + os.chmod(binary_path, 0o777) class Install(install): From 8d873f630fb08091ac9ee1114502062fcdebfff9 Mon Sep 17 00:00:00 2001 From: Ilya <37004806+skobellev@users.noreply.github.com> Date: Wed, 31 Mar 2021 13:09:45 +0300 Subject: [PATCH 10/81] feat P2P: add unary handler (#197) * Add unary handler * Add P2PContext to unary handler parameters Co-authored-by: Ilya Kobelev --- hivemind/p2p/p2p_daemon.py | 147 +++++++++++++++++++++++++++---------- tests/test_p2p_daemon.py | 62 +++++++++++++++- 2 files changed, 168 insertions(+), 41 deletions(-) diff --git a/hivemind/p2p/p2p_daemon.py b/hivemind/p2p/p2p_daemon.py index 1f441c5d1..a8f44550e 100644 --- a/hivemind/p2p/p2p_daemon.py +++ b/hivemind/p2p/p2p_daemon.py @@ -1,17 +1,27 @@ import asyncio -import contextlib import copy from pathlib import Path import pickle -import socket import subprocess import typing as tp import warnings +import google.protobuf from multiaddr import Multiaddr import p2pclient from libp2p.peer.id import ID +from hivemind.utils.networking import find_open_port + + +class P2PContext(object): + def __init__(self, ours_id, ours_port, handle_name): + self.peer_id = None + self.peer_addr = None + self.ours_id = ours_id + self.ours_port = ours_port + self.handle_name = handle_name + class P2P(object): """ @@ -26,11 +36,16 @@ class P2P(object): HEADER_LEN = 8 BYTEORDER = 'big' + class IncompleteRead(Exception): + pass + + class InterruptedError(Exception): + pass + def __init__(self): self._child = None self._listen_task = None self._server_stopped = asyncio.Event() - self._buffer = bytearray() @classmethod async def create(cls, *args, quic=1, tls=1, conn_manager=1, dht_client=1, @@ -89,50 +104,108 @@ def _assign_daemon_ports(self, host_port=None, daemon_listen_port=None): self._daemon_listen_port = find_open_port() @staticmethod - async def send_data(data, stream): - byte_str = pickle.dumps(data) + async def send_raw_data(byte_str, stream): request = len(byte_str).to_bytes(P2P.HEADER_LEN, P2P.BYTEORDER) + byte_str await stream.send_all(request) - class IncompleteRead(Exception): - pass + @staticmethod + async def send_data(data, stream): + await P2P.send_raw_data(pickle.dumps(data), stream) + + @staticmethod + async def send_protobuf(protobuf, out_proto_type, stream): + if type(protobuf) != out_proto_type: + error = TypeError('Unary handler returned protobuf of wrong type.') + await P2P.send_raw_data(pickle.dumps(error), stream) + raise error + await P2P.send_raw_data(protobuf.SerializeToString(), stream) - async def _receive_exactly(self, stream, n_bytes, max_bytes=1 << 16): - while len(self._buffer) < n_bytes: - data = await stream.receive_some(max_bytes) + @staticmethod + async def receive_exactly(stream, n_bytes, max_bytes=1 << 16): + buffer = bytearray() + while len(buffer) < n_bytes: + data = await stream.receive_some(min(max_bytes, n_bytes - len(buffer))) if len(data) == 0: raise P2P.IncompleteRead() - self._buffer.extend(data) - - result = self._buffer[:n_bytes] - self._buffer = self._buffer[n_bytes:] - return bytes(result) + buffer.extend(data) + return bytes(buffer) - async def receive_data(self, stream, max_bytes=(1 < 16)): - header = await self._receive_exactly(stream, P2P.HEADER_LEN) + @staticmethod + async def receive_raw_data(stream): + header = await P2P.receive_exactly(stream, P2P.HEADER_LEN) content_length = int.from_bytes(header, P2P.BYTEORDER) - data = await self._receive_exactly(stream, content_length) - return pickle.loads(data) + data = await P2P.receive_exactly(stream, content_length) + return data - def _handle_stream(self, handle): + @staticmethod + async def receive_data(stream): + return pickle.loads(await P2P.receive_raw_data(stream)) + + @staticmethod + async def receive_protobuf(in_proto_type, stream): + protobuf = in_proto_type() + protobuf.ParseFromString(await P2P.receive_raw_data(stream)) + return protobuf + + @staticmethod + def _handle_stream(handle): async def do_handle_stream(stream_info, stream): try: - request = await self.receive_data(stream) + request = await P2P.receive_data(stream) except P2P.IncompleteRead: warnings.warn("Incomplete read while receiving request from peer", RuntimeWarning) + await stream.close() return - finally: - stream.close() try: result = handle(request) - await self.send_data(result, stream) + await P2P.send_data(result, stream) except Exception as exc: - await self.send_data(exc, stream) + await P2P.send_data(exc, stream) finally: await stream.close() return do_handle_stream + @staticmethod + def _handle_unary_stream(handle, context, in_proto_type, out_proto_type): + async def watchdog(stream): + await stream.receive_some(max_bytes=1) + raise P2P.InterruptedError() + + async def do_handle_unary_stream(stream_info, stream): + try: + try: + request = await P2P.receive_protobuf(in_proto_type, stream) + except P2P.IncompleteRead: + warnings.warn("Incomplete read while receiving request from peer", + RuntimeWarning) + return + except google.protobuf.message.DecodeError as error: + warnings.warn(repr(error), RuntimeWarning) + return + + context.peer_id, context.peer_addr = stream_info.peer_id, stream_info.addr + done, pending = await asyncio.wait([watchdog(stream), handle(request, context)], + return_when=asyncio.FIRST_COMPLETED) + try: + result = done.pop().result() + await P2P.send_protobuf(result, out_proto_type, stream) + except P2P.InterruptedError: + pass + except Exception as exc: + await P2P.send_data(exc, stream) + finally: + pending_task = pending.pop() + pending_task.cancel() + try: + await pending_task + except asyncio.CancelledError: + pass + finally: + await stream.close() + + return do_handle_unary_stream + def start_listening(self): async def listen(): async with self._client.listen(): @@ -153,15 +226,21 @@ async def stop_listening(self): async def add_stream_handler(self, name, handle): if self._listen_task is None: self.start_listening() + await self._client.stream_handler(name, P2P._handle_stream(handle)) - await self._client.stream_handler(name, self._handle_stream(handle)) + async def add_unary_handler(self, name, handle, in_proto_type, out_proto_type): + if self._listen_task is None: + self.start_listening() + context = P2PContext(ours_id=self.id, ours_port=self._host_port, handle_name=name) + await self._client.stream_handler( + name, P2P._handle_unary_stream(handle, context, in_proto_type, out_proto_type)) async def call_peer_handler(self, peer_id, handler_name, input_data): libp2p_peer_id = ID.from_base58(peer_id) stream_info, stream = await self._client.stream_open(libp2p_peer_id, (handler_name,)) try: - await self.send_data(input_data, stream) - return await self.receive_data(stream) + await P2P.send_data(input_data, stream) + return await P2P.receive_data(stream) finally: await stream.close() @@ -183,15 +262,3 @@ def _make_process_args(self, *args, **kwargs) -> tp.List[str]: for key, value in kwargs.items() ) return proc_args - - -def find_open_port(params=(socket.AF_INET, socket.SOCK_STREAM), - opt=(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)): - """ Finds a tcp port that can be occupied with a socket with *params and use *opt options """ - try: - with contextlib.closing(socket.socket(*params)) as sock: - sock.bind(('', 0)) - sock.setsockopt(*opt) - return sock.getsockname()[1] - except Exception: - raise diff --git a/tests/test_p2p_daemon.py b/tests/test_p2p_daemon.py index 75fd51cdc..06814d244 100644 --- a/tests/test_p2p_daemon.py +++ b/tests/test_p2p_daemon.py @@ -2,11 +2,13 @@ import multiprocessing as mp import subprocess +from libp2p.peer.id import ID + import numpy as np import pytest -import hivemind.p2p from hivemind.p2p import P2P +from hivemind.proto import dht_pb2 RUNNING = 'running' NOT_RUNNING = 'not running' @@ -47,6 +49,64 @@ def handle_add(args): return result +@pytest.mark.parametrize( + 'should_cancel', [True, False] +) +@pytest.mark.asyncio +async def test_call_unary_handler(should_cancel, handle_name="handle"): + handler_cancelled = False + + async def ping_handler(request, context): + try: + await asyncio.sleep(2) + except asyncio.CancelledError: + nonlocal handler_cancelled + handler_cancelled = True + return dht_pb2.PingResponse( + peer=dht_pb2.NodeInfo( + node_id=context.ours_id.encode(), rpc_port=context.ours_port), + sender_endpoint=context.handle_name, available=True) + + server = await P2P.create() + server_pid = server._child.pid + await server.add_unary_handler(handle_name, ping_handler, dht_pb2.PingRequest, + dht_pb2.PingResponse) + assert is_process_running(server_pid) + + client = await P2P.create() + client_pid = client._child.pid + assert is_process_running(client_pid) + + ping_request = dht_pb2.PingRequest( + peer=dht_pb2.NodeInfo(node_id=client.id.encode(), rpc_port=client._host_port), + validate=True) + expected_response = dht_pb2.PingResponse( + peer=dht_pb2.NodeInfo(node_id=server.id.encode(), rpc_port=server._host_port), + sender_endpoint=handle_name, available=True) + + await asyncio.sleep(1) + libp2p_server_id = ID.from_base58(server.id) + stream_info, stream = await client._client.stream_open(libp2p_server_id, (handle_name,)) + + await P2P.send_raw_data(ping_request.SerializeToString(), stream) + + if should_cancel: + await stream.close() + await asyncio.sleep(1) + assert handler_cancelled + else: + result = await P2P.receive_protobuf(dht_pb2.PingResponse, stream) + assert result == expected_response + assert not handler_cancelled + + await server.stop_listening() + server.__del__() + assert not is_process_running(server_pid) + + client.__del__() + assert not is_process_running(client_pid) + + @pytest.mark.parametrize( "test_input,handle", [ From 3b5ce788282a4daa74f869a81f5a4d13c6d16df3 Mon Sep 17 00:00:00 2001 From: MaximKsh Date: Fri, 2 Apr 2021 22:59:06 +0300 Subject: [PATCH 11/81] Py libp2p bindings (#193) * #183 p2p daemon pybinding * #183 rename py bindings dir, fix imports and migrate tests * #183 move pb to hivemind.proto * #183 fix p2p tests * #183 remove config.py, move constants to classes * add docstrings and minor fixes --- hivemind/p2p/p2p_daemon.py | 75 +- hivemind/p2p/p2p_daemon_bindings/__init__.py | 0 hivemind/p2p/p2p_daemon_bindings/control.py | 211 +++++ .../p2p/p2p_daemon_bindings/datastructures.py | 186 +++++ hivemind/p2p/p2p_daemon_bindings/keys.py | 91 +++ hivemind/p2p/p2p_daemon_bindings/p2pclient.py | 75 ++ hivemind/p2p/p2p_daemon_bindings/utils.py | 72 ++ hivemind/proto/crypto.proto | 20 + hivemind/proto/p2pd.proto | 158 ++++ requirements.txt | 2 + tests/test_p2p_daemon.py | 48 +- tests/test_p2p_daemon_bindings.py | 769 ++++++++++++++++++ 12 files changed, 1648 insertions(+), 59 deletions(-) create mode 100644 hivemind/p2p/p2p_daemon_bindings/__init__.py create mode 100644 hivemind/p2p/p2p_daemon_bindings/control.py create mode 100644 hivemind/p2p/p2p_daemon_bindings/datastructures.py create mode 100644 hivemind/p2p/p2p_daemon_bindings/keys.py create mode 100644 hivemind/p2p/p2p_daemon_bindings/p2pclient.py create mode 100644 hivemind/p2p/p2p_daemon_bindings/utils.py create mode 100644 hivemind/proto/crypto.proto create mode 100644 hivemind/proto/p2pd.proto create mode 100644 tests/test_p2p_daemon_bindings.py diff --git a/hivemind/p2p/p2p_daemon.py b/hivemind/p2p/p2p_daemon.py index a8f44550e..af3185a44 100644 --- a/hivemind/p2p/p2p_daemon.py +++ b/hivemind/p2p/p2p_daemon.py @@ -8,8 +8,8 @@ import google.protobuf from multiaddr import Multiaddr -import p2pclient -from libp2p.peer.id import ID +import hivemind.p2p.p2p_daemon_bindings.p2pclient as p2pclient +from hivemind.p2p.p2p_daemon_bindings.datastructures import ID, StreamInfo from hivemind.utils.networking import find_open_port @@ -104,78 +104,81 @@ def _assign_daemon_ports(self, host_port=None, daemon_listen_port=None): self._daemon_listen_port = find_open_port() @staticmethod - async def send_raw_data(byte_str, stream): + async def send_raw_data(byte_str, writer): request = len(byte_str).to_bytes(P2P.HEADER_LEN, P2P.BYTEORDER) + byte_str - await stream.send_all(request) + writer.write(request) @staticmethod - async def send_data(data, stream): - await P2P.send_raw_data(pickle.dumps(data), stream) + async def send_data(data, writer): + await P2P.send_raw_data(pickle.dumps(data), writer) @staticmethod - async def send_protobuf(protobuf, out_proto_type, stream): + async def send_protobuf(protobuf, out_proto_type, writer): if type(protobuf) != out_proto_type: error = TypeError('Unary handler returned protobuf of wrong type.') - await P2P.send_raw_data(pickle.dumps(error), stream) + await P2P.send_raw_data(pickle.dumps(error), writer) raise error - await P2P.send_raw_data(protobuf.SerializeToString(), stream) + await P2P.send_raw_data(protobuf.SerializeToString(), writer) @staticmethod - async def receive_exactly(stream, n_bytes, max_bytes=1 << 16): + async def receive_exactly(reader, n_bytes, max_bytes=1 << 16): buffer = bytearray() while len(buffer) < n_bytes: - data = await stream.receive_some(min(max_bytes, n_bytes - len(buffer))) + data = await reader.read(min(max_bytes, n_bytes - len(buffer))) if len(data) == 0: raise P2P.IncompleteRead() buffer.extend(data) return bytes(buffer) @staticmethod - async def receive_raw_data(stream): - header = await P2P.receive_exactly(stream, P2P.HEADER_LEN) + async def receive_raw_data(reader): + header = await P2P.receive_exactly(reader, P2P.HEADER_LEN) content_length = int.from_bytes(header, P2P.BYTEORDER) - data = await P2P.receive_exactly(stream, content_length) + data = await P2P.receive_exactly(reader, content_length) return data @staticmethod - async def receive_data(stream): - return pickle.loads(await P2P.receive_raw_data(stream)) + async def receive_data(reader): + return pickle.loads(await P2P.receive_raw_data(reader)) @staticmethod - async def receive_protobuf(in_proto_type, stream): + async def receive_protobuf(in_proto_type, reader): protobuf = in_proto_type() - protobuf.ParseFromString(await P2P.receive_raw_data(stream)) + protobuf.ParseFromString(await P2P.receive_raw_data(reader)) return protobuf @staticmethod def _handle_stream(handle): - async def do_handle_stream(stream_info, stream): + async def do_handle_stream(stream_info, reader, writer): try: - request = await P2P.receive_data(stream) + request = await P2P.receive_data(reader) except P2P.IncompleteRead: warnings.warn("Incomplete read while receiving request from peer", RuntimeWarning) - await stream.close() + writer.close() return try: result = handle(request) - await P2P.send_data(result, stream) + await P2P.send_data(result, writer) except Exception as exc: - await P2P.send_data(exc, stream) + await P2P.send_data(exc, writer) finally: - await stream.close() + writer.close() return do_handle_stream @staticmethod def _handle_unary_stream(handle, context, in_proto_type, out_proto_type): - async def watchdog(stream): - await stream.receive_some(max_bytes=1) + async def watchdog(reader: asyncio.StreamReader): + await reader.read(n=1) raise P2P.InterruptedError() - async def do_handle_unary_stream(stream_info, stream): + async def do_handle_unary_stream( + stream_info: StreamInfo, + reader: asyncio.StreamReader, + writer: asyncio.StreamWriter) -> None: try: try: - request = await P2P.receive_protobuf(in_proto_type, stream) + request = await P2P.receive_protobuf(in_proto_type, reader) except P2P.IncompleteRead: warnings.warn("Incomplete read while receiving request from peer", RuntimeWarning) @@ -185,15 +188,15 @@ async def do_handle_unary_stream(stream_info, stream): return context.peer_id, context.peer_addr = stream_info.peer_id, stream_info.addr - done, pending = await asyncio.wait([watchdog(stream), handle(request, context)], + done, pending = await asyncio.wait([watchdog(reader), handle(request, context)], return_when=asyncio.FIRST_COMPLETED) try: result = done.pop().result() - await P2P.send_protobuf(result, out_proto_type, stream) + await P2P.send_protobuf(result, out_proto_type, writer) except P2P.InterruptedError: pass except Exception as exc: - await P2P.send_data(exc, stream) + await P2P.send_data(exc, writer) finally: pending_task = pending.pop() pending_task.cancel() @@ -202,7 +205,7 @@ async def do_handle_unary_stream(stream_info, stream): except asyncio.CancelledError: pass finally: - await stream.close() + writer.close() return do_handle_unary_stream @@ -237,12 +240,12 @@ async def add_unary_handler(self, name, handle, in_proto_type, out_proto_type): async def call_peer_handler(self, peer_id, handler_name, input_data): libp2p_peer_id = ID.from_base58(peer_id) - stream_info, stream = await self._client.stream_open(libp2p_peer_id, (handler_name,)) + stream_info, reader, writer = await self._client.stream_open(libp2p_peer_id, (handler_name,)) try: - await P2P.send_data(input_data, stream) - return await P2P.receive_data(stream) + await P2P.send_data(input_data, writer) + return await P2P.receive_data(reader) finally: - await stream.close() + writer.close() def __del__(self): self._kill_child() diff --git a/hivemind/p2p/p2p_daemon_bindings/__init__.py b/hivemind/p2p/p2p_daemon_bindings/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/hivemind/p2p/p2p_daemon_bindings/control.py b/hivemind/p2p/p2p_daemon_bindings/control.py new file mode 100644 index 000000000..df8aeaefa --- /dev/null +++ b/hivemind/p2p/p2p_daemon_bindings/control.py @@ -0,0 +1,211 @@ +import logging +from typing import AsyncIterator, Awaitable, Callable, Dict, Iterable, Sequence, Tuple + +import asyncio +from contextlib import asynccontextmanager +from multiaddr import Multiaddr, protocols +from hivemind.p2p.p2p_daemon_bindings.datastructures import PeerInfo, StreamInfo, ID +from hivemind.proto import p2pd_pb2 as p2pd_pb +from hivemind.p2p.p2p_daemon_bindings.utils import DispatchFailure, read_pbmsg_safe, write_pbmsg, raise_if_failed + +StreamHandler = Callable[[StreamInfo, asyncio.StreamReader, asyncio.StreamWriter], Awaitable[None]] + +_supported_conn_protocols = ( + protocols.P_IP4, + # protocols.P_IP6, + protocols.P_UNIX, +) + + +def parse_conn_protocol(maddr: Multiaddr) -> int: + proto_codes = set(proto.code for proto in maddr.protocols()) + proto_cand = proto_codes.intersection(_supported_conn_protocols) + if len(proto_cand) != 1: + supported_protos = ( + protocols.protocol_with_code(proto) for proto in _supported_conn_protocols + ) + raise ValueError( + f"connection protocol should be only one protocol out of {supported_protos}" + f", maddr={maddr}" + ) + return tuple(proto_cand)[0] + + +class DaemonConnector: + control_maddr: Multiaddr + logger = logging.getLogger("p2pclient.DaemonConnector") + DEFAULT_CONTROL_MADDR = "/unix/tmp/p2pd.sock" + + def __init__(self, control_maddr: Multiaddr = None) -> None: + if control_maddr is None: + control_maddr = Multiaddr(self.DEFAULT_CONTROL_MADDR) + self.control_maddr = control_maddr + + async def open_connection(self) -> (asyncio.StreamReader, asyncio.StreamWriter): + proto_code = parse_conn_protocol(self.control_maddr) + if proto_code == protocols.P_UNIX: + control_path = self.control_maddr.value_for_protocol(protocols.P_UNIX) + self.logger.debug( + "DaemonConnector %s opens connection to %s", self, self.control_maddr + ) + return await asyncio.open_unix_connection(control_path) + elif proto_code == protocols.P_IP4: + host = self.control_maddr.value_for_protocol(protocols.P_IP4) + port = int(self.control_maddr.value_for_protocol(protocols.P_TCP)) + return await asyncio.open_connection(host, port) + else: + raise ValueError( + f"protocol not supported: protocol={protocols.protocol_with_code(proto_code)}" + ) + + +class ControlClient: + listen_maddr: Multiaddr + daemon_connector: DaemonConnector + handlers: Dict[str, StreamHandler] + logger = logging.getLogger("p2pclient.ControlClient") + DEFAULT_LISTEN_MADDR = "/unix/tmp/p2pclient.sock" + + def __init__( + self, daemon_connector: DaemonConnector, listen_maddr: Multiaddr = None + ) -> None: + if listen_maddr is None: + listen_maddr = Multiaddr(self.DEFAULT_LISTEN_MADDR) + self.listen_maddr = listen_maddr + self.daemon_connector = daemon_connector + self.handlers = {} + + async def _handler(self, reader: asyncio.StreamReader, writer: asyncio.StreamWriter): + pb_stream_info = p2pd_pb.StreamInfo() # type: ignore + await read_pbmsg_safe(reader, pb_stream_info) + stream_info = StreamInfo.from_pb(pb_stream_info) + self.logger.info("New incoming stream: %s", stream_info) + try: + handler = self.handlers[stream_info.proto] + except KeyError as e: + # should never enter here... daemon should reject the stream for us. + writer.close() + raise DispatchFailure(e) + await handler(stream_info, reader, writer) + + @asynccontextmanager + async def listen(self) -> AsyncIterator["ControlClient"]: + proto_code = parse_conn_protocol(self.listen_maddr) + if proto_code == protocols.P_UNIX: + listen_path = self.listen_maddr.value_for_protocol(protocols.P_UNIX) + server = await asyncio.start_unix_server(self._handler, path=listen_path) + elif proto_code == protocols.P_IP4: + host = self.listen_maddr.value_for_protocol(protocols.P_IP4) + port = int(self.listen_maddr.value_for_protocol(protocols.P_TCP)) + server = await asyncio.start_server(self._handler, port=port, host=host) + else: + raise ValueError( + f"protocol not supported: protocol={protocols.protocol_with_code(proto_code)}" + ) + + async with server: + self.logger.info( + "DaemonConnector %s starts listening to %s", self, self.listen_maddr + ) + yield self + + self.logger.info("DaemonConnector %s closed", self) + + async def identify(self) -> Tuple[ID, Tuple[Multiaddr, ...]]: + reader, writer = await self.daemon_connector.open_connection() + req = p2pd_pb.Request(type=p2pd_pb.Request.IDENTIFY) + await write_pbmsg(writer, req) + + resp = p2pd_pb.Response() # type: ignore + await read_pbmsg_safe(reader, resp) + writer.close() + + raise_if_failed(resp) + peer_id_bytes = resp.identify.id + maddrs_bytes = resp.identify.addrs + + maddrs = tuple(Multiaddr(maddr_bytes) for maddr_bytes in maddrs_bytes) + peer_id = ID(peer_id_bytes) + + return peer_id, maddrs + + async def connect(self, peer_id: ID, maddrs: Iterable[Multiaddr]) -> None: + reader, writer = await self.daemon_connector.open_connection() + + maddrs_bytes = [i.to_bytes() for i in maddrs] + connect_req = p2pd_pb.ConnectRequest( + peer=peer_id.to_bytes(), addrs=maddrs_bytes + ) + req = p2pd_pb.Request(type=p2pd_pb.Request.CONNECT, connect=connect_req) + await write_pbmsg(writer, req) + + resp = p2pd_pb.Response() # type: ignore + await read_pbmsg_safe(reader, resp) + writer.close() + raise_if_failed(resp) + + async def list_peers(self) -> Tuple[PeerInfo, ...]: + req = p2pd_pb.Request(type=p2pd_pb.Request.LIST_PEERS) + reader, writer = await self.daemon_connector.open_connection() + await write_pbmsg(writer, req) + resp = p2pd_pb.Response() # type: ignore + await read_pbmsg_safe(reader, resp) + writer.close() + raise_if_failed(resp) + + peers = tuple(PeerInfo.from_pb(pinfo) for pinfo in resp.peers) + return peers + + async def disconnect(self, peer_id: ID) -> None: + disconnect_req = p2pd_pb.DisconnectRequest(peer=peer_id.to_bytes()) + req = p2pd_pb.Request( + type=p2pd_pb.Request.DISCONNECT, disconnect=disconnect_req + ) + reader, writer = await self.daemon_connector.open_connection() + await write_pbmsg(writer, req) + resp = p2pd_pb.Response() # type: ignore + await read_pbmsg_safe(reader, resp) + writer.close() + raise_if_failed(resp) + + async def stream_open( + self, peer_id: ID, protocols: Sequence[str] + ) -> Tuple[StreamInfo, asyncio.StreamReader, asyncio.StreamWriter]: + reader, writer = await self.daemon_connector.open_connection() + + stream_open_req = p2pd_pb.StreamOpenRequest( + peer=peer_id.to_bytes(), proto=list(protocols) + ) + req = p2pd_pb.Request( + type=p2pd_pb.Request.STREAM_OPEN, streamOpen=stream_open_req + ) + await write_pbmsg(writer, req) + + resp = p2pd_pb.Response() # type: ignore + await read_pbmsg_safe(reader, resp) + raise_if_failed(resp) + + pb_stream_info = resp.streamInfo + stream_info = StreamInfo.from_pb(pb_stream_info) + + return stream_info, reader, writer + + async def stream_handler(self, proto: str, handler_cb: StreamHandler) -> None: + reader, writer = await self.daemon_connector.open_connection() + + listen_path_maddr_bytes = self.listen_maddr.to_bytes() + stream_handler_req = p2pd_pb.StreamHandlerRequest( + addr=listen_path_maddr_bytes, proto=[proto] + ) + req = p2pd_pb.Request( + type=p2pd_pb.Request.STREAM_HANDLER, streamHandler=stream_handler_req + ) + await write_pbmsg(writer, req) + + resp = p2pd_pb.Response() # type: ignore + await read_pbmsg_safe(reader, resp) + writer.close() + raise_if_failed(resp) + + # if success, add the handler to the dict + self.handlers[proto] = handler_cb diff --git a/hivemind/p2p/p2p_daemon_bindings/datastructures.py b/hivemind/p2p/p2p_daemon_bindings/datastructures.py new file mode 100644 index 000000000..42351627c --- /dev/null +++ b/hivemind/p2p/p2p_daemon_bindings/datastructures.py @@ -0,0 +1,186 @@ +import hashlib +from typing import Union, List, Sequence, Any + +import base58 +import multihash + +from multiaddr import Multiaddr, protocols +from hivemind.proto import p2pd_pb2 + +from hivemind.p2p.p2p_daemon_bindings.keys import PublicKey + +# NOTE: On inlining... +# See: https://github.com/libp2p/specs/issues/138 +# NOTE: enabling to be interoperable w/ the Go implementation +ENABLE_INLINING = True +MAX_INLINE_KEY_LENGTH = 42 + +IDENTITY_MULTIHASH_CODE = 0x00 + +if ENABLE_INLINING: + + class IdentityHash: + _digest: bytes + + def __init__(self) -> None: + self._digest = bytearray() + + def update(self, input: bytes) -> None: + self._digest += input + + def digest(self) -> bytes: + return self._digest + + multihash.FuncReg.register( + IDENTITY_MULTIHASH_CODE, "identity", hash_new=lambda: IdentityHash() + ) + + +class ID: + _bytes: bytes + _xor_id: int = None + _b58_str: str = None + + def __init__(self, peer_id_bytes: bytes) -> None: + self._bytes = peer_id_bytes + + @property + def xor_id(self) -> int: + if not self._xor_id: + self._xor_id = int(sha256_digest(self._bytes).hex(), 16) + return self._xor_id + + def to_bytes(self) -> bytes: + return self._bytes + + def to_base58(self) -> str: + if not self._b58_str: + self._b58_str = base58.b58encode(self._bytes).decode() + return self._b58_str + + def __repr__(self) -> str: + return f"" + + __str__ = pretty = to_string = to_base58 + + def __eq__(self, other: object) -> bool: + if isinstance(other, str): + return self.to_base58() == other + elif isinstance(other, bytes): + return self._bytes == other + elif isinstance(other, ID): + return self._bytes == other._bytes + else: + return NotImplemented + + def __hash__(self) -> int: + return hash(self._bytes) + + @classmethod + def from_base58(cls, b58_encoded_peer_id_str: str) -> "ID": + peer_id_bytes = base58.b58decode(b58_encoded_peer_id_str) + pid = ID(peer_id_bytes) + return pid + + @classmethod + def from_pubkey(cls, key: PublicKey) -> "ID": + serialized_key = key.serialize() + algo = multihash.Func.sha2_256 + if ENABLE_INLINING and len(serialized_key) <= MAX_INLINE_KEY_LENGTH: + algo = IDENTITY_MULTIHASH_CODE + mh_digest = multihash.digest(serialized_key, algo) + return cls(mh_digest.encode()) + + +def sha256_digest(data: Union[str, bytes]) -> bytes: + if isinstance(data, str): + data = data.encode("utf8") + return hashlib.sha256(data).digest() + + +class StreamInfo: + peer_id: ID + addr: Multiaddr + proto: str + + def __init__(self, peer_id: ID, addr: Multiaddr, proto: str) -> None: + self.peer_id = peer_id + self.addr = addr + self.proto = proto + + def __repr__(self) -> str: + return ( + f"" + ) + + def to_pb(self) -> p2pd_pb2.StreamInfo: + pb_msg = p2pd_pb2.StreamInfo( + peer=self.peer_id.to_bytes(), addr=self.addr.to_bytes(), proto=self.proto + ) + return pb_msg + + @classmethod + def from_pb(cls, pb_msg: p2pd_pb2.StreamInfo) -> "StreamInfo": + stream_info = cls( + peer_id=ID(pb_msg.peer), addr=Multiaddr(pb_msg.addr), proto=pb_msg.proto + ) + return stream_info + + +class PeerInfoLibP2P: + peer_id: ID + addrs: List[Multiaddr] + + def __init__(self, peer_id: ID, addrs: Sequence[Multiaddr]) -> None: + self.peer_id = peer_id + self.addrs = list(addrs) + + def __eq__(self, other: Any) -> bool: + return ( + isinstance(other, PeerInfo) + and self.peer_id == other.peer_id + and self.addrs == other.addrs + ) + + +def info_from_p2p_addr(addr: Multiaddr) -> PeerInfoLibP2P: + if not addr: + raise InvalidAddrError("`addr` should not be `None`") + + parts = addr.split() + if not parts: + raise InvalidAddrError( + f"`parts`={parts} should at least have a protocol `P_P2P`" + ) + + p2p_part = parts[-1] + last_protocol_code = p2p_part.protocols()[0].code + if last_protocol_code != protocols.P_P2P: + raise InvalidAddrError( + f"The last protocol should be `P_P2P` instead of `{last_protocol_code}`" + ) + + # make sure the /p2p value parses as a peer.ID + peer_id_str: str = p2p_part.value_for_protocol(protocols.P_P2P) + peer_id: ID = ID.from_base58(peer_id_str) + + # we might have received just an / p2p part, which means there's no addr. + if len(parts) > 1: + addr = Multiaddr.join(*parts[:-1]) + + return PeerInfo(peer_id, [addr]) + + +class InvalidAddrError(ValueError): + pass + + +class PeerInfo(PeerInfoLibP2P): + @classmethod + def from_pb(cls, peer_info_pb: p2pd_pb2.PeerInfo) -> PeerInfoLibP2P: + peer_id = ID(peer_info_pb.id) + addrs = [Multiaddr(addr) for addr in peer_info_pb.addrs] + return PeerInfo(peer_id, addrs) + + def __str__(self): + return self.peer_id.pretty() + " " + ",".join(str(a) for a in self.addrs) diff --git a/hivemind/p2p/p2p_daemon_bindings/keys.py b/hivemind/p2p/p2p_daemon_bindings/keys.py new file mode 100644 index 000000000..01ec5ad55 --- /dev/null +++ b/hivemind/p2p/p2p_daemon_bindings/keys.py @@ -0,0 +1,91 @@ +from abc import ABC, abstractmethod +from dataclasses import dataclass +from enum import Enum, unique + +from hivemind.proto import crypto_pb2 as protobuf + + +@unique +class KeyType(Enum): + RSA = 0 + Ed25519 = 1 + Secp256k1 = 2 + ECDSA = 3 + ECC_P256 = 4 + + +class Key(ABC): + """A ``Key`` represents a cryptographic key.""" + + @abstractmethod + def to_bytes(self) -> bytes: + """Returns the byte representation of this key.""" + ... + + @abstractmethod + def get_type(self) -> KeyType: + """Returns the ``KeyType`` for ``self``.""" + ... + + def __eq__(self, other: object) -> bool: + if not isinstance(other, Key): + return NotImplemented + return self.to_bytes() == other.to_bytes() + + +class PublicKey(Key): + """A ``PublicKey`` represents a cryptographic public key.""" + + @abstractmethod + def verify(self, data: bytes, signature: bytes) -> bool: + """Verify that ``signature`` is the cryptographic signature of the hash + of ``data``.""" + ... + + def _serialize_to_protobuf(self) -> protobuf.PublicKey: + """Return the protobuf representation of this ``Key``.""" + key_type = self.get_type().value + data = self.to_bytes() + protobuf_key = protobuf.PublicKey(key_type=key_type, data=data) + return protobuf_key + + def serialize(self) -> bytes: + """Return the canonical serialization of this ``Key``.""" + return self._serialize_to_protobuf().SerializeToString() + + @classmethod + def deserialize_from_protobuf(cls, protobuf_data: bytes) -> protobuf.PublicKey: + return protobuf.PublicKey.FromString(protobuf_data) + + +class PrivateKey(Key): + """A ``PrivateKey`` represents a cryptographic private key.""" + + @abstractmethod + def sign(self, data: bytes) -> bytes: + ... + + @abstractmethod + def get_public_key(self) -> PublicKey: + ... + + def _serialize_to_protobuf(self) -> protobuf.PrivateKey: + """Return the protobuf representation of this ``Key``.""" + key_type = self.get_type().value + data = self.to_bytes() + protobuf_key = protobuf.PrivateKey(key_type=key_type, data=data) + return protobuf_key + + def serialize(self) -> bytes: + """Return the canonical serialization of this ``Key``.""" + return self._serialize_to_protobuf().SerializeToString() + + @classmethod + def deserialize_from_protobuf(cls, protobuf_data: bytes) -> protobuf.PrivateKey: + return protobuf.PrivateKey.FromString(protobuf_data) + + +@dataclass(frozen=True) +class KeyPair: + private_key: PrivateKey + public_key: PublicKey diff --git a/hivemind/p2p/p2p_daemon_bindings/p2pclient.py b/hivemind/p2p/p2p_daemon_bindings/p2pclient.py new file mode 100644 index 000000000..1dbcce960 --- /dev/null +++ b/hivemind/p2p/p2p_daemon_bindings/p2pclient.py @@ -0,0 +1,75 @@ +from typing import AsyncIterator, Iterable, Sequence, Tuple + +import asyncio +from hivemind.p2p.p2p_daemon_bindings.control import ControlClient, DaemonConnector, StreamHandler +from contextlib import asynccontextmanager +from multiaddr import Multiaddr +from hivemind.p2p.p2p_daemon_bindings.datastructures import PeerInfo, StreamInfo, ID + + +class Client: + control: ControlClient + + def __init__( + self, control_maddr: Multiaddr = None, listen_maddr: Multiaddr = None + ) -> None: + daemon_connector = DaemonConnector(control_maddr=control_maddr) + self.control = ControlClient( + daemon_connector=daemon_connector, listen_maddr=listen_maddr + ) + + @asynccontextmanager + async def listen(self) -> AsyncIterator["Client"]: + """ + Starts to listen incoming connections for handlers registered via stream_handler. + :return: + """ + async with self.control.listen(): + yield self + + async def identify(self) -> Tuple[ID, Tuple[Multiaddr, ...]]: + """ + Get current node peer id and list of addresses + """ + return await self.control.identify() + + async def connect(self, peer_id: ID, maddrs: Iterable[Multiaddr]) -> None: + """ + Connect to p2p node with specified addresses and peer id. + :peer_id: node peer id you want connect to + :maddrs: node multiaddresses you want connect to. Of course, it must be reachable. + """ + await self.control.connect(peer_id=peer_id, maddrs=maddrs) + + async def list_peers(self) -> Tuple[PeerInfo, ...]: + """ + Get list of peers that node connect to + """ + return await self.control.list_peers() + + async def disconnect(self, peer_id: ID) -> None: + """ + Disconnect from node with specified peer id + :peer_id: + """ + await self.control.disconnect(peer_id=peer_id) + + async def stream_open( + self, peer_id: ID, protocols: Sequence[str] + ) -> Tuple[StreamInfo, asyncio.StreamReader, asyncio.StreamWriter]: + """ + Open a stream to call other peer (with peer_id) handler for specified protocols + :peer_id: + :protocols: + :return: Returns tuple of stream info (info about connection to second peer) and reader/writer + """ + return await self.control.stream_open(peer_id=peer_id, protocols=protocols) + + async def stream_handler(self, proto: str, handler_cb: StreamHandler) -> None: + """ + Register a stream handler + :param proto: protocols that handler serves + :param handler_cb: handler callback + :return: + """ + await self.control.stream_handler(proto=proto, handler_cb=handler_cb) diff --git a/hivemind/p2p/p2p_daemon_bindings/utils.py b/hivemind/p2p/p2p_daemon_bindings/utils.py new file mode 100644 index 000000000..fa0e7cfd3 --- /dev/null +++ b/hivemind/p2p/p2p_daemon_bindings/utils.py @@ -0,0 +1,72 @@ +import asyncio + +from google.protobuf.message import Message as PBMessage + +from hivemind.proto import p2pd_pb2 as p2pd_pb + + +DEFAULT_MAX_BITS: int = 64 + + +class ControlFailure(Exception): + pass + + +class DispatchFailure(Exception): + pass + + +async def write_unsigned_varint( + stream: asyncio.StreamWriter, integer: int, max_bits: int = DEFAULT_MAX_BITS +) -> None: + max_int: int = 1 << max_bits + if integer < 0: + raise ValueError(f"negative integer: {integer}") + if integer >= max_int: + raise ValueError(f"integer too large: {integer}") + while True: + value: int = integer & 0x7F + integer >>= 7 + if integer != 0: + value |= 0x80 + byte = value.to_bytes(1, "big") + stream.write(byte) + if integer == 0: + break + + +async def read_unsigned_varint( + stream: asyncio.StreamReader, max_bits: int = DEFAULT_MAX_BITS +) -> int: + max_int: int = 1 << max_bits + iteration: int = 0 + result: int = 0 + has_next: bool = True + while has_next: + data = await stream.readexactly(1) + c = data[0] + value = c & 0x7F + result |= value << (iteration * 7) + has_next = (c & 0x80) != 0 + iteration += 1 + if result >= max_int: + raise ValueError(f"varint overflowed: {result}") + return result + + +def raise_if_failed(response: p2pd_pb.Response) -> None: + if response.type == p2pd_pb.Response.ERROR: + raise ControlFailure(f"connect failed. msg={response.error.msg}") + + +async def write_pbmsg(stream: asyncio.StreamWriter, pbmsg: PBMessage) -> None: + size = pbmsg.ByteSize() + await write_unsigned_varint(stream, size) + msg_bytes: bytes = pbmsg.SerializeToString() + stream.write(msg_bytes) + + +async def read_pbmsg_safe(stream: asyncio.StreamReader, pbmsg: PBMessage) -> None: + len_msg_bytes = await read_unsigned_varint(stream) + msg_bytes = await stream.readexactly(len_msg_bytes) + pbmsg.ParseFromString(msg_bytes) diff --git a/hivemind/proto/crypto.proto b/hivemind/proto/crypto.proto new file mode 100644 index 000000000..fe729a9d4 --- /dev/null +++ b/hivemind/proto/crypto.proto @@ -0,0 +1,20 @@ +syntax = "proto2"; + +package crypto.pb; + +enum KeyType { + RSA = 0; + Ed25519 = 1; + Secp256k1 = 2; + ECDSA = 3; +} + +message PublicKey { + required KeyType key_type = 1; + required bytes data = 2; +} + +message PrivateKey { + required KeyType key_type = 1; + required bytes data = 2; +} diff --git a/hivemind/proto/p2pd.proto b/hivemind/proto/p2pd.proto new file mode 100644 index 000000000..fec4a6ed7 --- /dev/null +++ b/hivemind/proto/p2pd.proto @@ -0,0 +1,158 @@ +syntax = "proto2"; + +package p2pclient.p2pd.pb; + +message Request { + enum Type { + IDENTIFY = 0; + CONNECT = 1; + STREAM_OPEN = 2; + STREAM_HANDLER = 3; + DHT = 4; + LIST_PEERS = 5; + CONNMANAGER = 6; + DISCONNECT = 7; + PUBSUB = 8; + } + + required Type type = 1; + + optional ConnectRequest connect = 2; + optional StreamOpenRequest streamOpen = 3; + optional StreamHandlerRequest streamHandler = 4; + optional DHTRequest dht = 5; + optional ConnManagerRequest connManager = 6; + optional DisconnectRequest disconnect = 7; + optional PSRequest pubsub = 8; +} + +message Response { + enum Type { + OK = 0; + ERROR = 1; + } + + required Type type = 1; + optional ErrorResponse error = 2; + optional StreamInfo streamInfo = 3; + optional IdentifyResponse identify = 4; + optional DHTResponse dht = 5; + repeated PeerInfo peers = 6; + optional PSResponse pubsub = 7; +} + +message IdentifyResponse { + required bytes id = 1; + repeated bytes addrs = 2; +} + +message ConnectRequest { + required bytes peer = 1; + repeated bytes addrs = 2; + optional int64 timeout = 3; +} + +message StreamOpenRequest { + required bytes peer = 1; + repeated string proto = 2; + optional int64 timeout = 3; +} + +message StreamHandlerRequest { + required bytes addr = 1; + repeated string proto = 2; +} + +message ErrorResponse { + required string msg = 1; +} + +message StreamInfo { + required bytes peer = 1; + required bytes addr = 2; + required string proto = 3; +} + +message DHTRequest { + enum Type { + FIND_PEER = 0; + FIND_PEERS_CONNECTED_TO_PEER = 1; + FIND_PROVIDERS = 2; + GET_CLOSEST_PEERS = 3; + GET_PUBLIC_KEY = 4; + GET_VALUE = 5; + SEARCH_VALUE = 6; + PUT_VALUE = 7; + PROVIDE = 8; + } + + required Type type = 1; + optional bytes peer = 2; + optional bytes cid = 3; + optional bytes key = 4; + optional bytes value = 5; + optional int32 count = 6; + optional int64 timeout = 7; +} + +message DHTResponse { + enum Type { + BEGIN = 0; + VALUE = 1; + END = 2; + } + + required Type type = 1; + optional PeerInfo peer = 2; + optional bytes value = 3; +} + +message PeerInfo { + required bytes id = 1; + repeated bytes addrs = 2; +} + +message ConnManagerRequest { + enum Type { + TAG_PEER = 0; + UNTAG_PEER = 1; + TRIM = 2; + } + + required Type type = 1; + + optional bytes peer = 2; + optional string tag = 3; + optional int64 weight = 4; +} + +message DisconnectRequest { + required bytes peer = 1; +} + +message PSRequest { + enum Type { + GET_TOPICS = 0; + LIST_PEERS = 1; + PUBLISH = 2; + SUBSCRIBE = 3; + } + + required Type type = 1; + optional string topic = 2; + optional bytes data = 3; +} + +message PSMessage { + optional bytes from_id = 1; + optional bytes data = 2; + optional bytes seqno = 3; + repeated string topicIDs = 4; + optional bytes signature = 5; + optional bytes key = 6; +} + +message PSResponse { + repeated string topics = 1; + repeated bytes peerIDs = 2; +} diff --git a/requirements.txt b/requirements.txt index 375a3ab60..36b418050 100644 --- a/requirements.txt +++ b/requirements.txt @@ -10,4 +10,6 @@ grpcio>=1.33.2 grpcio-tools>=1.33.2 protobuf>=3.12.2 configargparse>=1.2.3 +multiaddr==0.0.9 +pymultihash==0.8.2 cryptography>=3.4.6 diff --git a/tests/test_p2p_daemon.py b/tests/test_p2p_daemon.py index 06814d244..759b2eb2b 100644 --- a/tests/test_p2p_daemon.py +++ b/tests/test_p2p_daemon.py @@ -2,7 +2,7 @@ import multiprocessing as mp import subprocess -from libp2p.peer.id import ID +from hivemind.p2p.p2p_daemon_bindings.datastructures import ID import numpy as np import pytest @@ -86,16 +86,16 @@ async def ping_handler(request, context): await asyncio.sleep(1) libp2p_server_id = ID.from_base58(server.id) - stream_info, stream = await client._client.stream_open(libp2p_server_id, (handle_name,)) + stream_info, reader, writer = await client._client.stream_open(libp2p_server_id, (handle_name,)) - await P2P.send_raw_data(ping_request.SerializeToString(), stream) + await P2P.send_raw_data(ping_request.SerializeToString(), writer) if should_cancel: - await stream.close() + writer.close() await asyncio.sleep(1) assert handler_cancelled else: - result = await P2P.receive_protobuf(dht_pb2.PingResponse, stream) + result = await P2P.receive_protobuf(dht_pb2.PingResponse, reader) assert result == expected_response assert not handler_cancelled @@ -139,6 +139,25 @@ async def test_call_peer_single_process(test_input, handle, handler_name="handle assert not is_process_running(client_pid) +async def run_server(handler_name, server_side, client_side, response_received): + server = await P2P.create() + server_pid = server._child.pid + await server.add_stream_handler(handler_name, handle_square) + assert is_process_running(server_pid) + + server_side.send(server.id) + while response_received.value == 0: + await asyncio.sleep(0.5) + + await server.stop_listening() + server.__del__() + assert not is_process_running(server_pid) + + +def server_target(handler_name, server_side, client_side, response_received): + asyncio.run(run_server(handler_name, server_side, client_side, response_received)) + + @pytest.mark.asyncio async def test_call_peer_different_processes(): handler_name = "square" @@ -148,24 +167,7 @@ async def test_call_peer_different_processes(): response_received = mp.Value(np.ctypeslib.as_ctypes_type(np.int32)) response_received.value = 0 - async def run_server(): - server = await P2P.create() - server_pid = server._child.pid - await server.add_stream_handler(handler_name, handle_square) - assert is_process_running(server_pid) - - server_side.send(server.id) - while response_received.value == 0: - await asyncio.sleep(0.5) - - await server.stop_listening() - server.__del__() - assert not is_process_running(server_pid) - - def server_target(): - asyncio.run(run_server()) - - proc = mp.Process(target=server_target) + proc = mp.Process(target=server_target, args=(handler_name, server_side, client_side, response_received)) proc.start() client = await P2P.create() diff --git a/tests/test_p2p_daemon_bindings.py b/tests/test_p2p_daemon_bindings.py new file mode 100644 index 000000000..e9bc77213 --- /dev/null +++ b/tests/test_p2p_daemon_bindings.py @@ -0,0 +1,769 @@ +import asyncio +import functools +import io +import os +import subprocess +import time +import uuid +from contextlib import asynccontextmanager, AsyncExitStack +from typing import NamedTuple + +from google.protobuf.message import EncodeError +from multiaddr import Multiaddr, protocols + +import pytest + +from hivemind import find_open_port +from hivemind.p2p.p2p_daemon_bindings.control import parse_conn_protocol, DaemonConnector, ControlClient +from hivemind.p2p.p2p_daemon_bindings.p2pclient import Client +from hivemind.p2p.p2p_daemon_bindings.utils import ControlFailure, raise_if_failed, write_unsigned_varint, \ + read_unsigned_varint, read_pbmsg_safe, write_pbmsg +from hivemind.proto import p2pd_pb2 as p2pd_pb +from hivemind.p2p.p2p_daemon_bindings.datastructures import ID, StreamInfo, PeerInfo + + +def test_raise_if_failed_raises(): + resp = p2pd_pb.Response() + resp.type = p2pd_pb.Response.ERROR + with pytest.raises(ControlFailure): + raise_if_failed(resp) + + +def test_raise_if_failed_not_raises(): + resp = p2pd_pb.Response() + resp.type = p2pd_pb.Response.OK + raise_if_failed(resp) + + +pairs_int_varint_valid = ( + (0, b"\x00"), + (1, b"\x01"), + (128, b"\x80\x01"), + (2 ** 32, b"\x80\x80\x80\x80\x10"), + (2 ** 64 - 1, b"\xff\xff\xff\xff\xff\xff\xff\xff\xff\x01"), +) + +pairs_int_varint_overflow = ( + (2 ** 64, b"\x80\x80\x80\x80\x80\x80\x80\x80\x80\x02"), + (2 ** 64 + 1, b"\x81\x80\x80\x80\x80\x80\x80\x80\x80\x02"), + ( + 2 ** 128, + b"\x80\x80\x80\x80\x80\x80\x80\x80\x80\x80\x80\x80\x80\x80\x80\x80\x80\x80\x04", + ), +) + + +class MockReader(io.BytesIO): + async def readexactly(self, n): + await asyncio.sleep(0) + return self.read(n) + + +class MockWriter(io.BytesIO): + pass + + +class MockReaderWriter(MockReader, MockWriter): + pass + + +@pytest.mark.parametrize("integer, var_integer", pairs_int_varint_valid) +@pytest.mark.asyncio +async def test_write_unsigned_varint(integer, var_integer): + s = MockWriter() + await write_unsigned_varint(s, integer) + assert s.getvalue() == var_integer + + +@pytest.mark.parametrize("integer", tuple(i[0] for i in pairs_int_varint_overflow)) +@pytest.mark.asyncio +async def test_write_unsigned_varint_overflow(integer): + s = MockWriter() + with pytest.raises(ValueError): + await write_unsigned_varint(s, integer) + + +@pytest.mark.parametrize("integer", (-1, -(2 ** 32), -(2 ** 64), -(2 ** 128))) +@pytest.mark.asyncio +async def test_write_unsigned_varint_negative(integer): + s = MockWriter() + with pytest.raises(ValueError): + await write_unsigned_varint(s, integer) + + +@pytest.mark.parametrize("integer, var_integer", pairs_int_varint_valid) +@pytest.mark.asyncio +async def test_read_unsigned_varint(integer, var_integer): + s = MockReader(var_integer) + result = await read_unsigned_varint(s) + assert result == integer + + +@pytest.mark.parametrize("var_integer", tuple(i[1] for i in pairs_int_varint_overflow)) +@pytest.mark.asyncio +async def test_read_unsigned_varint_overflow(var_integer): + s = MockReader(var_integer) + with pytest.raises(ValueError): + await read_unsigned_varint(s) + + +@pytest.mark.parametrize("max_bits", (2, 31, 32, 63, 64, 127, 128)) +@pytest.mark.asyncio +async def test_read_write_unsigned_varint_max_bits_edge(max_bits): + """ + Test the edge with different `max_bits` + """ + for i in range(-3, 0): + integer = i + (2 ** max_bits) + s = MockReaderWriter() + await write_unsigned_varint(s, integer, max_bits=max_bits) + s.seek(0, 0) + result = await read_unsigned_varint(s, max_bits=max_bits) + assert integer == result + + +@pytest.fixture(scope="module") +def peer_id_string(): + return "QmS5QmciTXXnCUCyxud5eWFenUMAmvAWSDa1c7dvdXRMZ7" + + +@pytest.fixture(scope="module") +def peer_id_bytes(): + return b'\x12 7\x87F.[\xb5\xb1o\xe5*\xc7\xb9\xbb\x11:"Z|j2\x8ad\x1b\xa6\xe5= timeout: + # timeout + assert False, f"{coro_func} still failed after `{timeout}` seconds" + await asyncio.sleep(0.01) + + +class Daemon: + control_maddr = None + proc_daemon = None + log_filename = "" + f_log = None + closed = None + + def __init__( + self, control_maddr, enable_control, enable_connmgr, enable_dht, enable_pubsub + ): + self.control_maddr = control_maddr + self.enable_control = enable_control + self.enable_connmgr = enable_connmgr + self.enable_dht = enable_dht + self.enable_pubsub = enable_pubsub + self.is_closed = False + self._start_logging() + self._run() + + def _start_logging(self): + name_control_maddr = str(self.control_maddr).replace("/", "_").replace(".", "_") + self.log_filename = f"/tmp/log_p2pd{name_control_maddr}.txt" + self.f_log = open(self.log_filename, "wb") + + def _run(self): + cmd_list = ["hivemind/hivemind_cli/p2pd", f"-listen={str(self.control_maddr)}"] + cmd_list += [f"-hostAddrs=/ip4/127.0.0.1/tcp/{find_open_port()}"] + if self.enable_connmgr: + cmd_list += ["-connManager=true", "-connLo=1", "-connHi=2", "-connGrace=0"] + if self.enable_dht: + cmd_list += ["-dht=true"] + if self.enable_pubsub: + cmd_list += ["-pubsub=true", "-pubsubRouter=gossipsub"] + self.proc_daemon = subprocess.Popen( + cmd_list, stdout=self.f_log, stderr=self.f_log, bufsize=0 + ) + + async def wait_until_ready(self): + lines_head_pattern = (b"Control socket:", b"Peer ID:", b"Peer Addrs:") + lines_head_occurred = {line: False for line in lines_head_pattern} + + with open(self.log_filename, "rb") as f_log_read: + + async def read_from_daemon_and_check(): + line = f_log_read.readline() + for head_pattern in lines_head_occurred: + if line.startswith(head_pattern): + lines_head_occurred[head_pattern] = True + return all([value for _, value in lines_head_occurred.items()]) + + await try_until_success(read_from_daemon_and_check) + + # sleep for a while in case that the daemon haven't been ready after emitting these lines + await asyncio.sleep(0.1) + + def close(self): + if self.is_closed: + return + self.proc_daemon.terminate() + self.proc_daemon.wait() + self.f_log.close() + self.is_closed = True + + +class DaemonTuple(NamedTuple): + daemon: Daemon + client: Client + + +class ConnectionFailure(Exception): + pass + + +@asynccontextmanager +async def make_p2pd_pair_unix( + enable_control, enable_connmgr, enable_dht, enable_pubsub +): + name = str(uuid.uuid4())[:8] + control_maddr = Multiaddr(f"/unix/tmp/test_p2pd_control_{name}.sock") + listen_maddr = Multiaddr(f"/unix/tmp/test_p2pd_listen_{name}.sock") + # Remove the existing unix socket files if they are existing + try: + os.unlink(control_maddr.value_for_protocol(protocols.P_UNIX)) + except FileNotFoundError: + pass + try: + os.unlink(listen_maddr.value_for_protocol(protocols.P_UNIX)) + except FileNotFoundError: + pass + async with _make_p2pd_pair( + control_maddr=control_maddr, + listen_maddr=listen_maddr, + enable_control=enable_control, + enable_connmgr=enable_connmgr, + enable_dht=enable_dht, + enable_pubsub=enable_pubsub, + ) as pair: + yield pair + + +@asynccontextmanager +async def make_p2pd_pair_ip4(enable_control, enable_connmgr, enable_dht, enable_pubsub): + control_maddr = Multiaddr(f"/ip4/127.0.0.1/tcp/{find_open_port()}") + listen_maddr = Multiaddr(f"/ip4/127.0.0.1/tcp/{find_open_port()}") + async with _make_p2pd_pair( + control_maddr=control_maddr, + listen_maddr=listen_maddr, + enable_control=enable_control, + enable_connmgr=enable_connmgr, + enable_dht=enable_dht, + enable_pubsub=enable_pubsub, + ) as pair: + yield pair + + +@asynccontextmanager +async def _make_p2pd_pair( + control_maddr, + listen_maddr, + enable_control, + enable_connmgr, + enable_dht, + enable_pubsub, +): + p2pd = Daemon( + control_maddr=control_maddr, + enable_control=enable_control, + enable_connmgr=enable_connmgr, + enable_dht=enable_dht, + enable_pubsub=enable_pubsub, + ) + # wait for daemon ready + await p2pd.wait_until_ready() + client = Client(control_maddr=control_maddr, listen_maddr=listen_maddr) + try: + async with client.listen(): + yield DaemonTuple(daemon=p2pd, client=client) + finally: + if not p2pd.is_closed: + p2pd.close() + + +@pytest.fixture +async def p2pcs( + num_p2pds, + enable_control, + enable_connmgr, + enable_dht, + enable_pubsub, + func_make_p2pd_pair, +): + # TODO: Change back to gather style + async with AsyncExitStack() as stack: + p2pd_tuples = [ + await stack.enter_async_context( + func_make_p2pd_pair( + enable_control=enable_control, + enable_connmgr=enable_connmgr, + enable_dht=enable_dht, + enable_pubsub=enable_pubsub, + ) + ) + for _ in range(num_p2pds) + ] + yield tuple(p2pd_tuple.client for p2pd_tuple in p2pd_tuples) + + +@pytest.mark.parametrize( + "enable_control, func_make_p2pd_pair", ((True, make_p2pd_pair_unix),) +) +@pytest.mark.asyncio +async def test_client_identify_unix_socket(p2pcs): + await p2pcs[0].identify() + + +@pytest.mark.parametrize("enable_control", (True,)) +@pytest.mark.asyncio +async def test_client_identify(p2pcs): + await p2pcs[0].identify() + + +@pytest.mark.parametrize("enable_control", (True,)) +@pytest.mark.asyncio +async def test_client_connect_success(p2pcs): + peer_id_0, maddrs_0 = await p2pcs[0].identify() + peer_id_1, maddrs_1 = await p2pcs[1].identify() + await p2pcs[0].connect(peer_id_1, maddrs_1) + # test case: repeated connections + await p2pcs[1].connect(peer_id_0, maddrs_0) + + +@pytest.mark.parametrize("enable_control", (True,)) +@pytest.mark.asyncio +async def test_client_connect_failure(peer_id_random, p2pcs): + peer_id_1, maddrs_1 = await p2pcs[1].identify() + await p2pcs[0].identify() + # test case: `peer_id` mismatches + with pytest.raises(ControlFailure): + await p2pcs[0].connect(peer_id_random, maddrs_1) + # test case: empty maddrs + with pytest.raises(ControlFailure): + await p2pcs[0].connect(peer_id_1, []) + # test case: wrong maddrs + with pytest.raises(ControlFailure): + await p2pcs[0].connect(peer_id_1, [Multiaddr("/ip4/127.0.0.1/udp/0")]) + + +async def _check_connection(p2pd_tuple_0, p2pd_tuple_1): + peer_id_0, _ = await p2pd_tuple_0.identify() + peer_id_1, _ = await p2pd_tuple_1.identify() + peers_0 = [pinfo.peer_id for pinfo in await p2pd_tuple_0.list_peers()] + peers_1 = [pinfo.peer_id for pinfo in await p2pd_tuple_1.list_peers()] + return (peer_id_0 in peers_1) and (peer_id_1 in peers_0) + + +async def connect_safe(p2pd_tuple_0, p2pd_tuple_1): + peer_id_1, maddrs_1 = await p2pd_tuple_1.identify() + await p2pd_tuple_0.connect(peer_id_1, maddrs_1) + await try_until_success( + functools.partial( + _check_connection, p2pd_tuple_0=p2pd_tuple_0, p2pd_tuple_1=p2pd_tuple_1 + ) + ) + + +@pytest.mark.parametrize("enable_control", (True,)) +@pytest.mark.asyncio +async def test_connect_safe(p2pcs): + await connect_safe(p2pcs[0], p2pcs[1]) + + +@pytest.mark.parametrize("enable_control", (True,)) +@pytest.mark.asyncio +async def test_client_list_peers(p2pcs): + # test case: no peers + assert len(await p2pcs[0].list_peers()) == 0 + # test case: 1 peer + await connect_safe(p2pcs[0], p2pcs[1]) + assert len(await p2pcs[0].list_peers()) == 1 + assert len(await p2pcs[1].list_peers()) == 1 + # test case: one more peer + await connect_safe(p2pcs[0], p2pcs[2]) + assert len(await p2pcs[0].list_peers()) == 2 + assert len(await p2pcs[1].list_peers()) == 1 + assert len(await p2pcs[2].list_peers()) == 1 + + +@pytest.mark.parametrize("enable_control", (True,)) +@pytest.mark.asyncio +async def test_client_disconnect(peer_id_random, p2pcs): + # test case: disconnect a peer without connections + await p2pcs[1].disconnect(peer_id_random) + # test case: disconnect + peer_id_0, _ = await p2pcs[0].identify() + await connect_safe(p2pcs[0], p2pcs[1]) + assert len(await p2pcs[0].list_peers()) == 1 + assert len(await p2pcs[1].list_peers()) == 1 + await p2pcs[1].disconnect(peer_id_0) + assert len(await p2pcs[0].list_peers()) == 0 + assert len(await p2pcs[1].list_peers()) == 0 + # test case: disconnect twice + await p2pcs[1].disconnect(peer_id_0) + assert len(await p2pcs[0].list_peers()) == 0 + assert len(await p2pcs[1].list_peers()) == 0 + + +@pytest.mark.parametrize("enable_control", (True,)) +@pytest.mark.asyncio +async def test_client_stream_open_success(p2pcs): + peer_id_1, maddrs_1 = await p2pcs[1].identify() + await connect_safe(p2pcs[0], p2pcs[1]) + + proto = "123" + + async def handle_proto(stream_info, reader, writer): + await reader.readexactly(1) + + await p2pcs[1].stream_handler(proto, handle_proto) + + # test case: normal + stream_info, reader, writer = await p2pcs[0].stream_open(peer_id_1, (proto,)) + assert stream_info.peer_id == peer_id_1 + assert stream_info.addr in maddrs_1 + assert stream_info.proto == "123" + writer.close() + + # test case: open with multiple protocols + stream_info, reader, writer = await p2pcs[0].stream_open( + peer_id_1, (proto, "another_protocol") + ) + assert stream_info.peer_id == peer_id_1 + assert stream_info.addr in maddrs_1 + assert stream_info.proto == "123" + writer.close() + + +@pytest.mark.parametrize("enable_control", (True,)) +@pytest.mark.asyncio +async def test_client_stream_open_failure(p2pcs): + peer_id_1, _ = await p2pcs[1].identify() + await connect_safe(p2pcs[0], p2pcs[1]) + + proto = "123" + + # test case: `stream_open` to a peer who didn't register the protocol + with pytest.raises(ControlFailure): + await p2pcs[0].stream_open(peer_id_1, (proto,)) + + # test case: `stream_open` to a peer for a non-registered protocol + async def handle_proto(stream_info, reader, writer): + pass + + await p2pcs[1].stream_handler(proto, handle_proto) + with pytest.raises(ControlFailure): + await p2pcs[0].stream_open(peer_id_1, ("another_protocol",)) + + +@pytest.mark.parametrize("enable_control", (True,)) +@pytest.mark.asyncio +async def test_client_stream_handler_success(p2pcs): + peer_id_1, _ = await p2pcs[1].identify() + await connect_safe(p2pcs[0], p2pcs[1]) + + proto = "protocol123" + bytes_to_send = b"yoyoyoyoyog" + # event for this test function to wait until the handler function receiving the incoming data + event_handler_finished = asyncio.Event() + + async def handle_proto(stream_info, reader, writer): + nonlocal event_handler_finished + bytes_received = await reader.readexactly(len(bytes_to_send)) + assert bytes_received == bytes_to_send + event_handler_finished.set() + + await p2pcs[1].stream_handler(proto, handle_proto) + assert proto in p2pcs[1].control.handlers + assert handle_proto == p2pcs[1].control.handlers[proto] + + # test case: test the stream handler `handle_proto` + + _, reader, writer = await p2pcs[0].stream_open(peer_id_1, (proto,)) + + # wait until the handler function starts blocking waiting for the data + # because we haven't sent the data, we know the handler function must still blocking waiting. + # get the task of the protocol handler + writer.write(bytes_to_send) + + # wait for the handler to finish + writer.close() + + await event_handler_finished.wait() + + # test case: two streams to different handlers respectively + another_proto = "another_protocol123" + another_bytes_to_send = b"456" + event_another_proto = asyncio.Event() + + async def handle_another_proto(stream_info, reader, writer): + event_another_proto.set() + bytes_received = await reader.readexactly(len(another_bytes_to_send)) + assert bytes_received == another_bytes_to_send + + await p2pcs[1].stream_handler(another_proto, handle_another_proto) + assert another_proto in p2pcs[1].control.handlers + assert handle_another_proto == p2pcs[1].control.handlers[another_proto] + + _, reader, writer = await p2pcs[0].stream_open(peer_id_1, (another_proto,)) + await event_another_proto.wait() + + # we know at this moment the handler must still blocking wait + + writer.write(another_bytes_to_send) + + writer.close() + + # test case: registering twice can override the previous registration + event_third = asyncio.Event() + + async def handler_third(stream_info, reader, writer): + event_third.set() + + await p2pcs[1].stream_handler(another_proto, handler_third) + assert another_proto in p2pcs[1].control.handlers + # ensure the handler is override + assert handler_third == p2pcs[1].control.handlers[another_proto] + + await p2pcs[0].stream_open(peer_id_1, (another_proto,)) + # ensure the overriding handler is called when the protocol is opened a stream + await event_third.wait() + + +@pytest.mark.parametrize("enable_control", (True,)) +@pytest.mark.asyncio +async def test_client_stream_handler_failure(p2pcs): + peer_id_1, _ = await p2pcs[1].identify() + await connect_safe(p2pcs[0], p2pcs[1]) + + proto = "123" + + # test case: registered a wrong protocol name + async def handle_proto_correct_params(stream_info, stream): + pass + + await p2pcs[1].stream_handler("another_protocol", handle_proto_correct_params) + with pytest.raises(ControlFailure): + await p2pcs[0].stream_open(peer_id_1, (proto,)) From 2dbee5964cb77b7b7cf74403dbbba034c54b5ca6 Mon Sep 17 00:00:00 2001 From: MaximKsh Date: Wed, 7 Apr 2021 08:25:35 +0300 Subject: [PATCH 12/81] #204 P2P replica mode (#205) * #204 P2P replica mode * #204 rename replica->replicate --- hivemind/p2p/p2p_daemon.py | 14 +++++ tests/test_p2p_daemon.py | 115 +++++++++++++++++++++++++++++++------ 2 files changed, 111 insertions(+), 18 deletions(-) diff --git a/hivemind/p2p/p2p_daemon.py b/hivemind/p2p/p2p_daemon.py index af3185a44..6a05fd6d9 100644 --- a/hivemind/p2p/p2p_daemon.py +++ b/hivemind/p2p/p2p_daemon.py @@ -73,6 +73,20 @@ async def create(cls, *args, quic=1, tls=1, conn_manager=1, dht_client=1, break return self + @classmethod + async def replicate(cls, daemon_listen_port: int, host_port: int): + self = cls() + # There is no child under control + # Use external already running p2pd + self._child = None + self._assign_daemon_ports(host_port, daemon_listen_port) + self._client_listen_port = find_open_port() + self._client = p2pclient.Client( + Multiaddr(f'/ip4/127.0.0.1/tcp/{self._daemon_listen_port}'), + Multiaddr(f'/ip4/127.0.0.1/tcp/{self._client_listen_port}')) + await self._identify_client(0) + return self + def _initialize(self, proc_args: tp.List[str]) -> None: proc_args = copy.deepcopy(proc_args) proc_args.extend(self._make_process_args( diff --git a/tests/test_p2p_daemon.py b/tests/test_p2p_daemon.py index 759b2eb2b..5c1c9f211 100644 --- a/tests/test_p2p_daemon.py +++ b/tests/test_p2p_daemon.py @@ -1,6 +1,7 @@ import asyncio import multiprocessing as mp import subprocess +from functools import partial from hivemind.p2p.p2p_daemon_bindings.datastructures import ID @@ -27,6 +28,10 @@ def is_process_running(pid: int) -> bool: return subprocess.check_output(cmd, shell=True).decode('utf-8').strip() == RUNNING +async def replicate_if_needed(p2p: P2P, replicate: bool): + return await P2P.replicate(p2p._daemon_listen_port, p2p._host_port) if replicate else p2p + + @pytest.mark.asyncio async def test_daemon_killed_on_del(): p2p_daemon = await P2P.create() @@ -38,6 +43,21 @@ async def test_daemon_killed_on_del(): assert not is_process_running(child_pid) +@pytest.mark.asyncio +async def test_daemon_replica_does_not_affect_primary(): + p2p_daemon = await P2P.create() + p2p_replica = await P2P.replicate(p2p_daemon._daemon_listen_port, p2p_daemon._host_port) + + child_pid = p2p_daemon._child.pid + assert is_process_running(child_pid) + + p2p_replica.__del__() + assert is_process_running(child_pid) + + p2p_daemon.__del__() + assert not is_process_running(child_pid) + + def handle_square(x): return x ** 2 @@ -50,10 +70,15 @@ def handle_add(args): @pytest.mark.parametrize( - 'should_cancel', [True, False] + 'should_cancel,replicate', [ + (True, False), + (True, True), + (False, False), + (False, True), + ] ) @pytest.mark.asyncio -async def test_call_unary_handler(should_cancel, handle_name="handle"): +async def test_call_unary_handler(should_cancel, replicate, handle_name="handle"): handler_cancelled = False async def ping_handler(request, context): @@ -67,14 +92,16 @@ async def ping_handler(request, context): node_id=context.ours_id.encode(), rpc_port=context.ours_port), sender_endpoint=context.handle_name, available=True) - server = await P2P.create() - server_pid = server._child.pid + server_primary = await P2P.create() + server = await replicate_if_needed(server_primary, replicate) + server_pid = server_primary._child.pid await server.add_unary_handler(handle_name, ping_handler, dht_pb2.PingRequest, dht_pb2.PingResponse) assert is_process_running(server_pid) - client = await P2P.create() - client_pid = client._child.pid + client_primary = await P2P.create() + client = await replicate_if_needed(client_primary, replicate) + client_pid = client_primary._child.pid assert is_process_running(client_pid) ping_request = dht_pb2.PingRequest( @@ -100,10 +127,10 @@ async def ping_handler(request, context): assert not handler_cancelled await server.stop_listening() - server.__del__() + server_primary.__del__() assert not is_process_running(server_pid) - client.__del__() + client_primary.__del__() assert not is_process_running(client_pid) @@ -131,7 +158,6 @@ async def test_call_peer_single_process(test_input, handle, handler_name="handle result = await client.call_peer_handler(server.id, handler_name, test_input) assert result == handle(test_input) - await server.stop_listening() server.__del__() assert not is_process_running(server_pid) @@ -188,30 +214,83 @@ async def test_call_peer_different_processes(): @pytest.mark.parametrize( - "test_input,handle", + "test_input,handle,replicate", [ - pytest.param(np.random.randn(2, 3), handle_square, id="square"), - pytest.param([np.random.randn(2, 3), np.random.randn(2, 3)], handle_add, id="add"), + pytest.param(np.random.randn(2, 3), handle_square, False, id="square_primary"), + pytest.param(np.random.randn(2, 3), handle_square, True, id="square_replica"), + pytest.param([np.random.randn(2, 3), np.random.randn(2, 3)], handle_add, False, id="add_primary"), + pytest.param([np.random.randn(2, 3), np.random.randn(2, 3)], handle_add, True, id="add_replica"), ] ) @pytest.mark.asyncio -async def test_call_peer_numpy(test_input, handle, handler_name="handle"): - server = await P2P.create() +async def test_call_peer_numpy(test_input, handle, replicate, handler_name="handle"): + server_primary = await P2P.create() + server = await replicate_if_needed(server_primary, replicate) await server.add_stream_handler(handler_name, handle) - client = await P2P.create() + client_primary = await P2P.create() + client = await replicate_if_needed(client_primary, replicate) await asyncio.sleep(1) result = await client.call_peer_handler(server.id, handler_name, test_input) assert np.allclose(result, handle(test_input)) +@pytest.mark.parametrize( + "replicate", + [ + pytest.param(False, id="primary"), + pytest.param(True, id="replica"), + ] +) @pytest.mark.asyncio -async def test_call_peer_error(handler_name="handle"): - server = await P2P.create() +async def test_call_peer_error(replicate, handler_name="handle"): + server_primary = await P2P.create() + server = await replicate_if_needed(server_primary, replicate) await server.add_stream_handler(handler_name, handle_add) - client = await P2P.create() + client_primary = await P2P.create() + client = await replicate_if_needed(client_primary, replicate) await asyncio.sleep(1) result = await client.call_peer_handler(server.id, handler_name, [np.zeros((2, 3)), np.zeros((3, 2))]) assert type(result) == ValueError + + +@pytest.mark.asyncio +async def test_handlers_on_different_replicas(handler_name="handle"): + def handler(arg, key): + return key + + server_primary = await P2P.create() + server_id = server_primary.id + await server_primary.add_stream_handler(handler_name, partial(handler, key="primary")) + + server_replica1 = await replicate_if_needed(server_primary, True) + await server_replica1.add_stream_handler(handler_name + "1", partial(handler, key="replica1")) + + server_replica2 = await replicate_if_needed(server_primary, True) + await server_replica2.add_stream_handler(handler_name + "2", partial(handler, key="replica2")) + + client = await P2P.create() + await asyncio.sleep(1) + result = await client.call_peer_handler(server_id, handler_name, "") + assert result == "primary" + + result = await client.call_peer_handler(server_id, handler_name + "1", "") + assert result == "replica1" + + result = await client.call_peer_handler(server_id, handler_name + "2", "") + assert result == "replica2" + + await server_replica1.stop_listening() + await server_replica2.stop_listening() + + # Primary does not handle replicas protocols + with pytest.raises(P2P.IncompleteRead): + await client.call_peer_handler(server_id, handler_name + "1", "") + with pytest.raises(P2P.IncompleteRead): + await client.call_peer_handler(server_id, handler_name + "2", "") + + await server_primary.stop_listening() + server_primary.__del__() + client.__del__() From 133a6fcacc3bde0b98ec05e0b03abcbbc555d4da Mon Sep 17 00:00:00 2001 From: Denis Mazur Date: Mon, 1 Mar 2021 23:17:11 +0300 Subject: [PATCH 13/81] Compiling libp2p daemon on setup (#153) * add setup.py prototype * refactor --- setup.py | 56 +++++++++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 55 insertions(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 53cb6b77d..187d3ea43 100644 --- a/setup.py +++ b/setup.py @@ -2,6 +2,10 @@ import glob import os import re +import subprocess +import urllib.request +import tarfile +import tempfile from pkg_resources import parse_requirements from setuptools import setup, find_packages @@ -9,6 +13,19 @@ from setuptools.command.install import install +class cd: + """Context manager for changing the current working directory""" + def __init__(self, newPath): + self.newPath = os.path.expanduser(newPath) + + def __enter__(self): + self.savedPath = os.getcwd() + os.chdir(self.newPath) + + def __exit__(self, etype, value, traceback): + os.chdir(self.savedPath) + + def proto_compile(output_path): import grpc_tools.protoc @@ -28,6 +45,38 @@ def proto_compile(output_path): file.truncate() +def install_libp2p_daemon(): + # check go version: + try: + proc = subprocess.Popen(['go', 'version'], + stdout=subprocess.PIPE) + result, _ = proc.communicate() + result = result.decode('ascii', 'replace') + _, _, version, _ = result.split(' ') + version = version.lstrip('go') + + if version < "1.13": + raise EnvironmentError(f'newer version of go required: must be >= 1.13, found {version}') + + except FileNotFoundError: + raise FileNotFoundError('could not find golang installation') + + with tempfile.TemporaryDirectory() as tempdir: + url = 'https://github.com/libp2p/go-libp2p-daemon/archive/master.tar.gz' + dest = os.path.join(tempdir, 'libp2p-daemin.tar.gz') + urllib.request.urlretrieve(url, os.path.join(tempdir, dest)) + + tar = tarfile.open(dest, 'r:gz') + tar.extractall(tempdir) + tar.close() + + with cd(os.path.join(tempdir, 'go-libp2p-daemon-master')): + status = os.system('go install ./...') + if status: + raise RuntimeError('Failed to build or install libp2p-daemon:'\ + f' exited with status code :{status}') + + class ProtoCompileInstall(install): def run(self): proto_compile(os.path.join(self.build_lib, 'hivemind', 'proto')) @@ -40,6 +89,11 @@ def run(self): super().run() +class LibP2PInstall(install): + def run(self): + install_libp2p_daemon() + + here = os.path.abspath(os.path.dirname(__file__)) with open('requirements.txt') as requirements_file: @@ -63,7 +117,7 @@ def run(self): setup( name='hivemind', version=version_string, - cmdclass={'install': ProtoCompileInstall, 'develop': ProtoCompileDevelop}, + cmdclass={'install': ProtoCompileInstall, 'develop': ProtoCompileDevelop, 'libp2p': LibP2PInstall}, description='Decentralized deep learning in PyTorch', long_description='Decentralized deep learning in PyTorch. Built to train giant models on ' 'thousands of volunteers across the world.', From d3d1a9f40b91f2ef1ca2c74cd814f737048e3870 Mon Sep 17 00:00:00 2001 From: Ilya <37004806+skobellev@users.noreply.github.com> Date: Wed, 3 Mar 2021 12:25:17 +0300 Subject: [PATCH 14/81] feat: add p2p daemon (#164) * Add p2p daemon * Test p2p daemon exits correctly * Impose restriction on elapsed time Co-authored-by: Ilya Kobelev --- hivemind/__init__.py | 1 + hivemind/p2p/__init__.py | 1 + hivemind/p2p/p2p_daemon.py | 45 +++++++++++++++++++++++++++++++ tests/test_p2p_daemon.py | 55 ++++++++++++++++++++++++++++++++++++++ 4 files changed, 102 insertions(+) create mode 100644 hivemind/p2p/__init__.py create mode 100644 hivemind/p2p/p2p_daemon.py create mode 100644 tests/test_p2p_daemon.py diff --git a/hivemind/__init__.py b/hivemind/__init__.py index d058fc774..2d01ef538 100644 --- a/hivemind/__init__.py +++ b/hivemind/__init__.py @@ -1,5 +1,6 @@ from hivemind.client import * from hivemind.dht import * +from hivemind.p2p import * from hivemind.server import * from hivemind.utils import * from hivemind.optim import * diff --git a/hivemind/p2p/__init__.py b/hivemind/p2p/__init__.py new file mode 100644 index 000000000..6bae0b8bd --- /dev/null +++ b/hivemind/p2p/__init__.py @@ -0,0 +1 @@ +from hivemind.p2p.p2p_daemon import P2P diff --git a/hivemind/p2p/p2p_daemon.py b/hivemind/p2p/p2p_daemon.py new file mode 100644 index 000000000..3083c70e5 --- /dev/null +++ b/hivemind/p2p/p2p_daemon.py @@ -0,0 +1,45 @@ +import subprocess +import typing as tp + + +class P2P(object): + """ + Forks a child process and executes p2pd command with given arguments. + Sends SIGKILL to the child in destructor and on exit from contextmanager. + """ + + LIBP2P_CMD = 'p2pd' + + def __init__(self, *args, **kwargs): + self._child = subprocess.Popen(args=self._make_process_args(args, kwargs)) + try: + stdout, stderr = self._child.communicate(timeout=0.2) + except subprocess.TimeoutExpired: + pass + else: + raise RuntimeError(f'p2p daemon exited with stderr: {stderr}') + + def __enter__(self): + return self._child + + def __exit__(self, exc_type, exc_val, exc_tb): + self._kill_child() + + def __del__(self): + self._kill_child() + + def _kill_child(self): + if self._child.poll() is None: + self._child.kill() + self._child.wait() + + def _make_process_args(self, args: tp.Tuple[tp.Any], + kwargs: tp.Dict[str, tp.Any]) -> tp.List[str]: + proc_args = [self.LIBP2P_CMD] + proc_args.extend( + str(entry) for entry in args + ) + proc_args.extend( + f'-{key}={str(value)}' for key, value in kwargs.items() + ) + return proc_args diff --git a/tests/test_p2p_daemon.py b/tests/test_p2p_daemon.py new file mode 100644 index 000000000..ac57e9e2f --- /dev/null +++ b/tests/test_p2p_daemon.py @@ -0,0 +1,55 @@ +import subprocess +from time import perf_counter + +import pytest + +import hivemind.p2p +from hivemind.p2p import P2P + +RUNNING = 'running' +NOT_RUNNING = 'not running' +CHECK_PID_CMD = ''' +if ps -p {0} > /dev/null; +then + echo "{1}" +else + echo "{2}" +fi +''' + + +def is_process_running(pid: int) -> bool: + cmd = CHECK_PID_CMD.format(pid, RUNNING, NOT_RUNNING) + return subprocess.check_output(cmd, shell=True).decode('utf-8').strip() == RUNNING + + +@pytest.fixture() +def mock_p2p_class(): + P2P.LIBP2P_CMD = "sleep" + + +def test_daemon_killed_on_del(mock_p2p_class): + start = perf_counter() + p2p_daemon = P2P('10s') + + child_pid = p2p_daemon._child.pid + assert is_process_running(child_pid) + + del p2p_daemon + assert not is_process_running(child_pid) + assert perf_counter() - start < 1 + + +def test_daemon_killed_on_exit(mock_p2p_class): + start = perf_counter() + with P2P('10s') as daemon: + child_pid = daemon.pid + assert is_process_running(child_pid) + + assert not is_process_running(child_pid) + assert perf_counter() - start < 1 + + +def test_daemon_raises_on_faulty_args(): + with pytest.raises(RuntimeError): + P2P(faulty='argument') From b2ad1f19c58d0dd8518704dd41ad94ebe61caa36 Mon Sep 17 00:00:00 2001 From: Denis Mazur Date: Fri, 5 Mar 2021 05:19:39 +0300 Subject: [PATCH 15/81] compare golang versions using packaging.version --- setup.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/setup.py b/setup.py index 187d3ea43..f67235750 100644 --- a/setup.py +++ b/setup.py @@ -7,6 +7,7 @@ import tarfile import tempfile +from packaging import version from pkg_resources import parse_requirements from setuptools import setup, find_packages from setuptools.command.develop import develop @@ -52,10 +53,10 @@ def install_libp2p_daemon(): stdout=subprocess.PIPE) result, _ = proc.communicate() result = result.decode('ascii', 'replace') - _, _, version, _ = result.split(' ') - version = version.lstrip('go') + _, _, v, _ = result.split(' ') + v = v.lstrip('go') - if version < "1.13": + if version.parse(v) < version.parse("1.13"): raise EnvironmentError(f'newer version of go required: must be >= 1.13, found {version}') except FileNotFoundError: From de66acaf8dea7db9876ede9aca45d1f105167289 Mon Sep 17 00:00:00 2001 From: Denis Mazur Date: Tue, 9 Mar 2021 20:15:17 +0300 Subject: [PATCH 16/81] fix typo Co-authored-by: justheuristic --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index f67235750..d858d777c 100644 --- a/setup.py +++ b/setup.py @@ -64,7 +64,7 @@ def install_libp2p_daemon(): with tempfile.TemporaryDirectory() as tempdir: url = 'https://github.com/libp2p/go-libp2p-daemon/archive/master.tar.gz' - dest = os.path.join(tempdir, 'libp2p-daemin.tar.gz') + dest = os.path.join(tempdir, 'libp2p-daemon.tar.gz') urllib.request.urlretrieve(url, os.path.join(tempdir, dest)) tar = tarfile.open(dest, 'r:gz') From fa6699bd4258ecb42020380c8cd7e6edca989e56 Mon Sep 17 00:00:00 2001 From: Denis Mazur Date: Wed, 10 Mar 2021 01:31:19 +0300 Subject: [PATCH 17/81] move p2pd executable to hivemind/hivemind_cli --- setup.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/setup.py b/setup.py index d858d777c..36630cd27 100644 --- a/setup.py +++ b/setup.py @@ -14,6 +14,9 @@ from setuptools.command.install import install +here = os.path.abspath(os.path.dirname(__file__)) + + class cd: """Context manager for changing the current working directory""" def __init__(self, newPath): @@ -71,8 +74,8 @@ def install_libp2p_daemon(): tar.extractall(tempdir) tar.close() - with cd(os.path.join(tempdir, 'go-libp2p-daemon-master')): - status = os.system('go install ./...') + with cd(os.path.join(tempdir, 'go-libp2p-daemon-master', 'p2pd')): + status = os.system(f'go build -o {os.path.join(here, "hivemind/hivemind_cli", "p2pd")}') if status: raise RuntimeError('Failed to build or install libp2p-daemon:'\ f' exited with status code :{status}') @@ -95,7 +98,6 @@ def run(self): install_libp2p_daemon() -here = os.path.abspath(os.path.dirname(__file__)) with open('requirements.txt') as requirements_file: install_requires = list(map(str, parse_requirements(requirements_file))) From c0c16eb233ac64d02e11a6a511493a7d0e365d90 Mon Sep 17 00:00:00 2001 From: Denis Mazur Date: Wed, 10 Mar 2021 20:58:26 +0300 Subject: [PATCH 18/81] Rebase master onto libp2p (#179) * copytree implementation for py37 compatibility (#162) * copytree implementation for py37 compatibility * Running tests for python3.7 * Increment version * Python3.7 notions * Remove pickle.loads in averager (#160) * Security update: remove pickle.loads in averager * add py37 to circleci config Co-authored-by: Alexander Borzunov Co-authored-by: Max Ryabinin * Support edge cases for DHT key/subkey/value, add tests, update .gitignore for pb2 (#167) * fix bug with subkey equals zero * add autogenerated protobuf files to .gitignore * test store and get "tricky" values in dht * Fix the remaining tests for py37 (#166) * DecentralizedAverager is now compatible with python37's acyncio exception * the problem was: grpc.aio with python37 raised concurrent.futures.CancelledError in some cases; * we relied on isinstance(asyncio.CancelledError, Exception) == False * but isinstance(concurrent.futures.CancelledError, Exception) == True * DecentralizedAverager now shuts down if dereferenced in the main process * though it won't shutdown if dereferenced in forks for obvious reasons * HIVEMIND_THREADS now actually works * test_averaging now shuts down dht and averager instances to avoid leaking processes Co-authored-by: Max Ryabinin Co-authored-by: Max Ryabinin * Move Averager metadata serialization out of user scope (#168) * move metadata serialization outside user scope * test_overcrowded: reduce the default number of peers * Handle edge cases in DecentralizedAverager (#171) * move metadata serialization outside user scope * retry averager.step on network errors * raise AllreduceException on partial tensor * test split/combine tensors, combine corrupted stream Co-authored-by: Max Ryabinin * Fix a typo in quickstart.md (#174) * Serialize DHTID source with msgpack (#172) * Change DHTID serializer * Remove unused serializers * Add msgpack tuple serialization * Move CLI server launch script to hivemind/hivemind_cli (#173) * Cast environment variables to correct types * Compiling libp2p daemon on setup (#153) * add setup.py prototype * refactor * feat: add p2p daemon (#164) * Add p2p daemon * Test p2p daemon exits correctly * Impose restriction on elapsed time Co-authored-by: Ilya Kobelev * compare golang versions using packaging.version * fix typo Co-authored-by: justheuristic * move p2pd executable to hivemind/hivemind_cli Co-authored-by: Alexey Bukhtiyarov Co-authored-by: justheuristic Co-authored-by: Alexander Borzunov Co-authored-by: Max Ryabinin Co-authored-by: Michael Diskin Co-authored-by: romakail <36082689+romakail@users.noreply.github.com> Co-authored-by: Ilya <37004806+skobellev@users.noreply.github.com> Co-authored-by: Ilya Kobelev --- hivemind/client/averaging/matchmaking.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/hivemind/client/averaging/matchmaking.py b/hivemind/client/averaging/matchmaking.py index de20ebc02..b06318389 100644 --- a/hivemind/client/averaging/matchmaking.py +++ b/hivemind/client/averaging/matchmaking.py @@ -462,5 +462,13 @@ async def _declare_averager_periodically(self, key_manager: GroupKeyManager): looking_for_group=False) +def compute_schema_hash(tensors: Sequence[torch.Tensor]) -> bytes: + """ A hash that describes follower's tensor shapes, dtypes, devices, but not the actual values """ + schema_dicts = [{field_name: str(field_value) + for field_name, field_value in asdict(TensorDescriptor.from_tensor(tensor)).items()} + for tensor in tensors] + return DHTID.generate(source=schema_dicts).to_bytes() + + class MatchmakingException(Exception): """ An internal exception that marks undesired edge cases during averaging """ From 69ba66091a8df113f793e74e879e98a28627251c Mon Sep 17 00:00:00 2001 From: Denis Mazur Date: Fri, 19 Mar 2021 22:13:57 +0300 Subject: [PATCH 19/81] Fix LibP2P-Daemon installation in setup.py (#186) * Rename ProtoCompileInstall and ProtoCompileDevelop to Install and Develop * Install LibP2P-Daemon on setup install and setup develop * Install Golang in Circle CI builds * Add P2PD binary to gitignore --- .circleci/config.yml | 18 ++++++++++++ .gitignore | 3 ++ setup.py | 67 +++++++++++++++++++++++--------------------- 3 files changed, 56 insertions(+), 32 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index b1ab5978e..daaac9b6a 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -1,5 +1,10 @@ version: 2.1 +parameters: + go-version: + type: string + default: 1.16.2 + jobs: build-and-test-py37: docker: @@ -9,6 +14,11 @@ jobs: - restore_cache: keys: - py37-v1-{{ checksum "requirements.txt" }}-{{ checksum "requirements-dev.txt" }} + - v1-{{ checksum "requirements.txt" }}-{{ checksum "requirements-dev.txt" }} + - run: | + wget https://golang.org/dl/go<< pipeline.parameters.go-version >>.linux-amd64.tar.gz -O go.tar.gz + tar -C ~/ -xzf go.tar.gz + echo "export PATH=~/go/bin:$PATH" >> $BASH_ENV - run: pip install -r requirements.txt - run: pip install -r requirements-dev.txt - save_cache: @@ -29,6 +39,10 @@ jobs: - restore_cache: keys: - py38-v1-{{ checksum "requirements.txt" }}-{{ checksum "requirements-dev.txt" }} + - run: | + wget https://golang.org/dl/go<< pipeline.parameters.go-version >>.linux-amd64.tar.gz -O go.tar.gz + tar -C ~/ -xzf go.tar.gz + echo "export PATH=~/go/bin:$PATH" >> $BASH_ENV - run: pip install -r requirements.txt - run: pip install -r requirements-dev.txt - save_cache: @@ -49,6 +63,10 @@ jobs: - restore_cache: keys: - py39-v1-{{ checksum "requirements.txt" }}-{{ checksum "requirements-dev.txt" }} + - run: | + wget https://golang.org/dl/go<< pipeline.parameters.go-version >>.linux-amd64.tar.gz -O go.tar.gz + tar -C ~/ -xzf go.tar.gz + echo "export PATH=~/go/bin:$PATH" >> $BASH_ENV - run: pip install -r requirements.txt - run: pip install -r requirements-dev.txt - save_cache: diff --git a/.gitignore b/.gitignore index 965aa8972..61e239d1c 100644 --- a/.gitignore +++ b/.gitignore @@ -78,3 +78,6 @@ debian/files # protobuf stuff hivemind/proto/*_pb2* + +# libp2p-daemon binary +hivemind/hivemind_cli/p2pd diff --git a/setup.py b/setup.py index 36630cd27..6135feda7 100644 --- a/setup.py +++ b/setup.py @@ -6,6 +6,7 @@ import urllib.request import tarfile import tempfile +import hashlib from packaging import version from pkg_resources import parse_requirements @@ -13,21 +14,18 @@ from setuptools.command.develop import develop from setuptools.command.install import install +P2PD_VERSION = 'v0.3.1' +P2PD_CHECKSUM = '5094d094740f4e375afe80a5683b1bb2' here = os.path.abspath(os.path.dirname(__file__)) -class cd: - """Context manager for changing the current working directory""" - def __init__(self, newPath): - self.newPath = os.path.expanduser(newPath) - - def __enter__(self): - self.savedPath = os.getcwd() - os.chdir(self.newPath) - - def __exit__(self, etype, value, traceback): - os.chdir(self.savedPath) +def md5(fname, chunk_size=4096): + hash_md5 = hashlib.md5() + with open(fname, "rb") as f: + for chunk in iter(lambda: f.read(chunk_size), b""): + hash_md5.update(chunk) + return hash_md5.hexdigest() def proto_compile(output_path): @@ -49,8 +47,7 @@ def proto_compile(output_path): file.truncate() -def install_libp2p_daemon(): - # check go version: +def libp2p_build_install(): try: proc = subprocess.Popen(['go', 'version'], stdout=subprocess.PIPE) @@ -58,7 +55,7 @@ def install_libp2p_daemon(): result = result.decode('ascii', 'replace') _, _, v, _ = result.split(' ') v = v.lstrip('go') - + if version.parse(v) < version.parse("1.13"): raise EnvironmentError(f'newer version of go required: must be >= 1.13, found {version}') @@ -66,39 +63,45 @@ def install_libp2p_daemon(): raise FileNotFoundError('could not find golang installation') with tempfile.TemporaryDirectory() as tempdir: - url = 'https://github.com/libp2p/go-libp2p-daemon/archive/master.tar.gz' - dest = os.path.join(tempdir, 'libp2p-daemon.tar.gz') + url = f'https://github.com/learning-at-home/go-libp2p-daemon/archive/refs/tags/{P2PD_VERSION}.tar.gz' + dest = os.path.join(tempdir, 'libp2p-daemon.tar.gz') urllib.request.urlretrieve(url, os.path.join(tempdir, dest)) - + tar = tarfile.open(dest, 'r:gz') tar.extractall(tempdir) tar.close() - - with cd(os.path.join(tempdir, 'go-libp2p-daemon-master', 'p2pd')): - status = os.system(f'go build -o {os.path.join(here, "hivemind/hivemind_cli", "p2pd")}') - if status: - raise RuntimeError('Failed to build or install libp2p-daemon:'\ - f' exited with status code :{status}') + result = subprocess.run(['go', 'build', '-o', os.path.join(here, "hivemind/hivemind_cli", "p2pd")], + cwd=os.path.join(tempdir, f'go-libp2p-daemon-{P2PD_VERSION[1:]}', 'p2pd')) + if result.returncode: + raise RuntimeError('Failed to build or install libp2p-daemon:' + f' exited with status code :{result.returncode}') + + +def libp2p_download_install(): + install_path = os.path.join(here, 'hivemind/hivemind_cli/') + binary_path = os.path.join(install_path, 'p2pd') + if 'p2pd' not in os.listdir(install_path) or md5(binary_path) != P2PD_CHECKSUM: + print('Downloading Peer to Peer Daemon') + url = f'https://github.com/learning-at-home/go-libp2p-daemon/releases/download/{P2PD_VERSION}/p2pd' + urllib.request.urlretrieve(url, binary_path) + os.chmod(binary_path, 777) -class ProtoCompileInstall(install): + +class Install(install): def run(self): + libp2p_download_install() proto_compile(os.path.join(self.build_lib, 'hivemind', 'proto')) super().run() -class ProtoCompileDevelop(develop): +class Develop(develop): def run(self): + libp2p_build_install() proto_compile(os.path.join('hivemind', 'proto')) super().run() -class LibP2PInstall(install): - def run(self): - install_libp2p_daemon() - - - with open('requirements.txt') as requirements_file: install_requires = list(map(str, parse_requirements(requirements_file))) @@ -120,7 +123,7 @@ def run(self): setup( name='hivemind', version=version_string, - cmdclass={'install': ProtoCompileInstall, 'develop': ProtoCompileDevelop, 'libp2p': LibP2PInstall}, + cmdclass={'install': Install, 'develop': Develop}, description='Decentralized deep learning in PyTorch', long_description='Decentralized deep learning in PyTorch. Built to train giant models on ' 'thousands of volunteers across the world.', From a4652fc46372ada0c12ec2b3524a9f5c39f41000 Mon Sep 17 00:00:00 2001 From: Ilya <37004806+skobellev@users.noreply.github.com> Date: Sun, 21 Mar 2021 22:23:03 +0300 Subject: [PATCH 20/81] feat p2p_daemon: add API to call peer handle (#181) * Extend P2P api * Add tests for new api * Add p2pclient dependencies * Test P2P from different processes * Fix typo in tests * Add default initialization * Fix daemon ports assignment * Replace del with __del__ in tests * Read from input stream with receive_exactly Co-authored-by: Ilya Kobelev --- hivemind/p2p/p2p_daemon.py | 190 +++++++++++++++++++++++++++++++++---- tests/test_p2p_daemon.py | 142 +++++++++++++++++++++++---- 2 files changed, 292 insertions(+), 40 deletions(-) diff --git a/hivemind/p2p/p2p_daemon.py b/hivemind/p2p/p2p_daemon.py index 3083c70e5..1f441c5d1 100644 --- a/hivemind/p2p/p2p_daemon.py +++ b/hivemind/p2p/p2p_daemon.py @@ -1,45 +1,197 @@ +import asyncio +import contextlib +import copy +from pathlib import Path +import pickle +import socket import subprocess import typing as tp +import warnings + +from multiaddr import Multiaddr +import p2pclient +from libp2p.peer.id import ID class P2P(object): """ Forks a child process and executes p2pd command with given arguments. - Sends SIGKILL to the child in destructor and on exit from contextmanager. + Can be used for peer to peer communication and procedure calls. + Sends SIGKILL to the child in destructor. """ - LIBP2P_CMD = 'p2pd' + P2PD_RELATIVE_PATH = 'hivemind_cli/p2pd' + NUM_RETRIES = 3 + RETRY_DELAY = 0.4 + HEADER_LEN = 8 + BYTEORDER = 'big' - def __init__(self, *args, **kwargs): - self._child = subprocess.Popen(args=self._make_process_args(args, kwargs)) - try: - stdout, stderr = self._child.communicate(timeout=0.2) - except subprocess.TimeoutExpired: - pass - else: - raise RuntimeError(f'p2p daemon exited with stderr: {stderr}') + def __init__(self): + self._child = None + self._listen_task = None + self._server_stopped = asyncio.Event() + self._buffer = bytearray() - def __enter__(self): - return self._child + @classmethod + async def create(cls, *args, quic=1, tls=1, conn_manager=1, dht_client=1, + nat_port_map=True, auto_nat=True, bootstrap=True, + host_port: int = None, daemon_listen_port: int = None, **kwargs): + self = cls() + p2pd_path = Path(__file__).resolve().parents[1] / P2P.P2PD_RELATIVE_PATH + proc_args = self._make_process_args( + str(p2pd_path), *args, + quic=quic, tls=tls, connManager=conn_manager, + dhtClient=dht_client, natPortMap=nat_port_map, + autonat=auto_nat, b=bootstrap, **kwargs) + self._assign_daemon_ports(host_port, daemon_listen_port) + for try_count in range(self.NUM_RETRIES): + try: + self._initialize(proc_args) + await self._identify_client(P2P.RETRY_DELAY * (2 ** try_count)) + except Exception as exc: + warnings.warn("Failed to initialize p2p daemon: " + str(exc), RuntimeWarning) + self._kill_child() + if try_count == P2P.NUM_RETRIES - 1: + raise + self._assign_daemon_ports() + continue + break + return self - def __exit__(self, exc_type, exc_val, exc_tb): - self._kill_child() + def _initialize(self, proc_args: tp.List[str]) -> None: + proc_args = copy.deepcopy(proc_args) + proc_args.extend(self._make_process_args( + hostAddrs=f'/ip4/0.0.0.0/tcp/{self._host_port},/ip4/0.0.0.0/udp/{self._host_port}/quic', + listen=f'/ip4/127.0.0.1/tcp/{self._daemon_listen_port}' + )) + self._child = subprocess.Popen( + args=proc_args, + stdin=subprocess.PIPE, stdout=subprocess.PIPE, + stderr=subprocess.PIPE, encoding="utf8" + ) + self._client_listen_port = find_open_port() + self._client = p2pclient.Client( + Multiaddr(f'/ip4/127.0.0.1/tcp/{self._daemon_listen_port}'), + Multiaddr(f'/ip4/127.0.0.1/tcp/{self._client_listen_port}')) + + async def _identify_client(self, delay): + await asyncio.sleep(delay) + encoded = await self._client.identify() + self.id = encoded[0].to_base58() + + def _assign_daemon_ports(self, host_port=None, daemon_listen_port=None): + self._host_port, self._daemon_listen_port = host_port, daemon_listen_port + if host_port is None: + self._host_port = find_open_port() + if daemon_listen_port is None: + self._daemon_listen_port = find_open_port() + while self._daemon_listen_port == self._host_port: + self._daemon_listen_port = find_open_port() + + @staticmethod + async def send_data(data, stream): + byte_str = pickle.dumps(data) + request = len(byte_str).to_bytes(P2P.HEADER_LEN, P2P.BYTEORDER) + byte_str + await stream.send_all(request) + + class IncompleteRead(Exception): + pass + + async def _receive_exactly(self, stream, n_bytes, max_bytes=1 << 16): + while len(self._buffer) < n_bytes: + data = await stream.receive_some(max_bytes) + if len(data) == 0: + raise P2P.IncompleteRead() + self._buffer.extend(data) + + result = self._buffer[:n_bytes] + self._buffer = self._buffer[n_bytes:] + return bytes(result) + + async def receive_data(self, stream, max_bytes=(1 < 16)): + header = await self._receive_exactly(stream, P2P.HEADER_LEN) + content_length = int.from_bytes(header, P2P.BYTEORDER) + data = await self._receive_exactly(stream, content_length) + return pickle.loads(data) + + def _handle_stream(self, handle): + async def do_handle_stream(stream_info, stream): + try: + request = await self.receive_data(stream) + except P2P.IncompleteRead: + warnings.warn("Incomplete read while receiving request from peer", RuntimeWarning) + return + finally: + stream.close() + try: + result = handle(request) + await self.send_data(result, stream) + except Exception as exc: + await self.send_data(exc, stream) + finally: + await stream.close() + + return do_handle_stream + + def start_listening(self): + async def listen(): + async with self._client.listen(): + await self._server_stopped.wait() + + self._listen_task = asyncio.create_task(listen()) + + async def stop_listening(self): + if self._listen_task is not None: + self._server_stopped.set() + self._listen_task.cancel() + try: + await self._listen_task + except asyncio.CancelledError: + self._listen_task = None + self._server_stopped.clear() + + async def add_stream_handler(self, name, handle): + if self._listen_task is None: + self.start_listening() + + await self._client.stream_handler(name, self._handle_stream(handle)) + + async def call_peer_handler(self, peer_id, handler_name, input_data): + libp2p_peer_id = ID.from_base58(peer_id) + stream_info, stream = await self._client.stream_open(libp2p_peer_id, (handler_name,)) + try: + await self.send_data(input_data, stream) + return await self.receive_data(stream) + finally: + await stream.close() def __del__(self): self._kill_child() def _kill_child(self): - if self._child.poll() is None: + if self._child is not None and self._child.poll() is None: self._child.kill() self._child.wait() - def _make_process_args(self, args: tp.Tuple[tp.Any], - kwargs: tp.Dict[str, tp.Any]) -> tp.List[str]: - proc_args = [self.LIBP2P_CMD] + def _make_process_args(self, *args, **kwargs) -> tp.List[str]: + proc_args = [] proc_args.extend( str(entry) for entry in args ) proc_args.extend( - f'-{key}={str(value)}' for key, value in kwargs.items() + f'-{key}={value}' if value is not None else f'-{key}' + for key, value in kwargs.items() ) return proc_args + + +def find_open_port(params=(socket.AF_INET, socket.SOCK_STREAM), + opt=(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)): + """ Finds a tcp port that can be occupied with a socket with *params and use *opt options """ + try: + with contextlib.closing(socket.socket(*params)) as sock: + sock.bind(('', 0)) + sock.setsockopt(*opt) + return sock.getsockname()[1] + except Exception: + raise diff --git a/tests/test_p2p_daemon.py b/tests/test_p2p_daemon.py index ac57e9e2f..75fd51cdc 100644 --- a/tests/test_p2p_daemon.py +++ b/tests/test_p2p_daemon.py @@ -1,6 +1,8 @@ +import asyncio +import multiprocessing as mp import subprocess -from time import perf_counter +import numpy as np import pytest import hivemind.p2p @@ -23,33 +25,131 @@ def is_process_running(pid: int) -> bool: return subprocess.check_output(cmd, shell=True).decode('utf-8').strip() == RUNNING -@pytest.fixture() -def mock_p2p_class(): - P2P.LIBP2P_CMD = "sleep" - - -def test_daemon_killed_on_del(mock_p2p_class): - start = perf_counter() - p2p_daemon = P2P('10s') +@pytest.mark.asyncio +async def test_daemon_killed_on_del(): + p2p_daemon = await P2P.create() child_pid = p2p_daemon._child.pid assert is_process_running(child_pid) - del p2p_daemon + p2p_daemon.__del__() assert not is_process_running(child_pid) - assert perf_counter() - start < 1 -def test_daemon_killed_on_exit(mock_p2p_class): - start = perf_counter() - with P2P('10s') as daemon: - child_pid = daemon.pid - assert is_process_running(child_pid) +def handle_square(x): + return x ** 2 - assert not is_process_running(child_pid) - assert perf_counter() - start < 1 + +def handle_add(args): + result = args[0] + for i in range(1, len(args)): + result = result + args[i] + return result + + +@pytest.mark.parametrize( + "test_input,handle", + [ + pytest.param(10, handle_square, id="square_integer"), + pytest.param((1, 2), handle_add, id="add_integers"), + pytest.param(([1, 2, 3], [12, 13]), handle_add, id="add_lists"), + pytest.param(2, lambda x: x ** 3, id="lambda") + ] +) +@pytest.mark.asyncio +async def test_call_peer_single_process(test_input, handle, handler_name="handle"): + server = await P2P.create() + server_pid = server._child.pid + await server.add_stream_handler(handler_name, handle) + assert is_process_running(server_pid) + + client = await P2P.create() + client_pid = client._child.pid + assert is_process_running(client_pid) + + await asyncio.sleep(1) + result = await client.call_peer_handler(server.id, handler_name, test_input) + assert result == handle(test_input) + + await server.stop_listening() + server.__del__() + assert not is_process_running(server_pid) + + client.__del__() + assert not is_process_running(client_pid) + + +@pytest.mark.asyncio +async def test_call_peer_different_processes(): + handler_name = "square" + test_input = np.random.randn(2, 3) + + server_side, client_side = mp.Pipe() + response_received = mp.Value(np.ctypeslib.as_ctypes_type(np.int32)) + response_received.value = 0 + + async def run_server(): + server = await P2P.create() + server_pid = server._child.pid + await server.add_stream_handler(handler_name, handle_square) + assert is_process_running(server_pid) + + server_side.send(server.id) + while response_received.value == 0: + await asyncio.sleep(0.5) + + await server.stop_listening() + server.__del__() + assert not is_process_running(server_pid) + + def server_target(): + asyncio.run(run_server()) + + proc = mp.Process(target=server_target) + proc.start() + + client = await P2P.create() + client_pid = client._child.pid + assert is_process_running(client_pid) + + await asyncio.sleep(1) + peer_id = client_side.recv() + + result = await client.call_peer_handler(peer_id, handler_name, test_input) + assert np.allclose(result, handle_square(test_input)) + response_received.value = 1 + + client.__del__() + assert not is_process_running(client_pid) + + proc.join() -def test_daemon_raises_on_faulty_args(): - with pytest.raises(RuntimeError): - P2P(faulty='argument') +@pytest.mark.parametrize( + "test_input,handle", + [ + pytest.param(np.random.randn(2, 3), handle_square, id="square"), + pytest.param([np.random.randn(2, 3), np.random.randn(2, 3)], handle_add, id="add"), + ] +) +@pytest.mark.asyncio +async def test_call_peer_numpy(test_input, handle, handler_name="handle"): + server = await P2P.create() + await server.add_stream_handler(handler_name, handle) + client = await P2P.create() + + await asyncio.sleep(1) + result = await client.call_peer_handler(server.id, handler_name, test_input) + assert np.allclose(result, handle(test_input)) + + +@pytest.mark.asyncio +async def test_call_peer_error(handler_name="handle"): + server = await P2P.create() + await server.add_stream_handler(handler_name, handle_add) + client = await P2P.create() + + await asyncio.sleep(1) + result = await client.call_peer_handler(server.id, handler_name, + [np.zeros((2, 3)), np.zeros((3, 2))]) + assert type(result) == ValueError From 181f4b90e7ed3df8abb9ed1983d7f11b704ad7bc Mon Sep 17 00:00:00 2001 From: Denis Mazur Date: Thu, 25 Mar 2021 00:18:52 +0300 Subject: [PATCH 21/81] fix chmod permissions (#194) --- setup.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 6135feda7..0bc15830e 100644 --- a/setup.py +++ b/setup.py @@ -73,6 +73,7 @@ def libp2p_build_install(): result = subprocess.run(['go', 'build', '-o', os.path.join(here, "hivemind/hivemind_cli", "p2pd")], cwd=os.path.join(tempdir, f'go-libp2p-daemon-{P2PD_VERSION[1:]}', 'p2pd')) + if result.returncode: raise RuntimeError('Failed to build or install libp2p-daemon:' f' exited with status code :{result.returncode}') @@ -85,7 +86,7 @@ def libp2p_download_install(): print('Downloading Peer to Peer Daemon') url = f'https://github.com/learning-at-home/go-libp2p-daemon/releases/download/{P2PD_VERSION}/p2pd' urllib.request.urlretrieve(url, binary_path) - os.chmod(binary_path, 777) + os.chmod(binary_path, 0o777) class Install(install): From 246858288b46b101ab458e8d7c4daee7551dbd43 Mon Sep 17 00:00:00 2001 From: Ilya <37004806+skobellev@users.noreply.github.com> Date: Wed, 31 Mar 2021 13:09:45 +0300 Subject: [PATCH 22/81] feat P2P: add unary handler (#197) * Add unary handler * Add P2PContext to unary handler parameters Co-authored-by: Ilya Kobelev --- hivemind/p2p/p2p_daemon.py | 147 +++++++++++++++++++++++++++---------- tests/test_p2p_daemon.py | 62 +++++++++++++++- 2 files changed, 168 insertions(+), 41 deletions(-) diff --git a/hivemind/p2p/p2p_daemon.py b/hivemind/p2p/p2p_daemon.py index 1f441c5d1..a8f44550e 100644 --- a/hivemind/p2p/p2p_daemon.py +++ b/hivemind/p2p/p2p_daemon.py @@ -1,17 +1,27 @@ import asyncio -import contextlib import copy from pathlib import Path import pickle -import socket import subprocess import typing as tp import warnings +import google.protobuf from multiaddr import Multiaddr import p2pclient from libp2p.peer.id import ID +from hivemind.utils.networking import find_open_port + + +class P2PContext(object): + def __init__(self, ours_id, ours_port, handle_name): + self.peer_id = None + self.peer_addr = None + self.ours_id = ours_id + self.ours_port = ours_port + self.handle_name = handle_name + class P2P(object): """ @@ -26,11 +36,16 @@ class P2P(object): HEADER_LEN = 8 BYTEORDER = 'big' + class IncompleteRead(Exception): + pass + + class InterruptedError(Exception): + pass + def __init__(self): self._child = None self._listen_task = None self._server_stopped = asyncio.Event() - self._buffer = bytearray() @classmethod async def create(cls, *args, quic=1, tls=1, conn_manager=1, dht_client=1, @@ -89,50 +104,108 @@ def _assign_daemon_ports(self, host_port=None, daemon_listen_port=None): self._daemon_listen_port = find_open_port() @staticmethod - async def send_data(data, stream): - byte_str = pickle.dumps(data) + async def send_raw_data(byte_str, stream): request = len(byte_str).to_bytes(P2P.HEADER_LEN, P2P.BYTEORDER) + byte_str await stream.send_all(request) - class IncompleteRead(Exception): - pass + @staticmethod + async def send_data(data, stream): + await P2P.send_raw_data(pickle.dumps(data), stream) + + @staticmethod + async def send_protobuf(protobuf, out_proto_type, stream): + if type(protobuf) != out_proto_type: + error = TypeError('Unary handler returned protobuf of wrong type.') + await P2P.send_raw_data(pickle.dumps(error), stream) + raise error + await P2P.send_raw_data(protobuf.SerializeToString(), stream) - async def _receive_exactly(self, stream, n_bytes, max_bytes=1 << 16): - while len(self._buffer) < n_bytes: - data = await stream.receive_some(max_bytes) + @staticmethod + async def receive_exactly(stream, n_bytes, max_bytes=1 << 16): + buffer = bytearray() + while len(buffer) < n_bytes: + data = await stream.receive_some(min(max_bytes, n_bytes - len(buffer))) if len(data) == 0: raise P2P.IncompleteRead() - self._buffer.extend(data) - - result = self._buffer[:n_bytes] - self._buffer = self._buffer[n_bytes:] - return bytes(result) + buffer.extend(data) + return bytes(buffer) - async def receive_data(self, stream, max_bytes=(1 < 16)): - header = await self._receive_exactly(stream, P2P.HEADER_LEN) + @staticmethod + async def receive_raw_data(stream): + header = await P2P.receive_exactly(stream, P2P.HEADER_LEN) content_length = int.from_bytes(header, P2P.BYTEORDER) - data = await self._receive_exactly(stream, content_length) - return pickle.loads(data) + data = await P2P.receive_exactly(stream, content_length) + return data - def _handle_stream(self, handle): + @staticmethod + async def receive_data(stream): + return pickle.loads(await P2P.receive_raw_data(stream)) + + @staticmethod + async def receive_protobuf(in_proto_type, stream): + protobuf = in_proto_type() + protobuf.ParseFromString(await P2P.receive_raw_data(stream)) + return protobuf + + @staticmethod + def _handle_stream(handle): async def do_handle_stream(stream_info, stream): try: - request = await self.receive_data(stream) + request = await P2P.receive_data(stream) except P2P.IncompleteRead: warnings.warn("Incomplete read while receiving request from peer", RuntimeWarning) + await stream.close() return - finally: - stream.close() try: result = handle(request) - await self.send_data(result, stream) + await P2P.send_data(result, stream) except Exception as exc: - await self.send_data(exc, stream) + await P2P.send_data(exc, stream) finally: await stream.close() return do_handle_stream + @staticmethod + def _handle_unary_stream(handle, context, in_proto_type, out_proto_type): + async def watchdog(stream): + await stream.receive_some(max_bytes=1) + raise P2P.InterruptedError() + + async def do_handle_unary_stream(stream_info, stream): + try: + try: + request = await P2P.receive_protobuf(in_proto_type, stream) + except P2P.IncompleteRead: + warnings.warn("Incomplete read while receiving request from peer", + RuntimeWarning) + return + except google.protobuf.message.DecodeError as error: + warnings.warn(repr(error), RuntimeWarning) + return + + context.peer_id, context.peer_addr = stream_info.peer_id, stream_info.addr + done, pending = await asyncio.wait([watchdog(stream), handle(request, context)], + return_when=asyncio.FIRST_COMPLETED) + try: + result = done.pop().result() + await P2P.send_protobuf(result, out_proto_type, stream) + except P2P.InterruptedError: + pass + except Exception as exc: + await P2P.send_data(exc, stream) + finally: + pending_task = pending.pop() + pending_task.cancel() + try: + await pending_task + except asyncio.CancelledError: + pass + finally: + await stream.close() + + return do_handle_unary_stream + def start_listening(self): async def listen(): async with self._client.listen(): @@ -153,15 +226,21 @@ async def stop_listening(self): async def add_stream_handler(self, name, handle): if self._listen_task is None: self.start_listening() + await self._client.stream_handler(name, P2P._handle_stream(handle)) - await self._client.stream_handler(name, self._handle_stream(handle)) + async def add_unary_handler(self, name, handle, in_proto_type, out_proto_type): + if self._listen_task is None: + self.start_listening() + context = P2PContext(ours_id=self.id, ours_port=self._host_port, handle_name=name) + await self._client.stream_handler( + name, P2P._handle_unary_stream(handle, context, in_proto_type, out_proto_type)) async def call_peer_handler(self, peer_id, handler_name, input_data): libp2p_peer_id = ID.from_base58(peer_id) stream_info, stream = await self._client.stream_open(libp2p_peer_id, (handler_name,)) try: - await self.send_data(input_data, stream) - return await self.receive_data(stream) + await P2P.send_data(input_data, stream) + return await P2P.receive_data(stream) finally: await stream.close() @@ -183,15 +262,3 @@ def _make_process_args(self, *args, **kwargs) -> tp.List[str]: for key, value in kwargs.items() ) return proc_args - - -def find_open_port(params=(socket.AF_INET, socket.SOCK_STREAM), - opt=(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)): - """ Finds a tcp port that can be occupied with a socket with *params and use *opt options """ - try: - with contextlib.closing(socket.socket(*params)) as sock: - sock.bind(('', 0)) - sock.setsockopt(*opt) - return sock.getsockname()[1] - except Exception: - raise diff --git a/tests/test_p2p_daemon.py b/tests/test_p2p_daemon.py index 75fd51cdc..06814d244 100644 --- a/tests/test_p2p_daemon.py +++ b/tests/test_p2p_daemon.py @@ -2,11 +2,13 @@ import multiprocessing as mp import subprocess +from libp2p.peer.id import ID + import numpy as np import pytest -import hivemind.p2p from hivemind.p2p import P2P +from hivemind.proto import dht_pb2 RUNNING = 'running' NOT_RUNNING = 'not running' @@ -47,6 +49,64 @@ def handle_add(args): return result +@pytest.mark.parametrize( + 'should_cancel', [True, False] +) +@pytest.mark.asyncio +async def test_call_unary_handler(should_cancel, handle_name="handle"): + handler_cancelled = False + + async def ping_handler(request, context): + try: + await asyncio.sleep(2) + except asyncio.CancelledError: + nonlocal handler_cancelled + handler_cancelled = True + return dht_pb2.PingResponse( + peer=dht_pb2.NodeInfo( + node_id=context.ours_id.encode(), rpc_port=context.ours_port), + sender_endpoint=context.handle_name, available=True) + + server = await P2P.create() + server_pid = server._child.pid + await server.add_unary_handler(handle_name, ping_handler, dht_pb2.PingRequest, + dht_pb2.PingResponse) + assert is_process_running(server_pid) + + client = await P2P.create() + client_pid = client._child.pid + assert is_process_running(client_pid) + + ping_request = dht_pb2.PingRequest( + peer=dht_pb2.NodeInfo(node_id=client.id.encode(), rpc_port=client._host_port), + validate=True) + expected_response = dht_pb2.PingResponse( + peer=dht_pb2.NodeInfo(node_id=server.id.encode(), rpc_port=server._host_port), + sender_endpoint=handle_name, available=True) + + await asyncio.sleep(1) + libp2p_server_id = ID.from_base58(server.id) + stream_info, stream = await client._client.stream_open(libp2p_server_id, (handle_name,)) + + await P2P.send_raw_data(ping_request.SerializeToString(), stream) + + if should_cancel: + await stream.close() + await asyncio.sleep(1) + assert handler_cancelled + else: + result = await P2P.receive_protobuf(dht_pb2.PingResponse, stream) + assert result == expected_response + assert not handler_cancelled + + await server.stop_listening() + server.__del__() + assert not is_process_running(server_pid) + + client.__del__() + assert not is_process_running(client_pid) + + @pytest.mark.parametrize( "test_input,handle", [ From c672a239efab148f86ea9a541925734bccd6b95d Mon Sep 17 00:00:00 2001 From: MaximKsh Date: Fri, 2 Apr 2021 22:59:06 +0300 Subject: [PATCH 23/81] Py libp2p bindings (#193) * #183 p2p daemon pybinding * #183 rename py bindings dir, fix imports and migrate tests * #183 move pb to hivemind.proto * #183 fix p2p tests * #183 remove config.py, move constants to classes * add docstrings and minor fixes --- hivemind/p2p/p2p_daemon.py | 75 +- hivemind/p2p/p2p_daemon_bindings/__init__.py | 0 hivemind/p2p/p2p_daemon_bindings/control.py | 211 +++++ .../p2p/p2p_daemon_bindings/datastructures.py | 186 +++++ hivemind/p2p/p2p_daemon_bindings/keys.py | 91 +++ hivemind/p2p/p2p_daemon_bindings/p2pclient.py | 75 ++ hivemind/p2p/p2p_daemon_bindings/utils.py | 72 ++ hivemind/proto/crypto.proto | 20 + hivemind/proto/p2pd.proto | 158 ++++ requirements.txt | 2 + tests/test_p2p_daemon.py | 48 +- tests/test_p2p_daemon_bindings.py | 769 ++++++++++++++++++ 12 files changed, 1648 insertions(+), 59 deletions(-) create mode 100644 hivemind/p2p/p2p_daemon_bindings/__init__.py create mode 100644 hivemind/p2p/p2p_daemon_bindings/control.py create mode 100644 hivemind/p2p/p2p_daemon_bindings/datastructures.py create mode 100644 hivemind/p2p/p2p_daemon_bindings/keys.py create mode 100644 hivemind/p2p/p2p_daemon_bindings/p2pclient.py create mode 100644 hivemind/p2p/p2p_daemon_bindings/utils.py create mode 100644 hivemind/proto/crypto.proto create mode 100644 hivemind/proto/p2pd.proto create mode 100644 tests/test_p2p_daemon_bindings.py diff --git a/hivemind/p2p/p2p_daemon.py b/hivemind/p2p/p2p_daemon.py index a8f44550e..af3185a44 100644 --- a/hivemind/p2p/p2p_daemon.py +++ b/hivemind/p2p/p2p_daemon.py @@ -8,8 +8,8 @@ import google.protobuf from multiaddr import Multiaddr -import p2pclient -from libp2p.peer.id import ID +import hivemind.p2p.p2p_daemon_bindings.p2pclient as p2pclient +from hivemind.p2p.p2p_daemon_bindings.datastructures import ID, StreamInfo from hivemind.utils.networking import find_open_port @@ -104,78 +104,81 @@ def _assign_daemon_ports(self, host_port=None, daemon_listen_port=None): self._daemon_listen_port = find_open_port() @staticmethod - async def send_raw_data(byte_str, stream): + async def send_raw_data(byte_str, writer): request = len(byte_str).to_bytes(P2P.HEADER_LEN, P2P.BYTEORDER) + byte_str - await stream.send_all(request) + writer.write(request) @staticmethod - async def send_data(data, stream): - await P2P.send_raw_data(pickle.dumps(data), stream) + async def send_data(data, writer): + await P2P.send_raw_data(pickle.dumps(data), writer) @staticmethod - async def send_protobuf(protobuf, out_proto_type, stream): + async def send_protobuf(protobuf, out_proto_type, writer): if type(protobuf) != out_proto_type: error = TypeError('Unary handler returned protobuf of wrong type.') - await P2P.send_raw_data(pickle.dumps(error), stream) + await P2P.send_raw_data(pickle.dumps(error), writer) raise error - await P2P.send_raw_data(protobuf.SerializeToString(), stream) + await P2P.send_raw_data(protobuf.SerializeToString(), writer) @staticmethod - async def receive_exactly(stream, n_bytes, max_bytes=1 << 16): + async def receive_exactly(reader, n_bytes, max_bytes=1 << 16): buffer = bytearray() while len(buffer) < n_bytes: - data = await stream.receive_some(min(max_bytes, n_bytes - len(buffer))) + data = await reader.read(min(max_bytes, n_bytes - len(buffer))) if len(data) == 0: raise P2P.IncompleteRead() buffer.extend(data) return bytes(buffer) @staticmethod - async def receive_raw_data(stream): - header = await P2P.receive_exactly(stream, P2P.HEADER_LEN) + async def receive_raw_data(reader): + header = await P2P.receive_exactly(reader, P2P.HEADER_LEN) content_length = int.from_bytes(header, P2P.BYTEORDER) - data = await P2P.receive_exactly(stream, content_length) + data = await P2P.receive_exactly(reader, content_length) return data @staticmethod - async def receive_data(stream): - return pickle.loads(await P2P.receive_raw_data(stream)) + async def receive_data(reader): + return pickle.loads(await P2P.receive_raw_data(reader)) @staticmethod - async def receive_protobuf(in_proto_type, stream): + async def receive_protobuf(in_proto_type, reader): protobuf = in_proto_type() - protobuf.ParseFromString(await P2P.receive_raw_data(stream)) + protobuf.ParseFromString(await P2P.receive_raw_data(reader)) return protobuf @staticmethod def _handle_stream(handle): - async def do_handle_stream(stream_info, stream): + async def do_handle_stream(stream_info, reader, writer): try: - request = await P2P.receive_data(stream) + request = await P2P.receive_data(reader) except P2P.IncompleteRead: warnings.warn("Incomplete read while receiving request from peer", RuntimeWarning) - await stream.close() + writer.close() return try: result = handle(request) - await P2P.send_data(result, stream) + await P2P.send_data(result, writer) except Exception as exc: - await P2P.send_data(exc, stream) + await P2P.send_data(exc, writer) finally: - await stream.close() + writer.close() return do_handle_stream @staticmethod def _handle_unary_stream(handle, context, in_proto_type, out_proto_type): - async def watchdog(stream): - await stream.receive_some(max_bytes=1) + async def watchdog(reader: asyncio.StreamReader): + await reader.read(n=1) raise P2P.InterruptedError() - async def do_handle_unary_stream(stream_info, stream): + async def do_handle_unary_stream( + stream_info: StreamInfo, + reader: asyncio.StreamReader, + writer: asyncio.StreamWriter) -> None: try: try: - request = await P2P.receive_protobuf(in_proto_type, stream) + request = await P2P.receive_protobuf(in_proto_type, reader) except P2P.IncompleteRead: warnings.warn("Incomplete read while receiving request from peer", RuntimeWarning) @@ -185,15 +188,15 @@ async def do_handle_unary_stream(stream_info, stream): return context.peer_id, context.peer_addr = stream_info.peer_id, stream_info.addr - done, pending = await asyncio.wait([watchdog(stream), handle(request, context)], + done, pending = await asyncio.wait([watchdog(reader), handle(request, context)], return_when=asyncio.FIRST_COMPLETED) try: result = done.pop().result() - await P2P.send_protobuf(result, out_proto_type, stream) + await P2P.send_protobuf(result, out_proto_type, writer) except P2P.InterruptedError: pass except Exception as exc: - await P2P.send_data(exc, stream) + await P2P.send_data(exc, writer) finally: pending_task = pending.pop() pending_task.cancel() @@ -202,7 +205,7 @@ async def do_handle_unary_stream(stream_info, stream): except asyncio.CancelledError: pass finally: - await stream.close() + writer.close() return do_handle_unary_stream @@ -237,12 +240,12 @@ async def add_unary_handler(self, name, handle, in_proto_type, out_proto_type): async def call_peer_handler(self, peer_id, handler_name, input_data): libp2p_peer_id = ID.from_base58(peer_id) - stream_info, stream = await self._client.stream_open(libp2p_peer_id, (handler_name,)) + stream_info, reader, writer = await self._client.stream_open(libp2p_peer_id, (handler_name,)) try: - await P2P.send_data(input_data, stream) - return await P2P.receive_data(stream) + await P2P.send_data(input_data, writer) + return await P2P.receive_data(reader) finally: - await stream.close() + writer.close() def __del__(self): self._kill_child() diff --git a/hivemind/p2p/p2p_daemon_bindings/__init__.py b/hivemind/p2p/p2p_daemon_bindings/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/hivemind/p2p/p2p_daemon_bindings/control.py b/hivemind/p2p/p2p_daemon_bindings/control.py new file mode 100644 index 000000000..df8aeaefa --- /dev/null +++ b/hivemind/p2p/p2p_daemon_bindings/control.py @@ -0,0 +1,211 @@ +import logging +from typing import AsyncIterator, Awaitable, Callable, Dict, Iterable, Sequence, Tuple + +import asyncio +from contextlib import asynccontextmanager +from multiaddr import Multiaddr, protocols +from hivemind.p2p.p2p_daemon_bindings.datastructures import PeerInfo, StreamInfo, ID +from hivemind.proto import p2pd_pb2 as p2pd_pb +from hivemind.p2p.p2p_daemon_bindings.utils import DispatchFailure, read_pbmsg_safe, write_pbmsg, raise_if_failed + +StreamHandler = Callable[[StreamInfo, asyncio.StreamReader, asyncio.StreamWriter], Awaitable[None]] + +_supported_conn_protocols = ( + protocols.P_IP4, + # protocols.P_IP6, + protocols.P_UNIX, +) + + +def parse_conn_protocol(maddr: Multiaddr) -> int: + proto_codes = set(proto.code for proto in maddr.protocols()) + proto_cand = proto_codes.intersection(_supported_conn_protocols) + if len(proto_cand) != 1: + supported_protos = ( + protocols.protocol_with_code(proto) for proto in _supported_conn_protocols + ) + raise ValueError( + f"connection protocol should be only one protocol out of {supported_protos}" + f", maddr={maddr}" + ) + return tuple(proto_cand)[0] + + +class DaemonConnector: + control_maddr: Multiaddr + logger = logging.getLogger("p2pclient.DaemonConnector") + DEFAULT_CONTROL_MADDR = "/unix/tmp/p2pd.sock" + + def __init__(self, control_maddr: Multiaddr = None) -> None: + if control_maddr is None: + control_maddr = Multiaddr(self.DEFAULT_CONTROL_MADDR) + self.control_maddr = control_maddr + + async def open_connection(self) -> (asyncio.StreamReader, asyncio.StreamWriter): + proto_code = parse_conn_protocol(self.control_maddr) + if proto_code == protocols.P_UNIX: + control_path = self.control_maddr.value_for_protocol(protocols.P_UNIX) + self.logger.debug( + "DaemonConnector %s opens connection to %s", self, self.control_maddr + ) + return await asyncio.open_unix_connection(control_path) + elif proto_code == protocols.P_IP4: + host = self.control_maddr.value_for_protocol(protocols.P_IP4) + port = int(self.control_maddr.value_for_protocol(protocols.P_TCP)) + return await asyncio.open_connection(host, port) + else: + raise ValueError( + f"protocol not supported: protocol={protocols.protocol_with_code(proto_code)}" + ) + + +class ControlClient: + listen_maddr: Multiaddr + daemon_connector: DaemonConnector + handlers: Dict[str, StreamHandler] + logger = logging.getLogger("p2pclient.ControlClient") + DEFAULT_LISTEN_MADDR = "/unix/tmp/p2pclient.sock" + + def __init__( + self, daemon_connector: DaemonConnector, listen_maddr: Multiaddr = None + ) -> None: + if listen_maddr is None: + listen_maddr = Multiaddr(self.DEFAULT_LISTEN_MADDR) + self.listen_maddr = listen_maddr + self.daemon_connector = daemon_connector + self.handlers = {} + + async def _handler(self, reader: asyncio.StreamReader, writer: asyncio.StreamWriter): + pb_stream_info = p2pd_pb.StreamInfo() # type: ignore + await read_pbmsg_safe(reader, pb_stream_info) + stream_info = StreamInfo.from_pb(pb_stream_info) + self.logger.info("New incoming stream: %s", stream_info) + try: + handler = self.handlers[stream_info.proto] + except KeyError as e: + # should never enter here... daemon should reject the stream for us. + writer.close() + raise DispatchFailure(e) + await handler(stream_info, reader, writer) + + @asynccontextmanager + async def listen(self) -> AsyncIterator["ControlClient"]: + proto_code = parse_conn_protocol(self.listen_maddr) + if proto_code == protocols.P_UNIX: + listen_path = self.listen_maddr.value_for_protocol(protocols.P_UNIX) + server = await asyncio.start_unix_server(self._handler, path=listen_path) + elif proto_code == protocols.P_IP4: + host = self.listen_maddr.value_for_protocol(protocols.P_IP4) + port = int(self.listen_maddr.value_for_protocol(protocols.P_TCP)) + server = await asyncio.start_server(self._handler, port=port, host=host) + else: + raise ValueError( + f"protocol not supported: protocol={protocols.protocol_with_code(proto_code)}" + ) + + async with server: + self.logger.info( + "DaemonConnector %s starts listening to %s", self, self.listen_maddr + ) + yield self + + self.logger.info("DaemonConnector %s closed", self) + + async def identify(self) -> Tuple[ID, Tuple[Multiaddr, ...]]: + reader, writer = await self.daemon_connector.open_connection() + req = p2pd_pb.Request(type=p2pd_pb.Request.IDENTIFY) + await write_pbmsg(writer, req) + + resp = p2pd_pb.Response() # type: ignore + await read_pbmsg_safe(reader, resp) + writer.close() + + raise_if_failed(resp) + peer_id_bytes = resp.identify.id + maddrs_bytes = resp.identify.addrs + + maddrs = tuple(Multiaddr(maddr_bytes) for maddr_bytes in maddrs_bytes) + peer_id = ID(peer_id_bytes) + + return peer_id, maddrs + + async def connect(self, peer_id: ID, maddrs: Iterable[Multiaddr]) -> None: + reader, writer = await self.daemon_connector.open_connection() + + maddrs_bytes = [i.to_bytes() for i in maddrs] + connect_req = p2pd_pb.ConnectRequest( + peer=peer_id.to_bytes(), addrs=maddrs_bytes + ) + req = p2pd_pb.Request(type=p2pd_pb.Request.CONNECT, connect=connect_req) + await write_pbmsg(writer, req) + + resp = p2pd_pb.Response() # type: ignore + await read_pbmsg_safe(reader, resp) + writer.close() + raise_if_failed(resp) + + async def list_peers(self) -> Tuple[PeerInfo, ...]: + req = p2pd_pb.Request(type=p2pd_pb.Request.LIST_PEERS) + reader, writer = await self.daemon_connector.open_connection() + await write_pbmsg(writer, req) + resp = p2pd_pb.Response() # type: ignore + await read_pbmsg_safe(reader, resp) + writer.close() + raise_if_failed(resp) + + peers = tuple(PeerInfo.from_pb(pinfo) for pinfo in resp.peers) + return peers + + async def disconnect(self, peer_id: ID) -> None: + disconnect_req = p2pd_pb.DisconnectRequest(peer=peer_id.to_bytes()) + req = p2pd_pb.Request( + type=p2pd_pb.Request.DISCONNECT, disconnect=disconnect_req + ) + reader, writer = await self.daemon_connector.open_connection() + await write_pbmsg(writer, req) + resp = p2pd_pb.Response() # type: ignore + await read_pbmsg_safe(reader, resp) + writer.close() + raise_if_failed(resp) + + async def stream_open( + self, peer_id: ID, protocols: Sequence[str] + ) -> Tuple[StreamInfo, asyncio.StreamReader, asyncio.StreamWriter]: + reader, writer = await self.daemon_connector.open_connection() + + stream_open_req = p2pd_pb.StreamOpenRequest( + peer=peer_id.to_bytes(), proto=list(protocols) + ) + req = p2pd_pb.Request( + type=p2pd_pb.Request.STREAM_OPEN, streamOpen=stream_open_req + ) + await write_pbmsg(writer, req) + + resp = p2pd_pb.Response() # type: ignore + await read_pbmsg_safe(reader, resp) + raise_if_failed(resp) + + pb_stream_info = resp.streamInfo + stream_info = StreamInfo.from_pb(pb_stream_info) + + return stream_info, reader, writer + + async def stream_handler(self, proto: str, handler_cb: StreamHandler) -> None: + reader, writer = await self.daemon_connector.open_connection() + + listen_path_maddr_bytes = self.listen_maddr.to_bytes() + stream_handler_req = p2pd_pb.StreamHandlerRequest( + addr=listen_path_maddr_bytes, proto=[proto] + ) + req = p2pd_pb.Request( + type=p2pd_pb.Request.STREAM_HANDLER, streamHandler=stream_handler_req + ) + await write_pbmsg(writer, req) + + resp = p2pd_pb.Response() # type: ignore + await read_pbmsg_safe(reader, resp) + writer.close() + raise_if_failed(resp) + + # if success, add the handler to the dict + self.handlers[proto] = handler_cb diff --git a/hivemind/p2p/p2p_daemon_bindings/datastructures.py b/hivemind/p2p/p2p_daemon_bindings/datastructures.py new file mode 100644 index 000000000..42351627c --- /dev/null +++ b/hivemind/p2p/p2p_daemon_bindings/datastructures.py @@ -0,0 +1,186 @@ +import hashlib +from typing import Union, List, Sequence, Any + +import base58 +import multihash + +from multiaddr import Multiaddr, protocols +from hivemind.proto import p2pd_pb2 + +from hivemind.p2p.p2p_daemon_bindings.keys import PublicKey + +# NOTE: On inlining... +# See: https://github.com/libp2p/specs/issues/138 +# NOTE: enabling to be interoperable w/ the Go implementation +ENABLE_INLINING = True +MAX_INLINE_KEY_LENGTH = 42 + +IDENTITY_MULTIHASH_CODE = 0x00 + +if ENABLE_INLINING: + + class IdentityHash: + _digest: bytes + + def __init__(self) -> None: + self._digest = bytearray() + + def update(self, input: bytes) -> None: + self._digest += input + + def digest(self) -> bytes: + return self._digest + + multihash.FuncReg.register( + IDENTITY_MULTIHASH_CODE, "identity", hash_new=lambda: IdentityHash() + ) + + +class ID: + _bytes: bytes + _xor_id: int = None + _b58_str: str = None + + def __init__(self, peer_id_bytes: bytes) -> None: + self._bytes = peer_id_bytes + + @property + def xor_id(self) -> int: + if not self._xor_id: + self._xor_id = int(sha256_digest(self._bytes).hex(), 16) + return self._xor_id + + def to_bytes(self) -> bytes: + return self._bytes + + def to_base58(self) -> str: + if not self._b58_str: + self._b58_str = base58.b58encode(self._bytes).decode() + return self._b58_str + + def __repr__(self) -> str: + return f"" + + __str__ = pretty = to_string = to_base58 + + def __eq__(self, other: object) -> bool: + if isinstance(other, str): + return self.to_base58() == other + elif isinstance(other, bytes): + return self._bytes == other + elif isinstance(other, ID): + return self._bytes == other._bytes + else: + return NotImplemented + + def __hash__(self) -> int: + return hash(self._bytes) + + @classmethod + def from_base58(cls, b58_encoded_peer_id_str: str) -> "ID": + peer_id_bytes = base58.b58decode(b58_encoded_peer_id_str) + pid = ID(peer_id_bytes) + return pid + + @classmethod + def from_pubkey(cls, key: PublicKey) -> "ID": + serialized_key = key.serialize() + algo = multihash.Func.sha2_256 + if ENABLE_INLINING and len(serialized_key) <= MAX_INLINE_KEY_LENGTH: + algo = IDENTITY_MULTIHASH_CODE + mh_digest = multihash.digest(serialized_key, algo) + return cls(mh_digest.encode()) + + +def sha256_digest(data: Union[str, bytes]) -> bytes: + if isinstance(data, str): + data = data.encode("utf8") + return hashlib.sha256(data).digest() + + +class StreamInfo: + peer_id: ID + addr: Multiaddr + proto: str + + def __init__(self, peer_id: ID, addr: Multiaddr, proto: str) -> None: + self.peer_id = peer_id + self.addr = addr + self.proto = proto + + def __repr__(self) -> str: + return ( + f"" + ) + + def to_pb(self) -> p2pd_pb2.StreamInfo: + pb_msg = p2pd_pb2.StreamInfo( + peer=self.peer_id.to_bytes(), addr=self.addr.to_bytes(), proto=self.proto + ) + return pb_msg + + @classmethod + def from_pb(cls, pb_msg: p2pd_pb2.StreamInfo) -> "StreamInfo": + stream_info = cls( + peer_id=ID(pb_msg.peer), addr=Multiaddr(pb_msg.addr), proto=pb_msg.proto + ) + return stream_info + + +class PeerInfoLibP2P: + peer_id: ID + addrs: List[Multiaddr] + + def __init__(self, peer_id: ID, addrs: Sequence[Multiaddr]) -> None: + self.peer_id = peer_id + self.addrs = list(addrs) + + def __eq__(self, other: Any) -> bool: + return ( + isinstance(other, PeerInfo) + and self.peer_id == other.peer_id + and self.addrs == other.addrs + ) + + +def info_from_p2p_addr(addr: Multiaddr) -> PeerInfoLibP2P: + if not addr: + raise InvalidAddrError("`addr` should not be `None`") + + parts = addr.split() + if not parts: + raise InvalidAddrError( + f"`parts`={parts} should at least have a protocol `P_P2P`" + ) + + p2p_part = parts[-1] + last_protocol_code = p2p_part.protocols()[0].code + if last_protocol_code != protocols.P_P2P: + raise InvalidAddrError( + f"The last protocol should be `P_P2P` instead of `{last_protocol_code}`" + ) + + # make sure the /p2p value parses as a peer.ID + peer_id_str: str = p2p_part.value_for_protocol(protocols.P_P2P) + peer_id: ID = ID.from_base58(peer_id_str) + + # we might have received just an / p2p part, which means there's no addr. + if len(parts) > 1: + addr = Multiaddr.join(*parts[:-1]) + + return PeerInfo(peer_id, [addr]) + + +class InvalidAddrError(ValueError): + pass + + +class PeerInfo(PeerInfoLibP2P): + @classmethod + def from_pb(cls, peer_info_pb: p2pd_pb2.PeerInfo) -> PeerInfoLibP2P: + peer_id = ID(peer_info_pb.id) + addrs = [Multiaddr(addr) for addr in peer_info_pb.addrs] + return PeerInfo(peer_id, addrs) + + def __str__(self): + return self.peer_id.pretty() + " " + ",".join(str(a) for a in self.addrs) diff --git a/hivemind/p2p/p2p_daemon_bindings/keys.py b/hivemind/p2p/p2p_daemon_bindings/keys.py new file mode 100644 index 000000000..01ec5ad55 --- /dev/null +++ b/hivemind/p2p/p2p_daemon_bindings/keys.py @@ -0,0 +1,91 @@ +from abc import ABC, abstractmethod +from dataclasses import dataclass +from enum import Enum, unique + +from hivemind.proto import crypto_pb2 as protobuf + + +@unique +class KeyType(Enum): + RSA = 0 + Ed25519 = 1 + Secp256k1 = 2 + ECDSA = 3 + ECC_P256 = 4 + + +class Key(ABC): + """A ``Key`` represents a cryptographic key.""" + + @abstractmethod + def to_bytes(self) -> bytes: + """Returns the byte representation of this key.""" + ... + + @abstractmethod + def get_type(self) -> KeyType: + """Returns the ``KeyType`` for ``self``.""" + ... + + def __eq__(self, other: object) -> bool: + if not isinstance(other, Key): + return NotImplemented + return self.to_bytes() == other.to_bytes() + + +class PublicKey(Key): + """A ``PublicKey`` represents a cryptographic public key.""" + + @abstractmethod + def verify(self, data: bytes, signature: bytes) -> bool: + """Verify that ``signature`` is the cryptographic signature of the hash + of ``data``.""" + ... + + def _serialize_to_protobuf(self) -> protobuf.PublicKey: + """Return the protobuf representation of this ``Key``.""" + key_type = self.get_type().value + data = self.to_bytes() + protobuf_key = protobuf.PublicKey(key_type=key_type, data=data) + return protobuf_key + + def serialize(self) -> bytes: + """Return the canonical serialization of this ``Key``.""" + return self._serialize_to_protobuf().SerializeToString() + + @classmethod + def deserialize_from_protobuf(cls, protobuf_data: bytes) -> protobuf.PublicKey: + return protobuf.PublicKey.FromString(protobuf_data) + + +class PrivateKey(Key): + """A ``PrivateKey`` represents a cryptographic private key.""" + + @abstractmethod + def sign(self, data: bytes) -> bytes: + ... + + @abstractmethod + def get_public_key(self) -> PublicKey: + ... + + def _serialize_to_protobuf(self) -> protobuf.PrivateKey: + """Return the protobuf representation of this ``Key``.""" + key_type = self.get_type().value + data = self.to_bytes() + protobuf_key = protobuf.PrivateKey(key_type=key_type, data=data) + return protobuf_key + + def serialize(self) -> bytes: + """Return the canonical serialization of this ``Key``.""" + return self._serialize_to_protobuf().SerializeToString() + + @classmethod + def deserialize_from_protobuf(cls, protobuf_data: bytes) -> protobuf.PrivateKey: + return protobuf.PrivateKey.FromString(protobuf_data) + + +@dataclass(frozen=True) +class KeyPair: + private_key: PrivateKey + public_key: PublicKey diff --git a/hivemind/p2p/p2p_daemon_bindings/p2pclient.py b/hivemind/p2p/p2p_daemon_bindings/p2pclient.py new file mode 100644 index 000000000..1dbcce960 --- /dev/null +++ b/hivemind/p2p/p2p_daemon_bindings/p2pclient.py @@ -0,0 +1,75 @@ +from typing import AsyncIterator, Iterable, Sequence, Tuple + +import asyncio +from hivemind.p2p.p2p_daemon_bindings.control import ControlClient, DaemonConnector, StreamHandler +from contextlib import asynccontextmanager +from multiaddr import Multiaddr +from hivemind.p2p.p2p_daemon_bindings.datastructures import PeerInfo, StreamInfo, ID + + +class Client: + control: ControlClient + + def __init__( + self, control_maddr: Multiaddr = None, listen_maddr: Multiaddr = None + ) -> None: + daemon_connector = DaemonConnector(control_maddr=control_maddr) + self.control = ControlClient( + daemon_connector=daemon_connector, listen_maddr=listen_maddr + ) + + @asynccontextmanager + async def listen(self) -> AsyncIterator["Client"]: + """ + Starts to listen incoming connections for handlers registered via stream_handler. + :return: + """ + async with self.control.listen(): + yield self + + async def identify(self) -> Tuple[ID, Tuple[Multiaddr, ...]]: + """ + Get current node peer id and list of addresses + """ + return await self.control.identify() + + async def connect(self, peer_id: ID, maddrs: Iterable[Multiaddr]) -> None: + """ + Connect to p2p node with specified addresses and peer id. + :peer_id: node peer id you want connect to + :maddrs: node multiaddresses you want connect to. Of course, it must be reachable. + """ + await self.control.connect(peer_id=peer_id, maddrs=maddrs) + + async def list_peers(self) -> Tuple[PeerInfo, ...]: + """ + Get list of peers that node connect to + """ + return await self.control.list_peers() + + async def disconnect(self, peer_id: ID) -> None: + """ + Disconnect from node with specified peer id + :peer_id: + """ + await self.control.disconnect(peer_id=peer_id) + + async def stream_open( + self, peer_id: ID, protocols: Sequence[str] + ) -> Tuple[StreamInfo, asyncio.StreamReader, asyncio.StreamWriter]: + """ + Open a stream to call other peer (with peer_id) handler for specified protocols + :peer_id: + :protocols: + :return: Returns tuple of stream info (info about connection to second peer) and reader/writer + """ + return await self.control.stream_open(peer_id=peer_id, protocols=protocols) + + async def stream_handler(self, proto: str, handler_cb: StreamHandler) -> None: + """ + Register a stream handler + :param proto: protocols that handler serves + :param handler_cb: handler callback + :return: + """ + await self.control.stream_handler(proto=proto, handler_cb=handler_cb) diff --git a/hivemind/p2p/p2p_daemon_bindings/utils.py b/hivemind/p2p/p2p_daemon_bindings/utils.py new file mode 100644 index 000000000..fa0e7cfd3 --- /dev/null +++ b/hivemind/p2p/p2p_daemon_bindings/utils.py @@ -0,0 +1,72 @@ +import asyncio + +from google.protobuf.message import Message as PBMessage + +from hivemind.proto import p2pd_pb2 as p2pd_pb + + +DEFAULT_MAX_BITS: int = 64 + + +class ControlFailure(Exception): + pass + + +class DispatchFailure(Exception): + pass + + +async def write_unsigned_varint( + stream: asyncio.StreamWriter, integer: int, max_bits: int = DEFAULT_MAX_BITS +) -> None: + max_int: int = 1 << max_bits + if integer < 0: + raise ValueError(f"negative integer: {integer}") + if integer >= max_int: + raise ValueError(f"integer too large: {integer}") + while True: + value: int = integer & 0x7F + integer >>= 7 + if integer != 0: + value |= 0x80 + byte = value.to_bytes(1, "big") + stream.write(byte) + if integer == 0: + break + + +async def read_unsigned_varint( + stream: asyncio.StreamReader, max_bits: int = DEFAULT_MAX_BITS +) -> int: + max_int: int = 1 << max_bits + iteration: int = 0 + result: int = 0 + has_next: bool = True + while has_next: + data = await stream.readexactly(1) + c = data[0] + value = c & 0x7F + result |= value << (iteration * 7) + has_next = (c & 0x80) != 0 + iteration += 1 + if result >= max_int: + raise ValueError(f"varint overflowed: {result}") + return result + + +def raise_if_failed(response: p2pd_pb.Response) -> None: + if response.type == p2pd_pb.Response.ERROR: + raise ControlFailure(f"connect failed. msg={response.error.msg}") + + +async def write_pbmsg(stream: asyncio.StreamWriter, pbmsg: PBMessage) -> None: + size = pbmsg.ByteSize() + await write_unsigned_varint(stream, size) + msg_bytes: bytes = pbmsg.SerializeToString() + stream.write(msg_bytes) + + +async def read_pbmsg_safe(stream: asyncio.StreamReader, pbmsg: PBMessage) -> None: + len_msg_bytes = await read_unsigned_varint(stream) + msg_bytes = await stream.readexactly(len_msg_bytes) + pbmsg.ParseFromString(msg_bytes) diff --git a/hivemind/proto/crypto.proto b/hivemind/proto/crypto.proto new file mode 100644 index 000000000..fe729a9d4 --- /dev/null +++ b/hivemind/proto/crypto.proto @@ -0,0 +1,20 @@ +syntax = "proto2"; + +package crypto.pb; + +enum KeyType { + RSA = 0; + Ed25519 = 1; + Secp256k1 = 2; + ECDSA = 3; +} + +message PublicKey { + required KeyType key_type = 1; + required bytes data = 2; +} + +message PrivateKey { + required KeyType key_type = 1; + required bytes data = 2; +} diff --git a/hivemind/proto/p2pd.proto b/hivemind/proto/p2pd.proto new file mode 100644 index 000000000..fec4a6ed7 --- /dev/null +++ b/hivemind/proto/p2pd.proto @@ -0,0 +1,158 @@ +syntax = "proto2"; + +package p2pclient.p2pd.pb; + +message Request { + enum Type { + IDENTIFY = 0; + CONNECT = 1; + STREAM_OPEN = 2; + STREAM_HANDLER = 3; + DHT = 4; + LIST_PEERS = 5; + CONNMANAGER = 6; + DISCONNECT = 7; + PUBSUB = 8; + } + + required Type type = 1; + + optional ConnectRequest connect = 2; + optional StreamOpenRequest streamOpen = 3; + optional StreamHandlerRequest streamHandler = 4; + optional DHTRequest dht = 5; + optional ConnManagerRequest connManager = 6; + optional DisconnectRequest disconnect = 7; + optional PSRequest pubsub = 8; +} + +message Response { + enum Type { + OK = 0; + ERROR = 1; + } + + required Type type = 1; + optional ErrorResponse error = 2; + optional StreamInfo streamInfo = 3; + optional IdentifyResponse identify = 4; + optional DHTResponse dht = 5; + repeated PeerInfo peers = 6; + optional PSResponse pubsub = 7; +} + +message IdentifyResponse { + required bytes id = 1; + repeated bytes addrs = 2; +} + +message ConnectRequest { + required bytes peer = 1; + repeated bytes addrs = 2; + optional int64 timeout = 3; +} + +message StreamOpenRequest { + required bytes peer = 1; + repeated string proto = 2; + optional int64 timeout = 3; +} + +message StreamHandlerRequest { + required bytes addr = 1; + repeated string proto = 2; +} + +message ErrorResponse { + required string msg = 1; +} + +message StreamInfo { + required bytes peer = 1; + required bytes addr = 2; + required string proto = 3; +} + +message DHTRequest { + enum Type { + FIND_PEER = 0; + FIND_PEERS_CONNECTED_TO_PEER = 1; + FIND_PROVIDERS = 2; + GET_CLOSEST_PEERS = 3; + GET_PUBLIC_KEY = 4; + GET_VALUE = 5; + SEARCH_VALUE = 6; + PUT_VALUE = 7; + PROVIDE = 8; + } + + required Type type = 1; + optional bytes peer = 2; + optional bytes cid = 3; + optional bytes key = 4; + optional bytes value = 5; + optional int32 count = 6; + optional int64 timeout = 7; +} + +message DHTResponse { + enum Type { + BEGIN = 0; + VALUE = 1; + END = 2; + } + + required Type type = 1; + optional PeerInfo peer = 2; + optional bytes value = 3; +} + +message PeerInfo { + required bytes id = 1; + repeated bytes addrs = 2; +} + +message ConnManagerRequest { + enum Type { + TAG_PEER = 0; + UNTAG_PEER = 1; + TRIM = 2; + } + + required Type type = 1; + + optional bytes peer = 2; + optional string tag = 3; + optional int64 weight = 4; +} + +message DisconnectRequest { + required bytes peer = 1; +} + +message PSRequest { + enum Type { + GET_TOPICS = 0; + LIST_PEERS = 1; + PUBLISH = 2; + SUBSCRIBE = 3; + } + + required Type type = 1; + optional string topic = 2; + optional bytes data = 3; +} + +message PSMessage { + optional bytes from_id = 1; + optional bytes data = 2; + optional bytes seqno = 3; + repeated string topicIDs = 4; + optional bytes signature = 5; + optional bytes key = 6; +} + +message PSResponse { + repeated string topics = 1; + repeated bytes peerIDs = 2; +} diff --git a/requirements.txt b/requirements.txt index 375a3ab60..36b418050 100644 --- a/requirements.txt +++ b/requirements.txt @@ -10,4 +10,6 @@ grpcio>=1.33.2 grpcio-tools>=1.33.2 protobuf>=3.12.2 configargparse>=1.2.3 +multiaddr==0.0.9 +pymultihash==0.8.2 cryptography>=3.4.6 diff --git a/tests/test_p2p_daemon.py b/tests/test_p2p_daemon.py index 06814d244..759b2eb2b 100644 --- a/tests/test_p2p_daemon.py +++ b/tests/test_p2p_daemon.py @@ -2,7 +2,7 @@ import multiprocessing as mp import subprocess -from libp2p.peer.id import ID +from hivemind.p2p.p2p_daemon_bindings.datastructures import ID import numpy as np import pytest @@ -86,16 +86,16 @@ async def ping_handler(request, context): await asyncio.sleep(1) libp2p_server_id = ID.from_base58(server.id) - stream_info, stream = await client._client.stream_open(libp2p_server_id, (handle_name,)) + stream_info, reader, writer = await client._client.stream_open(libp2p_server_id, (handle_name,)) - await P2P.send_raw_data(ping_request.SerializeToString(), stream) + await P2P.send_raw_data(ping_request.SerializeToString(), writer) if should_cancel: - await stream.close() + writer.close() await asyncio.sleep(1) assert handler_cancelled else: - result = await P2P.receive_protobuf(dht_pb2.PingResponse, stream) + result = await P2P.receive_protobuf(dht_pb2.PingResponse, reader) assert result == expected_response assert not handler_cancelled @@ -139,6 +139,25 @@ async def test_call_peer_single_process(test_input, handle, handler_name="handle assert not is_process_running(client_pid) +async def run_server(handler_name, server_side, client_side, response_received): + server = await P2P.create() + server_pid = server._child.pid + await server.add_stream_handler(handler_name, handle_square) + assert is_process_running(server_pid) + + server_side.send(server.id) + while response_received.value == 0: + await asyncio.sleep(0.5) + + await server.stop_listening() + server.__del__() + assert not is_process_running(server_pid) + + +def server_target(handler_name, server_side, client_side, response_received): + asyncio.run(run_server(handler_name, server_side, client_side, response_received)) + + @pytest.mark.asyncio async def test_call_peer_different_processes(): handler_name = "square" @@ -148,24 +167,7 @@ async def test_call_peer_different_processes(): response_received = mp.Value(np.ctypeslib.as_ctypes_type(np.int32)) response_received.value = 0 - async def run_server(): - server = await P2P.create() - server_pid = server._child.pid - await server.add_stream_handler(handler_name, handle_square) - assert is_process_running(server_pid) - - server_side.send(server.id) - while response_received.value == 0: - await asyncio.sleep(0.5) - - await server.stop_listening() - server.__del__() - assert not is_process_running(server_pid) - - def server_target(): - asyncio.run(run_server()) - - proc = mp.Process(target=server_target) + proc = mp.Process(target=server_target, args=(handler_name, server_side, client_side, response_received)) proc.start() client = await P2P.create() diff --git a/tests/test_p2p_daemon_bindings.py b/tests/test_p2p_daemon_bindings.py new file mode 100644 index 000000000..e9bc77213 --- /dev/null +++ b/tests/test_p2p_daemon_bindings.py @@ -0,0 +1,769 @@ +import asyncio +import functools +import io +import os +import subprocess +import time +import uuid +from contextlib import asynccontextmanager, AsyncExitStack +from typing import NamedTuple + +from google.protobuf.message import EncodeError +from multiaddr import Multiaddr, protocols + +import pytest + +from hivemind import find_open_port +from hivemind.p2p.p2p_daemon_bindings.control import parse_conn_protocol, DaemonConnector, ControlClient +from hivemind.p2p.p2p_daemon_bindings.p2pclient import Client +from hivemind.p2p.p2p_daemon_bindings.utils import ControlFailure, raise_if_failed, write_unsigned_varint, \ + read_unsigned_varint, read_pbmsg_safe, write_pbmsg +from hivemind.proto import p2pd_pb2 as p2pd_pb +from hivemind.p2p.p2p_daemon_bindings.datastructures import ID, StreamInfo, PeerInfo + + +def test_raise_if_failed_raises(): + resp = p2pd_pb.Response() + resp.type = p2pd_pb.Response.ERROR + with pytest.raises(ControlFailure): + raise_if_failed(resp) + + +def test_raise_if_failed_not_raises(): + resp = p2pd_pb.Response() + resp.type = p2pd_pb.Response.OK + raise_if_failed(resp) + + +pairs_int_varint_valid = ( + (0, b"\x00"), + (1, b"\x01"), + (128, b"\x80\x01"), + (2 ** 32, b"\x80\x80\x80\x80\x10"), + (2 ** 64 - 1, b"\xff\xff\xff\xff\xff\xff\xff\xff\xff\x01"), +) + +pairs_int_varint_overflow = ( + (2 ** 64, b"\x80\x80\x80\x80\x80\x80\x80\x80\x80\x02"), + (2 ** 64 + 1, b"\x81\x80\x80\x80\x80\x80\x80\x80\x80\x02"), + ( + 2 ** 128, + b"\x80\x80\x80\x80\x80\x80\x80\x80\x80\x80\x80\x80\x80\x80\x80\x80\x80\x80\x04", + ), +) + + +class MockReader(io.BytesIO): + async def readexactly(self, n): + await asyncio.sleep(0) + return self.read(n) + + +class MockWriter(io.BytesIO): + pass + + +class MockReaderWriter(MockReader, MockWriter): + pass + + +@pytest.mark.parametrize("integer, var_integer", pairs_int_varint_valid) +@pytest.mark.asyncio +async def test_write_unsigned_varint(integer, var_integer): + s = MockWriter() + await write_unsigned_varint(s, integer) + assert s.getvalue() == var_integer + + +@pytest.mark.parametrize("integer", tuple(i[0] for i in pairs_int_varint_overflow)) +@pytest.mark.asyncio +async def test_write_unsigned_varint_overflow(integer): + s = MockWriter() + with pytest.raises(ValueError): + await write_unsigned_varint(s, integer) + + +@pytest.mark.parametrize("integer", (-1, -(2 ** 32), -(2 ** 64), -(2 ** 128))) +@pytest.mark.asyncio +async def test_write_unsigned_varint_negative(integer): + s = MockWriter() + with pytest.raises(ValueError): + await write_unsigned_varint(s, integer) + + +@pytest.mark.parametrize("integer, var_integer", pairs_int_varint_valid) +@pytest.mark.asyncio +async def test_read_unsigned_varint(integer, var_integer): + s = MockReader(var_integer) + result = await read_unsigned_varint(s) + assert result == integer + + +@pytest.mark.parametrize("var_integer", tuple(i[1] for i in pairs_int_varint_overflow)) +@pytest.mark.asyncio +async def test_read_unsigned_varint_overflow(var_integer): + s = MockReader(var_integer) + with pytest.raises(ValueError): + await read_unsigned_varint(s) + + +@pytest.mark.parametrize("max_bits", (2, 31, 32, 63, 64, 127, 128)) +@pytest.mark.asyncio +async def test_read_write_unsigned_varint_max_bits_edge(max_bits): + """ + Test the edge with different `max_bits` + """ + for i in range(-3, 0): + integer = i + (2 ** max_bits) + s = MockReaderWriter() + await write_unsigned_varint(s, integer, max_bits=max_bits) + s.seek(0, 0) + result = await read_unsigned_varint(s, max_bits=max_bits) + assert integer == result + + +@pytest.fixture(scope="module") +def peer_id_string(): + return "QmS5QmciTXXnCUCyxud5eWFenUMAmvAWSDa1c7dvdXRMZ7" + + +@pytest.fixture(scope="module") +def peer_id_bytes(): + return b'\x12 7\x87F.[\xb5\xb1o\xe5*\xc7\xb9\xbb\x11:"Z|j2\x8ad\x1b\xa6\xe5= timeout: + # timeout + assert False, f"{coro_func} still failed after `{timeout}` seconds" + await asyncio.sleep(0.01) + + +class Daemon: + control_maddr = None + proc_daemon = None + log_filename = "" + f_log = None + closed = None + + def __init__( + self, control_maddr, enable_control, enable_connmgr, enable_dht, enable_pubsub + ): + self.control_maddr = control_maddr + self.enable_control = enable_control + self.enable_connmgr = enable_connmgr + self.enable_dht = enable_dht + self.enable_pubsub = enable_pubsub + self.is_closed = False + self._start_logging() + self._run() + + def _start_logging(self): + name_control_maddr = str(self.control_maddr).replace("/", "_").replace(".", "_") + self.log_filename = f"/tmp/log_p2pd{name_control_maddr}.txt" + self.f_log = open(self.log_filename, "wb") + + def _run(self): + cmd_list = ["hivemind/hivemind_cli/p2pd", f"-listen={str(self.control_maddr)}"] + cmd_list += [f"-hostAddrs=/ip4/127.0.0.1/tcp/{find_open_port()}"] + if self.enable_connmgr: + cmd_list += ["-connManager=true", "-connLo=1", "-connHi=2", "-connGrace=0"] + if self.enable_dht: + cmd_list += ["-dht=true"] + if self.enable_pubsub: + cmd_list += ["-pubsub=true", "-pubsubRouter=gossipsub"] + self.proc_daemon = subprocess.Popen( + cmd_list, stdout=self.f_log, stderr=self.f_log, bufsize=0 + ) + + async def wait_until_ready(self): + lines_head_pattern = (b"Control socket:", b"Peer ID:", b"Peer Addrs:") + lines_head_occurred = {line: False for line in lines_head_pattern} + + with open(self.log_filename, "rb") as f_log_read: + + async def read_from_daemon_and_check(): + line = f_log_read.readline() + for head_pattern in lines_head_occurred: + if line.startswith(head_pattern): + lines_head_occurred[head_pattern] = True + return all([value for _, value in lines_head_occurred.items()]) + + await try_until_success(read_from_daemon_and_check) + + # sleep for a while in case that the daemon haven't been ready after emitting these lines + await asyncio.sleep(0.1) + + def close(self): + if self.is_closed: + return + self.proc_daemon.terminate() + self.proc_daemon.wait() + self.f_log.close() + self.is_closed = True + + +class DaemonTuple(NamedTuple): + daemon: Daemon + client: Client + + +class ConnectionFailure(Exception): + pass + + +@asynccontextmanager +async def make_p2pd_pair_unix( + enable_control, enable_connmgr, enable_dht, enable_pubsub +): + name = str(uuid.uuid4())[:8] + control_maddr = Multiaddr(f"/unix/tmp/test_p2pd_control_{name}.sock") + listen_maddr = Multiaddr(f"/unix/tmp/test_p2pd_listen_{name}.sock") + # Remove the existing unix socket files if they are existing + try: + os.unlink(control_maddr.value_for_protocol(protocols.P_UNIX)) + except FileNotFoundError: + pass + try: + os.unlink(listen_maddr.value_for_protocol(protocols.P_UNIX)) + except FileNotFoundError: + pass + async with _make_p2pd_pair( + control_maddr=control_maddr, + listen_maddr=listen_maddr, + enable_control=enable_control, + enable_connmgr=enable_connmgr, + enable_dht=enable_dht, + enable_pubsub=enable_pubsub, + ) as pair: + yield pair + + +@asynccontextmanager +async def make_p2pd_pair_ip4(enable_control, enable_connmgr, enable_dht, enable_pubsub): + control_maddr = Multiaddr(f"/ip4/127.0.0.1/tcp/{find_open_port()}") + listen_maddr = Multiaddr(f"/ip4/127.0.0.1/tcp/{find_open_port()}") + async with _make_p2pd_pair( + control_maddr=control_maddr, + listen_maddr=listen_maddr, + enable_control=enable_control, + enable_connmgr=enable_connmgr, + enable_dht=enable_dht, + enable_pubsub=enable_pubsub, + ) as pair: + yield pair + + +@asynccontextmanager +async def _make_p2pd_pair( + control_maddr, + listen_maddr, + enable_control, + enable_connmgr, + enable_dht, + enable_pubsub, +): + p2pd = Daemon( + control_maddr=control_maddr, + enable_control=enable_control, + enable_connmgr=enable_connmgr, + enable_dht=enable_dht, + enable_pubsub=enable_pubsub, + ) + # wait for daemon ready + await p2pd.wait_until_ready() + client = Client(control_maddr=control_maddr, listen_maddr=listen_maddr) + try: + async with client.listen(): + yield DaemonTuple(daemon=p2pd, client=client) + finally: + if not p2pd.is_closed: + p2pd.close() + + +@pytest.fixture +async def p2pcs( + num_p2pds, + enable_control, + enable_connmgr, + enable_dht, + enable_pubsub, + func_make_p2pd_pair, +): + # TODO: Change back to gather style + async with AsyncExitStack() as stack: + p2pd_tuples = [ + await stack.enter_async_context( + func_make_p2pd_pair( + enable_control=enable_control, + enable_connmgr=enable_connmgr, + enable_dht=enable_dht, + enable_pubsub=enable_pubsub, + ) + ) + for _ in range(num_p2pds) + ] + yield tuple(p2pd_tuple.client for p2pd_tuple in p2pd_tuples) + + +@pytest.mark.parametrize( + "enable_control, func_make_p2pd_pair", ((True, make_p2pd_pair_unix),) +) +@pytest.mark.asyncio +async def test_client_identify_unix_socket(p2pcs): + await p2pcs[0].identify() + + +@pytest.mark.parametrize("enable_control", (True,)) +@pytest.mark.asyncio +async def test_client_identify(p2pcs): + await p2pcs[0].identify() + + +@pytest.mark.parametrize("enable_control", (True,)) +@pytest.mark.asyncio +async def test_client_connect_success(p2pcs): + peer_id_0, maddrs_0 = await p2pcs[0].identify() + peer_id_1, maddrs_1 = await p2pcs[1].identify() + await p2pcs[0].connect(peer_id_1, maddrs_1) + # test case: repeated connections + await p2pcs[1].connect(peer_id_0, maddrs_0) + + +@pytest.mark.parametrize("enable_control", (True,)) +@pytest.mark.asyncio +async def test_client_connect_failure(peer_id_random, p2pcs): + peer_id_1, maddrs_1 = await p2pcs[1].identify() + await p2pcs[0].identify() + # test case: `peer_id` mismatches + with pytest.raises(ControlFailure): + await p2pcs[0].connect(peer_id_random, maddrs_1) + # test case: empty maddrs + with pytest.raises(ControlFailure): + await p2pcs[0].connect(peer_id_1, []) + # test case: wrong maddrs + with pytest.raises(ControlFailure): + await p2pcs[0].connect(peer_id_1, [Multiaddr("/ip4/127.0.0.1/udp/0")]) + + +async def _check_connection(p2pd_tuple_0, p2pd_tuple_1): + peer_id_0, _ = await p2pd_tuple_0.identify() + peer_id_1, _ = await p2pd_tuple_1.identify() + peers_0 = [pinfo.peer_id for pinfo in await p2pd_tuple_0.list_peers()] + peers_1 = [pinfo.peer_id for pinfo in await p2pd_tuple_1.list_peers()] + return (peer_id_0 in peers_1) and (peer_id_1 in peers_0) + + +async def connect_safe(p2pd_tuple_0, p2pd_tuple_1): + peer_id_1, maddrs_1 = await p2pd_tuple_1.identify() + await p2pd_tuple_0.connect(peer_id_1, maddrs_1) + await try_until_success( + functools.partial( + _check_connection, p2pd_tuple_0=p2pd_tuple_0, p2pd_tuple_1=p2pd_tuple_1 + ) + ) + + +@pytest.mark.parametrize("enable_control", (True,)) +@pytest.mark.asyncio +async def test_connect_safe(p2pcs): + await connect_safe(p2pcs[0], p2pcs[1]) + + +@pytest.mark.parametrize("enable_control", (True,)) +@pytest.mark.asyncio +async def test_client_list_peers(p2pcs): + # test case: no peers + assert len(await p2pcs[0].list_peers()) == 0 + # test case: 1 peer + await connect_safe(p2pcs[0], p2pcs[1]) + assert len(await p2pcs[0].list_peers()) == 1 + assert len(await p2pcs[1].list_peers()) == 1 + # test case: one more peer + await connect_safe(p2pcs[0], p2pcs[2]) + assert len(await p2pcs[0].list_peers()) == 2 + assert len(await p2pcs[1].list_peers()) == 1 + assert len(await p2pcs[2].list_peers()) == 1 + + +@pytest.mark.parametrize("enable_control", (True,)) +@pytest.mark.asyncio +async def test_client_disconnect(peer_id_random, p2pcs): + # test case: disconnect a peer without connections + await p2pcs[1].disconnect(peer_id_random) + # test case: disconnect + peer_id_0, _ = await p2pcs[0].identify() + await connect_safe(p2pcs[0], p2pcs[1]) + assert len(await p2pcs[0].list_peers()) == 1 + assert len(await p2pcs[1].list_peers()) == 1 + await p2pcs[1].disconnect(peer_id_0) + assert len(await p2pcs[0].list_peers()) == 0 + assert len(await p2pcs[1].list_peers()) == 0 + # test case: disconnect twice + await p2pcs[1].disconnect(peer_id_0) + assert len(await p2pcs[0].list_peers()) == 0 + assert len(await p2pcs[1].list_peers()) == 0 + + +@pytest.mark.parametrize("enable_control", (True,)) +@pytest.mark.asyncio +async def test_client_stream_open_success(p2pcs): + peer_id_1, maddrs_1 = await p2pcs[1].identify() + await connect_safe(p2pcs[0], p2pcs[1]) + + proto = "123" + + async def handle_proto(stream_info, reader, writer): + await reader.readexactly(1) + + await p2pcs[1].stream_handler(proto, handle_proto) + + # test case: normal + stream_info, reader, writer = await p2pcs[0].stream_open(peer_id_1, (proto,)) + assert stream_info.peer_id == peer_id_1 + assert stream_info.addr in maddrs_1 + assert stream_info.proto == "123" + writer.close() + + # test case: open with multiple protocols + stream_info, reader, writer = await p2pcs[0].stream_open( + peer_id_1, (proto, "another_protocol") + ) + assert stream_info.peer_id == peer_id_1 + assert stream_info.addr in maddrs_1 + assert stream_info.proto == "123" + writer.close() + + +@pytest.mark.parametrize("enable_control", (True,)) +@pytest.mark.asyncio +async def test_client_stream_open_failure(p2pcs): + peer_id_1, _ = await p2pcs[1].identify() + await connect_safe(p2pcs[0], p2pcs[1]) + + proto = "123" + + # test case: `stream_open` to a peer who didn't register the protocol + with pytest.raises(ControlFailure): + await p2pcs[0].stream_open(peer_id_1, (proto,)) + + # test case: `stream_open` to a peer for a non-registered protocol + async def handle_proto(stream_info, reader, writer): + pass + + await p2pcs[1].stream_handler(proto, handle_proto) + with pytest.raises(ControlFailure): + await p2pcs[0].stream_open(peer_id_1, ("another_protocol",)) + + +@pytest.mark.parametrize("enable_control", (True,)) +@pytest.mark.asyncio +async def test_client_stream_handler_success(p2pcs): + peer_id_1, _ = await p2pcs[1].identify() + await connect_safe(p2pcs[0], p2pcs[1]) + + proto = "protocol123" + bytes_to_send = b"yoyoyoyoyog" + # event for this test function to wait until the handler function receiving the incoming data + event_handler_finished = asyncio.Event() + + async def handle_proto(stream_info, reader, writer): + nonlocal event_handler_finished + bytes_received = await reader.readexactly(len(bytes_to_send)) + assert bytes_received == bytes_to_send + event_handler_finished.set() + + await p2pcs[1].stream_handler(proto, handle_proto) + assert proto in p2pcs[1].control.handlers + assert handle_proto == p2pcs[1].control.handlers[proto] + + # test case: test the stream handler `handle_proto` + + _, reader, writer = await p2pcs[0].stream_open(peer_id_1, (proto,)) + + # wait until the handler function starts blocking waiting for the data + # because we haven't sent the data, we know the handler function must still blocking waiting. + # get the task of the protocol handler + writer.write(bytes_to_send) + + # wait for the handler to finish + writer.close() + + await event_handler_finished.wait() + + # test case: two streams to different handlers respectively + another_proto = "another_protocol123" + another_bytes_to_send = b"456" + event_another_proto = asyncio.Event() + + async def handle_another_proto(stream_info, reader, writer): + event_another_proto.set() + bytes_received = await reader.readexactly(len(another_bytes_to_send)) + assert bytes_received == another_bytes_to_send + + await p2pcs[1].stream_handler(another_proto, handle_another_proto) + assert another_proto in p2pcs[1].control.handlers + assert handle_another_proto == p2pcs[1].control.handlers[another_proto] + + _, reader, writer = await p2pcs[0].stream_open(peer_id_1, (another_proto,)) + await event_another_proto.wait() + + # we know at this moment the handler must still blocking wait + + writer.write(another_bytes_to_send) + + writer.close() + + # test case: registering twice can override the previous registration + event_third = asyncio.Event() + + async def handler_third(stream_info, reader, writer): + event_third.set() + + await p2pcs[1].stream_handler(another_proto, handler_third) + assert another_proto in p2pcs[1].control.handlers + # ensure the handler is override + assert handler_third == p2pcs[1].control.handlers[another_proto] + + await p2pcs[0].stream_open(peer_id_1, (another_proto,)) + # ensure the overriding handler is called when the protocol is opened a stream + await event_third.wait() + + +@pytest.mark.parametrize("enable_control", (True,)) +@pytest.mark.asyncio +async def test_client_stream_handler_failure(p2pcs): + peer_id_1, _ = await p2pcs[1].identify() + await connect_safe(p2pcs[0], p2pcs[1]) + + proto = "123" + + # test case: registered a wrong protocol name + async def handle_proto_correct_params(stream_info, stream): + pass + + await p2pcs[1].stream_handler("another_protocol", handle_proto_correct_params) + with pytest.raises(ControlFailure): + await p2pcs[0].stream_open(peer_id_1, (proto,)) From d30f60ee0575cc47ab83c91fadabaeffc4da67b8 Mon Sep 17 00:00:00 2001 From: MaximKsh Date: Wed, 7 Apr 2021 08:25:35 +0300 Subject: [PATCH 24/81] #204 P2P replica mode (#205) * #204 P2P replica mode * #204 rename replica->replicate --- hivemind/p2p/p2p_daemon.py | 14 +++++ tests/test_p2p_daemon.py | 115 +++++++++++++++++++++++++++++++------ 2 files changed, 111 insertions(+), 18 deletions(-) diff --git a/hivemind/p2p/p2p_daemon.py b/hivemind/p2p/p2p_daemon.py index af3185a44..6a05fd6d9 100644 --- a/hivemind/p2p/p2p_daemon.py +++ b/hivemind/p2p/p2p_daemon.py @@ -73,6 +73,20 @@ async def create(cls, *args, quic=1, tls=1, conn_manager=1, dht_client=1, break return self + @classmethod + async def replicate(cls, daemon_listen_port: int, host_port: int): + self = cls() + # There is no child under control + # Use external already running p2pd + self._child = None + self._assign_daemon_ports(host_port, daemon_listen_port) + self._client_listen_port = find_open_port() + self._client = p2pclient.Client( + Multiaddr(f'/ip4/127.0.0.1/tcp/{self._daemon_listen_port}'), + Multiaddr(f'/ip4/127.0.0.1/tcp/{self._client_listen_port}')) + await self._identify_client(0) + return self + def _initialize(self, proc_args: tp.List[str]) -> None: proc_args = copy.deepcopy(proc_args) proc_args.extend(self._make_process_args( diff --git a/tests/test_p2p_daemon.py b/tests/test_p2p_daemon.py index 759b2eb2b..5c1c9f211 100644 --- a/tests/test_p2p_daemon.py +++ b/tests/test_p2p_daemon.py @@ -1,6 +1,7 @@ import asyncio import multiprocessing as mp import subprocess +from functools import partial from hivemind.p2p.p2p_daemon_bindings.datastructures import ID @@ -27,6 +28,10 @@ def is_process_running(pid: int) -> bool: return subprocess.check_output(cmd, shell=True).decode('utf-8').strip() == RUNNING +async def replicate_if_needed(p2p: P2P, replicate: bool): + return await P2P.replicate(p2p._daemon_listen_port, p2p._host_port) if replicate else p2p + + @pytest.mark.asyncio async def test_daemon_killed_on_del(): p2p_daemon = await P2P.create() @@ -38,6 +43,21 @@ async def test_daemon_killed_on_del(): assert not is_process_running(child_pid) +@pytest.mark.asyncio +async def test_daemon_replica_does_not_affect_primary(): + p2p_daemon = await P2P.create() + p2p_replica = await P2P.replicate(p2p_daemon._daemon_listen_port, p2p_daemon._host_port) + + child_pid = p2p_daemon._child.pid + assert is_process_running(child_pid) + + p2p_replica.__del__() + assert is_process_running(child_pid) + + p2p_daemon.__del__() + assert not is_process_running(child_pid) + + def handle_square(x): return x ** 2 @@ -50,10 +70,15 @@ def handle_add(args): @pytest.mark.parametrize( - 'should_cancel', [True, False] + 'should_cancel,replicate', [ + (True, False), + (True, True), + (False, False), + (False, True), + ] ) @pytest.mark.asyncio -async def test_call_unary_handler(should_cancel, handle_name="handle"): +async def test_call_unary_handler(should_cancel, replicate, handle_name="handle"): handler_cancelled = False async def ping_handler(request, context): @@ -67,14 +92,16 @@ async def ping_handler(request, context): node_id=context.ours_id.encode(), rpc_port=context.ours_port), sender_endpoint=context.handle_name, available=True) - server = await P2P.create() - server_pid = server._child.pid + server_primary = await P2P.create() + server = await replicate_if_needed(server_primary, replicate) + server_pid = server_primary._child.pid await server.add_unary_handler(handle_name, ping_handler, dht_pb2.PingRequest, dht_pb2.PingResponse) assert is_process_running(server_pid) - client = await P2P.create() - client_pid = client._child.pid + client_primary = await P2P.create() + client = await replicate_if_needed(client_primary, replicate) + client_pid = client_primary._child.pid assert is_process_running(client_pid) ping_request = dht_pb2.PingRequest( @@ -100,10 +127,10 @@ async def ping_handler(request, context): assert not handler_cancelled await server.stop_listening() - server.__del__() + server_primary.__del__() assert not is_process_running(server_pid) - client.__del__() + client_primary.__del__() assert not is_process_running(client_pid) @@ -131,7 +158,6 @@ async def test_call_peer_single_process(test_input, handle, handler_name="handle result = await client.call_peer_handler(server.id, handler_name, test_input) assert result == handle(test_input) - await server.stop_listening() server.__del__() assert not is_process_running(server_pid) @@ -188,30 +214,83 @@ async def test_call_peer_different_processes(): @pytest.mark.parametrize( - "test_input,handle", + "test_input,handle,replicate", [ - pytest.param(np.random.randn(2, 3), handle_square, id="square"), - pytest.param([np.random.randn(2, 3), np.random.randn(2, 3)], handle_add, id="add"), + pytest.param(np.random.randn(2, 3), handle_square, False, id="square_primary"), + pytest.param(np.random.randn(2, 3), handle_square, True, id="square_replica"), + pytest.param([np.random.randn(2, 3), np.random.randn(2, 3)], handle_add, False, id="add_primary"), + pytest.param([np.random.randn(2, 3), np.random.randn(2, 3)], handle_add, True, id="add_replica"), ] ) @pytest.mark.asyncio -async def test_call_peer_numpy(test_input, handle, handler_name="handle"): - server = await P2P.create() +async def test_call_peer_numpy(test_input, handle, replicate, handler_name="handle"): + server_primary = await P2P.create() + server = await replicate_if_needed(server_primary, replicate) await server.add_stream_handler(handler_name, handle) - client = await P2P.create() + client_primary = await P2P.create() + client = await replicate_if_needed(client_primary, replicate) await asyncio.sleep(1) result = await client.call_peer_handler(server.id, handler_name, test_input) assert np.allclose(result, handle(test_input)) +@pytest.mark.parametrize( + "replicate", + [ + pytest.param(False, id="primary"), + pytest.param(True, id="replica"), + ] +) @pytest.mark.asyncio -async def test_call_peer_error(handler_name="handle"): - server = await P2P.create() +async def test_call_peer_error(replicate, handler_name="handle"): + server_primary = await P2P.create() + server = await replicate_if_needed(server_primary, replicate) await server.add_stream_handler(handler_name, handle_add) - client = await P2P.create() + client_primary = await P2P.create() + client = await replicate_if_needed(client_primary, replicate) await asyncio.sleep(1) result = await client.call_peer_handler(server.id, handler_name, [np.zeros((2, 3)), np.zeros((3, 2))]) assert type(result) == ValueError + + +@pytest.mark.asyncio +async def test_handlers_on_different_replicas(handler_name="handle"): + def handler(arg, key): + return key + + server_primary = await P2P.create() + server_id = server_primary.id + await server_primary.add_stream_handler(handler_name, partial(handler, key="primary")) + + server_replica1 = await replicate_if_needed(server_primary, True) + await server_replica1.add_stream_handler(handler_name + "1", partial(handler, key="replica1")) + + server_replica2 = await replicate_if_needed(server_primary, True) + await server_replica2.add_stream_handler(handler_name + "2", partial(handler, key="replica2")) + + client = await P2P.create() + await asyncio.sleep(1) + result = await client.call_peer_handler(server_id, handler_name, "") + assert result == "primary" + + result = await client.call_peer_handler(server_id, handler_name + "1", "") + assert result == "replica1" + + result = await client.call_peer_handler(server_id, handler_name + "2", "") + assert result == "replica2" + + await server_replica1.stop_listening() + await server_replica2.stop_listening() + + # Primary does not handle replicas protocols + with pytest.raises(P2P.IncompleteRead): + await client.call_peer_handler(server_id, handler_name + "1", "") + with pytest.raises(P2P.IncompleteRead): + await client.call_peer_handler(server_id, handler_name + "2", "") + + await server_primary.stop_listening() + server_primary.__del__() + client.__del__() From c23cb85f77d3073c357d155402388b68386e2645 Mon Sep 17 00:00:00 2001 From: Maxim Kashirin Date: Wed, 21 Apr 2021 11:49:14 +0300 Subject: [PATCH 25/81] __del__ to shutdown in P2P --- hivemind/p2p/p2p_daemon.py | 23 ++++++++++++++++------- tests/test_p2p_daemon.py | 29 +++++++++++++++++------------ 2 files changed, 33 insertions(+), 19 deletions(-) diff --git a/hivemind/p2p/p2p_daemon.py b/hivemind/p2p/p2p_daemon.py index 6a05fd6d9..43fbd66ed 100644 --- a/hivemind/p2p/p2p_daemon.py +++ b/hivemind/p2p/p2p_daemon.py @@ -4,7 +4,6 @@ import pickle import subprocess import typing as tp -import warnings import google.protobuf from multiaddr import Multiaddr @@ -12,6 +11,10 @@ from hivemind.p2p.p2p_daemon_bindings.datastructures import ID, StreamInfo from hivemind.utils.networking import find_open_port +from hivemind.utils.logging import get_logger + + +logger = get_logger(__name__) class P2PContext(object): @@ -63,8 +66,8 @@ async def create(cls, *args, quic=1, tls=1, conn_manager=1, dht_client=1, try: self._initialize(proc_args) await self._identify_client(P2P.RETRY_DELAY * (2 ** try_count)) - except Exception as exc: - warnings.warn("Failed to initialize p2p daemon: " + str(exc), RuntimeWarning) + except Exception as e: + logger.debug(f"Failed to initialize p2p daemon: {e}", RuntimeWarning) self._kill_child() if try_count == P2P.NUM_RETRIES - 1: raise @@ -167,7 +170,7 @@ async def do_handle_stream(stream_info, reader, writer): try: request = await P2P.receive_data(reader) except P2P.IncompleteRead: - warnings.warn("Incomplete read while receiving request from peer", RuntimeWarning) + logger.debug("Incomplete read while receiving request from peer") writer.close() return try: @@ -194,11 +197,10 @@ async def do_handle_unary_stream( try: request = await P2P.receive_protobuf(in_proto_type, reader) except P2P.IncompleteRead: - warnings.warn("Incomplete read while receiving request from peer", - RuntimeWarning) + logger.debug("Incomplete read while receiving request from peer") return except google.protobuf.message.DecodeError as error: - warnings.warn(repr(error), RuntimeWarning) + logger.warning(repr(error)) return context.peer_id, context.peer_addr = stream_info.peer_id, stream_info.addr @@ -264,6 +266,13 @@ async def call_peer_handler(self, peer_id, handler_name, input_data): def __del__(self): self._kill_child() + @property + def is_alive(self): + return self._child.is_alive + + async def shutdown(self, timeout=None): + await asyncio.get_event_loop().run_in_executor(None, self._kill_child) + def _kill_child(self): if self._child is not None and self._child.poll() is None: self._child.kill() diff --git a/tests/test_p2p_daemon.py b/tests/test_p2p_daemon.py index 5c1c9f211..fd2597524 100644 --- a/tests/test_p2p_daemon.py +++ b/tests/test_p2p_daemon.py @@ -39,7 +39,7 @@ async def test_daemon_killed_on_del(): child_pid = p2p_daemon._child.pid assert is_process_running(child_pid) - p2p_daemon.__del__() + await p2p_daemon.shutdown() assert not is_process_running(child_pid) @@ -51,10 +51,10 @@ async def test_daemon_replica_does_not_affect_primary(): child_pid = p2p_daemon._child.pid assert is_process_running(child_pid) - p2p_replica.__del__() + await p2p_replica.shutdown() assert is_process_running(child_pid) - p2p_daemon.__del__() + await p2p_daemon.shutdown() assert not is_process_running(child_pid) @@ -127,10 +127,10 @@ async def ping_handler(request, context): assert not handler_cancelled await server.stop_listening() - server_primary.__del__() + await server_primary.shutdown() assert not is_process_running(server_pid) - client_primary.__del__() + await client_primary.shutdown() assert not is_process_running(client_pid) @@ -158,10 +158,10 @@ async def test_call_peer_single_process(test_input, handle, handler_name="handle result = await client.call_peer_handler(server.id, handler_name, test_input) assert result == handle(test_input) - server.__del__() + await server.shutdown() assert not is_process_running(server_pid) - client.__del__() + await client.shutdown() assert not is_process_running(client_pid) @@ -176,7 +176,7 @@ async def run_server(handler_name, server_side, client_side, response_received): await asyncio.sleep(0.5) await server.stop_listening() - server.__del__() + await server.shutdown() assert not is_process_running(server_pid) @@ -207,7 +207,7 @@ async def test_call_peer_different_processes(): assert np.allclose(result, handle_square(test_input)) response_received.value = 1 - client.__del__() + await client.shutdown() assert not is_process_running(client_pid) proc.join() @@ -231,9 +231,14 @@ async def test_call_peer_numpy(test_input, handle, replicate, handler_name="hand client = await replicate_if_needed(client_primary, replicate) await asyncio.sleep(1) + result = await client.call_peer_handler(server.id, handler_name, test_input) assert np.allclose(result, handle(test_input)) + await server.stop_listening() + await server_primary.shutdown() + await client_primary.shutdown() + @pytest.mark.parametrize( "replicate", @@ -272,7 +277,7 @@ def handler(arg, key): await server_replica2.add_stream_handler(handler_name + "2", partial(handler, key="replica2")) client = await P2P.create() - await asyncio.sleep(1) + await asyncio.sleep(2) result = await client.call_peer_handler(server_id, handler_name, "") assert result == "primary" @@ -292,5 +297,5 @@ def handler(arg, key): await client.call_peer_handler(server_id, handler_name + "2", "") await server_primary.stop_listening() - server_primary.__del__() - client.__del__() + await server_primary.shutdown() + await client.shutdown() From 26db677d3d00700c6541069a0916d5bf1aebcfdd Mon Sep 17 00:00:00 2001 From: Maxim Kashirin Date: Fri, 23 Apr 2021 11:06:16 +0300 Subject: [PATCH 26/81] review fixes --- hivemind/client/averaging/matchmaking.py | 8 ----- hivemind/p2p/p2p_daemon.py | 34 +++++++++++-------- hivemind/p2p/p2p_daemon_bindings/control.py | 6 ++++ .../p2p/p2p_daemon_bindings/datastructures.py | 13 ++++--- hivemind/p2p/p2p_daemon_bindings/keys.py | 6 ++++ hivemind/p2p/p2p_daemon_bindings/p2pclient.py | 16 +++++++-- hivemind/p2p/p2p_daemon_bindings/utils.py | 6 ++++ hivemind/proto/crypto.proto | 4 +++ hivemind/proto/p2pd.proto | 4 +++ setup.py | 21 ++++++------ 10 files changed, 78 insertions(+), 40 deletions(-) diff --git a/hivemind/client/averaging/matchmaking.py b/hivemind/client/averaging/matchmaking.py index b06318389..de20ebc02 100644 --- a/hivemind/client/averaging/matchmaking.py +++ b/hivemind/client/averaging/matchmaking.py @@ -462,13 +462,5 @@ async def _declare_averager_periodically(self, key_manager: GroupKeyManager): looking_for_group=False) -def compute_schema_hash(tensors: Sequence[torch.Tensor]) -> bytes: - """ A hash that describes follower's tensor shapes, dtypes, devices, but not the actual values """ - schema_dicts = [{field_name: str(field_value) - for field_name, field_value in asdict(TensorDescriptor.from_tensor(tensor)).items()} - for tensor in tensors] - return DHTID.generate(source=schema_dicts).to_bytes() - - class MatchmakingException(Exception): """ An internal exception that marks undesired edge cases during averaging """ diff --git a/hivemind/p2p/p2p_daemon.py b/hivemind/p2p/p2p_daemon.py index 43fbd66ed..7c0eb5918 100644 --- a/hivemind/p2p/p2p_daemon.py +++ b/hivemind/p2p/p2p_daemon.py @@ -1,29 +1,29 @@ import asyncio import copy -from pathlib import Path +import dataclasses import pickle import subprocess import typing as tp +from pathlib import Path import google.protobuf from multiaddr import Multiaddr + import hivemind.p2p.p2p_daemon_bindings.p2pclient as p2pclient from hivemind.p2p.p2p_daemon_bindings.datastructures import ID, StreamInfo - -from hivemind.utils.networking import find_open_port from hivemind.utils.logging import get_logger - +from hivemind.utils.networking import find_open_port logger = get_logger(__name__) +@dataclasses.dataclass(frozen=False) class P2PContext(object): - def __init__(self, ours_id, ours_port, handle_name): - self.peer_id = None - self.peer_addr = None - self.ours_id = ours_id - self.ours_port = ours_port - self.handle_name = handle_name + ours_id: str + ours_port: int + handle_name: str + peer_id: ID = None + peer_addr: Multiaddr = None class P2P(object): @@ -47,6 +47,7 @@ class InterruptedError(Exception): def __init__(self): self._child = None + self._alive = False self._listen_task = None self._server_stopped = asyncio.Event() @@ -67,7 +68,7 @@ async def create(cls, *args, quic=1, tls=1, conn_manager=1, dht_client=1, self._initialize(proc_args) await self._identify_client(P2P.RETRY_DELAY * (2 ** try_count)) except Exception as e: - logger.debug(f"Failed to initialize p2p daemon: {e}", RuntimeWarning) + logger.debug(f"Failed to initialize p2p daemon: {e}") self._kill_child() if try_count == P2P.NUM_RETRIES - 1: raise @@ -82,6 +83,7 @@ async def replicate(cls, daemon_listen_port: int, host_port: int): # There is no child under control # Use external already running p2pd self._child = None + self._alive = True self._assign_daemon_ports(host_port, daemon_listen_port) self._client_listen_port = find_open_port() self._client = p2pclient.Client( @@ -101,6 +103,7 @@ def _initialize(self, proc_args: tp.List[str]) -> None: stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE, encoding="utf8" ) + self._alive = True self._client_listen_port = find_open_port() self._client = p2pclient.Client( Multiaddr(f'/ip4/127.0.0.1/tcp/{self._daemon_listen_port}'), @@ -148,10 +151,10 @@ async def receive_exactly(reader, n_bytes, max_bytes=1 << 16): return bytes(buffer) @staticmethod - async def receive_raw_data(reader): - header = await P2P.receive_exactly(reader, P2P.HEADER_LEN) + async def receive_raw_data(reader: asyncio.StreamReader): + header = await reader.readexactly(P2P.HEADER_LEN) content_length = int.from_bytes(header, P2P.BYTEORDER) - data = await P2P.receive_exactly(reader, content_length) + data = await reader.readexactly(content_length) return data @staticmethod @@ -268,12 +271,13 @@ def __del__(self): @property def is_alive(self): - return self._child.is_alive + return self._alive async def shutdown(self, timeout=None): await asyncio.get_event_loop().run_in_executor(None, self._kill_child) def _kill_child(self): + self._alive = False if self._child is not None and self._child.poll() is None: self._child.kill() self._child.wait() diff --git a/hivemind/p2p/p2p_daemon_bindings/control.py b/hivemind/p2p/p2p_daemon_bindings/control.py index df8aeaefa..014ac674f 100644 --- a/hivemind/p2p/p2p_daemon_bindings/control.py +++ b/hivemind/p2p/p2p_daemon_bindings/control.py @@ -1,3 +1,9 @@ +""" +Originally taken from: https://github.com/mhchia/py-libp2p-daemon-bindings +Licence: MIT +Author: Kevin Mai-Husan Chia +""" + import logging from typing import AsyncIterator, Awaitable, Callable, Dict, Iterable, Sequence, Tuple diff --git a/hivemind/p2p/p2p_daemon_bindings/datastructures.py b/hivemind/p2p/p2p_daemon_bindings/datastructures.py index 42351627c..edc5ffc7c 100644 --- a/hivemind/p2p/p2p_daemon_bindings/datastructures.py +++ b/hivemind/p2p/p2p_daemon_bindings/datastructures.py @@ -1,13 +1,18 @@ +""" +Originally taken from: https://github.com/mhchia/py-libp2p-daemon-bindings +Licence: MIT +Author: Kevin Mai-Husan Chia +""" + import hashlib -from typing import Union, List, Sequence, Any +from typing import Any, List, Sequence, Union import base58 import multihash - from multiaddr import Multiaddr, protocols -from hivemind.proto import p2pd_pb2 from hivemind.p2p.p2p_daemon_bindings.keys import PublicKey +from hivemind.proto import p2pd_pb2 # NOTE: On inlining... # See: https://github.com/libp2p/specs/issues/138 @@ -32,7 +37,7 @@ def digest(self) -> bytes: return self._digest multihash.FuncReg.register( - IDENTITY_MULTIHASH_CODE, "identity", hash_new=lambda: IdentityHash() + IDENTITY_MULTIHASH_CODE, "identity", hash_new=IdentityHash ) diff --git a/hivemind/p2p/p2p_daemon_bindings/keys.py b/hivemind/p2p/p2p_daemon_bindings/keys.py index 01ec5ad55..84a106db0 100644 --- a/hivemind/p2p/p2p_daemon_bindings/keys.py +++ b/hivemind/p2p/p2p_daemon_bindings/keys.py @@ -1,3 +1,9 @@ +""" +Originally taken from: https://github.com/mhchia/py-libp2p-daemon-bindings +Licence: MIT +Author: Kevin Mai-Husan Chia +""" + from abc import ABC, abstractmethod from dataclasses import dataclass from enum import Enum, unique diff --git a/hivemind/p2p/p2p_daemon_bindings/p2pclient.py b/hivemind/p2p/p2p_daemon_bindings/p2pclient.py index 1dbcce960..baeab3612 100644 --- a/hivemind/p2p/p2p_daemon_bindings/p2pclient.py +++ b/hivemind/p2p/p2p_daemon_bindings/p2pclient.py @@ -1,10 +1,20 @@ -from typing import AsyncIterator, Iterable, Sequence, Tuple +""" +Originally taken from: https://github.com/mhchia/py-libp2p-daemon-bindings +Licence: MIT +Author: Kevin Mai-Husan Chia +""" import asyncio -from hivemind.p2p.p2p_daemon_bindings.control import ControlClient, DaemonConnector, StreamHandler from contextlib import asynccontextmanager +from typing import AsyncIterator, Iterable, Sequence, Tuple + from multiaddr import Multiaddr -from hivemind.p2p.p2p_daemon_bindings.datastructures import PeerInfo, StreamInfo, ID + +from hivemind.p2p.p2p_daemon_bindings.control import (ControlClient, + DaemonConnector, + StreamHandler) +from hivemind.p2p.p2p_daemon_bindings.datastructures import (ID, PeerInfo, + StreamInfo) class Client: diff --git a/hivemind/p2p/p2p_daemon_bindings/utils.py b/hivemind/p2p/p2p_daemon_bindings/utils.py index fa0e7cfd3..f567b33bb 100644 --- a/hivemind/p2p/p2p_daemon_bindings/utils.py +++ b/hivemind/p2p/p2p_daemon_bindings/utils.py @@ -1,3 +1,9 @@ +""" +Originally taken from: https://github.com/mhchia/py-libp2p-daemon-bindings +Licence: MIT +Author: Kevin Mai-Husan Chia +""" + import asyncio from google.protobuf.message import Message as PBMessage diff --git a/hivemind/proto/crypto.proto b/hivemind/proto/crypto.proto index fe729a9d4..1544252ee 100644 --- a/hivemind/proto/crypto.proto +++ b/hivemind/proto/crypto.proto @@ -1,3 +1,7 @@ +//Originally taken from: https://github.com/mhchia/py-libp2p-daemon-bindings +//Licence: MIT +//Author: Kevin Mai-Husan Chia + syntax = "proto2"; package crypto.pb; diff --git a/hivemind/proto/p2pd.proto b/hivemind/proto/p2pd.proto index fec4a6ed7..8eb3e7e17 100644 --- a/hivemind/proto/p2pd.proto +++ b/hivemind/proto/p2pd.proto @@ -1,3 +1,7 @@ +//Originally taken from: https://github.com/mhchia/py-libp2p-daemon-bindings +//Licence: MIT +//Author: Kevin Mai-Husan Chia + syntax = "proto2"; package p2pclient.p2pd.pb; diff --git a/setup.py b/setup.py index 0bc15830e..931324578 100644 --- a/setup.py +++ b/setup.py @@ -1,16 +1,16 @@ import codecs import glob +import hashlib import os import re import subprocess -import urllib.request import tarfile import tempfile -import hashlib +import urllib.request from packaging import version from pkg_resources import parse_requirements -from setuptools import setup, find_packages +from setuptools import find_packages, setup from setuptools.command.develop import develop from setuptools.command.install import install @@ -49,12 +49,11 @@ def proto_compile(output_path): def libp2p_build_install(): try: - proc = subprocess.Popen(['go', 'version'], - stdout=subprocess.PIPE) + proc = subprocess.Popen(['go', 'version'], stdout=subprocess.PIPE) result, _ = proc.communicate() result = result.decode('ascii', 'replace') - _, _, v, _ = result.split(' ') - v = v.lstrip('go') + m = re.search(r'^go version go([\d.]+)', result) + v = m.group(1) if version.parse(v) < version.parse("1.13"): raise EnvironmentError(f'newer version of go required: must be >= 1.13, found {version}') @@ -65,13 +64,13 @@ def libp2p_build_install(): with tempfile.TemporaryDirectory() as tempdir: url = f'https://github.com/learning-at-home/go-libp2p-daemon/archive/refs/tags/{P2PD_VERSION}.tar.gz' dest = os.path.join(tempdir, 'libp2p-daemon.tar.gz') - urllib.request.urlretrieve(url, os.path.join(tempdir, dest)) + urllib.request.urlretrieve(url, dest) tar = tarfile.open(dest, 'r:gz') tar.extractall(tempdir) tar.close() - result = subprocess.run(['go', 'build', '-o', os.path.join(here, "hivemind/hivemind_cli", "p2pd")], + result = subprocess.run(['go', 'build', '-o', os.path.join(here, "hivemind", "hivemind_cli", "p2pd")], cwd=os.path.join(tempdir, f'go-libp2p-daemon-{P2PD_VERSION[1:]}', 'p2pd')) if result.returncode: @@ -80,13 +79,15 @@ def libp2p_build_install(): def libp2p_download_install(): - install_path = os.path.join(here, 'hivemind/hivemind_cli/') + install_path = os.path.join(here, 'hivemind', 'hivemind_cli') binary_path = os.path.join(install_path, 'p2pd') if 'p2pd' not in os.listdir(install_path) or md5(binary_path) != P2PD_CHECKSUM: print('Downloading Peer to Peer Daemon') url = f'https://github.com/learning-at-home/go-libp2p-daemon/releases/download/{P2PD_VERSION}/p2pd' urllib.request.urlretrieve(url, binary_path) os.chmod(binary_path, 0o777) + if md5(binary_path) != P2PD_CHECKSUM: + raise RuntimeError(f'Downloaded p2pd binary from {url} does not match with md5 checksum') class Install(install): From 6856ee1ae2d1658fd47c5f228f82193ad08e8867 Mon Sep 17 00:00:00 2001 From: Maxim Kashirin Date: Fri, 23 Apr 2021 12:54:19 +0300 Subject: [PATCH 27/81] fix p2pd MD5 and fix that p2pd connects to ipfs --- hivemind/p2p/p2p_daemon.py | 70 ++++++++++++++++++++++++++---------- setup.py | 3 +- tests/test_p2p_daemon.py | 74 +++++++++++++++++++++++++++++--------- 3 files changed, 111 insertions(+), 36 deletions(-) diff --git a/hivemind/p2p/p2p_daemon.py b/hivemind/p2p/p2p_daemon.py index 7c0eb5918..8e84d0d07 100644 --- a/hivemind/p2p/p2p_daemon.py +++ b/hivemind/p2p/p2p_daemon.py @@ -39,9 +39,6 @@ class P2P(object): HEADER_LEN = 8 BYTEORDER = 'big' - class IncompleteRead(Exception): - pass - class InterruptedError(Exception): pass @@ -52,16 +49,27 @@ def __init__(self): self._server_stopped = asyncio.Event() @classmethod - async def create(cls, *args, quic=1, tls=1, conn_manager=1, dht_client=1, - nat_port_map=True, auto_nat=True, bootstrap=True, + async def create(cls, *args, quic=1, tls=1, conn_manager=1, dht_mode='dht_server', force_reachability=None, + nat_port_map=True, auto_nat=True, bootstrap=False, boostrap_peers=None, use_global_ipfs=False, host_port: int = None, daemon_listen_port: int = None, **kwargs): + if bootstrap and boostrap_peers is None and not use_global_ipfs: + raise AttributeError('Trying to create with bootstrap node without bootstrap nodes list. ' + 'It is very dangerous, because p2pd connects to global ipfs and it is very unstable. ' + 'If you really want this, pass use_global_ipfs=True') + if boostrap_peers is not None and use_global_ipfs: + raise AttributeError('Non empty boostrap_nodes and use_global_ipfs=True are incompatible.' + 'Choose one option: your nodes list (preferable) or global ipfs (very unstable)') + self = cls() p2pd_path = Path(__file__).resolve().parents[1] / P2P.P2PD_RELATIVE_PATH + bpeers = cls._make_bootstrap_peers(boostrap_peers) + dht = cls._make_dht_mode(dht_mode) + freachability = cls._make_force_reachability(force_reachability) proc_args = self._make_process_args( str(p2pd_path), *args, quic=quic, tls=tls, connManager=conn_manager, - dhtClient=dht_client, natPortMap=nat_port_map, - autonat=auto_nat, b=bootstrap, **kwargs) + natPortMap=nat_port_map, autonat=auto_nat, + b=bootstrap, **{**bpeers, **dht, **freachability, **kwargs}) self._assign_daemon_ports(host_port, daemon_listen_port) for try_count in range(self.NUM_RETRIES): try: @@ -92,6 +100,16 @@ async def replicate(cls, daemon_listen_port: int, host_port: int): await self._identify_client(0) return self + async def wait_peers_at_least(self, peers_num, attempts=3): + while attempts: + peers = await self._client.list_peers() + if len(peers) >= peers_num: + return + attempts -= 1 + await asyncio.sleep(1) + + raise RuntimeError('Not enough peers') + def _initialize(self, proc_args: tp.List[str]) -> None: proc_args = copy.deepcopy(proc_args) proc_args.extend(self._make_process_args( @@ -140,16 +158,6 @@ async def send_protobuf(protobuf, out_proto_type, writer): raise error await P2P.send_raw_data(protobuf.SerializeToString(), writer) - @staticmethod - async def receive_exactly(reader, n_bytes, max_bytes=1 << 16): - buffer = bytearray() - while len(buffer) < n_bytes: - data = await reader.read(min(max_bytes, n_bytes - len(buffer))) - if len(data) == 0: - raise P2P.IncompleteRead() - buffer.extend(data) - return bytes(buffer) - @staticmethod async def receive_raw_data(reader: asyncio.StreamReader): header = await reader.readexactly(P2P.HEADER_LEN) @@ -172,7 +180,7 @@ def _handle_stream(handle): async def do_handle_stream(stream_info, reader, writer): try: request = await P2P.receive_data(reader) - except P2P.IncompleteRead: + except asyncio.exceptions.IncompleteReadError: logger.debug("Incomplete read while receiving request from peer") writer.close() return @@ -199,7 +207,7 @@ async def do_handle_unary_stream( try: try: request = await P2P.receive_protobuf(in_proto_type, reader) - except P2P.IncompleteRead: + except asyncio.exceptions.IncompleteReadError: logger.debug("Incomplete read while receiving request from peer") return except google.protobuf.message.DecodeError as error: @@ -292,3 +300,27 @@ def _make_process_args(self, *args, **kwargs) -> tp.List[str]: for key, value in kwargs.items() ) return proc_args + + @staticmethod + def _make_bootstrap_peers(nodes): + if nodes is None: + return {} + return {'bootstrapPeers': ','.join(nodes)} + + @staticmethod + def _make_dht_mode(dht_mode): + if dht_mode == 'dht': + return {'dht': 1} + if dht_mode == 'dht_server': + return {'dhtServer': 1} + if dht_mode == 'dht_client': + return {'dhtClient': 1} + return {'dht': 0} + + @staticmethod + def _make_force_reachability(force_reachability): + if force_reachability == 'public': + return {'forceReachabilityPublic': 1} + if force_reachability == 'private': + return {'forceReachabilityPrivate': 1} + return {} diff --git a/setup.py b/setup.py index 931324578..cb1e0a6a5 100644 --- a/setup.py +++ b/setup.py @@ -15,7 +15,7 @@ from setuptools.command.install import install P2PD_VERSION = 'v0.3.1' -P2PD_CHECKSUM = '5094d094740f4e375afe80a5683b1bb2' +P2PD_CHECKSUM = '8810097959db720208cdc9f2945804a4' here = os.path.abspath(os.path.dirname(__file__)) @@ -86,6 +86,7 @@ def libp2p_download_install(): url = f'https://github.com/learning-at-home/go-libp2p-daemon/releases/download/{P2PD_VERSION}/p2pd' urllib.request.urlretrieve(url, binary_path) os.chmod(binary_path, 0o777) + print(md5(binary_path)) if md5(binary_path) != P2PD_CHECKSUM: raise RuntimeError(f'Downloaded p2pd binary from {url} does not match with md5 checksum') diff --git a/tests/test_p2p_daemon.py b/tests/test_p2p_daemon.py index fd2597524..2f13f91a9 100644 --- a/tests/test_p2p_daemon.py +++ b/tests/test_p2p_daemon.py @@ -2,6 +2,7 @@ import multiprocessing as mp import subprocess from functools import partial +from typing import List from hivemind.p2p.p2p_daemon_bindings.datastructures import ID @@ -32,6 +33,17 @@ async def replicate_if_needed(p2p: P2P, replicate: bool): return await P2P.replicate(p2p._daemon_listen_port, p2p._host_port) if replicate else p2p +def bootstrap_addr(host_port, id_): + return f'/ip4/127.0.0.1/tcp/{host_port}/p2p/{id_}' + + +def boostrap_from(daemons: List[P2P]) -> List[str]: + return [ + bootstrap_addr(d._host_port, d.id) + for d in daemons + ] + + @pytest.mark.asyncio async def test_daemon_killed_on_del(): p2p_daemon = await P2P.create() @@ -43,6 +55,22 @@ async def test_daemon_killed_on_del(): assert not is_process_running(child_pid) +@pytest.mark.asyncio +async def test_server_client_connection(): + server = await P2P.create() + peers = await server._client.list_peers() + assert len(peers) == 0 + + nodes = boostrap_from([server]) + client = await P2P.create(bootstrap=True, boostrap_peers=nodes) + await client.wait_peers_at_least(1) + + peers = await client._client.list_peers() + assert len(peers) == 1 + peers = await server._client.list_peers() + assert len(peers) == 1 + + @pytest.mark.asyncio async def test_daemon_replica_does_not_affect_primary(): p2p_daemon = await P2P.create() @@ -99,7 +127,8 @@ async def ping_handler(request, context): dht_pb2.PingResponse) assert is_process_running(server_pid) - client_primary = await P2P.create() + nodes = boostrap_from([server]) + client_primary = await P2P.create(bootstrap=True, boostrap_peers=nodes) client = await replicate_if_needed(client_primary, replicate) client_pid = client_primary._child.pid assert is_process_running(client_pid) @@ -111,7 +140,7 @@ async def ping_handler(request, context): peer=dht_pb2.NodeInfo(node_id=server.id.encode(), rpc_port=server._host_port), sender_endpoint=handle_name, available=True) - await asyncio.sleep(1) + await client.wait_peers_at_least(1) libp2p_server_id = ID.from_base58(server.id) stream_info, reader, writer = await client._client.stream_open(libp2p_server_id, (handle_name,)) @@ -150,14 +179,17 @@ async def test_call_peer_single_process(test_input, handle, handler_name="handle await server.add_stream_handler(handler_name, handle) assert is_process_running(server_pid) - client = await P2P.create() + nodes = boostrap_from([server]) + client = await P2P.create(bootstrap=True, boostrap_peers=nodes) client_pid = client._child.pid assert is_process_running(client_pid) - await asyncio.sleep(1) + await client.wait_peers_at_least(1) + result = await client.call_peer_handler(server.id, handler_name, test_input) assert result == handle(test_input) + await server.stop_listening() await server.shutdown() assert not is_process_running(server_pid) @@ -172,6 +204,7 @@ async def run_server(handler_name, server_side, client_side, response_received): assert is_process_running(server_pid) server_side.send(server.id) + server_side.send(server._host_port) while response_received.value == 0: await asyncio.sleep(0.5) @@ -196,12 +229,15 @@ async def test_call_peer_different_processes(): proc = mp.Process(target=server_target, args=(handler_name, server_side, client_side, response_received)) proc.start() - client = await P2P.create() + peer_id = client_side.recv() + peer_port = client_side.recv() + + nodes = [bootstrap_addr(peer_port, peer_id)] + client = await P2P.create(bootstrap=True, boostrap_peers=nodes) client_pid = client._child.pid assert is_process_running(client_pid) - await asyncio.sleep(1) - peer_id = client_side.recv() + await client.wait_peers_at_least(1) result = await client.call_peer_handler(peer_id, handler_name, test_input) assert np.allclose(result, handle_square(test_input)) @@ -227,10 +263,12 @@ async def test_call_peer_numpy(test_input, handle, replicate, handler_name="hand server_primary = await P2P.create() server = await replicate_if_needed(server_primary, replicate) await server.add_stream_handler(handler_name, handle) - client_primary = await P2P.create() + + nodes = boostrap_from([server]) + client_primary = await P2P.create(bootstrap=True, boostrap_peers=nodes) client = await replicate_if_needed(client_primary, replicate) - await asyncio.sleep(1) + await client.wait_peers_at_least(1) result = await client.call_peer_handler(server.id, handler_name, test_input) assert np.allclose(result, handle(test_input)) @@ -252,10 +290,12 @@ async def test_call_peer_error(replicate, handler_name="handle"): server_primary = await P2P.create() server = await replicate_if_needed(server_primary, replicate) await server.add_stream_handler(handler_name, handle_add) - client_primary = await P2P.create() + + nodes = boostrap_from([server]) + client_primary = await P2P.create(bootstrap=True, boostrap_peers=nodes) client = await replicate_if_needed(client_primary, replicate) - await asyncio.sleep(1) + await client.wait_peers_at_least(1) result = await client.call_peer_handler(server.id, handler_name, [np.zeros((2, 3)), np.zeros((3, 2))]) assert type(result) == ValueError @@ -266,7 +306,7 @@ async def test_handlers_on_different_replicas(handler_name="handle"): def handler(arg, key): return key - server_primary = await P2P.create() + server_primary = await P2P.create(bootstrap=False) server_id = server_primary.id await server_primary.add_stream_handler(handler_name, partial(handler, key="primary")) @@ -276,8 +316,10 @@ def handler(arg, key): server_replica2 = await replicate_if_needed(server_primary, True) await server_replica2.add_stream_handler(handler_name + "2", partial(handler, key="replica2")) - client = await P2P.create() - await asyncio.sleep(2) + nodes = boostrap_from([server_primary]) + client = await P2P.create(bootstrap=True, boostrap_peers=nodes) + await client.wait_peers_at_least(1) + result = await client.call_peer_handler(server_id, handler_name, "") assert result == "primary" @@ -291,9 +333,9 @@ def handler(arg, key): await server_replica2.stop_listening() # Primary does not handle replicas protocols - with pytest.raises(P2P.IncompleteRead): + with pytest.raises(asyncio.exceptions.IncompleteReadError): await client.call_peer_handler(server_id, handler_name + "1", "") - with pytest.raises(P2P.IncompleteRead): + with pytest.raises(asyncio.exceptions.IncompleteReadError): await client.call_peer_handler(server_id, handler_name + "2", "") await server_primary.stop_listening() From 67d43f2d9e5ea88f43dacc51886a2a79e34428c8 Mon Sep 17 00:00:00 2001 From: Maxim Kashirin Date: Fri, 23 Apr 2021 13:20:07 +0300 Subject: [PATCH 28/81] asyncio.IncompleteReadError --- hivemind/p2p/p2p_daemon.py | 4 ++-- tests/test_p2p_daemon.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/hivemind/p2p/p2p_daemon.py b/hivemind/p2p/p2p_daemon.py index 8e84d0d07..94e40680c 100644 --- a/hivemind/p2p/p2p_daemon.py +++ b/hivemind/p2p/p2p_daemon.py @@ -180,7 +180,7 @@ def _handle_stream(handle): async def do_handle_stream(stream_info, reader, writer): try: request = await P2P.receive_data(reader) - except asyncio.exceptions.IncompleteReadError: + except asyncio.IncompleteReadError: logger.debug("Incomplete read while receiving request from peer") writer.close() return @@ -207,7 +207,7 @@ async def do_handle_unary_stream( try: try: request = await P2P.receive_protobuf(in_proto_type, reader) - except asyncio.exceptions.IncompleteReadError: + except asyncio.IncompleteReadError: logger.debug("Incomplete read while receiving request from peer") return except google.protobuf.message.DecodeError as error: diff --git a/tests/test_p2p_daemon.py b/tests/test_p2p_daemon.py index 2f13f91a9..cb8016fa7 100644 --- a/tests/test_p2p_daemon.py +++ b/tests/test_p2p_daemon.py @@ -333,9 +333,9 @@ def handler(arg, key): await server_replica2.stop_listening() # Primary does not handle replicas protocols - with pytest.raises(asyncio.exceptions.IncompleteReadError): + with pytest.raises(asyncio.IncompleteReadError): await client.call_peer_handler(server_id, handler_name + "1", "") - with pytest.raises(asyncio.exceptions.IncompleteReadError): + with pytest.raises(asyncio.IncompleteReadError): await client.call_peer_handler(server_id, handler_name + "2", "") await server_primary.stop_listening() From 1d664f21d638dd8f27a866c9c04b4a4dd85a900a Mon Sep 17 00:00:00 2001 From: Maxim Kashirin Date: Tue, 27 Apr 2021 09:59:47 +0300 Subject: [PATCH 29/81] pr fixes: messagepack serialization, naming, etc --- hivemind/p2p/p2p_daemon.py | 20 ++-- hivemind/p2p/p2p_daemon_bindings/keys.py | 4 +- hivemind/proto/crypto.proto | 4 +- tests/test_p2p_daemon.py | 124 +++++++++++++++++------ tests/test_p2p_daemon_bindings.py | 24 ++--- 5 files changed, 120 insertions(+), 56 deletions(-) diff --git a/hivemind/p2p/p2p_daemon.py b/hivemind/p2p/p2p_daemon.py index 94e40680c..47a2610b1 100644 --- a/hivemind/p2p/p2p_daemon.py +++ b/hivemind/p2p/p2p_daemon.py @@ -1,7 +1,6 @@ import asyncio import copy import dataclasses -import pickle import subprocess import typing as tp from pathlib import Path @@ -11,6 +10,7 @@ import hivemind.p2p.p2p_daemon_bindings.p2pclient as p2pclient from hivemind.p2p.p2p_daemon_bindings.datastructures import ID, StreamInfo +from hivemind.utils import MSGPackSerializer from hivemind.utils.logging import get_logger from hivemind.utils.networking import find_open_port @@ -100,12 +100,11 @@ async def replicate(cls, daemon_listen_port: int, host_port: int): await self._identify_client(0) return self - async def wait_peers_at_least(self, peers_num, attempts=3): - while attempts: + async def wait_for_at_least_n_peers(self, n_peers, attempts=3): + for _ in range(attempts): peers = await self._client.list_peers() - if len(peers) >= peers_num: + if len(peers) >= n_peers: return - attempts -= 1 await asyncio.sleep(1) raise RuntimeError('Not enough peers') @@ -148,13 +147,14 @@ async def send_raw_data(byte_str, writer): @staticmethod async def send_data(data, writer): - await P2P.send_raw_data(pickle.dumps(data), writer) + raw_data = MSGPackSerializer.dumps(data) + await P2P.send_raw_data(raw_data, writer) @staticmethod async def send_protobuf(protobuf, out_proto_type, writer): if type(protobuf) != out_proto_type: error = TypeError('Unary handler returned protobuf of wrong type.') - await P2P.send_raw_data(pickle.dumps(error), writer) + await P2P.send_raw_data(MSGPackSerializer.dumps(error), writer) raise error await P2P.send_raw_data(protobuf.SerializeToString(), writer) @@ -167,7 +167,7 @@ async def receive_raw_data(reader: asyncio.StreamReader): @staticmethod async def receive_data(reader): - return pickle.loads(await P2P.receive_raw_data(reader)) + return MSGPackSerializer.loads(await P2P.receive_raw_data(reader)) @staticmethod async def receive_protobuf(in_proto_type, reader): @@ -188,7 +188,7 @@ async def do_handle_stream(stream_info, reader, writer): result = handle(request) await P2P.send_data(result, writer) except Exception as exc: - await P2P.send_data(exc, writer) + await P2P.send_data(str(exc), writer) finally: writer.close() @@ -223,7 +223,7 @@ async def do_handle_unary_stream( except P2P.InterruptedError: pass except Exception as exc: - await P2P.send_data(exc, writer) + await P2P.send_data(str(exc), writer) finally: pending_task = pending.pop() pending_task.cancel() diff --git a/hivemind/p2p/p2p_daemon_bindings/keys.py b/hivemind/p2p/p2p_daemon_bindings/keys.py index 84a106db0..763c3d76a 100644 --- a/hivemind/p2p/p2p_daemon_bindings/keys.py +++ b/hivemind/p2p/p2p_daemon_bindings/keys.py @@ -1,7 +1,7 @@ """ -Originally taken from: https://github.com/mhchia/py-libp2p-daemon-bindings +Originally taken from: https://github.com/libp2p/py-libp2p Licence: MIT -Author: Kevin Mai-Husan Chia +Author: Kevin Mai-Husan Chia and others """ from abc import ABC, abstractmethod diff --git a/hivemind/proto/crypto.proto b/hivemind/proto/crypto.proto index 1544252ee..e4a69f576 100644 --- a/hivemind/proto/crypto.proto +++ b/hivemind/proto/crypto.proto @@ -1,6 +1,6 @@ -//Originally taken from: https://github.com/mhchia/py-libp2p-daemon-bindings +//Originally taken from: https://github.com/libp2p/py-libp2p //Licence: MIT -//Author: Kevin Mai-Husan Chia +//Author: Kevin Mai-Husan Chia and others syntax = "proto2"; diff --git a/tests/test_p2p_daemon.py b/tests/test_p2p_daemon.py index cb8016fa7..4e37d757c 100644 --- a/tests/test_p2p_daemon.py +++ b/tests/test_p2p_daemon.py @@ -4,13 +4,17 @@ from functools import partial from typing import List +import torch + +from hivemind.utils.compression import serialize_torch_tensor, construct_torch_tensor, deserialize_torch_tensor + from hivemind.p2p.p2p_daemon_bindings.datastructures import ID import numpy as np import pytest from hivemind.p2p import P2P -from hivemind.proto import dht_pb2 +from hivemind.proto import dht_pb2, runtime_pb2 RUNNING = 'running' NOT_RUNNING = 'not running' @@ -25,8 +29,7 @@ def is_process_running(pid: int) -> bool: - cmd = CHECK_PID_CMD.format(pid, RUNNING, NOT_RUNNING) - return subprocess.check_output(cmd, shell=True).decode('utf-8').strip() == RUNNING + return subprocess.run(["ps", "-p", str(pid)], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL).returncode == 0 async def replicate_if_needed(p2p: P2P, replicate: bool): @@ -63,7 +66,7 @@ async def test_server_client_connection(): nodes = boostrap_from([server]) client = await P2P.create(bootstrap=True, boostrap_peers=nodes) - await client.wait_peers_at_least(1) + await client.wait_for_at_least_n_peers(1) peers = await client._client.list_peers() assert len(peers) == 1 @@ -97,6 +100,27 @@ def handle_add(args): return result +def handle_square_torch(x): + tensor = runtime_pb2.Tensor() + tensor.ParseFromString(x) + tensor = deserialize_torch_tensor(tensor) + result = tensor ** 2 + return serialize_torch_tensor(result).SerializeToString() + + +def handle_add_torch(args): + tensor = runtime_pb2.Tensor() + tensor.ParseFromString(args[0]) + result = deserialize_torch_tensor(tensor) + + for i in range(1, len(args)): + tensor = runtime_pb2.Tensor() + tensor.ParseFromString(args[i]) + result = result + deserialize_torch_tensor(tensor) + + return serialize_torch_tensor(result).SerializeToString() + + @pytest.mark.parametrize( 'should_cancel,replicate', [ (True, False), @@ -140,7 +164,7 @@ async def ping_handler(request, context): peer=dht_pb2.NodeInfo(node_id=server.id.encode(), rpc_port=server._host_port), sender_endpoint=handle_name, available=True) - await client.wait_peers_at_least(1) + await client.wait_for_at_least_n_peers(1) libp2p_server_id = ID.from_base58(server.id) stream_info, reader, writer = await client._client.stream_open(libp2p_server_id, (handle_name,)) @@ -184,7 +208,7 @@ async def test_call_peer_single_process(test_input, handle, handler_name="handle client_pid = client._child.pid assert is_process_running(client_pid) - await client.wait_peers_at_least(1) + await client.wait_for_at_least_n_peers(1) result = await client.call_peer_handler(server.id, handler_name, test_input) assert result == handle(test_input) @@ -220,7 +244,7 @@ def server_target(handler_name, server_side, client_side, response_received): @pytest.mark.asyncio async def test_call_peer_different_processes(): handler_name = "square" - test_input = np.random.randn(2, 3) + test_input = 2 server_side, client_side = mp.Pipe() response_received = mp.Value(np.ctypeslib.as_ctypes_type(np.int32)) @@ -237,10 +261,10 @@ async def test_call_peer_different_processes(): client_pid = client._child.pid assert is_process_running(client_pid) - await client.wait_peers_at_least(1) + await client.wait_for_at_least_n_peers(1) result = await client.call_peer_handler(peer_id, handler_name, test_input) - assert np.allclose(result, handle_square(test_input)) + assert np.allclose(result, test_input ** 2) response_received.value = 1 await client.shutdown() @@ -250,32 +274,67 @@ async def test_call_peer_different_processes(): @pytest.mark.parametrize( - "test_input,handle,replicate", + "test_input,expected", [ - pytest.param(np.random.randn(2, 3), handle_square, False, id="square_primary"), - pytest.param(np.random.randn(2, 3), handle_square, True, id="square_replica"), - pytest.param([np.random.randn(2, 3), np.random.randn(2, 3)], handle_add, False, id="add_primary"), - pytest.param([np.random.randn(2, 3), np.random.randn(2, 3)], handle_add, True, id="add_replica"), + pytest.param(torch.tensor([2]), torch.tensor(4)), + pytest.param( + torch.tensor([[1.0, 2.0], [0.5, 0.1]]), + torch.tensor([[1.0, 2.0], [0.5, 0.1]]) ** 2), ] ) @pytest.mark.asyncio -async def test_call_peer_numpy(test_input, handle, replicate, handler_name="handle"): - server_primary = await P2P.create() - server = await replicate_if_needed(server_primary, replicate) +async def test_call_peer_torch_square(test_input, expected, handler_name="handle"): + handle = handle_square_torch + server = await P2P.create() await server.add_stream_handler(handler_name, handle) nodes = boostrap_from([server]) - client_primary = await P2P.create(bootstrap=True, boostrap_peers=nodes) - client = await replicate_if_needed(client_primary, replicate) + client = await P2P.create(bootstrap=True, boostrap_peers=nodes) - await client.wait_peers_at_least(1) + await client.wait_for_at_least_n_peers(1) - result = await client.call_peer_handler(server.id, handler_name, test_input) - assert np.allclose(result, handle(test_input)) + inp = serialize_torch_tensor(test_input).SerializeToString() + resultPb = await client.call_peer_handler(server.id, handler_name, inp) + result = runtime_pb2.Tensor() + result.ParseFromString(resultPb) + result = deserialize_torch_tensor(result) + assert torch.allclose(result, expected) await server.stop_listening() - await server_primary.shutdown() - await client_primary.shutdown() + await server.shutdown() + await client.shutdown() + + +@pytest.mark.parametrize( + "test_input,expected", + [ + pytest.param([torch.tensor([1]), torch.tensor([2])], torch.tensor([3])), + pytest.param( + [torch.tensor([[0.1, 0.2], [0.3, 0.4]]), torch.tensor([[1.1, 1.2], [1.3, 1.4]])], + torch.tensor([[1.2, 1.4], [1.6, 1.8]])), + ] +) +@pytest.mark.asyncio +async def test_call_peer_torch_add(test_input, expected, handler_name="handle"): + handle = handle_add_torch + server = await P2P.create() + await server.add_stream_handler(handler_name, handle) + + nodes = boostrap_from([server]) + client = await P2P.create(bootstrap=True, boostrap_peers=nodes) + + await client.wait_for_at_least_n_peers(1) + + inp = [serialize_torch_tensor(i).SerializeToString() for i in test_input] + resultPb = await client.call_peer_handler(server.id, handler_name, inp) + result = runtime_pb2.Tensor() + result.ParseFromString(resultPb) + result = deserialize_torch_tensor(result) + assert torch.allclose(result, expected) + + await server.stop_listening() + await server.shutdown() + await client.shutdown() @pytest.mark.parametrize( @@ -289,16 +348,21 @@ async def test_call_peer_numpy(test_input, handle, replicate, handler_name="hand async def test_call_peer_error(replicate, handler_name="handle"): server_primary = await P2P.create() server = await replicate_if_needed(server_primary, replicate) - await server.add_stream_handler(handler_name, handle_add) + await server.add_stream_handler(handler_name, handle_add_torch) nodes = boostrap_from([server]) client_primary = await P2P.create(bootstrap=True, boostrap_peers=nodes) client = await replicate_if_needed(client_primary, replicate) - await client.wait_peers_at_least(1) - result = await client.call_peer_handler(server.id, handler_name, - [np.zeros((2, 3)), np.zeros((3, 2))]) - assert type(result) == ValueError + await client.wait_for_at_least_n_peers(1) + + inp = [serialize_torch_tensor(i).SerializeToString() for i in [torch.zeros((2, 3)), torch.zeros((3, 2))]] + result = await client.call_peer_handler(server.id, handler_name, inp) + assert type(result) == str + + await server.stop_listening() + await server_primary.shutdown() + await client_primary.shutdown() @pytest.mark.asyncio @@ -318,7 +382,7 @@ def handler(arg, key): nodes = boostrap_from([server_primary]) client = await P2P.create(bootstrap=True, boostrap_peers=nodes) - await client.wait_peers_at_least(1) + await client.wait_for_at_least_n_peers(1) result = await client.call_peer_handler(server_id, handler_name, "") assert result == "primary" diff --git a/tests/test_p2p_daemon_bindings.py b/tests/test_p2p_daemon_bindings.py index e9bc77213..9b0b1f966 100644 --- a/tests/test_p2p_daemon_bindings.py +++ b/tests/test_p2p_daemon_bindings.py @@ -35,7 +35,7 @@ def test_raise_if_failed_not_raises(): raise_if_failed(resp) -pairs_int_varint_valid = ( +pairs_int_serialized_valid = ( (0, b"\x00"), (1, b"\x01"), (128, b"\x80\x01"), @@ -43,7 +43,7 @@ def test_raise_if_failed_not_raises(): (2 ** 64 - 1, b"\xff\xff\xff\xff\xff\xff\xff\xff\xff\x01"), ) -pairs_int_varint_overflow = ( +pairs_int_serialized_overflow = ( (2 ** 64, b"\x80\x80\x80\x80\x80\x80\x80\x80\x80\x02"), (2 ** 64 + 1, b"\x81\x80\x80\x80\x80\x80\x80\x80\x80\x02"), ( @@ -67,15 +67,15 @@ class MockReaderWriter(MockReader, MockWriter): pass -@pytest.mark.parametrize("integer, var_integer", pairs_int_varint_valid) +@pytest.mark.parametrize("integer, serialized_integer", pairs_int_serialized_valid) @pytest.mark.asyncio -async def test_write_unsigned_varint(integer, var_integer): +async def test_write_unsigned_varint(integer, serialized_integer): s = MockWriter() await write_unsigned_varint(s, integer) - assert s.getvalue() == var_integer + assert s.getvalue() == serialized_integer -@pytest.mark.parametrize("integer", tuple(i[0] for i in pairs_int_varint_overflow)) +@pytest.mark.parametrize("integer", tuple(i[0] for i in pairs_int_serialized_overflow)) @pytest.mark.asyncio async def test_write_unsigned_varint_overflow(integer): s = MockWriter() @@ -91,18 +91,18 @@ async def test_write_unsigned_varint_negative(integer): await write_unsigned_varint(s, integer) -@pytest.mark.parametrize("integer, var_integer", pairs_int_varint_valid) +@pytest.mark.parametrize("integer, serialized_integer", pairs_int_serialized_valid) @pytest.mark.asyncio -async def test_read_unsigned_varint(integer, var_integer): - s = MockReader(var_integer) +async def test_read_unsigned_varint(integer, serialized_integer): + s = MockReader(serialized_integer) result = await read_unsigned_varint(s) assert result == integer -@pytest.mark.parametrize("var_integer", tuple(i[1] for i in pairs_int_varint_overflow)) +@pytest.mark.parametrize("serialized_integer", tuple(i[1] for i in pairs_int_serialized_overflow)) @pytest.mark.asyncio -async def test_read_unsigned_varint_overflow(var_integer): - s = MockReader(var_integer) +async def test_read_unsigned_varint_overflow(serialized_integer): + s = MockReader(serialized_integer) with pytest.raises(ValueError): await read_unsigned_varint(s) From cade39b6db92fa06419dd2903f5e8a043f51f45e Mon Sep 17 00:00:00 2001 From: Maxim Kashirin Date: Tue, 27 Apr 2021 17:50:40 +0300 Subject: [PATCH 30/81] remove unused constants --- tests/test_p2p_daemon.py | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/tests/test_p2p_daemon.py b/tests/test_p2p_daemon.py index 4e37d757c..a7dcb6223 100644 --- a/tests/test_p2p_daemon.py +++ b/tests/test_p2p_daemon.py @@ -16,17 +16,6 @@ from hivemind.p2p import P2P from hivemind.proto import dht_pb2, runtime_pb2 -RUNNING = 'running' -NOT_RUNNING = 'not running' -CHECK_PID_CMD = ''' -if ps -p {0} > /dev/null; -then - echo "{1}" -else - echo "{2}" -fi -''' - def is_process_running(pid: int) -> bool: return subprocess.run(["ps", "-p", str(pid)], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL).returncode == 0 From 5a057d5c0d4d9471f9fabbca0c3114ec06284ca0 Mon Sep 17 00:00:00 2001 From: Maxim Kashirin Date: Tue, 27 Apr 2021 17:53:01 +0300 Subject: [PATCH 31/81] remove unused import --- tests/test_p2p_daemon.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_p2p_daemon.py b/tests/test_p2p_daemon.py index a7dcb6223..7cf052556 100644 --- a/tests/test_p2p_daemon.py +++ b/tests/test_p2p_daemon.py @@ -6,7 +6,7 @@ import torch -from hivemind.utils.compression import serialize_torch_tensor, construct_torch_tensor, deserialize_torch_tensor +from hivemind.utils.compression import serialize_torch_tensor, deserialize_torch_tensor from hivemind.p2p.p2p_daemon_bindings.datastructures import ID From f73825126883354fd60c2c230683c673708c452d Mon Sep 17 00:00:00 2001 From: Maxim Kashirin Date: Wed, 28 Apr 2021 16:18:38 +0300 Subject: [PATCH 32/81] stream handler operates with bytes, unary handler works with errors --- hivemind/p2p/p2p_daemon.py | 61 +++++++++++++++----- hivemind/proto/p2pd.proto | 4 ++ tests/test_p2p_daemon.py | 112 +++++++++++++++++++++++++++---------- 3 files changed, 131 insertions(+), 46 deletions(-) diff --git a/hivemind/p2p/p2p_daemon.py b/hivemind/p2p/p2p_daemon.py index 47a2610b1..a14e270a7 100644 --- a/hivemind/p2p/p2p_daemon.py +++ b/hivemind/p2p/p2p_daemon.py @@ -10,6 +10,7 @@ import hivemind.p2p.p2p_daemon_bindings.p2pclient as p2pclient from hivemind.p2p.p2p_daemon_bindings.datastructures import ID, StreamInfo +from hivemind.proto import p2pd_pb2 from hivemind.utils import MSGPackSerializer from hivemind.utils.logging import get_logger from hivemind.utils.networking import find_open_port @@ -38,6 +39,9 @@ class P2P(object): RETRY_DELAY = 0.4 HEADER_LEN = 8 BYTEORDER = 'big' + PB_HEADER_LEN = 1 + RESULT_MESSAGE = int(0).to_bytes(PB_HEADER_LEN, BYTEORDER) + ERROR_MESSAGE = int(1).to_bytes(PB_HEADER_LEN, BYTEORDER) class InterruptedError(Exception): pass @@ -146,27 +150,41 @@ async def send_raw_data(byte_str, writer): writer.write(request) @staticmethod - async def send_data(data, writer): + async def send_message_pack(data, writer): raw_data = MSGPackSerializer.dumps(data) await P2P.send_raw_data(raw_data, writer) @staticmethod async def send_protobuf(protobuf, out_proto_type, writer): if type(protobuf) != out_proto_type: - error = TypeError('Unary handler returned protobuf of wrong type.') - await P2P.send_raw_data(MSGPackSerializer.dumps(error), writer) - raise error + raise TypeError('Unary handler returned protobuf of wrong type.') await P2P.send_raw_data(protobuf.SerializeToString(), writer) @staticmethod - async def receive_raw_data(reader: asyncio.StreamReader): - header = await reader.readexactly(P2P.HEADER_LEN) + async def send_protobuf_with_error(protobuf, out_proto_type, writer): + if type(protobuf) != out_proto_type: + raise TypeError('Unary handler returned protobuf of wrong type.') + if out_proto_type == p2pd_pb2.P2PRPCError: + await P2P.send_raw_data(P2P.ERROR_MESSAGE, writer) + else: + await P2P.send_raw_data(P2P.RESULT_MESSAGE, writer) + + await P2P.send_raw_data(protobuf.SerializeToString(), writer) + + @staticmethod + async def send_error_protobuf(protobuf, out_proto_type, writer): + await P2P.send_raw_data(P2P.RESULT_MESSAGE, writer) + await P2P.send_raw_data(protobuf.SerializeToString(), writer) + + @staticmethod + async def receive_raw_data(reader: asyncio.StreamReader, header_len=HEADER_LEN): + header = await reader.readexactly(header_len) content_length = int.from_bytes(header, P2P.BYTEORDER) data = await reader.readexactly(content_length) return data @staticmethod - async def receive_data(reader): + async def receive_message_pack(reader): return MSGPackSerializer.loads(await P2P.receive_raw_data(reader)) @staticmethod @@ -175,20 +193,32 @@ async def receive_protobuf(in_proto_type, reader): protobuf.ParseFromString(await P2P.receive_raw_data(reader)) return protobuf + @staticmethod + async def receive_protobuf_with_error(in_proto_type, reader): + msg_type = await P2P.receive_raw_data(reader) + if msg_type == P2P.RESULT_MESSAGE: + protobuf = in_proto_type() + protobuf.ParseFromString(await P2P.receive_raw_data(reader)) + return protobuf, None + elif msg_type == P2P.ERROR_MESSAGE: + protobuf = p2pd_pb2.P2PRPCError() + protobuf.ParseFromString(await P2P.receive_raw_data(reader)) + return None, protobuf + else: + raise TypeError('invalid protobuf message type') + @staticmethod def _handle_stream(handle): async def do_handle_stream(stream_info, reader, writer): try: - request = await P2P.receive_data(reader) + request = await P2P.receive_raw_data(reader) # receive raw data except asyncio.IncompleteReadError: logger.debug("Incomplete read while receiving request from peer") writer.close() return try: result = handle(request) - await P2P.send_data(result, writer) - except Exception as exc: - await P2P.send_data(str(exc), writer) + await P2P.send_raw_data(result, writer) finally: writer.close() @@ -219,11 +249,12 @@ async def do_handle_unary_stream( return_when=asyncio.FIRST_COMPLETED) try: result = done.pop().result() - await P2P.send_protobuf(result, out_proto_type, writer) + await P2P.send_protobuf_with_error(result, out_proto_type, writer) except P2P.InterruptedError: pass except Exception as exc: - await P2P.send_data(str(exc), writer) + error = p2pd_pb2.P2PRPCError(message=str(exc)) + await P2P.send_protobuf_with_error(error, p2pd_pb2.P2PRPCError, writer) finally: pending_task = pending.pop() pending_task.cancel() @@ -269,8 +300,8 @@ async def call_peer_handler(self, peer_id, handler_name, input_data): libp2p_peer_id = ID.from_base58(peer_id) stream_info, reader, writer = await self._client.stream_open(libp2p_peer_id, (handler_name,)) try: - await P2P.send_data(input_data, writer) - return await P2P.receive_data(reader) + await P2P.send_raw_data(input_data, writer) + return await P2P.receive_raw_data(reader) finally: writer.close() diff --git a/hivemind/proto/p2pd.proto b/hivemind/proto/p2pd.proto index 8eb3e7e17..f559bcb8d 100644 --- a/hivemind/proto/p2pd.proto +++ b/hivemind/proto/p2pd.proto @@ -160,3 +160,7 @@ message PSResponse { repeated string topics = 1; repeated bytes peerIDs = 2; } + +message P2PRPCError { + required string message = 1; +} \ No newline at end of file diff --git a/tests/test_p2p_daemon.py b/tests/test_p2p_daemon.py index 7cf052556..709387227 100644 --- a/tests/test_p2p_daemon.py +++ b/tests/test_p2p_daemon.py @@ -6,6 +6,7 @@ import torch +from hivemind.utils import MSGPackSerializer from hivemind.utils.compression import serialize_torch_tensor, deserialize_torch_tensor from hivemind.p2p.p2p_daemon_bindings.datastructures import ID @@ -79,14 +80,16 @@ async def test_daemon_replica_does_not_affect_primary(): def handle_square(x): - return x ** 2 + x = MSGPackSerializer.loads(x) + return MSGPackSerializer.dumps(x ** 2) def handle_add(args): + args = MSGPackSerializer.loads(args) result = args[0] for i in range(1, len(args)): result = result + args[i] - return result + return MSGPackSerializer.dumps(result) def handle_square_torch(x): @@ -98,6 +101,7 @@ def handle_square_torch(x): def handle_add_torch(args): + args = MSGPackSerializer.loads(args) tensor = runtime_pb2.Tensor() tensor.ParseFromString(args[0]) result = deserialize_torch_tensor(tensor) @@ -110,6 +114,13 @@ def handle_add_torch(args): return serialize_torch_tensor(result).SerializeToString() +def handle_add_torch_with_exc(args): + try: + return handle_add_torch(args) + except: + return b'something went wrong :(' + + @pytest.mark.parametrize( 'should_cancel,replicate', [ (True, False), @@ -157,14 +168,15 @@ async def ping_handler(request, context): libp2p_server_id = ID.from_base58(server.id) stream_info, reader, writer = await client._client.stream_open(libp2p_server_id, (handle_name,)) - await P2P.send_raw_data(ping_request.SerializeToString(), writer) + await P2P.send_protobuf(ping_request, dht_pb2.PingRequest, writer) if should_cancel: writer.close() await asyncio.sleep(1) assert handler_cancelled else: - result = await P2P.receive_protobuf(dht_pb2.PingResponse, reader) + result, err = await P2P.receive_protobuf_with_error(dht_pb2.PingResponse, reader) + assert err is None assert result == expected_response assert not handler_cancelled @@ -176,17 +188,49 @@ async def ping_handler(request, context): assert not is_process_running(client_pid) +@pytest.mark.asyncio +async def test_call_unary_handler_error(handle_name="handle"): + async def error_handler(request, context): + raise ValueError('boom') + + server = await P2P.create() + server_pid = server._child.pid + await server.add_unary_handler(handle_name, error_handler, dht_pb2.PingRequest, dht_pb2.PingResponse) + assert is_process_running(server_pid) + + nodes = boostrap_from([server]) + client = await P2P.create(bootstrap=True, boostrap_peers=nodes) + client_pid = client._child.pid + assert is_process_running(client_pid) + await client.wait_for_at_least_n_peers(1) + + ping_request = dht_pb2.PingRequest( + peer=dht_pb2.NodeInfo(node_id=client.id.encode(), rpc_port=client._host_port), + validate=True) + libp2p_server_id = ID.from_base58(server.id) + stream_info, reader, writer = await client._client.stream_open(libp2p_server_id, (handle_name,)) + + await P2P.send_protobuf(ping_request, dht_pb2.PingRequest, writer) + result, err = await P2P.receive_protobuf_with_error(dht_pb2.PingResponse, reader) + assert result is None + assert err.message == 'boom' + + await server.stop_listening() + await server.shutdown() + await client.shutdown() + + @pytest.mark.parametrize( - "test_input,handle", + "test_input,expected,handle", [ - pytest.param(10, handle_square, id="square_integer"), - pytest.param((1, 2), handle_add, id="add_integers"), - pytest.param(([1, 2, 3], [12, 13]), handle_add, id="add_lists"), - pytest.param(2, lambda x: x ** 3, id="lambda") + pytest.param(10, 100, handle_square, id="square_integer"), + pytest.param((1, 2), 3, handle_add, id="add_integers"), + pytest.param(([1, 2, 3], [12, 13]), [1, 2, 3, 12, 13], handle_add, id="add_lists"), + pytest.param(2, 8, lambda x: MSGPackSerializer.dumps(MSGPackSerializer.loads(x) ** 3), id="lambda") ] ) @pytest.mark.asyncio -async def test_call_peer_single_process(test_input, handle, handler_name="handle"): +async def test_call_peer_single_process(test_input, expected, handle, handler_name="handle"): server = await P2P.create() server_pid = server._child.pid await server.add_stream_handler(handler_name, handle) @@ -199,8 +243,10 @@ async def test_call_peer_single_process(test_input, handle, handler_name="handle await client.wait_for_at_least_n_peers(1) - result = await client.call_peer_handler(server.id, handler_name, test_input) - assert result == handle(test_input) + test_input_msgp = MSGPackSerializer.dumps(test_input) + result_msgp = await client.call_peer_handler(server.id, handler_name, test_input_msgp) + result = MSGPackSerializer.loads(result_msgp) + assert result == expected await server.stop_listening() await server.shutdown() @@ -252,7 +298,9 @@ async def test_call_peer_different_processes(): await client.wait_for_at_least_n_peers(1) - result = await client.call_peer_handler(peer_id, handler_name, test_input) + test_input_msgp = MSGPackSerializer.dumps(2) + result_msgp = await client.call_peer_handler(peer_id, handler_name, test_input_msgp) + result = MSGPackSerializer.loads(result_msgp) assert np.allclose(result, test_input ** 2) response_received.value = 1 @@ -283,9 +331,9 @@ async def test_call_peer_torch_square(test_input, expected, handler_name="handle await client.wait_for_at_least_n_peers(1) inp = serialize_torch_tensor(test_input).SerializeToString() - resultPb = await client.call_peer_handler(server.id, handler_name, inp) + result_pb = await client.call_peer_handler(server.id, handler_name, inp) result = runtime_pb2.Tensor() - result.ParseFromString(resultPb) + result.ParseFromString(result_pb) result = deserialize_torch_tensor(result) assert torch.allclose(result, expected) @@ -315,9 +363,10 @@ async def test_call_peer_torch_add(test_input, expected, handler_name="handle"): await client.wait_for_at_least_n_peers(1) inp = [serialize_torch_tensor(i).SerializeToString() for i in test_input] - resultPb = await client.call_peer_handler(server.id, handler_name, inp) + inp_msgp = MSGPackSerializer.dumps(inp) + result_pb = await client.call_peer_handler(server.id, handler_name, inp_msgp) result = runtime_pb2.Tensor() - result.ParseFromString(resultPb) + result.ParseFromString(result_pb) result = deserialize_torch_tensor(result) assert torch.allclose(result, expected) @@ -337,7 +386,7 @@ async def test_call_peer_torch_add(test_input, expected, handler_name="handle"): async def test_call_peer_error(replicate, handler_name="handle"): server_primary = await P2P.create() server = await replicate_if_needed(server_primary, replicate) - await server.add_stream_handler(handler_name, handle_add_torch) + await server.add_stream_handler(handler_name, handle_add_torch_with_exc) nodes = boostrap_from([server]) client_primary = await P2P.create(bootstrap=True, boostrap_peers=nodes) @@ -346,8 +395,9 @@ async def test_call_peer_error(replicate, handler_name="handle"): await client.wait_for_at_least_n_peers(1) inp = [serialize_torch_tensor(i).SerializeToString() for i in [torch.zeros((2, 3)), torch.zeros((3, 2))]] - result = await client.call_peer_handler(server.id, handler_name, inp) - assert type(result) == str + inp_msgp = MSGPackSerializer.dumps(inp) + result = await client.call_peer_handler(server.id, handler_name, inp_msgp) + assert result == b'something went wrong :(' await server.stop_listening() await server_primary.shutdown() @@ -361,35 +411,35 @@ def handler(arg, key): server_primary = await P2P.create(bootstrap=False) server_id = server_primary.id - await server_primary.add_stream_handler(handler_name, partial(handler, key="primary")) + await server_primary.add_stream_handler(handler_name, partial(handler, key=b'primary')) server_replica1 = await replicate_if_needed(server_primary, True) - await server_replica1.add_stream_handler(handler_name + "1", partial(handler, key="replica1")) + await server_replica1.add_stream_handler(handler_name + '1', partial(handler, key=b'replica1')) server_replica2 = await replicate_if_needed(server_primary, True) - await server_replica2.add_stream_handler(handler_name + "2", partial(handler, key="replica2")) + await server_replica2.add_stream_handler(handler_name + '2', partial(handler, key=b'replica2')) nodes = boostrap_from([server_primary]) client = await P2P.create(bootstrap=True, boostrap_peers=nodes) await client.wait_for_at_least_n_peers(1) - result = await client.call_peer_handler(server_id, handler_name, "") - assert result == "primary" + result = await client.call_peer_handler(server_id, handler_name, b'1') + assert result == b"primary" - result = await client.call_peer_handler(server_id, handler_name + "1", "") - assert result == "replica1" + result = await client.call_peer_handler(server_id, handler_name + '1', b'2') + assert result == b"replica1" - result = await client.call_peer_handler(server_id, handler_name + "2", "") - assert result == "replica2" + result = await client.call_peer_handler(server_id, handler_name + '2', b'3') + assert result == b"replica2" await server_replica1.stop_listening() await server_replica2.stop_listening() # Primary does not handle replicas protocols with pytest.raises(asyncio.IncompleteReadError): - await client.call_peer_handler(server_id, handler_name + "1", "") + await client.call_peer_handler(server_id, handler_name + '1', b'') with pytest.raises(asyncio.IncompleteReadError): - await client.call_peer_handler(server_id, handler_name + "2", "") + await client.call_peer_handler(server_id, handler_name + '2', b'') await server_primary.stop_listening() await server_primary.shutdown() From c46a835f6be5cb6caf7f58ec57682473b428c680 Mon Sep 17 00:00:00 2001 From: Maxim Kashirin Date: Wed, 28 Apr 2021 17:32:27 +0300 Subject: [PATCH 33/81] fix pr comments --- hivemind/p2p/p2p_daemon.py | 25 ++++++++++--------------- hivemind/proto/p2pd.proto | 2 +- tests/test_p2p_daemon.py | 6 +++--- 3 files changed, 14 insertions(+), 19 deletions(-) diff --git a/hivemind/p2p/p2p_daemon.py b/hivemind/p2p/p2p_daemon.py index a14e270a7..9b89fd5dc 100644 --- a/hivemind/p2p/p2p_daemon.py +++ b/hivemind/p2p/p2p_daemon.py @@ -40,8 +40,8 @@ class P2P(object): HEADER_LEN = 8 BYTEORDER = 'big' PB_HEADER_LEN = 1 - RESULT_MESSAGE = int(0).to_bytes(PB_HEADER_LEN, BYTEORDER) - ERROR_MESSAGE = int(1).to_bytes(PB_HEADER_LEN, BYTEORDER) + RESULT_MESSAGE = b'\x00' + ERROR_MESSAGE = b'\x01' class InterruptedError(Exception): pass @@ -161,21 +161,16 @@ async def send_protobuf(protobuf, out_proto_type, writer): await P2P.send_raw_data(protobuf.SerializeToString(), writer) @staticmethod - async def send_protobuf_with_error(protobuf, out_proto_type, writer): + async def send_protobuf_or_error(protobuf, out_proto_type, writer): if type(protobuf) != out_proto_type: raise TypeError('Unary handler returned protobuf of wrong type.') - if out_proto_type == p2pd_pb2.P2PRPCError: + if out_proto_type == p2pd_pb2.RPCError: await P2P.send_raw_data(P2P.ERROR_MESSAGE, writer) else: await P2P.send_raw_data(P2P.RESULT_MESSAGE, writer) await P2P.send_raw_data(protobuf.SerializeToString(), writer) - @staticmethod - async def send_error_protobuf(protobuf, out_proto_type, writer): - await P2P.send_raw_data(P2P.RESULT_MESSAGE, writer) - await P2P.send_raw_data(protobuf.SerializeToString(), writer) - @staticmethod async def receive_raw_data(reader: asyncio.StreamReader, header_len=HEADER_LEN): header = await reader.readexactly(header_len) @@ -194,14 +189,14 @@ async def receive_protobuf(in_proto_type, reader): return protobuf @staticmethod - async def receive_protobuf_with_error(in_proto_type, reader): + async def receive_protobuf_or_error(in_proto_type, reader): msg_type = await P2P.receive_raw_data(reader) if msg_type == P2P.RESULT_MESSAGE: protobuf = in_proto_type() protobuf.ParseFromString(await P2P.receive_raw_data(reader)) return protobuf, None elif msg_type == P2P.ERROR_MESSAGE: - protobuf = p2pd_pb2.P2PRPCError() + protobuf = p2pd_pb2.RPCError() protobuf.ParseFromString(await P2P.receive_raw_data(reader)) return None, protobuf else: @@ -211,7 +206,7 @@ async def receive_protobuf_with_error(in_proto_type, reader): def _handle_stream(handle): async def do_handle_stream(stream_info, reader, writer): try: - request = await P2P.receive_raw_data(reader) # receive raw data + request = await P2P.receive_raw_data(reader) except asyncio.IncompleteReadError: logger.debug("Incomplete read while receiving request from peer") writer.close() @@ -249,12 +244,12 @@ async def do_handle_unary_stream( return_when=asyncio.FIRST_COMPLETED) try: result = done.pop().result() - await P2P.send_protobuf_with_error(result, out_proto_type, writer) + await P2P.send_protobuf_or_error(result, out_proto_type, writer) except P2P.InterruptedError: pass except Exception as exc: - error = p2pd_pb2.P2PRPCError(message=str(exc)) - await P2P.send_protobuf_with_error(error, p2pd_pb2.P2PRPCError, writer) + error = p2pd_pb2.RPCError(message=str(exc)) + await P2P.send_protobuf_or_error(error, p2pd_pb2.RPCError, writer) finally: pending_task = pending.pop() pending_task.cancel() diff --git a/hivemind/proto/p2pd.proto b/hivemind/proto/p2pd.proto index f559bcb8d..dc65514e5 100644 --- a/hivemind/proto/p2pd.proto +++ b/hivemind/proto/p2pd.proto @@ -161,6 +161,6 @@ message PSResponse { repeated bytes peerIDs = 2; } -message P2PRPCError { +message RPCError { required string message = 1; } \ No newline at end of file diff --git a/tests/test_p2p_daemon.py b/tests/test_p2p_daemon.py index 709387227..4b9e40563 100644 --- a/tests/test_p2p_daemon.py +++ b/tests/test_p2p_daemon.py @@ -117,7 +117,7 @@ def handle_add_torch(args): def handle_add_torch_with_exc(args): try: return handle_add_torch(args) - except: + except Exception: return b'something went wrong :(' @@ -175,7 +175,7 @@ async def ping_handler(request, context): await asyncio.sleep(1) assert handler_cancelled else: - result, err = await P2P.receive_protobuf_with_error(dht_pb2.PingResponse, reader) + result, err = await P2P.receive_protobuf_or_error(dht_pb2.PingResponse, reader) assert err is None assert result == expected_response assert not handler_cancelled @@ -211,7 +211,7 @@ async def error_handler(request, context): stream_info, reader, writer = await client._client.stream_open(libp2p_server_id, (handle_name,)) await P2P.send_protobuf(ping_request, dht_pb2.PingRequest, writer) - result, err = await P2P.receive_protobuf_with_error(dht_pb2.PingResponse, reader) + result, err = await P2P.receive_protobuf_or_error(dht_pb2.PingResponse, reader) assert result is None assert err.message == 'boom' From 09b55e51f380e91868f63fc35807b67944b1ae07 Mon Sep 17 00:00:00 2001 From: Denis Mazur Date: Sun, 2 May 2021 03:57:24 +0300 Subject: [PATCH 34/81] fix setup.py --- setup.py | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/setup.py b/setup.py index cb1e0a6a5..6f17efc5f 100644 --- a/setup.py +++ b/setup.py @@ -3,6 +3,7 @@ import hashlib import os import re +import shlex import subprocess import tarfile import tempfile @@ -16,6 +17,8 @@ P2PD_VERSION = 'v0.3.1' P2PD_CHECKSUM = '8810097959db720208cdc9f2945804a4' +LIBP2P_TAR_URL = f'https://github.com/learning-at-home/go-libp2p-daemon/archive/refs/tags/{P2PD_VERSION}.tar.gz' + here = os.path.abspath(os.path.dirname(__file__)) @@ -62,20 +65,18 @@ def libp2p_build_install(): raise FileNotFoundError('could not find golang installation') with tempfile.TemporaryDirectory() as tempdir: - url = f'https://github.com/learning-at-home/go-libp2p-daemon/archive/refs/tags/{P2PD_VERSION}.tar.gz' dest = os.path.join(tempdir, 'libp2p-daemon.tar.gz') - urllib.request.urlretrieve(url, dest) + urllib.request.urlretrieve(LIBP2P_TAR_URL, dest) - tar = tarfile.open(dest, 'r:gz') - tar.extractall(tempdir) - tar.close() + with tarfile.open(dest, 'r:gz') as tar: + tar.extractall(tempdir) - result = subprocess.run(['go', 'build', '-o', os.path.join(here, "hivemind", "hivemind_cli", "p2pd")], - cwd=os.path.join(tempdir, f'go-libp2p-daemon-{P2PD_VERSION[1:]}', 'p2pd')) + result = subprocess.run(f'go build -o {shlex.quote(os.path.join(here, "hivemind", "hivemind_cli", "p2pd"))}', + cwd=os.path.join(tempdir, f'go-libp2p-daemon-{P2PD_VERSION[1:]}', 'p2pd'), shell=True) if result.returncode: raise RuntimeError('Failed to build or install libp2p-daemon:' - f' exited with status code :{result.returncode}') + f' exited with status code: {result.returncode}') def libp2p_download_install(): @@ -86,7 +87,6 @@ def libp2p_download_install(): url = f'https://github.com/learning-at-home/go-libp2p-daemon/releases/download/{P2PD_VERSION}/p2pd' urllib.request.urlretrieve(url, binary_path) os.chmod(binary_path, 0o777) - print(md5(binary_path)) if md5(binary_path) != P2PD_CHECKSUM: raise RuntimeError(f'Downloaded p2pd binary from {url} does not match with md5 checksum') @@ -94,7 +94,7 @@ def libp2p_download_install(): class Install(install): def run(self): libp2p_download_install() - proto_compile(os.path.join(self.build_lib, 'hivemind', 'proto')) + # proto_compile(os.path.join(self.build_lib, 'hivemind', 'proto')) super().run() From 6abd88fd0e3e21c25f72fee7e557e231c3ac429e Mon Sep 17 00:00:00 2001 From: Denis Mazur Date: Sun, 2 May 2021 04:19:12 +0300 Subject: [PATCH 35/81] replace popen with subprocess.run --- setup.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/setup.py b/setup.py index 6f17efc5f..7218f40c0 100644 --- a/setup.py +++ b/setup.py @@ -52,9 +52,7 @@ def proto_compile(output_path): def libp2p_build_install(): try: - proc = subprocess.Popen(['go', 'version'], stdout=subprocess.PIPE) - result, _ = proc.communicate() - result = result.decode('ascii', 'replace') + result = subprocess.run("go version", capture_output=True).stdout.decode('ascii', 'replace') m = re.search(r'^go version go([\d.]+)', result) v = m.group(1) From 269e9c526b48b47e60d41df83a220a87df931a64 Mon Sep 17 00:00:00 2001 From: Denis Mazur Date: Sun, 2 May 2021 04:28:48 +0300 Subject: [PATCH 36/81] fix setup.py --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 7218f40c0..030808e82 100644 --- a/setup.py +++ b/setup.py @@ -52,7 +52,7 @@ def proto_compile(output_path): def libp2p_build_install(): try: - result = subprocess.run("go version", capture_output=True).stdout.decode('ascii', 'replace') + result = subprocess.run("go version", capture_output=True, shell=True).stdout.decode('ascii', 'replace') m = re.search(r'^go version go([\d.]+)', result) v = m.group(1) From 2fce24e55d1b307f3e679cef58b9313115ad585e Mon Sep 17 00:00:00 2001 From: Denis Mazur Date: Sun, 2 May 2021 04:31:17 +0300 Subject: [PATCH 37/81] remove debug comment --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 030808e82..fe21524e2 100644 --- a/setup.py +++ b/setup.py @@ -92,7 +92,7 @@ def libp2p_download_install(): class Install(install): def run(self): libp2p_download_install() - # proto_compile(os.path.join(self.build_lib, 'hivemind', 'proto')) + proto_compile(os.path.join(self.build_lib, 'hivemind', 'proto')) super().run() From 53361c5c614b3cf47c3e6b9a7a61b376a90c60e9 Mon Sep 17 00:00:00 2001 From: Maxim Kashirin Date: Mon, 3 May 2021 14:15:11 +0300 Subject: [PATCH 38/81] fix comments in p2p and p2p_bindings --- hivemind/p2p/p2p_daemon.py | 219 ++++++++++-------- hivemind/p2p/p2p_daemon_bindings/control.py | 88 ++++--- .../p2p/p2p_daemon_bindings/datastructures.py | 97 ++++---- hivemind/p2p/p2p_daemon_bindings/p2pclient.py | 10 +- hivemind/p2p/p2p_daemon_bindings/utils.py | 1 - tests/test_p2p_daemon.py | 30 +-- tests/test_p2p_daemon_bindings.py | 34 +-- 7 files changed, 244 insertions(+), 235 deletions(-) diff --git a/hivemind/p2p/p2p_daemon.py b/hivemind/p2p/p2p_daemon.py index 9b89fd5dc..40e1b9f8b 100644 --- a/hivemind/p2p/p2p_daemon.py +++ b/hivemind/p2p/p2p_daemon.py @@ -1,15 +1,16 @@ import asyncio -import copy -import dataclasses -import subprocess -import typing as tp -from pathlib import Path +from copy import deepcopy +from dataclasses import dataclass +from importlib.resources import path +from subprocess import Popen +from typing import List, Optional import google.protobuf from multiaddr import Multiaddr +import hivemind.hivemind_cli as cli import hivemind.p2p.p2p_daemon_bindings.p2pclient as p2pclient -from hivemind.p2p.p2p_daemon_bindings.datastructures import ID, StreamInfo +from hivemind.p2p.p2p_daemon_bindings.datastructures import PeerID, StreamInfo from hivemind.proto import p2pd_pb2 from hivemind.utils import MSGPackSerializer from hivemind.utils.logging import get_logger @@ -18,33 +19,45 @@ logger = get_logger(__name__) -@dataclasses.dataclass(frozen=False) +P2PD_RELATIVE_PATH = 'hivemind_cli/p2pd' +NUM_RETRIES = 3 +RETRY_DELAY = 0.4 + + +class P2PInterruptedError(Exception): + pass + + +@dataclass(frozen=False) class P2PContext(object): - ours_id: str - ours_port: int + id: str + port: int handle_name: str - peer_id: ID = None + peer_id: PeerID = None peer_addr: Multiaddr = None -class P2P(object): +class P2P: """ Forks a child process and executes p2pd command with given arguments. Can be used for peer to peer communication and procedure calls. Sends SIGKILL to the child in destructor. """ - P2PD_RELATIVE_PATH = 'hivemind_cli/p2pd' - NUM_RETRIES = 3 - RETRY_DELAY = 0.4 HEADER_LEN = 8 BYTEORDER = 'big' PB_HEADER_LEN = 1 RESULT_MESSAGE = b'\x00' ERROR_MESSAGE = b'\x01' - - class InterruptedError(Exception): - pass + DHT_MODE_MAPPING = { + 'dht': {'dht': 1}, + 'dht_server': {'dhtServer': 1}, + 'dht_client': {'dhtClient': 1}, + } + FORCE_REACHABILITY_MAPPING = { + 'public': {'forceReachabilityPublic': 1}, + 'private': {'forceReachabilityPrivate': 1}, + } def __init__(self): self._child = None @@ -53,44 +66,74 @@ def __init__(self): self._server_stopped = asyncio.Event() @classmethod - async def create(cls, *args, quic=1, tls=1, conn_manager=1, dht_mode='dht_server', force_reachability=None, - nat_port_map=True, auto_nat=True, bootstrap=False, boostrap_peers=None, use_global_ipfs=False, - host_port: int = None, daemon_listen_port: int = None, **kwargs): - if bootstrap and boostrap_peers is None and not use_global_ipfs: - raise AttributeError('Trying to create with bootstrap node without bootstrap nodes list. ' - 'It is very dangerous, because p2pd connects to global ipfs and it is very unstable. ' - 'If you really want this, pass use_global_ipfs=True') - if boostrap_peers is not None and use_global_ipfs: - raise AttributeError('Non empty boostrap_nodes and use_global_ipfs=True are incompatible.' - 'Choose one option: your nodes list (preferable) or global ipfs (very unstable)') + async def create(cls, *args, quic: bool = True, tls: bool = True, conn_manager: bool = True, + dht_mode: str = 'dht_server', force_reachability: Optional[str] = None, + nat_port_map: bool = True, auto_nat: bool = True, bootstrap: bool = False, + bootstrap_peers: Optional[List[str]] = None, use_global_ipfs: bool = False, host_port: int = None, + daemon_listen_port: int = None, **kwargs): + """ + Start a new p2pd process and connect to it. + @param args: + @param quic: Enables the QUIC transport + @param tls: Enables TLS1.3 channel security protocol + @param conn_manager: Enables the Connection Manager + @param dht_mode: DHT mode (dht_client/dht_server/dht) + @param force_reachability: Force reachability mode (public/private) + @param nat_port_map: Enables NAT port mapping + @param auto_nat: Enables the AutoNAT service + @param bootstrap: Connects to bootstrap peers and bootstraps the dht if enabled + @param bootstrap_peers: List of bootstrap peers; defaults to the IPFS DHT peers + @param use_global_ipfs: Bootstrap to global ipfs (works only if bootstrap=True and bootstrap_peers=None) + @param host_port: port for p2p network + @param daemon_listen_port: port for connection daemon and client binding + @param kwargs: + @return: new wrapper for p2p daemon + """ + + assert not (bootstrap and bootstrap_peers is None and not use_global_ipfs), \ + 'Trying to create with bootstrap node without bootstrap nodes list. ' \ + 'It is very dangerous, because p2pd connects to global ipfs and it is very unstable. ' \ + 'If you really want this, pass use_global_ipfs=True' + assert not (bootstrap_peers is not None and use_global_ipfs), \ + 'Non empty boostrap_nodes and use_global_ipfs=True are incompatible.' \ + 'Choose one option: your nodes list (preferable) or global ipfs (very unstable)' self = cls() - p2pd_path = Path(__file__).resolve().parents[1] / P2P.P2PD_RELATIVE_PATH - bpeers = cls._make_bootstrap_peers(boostrap_peers) - dht = cls._make_dht_mode(dht_mode) - freachability = cls._make_force_reachability(force_reachability) + with path(cli, 'p2pd') as p: + p2pd_path = p + bootstrap_peers = cls._make_bootstrap_peers(bootstrap_peers) + dht = cls.DHT_MODE_MAPPING.get(dht_mode, {'dht': 0}) + force_reachability = cls.FORCE_REACHABILITY_MAPPING.get(force_reachability, {}) proc_args = self._make_process_args( str(p2pd_path), *args, quic=quic, tls=tls, connManager=conn_manager, natPortMap=nat_port_map, autonat=auto_nat, - b=bootstrap, **{**bpeers, **dht, **freachability, **kwargs}) + b=bootstrap, **{**bootstrap_peers, **dht, **force_reachability, **kwargs}) self._assign_daemon_ports(host_port, daemon_listen_port) - for try_count in range(self.NUM_RETRIES): + + for try_count in range(NUM_RETRIES): try: self._initialize(proc_args) - await self._identify_client(P2P.RETRY_DELAY * (2 ** try_count)) + await self._wait_for_client(RETRY_DELAY * (2 ** try_count)) + break except Exception as e: logger.debug(f"Failed to initialize p2p daemon: {e}") - self._kill_child() - if try_count == P2P.NUM_RETRIES - 1: + self._terminate() + if try_count == NUM_RETRIES - 1: raise self._assign_daemon_ports() - continue - break + return self @classmethod async def replicate(cls, daemon_listen_port: int, host_port: int): + """ + Connect to existing p2p daemon + @param host_port: port for p2p network + @param daemon_listen_port: port for connection daemon and client binding + @return: new wrapper for existing p2p daemon + """ + self = cls() # There is no child under control # Use external already running p2pd @@ -101,48 +144,45 @@ async def replicate(cls, daemon_listen_port: int, host_port: int): self._client = p2pclient.Client( Multiaddr(f'/ip4/127.0.0.1/tcp/{self._daemon_listen_port}'), Multiaddr(f'/ip4/127.0.0.1/tcp/{self._client_listen_port}')) - await self._identify_client(0) + await self._wait_for_client() return self - async def wait_for_at_least_n_peers(self, n_peers, attempts=3): + async def wait_for_at_least_n_peers(self, n_peers, attempts=3, delay=1): for _ in range(attempts): peers = await self._client.list_peers() if len(peers) >= n_peers: return - await asyncio.sleep(1) + await asyncio.sleep(delay) raise RuntimeError('Not enough peers') - def _initialize(self, proc_args: tp.List[str]) -> None: - proc_args = copy.deepcopy(proc_args) + def _initialize(self, proc_args: List[str]) -> None: + proc_args = deepcopy(proc_args) proc_args.extend(self._make_process_args( hostAddrs=f'/ip4/0.0.0.0/tcp/{self._host_port},/ip4/0.0.0.0/udp/{self._host_port}/quic', listen=f'/ip4/127.0.0.1/tcp/{self._daemon_listen_port}' )) - self._child = subprocess.Popen( - args=proc_args, - stdin=subprocess.PIPE, stdout=subprocess.PIPE, - stderr=subprocess.PIPE, encoding="utf8" - ) + self._child = Popen(args=proc_args, encoding="utf8") self._alive = True self._client_listen_port = find_open_port() self._client = p2pclient.Client( Multiaddr(f'/ip4/127.0.0.1/tcp/{self._daemon_listen_port}'), Multiaddr(f'/ip4/127.0.0.1/tcp/{self._client_listen_port}')) - async def _identify_client(self, delay): + async def _wait_for_client(self, delay=0): await asyncio.sleep(delay) encoded = await self._client.identify() self.id = encoded[0].to_base58() def _assign_daemon_ports(self, host_port=None, daemon_listen_port=None): - self._host_port, self._daemon_listen_port = host_port, daemon_listen_port if host_port is None: - self._host_port = find_open_port() + host_port = find_open_port() if daemon_listen_port is None: - self._daemon_listen_port = find_open_port() - while self._daemon_listen_port == self._host_port: - self._daemon_listen_port = find_open_port() + daemon_listen_port = find_open_port() + while daemon_listen_port == host_port: + daemon_listen_port = find_open_port() + + self._host_port, self._daemon_listen_port = host_port, daemon_listen_port @staticmethod async def send_raw_data(byte_str, writer): @@ -150,18 +190,12 @@ async def send_raw_data(byte_str, writer): writer.write(request) @staticmethod - async def send_message_pack(data, writer): + async def send_msgpack(data, writer): raw_data = MSGPackSerializer.dumps(data) await P2P.send_raw_data(raw_data, writer) @staticmethod async def send_protobuf(protobuf, out_proto_type, writer): - if type(protobuf) != out_proto_type: - raise TypeError('Unary handler returned protobuf of wrong type.') - await P2P.send_raw_data(protobuf.SerializeToString(), writer) - - @staticmethod - async def send_protobuf_or_error(protobuf, out_proto_type, writer): if type(protobuf) != out_proto_type: raise TypeError('Unary handler returned protobuf of wrong type.') if out_proto_type == p2pd_pb2.RPCError: @@ -179,17 +213,11 @@ async def receive_raw_data(reader: asyncio.StreamReader, header_len=HEADER_LEN): return data @staticmethod - async def receive_message_pack(reader): + async def receive_msgpack(reader): return MSGPackSerializer.loads(await P2P.receive_raw_data(reader)) @staticmethod async def receive_protobuf(in_proto_type, reader): - protobuf = in_proto_type() - protobuf.ParseFromString(await P2P.receive_raw_data(reader)) - return protobuf - - @staticmethod - async def receive_protobuf_or_error(in_proto_type, reader): msg_type = await P2P.receive_raw_data(reader) if msg_type == P2P.RESULT_MESSAGE: protobuf = in_proto_type() @@ -200,7 +228,7 @@ async def receive_protobuf_or_error(in_proto_type, reader): protobuf.ParseFromString(await P2P.receive_raw_data(reader)) return None, protobuf else: - raise TypeError('invalid protobuf message type') + raise TypeError('Invalid Protobuf message type') @staticmethod def _handle_stream(handle): @@ -223,7 +251,7 @@ async def do_handle_stream(stream_info, reader, writer): def _handle_unary_stream(handle, context, in_proto_type, out_proto_type): async def watchdog(reader: asyncio.StreamReader): await reader.read(n=1) - raise P2P.InterruptedError() + raise P2PInterruptedError() async def do_handle_unary_stream( stream_info: StreamInfo, @@ -236,7 +264,7 @@ async def do_handle_unary_stream( logger.debug("Incomplete read while receiving request from peer") return except google.protobuf.message.DecodeError as error: - logger.warning(repr(error)) + logger.exception(error) return context.peer_id, context.peer_addr = stream_info.peer_id, stream_info.addr @@ -244,12 +272,12 @@ async def do_handle_unary_stream( return_when=asyncio.FIRST_COMPLETED) try: result = done.pop().result() - await P2P.send_protobuf_or_error(result, out_proto_type, writer) - except P2P.InterruptedError: + await P2P.send_protobuf(result, out_proto_type, writer) + except P2PInterruptedError: pass except Exception as exc: error = p2pd_pb2.RPCError(message=str(exc)) - await P2P.send_protobuf_or_error(error, p2pd_pb2.RPCError, writer) + await P2P.send_protobuf(error, p2pd_pb2.RPCError, writer) finally: pending_task = pending.pop() pending_task.cancel() @@ -282,17 +310,17 @@ async def stop_listening(self): async def add_stream_handler(self, name, handle): if self._listen_task is None: self.start_listening() - await self._client.stream_handler(name, P2P._handle_stream(handle)) + await self._client.stream_handler(name, self._handle_stream(handle)) async def add_unary_handler(self, name, handle, in_proto_type, out_proto_type): if self._listen_task is None: self.start_listening() - context = P2PContext(ours_id=self.id, ours_port=self._host_port, handle_name=name) + context = P2PContext(id=self.id, port=self._host_port, handle_name=name) await self._client.stream_handler( name, P2P._handle_unary_stream(handle, context, in_proto_type, out_proto_type)) async def call_peer_handler(self, peer_id, handler_name, input_data): - libp2p_peer_id = ID.from_base58(peer_id) + libp2p_peer_id = PeerID.from_base58(peer_id) stream_info, reader, writer = await self._client.stream_open(libp2p_peer_id, (handler_name,)) try: await P2P.send_raw_data(input_data, writer) @@ -301,52 +329,41 @@ async def call_peer_handler(self, peer_id, handler_name, input_data): writer.close() def __del__(self): - self._kill_child() + self._terminate() @property def is_alive(self): return self._alive - async def shutdown(self, timeout=None): - await asyncio.get_event_loop().run_in_executor(None, self._kill_child) + async def shutdown(self): + await asyncio.get_event_loop().run_in_executor(None, self._terminate) - def _kill_child(self): + def _terminate(self): self._alive = False if self._child is not None and self._child.poll() is None: self._child.kill() self._child.wait() - def _make_process_args(self, *args, **kwargs) -> tp.List[str]: + @staticmethod + def _make_process_args(*args, **kwargs) -> List[str]: proc_args = [] proc_args.extend( str(entry) for entry in args ) proc_args.extend( - f'-{key}={value}' if value is not None else f'-{key}' + f'-{key}={P2P._convert_process_arg_type(value)}' if value is not None else f'-{key}' for key, value in kwargs.items() ) return proc_args + @staticmethod + def _convert_process_arg_type(val): + if isinstance(val, bool): + return 1 if val else 0 + return val + @staticmethod def _make_bootstrap_peers(nodes): if nodes is None: return {} return {'bootstrapPeers': ','.join(nodes)} - - @staticmethod - def _make_dht_mode(dht_mode): - if dht_mode == 'dht': - return {'dht': 1} - if dht_mode == 'dht_server': - return {'dhtServer': 1} - if dht_mode == 'dht_client': - return {'dhtClient': 1} - return {'dht': 0} - - @staticmethod - def _make_force_reachability(force_reachability): - if force_reachability == 'public': - return {'forceReachabilityPublic': 1} - if force_reachability == 'private': - return {'forceReachabilityPrivate': 1} - return {} diff --git a/hivemind/p2p/p2p_daemon_bindings/control.py b/hivemind/p2p/p2p_daemon_bindings/control.py index 014ac674f..ed8425bf6 100644 --- a/hivemind/p2p/p2p_daemon_bindings/control.py +++ b/hivemind/p2p/p2p_daemon_bindings/control.py @@ -4,88 +4,86 @@ Author: Kevin Mai-Husan Chia """ -import logging -from typing import AsyncIterator, Awaitable, Callable, Dict, Iterable, Sequence, Tuple - import asyncio +import logging from contextlib import asynccontextmanager +from typing import (AsyncIterator, Awaitable, Callable, Dict, Iterable, + Sequence, Tuple) + from multiaddr import Multiaddr, protocols -from hivemind.p2p.p2p_daemon_bindings.datastructures import PeerInfo, StreamInfo, ID + +from hivemind.p2p.p2p_daemon_bindings.datastructures import (PeerID, PeerInfo, + StreamInfo) +from hivemind.p2p.p2p_daemon_bindings.utils import (DispatchFailure, + raise_if_failed, + read_pbmsg_safe, + write_pbmsg) from hivemind.proto import p2pd_pb2 as p2pd_pb -from hivemind.p2p.p2p_daemon_bindings.utils import DispatchFailure, read_pbmsg_safe, write_pbmsg, raise_if_failed +from hivemind.utils.logging import get_logger StreamHandler = Callable[[StreamInfo, asyncio.StreamReader, asyncio.StreamWriter], Awaitable[None]] -_supported_conn_protocols = ( +SUPPORT_CONN_PROTOCOLS = ( protocols.P_IP4, # protocols.P_IP6, protocols.P_UNIX, ) +SUPPORTED_PROTOS = ( + protocols.protocol_with_code(proto) for proto in SUPPORT_CONN_PROTOCOLS +) def parse_conn_protocol(maddr: Multiaddr) -> int: proto_codes = set(proto.code for proto in maddr.protocols()) - proto_cand = proto_codes.intersection(_supported_conn_protocols) + proto_cand = proto_codes.intersection(SUPPORT_CONN_PROTOCOLS) if len(proto_cand) != 1: - supported_protos = ( - protocols.protocol_with_code(proto) for proto in _supported_conn_protocols - ) raise ValueError( - f"connection protocol should be only one protocol out of {supported_protos}" + f"connection protocol should be only one protocol out of {SUPPORTED_PROTOS}" f", maddr={maddr}" ) return tuple(proto_cand)[0] class DaemonConnector: - control_maddr: Multiaddr - logger = logging.getLogger("p2pclient.DaemonConnector") DEFAULT_CONTROL_MADDR = "/unix/tmp/p2pd.sock" - def __init__(self, control_maddr: Multiaddr = None) -> None: - if control_maddr is None: - control_maddr = Multiaddr(self.DEFAULT_CONTROL_MADDR) - self.control_maddr = control_maddr + def __init__(self, control_maddr: Multiaddr = Multiaddr(DEFAULT_CONTROL_MADDR)) -> None: + self.control_maddr: Multiaddr = control_maddr + self.proto_code: int = parse_conn_protocol(self.control_maddr) + self.logger = get_logger(__name__) async def open_connection(self) -> (asyncio.StreamReader, asyncio.StreamWriter): - proto_code = parse_conn_protocol(self.control_maddr) - if proto_code == protocols.P_UNIX: + if self.proto_code == protocols.P_UNIX: control_path = self.control_maddr.value_for_protocol(protocols.P_UNIX) self.logger.debug( "DaemonConnector %s opens connection to %s", self, self.control_maddr ) return await asyncio.open_unix_connection(control_path) - elif proto_code == protocols.P_IP4: + elif self.proto_code == protocols.P_IP4: host = self.control_maddr.value_for_protocol(protocols.P_IP4) port = int(self.control_maddr.value_for_protocol(protocols.P_TCP)) return await asyncio.open_connection(host, port) else: raise ValueError( - f"protocol not supported: protocol={protocols.protocol_with_code(proto_code)}" + f"Protocol not supported: {protocols.protocol_with_code(self.proto_code)}" ) class ControlClient: - listen_maddr: Multiaddr - daemon_connector: DaemonConnector - handlers: Dict[str, StreamHandler] - logger = logging.getLogger("p2pclient.ControlClient") DEFAULT_LISTEN_MADDR = "/unix/tmp/p2pclient.sock" - def __init__( - self, daemon_connector: DaemonConnector, listen_maddr: Multiaddr = None - ) -> None: - if listen_maddr is None: - listen_maddr = Multiaddr(self.DEFAULT_LISTEN_MADDR) - self.listen_maddr = listen_maddr - self.daemon_connector = daemon_connector - self.handlers = {} + def __init__(self, daemon_connector: DaemonConnector, + listen_maddr: Multiaddr = Multiaddr(DEFAULT_LISTEN_MADDR)) -> None: + self.listen_maddr: Multiaddr = listen_maddr + self.daemon_connector: DaemonConnector = daemon_connector + self.handlers: Dict[str, StreamHandler] = {} + self.logger = get_logger(__name__) async def _handler(self, reader: asyncio.StreamReader, writer: asyncio.StreamWriter): pb_stream_info = p2pd_pb.StreamInfo() # type: ignore await read_pbmsg_safe(reader, pb_stream_info) - stream_info = StreamInfo.from_pb(pb_stream_info) - self.logger.info("New incoming stream: %s", stream_info) + stream_info = StreamInfo.from_protobuf(pb_stream_info) + self.logger.debug(f"New incoming stream: {stream_info}") try: handler = self.handlers[stream_info.proto] except KeyError as e: @@ -110,14 +108,12 @@ async def listen(self) -> AsyncIterator["ControlClient"]: ) async with server: - self.logger.info( - "DaemonConnector %s starts listening to %s", self, self.listen_maddr - ) + self.logger.info(f"DaemonConnector {self} starts listening to {self.listen_maddr}") yield self - self.logger.info("DaemonConnector %s closed", self) + self.logger.info(f"DaemonConnector {self} closed") - async def identify(self) -> Tuple[ID, Tuple[Multiaddr, ...]]: + async def identify(self) -> Tuple[PeerID, Tuple[Multiaddr, ...]]: reader, writer = await self.daemon_connector.open_connection() req = p2pd_pb.Request(type=p2pd_pb.Request.IDENTIFY) await write_pbmsg(writer, req) @@ -131,11 +127,11 @@ async def identify(self) -> Tuple[ID, Tuple[Multiaddr, ...]]: maddrs_bytes = resp.identify.addrs maddrs = tuple(Multiaddr(maddr_bytes) for maddr_bytes in maddrs_bytes) - peer_id = ID(peer_id_bytes) + peer_id = PeerID(peer_id_bytes) return peer_id, maddrs - async def connect(self, peer_id: ID, maddrs: Iterable[Multiaddr]) -> None: + async def connect(self, peer_id: PeerID, maddrs: Iterable[Multiaddr]) -> None: reader, writer = await self.daemon_connector.open_connection() maddrs_bytes = [i.to_bytes() for i in maddrs] @@ -159,10 +155,10 @@ async def list_peers(self) -> Tuple[PeerInfo, ...]: writer.close() raise_if_failed(resp) - peers = tuple(PeerInfo.from_pb(pinfo) for pinfo in resp.peers) + peers = tuple(PeerInfo.from_protobuf(pinfo) for pinfo in resp.peers) return peers - async def disconnect(self, peer_id: ID) -> None: + async def disconnect(self, peer_id: PeerID) -> None: disconnect_req = p2pd_pb.DisconnectRequest(peer=peer_id.to_bytes()) req = p2pd_pb.Request( type=p2pd_pb.Request.DISCONNECT, disconnect=disconnect_req @@ -175,7 +171,7 @@ async def disconnect(self, peer_id: ID) -> None: raise_if_failed(resp) async def stream_open( - self, peer_id: ID, protocols: Sequence[str] + self, peer_id: PeerID, protocols: Sequence[str] ) -> Tuple[StreamInfo, asyncio.StreamReader, asyncio.StreamWriter]: reader, writer = await self.daemon_connector.open_connection() @@ -192,7 +188,7 @@ async def stream_open( raise_if_failed(resp) pb_stream_info = resp.streamInfo - stream_info = StreamInfo.from_pb(pb_stream_info) + stream_info = StreamInfo.from_protobuf(pb_stream_info) return stream_info, reader, writer diff --git a/hivemind/p2p/p2p_daemon_bindings/datastructures.py b/hivemind/p2p/p2p_daemon_bindings/datastructures.py index edc5ffc7c..224640d2f 100644 --- a/hivemind/p2p/p2p_daemon_bindings/datastructures.py +++ b/hivemind/p2p/p2p_daemon_bindings/datastructures.py @@ -5,7 +5,7 @@ """ import hashlib -from typing import Any, List, Sequence, Union +from typing import Any, List, Optional, Sequence, Union import base58 import multihash @@ -41,54 +41,54 @@ def digest(self) -> bytes: ) -class ID: - _bytes: bytes - _xor_id: int = None - _b58_str: str = None - +class PeerID: def __init__(self, peer_id_bytes: bytes) -> None: self._bytes = peer_id_bytes + self._xor_id: int = int(sha256_digest(self._bytes).hex(), 16) + self._b58_str: str = base58.b58encode(self._bytes).decode() @property def xor_id(self) -> int: - if not self._xor_id: - self._xor_id = int(sha256_digest(self._bytes).hex(), 16) return self._xor_id def to_bytes(self) -> bytes: return self._bytes def to_base58(self) -> str: - if not self._b58_str: - self._b58_str = base58.b58encode(self._bytes).decode() return self._b58_str def __repr__(self) -> str: - return f"" + return f"" + + def __str__(self): + return self.to_base58() + + def pretty(self): + return self.to_base58() - __str__ = pretty = to_string = to_base58 + def to_string(self): + return self.to_base58() def __eq__(self, other: object) -> bool: if isinstance(other, str): return self.to_base58() == other elif isinstance(other, bytes): return self._bytes == other - elif isinstance(other, ID): + elif isinstance(other, PeerID): return self._bytes == other._bytes else: - return NotImplemented + return False def __hash__(self) -> int: return hash(self._bytes) @classmethod - def from_base58(cls, b58_encoded_peer_id_str: str) -> "ID": - peer_id_bytes = base58.b58decode(b58_encoded_peer_id_str) - pid = ID(peer_id_bytes) - return pid + def from_base58(cls, base58_id: str) -> "PeerID": + peer_id_bytes = base58.b58decode(base58_id) + return cls(peer_id_bytes) @classmethod - def from_pubkey(cls, key: PublicKey) -> "ID": + def from_pubkey(cls, key: PublicKey) -> "PeerID": serialized_key = key.serialize() algo = multihash.Func.sha2_256 if ENABLE_INLINING and len(serialized_key) <= MAX_INLINE_KEY_LENGTH: @@ -104,39 +104,32 @@ def sha256_digest(data: Union[str, bytes]) -> bytes: class StreamInfo: - peer_id: ID - addr: Multiaddr - proto: str - - def __init__(self, peer_id: ID, addr: Multiaddr, proto: str) -> None: - self.peer_id = peer_id - self.addr = addr - self.proto = proto + def __init__(self, peer_id: PeerID, addr: Multiaddr, proto: str) -> None: + self.peer_id: PeerID = peer_id + self.addr: Multiaddr = addr + self.proto: str = proto def __repr__(self) -> str: return ( f"" ) - def to_pb(self) -> p2pd_pb2.StreamInfo: + def to_protobuf(self) -> p2pd_pb2.StreamInfo: pb_msg = p2pd_pb2.StreamInfo( peer=self.peer_id.to_bytes(), addr=self.addr.to_bytes(), proto=self.proto ) return pb_msg @classmethod - def from_pb(cls, pb_msg: p2pd_pb2.StreamInfo) -> "StreamInfo": + def from_protobuf(cls, pb_msg: p2pd_pb2.StreamInfo) -> "StreamInfo": stream_info = cls( - peer_id=ID(pb_msg.peer), addr=Multiaddr(pb_msg.addr), proto=pb_msg.proto + peer_id=PeerID(pb_msg.peer), addr=Multiaddr(pb_msg.addr), proto=pb_msg.proto ) return stream_info -class PeerInfoLibP2P: - peer_id: ID - addrs: List[Multiaddr] - - def __init__(self, peer_id: ID, addrs: Sequence[Multiaddr]) -> None: +class PeerInfo: + def __init__(self, peer_id: PeerID, addrs: Sequence[Multiaddr]) -> None: self.peer_id = peer_id self.addrs = list(addrs) @@ -147,9 +140,22 @@ def __eq__(self, other: Any) -> bool: and self.addrs == other.addrs ) + @classmethod + def from_protobuf(cls, peer_info_pb: p2pd_pb2.PeerInfo) -> "PeerInfo": + peer_id = PeerID(peer_info_pb.id) + addrs = [Multiaddr(addr) for addr in peer_info_pb.addrs] + return PeerInfo(peer_id, addrs) + + def __str__(self): + return f"{self.peer_id.pretty()} {','.join(str(a) for a in self.addrs)}" + + +class InvalidAddrError(ValueError): + pass -def info_from_p2p_addr(addr: Multiaddr) -> PeerInfoLibP2P: - if not addr: + +def info_from_p2p_addr(addr: Multiaddr) -> PeerInfo: + if addr is None: raise InvalidAddrError("`addr` should not be `None`") parts = addr.split() @@ -167,25 +173,10 @@ def info_from_p2p_addr(addr: Multiaddr) -> PeerInfoLibP2P: # make sure the /p2p value parses as a peer.ID peer_id_str: str = p2p_part.value_for_protocol(protocols.P_P2P) - peer_id: ID = ID.from_base58(peer_id_str) + peer_id = PeerID.from_base58(peer_id_str) # we might have received just an / p2p part, which means there's no addr. if len(parts) > 1: addr = Multiaddr.join(*parts[:-1]) return PeerInfo(peer_id, [addr]) - - -class InvalidAddrError(ValueError): - pass - - -class PeerInfo(PeerInfoLibP2P): - @classmethod - def from_pb(cls, peer_info_pb: p2pd_pb2.PeerInfo) -> PeerInfoLibP2P: - peer_id = ID(peer_info_pb.id) - addrs = [Multiaddr(addr) for addr in peer_info_pb.addrs] - return PeerInfo(peer_id, addrs) - - def __str__(self): - return self.peer_id.pretty() + " " + ",".join(str(a) for a in self.addrs) diff --git a/hivemind/p2p/p2p_daemon_bindings/p2pclient.py b/hivemind/p2p/p2p_daemon_bindings/p2pclient.py index baeab3612..d6b47c256 100644 --- a/hivemind/p2p/p2p_daemon_bindings/p2pclient.py +++ b/hivemind/p2p/p2p_daemon_bindings/p2pclient.py @@ -13,7 +13,7 @@ from hivemind.p2p.p2p_daemon_bindings.control import (ControlClient, DaemonConnector, StreamHandler) -from hivemind.p2p.p2p_daemon_bindings.datastructures import (ID, PeerInfo, +from hivemind.p2p.p2p_daemon_bindings.datastructures import (PeerID, PeerInfo, StreamInfo) @@ -37,13 +37,13 @@ async def listen(self) -> AsyncIterator["Client"]: async with self.control.listen(): yield self - async def identify(self) -> Tuple[ID, Tuple[Multiaddr, ...]]: + async def identify(self) -> Tuple[PeerID, Tuple[Multiaddr, ...]]: """ Get current node peer id and list of addresses """ return await self.control.identify() - async def connect(self, peer_id: ID, maddrs: Iterable[Multiaddr]) -> None: + async def connect(self, peer_id: PeerID, maddrs: Iterable[Multiaddr]) -> None: """ Connect to p2p node with specified addresses and peer id. :peer_id: node peer id you want connect to @@ -57,7 +57,7 @@ async def list_peers(self) -> Tuple[PeerInfo, ...]: """ return await self.control.list_peers() - async def disconnect(self, peer_id: ID) -> None: + async def disconnect(self, peer_id: PeerID) -> None: """ Disconnect from node with specified peer id :peer_id: @@ -65,7 +65,7 @@ async def disconnect(self, peer_id: ID) -> None: await self.control.disconnect(peer_id=peer_id) async def stream_open( - self, peer_id: ID, protocols: Sequence[str] + self, peer_id: PeerID, protocols: Sequence[str] ) -> Tuple[StreamInfo, asyncio.StreamReader, asyncio.StreamWriter]: """ Open a stream to call other peer (with peer_id) handler for specified protocols diff --git a/hivemind/p2p/p2p_daemon_bindings/utils.py b/hivemind/p2p/p2p_daemon_bindings/utils.py index f567b33bb..525bcc284 100644 --- a/hivemind/p2p/p2p_daemon_bindings/utils.py +++ b/hivemind/p2p/p2p_daemon_bindings/utils.py @@ -10,7 +10,6 @@ from hivemind.proto import p2pd_pb2 as p2pd_pb - DEFAULT_MAX_BITS: int = 64 diff --git a/tests/test_p2p_daemon.py b/tests/test_p2p_daemon.py index 4b9e40563..7fc52bd6a 100644 --- a/tests/test_p2p_daemon.py +++ b/tests/test_p2p_daemon.py @@ -9,7 +9,7 @@ from hivemind.utils import MSGPackSerializer from hivemind.utils.compression import serialize_torch_tensor, deserialize_torch_tensor -from hivemind.p2p.p2p_daemon_bindings.datastructures import ID +from hivemind.p2p.p2p_daemon_bindings.datastructures import PeerID import numpy as np import pytest @@ -55,7 +55,7 @@ async def test_server_client_connection(): assert len(peers) == 0 nodes = boostrap_from([server]) - client = await P2P.create(bootstrap=True, boostrap_peers=nodes) + client = await P2P.create(bootstrap=True, bootstrap_peers=nodes) await client.wait_for_at_least_n_peers(1) peers = await client._client.list_peers() @@ -141,7 +141,7 @@ async def ping_handler(request, context): handler_cancelled = True return dht_pb2.PingResponse( peer=dht_pb2.NodeInfo( - node_id=context.ours_id.encode(), rpc_port=context.ours_port), + node_id=context.id.encode(), rpc_port=context.port), sender_endpoint=context.handle_name, available=True) server_primary = await P2P.create() @@ -152,7 +152,7 @@ async def ping_handler(request, context): assert is_process_running(server_pid) nodes = boostrap_from([server]) - client_primary = await P2P.create(bootstrap=True, boostrap_peers=nodes) + client_primary = await P2P.create(bootstrap=True, bootstrap_peers=nodes) client = await replicate_if_needed(client_primary, replicate) client_pid = client_primary._child.pid assert is_process_running(client_pid) @@ -165,7 +165,7 @@ async def ping_handler(request, context): sender_endpoint=handle_name, available=True) await client.wait_for_at_least_n_peers(1) - libp2p_server_id = ID.from_base58(server.id) + libp2p_server_id = PeerID.from_base58(server.id) stream_info, reader, writer = await client._client.stream_open(libp2p_server_id, (handle_name,)) await P2P.send_protobuf(ping_request, dht_pb2.PingRequest, writer) @@ -175,7 +175,7 @@ async def ping_handler(request, context): await asyncio.sleep(1) assert handler_cancelled else: - result, err = await P2P.receive_protobuf_or_error(dht_pb2.PingResponse, reader) + result, err = await P2P.receive_protobuf(dht_pb2.PingResponse, reader) assert err is None assert result == expected_response assert not handler_cancelled @@ -199,7 +199,7 @@ async def error_handler(request, context): assert is_process_running(server_pid) nodes = boostrap_from([server]) - client = await P2P.create(bootstrap=True, boostrap_peers=nodes) + client = await P2P.create(bootstrap=True, bootstrap_peers=nodes) client_pid = client._child.pid assert is_process_running(client_pid) await client.wait_for_at_least_n_peers(1) @@ -207,11 +207,11 @@ async def error_handler(request, context): ping_request = dht_pb2.PingRequest( peer=dht_pb2.NodeInfo(node_id=client.id.encode(), rpc_port=client._host_port), validate=True) - libp2p_server_id = ID.from_base58(server.id) + libp2p_server_id = PeerID.from_base58(server.id) stream_info, reader, writer = await client._client.stream_open(libp2p_server_id, (handle_name,)) await P2P.send_protobuf(ping_request, dht_pb2.PingRequest, writer) - result, err = await P2P.receive_protobuf_or_error(dht_pb2.PingResponse, reader) + result, err = await P2P.receive_protobuf(dht_pb2.PingResponse, reader) assert result is None assert err.message == 'boom' @@ -237,7 +237,7 @@ async def test_call_peer_single_process(test_input, expected, handle, handler_na assert is_process_running(server_pid) nodes = boostrap_from([server]) - client = await P2P.create(bootstrap=True, boostrap_peers=nodes) + client = await P2P.create(bootstrap=True, bootstrap_peers=nodes) client_pid = client._child.pid assert is_process_running(client_pid) @@ -292,7 +292,7 @@ async def test_call_peer_different_processes(): peer_port = client_side.recv() nodes = [bootstrap_addr(peer_port, peer_id)] - client = await P2P.create(bootstrap=True, boostrap_peers=nodes) + client = await P2P.create(bootstrap=True, bootstrap_peers=nodes) client_pid = client._child.pid assert is_process_running(client_pid) @@ -326,7 +326,7 @@ async def test_call_peer_torch_square(test_input, expected, handler_name="handle await server.add_stream_handler(handler_name, handle) nodes = boostrap_from([server]) - client = await P2P.create(bootstrap=True, boostrap_peers=nodes) + client = await P2P.create(bootstrap=True, bootstrap_peers=nodes) await client.wait_for_at_least_n_peers(1) @@ -358,7 +358,7 @@ async def test_call_peer_torch_add(test_input, expected, handler_name="handle"): await server.add_stream_handler(handler_name, handle) nodes = boostrap_from([server]) - client = await P2P.create(bootstrap=True, boostrap_peers=nodes) + client = await P2P.create(bootstrap=True, bootstrap_peers=nodes) await client.wait_for_at_least_n_peers(1) @@ -389,7 +389,7 @@ async def test_call_peer_error(replicate, handler_name="handle"): await server.add_stream_handler(handler_name, handle_add_torch_with_exc) nodes = boostrap_from([server]) - client_primary = await P2P.create(bootstrap=True, boostrap_peers=nodes) + client_primary = await P2P.create(bootstrap=True, bootstrap_peers=nodes) client = await replicate_if_needed(client_primary, replicate) await client.wait_for_at_least_n_peers(1) @@ -420,7 +420,7 @@ def handler(arg, key): await server_replica2.add_stream_handler(handler_name + '2', partial(handler, key=b'replica2')) nodes = boostrap_from([server_primary]) - client = await P2P.create(bootstrap=True, boostrap_peers=nodes) + client = await P2P.create(bootstrap=True, bootstrap_peers=nodes) await client.wait_for_at_least_n_peers(1) result = await client.call_peer_handler(server_id, handler_name, b'1') diff --git a/tests/test_p2p_daemon_bindings.py b/tests/test_p2p_daemon_bindings.py index 9b0b1f966..052605494 100644 --- a/tests/test_p2p_daemon_bindings.py +++ b/tests/test_p2p_daemon_bindings.py @@ -5,21 +5,27 @@ import subprocess import time import uuid -from contextlib import asynccontextmanager, AsyncExitStack +from contextlib import AsyncExitStack, asynccontextmanager from typing import NamedTuple +import pytest from google.protobuf.message import EncodeError from multiaddr import Multiaddr, protocols -import pytest - from hivemind import find_open_port -from hivemind.p2p.p2p_daemon_bindings.control import parse_conn_protocol, DaemonConnector, ControlClient +from hivemind.p2p.p2p_daemon_bindings.control import (ControlClient, + DaemonConnector, + parse_conn_protocol) +from hivemind.p2p.p2p_daemon_bindings.datastructures import (PeerID, PeerInfo, + StreamInfo) from hivemind.p2p.p2p_daemon_bindings.p2pclient import Client -from hivemind.p2p.p2p_daemon_bindings.utils import ControlFailure, raise_if_failed, write_unsigned_varint, \ - read_unsigned_varint, read_pbmsg_safe, write_pbmsg +from hivemind.p2p.p2p_daemon_bindings.utils import (ControlFailure, + raise_if_failed, + read_pbmsg_safe, + read_unsigned_varint, + write_pbmsg, + write_unsigned_varint) from hivemind.proto import p2pd_pb2 as p2pd_pb -from hivemind.p2p.p2p_daemon_bindings.datastructures import ID, StreamInfo, PeerInfo def test_raise_if_failed_raises(): @@ -134,7 +140,7 @@ def peer_id_bytes(): @pytest.fixture(scope="module") def peer_id(peer_id_bytes): - return ID(peer_id_bytes) + return PeerID(peer_id_bytes) @pytest.fixture(scope="module") @@ -147,13 +153,13 @@ def test_peer_id(peer_id_string, peer_id_bytes, peer_id): assert peer_id.to_bytes() == peer_id_bytes assert peer_id.to_string() == peer_id_string # test initialized with string - peer_id_2 = ID.from_base58(peer_id_string) + peer_id_2 = PeerID.from_base58(peer_id_string) assert peer_id_2.to_bytes() == peer_id_bytes assert peer_id_2.to_string() == peer_id_string # test equal assert peer_id == peer_id_2 # test not equal - peer_id_3 = ID.from_base58("QmbmfNDEth7Ucvjuxiw3SP3E4PoJzbk7g4Ge6ZDigbCsNp") + peer_id_3 = PeerID.from_base58("QmbmfNDEth7Ucvjuxiw3SP3E4PoJzbk7g4Ge6ZDigbCsNp") assert peer_id != peer_id_3 @@ -165,12 +171,12 @@ def test_stream_info(peer_id, maddr): assert si.addr == maddr assert si.proto == proto # test case: `StreamInfo.to_pb` - pb_si = si.to_pb() + pb_si = si.to_protobuf() assert pb_si.peer == peer_id.to_bytes() assert pb_si.addr == maddr.to_bytes() assert pb_si.proto == si.proto # test case: `StreamInfo.from_pb` - si_1 = StreamInfo.from_pb(pb_si) + si_1 = StreamInfo.from_protobuf(pb_si) assert si_1.peer_id == peer_id assert si_1.addr == maddr assert si_1.proto == proto @@ -183,7 +189,7 @@ def test_peer_info(peer_id, maddr): assert pi.addrs == [maddr] # test case: `PeerInfo.from_pb` pi_pb = p2pd_pb.PeerInfo(id=peer_id.to_bytes(), addrs=[maddr.to_bytes()]) - pi_1 = PeerInfo.from_pb(pi_pb) + pi_1 = PeerInfo.from_protobuf(pi_pb) assert pi.peer_id == pi_1.peer_id assert pi.addrs == pi_1.addrs @@ -317,7 +323,7 @@ def num_p2pds(): @pytest.fixture(scope="module") def peer_id_random(): - return ID.from_base58("QmcgpsyWgH8Y8ajJz1Cu72KnS5uo2Aa2LpzU7kinSupNK1") + return PeerID.from_base58("QmcgpsyWgH8Y8ajJz1Cu72KnS5uo2Aa2LpzU7kinSupNK1") @pytest.fixture From ab4d3e4e2977870b7e3eeb5f89ef37ae870cef77 Mon Sep 17 00:00:00 2001 From: Maxim Kashirin Date: Mon, 3 May 2021 16:14:37 +0300 Subject: [PATCH 39/81] imports/rename constants/string formatting --- hivemind/p2p/p2p_daemon.py | 6 +++--- hivemind/p2p/p2p_daemon_bindings/control.py | 12 +++++------- .../p2p/p2p_daemon_bindings/datastructures.py | 2 +- tests/test_p2p_daemon.py | 18 +++++++++--------- 4 files changed, 18 insertions(+), 20 deletions(-) diff --git a/hivemind/p2p/p2p_daemon.py b/hivemind/p2p/p2p_daemon.py index 40e1b9f8b..ad0f1f943 100644 --- a/hivemind/p2p/p2p_daemon.py +++ b/hivemind/p2p/p2p_daemon.py @@ -19,7 +19,7 @@ logger = get_logger(__name__) -P2PD_RELATIVE_PATH = 'hivemind_cli/p2pd' +P2PD_FILENAME = 'p2pd' NUM_RETRIES = 3 RETRY_DELAY = 0.4 @@ -95,11 +95,11 @@ async def create(cls, *args, quic: bool = True, tls: bool = True, conn_manager: 'It is very dangerous, because p2pd connects to global ipfs and it is very unstable. ' \ 'If you really want this, pass use_global_ipfs=True' assert not (bootstrap_peers is not None and use_global_ipfs), \ - 'Non empty boostrap_nodes and use_global_ipfs=True are incompatible.' \ + 'Non empty bootstrap_nodes and use_global_ipfs=True are incompatible.' \ 'Choose one option: your nodes list (preferable) or global ipfs (very unstable)' self = cls() - with path(cli, 'p2pd') as p: + with path(cli, P2PD_FILENAME) as p: p2pd_path = p bootstrap_peers = cls._make_bootstrap_peers(bootstrap_peers) dht = cls.DHT_MODE_MAPPING.get(dht_mode, {'dht': 0}) diff --git a/hivemind/p2p/p2p_daemon_bindings/control.py b/hivemind/p2p/p2p_daemon_bindings/control.py index ed8425bf6..f0d48c9d2 100644 --- a/hivemind/p2p/p2p_daemon_bindings/control.py +++ b/hivemind/p2p/p2p_daemon_bindings/control.py @@ -5,7 +5,6 @@ """ import asyncio -import logging from contextlib import asynccontextmanager from typing import (AsyncIterator, Awaitable, Callable, Dict, Iterable, Sequence, Tuple) @@ -55,9 +54,7 @@ def __init__(self, control_maddr: Multiaddr = Multiaddr(DEFAULT_CONTROL_MADDR)) async def open_connection(self) -> (asyncio.StreamReader, asyncio.StreamWriter): if self.proto_code == protocols.P_UNIX: control_path = self.control_maddr.value_for_protocol(protocols.P_UNIX) - self.logger.debug( - "DaemonConnector %s opens connection to %s", self, self.control_maddr - ) + self.logger.debug(f"DaemonConnector {self} opens connection to {self.control_maddr}") return await asyncio.open_unix_connection(control_path) elif self.proto_code == protocols.P_IP4: host = self.control_maddr.value_for_protocol(protocols.P_IP4) @@ -72,8 +69,9 @@ async def open_connection(self) -> (asyncio.StreamReader, asyncio.StreamWriter): class ControlClient: DEFAULT_LISTEN_MADDR = "/unix/tmp/p2pclient.sock" - def __init__(self, daemon_connector: DaemonConnector, - listen_maddr: Multiaddr = Multiaddr(DEFAULT_LISTEN_MADDR)) -> None: + def __init__( + self, daemon_connector: DaemonConnector, listen_maddr: Multiaddr = Multiaddr(DEFAULT_LISTEN_MADDR) + ) -> None: self.listen_maddr: Multiaddr = listen_maddr self.daemon_connector: DaemonConnector = daemon_connector self.handlers: Dict[str, StreamHandler] = {} @@ -104,7 +102,7 @@ async def listen(self) -> AsyncIterator["ControlClient"]: server = await asyncio.start_server(self._handler, port=port, host=host) else: raise ValueError( - f"protocol not supported: protocol={protocols.protocol_with_code(proto_code)}" + f"Protocol not supported: {protocols.protocol_with_code(self.proto_code)}" ) async with server: diff --git a/hivemind/p2p/p2p_daemon_bindings/datastructures.py b/hivemind/p2p/p2p_daemon_bindings/datastructures.py index 224640d2f..ddbcb3b02 100644 --- a/hivemind/p2p/p2p_daemon_bindings/datastructures.py +++ b/hivemind/p2p/p2p_daemon_bindings/datastructures.py @@ -5,7 +5,7 @@ """ import hashlib -from typing import Any, List, Optional, Sequence, Union +from typing import Any, Sequence, Union import base58 import multihash diff --git a/tests/test_p2p_daemon.py b/tests/test_p2p_daemon.py index 7fc52bd6a..c1f917dc4 100644 --- a/tests/test_p2p_daemon.py +++ b/tests/test_p2p_daemon.py @@ -30,7 +30,7 @@ def bootstrap_addr(host_port, id_): return f'/ip4/127.0.0.1/tcp/{host_port}/p2p/{id_}' -def boostrap_from(daemons: List[P2P]) -> List[str]: +def bootstrap_from(daemons: List[P2P]) -> List[str]: return [ bootstrap_addr(d._host_port, d.id) for d in daemons @@ -54,7 +54,7 @@ async def test_server_client_connection(): peers = await server._client.list_peers() assert len(peers) == 0 - nodes = boostrap_from([server]) + nodes = bootstrap_from([server]) client = await P2P.create(bootstrap=True, bootstrap_peers=nodes) await client.wait_for_at_least_n_peers(1) @@ -151,7 +151,7 @@ async def ping_handler(request, context): dht_pb2.PingResponse) assert is_process_running(server_pid) - nodes = boostrap_from([server]) + nodes = bootstrap_from([server]) client_primary = await P2P.create(bootstrap=True, bootstrap_peers=nodes) client = await replicate_if_needed(client_primary, replicate) client_pid = client_primary._child.pid @@ -198,7 +198,7 @@ async def error_handler(request, context): await server.add_unary_handler(handle_name, error_handler, dht_pb2.PingRequest, dht_pb2.PingResponse) assert is_process_running(server_pid) - nodes = boostrap_from([server]) + nodes = bootstrap_from([server]) client = await P2P.create(bootstrap=True, bootstrap_peers=nodes) client_pid = client._child.pid assert is_process_running(client_pid) @@ -236,7 +236,7 @@ async def test_call_peer_single_process(test_input, expected, handle, handler_na await server.add_stream_handler(handler_name, handle) assert is_process_running(server_pid) - nodes = boostrap_from([server]) + nodes = bootstrap_from([server]) client = await P2P.create(bootstrap=True, bootstrap_peers=nodes) client_pid = client._child.pid assert is_process_running(client_pid) @@ -325,7 +325,7 @@ async def test_call_peer_torch_square(test_input, expected, handler_name="handle server = await P2P.create() await server.add_stream_handler(handler_name, handle) - nodes = boostrap_from([server]) + nodes = bootstrap_from([server]) client = await P2P.create(bootstrap=True, bootstrap_peers=nodes) await client.wait_for_at_least_n_peers(1) @@ -357,7 +357,7 @@ async def test_call_peer_torch_add(test_input, expected, handler_name="handle"): server = await P2P.create() await server.add_stream_handler(handler_name, handle) - nodes = boostrap_from([server]) + nodes = bootstrap_from([server]) client = await P2P.create(bootstrap=True, bootstrap_peers=nodes) await client.wait_for_at_least_n_peers(1) @@ -388,7 +388,7 @@ async def test_call_peer_error(replicate, handler_name="handle"): server = await replicate_if_needed(server_primary, replicate) await server.add_stream_handler(handler_name, handle_add_torch_with_exc) - nodes = boostrap_from([server]) + nodes = bootstrap_from([server]) client_primary = await P2P.create(bootstrap=True, bootstrap_peers=nodes) client = await replicate_if_needed(client_primary, replicate) @@ -419,7 +419,7 @@ def handler(arg, key): server_replica2 = await replicate_if_needed(server_primary, True) await server_replica2.add_stream_handler(handler_name + '2', partial(handler, key=b'replica2')) - nodes = boostrap_from([server_primary]) + nodes = bootstrap_from([server_primary]) client = await P2P.create(bootstrap=True, bootstrap_peers=nodes) await client.wait_for_at_least_n_peers(1) From a021772973ebf232983736571d7d5545caf18cf7 Mon Sep 17 00:00:00 2001 From: Maxim Kashirin Date: Tue, 4 May 2021 00:49:22 +0300 Subject: [PATCH 40/81] reST docstring --- hivemind/p2p/p2p_daemon.py | 36 ++++++++++++++++++------------------ 1 file changed, 18 insertions(+), 18 deletions(-) diff --git a/hivemind/p2p/p2p_daemon.py b/hivemind/p2p/p2p_daemon.py index ad0f1f943..fa521716f 100644 --- a/hivemind/p2p/p2p_daemon.py +++ b/hivemind/p2p/p2p_daemon.py @@ -73,21 +73,21 @@ async def create(cls, *args, quic: bool = True, tls: bool = True, conn_manager: daemon_listen_port: int = None, **kwargs): """ Start a new p2pd process and connect to it. - @param args: - @param quic: Enables the QUIC transport - @param tls: Enables TLS1.3 channel security protocol - @param conn_manager: Enables the Connection Manager - @param dht_mode: DHT mode (dht_client/dht_server/dht) - @param force_reachability: Force reachability mode (public/private) - @param nat_port_map: Enables NAT port mapping - @param auto_nat: Enables the AutoNAT service - @param bootstrap: Connects to bootstrap peers and bootstraps the dht if enabled - @param bootstrap_peers: List of bootstrap peers; defaults to the IPFS DHT peers - @param use_global_ipfs: Bootstrap to global ipfs (works only if bootstrap=True and bootstrap_peers=None) - @param host_port: port for p2p network - @param daemon_listen_port: port for connection daemon and client binding - @param kwargs: - @return: new wrapper for p2p daemon + :param args: + :param quic: Enables the QUIC transport + :param tls: Enables TLS1.3 channel security protocol + :param conn_manager: Enables the Connection Manager + :param dht_mode: DHT mode (dht_client/dht_server/dht) + :param force_reachability: Force reachability mode (public/private) + :param nat_port_map: Enables NAT port mapping + :param auto_nat: Enables the AutoNAT service + :param bootstrap: Connects to bootstrap peers and bootstraps the dht if enabled + :param bootstrap_peers: List of bootstrap peers; defaults to the IPFS DHT peers + :param use_global_ipfs: Bootstrap to global ipfs (works only if bootstrap=True and bootstrap_peers=None) + :param host_port: port for p2p network + :param daemon_listen_port: port for connection daemon and client binding + :param kwargs: + :return: new wrapper for p2p daemon """ assert not (bootstrap and bootstrap_peers is None and not use_global_ipfs), \ @@ -129,9 +129,9 @@ async def create(cls, *args, quic: bool = True, tls: bool = True, conn_manager: async def replicate(cls, daemon_listen_port: int, host_port: int): """ Connect to existing p2p daemon - @param host_port: port for p2p network - @param daemon_listen_port: port for connection daemon and client binding - @return: new wrapper for existing p2p daemon + :param daemon_listen_port: port for connection daemon and client binding + :param host_port: port for p2p network + :return: new wrapper for existing p2p daemon """ self = cls() From 23357e1d7c22d39427bd6ccfce2df41ff384fbf1 Mon Sep 17 00:00:00 2001 From: Maxim Kashirin Date: Wed, 12 May 2021 19:54:47 +0300 Subject: [PATCH 41/81] pr fixes --- hivemind/p2p/p2p_daemon_bindings/control.py | 21 +- .../p2p/p2p_daemon_bindings/datastructures.py | 22 +- hivemind/p2p/p2p_daemon_bindings/keys.py | 97 ----- hivemind/p2p/p2p_daemon_bindings/p2pclient.py | 6 +- hivemind/p2p/p2p_daemon_bindings/utils.py | 24 +- hivemind/proto/crypto.proto | 24 -- hivemind/proto/p2pd.proto | 2 +- requirements.txt | 4 +- setup.py | 4 +- tests/test_p2p_daemon.py | 16 +- tests/test_p2p_daemon_bindings.py | 368 +++--------------- tests/test_utils/__init__.py | 192 +++++++++ 12 files changed, 287 insertions(+), 493 deletions(-) delete mode 100644 hivemind/p2p/p2p_daemon_bindings/keys.py delete mode 100644 hivemind/proto/crypto.proto diff --git a/hivemind/p2p/p2p_daemon_bindings/control.py b/hivemind/p2p/p2p_daemon_bindings/control.py index f0d48c9d2..2002338a2 100644 --- a/hivemind/p2p/p2p_daemon_bindings/control.py +++ b/hivemind/p2p/p2p_daemon_bindings/control.py @@ -30,6 +30,7 @@ SUPPORTED_PROTOS = ( protocols.protocol_with_code(proto) for proto in SUPPORT_CONN_PROTOCOLS ) +logger = get_logger(__name__) def parse_conn_protocol(maddr: Multiaddr) -> int: @@ -47,14 +48,13 @@ class DaemonConnector: DEFAULT_CONTROL_MADDR = "/unix/tmp/p2pd.sock" def __init__(self, control_maddr: Multiaddr = Multiaddr(DEFAULT_CONTROL_MADDR)) -> None: - self.control_maddr: Multiaddr = control_maddr - self.proto_code: int = parse_conn_protocol(self.control_maddr) - self.logger = get_logger(__name__) + self.control_maddr = control_maddr + self.proto_code = parse_conn_protocol(self.control_maddr) async def open_connection(self) -> (asyncio.StreamReader, asyncio.StreamWriter): if self.proto_code == protocols.P_UNIX: control_path = self.control_maddr.value_for_protocol(protocols.P_UNIX) - self.logger.debug(f"DaemonConnector {self} opens connection to {self.control_maddr}") + logger.debug(f"DaemonConnector {self} opens connection to {self.control_maddr}") return await asyncio.open_unix_connection(control_path) elif self.proto_code == protocols.P_IP4: host = self.control_maddr.value_for_protocol(protocols.P_IP4) @@ -72,16 +72,15 @@ class ControlClient: def __init__( self, daemon_connector: DaemonConnector, listen_maddr: Multiaddr = Multiaddr(DEFAULT_LISTEN_MADDR) ) -> None: - self.listen_maddr: Multiaddr = listen_maddr - self.daemon_connector: DaemonConnector = daemon_connector + self.listen_maddr = listen_maddr + self.daemon_connector = daemon_connector self.handlers: Dict[str, StreamHandler] = {} - self.logger = get_logger(__name__) async def _handler(self, reader: asyncio.StreamReader, writer: asyncio.StreamWriter): pb_stream_info = p2pd_pb.StreamInfo() # type: ignore await read_pbmsg_safe(reader, pb_stream_info) stream_info = StreamInfo.from_protobuf(pb_stream_info) - self.logger.debug(f"New incoming stream: {stream_info}") + logger.debug(f"New incoming stream: {stream_info}") try: handler = self.handlers[stream_info.proto] except KeyError as e: @@ -102,14 +101,14 @@ async def listen(self) -> AsyncIterator["ControlClient"]: server = await asyncio.start_server(self._handler, port=port, host=host) else: raise ValueError( - f"Protocol not supported: {protocols.protocol_with_code(self.proto_code)}" + f"Protocol not supported: {protocols.protocol_with_code(proto_code)}" ) async with server: - self.logger.info(f"DaemonConnector {self} starts listening to {self.listen_maddr}") + logger.info(f"DaemonConnector {self} starts listening to {self.listen_maddr}") yield self - self.logger.info(f"DaemonConnector {self} closed") + logger.info(f"DaemonConnector {self} closed") async def identify(self) -> Tuple[PeerID, Tuple[Multiaddr, ...]]: reader, writer = await self.daemon_connector.open_connection() diff --git a/hivemind/p2p/p2p_daemon_bindings/datastructures.py b/hivemind/p2p/p2p_daemon_bindings/datastructures.py index ddbcb3b02..dab58c408 100644 --- a/hivemind/p2p/p2p_daemon_bindings/datastructures.py +++ b/hivemind/p2p/p2p_daemon_bindings/datastructures.py @@ -11,7 +11,6 @@ import multihash from multiaddr import Multiaddr, protocols -from hivemind.p2p.p2p_daemon_bindings.keys import PublicKey from hivemind.proto import p2pd_pb2 # NOTE: On inlining... @@ -25,8 +24,6 @@ if ENABLE_INLINING: class IdentityHash: - _digest: bytes - def __init__(self) -> None: self._digest = bytearray() @@ -44,8 +41,8 @@ def digest(self) -> bytes: class PeerID: def __init__(self, peer_id_bytes: bytes) -> None: self._bytes = peer_id_bytes - self._xor_id: int = int(sha256_digest(self._bytes).hex(), 16) - self._b58_str: str = base58.b58encode(self._bytes).decode() + self._xor_id = int(sha256_digest(self._bytes).hex(), 16) + self._b58_str = base58.b58encode(self._bytes).decode() @property def xor_id(self) -> int: @@ -87,15 +84,6 @@ def from_base58(cls, base58_id: str) -> "PeerID": peer_id_bytes = base58.b58decode(base58_id) return cls(peer_id_bytes) - @classmethod - def from_pubkey(cls, key: PublicKey) -> "PeerID": - serialized_key = key.serialize() - algo = multihash.Func.sha2_256 - if ENABLE_INLINING and len(serialized_key) <= MAX_INLINE_KEY_LENGTH: - algo = IDENTITY_MULTIHASH_CODE - mh_digest = multihash.digest(serialized_key, algo) - return cls(mh_digest.encode()) - def sha256_digest(data: Union[str, bytes]) -> bytes: if isinstance(data, str): @@ -105,9 +93,9 @@ def sha256_digest(data: Union[str, bytes]) -> bytes: class StreamInfo: def __init__(self, peer_id: PeerID, addr: Multiaddr, proto: str) -> None: - self.peer_id: PeerID = peer_id - self.addr: Multiaddr = addr - self.proto: str = proto + self.peer_id = peer_id + self.addr = addr + self.proto = proto def __repr__(self) -> str: return ( diff --git a/hivemind/p2p/p2p_daemon_bindings/keys.py b/hivemind/p2p/p2p_daemon_bindings/keys.py deleted file mode 100644 index 763c3d76a..000000000 --- a/hivemind/p2p/p2p_daemon_bindings/keys.py +++ /dev/null @@ -1,97 +0,0 @@ -""" -Originally taken from: https://github.com/libp2p/py-libp2p -Licence: MIT -Author: Kevin Mai-Husan Chia and others -""" - -from abc import ABC, abstractmethod -from dataclasses import dataclass -from enum import Enum, unique - -from hivemind.proto import crypto_pb2 as protobuf - - -@unique -class KeyType(Enum): - RSA = 0 - Ed25519 = 1 - Secp256k1 = 2 - ECDSA = 3 - ECC_P256 = 4 - - -class Key(ABC): - """A ``Key`` represents a cryptographic key.""" - - @abstractmethod - def to_bytes(self) -> bytes: - """Returns the byte representation of this key.""" - ... - - @abstractmethod - def get_type(self) -> KeyType: - """Returns the ``KeyType`` for ``self``.""" - ... - - def __eq__(self, other: object) -> bool: - if not isinstance(other, Key): - return NotImplemented - return self.to_bytes() == other.to_bytes() - - -class PublicKey(Key): - """A ``PublicKey`` represents a cryptographic public key.""" - - @abstractmethod - def verify(self, data: bytes, signature: bytes) -> bool: - """Verify that ``signature`` is the cryptographic signature of the hash - of ``data``.""" - ... - - def _serialize_to_protobuf(self) -> protobuf.PublicKey: - """Return the protobuf representation of this ``Key``.""" - key_type = self.get_type().value - data = self.to_bytes() - protobuf_key = protobuf.PublicKey(key_type=key_type, data=data) - return protobuf_key - - def serialize(self) -> bytes: - """Return the canonical serialization of this ``Key``.""" - return self._serialize_to_protobuf().SerializeToString() - - @classmethod - def deserialize_from_protobuf(cls, protobuf_data: bytes) -> protobuf.PublicKey: - return protobuf.PublicKey.FromString(protobuf_data) - - -class PrivateKey(Key): - """A ``PrivateKey`` represents a cryptographic private key.""" - - @abstractmethod - def sign(self, data: bytes) -> bytes: - ... - - @abstractmethod - def get_public_key(self) -> PublicKey: - ... - - def _serialize_to_protobuf(self) -> protobuf.PrivateKey: - """Return the protobuf representation of this ``Key``.""" - key_type = self.get_type().value - data = self.to_bytes() - protobuf_key = protobuf.PrivateKey(key_type=key_type, data=data) - return protobuf_key - - def serialize(self) -> bytes: - """Return the canonical serialization of this ``Key``.""" - return self._serialize_to_protobuf().SerializeToString() - - @classmethod - def deserialize_from_protobuf(cls, protobuf_data: bytes) -> protobuf.PrivateKey: - return protobuf.PrivateKey.FromString(protobuf_data) - - -@dataclass(frozen=True) -class KeyPair: - private_key: PrivateKey - public_key: PublicKey diff --git a/hivemind/p2p/p2p_daemon_bindings/p2pclient.py b/hivemind/p2p/p2p_daemon_bindings/p2pclient.py index d6b47c256..c1fe97808 100644 --- a/hivemind/p2p/p2p_daemon_bindings/p2pclient.py +++ b/hivemind/p2p/p2p_daemon_bindings/p2pclient.py @@ -60,7 +60,7 @@ async def list_peers(self) -> Tuple[PeerInfo, ...]: async def disconnect(self, peer_id: PeerID) -> None: """ Disconnect from node with specified peer id - :peer_id: + :peer_id: node peer id you want disconnect from """ await self.control.disconnect(peer_id=peer_id) @@ -69,8 +69,8 @@ async def stream_open( ) -> Tuple[StreamInfo, asyncio.StreamReader, asyncio.StreamWriter]: """ Open a stream to call other peer (with peer_id) handler for specified protocols - :peer_id: - :protocols: + :peer_id: other peer id + :protocols: list of protocols for other peer handling :return: Returns tuple of stream info (info about connection to second peer) and reader/writer """ return await self.control.stream_open(peer_id=peer_id, protocols=protocols) diff --git a/hivemind/p2p/p2p_daemon_bindings/utils.py b/hivemind/p2p/p2p_daemon_bindings/utils.py index 525bcc284..2a0d5b97c 100644 --- a/hivemind/p2p/p2p_daemon_bindings/utils.py +++ b/hivemind/p2p/p2p_daemon_bindings/utils.py @@ -21,16 +21,14 @@ class DispatchFailure(Exception): pass -async def write_unsigned_varint( - stream: asyncio.StreamWriter, integer: int, max_bits: int = DEFAULT_MAX_BITS -) -> None: - max_int: int = 1 << max_bits +async def write_unsigned_varint(stream: asyncio.StreamWriter, integer: int, max_bits: int = DEFAULT_MAX_BITS) -> None: + max_int = 1 << max_bits if integer < 0: raise ValueError(f"negative integer: {integer}") if integer >= max_int: raise ValueError(f"integer too large: {integer}") while True: - value: int = integer & 0x7F + value = integer & 0x7F integer >>= 7 if integer != 0: value |= 0x80 @@ -40,13 +38,11 @@ async def write_unsigned_varint( break -async def read_unsigned_varint( - stream: asyncio.StreamReader, max_bits: int = DEFAULT_MAX_BITS -) -> int: - max_int: int = 1 << max_bits - iteration: int = 0 - result: int = 0 - has_next: bool = True +async def read_unsigned_varint(stream: asyncio.StreamReader, max_bits: int = DEFAULT_MAX_BITS) -> int: + max_int = 1 << max_bits + iteration = 0 + result = 0 + has_next = True while has_next: data = await stream.readexactly(1) c = data[0] @@ -55,13 +51,13 @@ async def read_unsigned_varint( has_next = (c & 0x80) != 0 iteration += 1 if result >= max_int: - raise ValueError(f"varint overflowed: {result}") + raise ValueError(f"Varint overflowed: {result}") return result def raise_if_failed(response: p2pd_pb.Response) -> None: if response.type == p2pd_pb.Response.ERROR: - raise ControlFailure(f"connect failed. msg={response.error.msg}") + raise ControlFailure(f"Connect failed. msg={response.error.msg}") async def write_pbmsg(stream: asyncio.StreamWriter, pbmsg: PBMessage) -> None: diff --git a/hivemind/proto/crypto.proto b/hivemind/proto/crypto.proto deleted file mode 100644 index e4a69f576..000000000 --- a/hivemind/proto/crypto.proto +++ /dev/null @@ -1,24 +0,0 @@ -//Originally taken from: https://github.com/libp2p/py-libp2p -//Licence: MIT -//Author: Kevin Mai-Husan Chia and others - -syntax = "proto2"; - -package crypto.pb; - -enum KeyType { - RSA = 0; - Ed25519 = 1; - Secp256k1 = 2; - ECDSA = 3; -} - -message PublicKey { - required KeyType key_type = 1; - required bytes data = 2; -} - -message PrivateKey { - required KeyType key_type = 1; - required bytes data = 2; -} diff --git a/hivemind/proto/p2pd.proto b/hivemind/proto/p2pd.proto index dc65514e5..373c6d8e9 100644 --- a/hivemind/proto/p2pd.proto +++ b/hivemind/proto/p2pd.proto @@ -163,4 +163,4 @@ message PSResponse { message RPCError { required string message = 1; -} \ No newline at end of file +} diff --git a/requirements.txt b/requirements.txt index 36b418050..78e07703b 100644 --- a/requirements.txt +++ b/requirements.txt @@ -10,6 +10,6 @@ grpcio>=1.33.2 grpcio-tools>=1.33.2 protobuf>=3.12.2 configargparse>=1.2.3 -multiaddr==0.0.9 -pymultihash==0.8.2 +multiaddr>=0.0.9 +pymultihash>=0.8.2 cryptography>=3.4.6 diff --git a/setup.py b/setup.py index fe21524e2..2cdef0d36 100644 --- a/setup.py +++ b/setup.py @@ -57,10 +57,10 @@ def libp2p_build_install(): v = m.group(1) if version.parse(v) < version.parse("1.13"): - raise EnvironmentError(f'newer version of go required: must be >= 1.13, found {version}') + raise EnvironmentError(f'Newer version of go required: must be >= 1.13, found {version}') except FileNotFoundError: - raise FileNotFoundError('could not find golang installation') + raise FileNotFoundError('Could not find golang installation') with tempfile.TemporaryDirectory() as tempdir: dest = os.path.join(tempdir, 'libp2p-daemon.tar.gz') diff --git a/tests/test_p2p_daemon.py b/tests/test_p2p_daemon.py index c1f917dc4..f8464d2a7 100644 --- a/tests/test_p2p_daemon.py +++ b/tests/test_p2p_daemon.py @@ -4,18 +4,15 @@ from functools import partial from typing import List -import torch - -from hivemind.utils import MSGPackSerializer -from hivemind.utils.compression import serialize_torch_tensor, deserialize_torch_tensor - -from hivemind.p2p.p2p_daemon_bindings.datastructures import PeerID - import numpy as np import pytest +import torch from hivemind.p2p import P2P +from hivemind.p2p.p2p_daemon_bindings.datastructures import PeerID from hivemind.proto import dht_pb2, runtime_pb2 +from hivemind.utils import MSGPackSerializer +from hivemind.utils.compression import deserialize_torch_tensor, serialize_torch_tensor def is_process_running(pid: int) -> bool: @@ -31,10 +28,7 @@ def bootstrap_addr(host_port, id_): def bootstrap_from(daemons: List[P2P]) -> List[str]: - return [ - bootstrap_addr(d._host_port, d.id) - for d in daemons - ] + return [bootstrap_addr(d._host_port, d.id) for d in daemons] @pytest.mark.asyncio diff --git a/tests/test_p2p_daemon_bindings.py b/tests/test_p2p_daemon_bindings.py index 052605494..7f87cbd88 100644 --- a/tests/test_p2p_daemon_bindings.py +++ b/tests/test_p2p_daemon_bindings.py @@ -1,31 +1,17 @@ import asyncio -import functools import io -import os -import subprocess -import time -import uuid -from contextlib import AsyncExitStack, asynccontextmanager -from typing import NamedTuple +from contextlib import AsyncExitStack import pytest from google.protobuf.message import EncodeError from multiaddr import Multiaddr, protocols -from hivemind import find_open_port -from hivemind.p2p.p2p_daemon_bindings.control import (ControlClient, - DaemonConnector, - parse_conn_protocol) -from hivemind.p2p.p2p_daemon_bindings.datastructures import (PeerID, PeerInfo, - StreamInfo) -from hivemind.p2p.p2p_daemon_bindings.p2pclient import Client -from hivemind.p2p.p2p_daemon_bindings.utils import (ControlFailure, - raise_if_failed, - read_pbmsg_safe, - read_unsigned_varint, - write_pbmsg, - write_unsigned_varint) +from hivemind.p2p.p2p_daemon_bindings.control import ControlClient, DaemonConnector, parse_conn_protocol +from hivemind.p2p.p2p_daemon_bindings.datastructures import PeerID, PeerInfo, StreamInfo +from hivemind.p2p.p2p_daemon_bindings.utils import (ControlFailure, raise_if_failed, read_pbmsg_safe, + read_unsigned_varint, write_pbmsg, write_unsigned_varint) from hivemind.proto import p2pd_pb2 as p2pd_pb +from test_utils import make_p2pd_pair_ip4, connect_safe def test_raise_if_failed_raises(): @@ -41,7 +27,7 @@ def test_raise_if_failed_not_raises(): raise_if_failed(resp) -pairs_int_serialized_valid = ( +PAIRS_INT_SERIALIZED_VALID = ( (0, b"\x00"), (1, b"\x01"), (128, b"\x80\x01"), @@ -49,7 +35,7 @@ def test_raise_if_failed_not_raises(): (2 ** 64 - 1, b"\xff\xff\xff\xff\xff\xff\xff\xff\xff\x01"), ) -pairs_int_serialized_overflow = ( +PAIRS_INT_SERIALIZED_OVERFLOW = ( (2 ** 64, b"\x80\x80\x80\x80\x80\x80\x80\x80\x80\x02"), (2 ** 64 + 1, b"\x81\x80\x80\x80\x80\x80\x80\x80\x80\x02"), ( @@ -58,6 +44,18 @@ def test_raise_if_failed_not_raises(): ), ) +PEER_ID_STRING = "QmS5QmciTXXnCUCyxud5eWFenUMAmvAWSDa1c7dvdXRMZ7" +PEER_ID_BYTES = b'\x12 7\x87F.[\xb5\xb1o\xe5*\xc7\xb9\xbb\x11:"Z|j2\x8ad\x1b\xa6\xe5= timeout: - # timeout - assert False, f"{coro_func} still failed after `{timeout}` seconds" - await asyncio.sleep(0.01) - - -class Daemon: - control_maddr = None - proc_daemon = None - log_filename = "" - f_log = None - closed = None - - def __init__( - self, control_maddr, enable_control, enable_connmgr, enable_dht, enable_pubsub - ): - self.control_maddr = control_maddr - self.enable_control = enable_control - self.enable_connmgr = enable_connmgr - self.enable_dht = enable_dht - self.enable_pubsub = enable_pubsub - self.is_closed = False - self._start_logging() - self._run() - - def _start_logging(self): - name_control_maddr = str(self.control_maddr).replace("/", "_").replace(".", "_") - self.log_filename = f"/tmp/log_p2pd{name_control_maddr}.txt" - self.f_log = open(self.log_filename, "wb") - - def _run(self): - cmd_list = ["hivemind/hivemind_cli/p2pd", f"-listen={str(self.control_maddr)}"] - cmd_list += [f"-hostAddrs=/ip4/127.0.0.1/tcp/{find_open_port()}"] - if self.enable_connmgr: - cmd_list += ["-connManager=true", "-connLo=1", "-connHi=2", "-connGrace=0"] - if self.enable_dht: - cmd_list += ["-dht=true"] - if self.enable_pubsub: - cmd_list += ["-pubsub=true", "-pubsubRouter=gossipsub"] - self.proc_daemon = subprocess.Popen( - cmd_list, stdout=self.f_log, stderr=self.f_log, bufsize=0 - ) - - async def wait_until_ready(self): - lines_head_pattern = (b"Control socket:", b"Peer ID:", b"Peer Addrs:") - lines_head_occurred = {line: False for line in lines_head_pattern} - - with open(self.log_filename, "rb") as f_log_read: - - async def read_from_daemon_and_check(): - line = f_log_read.readline() - for head_pattern in lines_head_occurred: - if line.startswith(head_pattern): - lines_head_occurred[head_pattern] = True - return all([value for _, value in lines_head_occurred.items()]) - - await try_until_success(read_from_daemon_and_check) - - # sleep for a while in case that the daemon haven't been ready after emitting these lines - await asyncio.sleep(0.1) - - def close(self): - if self.is_closed: - return - self.proc_daemon.terminate() - self.proc_daemon.wait() - self.f_log.close() - self.is_closed = True - - -class DaemonTuple(NamedTuple): - daemon: Daemon - client: Client - - -class ConnectionFailure(Exception): - pass - - -@asynccontextmanager -async def make_p2pd_pair_unix( - enable_control, enable_connmgr, enable_dht, enable_pubsub -): - name = str(uuid.uuid4())[:8] - control_maddr = Multiaddr(f"/unix/tmp/test_p2pd_control_{name}.sock") - listen_maddr = Multiaddr(f"/unix/tmp/test_p2pd_listen_{name}.sock") - # Remove the existing unix socket files if they are existing - try: - os.unlink(control_maddr.value_for_protocol(protocols.P_UNIX)) - except FileNotFoundError: - pass - try: - os.unlink(listen_maddr.value_for_protocol(protocols.P_UNIX)) - except FileNotFoundError: - pass - async with _make_p2pd_pair( - control_maddr=control_maddr, - listen_maddr=listen_maddr, - enable_control=enable_control, - enable_connmgr=enable_connmgr, - enable_dht=enable_dht, - enable_pubsub=enable_pubsub, - ) as pair: - yield pair - - -@asynccontextmanager -async def make_p2pd_pair_ip4(enable_control, enable_connmgr, enable_dht, enable_pubsub): - control_maddr = Multiaddr(f"/ip4/127.0.0.1/tcp/{find_open_port()}") - listen_maddr = Multiaddr(f"/ip4/127.0.0.1/tcp/{find_open_port()}") - async with _make_p2pd_pair( - control_maddr=control_maddr, - listen_maddr=listen_maddr, - enable_control=enable_control, - enable_connmgr=enable_connmgr, - enable_dht=enable_dht, - enable_pubsub=enable_pubsub, - ) as pair: - yield pair - - -@asynccontextmanager -async def _make_p2pd_pair( - control_maddr, - listen_maddr, - enable_control, - enable_connmgr, - enable_dht, - enable_pubsub, -): - p2pd = Daemon( - control_maddr=control_maddr, - enable_control=enable_control, - enable_connmgr=enable_connmgr, - enable_dht=enable_dht, - enable_pubsub=enable_pubsub, - ) - # wait for daemon ready - await p2pd.wait_until_ready() - client = Client(control_maddr=control_maddr, listen_maddr=listen_maddr) - try: - async with client.listen(): - yield DaemonTuple(daemon=p2pd, client=client) - finally: - if not p2pd.is_closed: - p2pd.close() - - -@pytest.fixture -async def p2pcs( - num_p2pds, - enable_control, - enable_connmgr, - enable_dht, - enable_pubsub, - func_make_p2pd_pair, -): +async def p2pcs(): # TODO: Change back to gather style async with AsyncExitStack() as stack: p2pd_tuples = [ await stack.enter_async_context( - func_make_p2pd_pair( - enable_control=enable_control, - enable_connmgr=enable_connmgr, - enable_dht=enable_dht, - enable_pubsub=enable_pubsub, + FUNC_MAKE_P2PD_PAIR( + enable_control=ENABLE_CONTROL, + enable_connmgr=ENABLE_CONNMGR, + enable_dht=ENABLE_DHT, + enable_pubsub=ENABLE_PUBSUB, ) ) - for _ in range(num_p2pds) + for _ in range(NUM_P2PDS) ] yield tuple(p2pd_tuple.client for p2pd_tuple in p2pd_tuples) -@pytest.mark.parametrize( - "enable_control, func_make_p2pd_pair", ((True, make_p2pd_pair_unix),) -) @pytest.mark.asyncio async def test_client_identify_unix_socket(p2pcs): await p2pcs[0].identify() -@pytest.mark.parametrize("enable_control", (True,)) @pytest.mark.asyncio async def test_client_identify(p2pcs): await p2pcs[0].identify() -@pytest.mark.parametrize("enable_control", (True,)) @pytest.mark.asyncio async def test_client_connect_success(p2pcs): peer_id_0, maddrs_0 = await p2pcs[0].identify() @@ -558,14 +330,13 @@ async def test_client_connect_success(p2pcs): await p2pcs[1].connect(peer_id_0, maddrs_0) -@pytest.mark.parametrize("enable_control", (True,)) @pytest.mark.asyncio -async def test_client_connect_failure(peer_id_random, p2pcs): +async def test_client_connect_failure(p2pcs): peer_id_1, maddrs_1 = await p2pcs[1].identify() await p2pcs[0].identify() # test case: `peer_id` mismatches with pytest.raises(ControlFailure): - await p2pcs[0].connect(peer_id_random, maddrs_1) + await p2pcs[0].connect(PEER_ID_RANDOM, maddrs_1) # test case: empty maddrs with pytest.raises(ControlFailure): await p2pcs[0].connect(peer_id_1, []) @@ -574,31 +345,11 @@ async def test_client_connect_failure(peer_id_random, p2pcs): await p2pcs[0].connect(peer_id_1, [Multiaddr("/ip4/127.0.0.1/udp/0")]) -async def _check_connection(p2pd_tuple_0, p2pd_tuple_1): - peer_id_0, _ = await p2pd_tuple_0.identify() - peer_id_1, _ = await p2pd_tuple_1.identify() - peers_0 = [pinfo.peer_id for pinfo in await p2pd_tuple_0.list_peers()] - peers_1 = [pinfo.peer_id for pinfo in await p2pd_tuple_1.list_peers()] - return (peer_id_0 in peers_1) and (peer_id_1 in peers_0) - - -async def connect_safe(p2pd_tuple_0, p2pd_tuple_1): - peer_id_1, maddrs_1 = await p2pd_tuple_1.identify() - await p2pd_tuple_0.connect(peer_id_1, maddrs_1) - await try_until_success( - functools.partial( - _check_connection, p2pd_tuple_0=p2pd_tuple_0, p2pd_tuple_1=p2pd_tuple_1 - ) - ) - - -@pytest.mark.parametrize("enable_control", (True,)) @pytest.mark.asyncio async def test_connect_safe(p2pcs): await connect_safe(p2pcs[0], p2pcs[1]) -@pytest.mark.parametrize("enable_control", (True,)) @pytest.mark.asyncio async def test_client_list_peers(p2pcs): # test case: no peers @@ -614,11 +365,10 @@ async def test_client_list_peers(p2pcs): assert len(await p2pcs[2].list_peers()) == 1 -@pytest.mark.parametrize("enable_control", (True,)) @pytest.mark.asyncio -async def test_client_disconnect(peer_id_random, p2pcs): +async def test_client_disconnect(p2pcs): # test case: disconnect a peer without connections - await p2pcs[1].disconnect(peer_id_random) + await p2pcs[1].disconnect(PEER_ID_RANDOM) # test case: disconnect peer_id_0, _ = await p2pcs[0].identify() await connect_safe(p2pcs[0], p2pcs[1]) @@ -633,7 +383,6 @@ async def test_client_disconnect(peer_id_random, p2pcs): assert len(await p2pcs[1].list_peers()) == 0 -@pytest.mark.parametrize("enable_control", (True,)) @pytest.mark.asyncio async def test_client_stream_open_success(p2pcs): peer_id_1, maddrs_1 = await p2pcs[1].identify() @@ -663,7 +412,6 @@ async def handle_proto(stream_info, reader, writer): writer.close() -@pytest.mark.parametrize("enable_control", (True,)) @pytest.mark.asyncio async def test_client_stream_open_failure(p2pcs): peer_id_1, _ = await p2pcs[1].identify() @@ -684,7 +432,6 @@ async def handle_proto(stream_info, reader, writer): await p2pcs[0].stream_open(peer_id_1, ("another_protocol",)) -@pytest.mark.parametrize("enable_control", (True,)) @pytest.mark.asyncio async def test_client_stream_handler_success(p2pcs): peer_id_1, _ = await p2pcs[1].identify() @@ -758,7 +505,6 @@ async def handler_third(stream_info, reader, writer): await event_third.wait() -@pytest.mark.parametrize("enable_control", (True,)) @pytest.mark.asyncio async def test_client_stream_handler_failure(p2pcs): peer_id_1, _ = await p2pcs[1].identify() diff --git a/tests/test_utils/__init__.py b/tests/test_utils/__init__.py index b30674ed7..7dab79902 100644 --- a/tests/test_utils/__init__.py +++ b/tests/test_utils/__init__.py @@ -1,4 +1,17 @@ +import asyncio +import functools +import os +import subprocess +import time +import uuid +from contextlib import asynccontextmanager +from typing import NamedTuple + import torch +from multiaddr import Multiaddr, protocols + +from hivemind import find_open_port +from hivemind.p2p.p2p_daemon_bindings.p2pclient import Client def print_device_info(device=None): @@ -12,3 +25,182 @@ def print_device_info(device=None): print('Memory Usage:') print('Allocated:', round(torch.cuda.memory_allocated(0) / 1024 ** 3, 1), 'GB') print('Cached: ', round(torch.cuda.memory_cached(0) / 1024 ** 3, 1), 'GB') + + +TIMEOUT_DURATION = 30 # seconds + + +async def try_until_success(coro_func, timeout=TIMEOUT_DURATION): + """ + Keep running ``coro_func`` until the time is out. + All arguments of ``coro_func`` should be filled, i.e. it should be called without arguments. + """ + t_start = time.monotonic() + while True: + result = await coro_func() + if result: + break + if (time.monotonic() - t_start) >= timeout: + # timeout + assert False, f"{coro_func} still failed after `{timeout}` seconds" + await asyncio.sleep(0.01) + + +class Daemon: + control_maddr = None + proc_daemon = None + log_filename = "" + f_log = None + closed = None + + def __init__( + self, control_maddr, enable_control, enable_connmgr, enable_dht, enable_pubsub + ): + self.control_maddr = control_maddr + self.enable_control = enable_control + self.enable_connmgr = enable_connmgr + self.enable_dht = enable_dht + self.enable_pubsub = enable_pubsub + self.is_closed = False + self._start_logging() + self._run() + + def _start_logging(self): + name_control_maddr = str(self.control_maddr).replace("/", "_").replace(".", "_") + self.log_filename = f"/tmp/log_p2pd{name_control_maddr}.txt" + self.f_log = open(self.log_filename, "wb") + + def _run(self): + cmd_list = ["hivemind/hivemind_cli/p2pd", f"-listen={str(self.control_maddr)}"] + cmd_list += [f"-hostAddrs=/ip4/127.0.0.1/tcp/{find_open_port()}"] + if self.enable_connmgr: + cmd_list += ["-connManager=true", "-connLo=1", "-connHi=2", "-connGrace=0"] + if self.enable_dht: + cmd_list += ["-dht=true"] + if self.enable_pubsub: + cmd_list += ["-pubsub=true", "-pubsubRouter=gossipsub"] + self.proc_daemon = subprocess.Popen( + cmd_list, stdout=self.f_log, stderr=self.f_log, bufsize=0 + ) + + async def wait_until_ready(self): + lines_head_pattern = (b"Control socket:", b"Peer ID:", b"Peer Addrs:") + lines_head_occurred = {line: False for line in lines_head_pattern} + + with open(self.log_filename, "rb") as f_log_read: + + async def read_from_daemon_and_check(): + line = f_log_read.readline() + for head_pattern in lines_head_occurred: + if line.startswith(head_pattern): + lines_head_occurred[head_pattern] = True + return all([value for _, value in lines_head_occurred.items()]) + + await try_until_success(read_from_daemon_and_check) + + # sleep for a while in case that the daemon haven't been ready after emitting these lines + await asyncio.sleep(0.1) + + def close(self): + if self.is_closed: + return + self.proc_daemon.terminate() + self.proc_daemon.wait() + self.f_log.close() + self.is_closed = True + + +class DaemonTuple(NamedTuple): + daemon: Daemon + client: Client + + +class ConnectionFailure(Exception): + pass + + +@asynccontextmanager +async def make_p2pd_pair_unix( + enable_control, enable_connmgr, enable_dht, enable_pubsub +): + name = str(uuid.uuid4())[:8] + control_maddr = Multiaddr(f"/unix/tmp/test_p2pd_control_{name}.sock") + listen_maddr = Multiaddr(f"/unix/tmp/test_p2pd_listen_{name}.sock") + # Remove the existing unix socket files if they are existing + try: + os.unlink(control_maddr.value_for_protocol(protocols.P_UNIX)) + except FileNotFoundError: + pass + try: + os.unlink(listen_maddr.value_for_protocol(protocols.P_UNIX)) + except FileNotFoundError: + pass + async with _make_p2pd_pair( + control_maddr=control_maddr, + listen_maddr=listen_maddr, + enable_control=enable_control, + enable_connmgr=enable_connmgr, + enable_dht=enable_dht, + enable_pubsub=enable_pubsub, + ) as pair: + yield pair + + +@asynccontextmanager +async def make_p2pd_pair_ip4(enable_control, enable_connmgr, enable_dht, enable_pubsub): + control_maddr = Multiaddr(f"/ip4/127.0.0.1/tcp/{find_open_port()}") + listen_maddr = Multiaddr(f"/ip4/127.0.0.1/tcp/{find_open_port()}") + async with _make_p2pd_pair( + control_maddr=control_maddr, + listen_maddr=listen_maddr, + enable_control=enable_control, + enable_connmgr=enable_connmgr, + enable_dht=enable_dht, + enable_pubsub=enable_pubsub, + ) as pair: + yield pair + + +@asynccontextmanager +async def _make_p2pd_pair( + control_maddr, + listen_maddr, + enable_control, + enable_connmgr, + enable_dht, + enable_pubsub, +): + p2pd = Daemon( + control_maddr=control_maddr, + enable_control=enable_control, + enable_connmgr=enable_connmgr, + enable_dht=enable_dht, + enable_pubsub=enable_pubsub, + ) + # wait for daemon ready + await p2pd.wait_until_ready() + client = Client(control_maddr=control_maddr, listen_maddr=listen_maddr) + try: + async with client.listen(): + yield DaemonTuple(daemon=p2pd, client=client) + finally: + if not p2pd.is_closed: + p2pd.close() + + +async def _check_connection(p2pd_tuple_0, p2pd_tuple_1): + peer_id_0, _ = await p2pd_tuple_0.identify() + peer_id_1, _ = await p2pd_tuple_1.identify() + peers_0 = [pinfo.peer_id for pinfo in await p2pd_tuple_0.list_peers()] + peers_1 = [pinfo.peer_id for pinfo in await p2pd_tuple_1.list_peers()] + return (peer_id_0 in peers_1) and (peer_id_1 in peers_0) + + +async def connect_safe(p2pd_tuple_0, p2pd_tuple_1): + peer_id_1, maddrs_1 = await p2pd_tuple_1.identify() + await p2pd_tuple_0.connect(peer_id_1, maddrs_1) + await try_until_success( + functools.partial( + _check_connection, p2pd_tuple_0=p2pd_tuple_0, p2pd_tuple_1=p2pd_tuple_1 + ) + ) \ No newline at end of file From fd39c2c06ee039372bee2a26f020e93ebec1521e Mon Sep 17 00:00:00 2001 From: Maxim Kashirin Date: Wed, 12 May 2021 19:57:30 +0300 Subject: [PATCH 42/81] remove obvious comments --- tests/test_p2p_daemon_bindings.py | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) diff --git a/tests/test_p2p_daemon_bindings.py b/tests/test_p2p_daemon_bindings.py index 7f87cbd88..9d7c203cf 100644 --- a/tests/test_p2p_daemon_bindings.py +++ b/tests/test_p2p_daemon_bindings.py @@ -127,33 +127,27 @@ async def test_read_write_unsigned_varint_max_bits_edge(max_bits): def test_peer_id(): - # test initialized with bytes assert PEER_ID.to_bytes() == PEER_ID_BYTES assert PEER_ID.to_string() == PEER_ID_STRING - # test initialized with string + peer_id_2 = PeerID.from_base58(PEER_ID_STRING) assert peer_id_2.to_bytes() == PEER_ID_BYTES assert peer_id_2.to_string() == PEER_ID_STRING - # test equal assert PEER_ID == peer_id_2 - # test not equal peer_id_3 = PeerID.from_base58("QmbmfNDEth7Ucvjuxiw3SP3E4PoJzbk7g4Ge6ZDigbCsNp") assert PEER_ID != peer_id_3 def test_stream_info(): proto = "123" - # test case: `StreamInfo.__init__` si = StreamInfo(PEER_ID, MADDR, proto) assert si.peer_id == PEER_ID assert si.addr == MADDR assert si.proto == proto - # test case: `StreamInfo.to_pb` pb_si = si.to_protobuf() assert pb_si.peer == PEER_ID.to_bytes() assert pb_si.addr == MADDR.to_bytes() assert pb_si.proto == si.proto - # test case: `StreamInfo.from_pb` si_1 = StreamInfo.from_protobuf(pb_si) assert si_1.peer_id == PEER_ID assert si_1.addr == MADDR @@ -162,10 +156,8 @@ def test_stream_info(): def test_peer_info(): pi = PeerInfo(PEER_ID, [MADDR]) - # test case: `PeerInfo.__init__` assert pi.peer_id == PEER_ID assert pi.addrs == [MADDR] - # test case: `PeerInfo.from_pb` pi_pb = p2pd_pb.PeerInfo(id=PEER_ID.to_bytes(), addrs=[MADDR.to_bytes()]) pi_1 = PeerInfo.from_protobuf(pi_pb) assert pi.peer_id == pi_1.peer_id From 5575d206192d6f0ba507a3fa03f043e5a2467656 Mon Sep 17 00:00:00 2001 From: Maxim Kashirin Date: Thu, 13 May 2021 08:50:56 +0300 Subject: [PATCH 43/81] raw bytes to pb creation --- tests/test_p2p_daemon_bindings.py | 78 ++++++++++++++++++++++++------- 1 file changed, 62 insertions(+), 16 deletions(-) diff --git a/tests/test_p2p_daemon_bindings.py b/tests/test_p2p_daemon_bindings.py index 9d7c203cf..172bded54 100644 --- a/tests/test_p2p_daemon_bindings.py +++ b/tests/test_p2p_daemon_bindings.py @@ -212,9 +212,30 @@ def test_control_client_ctor_default_listen_maddr(): @pytest.mark.parametrize( "msg_bytes", ( - b'\x08\x00"R\n"\x12 F\xec\xd3p0X\xbeT\x95p^\xc8{\xc8\x13\xa3\x9c\x84d\x0b\x1b\xbb\xa0P\x98w\xc1\xb3\x981i\x16\x12\x02\xa2\x02\x12\x08\x04\x7f\x00\x00\x01\x06\xc7\xb6\x12\x08\x04\xc0\xa8\n\x87\x06\xc7\xb6\x12\x14)\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x01\x06\xc7\xb7', # noqa: E501 - b'\x08\x00"R\n"\x12 \xd0\xf0 \x9a\xc6v\xa6\xd3;\xcac|\x95\x94\xa0\xe6:\nM\xc53T\x0e\xf0\x89\x8e(\x0c\xb9\xf7\\\xa5\x12\x02\xa2\x02\x12\x08\x04\x7f\x00\x00\x01\x06\xc9%\x12\x08\x04\xc0\xa8\n\x87\x06\xc9%\x12\x14)\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x01\x06\xc9&', # noqa: E501 - b'\x08\x00"R\n"\x12 \xc3\xc3\xee\x18i\x8a\xde\x13\xa9y\x905\xeb\xcb\xa4\xd07\x14\xbe\xf4\xf8\x1b\xe8[g94\x94\xe3f\x18\xa9\x12\x02\xa2\x02\x12\x08\x04\x7f\x00\x00\x01\x06\xc9`\x12\x08\x04\xc0\xa8\n\x87\x06\xc9`\x12\x14)\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x01\x06\xc9a', # noqa: E501 + p2pd_pb.Response( + type=p2pd_pb.Response.Type.OK, + identify=p2pd_pb.IdentifyResponse( + id=PeerID.from_base58('QmT7WhTne9zBLfAgAJt9aiZ8jZ5BxJGowRubxsHYmnyzUd').to_bytes(), + addrs=[Multiaddr('/p2p-circuit').to_bytes(), Multiaddr('/ip4/127.0.0.1/tcp/51126').to_bytes(), + Multiaddr('/ip4/192.168.10.135/tcp/51126').to_bytes(), + Multiaddr('/ip6/::1/tcp/51127').to_bytes()] + )).SerializeToString(), + p2pd_pb.Response( + type=p2pd_pb.Response.Type.OK, + identify=p2pd_pb.IdentifyResponse( + id=PeerID.from_base58('QmcQFt2MFfCZ9AxzUCNrk4k7TtMdZZvAAteaA6tHpBKdrk').to_bytes(), + addrs=[Multiaddr('/p2p-circuit').to_bytes(), Multiaddr('/ip4/127.0.0.1/tcp/51493').to_bytes(), + Multiaddr('/ip4/192.168.10.135/tcp/51493').to_bytes(), + Multiaddr('/ip6/::1/tcp/51494').to_bytes()] + )).SerializeToString(), + p2pd_pb.Response( + type=p2pd_pb.Response.Type.OK, + identify=p2pd_pb.IdentifyResponse( + id=PeerID.from_base58('QmbWqVVoz7v9LS9ZUQAhyyfdFJY3iU8ZrUY3XQozoTA5cc').to_bytes(), + addrs=[Multiaddr('/p2p-circuit').to_bytes(), Multiaddr('/ip4/127.0.0.1/tcp/51552').to_bytes(), + Multiaddr('/ip4/192.168.10.135/tcp/51552').to_bytes(), + Multiaddr('/ip6/::1/tcp/51553').to_bytes()] + )).SerializeToString(), ), # give test cases ids to prevent bytes from ruining the terminal ids=("pb example Response 0", "pb example Response 1", "pb example Response 2"), @@ -232,24 +253,46 @@ async def test_read_pbmsg_safe_valid(msg_bytes): @pytest.mark.parametrize( - "pb_msg, msg_bytes", + "pb_type, pb_msg", ( ( - p2pd_pb.Response(), - b'Z\x08\x00*V\x08\x01\x12R\n"\x12 \x03\x8d\xf5\xd4(/#\xd6\xed\xa5\x1bU\xb8s\x8c\xfa\xad\xfc{\x04\xe3\xecw\xdeK\xc9,\xfe\x9c\x00:\xc8\x12\x02\xa2\x02\x12\x08\x04\x7f\x00\x00\x01\x06\xdea\x12\x08\x04\xc0\xa8\n\x87\x06\xdea\x12\x14)\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x01\x06\xdeb', # noqa: E501 + p2pd_pb.Response, + p2pd_pb.Response( + type=p2pd_pb.Response.Type.OK, + dht=p2pd_pb.DHTResponse( + type=p2pd_pb.DHTResponse.Type.VALUE, + peer=p2pd_pb.PeerInfo( + id=PeerID.from_base58('QmNaXUy78W9moQ9APCoKaTtPjLcEJPN9hRBCqErY7o2fQs').to_bytes(), + addrs=[Multiaddr('/p2p-circuit').to_bytes(), Multiaddr('/ip4/127.0.0.1/tcp/56929').to_bytes(), + Multiaddr('/ip4/192.168.10.135/tcp/56929').to_bytes(), + Multiaddr('/ip6/::1/tcp/56930').to_bytes()] + ) + ) + ), ), - (p2pd_pb.Request(), b"\x02\x08\x05"), + (p2pd_pb.Request, p2pd_pb.Request(type=p2pd_pb.Request.Type.LIST_PEERS)), ( - p2pd_pb.DHTRequest(), - b'&\x08\x00\x12"\x12 \xd5\x0b\x18/\x9e\xa5G\x06.\xdd\xebW\xf0N\xf5\x0eW\xd3\xec\xdf\x06\x02\xe2\x89\x1e\xf0\xbb.\xc0\xbdE\xb8', # noqa: E501 + p2pd_pb.DHTRequest, + p2pd_pb.DHTRequest(type=p2pd_pb.DHTRequest.Type.FIND_PEER, + peer=PeerID.from_base58('QmcgHMuEhqdLHDVeNjiCGU7Ds6E7xK3f4amgiwHNPKKn7R').to_bytes()), ), ( - p2pd_pb.DHTResponse(), - b'V\x08\x01\x12R\n"\x12 wy\xe2\xfa\x11\x9e\xe2\x84X]\x84\xf8\x98\xba\x8c\x8cQ\xd7,\xb59\x1e!G\x92\x86G{\x141\xe9\x1b\x12\x02\xa2\x02\x12\x08\x04\x7f\x00\x00\x01\x06\xdeA\x12\x08\x04\xc0\xa8\n\x87\x06\xdeA\x12\x14)\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x01\x06\xdeB', # noqa: E501 + p2pd_pb.DHTResponse, + p2pd_pb.DHTResponse( + type=p2pd_pb.DHTResponse.Type.VALUE, + peer=p2pd_pb.PeerInfo( + id=PeerID.from_base58('QmWP32GhEyXVQsLXFvV81eadDC8zQRZxZvJK359rXxLquk').to_bytes(), + addrs=[Multiaddr('/p2p-circuit').to_bytes(), Multiaddr('/ip4/127.0.0.1/tcp/56897').to_bytes(), + Multiaddr('/ip4/192.168.10.135/tcp/56897').to_bytes(), + Multiaddr('/ip6/::1/tcp/56898').to_bytes()] + ) + ), ), ( - p2pd_pb.StreamInfo(), - b';\n"\x12 \xf6\x9e=\x9f\xc1J\xfe\x02\x93k!S\x80\xa0\xcc(s\xea&\xbe\xed\x9274qTI\xc1\xf7\xb6\xbd7\x12\x08\x04\x7f\x00\x00\x01\x06\xde\xc5\x1a\x0bprotocol123', # noqa: E501 + p2pd_pb.StreamInfo, + p2pd_pb.StreamInfo(peer=PeerID.from_base58('QmewLxB46MftfxQiunRgJo2W8nW4Lh5NLEkRohkHhJ4wW6').to_bytes(), + addr=Multiaddr('/ip4/127.0.0.1/tcp/57029').to_bytes(), + proto=b'protocol123'), ), ), ids=( @@ -261,11 +304,14 @@ async def test_read_pbmsg_safe_valid(msg_bytes): ), ) @pytest.mark.asyncio -async def test_write_pbmsg(pb_msg, msg_bytes): +async def test_write_pbmsg(pb_type, pb_msg): + msg_bytes = bytes(chr(pb_msg.ByteSize()), 'utf-8') + pb_msg.SerializeToString() + pb_obj = pb_type() + s_read = MockReaderWriter(msg_bytes) - await read_pbmsg_safe(s_read, pb_msg) + await read_pbmsg_safe(s_read, pb_obj) s_write = MockReaderWriter() - await write_pbmsg(s_write, pb_msg) + await write_pbmsg(s_write, pb_obj) assert msg_bytes == s_write.getvalue() From 295a20fdfd38da349f939eccee66567cb2c375da Mon Sep 17 00:00:00 2001 From: Denis Mazur Date: Mon, 1 Mar 2021 23:17:11 +0300 Subject: [PATCH 44/81] Compiling libp2p daemon on setup (#153) * add setup.py prototype * refactor --- setup.py | 56 +++++++++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 55 insertions(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 53cb6b77d..187d3ea43 100644 --- a/setup.py +++ b/setup.py @@ -2,6 +2,10 @@ import glob import os import re +import subprocess +import urllib.request +import tarfile +import tempfile from pkg_resources import parse_requirements from setuptools import setup, find_packages @@ -9,6 +13,19 @@ from setuptools.command.install import install +class cd: + """Context manager for changing the current working directory""" + def __init__(self, newPath): + self.newPath = os.path.expanduser(newPath) + + def __enter__(self): + self.savedPath = os.getcwd() + os.chdir(self.newPath) + + def __exit__(self, etype, value, traceback): + os.chdir(self.savedPath) + + def proto_compile(output_path): import grpc_tools.protoc @@ -28,6 +45,38 @@ def proto_compile(output_path): file.truncate() +def install_libp2p_daemon(): + # check go version: + try: + proc = subprocess.Popen(['go', 'version'], + stdout=subprocess.PIPE) + result, _ = proc.communicate() + result = result.decode('ascii', 'replace') + _, _, version, _ = result.split(' ') + version = version.lstrip('go') + + if version < "1.13": + raise EnvironmentError(f'newer version of go required: must be >= 1.13, found {version}') + + except FileNotFoundError: + raise FileNotFoundError('could not find golang installation') + + with tempfile.TemporaryDirectory() as tempdir: + url = 'https://github.com/libp2p/go-libp2p-daemon/archive/master.tar.gz' + dest = os.path.join(tempdir, 'libp2p-daemin.tar.gz') + urllib.request.urlretrieve(url, os.path.join(tempdir, dest)) + + tar = tarfile.open(dest, 'r:gz') + tar.extractall(tempdir) + tar.close() + + with cd(os.path.join(tempdir, 'go-libp2p-daemon-master')): + status = os.system('go install ./...') + if status: + raise RuntimeError('Failed to build or install libp2p-daemon:'\ + f' exited with status code :{status}') + + class ProtoCompileInstall(install): def run(self): proto_compile(os.path.join(self.build_lib, 'hivemind', 'proto')) @@ -40,6 +89,11 @@ def run(self): super().run() +class LibP2PInstall(install): + def run(self): + install_libp2p_daemon() + + here = os.path.abspath(os.path.dirname(__file__)) with open('requirements.txt') as requirements_file: @@ -63,7 +117,7 @@ def run(self): setup( name='hivemind', version=version_string, - cmdclass={'install': ProtoCompileInstall, 'develop': ProtoCompileDevelop}, + cmdclass={'install': ProtoCompileInstall, 'develop': ProtoCompileDevelop, 'libp2p': LibP2PInstall}, description='Decentralized deep learning in PyTorch', long_description='Decentralized deep learning in PyTorch. Built to train giant models on ' 'thousands of volunteers across the world.', From 67b2d4446cde18301eda003a88c7b241fd8f1dff Mon Sep 17 00:00:00 2001 From: Ilya <37004806+skobellev@users.noreply.github.com> Date: Wed, 3 Mar 2021 12:25:17 +0300 Subject: [PATCH 45/81] feat: add p2p daemon (#164) * Add p2p daemon * Test p2p daemon exits correctly * Impose restriction on elapsed time Co-authored-by: Ilya Kobelev --- hivemind/__init__.py | 1 + hivemind/p2p/__init__.py | 1 + hivemind/p2p/p2p_daemon.py | 45 +++++++++++++++++++++++++++++++ tests/test_p2p_daemon.py | 55 ++++++++++++++++++++++++++++++++++++++ 4 files changed, 102 insertions(+) create mode 100644 hivemind/p2p/__init__.py create mode 100644 hivemind/p2p/p2p_daemon.py create mode 100644 tests/test_p2p_daemon.py diff --git a/hivemind/__init__.py b/hivemind/__init__.py index ebbfa0588..3fdfc625b 100644 --- a/hivemind/__init__.py +++ b/hivemind/__init__.py @@ -1,5 +1,6 @@ from hivemind.client import * from hivemind.dht import * +from hivemind.p2p import * from hivemind.server import * from hivemind.utils import * from hivemind.optim import * diff --git a/hivemind/p2p/__init__.py b/hivemind/p2p/__init__.py new file mode 100644 index 000000000..6bae0b8bd --- /dev/null +++ b/hivemind/p2p/__init__.py @@ -0,0 +1 @@ +from hivemind.p2p.p2p_daemon import P2P diff --git a/hivemind/p2p/p2p_daemon.py b/hivemind/p2p/p2p_daemon.py new file mode 100644 index 000000000..3083c70e5 --- /dev/null +++ b/hivemind/p2p/p2p_daemon.py @@ -0,0 +1,45 @@ +import subprocess +import typing as tp + + +class P2P(object): + """ + Forks a child process and executes p2pd command with given arguments. + Sends SIGKILL to the child in destructor and on exit from contextmanager. + """ + + LIBP2P_CMD = 'p2pd' + + def __init__(self, *args, **kwargs): + self._child = subprocess.Popen(args=self._make_process_args(args, kwargs)) + try: + stdout, stderr = self._child.communicate(timeout=0.2) + except subprocess.TimeoutExpired: + pass + else: + raise RuntimeError(f'p2p daemon exited with stderr: {stderr}') + + def __enter__(self): + return self._child + + def __exit__(self, exc_type, exc_val, exc_tb): + self._kill_child() + + def __del__(self): + self._kill_child() + + def _kill_child(self): + if self._child.poll() is None: + self._child.kill() + self._child.wait() + + def _make_process_args(self, args: tp.Tuple[tp.Any], + kwargs: tp.Dict[str, tp.Any]) -> tp.List[str]: + proc_args = [self.LIBP2P_CMD] + proc_args.extend( + str(entry) for entry in args + ) + proc_args.extend( + f'-{key}={str(value)}' for key, value in kwargs.items() + ) + return proc_args diff --git a/tests/test_p2p_daemon.py b/tests/test_p2p_daemon.py new file mode 100644 index 000000000..ac57e9e2f --- /dev/null +++ b/tests/test_p2p_daemon.py @@ -0,0 +1,55 @@ +import subprocess +from time import perf_counter + +import pytest + +import hivemind.p2p +from hivemind.p2p import P2P + +RUNNING = 'running' +NOT_RUNNING = 'not running' +CHECK_PID_CMD = ''' +if ps -p {0} > /dev/null; +then + echo "{1}" +else + echo "{2}" +fi +''' + + +def is_process_running(pid: int) -> bool: + cmd = CHECK_PID_CMD.format(pid, RUNNING, NOT_RUNNING) + return subprocess.check_output(cmd, shell=True).decode('utf-8').strip() == RUNNING + + +@pytest.fixture() +def mock_p2p_class(): + P2P.LIBP2P_CMD = "sleep" + + +def test_daemon_killed_on_del(mock_p2p_class): + start = perf_counter() + p2p_daemon = P2P('10s') + + child_pid = p2p_daemon._child.pid + assert is_process_running(child_pid) + + del p2p_daemon + assert not is_process_running(child_pid) + assert perf_counter() - start < 1 + + +def test_daemon_killed_on_exit(mock_p2p_class): + start = perf_counter() + with P2P('10s') as daemon: + child_pid = daemon.pid + assert is_process_running(child_pid) + + assert not is_process_running(child_pid) + assert perf_counter() - start < 1 + + +def test_daemon_raises_on_faulty_args(): + with pytest.raises(RuntimeError): + P2P(faulty='argument') From e6c92777d8602e4d49977ac47943b74962d82b27 Mon Sep 17 00:00:00 2001 From: Denis Mazur Date: Fri, 5 Mar 2021 05:19:39 +0300 Subject: [PATCH 46/81] compare golang versions using packaging.version --- setup.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/setup.py b/setup.py index 187d3ea43..f67235750 100644 --- a/setup.py +++ b/setup.py @@ -7,6 +7,7 @@ import tarfile import tempfile +from packaging import version from pkg_resources import parse_requirements from setuptools import setup, find_packages from setuptools.command.develop import develop @@ -52,10 +53,10 @@ def install_libp2p_daemon(): stdout=subprocess.PIPE) result, _ = proc.communicate() result = result.decode('ascii', 'replace') - _, _, version, _ = result.split(' ') - version = version.lstrip('go') + _, _, v, _ = result.split(' ') + v = v.lstrip('go') - if version < "1.13": + if version.parse(v) < version.parse("1.13"): raise EnvironmentError(f'newer version of go required: must be >= 1.13, found {version}') except FileNotFoundError: From b454acf90640fa3ecacc1d99fe256e61ab779099 Mon Sep 17 00:00:00 2001 From: Denis Mazur Date: Tue, 9 Mar 2021 20:15:17 +0300 Subject: [PATCH 47/81] fix typo Co-authored-by: justheuristic --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index f67235750..d858d777c 100644 --- a/setup.py +++ b/setup.py @@ -64,7 +64,7 @@ def install_libp2p_daemon(): with tempfile.TemporaryDirectory() as tempdir: url = 'https://github.com/libp2p/go-libp2p-daemon/archive/master.tar.gz' - dest = os.path.join(tempdir, 'libp2p-daemin.tar.gz') + dest = os.path.join(tempdir, 'libp2p-daemon.tar.gz') urllib.request.urlretrieve(url, os.path.join(tempdir, dest)) tar = tarfile.open(dest, 'r:gz') From fc3e2b3716ff008dade8d6be4855735d51724e1b Mon Sep 17 00:00:00 2001 From: Denis Mazur Date: Wed, 10 Mar 2021 01:31:19 +0300 Subject: [PATCH 48/81] move p2pd executable to hivemind/hivemind_cli --- setup.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/setup.py b/setup.py index d858d777c..36630cd27 100644 --- a/setup.py +++ b/setup.py @@ -14,6 +14,9 @@ from setuptools.command.install import install +here = os.path.abspath(os.path.dirname(__file__)) + + class cd: """Context manager for changing the current working directory""" def __init__(self, newPath): @@ -71,8 +74,8 @@ def install_libp2p_daemon(): tar.extractall(tempdir) tar.close() - with cd(os.path.join(tempdir, 'go-libp2p-daemon-master')): - status = os.system('go install ./...') + with cd(os.path.join(tempdir, 'go-libp2p-daemon-master', 'p2pd')): + status = os.system(f'go build -o {os.path.join(here, "hivemind/hivemind_cli", "p2pd")}') if status: raise RuntimeError('Failed to build or install libp2p-daemon:'\ f' exited with status code :{status}') @@ -95,7 +98,6 @@ def run(self): install_libp2p_daemon() -here = os.path.abspath(os.path.dirname(__file__)) with open('requirements.txt') as requirements_file: install_requires = list(map(str, parse_requirements(requirements_file))) From decd6d791186d606a685f941e95b3c207e67e605 Mon Sep 17 00:00:00 2001 From: Denis Mazur Date: Wed, 10 Mar 2021 20:58:26 +0300 Subject: [PATCH 49/81] Rebase master onto libp2p (#179) * copytree implementation for py37 compatibility (#162) * copytree implementation for py37 compatibility * Running tests for python3.7 * Increment version * Python3.7 notions * Remove pickle.loads in averager (#160) * Security update: remove pickle.loads in averager * add py37 to circleci config Co-authored-by: Alexander Borzunov Co-authored-by: Max Ryabinin * Support edge cases for DHT key/subkey/value, add tests, update .gitignore for pb2 (#167) * fix bug with subkey equals zero * add autogenerated protobuf files to .gitignore * test store and get "tricky" values in dht * Fix the remaining tests for py37 (#166) * DecentralizedAverager is now compatible with python37's acyncio exception * the problem was: grpc.aio with python37 raised concurrent.futures.CancelledError in some cases; * we relied on isinstance(asyncio.CancelledError, Exception) == False * but isinstance(concurrent.futures.CancelledError, Exception) == True * DecentralizedAverager now shuts down if dereferenced in the main process * though it won't shutdown if dereferenced in forks for obvious reasons * HIVEMIND_THREADS now actually works * test_averaging now shuts down dht and averager instances to avoid leaking processes Co-authored-by: Max Ryabinin Co-authored-by: Max Ryabinin * Move Averager metadata serialization out of user scope (#168) * move metadata serialization outside user scope * test_overcrowded: reduce the default number of peers * Handle edge cases in DecentralizedAverager (#171) * move metadata serialization outside user scope * retry averager.step on network errors * raise AllreduceException on partial tensor * test split/combine tensors, combine corrupted stream Co-authored-by: Max Ryabinin * Fix a typo in quickstart.md (#174) * Serialize DHTID source with msgpack (#172) * Change DHTID serializer * Remove unused serializers * Add msgpack tuple serialization * Move CLI server launch script to hivemind/hivemind_cli (#173) * Cast environment variables to correct types * Compiling libp2p daemon on setup (#153) * add setup.py prototype * refactor * feat: add p2p daemon (#164) * Add p2p daemon * Test p2p daemon exits correctly * Impose restriction on elapsed time Co-authored-by: Ilya Kobelev * compare golang versions using packaging.version * fix typo Co-authored-by: justheuristic * move p2pd executable to hivemind/hivemind_cli Co-authored-by: Alexey Bukhtiyarov Co-authored-by: justheuristic Co-authored-by: Alexander Borzunov Co-authored-by: Max Ryabinin Co-authored-by: Michael Diskin Co-authored-by: romakail <36082689+romakail@users.noreply.github.com> Co-authored-by: Ilya <37004806+skobellev@users.noreply.github.com> Co-authored-by: Ilya Kobelev --- hivemind/client/averaging/matchmaking.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/hivemind/client/averaging/matchmaking.py b/hivemind/client/averaging/matchmaking.py index 8ec866e51..c711f49a6 100644 --- a/hivemind/client/averaging/matchmaking.py +++ b/hivemind/client/averaging/matchmaking.py @@ -467,5 +467,13 @@ async def _declare_averager_periodically(self, key_manager: GroupKeyManager): looking_for_group=False) +def compute_schema_hash(tensors: Sequence[torch.Tensor]) -> bytes: + """ A hash that describes follower's tensor shapes, dtypes, devices, but not the actual values """ + schema_dicts = [{field_name: str(field_value) + for field_name, field_value in asdict(TensorDescriptor.from_tensor(tensor)).items()} + for tensor in tensors] + return DHTID.generate(source=schema_dicts).to_bytes() + + class MatchmakingException(Exception): """ An internal exception that marks undesired edge cases during averaging """ From c0e6c824ef2a51a9f7dde5f8c222aff258719722 Mon Sep 17 00:00:00 2001 From: Denis Mazur Date: Fri, 19 Mar 2021 22:13:57 +0300 Subject: [PATCH 50/81] Fix LibP2P-Daemon installation in setup.py (#186) * Rename ProtoCompileInstall and ProtoCompileDevelop to Install and Develop * Install LibP2P-Daemon on setup install and setup develop * Install Golang in Circle CI builds * Add P2PD binary to gitignore --- .circleci/config.yml | 18 ++++++++++++ .gitignore | 3 ++ setup.py | 67 +++++++++++++++++++++++--------------------- 3 files changed, 56 insertions(+), 32 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index b1ab5978e..daaac9b6a 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -1,5 +1,10 @@ version: 2.1 +parameters: + go-version: + type: string + default: 1.16.2 + jobs: build-and-test-py37: docker: @@ -9,6 +14,11 @@ jobs: - restore_cache: keys: - py37-v1-{{ checksum "requirements.txt" }}-{{ checksum "requirements-dev.txt" }} + - v1-{{ checksum "requirements.txt" }}-{{ checksum "requirements-dev.txt" }} + - run: | + wget https://golang.org/dl/go<< pipeline.parameters.go-version >>.linux-amd64.tar.gz -O go.tar.gz + tar -C ~/ -xzf go.tar.gz + echo "export PATH=~/go/bin:$PATH" >> $BASH_ENV - run: pip install -r requirements.txt - run: pip install -r requirements-dev.txt - save_cache: @@ -29,6 +39,10 @@ jobs: - restore_cache: keys: - py38-v1-{{ checksum "requirements.txt" }}-{{ checksum "requirements-dev.txt" }} + - run: | + wget https://golang.org/dl/go<< pipeline.parameters.go-version >>.linux-amd64.tar.gz -O go.tar.gz + tar -C ~/ -xzf go.tar.gz + echo "export PATH=~/go/bin:$PATH" >> $BASH_ENV - run: pip install -r requirements.txt - run: pip install -r requirements-dev.txt - save_cache: @@ -49,6 +63,10 @@ jobs: - restore_cache: keys: - py39-v1-{{ checksum "requirements.txt" }}-{{ checksum "requirements-dev.txt" }} + - run: | + wget https://golang.org/dl/go<< pipeline.parameters.go-version >>.linux-amd64.tar.gz -O go.tar.gz + tar -C ~/ -xzf go.tar.gz + echo "export PATH=~/go/bin:$PATH" >> $BASH_ENV - run: pip install -r requirements.txt - run: pip install -r requirements-dev.txt - save_cache: diff --git a/.gitignore b/.gitignore index 965aa8972..61e239d1c 100644 --- a/.gitignore +++ b/.gitignore @@ -78,3 +78,6 @@ debian/files # protobuf stuff hivemind/proto/*_pb2* + +# libp2p-daemon binary +hivemind/hivemind_cli/p2pd diff --git a/setup.py b/setup.py index 36630cd27..6135feda7 100644 --- a/setup.py +++ b/setup.py @@ -6,6 +6,7 @@ import urllib.request import tarfile import tempfile +import hashlib from packaging import version from pkg_resources import parse_requirements @@ -13,21 +14,18 @@ from setuptools.command.develop import develop from setuptools.command.install import install +P2PD_VERSION = 'v0.3.1' +P2PD_CHECKSUM = '5094d094740f4e375afe80a5683b1bb2' here = os.path.abspath(os.path.dirname(__file__)) -class cd: - """Context manager for changing the current working directory""" - def __init__(self, newPath): - self.newPath = os.path.expanduser(newPath) - - def __enter__(self): - self.savedPath = os.getcwd() - os.chdir(self.newPath) - - def __exit__(self, etype, value, traceback): - os.chdir(self.savedPath) +def md5(fname, chunk_size=4096): + hash_md5 = hashlib.md5() + with open(fname, "rb") as f: + for chunk in iter(lambda: f.read(chunk_size), b""): + hash_md5.update(chunk) + return hash_md5.hexdigest() def proto_compile(output_path): @@ -49,8 +47,7 @@ def proto_compile(output_path): file.truncate() -def install_libp2p_daemon(): - # check go version: +def libp2p_build_install(): try: proc = subprocess.Popen(['go', 'version'], stdout=subprocess.PIPE) @@ -58,7 +55,7 @@ def install_libp2p_daemon(): result = result.decode('ascii', 'replace') _, _, v, _ = result.split(' ') v = v.lstrip('go') - + if version.parse(v) < version.parse("1.13"): raise EnvironmentError(f'newer version of go required: must be >= 1.13, found {version}') @@ -66,39 +63,45 @@ def install_libp2p_daemon(): raise FileNotFoundError('could not find golang installation') with tempfile.TemporaryDirectory() as tempdir: - url = 'https://github.com/libp2p/go-libp2p-daemon/archive/master.tar.gz' - dest = os.path.join(tempdir, 'libp2p-daemon.tar.gz') + url = f'https://github.com/learning-at-home/go-libp2p-daemon/archive/refs/tags/{P2PD_VERSION}.tar.gz' + dest = os.path.join(tempdir, 'libp2p-daemon.tar.gz') urllib.request.urlretrieve(url, os.path.join(tempdir, dest)) - + tar = tarfile.open(dest, 'r:gz') tar.extractall(tempdir) tar.close() - - with cd(os.path.join(tempdir, 'go-libp2p-daemon-master', 'p2pd')): - status = os.system(f'go build -o {os.path.join(here, "hivemind/hivemind_cli", "p2pd")}') - if status: - raise RuntimeError('Failed to build or install libp2p-daemon:'\ - f' exited with status code :{status}') + result = subprocess.run(['go', 'build', '-o', os.path.join(here, "hivemind/hivemind_cli", "p2pd")], + cwd=os.path.join(tempdir, f'go-libp2p-daemon-{P2PD_VERSION[1:]}', 'p2pd')) + if result.returncode: + raise RuntimeError('Failed to build or install libp2p-daemon:' + f' exited with status code :{result.returncode}') + + +def libp2p_download_install(): + install_path = os.path.join(here, 'hivemind/hivemind_cli/') + binary_path = os.path.join(install_path, 'p2pd') + if 'p2pd' not in os.listdir(install_path) or md5(binary_path) != P2PD_CHECKSUM: + print('Downloading Peer to Peer Daemon') + url = f'https://github.com/learning-at-home/go-libp2p-daemon/releases/download/{P2PD_VERSION}/p2pd' + urllib.request.urlretrieve(url, binary_path) + os.chmod(binary_path, 777) -class ProtoCompileInstall(install): + +class Install(install): def run(self): + libp2p_download_install() proto_compile(os.path.join(self.build_lib, 'hivemind', 'proto')) super().run() -class ProtoCompileDevelop(develop): +class Develop(develop): def run(self): + libp2p_build_install() proto_compile(os.path.join('hivemind', 'proto')) super().run() -class LibP2PInstall(install): - def run(self): - install_libp2p_daemon() - - - with open('requirements.txt') as requirements_file: install_requires = list(map(str, parse_requirements(requirements_file))) @@ -120,7 +123,7 @@ def run(self): setup( name='hivemind', version=version_string, - cmdclass={'install': ProtoCompileInstall, 'develop': ProtoCompileDevelop, 'libp2p': LibP2PInstall}, + cmdclass={'install': Install, 'develop': Develop}, description='Decentralized deep learning in PyTorch', long_description='Decentralized deep learning in PyTorch. Built to train giant models on ' 'thousands of volunteers across the world.', From a66ef9b6cca24b8499219efe5a101a5fefab8e1e Mon Sep 17 00:00:00 2001 From: Ilya <37004806+skobellev@users.noreply.github.com> Date: Sun, 21 Mar 2021 22:23:03 +0300 Subject: [PATCH 51/81] feat p2p_daemon: add API to call peer handle (#181) * Extend P2P api * Add tests for new api * Add p2pclient dependencies * Test P2P from different processes * Fix typo in tests * Add default initialization * Fix daemon ports assignment * Replace del with __del__ in tests * Read from input stream with receive_exactly Co-authored-by: Ilya Kobelev --- hivemind/p2p/p2p_daemon.py | 190 +++++++++++++++++++++++++++++++++---- tests/test_p2p_daemon.py | 142 +++++++++++++++++++++++---- 2 files changed, 292 insertions(+), 40 deletions(-) diff --git a/hivemind/p2p/p2p_daemon.py b/hivemind/p2p/p2p_daemon.py index 3083c70e5..1f441c5d1 100644 --- a/hivemind/p2p/p2p_daemon.py +++ b/hivemind/p2p/p2p_daemon.py @@ -1,45 +1,197 @@ +import asyncio +import contextlib +import copy +from pathlib import Path +import pickle +import socket import subprocess import typing as tp +import warnings + +from multiaddr import Multiaddr +import p2pclient +from libp2p.peer.id import ID class P2P(object): """ Forks a child process and executes p2pd command with given arguments. - Sends SIGKILL to the child in destructor and on exit from contextmanager. + Can be used for peer to peer communication and procedure calls. + Sends SIGKILL to the child in destructor. """ - LIBP2P_CMD = 'p2pd' + P2PD_RELATIVE_PATH = 'hivemind_cli/p2pd' + NUM_RETRIES = 3 + RETRY_DELAY = 0.4 + HEADER_LEN = 8 + BYTEORDER = 'big' - def __init__(self, *args, **kwargs): - self._child = subprocess.Popen(args=self._make_process_args(args, kwargs)) - try: - stdout, stderr = self._child.communicate(timeout=0.2) - except subprocess.TimeoutExpired: - pass - else: - raise RuntimeError(f'p2p daemon exited with stderr: {stderr}') + def __init__(self): + self._child = None + self._listen_task = None + self._server_stopped = asyncio.Event() + self._buffer = bytearray() - def __enter__(self): - return self._child + @classmethod + async def create(cls, *args, quic=1, tls=1, conn_manager=1, dht_client=1, + nat_port_map=True, auto_nat=True, bootstrap=True, + host_port: int = None, daemon_listen_port: int = None, **kwargs): + self = cls() + p2pd_path = Path(__file__).resolve().parents[1] / P2P.P2PD_RELATIVE_PATH + proc_args = self._make_process_args( + str(p2pd_path), *args, + quic=quic, tls=tls, connManager=conn_manager, + dhtClient=dht_client, natPortMap=nat_port_map, + autonat=auto_nat, b=bootstrap, **kwargs) + self._assign_daemon_ports(host_port, daemon_listen_port) + for try_count in range(self.NUM_RETRIES): + try: + self._initialize(proc_args) + await self._identify_client(P2P.RETRY_DELAY * (2 ** try_count)) + except Exception as exc: + warnings.warn("Failed to initialize p2p daemon: " + str(exc), RuntimeWarning) + self._kill_child() + if try_count == P2P.NUM_RETRIES - 1: + raise + self._assign_daemon_ports() + continue + break + return self - def __exit__(self, exc_type, exc_val, exc_tb): - self._kill_child() + def _initialize(self, proc_args: tp.List[str]) -> None: + proc_args = copy.deepcopy(proc_args) + proc_args.extend(self._make_process_args( + hostAddrs=f'/ip4/0.0.0.0/tcp/{self._host_port},/ip4/0.0.0.0/udp/{self._host_port}/quic', + listen=f'/ip4/127.0.0.1/tcp/{self._daemon_listen_port}' + )) + self._child = subprocess.Popen( + args=proc_args, + stdin=subprocess.PIPE, stdout=subprocess.PIPE, + stderr=subprocess.PIPE, encoding="utf8" + ) + self._client_listen_port = find_open_port() + self._client = p2pclient.Client( + Multiaddr(f'/ip4/127.0.0.1/tcp/{self._daemon_listen_port}'), + Multiaddr(f'/ip4/127.0.0.1/tcp/{self._client_listen_port}')) + + async def _identify_client(self, delay): + await asyncio.sleep(delay) + encoded = await self._client.identify() + self.id = encoded[0].to_base58() + + def _assign_daemon_ports(self, host_port=None, daemon_listen_port=None): + self._host_port, self._daemon_listen_port = host_port, daemon_listen_port + if host_port is None: + self._host_port = find_open_port() + if daemon_listen_port is None: + self._daemon_listen_port = find_open_port() + while self._daemon_listen_port == self._host_port: + self._daemon_listen_port = find_open_port() + + @staticmethod + async def send_data(data, stream): + byte_str = pickle.dumps(data) + request = len(byte_str).to_bytes(P2P.HEADER_LEN, P2P.BYTEORDER) + byte_str + await stream.send_all(request) + + class IncompleteRead(Exception): + pass + + async def _receive_exactly(self, stream, n_bytes, max_bytes=1 << 16): + while len(self._buffer) < n_bytes: + data = await stream.receive_some(max_bytes) + if len(data) == 0: + raise P2P.IncompleteRead() + self._buffer.extend(data) + + result = self._buffer[:n_bytes] + self._buffer = self._buffer[n_bytes:] + return bytes(result) + + async def receive_data(self, stream, max_bytes=(1 < 16)): + header = await self._receive_exactly(stream, P2P.HEADER_LEN) + content_length = int.from_bytes(header, P2P.BYTEORDER) + data = await self._receive_exactly(stream, content_length) + return pickle.loads(data) + + def _handle_stream(self, handle): + async def do_handle_stream(stream_info, stream): + try: + request = await self.receive_data(stream) + except P2P.IncompleteRead: + warnings.warn("Incomplete read while receiving request from peer", RuntimeWarning) + return + finally: + stream.close() + try: + result = handle(request) + await self.send_data(result, stream) + except Exception as exc: + await self.send_data(exc, stream) + finally: + await stream.close() + + return do_handle_stream + + def start_listening(self): + async def listen(): + async with self._client.listen(): + await self._server_stopped.wait() + + self._listen_task = asyncio.create_task(listen()) + + async def stop_listening(self): + if self._listen_task is not None: + self._server_stopped.set() + self._listen_task.cancel() + try: + await self._listen_task + except asyncio.CancelledError: + self._listen_task = None + self._server_stopped.clear() + + async def add_stream_handler(self, name, handle): + if self._listen_task is None: + self.start_listening() + + await self._client.stream_handler(name, self._handle_stream(handle)) + + async def call_peer_handler(self, peer_id, handler_name, input_data): + libp2p_peer_id = ID.from_base58(peer_id) + stream_info, stream = await self._client.stream_open(libp2p_peer_id, (handler_name,)) + try: + await self.send_data(input_data, stream) + return await self.receive_data(stream) + finally: + await stream.close() def __del__(self): self._kill_child() def _kill_child(self): - if self._child.poll() is None: + if self._child is not None and self._child.poll() is None: self._child.kill() self._child.wait() - def _make_process_args(self, args: tp.Tuple[tp.Any], - kwargs: tp.Dict[str, tp.Any]) -> tp.List[str]: - proc_args = [self.LIBP2P_CMD] + def _make_process_args(self, *args, **kwargs) -> tp.List[str]: + proc_args = [] proc_args.extend( str(entry) for entry in args ) proc_args.extend( - f'-{key}={str(value)}' for key, value in kwargs.items() + f'-{key}={value}' if value is not None else f'-{key}' + for key, value in kwargs.items() ) return proc_args + + +def find_open_port(params=(socket.AF_INET, socket.SOCK_STREAM), + opt=(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)): + """ Finds a tcp port that can be occupied with a socket with *params and use *opt options """ + try: + with contextlib.closing(socket.socket(*params)) as sock: + sock.bind(('', 0)) + sock.setsockopt(*opt) + return sock.getsockname()[1] + except Exception: + raise diff --git a/tests/test_p2p_daemon.py b/tests/test_p2p_daemon.py index ac57e9e2f..75fd51cdc 100644 --- a/tests/test_p2p_daemon.py +++ b/tests/test_p2p_daemon.py @@ -1,6 +1,8 @@ +import asyncio +import multiprocessing as mp import subprocess -from time import perf_counter +import numpy as np import pytest import hivemind.p2p @@ -23,33 +25,131 @@ def is_process_running(pid: int) -> bool: return subprocess.check_output(cmd, shell=True).decode('utf-8').strip() == RUNNING -@pytest.fixture() -def mock_p2p_class(): - P2P.LIBP2P_CMD = "sleep" - - -def test_daemon_killed_on_del(mock_p2p_class): - start = perf_counter() - p2p_daemon = P2P('10s') +@pytest.mark.asyncio +async def test_daemon_killed_on_del(): + p2p_daemon = await P2P.create() child_pid = p2p_daemon._child.pid assert is_process_running(child_pid) - del p2p_daemon + p2p_daemon.__del__() assert not is_process_running(child_pid) - assert perf_counter() - start < 1 -def test_daemon_killed_on_exit(mock_p2p_class): - start = perf_counter() - with P2P('10s') as daemon: - child_pid = daemon.pid - assert is_process_running(child_pid) +def handle_square(x): + return x ** 2 - assert not is_process_running(child_pid) - assert perf_counter() - start < 1 + +def handle_add(args): + result = args[0] + for i in range(1, len(args)): + result = result + args[i] + return result + + +@pytest.mark.parametrize( + "test_input,handle", + [ + pytest.param(10, handle_square, id="square_integer"), + pytest.param((1, 2), handle_add, id="add_integers"), + pytest.param(([1, 2, 3], [12, 13]), handle_add, id="add_lists"), + pytest.param(2, lambda x: x ** 3, id="lambda") + ] +) +@pytest.mark.asyncio +async def test_call_peer_single_process(test_input, handle, handler_name="handle"): + server = await P2P.create() + server_pid = server._child.pid + await server.add_stream_handler(handler_name, handle) + assert is_process_running(server_pid) + + client = await P2P.create() + client_pid = client._child.pid + assert is_process_running(client_pid) + + await asyncio.sleep(1) + result = await client.call_peer_handler(server.id, handler_name, test_input) + assert result == handle(test_input) + + await server.stop_listening() + server.__del__() + assert not is_process_running(server_pid) + + client.__del__() + assert not is_process_running(client_pid) + + +@pytest.mark.asyncio +async def test_call_peer_different_processes(): + handler_name = "square" + test_input = np.random.randn(2, 3) + + server_side, client_side = mp.Pipe() + response_received = mp.Value(np.ctypeslib.as_ctypes_type(np.int32)) + response_received.value = 0 + + async def run_server(): + server = await P2P.create() + server_pid = server._child.pid + await server.add_stream_handler(handler_name, handle_square) + assert is_process_running(server_pid) + + server_side.send(server.id) + while response_received.value == 0: + await asyncio.sleep(0.5) + + await server.stop_listening() + server.__del__() + assert not is_process_running(server_pid) + + def server_target(): + asyncio.run(run_server()) + + proc = mp.Process(target=server_target) + proc.start() + + client = await P2P.create() + client_pid = client._child.pid + assert is_process_running(client_pid) + + await asyncio.sleep(1) + peer_id = client_side.recv() + + result = await client.call_peer_handler(peer_id, handler_name, test_input) + assert np.allclose(result, handle_square(test_input)) + response_received.value = 1 + + client.__del__() + assert not is_process_running(client_pid) + + proc.join() -def test_daemon_raises_on_faulty_args(): - with pytest.raises(RuntimeError): - P2P(faulty='argument') +@pytest.mark.parametrize( + "test_input,handle", + [ + pytest.param(np.random.randn(2, 3), handle_square, id="square"), + pytest.param([np.random.randn(2, 3), np.random.randn(2, 3)], handle_add, id="add"), + ] +) +@pytest.mark.asyncio +async def test_call_peer_numpy(test_input, handle, handler_name="handle"): + server = await P2P.create() + await server.add_stream_handler(handler_name, handle) + client = await P2P.create() + + await asyncio.sleep(1) + result = await client.call_peer_handler(server.id, handler_name, test_input) + assert np.allclose(result, handle(test_input)) + + +@pytest.mark.asyncio +async def test_call_peer_error(handler_name="handle"): + server = await P2P.create() + await server.add_stream_handler(handler_name, handle_add) + client = await P2P.create() + + await asyncio.sleep(1) + result = await client.call_peer_handler(server.id, handler_name, + [np.zeros((2, 3)), np.zeros((3, 2))]) + assert type(result) == ValueError From 2d50b765cf2ee42571d85d10a8129c45d7d24a7f Mon Sep 17 00:00:00 2001 From: Denis Mazur Date: Thu, 25 Mar 2021 00:18:52 +0300 Subject: [PATCH 52/81] fix chmod permissions (#194) --- setup.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 6135feda7..0bc15830e 100644 --- a/setup.py +++ b/setup.py @@ -73,6 +73,7 @@ def libp2p_build_install(): result = subprocess.run(['go', 'build', '-o', os.path.join(here, "hivemind/hivemind_cli", "p2pd")], cwd=os.path.join(tempdir, f'go-libp2p-daemon-{P2PD_VERSION[1:]}', 'p2pd')) + if result.returncode: raise RuntimeError('Failed to build or install libp2p-daemon:' f' exited with status code :{result.returncode}') @@ -85,7 +86,7 @@ def libp2p_download_install(): print('Downloading Peer to Peer Daemon') url = f'https://github.com/learning-at-home/go-libp2p-daemon/releases/download/{P2PD_VERSION}/p2pd' urllib.request.urlretrieve(url, binary_path) - os.chmod(binary_path, 777) + os.chmod(binary_path, 0o777) class Install(install): From 129a370dec18a82fdce5cf41d44ec2fa71433fbd Mon Sep 17 00:00:00 2001 From: Ilya <37004806+skobellev@users.noreply.github.com> Date: Wed, 31 Mar 2021 13:09:45 +0300 Subject: [PATCH 53/81] feat P2P: add unary handler (#197) * Add unary handler * Add P2PContext to unary handler parameters Co-authored-by: Ilya Kobelev --- hivemind/p2p/p2p_daemon.py | 147 +++++++++++++++++++++++++++---------- tests/test_p2p_daemon.py | 62 +++++++++++++++- 2 files changed, 168 insertions(+), 41 deletions(-) diff --git a/hivemind/p2p/p2p_daemon.py b/hivemind/p2p/p2p_daemon.py index 1f441c5d1..a8f44550e 100644 --- a/hivemind/p2p/p2p_daemon.py +++ b/hivemind/p2p/p2p_daemon.py @@ -1,17 +1,27 @@ import asyncio -import contextlib import copy from pathlib import Path import pickle -import socket import subprocess import typing as tp import warnings +import google.protobuf from multiaddr import Multiaddr import p2pclient from libp2p.peer.id import ID +from hivemind.utils.networking import find_open_port + + +class P2PContext(object): + def __init__(self, ours_id, ours_port, handle_name): + self.peer_id = None + self.peer_addr = None + self.ours_id = ours_id + self.ours_port = ours_port + self.handle_name = handle_name + class P2P(object): """ @@ -26,11 +36,16 @@ class P2P(object): HEADER_LEN = 8 BYTEORDER = 'big' + class IncompleteRead(Exception): + pass + + class InterruptedError(Exception): + pass + def __init__(self): self._child = None self._listen_task = None self._server_stopped = asyncio.Event() - self._buffer = bytearray() @classmethod async def create(cls, *args, quic=1, tls=1, conn_manager=1, dht_client=1, @@ -89,50 +104,108 @@ def _assign_daemon_ports(self, host_port=None, daemon_listen_port=None): self._daemon_listen_port = find_open_port() @staticmethod - async def send_data(data, stream): - byte_str = pickle.dumps(data) + async def send_raw_data(byte_str, stream): request = len(byte_str).to_bytes(P2P.HEADER_LEN, P2P.BYTEORDER) + byte_str await stream.send_all(request) - class IncompleteRead(Exception): - pass + @staticmethod + async def send_data(data, stream): + await P2P.send_raw_data(pickle.dumps(data), stream) + + @staticmethod + async def send_protobuf(protobuf, out_proto_type, stream): + if type(protobuf) != out_proto_type: + error = TypeError('Unary handler returned protobuf of wrong type.') + await P2P.send_raw_data(pickle.dumps(error), stream) + raise error + await P2P.send_raw_data(protobuf.SerializeToString(), stream) - async def _receive_exactly(self, stream, n_bytes, max_bytes=1 << 16): - while len(self._buffer) < n_bytes: - data = await stream.receive_some(max_bytes) + @staticmethod + async def receive_exactly(stream, n_bytes, max_bytes=1 << 16): + buffer = bytearray() + while len(buffer) < n_bytes: + data = await stream.receive_some(min(max_bytes, n_bytes - len(buffer))) if len(data) == 0: raise P2P.IncompleteRead() - self._buffer.extend(data) - - result = self._buffer[:n_bytes] - self._buffer = self._buffer[n_bytes:] - return bytes(result) + buffer.extend(data) + return bytes(buffer) - async def receive_data(self, stream, max_bytes=(1 < 16)): - header = await self._receive_exactly(stream, P2P.HEADER_LEN) + @staticmethod + async def receive_raw_data(stream): + header = await P2P.receive_exactly(stream, P2P.HEADER_LEN) content_length = int.from_bytes(header, P2P.BYTEORDER) - data = await self._receive_exactly(stream, content_length) - return pickle.loads(data) + data = await P2P.receive_exactly(stream, content_length) + return data - def _handle_stream(self, handle): + @staticmethod + async def receive_data(stream): + return pickle.loads(await P2P.receive_raw_data(stream)) + + @staticmethod + async def receive_protobuf(in_proto_type, stream): + protobuf = in_proto_type() + protobuf.ParseFromString(await P2P.receive_raw_data(stream)) + return protobuf + + @staticmethod + def _handle_stream(handle): async def do_handle_stream(stream_info, stream): try: - request = await self.receive_data(stream) + request = await P2P.receive_data(stream) except P2P.IncompleteRead: warnings.warn("Incomplete read while receiving request from peer", RuntimeWarning) + await stream.close() return - finally: - stream.close() try: result = handle(request) - await self.send_data(result, stream) + await P2P.send_data(result, stream) except Exception as exc: - await self.send_data(exc, stream) + await P2P.send_data(exc, stream) finally: await stream.close() return do_handle_stream + @staticmethod + def _handle_unary_stream(handle, context, in_proto_type, out_proto_type): + async def watchdog(stream): + await stream.receive_some(max_bytes=1) + raise P2P.InterruptedError() + + async def do_handle_unary_stream(stream_info, stream): + try: + try: + request = await P2P.receive_protobuf(in_proto_type, stream) + except P2P.IncompleteRead: + warnings.warn("Incomplete read while receiving request from peer", + RuntimeWarning) + return + except google.protobuf.message.DecodeError as error: + warnings.warn(repr(error), RuntimeWarning) + return + + context.peer_id, context.peer_addr = stream_info.peer_id, stream_info.addr + done, pending = await asyncio.wait([watchdog(stream), handle(request, context)], + return_when=asyncio.FIRST_COMPLETED) + try: + result = done.pop().result() + await P2P.send_protobuf(result, out_proto_type, stream) + except P2P.InterruptedError: + pass + except Exception as exc: + await P2P.send_data(exc, stream) + finally: + pending_task = pending.pop() + pending_task.cancel() + try: + await pending_task + except asyncio.CancelledError: + pass + finally: + await stream.close() + + return do_handle_unary_stream + def start_listening(self): async def listen(): async with self._client.listen(): @@ -153,15 +226,21 @@ async def stop_listening(self): async def add_stream_handler(self, name, handle): if self._listen_task is None: self.start_listening() + await self._client.stream_handler(name, P2P._handle_stream(handle)) - await self._client.stream_handler(name, self._handle_stream(handle)) + async def add_unary_handler(self, name, handle, in_proto_type, out_proto_type): + if self._listen_task is None: + self.start_listening() + context = P2PContext(ours_id=self.id, ours_port=self._host_port, handle_name=name) + await self._client.stream_handler( + name, P2P._handle_unary_stream(handle, context, in_proto_type, out_proto_type)) async def call_peer_handler(self, peer_id, handler_name, input_data): libp2p_peer_id = ID.from_base58(peer_id) stream_info, stream = await self._client.stream_open(libp2p_peer_id, (handler_name,)) try: - await self.send_data(input_data, stream) - return await self.receive_data(stream) + await P2P.send_data(input_data, stream) + return await P2P.receive_data(stream) finally: await stream.close() @@ -183,15 +262,3 @@ def _make_process_args(self, *args, **kwargs) -> tp.List[str]: for key, value in kwargs.items() ) return proc_args - - -def find_open_port(params=(socket.AF_INET, socket.SOCK_STREAM), - opt=(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)): - """ Finds a tcp port that can be occupied with a socket with *params and use *opt options """ - try: - with contextlib.closing(socket.socket(*params)) as sock: - sock.bind(('', 0)) - sock.setsockopt(*opt) - return sock.getsockname()[1] - except Exception: - raise diff --git a/tests/test_p2p_daemon.py b/tests/test_p2p_daemon.py index 75fd51cdc..06814d244 100644 --- a/tests/test_p2p_daemon.py +++ b/tests/test_p2p_daemon.py @@ -2,11 +2,13 @@ import multiprocessing as mp import subprocess +from libp2p.peer.id import ID + import numpy as np import pytest -import hivemind.p2p from hivemind.p2p import P2P +from hivemind.proto import dht_pb2 RUNNING = 'running' NOT_RUNNING = 'not running' @@ -47,6 +49,64 @@ def handle_add(args): return result +@pytest.mark.parametrize( + 'should_cancel', [True, False] +) +@pytest.mark.asyncio +async def test_call_unary_handler(should_cancel, handle_name="handle"): + handler_cancelled = False + + async def ping_handler(request, context): + try: + await asyncio.sleep(2) + except asyncio.CancelledError: + nonlocal handler_cancelled + handler_cancelled = True + return dht_pb2.PingResponse( + peer=dht_pb2.NodeInfo( + node_id=context.ours_id.encode(), rpc_port=context.ours_port), + sender_endpoint=context.handle_name, available=True) + + server = await P2P.create() + server_pid = server._child.pid + await server.add_unary_handler(handle_name, ping_handler, dht_pb2.PingRequest, + dht_pb2.PingResponse) + assert is_process_running(server_pid) + + client = await P2P.create() + client_pid = client._child.pid + assert is_process_running(client_pid) + + ping_request = dht_pb2.PingRequest( + peer=dht_pb2.NodeInfo(node_id=client.id.encode(), rpc_port=client._host_port), + validate=True) + expected_response = dht_pb2.PingResponse( + peer=dht_pb2.NodeInfo(node_id=server.id.encode(), rpc_port=server._host_port), + sender_endpoint=handle_name, available=True) + + await asyncio.sleep(1) + libp2p_server_id = ID.from_base58(server.id) + stream_info, stream = await client._client.stream_open(libp2p_server_id, (handle_name,)) + + await P2P.send_raw_data(ping_request.SerializeToString(), stream) + + if should_cancel: + await stream.close() + await asyncio.sleep(1) + assert handler_cancelled + else: + result = await P2P.receive_protobuf(dht_pb2.PingResponse, stream) + assert result == expected_response + assert not handler_cancelled + + await server.stop_listening() + server.__del__() + assert not is_process_running(server_pid) + + client.__del__() + assert not is_process_running(client_pid) + + @pytest.mark.parametrize( "test_input,handle", [ From 219ff0da25bea07975bbdd796bd07f9b0a9672e1 Mon Sep 17 00:00:00 2001 From: MaximKsh Date: Fri, 2 Apr 2021 22:59:06 +0300 Subject: [PATCH 54/81] Py libp2p bindings (#193) * #183 p2p daemon pybinding * #183 rename py bindings dir, fix imports and migrate tests * #183 move pb to hivemind.proto * #183 fix p2p tests * #183 remove config.py, move constants to classes * add docstrings and minor fixes --- hivemind/p2p/p2p_daemon.py | 75 +- hivemind/p2p/p2p_daemon_bindings/__init__.py | 0 hivemind/p2p/p2p_daemon_bindings/control.py | 211 +++++ .../p2p/p2p_daemon_bindings/datastructures.py | 186 +++++ hivemind/p2p/p2p_daemon_bindings/keys.py | 91 +++ hivemind/p2p/p2p_daemon_bindings/p2pclient.py | 75 ++ hivemind/p2p/p2p_daemon_bindings/utils.py | 72 ++ hivemind/proto/crypto.proto | 20 + hivemind/proto/p2pd.proto | 158 ++++ requirements.txt | 2 + tests/test_p2p_daemon.py | 48 +- tests/test_p2p_daemon_bindings.py | 769 ++++++++++++++++++ 12 files changed, 1648 insertions(+), 59 deletions(-) create mode 100644 hivemind/p2p/p2p_daemon_bindings/__init__.py create mode 100644 hivemind/p2p/p2p_daemon_bindings/control.py create mode 100644 hivemind/p2p/p2p_daemon_bindings/datastructures.py create mode 100644 hivemind/p2p/p2p_daemon_bindings/keys.py create mode 100644 hivemind/p2p/p2p_daemon_bindings/p2pclient.py create mode 100644 hivemind/p2p/p2p_daemon_bindings/utils.py create mode 100644 hivemind/proto/crypto.proto create mode 100644 hivemind/proto/p2pd.proto create mode 100644 tests/test_p2p_daemon_bindings.py diff --git a/hivemind/p2p/p2p_daemon.py b/hivemind/p2p/p2p_daemon.py index a8f44550e..af3185a44 100644 --- a/hivemind/p2p/p2p_daemon.py +++ b/hivemind/p2p/p2p_daemon.py @@ -8,8 +8,8 @@ import google.protobuf from multiaddr import Multiaddr -import p2pclient -from libp2p.peer.id import ID +import hivemind.p2p.p2p_daemon_bindings.p2pclient as p2pclient +from hivemind.p2p.p2p_daemon_bindings.datastructures import ID, StreamInfo from hivemind.utils.networking import find_open_port @@ -104,78 +104,81 @@ def _assign_daemon_ports(self, host_port=None, daemon_listen_port=None): self._daemon_listen_port = find_open_port() @staticmethod - async def send_raw_data(byte_str, stream): + async def send_raw_data(byte_str, writer): request = len(byte_str).to_bytes(P2P.HEADER_LEN, P2P.BYTEORDER) + byte_str - await stream.send_all(request) + writer.write(request) @staticmethod - async def send_data(data, stream): - await P2P.send_raw_data(pickle.dumps(data), stream) + async def send_data(data, writer): + await P2P.send_raw_data(pickle.dumps(data), writer) @staticmethod - async def send_protobuf(protobuf, out_proto_type, stream): + async def send_protobuf(protobuf, out_proto_type, writer): if type(protobuf) != out_proto_type: error = TypeError('Unary handler returned protobuf of wrong type.') - await P2P.send_raw_data(pickle.dumps(error), stream) + await P2P.send_raw_data(pickle.dumps(error), writer) raise error - await P2P.send_raw_data(protobuf.SerializeToString(), stream) + await P2P.send_raw_data(protobuf.SerializeToString(), writer) @staticmethod - async def receive_exactly(stream, n_bytes, max_bytes=1 << 16): + async def receive_exactly(reader, n_bytes, max_bytes=1 << 16): buffer = bytearray() while len(buffer) < n_bytes: - data = await stream.receive_some(min(max_bytes, n_bytes - len(buffer))) + data = await reader.read(min(max_bytes, n_bytes - len(buffer))) if len(data) == 0: raise P2P.IncompleteRead() buffer.extend(data) return bytes(buffer) @staticmethod - async def receive_raw_data(stream): - header = await P2P.receive_exactly(stream, P2P.HEADER_LEN) + async def receive_raw_data(reader): + header = await P2P.receive_exactly(reader, P2P.HEADER_LEN) content_length = int.from_bytes(header, P2P.BYTEORDER) - data = await P2P.receive_exactly(stream, content_length) + data = await P2P.receive_exactly(reader, content_length) return data @staticmethod - async def receive_data(stream): - return pickle.loads(await P2P.receive_raw_data(stream)) + async def receive_data(reader): + return pickle.loads(await P2P.receive_raw_data(reader)) @staticmethod - async def receive_protobuf(in_proto_type, stream): + async def receive_protobuf(in_proto_type, reader): protobuf = in_proto_type() - protobuf.ParseFromString(await P2P.receive_raw_data(stream)) + protobuf.ParseFromString(await P2P.receive_raw_data(reader)) return protobuf @staticmethod def _handle_stream(handle): - async def do_handle_stream(stream_info, stream): + async def do_handle_stream(stream_info, reader, writer): try: - request = await P2P.receive_data(stream) + request = await P2P.receive_data(reader) except P2P.IncompleteRead: warnings.warn("Incomplete read while receiving request from peer", RuntimeWarning) - await stream.close() + writer.close() return try: result = handle(request) - await P2P.send_data(result, stream) + await P2P.send_data(result, writer) except Exception as exc: - await P2P.send_data(exc, stream) + await P2P.send_data(exc, writer) finally: - await stream.close() + writer.close() return do_handle_stream @staticmethod def _handle_unary_stream(handle, context, in_proto_type, out_proto_type): - async def watchdog(stream): - await stream.receive_some(max_bytes=1) + async def watchdog(reader: asyncio.StreamReader): + await reader.read(n=1) raise P2P.InterruptedError() - async def do_handle_unary_stream(stream_info, stream): + async def do_handle_unary_stream( + stream_info: StreamInfo, + reader: asyncio.StreamReader, + writer: asyncio.StreamWriter) -> None: try: try: - request = await P2P.receive_protobuf(in_proto_type, stream) + request = await P2P.receive_protobuf(in_proto_type, reader) except P2P.IncompleteRead: warnings.warn("Incomplete read while receiving request from peer", RuntimeWarning) @@ -185,15 +188,15 @@ async def do_handle_unary_stream(stream_info, stream): return context.peer_id, context.peer_addr = stream_info.peer_id, stream_info.addr - done, pending = await asyncio.wait([watchdog(stream), handle(request, context)], + done, pending = await asyncio.wait([watchdog(reader), handle(request, context)], return_when=asyncio.FIRST_COMPLETED) try: result = done.pop().result() - await P2P.send_protobuf(result, out_proto_type, stream) + await P2P.send_protobuf(result, out_proto_type, writer) except P2P.InterruptedError: pass except Exception as exc: - await P2P.send_data(exc, stream) + await P2P.send_data(exc, writer) finally: pending_task = pending.pop() pending_task.cancel() @@ -202,7 +205,7 @@ async def do_handle_unary_stream(stream_info, stream): except asyncio.CancelledError: pass finally: - await stream.close() + writer.close() return do_handle_unary_stream @@ -237,12 +240,12 @@ async def add_unary_handler(self, name, handle, in_proto_type, out_proto_type): async def call_peer_handler(self, peer_id, handler_name, input_data): libp2p_peer_id = ID.from_base58(peer_id) - stream_info, stream = await self._client.stream_open(libp2p_peer_id, (handler_name,)) + stream_info, reader, writer = await self._client.stream_open(libp2p_peer_id, (handler_name,)) try: - await P2P.send_data(input_data, stream) - return await P2P.receive_data(stream) + await P2P.send_data(input_data, writer) + return await P2P.receive_data(reader) finally: - await stream.close() + writer.close() def __del__(self): self._kill_child() diff --git a/hivemind/p2p/p2p_daemon_bindings/__init__.py b/hivemind/p2p/p2p_daemon_bindings/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/hivemind/p2p/p2p_daemon_bindings/control.py b/hivemind/p2p/p2p_daemon_bindings/control.py new file mode 100644 index 000000000..df8aeaefa --- /dev/null +++ b/hivemind/p2p/p2p_daemon_bindings/control.py @@ -0,0 +1,211 @@ +import logging +from typing import AsyncIterator, Awaitable, Callable, Dict, Iterable, Sequence, Tuple + +import asyncio +from contextlib import asynccontextmanager +from multiaddr import Multiaddr, protocols +from hivemind.p2p.p2p_daemon_bindings.datastructures import PeerInfo, StreamInfo, ID +from hivemind.proto import p2pd_pb2 as p2pd_pb +from hivemind.p2p.p2p_daemon_bindings.utils import DispatchFailure, read_pbmsg_safe, write_pbmsg, raise_if_failed + +StreamHandler = Callable[[StreamInfo, asyncio.StreamReader, asyncio.StreamWriter], Awaitable[None]] + +_supported_conn_protocols = ( + protocols.P_IP4, + # protocols.P_IP6, + protocols.P_UNIX, +) + + +def parse_conn_protocol(maddr: Multiaddr) -> int: + proto_codes = set(proto.code for proto in maddr.protocols()) + proto_cand = proto_codes.intersection(_supported_conn_protocols) + if len(proto_cand) != 1: + supported_protos = ( + protocols.protocol_with_code(proto) for proto in _supported_conn_protocols + ) + raise ValueError( + f"connection protocol should be only one protocol out of {supported_protos}" + f", maddr={maddr}" + ) + return tuple(proto_cand)[0] + + +class DaemonConnector: + control_maddr: Multiaddr + logger = logging.getLogger("p2pclient.DaemonConnector") + DEFAULT_CONTROL_MADDR = "/unix/tmp/p2pd.sock" + + def __init__(self, control_maddr: Multiaddr = None) -> None: + if control_maddr is None: + control_maddr = Multiaddr(self.DEFAULT_CONTROL_MADDR) + self.control_maddr = control_maddr + + async def open_connection(self) -> (asyncio.StreamReader, asyncio.StreamWriter): + proto_code = parse_conn_protocol(self.control_maddr) + if proto_code == protocols.P_UNIX: + control_path = self.control_maddr.value_for_protocol(protocols.P_UNIX) + self.logger.debug( + "DaemonConnector %s opens connection to %s", self, self.control_maddr + ) + return await asyncio.open_unix_connection(control_path) + elif proto_code == protocols.P_IP4: + host = self.control_maddr.value_for_protocol(protocols.P_IP4) + port = int(self.control_maddr.value_for_protocol(protocols.P_TCP)) + return await asyncio.open_connection(host, port) + else: + raise ValueError( + f"protocol not supported: protocol={protocols.protocol_with_code(proto_code)}" + ) + + +class ControlClient: + listen_maddr: Multiaddr + daemon_connector: DaemonConnector + handlers: Dict[str, StreamHandler] + logger = logging.getLogger("p2pclient.ControlClient") + DEFAULT_LISTEN_MADDR = "/unix/tmp/p2pclient.sock" + + def __init__( + self, daemon_connector: DaemonConnector, listen_maddr: Multiaddr = None + ) -> None: + if listen_maddr is None: + listen_maddr = Multiaddr(self.DEFAULT_LISTEN_MADDR) + self.listen_maddr = listen_maddr + self.daemon_connector = daemon_connector + self.handlers = {} + + async def _handler(self, reader: asyncio.StreamReader, writer: asyncio.StreamWriter): + pb_stream_info = p2pd_pb.StreamInfo() # type: ignore + await read_pbmsg_safe(reader, pb_stream_info) + stream_info = StreamInfo.from_pb(pb_stream_info) + self.logger.info("New incoming stream: %s", stream_info) + try: + handler = self.handlers[stream_info.proto] + except KeyError as e: + # should never enter here... daemon should reject the stream for us. + writer.close() + raise DispatchFailure(e) + await handler(stream_info, reader, writer) + + @asynccontextmanager + async def listen(self) -> AsyncIterator["ControlClient"]: + proto_code = parse_conn_protocol(self.listen_maddr) + if proto_code == protocols.P_UNIX: + listen_path = self.listen_maddr.value_for_protocol(protocols.P_UNIX) + server = await asyncio.start_unix_server(self._handler, path=listen_path) + elif proto_code == protocols.P_IP4: + host = self.listen_maddr.value_for_protocol(protocols.P_IP4) + port = int(self.listen_maddr.value_for_protocol(protocols.P_TCP)) + server = await asyncio.start_server(self._handler, port=port, host=host) + else: + raise ValueError( + f"protocol not supported: protocol={protocols.protocol_with_code(proto_code)}" + ) + + async with server: + self.logger.info( + "DaemonConnector %s starts listening to %s", self, self.listen_maddr + ) + yield self + + self.logger.info("DaemonConnector %s closed", self) + + async def identify(self) -> Tuple[ID, Tuple[Multiaddr, ...]]: + reader, writer = await self.daemon_connector.open_connection() + req = p2pd_pb.Request(type=p2pd_pb.Request.IDENTIFY) + await write_pbmsg(writer, req) + + resp = p2pd_pb.Response() # type: ignore + await read_pbmsg_safe(reader, resp) + writer.close() + + raise_if_failed(resp) + peer_id_bytes = resp.identify.id + maddrs_bytes = resp.identify.addrs + + maddrs = tuple(Multiaddr(maddr_bytes) for maddr_bytes in maddrs_bytes) + peer_id = ID(peer_id_bytes) + + return peer_id, maddrs + + async def connect(self, peer_id: ID, maddrs: Iterable[Multiaddr]) -> None: + reader, writer = await self.daemon_connector.open_connection() + + maddrs_bytes = [i.to_bytes() for i in maddrs] + connect_req = p2pd_pb.ConnectRequest( + peer=peer_id.to_bytes(), addrs=maddrs_bytes + ) + req = p2pd_pb.Request(type=p2pd_pb.Request.CONNECT, connect=connect_req) + await write_pbmsg(writer, req) + + resp = p2pd_pb.Response() # type: ignore + await read_pbmsg_safe(reader, resp) + writer.close() + raise_if_failed(resp) + + async def list_peers(self) -> Tuple[PeerInfo, ...]: + req = p2pd_pb.Request(type=p2pd_pb.Request.LIST_PEERS) + reader, writer = await self.daemon_connector.open_connection() + await write_pbmsg(writer, req) + resp = p2pd_pb.Response() # type: ignore + await read_pbmsg_safe(reader, resp) + writer.close() + raise_if_failed(resp) + + peers = tuple(PeerInfo.from_pb(pinfo) for pinfo in resp.peers) + return peers + + async def disconnect(self, peer_id: ID) -> None: + disconnect_req = p2pd_pb.DisconnectRequest(peer=peer_id.to_bytes()) + req = p2pd_pb.Request( + type=p2pd_pb.Request.DISCONNECT, disconnect=disconnect_req + ) + reader, writer = await self.daemon_connector.open_connection() + await write_pbmsg(writer, req) + resp = p2pd_pb.Response() # type: ignore + await read_pbmsg_safe(reader, resp) + writer.close() + raise_if_failed(resp) + + async def stream_open( + self, peer_id: ID, protocols: Sequence[str] + ) -> Tuple[StreamInfo, asyncio.StreamReader, asyncio.StreamWriter]: + reader, writer = await self.daemon_connector.open_connection() + + stream_open_req = p2pd_pb.StreamOpenRequest( + peer=peer_id.to_bytes(), proto=list(protocols) + ) + req = p2pd_pb.Request( + type=p2pd_pb.Request.STREAM_OPEN, streamOpen=stream_open_req + ) + await write_pbmsg(writer, req) + + resp = p2pd_pb.Response() # type: ignore + await read_pbmsg_safe(reader, resp) + raise_if_failed(resp) + + pb_stream_info = resp.streamInfo + stream_info = StreamInfo.from_pb(pb_stream_info) + + return stream_info, reader, writer + + async def stream_handler(self, proto: str, handler_cb: StreamHandler) -> None: + reader, writer = await self.daemon_connector.open_connection() + + listen_path_maddr_bytes = self.listen_maddr.to_bytes() + stream_handler_req = p2pd_pb.StreamHandlerRequest( + addr=listen_path_maddr_bytes, proto=[proto] + ) + req = p2pd_pb.Request( + type=p2pd_pb.Request.STREAM_HANDLER, streamHandler=stream_handler_req + ) + await write_pbmsg(writer, req) + + resp = p2pd_pb.Response() # type: ignore + await read_pbmsg_safe(reader, resp) + writer.close() + raise_if_failed(resp) + + # if success, add the handler to the dict + self.handlers[proto] = handler_cb diff --git a/hivemind/p2p/p2p_daemon_bindings/datastructures.py b/hivemind/p2p/p2p_daemon_bindings/datastructures.py new file mode 100644 index 000000000..42351627c --- /dev/null +++ b/hivemind/p2p/p2p_daemon_bindings/datastructures.py @@ -0,0 +1,186 @@ +import hashlib +from typing import Union, List, Sequence, Any + +import base58 +import multihash + +from multiaddr import Multiaddr, protocols +from hivemind.proto import p2pd_pb2 + +from hivemind.p2p.p2p_daemon_bindings.keys import PublicKey + +# NOTE: On inlining... +# See: https://github.com/libp2p/specs/issues/138 +# NOTE: enabling to be interoperable w/ the Go implementation +ENABLE_INLINING = True +MAX_INLINE_KEY_LENGTH = 42 + +IDENTITY_MULTIHASH_CODE = 0x00 + +if ENABLE_INLINING: + + class IdentityHash: + _digest: bytes + + def __init__(self) -> None: + self._digest = bytearray() + + def update(self, input: bytes) -> None: + self._digest += input + + def digest(self) -> bytes: + return self._digest + + multihash.FuncReg.register( + IDENTITY_MULTIHASH_CODE, "identity", hash_new=lambda: IdentityHash() + ) + + +class ID: + _bytes: bytes + _xor_id: int = None + _b58_str: str = None + + def __init__(self, peer_id_bytes: bytes) -> None: + self._bytes = peer_id_bytes + + @property + def xor_id(self) -> int: + if not self._xor_id: + self._xor_id = int(sha256_digest(self._bytes).hex(), 16) + return self._xor_id + + def to_bytes(self) -> bytes: + return self._bytes + + def to_base58(self) -> str: + if not self._b58_str: + self._b58_str = base58.b58encode(self._bytes).decode() + return self._b58_str + + def __repr__(self) -> str: + return f"" + + __str__ = pretty = to_string = to_base58 + + def __eq__(self, other: object) -> bool: + if isinstance(other, str): + return self.to_base58() == other + elif isinstance(other, bytes): + return self._bytes == other + elif isinstance(other, ID): + return self._bytes == other._bytes + else: + return NotImplemented + + def __hash__(self) -> int: + return hash(self._bytes) + + @classmethod + def from_base58(cls, b58_encoded_peer_id_str: str) -> "ID": + peer_id_bytes = base58.b58decode(b58_encoded_peer_id_str) + pid = ID(peer_id_bytes) + return pid + + @classmethod + def from_pubkey(cls, key: PublicKey) -> "ID": + serialized_key = key.serialize() + algo = multihash.Func.sha2_256 + if ENABLE_INLINING and len(serialized_key) <= MAX_INLINE_KEY_LENGTH: + algo = IDENTITY_MULTIHASH_CODE + mh_digest = multihash.digest(serialized_key, algo) + return cls(mh_digest.encode()) + + +def sha256_digest(data: Union[str, bytes]) -> bytes: + if isinstance(data, str): + data = data.encode("utf8") + return hashlib.sha256(data).digest() + + +class StreamInfo: + peer_id: ID + addr: Multiaddr + proto: str + + def __init__(self, peer_id: ID, addr: Multiaddr, proto: str) -> None: + self.peer_id = peer_id + self.addr = addr + self.proto = proto + + def __repr__(self) -> str: + return ( + f"" + ) + + def to_pb(self) -> p2pd_pb2.StreamInfo: + pb_msg = p2pd_pb2.StreamInfo( + peer=self.peer_id.to_bytes(), addr=self.addr.to_bytes(), proto=self.proto + ) + return pb_msg + + @classmethod + def from_pb(cls, pb_msg: p2pd_pb2.StreamInfo) -> "StreamInfo": + stream_info = cls( + peer_id=ID(pb_msg.peer), addr=Multiaddr(pb_msg.addr), proto=pb_msg.proto + ) + return stream_info + + +class PeerInfoLibP2P: + peer_id: ID + addrs: List[Multiaddr] + + def __init__(self, peer_id: ID, addrs: Sequence[Multiaddr]) -> None: + self.peer_id = peer_id + self.addrs = list(addrs) + + def __eq__(self, other: Any) -> bool: + return ( + isinstance(other, PeerInfo) + and self.peer_id == other.peer_id + and self.addrs == other.addrs + ) + + +def info_from_p2p_addr(addr: Multiaddr) -> PeerInfoLibP2P: + if not addr: + raise InvalidAddrError("`addr` should not be `None`") + + parts = addr.split() + if not parts: + raise InvalidAddrError( + f"`parts`={parts} should at least have a protocol `P_P2P`" + ) + + p2p_part = parts[-1] + last_protocol_code = p2p_part.protocols()[0].code + if last_protocol_code != protocols.P_P2P: + raise InvalidAddrError( + f"The last protocol should be `P_P2P` instead of `{last_protocol_code}`" + ) + + # make sure the /p2p value parses as a peer.ID + peer_id_str: str = p2p_part.value_for_protocol(protocols.P_P2P) + peer_id: ID = ID.from_base58(peer_id_str) + + # we might have received just an / p2p part, which means there's no addr. + if len(parts) > 1: + addr = Multiaddr.join(*parts[:-1]) + + return PeerInfo(peer_id, [addr]) + + +class InvalidAddrError(ValueError): + pass + + +class PeerInfo(PeerInfoLibP2P): + @classmethod + def from_pb(cls, peer_info_pb: p2pd_pb2.PeerInfo) -> PeerInfoLibP2P: + peer_id = ID(peer_info_pb.id) + addrs = [Multiaddr(addr) for addr in peer_info_pb.addrs] + return PeerInfo(peer_id, addrs) + + def __str__(self): + return self.peer_id.pretty() + " " + ",".join(str(a) for a in self.addrs) diff --git a/hivemind/p2p/p2p_daemon_bindings/keys.py b/hivemind/p2p/p2p_daemon_bindings/keys.py new file mode 100644 index 000000000..01ec5ad55 --- /dev/null +++ b/hivemind/p2p/p2p_daemon_bindings/keys.py @@ -0,0 +1,91 @@ +from abc import ABC, abstractmethod +from dataclasses import dataclass +from enum import Enum, unique + +from hivemind.proto import crypto_pb2 as protobuf + + +@unique +class KeyType(Enum): + RSA = 0 + Ed25519 = 1 + Secp256k1 = 2 + ECDSA = 3 + ECC_P256 = 4 + + +class Key(ABC): + """A ``Key`` represents a cryptographic key.""" + + @abstractmethod + def to_bytes(self) -> bytes: + """Returns the byte representation of this key.""" + ... + + @abstractmethod + def get_type(self) -> KeyType: + """Returns the ``KeyType`` for ``self``.""" + ... + + def __eq__(self, other: object) -> bool: + if not isinstance(other, Key): + return NotImplemented + return self.to_bytes() == other.to_bytes() + + +class PublicKey(Key): + """A ``PublicKey`` represents a cryptographic public key.""" + + @abstractmethod + def verify(self, data: bytes, signature: bytes) -> bool: + """Verify that ``signature`` is the cryptographic signature of the hash + of ``data``.""" + ... + + def _serialize_to_protobuf(self) -> protobuf.PublicKey: + """Return the protobuf representation of this ``Key``.""" + key_type = self.get_type().value + data = self.to_bytes() + protobuf_key = protobuf.PublicKey(key_type=key_type, data=data) + return protobuf_key + + def serialize(self) -> bytes: + """Return the canonical serialization of this ``Key``.""" + return self._serialize_to_protobuf().SerializeToString() + + @classmethod + def deserialize_from_protobuf(cls, protobuf_data: bytes) -> protobuf.PublicKey: + return protobuf.PublicKey.FromString(protobuf_data) + + +class PrivateKey(Key): + """A ``PrivateKey`` represents a cryptographic private key.""" + + @abstractmethod + def sign(self, data: bytes) -> bytes: + ... + + @abstractmethod + def get_public_key(self) -> PublicKey: + ... + + def _serialize_to_protobuf(self) -> protobuf.PrivateKey: + """Return the protobuf representation of this ``Key``.""" + key_type = self.get_type().value + data = self.to_bytes() + protobuf_key = protobuf.PrivateKey(key_type=key_type, data=data) + return protobuf_key + + def serialize(self) -> bytes: + """Return the canonical serialization of this ``Key``.""" + return self._serialize_to_protobuf().SerializeToString() + + @classmethod + def deserialize_from_protobuf(cls, protobuf_data: bytes) -> protobuf.PrivateKey: + return protobuf.PrivateKey.FromString(protobuf_data) + + +@dataclass(frozen=True) +class KeyPair: + private_key: PrivateKey + public_key: PublicKey diff --git a/hivemind/p2p/p2p_daemon_bindings/p2pclient.py b/hivemind/p2p/p2p_daemon_bindings/p2pclient.py new file mode 100644 index 000000000..1dbcce960 --- /dev/null +++ b/hivemind/p2p/p2p_daemon_bindings/p2pclient.py @@ -0,0 +1,75 @@ +from typing import AsyncIterator, Iterable, Sequence, Tuple + +import asyncio +from hivemind.p2p.p2p_daemon_bindings.control import ControlClient, DaemonConnector, StreamHandler +from contextlib import asynccontextmanager +from multiaddr import Multiaddr +from hivemind.p2p.p2p_daemon_bindings.datastructures import PeerInfo, StreamInfo, ID + + +class Client: + control: ControlClient + + def __init__( + self, control_maddr: Multiaddr = None, listen_maddr: Multiaddr = None + ) -> None: + daemon_connector = DaemonConnector(control_maddr=control_maddr) + self.control = ControlClient( + daemon_connector=daemon_connector, listen_maddr=listen_maddr + ) + + @asynccontextmanager + async def listen(self) -> AsyncIterator["Client"]: + """ + Starts to listen incoming connections for handlers registered via stream_handler. + :return: + """ + async with self.control.listen(): + yield self + + async def identify(self) -> Tuple[ID, Tuple[Multiaddr, ...]]: + """ + Get current node peer id and list of addresses + """ + return await self.control.identify() + + async def connect(self, peer_id: ID, maddrs: Iterable[Multiaddr]) -> None: + """ + Connect to p2p node with specified addresses and peer id. + :peer_id: node peer id you want connect to + :maddrs: node multiaddresses you want connect to. Of course, it must be reachable. + """ + await self.control.connect(peer_id=peer_id, maddrs=maddrs) + + async def list_peers(self) -> Tuple[PeerInfo, ...]: + """ + Get list of peers that node connect to + """ + return await self.control.list_peers() + + async def disconnect(self, peer_id: ID) -> None: + """ + Disconnect from node with specified peer id + :peer_id: + """ + await self.control.disconnect(peer_id=peer_id) + + async def stream_open( + self, peer_id: ID, protocols: Sequence[str] + ) -> Tuple[StreamInfo, asyncio.StreamReader, asyncio.StreamWriter]: + """ + Open a stream to call other peer (with peer_id) handler for specified protocols + :peer_id: + :protocols: + :return: Returns tuple of stream info (info about connection to second peer) and reader/writer + """ + return await self.control.stream_open(peer_id=peer_id, protocols=protocols) + + async def stream_handler(self, proto: str, handler_cb: StreamHandler) -> None: + """ + Register a stream handler + :param proto: protocols that handler serves + :param handler_cb: handler callback + :return: + """ + await self.control.stream_handler(proto=proto, handler_cb=handler_cb) diff --git a/hivemind/p2p/p2p_daemon_bindings/utils.py b/hivemind/p2p/p2p_daemon_bindings/utils.py new file mode 100644 index 000000000..fa0e7cfd3 --- /dev/null +++ b/hivemind/p2p/p2p_daemon_bindings/utils.py @@ -0,0 +1,72 @@ +import asyncio + +from google.protobuf.message import Message as PBMessage + +from hivemind.proto import p2pd_pb2 as p2pd_pb + + +DEFAULT_MAX_BITS: int = 64 + + +class ControlFailure(Exception): + pass + + +class DispatchFailure(Exception): + pass + + +async def write_unsigned_varint( + stream: asyncio.StreamWriter, integer: int, max_bits: int = DEFAULT_MAX_BITS +) -> None: + max_int: int = 1 << max_bits + if integer < 0: + raise ValueError(f"negative integer: {integer}") + if integer >= max_int: + raise ValueError(f"integer too large: {integer}") + while True: + value: int = integer & 0x7F + integer >>= 7 + if integer != 0: + value |= 0x80 + byte = value.to_bytes(1, "big") + stream.write(byte) + if integer == 0: + break + + +async def read_unsigned_varint( + stream: asyncio.StreamReader, max_bits: int = DEFAULT_MAX_BITS +) -> int: + max_int: int = 1 << max_bits + iteration: int = 0 + result: int = 0 + has_next: bool = True + while has_next: + data = await stream.readexactly(1) + c = data[0] + value = c & 0x7F + result |= value << (iteration * 7) + has_next = (c & 0x80) != 0 + iteration += 1 + if result >= max_int: + raise ValueError(f"varint overflowed: {result}") + return result + + +def raise_if_failed(response: p2pd_pb.Response) -> None: + if response.type == p2pd_pb.Response.ERROR: + raise ControlFailure(f"connect failed. msg={response.error.msg}") + + +async def write_pbmsg(stream: asyncio.StreamWriter, pbmsg: PBMessage) -> None: + size = pbmsg.ByteSize() + await write_unsigned_varint(stream, size) + msg_bytes: bytes = pbmsg.SerializeToString() + stream.write(msg_bytes) + + +async def read_pbmsg_safe(stream: asyncio.StreamReader, pbmsg: PBMessage) -> None: + len_msg_bytes = await read_unsigned_varint(stream) + msg_bytes = await stream.readexactly(len_msg_bytes) + pbmsg.ParseFromString(msg_bytes) diff --git a/hivemind/proto/crypto.proto b/hivemind/proto/crypto.proto new file mode 100644 index 000000000..fe729a9d4 --- /dev/null +++ b/hivemind/proto/crypto.proto @@ -0,0 +1,20 @@ +syntax = "proto2"; + +package crypto.pb; + +enum KeyType { + RSA = 0; + Ed25519 = 1; + Secp256k1 = 2; + ECDSA = 3; +} + +message PublicKey { + required KeyType key_type = 1; + required bytes data = 2; +} + +message PrivateKey { + required KeyType key_type = 1; + required bytes data = 2; +} diff --git a/hivemind/proto/p2pd.proto b/hivemind/proto/p2pd.proto new file mode 100644 index 000000000..fec4a6ed7 --- /dev/null +++ b/hivemind/proto/p2pd.proto @@ -0,0 +1,158 @@ +syntax = "proto2"; + +package p2pclient.p2pd.pb; + +message Request { + enum Type { + IDENTIFY = 0; + CONNECT = 1; + STREAM_OPEN = 2; + STREAM_HANDLER = 3; + DHT = 4; + LIST_PEERS = 5; + CONNMANAGER = 6; + DISCONNECT = 7; + PUBSUB = 8; + } + + required Type type = 1; + + optional ConnectRequest connect = 2; + optional StreamOpenRequest streamOpen = 3; + optional StreamHandlerRequest streamHandler = 4; + optional DHTRequest dht = 5; + optional ConnManagerRequest connManager = 6; + optional DisconnectRequest disconnect = 7; + optional PSRequest pubsub = 8; +} + +message Response { + enum Type { + OK = 0; + ERROR = 1; + } + + required Type type = 1; + optional ErrorResponse error = 2; + optional StreamInfo streamInfo = 3; + optional IdentifyResponse identify = 4; + optional DHTResponse dht = 5; + repeated PeerInfo peers = 6; + optional PSResponse pubsub = 7; +} + +message IdentifyResponse { + required bytes id = 1; + repeated bytes addrs = 2; +} + +message ConnectRequest { + required bytes peer = 1; + repeated bytes addrs = 2; + optional int64 timeout = 3; +} + +message StreamOpenRequest { + required bytes peer = 1; + repeated string proto = 2; + optional int64 timeout = 3; +} + +message StreamHandlerRequest { + required bytes addr = 1; + repeated string proto = 2; +} + +message ErrorResponse { + required string msg = 1; +} + +message StreamInfo { + required bytes peer = 1; + required bytes addr = 2; + required string proto = 3; +} + +message DHTRequest { + enum Type { + FIND_PEER = 0; + FIND_PEERS_CONNECTED_TO_PEER = 1; + FIND_PROVIDERS = 2; + GET_CLOSEST_PEERS = 3; + GET_PUBLIC_KEY = 4; + GET_VALUE = 5; + SEARCH_VALUE = 6; + PUT_VALUE = 7; + PROVIDE = 8; + } + + required Type type = 1; + optional bytes peer = 2; + optional bytes cid = 3; + optional bytes key = 4; + optional bytes value = 5; + optional int32 count = 6; + optional int64 timeout = 7; +} + +message DHTResponse { + enum Type { + BEGIN = 0; + VALUE = 1; + END = 2; + } + + required Type type = 1; + optional PeerInfo peer = 2; + optional bytes value = 3; +} + +message PeerInfo { + required bytes id = 1; + repeated bytes addrs = 2; +} + +message ConnManagerRequest { + enum Type { + TAG_PEER = 0; + UNTAG_PEER = 1; + TRIM = 2; + } + + required Type type = 1; + + optional bytes peer = 2; + optional string tag = 3; + optional int64 weight = 4; +} + +message DisconnectRequest { + required bytes peer = 1; +} + +message PSRequest { + enum Type { + GET_TOPICS = 0; + LIST_PEERS = 1; + PUBLISH = 2; + SUBSCRIBE = 3; + } + + required Type type = 1; + optional string topic = 2; + optional bytes data = 3; +} + +message PSMessage { + optional bytes from_id = 1; + optional bytes data = 2; + optional bytes seqno = 3; + repeated string topicIDs = 4; + optional bytes signature = 5; + optional bytes key = 6; +} + +message PSResponse { + repeated string topics = 1; + repeated bytes peerIDs = 2; +} diff --git a/requirements.txt b/requirements.txt index 7e2e84d93..b0011a653 100644 --- a/requirements.txt +++ b/requirements.txt @@ -10,5 +10,7 @@ grpcio>=1.33.2 grpcio-tools>=1.33.2 protobuf>=3.12.2 configargparse>=1.2.3 +multiaddr==0.0.9 +pymultihash==0.8.2 cryptography>=3.4.6 pydantic>=1.8.1 diff --git a/tests/test_p2p_daemon.py b/tests/test_p2p_daemon.py index 06814d244..759b2eb2b 100644 --- a/tests/test_p2p_daemon.py +++ b/tests/test_p2p_daemon.py @@ -2,7 +2,7 @@ import multiprocessing as mp import subprocess -from libp2p.peer.id import ID +from hivemind.p2p.p2p_daemon_bindings.datastructures import ID import numpy as np import pytest @@ -86,16 +86,16 @@ async def ping_handler(request, context): await asyncio.sleep(1) libp2p_server_id = ID.from_base58(server.id) - stream_info, stream = await client._client.stream_open(libp2p_server_id, (handle_name,)) + stream_info, reader, writer = await client._client.stream_open(libp2p_server_id, (handle_name,)) - await P2P.send_raw_data(ping_request.SerializeToString(), stream) + await P2P.send_raw_data(ping_request.SerializeToString(), writer) if should_cancel: - await stream.close() + writer.close() await asyncio.sleep(1) assert handler_cancelled else: - result = await P2P.receive_protobuf(dht_pb2.PingResponse, stream) + result = await P2P.receive_protobuf(dht_pb2.PingResponse, reader) assert result == expected_response assert not handler_cancelled @@ -139,6 +139,25 @@ async def test_call_peer_single_process(test_input, handle, handler_name="handle assert not is_process_running(client_pid) +async def run_server(handler_name, server_side, client_side, response_received): + server = await P2P.create() + server_pid = server._child.pid + await server.add_stream_handler(handler_name, handle_square) + assert is_process_running(server_pid) + + server_side.send(server.id) + while response_received.value == 0: + await asyncio.sleep(0.5) + + await server.stop_listening() + server.__del__() + assert not is_process_running(server_pid) + + +def server_target(handler_name, server_side, client_side, response_received): + asyncio.run(run_server(handler_name, server_side, client_side, response_received)) + + @pytest.mark.asyncio async def test_call_peer_different_processes(): handler_name = "square" @@ -148,24 +167,7 @@ async def test_call_peer_different_processes(): response_received = mp.Value(np.ctypeslib.as_ctypes_type(np.int32)) response_received.value = 0 - async def run_server(): - server = await P2P.create() - server_pid = server._child.pid - await server.add_stream_handler(handler_name, handle_square) - assert is_process_running(server_pid) - - server_side.send(server.id) - while response_received.value == 0: - await asyncio.sleep(0.5) - - await server.stop_listening() - server.__del__() - assert not is_process_running(server_pid) - - def server_target(): - asyncio.run(run_server()) - - proc = mp.Process(target=server_target) + proc = mp.Process(target=server_target, args=(handler_name, server_side, client_side, response_received)) proc.start() client = await P2P.create() diff --git a/tests/test_p2p_daemon_bindings.py b/tests/test_p2p_daemon_bindings.py new file mode 100644 index 000000000..e9bc77213 --- /dev/null +++ b/tests/test_p2p_daemon_bindings.py @@ -0,0 +1,769 @@ +import asyncio +import functools +import io +import os +import subprocess +import time +import uuid +from contextlib import asynccontextmanager, AsyncExitStack +from typing import NamedTuple + +from google.protobuf.message import EncodeError +from multiaddr import Multiaddr, protocols + +import pytest + +from hivemind import find_open_port +from hivemind.p2p.p2p_daemon_bindings.control import parse_conn_protocol, DaemonConnector, ControlClient +from hivemind.p2p.p2p_daemon_bindings.p2pclient import Client +from hivemind.p2p.p2p_daemon_bindings.utils import ControlFailure, raise_if_failed, write_unsigned_varint, \ + read_unsigned_varint, read_pbmsg_safe, write_pbmsg +from hivemind.proto import p2pd_pb2 as p2pd_pb +from hivemind.p2p.p2p_daemon_bindings.datastructures import ID, StreamInfo, PeerInfo + + +def test_raise_if_failed_raises(): + resp = p2pd_pb.Response() + resp.type = p2pd_pb.Response.ERROR + with pytest.raises(ControlFailure): + raise_if_failed(resp) + + +def test_raise_if_failed_not_raises(): + resp = p2pd_pb.Response() + resp.type = p2pd_pb.Response.OK + raise_if_failed(resp) + + +pairs_int_varint_valid = ( + (0, b"\x00"), + (1, b"\x01"), + (128, b"\x80\x01"), + (2 ** 32, b"\x80\x80\x80\x80\x10"), + (2 ** 64 - 1, b"\xff\xff\xff\xff\xff\xff\xff\xff\xff\x01"), +) + +pairs_int_varint_overflow = ( + (2 ** 64, b"\x80\x80\x80\x80\x80\x80\x80\x80\x80\x02"), + (2 ** 64 + 1, b"\x81\x80\x80\x80\x80\x80\x80\x80\x80\x02"), + ( + 2 ** 128, + b"\x80\x80\x80\x80\x80\x80\x80\x80\x80\x80\x80\x80\x80\x80\x80\x80\x80\x80\x04", + ), +) + + +class MockReader(io.BytesIO): + async def readexactly(self, n): + await asyncio.sleep(0) + return self.read(n) + + +class MockWriter(io.BytesIO): + pass + + +class MockReaderWriter(MockReader, MockWriter): + pass + + +@pytest.mark.parametrize("integer, var_integer", pairs_int_varint_valid) +@pytest.mark.asyncio +async def test_write_unsigned_varint(integer, var_integer): + s = MockWriter() + await write_unsigned_varint(s, integer) + assert s.getvalue() == var_integer + + +@pytest.mark.parametrize("integer", tuple(i[0] for i in pairs_int_varint_overflow)) +@pytest.mark.asyncio +async def test_write_unsigned_varint_overflow(integer): + s = MockWriter() + with pytest.raises(ValueError): + await write_unsigned_varint(s, integer) + + +@pytest.mark.parametrize("integer", (-1, -(2 ** 32), -(2 ** 64), -(2 ** 128))) +@pytest.mark.asyncio +async def test_write_unsigned_varint_negative(integer): + s = MockWriter() + with pytest.raises(ValueError): + await write_unsigned_varint(s, integer) + + +@pytest.mark.parametrize("integer, var_integer", pairs_int_varint_valid) +@pytest.mark.asyncio +async def test_read_unsigned_varint(integer, var_integer): + s = MockReader(var_integer) + result = await read_unsigned_varint(s) + assert result == integer + + +@pytest.mark.parametrize("var_integer", tuple(i[1] for i in pairs_int_varint_overflow)) +@pytest.mark.asyncio +async def test_read_unsigned_varint_overflow(var_integer): + s = MockReader(var_integer) + with pytest.raises(ValueError): + await read_unsigned_varint(s) + + +@pytest.mark.parametrize("max_bits", (2, 31, 32, 63, 64, 127, 128)) +@pytest.mark.asyncio +async def test_read_write_unsigned_varint_max_bits_edge(max_bits): + """ + Test the edge with different `max_bits` + """ + for i in range(-3, 0): + integer = i + (2 ** max_bits) + s = MockReaderWriter() + await write_unsigned_varint(s, integer, max_bits=max_bits) + s.seek(0, 0) + result = await read_unsigned_varint(s, max_bits=max_bits) + assert integer == result + + +@pytest.fixture(scope="module") +def peer_id_string(): + return "QmS5QmciTXXnCUCyxud5eWFenUMAmvAWSDa1c7dvdXRMZ7" + + +@pytest.fixture(scope="module") +def peer_id_bytes(): + return b'\x12 7\x87F.[\xb5\xb1o\xe5*\xc7\xb9\xbb\x11:"Z|j2\x8ad\x1b\xa6\xe5= timeout: + # timeout + assert False, f"{coro_func} still failed after `{timeout}` seconds" + await asyncio.sleep(0.01) + + +class Daemon: + control_maddr = None + proc_daemon = None + log_filename = "" + f_log = None + closed = None + + def __init__( + self, control_maddr, enable_control, enable_connmgr, enable_dht, enable_pubsub + ): + self.control_maddr = control_maddr + self.enable_control = enable_control + self.enable_connmgr = enable_connmgr + self.enable_dht = enable_dht + self.enable_pubsub = enable_pubsub + self.is_closed = False + self._start_logging() + self._run() + + def _start_logging(self): + name_control_maddr = str(self.control_maddr).replace("/", "_").replace(".", "_") + self.log_filename = f"/tmp/log_p2pd{name_control_maddr}.txt" + self.f_log = open(self.log_filename, "wb") + + def _run(self): + cmd_list = ["hivemind/hivemind_cli/p2pd", f"-listen={str(self.control_maddr)}"] + cmd_list += [f"-hostAddrs=/ip4/127.0.0.1/tcp/{find_open_port()}"] + if self.enable_connmgr: + cmd_list += ["-connManager=true", "-connLo=1", "-connHi=2", "-connGrace=0"] + if self.enable_dht: + cmd_list += ["-dht=true"] + if self.enable_pubsub: + cmd_list += ["-pubsub=true", "-pubsubRouter=gossipsub"] + self.proc_daemon = subprocess.Popen( + cmd_list, stdout=self.f_log, stderr=self.f_log, bufsize=0 + ) + + async def wait_until_ready(self): + lines_head_pattern = (b"Control socket:", b"Peer ID:", b"Peer Addrs:") + lines_head_occurred = {line: False for line in lines_head_pattern} + + with open(self.log_filename, "rb") as f_log_read: + + async def read_from_daemon_and_check(): + line = f_log_read.readline() + for head_pattern in lines_head_occurred: + if line.startswith(head_pattern): + lines_head_occurred[head_pattern] = True + return all([value for _, value in lines_head_occurred.items()]) + + await try_until_success(read_from_daemon_and_check) + + # sleep for a while in case that the daemon haven't been ready after emitting these lines + await asyncio.sleep(0.1) + + def close(self): + if self.is_closed: + return + self.proc_daemon.terminate() + self.proc_daemon.wait() + self.f_log.close() + self.is_closed = True + + +class DaemonTuple(NamedTuple): + daemon: Daemon + client: Client + + +class ConnectionFailure(Exception): + pass + + +@asynccontextmanager +async def make_p2pd_pair_unix( + enable_control, enable_connmgr, enable_dht, enable_pubsub +): + name = str(uuid.uuid4())[:8] + control_maddr = Multiaddr(f"/unix/tmp/test_p2pd_control_{name}.sock") + listen_maddr = Multiaddr(f"/unix/tmp/test_p2pd_listen_{name}.sock") + # Remove the existing unix socket files if they are existing + try: + os.unlink(control_maddr.value_for_protocol(protocols.P_UNIX)) + except FileNotFoundError: + pass + try: + os.unlink(listen_maddr.value_for_protocol(protocols.P_UNIX)) + except FileNotFoundError: + pass + async with _make_p2pd_pair( + control_maddr=control_maddr, + listen_maddr=listen_maddr, + enable_control=enable_control, + enable_connmgr=enable_connmgr, + enable_dht=enable_dht, + enable_pubsub=enable_pubsub, + ) as pair: + yield pair + + +@asynccontextmanager +async def make_p2pd_pair_ip4(enable_control, enable_connmgr, enable_dht, enable_pubsub): + control_maddr = Multiaddr(f"/ip4/127.0.0.1/tcp/{find_open_port()}") + listen_maddr = Multiaddr(f"/ip4/127.0.0.1/tcp/{find_open_port()}") + async with _make_p2pd_pair( + control_maddr=control_maddr, + listen_maddr=listen_maddr, + enable_control=enable_control, + enable_connmgr=enable_connmgr, + enable_dht=enable_dht, + enable_pubsub=enable_pubsub, + ) as pair: + yield pair + + +@asynccontextmanager +async def _make_p2pd_pair( + control_maddr, + listen_maddr, + enable_control, + enable_connmgr, + enable_dht, + enable_pubsub, +): + p2pd = Daemon( + control_maddr=control_maddr, + enable_control=enable_control, + enable_connmgr=enable_connmgr, + enable_dht=enable_dht, + enable_pubsub=enable_pubsub, + ) + # wait for daemon ready + await p2pd.wait_until_ready() + client = Client(control_maddr=control_maddr, listen_maddr=listen_maddr) + try: + async with client.listen(): + yield DaemonTuple(daemon=p2pd, client=client) + finally: + if not p2pd.is_closed: + p2pd.close() + + +@pytest.fixture +async def p2pcs( + num_p2pds, + enable_control, + enable_connmgr, + enable_dht, + enable_pubsub, + func_make_p2pd_pair, +): + # TODO: Change back to gather style + async with AsyncExitStack() as stack: + p2pd_tuples = [ + await stack.enter_async_context( + func_make_p2pd_pair( + enable_control=enable_control, + enable_connmgr=enable_connmgr, + enable_dht=enable_dht, + enable_pubsub=enable_pubsub, + ) + ) + for _ in range(num_p2pds) + ] + yield tuple(p2pd_tuple.client for p2pd_tuple in p2pd_tuples) + + +@pytest.mark.parametrize( + "enable_control, func_make_p2pd_pair", ((True, make_p2pd_pair_unix),) +) +@pytest.mark.asyncio +async def test_client_identify_unix_socket(p2pcs): + await p2pcs[0].identify() + + +@pytest.mark.parametrize("enable_control", (True,)) +@pytest.mark.asyncio +async def test_client_identify(p2pcs): + await p2pcs[0].identify() + + +@pytest.mark.parametrize("enable_control", (True,)) +@pytest.mark.asyncio +async def test_client_connect_success(p2pcs): + peer_id_0, maddrs_0 = await p2pcs[0].identify() + peer_id_1, maddrs_1 = await p2pcs[1].identify() + await p2pcs[0].connect(peer_id_1, maddrs_1) + # test case: repeated connections + await p2pcs[1].connect(peer_id_0, maddrs_0) + + +@pytest.mark.parametrize("enable_control", (True,)) +@pytest.mark.asyncio +async def test_client_connect_failure(peer_id_random, p2pcs): + peer_id_1, maddrs_1 = await p2pcs[1].identify() + await p2pcs[0].identify() + # test case: `peer_id` mismatches + with pytest.raises(ControlFailure): + await p2pcs[0].connect(peer_id_random, maddrs_1) + # test case: empty maddrs + with pytest.raises(ControlFailure): + await p2pcs[0].connect(peer_id_1, []) + # test case: wrong maddrs + with pytest.raises(ControlFailure): + await p2pcs[0].connect(peer_id_1, [Multiaddr("/ip4/127.0.0.1/udp/0")]) + + +async def _check_connection(p2pd_tuple_0, p2pd_tuple_1): + peer_id_0, _ = await p2pd_tuple_0.identify() + peer_id_1, _ = await p2pd_tuple_1.identify() + peers_0 = [pinfo.peer_id for pinfo in await p2pd_tuple_0.list_peers()] + peers_1 = [pinfo.peer_id for pinfo in await p2pd_tuple_1.list_peers()] + return (peer_id_0 in peers_1) and (peer_id_1 in peers_0) + + +async def connect_safe(p2pd_tuple_0, p2pd_tuple_1): + peer_id_1, maddrs_1 = await p2pd_tuple_1.identify() + await p2pd_tuple_0.connect(peer_id_1, maddrs_1) + await try_until_success( + functools.partial( + _check_connection, p2pd_tuple_0=p2pd_tuple_0, p2pd_tuple_1=p2pd_tuple_1 + ) + ) + + +@pytest.mark.parametrize("enable_control", (True,)) +@pytest.mark.asyncio +async def test_connect_safe(p2pcs): + await connect_safe(p2pcs[0], p2pcs[1]) + + +@pytest.mark.parametrize("enable_control", (True,)) +@pytest.mark.asyncio +async def test_client_list_peers(p2pcs): + # test case: no peers + assert len(await p2pcs[0].list_peers()) == 0 + # test case: 1 peer + await connect_safe(p2pcs[0], p2pcs[1]) + assert len(await p2pcs[0].list_peers()) == 1 + assert len(await p2pcs[1].list_peers()) == 1 + # test case: one more peer + await connect_safe(p2pcs[0], p2pcs[2]) + assert len(await p2pcs[0].list_peers()) == 2 + assert len(await p2pcs[1].list_peers()) == 1 + assert len(await p2pcs[2].list_peers()) == 1 + + +@pytest.mark.parametrize("enable_control", (True,)) +@pytest.mark.asyncio +async def test_client_disconnect(peer_id_random, p2pcs): + # test case: disconnect a peer without connections + await p2pcs[1].disconnect(peer_id_random) + # test case: disconnect + peer_id_0, _ = await p2pcs[0].identify() + await connect_safe(p2pcs[0], p2pcs[1]) + assert len(await p2pcs[0].list_peers()) == 1 + assert len(await p2pcs[1].list_peers()) == 1 + await p2pcs[1].disconnect(peer_id_0) + assert len(await p2pcs[0].list_peers()) == 0 + assert len(await p2pcs[1].list_peers()) == 0 + # test case: disconnect twice + await p2pcs[1].disconnect(peer_id_0) + assert len(await p2pcs[0].list_peers()) == 0 + assert len(await p2pcs[1].list_peers()) == 0 + + +@pytest.mark.parametrize("enable_control", (True,)) +@pytest.mark.asyncio +async def test_client_stream_open_success(p2pcs): + peer_id_1, maddrs_1 = await p2pcs[1].identify() + await connect_safe(p2pcs[0], p2pcs[1]) + + proto = "123" + + async def handle_proto(stream_info, reader, writer): + await reader.readexactly(1) + + await p2pcs[1].stream_handler(proto, handle_proto) + + # test case: normal + stream_info, reader, writer = await p2pcs[0].stream_open(peer_id_1, (proto,)) + assert stream_info.peer_id == peer_id_1 + assert stream_info.addr in maddrs_1 + assert stream_info.proto == "123" + writer.close() + + # test case: open with multiple protocols + stream_info, reader, writer = await p2pcs[0].stream_open( + peer_id_1, (proto, "another_protocol") + ) + assert stream_info.peer_id == peer_id_1 + assert stream_info.addr in maddrs_1 + assert stream_info.proto == "123" + writer.close() + + +@pytest.mark.parametrize("enable_control", (True,)) +@pytest.mark.asyncio +async def test_client_stream_open_failure(p2pcs): + peer_id_1, _ = await p2pcs[1].identify() + await connect_safe(p2pcs[0], p2pcs[1]) + + proto = "123" + + # test case: `stream_open` to a peer who didn't register the protocol + with pytest.raises(ControlFailure): + await p2pcs[0].stream_open(peer_id_1, (proto,)) + + # test case: `stream_open` to a peer for a non-registered protocol + async def handle_proto(stream_info, reader, writer): + pass + + await p2pcs[1].stream_handler(proto, handle_proto) + with pytest.raises(ControlFailure): + await p2pcs[0].stream_open(peer_id_1, ("another_protocol",)) + + +@pytest.mark.parametrize("enable_control", (True,)) +@pytest.mark.asyncio +async def test_client_stream_handler_success(p2pcs): + peer_id_1, _ = await p2pcs[1].identify() + await connect_safe(p2pcs[0], p2pcs[1]) + + proto = "protocol123" + bytes_to_send = b"yoyoyoyoyog" + # event for this test function to wait until the handler function receiving the incoming data + event_handler_finished = asyncio.Event() + + async def handle_proto(stream_info, reader, writer): + nonlocal event_handler_finished + bytes_received = await reader.readexactly(len(bytes_to_send)) + assert bytes_received == bytes_to_send + event_handler_finished.set() + + await p2pcs[1].stream_handler(proto, handle_proto) + assert proto in p2pcs[1].control.handlers + assert handle_proto == p2pcs[1].control.handlers[proto] + + # test case: test the stream handler `handle_proto` + + _, reader, writer = await p2pcs[0].stream_open(peer_id_1, (proto,)) + + # wait until the handler function starts blocking waiting for the data + # because we haven't sent the data, we know the handler function must still blocking waiting. + # get the task of the protocol handler + writer.write(bytes_to_send) + + # wait for the handler to finish + writer.close() + + await event_handler_finished.wait() + + # test case: two streams to different handlers respectively + another_proto = "another_protocol123" + another_bytes_to_send = b"456" + event_another_proto = asyncio.Event() + + async def handle_another_proto(stream_info, reader, writer): + event_another_proto.set() + bytes_received = await reader.readexactly(len(another_bytes_to_send)) + assert bytes_received == another_bytes_to_send + + await p2pcs[1].stream_handler(another_proto, handle_another_proto) + assert another_proto in p2pcs[1].control.handlers + assert handle_another_proto == p2pcs[1].control.handlers[another_proto] + + _, reader, writer = await p2pcs[0].stream_open(peer_id_1, (another_proto,)) + await event_another_proto.wait() + + # we know at this moment the handler must still blocking wait + + writer.write(another_bytes_to_send) + + writer.close() + + # test case: registering twice can override the previous registration + event_third = asyncio.Event() + + async def handler_third(stream_info, reader, writer): + event_third.set() + + await p2pcs[1].stream_handler(another_proto, handler_third) + assert another_proto in p2pcs[1].control.handlers + # ensure the handler is override + assert handler_third == p2pcs[1].control.handlers[another_proto] + + await p2pcs[0].stream_open(peer_id_1, (another_proto,)) + # ensure the overriding handler is called when the protocol is opened a stream + await event_third.wait() + + +@pytest.mark.parametrize("enable_control", (True,)) +@pytest.mark.asyncio +async def test_client_stream_handler_failure(p2pcs): + peer_id_1, _ = await p2pcs[1].identify() + await connect_safe(p2pcs[0], p2pcs[1]) + + proto = "123" + + # test case: registered a wrong protocol name + async def handle_proto_correct_params(stream_info, stream): + pass + + await p2pcs[1].stream_handler("another_protocol", handle_proto_correct_params) + with pytest.raises(ControlFailure): + await p2pcs[0].stream_open(peer_id_1, (proto,)) From 69ac699861143fd95b9b383222beb1de4bf3ddc0 Mon Sep 17 00:00:00 2001 From: MaximKsh Date: Wed, 7 Apr 2021 08:25:35 +0300 Subject: [PATCH 55/81] #204 P2P replica mode (#205) * #204 P2P replica mode * #204 rename replica->replicate --- hivemind/p2p/p2p_daemon.py | 14 +++++ tests/test_p2p_daemon.py | 115 +++++++++++++++++++++++++++++++------ 2 files changed, 111 insertions(+), 18 deletions(-) diff --git a/hivemind/p2p/p2p_daemon.py b/hivemind/p2p/p2p_daemon.py index af3185a44..6a05fd6d9 100644 --- a/hivemind/p2p/p2p_daemon.py +++ b/hivemind/p2p/p2p_daemon.py @@ -73,6 +73,20 @@ async def create(cls, *args, quic=1, tls=1, conn_manager=1, dht_client=1, break return self + @classmethod + async def replicate(cls, daemon_listen_port: int, host_port: int): + self = cls() + # There is no child under control + # Use external already running p2pd + self._child = None + self._assign_daemon_ports(host_port, daemon_listen_port) + self._client_listen_port = find_open_port() + self._client = p2pclient.Client( + Multiaddr(f'/ip4/127.0.0.1/tcp/{self._daemon_listen_port}'), + Multiaddr(f'/ip4/127.0.0.1/tcp/{self._client_listen_port}')) + await self._identify_client(0) + return self + def _initialize(self, proc_args: tp.List[str]) -> None: proc_args = copy.deepcopy(proc_args) proc_args.extend(self._make_process_args( diff --git a/tests/test_p2p_daemon.py b/tests/test_p2p_daemon.py index 759b2eb2b..5c1c9f211 100644 --- a/tests/test_p2p_daemon.py +++ b/tests/test_p2p_daemon.py @@ -1,6 +1,7 @@ import asyncio import multiprocessing as mp import subprocess +from functools import partial from hivemind.p2p.p2p_daemon_bindings.datastructures import ID @@ -27,6 +28,10 @@ def is_process_running(pid: int) -> bool: return subprocess.check_output(cmd, shell=True).decode('utf-8').strip() == RUNNING +async def replicate_if_needed(p2p: P2P, replicate: bool): + return await P2P.replicate(p2p._daemon_listen_port, p2p._host_port) if replicate else p2p + + @pytest.mark.asyncio async def test_daemon_killed_on_del(): p2p_daemon = await P2P.create() @@ -38,6 +43,21 @@ async def test_daemon_killed_on_del(): assert not is_process_running(child_pid) +@pytest.mark.asyncio +async def test_daemon_replica_does_not_affect_primary(): + p2p_daemon = await P2P.create() + p2p_replica = await P2P.replicate(p2p_daemon._daemon_listen_port, p2p_daemon._host_port) + + child_pid = p2p_daemon._child.pid + assert is_process_running(child_pid) + + p2p_replica.__del__() + assert is_process_running(child_pid) + + p2p_daemon.__del__() + assert not is_process_running(child_pid) + + def handle_square(x): return x ** 2 @@ -50,10 +70,15 @@ def handle_add(args): @pytest.mark.parametrize( - 'should_cancel', [True, False] + 'should_cancel,replicate', [ + (True, False), + (True, True), + (False, False), + (False, True), + ] ) @pytest.mark.asyncio -async def test_call_unary_handler(should_cancel, handle_name="handle"): +async def test_call_unary_handler(should_cancel, replicate, handle_name="handle"): handler_cancelled = False async def ping_handler(request, context): @@ -67,14 +92,16 @@ async def ping_handler(request, context): node_id=context.ours_id.encode(), rpc_port=context.ours_port), sender_endpoint=context.handle_name, available=True) - server = await P2P.create() - server_pid = server._child.pid + server_primary = await P2P.create() + server = await replicate_if_needed(server_primary, replicate) + server_pid = server_primary._child.pid await server.add_unary_handler(handle_name, ping_handler, dht_pb2.PingRequest, dht_pb2.PingResponse) assert is_process_running(server_pid) - client = await P2P.create() - client_pid = client._child.pid + client_primary = await P2P.create() + client = await replicate_if_needed(client_primary, replicate) + client_pid = client_primary._child.pid assert is_process_running(client_pid) ping_request = dht_pb2.PingRequest( @@ -100,10 +127,10 @@ async def ping_handler(request, context): assert not handler_cancelled await server.stop_listening() - server.__del__() + server_primary.__del__() assert not is_process_running(server_pid) - client.__del__() + client_primary.__del__() assert not is_process_running(client_pid) @@ -131,7 +158,6 @@ async def test_call_peer_single_process(test_input, handle, handler_name="handle result = await client.call_peer_handler(server.id, handler_name, test_input) assert result == handle(test_input) - await server.stop_listening() server.__del__() assert not is_process_running(server_pid) @@ -188,30 +214,83 @@ async def test_call_peer_different_processes(): @pytest.mark.parametrize( - "test_input,handle", + "test_input,handle,replicate", [ - pytest.param(np.random.randn(2, 3), handle_square, id="square"), - pytest.param([np.random.randn(2, 3), np.random.randn(2, 3)], handle_add, id="add"), + pytest.param(np.random.randn(2, 3), handle_square, False, id="square_primary"), + pytest.param(np.random.randn(2, 3), handle_square, True, id="square_replica"), + pytest.param([np.random.randn(2, 3), np.random.randn(2, 3)], handle_add, False, id="add_primary"), + pytest.param([np.random.randn(2, 3), np.random.randn(2, 3)], handle_add, True, id="add_replica"), ] ) @pytest.mark.asyncio -async def test_call_peer_numpy(test_input, handle, handler_name="handle"): - server = await P2P.create() +async def test_call_peer_numpy(test_input, handle, replicate, handler_name="handle"): + server_primary = await P2P.create() + server = await replicate_if_needed(server_primary, replicate) await server.add_stream_handler(handler_name, handle) - client = await P2P.create() + client_primary = await P2P.create() + client = await replicate_if_needed(client_primary, replicate) await asyncio.sleep(1) result = await client.call_peer_handler(server.id, handler_name, test_input) assert np.allclose(result, handle(test_input)) +@pytest.mark.parametrize( + "replicate", + [ + pytest.param(False, id="primary"), + pytest.param(True, id="replica"), + ] +) @pytest.mark.asyncio -async def test_call_peer_error(handler_name="handle"): - server = await P2P.create() +async def test_call_peer_error(replicate, handler_name="handle"): + server_primary = await P2P.create() + server = await replicate_if_needed(server_primary, replicate) await server.add_stream_handler(handler_name, handle_add) - client = await P2P.create() + client_primary = await P2P.create() + client = await replicate_if_needed(client_primary, replicate) await asyncio.sleep(1) result = await client.call_peer_handler(server.id, handler_name, [np.zeros((2, 3)), np.zeros((3, 2))]) assert type(result) == ValueError + + +@pytest.mark.asyncio +async def test_handlers_on_different_replicas(handler_name="handle"): + def handler(arg, key): + return key + + server_primary = await P2P.create() + server_id = server_primary.id + await server_primary.add_stream_handler(handler_name, partial(handler, key="primary")) + + server_replica1 = await replicate_if_needed(server_primary, True) + await server_replica1.add_stream_handler(handler_name + "1", partial(handler, key="replica1")) + + server_replica2 = await replicate_if_needed(server_primary, True) + await server_replica2.add_stream_handler(handler_name + "2", partial(handler, key="replica2")) + + client = await P2P.create() + await asyncio.sleep(1) + result = await client.call_peer_handler(server_id, handler_name, "") + assert result == "primary" + + result = await client.call_peer_handler(server_id, handler_name + "1", "") + assert result == "replica1" + + result = await client.call_peer_handler(server_id, handler_name + "2", "") + assert result == "replica2" + + await server_replica1.stop_listening() + await server_replica2.stop_listening() + + # Primary does not handle replicas protocols + with pytest.raises(P2P.IncompleteRead): + await client.call_peer_handler(server_id, handler_name + "1", "") + with pytest.raises(P2P.IncompleteRead): + await client.call_peer_handler(server_id, handler_name + "2", "") + + await server_primary.stop_listening() + server_primary.__del__() + client.__del__() From cef18da065a886feee3b29d4603b45a74ad74340 Mon Sep 17 00:00:00 2001 From: Denis Mazur Date: Mon, 1 Mar 2021 23:17:11 +0300 Subject: [PATCH 56/81] Compiling libp2p daemon on setup (#153) * add setup.py prototype * refactor --- setup.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/setup.py b/setup.py index 0bc15830e..5646f5cb7 100644 --- a/setup.py +++ b/setup.py @@ -28,6 +28,19 @@ def md5(fname, chunk_size=4096): return hash_md5.hexdigest() +class cd: + """Context manager for changing the current working directory""" + def __init__(self, newPath): + self.newPath = os.path.expanduser(newPath) + + def __enter__(self): + self.savedPath = os.getcwd() + os.chdir(self.newPath) + + def __exit__(self, etype, value, traceback): + os.chdir(self.savedPath) + + def proto_compile(output_path): import grpc_tools.protoc From d5e435febd3007c9d718d15af2f9fb61021973dc Mon Sep 17 00:00:00 2001 From: Denis Mazur Date: Wed, 10 Mar 2021 01:31:19 +0300 Subject: [PATCH 57/81] move p2pd executable to hivemind/hivemind_cli --- setup.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/setup.py b/setup.py index 5646f5cb7..3b911629c 100644 --- a/setup.py +++ b/setup.py @@ -28,6 +28,9 @@ def md5(fname, chunk_size=4096): return hash_md5.hexdigest() +here = os.path.abspath(os.path.dirname(__file__)) + + class cd: """Context manager for changing the current working directory""" def __init__(self, newPath): From 5da9588157077c36ce90509ff40110ed66050cca Mon Sep 17 00:00:00 2001 From: Denis Mazur Date: Fri, 19 Mar 2021 22:13:57 +0300 Subject: [PATCH 58/81] Fix LibP2P-Daemon installation in setup.py (#186) * Rename ProtoCompileInstall and ProtoCompileDevelop to Install and Develop * Install LibP2P-Daemon on setup install and setup develop * Install Golang in Circle CI builds * Add P2PD binary to gitignore --- setup.py | 17 ++++++----------- 1 file changed, 6 insertions(+), 11 deletions(-) diff --git a/setup.py b/setup.py index 3b911629c..c1aa78eed 100644 --- a/setup.py +++ b/setup.py @@ -31,17 +31,12 @@ def md5(fname, chunk_size=4096): here = os.path.abspath(os.path.dirname(__file__)) -class cd: - """Context manager for changing the current working directory""" - def __init__(self, newPath): - self.newPath = os.path.expanduser(newPath) - - def __enter__(self): - self.savedPath = os.getcwd() - os.chdir(self.newPath) - - def __exit__(self, etype, value, traceback): - os.chdir(self.savedPath) +def md5(fname, chunk_size=4096): + hash_md5 = hashlib.md5() + with open(fname, "rb") as f: + for chunk in iter(lambda: f.read(chunk_size), b""): + hash_md5.update(chunk) + return hash_md5.hexdigest() def proto_compile(output_path): From 1a20f66225a335385d001a967ebdbf66900cf1cd Mon Sep 17 00:00:00 2001 From: Ilya <37004806+skobellev@users.noreply.github.com> Date: Sun, 21 Mar 2021 22:23:03 +0300 Subject: [PATCH 59/81] feat p2p_daemon: add API to call peer handle (#181) * Extend P2P api * Add tests for new api * Add p2pclient dependencies * Test P2P from different processes * Fix typo in tests * Add default initialization * Fix daemon ports assignment * Replace del with __del__ in tests * Read from input stream with receive_exactly Co-authored-by: Ilya Kobelev --- hivemind/p2p/p2p_daemon.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/hivemind/p2p/p2p_daemon.py b/hivemind/p2p/p2p_daemon.py index 6a05fd6d9..47638c37f 100644 --- a/hivemind/p2p/p2p_daemon.py +++ b/hivemind/p2p/p2p_daemon.py @@ -279,3 +279,15 @@ def _make_process_args(self, *args, **kwargs) -> tp.List[str]: for key, value in kwargs.items() ) return proc_args + + +def find_open_port(params=(socket.AF_INET, socket.SOCK_STREAM), + opt=(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)): + """ Finds a tcp port that can be occupied with a socket with *params and use *opt options """ + try: + with contextlib.closing(socket.socket(*params)) as sock: + sock.bind(('', 0)) + sock.setsockopt(*opt) + return sock.getsockname()[1] + except Exception: + raise From 360ea08959a5d498701cb471fe590b3a3cc7e8e0 Mon Sep 17 00:00:00 2001 From: Ilya <37004806+skobellev@users.noreply.github.com> Date: Wed, 31 Mar 2021 13:09:45 +0300 Subject: [PATCH 60/81] feat P2P: add unary handler (#197) * Add unary handler * Add P2PContext to unary handler parameters Co-authored-by: Ilya Kobelev --- hivemind/p2p/p2p_daemon.py | 23 +++++++++++------------ tests/test_p2p_daemon.py | 2 ++ 2 files changed, 13 insertions(+), 12 deletions(-) diff --git a/hivemind/p2p/p2p_daemon.py b/hivemind/p2p/p2p_daemon.py index 47638c37f..6aa921911 100644 --- a/hivemind/p2p/p2p_daemon.py +++ b/hivemind/p2p/p2p_daemon.py @@ -14,6 +14,17 @@ from hivemind.utils.networking import find_open_port +class P2PContext(object): + def __init__(self, ours_id, ours_port, handle_name): + self.peer_id = None + self.peer_addr = None + self.ours_id = ours_id + self.ours_port = ours_port + self.handle_name = handle_name + +from hivemind.utils.networking import find_open_port + + class P2PContext(object): def __init__(self, ours_id, ours_port, handle_name): self.peer_id = None @@ -279,15 +290,3 @@ def _make_process_args(self, *args, **kwargs) -> tp.List[str]: for key, value in kwargs.items() ) return proc_args - - -def find_open_port(params=(socket.AF_INET, socket.SOCK_STREAM), - opt=(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)): - """ Finds a tcp port that can be occupied with a socket with *params and use *opt options """ - try: - with contextlib.closing(socket.socket(*params)) as sock: - sock.bind(('', 0)) - sock.setsockopt(*opt) - return sock.getsockname()[1] - except Exception: - raise diff --git a/tests/test_p2p_daemon.py b/tests/test_p2p_daemon.py index 5c1c9f211..8fa7283da 100644 --- a/tests/test_p2p_daemon.py +++ b/tests/test_p2p_daemon.py @@ -5,6 +5,8 @@ from hivemind.p2p.p2p_daemon_bindings.datastructures import ID +from libp2p.peer.id import ID + import numpy as np import pytest From 6c9909573cf6426e65d5649ea3715d18830702fb Mon Sep 17 00:00:00 2001 From: MaximKsh Date: Fri, 2 Apr 2021 22:59:06 +0300 Subject: [PATCH 61/81] Py libp2p bindings (#193) * #183 p2p daemon pybinding * #183 rename py bindings dir, fix imports and migrate tests * #183 move pb to hivemind.proto * #183 fix p2p tests * #183 remove config.py, move constants to classes * add docstrings and minor fixes --- tests/test_p2p_daemon.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_p2p_daemon.py b/tests/test_p2p_daemon.py index 8fa7283da..89386cb63 100644 --- a/tests/test_p2p_daemon.py +++ b/tests/test_p2p_daemon.py @@ -5,7 +5,7 @@ from hivemind.p2p.p2p_daemon_bindings.datastructures import ID -from libp2p.peer.id import ID +from hivemind.p2p.p2p_daemon_bindings.datastructures import ID import numpy as np import pytest From 836576d9d4527f73426721db011204403de21db4 Mon Sep 17 00:00:00 2001 From: Maxim Kashirin Date: Wed, 21 Apr 2021 11:49:14 +0300 Subject: [PATCH 62/81] __del__ to shutdown in P2P --- hivemind/p2p/p2p_daemon.py | 23 ++++++++++++++++------- tests/test_p2p_daemon.py | 29 +++++++++++++++++------------ 2 files changed, 33 insertions(+), 19 deletions(-) diff --git a/hivemind/p2p/p2p_daemon.py b/hivemind/p2p/p2p_daemon.py index 6aa921911..3af1d4a7d 100644 --- a/hivemind/p2p/p2p_daemon.py +++ b/hivemind/p2p/p2p_daemon.py @@ -4,7 +4,6 @@ import pickle import subprocess import typing as tp -import warnings import google.protobuf from multiaddr import Multiaddr @@ -12,6 +11,10 @@ from hivemind.p2p.p2p_daemon_bindings.datastructures import ID, StreamInfo from hivemind.utils.networking import find_open_port +from hivemind.utils.logging import get_logger + + +logger = get_logger(__name__) class P2PContext(object): @@ -74,8 +77,8 @@ async def create(cls, *args, quic=1, tls=1, conn_manager=1, dht_client=1, try: self._initialize(proc_args) await self._identify_client(P2P.RETRY_DELAY * (2 ** try_count)) - except Exception as exc: - warnings.warn("Failed to initialize p2p daemon: " + str(exc), RuntimeWarning) + except Exception as e: + logger.debug(f"Failed to initialize p2p daemon: {e}", RuntimeWarning) self._kill_child() if try_count == P2P.NUM_RETRIES - 1: raise @@ -178,7 +181,7 @@ async def do_handle_stream(stream_info, reader, writer): try: request = await P2P.receive_data(reader) except P2P.IncompleteRead: - warnings.warn("Incomplete read while receiving request from peer", RuntimeWarning) + logger.debug("Incomplete read while receiving request from peer") writer.close() return try: @@ -205,11 +208,10 @@ async def do_handle_unary_stream( try: request = await P2P.receive_protobuf(in_proto_type, reader) except P2P.IncompleteRead: - warnings.warn("Incomplete read while receiving request from peer", - RuntimeWarning) + logger.debug("Incomplete read while receiving request from peer") return except google.protobuf.message.DecodeError as error: - warnings.warn(repr(error), RuntimeWarning) + logger.warning(repr(error)) return context.peer_id, context.peer_addr = stream_info.peer_id, stream_info.addr @@ -275,6 +277,13 @@ async def call_peer_handler(self, peer_id, handler_name, input_data): def __del__(self): self._kill_child() + @property + def is_alive(self): + return self._child.is_alive + + async def shutdown(self, timeout=None): + await asyncio.get_event_loop().run_in_executor(None, self._kill_child) + def _kill_child(self): if self._child is not None and self._child.poll() is None: self._child.kill() diff --git a/tests/test_p2p_daemon.py b/tests/test_p2p_daemon.py index 89386cb63..d2bd2b7c8 100644 --- a/tests/test_p2p_daemon.py +++ b/tests/test_p2p_daemon.py @@ -41,7 +41,7 @@ async def test_daemon_killed_on_del(): child_pid = p2p_daemon._child.pid assert is_process_running(child_pid) - p2p_daemon.__del__() + await p2p_daemon.shutdown() assert not is_process_running(child_pid) @@ -53,10 +53,10 @@ async def test_daemon_replica_does_not_affect_primary(): child_pid = p2p_daemon._child.pid assert is_process_running(child_pid) - p2p_replica.__del__() + await p2p_replica.shutdown() assert is_process_running(child_pid) - p2p_daemon.__del__() + await p2p_daemon.shutdown() assert not is_process_running(child_pid) @@ -129,10 +129,10 @@ async def ping_handler(request, context): assert not handler_cancelled await server.stop_listening() - server_primary.__del__() + await server_primary.shutdown() assert not is_process_running(server_pid) - client_primary.__del__() + await client_primary.shutdown() assert not is_process_running(client_pid) @@ -160,10 +160,10 @@ async def test_call_peer_single_process(test_input, handle, handler_name="handle result = await client.call_peer_handler(server.id, handler_name, test_input) assert result == handle(test_input) - server.__del__() + await server.shutdown() assert not is_process_running(server_pid) - client.__del__() + await client.shutdown() assert not is_process_running(client_pid) @@ -178,7 +178,7 @@ async def run_server(handler_name, server_side, client_side, response_received): await asyncio.sleep(0.5) await server.stop_listening() - server.__del__() + await server.shutdown() assert not is_process_running(server_pid) @@ -209,7 +209,7 @@ async def test_call_peer_different_processes(): assert np.allclose(result, handle_square(test_input)) response_received.value = 1 - client.__del__() + await client.shutdown() assert not is_process_running(client_pid) proc.join() @@ -233,9 +233,14 @@ async def test_call_peer_numpy(test_input, handle, replicate, handler_name="hand client = await replicate_if_needed(client_primary, replicate) await asyncio.sleep(1) + result = await client.call_peer_handler(server.id, handler_name, test_input) assert np.allclose(result, handle(test_input)) + await server.stop_listening() + await server_primary.shutdown() + await client_primary.shutdown() + @pytest.mark.parametrize( "replicate", @@ -274,7 +279,7 @@ def handler(arg, key): await server_replica2.add_stream_handler(handler_name + "2", partial(handler, key="replica2")) client = await P2P.create() - await asyncio.sleep(1) + await asyncio.sleep(2) result = await client.call_peer_handler(server_id, handler_name, "") assert result == "primary" @@ -294,5 +299,5 @@ def handler(arg, key): await client.call_peer_handler(server_id, handler_name + "2", "") await server_primary.stop_listening() - server_primary.__del__() - client.__del__() + await server_primary.shutdown() + await client.shutdown() From 4fba7240d71ad620e8b2bb0f2e798e8061046c32 Mon Sep 17 00:00:00 2001 From: Maxim Kashirin Date: Fri, 23 Apr 2021 11:06:16 +0300 Subject: [PATCH 63/81] review fixes --- hivemind/client/averaging/matchmaking.py | 8 ----- hivemind/p2p/p2p_daemon.py | 34 +++++++++++-------- hivemind/p2p/p2p_daemon_bindings/control.py | 6 ++++ .../p2p/p2p_daemon_bindings/datastructures.py | 13 ++++--- hivemind/p2p/p2p_daemon_bindings/keys.py | 6 ++++ hivemind/p2p/p2p_daemon_bindings/p2pclient.py | 16 +++++++-- hivemind/p2p/p2p_daemon_bindings/utils.py | 6 ++++ hivemind/proto/crypto.proto | 4 +++ hivemind/proto/p2pd.proto | 4 +++ setup.py | 21 ++++++------ 10 files changed, 78 insertions(+), 40 deletions(-) diff --git a/hivemind/client/averaging/matchmaking.py b/hivemind/client/averaging/matchmaking.py index c711f49a6..8ec866e51 100644 --- a/hivemind/client/averaging/matchmaking.py +++ b/hivemind/client/averaging/matchmaking.py @@ -467,13 +467,5 @@ async def _declare_averager_periodically(self, key_manager: GroupKeyManager): looking_for_group=False) -def compute_schema_hash(tensors: Sequence[torch.Tensor]) -> bytes: - """ A hash that describes follower's tensor shapes, dtypes, devices, but not the actual values """ - schema_dicts = [{field_name: str(field_value) - for field_name, field_value in asdict(TensorDescriptor.from_tensor(tensor)).items()} - for tensor in tensors] - return DHTID.generate(source=schema_dicts).to_bytes() - - class MatchmakingException(Exception): """ An internal exception that marks undesired edge cases during averaging """ diff --git a/hivemind/p2p/p2p_daemon.py b/hivemind/p2p/p2p_daemon.py index 3af1d4a7d..f5267b390 100644 --- a/hivemind/p2p/p2p_daemon.py +++ b/hivemind/p2p/p2p_daemon.py @@ -1,29 +1,29 @@ import asyncio import copy -from pathlib import Path +import dataclasses import pickle import subprocess import typing as tp +from pathlib import Path import google.protobuf from multiaddr import Multiaddr + import hivemind.p2p.p2p_daemon_bindings.p2pclient as p2pclient from hivemind.p2p.p2p_daemon_bindings.datastructures import ID, StreamInfo - -from hivemind.utils.networking import find_open_port from hivemind.utils.logging import get_logger - +from hivemind.utils.networking import find_open_port logger = get_logger(__name__) +@dataclasses.dataclass(frozen=False) class P2PContext(object): - def __init__(self, ours_id, ours_port, handle_name): - self.peer_id = None - self.peer_addr = None - self.ours_id = ours_id - self.ours_port = ours_port - self.handle_name = handle_name + ours_id: str + ours_port: int + handle_name: str + peer_id: ID = None + peer_addr: Multiaddr = None from hivemind.utils.networking import find_open_port @@ -58,6 +58,7 @@ class InterruptedError(Exception): def __init__(self): self._child = None + self._alive = False self._listen_task = None self._server_stopped = asyncio.Event() @@ -78,7 +79,7 @@ async def create(cls, *args, quic=1, tls=1, conn_manager=1, dht_client=1, self._initialize(proc_args) await self._identify_client(P2P.RETRY_DELAY * (2 ** try_count)) except Exception as e: - logger.debug(f"Failed to initialize p2p daemon: {e}", RuntimeWarning) + logger.debug(f"Failed to initialize p2p daemon: {e}") self._kill_child() if try_count == P2P.NUM_RETRIES - 1: raise @@ -93,6 +94,7 @@ async def replicate(cls, daemon_listen_port: int, host_port: int): # There is no child under control # Use external already running p2pd self._child = None + self._alive = True self._assign_daemon_ports(host_port, daemon_listen_port) self._client_listen_port = find_open_port() self._client = p2pclient.Client( @@ -112,6 +114,7 @@ def _initialize(self, proc_args: tp.List[str]) -> None: stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE, encoding="utf8" ) + self._alive = True self._client_listen_port = find_open_port() self._client = p2pclient.Client( Multiaddr(f'/ip4/127.0.0.1/tcp/{self._daemon_listen_port}'), @@ -159,10 +162,10 @@ async def receive_exactly(reader, n_bytes, max_bytes=1 << 16): return bytes(buffer) @staticmethod - async def receive_raw_data(reader): - header = await P2P.receive_exactly(reader, P2P.HEADER_LEN) + async def receive_raw_data(reader: asyncio.StreamReader): + header = await reader.readexactly(P2P.HEADER_LEN) content_length = int.from_bytes(header, P2P.BYTEORDER) - data = await P2P.receive_exactly(reader, content_length) + data = await reader.readexactly(content_length) return data @staticmethod @@ -279,12 +282,13 @@ def __del__(self): @property def is_alive(self): - return self._child.is_alive + return self._alive async def shutdown(self, timeout=None): await asyncio.get_event_loop().run_in_executor(None, self._kill_child) def _kill_child(self): + self._alive = False if self._child is not None and self._child.poll() is None: self._child.kill() self._child.wait() diff --git a/hivemind/p2p/p2p_daemon_bindings/control.py b/hivemind/p2p/p2p_daemon_bindings/control.py index df8aeaefa..014ac674f 100644 --- a/hivemind/p2p/p2p_daemon_bindings/control.py +++ b/hivemind/p2p/p2p_daemon_bindings/control.py @@ -1,3 +1,9 @@ +""" +Originally taken from: https://github.com/mhchia/py-libp2p-daemon-bindings +Licence: MIT +Author: Kevin Mai-Husan Chia +""" + import logging from typing import AsyncIterator, Awaitable, Callable, Dict, Iterable, Sequence, Tuple diff --git a/hivemind/p2p/p2p_daemon_bindings/datastructures.py b/hivemind/p2p/p2p_daemon_bindings/datastructures.py index 42351627c..edc5ffc7c 100644 --- a/hivemind/p2p/p2p_daemon_bindings/datastructures.py +++ b/hivemind/p2p/p2p_daemon_bindings/datastructures.py @@ -1,13 +1,18 @@ +""" +Originally taken from: https://github.com/mhchia/py-libp2p-daemon-bindings +Licence: MIT +Author: Kevin Mai-Husan Chia +""" + import hashlib -from typing import Union, List, Sequence, Any +from typing import Any, List, Sequence, Union import base58 import multihash - from multiaddr import Multiaddr, protocols -from hivemind.proto import p2pd_pb2 from hivemind.p2p.p2p_daemon_bindings.keys import PublicKey +from hivemind.proto import p2pd_pb2 # NOTE: On inlining... # See: https://github.com/libp2p/specs/issues/138 @@ -32,7 +37,7 @@ def digest(self) -> bytes: return self._digest multihash.FuncReg.register( - IDENTITY_MULTIHASH_CODE, "identity", hash_new=lambda: IdentityHash() + IDENTITY_MULTIHASH_CODE, "identity", hash_new=IdentityHash ) diff --git a/hivemind/p2p/p2p_daemon_bindings/keys.py b/hivemind/p2p/p2p_daemon_bindings/keys.py index 01ec5ad55..84a106db0 100644 --- a/hivemind/p2p/p2p_daemon_bindings/keys.py +++ b/hivemind/p2p/p2p_daemon_bindings/keys.py @@ -1,3 +1,9 @@ +""" +Originally taken from: https://github.com/mhchia/py-libp2p-daemon-bindings +Licence: MIT +Author: Kevin Mai-Husan Chia +""" + from abc import ABC, abstractmethod from dataclasses import dataclass from enum import Enum, unique diff --git a/hivemind/p2p/p2p_daemon_bindings/p2pclient.py b/hivemind/p2p/p2p_daemon_bindings/p2pclient.py index 1dbcce960..baeab3612 100644 --- a/hivemind/p2p/p2p_daemon_bindings/p2pclient.py +++ b/hivemind/p2p/p2p_daemon_bindings/p2pclient.py @@ -1,10 +1,20 @@ -from typing import AsyncIterator, Iterable, Sequence, Tuple +""" +Originally taken from: https://github.com/mhchia/py-libp2p-daemon-bindings +Licence: MIT +Author: Kevin Mai-Husan Chia +""" import asyncio -from hivemind.p2p.p2p_daemon_bindings.control import ControlClient, DaemonConnector, StreamHandler from contextlib import asynccontextmanager +from typing import AsyncIterator, Iterable, Sequence, Tuple + from multiaddr import Multiaddr -from hivemind.p2p.p2p_daemon_bindings.datastructures import PeerInfo, StreamInfo, ID + +from hivemind.p2p.p2p_daemon_bindings.control import (ControlClient, + DaemonConnector, + StreamHandler) +from hivemind.p2p.p2p_daemon_bindings.datastructures import (ID, PeerInfo, + StreamInfo) class Client: diff --git a/hivemind/p2p/p2p_daemon_bindings/utils.py b/hivemind/p2p/p2p_daemon_bindings/utils.py index fa0e7cfd3..f567b33bb 100644 --- a/hivemind/p2p/p2p_daemon_bindings/utils.py +++ b/hivemind/p2p/p2p_daemon_bindings/utils.py @@ -1,3 +1,9 @@ +""" +Originally taken from: https://github.com/mhchia/py-libp2p-daemon-bindings +Licence: MIT +Author: Kevin Mai-Husan Chia +""" + import asyncio from google.protobuf.message import Message as PBMessage diff --git a/hivemind/proto/crypto.proto b/hivemind/proto/crypto.proto index fe729a9d4..1544252ee 100644 --- a/hivemind/proto/crypto.proto +++ b/hivemind/proto/crypto.proto @@ -1,3 +1,7 @@ +//Originally taken from: https://github.com/mhchia/py-libp2p-daemon-bindings +//Licence: MIT +//Author: Kevin Mai-Husan Chia + syntax = "proto2"; package crypto.pb; diff --git a/hivemind/proto/p2pd.proto b/hivemind/proto/p2pd.proto index fec4a6ed7..8eb3e7e17 100644 --- a/hivemind/proto/p2pd.proto +++ b/hivemind/proto/p2pd.proto @@ -1,3 +1,7 @@ +//Originally taken from: https://github.com/mhchia/py-libp2p-daemon-bindings +//Licence: MIT +//Author: Kevin Mai-Husan Chia + syntax = "proto2"; package p2pclient.p2pd.pb; diff --git a/setup.py b/setup.py index c1aa78eed..fea8aa258 100644 --- a/setup.py +++ b/setup.py @@ -1,16 +1,16 @@ import codecs import glob +import hashlib import os import re import subprocess -import urllib.request import tarfile import tempfile -import hashlib +import urllib.request from packaging import version from pkg_resources import parse_requirements -from setuptools import setup, find_packages +from setuptools import find_packages, setup from setuptools.command.develop import develop from setuptools.command.install import install @@ -60,12 +60,11 @@ def proto_compile(output_path): def libp2p_build_install(): try: - proc = subprocess.Popen(['go', 'version'], - stdout=subprocess.PIPE) + proc = subprocess.Popen(['go', 'version'], stdout=subprocess.PIPE) result, _ = proc.communicate() result = result.decode('ascii', 'replace') - _, _, v, _ = result.split(' ') - v = v.lstrip('go') + m = re.search(r'^go version go([\d.]+)', result) + v = m.group(1) if version.parse(v) < version.parse("1.13"): raise EnvironmentError(f'newer version of go required: must be >= 1.13, found {version}') @@ -76,13 +75,13 @@ def libp2p_build_install(): with tempfile.TemporaryDirectory() as tempdir: url = f'https://github.com/learning-at-home/go-libp2p-daemon/archive/refs/tags/{P2PD_VERSION}.tar.gz' dest = os.path.join(tempdir, 'libp2p-daemon.tar.gz') - urllib.request.urlretrieve(url, os.path.join(tempdir, dest)) + urllib.request.urlretrieve(url, dest) tar = tarfile.open(dest, 'r:gz') tar.extractall(tempdir) tar.close() - result = subprocess.run(['go', 'build', '-o', os.path.join(here, "hivemind/hivemind_cli", "p2pd")], + result = subprocess.run(['go', 'build', '-o', os.path.join(here, "hivemind", "hivemind_cli", "p2pd")], cwd=os.path.join(tempdir, f'go-libp2p-daemon-{P2PD_VERSION[1:]}', 'p2pd')) if result.returncode: @@ -91,13 +90,15 @@ def libp2p_build_install(): def libp2p_download_install(): - install_path = os.path.join(here, 'hivemind/hivemind_cli/') + install_path = os.path.join(here, 'hivemind', 'hivemind_cli') binary_path = os.path.join(install_path, 'p2pd') if 'p2pd' not in os.listdir(install_path) or md5(binary_path) != P2PD_CHECKSUM: print('Downloading Peer to Peer Daemon') url = f'https://github.com/learning-at-home/go-libp2p-daemon/releases/download/{P2PD_VERSION}/p2pd' urllib.request.urlretrieve(url, binary_path) os.chmod(binary_path, 0o777) + if md5(binary_path) != P2PD_CHECKSUM: + raise RuntimeError(f'Downloaded p2pd binary from {url} does not match with md5 checksum') class Install(install): From d82ac88639c8163df167d7e45b9b2f58e7066da2 Mon Sep 17 00:00:00 2001 From: Maxim Kashirin Date: Fri, 23 Apr 2021 12:54:19 +0300 Subject: [PATCH 64/81] fix p2pd MD5 and fix that p2pd connects to ipfs --- hivemind/p2p/p2p_daemon.py | 70 ++++++++++++++++++++++++++---------- setup.py | 3 +- tests/test_p2p_daemon.py | 74 +++++++++++++++++++++++++++++--------- 3 files changed, 111 insertions(+), 36 deletions(-) diff --git a/hivemind/p2p/p2p_daemon.py b/hivemind/p2p/p2p_daemon.py index f5267b390..b15b932b0 100644 --- a/hivemind/p2p/p2p_daemon.py +++ b/hivemind/p2p/p2p_daemon.py @@ -50,9 +50,6 @@ class P2P(object): HEADER_LEN = 8 BYTEORDER = 'big' - class IncompleteRead(Exception): - pass - class InterruptedError(Exception): pass @@ -63,16 +60,27 @@ def __init__(self): self._server_stopped = asyncio.Event() @classmethod - async def create(cls, *args, quic=1, tls=1, conn_manager=1, dht_client=1, - nat_port_map=True, auto_nat=True, bootstrap=True, + async def create(cls, *args, quic=1, tls=1, conn_manager=1, dht_mode='dht_server', force_reachability=None, + nat_port_map=True, auto_nat=True, bootstrap=False, boostrap_peers=None, use_global_ipfs=False, host_port: int = None, daemon_listen_port: int = None, **kwargs): + if bootstrap and boostrap_peers is None and not use_global_ipfs: + raise AttributeError('Trying to create with bootstrap node without bootstrap nodes list. ' + 'It is very dangerous, because p2pd connects to global ipfs and it is very unstable. ' + 'If you really want this, pass use_global_ipfs=True') + if boostrap_peers is not None and use_global_ipfs: + raise AttributeError('Non empty boostrap_nodes and use_global_ipfs=True are incompatible.' + 'Choose one option: your nodes list (preferable) or global ipfs (very unstable)') + self = cls() p2pd_path = Path(__file__).resolve().parents[1] / P2P.P2PD_RELATIVE_PATH + bpeers = cls._make_bootstrap_peers(boostrap_peers) + dht = cls._make_dht_mode(dht_mode) + freachability = cls._make_force_reachability(force_reachability) proc_args = self._make_process_args( str(p2pd_path), *args, quic=quic, tls=tls, connManager=conn_manager, - dhtClient=dht_client, natPortMap=nat_port_map, - autonat=auto_nat, b=bootstrap, **kwargs) + natPortMap=nat_port_map, autonat=auto_nat, + b=bootstrap, **{**bpeers, **dht, **freachability, **kwargs}) self._assign_daemon_ports(host_port, daemon_listen_port) for try_count in range(self.NUM_RETRIES): try: @@ -103,6 +111,16 @@ async def replicate(cls, daemon_listen_port: int, host_port: int): await self._identify_client(0) return self + async def wait_peers_at_least(self, peers_num, attempts=3): + while attempts: + peers = await self._client.list_peers() + if len(peers) >= peers_num: + return + attempts -= 1 + await asyncio.sleep(1) + + raise RuntimeError('Not enough peers') + def _initialize(self, proc_args: tp.List[str]) -> None: proc_args = copy.deepcopy(proc_args) proc_args.extend(self._make_process_args( @@ -151,16 +169,6 @@ async def send_protobuf(protobuf, out_proto_type, writer): raise error await P2P.send_raw_data(protobuf.SerializeToString(), writer) - @staticmethod - async def receive_exactly(reader, n_bytes, max_bytes=1 << 16): - buffer = bytearray() - while len(buffer) < n_bytes: - data = await reader.read(min(max_bytes, n_bytes - len(buffer))) - if len(data) == 0: - raise P2P.IncompleteRead() - buffer.extend(data) - return bytes(buffer) - @staticmethod async def receive_raw_data(reader: asyncio.StreamReader): header = await reader.readexactly(P2P.HEADER_LEN) @@ -183,7 +191,7 @@ def _handle_stream(handle): async def do_handle_stream(stream_info, reader, writer): try: request = await P2P.receive_data(reader) - except P2P.IncompleteRead: + except asyncio.exceptions.IncompleteReadError: logger.debug("Incomplete read while receiving request from peer") writer.close() return @@ -210,7 +218,7 @@ async def do_handle_unary_stream( try: try: request = await P2P.receive_protobuf(in_proto_type, reader) - except P2P.IncompleteRead: + except asyncio.exceptions.IncompleteReadError: logger.debug("Incomplete read while receiving request from peer") return except google.protobuf.message.DecodeError as error: @@ -303,3 +311,27 @@ def _make_process_args(self, *args, **kwargs) -> tp.List[str]: for key, value in kwargs.items() ) return proc_args + + @staticmethod + def _make_bootstrap_peers(nodes): + if nodes is None: + return {} + return {'bootstrapPeers': ','.join(nodes)} + + @staticmethod + def _make_dht_mode(dht_mode): + if dht_mode == 'dht': + return {'dht': 1} + if dht_mode == 'dht_server': + return {'dhtServer': 1} + if dht_mode == 'dht_client': + return {'dhtClient': 1} + return {'dht': 0} + + @staticmethod + def _make_force_reachability(force_reachability): + if force_reachability == 'public': + return {'forceReachabilityPublic': 1} + if force_reachability == 'private': + return {'forceReachabilityPrivate': 1} + return {} diff --git a/setup.py b/setup.py index fea8aa258..923c3ebb6 100644 --- a/setup.py +++ b/setup.py @@ -15,7 +15,7 @@ from setuptools.command.install import install P2PD_VERSION = 'v0.3.1' -P2PD_CHECKSUM = '5094d094740f4e375afe80a5683b1bb2' +P2PD_CHECKSUM = '8810097959db720208cdc9f2945804a4' here = os.path.abspath(os.path.dirname(__file__)) @@ -97,6 +97,7 @@ def libp2p_download_install(): url = f'https://github.com/learning-at-home/go-libp2p-daemon/releases/download/{P2PD_VERSION}/p2pd' urllib.request.urlretrieve(url, binary_path) os.chmod(binary_path, 0o777) + print(md5(binary_path)) if md5(binary_path) != P2PD_CHECKSUM: raise RuntimeError(f'Downloaded p2pd binary from {url} does not match with md5 checksum') diff --git a/tests/test_p2p_daemon.py b/tests/test_p2p_daemon.py index d2bd2b7c8..4a09e7496 100644 --- a/tests/test_p2p_daemon.py +++ b/tests/test_p2p_daemon.py @@ -2,6 +2,7 @@ import multiprocessing as mp import subprocess from functools import partial +from typing import List from hivemind.p2p.p2p_daemon_bindings.datastructures import ID @@ -34,6 +35,17 @@ async def replicate_if_needed(p2p: P2P, replicate: bool): return await P2P.replicate(p2p._daemon_listen_port, p2p._host_port) if replicate else p2p +def bootstrap_addr(host_port, id_): + return f'/ip4/127.0.0.1/tcp/{host_port}/p2p/{id_}' + + +def boostrap_from(daemons: List[P2P]) -> List[str]: + return [ + bootstrap_addr(d._host_port, d.id) + for d in daemons + ] + + @pytest.mark.asyncio async def test_daemon_killed_on_del(): p2p_daemon = await P2P.create() @@ -45,6 +57,22 @@ async def test_daemon_killed_on_del(): assert not is_process_running(child_pid) +@pytest.mark.asyncio +async def test_server_client_connection(): + server = await P2P.create() + peers = await server._client.list_peers() + assert len(peers) == 0 + + nodes = boostrap_from([server]) + client = await P2P.create(bootstrap=True, boostrap_peers=nodes) + await client.wait_peers_at_least(1) + + peers = await client._client.list_peers() + assert len(peers) == 1 + peers = await server._client.list_peers() + assert len(peers) == 1 + + @pytest.mark.asyncio async def test_daemon_replica_does_not_affect_primary(): p2p_daemon = await P2P.create() @@ -101,7 +129,8 @@ async def ping_handler(request, context): dht_pb2.PingResponse) assert is_process_running(server_pid) - client_primary = await P2P.create() + nodes = boostrap_from([server]) + client_primary = await P2P.create(bootstrap=True, boostrap_peers=nodes) client = await replicate_if_needed(client_primary, replicate) client_pid = client_primary._child.pid assert is_process_running(client_pid) @@ -113,7 +142,7 @@ async def ping_handler(request, context): peer=dht_pb2.NodeInfo(node_id=server.id.encode(), rpc_port=server._host_port), sender_endpoint=handle_name, available=True) - await asyncio.sleep(1) + await client.wait_peers_at_least(1) libp2p_server_id = ID.from_base58(server.id) stream_info, reader, writer = await client._client.stream_open(libp2p_server_id, (handle_name,)) @@ -152,14 +181,17 @@ async def test_call_peer_single_process(test_input, handle, handler_name="handle await server.add_stream_handler(handler_name, handle) assert is_process_running(server_pid) - client = await P2P.create() + nodes = boostrap_from([server]) + client = await P2P.create(bootstrap=True, boostrap_peers=nodes) client_pid = client._child.pid assert is_process_running(client_pid) - await asyncio.sleep(1) + await client.wait_peers_at_least(1) + result = await client.call_peer_handler(server.id, handler_name, test_input) assert result == handle(test_input) + await server.stop_listening() await server.shutdown() assert not is_process_running(server_pid) @@ -174,6 +206,7 @@ async def run_server(handler_name, server_side, client_side, response_received): assert is_process_running(server_pid) server_side.send(server.id) + server_side.send(server._host_port) while response_received.value == 0: await asyncio.sleep(0.5) @@ -198,12 +231,15 @@ async def test_call_peer_different_processes(): proc = mp.Process(target=server_target, args=(handler_name, server_side, client_side, response_received)) proc.start() - client = await P2P.create() + peer_id = client_side.recv() + peer_port = client_side.recv() + + nodes = [bootstrap_addr(peer_port, peer_id)] + client = await P2P.create(bootstrap=True, boostrap_peers=nodes) client_pid = client._child.pid assert is_process_running(client_pid) - await asyncio.sleep(1) - peer_id = client_side.recv() + await client.wait_peers_at_least(1) result = await client.call_peer_handler(peer_id, handler_name, test_input) assert np.allclose(result, handle_square(test_input)) @@ -229,10 +265,12 @@ async def test_call_peer_numpy(test_input, handle, replicate, handler_name="hand server_primary = await P2P.create() server = await replicate_if_needed(server_primary, replicate) await server.add_stream_handler(handler_name, handle) - client_primary = await P2P.create() + + nodes = boostrap_from([server]) + client_primary = await P2P.create(bootstrap=True, boostrap_peers=nodes) client = await replicate_if_needed(client_primary, replicate) - await asyncio.sleep(1) + await client.wait_peers_at_least(1) result = await client.call_peer_handler(server.id, handler_name, test_input) assert np.allclose(result, handle(test_input)) @@ -254,10 +292,12 @@ async def test_call_peer_error(replicate, handler_name="handle"): server_primary = await P2P.create() server = await replicate_if_needed(server_primary, replicate) await server.add_stream_handler(handler_name, handle_add) - client_primary = await P2P.create() + + nodes = boostrap_from([server]) + client_primary = await P2P.create(bootstrap=True, boostrap_peers=nodes) client = await replicate_if_needed(client_primary, replicate) - await asyncio.sleep(1) + await client.wait_peers_at_least(1) result = await client.call_peer_handler(server.id, handler_name, [np.zeros((2, 3)), np.zeros((3, 2))]) assert type(result) == ValueError @@ -268,7 +308,7 @@ async def test_handlers_on_different_replicas(handler_name="handle"): def handler(arg, key): return key - server_primary = await P2P.create() + server_primary = await P2P.create(bootstrap=False) server_id = server_primary.id await server_primary.add_stream_handler(handler_name, partial(handler, key="primary")) @@ -278,8 +318,10 @@ def handler(arg, key): server_replica2 = await replicate_if_needed(server_primary, True) await server_replica2.add_stream_handler(handler_name + "2", partial(handler, key="replica2")) - client = await P2P.create() - await asyncio.sleep(2) + nodes = boostrap_from([server_primary]) + client = await P2P.create(bootstrap=True, boostrap_peers=nodes) + await client.wait_peers_at_least(1) + result = await client.call_peer_handler(server_id, handler_name, "") assert result == "primary" @@ -293,9 +335,9 @@ def handler(arg, key): await server_replica2.stop_listening() # Primary does not handle replicas protocols - with pytest.raises(P2P.IncompleteRead): + with pytest.raises(asyncio.exceptions.IncompleteReadError): await client.call_peer_handler(server_id, handler_name + "1", "") - with pytest.raises(P2P.IncompleteRead): + with pytest.raises(asyncio.exceptions.IncompleteReadError): await client.call_peer_handler(server_id, handler_name + "2", "") await server_primary.stop_listening() From 462bf2ba2557d644bd358e2ba82493af904e0e32 Mon Sep 17 00:00:00 2001 From: Maxim Kashirin Date: Fri, 23 Apr 2021 13:20:07 +0300 Subject: [PATCH 65/81] asyncio.IncompleteReadError --- hivemind/p2p/p2p_daemon.py | 4 ++-- tests/test_p2p_daemon.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/hivemind/p2p/p2p_daemon.py b/hivemind/p2p/p2p_daemon.py index b15b932b0..856c53f75 100644 --- a/hivemind/p2p/p2p_daemon.py +++ b/hivemind/p2p/p2p_daemon.py @@ -191,7 +191,7 @@ def _handle_stream(handle): async def do_handle_stream(stream_info, reader, writer): try: request = await P2P.receive_data(reader) - except asyncio.exceptions.IncompleteReadError: + except asyncio.IncompleteReadError: logger.debug("Incomplete read while receiving request from peer") writer.close() return @@ -218,7 +218,7 @@ async def do_handle_unary_stream( try: try: request = await P2P.receive_protobuf(in_proto_type, reader) - except asyncio.exceptions.IncompleteReadError: + except asyncio.IncompleteReadError: logger.debug("Incomplete read while receiving request from peer") return except google.protobuf.message.DecodeError as error: diff --git a/tests/test_p2p_daemon.py b/tests/test_p2p_daemon.py index 4a09e7496..c20bd6447 100644 --- a/tests/test_p2p_daemon.py +++ b/tests/test_p2p_daemon.py @@ -335,9 +335,9 @@ def handler(arg, key): await server_replica2.stop_listening() # Primary does not handle replicas protocols - with pytest.raises(asyncio.exceptions.IncompleteReadError): + with pytest.raises(asyncio.IncompleteReadError): await client.call_peer_handler(server_id, handler_name + "1", "") - with pytest.raises(asyncio.exceptions.IncompleteReadError): + with pytest.raises(asyncio.IncompleteReadError): await client.call_peer_handler(server_id, handler_name + "2", "") await server_primary.stop_listening() From b2d9aa2cad6b6b4fbd7a115f66450ce6093fbe71 Mon Sep 17 00:00:00 2001 From: Maxim Kashirin Date: Tue, 27 Apr 2021 09:59:47 +0300 Subject: [PATCH 66/81] pr fixes: messagepack serialization, naming, etc --- hivemind/p2p/p2p_daemon.py | 20 ++-- hivemind/p2p/p2p_daemon_bindings/keys.py | 4 +- hivemind/proto/crypto.proto | 4 +- tests/test_p2p_daemon.py | 124 +++++++++++++++++------ tests/test_p2p_daemon_bindings.py | 24 ++--- 5 files changed, 120 insertions(+), 56 deletions(-) diff --git a/hivemind/p2p/p2p_daemon.py b/hivemind/p2p/p2p_daemon.py index 856c53f75..c4fba0ab2 100644 --- a/hivemind/p2p/p2p_daemon.py +++ b/hivemind/p2p/p2p_daemon.py @@ -1,7 +1,6 @@ import asyncio import copy import dataclasses -import pickle import subprocess import typing as tp from pathlib import Path @@ -11,6 +10,7 @@ import hivemind.p2p.p2p_daemon_bindings.p2pclient as p2pclient from hivemind.p2p.p2p_daemon_bindings.datastructures import ID, StreamInfo +from hivemind.utils import MSGPackSerializer from hivemind.utils.logging import get_logger from hivemind.utils.networking import find_open_port @@ -111,12 +111,11 @@ async def replicate(cls, daemon_listen_port: int, host_port: int): await self._identify_client(0) return self - async def wait_peers_at_least(self, peers_num, attempts=3): - while attempts: + async def wait_for_at_least_n_peers(self, n_peers, attempts=3): + for _ in range(attempts): peers = await self._client.list_peers() - if len(peers) >= peers_num: + if len(peers) >= n_peers: return - attempts -= 1 await asyncio.sleep(1) raise RuntimeError('Not enough peers') @@ -159,13 +158,14 @@ async def send_raw_data(byte_str, writer): @staticmethod async def send_data(data, writer): - await P2P.send_raw_data(pickle.dumps(data), writer) + raw_data = MSGPackSerializer.dumps(data) + await P2P.send_raw_data(raw_data, writer) @staticmethod async def send_protobuf(protobuf, out_proto_type, writer): if type(protobuf) != out_proto_type: error = TypeError('Unary handler returned protobuf of wrong type.') - await P2P.send_raw_data(pickle.dumps(error), writer) + await P2P.send_raw_data(MSGPackSerializer.dumps(error), writer) raise error await P2P.send_raw_data(protobuf.SerializeToString(), writer) @@ -178,7 +178,7 @@ async def receive_raw_data(reader: asyncio.StreamReader): @staticmethod async def receive_data(reader): - return pickle.loads(await P2P.receive_raw_data(reader)) + return MSGPackSerializer.loads(await P2P.receive_raw_data(reader)) @staticmethod async def receive_protobuf(in_proto_type, reader): @@ -199,7 +199,7 @@ async def do_handle_stream(stream_info, reader, writer): result = handle(request) await P2P.send_data(result, writer) except Exception as exc: - await P2P.send_data(exc, writer) + await P2P.send_data(str(exc), writer) finally: writer.close() @@ -234,7 +234,7 @@ async def do_handle_unary_stream( except P2P.InterruptedError: pass except Exception as exc: - await P2P.send_data(exc, writer) + await P2P.send_data(str(exc), writer) finally: pending_task = pending.pop() pending_task.cancel() diff --git a/hivemind/p2p/p2p_daemon_bindings/keys.py b/hivemind/p2p/p2p_daemon_bindings/keys.py index 84a106db0..763c3d76a 100644 --- a/hivemind/p2p/p2p_daemon_bindings/keys.py +++ b/hivemind/p2p/p2p_daemon_bindings/keys.py @@ -1,7 +1,7 @@ """ -Originally taken from: https://github.com/mhchia/py-libp2p-daemon-bindings +Originally taken from: https://github.com/libp2p/py-libp2p Licence: MIT -Author: Kevin Mai-Husan Chia +Author: Kevin Mai-Husan Chia and others """ from abc import ABC, abstractmethod diff --git a/hivemind/proto/crypto.proto b/hivemind/proto/crypto.proto index 1544252ee..e4a69f576 100644 --- a/hivemind/proto/crypto.proto +++ b/hivemind/proto/crypto.proto @@ -1,6 +1,6 @@ -//Originally taken from: https://github.com/mhchia/py-libp2p-daemon-bindings +//Originally taken from: https://github.com/libp2p/py-libp2p //Licence: MIT -//Author: Kevin Mai-Husan Chia +//Author: Kevin Mai-Husan Chia and others syntax = "proto2"; diff --git a/tests/test_p2p_daemon.py b/tests/test_p2p_daemon.py index c20bd6447..b09acc8c0 100644 --- a/tests/test_p2p_daemon.py +++ b/tests/test_p2p_daemon.py @@ -4,6 +4,10 @@ from functools import partial from typing import List +import torch + +from hivemind.utils.compression import serialize_torch_tensor, construct_torch_tensor, deserialize_torch_tensor + from hivemind.p2p.p2p_daemon_bindings.datastructures import ID from hivemind.p2p.p2p_daemon_bindings.datastructures import ID @@ -12,7 +16,7 @@ import pytest from hivemind.p2p import P2P -from hivemind.proto import dht_pb2 +from hivemind.proto import dht_pb2, runtime_pb2 RUNNING = 'running' NOT_RUNNING = 'not running' @@ -27,8 +31,7 @@ def is_process_running(pid: int) -> bool: - cmd = CHECK_PID_CMD.format(pid, RUNNING, NOT_RUNNING) - return subprocess.check_output(cmd, shell=True).decode('utf-8').strip() == RUNNING + return subprocess.run(["ps", "-p", str(pid)], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL).returncode == 0 async def replicate_if_needed(p2p: P2P, replicate: bool): @@ -65,7 +68,7 @@ async def test_server_client_connection(): nodes = boostrap_from([server]) client = await P2P.create(bootstrap=True, boostrap_peers=nodes) - await client.wait_peers_at_least(1) + await client.wait_for_at_least_n_peers(1) peers = await client._client.list_peers() assert len(peers) == 1 @@ -99,6 +102,27 @@ def handle_add(args): return result +def handle_square_torch(x): + tensor = runtime_pb2.Tensor() + tensor.ParseFromString(x) + tensor = deserialize_torch_tensor(tensor) + result = tensor ** 2 + return serialize_torch_tensor(result).SerializeToString() + + +def handle_add_torch(args): + tensor = runtime_pb2.Tensor() + tensor.ParseFromString(args[0]) + result = deserialize_torch_tensor(tensor) + + for i in range(1, len(args)): + tensor = runtime_pb2.Tensor() + tensor.ParseFromString(args[i]) + result = result + deserialize_torch_tensor(tensor) + + return serialize_torch_tensor(result).SerializeToString() + + @pytest.mark.parametrize( 'should_cancel,replicate', [ (True, False), @@ -142,7 +166,7 @@ async def ping_handler(request, context): peer=dht_pb2.NodeInfo(node_id=server.id.encode(), rpc_port=server._host_port), sender_endpoint=handle_name, available=True) - await client.wait_peers_at_least(1) + await client.wait_for_at_least_n_peers(1) libp2p_server_id = ID.from_base58(server.id) stream_info, reader, writer = await client._client.stream_open(libp2p_server_id, (handle_name,)) @@ -186,7 +210,7 @@ async def test_call_peer_single_process(test_input, handle, handler_name="handle client_pid = client._child.pid assert is_process_running(client_pid) - await client.wait_peers_at_least(1) + await client.wait_for_at_least_n_peers(1) result = await client.call_peer_handler(server.id, handler_name, test_input) assert result == handle(test_input) @@ -222,7 +246,7 @@ def server_target(handler_name, server_side, client_side, response_received): @pytest.mark.asyncio async def test_call_peer_different_processes(): handler_name = "square" - test_input = np.random.randn(2, 3) + test_input = 2 server_side, client_side = mp.Pipe() response_received = mp.Value(np.ctypeslib.as_ctypes_type(np.int32)) @@ -239,10 +263,10 @@ async def test_call_peer_different_processes(): client_pid = client._child.pid assert is_process_running(client_pid) - await client.wait_peers_at_least(1) + await client.wait_for_at_least_n_peers(1) result = await client.call_peer_handler(peer_id, handler_name, test_input) - assert np.allclose(result, handle_square(test_input)) + assert np.allclose(result, test_input ** 2) response_received.value = 1 await client.shutdown() @@ -252,32 +276,67 @@ async def test_call_peer_different_processes(): @pytest.mark.parametrize( - "test_input,handle,replicate", + "test_input,expected", [ - pytest.param(np.random.randn(2, 3), handle_square, False, id="square_primary"), - pytest.param(np.random.randn(2, 3), handle_square, True, id="square_replica"), - pytest.param([np.random.randn(2, 3), np.random.randn(2, 3)], handle_add, False, id="add_primary"), - pytest.param([np.random.randn(2, 3), np.random.randn(2, 3)], handle_add, True, id="add_replica"), + pytest.param(torch.tensor([2]), torch.tensor(4)), + pytest.param( + torch.tensor([[1.0, 2.0], [0.5, 0.1]]), + torch.tensor([[1.0, 2.0], [0.5, 0.1]]) ** 2), ] ) @pytest.mark.asyncio -async def test_call_peer_numpy(test_input, handle, replicate, handler_name="handle"): - server_primary = await P2P.create() - server = await replicate_if_needed(server_primary, replicate) +async def test_call_peer_torch_square(test_input, expected, handler_name="handle"): + handle = handle_square_torch + server = await P2P.create() await server.add_stream_handler(handler_name, handle) nodes = boostrap_from([server]) - client_primary = await P2P.create(bootstrap=True, boostrap_peers=nodes) - client = await replicate_if_needed(client_primary, replicate) + client = await P2P.create(bootstrap=True, boostrap_peers=nodes) - await client.wait_peers_at_least(1) + await client.wait_for_at_least_n_peers(1) - result = await client.call_peer_handler(server.id, handler_name, test_input) - assert np.allclose(result, handle(test_input)) + inp = serialize_torch_tensor(test_input).SerializeToString() + resultPb = await client.call_peer_handler(server.id, handler_name, inp) + result = runtime_pb2.Tensor() + result.ParseFromString(resultPb) + result = deserialize_torch_tensor(result) + assert torch.allclose(result, expected) await server.stop_listening() - await server_primary.shutdown() - await client_primary.shutdown() + await server.shutdown() + await client.shutdown() + + +@pytest.mark.parametrize( + "test_input,expected", + [ + pytest.param([torch.tensor([1]), torch.tensor([2])], torch.tensor([3])), + pytest.param( + [torch.tensor([[0.1, 0.2], [0.3, 0.4]]), torch.tensor([[1.1, 1.2], [1.3, 1.4]])], + torch.tensor([[1.2, 1.4], [1.6, 1.8]])), + ] +) +@pytest.mark.asyncio +async def test_call_peer_torch_add(test_input, expected, handler_name="handle"): + handle = handle_add_torch + server = await P2P.create() + await server.add_stream_handler(handler_name, handle) + + nodes = boostrap_from([server]) + client = await P2P.create(bootstrap=True, boostrap_peers=nodes) + + await client.wait_for_at_least_n_peers(1) + + inp = [serialize_torch_tensor(i).SerializeToString() for i in test_input] + resultPb = await client.call_peer_handler(server.id, handler_name, inp) + result = runtime_pb2.Tensor() + result.ParseFromString(resultPb) + result = deserialize_torch_tensor(result) + assert torch.allclose(result, expected) + + await server.stop_listening() + await server.shutdown() + await client.shutdown() @pytest.mark.parametrize( @@ -291,16 +350,21 @@ async def test_call_peer_numpy(test_input, handle, replicate, handler_name="hand async def test_call_peer_error(replicate, handler_name="handle"): server_primary = await P2P.create() server = await replicate_if_needed(server_primary, replicate) - await server.add_stream_handler(handler_name, handle_add) + await server.add_stream_handler(handler_name, handle_add_torch) nodes = boostrap_from([server]) client_primary = await P2P.create(bootstrap=True, boostrap_peers=nodes) client = await replicate_if_needed(client_primary, replicate) - await client.wait_peers_at_least(1) - result = await client.call_peer_handler(server.id, handler_name, - [np.zeros((2, 3)), np.zeros((3, 2))]) - assert type(result) == ValueError + await client.wait_for_at_least_n_peers(1) + + inp = [serialize_torch_tensor(i).SerializeToString() for i in [torch.zeros((2, 3)), torch.zeros((3, 2))]] + result = await client.call_peer_handler(server.id, handler_name, inp) + assert type(result) == str + + await server.stop_listening() + await server_primary.shutdown() + await client_primary.shutdown() @pytest.mark.asyncio @@ -320,7 +384,7 @@ def handler(arg, key): nodes = boostrap_from([server_primary]) client = await P2P.create(bootstrap=True, boostrap_peers=nodes) - await client.wait_peers_at_least(1) + await client.wait_for_at_least_n_peers(1) result = await client.call_peer_handler(server_id, handler_name, "") assert result == "primary" diff --git a/tests/test_p2p_daemon_bindings.py b/tests/test_p2p_daemon_bindings.py index e9bc77213..9b0b1f966 100644 --- a/tests/test_p2p_daemon_bindings.py +++ b/tests/test_p2p_daemon_bindings.py @@ -35,7 +35,7 @@ def test_raise_if_failed_not_raises(): raise_if_failed(resp) -pairs_int_varint_valid = ( +pairs_int_serialized_valid = ( (0, b"\x00"), (1, b"\x01"), (128, b"\x80\x01"), @@ -43,7 +43,7 @@ def test_raise_if_failed_not_raises(): (2 ** 64 - 1, b"\xff\xff\xff\xff\xff\xff\xff\xff\xff\x01"), ) -pairs_int_varint_overflow = ( +pairs_int_serialized_overflow = ( (2 ** 64, b"\x80\x80\x80\x80\x80\x80\x80\x80\x80\x02"), (2 ** 64 + 1, b"\x81\x80\x80\x80\x80\x80\x80\x80\x80\x02"), ( @@ -67,15 +67,15 @@ class MockReaderWriter(MockReader, MockWriter): pass -@pytest.mark.parametrize("integer, var_integer", pairs_int_varint_valid) +@pytest.mark.parametrize("integer, serialized_integer", pairs_int_serialized_valid) @pytest.mark.asyncio -async def test_write_unsigned_varint(integer, var_integer): +async def test_write_unsigned_varint(integer, serialized_integer): s = MockWriter() await write_unsigned_varint(s, integer) - assert s.getvalue() == var_integer + assert s.getvalue() == serialized_integer -@pytest.mark.parametrize("integer", tuple(i[0] for i in pairs_int_varint_overflow)) +@pytest.mark.parametrize("integer", tuple(i[0] for i in pairs_int_serialized_overflow)) @pytest.mark.asyncio async def test_write_unsigned_varint_overflow(integer): s = MockWriter() @@ -91,18 +91,18 @@ async def test_write_unsigned_varint_negative(integer): await write_unsigned_varint(s, integer) -@pytest.mark.parametrize("integer, var_integer", pairs_int_varint_valid) +@pytest.mark.parametrize("integer, serialized_integer", pairs_int_serialized_valid) @pytest.mark.asyncio -async def test_read_unsigned_varint(integer, var_integer): - s = MockReader(var_integer) +async def test_read_unsigned_varint(integer, serialized_integer): + s = MockReader(serialized_integer) result = await read_unsigned_varint(s) assert result == integer -@pytest.mark.parametrize("var_integer", tuple(i[1] for i in pairs_int_varint_overflow)) +@pytest.mark.parametrize("serialized_integer", tuple(i[1] for i in pairs_int_serialized_overflow)) @pytest.mark.asyncio -async def test_read_unsigned_varint_overflow(var_integer): - s = MockReader(var_integer) +async def test_read_unsigned_varint_overflow(serialized_integer): + s = MockReader(serialized_integer) with pytest.raises(ValueError): await read_unsigned_varint(s) From 4b859904f83142a6b1cb79312ebf4a02c1487f79 Mon Sep 17 00:00:00 2001 From: Maxim Kashirin Date: Tue, 27 Apr 2021 17:50:40 +0300 Subject: [PATCH 67/81] remove unused constants --- tests/test_p2p_daemon.py | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/tests/test_p2p_daemon.py b/tests/test_p2p_daemon.py index b09acc8c0..7581d8b73 100644 --- a/tests/test_p2p_daemon.py +++ b/tests/test_p2p_daemon.py @@ -18,17 +18,6 @@ from hivemind.p2p import P2P from hivemind.proto import dht_pb2, runtime_pb2 -RUNNING = 'running' -NOT_RUNNING = 'not running' -CHECK_PID_CMD = ''' -if ps -p {0} > /dev/null; -then - echo "{1}" -else - echo "{2}" -fi -''' - def is_process_running(pid: int) -> bool: return subprocess.run(["ps", "-p", str(pid)], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL).returncode == 0 From fbe5c9ac778be339226dd0aa71687008d4720a66 Mon Sep 17 00:00:00 2001 From: Maxim Kashirin Date: Tue, 27 Apr 2021 17:53:01 +0300 Subject: [PATCH 68/81] remove unused import --- tests/test_p2p_daemon.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_p2p_daemon.py b/tests/test_p2p_daemon.py index 7581d8b73..1432ef2fc 100644 --- a/tests/test_p2p_daemon.py +++ b/tests/test_p2p_daemon.py @@ -6,7 +6,7 @@ import torch -from hivemind.utils.compression import serialize_torch_tensor, construct_torch_tensor, deserialize_torch_tensor +from hivemind.utils.compression import serialize_torch_tensor, deserialize_torch_tensor from hivemind.p2p.p2p_daemon_bindings.datastructures import ID From 19457cfdaa01e67ddd9b1d245fb424d32fc0cd96 Mon Sep 17 00:00:00 2001 From: Maxim Kashirin Date: Wed, 28 Apr 2021 16:18:38 +0300 Subject: [PATCH 69/81] stream handler operates with bytes, unary handler works with errors --- hivemind/p2p/p2p_daemon.py | 61 +++++++++++++++----- hivemind/proto/p2pd.proto | 4 ++ tests/test_p2p_daemon.py | 112 +++++++++++++++++++++++++++---------- 3 files changed, 131 insertions(+), 46 deletions(-) diff --git a/hivemind/p2p/p2p_daemon.py b/hivemind/p2p/p2p_daemon.py index c4fba0ab2..a31cbdffa 100644 --- a/hivemind/p2p/p2p_daemon.py +++ b/hivemind/p2p/p2p_daemon.py @@ -10,6 +10,7 @@ import hivemind.p2p.p2p_daemon_bindings.p2pclient as p2pclient from hivemind.p2p.p2p_daemon_bindings.datastructures import ID, StreamInfo +from hivemind.proto import p2pd_pb2 from hivemind.utils import MSGPackSerializer from hivemind.utils.logging import get_logger from hivemind.utils.networking import find_open_port @@ -49,6 +50,9 @@ class P2P(object): RETRY_DELAY = 0.4 HEADER_LEN = 8 BYTEORDER = 'big' + PB_HEADER_LEN = 1 + RESULT_MESSAGE = int(0).to_bytes(PB_HEADER_LEN, BYTEORDER) + ERROR_MESSAGE = int(1).to_bytes(PB_HEADER_LEN, BYTEORDER) class InterruptedError(Exception): pass @@ -157,27 +161,41 @@ async def send_raw_data(byte_str, writer): writer.write(request) @staticmethod - async def send_data(data, writer): + async def send_message_pack(data, writer): raw_data = MSGPackSerializer.dumps(data) await P2P.send_raw_data(raw_data, writer) @staticmethod async def send_protobuf(protobuf, out_proto_type, writer): if type(protobuf) != out_proto_type: - error = TypeError('Unary handler returned protobuf of wrong type.') - await P2P.send_raw_data(MSGPackSerializer.dumps(error), writer) - raise error + raise TypeError('Unary handler returned protobuf of wrong type.') await P2P.send_raw_data(protobuf.SerializeToString(), writer) @staticmethod - async def receive_raw_data(reader: asyncio.StreamReader): - header = await reader.readexactly(P2P.HEADER_LEN) + async def send_protobuf_with_error(protobuf, out_proto_type, writer): + if type(protobuf) != out_proto_type: + raise TypeError('Unary handler returned protobuf of wrong type.') + if out_proto_type == p2pd_pb2.P2PRPCError: + await P2P.send_raw_data(P2P.ERROR_MESSAGE, writer) + else: + await P2P.send_raw_data(P2P.RESULT_MESSAGE, writer) + + await P2P.send_raw_data(protobuf.SerializeToString(), writer) + + @staticmethod + async def send_error_protobuf(protobuf, out_proto_type, writer): + await P2P.send_raw_data(P2P.RESULT_MESSAGE, writer) + await P2P.send_raw_data(protobuf.SerializeToString(), writer) + + @staticmethod + async def receive_raw_data(reader: asyncio.StreamReader, header_len=HEADER_LEN): + header = await reader.readexactly(header_len) content_length = int.from_bytes(header, P2P.BYTEORDER) data = await reader.readexactly(content_length) return data @staticmethod - async def receive_data(reader): + async def receive_message_pack(reader): return MSGPackSerializer.loads(await P2P.receive_raw_data(reader)) @staticmethod @@ -186,20 +204,32 @@ async def receive_protobuf(in_proto_type, reader): protobuf.ParseFromString(await P2P.receive_raw_data(reader)) return protobuf + @staticmethod + async def receive_protobuf_with_error(in_proto_type, reader): + msg_type = await P2P.receive_raw_data(reader) + if msg_type == P2P.RESULT_MESSAGE: + protobuf = in_proto_type() + protobuf.ParseFromString(await P2P.receive_raw_data(reader)) + return protobuf, None + elif msg_type == P2P.ERROR_MESSAGE: + protobuf = p2pd_pb2.P2PRPCError() + protobuf.ParseFromString(await P2P.receive_raw_data(reader)) + return None, protobuf + else: + raise TypeError('invalid protobuf message type') + @staticmethod def _handle_stream(handle): async def do_handle_stream(stream_info, reader, writer): try: - request = await P2P.receive_data(reader) + request = await P2P.receive_raw_data(reader) # receive raw data except asyncio.IncompleteReadError: logger.debug("Incomplete read while receiving request from peer") writer.close() return try: result = handle(request) - await P2P.send_data(result, writer) - except Exception as exc: - await P2P.send_data(str(exc), writer) + await P2P.send_raw_data(result, writer) finally: writer.close() @@ -230,11 +260,12 @@ async def do_handle_unary_stream( return_when=asyncio.FIRST_COMPLETED) try: result = done.pop().result() - await P2P.send_protobuf(result, out_proto_type, writer) + await P2P.send_protobuf_with_error(result, out_proto_type, writer) except P2P.InterruptedError: pass except Exception as exc: - await P2P.send_data(str(exc), writer) + error = p2pd_pb2.P2PRPCError(message=str(exc)) + await P2P.send_protobuf_with_error(error, p2pd_pb2.P2PRPCError, writer) finally: pending_task = pending.pop() pending_task.cancel() @@ -280,8 +311,8 @@ async def call_peer_handler(self, peer_id, handler_name, input_data): libp2p_peer_id = ID.from_base58(peer_id) stream_info, reader, writer = await self._client.stream_open(libp2p_peer_id, (handler_name,)) try: - await P2P.send_data(input_data, writer) - return await P2P.receive_data(reader) + await P2P.send_raw_data(input_data, writer) + return await P2P.receive_raw_data(reader) finally: writer.close() diff --git a/hivemind/proto/p2pd.proto b/hivemind/proto/p2pd.proto index 8eb3e7e17..f559bcb8d 100644 --- a/hivemind/proto/p2pd.proto +++ b/hivemind/proto/p2pd.proto @@ -160,3 +160,7 @@ message PSResponse { repeated string topics = 1; repeated bytes peerIDs = 2; } + +message P2PRPCError { + required string message = 1; +} \ No newline at end of file diff --git a/tests/test_p2p_daemon.py b/tests/test_p2p_daemon.py index 1432ef2fc..37724b6bb 100644 --- a/tests/test_p2p_daemon.py +++ b/tests/test_p2p_daemon.py @@ -6,6 +6,7 @@ import torch +from hivemind.utils import MSGPackSerializer from hivemind.utils.compression import serialize_torch_tensor, deserialize_torch_tensor from hivemind.p2p.p2p_daemon_bindings.datastructures import ID @@ -81,14 +82,16 @@ async def test_daemon_replica_does_not_affect_primary(): def handle_square(x): - return x ** 2 + x = MSGPackSerializer.loads(x) + return MSGPackSerializer.dumps(x ** 2) def handle_add(args): + args = MSGPackSerializer.loads(args) result = args[0] for i in range(1, len(args)): result = result + args[i] - return result + return MSGPackSerializer.dumps(result) def handle_square_torch(x): @@ -100,6 +103,7 @@ def handle_square_torch(x): def handle_add_torch(args): + args = MSGPackSerializer.loads(args) tensor = runtime_pb2.Tensor() tensor.ParseFromString(args[0]) result = deserialize_torch_tensor(tensor) @@ -112,6 +116,13 @@ def handle_add_torch(args): return serialize_torch_tensor(result).SerializeToString() +def handle_add_torch_with_exc(args): + try: + return handle_add_torch(args) + except: + return b'something went wrong :(' + + @pytest.mark.parametrize( 'should_cancel,replicate', [ (True, False), @@ -159,14 +170,15 @@ async def ping_handler(request, context): libp2p_server_id = ID.from_base58(server.id) stream_info, reader, writer = await client._client.stream_open(libp2p_server_id, (handle_name,)) - await P2P.send_raw_data(ping_request.SerializeToString(), writer) + await P2P.send_protobuf(ping_request, dht_pb2.PingRequest, writer) if should_cancel: writer.close() await asyncio.sleep(1) assert handler_cancelled else: - result = await P2P.receive_protobuf(dht_pb2.PingResponse, reader) + result, err = await P2P.receive_protobuf_with_error(dht_pb2.PingResponse, reader) + assert err is None assert result == expected_response assert not handler_cancelled @@ -178,17 +190,49 @@ async def ping_handler(request, context): assert not is_process_running(client_pid) +@pytest.mark.asyncio +async def test_call_unary_handler_error(handle_name="handle"): + async def error_handler(request, context): + raise ValueError('boom') + + server = await P2P.create() + server_pid = server._child.pid + await server.add_unary_handler(handle_name, error_handler, dht_pb2.PingRequest, dht_pb2.PingResponse) + assert is_process_running(server_pid) + + nodes = boostrap_from([server]) + client = await P2P.create(bootstrap=True, boostrap_peers=nodes) + client_pid = client._child.pid + assert is_process_running(client_pid) + await client.wait_for_at_least_n_peers(1) + + ping_request = dht_pb2.PingRequest( + peer=dht_pb2.NodeInfo(node_id=client.id.encode(), rpc_port=client._host_port), + validate=True) + libp2p_server_id = ID.from_base58(server.id) + stream_info, reader, writer = await client._client.stream_open(libp2p_server_id, (handle_name,)) + + await P2P.send_protobuf(ping_request, dht_pb2.PingRequest, writer) + result, err = await P2P.receive_protobuf_with_error(dht_pb2.PingResponse, reader) + assert result is None + assert err.message == 'boom' + + await server.stop_listening() + await server.shutdown() + await client.shutdown() + + @pytest.mark.parametrize( - "test_input,handle", + "test_input,expected,handle", [ - pytest.param(10, handle_square, id="square_integer"), - pytest.param((1, 2), handle_add, id="add_integers"), - pytest.param(([1, 2, 3], [12, 13]), handle_add, id="add_lists"), - pytest.param(2, lambda x: x ** 3, id="lambda") + pytest.param(10, 100, handle_square, id="square_integer"), + pytest.param((1, 2), 3, handle_add, id="add_integers"), + pytest.param(([1, 2, 3], [12, 13]), [1, 2, 3, 12, 13], handle_add, id="add_lists"), + pytest.param(2, 8, lambda x: MSGPackSerializer.dumps(MSGPackSerializer.loads(x) ** 3), id="lambda") ] ) @pytest.mark.asyncio -async def test_call_peer_single_process(test_input, handle, handler_name="handle"): +async def test_call_peer_single_process(test_input, expected, handle, handler_name="handle"): server = await P2P.create() server_pid = server._child.pid await server.add_stream_handler(handler_name, handle) @@ -201,8 +245,10 @@ async def test_call_peer_single_process(test_input, handle, handler_name="handle await client.wait_for_at_least_n_peers(1) - result = await client.call_peer_handler(server.id, handler_name, test_input) - assert result == handle(test_input) + test_input_msgp = MSGPackSerializer.dumps(test_input) + result_msgp = await client.call_peer_handler(server.id, handler_name, test_input_msgp) + result = MSGPackSerializer.loads(result_msgp) + assert result == expected await server.stop_listening() await server.shutdown() @@ -254,7 +300,9 @@ async def test_call_peer_different_processes(): await client.wait_for_at_least_n_peers(1) - result = await client.call_peer_handler(peer_id, handler_name, test_input) + test_input_msgp = MSGPackSerializer.dumps(2) + result_msgp = await client.call_peer_handler(peer_id, handler_name, test_input_msgp) + result = MSGPackSerializer.loads(result_msgp) assert np.allclose(result, test_input ** 2) response_received.value = 1 @@ -285,9 +333,9 @@ async def test_call_peer_torch_square(test_input, expected, handler_name="handle await client.wait_for_at_least_n_peers(1) inp = serialize_torch_tensor(test_input).SerializeToString() - resultPb = await client.call_peer_handler(server.id, handler_name, inp) + result_pb = await client.call_peer_handler(server.id, handler_name, inp) result = runtime_pb2.Tensor() - result.ParseFromString(resultPb) + result.ParseFromString(result_pb) result = deserialize_torch_tensor(result) assert torch.allclose(result, expected) @@ -317,9 +365,10 @@ async def test_call_peer_torch_add(test_input, expected, handler_name="handle"): await client.wait_for_at_least_n_peers(1) inp = [serialize_torch_tensor(i).SerializeToString() for i in test_input] - resultPb = await client.call_peer_handler(server.id, handler_name, inp) + inp_msgp = MSGPackSerializer.dumps(inp) + result_pb = await client.call_peer_handler(server.id, handler_name, inp_msgp) result = runtime_pb2.Tensor() - result.ParseFromString(resultPb) + result.ParseFromString(result_pb) result = deserialize_torch_tensor(result) assert torch.allclose(result, expected) @@ -339,7 +388,7 @@ async def test_call_peer_torch_add(test_input, expected, handler_name="handle"): async def test_call_peer_error(replicate, handler_name="handle"): server_primary = await P2P.create() server = await replicate_if_needed(server_primary, replicate) - await server.add_stream_handler(handler_name, handle_add_torch) + await server.add_stream_handler(handler_name, handle_add_torch_with_exc) nodes = boostrap_from([server]) client_primary = await P2P.create(bootstrap=True, boostrap_peers=nodes) @@ -348,8 +397,9 @@ async def test_call_peer_error(replicate, handler_name="handle"): await client.wait_for_at_least_n_peers(1) inp = [serialize_torch_tensor(i).SerializeToString() for i in [torch.zeros((2, 3)), torch.zeros((3, 2))]] - result = await client.call_peer_handler(server.id, handler_name, inp) - assert type(result) == str + inp_msgp = MSGPackSerializer.dumps(inp) + result = await client.call_peer_handler(server.id, handler_name, inp_msgp) + assert result == b'something went wrong :(' await server.stop_listening() await server_primary.shutdown() @@ -363,35 +413,35 @@ def handler(arg, key): server_primary = await P2P.create(bootstrap=False) server_id = server_primary.id - await server_primary.add_stream_handler(handler_name, partial(handler, key="primary")) + await server_primary.add_stream_handler(handler_name, partial(handler, key=b'primary')) server_replica1 = await replicate_if_needed(server_primary, True) - await server_replica1.add_stream_handler(handler_name + "1", partial(handler, key="replica1")) + await server_replica1.add_stream_handler(handler_name + '1', partial(handler, key=b'replica1')) server_replica2 = await replicate_if_needed(server_primary, True) - await server_replica2.add_stream_handler(handler_name + "2", partial(handler, key="replica2")) + await server_replica2.add_stream_handler(handler_name + '2', partial(handler, key=b'replica2')) nodes = boostrap_from([server_primary]) client = await P2P.create(bootstrap=True, boostrap_peers=nodes) await client.wait_for_at_least_n_peers(1) - result = await client.call_peer_handler(server_id, handler_name, "") - assert result == "primary" + result = await client.call_peer_handler(server_id, handler_name, b'1') + assert result == b"primary" - result = await client.call_peer_handler(server_id, handler_name + "1", "") - assert result == "replica1" + result = await client.call_peer_handler(server_id, handler_name + '1', b'2') + assert result == b"replica1" - result = await client.call_peer_handler(server_id, handler_name + "2", "") - assert result == "replica2" + result = await client.call_peer_handler(server_id, handler_name + '2', b'3') + assert result == b"replica2" await server_replica1.stop_listening() await server_replica2.stop_listening() # Primary does not handle replicas protocols with pytest.raises(asyncio.IncompleteReadError): - await client.call_peer_handler(server_id, handler_name + "1", "") + await client.call_peer_handler(server_id, handler_name + '1', b'') with pytest.raises(asyncio.IncompleteReadError): - await client.call_peer_handler(server_id, handler_name + "2", "") + await client.call_peer_handler(server_id, handler_name + '2', b'') await server_primary.stop_listening() await server_primary.shutdown() From d8104ac869ba97a83c975898ede7772ed0792406 Mon Sep 17 00:00:00 2001 From: Maxim Kashirin Date: Wed, 28 Apr 2021 17:32:27 +0300 Subject: [PATCH 70/81] fix pr comments --- hivemind/p2p/p2p_daemon.py | 25 ++++++++++--------------- hivemind/proto/p2pd.proto | 2 +- tests/test_p2p_daemon.py | 6 +++--- 3 files changed, 14 insertions(+), 19 deletions(-) diff --git a/hivemind/p2p/p2p_daemon.py b/hivemind/p2p/p2p_daemon.py index a31cbdffa..81b5942ba 100644 --- a/hivemind/p2p/p2p_daemon.py +++ b/hivemind/p2p/p2p_daemon.py @@ -51,8 +51,8 @@ class P2P(object): HEADER_LEN = 8 BYTEORDER = 'big' PB_HEADER_LEN = 1 - RESULT_MESSAGE = int(0).to_bytes(PB_HEADER_LEN, BYTEORDER) - ERROR_MESSAGE = int(1).to_bytes(PB_HEADER_LEN, BYTEORDER) + RESULT_MESSAGE = b'\x00' + ERROR_MESSAGE = b'\x01' class InterruptedError(Exception): pass @@ -172,21 +172,16 @@ async def send_protobuf(protobuf, out_proto_type, writer): await P2P.send_raw_data(protobuf.SerializeToString(), writer) @staticmethod - async def send_protobuf_with_error(protobuf, out_proto_type, writer): + async def send_protobuf_or_error(protobuf, out_proto_type, writer): if type(protobuf) != out_proto_type: raise TypeError('Unary handler returned protobuf of wrong type.') - if out_proto_type == p2pd_pb2.P2PRPCError: + if out_proto_type == p2pd_pb2.RPCError: await P2P.send_raw_data(P2P.ERROR_MESSAGE, writer) else: await P2P.send_raw_data(P2P.RESULT_MESSAGE, writer) await P2P.send_raw_data(protobuf.SerializeToString(), writer) - @staticmethod - async def send_error_protobuf(protobuf, out_proto_type, writer): - await P2P.send_raw_data(P2P.RESULT_MESSAGE, writer) - await P2P.send_raw_data(protobuf.SerializeToString(), writer) - @staticmethod async def receive_raw_data(reader: asyncio.StreamReader, header_len=HEADER_LEN): header = await reader.readexactly(header_len) @@ -205,14 +200,14 @@ async def receive_protobuf(in_proto_type, reader): return protobuf @staticmethod - async def receive_protobuf_with_error(in_proto_type, reader): + async def receive_protobuf_or_error(in_proto_type, reader): msg_type = await P2P.receive_raw_data(reader) if msg_type == P2P.RESULT_MESSAGE: protobuf = in_proto_type() protobuf.ParseFromString(await P2P.receive_raw_data(reader)) return protobuf, None elif msg_type == P2P.ERROR_MESSAGE: - protobuf = p2pd_pb2.P2PRPCError() + protobuf = p2pd_pb2.RPCError() protobuf.ParseFromString(await P2P.receive_raw_data(reader)) return None, protobuf else: @@ -222,7 +217,7 @@ async def receive_protobuf_with_error(in_proto_type, reader): def _handle_stream(handle): async def do_handle_stream(stream_info, reader, writer): try: - request = await P2P.receive_raw_data(reader) # receive raw data + request = await P2P.receive_raw_data(reader) except asyncio.IncompleteReadError: logger.debug("Incomplete read while receiving request from peer") writer.close() @@ -260,12 +255,12 @@ async def do_handle_unary_stream( return_when=asyncio.FIRST_COMPLETED) try: result = done.pop().result() - await P2P.send_protobuf_with_error(result, out_proto_type, writer) + await P2P.send_protobuf_or_error(result, out_proto_type, writer) except P2P.InterruptedError: pass except Exception as exc: - error = p2pd_pb2.P2PRPCError(message=str(exc)) - await P2P.send_protobuf_with_error(error, p2pd_pb2.P2PRPCError, writer) + error = p2pd_pb2.RPCError(message=str(exc)) + await P2P.send_protobuf_or_error(error, p2pd_pb2.RPCError, writer) finally: pending_task = pending.pop() pending_task.cancel() diff --git a/hivemind/proto/p2pd.proto b/hivemind/proto/p2pd.proto index f559bcb8d..dc65514e5 100644 --- a/hivemind/proto/p2pd.proto +++ b/hivemind/proto/p2pd.proto @@ -161,6 +161,6 @@ message PSResponse { repeated bytes peerIDs = 2; } -message P2PRPCError { +message RPCError { required string message = 1; } \ No newline at end of file diff --git a/tests/test_p2p_daemon.py b/tests/test_p2p_daemon.py index 37724b6bb..d5ea6df8c 100644 --- a/tests/test_p2p_daemon.py +++ b/tests/test_p2p_daemon.py @@ -119,7 +119,7 @@ def handle_add_torch(args): def handle_add_torch_with_exc(args): try: return handle_add_torch(args) - except: + except Exception: return b'something went wrong :(' @@ -177,7 +177,7 @@ async def ping_handler(request, context): await asyncio.sleep(1) assert handler_cancelled else: - result, err = await P2P.receive_protobuf_with_error(dht_pb2.PingResponse, reader) + result, err = await P2P.receive_protobuf_or_error(dht_pb2.PingResponse, reader) assert err is None assert result == expected_response assert not handler_cancelled @@ -213,7 +213,7 @@ async def error_handler(request, context): stream_info, reader, writer = await client._client.stream_open(libp2p_server_id, (handle_name,)) await P2P.send_protobuf(ping_request, dht_pb2.PingRequest, writer) - result, err = await P2P.receive_protobuf_with_error(dht_pb2.PingResponse, reader) + result, err = await P2P.receive_protobuf_or_error(dht_pb2.PingResponse, reader) assert result is None assert err.message == 'boom' From 5feb5f83174690701482e9ddba0a45d3bf758a88 Mon Sep 17 00:00:00 2001 From: Denis Mazur Date: Sun, 2 May 2021 03:57:24 +0300 Subject: [PATCH 71/81] fix setup.py --- setup.py | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/setup.py b/setup.py index 923c3ebb6..5e34dd4cd 100644 --- a/setup.py +++ b/setup.py @@ -3,6 +3,7 @@ import hashlib import os import re +import shlex import subprocess import tarfile import tempfile @@ -16,6 +17,8 @@ P2PD_VERSION = 'v0.3.1' P2PD_CHECKSUM = '8810097959db720208cdc9f2945804a4' +LIBP2P_TAR_URL = f'https://github.com/learning-at-home/go-libp2p-daemon/archive/refs/tags/{P2PD_VERSION}.tar.gz' + here = os.path.abspath(os.path.dirname(__file__)) @@ -73,20 +76,18 @@ def libp2p_build_install(): raise FileNotFoundError('could not find golang installation') with tempfile.TemporaryDirectory() as tempdir: - url = f'https://github.com/learning-at-home/go-libp2p-daemon/archive/refs/tags/{P2PD_VERSION}.tar.gz' dest = os.path.join(tempdir, 'libp2p-daemon.tar.gz') - urllib.request.urlretrieve(url, dest) + urllib.request.urlretrieve(LIBP2P_TAR_URL, dest) - tar = tarfile.open(dest, 'r:gz') - tar.extractall(tempdir) - tar.close() + with tarfile.open(dest, 'r:gz') as tar: + tar.extractall(tempdir) - result = subprocess.run(['go', 'build', '-o', os.path.join(here, "hivemind", "hivemind_cli", "p2pd")], - cwd=os.path.join(tempdir, f'go-libp2p-daemon-{P2PD_VERSION[1:]}', 'p2pd')) + result = subprocess.run(f'go build -o {shlex.quote(os.path.join(here, "hivemind", "hivemind_cli", "p2pd"))}', + cwd=os.path.join(tempdir, f'go-libp2p-daemon-{P2PD_VERSION[1:]}', 'p2pd'), shell=True) if result.returncode: raise RuntimeError('Failed to build or install libp2p-daemon:' - f' exited with status code :{result.returncode}') + f' exited with status code: {result.returncode}') def libp2p_download_install(): @@ -97,7 +98,6 @@ def libp2p_download_install(): url = f'https://github.com/learning-at-home/go-libp2p-daemon/releases/download/{P2PD_VERSION}/p2pd' urllib.request.urlretrieve(url, binary_path) os.chmod(binary_path, 0o777) - print(md5(binary_path)) if md5(binary_path) != P2PD_CHECKSUM: raise RuntimeError(f'Downloaded p2pd binary from {url} does not match with md5 checksum') @@ -105,7 +105,7 @@ def libp2p_download_install(): class Install(install): def run(self): libp2p_download_install() - proto_compile(os.path.join(self.build_lib, 'hivemind', 'proto')) + # proto_compile(os.path.join(self.build_lib, 'hivemind', 'proto')) super().run() From 6b0a6448c63127e09bf69e1567a449175d71cc21 Mon Sep 17 00:00:00 2001 From: Denis Mazur Date: Sun, 2 May 2021 04:19:12 +0300 Subject: [PATCH 72/81] replace popen with subprocess.run --- setup.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/setup.py b/setup.py index 5e34dd4cd..1d526c3fa 100644 --- a/setup.py +++ b/setup.py @@ -63,9 +63,7 @@ def proto_compile(output_path): def libp2p_build_install(): try: - proc = subprocess.Popen(['go', 'version'], stdout=subprocess.PIPE) - result, _ = proc.communicate() - result = result.decode('ascii', 'replace') + result = subprocess.run("go version", capture_output=True).stdout.decode('ascii', 'replace') m = re.search(r'^go version go([\d.]+)', result) v = m.group(1) From fe0032ea0a79f15fad4424f202b407da9a831ae5 Mon Sep 17 00:00:00 2001 From: Denis Mazur Date: Sun, 2 May 2021 04:28:48 +0300 Subject: [PATCH 73/81] fix setup.py --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 1d526c3fa..67ced08f6 100644 --- a/setup.py +++ b/setup.py @@ -63,7 +63,7 @@ def proto_compile(output_path): def libp2p_build_install(): try: - result = subprocess.run("go version", capture_output=True).stdout.decode('ascii', 'replace') + result = subprocess.run("go version", capture_output=True, shell=True).stdout.decode('ascii', 'replace') m = re.search(r'^go version go([\d.]+)', result) v = m.group(1) From b5a358c33104e1d3f2623d8d8e66f2542a5648fa Mon Sep 17 00:00:00 2001 From: Denis Mazur Date: Sun, 2 May 2021 04:31:17 +0300 Subject: [PATCH 74/81] remove debug comment --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 67ced08f6..2723a3f66 100644 --- a/setup.py +++ b/setup.py @@ -103,7 +103,7 @@ def libp2p_download_install(): class Install(install): def run(self): libp2p_download_install() - # proto_compile(os.path.join(self.build_lib, 'hivemind', 'proto')) + proto_compile(os.path.join(self.build_lib, 'hivemind', 'proto')) super().run() From 99feece9dc50ed55be707a4fdbf7f102d4db0a19 Mon Sep 17 00:00:00 2001 From: Maxim Kashirin Date: Mon, 3 May 2021 14:15:11 +0300 Subject: [PATCH 75/81] fix comments in p2p and p2p_bindings --- hivemind/p2p/p2p_daemon.py | 219 ++++++++++-------- hivemind/p2p/p2p_daemon_bindings/control.py | 88 ++++--- .../p2p/p2p_daemon_bindings/datastructures.py | 97 ++++---- hivemind/p2p/p2p_daemon_bindings/p2pclient.py | 10 +- hivemind/p2p/p2p_daemon_bindings/utils.py | 1 - tests/test_p2p_daemon.py | 30 +-- tests/test_p2p_daemon_bindings.py | 34 +-- 7 files changed, 244 insertions(+), 235 deletions(-) diff --git a/hivemind/p2p/p2p_daemon.py b/hivemind/p2p/p2p_daemon.py index 81b5942ba..aa5cbd36c 100644 --- a/hivemind/p2p/p2p_daemon.py +++ b/hivemind/p2p/p2p_daemon.py @@ -1,15 +1,16 @@ import asyncio -import copy -import dataclasses -import subprocess -import typing as tp -from pathlib import Path +from copy import deepcopy +from dataclasses import dataclass +from importlib.resources import path +from subprocess import Popen +from typing import List, Optional import google.protobuf from multiaddr import Multiaddr +import hivemind.hivemind_cli as cli import hivemind.p2p.p2p_daemon_bindings.p2pclient as p2pclient -from hivemind.p2p.p2p_daemon_bindings.datastructures import ID, StreamInfo +from hivemind.p2p.p2p_daemon_bindings.datastructures import PeerID, StreamInfo from hivemind.proto import p2pd_pb2 from hivemind.utils import MSGPackSerializer from hivemind.utils.logging import get_logger @@ -18,12 +19,21 @@ logger = get_logger(__name__) -@dataclasses.dataclass(frozen=False) +P2PD_RELATIVE_PATH = 'hivemind_cli/p2pd' +NUM_RETRIES = 3 +RETRY_DELAY = 0.4 + + +class P2PInterruptedError(Exception): + pass + + +@dataclass(frozen=False) class P2PContext(object): - ours_id: str - ours_port: int + id: str + port: int handle_name: str - peer_id: ID = None + peer_id: PeerID = None peer_addr: Multiaddr = None from hivemind.utils.networking import find_open_port @@ -38,24 +48,27 @@ def __init__(self, ours_id, ours_port, handle_name): self.handle_name = handle_name -class P2P(object): +class P2P: """ Forks a child process and executes p2pd command with given arguments. Can be used for peer to peer communication and procedure calls. Sends SIGKILL to the child in destructor. """ - P2PD_RELATIVE_PATH = 'hivemind_cli/p2pd' - NUM_RETRIES = 3 - RETRY_DELAY = 0.4 HEADER_LEN = 8 BYTEORDER = 'big' PB_HEADER_LEN = 1 RESULT_MESSAGE = b'\x00' ERROR_MESSAGE = b'\x01' - - class InterruptedError(Exception): - pass + DHT_MODE_MAPPING = { + 'dht': {'dht': 1}, + 'dht_server': {'dhtServer': 1}, + 'dht_client': {'dhtClient': 1}, + } + FORCE_REACHABILITY_MAPPING = { + 'public': {'forceReachabilityPublic': 1}, + 'private': {'forceReachabilityPrivate': 1}, + } def __init__(self): self._child = None @@ -64,44 +77,74 @@ def __init__(self): self._server_stopped = asyncio.Event() @classmethod - async def create(cls, *args, quic=1, tls=1, conn_manager=1, dht_mode='dht_server', force_reachability=None, - nat_port_map=True, auto_nat=True, bootstrap=False, boostrap_peers=None, use_global_ipfs=False, - host_port: int = None, daemon_listen_port: int = None, **kwargs): - if bootstrap and boostrap_peers is None and not use_global_ipfs: - raise AttributeError('Trying to create with bootstrap node without bootstrap nodes list. ' - 'It is very dangerous, because p2pd connects to global ipfs and it is very unstable. ' - 'If you really want this, pass use_global_ipfs=True') - if boostrap_peers is not None and use_global_ipfs: - raise AttributeError('Non empty boostrap_nodes and use_global_ipfs=True are incompatible.' - 'Choose one option: your nodes list (preferable) or global ipfs (very unstable)') + async def create(cls, *args, quic: bool = True, tls: bool = True, conn_manager: bool = True, + dht_mode: str = 'dht_server', force_reachability: Optional[str] = None, + nat_port_map: bool = True, auto_nat: bool = True, bootstrap: bool = False, + bootstrap_peers: Optional[List[str]] = None, use_global_ipfs: bool = False, host_port: int = None, + daemon_listen_port: int = None, **kwargs): + """ + Start a new p2pd process and connect to it. + @param args: + @param quic: Enables the QUIC transport + @param tls: Enables TLS1.3 channel security protocol + @param conn_manager: Enables the Connection Manager + @param dht_mode: DHT mode (dht_client/dht_server/dht) + @param force_reachability: Force reachability mode (public/private) + @param nat_port_map: Enables NAT port mapping + @param auto_nat: Enables the AutoNAT service + @param bootstrap: Connects to bootstrap peers and bootstraps the dht if enabled + @param bootstrap_peers: List of bootstrap peers; defaults to the IPFS DHT peers + @param use_global_ipfs: Bootstrap to global ipfs (works only if bootstrap=True and bootstrap_peers=None) + @param host_port: port for p2p network + @param daemon_listen_port: port for connection daemon and client binding + @param kwargs: + @return: new wrapper for p2p daemon + """ + + assert not (bootstrap and bootstrap_peers is None and not use_global_ipfs), \ + 'Trying to create with bootstrap node without bootstrap nodes list. ' \ + 'It is very dangerous, because p2pd connects to global ipfs and it is very unstable. ' \ + 'If you really want this, pass use_global_ipfs=True' + assert not (bootstrap_peers is not None and use_global_ipfs), \ + 'Non empty boostrap_nodes and use_global_ipfs=True are incompatible.' \ + 'Choose one option: your nodes list (preferable) or global ipfs (very unstable)' self = cls() - p2pd_path = Path(__file__).resolve().parents[1] / P2P.P2PD_RELATIVE_PATH - bpeers = cls._make_bootstrap_peers(boostrap_peers) - dht = cls._make_dht_mode(dht_mode) - freachability = cls._make_force_reachability(force_reachability) + with path(cli, 'p2pd') as p: + p2pd_path = p + bootstrap_peers = cls._make_bootstrap_peers(bootstrap_peers) + dht = cls.DHT_MODE_MAPPING.get(dht_mode, {'dht': 0}) + force_reachability = cls.FORCE_REACHABILITY_MAPPING.get(force_reachability, {}) proc_args = self._make_process_args( str(p2pd_path), *args, quic=quic, tls=tls, connManager=conn_manager, natPortMap=nat_port_map, autonat=auto_nat, - b=bootstrap, **{**bpeers, **dht, **freachability, **kwargs}) + b=bootstrap, **{**bootstrap_peers, **dht, **force_reachability, **kwargs}) self._assign_daemon_ports(host_port, daemon_listen_port) - for try_count in range(self.NUM_RETRIES): + + for try_count in range(NUM_RETRIES): try: self._initialize(proc_args) - await self._identify_client(P2P.RETRY_DELAY * (2 ** try_count)) + await self._wait_for_client(RETRY_DELAY * (2 ** try_count)) + break except Exception as e: logger.debug(f"Failed to initialize p2p daemon: {e}") - self._kill_child() - if try_count == P2P.NUM_RETRIES - 1: + self._terminate() + if try_count == NUM_RETRIES - 1: raise self._assign_daemon_ports() - continue - break + return self @classmethod async def replicate(cls, daemon_listen_port: int, host_port: int): + """ + Connect to existing p2p daemon + @param host_port: port for p2p network + @param daemon_listen_port: port for connection daemon and client binding + @return: new wrapper for existing p2p daemon + """ + self = cls() # There is no child under control # Use external already running p2pd @@ -112,48 +155,45 @@ async def replicate(cls, daemon_listen_port: int, host_port: int): self._client = p2pclient.Client( Multiaddr(f'/ip4/127.0.0.1/tcp/{self._daemon_listen_port}'), Multiaddr(f'/ip4/127.0.0.1/tcp/{self._client_listen_port}')) - await self._identify_client(0) + await self._wait_for_client() return self - async def wait_for_at_least_n_peers(self, n_peers, attempts=3): + async def wait_for_at_least_n_peers(self, n_peers, attempts=3, delay=1): for _ in range(attempts): peers = await self._client.list_peers() if len(peers) >= n_peers: return - await asyncio.sleep(1) + await asyncio.sleep(delay) raise RuntimeError('Not enough peers') - def _initialize(self, proc_args: tp.List[str]) -> None: - proc_args = copy.deepcopy(proc_args) + def _initialize(self, proc_args: List[str]) -> None: + proc_args = deepcopy(proc_args) proc_args.extend(self._make_process_args( hostAddrs=f'/ip4/0.0.0.0/tcp/{self._host_port},/ip4/0.0.0.0/udp/{self._host_port}/quic', listen=f'/ip4/127.0.0.1/tcp/{self._daemon_listen_port}' )) - self._child = subprocess.Popen( - args=proc_args, - stdin=subprocess.PIPE, stdout=subprocess.PIPE, - stderr=subprocess.PIPE, encoding="utf8" - ) + self._child = Popen(args=proc_args, encoding="utf8") self._alive = True self._client_listen_port = find_open_port() self._client = p2pclient.Client( Multiaddr(f'/ip4/127.0.0.1/tcp/{self._daemon_listen_port}'), Multiaddr(f'/ip4/127.0.0.1/tcp/{self._client_listen_port}')) - async def _identify_client(self, delay): + async def _wait_for_client(self, delay=0): await asyncio.sleep(delay) encoded = await self._client.identify() self.id = encoded[0].to_base58() def _assign_daemon_ports(self, host_port=None, daemon_listen_port=None): - self._host_port, self._daemon_listen_port = host_port, daemon_listen_port if host_port is None: - self._host_port = find_open_port() + host_port = find_open_port() if daemon_listen_port is None: - self._daemon_listen_port = find_open_port() - while self._daemon_listen_port == self._host_port: - self._daemon_listen_port = find_open_port() + daemon_listen_port = find_open_port() + while daemon_listen_port == host_port: + daemon_listen_port = find_open_port() + + self._host_port, self._daemon_listen_port = host_port, daemon_listen_port @staticmethod async def send_raw_data(byte_str, writer): @@ -161,18 +201,12 @@ async def send_raw_data(byte_str, writer): writer.write(request) @staticmethod - async def send_message_pack(data, writer): + async def send_msgpack(data, writer): raw_data = MSGPackSerializer.dumps(data) await P2P.send_raw_data(raw_data, writer) @staticmethod async def send_protobuf(protobuf, out_proto_type, writer): - if type(protobuf) != out_proto_type: - raise TypeError('Unary handler returned protobuf of wrong type.') - await P2P.send_raw_data(protobuf.SerializeToString(), writer) - - @staticmethod - async def send_protobuf_or_error(protobuf, out_proto_type, writer): if type(protobuf) != out_proto_type: raise TypeError('Unary handler returned protobuf of wrong type.') if out_proto_type == p2pd_pb2.RPCError: @@ -190,17 +224,11 @@ async def receive_raw_data(reader: asyncio.StreamReader, header_len=HEADER_LEN): return data @staticmethod - async def receive_message_pack(reader): + async def receive_msgpack(reader): return MSGPackSerializer.loads(await P2P.receive_raw_data(reader)) @staticmethod async def receive_protobuf(in_proto_type, reader): - protobuf = in_proto_type() - protobuf.ParseFromString(await P2P.receive_raw_data(reader)) - return protobuf - - @staticmethod - async def receive_protobuf_or_error(in_proto_type, reader): msg_type = await P2P.receive_raw_data(reader) if msg_type == P2P.RESULT_MESSAGE: protobuf = in_proto_type() @@ -211,7 +239,7 @@ async def receive_protobuf_or_error(in_proto_type, reader): protobuf.ParseFromString(await P2P.receive_raw_data(reader)) return None, protobuf else: - raise TypeError('invalid protobuf message type') + raise TypeError('Invalid Protobuf message type') @staticmethod def _handle_stream(handle): @@ -234,7 +262,7 @@ async def do_handle_stream(stream_info, reader, writer): def _handle_unary_stream(handle, context, in_proto_type, out_proto_type): async def watchdog(reader: asyncio.StreamReader): await reader.read(n=1) - raise P2P.InterruptedError() + raise P2PInterruptedError() async def do_handle_unary_stream( stream_info: StreamInfo, @@ -247,7 +275,7 @@ async def do_handle_unary_stream( logger.debug("Incomplete read while receiving request from peer") return except google.protobuf.message.DecodeError as error: - logger.warning(repr(error)) + logger.exception(error) return context.peer_id, context.peer_addr = stream_info.peer_id, stream_info.addr @@ -255,12 +283,12 @@ async def do_handle_unary_stream( return_when=asyncio.FIRST_COMPLETED) try: result = done.pop().result() - await P2P.send_protobuf_or_error(result, out_proto_type, writer) - except P2P.InterruptedError: + await P2P.send_protobuf(result, out_proto_type, writer) + except P2PInterruptedError: pass except Exception as exc: error = p2pd_pb2.RPCError(message=str(exc)) - await P2P.send_protobuf_or_error(error, p2pd_pb2.RPCError, writer) + await P2P.send_protobuf(error, p2pd_pb2.RPCError, writer) finally: pending_task = pending.pop() pending_task.cancel() @@ -293,17 +321,17 @@ async def stop_listening(self): async def add_stream_handler(self, name, handle): if self._listen_task is None: self.start_listening() - await self._client.stream_handler(name, P2P._handle_stream(handle)) + await self._client.stream_handler(name, self._handle_stream(handle)) async def add_unary_handler(self, name, handle, in_proto_type, out_proto_type): if self._listen_task is None: self.start_listening() - context = P2PContext(ours_id=self.id, ours_port=self._host_port, handle_name=name) + context = P2PContext(id=self.id, port=self._host_port, handle_name=name) await self._client.stream_handler( name, P2P._handle_unary_stream(handle, context, in_proto_type, out_proto_type)) async def call_peer_handler(self, peer_id, handler_name, input_data): - libp2p_peer_id = ID.from_base58(peer_id) + libp2p_peer_id = PeerID.from_base58(peer_id) stream_info, reader, writer = await self._client.stream_open(libp2p_peer_id, (handler_name,)) try: await P2P.send_raw_data(input_data, writer) @@ -312,52 +340,41 @@ async def call_peer_handler(self, peer_id, handler_name, input_data): writer.close() def __del__(self): - self._kill_child() + self._terminate() @property def is_alive(self): return self._alive - async def shutdown(self, timeout=None): - await asyncio.get_event_loop().run_in_executor(None, self._kill_child) + async def shutdown(self): + await asyncio.get_event_loop().run_in_executor(None, self._terminate) - def _kill_child(self): + def _terminate(self): self._alive = False if self._child is not None and self._child.poll() is None: self._child.kill() self._child.wait() - def _make_process_args(self, *args, **kwargs) -> tp.List[str]: + @staticmethod + def _make_process_args(*args, **kwargs) -> List[str]: proc_args = [] proc_args.extend( str(entry) for entry in args ) proc_args.extend( - f'-{key}={value}' if value is not None else f'-{key}' + f'-{key}={P2P._convert_process_arg_type(value)}' if value is not None else f'-{key}' for key, value in kwargs.items() ) return proc_args + @staticmethod + def _convert_process_arg_type(val): + if isinstance(val, bool): + return 1 if val else 0 + return val + @staticmethod def _make_bootstrap_peers(nodes): if nodes is None: return {} return {'bootstrapPeers': ','.join(nodes)} - - @staticmethod - def _make_dht_mode(dht_mode): - if dht_mode == 'dht': - return {'dht': 1} - if dht_mode == 'dht_server': - return {'dhtServer': 1} - if dht_mode == 'dht_client': - return {'dhtClient': 1} - return {'dht': 0} - - @staticmethod - def _make_force_reachability(force_reachability): - if force_reachability == 'public': - return {'forceReachabilityPublic': 1} - if force_reachability == 'private': - return {'forceReachabilityPrivate': 1} - return {} diff --git a/hivemind/p2p/p2p_daemon_bindings/control.py b/hivemind/p2p/p2p_daemon_bindings/control.py index 014ac674f..ed8425bf6 100644 --- a/hivemind/p2p/p2p_daemon_bindings/control.py +++ b/hivemind/p2p/p2p_daemon_bindings/control.py @@ -4,88 +4,86 @@ Author: Kevin Mai-Husan Chia """ -import logging -from typing import AsyncIterator, Awaitable, Callable, Dict, Iterable, Sequence, Tuple - import asyncio +import logging from contextlib import asynccontextmanager +from typing import (AsyncIterator, Awaitable, Callable, Dict, Iterable, + Sequence, Tuple) + from multiaddr import Multiaddr, protocols -from hivemind.p2p.p2p_daemon_bindings.datastructures import PeerInfo, StreamInfo, ID + +from hivemind.p2p.p2p_daemon_bindings.datastructures import (PeerID, PeerInfo, + StreamInfo) +from hivemind.p2p.p2p_daemon_bindings.utils import (DispatchFailure, + raise_if_failed, + read_pbmsg_safe, + write_pbmsg) from hivemind.proto import p2pd_pb2 as p2pd_pb -from hivemind.p2p.p2p_daemon_bindings.utils import DispatchFailure, read_pbmsg_safe, write_pbmsg, raise_if_failed +from hivemind.utils.logging import get_logger StreamHandler = Callable[[StreamInfo, asyncio.StreamReader, asyncio.StreamWriter], Awaitable[None]] -_supported_conn_protocols = ( +SUPPORT_CONN_PROTOCOLS = ( protocols.P_IP4, # protocols.P_IP6, protocols.P_UNIX, ) +SUPPORTED_PROTOS = ( + protocols.protocol_with_code(proto) for proto in SUPPORT_CONN_PROTOCOLS +) def parse_conn_protocol(maddr: Multiaddr) -> int: proto_codes = set(proto.code for proto in maddr.protocols()) - proto_cand = proto_codes.intersection(_supported_conn_protocols) + proto_cand = proto_codes.intersection(SUPPORT_CONN_PROTOCOLS) if len(proto_cand) != 1: - supported_protos = ( - protocols.protocol_with_code(proto) for proto in _supported_conn_protocols - ) raise ValueError( - f"connection protocol should be only one protocol out of {supported_protos}" + f"connection protocol should be only one protocol out of {SUPPORTED_PROTOS}" f", maddr={maddr}" ) return tuple(proto_cand)[0] class DaemonConnector: - control_maddr: Multiaddr - logger = logging.getLogger("p2pclient.DaemonConnector") DEFAULT_CONTROL_MADDR = "/unix/tmp/p2pd.sock" - def __init__(self, control_maddr: Multiaddr = None) -> None: - if control_maddr is None: - control_maddr = Multiaddr(self.DEFAULT_CONTROL_MADDR) - self.control_maddr = control_maddr + def __init__(self, control_maddr: Multiaddr = Multiaddr(DEFAULT_CONTROL_MADDR)) -> None: + self.control_maddr: Multiaddr = control_maddr + self.proto_code: int = parse_conn_protocol(self.control_maddr) + self.logger = get_logger(__name__) async def open_connection(self) -> (asyncio.StreamReader, asyncio.StreamWriter): - proto_code = parse_conn_protocol(self.control_maddr) - if proto_code == protocols.P_UNIX: + if self.proto_code == protocols.P_UNIX: control_path = self.control_maddr.value_for_protocol(protocols.P_UNIX) self.logger.debug( "DaemonConnector %s opens connection to %s", self, self.control_maddr ) return await asyncio.open_unix_connection(control_path) - elif proto_code == protocols.P_IP4: + elif self.proto_code == protocols.P_IP4: host = self.control_maddr.value_for_protocol(protocols.P_IP4) port = int(self.control_maddr.value_for_protocol(protocols.P_TCP)) return await asyncio.open_connection(host, port) else: raise ValueError( - f"protocol not supported: protocol={protocols.protocol_with_code(proto_code)}" + f"Protocol not supported: {protocols.protocol_with_code(self.proto_code)}" ) class ControlClient: - listen_maddr: Multiaddr - daemon_connector: DaemonConnector - handlers: Dict[str, StreamHandler] - logger = logging.getLogger("p2pclient.ControlClient") DEFAULT_LISTEN_MADDR = "/unix/tmp/p2pclient.sock" - def __init__( - self, daemon_connector: DaemonConnector, listen_maddr: Multiaddr = None - ) -> None: - if listen_maddr is None: - listen_maddr = Multiaddr(self.DEFAULT_LISTEN_MADDR) - self.listen_maddr = listen_maddr - self.daemon_connector = daemon_connector - self.handlers = {} + def __init__(self, daemon_connector: DaemonConnector, + listen_maddr: Multiaddr = Multiaddr(DEFAULT_LISTEN_MADDR)) -> None: + self.listen_maddr: Multiaddr = listen_maddr + self.daemon_connector: DaemonConnector = daemon_connector + self.handlers: Dict[str, StreamHandler] = {} + self.logger = get_logger(__name__) async def _handler(self, reader: asyncio.StreamReader, writer: asyncio.StreamWriter): pb_stream_info = p2pd_pb.StreamInfo() # type: ignore await read_pbmsg_safe(reader, pb_stream_info) - stream_info = StreamInfo.from_pb(pb_stream_info) - self.logger.info("New incoming stream: %s", stream_info) + stream_info = StreamInfo.from_protobuf(pb_stream_info) + self.logger.debug(f"New incoming stream: {stream_info}") try: handler = self.handlers[stream_info.proto] except KeyError as e: @@ -110,14 +108,12 @@ async def listen(self) -> AsyncIterator["ControlClient"]: ) async with server: - self.logger.info( - "DaemonConnector %s starts listening to %s", self, self.listen_maddr - ) + self.logger.info(f"DaemonConnector {self} starts listening to {self.listen_maddr}") yield self - self.logger.info("DaemonConnector %s closed", self) + self.logger.info(f"DaemonConnector {self} closed") - async def identify(self) -> Tuple[ID, Tuple[Multiaddr, ...]]: + async def identify(self) -> Tuple[PeerID, Tuple[Multiaddr, ...]]: reader, writer = await self.daemon_connector.open_connection() req = p2pd_pb.Request(type=p2pd_pb.Request.IDENTIFY) await write_pbmsg(writer, req) @@ -131,11 +127,11 @@ async def identify(self) -> Tuple[ID, Tuple[Multiaddr, ...]]: maddrs_bytes = resp.identify.addrs maddrs = tuple(Multiaddr(maddr_bytes) for maddr_bytes in maddrs_bytes) - peer_id = ID(peer_id_bytes) + peer_id = PeerID(peer_id_bytes) return peer_id, maddrs - async def connect(self, peer_id: ID, maddrs: Iterable[Multiaddr]) -> None: + async def connect(self, peer_id: PeerID, maddrs: Iterable[Multiaddr]) -> None: reader, writer = await self.daemon_connector.open_connection() maddrs_bytes = [i.to_bytes() for i in maddrs] @@ -159,10 +155,10 @@ async def list_peers(self) -> Tuple[PeerInfo, ...]: writer.close() raise_if_failed(resp) - peers = tuple(PeerInfo.from_pb(pinfo) for pinfo in resp.peers) + peers = tuple(PeerInfo.from_protobuf(pinfo) for pinfo in resp.peers) return peers - async def disconnect(self, peer_id: ID) -> None: + async def disconnect(self, peer_id: PeerID) -> None: disconnect_req = p2pd_pb.DisconnectRequest(peer=peer_id.to_bytes()) req = p2pd_pb.Request( type=p2pd_pb.Request.DISCONNECT, disconnect=disconnect_req @@ -175,7 +171,7 @@ async def disconnect(self, peer_id: ID) -> None: raise_if_failed(resp) async def stream_open( - self, peer_id: ID, protocols: Sequence[str] + self, peer_id: PeerID, protocols: Sequence[str] ) -> Tuple[StreamInfo, asyncio.StreamReader, asyncio.StreamWriter]: reader, writer = await self.daemon_connector.open_connection() @@ -192,7 +188,7 @@ async def stream_open( raise_if_failed(resp) pb_stream_info = resp.streamInfo - stream_info = StreamInfo.from_pb(pb_stream_info) + stream_info = StreamInfo.from_protobuf(pb_stream_info) return stream_info, reader, writer diff --git a/hivemind/p2p/p2p_daemon_bindings/datastructures.py b/hivemind/p2p/p2p_daemon_bindings/datastructures.py index edc5ffc7c..224640d2f 100644 --- a/hivemind/p2p/p2p_daemon_bindings/datastructures.py +++ b/hivemind/p2p/p2p_daemon_bindings/datastructures.py @@ -5,7 +5,7 @@ """ import hashlib -from typing import Any, List, Sequence, Union +from typing import Any, List, Optional, Sequence, Union import base58 import multihash @@ -41,54 +41,54 @@ def digest(self) -> bytes: ) -class ID: - _bytes: bytes - _xor_id: int = None - _b58_str: str = None - +class PeerID: def __init__(self, peer_id_bytes: bytes) -> None: self._bytes = peer_id_bytes + self._xor_id: int = int(sha256_digest(self._bytes).hex(), 16) + self._b58_str: str = base58.b58encode(self._bytes).decode() @property def xor_id(self) -> int: - if not self._xor_id: - self._xor_id = int(sha256_digest(self._bytes).hex(), 16) return self._xor_id def to_bytes(self) -> bytes: return self._bytes def to_base58(self) -> str: - if not self._b58_str: - self._b58_str = base58.b58encode(self._bytes).decode() return self._b58_str def __repr__(self) -> str: - return f"" + return f"" + + def __str__(self): + return self.to_base58() + + def pretty(self): + return self.to_base58() - __str__ = pretty = to_string = to_base58 + def to_string(self): + return self.to_base58() def __eq__(self, other: object) -> bool: if isinstance(other, str): return self.to_base58() == other elif isinstance(other, bytes): return self._bytes == other - elif isinstance(other, ID): + elif isinstance(other, PeerID): return self._bytes == other._bytes else: - return NotImplemented + return False def __hash__(self) -> int: return hash(self._bytes) @classmethod - def from_base58(cls, b58_encoded_peer_id_str: str) -> "ID": - peer_id_bytes = base58.b58decode(b58_encoded_peer_id_str) - pid = ID(peer_id_bytes) - return pid + def from_base58(cls, base58_id: str) -> "PeerID": + peer_id_bytes = base58.b58decode(base58_id) + return cls(peer_id_bytes) @classmethod - def from_pubkey(cls, key: PublicKey) -> "ID": + def from_pubkey(cls, key: PublicKey) -> "PeerID": serialized_key = key.serialize() algo = multihash.Func.sha2_256 if ENABLE_INLINING and len(serialized_key) <= MAX_INLINE_KEY_LENGTH: @@ -104,39 +104,32 @@ def sha256_digest(data: Union[str, bytes]) -> bytes: class StreamInfo: - peer_id: ID - addr: Multiaddr - proto: str - - def __init__(self, peer_id: ID, addr: Multiaddr, proto: str) -> None: - self.peer_id = peer_id - self.addr = addr - self.proto = proto + def __init__(self, peer_id: PeerID, addr: Multiaddr, proto: str) -> None: + self.peer_id: PeerID = peer_id + self.addr: Multiaddr = addr + self.proto: str = proto def __repr__(self) -> str: return ( f"" ) - def to_pb(self) -> p2pd_pb2.StreamInfo: + def to_protobuf(self) -> p2pd_pb2.StreamInfo: pb_msg = p2pd_pb2.StreamInfo( peer=self.peer_id.to_bytes(), addr=self.addr.to_bytes(), proto=self.proto ) return pb_msg @classmethod - def from_pb(cls, pb_msg: p2pd_pb2.StreamInfo) -> "StreamInfo": + def from_protobuf(cls, pb_msg: p2pd_pb2.StreamInfo) -> "StreamInfo": stream_info = cls( - peer_id=ID(pb_msg.peer), addr=Multiaddr(pb_msg.addr), proto=pb_msg.proto + peer_id=PeerID(pb_msg.peer), addr=Multiaddr(pb_msg.addr), proto=pb_msg.proto ) return stream_info -class PeerInfoLibP2P: - peer_id: ID - addrs: List[Multiaddr] - - def __init__(self, peer_id: ID, addrs: Sequence[Multiaddr]) -> None: +class PeerInfo: + def __init__(self, peer_id: PeerID, addrs: Sequence[Multiaddr]) -> None: self.peer_id = peer_id self.addrs = list(addrs) @@ -147,9 +140,22 @@ def __eq__(self, other: Any) -> bool: and self.addrs == other.addrs ) + @classmethod + def from_protobuf(cls, peer_info_pb: p2pd_pb2.PeerInfo) -> "PeerInfo": + peer_id = PeerID(peer_info_pb.id) + addrs = [Multiaddr(addr) for addr in peer_info_pb.addrs] + return PeerInfo(peer_id, addrs) + + def __str__(self): + return f"{self.peer_id.pretty()} {','.join(str(a) for a in self.addrs)}" + + +class InvalidAddrError(ValueError): + pass -def info_from_p2p_addr(addr: Multiaddr) -> PeerInfoLibP2P: - if not addr: + +def info_from_p2p_addr(addr: Multiaddr) -> PeerInfo: + if addr is None: raise InvalidAddrError("`addr` should not be `None`") parts = addr.split() @@ -167,25 +173,10 @@ def info_from_p2p_addr(addr: Multiaddr) -> PeerInfoLibP2P: # make sure the /p2p value parses as a peer.ID peer_id_str: str = p2p_part.value_for_protocol(protocols.P_P2P) - peer_id: ID = ID.from_base58(peer_id_str) + peer_id = PeerID.from_base58(peer_id_str) # we might have received just an / p2p part, which means there's no addr. if len(parts) > 1: addr = Multiaddr.join(*parts[:-1]) return PeerInfo(peer_id, [addr]) - - -class InvalidAddrError(ValueError): - pass - - -class PeerInfo(PeerInfoLibP2P): - @classmethod - def from_pb(cls, peer_info_pb: p2pd_pb2.PeerInfo) -> PeerInfoLibP2P: - peer_id = ID(peer_info_pb.id) - addrs = [Multiaddr(addr) for addr in peer_info_pb.addrs] - return PeerInfo(peer_id, addrs) - - def __str__(self): - return self.peer_id.pretty() + " " + ",".join(str(a) for a in self.addrs) diff --git a/hivemind/p2p/p2p_daemon_bindings/p2pclient.py b/hivemind/p2p/p2p_daemon_bindings/p2pclient.py index baeab3612..d6b47c256 100644 --- a/hivemind/p2p/p2p_daemon_bindings/p2pclient.py +++ b/hivemind/p2p/p2p_daemon_bindings/p2pclient.py @@ -13,7 +13,7 @@ from hivemind.p2p.p2p_daemon_bindings.control import (ControlClient, DaemonConnector, StreamHandler) -from hivemind.p2p.p2p_daemon_bindings.datastructures import (ID, PeerInfo, +from hivemind.p2p.p2p_daemon_bindings.datastructures import (PeerID, PeerInfo, StreamInfo) @@ -37,13 +37,13 @@ async def listen(self) -> AsyncIterator["Client"]: async with self.control.listen(): yield self - async def identify(self) -> Tuple[ID, Tuple[Multiaddr, ...]]: + async def identify(self) -> Tuple[PeerID, Tuple[Multiaddr, ...]]: """ Get current node peer id and list of addresses """ return await self.control.identify() - async def connect(self, peer_id: ID, maddrs: Iterable[Multiaddr]) -> None: + async def connect(self, peer_id: PeerID, maddrs: Iterable[Multiaddr]) -> None: """ Connect to p2p node with specified addresses and peer id. :peer_id: node peer id you want connect to @@ -57,7 +57,7 @@ async def list_peers(self) -> Tuple[PeerInfo, ...]: """ return await self.control.list_peers() - async def disconnect(self, peer_id: ID) -> None: + async def disconnect(self, peer_id: PeerID) -> None: """ Disconnect from node with specified peer id :peer_id: @@ -65,7 +65,7 @@ async def disconnect(self, peer_id: ID) -> None: await self.control.disconnect(peer_id=peer_id) async def stream_open( - self, peer_id: ID, protocols: Sequence[str] + self, peer_id: PeerID, protocols: Sequence[str] ) -> Tuple[StreamInfo, asyncio.StreamReader, asyncio.StreamWriter]: """ Open a stream to call other peer (with peer_id) handler for specified protocols diff --git a/hivemind/p2p/p2p_daemon_bindings/utils.py b/hivemind/p2p/p2p_daemon_bindings/utils.py index f567b33bb..525bcc284 100644 --- a/hivemind/p2p/p2p_daemon_bindings/utils.py +++ b/hivemind/p2p/p2p_daemon_bindings/utils.py @@ -10,7 +10,6 @@ from hivemind.proto import p2pd_pb2 as p2pd_pb - DEFAULT_MAX_BITS: int = 64 diff --git a/tests/test_p2p_daemon.py b/tests/test_p2p_daemon.py index d5ea6df8c..2772d792f 100644 --- a/tests/test_p2p_daemon.py +++ b/tests/test_p2p_daemon.py @@ -9,7 +9,7 @@ from hivemind.utils import MSGPackSerializer from hivemind.utils.compression import serialize_torch_tensor, deserialize_torch_tensor -from hivemind.p2p.p2p_daemon_bindings.datastructures import ID +from hivemind.p2p.p2p_daemon_bindings.datastructures import PeerID from hivemind.p2p.p2p_daemon_bindings.datastructures import ID @@ -57,7 +57,7 @@ async def test_server_client_connection(): assert len(peers) == 0 nodes = boostrap_from([server]) - client = await P2P.create(bootstrap=True, boostrap_peers=nodes) + client = await P2P.create(bootstrap=True, bootstrap_peers=nodes) await client.wait_for_at_least_n_peers(1) peers = await client._client.list_peers() @@ -143,7 +143,7 @@ async def ping_handler(request, context): handler_cancelled = True return dht_pb2.PingResponse( peer=dht_pb2.NodeInfo( - node_id=context.ours_id.encode(), rpc_port=context.ours_port), + node_id=context.id.encode(), rpc_port=context.port), sender_endpoint=context.handle_name, available=True) server_primary = await P2P.create() @@ -154,7 +154,7 @@ async def ping_handler(request, context): assert is_process_running(server_pid) nodes = boostrap_from([server]) - client_primary = await P2P.create(bootstrap=True, boostrap_peers=nodes) + client_primary = await P2P.create(bootstrap=True, bootstrap_peers=nodes) client = await replicate_if_needed(client_primary, replicate) client_pid = client_primary._child.pid assert is_process_running(client_pid) @@ -167,7 +167,7 @@ async def ping_handler(request, context): sender_endpoint=handle_name, available=True) await client.wait_for_at_least_n_peers(1) - libp2p_server_id = ID.from_base58(server.id) + libp2p_server_id = PeerID.from_base58(server.id) stream_info, reader, writer = await client._client.stream_open(libp2p_server_id, (handle_name,)) await P2P.send_protobuf(ping_request, dht_pb2.PingRequest, writer) @@ -177,7 +177,7 @@ async def ping_handler(request, context): await asyncio.sleep(1) assert handler_cancelled else: - result, err = await P2P.receive_protobuf_or_error(dht_pb2.PingResponse, reader) + result, err = await P2P.receive_protobuf(dht_pb2.PingResponse, reader) assert err is None assert result == expected_response assert not handler_cancelled @@ -201,7 +201,7 @@ async def error_handler(request, context): assert is_process_running(server_pid) nodes = boostrap_from([server]) - client = await P2P.create(bootstrap=True, boostrap_peers=nodes) + client = await P2P.create(bootstrap=True, bootstrap_peers=nodes) client_pid = client._child.pid assert is_process_running(client_pid) await client.wait_for_at_least_n_peers(1) @@ -209,11 +209,11 @@ async def error_handler(request, context): ping_request = dht_pb2.PingRequest( peer=dht_pb2.NodeInfo(node_id=client.id.encode(), rpc_port=client._host_port), validate=True) - libp2p_server_id = ID.from_base58(server.id) + libp2p_server_id = PeerID.from_base58(server.id) stream_info, reader, writer = await client._client.stream_open(libp2p_server_id, (handle_name,)) await P2P.send_protobuf(ping_request, dht_pb2.PingRequest, writer) - result, err = await P2P.receive_protobuf_or_error(dht_pb2.PingResponse, reader) + result, err = await P2P.receive_protobuf(dht_pb2.PingResponse, reader) assert result is None assert err.message == 'boom' @@ -239,7 +239,7 @@ async def test_call_peer_single_process(test_input, expected, handle, handler_na assert is_process_running(server_pid) nodes = boostrap_from([server]) - client = await P2P.create(bootstrap=True, boostrap_peers=nodes) + client = await P2P.create(bootstrap=True, bootstrap_peers=nodes) client_pid = client._child.pid assert is_process_running(client_pid) @@ -294,7 +294,7 @@ async def test_call_peer_different_processes(): peer_port = client_side.recv() nodes = [bootstrap_addr(peer_port, peer_id)] - client = await P2P.create(bootstrap=True, boostrap_peers=nodes) + client = await P2P.create(bootstrap=True, bootstrap_peers=nodes) client_pid = client._child.pid assert is_process_running(client_pid) @@ -328,7 +328,7 @@ async def test_call_peer_torch_square(test_input, expected, handler_name="handle await server.add_stream_handler(handler_name, handle) nodes = boostrap_from([server]) - client = await P2P.create(bootstrap=True, boostrap_peers=nodes) + client = await P2P.create(bootstrap=True, bootstrap_peers=nodes) await client.wait_for_at_least_n_peers(1) @@ -360,7 +360,7 @@ async def test_call_peer_torch_add(test_input, expected, handler_name="handle"): await server.add_stream_handler(handler_name, handle) nodes = boostrap_from([server]) - client = await P2P.create(bootstrap=True, boostrap_peers=nodes) + client = await P2P.create(bootstrap=True, bootstrap_peers=nodes) await client.wait_for_at_least_n_peers(1) @@ -391,7 +391,7 @@ async def test_call_peer_error(replicate, handler_name="handle"): await server.add_stream_handler(handler_name, handle_add_torch_with_exc) nodes = boostrap_from([server]) - client_primary = await P2P.create(bootstrap=True, boostrap_peers=nodes) + client_primary = await P2P.create(bootstrap=True, bootstrap_peers=nodes) client = await replicate_if_needed(client_primary, replicate) await client.wait_for_at_least_n_peers(1) @@ -422,7 +422,7 @@ def handler(arg, key): await server_replica2.add_stream_handler(handler_name + '2', partial(handler, key=b'replica2')) nodes = boostrap_from([server_primary]) - client = await P2P.create(bootstrap=True, boostrap_peers=nodes) + client = await P2P.create(bootstrap=True, bootstrap_peers=nodes) await client.wait_for_at_least_n_peers(1) result = await client.call_peer_handler(server_id, handler_name, b'1') diff --git a/tests/test_p2p_daemon_bindings.py b/tests/test_p2p_daemon_bindings.py index 9b0b1f966..052605494 100644 --- a/tests/test_p2p_daemon_bindings.py +++ b/tests/test_p2p_daemon_bindings.py @@ -5,21 +5,27 @@ import subprocess import time import uuid -from contextlib import asynccontextmanager, AsyncExitStack +from contextlib import AsyncExitStack, asynccontextmanager from typing import NamedTuple +import pytest from google.protobuf.message import EncodeError from multiaddr import Multiaddr, protocols -import pytest - from hivemind import find_open_port -from hivemind.p2p.p2p_daemon_bindings.control import parse_conn_protocol, DaemonConnector, ControlClient +from hivemind.p2p.p2p_daemon_bindings.control import (ControlClient, + DaemonConnector, + parse_conn_protocol) +from hivemind.p2p.p2p_daemon_bindings.datastructures import (PeerID, PeerInfo, + StreamInfo) from hivemind.p2p.p2p_daemon_bindings.p2pclient import Client -from hivemind.p2p.p2p_daemon_bindings.utils import ControlFailure, raise_if_failed, write_unsigned_varint, \ - read_unsigned_varint, read_pbmsg_safe, write_pbmsg +from hivemind.p2p.p2p_daemon_bindings.utils import (ControlFailure, + raise_if_failed, + read_pbmsg_safe, + read_unsigned_varint, + write_pbmsg, + write_unsigned_varint) from hivemind.proto import p2pd_pb2 as p2pd_pb -from hivemind.p2p.p2p_daemon_bindings.datastructures import ID, StreamInfo, PeerInfo def test_raise_if_failed_raises(): @@ -134,7 +140,7 @@ def peer_id_bytes(): @pytest.fixture(scope="module") def peer_id(peer_id_bytes): - return ID(peer_id_bytes) + return PeerID(peer_id_bytes) @pytest.fixture(scope="module") @@ -147,13 +153,13 @@ def test_peer_id(peer_id_string, peer_id_bytes, peer_id): assert peer_id.to_bytes() == peer_id_bytes assert peer_id.to_string() == peer_id_string # test initialized with string - peer_id_2 = ID.from_base58(peer_id_string) + peer_id_2 = PeerID.from_base58(peer_id_string) assert peer_id_2.to_bytes() == peer_id_bytes assert peer_id_2.to_string() == peer_id_string # test equal assert peer_id == peer_id_2 # test not equal - peer_id_3 = ID.from_base58("QmbmfNDEth7Ucvjuxiw3SP3E4PoJzbk7g4Ge6ZDigbCsNp") + peer_id_3 = PeerID.from_base58("QmbmfNDEth7Ucvjuxiw3SP3E4PoJzbk7g4Ge6ZDigbCsNp") assert peer_id != peer_id_3 @@ -165,12 +171,12 @@ def test_stream_info(peer_id, maddr): assert si.addr == maddr assert si.proto == proto # test case: `StreamInfo.to_pb` - pb_si = si.to_pb() + pb_si = si.to_protobuf() assert pb_si.peer == peer_id.to_bytes() assert pb_si.addr == maddr.to_bytes() assert pb_si.proto == si.proto # test case: `StreamInfo.from_pb` - si_1 = StreamInfo.from_pb(pb_si) + si_1 = StreamInfo.from_protobuf(pb_si) assert si_1.peer_id == peer_id assert si_1.addr == maddr assert si_1.proto == proto @@ -183,7 +189,7 @@ def test_peer_info(peer_id, maddr): assert pi.addrs == [maddr] # test case: `PeerInfo.from_pb` pi_pb = p2pd_pb.PeerInfo(id=peer_id.to_bytes(), addrs=[maddr.to_bytes()]) - pi_1 = PeerInfo.from_pb(pi_pb) + pi_1 = PeerInfo.from_protobuf(pi_pb) assert pi.peer_id == pi_1.peer_id assert pi.addrs == pi_1.addrs @@ -317,7 +323,7 @@ def num_p2pds(): @pytest.fixture(scope="module") def peer_id_random(): - return ID.from_base58("QmcgpsyWgH8Y8ajJz1Cu72KnS5uo2Aa2LpzU7kinSupNK1") + return PeerID.from_base58("QmcgpsyWgH8Y8ajJz1Cu72KnS5uo2Aa2LpzU7kinSupNK1") @pytest.fixture From e92aa9d73d9b28869ad193c9db6d9b71ceb4cd8e Mon Sep 17 00:00:00 2001 From: Maxim Kashirin Date: Mon, 3 May 2021 16:14:37 +0300 Subject: [PATCH 76/81] imports/rename constants/string formatting --- hivemind/p2p/p2p_daemon.py | 6 +++--- hivemind/p2p/p2p_daemon_bindings/control.py | 12 +++++------- .../p2p/p2p_daemon_bindings/datastructures.py | 2 +- tests/test_p2p_daemon.py | 18 +++++++++--------- 4 files changed, 18 insertions(+), 20 deletions(-) diff --git a/hivemind/p2p/p2p_daemon.py b/hivemind/p2p/p2p_daemon.py index aa5cbd36c..bc7d1f91c 100644 --- a/hivemind/p2p/p2p_daemon.py +++ b/hivemind/p2p/p2p_daemon.py @@ -19,7 +19,7 @@ logger = get_logger(__name__) -P2PD_RELATIVE_PATH = 'hivemind_cli/p2pd' +P2PD_FILENAME = 'p2pd' NUM_RETRIES = 3 RETRY_DELAY = 0.4 @@ -106,11 +106,11 @@ async def create(cls, *args, quic: bool = True, tls: bool = True, conn_manager: 'It is very dangerous, because p2pd connects to global ipfs and it is very unstable. ' \ 'If you really want this, pass use_global_ipfs=True' assert not (bootstrap_peers is not None and use_global_ipfs), \ - 'Non empty boostrap_nodes and use_global_ipfs=True are incompatible.' \ + 'Non empty bootstrap_nodes and use_global_ipfs=True are incompatible.' \ 'Choose one option: your nodes list (preferable) or global ipfs (very unstable)' self = cls() - with path(cli, 'p2pd') as p: + with path(cli, P2PD_FILENAME) as p: p2pd_path = p bootstrap_peers = cls._make_bootstrap_peers(bootstrap_peers) dht = cls.DHT_MODE_MAPPING.get(dht_mode, {'dht': 0}) diff --git a/hivemind/p2p/p2p_daemon_bindings/control.py b/hivemind/p2p/p2p_daemon_bindings/control.py index ed8425bf6..f0d48c9d2 100644 --- a/hivemind/p2p/p2p_daemon_bindings/control.py +++ b/hivemind/p2p/p2p_daemon_bindings/control.py @@ -5,7 +5,6 @@ """ import asyncio -import logging from contextlib import asynccontextmanager from typing import (AsyncIterator, Awaitable, Callable, Dict, Iterable, Sequence, Tuple) @@ -55,9 +54,7 @@ def __init__(self, control_maddr: Multiaddr = Multiaddr(DEFAULT_CONTROL_MADDR)) async def open_connection(self) -> (asyncio.StreamReader, asyncio.StreamWriter): if self.proto_code == protocols.P_UNIX: control_path = self.control_maddr.value_for_protocol(protocols.P_UNIX) - self.logger.debug( - "DaemonConnector %s opens connection to %s", self, self.control_maddr - ) + self.logger.debug(f"DaemonConnector {self} opens connection to {self.control_maddr}") return await asyncio.open_unix_connection(control_path) elif self.proto_code == protocols.P_IP4: host = self.control_maddr.value_for_protocol(protocols.P_IP4) @@ -72,8 +69,9 @@ async def open_connection(self) -> (asyncio.StreamReader, asyncio.StreamWriter): class ControlClient: DEFAULT_LISTEN_MADDR = "/unix/tmp/p2pclient.sock" - def __init__(self, daemon_connector: DaemonConnector, - listen_maddr: Multiaddr = Multiaddr(DEFAULT_LISTEN_MADDR)) -> None: + def __init__( + self, daemon_connector: DaemonConnector, listen_maddr: Multiaddr = Multiaddr(DEFAULT_LISTEN_MADDR) + ) -> None: self.listen_maddr: Multiaddr = listen_maddr self.daemon_connector: DaemonConnector = daemon_connector self.handlers: Dict[str, StreamHandler] = {} @@ -104,7 +102,7 @@ async def listen(self) -> AsyncIterator["ControlClient"]: server = await asyncio.start_server(self._handler, port=port, host=host) else: raise ValueError( - f"protocol not supported: protocol={protocols.protocol_with_code(proto_code)}" + f"Protocol not supported: {protocols.protocol_with_code(self.proto_code)}" ) async with server: diff --git a/hivemind/p2p/p2p_daemon_bindings/datastructures.py b/hivemind/p2p/p2p_daemon_bindings/datastructures.py index 224640d2f..ddbcb3b02 100644 --- a/hivemind/p2p/p2p_daemon_bindings/datastructures.py +++ b/hivemind/p2p/p2p_daemon_bindings/datastructures.py @@ -5,7 +5,7 @@ """ import hashlib -from typing import Any, List, Optional, Sequence, Union +from typing import Any, Sequence, Union import base58 import multihash diff --git a/tests/test_p2p_daemon.py b/tests/test_p2p_daemon.py index 2772d792f..d174f7bfa 100644 --- a/tests/test_p2p_daemon.py +++ b/tests/test_p2p_daemon.py @@ -32,7 +32,7 @@ def bootstrap_addr(host_port, id_): return f'/ip4/127.0.0.1/tcp/{host_port}/p2p/{id_}' -def boostrap_from(daemons: List[P2P]) -> List[str]: +def bootstrap_from(daemons: List[P2P]) -> List[str]: return [ bootstrap_addr(d._host_port, d.id) for d in daemons @@ -56,7 +56,7 @@ async def test_server_client_connection(): peers = await server._client.list_peers() assert len(peers) == 0 - nodes = boostrap_from([server]) + nodes = bootstrap_from([server]) client = await P2P.create(bootstrap=True, bootstrap_peers=nodes) await client.wait_for_at_least_n_peers(1) @@ -153,7 +153,7 @@ async def ping_handler(request, context): dht_pb2.PingResponse) assert is_process_running(server_pid) - nodes = boostrap_from([server]) + nodes = bootstrap_from([server]) client_primary = await P2P.create(bootstrap=True, bootstrap_peers=nodes) client = await replicate_if_needed(client_primary, replicate) client_pid = client_primary._child.pid @@ -200,7 +200,7 @@ async def error_handler(request, context): await server.add_unary_handler(handle_name, error_handler, dht_pb2.PingRequest, dht_pb2.PingResponse) assert is_process_running(server_pid) - nodes = boostrap_from([server]) + nodes = bootstrap_from([server]) client = await P2P.create(bootstrap=True, bootstrap_peers=nodes) client_pid = client._child.pid assert is_process_running(client_pid) @@ -238,7 +238,7 @@ async def test_call_peer_single_process(test_input, expected, handle, handler_na await server.add_stream_handler(handler_name, handle) assert is_process_running(server_pid) - nodes = boostrap_from([server]) + nodes = bootstrap_from([server]) client = await P2P.create(bootstrap=True, bootstrap_peers=nodes) client_pid = client._child.pid assert is_process_running(client_pid) @@ -327,7 +327,7 @@ async def test_call_peer_torch_square(test_input, expected, handler_name="handle server = await P2P.create() await server.add_stream_handler(handler_name, handle) - nodes = boostrap_from([server]) + nodes = bootstrap_from([server]) client = await P2P.create(bootstrap=True, bootstrap_peers=nodes) await client.wait_for_at_least_n_peers(1) @@ -359,7 +359,7 @@ async def test_call_peer_torch_add(test_input, expected, handler_name="handle"): server = await P2P.create() await server.add_stream_handler(handler_name, handle) - nodes = boostrap_from([server]) + nodes = bootstrap_from([server]) client = await P2P.create(bootstrap=True, bootstrap_peers=nodes) await client.wait_for_at_least_n_peers(1) @@ -390,7 +390,7 @@ async def test_call_peer_error(replicate, handler_name="handle"): server = await replicate_if_needed(server_primary, replicate) await server.add_stream_handler(handler_name, handle_add_torch_with_exc) - nodes = boostrap_from([server]) + nodes = bootstrap_from([server]) client_primary = await P2P.create(bootstrap=True, bootstrap_peers=nodes) client = await replicate_if_needed(client_primary, replicate) @@ -421,7 +421,7 @@ def handler(arg, key): server_replica2 = await replicate_if_needed(server_primary, True) await server_replica2.add_stream_handler(handler_name + '2', partial(handler, key=b'replica2')) - nodes = boostrap_from([server_primary]) + nodes = bootstrap_from([server_primary]) client = await P2P.create(bootstrap=True, bootstrap_peers=nodes) await client.wait_for_at_least_n_peers(1) From 550fd4b4849974416082588fa6c80b0ff2e653b4 Mon Sep 17 00:00:00 2001 From: Maxim Kashirin Date: Tue, 4 May 2021 00:49:22 +0300 Subject: [PATCH 77/81] reST docstring --- hivemind/p2p/p2p_daemon.py | 36 ++++++++++++++++++------------------ 1 file changed, 18 insertions(+), 18 deletions(-) diff --git a/hivemind/p2p/p2p_daemon.py b/hivemind/p2p/p2p_daemon.py index bc7d1f91c..6b263f68e 100644 --- a/hivemind/p2p/p2p_daemon.py +++ b/hivemind/p2p/p2p_daemon.py @@ -84,21 +84,21 @@ async def create(cls, *args, quic: bool = True, tls: bool = True, conn_manager: daemon_listen_port: int = None, **kwargs): """ Start a new p2pd process and connect to it. - @param args: - @param quic: Enables the QUIC transport - @param tls: Enables TLS1.3 channel security protocol - @param conn_manager: Enables the Connection Manager - @param dht_mode: DHT mode (dht_client/dht_server/dht) - @param force_reachability: Force reachability mode (public/private) - @param nat_port_map: Enables NAT port mapping - @param auto_nat: Enables the AutoNAT service - @param bootstrap: Connects to bootstrap peers and bootstraps the dht if enabled - @param bootstrap_peers: List of bootstrap peers; defaults to the IPFS DHT peers - @param use_global_ipfs: Bootstrap to global ipfs (works only if bootstrap=True and bootstrap_peers=None) - @param host_port: port for p2p network - @param daemon_listen_port: port for connection daemon and client binding - @param kwargs: - @return: new wrapper for p2p daemon + :param args: + :param quic: Enables the QUIC transport + :param tls: Enables TLS1.3 channel security protocol + :param conn_manager: Enables the Connection Manager + :param dht_mode: DHT mode (dht_client/dht_server/dht) + :param force_reachability: Force reachability mode (public/private) + :param nat_port_map: Enables NAT port mapping + :param auto_nat: Enables the AutoNAT service + :param bootstrap: Connects to bootstrap peers and bootstraps the dht if enabled + :param bootstrap_peers: List of bootstrap peers; defaults to the IPFS DHT peers + :param use_global_ipfs: Bootstrap to global ipfs (works only if bootstrap=True and bootstrap_peers=None) + :param host_port: port for p2p network + :param daemon_listen_port: port for connection daemon and client binding + :param kwargs: + :return: new wrapper for p2p daemon """ assert not (bootstrap and bootstrap_peers is None and not use_global_ipfs), \ @@ -140,9 +140,9 @@ async def create(cls, *args, quic: bool = True, tls: bool = True, conn_manager: async def replicate(cls, daemon_listen_port: int, host_port: int): """ Connect to existing p2p daemon - @param host_port: port for p2p network - @param daemon_listen_port: port for connection daemon and client binding - @return: new wrapper for existing p2p daemon + :param daemon_listen_port: port for connection daemon and client binding + :param host_port: port for p2p network + :return: new wrapper for existing p2p daemon """ self = cls() From 450ddc838fc5e2fc9b3a9e3d3825da88a1eb81ca Mon Sep 17 00:00:00 2001 From: Maxim Kashirin Date: Wed, 12 May 2021 19:54:47 +0300 Subject: [PATCH 78/81] pr fixes --- hivemind/p2p/p2p_daemon_bindings/control.py | 21 +- .../p2p/p2p_daemon_bindings/datastructures.py | 22 +- hivemind/p2p/p2p_daemon_bindings/keys.py | 97 ----- hivemind/p2p/p2p_daemon_bindings/p2pclient.py | 6 +- hivemind/p2p/p2p_daemon_bindings/utils.py | 24 +- hivemind/proto/crypto.proto | 24 -- hivemind/proto/p2pd.proto | 2 +- requirements.txt | 4 +- setup.py | 4 +- tests/test_p2p_daemon.py | 9 +- tests/test_p2p_daemon_bindings.py | 368 +++--------------- tests/test_utils/__init__.py | 192 +++++++++ 12 files changed, 287 insertions(+), 486 deletions(-) delete mode 100644 hivemind/p2p/p2p_daemon_bindings/keys.py delete mode 100644 hivemind/proto/crypto.proto create mode 100644 tests/test_utils/__init__.py diff --git a/hivemind/p2p/p2p_daemon_bindings/control.py b/hivemind/p2p/p2p_daemon_bindings/control.py index f0d48c9d2..2002338a2 100644 --- a/hivemind/p2p/p2p_daemon_bindings/control.py +++ b/hivemind/p2p/p2p_daemon_bindings/control.py @@ -30,6 +30,7 @@ SUPPORTED_PROTOS = ( protocols.protocol_with_code(proto) for proto in SUPPORT_CONN_PROTOCOLS ) +logger = get_logger(__name__) def parse_conn_protocol(maddr: Multiaddr) -> int: @@ -47,14 +48,13 @@ class DaemonConnector: DEFAULT_CONTROL_MADDR = "/unix/tmp/p2pd.sock" def __init__(self, control_maddr: Multiaddr = Multiaddr(DEFAULT_CONTROL_MADDR)) -> None: - self.control_maddr: Multiaddr = control_maddr - self.proto_code: int = parse_conn_protocol(self.control_maddr) - self.logger = get_logger(__name__) + self.control_maddr = control_maddr + self.proto_code = parse_conn_protocol(self.control_maddr) async def open_connection(self) -> (asyncio.StreamReader, asyncio.StreamWriter): if self.proto_code == protocols.P_UNIX: control_path = self.control_maddr.value_for_protocol(protocols.P_UNIX) - self.logger.debug(f"DaemonConnector {self} opens connection to {self.control_maddr}") + logger.debug(f"DaemonConnector {self} opens connection to {self.control_maddr}") return await asyncio.open_unix_connection(control_path) elif self.proto_code == protocols.P_IP4: host = self.control_maddr.value_for_protocol(protocols.P_IP4) @@ -72,16 +72,15 @@ class ControlClient: def __init__( self, daemon_connector: DaemonConnector, listen_maddr: Multiaddr = Multiaddr(DEFAULT_LISTEN_MADDR) ) -> None: - self.listen_maddr: Multiaddr = listen_maddr - self.daemon_connector: DaemonConnector = daemon_connector + self.listen_maddr = listen_maddr + self.daemon_connector = daemon_connector self.handlers: Dict[str, StreamHandler] = {} - self.logger = get_logger(__name__) async def _handler(self, reader: asyncio.StreamReader, writer: asyncio.StreamWriter): pb_stream_info = p2pd_pb.StreamInfo() # type: ignore await read_pbmsg_safe(reader, pb_stream_info) stream_info = StreamInfo.from_protobuf(pb_stream_info) - self.logger.debug(f"New incoming stream: {stream_info}") + logger.debug(f"New incoming stream: {stream_info}") try: handler = self.handlers[stream_info.proto] except KeyError as e: @@ -102,14 +101,14 @@ async def listen(self) -> AsyncIterator["ControlClient"]: server = await asyncio.start_server(self._handler, port=port, host=host) else: raise ValueError( - f"Protocol not supported: {protocols.protocol_with_code(self.proto_code)}" + f"Protocol not supported: {protocols.protocol_with_code(proto_code)}" ) async with server: - self.logger.info(f"DaemonConnector {self} starts listening to {self.listen_maddr}") + logger.info(f"DaemonConnector {self} starts listening to {self.listen_maddr}") yield self - self.logger.info(f"DaemonConnector {self} closed") + logger.info(f"DaemonConnector {self} closed") async def identify(self) -> Tuple[PeerID, Tuple[Multiaddr, ...]]: reader, writer = await self.daemon_connector.open_connection() diff --git a/hivemind/p2p/p2p_daemon_bindings/datastructures.py b/hivemind/p2p/p2p_daemon_bindings/datastructures.py index ddbcb3b02..dab58c408 100644 --- a/hivemind/p2p/p2p_daemon_bindings/datastructures.py +++ b/hivemind/p2p/p2p_daemon_bindings/datastructures.py @@ -11,7 +11,6 @@ import multihash from multiaddr import Multiaddr, protocols -from hivemind.p2p.p2p_daemon_bindings.keys import PublicKey from hivemind.proto import p2pd_pb2 # NOTE: On inlining... @@ -25,8 +24,6 @@ if ENABLE_INLINING: class IdentityHash: - _digest: bytes - def __init__(self) -> None: self._digest = bytearray() @@ -44,8 +41,8 @@ def digest(self) -> bytes: class PeerID: def __init__(self, peer_id_bytes: bytes) -> None: self._bytes = peer_id_bytes - self._xor_id: int = int(sha256_digest(self._bytes).hex(), 16) - self._b58_str: str = base58.b58encode(self._bytes).decode() + self._xor_id = int(sha256_digest(self._bytes).hex(), 16) + self._b58_str = base58.b58encode(self._bytes).decode() @property def xor_id(self) -> int: @@ -87,15 +84,6 @@ def from_base58(cls, base58_id: str) -> "PeerID": peer_id_bytes = base58.b58decode(base58_id) return cls(peer_id_bytes) - @classmethod - def from_pubkey(cls, key: PublicKey) -> "PeerID": - serialized_key = key.serialize() - algo = multihash.Func.sha2_256 - if ENABLE_INLINING and len(serialized_key) <= MAX_INLINE_KEY_LENGTH: - algo = IDENTITY_MULTIHASH_CODE - mh_digest = multihash.digest(serialized_key, algo) - return cls(mh_digest.encode()) - def sha256_digest(data: Union[str, bytes]) -> bytes: if isinstance(data, str): @@ -105,9 +93,9 @@ def sha256_digest(data: Union[str, bytes]) -> bytes: class StreamInfo: def __init__(self, peer_id: PeerID, addr: Multiaddr, proto: str) -> None: - self.peer_id: PeerID = peer_id - self.addr: Multiaddr = addr - self.proto: str = proto + self.peer_id = peer_id + self.addr = addr + self.proto = proto def __repr__(self) -> str: return ( diff --git a/hivemind/p2p/p2p_daemon_bindings/keys.py b/hivemind/p2p/p2p_daemon_bindings/keys.py deleted file mode 100644 index 763c3d76a..000000000 --- a/hivemind/p2p/p2p_daemon_bindings/keys.py +++ /dev/null @@ -1,97 +0,0 @@ -""" -Originally taken from: https://github.com/libp2p/py-libp2p -Licence: MIT -Author: Kevin Mai-Husan Chia and others -""" - -from abc import ABC, abstractmethod -from dataclasses import dataclass -from enum import Enum, unique - -from hivemind.proto import crypto_pb2 as protobuf - - -@unique -class KeyType(Enum): - RSA = 0 - Ed25519 = 1 - Secp256k1 = 2 - ECDSA = 3 - ECC_P256 = 4 - - -class Key(ABC): - """A ``Key`` represents a cryptographic key.""" - - @abstractmethod - def to_bytes(self) -> bytes: - """Returns the byte representation of this key.""" - ... - - @abstractmethod - def get_type(self) -> KeyType: - """Returns the ``KeyType`` for ``self``.""" - ... - - def __eq__(self, other: object) -> bool: - if not isinstance(other, Key): - return NotImplemented - return self.to_bytes() == other.to_bytes() - - -class PublicKey(Key): - """A ``PublicKey`` represents a cryptographic public key.""" - - @abstractmethod - def verify(self, data: bytes, signature: bytes) -> bool: - """Verify that ``signature`` is the cryptographic signature of the hash - of ``data``.""" - ... - - def _serialize_to_protobuf(self) -> protobuf.PublicKey: - """Return the protobuf representation of this ``Key``.""" - key_type = self.get_type().value - data = self.to_bytes() - protobuf_key = protobuf.PublicKey(key_type=key_type, data=data) - return protobuf_key - - def serialize(self) -> bytes: - """Return the canonical serialization of this ``Key``.""" - return self._serialize_to_protobuf().SerializeToString() - - @classmethod - def deserialize_from_protobuf(cls, protobuf_data: bytes) -> protobuf.PublicKey: - return protobuf.PublicKey.FromString(protobuf_data) - - -class PrivateKey(Key): - """A ``PrivateKey`` represents a cryptographic private key.""" - - @abstractmethod - def sign(self, data: bytes) -> bytes: - ... - - @abstractmethod - def get_public_key(self) -> PublicKey: - ... - - def _serialize_to_protobuf(self) -> protobuf.PrivateKey: - """Return the protobuf representation of this ``Key``.""" - key_type = self.get_type().value - data = self.to_bytes() - protobuf_key = protobuf.PrivateKey(key_type=key_type, data=data) - return protobuf_key - - def serialize(self) -> bytes: - """Return the canonical serialization of this ``Key``.""" - return self._serialize_to_protobuf().SerializeToString() - - @classmethod - def deserialize_from_protobuf(cls, protobuf_data: bytes) -> protobuf.PrivateKey: - return protobuf.PrivateKey.FromString(protobuf_data) - - -@dataclass(frozen=True) -class KeyPair: - private_key: PrivateKey - public_key: PublicKey diff --git a/hivemind/p2p/p2p_daemon_bindings/p2pclient.py b/hivemind/p2p/p2p_daemon_bindings/p2pclient.py index d6b47c256..c1fe97808 100644 --- a/hivemind/p2p/p2p_daemon_bindings/p2pclient.py +++ b/hivemind/p2p/p2p_daemon_bindings/p2pclient.py @@ -60,7 +60,7 @@ async def list_peers(self) -> Tuple[PeerInfo, ...]: async def disconnect(self, peer_id: PeerID) -> None: """ Disconnect from node with specified peer id - :peer_id: + :peer_id: node peer id you want disconnect from """ await self.control.disconnect(peer_id=peer_id) @@ -69,8 +69,8 @@ async def stream_open( ) -> Tuple[StreamInfo, asyncio.StreamReader, asyncio.StreamWriter]: """ Open a stream to call other peer (with peer_id) handler for specified protocols - :peer_id: - :protocols: + :peer_id: other peer id + :protocols: list of protocols for other peer handling :return: Returns tuple of stream info (info about connection to second peer) and reader/writer """ return await self.control.stream_open(peer_id=peer_id, protocols=protocols) diff --git a/hivemind/p2p/p2p_daemon_bindings/utils.py b/hivemind/p2p/p2p_daemon_bindings/utils.py index 525bcc284..2a0d5b97c 100644 --- a/hivemind/p2p/p2p_daemon_bindings/utils.py +++ b/hivemind/p2p/p2p_daemon_bindings/utils.py @@ -21,16 +21,14 @@ class DispatchFailure(Exception): pass -async def write_unsigned_varint( - stream: asyncio.StreamWriter, integer: int, max_bits: int = DEFAULT_MAX_BITS -) -> None: - max_int: int = 1 << max_bits +async def write_unsigned_varint(stream: asyncio.StreamWriter, integer: int, max_bits: int = DEFAULT_MAX_BITS) -> None: + max_int = 1 << max_bits if integer < 0: raise ValueError(f"negative integer: {integer}") if integer >= max_int: raise ValueError(f"integer too large: {integer}") while True: - value: int = integer & 0x7F + value = integer & 0x7F integer >>= 7 if integer != 0: value |= 0x80 @@ -40,13 +38,11 @@ async def write_unsigned_varint( break -async def read_unsigned_varint( - stream: asyncio.StreamReader, max_bits: int = DEFAULT_MAX_BITS -) -> int: - max_int: int = 1 << max_bits - iteration: int = 0 - result: int = 0 - has_next: bool = True +async def read_unsigned_varint(stream: asyncio.StreamReader, max_bits: int = DEFAULT_MAX_BITS) -> int: + max_int = 1 << max_bits + iteration = 0 + result = 0 + has_next = True while has_next: data = await stream.readexactly(1) c = data[0] @@ -55,13 +51,13 @@ async def read_unsigned_varint( has_next = (c & 0x80) != 0 iteration += 1 if result >= max_int: - raise ValueError(f"varint overflowed: {result}") + raise ValueError(f"Varint overflowed: {result}") return result def raise_if_failed(response: p2pd_pb.Response) -> None: if response.type == p2pd_pb.Response.ERROR: - raise ControlFailure(f"connect failed. msg={response.error.msg}") + raise ControlFailure(f"Connect failed. msg={response.error.msg}") async def write_pbmsg(stream: asyncio.StreamWriter, pbmsg: PBMessage) -> None: diff --git a/hivemind/proto/crypto.proto b/hivemind/proto/crypto.proto deleted file mode 100644 index e4a69f576..000000000 --- a/hivemind/proto/crypto.proto +++ /dev/null @@ -1,24 +0,0 @@ -//Originally taken from: https://github.com/libp2p/py-libp2p -//Licence: MIT -//Author: Kevin Mai-Husan Chia and others - -syntax = "proto2"; - -package crypto.pb; - -enum KeyType { - RSA = 0; - Ed25519 = 1; - Secp256k1 = 2; - ECDSA = 3; -} - -message PublicKey { - required KeyType key_type = 1; - required bytes data = 2; -} - -message PrivateKey { - required KeyType key_type = 1; - required bytes data = 2; -} diff --git a/hivemind/proto/p2pd.proto b/hivemind/proto/p2pd.proto index dc65514e5..373c6d8e9 100644 --- a/hivemind/proto/p2pd.proto +++ b/hivemind/proto/p2pd.proto @@ -163,4 +163,4 @@ message PSResponse { message RPCError { required string message = 1; -} \ No newline at end of file +} diff --git a/requirements.txt b/requirements.txt index b0011a653..9ce07c9eb 100644 --- a/requirements.txt +++ b/requirements.txt @@ -10,7 +10,7 @@ grpcio>=1.33.2 grpcio-tools>=1.33.2 protobuf>=3.12.2 configargparse>=1.2.3 -multiaddr==0.0.9 -pymultihash==0.8.2 +multiaddr>=0.0.9 +pymultihash>=0.8.2 cryptography>=3.4.6 pydantic>=1.8.1 diff --git a/setup.py b/setup.py index 2723a3f66..236fde563 100644 --- a/setup.py +++ b/setup.py @@ -68,10 +68,10 @@ def libp2p_build_install(): v = m.group(1) if version.parse(v) < version.parse("1.13"): - raise EnvironmentError(f'newer version of go required: must be >= 1.13, found {version}') + raise EnvironmentError(f'Newer version of go required: must be >= 1.13, found {version}') except FileNotFoundError: - raise FileNotFoundError('could not find golang installation') + raise FileNotFoundError('Could not find golang installation') with tempfile.TemporaryDirectory() as tempdir: dest = os.path.join(tempdir, 'libp2p-daemon.tar.gz') diff --git a/tests/test_p2p_daemon.py b/tests/test_p2p_daemon.py index d174f7bfa..3e38d72eb 100644 --- a/tests/test_p2p_daemon.py +++ b/tests/test_p2p_daemon.py @@ -15,9 +15,13 @@ import numpy as np import pytest +import torch from hivemind.p2p import P2P +from hivemind.p2p.p2p_daemon_bindings.datastructures import PeerID from hivemind.proto import dht_pb2, runtime_pb2 +from hivemind.utils import MSGPackSerializer +from hivemind.utils.compression import deserialize_torch_tensor, serialize_torch_tensor def is_process_running(pid: int) -> bool: @@ -33,10 +37,7 @@ def bootstrap_addr(host_port, id_): def bootstrap_from(daemons: List[P2P]) -> List[str]: - return [ - bootstrap_addr(d._host_port, d.id) - for d in daemons - ] + return [bootstrap_addr(d._host_port, d.id) for d in daemons] @pytest.mark.asyncio diff --git a/tests/test_p2p_daemon_bindings.py b/tests/test_p2p_daemon_bindings.py index 052605494..7f87cbd88 100644 --- a/tests/test_p2p_daemon_bindings.py +++ b/tests/test_p2p_daemon_bindings.py @@ -1,31 +1,17 @@ import asyncio -import functools import io -import os -import subprocess -import time -import uuid -from contextlib import AsyncExitStack, asynccontextmanager -from typing import NamedTuple +from contextlib import AsyncExitStack import pytest from google.protobuf.message import EncodeError from multiaddr import Multiaddr, protocols -from hivemind import find_open_port -from hivemind.p2p.p2p_daemon_bindings.control import (ControlClient, - DaemonConnector, - parse_conn_protocol) -from hivemind.p2p.p2p_daemon_bindings.datastructures import (PeerID, PeerInfo, - StreamInfo) -from hivemind.p2p.p2p_daemon_bindings.p2pclient import Client -from hivemind.p2p.p2p_daemon_bindings.utils import (ControlFailure, - raise_if_failed, - read_pbmsg_safe, - read_unsigned_varint, - write_pbmsg, - write_unsigned_varint) +from hivemind.p2p.p2p_daemon_bindings.control import ControlClient, DaemonConnector, parse_conn_protocol +from hivemind.p2p.p2p_daemon_bindings.datastructures import PeerID, PeerInfo, StreamInfo +from hivemind.p2p.p2p_daemon_bindings.utils import (ControlFailure, raise_if_failed, read_pbmsg_safe, + read_unsigned_varint, write_pbmsg, write_unsigned_varint) from hivemind.proto import p2pd_pb2 as p2pd_pb +from test_utils import make_p2pd_pair_ip4, connect_safe def test_raise_if_failed_raises(): @@ -41,7 +27,7 @@ def test_raise_if_failed_not_raises(): raise_if_failed(resp) -pairs_int_serialized_valid = ( +PAIRS_INT_SERIALIZED_VALID = ( (0, b"\x00"), (1, b"\x01"), (128, b"\x80\x01"), @@ -49,7 +35,7 @@ def test_raise_if_failed_not_raises(): (2 ** 64 - 1, b"\xff\xff\xff\xff\xff\xff\xff\xff\xff\x01"), ) -pairs_int_serialized_overflow = ( +PAIRS_INT_SERIALIZED_OVERFLOW = ( (2 ** 64, b"\x80\x80\x80\x80\x80\x80\x80\x80\x80\x02"), (2 ** 64 + 1, b"\x81\x80\x80\x80\x80\x80\x80\x80\x80\x02"), ( @@ -58,6 +44,18 @@ def test_raise_if_failed_not_raises(): ), ) +PEER_ID_STRING = "QmS5QmciTXXnCUCyxud5eWFenUMAmvAWSDa1c7dvdXRMZ7" +PEER_ID_BYTES = b'\x12 7\x87F.[\xb5\xb1o\xe5*\xc7\xb9\xbb\x11:"Z|j2\x8ad\x1b\xa6\xe5= timeout: - # timeout - assert False, f"{coro_func} still failed after `{timeout}` seconds" - await asyncio.sleep(0.01) - - -class Daemon: - control_maddr = None - proc_daemon = None - log_filename = "" - f_log = None - closed = None - - def __init__( - self, control_maddr, enable_control, enable_connmgr, enable_dht, enable_pubsub - ): - self.control_maddr = control_maddr - self.enable_control = enable_control - self.enable_connmgr = enable_connmgr - self.enable_dht = enable_dht - self.enable_pubsub = enable_pubsub - self.is_closed = False - self._start_logging() - self._run() - - def _start_logging(self): - name_control_maddr = str(self.control_maddr).replace("/", "_").replace(".", "_") - self.log_filename = f"/tmp/log_p2pd{name_control_maddr}.txt" - self.f_log = open(self.log_filename, "wb") - - def _run(self): - cmd_list = ["hivemind/hivemind_cli/p2pd", f"-listen={str(self.control_maddr)}"] - cmd_list += [f"-hostAddrs=/ip4/127.0.0.1/tcp/{find_open_port()}"] - if self.enable_connmgr: - cmd_list += ["-connManager=true", "-connLo=1", "-connHi=2", "-connGrace=0"] - if self.enable_dht: - cmd_list += ["-dht=true"] - if self.enable_pubsub: - cmd_list += ["-pubsub=true", "-pubsubRouter=gossipsub"] - self.proc_daemon = subprocess.Popen( - cmd_list, stdout=self.f_log, stderr=self.f_log, bufsize=0 - ) - - async def wait_until_ready(self): - lines_head_pattern = (b"Control socket:", b"Peer ID:", b"Peer Addrs:") - lines_head_occurred = {line: False for line in lines_head_pattern} - - with open(self.log_filename, "rb") as f_log_read: - - async def read_from_daemon_and_check(): - line = f_log_read.readline() - for head_pattern in lines_head_occurred: - if line.startswith(head_pattern): - lines_head_occurred[head_pattern] = True - return all([value for _, value in lines_head_occurred.items()]) - - await try_until_success(read_from_daemon_and_check) - - # sleep for a while in case that the daemon haven't been ready after emitting these lines - await asyncio.sleep(0.1) - - def close(self): - if self.is_closed: - return - self.proc_daemon.terminate() - self.proc_daemon.wait() - self.f_log.close() - self.is_closed = True - - -class DaemonTuple(NamedTuple): - daemon: Daemon - client: Client - - -class ConnectionFailure(Exception): - pass - - -@asynccontextmanager -async def make_p2pd_pair_unix( - enable_control, enable_connmgr, enable_dht, enable_pubsub -): - name = str(uuid.uuid4())[:8] - control_maddr = Multiaddr(f"/unix/tmp/test_p2pd_control_{name}.sock") - listen_maddr = Multiaddr(f"/unix/tmp/test_p2pd_listen_{name}.sock") - # Remove the existing unix socket files if they are existing - try: - os.unlink(control_maddr.value_for_protocol(protocols.P_UNIX)) - except FileNotFoundError: - pass - try: - os.unlink(listen_maddr.value_for_protocol(protocols.P_UNIX)) - except FileNotFoundError: - pass - async with _make_p2pd_pair( - control_maddr=control_maddr, - listen_maddr=listen_maddr, - enable_control=enable_control, - enable_connmgr=enable_connmgr, - enable_dht=enable_dht, - enable_pubsub=enable_pubsub, - ) as pair: - yield pair - - -@asynccontextmanager -async def make_p2pd_pair_ip4(enable_control, enable_connmgr, enable_dht, enable_pubsub): - control_maddr = Multiaddr(f"/ip4/127.0.0.1/tcp/{find_open_port()}") - listen_maddr = Multiaddr(f"/ip4/127.0.0.1/tcp/{find_open_port()}") - async with _make_p2pd_pair( - control_maddr=control_maddr, - listen_maddr=listen_maddr, - enable_control=enable_control, - enable_connmgr=enable_connmgr, - enable_dht=enable_dht, - enable_pubsub=enable_pubsub, - ) as pair: - yield pair - - -@asynccontextmanager -async def _make_p2pd_pair( - control_maddr, - listen_maddr, - enable_control, - enable_connmgr, - enable_dht, - enable_pubsub, -): - p2pd = Daemon( - control_maddr=control_maddr, - enable_control=enable_control, - enable_connmgr=enable_connmgr, - enable_dht=enable_dht, - enable_pubsub=enable_pubsub, - ) - # wait for daemon ready - await p2pd.wait_until_ready() - client = Client(control_maddr=control_maddr, listen_maddr=listen_maddr) - try: - async with client.listen(): - yield DaemonTuple(daemon=p2pd, client=client) - finally: - if not p2pd.is_closed: - p2pd.close() - - -@pytest.fixture -async def p2pcs( - num_p2pds, - enable_control, - enable_connmgr, - enable_dht, - enable_pubsub, - func_make_p2pd_pair, -): +async def p2pcs(): # TODO: Change back to gather style async with AsyncExitStack() as stack: p2pd_tuples = [ await stack.enter_async_context( - func_make_p2pd_pair( - enable_control=enable_control, - enable_connmgr=enable_connmgr, - enable_dht=enable_dht, - enable_pubsub=enable_pubsub, + FUNC_MAKE_P2PD_PAIR( + enable_control=ENABLE_CONTROL, + enable_connmgr=ENABLE_CONNMGR, + enable_dht=ENABLE_DHT, + enable_pubsub=ENABLE_PUBSUB, ) ) - for _ in range(num_p2pds) + for _ in range(NUM_P2PDS) ] yield tuple(p2pd_tuple.client for p2pd_tuple in p2pd_tuples) -@pytest.mark.parametrize( - "enable_control, func_make_p2pd_pair", ((True, make_p2pd_pair_unix),) -) @pytest.mark.asyncio async def test_client_identify_unix_socket(p2pcs): await p2pcs[0].identify() -@pytest.mark.parametrize("enable_control", (True,)) @pytest.mark.asyncio async def test_client_identify(p2pcs): await p2pcs[0].identify() -@pytest.mark.parametrize("enable_control", (True,)) @pytest.mark.asyncio async def test_client_connect_success(p2pcs): peer_id_0, maddrs_0 = await p2pcs[0].identify() @@ -558,14 +330,13 @@ async def test_client_connect_success(p2pcs): await p2pcs[1].connect(peer_id_0, maddrs_0) -@pytest.mark.parametrize("enable_control", (True,)) @pytest.mark.asyncio -async def test_client_connect_failure(peer_id_random, p2pcs): +async def test_client_connect_failure(p2pcs): peer_id_1, maddrs_1 = await p2pcs[1].identify() await p2pcs[0].identify() # test case: `peer_id` mismatches with pytest.raises(ControlFailure): - await p2pcs[0].connect(peer_id_random, maddrs_1) + await p2pcs[0].connect(PEER_ID_RANDOM, maddrs_1) # test case: empty maddrs with pytest.raises(ControlFailure): await p2pcs[0].connect(peer_id_1, []) @@ -574,31 +345,11 @@ async def test_client_connect_failure(peer_id_random, p2pcs): await p2pcs[0].connect(peer_id_1, [Multiaddr("/ip4/127.0.0.1/udp/0")]) -async def _check_connection(p2pd_tuple_0, p2pd_tuple_1): - peer_id_0, _ = await p2pd_tuple_0.identify() - peer_id_1, _ = await p2pd_tuple_1.identify() - peers_0 = [pinfo.peer_id for pinfo in await p2pd_tuple_0.list_peers()] - peers_1 = [pinfo.peer_id for pinfo in await p2pd_tuple_1.list_peers()] - return (peer_id_0 in peers_1) and (peer_id_1 in peers_0) - - -async def connect_safe(p2pd_tuple_0, p2pd_tuple_1): - peer_id_1, maddrs_1 = await p2pd_tuple_1.identify() - await p2pd_tuple_0.connect(peer_id_1, maddrs_1) - await try_until_success( - functools.partial( - _check_connection, p2pd_tuple_0=p2pd_tuple_0, p2pd_tuple_1=p2pd_tuple_1 - ) - ) - - -@pytest.mark.parametrize("enable_control", (True,)) @pytest.mark.asyncio async def test_connect_safe(p2pcs): await connect_safe(p2pcs[0], p2pcs[1]) -@pytest.mark.parametrize("enable_control", (True,)) @pytest.mark.asyncio async def test_client_list_peers(p2pcs): # test case: no peers @@ -614,11 +365,10 @@ async def test_client_list_peers(p2pcs): assert len(await p2pcs[2].list_peers()) == 1 -@pytest.mark.parametrize("enable_control", (True,)) @pytest.mark.asyncio -async def test_client_disconnect(peer_id_random, p2pcs): +async def test_client_disconnect(p2pcs): # test case: disconnect a peer without connections - await p2pcs[1].disconnect(peer_id_random) + await p2pcs[1].disconnect(PEER_ID_RANDOM) # test case: disconnect peer_id_0, _ = await p2pcs[0].identify() await connect_safe(p2pcs[0], p2pcs[1]) @@ -633,7 +383,6 @@ async def test_client_disconnect(peer_id_random, p2pcs): assert len(await p2pcs[1].list_peers()) == 0 -@pytest.mark.parametrize("enable_control", (True,)) @pytest.mark.asyncio async def test_client_stream_open_success(p2pcs): peer_id_1, maddrs_1 = await p2pcs[1].identify() @@ -663,7 +412,6 @@ async def handle_proto(stream_info, reader, writer): writer.close() -@pytest.mark.parametrize("enable_control", (True,)) @pytest.mark.asyncio async def test_client_stream_open_failure(p2pcs): peer_id_1, _ = await p2pcs[1].identify() @@ -684,7 +432,6 @@ async def handle_proto(stream_info, reader, writer): await p2pcs[0].stream_open(peer_id_1, ("another_protocol",)) -@pytest.mark.parametrize("enable_control", (True,)) @pytest.mark.asyncio async def test_client_stream_handler_success(p2pcs): peer_id_1, _ = await p2pcs[1].identify() @@ -758,7 +505,6 @@ async def handler_third(stream_info, reader, writer): await event_third.wait() -@pytest.mark.parametrize("enable_control", (True,)) @pytest.mark.asyncio async def test_client_stream_handler_failure(p2pcs): peer_id_1, _ = await p2pcs[1].identify() diff --git a/tests/test_utils/__init__.py b/tests/test_utils/__init__.py new file mode 100644 index 000000000..7d044f8fb --- /dev/null +++ b/tests/test_utils/__init__.py @@ -0,0 +1,192 @@ +import asyncio +import functools +import os +import subprocess +import time +import uuid +from contextlib import asynccontextmanager +from typing import NamedTuple + +from multiaddr import Multiaddr, protocols + +from hivemind import find_open_port +from hivemind.p2p.p2p_daemon_bindings.p2pclient import Client + + +TIMEOUT_DURATION = 30 # seconds + + +async def try_until_success(coro_func, timeout=TIMEOUT_DURATION): + """ + Keep running ``coro_func`` until the time is out. + All arguments of ``coro_func`` should be filled, i.e. it should be called without arguments. + """ + t_start = time.monotonic() + while True: + result = await coro_func() + if result: + break + if (time.monotonic() - t_start) >= timeout: + # timeout + assert False, f"{coro_func} still failed after `{timeout}` seconds" + await asyncio.sleep(0.01) + + +class Daemon: + control_maddr = None + proc_daemon = None + log_filename = "" + f_log = None + closed = None + + def __init__( + self, control_maddr, enable_control, enable_connmgr, enable_dht, enable_pubsub + ): + self.control_maddr = control_maddr + self.enable_control = enable_control + self.enable_connmgr = enable_connmgr + self.enable_dht = enable_dht + self.enable_pubsub = enable_pubsub + self.is_closed = False + self._start_logging() + self._run() + + def _start_logging(self): + name_control_maddr = str(self.control_maddr).replace("/", "_").replace(".", "_") + self.log_filename = f"/tmp/log_p2pd{name_control_maddr}.txt" + self.f_log = open(self.log_filename, "wb") + + def _run(self): + cmd_list = ["hivemind/hivemind_cli/p2pd", f"-listen={str(self.control_maddr)}"] + cmd_list += [f"-hostAddrs=/ip4/127.0.0.1/tcp/{find_open_port()}"] + if self.enable_connmgr: + cmd_list += ["-connManager=true", "-connLo=1", "-connHi=2", "-connGrace=0"] + if self.enable_dht: + cmd_list += ["-dht=true"] + if self.enable_pubsub: + cmd_list += ["-pubsub=true", "-pubsubRouter=gossipsub"] + self.proc_daemon = subprocess.Popen( + cmd_list, stdout=self.f_log, stderr=self.f_log, bufsize=0 + ) + + async def wait_until_ready(self): + lines_head_pattern = (b"Control socket:", b"Peer ID:", b"Peer Addrs:") + lines_head_occurred = {line: False for line in lines_head_pattern} + + with open(self.log_filename, "rb") as f_log_read: + + async def read_from_daemon_and_check(): + line = f_log_read.readline() + for head_pattern in lines_head_occurred: + if line.startswith(head_pattern): + lines_head_occurred[head_pattern] = True + return all([value for _, value in lines_head_occurred.items()]) + + await try_until_success(read_from_daemon_and_check) + + # sleep for a while in case that the daemon haven't been ready after emitting these lines + await asyncio.sleep(0.1) + + def close(self): + if self.is_closed: + return + self.proc_daemon.terminate() + self.proc_daemon.wait() + self.f_log.close() + self.is_closed = True + + +class DaemonTuple(NamedTuple): + daemon: Daemon + client: Client + + +class ConnectionFailure(Exception): + pass + + +@asynccontextmanager +async def make_p2pd_pair_unix( + enable_control, enable_connmgr, enable_dht, enable_pubsub +): + name = str(uuid.uuid4())[:8] + control_maddr = Multiaddr(f"/unix/tmp/test_p2pd_control_{name}.sock") + listen_maddr = Multiaddr(f"/unix/tmp/test_p2pd_listen_{name}.sock") + # Remove the existing unix socket files if they are existing + try: + os.unlink(control_maddr.value_for_protocol(protocols.P_UNIX)) + except FileNotFoundError: + pass + try: + os.unlink(listen_maddr.value_for_protocol(protocols.P_UNIX)) + except FileNotFoundError: + pass + async with _make_p2pd_pair( + control_maddr=control_maddr, + listen_maddr=listen_maddr, + enable_control=enable_control, + enable_connmgr=enable_connmgr, + enable_dht=enable_dht, + enable_pubsub=enable_pubsub, + ) as pair: + yield pair + + +@asynccontextmanager +async def make_p2pd_pair_ip4(enable_control, enable_connmgr, enable_dht, enable_pubsub): + control_maddr = Multiaddr(f"/ip4/127.0.0.1/tcp/{find_open_port()}") + listen_maddr = Multiaddr(f"/ip4/127.0.0.1/tcp/{find_open_port()}") + async with _make_p2pd_pair( + control_maddr=control_maddr, + listen_maddr=listen_maddr, + enable_control=enable_control, + enable_connmgr=enable_connmgr, + enable_dht=enable_dht, + enable_pubsub=enable_pubsub, + ) as pair: + yield pair + + +@asynccontextmanager +async def _make_p2pd_pair( + control_maddr, + listen_maddr, + enable_control, + enable_connmgr, + enable_dht, + enable_pubsub, +): + p2pd = Daemon( + control_maddr=control_maddr, + enable_control=enable_control, + enable_connmgr=enable_connmgr, + enable_dht=enable_dht, + enable_pubsub=enable_pubsub, + ) + # wait for daemon ready + await p2pd.wait_until_ready() + client = Client(control_maddr=control_maddr, listen_maddr=listen_maddr) + try: + async with client.listen(): + yield DaemonTuple(daemon=p2pd, client=client) + finally: + if not p2pd.is_closed: + p2pd.close() + + +async def _check_connection(p2pd_tuple_0, p2pd_tuple_1): + peer_id_0, _ = await p2pd_tuple_0.identify() + peer_id_1, _ = await p2pd_tuple_1.identify() + peers_0 = [pinfo.peer_id for pinfo in await p2pd_tuple_0.list_peers()] + peers_1 = [pinfo.peer_id for pinfo in await p2pd_tuple_1.list_peers()] + return (peer_id_0 in peers_1) and (peer_id_1 in peers_0) + + +async def connect_safe(p2pd_tuple_0, p2pd_tuple_1): + peer_id_1, maddrs_1 = await p2pd_tuple_1.identify() + await p2pd_tuple_0.connect(peer_id_1, maddrs_1) + await try_until_success( + functools.partial( + _check_connection, p2pd_tuple_0=p2pd_tuple_0, p2pd_tuple_1=p2pd_tuple_1 + ) + ) \ No newline at end of file From 16e54cad39423a659bb725b438d75cc0f9315e3c Mon Sep 17 00:00:00 2001 From: Maxim Kashirin Date: Wed, 12 May 2021 19:57:30 +0300 Subject: [PATCH 79/81] remove obvious comments --- tests/test_p2p_daemon_bindings.py | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) diff --git a/tests/test_p2p_daemon_bindings.py b/tests/test_p2p_daemon_bindings.py index 7f87cbd88..9d7c203cf 100644 --- a/tests/test_p2p_daemon_bindings.py +++ b/tests/test_p2p_daemon_bindings.py @@ -127,33 +127,27 @@ async def test_read_write_unsigned_varint_max_bits_edge(max_bits): def test_peer_id(): - # test initialized with bytes assert PEER_ID.to_bytes() == PEER_ID_BYTES assert PEER_ID.to_string() == PEER_ID_STRING - # test initialized with string + peer_id_2 = PeerID.from_base58(PEER_ID_STRING) assert peer_id_2.to_bytes() == PEER_ID_BYTES assert peer_id_2.to_string() == PEER_ID_STRING - # test equal assert PEER_ID == peer_id_2 - # test not equal peer_id_3 = PeerID.from_base58("QmbmfNDEth7Ucvjuxiw3SP3E4PoJzbk7g4Ge6ZDigbCsNp") assert PEER_ID != peer_id_3 def test_stream_info(): proto = "123" - # test case: `StreamInfo.__init__` si = StreamInfo(PEER_ID, MADDR, proto) assert si.peer_id == PEER_ID assert si.addr == MADDR assert si.proto == proto - # test case: `StreamInfo.to_pb` pb_si = si.to_protobuf() assert pb_si.peer == PEER_ID.to_bytes() assert pb_si.addr == MADDR.to_bytes() assert pb_si.proto == si.proto - # test case: `StreamInfo.from_pb` si_1 = StreamInfo.from_protobuf(pb_si) assert si_1.peer_id == PEER_ID assert si_1.addr == MADDR @@ -162,10 +156,8 @@ def test_stream_info(): def test_peer_info(): pi = PeerInfo(PEER_ID, [MADDR]) - # test case: `PeerInfo.__init__` assert pi.peer_id == PEER_ID assert pi.addrs == [MADDR] - # test case: `PeerInfo.from_pb` pi_pb = p2pd_pb.PeerInfo(id=PEER_ID.to_bytes(), addrs=[MADDR.to_bytes()]) pi_1 = PeerInfo.from_protobuf(pi_pb) assert pi.peer_id == pi_1.peer_id From 3b42ae8f8d0025b197609cfec5f0b6733999e0c3 Mon Sep 17 00:00:00 2001 From: Maxim Kashirin Date: Thu, 13 May 2021 08:50:56 +0300 Subject: [PATCH 80/81] raw bytes to pb creation --- tests/test_p2p_daemon_bindings.py | 78 ++++++++++++++++++++++++------- 1 file changed, 62 insertions(+), 16 deletions(-) diff --git a/tests/test_p2p_daemon_bindings.py b/tests/test_p2p_daemon_bindings.py index 9d7c203cf..172bded54 100644 --- a/tests/test_p2p_daemon_bindings.py +++ b/tests/test_p2p_daemon_bindings.py @@ -212,9 +212,30 @@ def test_control_client_ctor_default_listen_maddr(): @pytest.mark.parametrize( "msg_bytes", ( - b'\x08\x00"R\n"\x12 F\xec\xd3p0X\xbeT\x95p^\xc8{\xc8\x13\xa3\x9c\x84d\x0b\x1b\xbb\xa0P\x98w\xc1\xb3\x981i\x16\x12\x02\xa2\x02\x12\x08\x04\x7f\x00\x00\x01\x06\xc7\xb6\x12\x08\x04\xc0\xa8\n\x87\x06\xc7\xb6\x12\x14)\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x01\x06\xc7\xb7', # noqa: E501 - b'\x08\x00"R\n"\x12 \xd0\xf0 \x9a\xc6v\xa6\xd3;\xcac|\x95\x94\xa0\xe6:\nM\xc53T\x0e\xf0\x89\x8e(\x0c\xb9\xf7\\\xa5\x12\x02\xa2\x02\x12\x08\x04\x7f\x00\x00\x01\x06\xc9%\x12\x08\x04\xc0\xa8\n\x87\x06\xc9%\x12\x14)\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x01\x06\xc9&', # noqa: E501 - b'\x08\x00"R\n"\x12 \xc3\xc3\xee\x18i\x8a\xde\x13\xa9y\x905\xeb\xcb\xa4\xd07\x14\xbe\xf4\xf8\x1b\xe8[g94\x94\xe3f\x18\xa9\x12\x02\xa2\x02\x12\x08\x04\x7f\x00\x00\x01\x06\xc9`\x12\x08\x04\xc0\xa8\n\x87\x06\xc9`\x12\x14)\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x01\x06\xc9a', # noqa: E501 + p2pd_pb.Response( + type=p2pd_pb.Response.Type.OK, + identify=p2pd_pb.IdentifyResponse( + id=PeerID.from_base58('QmT7WhTne9zBLfAgAJt9aiZ8jZ5BxJGowRubxsHYmnyzUd').to_bytes(), + addrs=[Multiaddr('/p2p-circuit').to_bytes(), Multiaddr('/ip4/127.0.0.1/tcp/51126').to_bytes(), + Multiaddr('/ip4/192.168.10.135/tcp/51126').to_bytes(), + Multiaddr('/ip6/::1/tcp/51127').to_bytes()] + )).SerializeToString(), + p2pd_pb.Response( + type=p2pd_pb.Response.Type.OK, + identify=p2pd_pb.IdentifyResponse( + id=PeerID.from_base58('QmcQFt2MFfCZ9AxzUCNrk4k7TtMdZZvAAteaA6tHpBKdrk').to_bytes(), + addrs=[Multiaddr('/p2p-circuit').to_bytes(), Multiaddr('/ip4/127.0.0.1/tcp/51493').to_bytes(), + Multiaddr('/ip4/192.168.10.135/tcp/51493').to_bytes(), + Multiaddr('/ip6/::1/tcp/51494').to_bytes()] + )).SerializeToString(), + p2pd_pb.Response( + type=p2pd_pb.Response.Type.OK, + identify=p2pd_pb.IdentifyResponse( + id=PeerID.from_base58('QmbWqVVoz7v9LS9ZUQAhyyfdFJY3iU8ZrUY3XQozoTA5cc').to_bytes(), + addrs=[Multiaddr('/p2p-circuit').to_bytes(), Multiaddr('/ip4/127.0.0.1/tcp/51552').to_bytes(), + Multiaddr('/ip4/192.168.10.135/tcp/51552').to_bytes(), + Multiaddr('/ip6/::1/tcp/51553').to_bytes()] + )).SerializeToString(), ), # give test cases ids to prevent bytes from ruining the terminal ids=("pb example Response 0", "pb example Response 1", "pb example Response 2"), @@ -232,24 +253,46 @@ async def test_read_pbmsg_safe_valid(msg_bytes): @pytest.mark.parametrize( - "pb_msg, msg_bytes", + "pb_type, pb_msg", ( ( - p2pd_pb.Response(), - b'Z\x08\x00*V\x08\x01\x12R\n"\x12 \x03\x8d\xf5\xd4(/#\xd6\xed\xa5\x1bU\xb8s\x8c\xfa\xad\xfc{\x04\xe3\xecw\xdeK\xc9,\xfe\x9c\x00:\xc8\x12\x02\xa2\x02\x12\x08\x04\x7f\x00\x00\x01\x06\xdea\x12\x08\x04\xc0\xa8\n\x87\x06\xdea\x12\x14)\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x01\x06\xdeb', # noqa: E501 + p2pd_pb.Response, + p2pd_pb.Response( + type=p2pd_pb.Response.Type.OK, + dht=p2pd_pb.DHTResponse( + type=p2pd_pb.DHTResponse.Type.VALUE, + peer=p2pd_pb.PeerInfo( + id=PeerID.from_base58('QmNaXUy78W9moQ9APCoKaTtPjLcEJPN9hRBCqErY7o2fQs').to_bytes(), + addrs=[Multiaddr('/p2p-circuit').to_bytes(), Multiaddr('/ip4/127.0.0.1/tcp/56929').to_bytes(), + Multiaddr('/ip4/192.168.10.135/tcp/56929').to_bytes(), + Multiaddr('/ip6/::1/tcp/56930').to_bytes()] + ) + ) + ), ), - (p2pd_pb.Request(), b"\x02\x08\x05"), + (p2pd_pb.Request, p2pd_pb.Request(type=p2pd_pb.Request.Type.LIST_PEERS)), ( - p2pd_pb.DHTRequest(), - b'&\x08\x00\x12"\x12 \xd5\x0b\x18/\x9e\xa5G\x06.\xdd\xebW\xf0N\xf5\x0eW\xd3\xec\xdf\x06\x02\xe2\x89\x1e\xf0\xbb.\xc0\xbdE\xb8', # noqa: E501 + p2pd_pb.DHTRequest, + p2pd_pb.DHTRequest(type=p2pd_pb.DHTRequest.Type.FIND_PEER, + peer=PeerID.from_base58('QmcgHMuEhqdLHDVeNjiCGU7Ds6E7xK3f4amgiwHNPKKn7R').to_bytes()), ), ( - p2pd_pb.DHTResponse(), - b'V\x08\x01\x12R\n"\x12 wy\xe2\xfa\x11\x9e\xe2\x84X]\x84\xf8\x98\xba\x8c\x8cQ\xd7,\xb59\x1e!G\x92\x86G{\x141\xe9\x1b\x12\x02\xa2\x02\x12\x08\x04\x7f\x00\x00\x01\x06\xdeA\x12\x08\x04\xc0\xa8\n\x87\x06\xdeA\x12\x14)\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x01\x06\xdeB', # noqa: E501 + p2pd_pb.DHTResponse, + p2pd_pb.DHTResponse( + type=p2pd_pb.DHTResponse.Type.VALUE, + peer=p2pd_pb.PeerInfo( + id=PeerID.from_base58('QmWP32GhEyXVQsLXFvV81eadDC8zQRZxZvJK359rXxLquk').to_bytes(), + addrs=[Multiaddr('/p2p-circuit').to_bytes(), Multiaddr('/ip4/127.0.0.1/tcp/56897').to_bytes(), + Multiaddr('/ip4/192.168.10.135/tcp/56897').to_bytes(), + Multiaddr('/ip6/::1/tcp/56898').to_bytes()] + ) + ), ), ( - p2pd_pb.StreamInfo(), - b';\n"\x12 \xf6\x9e=\x9f\xc1J\xfe\x02\x93k!S\x80\xa0\xcc(s\xea&\xbe\xed\x9274qTI\xc1\xf7\xb6\xbd7\x12\x08\x04\x7f\x00\x00\x01\x06\xde\xc5\x1a\x0bprotocol123', # noqa: E501 + p2pd_pb.StreamInfo, + p2pd_pb.StreamInfo(peer=PeerID.from_base58('QmewLxB46MftfxQiunRgJo2W8nW4Lh5NLEkRohkHhJ4wW6').to_bytes(), + addr=Multiaddr('/ip4/127.0.0.1/tcp/57029').to_bytes(), + proto=b'protocol123'), ), ), ids=( @@ -261,11 +304,14 @@ async def test_read_pbmsg_safe_valid(msg_bytes): ), ) @pytest.mark.asyncio -async def test_write_pbmsg(pb_msg, msg_bytes): +async def test_write_pbmsg(pb_type, pb_msg): + msg_bytes = bytes(chr(pb_msg.ByteSize()), 'utf-8') + pb_msg.SerializeToString() + pb_obj = pb_type() + s_read = MockReaderWriter(msg_bytes) - await read_pbmsg_safe(s_read, pb_msg) + await read_pbmsg_safe(s_read, pb_obj) s_write = MockReaderWriter() - await write_pbmsg(s_write, pb_msg) + await write_pbmsg(s_write, pb_obj) assert msg_bytes == s_write.getvalue() From 3229e6f629f2729594a799f4cb1e59e0a83f2325 Mon Sep 17 00:00:00 2001 From: MaximKsh Date: Tue, 1 Jun 2021 16:58:29 +0300 Subject: [PATCH 81/81] Update tests/test_utils/__init__.py add newline Co-authored-by: justheuristic --- tests/test_utils/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_utils/__init__.py b/tests/test_utils/__init__.py index 7d044f8fb..622dd479f 100644 --- a/tests/test_utils/__init__.py +++ b/tests/test_utils/__init__.py @@ -189,4 +189,4 @@ async def connect_safe(p2pd_tuple_0, p2pd_tuple_1): functools.partial( _check_connection, p2pd_tuple_0=p2pd_tuple_0, p2pd_tuple_1=p2pd_tuple_1 ) - ) \ No newline at end of file + )