Fix LibP2P-Daemon installation in setup.py

.circleci/config.yml

@@ -9,6 +9,10 @@ jobs:
       - restore_cache:
           keys:
             - v1-{{ checksum "requirements.txt" }}-{{ checksum "requirements-dev.txt" }}
+      - run: |
+          wget https://golang.org/dl/go1.16.2.linux-amd64.tar.gz
+          tar -C ~/ -xzf go1.16.2.linux-amd64.tar.gz
+          echo "export PATH=~/go/bin:$PATH" >> $BASH_ENV
       - run: pip install -r requirements.txt
       - run: pip install -r requirements-dev.txt
       - save_cache:
@@ -29,6 +33,10 @@ jobs:
       - restore_cache:
           keys:
             - v1-{{ checksum "requirements.txt" }}-{{ checksum "requirements-dev.txt" }}
+      - run: |
+          wget https://golang.org/dl/go1.16.2.linux-amd64.tar.gz
+          tar -C ~/ -xzf go1.16.2.linux-amd64.tar.gz
+          echo "export PATH=~/go/bin:$PATH" >> $BASH_ENV
       - run: pip install -r requirements.txt
       - run: pip install -r requirements-dev.txt
       - save_cache:
@@ -49,6 +57,10 @@ jobs:
       - restore_cache:
           keys:
             - v1-{{ checksum "requirements.txt" }}-{{ checksum "requirements-dev.txt" }}
+      - run: |
+          wget https://golang.org/dl/go1.16.2.linux-amd64.tar.gz
+          tar -C ~/ -xzf go1.16.2.linux-amd64.tar.gz
+          echo "export PATH=~/go/bin:$PATH" >> $BASH_ENV
       - run: pip install -r requirements.txt
       - run: pip install -r requirements-dev.txt
       - save_cache:

.gitignore

@@ -78,3 +78,6 @@ debian/files
 
 # protobuf stuff
 hivemind/proto/*_pb2*
+
+# libp2p-daemon binary
+hivemind/hivemind_cli/p2pd

setup.py

@@ -6,6 +6,8 @@
 import urllib.request
 import tarfile
 import tempfile
+import json
+import hashlib
 
 from packaging import version
 from pkg_resources import parse_requirements
@@ -14,13 +16,16 @@
 from setuptools.command.install import install
 
 
+P2PD_VERSION = 'v0.3.1'
+P2PD_CHECKSUM = '5094d094740f4e375afe80a5683b1bb2'
+
 here = os.path.abspath(os.path.dirname(__file__))
 
 
 class cd:
     """Context manager for changing the current working directory"""
-    def __init__(self, newPath):
-        self.newPath = os.path.expanduser(newPath)
+    def __init__(self, new_path):
+        self.newPath = os.path.expanduser(new_path)
 
     def __enter__(self):
         self.savedPath = os.getcwd()
@@ -30,6 +35,14 @@ def __exit__(self, etype, value, traceback):
         os.chdir(self.savedPath)
 
 
+def md5(fname):
+    hash_md5 = hashlib.md5()
+    with open(fname, "rb") as f:
+        for chunk in iter(lambda: f.read(4096), b""):
+            hash_md5.update(chunk)
+    return hash_md5.hexdigest()
+
+
 def proto_compile(output_path):
     import grpc_tools.protoc
 
@@ -49,16 +62,15 @@ def proto_compile(output_path):
             file.truncate()
 
 
-def install_libp2p_daemon():
-    # check go version:
+def libp2p_build_install():
     try:
         proc = subprocess.Popen(['go', 'version'],
                                 stdout=subprocess.PIPE)
         result, _ = proc.communicate()
         result = result.decode('ascii', 'replace')
         _, _, v, _ = result.split(' ')
         v = v.lstrip('go')
-    
+
         if version.parse(v) < version.parse("1.13"):
             raise EnvironmentError(f'newer version of go required: must be >= 1.13, found {version}')
 
@@ -67,38 +79,44 @@ def install_libp2p_daemon():
 
     with tempfile.TemporaryDirectory() as tempdir:
         url = 'https://github.com/libp2p/go-libp2p-daemon/archive/master.tar.gz'
-        dest = os.path.join(tempdir, 'libp2p-daemon.tar.gz')   
+        dest = os.path.join(tempdir, 'libp2p-daemon.tar.gz')
         urllib.request.urlretrieve(url, os.path.join(tempdir, dest))
-            
+
         tar = tarfile.open(dest, 'r:gz')
         tar.extractall(tempdir)
         tar.close()
-            
+
         with cd(os.path.join(tempdir, 'go-libp2p-daemon-master', 'p2pd')):
             status = os.system(f'go build -o {os.path.join(here, "hivemind/hivemind_cli", "p2pd")}')
             if status:
-                raise RuntimeError('Failed to build or install libp2p-daemon:'\
+                raise RuntimeError('Failed to build or install libp2p-daemon:'
                                    f' exited with status code :{status}')
 
 
-class ProtoCompileInstall(install):
+def libp2p_download_install():
+    install_path = os.path.join(here, 'hivemind/hivemind_cli/')
+    binary_path = os.path.join(install_path, 'p2pd')
+    if 'p2pd' not in os.listdir(install_path) or md5(binary_path) != P2PD_CHECKSUM:
+        print('Downloading Peer to Peer Daemon')
+        url = f'https://github.com/learning-at-home/go-libp2p-daemon/releases/download/{P2PD_VERSION}/p2pd'
+        urllib.request.urlretrieve(url, binary_path)
+        os.chmod(binary_path, 777)
+
+
+class Install(install):
     def run(self):
+        libp2p_download_install()
         proto_compile(os.path.join(self.build_lib, 'hivemind', 'proto'))
         super().run()
 
 
-class ProtoCompileDevelop(develop):
+class Develop(develop):
     def run(self):
+        libp2p_build_install()
         proto_compile(os.path.join('hivemind', 'proto'))
         super().run()
 
 
-class LibP2PInstall(install):
-    def run(self):
-        install_libp2p_daemon()
-
-
-
 with open('requirements.txt') as requirements_file:
     install_requires = list(map(str, parse_requirements(requirements_file)))
 
@@ -120,7 +138,7 @@ def run(self):
 setup(
     name='hivemind',
     version=version_string,
-    cmdclass={'install': ProtoCompileInstall, 'develop': ProtoCompileDevelop, 'libp2p': LibP2PInstall},
+    cmdclass={'install': Install, 'develop': Develop},
     description='Decentralized deep learning in PyTorch',
     long_description='Decentralized deep learning in PyTorch. Built to train giant models on '
                      'thousands of volunteers across the world.',