Chia DB download via torrent. GPU pass-thru configuration for workers.

guydavis · Mar 20, 2023 · f0fdf81 · f0fdf81
1 parent 7046c1e
commit f0fdf81
Show file tree

Hide file tree

Showing 5 changed files with 110 additions and 9 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -7,6 +7,7 @@ All notable changes to this project will be documented in this file. The format
  - Support for running a blockchain timelord with environment variable `mode` set to `fullnode,timelord`.
  - Optional config setting to restart fork fullnodes if they consume too much memory.
 ## Changed
+ - On fresh install, optionally, via libtorrent, download a recent [Chia database checkpoint](https://www.chia.net/downloads/).
  - Fixes for Gigahorse GPU plotting including 64 GB RAM mode using the `tmp3` SSD plotting path.
  - Fix for Gigahorse Alerts, please reset earlier broken settings file from [this one](https://github.com/guydavis/machinaris/blob/develop/config/chiadog/gigahorse.sample.yaml).
  - Fix for container stop signal to cleanly shutdown forks too, as was already happening for Chia.

diff --git a/docker/dockerfile-jammy.base b/docker/dockerfile-jammy.base
@@ -31,6 +31,7 @@ RUN apt-get update \
 		python3.10-venv \
 		python3-dev \
 		python3-pip \
+		python3-libtorrent \ 
 		python-is-python3 \
 		smartmontools \
 		sqlite3 \

diff --git a/scripts/chiadb_download.py b/scripts/chiadb_download.py
@@ -0,0 +1,38 @@
+#
+# Via libtorrent, downloads the recent database checkpoint from https://www.chia.net/downloads/
+#
+
+import sys
+import time
+import traceback
+
+import libtorrent as lt
+
+try:
+    ses = lt.session({'listen_interfaces': '0.0.0.0:6881'})
+
+    info = lt.torrent_info(sys.argv[1])
+    h = ses.add_torrent({'ti': info, 'save_path': '.'})
+    s = h.status()
+    print('Starting torrent download of: ', s.name)
+
+    while (not s.is_seeding):
+        s = h.status()
+
+        print('\r%.2f%% complete (down: %.1f kB/s up: %.1f kB/s peers: %d) %s' % (
+            s.progress * 100, s.download_rate / 1000, s.upload_rate / 1000,
+            s.num_peers, s.state), end=' ')
+
+        #alerts = ses.pop_alerts()
+        #for a in alerts:
+        #    if a.category() & lt.alert.category_t.error_notification:
+        #        print(a)
+
+        sys.stdout.flush()
+
+        time.sleep(1)
+
+    print(h.status().name, 'Database download complete.')
+except:
+    print(h.status().name, 'Database download failed.')
+    traceback.print_exc()
diff --git a/scripts/forks/chia_launch.sh b/scripts/forks/chia_launch.sh
@@ -36,15 +36,16 @@ if [[ "${blockchain_db_download}" == 'true' ]] \
   echo "Downloading Chia blockchain DB (many GBs in size) on first launch..."
   echo "Please be patient as takes minutes now, but saves days of syncing time later."
   mkdir -p /root/.chia/mainnet/db/chia && cd /root/.chia/mainnet/db/chia
-  # Latest Blockchain DB download from direct from https://sweetchia.com/
-  db_url=$(curl -s https://sweetchia.com | grep -Po "https:.*/blockchain_v2_mainnet-\d{4}-\d{2}-\d{2}-\d{4}.7z" | shuf -n 1)
-  echo "Please be patient! Downloading blockchain database from: "
-  echo "    ${db_url}"
-  curl -skLJ -O ${db_url}
-  p7zip --decompress --force blockchain_v2_mainnet*.7z
+  # Latest Blockchain DB download
+  torrent=$(curl -s https://www.chia.net/downloads/ | grep -Po "https:.*/blockchain_v2_mainnet.\d{4}-\d{2}-\d{2}.sqlite.gz.torrent")
+  echo "Please be patient! Downloading blockchain database (via libtorrent) from: "
+  echo "    ${torrent}"
+  curl -kLJ -O ${torrent}
+  python /machinaris/scripts/chiadb_download.py $PWD/$torrent
+  gunzip *.gz
   cd /root/.chia/mainnet/db
   mv /root/.chia/mainnet/db/chia/blockchain_v2_mainnet.sqlite .
-  rm -rf /root/.chia/mainnet/db/chia
+  rm -rf /root/.chia/mainnet/db/chia 
 fi
 
 mkdir -p /root/.chia/mainnet/log

diff --git a/web/templates/worker_launch.html b/web/templates/worker_launch.html
@@ -363,6 +363,8 @@
         function updateDockerCompose(errors) {
             var line_end = '\n';
             var os = getRadioButtonValue('os')
+            var gpu_type = $('#gpu_type').val()
+            var gpu_blockchain = $('#gpu_blockchain').val()
             cmd = "version: '3.7'" + line_end;
             cmd += "services:"  + line_end + line_end;
 
@@ -483,6 +485,40 @@
                     if (remote_windows_shares.length > 0) {
                         cmd += '            - remote_shares=' + remote_windows_shares.join() + line_end;
                     }
+                    // Add new environment variables above this section as it has both env vars and runtime and devices etc
+                    if (blockchain == gpu_blockchain) {
+                        if (gpu_type == 'nvidia') {
+                            cmd += '            - OPENCL_GPU=nvidia' + line_end;
+                            cmd += '            - NVIDIA_VISIBLE_DEVICES=0' + line_end;
+                            cmd += '            - NVIDIA_DRIVER_CAPABILITIES=compute,utility' + line_end;
+                            cmd += '        runtime: nvidia' + line_end;
+                        }
+                        else if (gpu_type == 'amd') {
+                            if (gpu_blockchain == 'chia') {
+                                errors.push("{{_('Currently Chia and Bladebit require a Nvidia GPU.  Other cards are not supported unfortunately.')}}");
+                            }
+                            else {
+                                cmd += '            - OPENCL_GPU=amd' + line_end;
+                                cmd += '            - ROC_ENABLE_PRE_VEGA=1' + line_end;
+                                cmd += '        devices:' + line_end;
+                                cmd += '            - "/dev/kfd:/dev/kfd"' + line_end;
+                                cmd += '            - "/dev/dri/:/dev/dri/"' + line_end;
+                            }
+                        }
+                        else if (gpu_type == 'intel') {
+                            if (gpu_blockchain == 'chia') {
+                                errors.push("{{_('Currently Chia and Bladebit require a Nvidia GPU.  Other cards are not supported unfortunately.')}}");
+                            }
+                            else if (gpu_blockchain == 'gigahorse') {
+                                errors.push("{{_('Currently Gigahorse requires a Nvidia or AMD GPU.  Intel cards are not supported unfortunately.')}}");
+                            }
+                            else {
+                                cmd += '            - OPENCL_GPU=intel' + line_end;
+                                cmd += '        devices:' + line_end;
+                                cmd += '            - "/dev/dri/:/dev/dri/"' + line_end;
+                            }
+                        }
+                    }
                     cmd += '        ports:' + line_end;
                     if (blockchain == "chia") {
                         cmd += '            - 8926:8926' + line_end;
@@ -940,6 +976,32 @@ <h4>{{_('Volume Mounts')}}</h4>
                             class="fs-4 bi-dash text-white text-right"></i></a>
                 </div>
             </div>
+            <div class="row mb-3" id="gpus_row">
+                <div class="col offset">
+                    <h4>{{_('Graphics Devices')}} <a href="https://github.com/guydavis/machinaris/wiki/GPUs" target="_blank"><i class="fs-4 bi-question-circle text-white"></i></a></h4>
+                    <p>{{_('Optionally, if your system has an available and supported GPU, you can pass it thru to the blockchain\'s Docker container. Only Chia, Gigahorse, and MMX blockchains support certain GPUs.')}}</p>
+                    <div class="row">
+                        <div class="col">
+                            <label for="gpu_type" class="form-label">{{_('GPU Type:')}}</label>
+                            <select class="form-select" id="gpu_type" name="gpu_type" aria-label="gpu_type" onchange="updateDocker()">
+                                <option value="none" selected>{{_('None')}}</option>
+                                <option value="nvidia">{{_('Nvidia')}}</option>
+                                <option value="amd">{{_('AMD')}}</option>
+                                <option value="intel">{{_('Intel')}}</option>
+                            </select>
+                        </div>
+                        <div class="col">
+                            <label for="gpu_blockchain" class="form-label">{{_('Blockchain:')}}</label>
+                            <select class="form-select" id="gpu_blockchain" name="gpu_blockchain" aria-label="gpu_blockchain" onchange="updateDocker()">
+                                <option value="none" selected>{{_('None')}}</option>
+                                <option value="chia">{{_('Chia')}}</option>
+                                <option value="gigahorse">{{_('Gigahorse')}}</option>
+                                <option value="mmx">{{_('MMX')}}</option>
+                            </select>
+                        </div>
+                    </div>
+                </div>
+            </div>
         </div>
     </form>
     <div class="rounded-3 small" id="docker_compose">
@@ -950,8 +1012,6 @@ <h4>{{_('Volume Mounts')}}</h4>
         <h2>{{_('Docker Compose')}}</h2>
         <pre id="compose"></pre>
         <br/>
-        <small><i>{{_('NOTE: Passing a GPU thru into the container requires additional config.  Please see the ')}}
-            <a class='text-white' href="https://github.com/guydavis/machinaris/wiki/MMX#can-i-use-my-gpu" target="_blank">wiki.</a></i></small>
     </div>
 
     <script src="{{ url_for('static', filename='3rd_party/bootstrap.bundle.min.js') }}"></script>