Merge branch 'development' into interactive-noauto-start

.github/ISSUE_TEMPLATE/bug_report.md

@@ -0,0 +1,43 @@
+---
+name: Bug report
+about: Create a report to help us improve
+title: ''
+labels: 'bug'
+assignees: ''
+
+---
+
+<!--
+**Should I file this?**
+Use this template to report *observed bugs in current Plotman*.  Do not use this to request tech support or troubleshooting; we are unable to provide that support here, so please use the user channel(s) on Keybase, or GitHub Discussions instead.  Do not use this to report issues with the core Chia plotter.  Because the `development` branch often contains new fixes not yet released, you may also want to try installing it to see if it addresses your issue (you can do this by following the install instructions but specifying `@development` instead of `@main`).
+-->
+
+**Describe the bug**
+<!-- A clear and concise description of what the bug is. -->
+
+**To Reproduce**
+
+Steps to reproduce the behavior, e.g.:
+1. Set up config with '...'
+2. Run other programs '....'
+3. Run Plotman with '....'
+4. See error
+
+**Expected behavior**
+<!-- A clear and concise description of what you expected to happen, even if it seems obvious. -->
+
+**System setup:**
+ - OS: [e.g. Ubuntu, iOS, ...]
+ - Method of archiving (e.g., none, rsyncd, rsync, ...)
+
+**Config**
+<details> <summary>full configuration</summary>
+
+```yaml
+# paste your complete configuration file contents here.
+```
+
+</details>
+
+**Additional context & screenshots**
+<!-- Add any other context about the problem here, including screenshots. -->

.github/ISSUE_TEMPLATE/config.yml

@@ -0,0 +1,8 @@
+contact_links:
+  - about: Ask a general question or request support in Discussions
+    name: Ask for Support
+    url: >-
+      https://github.com/ericaltendorf/plotman/discussions/new
+  - about: Get support on the Chia Keybase's dedicated plotman channel.
+    name: Join the Keybase.io plotman channel
+    url: 'https://keybase.io/team/chia_network.public'

.github/ISSUE_TEMPLATE/request.md

@@ -0,0 +1,17 @@
+---
+name: Feature request
+about: Log a request for a new feature or enhancement
+title: ''
+labels: 'enhancement'
+assignees: ''
+
+---
+
+**Describe the request**
+<!-- A clear and concise description of your suggested enhancement, as well as some basic rationale to allow us to balance the effort/reward for the work, if possible. -->
+
+**Images**
+<!-- If applicable (but not required), add diagrams or wireframes to help explain your request if it is UI/UX related. -->
+
+**Additional comments**
+<!-- Add any other comments here. -->

README.md

@@ -222,6 +222,22 @@ Installation for Linux and macOS:
 4. That's it! You can now run Plotman by typing `plotman version` to verify its version.
    Run `plotman --help` to learn about the available commands.
 
+*Note:* If you see `ModuleNotFoundError: No module named 'readline'` when using `plotman` on [RHEL based linux](https://github.com/ericaltendorf/plotman/issues/195) after installing using [chia's guide](https://github.com/Chia-Network/chia-blockchain/wiki/INSTALL#centos--red-hat--fedora), install `readline-devel` then reinstall chia starting at compiling python in a new build environment; or consider using a project like `pyenv`.
+
+## Basic Usage:
+
+1. Install
+
+2. Generate initial config
+
+3. Configure (default location can be found with `plotman config path`). Options explained in the default config file (step 2)
+
+4. Create log directory specified in `directories: { log: "" }`
+
+5. Start plotman: `plotman plot` or `plotman interactive`
+
+6. Check status: `plotman status`
+
 ### Development note:
 
 If you are forking Plotman, simply replace the installation step with `pip install --editable .[dev]` from the project root directory to install *your* version of plotman with test and development extras.

VERSION

@@ -1 +1 @@
-0.2
+0.3.1+dev

setup.cfg

@@ -36,15 +36,15 @@ package_dir=
     =src
 packages=find:
 install_requires =
-    appdirs
-    attrs
-    click
-    desert
-    marshmallow
-    pendulum
+    appdirs ~= 1.4
+    attrs == 21.2
+    click ~= 7.1
+    desert == 2020.11.18
+    marshmallow ~= 3.12
+    pendulum ~= 2.1
     psutil ~= 5.8
-    pyyaml
-    texttable
+    pyyaml ~= 5.4
+    texttable ~= 1.6
 
 [options.packages.find]
 where=src

src/plotman/__main__.py

@@ -4,7 +4,7 @@
 
 
 """Plotman module launcher.
-This is a shim that allows you to run plotman via 
+This is a shim that allows you to run plotman via
     python3 -m plotman
 """
 plotman.main()

src/plotman/_tests/configuration_test.py

@@ -69,3 +69,33 @@ def test_loads_without_user_interface(config_text):
     reloaded_yaml = configuration.get_validated_configs(stripped_config_text, '')
 
     assert reloaded_yaml.user_interface == configuration.UserInterface()
+
+
+def test_get_dst_directories_gets_dst():
+    tmp = ['/tmp']
+    dst = ['/dst0', '/dst1']
+    directories = configuration.Directories(log='', tmp=tmp, dst=dst)
+
+    assert directories.get_dst_directories() == dst
+
+
+def test_get_dst_directories_gets_tmp():
+    tmp = ['/tmp']
+    directories = configuration.Directories(log='', tmp=tmp)
+
+    assert directories.get_dst_directories() == tmp
+
+
+def test_dst_is_dst():
+    tmp = ['/tmp']
+    dst = ['/dst0', '/dst1']
+    directories = configuration.Directories(log='', tmp=tmp, dst=dst)
+
+    assert not directories.dst_is_tmp()
+
+
+def test_dst_is_tmp():
+    tmp = ['/tmp']
+    directories = configuration.Directories(log='', tmp=tmp)
+
+    assert directories.dst_is_tmp()

src/plotman/_tests/manager_test.py

@@ -85,7 +85,7 @@ def test_dstdirs_to_furthest_phase():
             { '/plots1' : (1, 5),
               '/plots2' : (3, 1),
               '/plots3' : (4, 1) } )
-           
+
 
 def test_dstdirs_to_youngest_phase():
     all_jobs = [ job_w_dstdir_phase('/plots1', (1, 5)),

src/plotman/_tests/plot_util_test.py

@@ -10,7 +10,10 @@ def test_human_format():
     assert (plot_util.human_format(3442000000, 0) == '3G')
     assert (plot_util.human_format(3542000, 2) == '3.54M')
     assert (plot_util.human_format(354, 0) == '354')
+    assert (plot_util.human_format(354, 0, True) == '354')
     assert (plot_util.human_format(354, 2) == '354.00')
+    assert (plot_util.human_format(422399296143, 2) == '422.40G')
+    assert (plot_util.human_format(422399296143, 2, True) == '393.39Gi')
 
 def test_time_format():
     assert (plot_util.time_format(34) == '34s')
@@ -54,3 +57,10 @@ def test_list_k32_plots(fs: pyfakefs.fake_filesystem.FakeFilesystem):
             [ '/t/plot-k32-0.plot',
               '/t/plot-k32-1.plot',
               '/t/plot-k32-5.plot' ] )
+
+
+def test_get_plotsize():
+    assert (
+        [659272492, 107287518791, 221143636517, 455373353413, 936816632588]
+        == [plot_util.get_plotsize(n) for n in [25, 32, 33, 34, 35]]
+    )

src/plotman/_tests/reporting_test.py

@@ -28,7 +28,7 @@ def test_job_viz_empty():
 @patch('plotman.job.Job')
 def job_w_phase(ph, MockJob):
     j = MockJob()
-    j.progress.return_value = ph
+    j.progress.return_value = job.Phase.from_tuple(ph)
     return j
 
 def test_job_viz_positions():

src/plotman/archive.py

@@ -2,6 +2,7 @@
 import contextlib
 import math
 import os
+import posixpath
 import random
 import re
 import subprocess
@@ -16,16 +17,16 @@
 # TODO : write-protect and delete-protect archived plots
 
 def spawn_archive_process(dir_cfg, all_jobs):
-    '''Spawns a new archive process using the command created 
+    '''Spawns a new archive process using the command created
     in the archive() function. Returns archiving status and a log message to print.'''
 
     log_message = None
     archiving_status = None
-    
+
     # Look for running archive jobs.  Be robust to finding more than one
     # even though the scheduler should only run one at a time.
     arch_jobs = get_running_archive_jobs(dir_cfg.archive)
-    
+
     if not arch_jobs:
         (should_start, status_or_cmd) = archive(dir_cfg, all_jobs)
         if not should_start:
@@ -37,7 +38,7 @@ def spawn_archive_process(dir_cfg, all_jobs):
                     shell=True,
                     stdout=subprocess.DEVNULL,
                     stderr=subprocess.STDOUT,
-                    start_new_session=True) 
+                    start_new_session=True)
             log_message = 'Starting archive: ' + cmd
             # At least for now it seems that even if we get a new running
             # archive jobs list it doesn't contain the new rsync process.
@@ -51,7 +52,7 @@ def spawn_archive_process(dir_cfg, all_jobs):
         archiving_status = 'pid: ' + ', '.join(map(str, arch_jobs))
 
     return archiving_status, log_message
-            
+
 def compute_priority(phase, gb_free, n_plots):
     # All these values are designed around dst buffer dirs of about
     # ~2TB size and containing k32 plots.  TODO: Generalize, and
@@ -71,7 +72,7 @@ def compute_priority(phase, gb_free, n_plots):
             priority -= 16
         elif (phase >= job.Phase(3, 7)):
             priority -= 32
-        
+
     # If a drive is getting full, we should prioritize it
     if (gb_free < 1000):
         priority += 1 + int((1000 - gb_free) / 100)
@@ -87,7 +88,7 @@ def compute_priority(phase, gb_free, n_plots):
 def get_archdir_freebytes(arch_cfg):
     archdir_freebytes = {}
     df_cmd = ('ssh %s@%s df -aBK | grep " %s/"' %
-        (arch_cfg.rsyncd_user, arch_cfg.rsyncd_host, arch_cfg.rsyncd_path) )
+        (arch_cfg.rsyncd_user, arch_cfg.rsyncd_host, posixpath.normpath(arch_cfg.rsyncd_path)) )
     with subprocess.Popen(df_cmd, shell=True, stdout=subprocess.PIPE) as proc:
         for line in proc.stdout.readlines():
             fields = line.split()
@@ -124,7 +125,7 @@ def get_running_archive_jobs(arch_cfg):
 
 def archive(dir_cfg, all_jobs):
     '''Configure one archive job.  Needs to know all jobs so it can avoid IO
-    contention on the plotting dstdir drives.  Returns either (False, <reason>) 
+    contention on the plotting dstdir drives.  Returns either (False, <reason>)
     if we should not execute an archive job or (True, <cmd>) with the archive
     command if we should.'''
     if dir_cfg.archive is None:
@@ -133,13 +134,13 @@ def archive(dir_cfg, all_jobs):
     dir2ph = manager.dstdirs_to_furthest_phase(all_jobs)
     best_priority = -100000000
     chosen_plot = None
-
-    for d in dir_cfg.dst:
-        ph = dir2ph.get(d, (0, 0))
+    dst_dir = dir_cfg.get_dst_directories()
+    for d in dst_dir:
+        ph = dir2ph.get(d, job.Phase(0, 0))
         dir_plots = plot_util.list_k32_plots(d)
         gb_free = plot_util.df_b(d) / plot_util.GB
         n_plots = len(dir_plots)
-        priority = compute_priority(ph, gb_free, n_plots) 
+        priority = compute_priority(ph, gb_free, n_plots)
         if priority >= best_priority and dir_plots:
             best_priority = priority
             chosen_plot = dir_plots[0]
@@ -156,22 +157,22 @@ def archive(dir_cfg, all_jobs):
     archdir_freebytes = get_archdir_freebytes(dir_cfg.archive)
     if not archdir_freebytes:
         return(False, 'No free archive dirs found.')
-    
+
     archdir = ''
-    available = [(d, space) for (d, space) in archdir_freebytes.items() if 
+    available = [(d, space) for (d, space) in archdir_freebytes.items() if
                  space > 1.2 * plot_util.get_k32_plotsize()]
     if len(available) > 0:
         index = min(dir_cfg.archive.index, len(available) - 1)
         (archdir, freespace) = sorted(available)[index]
 
     if not archdir:
         return(False, 'No archive directories found with enough free space')
-    
+
     msg = 'Found %s with ~%d GB free' % (archdir, freespace / plot_util.GB)
 
     bwlimit = dir_cfg.archive.rsyncd_bwlimit
     throttle_arg = ('--bwlimit=%d' % bwlimit) if bwlimit else ''
-    cmd = ('rsync %s --no-compress --remove-source-files -P %s %s' %
+    cmd = ('rsync %s --compress-level=0 --remove-source-files -P %s %s' %
             (throttle_arg, chosen_plot, rsync_dest(dir_cfg.archive, archdir)))
 
     return (True, cmd)