From af115b656ba843d314db4ab33e919dd3c2fe4d41 Mon Sep 17 00:00:00 2001
From: Kyle Altendorf <sda@fstab.net>
Date: Thu, 15 Apr 2021 23:10:06 -0400
Subject: [PATCH 01/86] Use chia's own 'chia plots create' argument parser

---
 setup.cfg          |  2 ++
 src/plotman/job.py | 62 +++++++++++++---------------------------------
 tox.ini            |  1 +
 3 files changed, 20 insertions(+), 45 deletions(-)

diff --git a/setup.cfg b/setup.cfg
index caebb8b0..df00d2d2 100644
--- a/setup.cfg
+++ b/setup.cfg
@@ -46,6 +46,8 @@ console_scripts =
     plotman = plotman.plotman:main
 
 [options.extras_require]
+chia =
+    chia-blockchain >= 1.0.4
 dev =
     %(test)s
     isort
diff --git a/src/plotman/job.py b/src/plotman/job.py
index 64cdb5d1..4083fb53 100644
--- a/src/plotman/job.py
+++ b/src/plotman/job.py
@@ -12,6 +12,7 @@
 from enum import Enum, auto
 from subprocess import call
 
+import chia.cmds.plots
 import pendulum
 import psutil
 
@@ -57,15 +58,6 @@ def parse_chia_plot_time(s):
 class Job:
     'Represents a plotter job'
 
-    # These are constants, not updated during a run.
-    k = 0
-    r = 0
-    u = 0
-    b = 0
-    n = 0  # probably not used
-    tmpdir = ''
-    tmp2dir = ''
-    dstdir = ''
     logfile = ''
     jobfile = ''
     job_id = 0
@@ -110,34 +102,14 @@ def __init__(self, proc, logroot):
             assert 'chia' in args[1]
             assert 'plots' == args[2]
             assert 'create' == args[3]
-            args_iter = iter(cmdline_argfix(args[4:]))
-            for arg in args_iter:
-                val = None if arg in {'-e', '--nobitfield', '-h', '--help', '--override-k'} else next(args_iter)
-                if arg in {'-k', '--size'}:
-                    self.k = val
-                elif arg in {'-r', '--num_threads'}:
-                    self.r = val
-                elif arg in {'-b', '--buffer'}:
-                    self.b = val
-                elif arg in {'-u', '--buckets'}:
-                    self.u = val
-                elif arg in {'-t', '--tmp_dir'}:
-                    self.tmpdir = val
-                elif arg in {'-2', '--tmp2_dir'}:
-                    self.tmp2dir = val
-                elif arg in {'-d', '--final_dir'}:
-                    self.dstdir = val
-                elif arg in {'-n', '--num'}:
-                    self.n = val
-                elif arg in {'-h', '--help'}:
-                    self.help = True
-                elif arg in {'-e', '--nobitfield', '-f', '--farmer_public_key', '-p', '--pool_public_key'}:
-                    pass
-                    # TODO: keep track of these
-                elif arg == '--override-k':
-                    pass
-                else:
-                    print('Warning: unrecognized args: %s %s' % (arg, val))
+
+            self.args = chia.cmds.plots.create_cmd.make_context(info_name='', args=args[4:])
+
+            plot_cwd = self.proc.cwd()
+            self.args['tmpdir'] = os.path.join(plot_cwd, self.args['tmpdir'])
+            if self.args['tmp2dir'] is not None:
+                self.args['tmp2dir'] = os.path.join(plot_cwd, args['tmp2dir'])
+            self.args['dstdir'] = os.path.join(plot_cwd, self.args['dstdir'])
 
             # Find logfile (whatever file is open under the log root).  The
             # file may be open more than once, e.g. for STDOUT and STDERR.
@@ -262,14 +234,14 @@ def plot_id_prefix(self):
     def status_str_long(self):
         return '{plot_id}\nk={k} r={r} b={b} u={u}\npid:{pid}\ntmp:{tmp}\ntmp2:{tmp2}\ndst:{dst}\nlogfile:{logfile}'.format(
             plot_id = self.plot_id,
-            k = self.k,
-            r = self.r,
-            b = self.b,
-            u = self.u,
+            k = self.args['size'],
+            r = self.args['num_threads'],
+            b = self.args['buffer'],
+            u = self.args['buckets'],
             pid = self.proc.pid,
-            tmp = self.tmpdir,
-            tmp2 = self.tmp2dir,
-            dst = self.dstdir,
+            tmp = self.args['tmpdir'],
+            tmp2 = self.args['tmp2dir'],
+            dst = self.args['dstdir'],
             plotid = self.plot_id,
             logfile = self.logfile
             )
@@ -332,7 +304,7 @@ def get_temp_files(self):
         # Prevent duplicate file paths by using set.
         temp_files = set([])
         for f in self.proc.open_files():
-            if self.tmpdir in f.path or self.tmp2dir in f.path or self.dstdir in f.path:
+            if self.args['tmpdir'] in f.path or self.args['tmp2dir'] in f.path or self.args['dstdir'] in f.path:
                 temp_files.add(f.path)
         return temp_files
 
diff --git a/tox.ini b/tox.ini
index 81081aaa..bf228508 100644
--- a/tox.ini
+++ b/tox.ini
@@ -6,6 +6,7 @@ changedir = {envtmpdir}
 
 [testenv:test-py{36,37,38,39,py36,py37}]
 extras =
+    chia
     test
 commands =
     pytest --capture=no --verbose --pyargs plotman

From 83349319dc845b6ff0696bfb91dfd688ea606550 Mon Sep 17 00:00:00 2001
From: Kyle Altendorf <sda@fstab.net>
Date: Thu, 15 Apr 2021 23:16:57 -0400
Subject: [PATCH 02/86] Manually handle --help

---
 src/plotman/job.py | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/src/plotman/job.py b/src/plotman/job.py
index 4083fb53..f10a51f4 100644
--- a/src/plotman/job.py
+++ b/src/plotman/job.py
@@ -63,7 +63,6 @@ class Job:
     job_id = 0
     plot_id = '--------'
     proc = None   # will get a psutil.Process
-    help = False
 
     # These are dynamic, cached, and need to be udpated periodically
     phase = (None, None)   # Phase/subphase
@@ -103,6 +102,12 @@ def __init__(self, proc, logroot):
             assert 'plots' == args[2]
             assert 'create' == args[3]
 
+            if '--help' in args:
+                args = [arg for arg in args if arg != '--help']
+                self.help = True
+            else:
+                self.help = False
+
             self.args = chia.cmds.plots.create_cmd.make_context(info_name='', args=args[4:])
 
             plot_cwd = self.proc.cwd()

From be7c4e5a21c33ca714421368d8731ef37c24c233 Mon Sep 17 00:00:00 2001
From: Kyle Altendorf <sda@fstab.net>
Date: Thu, 15 Apr 2021 23:40:32 -0400
Subject: [PATCH 03/86] Change less

---
 src/plotman/job.py | 54 +++++++++++++++++++++++++++++++++++-----------
 1 file changed, 42 insertions(+), 12 deletions(-)

diff --git a/src/plotman/job.py b/src/plotman/job.py
index f10a51f4..3bfc668c 100644
--- a/src/plotman/job.py
+++ b/src/plotman/job.py
@@ -110,11 +110,41 @@ def __init__(self, proc, logroot):
 
             self.args = chia.cmds.plots.create_cmd.make_context(info_name='', args=args[4:])
 
+            # an example as of 1.0.5
+            # {
+            #     'size': 32,
+            #     'num_threads': 4,
+            #     'buckets': 128,
+            #     'buffer': 6000,
+            #     'tmp_dir': '/farm/yards/901',
+            #     'final_dir': '/farm/wagons/801',
+            #     'override_k': False,
+            #     'num': 1,
+            #     'alt_fingerprint': None,
+            #     'pool_contract_address': None,
+            #     'farmer_public_key': None,
+            #     'pool_public_key': None,
+            #     'tmp2_dir': None,
+            #     'plotid': None,
+            #     'memo': None,
+            #     'nobitfield': False,
+            #     'exclude_final_dir': False,
+            # }
+
+            self.k = self.args['size']
+            self.r = self.args['num_threads']
+            self.u = self.args['buckets']
+            self.b = self.args['buffer']
+            self.n = self.args['num']
+            self.tmpdir = self.args['tmp_dir']
+            self.tmp2dir = self.args['tmp2_dir']
+            self.dstdir = self.args['final_dir']
+
             plot_cwd = self.proc.cwd()
-            self.args['tmpdir'] = os.path.join(plot_cwd, self.args['tmpdir'])
-            if self.args['tmp2dir'] is not None:
-                self.args['tmp2dir'] = os.path.join(plot_cwd, args['tmp2dir'])
-            self.args['dstdir'] = os.path.join(plot_cwd, self.args['dstdir'])
+            self.tmpdir = os.path.join(plot_cwd, self.tmpdir)
+            if self.tmp2dir != '':
+                self.tmp2dir = os.path.join(plot_cwd, self.tmp2dir)
+            self.dstdir = os.path.join(plot_cwd, self.dstdir)
 
             # Find logfile (whatever file is open under the log root).  The
             # file may be open more than once, e.g. for STDOUT and STDERR.
@@ -239,14 +269,14 @@ def plot_id_prefix(self):
     def status_str_long(self):
         return '{plot_id}\nk={k} r={r} b={b} u={u}\npid:{pid}\ntmp:{tmp}\ntmp2:{tmp2}\ndst:{dst}\nlogfile:{logfile}'.format(
             plot_id = self.plot_id,
-            k = self.args['size'],
-            r = self.args['num_threads'],
-            b = self.args['buffer'],
-            u = self.args['buckets'],
+            k = self.k,
+            r = self.r,
+            b = self.b,
+            u = self.u,
             pid = self.proc.pid,
-            tmp = self.args['tmpdir'],
-            tmp2 = self.args['tmp2dir'],
-            dst = self.args['dstdir'],
+            tmp = self.tmpdir,
+            tmp2 = self.tmp2dir,
+            dst = self.dstdir,
             plotid = self.plot_id,
             logfile = self.logfile
             )
@@ -309,7 +339,7 @@ def get_temp_files(self):
         # Prevent duplicate file paths by using set.
         temp_files = set([])
         for f in self.proc.open_files():
-            if self.args['tmpdir'] in f.path or self.args['tmp2dir'] in f.path or self.args['dstdir'] in f.path:
+            if self.tmpdir in f.path or self.tmp2dir in f.path or self.dstdir in f.path:
                 temp_files.add(f.path)
         return temp_files
 

From 17a2b83285ce0db7f4e32c40875ead515ef58100 Mon Sep 17 00:00:00 2001
From: Kyle Altendorf <sda@fstab.net>
Date: Thu, 15 Apr 2021 23:41:14 -0400
Subject: [PATCH 04/86] fix: use .params

---
 src/plotman/job.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/plotman/job.py b/src/plotman/job.py
index 3bfc668c..05620927 100644
--- a/src/plotman/job.py
+++ b/src/plotman/job.py
@@ -108,7 +108,7 @@ def __init__(self, proc, logroot):
             else:
                 self.help = False
 
-            self.args = chia.cmds.plots.create_cmd.make_context(info_name='', args=args[4:])
+            self.args = chia.cmds.plots.create_cmd.make_context(info_name='', args=args[4:]).params
 
             # an example as of 1.0.5
             # {

From 30ad3f67614fce29d20a3c3eae18df99e11ca5c2 Mon Sep 17 00:00:00 2001
From: Kyle Altendorf <sda@fstab.net>
Date: Fri, 16 Apr 2021 03:42:51 +0000
Subject: [PATCH 05/86] fix: tmp2dir is optionally None now

---
 src/plotman/job.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/plotman/job.py b/src/plotman/job.py
index 05620927..68a871ff 100644
--- a/src/plotman/job.py
+++ b/src/plotman/job.py
@@ -142,7 +142,7 @@ def __init__(self, proc, logroot):
 
             plot_cwd = self.proc.cwd()
             self.tmpdir = os.path.join(plot_cwd, self.tmpdir)
-            if self.tmp2dir != '':
+            if self.tmp2dir is not None:
                 self.tmp2dir = os.path.join(plot_cwd, self.tmp2dir)
             self.dstdir = os.path.join(plot_cwd, self.dstdir)
 

From 18cd48003bdb4a29d5a4a5cc4e37a1b7eed36f1a Mon Sep 17 00:00:00 2001
From: Kyle Altendorf <sda@fstab.net>
Date: Fri, 16 Apr 2021 22:40:42 -0400
Subject: [PATCH 06/86] fix: make user_interface configuration optional

---
 src/plotman/_tests/configuration_test.py | 15 +++++++++++++++
 src/plotman/configuration.py             |  6 +++---
 2 files changed, 18 insertions(+), 3 deletions(-)

diff --git a/src/plotman/_tests/configuration_test.py b/src/plotman/_tests/configuration_test.py
index 2f197efa..0be0c990 100644
--- a/src/plotman/_tests/configuration_test.py
+++ b/src/plotman/_tests/configuration_test.py
@@ -50,3 +50,18 @@ def test_get_validated_configs__missing(mocker, config_path):
         f"No 'plotman.yaml' file exists at expected location: '{nonexistent_config}'. To generate "
         f"default config file, run: 'plotman config generate'"
     )
+
+
+def test_loads_without_user_interface(mocker, config_path, tmp_path):
+    with open(config_path, "r") as file:
+        loaded_yaml = yaml.load(file, Loader=yaml.SafeLoader)
+
+    del loaded_yaml["user_interface"]
+
+    temporary_configuration_path = tmp_path.joinpath("config.yaml")
+    temporary_configuration_path.write_text(yaml.safe_dump(loaded_yaml))
+
+    mocker.patch("plotman.configuration.get_path", return_value=temporary_configuration_path)
+    reloaded_yaml = configuration.get_validated_configs()
+
+    assert reloaded_yaml.user_interface == configuration.UserInterface()
diff --git a/src/plotman/configuration.py b/src/plotman/configuration.py
index e8eaa905..4efbfac6 100644
--- a/src/plotman/configuration.py
+++ b/src/plotman/configuration.py
@@ -1,4 +1,4 @@
-from dataclasses import dataclass
+from dataclasses import dataclass, field
 from typing import Dict, List, Optional
 
 import appdirs
@@ -82,11 +82,11 @@ class Plotting:
 
 @dataclass
 class UserInterface:
-    use_stty_size: bool
+    use_stty_size: bool = True
 
 @dataclass
 class PlotmanConfig:
-    user_interface: UserInterface
     directories: Directories
     scheduling: Scheduling
     plotting: Plotting
+    user_interface: UserInterface = field(default_factory=UserInterface)

From 4c8f2007e655f2aaba0cef1966bb4be4dadce1a9 Mon Sep 17 00:00:00 2001
From: Kyle Altendorf <sda@fstab.net>
Date: Sat, 17 Apr 2021 00:05:21 -0400
Subject: [PATCH 07/86] fix: handle -h as well

---
 src/plotman/job.py | 36 +++++++++++++++++++++++-------------
 1 file changed, 23 insertions(+), 13 deletions(-)

diff --git a/src/plotman/job.py b/src/plotman/job.py
index 68a871ff..18d116f0 100644
--- a/src/plotman/job.py
+++ b/src/plotman/job.py
@@ -95,20 +95,30 @@ def __init__(self, proc, logroot):
 
         with self.proc.oneshot():
             # Parse command line args
-            args = self.proc.cmdline()
-            assert len(args) > 4
-            assert 'python' in args[0]
-            assert 'chia' in args[1]
-            assert 'plots' == args[2]
-            assert 'create' == args[3]
-
-            if '--help' in args:
-                args = [arg for arg in args if arg != '--help']
-                self.help = True
-            else:
-                self.help = False
+            command_line = self.proc.cmdline()
+            assert len(command_line) > 4
+            assert 'python' in command_line[0]
+            assert 'chia' in command_line[1]
+            assert 'plots' == command_line[2]
+            assert 'create' == command_line[3]
+
+            command_arguments = command_line[4:]
+            command = chia.cmds.plots.create_cmd
+            context = command.make_context(
+                info_name='',
+                args=command_arguments,
+                resilient_parsing=True,
+            )
+
+            # nice idea, but this doesn't include -h
+            # help_option_names = command.get_help_option_names(ctx=context)
+            help_option_names = {'--help', '-h'}
+
+            self.help = any(
+                argument in help_option_names for argument in command_arguments
+            )
 
-            self.args = chia.cmds.plots.create_cmd.make_context(info_name='', args=args[4:]).params
+            self.args = context.params
 
             # an example as of 1.0.5
             # {

From f3114a5edfb4962a00771845036b5dd289726f68 Mon Sep 17 00:00:00 2001
From: Kyle Altendorf <sda@fstab.net>
Date: Sat, 17 Apr 2021 04:25:17 +0000
Subject: [PATCH 08/86] fix: get defaults back

---
 src/plotman/job.py | 22 +++++++++++-----------
 1 file changed, 11 insertions(+), 11 deletions(-)

diff --git a/src/plotman/job.py b/src/plotman/job.py
index 18d116f0..75392667 100644
--- a/src/plotman/job.py
+++ b/src/plotman/job.py
@@ -102,21 +102,21 @@ def __init__(self, proc, logroot):
             assert 'plots' == command_line[2]
             assert 'create' == command_line[3]
 
-            command_arguments = command_line[4:]
-            command = chia.cmds.plots.create_cmd
-            context = command.make_context(
-                info_name='',
-                args=command_arguments,
-                resilient_parsing=True,
-            )
-
             # nice idea, but this doesn't include -h
             # help_option_names = command.get_help_option_names(ctx=context)
             help_option_names = {'--help', '-h'}
 
-            self.help = any(
-                argument in help_option_names for argument in command_arguments
-            )
+            all_command_arguments = command_line[4:]
+            command_arguments = [
+                argument
+                for argument in all_command_arguments
+                if argument not in help_option_names
+            ]
+
+            self.help = len(all_command_arguments) > len(command_arguments)
+
+            command = chia.cmds.plots.create_cmd
+            context = command.make_context(info_name='', args=command_arguments)
 
             self.args = context.params
 

From 8dde509c6c6fd4c44326eb004f2b57492149cc04 Mon Sep 17 00:00:00 2001
From: Kyle Altendorf <sda@fstab.net>
Date: Sat, 17 Apr 2021 10:38:34 -0400
Subject: [PATCH 09/86] refactor: isolate chia cli parsing and test it

---
 src/plotman/_tests/job_test.py | 62 +++++++++++++++++++++++++++++++++
 src/plotman/job.py             | 63 ++++++++++++++++++++--------------
 2 files changed, 100 insertions(+), 25 deletions(-)

diff --git a/src/plotman/_tests/job_test.py b/src/plotman/_tests/job_test.py
index 46a28cda..e1052f26 100644
--- a/src/plotman/_tests/job_test.py
+++ b/src/plotman/_tests/job_test.py
@@ -54,3 +54,65 @@ def test_job_parses_time_with_non_english_locale(logfile_path, locale_name):
         job.Job.init_from_logfile(self=faux_job_with_logfile)
 
     assert faux_job_with_logfile.start_time == log_file_time
+
+
+@pytest.mark.parametrize(
+    argnames=['arguments'],
+    argvalues=[
+        [['-h']],
+        [['--help']],
+        [['-k', '32']],
+        [['-k32']],
+        [['-k', '32', '--help']],
+    ],
+)
+def test_chia_plots_create_parsing_does_not_fail(arguments):
+    job.parse_chia_plots_create_command_line(
+        command_line=['python', 'chia', 'plots', 'create', *arguments],
+    )
+
+
+@pytest.mark.parametrize(
+    argnames=['arguments'],
+    argvalues=[
+        [['-h']],
+        [['--help']],
+        [['-k', '32', '--help']],
+    ],
+)
+def test_chia_plots_create_parsing_does_not_fail(arguments):
+    job.parse_chia_plots_create_command_line(
+        command_line=['python', 'chia', 'plots', 'create', *arguments],
+    )
+
+
+@pytest.mark.parametrize(
+    argnames=['arguments'],
+    argvalues=[
+        [['-h']],
+        [['--help']],
+        [['-k', '32', '--help']],
+    ],
+)
+def test_chia_plots_create_parsing_detects_help(arguments):
+    parsed = job.parse_chia_plots_create_command_line(
+        command_line=['python', 'chia', 'plots', 'create', *arguments],
+    )
+
+    assert parsed.help
+
+
+@pytest.mark.parametrize(
+    argnames=['arguments'],
+    argvalues=[
+        [[]],
+        [['-k32']],
+        [['-k', '32']],
+    ],
+)
+def test_chia_plots_create_parsing_detects_not_help(arguments):
+    parsed = job.parse_chia_plots_create_command_line(
+        command_line=['python', 'chia', 'plots', 'create', *arguments],
+    )
+
+    assert not parsed.help
diff --git a/src/plotman/job.py b/src/plotman/job.py
index 75392667..48e48a48 100644
--- a/src/plotman/job.py
+++ b/src/plotman/job.py
@@ -53,6 +53,39 @@ def parse_chia_plot_time(s):
     # This will grow to try ISO8601 as well for when Chia logs that way
     return pendulum.from_format(s, 'ddd MMM DD HH:mm:ss YYYY', locale='en', tz=None)
 
+class ParsedChiaPlotsCreateCommand:
+    def __init__(self, help, parameters):
+        self.help = help
+        self.parameters = parameters
+
+def parse_chia_plots_create_command_line(command_line):
+    # Parse command line args
+    assert len(command_line) >= 4
+    assert 'python' in command_line[0]
+    assert 'chia' in command_line[1]
+    assert 'plots' == command_line[2]
+    assert 'create' == command_line[3]
+
+    all_command_arguments = command_line[4:]
+
+    # nice idea, but this doesn't include -h
+    # help_option_names = command.get_help_option_names(ctx=context)
+    help_option_names = {'--help', '-h'}
+
+    command_arguments = [
+        argument
+        for argument in all_command_arguments
+        if argument not in help_option_names
+    ]
+
+    command = chia.cmds.plots.create_cmd
+    context = command.make_context(info_name='', args=list(command_arguments))
+
+    return ParsedChiaPlotsCreateCommand(
+        help=len(all_command_arguments) > len(command_arguments),
+        parameters=context.params,
+    )
+
 # TODO: be more principled and explicit about what we cache vs. what we look up
 # dynamically from the logfile
 class Job:
@@ -94,31 +127,11 @@ def __init__(self, proc, logroot):
         self.proc = proc
 
         with self.proc.oneshot():
-            # Parse command line args
-            command_line = self.proc.cmdline()
-            assert len(command_line) > 4
-            assert 'python' in command_line[0]
-            assert 'chia' in command_line[1]
-            assert 'plots' == command_line[2]
-            assert 'create' == command_line[3]
-
-            # nice idea, but this doesn't include -h
-            # help_option_names = command.get_help_option_names(ctx=context)
-            help_option_names = {'--help', '-h'}
-
-            all_command_arguments = command_line[4:]
-            command_arguments = [
-                argument
-                for argument in all_command_arguments
-                if argument not in help_option_names
-            ]
-
-            self.help = len(all_command_arguments) > len(command_arguments)
-
-            command = chia.cmds.plots.create_cmd
-            context = command.make_context(info_name='', args=command_arguments)
-
-            self.args = context.params
+            parsed_command = parse_chia_plots_create_command_line(
+                command_line=self.proc.cmdline(),
+            )
+            self.help = parsed_command.help
+            self.args = parsed_command.parameters
 
             # an example as of 1.0.5
             # {

From ef87c2b4b541984d2db73707dc968d5ccb21e9a1 Mon Sep 17 00:00:00 2001
From: Kyle Altendorf <sda@fstab.net>
Date: Sat, 17 Apr 2021 21:15:14 -0400
Subject: [PATCH 10/86] refactor: some more testing and adjusting

---
 src/plotman/_tests/job_test.py | 22 ++++++++++++++++++++++
 src/plotman/job.py             | 10 +++++-----
 2 files changed, 27 insertions(+), 5 deletions(-)

diff --git a/src/plotman/_tests/job_test.py b/src/plotman/_tests/job_test.py
index e1052f26..3ce250d1 100644
--- a/src/plotman/_tests/job_test.py
+++ b/src/plotman/_tests/job_test.py
@@ -65,6 +65,7 @@ def test_job_parses_time_with_non_english_locale(logfile_path, locale_name):
         [['-k32']],
         [['-k', '32', '--help']],
     ],
+    ids=str,
 )
 def test_chia_plots_create_parsing_does_not_fail(arguments):
     job.parse_chia_plots_create_command_line(
@@ -79,6 +80,7 @@ def test_chia_plots_create_parsing_does_not_fail(arguments):
         [['--help']],
         [['-k', '32', '--help']],
     ],
+    ids=str,
 )
 def test_chia_plots_create_parsing_does_not_fail(arguments):
     job.parse_chia_plots_create_command_line(
@@ -93,6 +95,7 @@ def test_chia_plots_create_parsing_does_not_fail(arguments):
         [['--help']],
         [['-k', '32', '--help']],
     ],
+    ids=str,
 )
 def test_chia_plots_create_parsing_detects_help(arguments):
     parsed = job.parse_chia_plots_create_command_line(
@@ -109,6 +112,7 @@ def test_chia_plots_create_parsing_detects_help(arguments):
         [['-k32']],
         [['-k', '32']],
     ],
+    ids=str,
 )
 def test_chia_plots_create_parsing_detects_not_help(arguments):
     parsed = job.parse_chia_plots_create_command_line(
@@ -116,3 +120,21 @@ def test_chia_plots_create_parsing_detects_not_help(arguments):
     )
 
     assert not parsed.help
+
+
+@pytest.mark.parametrize(
+    argnames=['arguments'],
+    argvalues=[
+        [[]],
+        [['-k32']],
+        [['-k', '32']],
+        [['--size', '32']],
+    ],
+    ids=str,
+)
+def test_chia_plots_create_parsing_handles_argument_forms(arguments):
+    parsed = job.parse_chia_plots_create_command_line(
+        command_line=['python', 'chia', 'plots', 'create', *arguments],
+    )
+
+    assert parsed.parameters['size'] == 32
diff --git a/src/plotman/job.py b/src/plotman/job.py
index 48e48a48..8dc393d0 100644
--- a/src/plotman/job.py
+++ b/src/plotman/job.py
@@ -53,11 +53,6 @@ def parse_chia_plot_time(s):
     # This will grow to try ISO8601 as well for when Chia logs that way
     return pendulum.from_format(s, 'ddd MMM DD HH:mm:ss YYYY', locale='en', tz=None)
 
-class ParsedChiaPlotsCreateCommand:
-    def __init__(self, help, parameters):
-        self.help = help
-        self.parameters = parameters
-
 def parse_chia_plots_create_command_line(command_line):
     # Parse command line args
     assert len(command_line) >= 4
@@ -86,6 +81,11 @@ def parse_chia_plots_create_command_line(command_line):
         parameters=context.params,
     )
 
+class ParsedChiaPlotsCreateCommand:
+    def __init__(self, help, parameters):
+        self.help = help
+        self.parameters = parameters
+
 # TODO: be more principled and explicit about what we cache vs. what we look up
 # dynamically from the logfile
 class Job:

From a4737c0bbe12936fceb084d0c4b2e78bd9f55104 Mon Sep 17 00:00:00 2001
From: Kyle Altendorf <sda@fstab.net>
Date: Sat, 17 Apr 2021 21:28:53 -0400
Subject: [PATCH 11/86] fix: skip jobs whose arguments fail to be parsed

---
 setup.cfg                      |   1 +
 src/plotman/_tests/job_test.py |  29 +++----
 src/plotman/job.py             | 149 ++++++++++++++++++---------------
 3 files changed, 99 insertions(+), 80 deletions(-)

diff --git a/setup.cfg b/setup.cfg
index a8b9e4f5..bc8fb59c 100644
--- a/setup.cfg
+++ b/setup.cfg
@@ -34,6 +34,7 @@ package_dir=
 packages=find:
 install_requires =
     appdirs
+    click
     desert
     marshmallow
     pendulum
diff --git a/src/plotman/_tests/job_test.py b/src/plotman/_tests/job_test.py
index 3ce250d1..ecac7a88 100644
--- a/src/plotman/_tests/job_test.py
+++ b/src/plotman/_tests/job_test.py
@@ -82,27 +82,29 @@ def test_chia_plots_create_parsing_does_not_fail(arguments):
     ],
     ids=str,
 )
-def test_chia_plots_create_parsing_does_not_fail(arguments):
-    job.parse_chia_plots_create_command_line(
+def test_chia_plots_create_parsing_detects_help(arguments):
+    parsed = job.parse_chia_plots_create_command_line(
         command_line=['python', 'chia', 'plots', 'create', *arguments],
     )
 
+    assert parsed.help
+
 
 @pytest.mark.parametrize(
     argnames=['arguments'],
     argvalues=[
-        [['-h']],
-        [['--help']],
-        [['-k', '32', '--help']],
+        [[]],
+        [['-k32']],
+        [['-k', '32']],
     ],
     ids=str,
 )
-def test_chia_plots_create_parsing_detects_help(arguments):
+def test_chia_plots_create_parsing_detects_not_help(arguments):
     parsed = job.parse_chia_plots_create_command_line(
         command_line=['python', 'chia', 'plots', 'create', *arguments],
     )
 
-    assert parsed.help
+    assert not parsed.help
 
 
 @pytest.mark.parametrize(
@@ -111,24 +113,23 @@ def test_chia_plots_create_parsing_detects_help(arguments):
         [[]],
         [['-k32']],
         [['-k', '32']],
+        [['--size', '32']],
     ],
     ids=str,
 )
-def test_chia_plots_create_parsing_detects_not_help(arguments):
+def test_chia_plots_create_parsing_handles_argument_forms(arguments):
     parsed = job.parse_chia_plots_create_command_line(
         command_line=['python', 'chia', 'plots', 'create', *arguments],
     )
 
-    assert not parsed.help
+    assert parsed.parameters['size'] == 32
 
 
 @pytest.mark.parametrize(
     argnames=['arguments'],
     argvalues=[
-        [[]],
-        [['-k32']],
-        [['-k', '32']],
-        [['--size', '32']],
+        [['--size32']],
+        [['--not-an-actual-option']],
     ],
     ids=str,
 )
@@ -137,4 +138,4 @@ def test_chia_plots_create_parsing_handles_argument_forms(arguments):
         command_line=['python', 'chia', 'plots', 'create', *arguments],
     )
 
-    assert parsed.parameters['size'] == 32
+    assert parsed.error is not None
diff --git a/src/plotman/job.py b/src/plotman/job.py
index 8dc393d0..0ae3af27 100644
--- a/src/plotman/job.py
+++ b/src/plotman/job.py
@@ -13,6 +13,7 @@
 from subprocess import call
 
 import chia.cmds.plots
+import click
 import pendulum
 import psutil
 
@@ -74,15 +75,24 @@ def parse_chia_plots_create_command_line(command_line):
     ]
 
     command = chia.cmds.plots.create_cmd
-    context = command.make_context(info_name='', args=list(command_arguments))
+    try:
+        context = command.make_context(info_name='', args=list(command_arguments))
+    except click.ClickException as e:
+        error = e
+        params = {}
+    else:
+        error = None
+        params = context.params
 
     return ParsedChiaPlotsCreateCommand(
+        error=error,
         help=len(all_command_arguments) > len(command_arguments),
-        parameters=context.params,
+        parameters=params,
     )
 
 class ParsedChiaPlotsCreateCommand:
-    def __init__(self, help, parameters):
+    def __init__(self, error, help, parameters):
+        self.error = error
         self.help = help
         self.parameters = parameters
 
@@ -115,78 +125,85 @@ def get_running_jobs(logroot, cached_jobs=()):
                     if proc.pid in cached_jobs_by_pid.keys():
                         jobs.append(cached_jobs_by_pid[proc.pid])  # Copy from cache
                     else:
-                        job = Job(proc, logroot)
-                        if not job.help:
+                        with proc.oneshot():
+                            parsed_command = parse_chia_plots_create_command_line(
+                                command_line=proc.cmdline(),
+                            )
+                            if parsed_command.error is not None:
+                                continue
+                            if job.help:
+                                continue
+                            job = Job(
+                                proc=proc,
+                                parsed_command=parsed_command,
+                                logroot=logroot,
+                            )
                             jobs.append(job)
 
         return jobs
 
  
-    def __init__(self, proc, logroot):
+    def __init__(self, proc, parsed_command, logroot):
         '''Initialize from an existing psutil.Process object.  must know logroot in order to understand open files'''
         self.proc = proc
 
-        with self.proc.oneshot():
-            parsed_command = parse_chia_plots_create_command_line(
-                command_line=self.proc.cmdline(),
-            )
-            self.help = parsed_command.help
-            self.args = parsed_command.parameters
-
-            # an example as of 1.0.5
-            # {
-            #     'size': 32,
-            #     'num_threads': 4,
-            #     'buckets': 128,
-            #     'buffer': 6000,
-            #     'tmp_dir': '/farm/yards/901',
-            #     'final_dir': '/farm/wagons/801',
-            #     'override_k': False,
-            #     'num': 1,
-            #     'alt_fingerprint': None,
-            #     'pool_contract_address': None,
-            #     'farmer_public_key': None,
-            #     'pool_public_key': None,
-            #     'tmp2_dir': None,
-            #     'plotid': None,
-            #     'memo': None,
-            #     'nobitfield': False,
-            #     'exclude_final_dir': False,
-            # }
-
-            self.k = self.args['size']
-            self.r = self.args['num_threads']
-            self.u = self.args['buckets']
-            self.b = self.args['buffer']
-            self.n = self.args['num']
-            self.tmpdir = self.args['tmp_dir']
-            self.tmp2dir = self.args['tmp2_dir']
-            self.dstdir = self.args['final_dir']
-
-            plot_cwd = self.proc.cwd()
-            self.tmpdir = os.path.join(plot_cwd, self.tmpdir)
-            if self.tmp2dir is not None:
-                self.tmp2dir = os.path.join(plot_cwd, self.tmp2dir)
-            self.dstdir = os.path.join(plot_cwd, self.dstdir)
-
-            # Find logfile (whatever file is open under the log root).  The
-            # file may be open more than once, e.g. for STDOUT and STDERR.
+        self.help = parsed_command.help
+        self.args = parsed_command.parameters
+
+        # an example as of 1.0.5
+        # {
+        #     'size': 32,
+        #     'num_threads': 4,
+        #     'buckets': 128,
+        #     'buffer': 6000,
+        #     'tmp_dir': '/farm/yards/901',
+        #     'final_dir': '/farm/wagons/801',
+        #     'override_k': False,
+        #     'num': 1,
+        #     'alt_fingerprint': None,
+        #     'pool_contract_address': None,
+        #     'farmer_public_key': None,
+        #     'pool_public_key': None,
+        #     'tmp2_dir': None,
+        #     'plotid': None,
+        #     'memo': None,
+        #     'nobitfield': False,
+        #     'exclude_final_dir': False,
+        # }
+
+        self.k = self.args['size']
+        self.r = self.args['num_threads']
+        self.u = self.args['buckets']
+        self.b = self.args['buffer']
+        self.n = self.args['num']
+        self.tmpdir = self.args['tmp_dir']
+        self.tmp2dir = self.args['tmp2_dir']
+        self.dstdir = self.args['final_dir']
+
+        plot_cwd = self.proc.cwd()
+        self.tmpdir = os.path.join(plot_cwd, self.tmpdir)
+        if self.tmp2dir is not None:
+            self.tmp2dir = os.path.join(plot_cwd, self.tmp2dir)
+        self.dstdir = os.path.join(plot_cwd, self.dstdir)
+
+        # Find logfile (whatever file is open under the log root).  The
+        # file may be open more than once, e.g. for STDOUT and STDERR.
+        for f in self.proc.open_files():
+            if logroot in f.path:
+                if self.logfile:
+                    assert self.logfile == f.path
+                else:
+                    self.logfile = f.path
+                break
+
+        if self.logfile:
+            # Initialize data that needs to be loaded from the logfile
+            self.init_from_logfile()
+        else:
+            print('Found plotting process PID {pid}, but could not find '
+                    'logfile in its open files:'.format(pid = self.proc.pid))
             for f in self.proc.open_files():
-                if logroot in f.path:
-                    if self.logfile:
-                        assert self.logfile == f.path
-                    else:
-                        self.logfile = f.path
-                    break
-
-            if self.logfile:
-                # Initialize data that needs to be loaded from the logfile
-                self.init_from_logfile()
-            else:
-                print('Found plotting process PID {pid}, but could not find '
-                        'logfile in its open files:'.format(pid = self.proc.pid))
-                for f in self.proc.open_files():
-                    print(f.path)
+                print(f.path)
 
 
 

From 0fce189ef2a77f9f00e498b8661cf262764c0bb6 Mon Sep 17 00:00:00 2001
From: 13thProgression <rookie@13thprogression.com>
Date: Mon, 19 Apr 2021 14:43:21 -0600
Subject: [PATCH 12/86] Include job age any time that a plot is held off;
 Insert the last delay message into the log if it isn't a stagger

---
 src/plotman/interactive.py | 7 +++++++
 src/plotman/manager.py     | 4 ++--
 2 files changed, 9 insertions(+), 2 deletions(-)

diff --git a/src/plotman/interactive.py b/src/plotman/interactive.py
index 321aa14c..f6dafb68 100644
--- a/src/plotman/interactive.py
+++ b/src/plotman/interactive.py
@@ -87,6 +87,7 @@ def curses_main(stdscr):
     pressed_key = ''   # For debugging
 
     archdir_freebytes = None
+    aging_reason = None
 
     while True:
 
@@ -113,10 +114,16 @@ def curses_main(stdscr):
                     cfg.directories, cfg.scheduling, cfg.plotting
                 )
                 if (started):
+                    if aging_reason is not None:
+                        log.log(aging_reason)
+                        aging_reason = None
                     log.log(msg)
                     plotting_status = '<just started job>'
                     jobs = Job.get_running_jobs(cfg.directories.log, cached_jobs=jobs)
                 else:
+                    # If a plot is delayed for any reason other than stagger, log it
+                    if msg.find("stagger") < 0:
+                        aging_reason = msg
                     plotting_status = msg
 
             if archiving_configured:
diff --git a/src/plotman/manager.py b/src/plotman/manager.py
index 19a45d63..9463bdd4 100644
--- a/src/plotman/manager.py
+++ b/src/plotman/manager.py
@@ -77,7 +77,7 @@ def maybe_start_new_plot(dir_cfg, sched_cfg, plotting_cfg):
     if (youngest_job_age < global_stagger):
         wait_reason = 'stagger (%ds/%ds)' % (youngest_job_age, global_stagger)
     elif len(jobs) >= sched_cfg.global_max_jobs:
-        wait_reason = 'max jobs (%d)' % sched_cfg.global_max_jobs
+        wait_reason = 'max jobs (%d) - (%ds/%ds)' % (sched_cfg.global_max_jobs, youngest_job_age, global_stagger)
     else:
         tmp_to_all_phases = [(d, job.job_phases_for_tmpdir(d, jobs)) for d in dir_cfg.tmp]
         eligible = [ (d, phases) for (d, phases) in tmp_to_all_phases
@@ -86,7 +86,7 @@ def maybe_start_new_plot(dir_cfg, sched_cfg, plotting_cfg):
                 for (d, phases) in eligible ]
         
         if not eligible:
-            wait_reason = 'no eligible tempdirs'
+            wait_reason = 'no eligible tempdirs (%ds/%ds)' % (youngest_job_age, global_stagger)
         else:
             # Plot to oldest tmpdir.
             tmpdir = max(rankable, key=operator.itemgetter(1))[0]

From 7ad30b99f1d5ccac8a20854c2cfa801fdad0db54 Mon Sep 17 00:00:00 2001
From: Kyle Altendorf <sda@fstab.net>
Date: Mon, 19 Apr 2021 22:19:47 -0400
Subject: [PATCH 13/86] chore: remove unused threading imports

---
 src/plotman/interactive.py | 1 -
 src/plotman/job.py         | 1 -
 src/plotman/manager.py     | 1 -
 3 files changed, 3 deletions(-)

diff --git a/src/plotman/interactive.py b/src/plotman/interactive.py
index 321aa14c..d10a2b22 100644
--- a/src/plotman/interactive.py
+++ b/src/plotman/interactive.py
@@ -4,7 +4,6 @@
 import math
 import os
 import subprocess
-import threading
 
 from plotman import archive, configuration, manager, reporting
 from plotman.job import Job
diff --git a/src/plotman/job.py b/src/plotman/job.py
index 6314d168..14c9af59 100644
--- a/src/plotman/job.py
+++ b/src/plotman/job.py
@@ -6,7 +6,6 @@
 import random
 import re
 import sys
-import threading
 import time
 from datetime import datetime
 from enum import Enum, auto
diff --git a/src/plotman/manager.py b/src/plotman/manager.py
index 19a45d63..a5c9a665 100644
--- a/src/plotman/manager.py
+++ b/src/plotman/manager.py
@@ -6,7 +6,6 @@
 import readline  # For nice CLI
 import subprocess
 import sys
-import threading
 import time
 from datetime import datetime
 

From 845750a9c833b4fae2aa627744e2a29abbbbdeaf Mon Sep 17 00:00:00 2001
From: Kyle Altendorf <sda@fstab.net>
Date: Tue, 20 Apr 2021 00:26:06 -0400
Subject: [PATCH 14/86] ci: also build on pushes to main

---
 .github/workflows/ci.yml | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index dfa08abb..797469aa 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -2,7 +2,9 @@ name: CI
 
 on:
   push:
-    branches: [ development ]
+    branches:
+      - main
+      - development
     tags: [ "**" ]
   pull_request:
     branches: [ "**" ]

From 7e939271eba0c8e100c506a06cf09c5dd60a1b99 Mon Sep 17 00:00:00 2001
From: Kyle Altendorf <sda@fstab.net>
Date: Tue, 20 Apr 2021 08:18:33 -0400
Subject: [PATCH 15/86] fix: only report archive dirs if configured

---
 src/plotman/reporting.py | 18 +++++++++++-------
 1 file changed, 11 insertions(+), 7 deletions(-)

diff --git a/src/plotman/reporting.py b/src/plotman/reporting.py
index 80af0b55..cc9c803c 100644
--- a/src/plotman/reporting.py
+++ b/src/plotman/reporting.py
@@ -186,10 +186,14 @@ def arch_dir_report(archdir_freebytes, width, prefix=''):
 
 # TODO: remove this
 def dirs_report(jobs, dir_cfg, sched_cfg, width):
-    return (
-        tmp_dir_report(jobs, dir_cfg, sched_cfg, width) + '\n' +
-        dst_dir_report(jobs, dir_cfg.dst, width) + '\n' +
-        'archive dirs free space:\n' +
-        arch_dir_report(archive.get_archdir_freebytes(dir_cfg.archive), width) + '\n'
-    )
-
+    reports = [
+        tmp_dir_report(jobs, dir_cfg, sched_cfg, width),
+        dst_dir_report(jobs, dir_cfg.dst, width),
+    ]
+    if dir_cfg.archive is not None:
+        reports.extend([
+            'archive dirs free space:',
+            arch_dir_report(archive.get_archdir_freebytes(dir_cfg.archive), width),
+        ])
+
+    return '\n'.join(reports) + '\n'

From 1112b3d0ba23ae5fe91526c92c9e4b269fa1c57c Mon Sep 17 00:00:00 2001
From: David Taylor <david@taylorhq.com>
Date: Tue, 20 Apr 2021 23:36:55 +0100
Subject: [PATCH 16/86] add support for chia installed via macOS dmg installer

The official macOS dmg installation makes `chia` available as a standalone binary. That means the `cmdline` does not include `python` in the first position. This commit updates the cmdline parsing logic to allow for `chia` binaries like this.
---
 README.md          |  9 ++++++---
 src/plotman/job.py | 22 ++++++++++++----------
 2 files changed, 18 insertions(+), 13 deletions(-)

diff --git a/README.md b/README.md
index b298e7ca..3216f800 100644
--- a/README.md
+++ b/README.md
@@ -181,11 +181,14 @@ To display the current location of your `plotman.yaml` file and check if it exis
 
 ## Installation
 
-Installation for Linux:
+Installation for Linux and macOS:
 
 1. Plotman assumes that a functioning [Chia](https://github.com/Chia-Network/chia-blockchain)
-   installation is present on the system. Activate your `chia` environment by typing
-   `source /path/to/your/chia/install/activate`.
+   installation is present on the system.
+      - Linux: Activate your `chia` environment by typing
+        `source /path/to/your/chia/install/activate`.
+      - macOS: Follow [these instructions](https://github.com/Chia-Network/chia-blockchain/wiki/CLI-Commands-Reference#mac)
+        to add the `chia` binary to the PATH
 2. Then, install Plotman using the following command:
    ```shell
     > pip install --force-reinstall git+https://github.com/ericaltendorf/plotman@main
diff --git a/src/plotman/job.py b/src/plotman/job.py
index 6314d168..2d8d01f6 100644
--- a/src/plotman/job.py
+++ b/src/plotman/job.py
@@ -25,12 +25,13 @@ def job_phases_for_dstdir(d, all_jobs):
     return sorted([j.progress() for j in all_jobs if j.dstdir == d])
 
 def is_plotting_cmdline(cmdline):
+    if cmdline and 'python' in cmdline[0]:
+        cmdline = cmdline[1:]
     return (
-        len(cmdline) >= 4
-        and 'python' in cmdline[0]
-        and cmdline[1].endswith('/chia')
-        and 'plots' == cmdline[2]
-        and 'create' == cmdline[3]
+        len(cmdline) >= 3
+        and cmdline[0].endswith("chia")
+        and 'plots' == cmdline[1]
+        and 'create' == cmdline[2]
     )
 
 # This is a cmdline argument fix for https://github.com/ericaltendorf/plotman/issues/41
@@ -105,12 +106,13 @@ def __init__(self, proc, logroot):
         with self.proc.oneshot():
             # Parse command line args
             args = self.proc.cmdline()
+            if 'python' in args[0]:
+                args = args[1:]
             assert len(args) > 4
-            assert 'python' in args[0]
-            assert 'chia' in args[1]
-            assert 'plots' == args[2]
-            assert 'create' == args[3]
-            args_iter = iter(cmdline_argfix(args[4:]))
+            assert 'chia' in args[0]
+            assert 'plots' == args[1]
+            assert 'create' == args[2]
+            args_iter = iter(cmdline_argfix(args[3:]))
             for arg in args_iter:
                 val = None if arg in {'-e', '--nobitfield', '-h', '--help', '--override-k'} else next(args_iter)
                 if arg in {'-k', '--size'}:

From 741f4db43abea8acbf04aefd0cce5179ca2d2474 Mon Sep 17 00:00:00 2001
From: BasilHorowt <82249557+BasilHorowt@users.noreply.github.com>
Date: Tue, 20 Apr 2021 23:19:02 -0700
Subject: [PATCH 17/86] Update process detection logic to work on OSX

the arguments passed to this function looked like this on OSX:

`['/Library/Frameworks/Python.framework/Versions/3.9/Resources/Python.app/Contents/MacOS/Python', '/Users/chia/chia-blockchain/venv/bin/chia', 'plots', 'create', '-k', '32', '-r', '2', '-u', '128', '-b', '4608', '-t', '/Volumes/plotting2tb', '-d', '/Volumes/16tb']`

The first argument did not contain a lowercase 'python', so we can lowercase the string before comparison to make the check work more generally.
---
 src/plotman/job.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/plotman/job.py b/src/plotman/job.py
index 6314d168..0f410d06 100644
--- a/src/plotman/job.py
+++ b/src/plotman/job.py
@@ -27,7 +27,7 @@ def job_phases_for_dstdir(d, all_jobs):
 def is_plotting_cmdline(cmdline):
     return (
         len(cmdline) >= 4
-        and 'python' in cmdline[0]
+        and 'python' in cmdline[0].lower()
         and cmdline[1].endswith('/chia')
         and 'plots' == cmdline[2]
         and 'create' == cmdline[3]

From 6ad536d0f482a92292a8fad22411e83bc3cc2f68 Mon Sep 17 00:00:00 2001
From: BasilHorowt <82249557+BasilHorowt@users.noreply.github.com>
Date: Tue, 20 Apr 2021 23:25:18 -0700
Subject: [PATCH 18/86] fixup! Update process detection logic to work on OSX

---
 src/plotman/job.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/plotman/job.py b/src/plotman/job.py
index 0f410d06..5a34ce60 100644
--- a/src/plotman/job.py
+++ b/src/plotman/job.py
@@ -106,7 +106,7 @@ def __init__(self, proc, logroot):
             # Parse command line args
             args = self.proc.cmdline()
             assert len(args) > 4
-            assert 'python' in args[0]
+            assert 'python' in args[0].lower()
             assert 'chia' in args[1]
             assert 'plots' == args[2]
             assert 'create' == args[3]

From 2b4fcc5a2600161b7852a71cdc27940f26af81d3 Mon Sep 17 00:00:00 2001
From: Kyle Altendorf <sda@fstab.net>
Date: Wed, 21 Apr 2021 23:03:41 -0400
Subject: [PATCH 19/86] chore: remove unused code

---
 src/plotman/job.py | 15 ---------------
 1 file changed, 15 deletions(-)

diff --git a/src/plotman/job.py b/src/plotman/job.py
index 3f1fd1fb..4e0926b2 100644
--- a/src/plotman/job.py
+++ b/src/plotman/job.py
@@ -34,21 +34,6 @@ def is_plotting_cmdline(cmdline):
         and 'create' == cmdline[3]
     )
 
-# This is a cmdline argument fix for https://github.com/ericaltendorf/plotman/issues/41
-def cmdline_argfix(cmdline):
-    known_keys = 'krbut2dne'
-    for i in cmdline:
-        # If the argument starts with dash and a known key and is longer than 2,
-        # then an argument is passed with no space between its key and value.
-        # This is POSIX compliant but the arg parser was tripping over it.
-        # In these cases, splitting that item up in separate key and value
-        # elements results in a `cmdline` list that is correctly formatted.
-        if i[0]=='-' and i[1] in known_keys and len(i)>2:
-            yield i[0:2]  # key
-            yield i[2:]  # value
-        else:
-            yield i
-
 def parse_chia_plot_time(s):
     # This will grow to try ISO8601 as well for when Chia logs that way
     return pendulum.from_format(s, 'ddd MMM DD HH:mm:ss YYYY', locale='en', tz=None)

From 7229b43a8818936f528daf59adda845ffe02eea3 Mon Sep 17 00:00:00 2001
From: David Taylor <david@taylorhq.com>
Date: Thu, 22 Apr 2021 10:01:02 +0100
Subject: [PATCH 20/86] Update README.md

Co-authored-by: Kyle Altendorf <sda@fstab.net>
---
 README.md | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/README.md b/README.md
index 3216f800..b53e4244 100644
--- a/README.md
+++ b/README.md
@@ -185,10 +185,10 @@ Installation for Linux and macOS:
 
 1. Plotman assumes that a functioning [Chia](https://github.com/Chia-Network/chia-blockchain)
    installation is present on the system.
-      - Linux: Activate your `chia` environment by typing
+      - virtual environment (Linux, macOS): Activate your `chia` environment by typing
         `source /path/to/your/chia/install/activate`.
-      - macOS: Follow [these instructions](https://github.com/Chia-Network/chia-blockchain/wiki/CLI-Commands-Reference#mac)
-        to add the `chia` binary to the PATH
+      - dmg (macOS): Follow [these instructions](https://github.com/Chia-Network/chia-blockchain/wiki/CLI-Commands-Reference#mac)
+        to add the `chia` binary to the `PATH`
 2. Then, install Plotman using the following command:
    ```shell
     > pip install --force-reinstall git+https://github.com/ericaltendorf/plotman@main

From 5200b13ac0663dd8873612c86977e7b44bd7fb93 Mon Sep 17 00:00:00 2001
From: Kyle Altendorf <sda@fstab.net>
Date: Thu, 22 Apr 2021 20:14:44 -0400
Subject: [PATCH 21/86] fix: handle AccessDenied in report for macOS

---
 src/plotman/reporting.py | 31 ++++++++++++++++---------------
 1 file changed, 16 insertions(+), 15 deletions(-)

diff --git a/src/plotman/reporting.py b/src/plotman/reporting.py
index 80af0b55..f89c4a4a 100644
--- a/src/plotman/reporting.py
+++ b/src/plotman/reporting.py
@@ -95,21 +95,22 @@ def status_report(jobs, width, height=None, tmp_prefix='', dst_prefix=''):
         # Regular row
         else:
             try:
-                row = [j.plot_id[:8],
-                    j.k,
-                    abbr_path(j.tmpdir, tmp_prefix),
-                    abbr_path(j.dstdir, dst_prefix),
-                    plot_util.time_format(j.get_time_wall()),
-                    phase_str(j.progress()),
-                    plot_util.human_format(j.get_tmp_usage(), 0),
-                    j.proc.pid,
-                    j.get_run_status(),
-                    plot_util.human_format(j.get_mem_usage(), 1),
-                    plot_util.time_format(j.get_time_user()),
-                    plot_util.time_format(j.get_time_sys()),
-                    plot_util.time_format(j.get_time_iowait())
-                    ]
-            except psutil.NoSuchProcess:
+                with j.proc.one_shot():
+                    row = [j.plot_id[:8],
+                        j.k,
+                        abbr_path(j.tmpdir, tmp_prefix),
+                        abbr_path(j.dstdir, dst_prefix),
+                        plot_util.time_format(j.get_time_wall()),
+                        phase_str(j.progress()),
+                        plot_util.human_format(j.get_tmp_usage(), 0),
+                        j.proc.pid,
+                        j.get_run_status(),
+                        plot_util.human_format(j.get_mem_usage(), 1),
+                        plot_util.time_format(j.get_time_user()),
+                        plot_util.time_format(j.get_time_sys()),
+                        plot_util.time_format(j.get_time_iowait())
+                        ]
+            except psutil.NoSuchProcess, psutil.AccessDenied:
                 # In case the job has disappeared
                 row = [j.plot_id[:8]] + (['--'] * 12)
 

From a390c35b8ddb601531c8f9fd91a28126ab98229a Mon Sep 17 00:00:00 2001
From: Kyle Altendorf <sda@fstab.net>
Date: Thu, 22 Apr 2021 20:19:16 -0400
Subject: [PATCH 22/86] chore: correct multi-type exception handling syntax

---
 src/plotman/reporting.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/plotman/reporting.py b/src/plotman/reporting.py
index f89c4a4a..0b41a64e 100644
--- a/src/plotman/reporting.py
+++ b/src/plotman/reporting.py
@@ -110,7 +110,7 @@ def status_report(jobs, width, height=None, tmp_prefix='', dst_prefix=''):
                         plot_util.time_format(j.get_time_sys()),
                         plot_util.time_format(j.get_time_iowait())
                         ]
-            except psutil.NoSuchProcess, psutil.AccessDenied:
+            except (psutil.NoSuchProcess, psutil.AccessDenied):
                 # In case the job has disappeared
                 row = [j.plot_id[:8]] + (['--'] * 12)
 

From 97dbc608db4e0c34ab46cce9fe91ae102dcf06d5 Mon Sep 17 00:00:00 2001
From: Kyle Altendorf <sda@fstab.net>
Date: Fri, 23 Apr 2021 03:24:20 +0000
Subject: [PATCH 23/86] raise TerminalTooSmallError

---
 src/plotman/interactive.py | 10 +++++++++-
 1 file changed, 9 insertions(+), 1 deletion(-)

diff --git a/src/plotman/interactive.py b/src/plotman/interactive.py
index d10a2b22..73464e41 100644
--- a/src/plotman/interactive.py
+++ b/src/plotman/interactive.py
@@ -9,6 +9,9 @@
 from plotman.job import Job
 
 
+class TerminalTooSmallError(Exception):
+    pass
+
 class Log:
     def __init__(self):
         self.entries = []
@@ -349,4 +352,9 @@ def run_interactive():
     code = locale.getpreferredencoding()
     # Then use code as the encoding for str.encode() calls.
 
-    curses.wrapper(curses_main)
+    try:
+        curses.wrapper(curses_main)
+    except curses.error as e:
+        raise TerminalTooSmallError(
+            "Your terminal may be too small, try making it bigger.",
+        ) from e

From 242a8346acc6c6626bb0e13f5f96fa47b94a404a Mon Sep 17 00:00:00 2001
From: Kyle Altendorf <sda@fstab.net>
Date: Fri, 23 Apr 2021 23:45:07 +0000
Subject: [PATCH 24/86] chore: fix mismerge

---
 src/plotman/job.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/plotman/job.py b/src/plotman/job.py
index 60d7697c..f26d3920 100644
--- a/src/plotman/job.py
+++ b/src/plotman/job.py
@@ -118,13 +118,13 @@ def get_running_jobs(logroot, cached_jobs=()):
                             )
                             if parsed_command.error is not None:
                                 continue
-                            if job.help:
-                                continue
                             job = Job(
                                 proc=proc,
                                 parsed_command=parsed_command,
                                 logroot=logroot,
                             )
+                            if job.help:
+                                continue
                             jobs.append(job)
 
         return jobs

From 2d15d25b2eaa0a815524bf3cccac2abdfb7ff3b2 Mon Sep 17 00:00:00 2001
From: BasilHorowt <82249557+BasilHorowt@users.noreply.github.com>
Date: Sat, 24 Apr 2021 21:17:03 -0700
Subject: [PATCH 25/86] Update default buffer option to 3389 to match Chia
 default

As per the [previous PR](https://github.com/ericaltendorf/plotman/pull/139), Plotman's default config should track the default Chia options to make things simpler.  As per 1.1.2, Chia actually defaults to 3389 for the buffer: https://github.com/Chia-Network/chia-blockchain/commit/d0649fb57407cfffbaec7baeba86fd76ecb51770
---
 src/plotman/resources/plotman.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/plotman/resources/plotman.yaml b/src/plotman/resources/plotman.yaml
index b2b69cb3..1b644c28 100644
--- a/src/plotman/resources/plotman.yaml
+++ b/src/plotman/resources/plotman.yaml
@@ -112,7 +112,7 @@ plotting:
         e: False             # Use -e plotting option
         n_threads: 2         # Threads per job
         n_buckets: 128       # Number of buckets to split data into
-        job_buffer: 4608     # Per job memory
+        job_buffer: 3389     # Per job memory
         # If specified, pass through to the -f and -p options.  See CLI reference.
         #   farmer_pk: ...
         #   pool_pk: ...

From a43f9b2c28bcafc39a5a4eaf1cb52352857bf883 Mon Sep 17 00:00:00 2001
From: Kyle Altendorf <sda@fstab.net>
Date: Sun, 25 Apr 2021 10:18:31 -0400
Subject: [PATCH 26/86] fix: correct error message when kill does not find jobs

---
 src/plotman/plotman.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/plotman/plotman.py b/src/plotman/plotman.py
index 77602d69..b5aa3108 100755
--- a/src/plotman/plotman.py
+++ b/src/plotman/plotman.py
@@ -202,9 +202,9 @@ def main():
                 # TODO: allow multiple idprefixes, not just take the first
                 selected = manager.select_jobs_by_partial_id(jobs, args.idprefix[0])
                 if (len(selected) == 0):
-                    print('Error: %s matched no jobs.' % id_spec)
+                    print('Error: %s matched no jobs.' % args.idprefix[0])
                 elif len(selected) > 1:
-                    print('Error: "%s" matched multiple jobs:' % id_spec)
+                    print('Error: "%s" matched multiple jobs:' % args.idprefix[0])
                     for j in selected:
                         print('  %s' % j.plot_id)
                     selected = []

From d8a11e84530285dcab43696b9d8e7221d7f6e1aa Mon Sep 17 00:00:00 2001
From: Kyle Altendorf <sda@fstab.net>
Date: Sun, 25 Apr 2021 19:51:22 -0400
Subject: [PATCH 27/86] fix: ignore jobs without phases when checking dstdirs

---
 src/plotman/manager.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/plotman/manager.py b/src/plotman/manager.py
index a5c9a665..6a1c7761 100644
--- a/src/plotman/manager.py
+++ b/src/plotman/manager.py
@@ -92,7 +92,7 @@ def maybe_start_new_plot(dir_cfg, sched_cfg, plotting_cfg):
 
             # Select the dst dir least recently selected
             dir2ph = { d:ph for (d, ph) in dstdirs_to_youngest_phase(jobs).items()
-                      if d in dir_cfg.dst }
+                      if d in dir_cfg.dst and ph is not None}
             unused_dirs = [d for d in dir_cfg.dst if d not in dir2ph.keys()]
             dstdir = ''
             if unused_dirs: 

From da3583944f9bb86e2a246ac9cfc06a16c1339623 Mon Sep 17 00:00:00 2001
From: Kyle Altendorf <sda@fstab.net>
Date: Sun, 25 Apr 2021 20:09:04 -0400
Subject: [PATCH 28/86] fix: also skip Nones in dstdirs_to_youngest_phase()

---
 src/plotman/manager.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/src/plotman/manager.py b/src/plotman/manager.py
index 6a1c7761..4e7bb6cb 100644
--- a/src/plotman/manager.py
+++ b/src/plotman/manager.py
@@ -36,6 +36,8 @@ def dstdirs_to_youngest_phase(all_jobs):
        that is emitting to that dst dir.'''
     result = {}
     for j in all_jobs:
+        if j.dstdir is None:
+            continue
         if not j.dstdir in result.keys() or result[j.dstdir] > j.progress():
             result[j.dstdir] = j.progress()
     return result

From 9d2a4354ac580725e1be68d6e54b9cfafd4fc280 Mon Sep 17 00:00:00 2001
From: Jiahui Zhu <1284663+ilazycat@users.noreply.github.com>
Date: Mon, 26 Apr 2021 13:03:21 +0800
Subject: [PATCH 29/86] fix: support -a, --alt_fingerprint parameter

Change-Id: Ic1b7a8eb9e1567bab907bb60ea6887d6c7049207
---
 src/plotman/job.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/src/plotman/job.py b/src/plotman/job.py
index 29e706f2..97115be1 100644
--- a/src/plotman/job.py
+++ b/src/plotman/job.py
@@ -35,7 +35,7 @@ def is_plotting_cmdline(cmdline):
 
 # This is a cmdline argument fix for https://github.com/ericaltendorf/plotman/issues/41
 def cmdline_argfix(cmdline):
-    known_keys = 'krbut2dne'
+    known_keys = 'krbut2dnea'
     for i in cmdline:
         # If the argument starts with dash and a known key and is longer than 2,
         # then an argument is passed with no space between its key and value.
@@ -97,7 +97,7 @@ def get_running_jobs(logroot, cached_jobs=()):
 
         return jobs
 
- 
+
     def __init__(self, proc, logroot):
         '''Initialize from an existing psutil.Process object.  must know logroot in order to understand open files'''
         self.proc = proc
@@ -132,7 +132,7 @@ def __init__(self, proc, logroot):
                     self.n = val
                 elif arg in {'-h', '--help'}:
                     self.help = True
-                elif arg in {'-e', '--nobitfield', '-f', '--farmer_public_key', '-p', '--pool_public_key'}:
+                elif arg in {'-e', '--nobitfield', '-f', '--farmer_public_key', '-p', '--pool_public_key', '-a', '--alt_fingerprint'}:
                     pass
                     # TODO: keep track of these
                 elif arg == '--override-k':

From 26309512d442b2ca3486ec13245262c6f6a972b6 Mon Sep 17 00:00:00 2001
From: Kyle Altendorf <sda@fstab.net>
Date: Mon, 26 Apr 2021 07:58:45 -0400
Subject: [PATCH 30/86] fix: skip missing files in plot_util.list_k32_plots()

Perhaps it would be good to use os.scandir() but this is the direct quick fix.  Presumably this plot was removed by rsync between listing the directory and checking its size.

```
Traceback (most recent call last):
  File "/home/chia/.local/bin/plotman", line 33, in <module>
    sys.exit(load_entry_point('plotman', 'console_scripts', 'plotman')())
  File "/farm/plotman/src/plotman/plotman.py", line 171, in main
    interactive.run_interactive()
  File "/farm/plotman/src/plotman/interactive.py", line 352, in run_interactive
    curses.wrapper(curses_main)
  File "/home/chia/.pyenv/versions/3.9.2/lib/python3.9/curses/__init__.py", line 94, in wrapper
    return func(stdscr, *args, **kwds)
  File "/farm/plotman/src/plotman/interactive.py", line 187, in curses_main
    dst_report = reporting.dst_dir_report(
  File "/farm/plotman/src/plotman/reporting.py", line 160, in dst_dir_report
    dir_plots = plot_util.list_k32_plots(d)
  File "/farm/plotman/src/plotman/plot_util.py", line 54, in list_k32_plots
    if os.stat(plot).st_size > (0.95 * get_k32_plotsize()):
FileNotFoundError: [Errno 2] No such file or directory: '/farm/yards/900/plot-k32-2021-04-25-17-50-527165ae8b5ddc95cf5853fec7dfaaf6672246b3eec369458f84e28992b1602a.plot'
```
---
 src/plotman/plot_util.py | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/src/plotman/plot_util.py b/src/plotman/plot_util.py
index 59ef0fad..ca24ae08 100644
--- a/src/plotman/plot_util.py
+++ b/src/plotman/plot_util.py
@@ -51,8 +51,11 @@ def list_k32_plots(d):
     for plot in os.listdir(d):
         if re.match(r'^plot-k32-.*plot$', plot):
             plot = os.path.join(d, plot)
-            if os.stat(plot).st_size > (0.95 * get_k32_plotsize()):
-                plots.append(plot)
+            try:
+                if os.stat(plot).st_size > (0.95 * get_k32_plotsize()):
+                    plots.append(plot)
+            except FileNotFoundError:
+                continue
     
     return plots
 

From 9352d9cc8ad86107bb723693a7e63c136f6449a5 Mon Sep 17 00:00:00 2001
From: Clark Meyer <clarkmeyer@Clarks-MBP.chia.local>
Date: Mon, 26 Apr 2021 16:45:58 -0700
Subject: [PATCH 31/86] Created spawn_archive_process function

---
 src/plotman/archive.py     | 27 +++++++++++++++++++++++++++
 src/plotman/interactive.py | 22 +++-------------------
 src/plotman/plotman.py     |  6 +++++-
 3 files changed, 35 insertions(+), 20 deletions(-)

diff --git a/src/plotman/archive.py b/src/plotman/archive.py
index 17a29af0..ff3c4797 100644
--- a/src/plotman/archive.py
+++ b/src/plotman/archive.py
@@ -15,6 +15,33 @@
 
 # TODO : write-protect and delete-protect archived plots
 
+def spawn_archive_process(dir_cfg, all_jobs):
+    '''Spawns a new archive process using the command created 
+    in the archive() function. Returns archiving status and a log message to print.'''
+
+    log_message = None
+
+    # Look for running archive jobs.  Be robust to finding more than one
+    # even though the scheduler should only run one at a time.
+    arch_jobs = get_running_archive_jobs(dir_cfg.archive)
+    
+    if arch_jobs:
+        archiving_status = 'pid: ' + ', '.join(map(str, arch_jobs))
+    else:
+        (should_start, status_or_cmd) = archive(dir_cfg, all_jobs)
+        if not should_start:
+            archiving_status = status_or_cmd
+        else:
+            cmd = status_or_cmd
+            # TODO: do something useful with output instead of DEVNULL
+            p = subprocess.Popen(cmd,
+                    shell=True,
+                    stdout=subprocess.DEVNULL,
+                    stderr=subprocess.STDOUT,
+                    start_new_session=True) 
+            log_message = 'Starting archive: ' + cmd
+    return archiving_status, log_message
+            
 def compute_priority(phase, gb_free, n_plots):
     # All these values are designed around dst buffer dirs of about
     # ~2TB size and containing k32 plots.  TODO: Generalize, and
diff --git a/src/plotman/interactive.py b/src/plotman/interactive.py
index d10a2b22..1597bb82 100644
--- a/src/plotman/interactive.py
+++ b/src/plotman/interactive.py
@@ -120,25 +120,9 @@ def curses_main(stdscr):
 
             if archiving_configured:
                 if archiving_active:
-                    # Look for running archive jobs.  Be robust to finding more than one
-                    # even though the scheduler should only run one at a time.
-                    arch_jobs = archive.get_running_archive_jobs(cfg.directories.archive)
-                    if arch_jobs:
-                        archiving_status = 'pid: ' + ', '.join(map(str, arch_jobs))
-                    else:
-                        (should_start, status_or_cmd) = archive.archive(cfg.directories, jobs)
-                        if not should_start:
-                            archiving_status = status_or_cmd
-                        else:
-                            cmd = status_or_cmd
-                            log.log('Starting archive: ' + cmd)
-
-                            # TODO: do something useful with output instead of DEVNULL
-                            p = subprocess.Popen(cmd,
-                                    shell=True,
-                                    stdout=subprocess.DEVNULL,
-                                    stderr=subprocess.STDOUT,
-                                    start_new_session=True)
+                    archiving_status, log_message = archive.spawn_archive_process(cfg.directories, jobs)
+                    if log_message:
+                        log.log(log_message)
 
                 archdir_freebytes = archive.get_archdir_freebytes(cfg.directories.archive)
 
diff --git a/src/plotman/plotman.py b/src/plotman/plotman.py
index 77602d69..2dd34579 100755
--- a/src/plotman/plotman.py
+++ b/src/plotman/plotman.py
@@ -180,7 +180,11 @@ def main():
                     time.sleep(60)
                     jobs = Job.get_running_jobs(cfg.directories.log)
                 firstit = False
-                archive.archive(cfg.directories, jobs)
+
+                archiving_status, log_message = archive.spawn_archive_process(cfg.directories, jobs)
+                if log_message:
+                    print(log_message)
+
 
         # Debugging: show the destination drive usage schedule
         elif args.cmd == 'dsched':

From 65d8a4af8af8308a07d25b0a1f2053a5cce731a7 Mon Sep 17 00:00:00 2001
From: Clark Meyer <clarkmeyer@Clarks-MBP.chia.local>
Date: Mon, 26 Apr 2021 17:11:10 -0700
Subject: [PATCH 32/86] archiving_status = None

---
 src/plotman/archive.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/src/plotman/archive.py b/src/plotman/archive.py
index ff3c4797..6d9f15f8 100644
--- a/src/plotman/archive.py
+++ b/src/plotman/archive.py
@@ -20,7 +20,8 @@ def spawn_archive_process(dir_cfg, all_jobs):
     in the archive() function. Returns archiving status and a log message to print.'''
 
     log_message = None
-
+    archiving_status = None
+    
     # Look for running archive jobs.  Be robust to finding more than one
     # even though the scheduler should only run one at a time.
     arch_jobs = get_running_archive_jobs(dir_cfg.archive)

From 2e72254d048564f03c1da4b5909b4332925be610 Mon Sep 17 00:00:00 2001
From: Basil Horowt <82249557+BasilHorowt@users.noreply.github.com>
Date: Mon, 26 Apr 2021 22:23:46 -0700
Subject: [PATCH 33/86] Improve compatibility and precision of date information
 in log filenames
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

There a few reasons that the current log filenames may not be ideal:

1) They contain `:` which breaks in Windows and is shown as `/` in OSX GUI ([#144](https://github.com/ericaltendorf/plotman/issues/144))
2) They don't contain any more precision than seconds, so with many concurrent plots being created (or error situations), naming collisions can occur
3) They don't contain any timezone information which makes them not fully comprehensive

This potentially closes https://github.com/ericaltendorf/plotman/issues/144

Note: As discussed on chat with @altendky, we settled on `+HH_MM` / `-HH_MM` for the timezone part (`±[hh]:[mm]` with the same `:` -> `_` done for times).  it seems that `strftime` provides the offset as one component via `%z`, without the option to use a `HH:MM` format.  For now I have left it without a separator, but it would be trivial to insert a `_` if it is desired. Thoughts?
---
 src/plotman/_tests/job_test.py                                 | 2 +-
 ...-04-04-19:00:47.log => 2021-04-04T19_00_47.681088-0400.log} | 0
 ...04-19:00:47.notes => 2021-04-04T19_00_47.681088-0400.notes} | 0
 src/plotman/manager.py                                         | 3 ++-
 4 files changed, 3 insertions(+), 2 deletions(-)
 rename src/plotman/_tests/resources/{2021-04-04-19:00:47.log => 2021-04-04T19_00_47.681088-0400.log} (100%)
 rename src/plotman/_tests/resources/{2021-04-04-19:00:47.notes => 2021-04-04T19_00_47.681088-0400.notes} (100%)

diff --git a/src/plotman/_tests/job_test.py b/src/plotman/_tests/job_test.py
index 04727a92..ce6ff5e0 100644
--- a/src/plotman/_tests/job_test.py
+++ b/src/plotman/_tests/job_test.py
@@ -21,7 +21,7 @@ def update_from_logfile(self):
 
 @pytest.fixture(name='logfile_path')
 def logfile_fixture(tmp_path):
-    log_name = '2021-04-04-19:00:47.log'
+    log_name = '2021-04-04T19_00_47.681088-0400.log'
     log_contents = importlib.resources.read_binary(resources, log_name)
     log_file_path = tmp_path.joinpath(log_name)
     log_file_path.write_bytes(log_contents)
diff --git a/src/plotman/_tests/resources/2021-04-04-19:00:47.log b/src/plotman/_tests/resources/2021-04-04T19_00_47.681088-0400.log
similarity index 100%
rename from src/plotman/_tests/resources/2021-04-04-19:00:47.log
rename to src/plotman/_tests/resources/2021-04-04T19_00_47.681088-0400.log
diff --git a/src/plotman/_tests/resources/2021-04-04-19:00:47.notes b/src/plotman/_tests/resources/2021-04-04T19_00_47.681088-0400.notes
similarity index 100%
rename from src/plotman/_tests/resources/2021-04-04-19:00:47.notes
rename to src/plotman/_tests/resources/2021-04-04T19_00_47.681088-0400.notes
diff --git a/src/plotman/manager.py b/src/plotman/manager.py
index a5c9a665..e508baba 100644
--- a/src/plotman/manager.py
+++ b/src/plotman/manager.py
@@ -9,6 +9,7 @@
 import time
 from datetime import datetime
 
+import pendulum
 import psutil
 
 # Plotman libraries
@@ -101,7 +102,7 @@ def maybe_start_new_plot(dir_cfg, sched_cfg, plotting_cfg):
                 dstdir = max(dir2ph, key=dir2ph.get)
 
             logfile = os.path.join(
-                dir_cfg.log, datetime.now().strftime('%Y-%m-%d-%H:%M:%S.log')
+                dir_cfg.log, pendulum.now().strftime('%Y-%m-%dT%H_%M_%S.%f%z.log')
             )
 
             plot_args = ['chia', 'plots', 'create',

From 6b42d8baa164683ebe63af9417f62900aa888923 Mon Sep 17 00:00:00 2001
From: BasilHorowt <82249557+BasilHorowt@users.noreply.github.com>
Date: Tue, 27 Apr 2021 21:25:35 -0700
Subject: [PATCH 34/86] Use isoformat instead of custom built format string

Co-authored-by: Kyle Altendorf <sda@fstab.net>
---
 src/plotman/manager.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/plotman/manager.py b/src/plotman/manager.py
index e508baba..e6a98585 100644
--- a/src/plotman/manager.py
+++ b/src/plotman/manager.py
@@ -102,7 +102,7 @@ def maybe_start_new_plot(dir_cfg, sched_cfg, plotting_cfg):
                 dstdir = max(dir2ph, key=dir2ph.get)
 
             logfile = os.path.join(
-                dir_cfg.log, pendulum.now().strftime('%Y-%m-%dT%H_%M_%S.%f%z.log')
+                dir_cfg.log, pendulum.now().isoformat(timespec='microseconds').replace(':', '_') + '.log'
             )
 
             plot_args = ['chia', 'plots', 'create',

From 00f068022326c22c9c86aa5e0a17dc82bdef7974 Mon Sep 17 00:00:00 2001
From: Kyle Altendorf <sda@fstab.net>
Date: Wed, 28 Apr 2021 00:52:57 -0400
Subject: [PATCH 35/86] fix: typo _

---
 src/plotman/reporting.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/plotman/reporting.py b/src/plotman/reporting.py
index 0b41a64e..59962152 100644
--- a/src/plotman/reporting.py
+++ b/src/plotman/reporting.py
@@ -95,7 +95,7 @@ def status_report(jobs, width, height=None, tmp_prefix='', dst_prefix=''):
         # Regular row
         else:
             try:
-                with j.proc.one_shot():
+                with j.proc.oneshot():
                     row = [j.plot_id[:8],
                         j.k,
                         abbr_path(j.tmpdir, tmp_prefix),

From 8cc44fe75edb1b3009092083ae81c1c656cafd17 Mon Sep 17 00:00:00 2001
From: artorantala <arto.rantala@ekahau.com>
Date: Thu, 29 Apr 2021 19:42:40 +0300
Subject: [PATCH 36/86] Edit default config to add helpful documentation

I had a confusing time setting up the archiving. I added some key comments that would have been very useful to me in figuring out how the archival works.
---
 src/plotman/resources/plotman.yaml | 11 +++++++----
 1 file changed, 7 insertions(+), 4 deletions(-)

diff --git a/src/plotman/resources/plotman.yaml b/src/plotman/resources/plotman.yaml
index 845218b9..d0960369 100644
--- a/src/plotman/resources/plotman.yaml
+++ b/src/plotman/resources/plotman.yaml
@@ -59,11 +59,14 @@ directories:
         # archiving operation, comment this section out.
         #
         # Currently archival depends on an rsync daemon running on the remote
-        # host, and that the module is configured to match the local path.
-        # See code for details.
+        # host.
+        # The archival also uses ssh to connect to the remote host and check 
+        # for available directories. Set up ssh keys on the remote host to 
+        # allow public key login from rsyncd_user.
         archive:
-                rsyncd_module: plots
-                rsyncd_path: /plots
+                rsyncd_module: plots # Define this in remote rsyncd.conf.
+                rsyncd_path: /plots # This is used via ssh. Should match path 
+                                    # defined in the module referenced above.
                 rsyncd_bwlimit: 80000  # Bandwidth limit in KB/s
                 rsyncd_host: myfarmer
                 rsyncd_user: chia

From 8b2520463b62cc664e8fc2233efef1252ddc94cd Mon Sep 17 00:00:00 2001
From: Roy Natian <1146050+royimous@users.noreply.github.com>
Date: Thu, 29 Apr 2021 19:32:35 -0400
Subject: [PATCH 37/86] Clarifying scheduling settings

Added some detail that makes it clearer what this does.
---
 src/plotman/resources/plotman.yaml | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/src/plotman/resources/plotman.yaml b/src/plotman/resources/plotman.yaml
index 845218b9..99790894 100644
--- a/src/plotman/resources/plotman.yaml
+++ b/src/plotman/resources/plotman.yaml
@@ -81,11 +81,14 @@ directories:
 # Plotting scheduling parameters
 scheduling:
         # Run a job on a particular temp dir only if the number of existing jobs
-        # before tmpdir_stagger_phase_major tmpdir_stagger_phase_minor
+        # before [tmpdir_stagger_phase_major : tmpdir_stagger_phase_minor]
         # is less than tmpdir_stagger_phase_limit.
         # Phase major corresponds to the plot phase, phase minor corresponds to
         # the table or table pair in sequence, phase limit corresponds to
-        # the number of plots allowed before [phase major, phase minor]
+        # the number of plots allowed before [phase major : phase minor].
+        # e.g, with default settings, a new plot will start only when your plot
+        # reaches phase [2 : 1] on your temp drive. This setting takes precidence 
+        # over global_stagger_m 
         tmpdir_stagger_phase_major: 2
         tmpdir_stagger_phase_minor: 1
         # Optional: default is 1

From c0fa13a69e8a7a35a3be4ae0012c397275031338 Mon Sep 17 00:00:00 2001
From: Kyle Altendorf <sda@fstab.net>
Date: Thu, 29 Apr 2021 22:06:10 -0400
Subject: [PATCH 38/86] refactor: copy in chia plots create click options

---
 LICENSE-chia-blockchain        | 201 +++++++++++++++++++++++++++++++++
 setup.cfg                      |   2 -
 src/plotman/_tests/job_test.py |   2 +-
 src/plotman/chia.py            |  83 ++++++++++++++
 src/plotman/job.py             |   8 +-
 tox.ini                        |   1 -
 6 files changed, 291 insertions(+), 6 deletions(-)
 create mode 100644 LICENSE-chia-blockchain
 create mode 100644 src/plotman/chia.py

diff --git a/LICENSE-chia-blockchain b/LICENSE-chia-blockchain
new file mode 100644
index 00000000..261eeb9e
--- /dev/null
+++ b/LICENSE-chia-blockchain
@@ -0,0 +1,201 @@
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "[]"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+   Copyright [yyyy] [name of copyright owner]
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
diff --git a/setup.cfg b/setup.cfg
index 9e9720d6..ed151143 100644
--- a/setup.cfg
+++ b/setup.cfg
@@ -53,8 +53,6 @@ console_scripts =
     plotman = plotman.plotman:main
 
 [options.extras_require]
-chia =
-    chia-blockchain >= 1.0.4
 dev =
     %(test)s
     isort
diff --git a/src/plotman/_tests/job_test.py b/src/plotman/_tests/job_test.py
index 6e607050..25840723 100644
--- a/src/plotman/_tests/job_test.py
+++ b/src/plotman/_tests/job_test.py
@@ -133,7 +133,7 @@ def test_chia_plots_create_parsing_handles_argument_forms(arguments):
     ],
     ids=str,
 )
-def test_chia_plots_create_parsing_handles_argument_forms(arguments):
+def test_chia_plots_create_parsing_identifies_errors(arguments):
     parsed = job.parse_chia_plots_create_command_line(
         command_line=['python', 'chia', 'plots', 'create', *arguments],
     )
diff --git a/src/plotman/chia.py b/src/plotman/chia.py
new file mode 100644
index 00000000..283594d9
--- /dev/null
+++ b/src/plotman/chia.py
@@ -0,0 +1,83 @@
+import functools
+
+import click
+from pathlib import Path
+
+
+class Commands:
+    def __init__(self):
+        self.by_version = {}
+
+    def register(self, version):
+        if version in self.by_version:
+            raise Exception(f'Version already registered: {version!r}')
+        if not isinstance(version, tuple):
+            raise Exception(f'Version must be a tuple: {version!r}')
+
+        return functools.partial(self._decorator, version=version)
+
+    def _decorator(self, command, *, version):
+        self.by_version[version] = command
+        # self.by_version = dict(sorted(self.by_version.items()))
+
+    def __getitem__(self, item):
+        return self.by_version[item]
+
+    def latest_command(self):
+        return max(self.by_version.items())[1]
+
+
+commands = Commands()
+
+
+@commands.register(version=(1, 1, 2))
+@click.command()
+# https://github.com/Chia-Network/chia-blockchain/blob/v1.1.2/LICENSE
+# https://github.com/Chia-Network/chia-blockchain/blob/v1.1.2/chia/cmds/plots.py#L39-L83
+@click.option("-k", "--size", help="Plot size", type=int, default=32, show_default=True)
+@click.option("--override-k", help="Force size smaller than 32", default=False, show_default=True, is_flag=True)
+@click.option("-n", "--num", help="Number of plots or challenges", type=int, default=1, show_default=True)
+@click.option("-b", "--buffer", help="Megabytes for sort/plot buffer", type=int, default=4608, show_default=True)
+@click.option("-r", "--num_threads", help="Number of threads to use", type=int, default=2, show_default=True)
+@click.option("-u", "--buckets", help="Number of buckets", type=int, default=128, show_default=True)
+@click.option(
+    "-a",
+    "--alt_fingerprint",
+    type=int,
+    default=None,
+    help="Enter the alternative fingerprint of the key you want to use",
+)
+@click.option(
+    "-c",
+    "--pool_contract_address",
+    type=str,
+    default=None,
+    help="Address of where the pool reward will be sent to. Only used if alt_fingerprint and pool public key are None",
+)
+@click.option("-f", "--farmer_public_key", help="Hex farmer public key", type=str, default=None)
+@click.option("-p", "--pool_public_key", help="Hex public key of pool", type=str, default=None)
+@click.option(
+    "-t",
+    "--tmp_dir",
+    help="Temporary directory for plotting files",
+    type=click.Path(),
+    default=Path("."),
+    show_default=True,
+)
+@click.option("-2", "--tmp2_dir", help="Second temporary directory for plotting files", type=click.Path(), default=None)
+@click.option(
+    "-d",
+    "--final_dir",
+    help="Final directory for plots (relative or absolute)",
+    type=click.Path(),
+    default=Path("."),
+    show_default=True,
+)
+@click.option("-i", "--plotid", help="PlotID in hex for reproducing plots (debugging only)", type=str, default=None)
+@click.option("-m", "--memo", help="Memo in hex for reproducing plots (debugging only)", type=str, default=None)
+@click.option("-e", "--nobitfield", help="Disable bitfield", default=False, is_flag=True)
+@click.option(
+    "-x", "--exclude_final_dir", help="Skips adding [final dir] to harvester for farming", default=False, is_flag=True
+)
+def _cli():
+    pass
diff --git a/src/plotman/job.py b/src/plotman/job.py
index 065bc10b..8150d400 100644
--- a/src/plotman/job.py
+++ b/src/plotman/job.py
@@ -11,11 +11,12 @@
 from enum import Enum, auto
 from subprocess import call
 
-import chia.cmds.plots
 import click
 import pendulum
 import psutil
 
+from plotman import chia
+
 
 def job_phases_for_tmpdir(d, all_jobs):
     '''Return phase 2-tuples for jobs running on tmpdir d'''
@@ -61,7 +62,10 @@ def parse_chia_plots_create_command_line(command_line):
         if argument not in help_option_names
     ]
 
-    command = chia.cmds.plots.create_cmd
+    # TODO: We could at some point do chia version detection and pick the
+    #       associated command.  For now we'll just use the latest one we have
+    #       copied.
+    command = chia.commands.latest_command()
     try:
         context = command.make_context(info_name='', args=list(command_arguments))
     except click.ClickException as e:
diff --git a/tox.ini b/tox.ini
index 3d35c407..bd7e875e 100644
--- a/tox.ini
+++ b/tox.ini
@@ -6,7 +6,6 @@ changedir = {envtmpdir}
 
 [testenv:test-py{37,38,39}]
 extras =
-    chia
     test
 commands =
     pytest --capture=no --verbose --pyargs plotman

From 1561c6fc38ded2c813d7bb3669dd7cd07011ab5e Mon Sep 17 00:00:00 2001
From: Kyle Altendorf <sda@fstab.net>
Date: Thu, 29 Apr 2021 22:14:32 -0400
Subject: [PATCH 39/86] chore: add chia-blockchain license file to manifest

---
 MANIFEST.in | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/MANIFEST.in b/MANIFEST.in
index c6146883..8a034f2d 100644
--- a/MANIFEST.in
+++ b/MANIFEST.in
@@ -1,5 +1,5 @@
 include CHANGELOG.md
-include LICENSE
+include LICENSE*
 include README.md
 include *.md
 include VERSION

From ae8ace6b55fa8bff7bfc567501046a64d4ce985a Mon Sep 17 00:00:00 2001
From: Kyle Altendorf <sda@fstab.net>
Date: Thu, 29 Apr 2021 23:01:12 -0400
Subject: [PATCH 40/86] fix: archiving status could be None

---
 src/plotman/archive.py     | 4 ++--
 src/plotman/interactive.py | 6 +++++-
 src/plotman/plotman.py     | 6 +++++-
 3 files changed, 12 insertions(+), 4 deletions(-)

diff --git a/src/plotman/archive.py b/src/plotman/archive.py
index 6d9f15f8..727cf4a8 100644
--- a/src/plotman/archive.py
+++ b/src/plotman/archive.py
@@ -15,12 +15,12 @@
 
 # TODO : write-protect and delete-protect archived plots
 
-def spawn_archive_process(dir_cfg, all_jobs):
+def spawn_archive_process(dir_cfg, all_jobs, previous_archiving_status):
     '''Spawns a new archive process using the command created 
     in the archive() function. Returns archiving status and a log message to print.'''
 
     log_message = None
-    archiving_status = None
+    archiving_status = previous_archiving_status
     
     # Look for running archive jobs.  Be robust to finding more than one
     # even though the scheduler should only run one at a time.
diff --git a/src/plotman/interactive.py b/src/plotman/interactive.py
index 521db4c9..ad0ea187 100644
--- a/src/plotman/interactive.py
+++ b/src/plotman/interactive.py
@@ -123,7 +123,11 @@ def curses_main(stdscr):
 
             if archiving_configured:
                 if archiving_active:
-                    archiving_status, log_message = archive.spawn_archive_process(cfg.directories, jobs)
+                    archiving_status, log_message = archive.spawn_archive_process(
+                        cfg.directories,
+                        jobs,
+                        previous_archiving_status=archiving_status,
+                    )
                     if log_message:
                         log.log(log_message)
 
diff --git a/src/plotman/plotman.py b/src/plotman/plotman.py
index 3319cae8..59514bf5 100755
--- a/src/plotman/plotman.py
+++ b/src/plotman/plotman.py
@@ -181,7 +181,11 @@ def main():
                     jobs = Job.get_running_jobs(cfg.directories.log)
                 firstit = False
 
-                archiving_status, log_message = archive.spawn_archive_process(cfg.directories, jobs)
+                archiving_status, log_message = archive.spawn_archive_process(
+                    cfg.directories,
+                    jobs,
+                    previous_archiving_status=archiving_status,
+                )
                 if log_message:
                     print(log_message)
 

From 6404bb50943ef3b51c69bed7535767a7dafb178b Mon Sep 17 00:00:00 2001
From: Kyle Altendorf <sda@fstab.net>
Date: Thu, 29 Apr 2021 23:18:08 -0400
Subject: [PATCH 41/86] Revert "fix: archiving status could be None"

This reverts commit ae8ace6b55fa8bff7bfc567501046a64d4ce985a.
---
 src/plotman/archive.py     | 4 ++--
 src/plotman/interactive.py | 6 +-----
 src/plotman/plotman.py     | 6 +-----
 3 files changed, 4 insertions(+), 12 deletions(-)

diff --git a/src/plotman/archive.py b/src/plotman/archive.py
index 727cf4a8..6d9f15f8 100644
--- a/src/plotman/archive.py
+++ b/src/plotman/archive.py
@@ -15,12 +15,12 @@
 
 # TODO : write-protect and delete-protect archived plots
 
-def spawn_archive_process(dir_cfg, all_jobs, previous_archiving_status):
+def spawn_archive_process(dir_cfg, all_jobs):
     '''Spawns a new archive process using the command created 
     in the archive() function. Returns archiving status and a log message to print.'''
 
     log_message = None
-    archiving_status = previous_archiving_status
+    archiving_status = None
     
     # Look for running archive jobs.  Be robust to finding more than one
     # even though the scheduler should only run one at a time.
diff --git a/src/plotman/interactive.py b/src/plotman/interactive.py
index ad0ea187..521db4c9 100644
--- a/src/plotman/interactive.py
+++ b/src/plotman/interactive.py
@@ -123,11 +123,7 @@ def curses_main(stdscr):
 
             if archiving_configured:
                 if archiving_active:
-                    archiving_status, log_message = archive.spawn_archive_process(
-                        cfg.directories,
-                        jobs,
-                        previous_archiving_status=archiving_status,
-                    )
+                    archiving_status, log_message = archive.spawn_archive_process(cfg.directories, jobs)
                     if log_message:
                         log.log(log_message)
 
diff --git a/src/plotman/plotman.py b/src/plotman/plotman.py
index 59514bf5..3319cae8 100755
--- a/src/plotman/plotman.py
+++ b/src/plotman/plotman.py
@@ -181,11 +181,7 @@ def main():
                     jobs = Job.get_running_jobs(cfg.directories.log)
                 firstit = False
 
-                archiving_status, log_message = archive.spawn_archive_process(
-                    cfg.directories,
-                    jobs,
-                    previous_archiving_status=archiving_status,
-                )
+                archiving_status, log_message = archive.spawn_archive_process(cfg.directories, jobs)
                 if log_message:
                     print(log_message)
 

From c89356d42cc33c685ba7aaac0da6df077291d7f6 Mon Sep 17 00:00:00 2001
From: Kyle Altendorf <sda@fstab.net>
Date: Thu, 29 Apr 2021 23:22:49 -0400
Subject: [PATCH 42/86] fix: archiving status could be None (maybe more
 cleanly)

---
 src/plotman/archive.py | 9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)

diff --git a/src/plotman/archive.py b/src/plotman/archive.py
index 6d9f15f8..508b10b2 100644
--- a/src/plotman/archive.py
+++ b/src/plotman/archive.py
@@ -26,9 +26,7 @@ def spawn_archive_process(dir_cfg, all_jobs):
     # even though the scheduler should only run one at a time.
     arch_jobs = get_running_archive_jobs(dir_cfg.archive)
     
-    if arch_jobs:
-        archiving_status = 'pid: ' + ', '.join(map(str, arch_jobs))
-    else:
+    if not arch_jobs:
         (should_start, status_or_cmd) = archive(dir_cfg, all_jobs)
         if not should_start:
             archiving_status = status_or_cmd
@@ -41,6 +39,11 @@ def spawn_archive_process(dir_cfg, all_jobs):
                     stderr=subprocess.STDOUT,
                     start_new_session=True) 
             log_message = 'Starting archive: ' + cmd
+            arch_jobs = get_running_archive_jobs(dir_cfg.archive)
+
+    if archiving_status is None:
+        archiving_status = 'pid: ' + ', '.join(map(str, arch_jobs))
+
     return archiving_status, log_message
             
 def compute_priority(phase, gb_free, n_plots):

From 2c0be2203fa10dff6a1e437df33bdc83233ef56b Mon Sep 17 00:00:00 2001
From: Kyle Altendorf <sda@fstab.net>
Date: Thu, 29 Apr 2021 23:45:23 -0400
Subject: [PATCH 43/86] fix: add \`<pending>\` to avoid an empty archive pid
 list

---
 src/plotman/archive.py | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/src/plotman/archive.py b/src/plotman/archive.py
index 508b10b2..bfd757ee 100644
--- a/src/plotman/archive.py
+++ b/src/plotman/archive.py
@@ -39,7 +39,13 @@ def spawn_archive_process(dir_cfg, all_jobs):
                     stderr=subprocess.STDOUT,
                     start_new_session=True) 
             log_message = 'Starting archive: ' + cmd
-            arch_jobs = get_running_archive_jobs(dir_cfg.archive)
+            # At least for now it seems that even if we get a new running
+            # archive jobs list it doesn't contain the new rsync process.
+            # My guess is that this is because the bash in the middle due to
+            # shell=True is still starting up and really hasn't launched the
+            # new rsync process yet.  So, just put a placeholder here.  It
+            # will get filled on the next cycle.
+            arch_jobs.append('<pending>')
 
     if archiving_status is None:
         archiving_status = 'pid: ' + ', '.join(map(str, arch_jobs))

From 2b2d6f41e85f7beae3a054435871e3dae05edb93 Mon Sep 17 00:00:00 2001
From: Kyle Altendorf <sda@fstab.net>
Date: Fri, 30 Apr 2021 13:09:15 -0400
Subject: [PATCH 44/86] Update LICENSE-chia-blockchain

---
 LICENSE-chia-blockchain | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/LICENSE-chia-blockchain b/LICENSE-chia-blockchain
index 261eeb9e..ee81ae2a 100644
--- a/LICENSE-chia-blockchain
+++ b/LICENSE-chia-blockchain
@@ -186,7 +186,7 @@
       same "printed page" as the copyright notice for easier
       identification within third-party archives.
 
-   Copyright [yyyy] [name of copyright owner]
+   Copyright 2021 Chia Network
 
    Licensed under the Apache License, Version 2.0 (the "License");
    you may not use this file except in compliance with the License.

From 37306bf8e2e29b81159819ae4c267ed03d5ce855 Mon Sep 17 00:00:00 2001
From: Kyle Altendorf <sda@fstab.net>
Date: Fri, 30 Apr 2021 20:47:59 -0400
Subject: [PATCH 45/86] ci: add coverage (preliminary)

---
 .github/workflows/ci.yml | 81 +++++++++++++++++++++++++++++++++++++++-
 setup.cfg                |  7 +++-
 tox.ini                  | 12 +++++-
 3 files changed, 96 insertions(+), 4 deletions(-)

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 797469aa..67381de7 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -15,7 +15,8 @@ defaults:
 
 jobs:
   build:
-    name: ${{ matrix.task.name}} - ${{ matrix.os.name }} ${{ matrix.python.name }}
+    # Should match JOB_NAME below
+    name: ${{ matrix.task.name }} - ${{ matrix.os.name }} ${{ matrix.python.name }} ${{ matrix.arch.name }}
     runs-on: ${{ matrix.os.runs-on }}
     strategy:
       fail-fast: false
@@ -32,6 +33,10 @@ jobs:
           - name: Build
             tox: build
 
+    env:
+      # Should match name above
+      JOB_NAME: ${{ matrix.task.name }} - ${{ matrix.os.name }} ${{ matrix.python.name }} ${{ matrix.arch.name }}
+
     steps:
     - uses: actions/checkout@v2
       with:
@@ -73,7 +78,8 @@ jobs:
         path: dist/
 
   test:
-    name: ${{ matrix.task.name}} - ${{ matrix.os.name }} ${{ matrix.python.name }}
+    # Should match JOB_NAME below
+    name: ${{ matrix.task.name }} - ${{ matrix.os.name }} ${{ matrix.python.name }} ${{ matrix.arch.name }}
     runs-on: ${{ matrix.os.runs-on }}
     needs:
       - build
@@ -113,6 +119,8 @@ jobs:
 
 
     env:
+      # Should match name above
+      JOB_NAME: ${{ matrix.task.name }} - ${{ matrix.os.name }} ${{ matrix.python.name }} ${{ matrix.arch.name }}
       TOXENV: ${{ matrix.task.tox }}${{ fromJSON('["", "-"]')[matrix.python.tox != null] }}${{ matrix.python.tox }}
 
     steps:
@@ -159,6 +167,75 @@ jobs:
       run: |
         tox --skip-pkg-install
 
+    - name: Coverage Processing
+      if: matrix.task.coverage
+      run: |
+        mkdir coverage_reports
+        cp .coverage "coverage_reports/coverage.${{ env.JOB_NAME }}"
+        cp coverage.xml "coverage_reports/coverage.${{ env.JOB_NAME }}.xml"
+
+    - name: Publish Coverage
+      if: matrix.task.coverage
+      uses: actions/upload-artifact@v2
+      with:
+        name: coverage
+        path: coverage_reports/*
+
+  check:
+    # Should match JOB_NAME below
+    name: ${{ matrix.task.name }} - ${{ matrix.os.name }} ${{ matrix.python.name }} ${{ matrix.arch.name }}
+    runs-on: ${{ matrix.os.runs-on }}
+    container: ${{ matrix.os.container[matrix.python.docker] }}
+    strategy:
+      fail-fast: false
+      matrix:
+        task:
+          - name: Coverage
+            tox: combined-coverage
+            download_coverage: true
+        os:
+          - name: Linux
+            runs-on: ubuntu-latest
+            python_platform: linux
+            container:
+              3.8: docker://python:3.8-buster
+        python:
+          - name: CPython 3.8
+            tox: py38
+            action: 3.8
+            docker: 3.8
+
+    env:
+      # Should match name above
+      JOB_NAME: ${{ matrix.task.name }} - ${{ matrix.os.name }} ${{ matrix.python.name }} ${{ matrix.arch.name }}
+
+    steps:
+      - uses: actions/checkout@v2
+        with:
+          fetch-depth: 0
+
+      - name: Install tox
+        run: |
+          pip install --upgrade pip setuptools wheel
+          pip install --upgrade tox
+
+      - uses: twisted/python-info-action@v1.0.1
+
+      - name: Download Coverage
+        if: matrix.task.download_coverage
+        uses: actions/download-artifact@v2
+        with:
+          name: coverage
+          path: coverage_reports
+
+      - name: Setup tox environment
+        run: |
+          tox -vv --notest -e ${{ matrix.task.tox }}
+
+      - name: Test
+        run: |
+          tox -e ${{ matrix.task.tox }}
+
   all:
     name: All
     runs-on: ubuntu-latest
diff --git a/setup.cfg b/setup.cfg
index 092ae2b3..4d754ddc 100644
--- a/setup.cfg
+++ b/setup.cfg
@@ -52,12 +52,17 @@ console_scripts =
     plotman = plotman.plotman:main
 
 [options.extras_require]
+coverage =
+    coverage
+    diff-cover
 dev =
     %(test)s
     isort
-test = 
+test =
+    %(coverage)s
     check-manifest
     pytest
+    pytest-cov
     pytest-mock
     pyfakefs
 
diff --git a/tox.ini b/tox.ini
index bd7e875e..9c15a0d3 100644
--- a/tox.ini
+++ b/tox.ini
@@ -8,10 +8,20 @@ changedir = {envtmpdir}
 extras =
     test
 commands =
-    pytest --capture=no --verbose --pyargs plotman
+    pytest --capture=no --verbose --cov=plotman --cov-report=xml --pyargs plotman
 
 [testenv:check]
 extras =
     test
 commands =
     check-manifest --verbose {toxinidir}
+
+[testenv:combined-coverage]
+allowlist_externals =
+    ls
+extras =
+    coverage
+commands =
+    ls -la coverage_reports
+    coverage combine coverage_reports
+    coverage report --fail-under=85 --ignore-errors

From 4805e2c0c509d694aa28431a25b43fd18fa008d6 Mon Sep 17 00:00:00 2001
From: Kyle Altendorf <sda@fstab.net>
Date: Fri, 30 Apr 2021 20:54:15 -0400
Subject: [PATCH 46/86] ci: check needs test

---
 .github/workflows/ci.yml | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 67381de7..b9ce2371 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -185,6 +185,8 @@ jobs:
     # Should match JOB_NAME below
     name: ${{ matrix.task.name }} - ${{ matrix.os.name }} ${{ matrix.python.name }} ${{ matrix.arch.name }}
     runs-on: ${{ matrix.os.runs-on }}
+    needs:
+      - test
     container: ${{ matrix.os.container[matrix.python.docker] }}
     strategy:
       fail-fast: false

From 03ce5eee92fc50efff2fd2d8b006f2a98683a49b Mon Sep 17 00:00:00 2001
From: Kyle Altendorf <sda@fstab.net>
Date: Fri, 30 Apr 2021 20:59:56 -0400
Subject: [PATCH 47/86] ci: enable coverage for the tests

---
 .github/workflows/ci.yml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index b9ce2371..abc33cc7 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -106,6 +106,7 @@ jobs:
         task:
           - name: Test
             tox: test
+            coverage: true
         include:
           - task:
               name: Check

From 1f5cfaa5f494d4e3f4f85ea5ac586e2198f469b4 Mon Sep 17 00:00:00 2001
From: Kyle Altendorf <sda@fstab.net>
Date: Fri, 30 Apr 2021 21:08:03 -0400
Subject: [PATCH 48/86] ci: add .coveragerc

---
 .coveragerc | 18 ++++++++++++++++++
 1 file changed, 18 insertions(+)
 create mode 100644 .coveragerc

diff --git a/.coveragerc b/.coveragerc
new file mode 100644
index 00000000..44de54c2
--- /dev/null
+++ b/.coveragerc
@@ -0,0 +1,18 @@
+[paths]
+source =
+   src
+
+[report]
+precision = 1
+exclude_lines =
+  pragma: no cover
+  abc\.abstractmethod
+  typing\.overload
+  if typing.TYPE_CHECKING:
+  ^\s*pass\s*$
+  ^\s*...\s*$
+
+[run]
+branch = True
+source =
+    plotman

From 7b20ba8e0ad6cc52acae4921d563d7599aafcfb8 Mon Sep 17 00:00:00 2001
From: Kyle Altendorf <sda@fstab.net>
Date: Fri, 30 Apr 2021 21:13:46 -0400
Subject: [PATCH 49/86] chore: add .coveragerc to manifest

---
 MANIFEST.in | 1 +
 1 file changed, 1 insertion(+)

diff --git a/MANIFEST.in b/MANIFEST.in
index c6146883..ad05e1e7 100644
--- a/MANIFEST.in
+++ b/MANIFEST.in
@@ -4,6 +4,7 @@ include README.md
 include *.md
 include VERSION
 include tox.ini
+include .coveragerc
 recursive-include src *.py
 recursive-include src/plotman/_tests/resources *
 recursive-include src/plotman/resources *

From 524eb3e5e3c28b5317f70e62482e96e05a26f3c5 Mon Sep 17 00:00:00 2001
From: Kyle Altendorf <sda@fstab.net>
Date: Fri, 30 Apr 2021 21:21:55 -0400
Subject: [PATCH 50/86] ci: help with coverage file locations

---
 tox.ini | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/tox.ini b/tox.ini
index 9c15a0d3..4b7a524d 100644
--- a/tox.ini
+++ b/tox.ini
@@ -3,12 +3,15 @@ envlist = test-py{37,38,39}
 
 [testenv]
 changedir = {envtmpdir}
+setenv =
+    COVERAGE_FILE={toxinidir}/.coverage
 
 [testenv:test-py{37,38,39}]
 extras =
     test
 commands =
-    pytest --capture=no --verbose --cov=plotman --cov-report=xml --pyargs plotman
+    pytest --capture=no --verbose --cov=plotman --cov-report term-missing --cov-report=xml --pyargs plotman
+    cp .coverage coverage.xml {toxinidir}/
 
 [testenv:check]
 extras =

From e3233dda2b31b4fe837ac3a8dc2b77cfc458f844 Mon Sep 17 00:00:00 2001
From: Kyle Altendorf <sda@fstab.net>
Date: Fri, 30 Apr 2021 21:26:58 -0400
Subject: [PATCH 51/86] ci: more coverage stuff

---
 .github/workflows/ci.yml |  2 +-
 tox.ini                  | 14 ++++++--------
 2 files changed, 7 insertions(+), 9 deletions(-)

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index abc33cc7..02977fe5 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -194,7 +194,7 @@ jobs:
       matrix:
         task:
           - name: Coverage
-            tox: combined-coverage
+            tox: check-coverage
             download_coverage: true
         os:
           - name: Linux
diff --git a/tox.ini b/tox.ini
index 4b7a524d..33c95a39 100644
--- a/tox.ini
+++ b/tox.ini
@@ -19,12 +19,10 @@ extras =
 commands =
     check-manifest --verbose {toxinidir}
 
-[testenv:combined-coverage]
-allowlist_externals =
-    ls
-extras =
-    coverage
+[testenv:check-coverage]
+changedir = {toxinidir}
 commands =
-    ls -la coverage_reports
-    coverage combine coverage_reports
-    coverage report --fail-under=85 --ignore-errors
+    coverage combine coverage_reports/
+    coverage xml -o coverage.xml
+    coverage report --fail-under=90 --ignore-errors
+    diff-cover --fail-under=100 {posargs:--compare-branch=development} coverage.xml

From caaefcd581887070658431fb01bcc9965102fdc3 Mon Sep 17 00:00:00 2001
From: Kyle Altendorf <sda@fstab.net>
Date: Fri, 30 Apr 2021 21:34:19 -0400
Subject: [PATCH 52/86] ci: exploration

---
 tox.ini | 1 +
 1 file changed, 1 insertion(+)

diff --git a/tox.ini b/tox.ini
index 33c95a39..8c5623be 100644
--- a/tox.ini
+++ b/tox.ini
@@ -11,6 +11,7 @@ extras =
     test
 commands =
     pytest --capture=no --verbose --cov=plotman --cov-report term-missing --cov-report=xml --pyargs plotman
+    ls -la . {toxinidir}
     cp .coverage coverage.xml {toxinidir}/
 
 [testenv:check]

From d12a1d0c654e32a69b6d2d0d3cb31ed77990a5ff Mon Sep 17 00:00:00 2001
From: Kyle Altendorf <sda@fstab.net>
Date: Fri, 30 Apr 2021 21:39:15 -0400
Subject: [PATCH 53/86] ci: maybe

---
 tox.ini | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/tox.ini b/tox.ini
index 8c5623be..e60faa1d 100644
--- a/tox.ini
+++ b/tox.ini
@@ -10,9 +10,7 @@ setenv =
 extras =
     test
 commands =
-    pytest --capture=no --verbose --cov=plotman --cov-report term-missing --cov-report=xml --pyargs plotman
-    ls -la . {toxinidir}
-    cp .coverage coverage.xml {toxinidir}/
+    pytest --capture=no --verbose --cov=plotman --cov-report=term-missing --cov-report=xml:{toxinidir}/coverage.xml --pyargs plotman
 
 [testenv:check]
 extras =

From a92658874130b94417b5045d389335eaba871007 Mon Sep 17 00:00:00 2001
From: Kyle Altendorf <sda@fstab.net>
Date: Fri, 30 Apr 2021 21:45:14 -0400
Subject: [PATCH 54/86] ci: require coverage checks jobs

---
 .github/workflows/ci.yml | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 02977fe5..81eaac8f 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -182,7 +182,7 @@ jobs:
         name: coverage
         path: coverage_reports/*
 
-  check:
+  coverage:
     # Should match JOB_NAME below
     name: ${{ matrix.task.name }} - ${{ matrix.os.name }} ${{ matrix.python.name }} ${{ matrix.arch.name }}
     runs-on: ${{ matrix.os.runs-on }}
@@ -245,6 +245,7 @@ jobs:
     needs:
       - build
       - test
+      - coverage
     steps:
       - name: This
         shell: python

From 528c9d597df11854ebec3f6f2192a097960d0541 Mon Sep 17 00:00:00 2001
From: Kyle Altendorf <sda@fstab.net>
Date: Fri, 30 Apr 2021 21:45:42 -0400
Subject: [PATCH 55/86] ci: actualy install coverage...

---
 tox.ini | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/tox.ini b/tox.ini
index e60faa1d..980763ff 100644
--- a/tox.ini
+++ b/tox.ini
@@ -20,6 +20,8 @@ commands =
 
 [testenv:check-coverage]
 changedir = {toxinidir}
+extras =
+    coverage
 commands =
     coverage combine coverage_reports/
     coverage xml -o coverage.xml

From 0b308233a66e1aff605342d65a88d1a8ba241809 Mon Sep 17 00:00:00 2001
From: Kyle Altendorf <sda@fstab.net>
Date: Fri, 30 Apr 2021 21:55:25 -0400
Subject: [PATCH 56/86] ci: explore

---
 tox.ini | 1 +
 1 file changed, 1 insertion(+)

diff --git a/tox.ini b/tox.ini
index 980763ff..e89af809 100644
--- a/tox.ini
+++ b/tox.ini
@@ -23,6 +23,7 @@ changedir = {toxinidir}
 extras =
     coverage
 commands =
+    ls -laR coverage_reports/
     coverage combine coverage_reports/
     coverage xml -o coverage.xml
     coverage report --fail-under=90 --ignore-errors

From afd3e24707de962529ea3149392b35db21b924be Mon Sep 17 00:00:00 2001
From: Kyle Altendorf <sda@fstab.net>
Date: Fri, 30 Apr 2021 22:10:41 -0400
Subject: [PATCH 57/86] ci: hmm

---
 .github/workflows/ci.yml | 14 +++++++-------
 1 file changed, 7 insertions(+), 7 deletions(-)

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 81eaac8f..7dc401b6 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -16,7 +16,7 @@ defaults:
 jobs:
   build:
     # Should match JOB_NAME below
-    name: ${{ matrix.task.name }} - ${{ matrix.os.name }} ${{ matrix.python.name }} ${{ matrix.arch.name }}
+    name: ${{ matrix.task.name }} - ${{ matrix.os.name }} ${{ matrix.python.name }}
     runs-on: ${{ matrix.os.runs-on }}
     strategy:
       fail-fast: false
@@ -35,7 +35,7 @@ jobs:
 
     env:
       # Should match name above
-      JOB_NAME: ${{ matrix.task.name }} - ${{ matrix.os.name }} ${{ matrix.python.name }} ${{ matrix.arch.name }}
+      JOB_NAME: ${{ matrix.task.name }} - ${{ matrix.os.name }} ${{ matrix.python.name }}
 
     steps:
     - uses: actions/checkout@v2
@@ -79,7 +79,7 @@ jobs:
 
   test:
     # Should match JOB_NAME below
-    name: ${{ matrix.task.name }} - ${{ matrix.os.name }} ${{ matrix.python.name }} ${{ matrix.arch.name }}
+    name: ${{ matrix.task.name }} - ${{ matrix.os.name }} ${{ matrix.python.name }}
     runs-on: ${{ matrix.os.runs-on }}
     needs:
       - build
@@ -121,7 +121,7 @@ jobs:
 
     env:
       # Should match name above
-      JOB_NAME: ${{ matrix.task.name }} - ${{ matrix.os.name }} ${{ matrix.python.name }} ${{ matrix.arch.name }}
+      JOB_NAME: ${{ matrix.task.name }} - ${{ matrix.os.name }} ${{ matrix.python.name }}
       TOXENV: ${{ matrix.task.tox }}${{ fromJSON('["", "-"]')[matrix.python.tox != null] }}${{ matrix.python.tox }}
 
     steps:
@@ -172,7 +172,7 @@ jobs:
       if: matrix.task.coverage
       run: |
         mkdir coverage_reports
-        cp .coverage "coverage_reports/coverage.${{ env.JOB_NAME }}"
+        cp .coverage "coverage_reports/.coverage.${{ env.JOB_NAME }}"
         cp coverage.xml "coverage_reports/coverage.${{ env.JOB_NAME }}.xml"
 
     - name: Publish Coverage
@@ -184,7 +184,7 @@ jobs:
 
   coverage:
     # Should match JOB_NAME below
-    name: ${{ matrix.task.name }} - ${{ matrix.os.name }} ${{ matrix.python.name }} ${{ matrix.arch.name }}
+    name: ${{ matrix.task.name }} - ${{ matrix.os.name }} ${{ matrix.python.name }}
     runs-on: ${{ matrix.os.runs-on }}
     needs:
       - test
@@ -210,7 +210,7 @@ jobs:
 
     env:
       # Should match name above
-      JOB_NAME: ${{ matrix.task.name }} - ${{ matrix.os.name }} ${{ matrix.python.name }} ${{ matrix.arch.name }}
+      JOB_NAME: ${{ matrix.task.name }} - ${{ matrix.os.name }} ${{ matrix.python.name }}
 
     steps:
       - uses: actions/checkout@v2

From 1a04bd760b2a72aaad35abfe4a69e9886c568134 Mon Sep 17 00:00:00 2001
From: Kyle Altendorf <sda@fstab.net>
Date: Fri, 30 Apr 2021 22:17:38 -0400
Subject: [PATCH 58/86] ci: coverage include   */site-packages

---
 .coveragerc | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.coveragerc b/.coveragerc
index 44de54c2..e6661885 100644
--- a/.coveragerc
+++ b/.coveragerc
@@ -1,6 +1,7 @@
 [paths]
 source =
    src
+   */site-packages
 
 [report]
 precision = 1

From 8798e53d3dcccd6f7ba939fddaa5c5b7b30b3811 Mon Sep 17 00:00:00 2001
From: Kyle Altendorf <sda@fstab.net>
Date: Fri, 30 Apr 2021 22:25:07 -0400
Subject: [PATCH 59/86] ci: allow 35% coverage

---
 tox.ini | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/tox.ini b/tox.ini
index e89af809..3f85064e 100644
--- a/tox.ini
+++ b/tox.ini
@@ -23,8 +23,7 @@ changedir = {toxinidir}
 extras =
     coverage
 commands =
-    ls -laR coverage_reports/
     coverage combine coverage_reports/
     coverage xml -o coverage.xml
-    coverage report --fail-under=90 --ignore-errors
+    coverage report --fail-under=35 --ignore-errors
     diff-cover --fail-under=100 {posargs:--compare-branch=development} coverage.xml

From e4c5663afd54d044e4f60b7c0e914dc03ff8b295 Mon Sep 17 00:00:00 2001
From: Kyle Altendorf <sda@fstab.net>
Date: Fri, 30 Apr 2021 22:51:00 -0400
Subject: [PATCH 60/86] ci: rework

---
 .github/workflows/ci.yml | 113 +++++++++++++++++++++++++--------------
 1 file changed, 72 insertions(+), 41 deletions(-)

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 7dc401b6..67ab104f 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -188,56 +188,87 @@ jobs:
     runs-on: ${{ matrix.os.runs-on }}
     needs:
       - test
-    container: ${{ matrix.os.container[matrix.python.docker] }}
     strategy:
       fail-fast: false
       matrix:
-        task:
-          - name: Coverage
-            tox: check-coverage
-            download_coverage: true
-        os:
-          - name: Linux
-            runs-on: ubuntu-latest
-            python_platform: linux
-            container:
-              3.8: docker://python:3.8-buster
-        python:
-          - name: CPython 3.8
-            tox: py38
-            action: 3.8
-            docker: 3.8
+        include:
+          - os:
+              name: Linux
+              runs-on: ubuntu-latest
+            python:
+              name: CPython 3.8
+              action: 3.8
+            task:
+              name: Coverage
+              tox: check-coverage
+              coverage: false
+              download_coverage: true
 
     env:
       # Should match name above
       JOB_NAME: ${{ matrix.task.name }} - ${{ matrix.os.name }} ${{ matrix.python.name }}
+      TOXENV: ${{ matrix.task.tox }}${{ fromJSON('["", "-"]')[matrix.task.tox != null && matrix.python.tox != null] }}${{ matrix.python.tox }}
 
     steps:
-      - uses: actions/checkout@v2
-        with:
-          fetch-depth: 0
-
-      - name: Install tox
-        run: |
-          pip install --upgrade pip setuptools wheel
-          pip install --upgrade tox
-
-      - uses: twisted/python-info-action@v1.0.1
-
-      - name: Download Coverage
-        if: matrix.task.download_coverage
-        uses: actions/download-artifact@v2
-        with:
-          name: coverage
-          path: coverage_reports
-
-      - name: Setup tox environment
-        run: |
-          tox -vv --notest -e ${{ matrix.task.tox }}
-
-      - name: Test
-        run: |
-          tox -e ${{ matrix.task.tox }}
+    - uses: actions/checkout@v2
+      with:
+        fetch-depth: 0
+        ref: ${{ github.event.pull_request.head.sha }}
+
+    - name: Download package files
+      uses: actions/download-artifact@v2
+      with:
+        name: dist
+        path: dist/
+
+    - name: Download Coverage
+      if: matrix.task.download_coverage
+      uses: actions/download-artifact@v2
+      with:
+        name: coverage
+        path: coverage_reports
+
+    - name: Set up ${{ matrix.python.name }}
+      uses: actions/setup-python@v2
+      with:
+        python-version: ${{ matrix.python.action }}
+
+    - name: Install dependencies
+      run: |
+        python -m pip install --upgrade pip setuptools wheel
+        pip install tox
+
+    - name: Prepare tox environment
+      run: |
+        tox --notest --installpkg dist/*.whl
+
+    - name: Runner info
+      uses: twisted/python-info-action@v1
+
+    - name: Tox info
+      uses: twisted/python-info-action@v1
+      with:
+        python-path: .tox/${{ env.TOXENV }}/*/python
+
+    - name: Run tox environment
+      env:
+        BASE_REF: ${{ fromJSON(format('[{0}, {1}]', toJSON(github.event.before), toJSON(format('origin/{0}', github.base_ref))))[github.base_ref != ''] }}
+      run: |
+        tox --skip-pkg-install -- --compare-branch="${BASE_REF}"
+
+    - name: Coverage Processing
+      if: always()
+      run: |
+        mkdir all_coverage_report
+        cp .coverage "all_coverage_report/.coverage.all"
+        cp coverage.xml "all_coverage_report/coverage.all.xml"
+
+    - name: Upload Coverage
+      if: always()
+      uses: actions/upload-artifact@v2
+      with:
+        name: coverage
+        path: all_coverage_report/*
 
   all:
     name: All

From 5e18bc5a851733d0da7a67fbbe72289f74c3bc7d Mon Sep 17 00:00:00 2001
From: Kyle Altendorf <sda@fstab.net>
Date: Fri, 30 Apr 2021 23:08:09 -0400
Subject: [PATCH 61/86] ci:  --show-missing

---
 tox.ini | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tox.ini b/tox.ini
index 3f85064e..74aec0d5 100644
--- a/tox.ini
+++ b/tox.ini
@@ -25,5 +25,5 @@ extras =
 commands =
     coverage combine coverage_reports/
     coverage xml -o coverage.xml
-    coverage report --fail-under=35 --ignore-errors
+    coverage report --fail-under=35 --ignore-errors --show-missing
     diff-cover --fail-under=100 {posargs:--compare-branch=development} coverage.xml

From 3921ab93299c1990a55c1e8a3c7cf4b28bf3adc3 Mon Sep 17 00:00:00 2001
From: Kyle Altendorf <sda@fstab.net>
Date: Sat, 1 May 2021 23:38:27 -0400
Subject: [PATCH 62/86] doc: add markers for start and end of copied code

---
 src/plotman/chia.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/src/plotman/chia.py b/src/plotman/chia.py
index 283594d9..0c90501f 100644
--- a/src/plotman/chia.py
+++ b/src/plotman/chia.py
@@ -34,6 +34,7 @@ def latest_command(self):
 @click.command()
 # https://github.com/Chia-Network/chia-blockchain/blob/v1.1.2/LICENSE
 # https://github.com/Chia-Network/chia-blockchain/blob/v1.1.2/chia/cmds/plots.py#L39-L83
+# start copied code
 @click.option("-k", "--size", help="Plot size", type=int, default=32, show_default=True)
 @click.option("--override-k", help="Force size smaller than 32", default=False, show_default=True, is_flag=True)
 @click.option("-n", "--num", help="Number of plots or challenges", type=int, default=1, show_default=True)
@@ -79,5 +80,6 @@ def latest_command(self):
 @click.option(
     "-x", "--exclude_final_dir", help="Skips adding [final dir] to harvester for farming", default=False, is_flag=True
 )
+# end copied code
 def _cli():
     pass

From 1b5db4e342b9ec1f7910663a453aec3a97ba51a6 Mon Sep 17 00:00:00 2001
From: Kyle Altendorf <sda@fstab.net>
Date: Sat, 1 May 2021 23:39:10 -0400
Subject: [PATCH 63/86] chore: copy in chia plots create cli parser for 1.1.3

---
 src/plotman/chia.py | 55 +++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 55 insertions(+)

diff --git a/src/plotman/chia.py b/src/plotman/chia.py
index 0c90501f..a9e4f9ef 100644
--- a/src/plotman/chia.py
+++ b/src/plotman/chia.py
@@ -83,3 +83,58 @@ def latest_command(self):
 # end copied code
 def _cli():
     pass
+
+
+@commands.register(version=(1, 1, 3))
+@click.command()
+# https://github.com/Chia-Network/chia-blockchain/blob/v1.1.3/LICENSE
+# https://github.com/Chia-Network/chia-blockchain/blob/v1.1.3/chia/cmds/plots.py#L39-L83
+# start copied code
+@click.option("-k", "--size", help="Plot size", type=int, default=32, show_default=True)
+@click.option("--override-k", help="Force size smaller than 32", default=False, show_default=True, is_flag=True)
+@click.option("-n", "--num", help="Number of plots or challenges", type=int, default=1, show_default=True)
+@click.option("-b", "--buffer", help="Megabytes for sort/plot buffer", type=int, default=4608, show_default=True)
+@click.option("-r", "--num_threads", help="Number of threads to use", type=int, default=2, show_default=True)
+@click.option("-u", "--buckets", help="Number of buckets", type=int, default=128, show_default=True)
+@click.option(
+    "-a",
+    "--alt_fingerprint",
+    type=int,
+    default=None,
+    help="Enter the alternative fingerprint of the key you want to use",
+)
+@click.option(
+    "-c",
+    "--pool_contract_address",
+    type=str,
+    default=None,
+    help="Address of where the pool reward will be sent to. Only used if alt_fingerprint and pool public key are None",
+)
+@click.option("-f", "--farmer_public_key", help="Hex farmer public key", type=str, default=None)
+@click.option("-p", "--pool_public_key", help="Hex public key of pool", type=str, default=None)
+@click.option(
+    "-t",
+    "--tmp_dir",
+    help="Temporary directory for plotting files",
+    type=click.Path(),
+    default=Path("."),
+    show_default=True,
+)
+@click.option("-2", "--tmp2_dir", help="Second temporary directory for plotting files", type=click.Path(), default=None)
+@click.option(
+    "-d",
+    "--final_dir",
+    help="Final directory for plots (relative or absolute)",
+    type=click.Path(),
+    default=Path("."),
+    show_default=True,
+)
+@click.option("-i", "--plotid", help="PlotID in hex for reproducing plots (debugging only)", type=str, default=None)
+@click.option("-m", "--memo", help="Memo in hex for reproducing plots (debugging only)", type=str, default=None)
+@click.option("-e", "--nobitfield", help="Disable bitfield", default=False, is_flag=True)
+@click.option(
+    "-x", "--exclude_final_dir", help="Skips adding [final dir] to harvester for farming", default=False, is_flag=True
+)
+# end copied code
+def _cli():
+    pass

From e784731b798eaaa9de3998d1fed4d62eb6713c15 Mon Sep 17 00:00:00 2001
From: Kyle Altendorf <sda@fstab.net>
Date: Sat, 1 May 2021 23:50:37 -0400
Subject: [PATCH 64/86] doc: add MAINTENANCE.md and a description fo the chia
 code copying

---
 MAINTENANCE.md | 19 +++++++++++++++++++
 1 file changed, 19 insertions(+)
 create mode 100644 MAINTENANCE.md

diff --git a/MAINTENANCE.md b/MAINTENANCE.md
new file mode 100644
index 00000000..825c273e
--- /dev/null
+++ b/MAINTENANCE.md
@@ -0,0 +1,19 @@
+# Maintenance
+
+## Overview
+
+This document holds guidance on maintaining aspects of plotman.
+
+## The `chia plots create` CLI parsing code
+
+In [src/plotman/chia.py](src/plotman/chia.py) there is code copied from the `chia plots create` subcommand's CLI parser definition.
+When new versions of `chia-blockchain` are released, their interface code should be added to plotman.
+plotman commit [1b5db4e](https://github.com/altendky/plotman/commit/1b5db4e342b9ec1f7910663a453aec3a97ba51a6) provides an example of adding a new version.
+
+In many cases, copying code is a poor choice.
+It is believed that in this case it is appropriate since the chia code that plotman could import is not necessarily the code that is parsing the plotting process command lines anyways.
+The chia command could come from another Python environment, a system package, a `.dmg`, etc.
+This approach also offers future potential of using the proper version of parsing for the specific plot process being inspected.
+Finally, this alleviates dealing with the dependency on the `chia-blockchain` package.
+In generally, using dependencies is good.
+This seems to be an exceptional case.

From 28f5b41229caea8160197570201ca26ab591c059 Mon Sep 17 00:00:00 2001
From: Kyle Altendorf <sda@fstab.net>
Date: Sat, 1 May 2021 23:52:01 -0400
Subject: [PATCH 65/86] doc: change code copying example link to use
 ericaltendorf repo

---
 MAINTENANCE.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/MAINTENANCE.md b/MAINTENANCE.md
index 825c273e..b8b17c2d 100644
--- a/MAINTENANCE.md
+++ b/MAINTENANCE.md
@@ -8,7 +8,7 @@ This document holds guidance on maintaining aspects of plotman.
 
 In [src/plotman/chia.py](src/plotman/chia.py) there is code copied from the `chia plots create` subcommand's CLI parser definition.
 When new versions of `chia-blockchain` are released, their interface code should be added to plotman.
-plotman commit [1b5db4e](https://github.com/altendky/plotman/commit/1b5db4e342b9ec1f7910663a453aec3a97ba51a6) provides an example of adding a new version.
+plotman commit [1b5db4e](https://github.com/ericaltendorf/plotman/commit/1b5db4e342b9ec1f7910663a453aec3a97ba51a6) provides an example of adding a new version.
 
 In many cases, copying code is a poor choice.
 It is believed that in this case it is appropriate since the chia code that plotman could import is not necessarily the code that is parsing the plotting process command lines anyways.

From 808073a36ae586d72fb3c2f0c582923c7cf6e1cb Mon Sep 17 00:00:00 2001
From: Kyle Altendorf <sda@fstab.net>
Date: Sun, 2 May 2021 17:51:27 -0400
Subject: [PATCH 66/86] ci: do not require that coverage pass

---
 .github/workflows/ci.yml | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 67ab104f..f2c0f406 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -276,7 +276,8 @@ jobs:
     needs:
       - build
       - test
-      - coverage
+      # TODO: make this required when we have a better testing situation
+      # - coverage
     steps:
       - name: This
         shell: python

From 418d62b34c38a532028428e1d51c8088b7558d4a Mon Sep 17 00:00:00 2001
From: Kyle Altendorf <sda@fstab.net>
Date: Sun, 2 May 2021 22:16:29 -0400
Subject: [PATCH 67/86] fix: explain failure to open new log files

---
 src/plotman/manager.py | 26 +++++++++++++++++++++-----
 1 file changed, 21 insertions(+), 5 deletions(-)

diff --git a/src/plotman/manager.py b/src/plotman/manager.py
index 1116ea98..06ed2947 100644
--- a/src/plotman/manager.py
+++ b/src/plotman/manager.py
@@ -126,11 +126,27 @@ def maybe_start_new_plot(dir_cfg, sched_cfg, plotting_cfg):
 
             logmsg = ('Starting plot job: %s ; logging to %s' % (' '.join(plot_args), logfile))
 
-            # start_new_sessions to make the job independent of this controlling tty.
-            p = subprocess.Popen(plot_args,
-                stdout=open(logfile, 'w'),
-                stderr=subprocess.STDOUT,
-                start_new_session=True)
+            try:
+                open_log_file = open(logfile, 'w')
+            except FileNotFoundError as e:
+                message = (
+                    f'Unable to open log file.  Verify that the directory exists'
+                    f' and has proper write permissions: {logfile!r}'
+                )
+                raise Exception(message) from e
+
+            # Preferably, do not add any code between the try block above
+            # and the with block below.  IOW, this space intentionally left
+            # blank...  As is, this provides a good chance that our handle
+            # of the log file will get closed explicitly while still
+            # allowing handling of just the log file opening error.
+                
+            with open_log_file:
+                # start_new_sessions to make the job independent of this controlling tty.
+                p = subprocess.Popen(plot_args,
+                    stdout=open_log_file,
+                    stderr=subprocess.STDOUT,
+                    start_new_session=True)
 
             psutil.Process(p.pid).nice(15)
             return (True, logmsg)

From a06ab34b3aca0644fea6fa385b568e777cb80dc5 Mon Sep 17 00:00:00 2001
From: Kyle Altendorf <sda@fstab.net>
Date: Sun, 2 May 2021 22:31:12 -0400
Subject: [PATCH 68/86] fix: avoid creation of multiple plot processes logging
 to a single file

---
 src/plotman/manager.py | 17 ++++++++++++++++-
 1 file changed, 16 insertions(+), 1 deletion(-)

diff --git a/src/plotman/manager.py b/src/plotman/manager.py
index 1116ea98..86d69903 100644
--- a/src/plotman/manager.py
+++ b/src/plotman/manager.py
@@ -126,9 +126,24 @@ def maybe_start_new_plot(dir_cfg, sched_cfg, plotting_cfg):
 
             logmsg = ('Starting plot job: %s ; logging to %s' % (' '.join(plot_args), logfile))
 
+            try:
+                open_log_file = open(logfile, 'x')
+            except FileExistsError:
+                # The desired log file name already exists.  Most likely another
+                # plotman process already launched a new process in response to
+                # the same scenario that triggered us.  Let's at least not
+                # confuse things further by having two plotting processes
+                # logging to the same file.  If we really should launch another
+                # plotting process, we'll get it at the next check cycle anyways.
+                message = (
+                    f'Plot log file already exists, skipping attempt to start a'
+                    f' new plot: {logfile!r}'
+                )
+                return (False, logmsg)
+
             # start_new_sessions to make the job independent of this controlling tty.
             p = subprocess.Popen(plot_args,
-                stdout=open(logfile, 'w'),
+                stdout=open_log_file,
                 stderr=subprocess.STDOUT,
                 start_new_session=True)
 

From 1e3fd653f015bb950905cbedf4a3ce7b13bbf326 Mon Sep 17 00:00:00 2001
From: ericaltendorf <ericaltendorf@gmail.com>
Date: Mon, 3 May 2021 10:25:30 -0700
Subject: [PATCH 69/86] feat: collapse split tmpdir tables into one table

---
 src/plotman/interactive.py | 35 ++++++++++-------------------------
 1 file changed, 10 insertions(+), 25 deletions(-)

diff --git a/src/plotman/interactive.py b/src/plotman/interactive.py
index 4aa85d04..d5f32319 100644
--- a/src/plotman/interactive.py
+++ b/src/plotman/interactive.py
@@ -171,13 +171,10 @@ def curses_main(stdscr):
             arch_prefix = cfg.directories.archive.rsyncd_path
 
         n_tmpdirs = len(cfg.directories.tmp)
-        n_tmpdirs_half = int(n_tmpdirs / 2)
 
         # Directory reports.
-        tmp_report_1 = reporting.tmp_dir_report(
-            jobs, cfg.directories, cfg.scheduling, n_cols, 0, n_tmpdirs_half, tmp_prefix)
-        tmp_report_2 = reporting.tmp_dir_report(
-            jobs, cfg.directories, cfg.scheduling, n_cols, n_tmpdirs_half, n_tmpdirs, tmp_prefix)
+        tmp_report = reporting.tmp_dir_report(
+            jobs, cfg.directories, cfg.scheduling, n_cols, 0, n_tmpdirs, tmp_prefix)
         dst_report = reporting.dst_dir_report(
             jobs, cfg.directories.dst, n_cols, dst_prefix)
         if archiving_configured:
@@ -191,10 +188,8 @@ def curses_main(stdscr):
         # Layout
         #
             
-        tmp_h = max(len(tmp_report_1.splitlines()),
-                    len(tmp_report_2.splitlines()))
-        tmp_w = len(max(tmp_report_1.splitlines() +
-                        tmp_report_2.splitlines(), key=len)) + 1
+        tmp_h = len(tmp_report.splitlines())
+        tmp_w = len(max(tmp_report.splitlines(), key=len)) + 1
         dst_h = len(dst_report.splitlines())
         dst_w = len(max(dst_report.splitlines(), key=len)) + 1
         arch_h = len(arch_report.splitlines()) + 1
@@ -266,28 +261,19 @@ def curses_main(stdscr):
         jobs_win.chgat(0, 0, curses.A_REVERSE)
 
         # Dirs
-        tmpwin_12_gutter = 3
         tmpwin_dstwin_gutter = 6
 
         maxtd_h = max([tmp_h, dst_h])
 
-        tmpwin_1 = curses.newwin(
+        tmpwin = curses.newwin(
                     tmp_h, tmp_w,
-                    dirs_pos + int((maxtd_h - tmp_h) / 2), 0)
-        tmpwin_1.addstr(tmp_report_1)
-
-        tmpwin_2 = curses.newwin(
-                    tmp_h, tmp_w,
-                    dirs_pos + int((maxtd_h - tmp_h) / 2),
-                    tmp_w + tmpwin_12_gutter)
-        tmpwin_2.addstr(tmp_report_2)
-
-        tmpwin_1.chgat(0, 0, curses.A_REVERSE)
-        tmpwin_2.chgat(0, 0, curses.A_REVERSE)
+                    dirs_pos + int(maxtd_h - tmp_h), 0)
+        tmpwin.addstr(tmp_report)
+        tmpwin.chgat(0, 0, curses.A_REVERSE)
 
         dstwin = curses.newwin(
                 dst_h, dst_w,
-                dirs_pos + int((maxtd_h - dst_h) / 2), 2 * tmp_w + tmpwin_12_gutter + tmpwin_dstwin_gutter)
+                dirs_pos + int((maxtd_h - dst_h) / 2), tmp_w + tmpwin_dstwin_gutter)
         dstwin.addstr(dst_report)
         dstwin.chgat(0, 0, curses.A_REVERSE)
 
@@ -305,8 +291,7 @@ def curses_main(stdscr):
         stdscr.noutrefresh()
         header_win.noutrefresh()
         jobs_win.noutrefresh()
-        tmpwin_1.noutrefresh()
-        tmpwin_2.noutrefresh()
+        tmpwin.noutrefresh()
         dstwin.noutrefresh()
         archwin.noutrefresh()
         log_win.noutrefresh()

From 5ace5e7ecb985708ca9328c1dada44926d197e36 Mon Sep 17 00:00:00 2001
From: ericaltendorf <ericaltendorf@gmail.com>
Date: Mon, 3 May 2021 10:50:28 -0700
Subject: [PATCH 70/86] feat: abbreviate tmpdir phase list too

---
 src/plotman/reporting.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/plotman/reporting.py b/src/plotman/reporting.py
index c56c9a1b..633e4c69 100644
--- a/src/plotman/reporting.py
+++ b/src/plotman/reporting.py
@@ -136,7 +136,7 @@ def tmp_dir_report(jobs, dir_cfg, sched_cfg, width, start_row=None, end_row=None
             continue
         phases = sorted(job.job_phases_for_tmpdir(d, jobs))
         ready = manager.phases_permit_new_job(phases, d, sched_cfg, dir_cfg)
-        row = [abbr_path(d, prefix), 'OK' if ready else '--', phases_str(phases)]
+        row = [abbr_path(d, prefix), 'OK' if ready else '--', phases_str(phases, 5)]
         tab.add_row(row)
 
     tab.set_max_width(width)

From 2512be3db0ddde13a9b0b10afb170fd9d84765f4 Mon Sep 17 00:00:00 2001
From: Marcos R <marcos.ramos@gmail.com>
Date: Tue, 4 May 2021 22:27:24 -0700
Subject: [PATCH 71/86] Add a link to the Archiving wiki to help new users

---
 src/plotman/resources/plotman.yaml | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/src/plotman/resources/plotman.yaml b/src/plotman/resources/plotman.yaml
index 845218b9..114385c9 100644
--- a/src/plotman/resources/plotman.yaml
+++ b/src/plotman/resources/plotman.yaml
@@ -61,6 +61,8 @@ directories:
         # Currently archival depends on an rsync daemon running on the remote
         # host, and that the module is configured to match the local path.
         # See code for details.
+        #
+        # Complete example: https://github.com/ericaltendorf/plotman/wiki/Archiving
         archive:
                 rsyncd_module: plots
                 rsyncd_path: /plots

From 54d870daab7909c7732311433f9429e87fc5661d Mon Sep 17 00:00:00 2001
From: Kyle Altendorf <sda@fstab.net>
Date: Fri, 7 May 2021 01:07:08 -0400
Subject: [PATCH 72/86] fix: rank unknown phases

---
 src/plotman/manager.py | 13 +++++++++++--
 1 file changed, 11 insertions(+), 2 deletions(-)

diff --git a/src/plotman/manager.py b/src/plotman/manager.py
index dd7444a5..dc18f644 100644
--- a/src/plotman/manager.py
+++ b/src/plotman/manager.py
@@ -84,8 +84,17 @@ def maybe_start_new_plot(dir_cfg, sched_cfg, plotting_cfg):
         tmp_to_all_phases = [(d, job.job_phases_for_tmpdir(d, jobs)) for d in dir_cfg.tmp]
         eligible = [ (d, phases) for (d, phases) in tmp_to_all_phases
                 if phases_permit_new_job(phases, d, sched_cfg, dir_cfg) ]
-        rankable = [ (d, phases[0]) if phases else (d, (999, 999))
-                for (d, phases) in eligible ]
+
+        rankable = []
+        for d, phases in eligible:
+            if len(phases) == 0:
+                phase = (999, 999)
+            else:
+                if None in phases[0]:
+                    phase = (888, 888)
+                else:
+                    phase = phases[0]
+            rankable.append((d, phase))
         
         if not eligible:
             wait_reason = 'no eligible tempdirs (%ds/%ds)' % (youngest_job_age, global_stagger)

From 6a8fe8a52c7c08b7127aa72a0d02129beec081fc Mon Sep 17 00:00:00 2001
From: Ed-Yang <edwardyangyang@hotmail.com>
Date: Fri, 7 May 2021 14:47:39 +0800
Subject: [PATCH 73/86] unicode filename in archive drive

---
 .gitignore             | 2 ++
 src/plotman/archive.py | 2 +-
 2 files changed, 3 insertions(+), 1 deletion(-)

diff --git a/.gitignore b/.gitignore
index 82adb58b..16303851 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,2 +1,4 @@
 __pycache__
 venv
+.DS_Store
+.vscode
diff --git a/src/plotman/archive.py b/src/plotman/archive.py
index 17a29af0..f6cfbfb1 100644
--- a/src/plotman/archive.py
+++ b/src/plotman/archive.py
@@ -58,7 +58,7 @@ def get_archdir_freebytes(arch_cfg):
                 # not actually mounted
                 continue
             freebytes = int(fields[3][:-1]) * 1024  # Strip the final 'K'
-            archdir = (fields[5]).decode('ascii')
+            archdir = (fields[5]).decode('utf-8')
             archdir_freebytes[archdir] = freebytes
     return archdir_freebytes
 

From 8dc476c024e6b5eff4a2e4952af647deb2ac7ede Mon Sep 17 00:00:00 2001
From: Kyle Altendorf <sda@fstab.net>
Date: Fri, 7 May 2021 22:06:13 -0400
Subject: [PATCH 74/86] switch to attrs.frozen

---
 setup.cfg                    |  1 +
 src/plotman/configuration.py | 20 ++++++++++----------
 2 files changed, 11 insertions(+), 10 deletions(-)

diff --git a/setup.cfg b/setup.cfg
index 9c10c8f6..87f539b9 100644
--- a/setup.cfg
+++ b/setup.cfg
@@ -37,6 +37,7 @@ package_dir=
 packages=find:
 install_requires =
     appdirs
+    attrs
     click
     desert
     marshmallow
diff --git a/src/plotman/configuration.py b/src/plotman/configuration.py
index 9e92ea5e..df15b8ea 100644
--- a/src/plotman/configuration.py
+++ b/src/plotman/configuration.py
@@ -1,7 +1,7 @@
-from dataclasses import dataclass, field
 from typing import Dict, List, Optional
 
 import appdirs
+import attr
 import desert
 import marshmallow
 import yaml
@@ -17,7 +17,7 @@ def get_path():
 
 
 def get_validated_configs():
-    """Return a validated instance of the PlotmanConfig dataclass with data from plotman.yaml
+    """Return a validated instance of PlotmanConfig with data from plotman.yaml
 
     :raises ConfigurationException: Raised when plotman.yaml is either missing or malformed
     """
@@ -38,7 +38,7 @@ def get_validated_configs():
 
 # Data models used to deserializing/formatting plotman.yaml files.
 
-@dataclass
+@attr.frozen
 class Archive:
     rsyncd_module: str
     rsyncd_path: str
@@ -47,11 +47,11 @@ class Archive:
     rsyncd_user: str
     index: int = 0  # If not explicit, "index" will default to 0
 
-@dataclass
+@attr.frozen
 class TmpOverrides:
     tmpdir_max_jobs: Optional[int] = None
 
-@dataclass
+@attr.frozen
 class Directories:
     log: str
     tmp: List[str]
@@ -60,7 +60,7 @@ class Directories:
     tmp_overrides: Optional[Dict[str, TmpOverrides]] = None
     archive: Optional[Archive] = None
 
-@dataclass
+@attr.frozen
 class Scheduling:
     global_max_jobs: int
     global_stagger_m: int
@@ -70,7 +70,7 @@ class Scheduling:
     tmpdir_stagger_phase_minor: int
     tmpdir_stagger_phase_limit: int = 1  # If not explicit, "tmpdir_stagger_phase_limit" will default to 1
 
-@dataclass
+@attr.frozen
 class Plotting:
     k: int
     e: bool
@@ -80,13 +80,13 @@ class Plotting:
     farmer_pk: Optional[str] = None
     pool_pk: Optional[str] = None
 
-@dataclass
+@attr.frozen
 class UserInterface:
     use_stty_size: bool = True
 
-@dataclass
+@attr.frozen
 class PlotmanConfig:
     directories: Directories
     scheduling: Scheduling
     plotting: Plotting
-    user_interface: UserInterface = field(default_factory=UserInterface)
+    user_interface: UserInterface = attr.ib(factory=UserInterface)

From 394ccf873459f9a20e32072df18a34ff2e224424 Mon Sep 17 00:00:00 2001
From: Kyle Altendorf <sda@fstab.net>
Date: Fri, 7 May 2021 22:33:19 -0400
Subject: [PATCH 75/86] fix: disable rsync compression

---
 src/plotman/archive.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/plotman/archive.py b/src/plotman/archive.py
index 01f91693..62059ec0 100644
--- a/src/plotman/archive.py
+++ b/src/plotman/archive.py
@@ -171,7 +171,7 @@ def archive(dir_cfg, all_jobs):
 
     bwlimit = dir_cfg.archive.rsyncd_bwlimit
     throttle_arg = ('--bwlimit=%d' % bwlimit) if bwlimit else ''
-    cmd = ('rsync %s --remove-source-files -P %s %s' %
+    cmd = ('rsync %s --no-compress --remove-source-files -P %s %s' %
             (throttle_arg, chosen_plot, rsync_dest(dir_cfg.archive, archdir)))
 
     return (True, cmd)

From 2c2563a26d374b5e9955485e2af8f93a6f150a1c Mon Sep 17 00:00:00 2001
From: Kyle Altendorf <sda@fstab.net>
Date: Sat, 8 May 2021 00:22:15 -0400
Subject: [PATCH 76/86] refactor: configuration I/O to the outside

---
 setup.cfg                                |  1 -
 src/plotman/_tests/configuration_test.py | 45 +++++++++---------------
 src/plotman/configuration.py             | 30 ++++++++++------
 src/plotman/interactive.py               |  4 ++-
 src/plotman/plotman.py                   |  4 ++-
 5 files changed, 43 insertions(+), 41 deletions(-)

diff --git a/setup.cfg b/setup.cfg
index 87f539b9..98e6f84f 100644
--- a/setup.cfg
+++ b/setup.cfg
@@ -65,7 +65,6 @@ test =
     check-manifest
     pytest
     pytest-cov
-    pytest-mock
     pyfakefs
 
 [options.data_files]
diff --git a/src/plotman/_tests/configuration_test.py b/src/plotman/_tests/configuration_test.py
index 6ad2abb9..f0a548b6 100644
--- a/src/plotman/_tests/configuration_test.py
+++ b/src/plotman/_tests/configuration_test.py
@@ -8,60 +8,49 @@
 from plotman import resources as plotman_resources
 
 
-@pytest.fixture(name='config_path')
-def config_fixture(tmp_path):
-    """Return direct path to plotman.yaml"""
-    with importlib.resources.path(plotman_resources, "plotman.yaml") as path:
-        yield path
+@pytest.fixture(name='config_text')
+def config_text_fixture():
+    return importlib.resources.read_text(plotman_resources, "plotman.yaml")
 
 
-def test_get_validated_configs__default(mocker, config_path):
+def test_get_validated_configs__default(config_text):
     """Check that get_validated_configs() works with default/example plotman.yaml file."""
-    mocker.patch("plotman.configuration.get_path", return_value=config_path)
-    res = configuration.get_validated_configs()
+    res = configuration.get_validated_configs(config_text, '')
     assert isinstance(res, configuration.PlotmanConfig)
 
 
-def test_get_validated_configs__malformed(mocker, config_path):
+def test_get_validated_configs__malformed(config_text):
     """Check that get_validated_configs() raises exception with invalid plotman.yaml contents."""
-    mocker.patch("plotman.configuration.get_path", return_value=config_path)
-    with open(configuration.get_path(), "r") as file:
-        loaded_yaml = yaml.load(file, Loader=yaml.SafeLoader)
+    loaded_yaml = yaml.load(config_text, Loader=yaml.SafeLoader)
 
     # Purposefully malform the contents of loaded_yaml by changing tmp from List[str] --> str
     loaded_yaml["directories"]["tmp"] = "/mnt/tmp/00"
-    mocker.patch("yaml.load", return_value=loaded_yaml)
+    malformed_config_text = yaml.dump(loaded_yaml, Dumper=yaml.SafeDumper)
 
     with pytest.raises(configuration.ConfigurationException) as exc_info:
-        configuration.get_validated_configs()
+        configuration.get_validated_configs(malformed_config_text, '/the_path')
 
-    assert exc_info.value.args[0] == f"Config file at: '{configuration.get_path()}' is malformed"
+    assert exc_info.value.args[0] == f"Config file at: '/the_path' is malformed"
 
 
-def test_get_validated_configs__missing(mocker, config_path):
+def test_get_validated_configs__missing():
     """Check that get_validated_configs() raises exception when plotman.yaml does not exist."""
-    nonexistent_config = config_path.with_name("plotman2.yaml")
-    mocker.patch("plotman.configuration.get_path", return_value=nonexistent_config)
-
     with pytest.raises(configuration.ConfigurationException) as exc_info:
-        configuration.get_validated_configs()
+        configuration.read_configuration_text('/invalid_path')
 
     assert exc_info.value.args[0] == (
-        f"No 'plotman.yaml' file exists at expected location: '{nonexistent_config}'. To generate "
+        f"No 'plotman.yaml' file exists at expected location: '/invalid_path'. To generate "
         f"default config file, run: 'plotman config generate'"
     )
 
 
-def test_loads_without_user_interface(mocker, config_path, tmp_path):
-    with open(config_path, "r") as file:
-        loaded_yaml = yaml.load(file, Loader=yaml.SafeLoader)
+def test_loads_without_user_interface(config_text):
+    loaded_yaml = yaml.load(config_text, Loader=yaml.SafeLoader)
 
     del loaded_yaml["user_interface"]
 
-    temporary_configuration_path = tmp_path.joinpath("config.yaml")
-    temporary_configuration_path.write_text(yaml.safe_dump(loaded_yaml))
+    stripped_config_text = yaml.dump(loaded_yaml, Dumper=yaml.SafeDumper)
 
-    mocker.patch("plotman.configuration.get_path", return_value=temporary_configuration_path)
-    reloaded_yaml = configuration.get_validated_configs()
+    reloaded_yaml = configuration.get_validated_configs(stripped_config_text, '')
 
     assert reloaded_yaml.user_interface == configuration.UserInterface()
diff --git a/src/plotman/configuration.py b/src/plotman/configuration.py
index df15b8ea..2434456e 100644
--- a/src/plotman/configuration.py
+++ b/src/plotman/configuration.py
@@ -1,3 +1,4 @@
+import contextlib
 from typing import Dict, List, Optional
 
 import appdirs
@@ -16,24 +17,33 @@ def get_path():
     return appdirs.user_config_dir("plotman") + "/plotman.yaml"
 
 
-def get_validated_configs():
+def read_configuration_text(config_path):
+    try:
+        with open(config_path, "r") as file:
+            return file.read()
+    except FileNotFoundError as e:
+        raise ConfigurationException(
+            f"No 'plotman.yaml' file exists at expected location: '{config_path}'. To generate "
+            f"default config file, run: 'plotman config generate'"
+        ) from e
+
+
+def get_validated_configs(config_text, config_path):
     """Return a validated instance of PlotmanConfig with data from plotman.yaml
 
     :raises ConfigurationException: Raised when plotman.yaml is either missing or malformed
     """
     schema = desert.schema(PlotmanConfig)
-    config_file_path = get_path()
+    config_objects = yaml.load(config_text, Loader=yaml.SafeLoader)
+
     try:
-        with open(config_file_path, "r") as file:
-            config_file = yaml.load(file, Loader=yaml.SafeLoader)
-            return schema.load(config_file)
-    except FileNotFoundError as e:
+        loaded = schema.load(config_objects)
+    except marshmallow.exceptions.ValidationError as e:
         raise ConfigurationException(
-            f"No 'plotman.yaml' file exists at expected location: '{config_file_path}'. To generate "
-            f"default config file, run: 'plotman config generate'"
+            f"Config file at: '{config_path}' is malformed"
         ) from e
-    except marshmallow.exceptions.ValidationError as e:
-        raise ConfigurationException(f"Config file at: '{config_file_path}' is malformed") from e
+
+    return loaded
 
 
 # Data models used to deserializing/formatting plotman.yaml files.
diff --git a/src/plotman/interactive.py b/src/plotman/interactive.py
index d5f32319..5835b18d 100644
--- a/src/plotman/interactive.py
+++ b/src/plotman/interactive.py
@@ -65,7 +65,9 @@ def archiving_status_msg(configured, active, status):
 def curses_main(stdscr):
     log = Log()
 
-    cfg = configuration.get_validated_configs()
+    config_path = configuration.get_path()
+    config_text = configuration.read_configuration_text(config_path)
+    cfg = configuration.get_validated_configs(config_text, config_path)
 
     plotting_active = True
     archiving_configured = cfg.directories.archive is not None
diff --git a/src/plotman/plotman.py b/src/plotman/plotman.py
index 3319cae8..d3468735 100755
--- a/src/plotman/plotman.py
+++ b/src/plotman/plotman.py
@@ -132,7 +132,9 @@ def main():
             print("No action requested, add 'generate' or 'path'.")
             return
 
-    cfg = configuration.get_validated_configs()
+    config_path = configuration.get_path()
+    config_text = configuration.read_configuration_text(config_path)
+    cfg = configuration.get_validated_configs(config_text, config_path)
 
     #
     # Stay alive, spawning plot jobs

From cdbb6e3e179fe70082ffceff86019c71bb975760 Mon Sep 17 00:00:00 2001
From: Kyle Altendorf <sda@fstab.net>
Date: Sat, 8 May 2021 01:22:09 -0400
Subject: [PATCH 77/86] refactor: introduce Phase class

---
 src/plotman/_tests/reporting_test.py | 17 ++++++------
 src/plotman/job.py                   | 39 ++++++++++++++++++++++++----
 src/plotman/reporting.py             |  9 ++++---
 3 files changed, 48 insertions(+), 17 deletions(-)

diff --git a/src/plotman/_tests/reporting_test.py b/src/plotman/_tests/reporting_test.py
index b46dd873..2013e2a9 100644
--- a/src/plotman/_tests/reporting_test.py
+++ b/src/plotman/_tests/reporting_test.py
@@ -3,23 +3,24 @@
 from unittest.mock import patch
 
 from plotman import reporting
+from plotman import job
 
 
 def test_phases_str_basic():
-    assert(reporting.phases_str([(1,2), (2,3), (3,4), (4,0)]) ==
-            '1:2 2:3 3:4 4:0')
+    phases = job.Phase.list_from_tuples([(1,2), (2,3), (3,4), (4,0)])
+    assert reporting.phases_str(phases) == '1:2 2:3 3:4 4:0'
 
 def test_phases_str_elipsis_1():
-    assert(reporting.phases_str([(1,2), (2,3), (3,4), (4,0)], 3) ==
-            '1:2 [+1] 3:4 4:0')
+    phases = job.Phase.list_from_tuples([(1,2), (2,3), (3,4), (4,0)])
+    assert reporting.phases_str(phases, 3) == '1:2 [+1] 3:4 4:0'
 
 def test_phases_str_elipsis_2():
-    assert(reporting.phases_str([(1,2), (2,3), (3,4), (4,0)], 2) ==
-            '1:2 [+2] 4:0')
+    phases = job.Phase.list_from_tuples([(1,2), (2,3), (3,4), (4,0)])
+    assert reporting.phases_str(phases, 2) == '1:2 [+2] 4:0'
 
 def test_phases_str_none():
-    assert(reporting.phases_str([(None, None), (2, None), (3, 0)]) ==
-            '?:? 2:? 3:0')
+    phases = job.Phase.list_from_tuples([(None, None), (3, 0)])
+    assert reporting.phases_str(phases) == '?:? 3:0'
 
 def test_job_viz_empty():
     assert(reporting.job_viz([]) == '1        2        3       4 ')
diff --git a/src/plotman/job.py b/src/plotman/job.py
index 8150d400..9b684c1b 100644
--- a/src/plotman/job.py
+++ b/src/plotman/job.py
@@ -11,6 +11,7 @@
 from enum import Enum, auto
 from subprocess import call
 
+import attr
 import click
 import pendulum
 import psutil
@@ -87,6 +88,35 @@ def __init__(self, error, help, parameters):
         self.help = help
         self.parameters = parameters
 
+@attr.frozen(order=False)
+class Phase:
+    major: int = 0
+    minor: int = 0
+    known: bool = True
+
+    def __lt__(self, other):
+        return (
+            (self.unknown, self.major, self.minor)
+            < (other.unknown, other.major, other.minor)
+        )
+
+    @classmethod
+    def from_tuple(cls, t):
+        if len(t) != 2:
+            raise Exception(f'phase must be created from 2-tuple: {t!r}')
+
+        if None in t and not t[0] is t[1]:
+            raise Exception(f'phase can not be partially known: {t!r}')
+
+        if t[0] is None:
+            return cls(known=False)
+
+        return cls(major=t[0], minor=t[1])
+
+    @classmethod
+    def list_from_tuples(cls, l):
+        return [cls.from_tuple(t) for t in l]
+
 # TODO: be more principled and explicit about what we cache vs. what we look up
 # dynamically from the logfile
 class Job:
@@ -98,9 +128,6 @@ class Job:
     plot_id = '--------'
     proc = None   # will get a psutil.Process
 
-    # These are dynamic, cached, and need to be udpated periodically
-    phase = (None, None)   # Phase/subphase
-
     def get_running_jobs(logroot, cached_jobs=()):
         '''Return a list of running plot jobs.  If a cache of preexisting jobs is provided,
            reuse those previous jobs without updating their information.  Always look for
@@ -137,6 +164,8 @@ def get_running_jobs(logroot, cached_jobs=()):
     def __init__(self, proc, parsed_command, logroot):
         '''Initialize from an existing psutil.Process object.  must know logroot in order to understand open files'''
         self.proc = proc
+        # These are dynamic, cached, and need to be udpated periodically
+        self.phase = Phase(known=False)
 
         self.help = parsed_command.help
         self.args = parsed_command.parameters
@@ -285,9 +314,9 @@ def set_phase_from_logfile(self):
 
         if phase_subphases:
             phase = max(phase_subphases.keys())
-            self.phase = (phase, phase_subphases[phase])
+            self.phase = Phase(major=phase, minor=phase_subphases[phase])
         else:
-            self.phase = (0, 0)
+            self.phase = Phase(major=0, minor=0)
 
     def progress(self):
         '''Return a 2-tuple with the job phase and subphase (by reading the logfile)'''
diff --git a/src/plotman/reporting.py b/src/plotman/reporting.py
index 633e4c69..19f4c087 100644
--- a/src/plotman/reporting.py
+++ b/src/plotman/reporting.py
@@ -13,10 +13,11 @@ def abbr_path(path, putative_prefix):
     else:
         return path
     
-def phase_str(phase_pair):
-    (ph, subph) = phase_pair
-    return ((str(ph) if ph is not None else '?') + ':'
-            + (str(subph) if subph is not None else '?'))
+def phase_str(phase):
+    if not phase.known:
+        return '?:?'
+
+    return f'{phase.major}:{phase.minor}'
 
 def phases_str(phases, max_num=None):
     '''Take a list of phase-subphase pairs and return them as a compact string'''

From 792c3fcad43526a8071519ff93b333ae9f72d757 Mon Sep 17 00:00:00 2001
From: Kyle Altendorf <sda@fstab.net>
Date: Sat, 8 May 2021 01:34:03 -0400
Subject: [PATCH 78/86] refactor: go back a bit

---
 src/plotman/manager.py | 13 ++-----------
 1 file changed, 2 insertions(+), 11 deletions(-)

diff --git a/src/plotman/manager.py b/src/plotman/manager.py
index dc18f644..20800a6a 100644
--- a/src/plotman/manager.py
+++ b/src/plotman/manager.py
@@ -84,17 +84,8 @@ def maybe_start_new_plot(dir_cfg, sched_cfg, plotting_cfg):
         tmp_to_all_phases = [(d, job.job_phases_for_tmpdir(d, jobs)) for d in dir_cfg.tmp]
         eligible = [ (d, phases) for (d, phases) in tmp_to_all_phases
                 if phases_permit_new_job(phases, d, sched_cfg, dir_cfg) ]
-
-        rankable = []
-        for d, phases in eligible:
-            if len(phases) == 0:
-                phase = (999, 999)
-            else:
-                if None in phases[0]:
-                    phase = (888, 888)
-                else:
-                    phase = phases[0]
-            rankable.append((d, phase))
+        rankable = [ (d, phases[0]) if phases else (d, job.Phase(known=False))
+                for (d, phases) in eligible ]
         
         if not eligible:
             wait_reason = 'no eligible tempdirs (%ds/%ds)' % (youngest_job_age, global_stagger)

From bcc2a2a958bb0190f87e8d5ad10a4478c0aac833 Mon Sep 17 00:00:00 2001
From: Kyle Altendorf <sda@fstab.net>
Date: Sat, 8 May 2021 01:36:29 -0400
Subject: [PATCH 79/86] fix: known not unknown

---
 src/plotman/job.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/plotman/job.py b/src/plotman/job.py
index 9b684c1b..205d3509 100644
--- a/src/plotman/job.py
+++ b/src/plotman/job.py
@@ -96,8 +96,8 @@ class Phase:
 
     def __lt__(self, other):
         return (
-            (self.unknown, self.major, self.minor)
-            < (other.unknown, other.major, other.minor)
+            (not self.known, self.major, self.minor)
+            < (not other.known, other.major, other.minor)
         )
 
     @classmethod

From f5e4c34fb33a74f954e56ed82ff3dea54f3b8d57 Mon Sep 17 00:00:00 2001
From: Kyle Altendorf <sda@fstab.net>
Date: Sat, 8 May 2021 01:41:20 -0400
Subject: [PATCH 80/86] fix: more phase corners

---
 src/plotman/archive.py | 12 ++++++------
 src/plotman/manager.py |  7 +++++--
 2 files changed, 11 insertions(+), 8 deletions(-)

diff --git a/src/plotman/archive.py b/src/plotman/archive.py
index 62059ec0..059a48e5 100644
--- a/src/plotman/archive.py
+++ b/src/plotman/archive.py
@@ -11,7 +11,7 @@
 import psutil
 import texttable as tt
 
-from plotman import manager, plot_util
+from plotman import job, manager, plot_util
 
 # TODO : write-protect and delete-protect archived plots
 
@@ -62,14 +62,14 @@ def compute_priority(phase, gb_free, n_plots):
     # To avoid concurrent IO, we should not touch drives that
     # are about to receive a new plot.  If we don't know the phase,
     # ignore.
-    if (phase[0] and phase[1]):
-        if (phase == (3, 4)):
+    if (phase.known):
+        if (phase == job.Phase(3, 4)):
             priority -= 4
-        elif (phase == (3, 5)):
+        elif (phase == job.Phase(3, 5)):
             priority -= 8
-        elif (phase == (3, 6)):
+        elif (phase == job.Phase(3, 6)):
             priority -= 16
-        elif (phase >= (3, 7)):
+        elif (phase >= job.Phase(3, 7)):
             priority -= 32
         
     # If a drive is getting full, we should prioritize it
diff --git a/src/plotman/manager.py b/src/plotman/manager.py
index 20800a6a..e0ed2851 100644
--- a/src/plotman/manager.py
+++ b/src/plotman/manager.py
@@ -47,12 +47,15 @@ def phases_permit_new_job(phases, d, sched_cfg, dir_cfg):
     '''Scheduling logic: return True if it's OK to start a new job on a tmp dir
        with existing jobs in the provided phases.'''
     # Filter unknown-phase jobs
-    phases = [ph for ph in phases if ph[0] is not None and ph[1] is not None]
+    phases = [ph for ph in phases if ph.known]
 
     if len(phases) == 0:
         return True
 
-    milestone = (sched_cfg.tmpdir_stagger_phase_major, sched_cfg.tmpdir_stagger_phase_minor)
+    milestone = job.Phase(
+        major=sched_cfg.tmpdir_stagger_phase_major,
+        minor=sched_cfg.tmpdir_stagger_phase_minor,
+    )
     # tmpdir_stagger_phase_limit default is 1, as declared in configuration.py
     if len([p for p in phases if p < milestone]) >= sched_cfg.tmpdir_stagger_phase_limit:
         return False

From e37f50a05194c49c7206be3f5a43b056268bfef4 Mon Sep 17 00:00:00 2001
From: Kyle Altendorf <sda@fstab.net>
Date: Sat, 8 May 2021 02:01:22 -0400
Subject: [PATCH 81/86] fix: and yet more

---
 src/plotman/_tests/archive_test.py |  6 +++---
 src/plotman/_tests/manager_test.py | 21 ++++++++++++++-------
 src/plotman/job.py                 |  2 ++
 3 files changed, 19 insertions(+), 10 deletions(-)

diff --git a/src/plotman/_tests/archive_test.py b/src/plotman/_tests/archive_test.py
index 9caa2abe..4b98ac90 100755
--- a/src/plotman/_tests/archive_test.py
+++ b/src/plotman/_tests/archive_test.py
@@ -1,9 +1,9 @@
-from plotman import archive, configuration, manager
+from plotman import archive, configuration, job, manager
 
 
 def test_compute_priority():
-    assert (archive.compute_priority( (3, 1), 1000, 10) >
-            archive.compute_priority( (3, 6), 1000, 10) )
+    assert (archive.compute_priority( job.Phase(major=3, minor=1), 1000, 10) >
+            archive.compute_priority( job.Phase(major=3, minor=6), 1000, 10) )
 
 def test_rsync_dest():
     arch_dir = '/plotdir/012'
diff --git a/src/plotman/_tests/manager_test.py b/src/plotman/_tests/manager_test.py
index 2f425955..cca37596 100755
--- a/src/plotman/_tests/manager_test.py
+++ b/src/plotman/_tests/manager_test.py
@@ -27,30 +27,37 @@ def dir_cfg():
     )
 
 def test_permit_new_job_post_milestone(sched_cfg, dir_cfg):
+    phases = job.Phase.list_from_tuples([ (3, 8), (4, 1) ])
     assert manager.phases_permit_new_job(
-        [ (3, 8), (4, 1) ], '/mnt/tmp/00', sched_cfg, dir_cfg)
+        phases, '/mnt/tmp/00', sched_cfg, dir_cfg)
 
 def test_permit_new_job_pre_milestone(sched_cfg, dir_cfg):
+    phases = job.Phase.list_from_tuples([ (2, 3), (4, 1) ])
     assert not manager.phases_permit_new_job(
-        [ (2, 3), (4, 1) ], '/mnt/tmp/00', sched_cfg, dir_cfg)
+        phases, '/mnt/tmp/00', sched_cfg, dir_cfg)
 
 def test_permit_new_job_too_many_jobs(sched_cfg, dir_cfg):
+    phases = job.Phase.list_from_tuples([ (3, 1), (3, 2), (3, 3) ])
     assert not manager.phases_permit_new_job(
-        [ (3, 1), (3, 2), (3, 3) ], '/mnt/tmp/00', sched_cfg, dir_cfg)
+        phases, '/mnt/tmp/00', sched_cfg, dir_cfg)
 
 def test_permit_new_job_too_many_jobs_zerophase(sched_cfg, dir_cfg):
+    phases = job.Phase.list_from_tuples([ (3, 0), (3, 1), (3, 3) ])
     assert not manager.phases_permit_new_job(
-        [ (3, 0), (3, 1), (3, 3) ], '/mnt/tmp/00', sched_cfg, dir_cfg)
+        phases, '/mnt/tmp/00', sched_cfg, dir_cfg)
 
 def test_permit_new_job_too_many_jobs_nonephase(sched_cfg, dir_cfg):
+    phases = job.Phase.list_from_tuples([ (None, None), (3, 1), (3, 3) ])
     assert manager.phases_permit_new_job(
-        [ (None, None), (3, 1), (3, 3) ], '/mnt/tmp/00', sched_cfg, dir_cfg)
+        phases, '/mnt/tmp/00', sched_cfg, dir_cfg)
 
 def test_permit_new_job_override_tmp_dir(sched_cfg, dir_cfg):
+    phases = job.Phase.list_from_tuples([ (3, 1), (3, 2), (3, 3) ])
     assert manager.phases_permit_new_job(
-        [ (3, 1), (3, 2), (3, 3) ], '/mnt/tmp/04', sched_cfg, dir_cfg)
+        phases, '/mnt/tmp/04', sched_cfg, dir_cfg)
+    phases = job.Phase.list_from_tuples([ (3, 1), (3, 2), (3, 3), (3, 6) ])
     assert not manager.phases_permit_new_job(
-        [ (3, 1), (3, 2), (3, 3), (3, 6) ], '/mnt/tmp/04', sched_cfg,
+        phases, '/mnt/tmp/04', sched_cfg,
         dir_cfg)
 
 @patch('plotman.job.Job')
diff --git a/src/plotman/job.py b/src/plotman/job.py
index 205d3509..5116c305 100644
--- a/src/plotman/job.py
+++ b/src/plotman/job.py
@@ -1,6 +1,7 @@
 # TODO do we use all these?
 import argparse
 import contextlib
+import functools
 import logging
 import os
 import random
@@ -88,6 +89,7 @@ def __init__(self, error, help, parameters):
         self.help = help
         self.parameters = parameters
 
+@functools.total_ordering
 @attr.frozen(order=False)
 class Phase:
     major: int = 0

From 6e66c2ef2e3efdc1c9a7223181f4b659877de8ed Mon Sep 17 00:00:00 2001
From: Kyle Altendorf <sda@fstab.net>
Date: Sat, 8 May 2021 13:17:18 -0400
Subject: [PATCH 82/86] fix: handle None dirs when checking for tmp files

---
 src/plotman/job.py | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/src/plotman/job.py b/src/plotman/job.py
index 8150d400..6092115a 100644
--- a/src/plotman/job.py
+++ b/src/plotman/job.py
@@ -370,7 +370,11 @@ def get_temp_files(self):
         # Prevent duplicate file paths by using set.
         temp_files = set([])
         for f in self.proc.open_files():
-            if self.tmpdir in f.path or self.tmp2dir in f.path or self.dstdir in f.path:
+            if any(
+                dir in f.path
+                for dir in [self.tmpdir, self.tmp2dir, self.dstdir]
+                if dir is not None
+            ):
                 temp_files.add(f.path)
         return temp_files
 

From 0157abc724ebde8bc47017f13ed8bd2fa583fb24 Mon Sep 17 00:00:00 2001
From: Kyle Altendorf <sda@fstab.net>
Date: Mon, 10 May 2021 21:08:12 -0400
Subject: [PATCH 83/86] fix: another tuple -> job.Phase

---
 src/plotman/archive.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/plotman/archive.py b/src/plotman/archive.py
index 059a48e5..c5b389b7 100644
--- a/src/plotman/archive.py
+++ b/src/plotman/archive.py
@@ -135,7 +135,7 @@ def archive(dir_cfg, all_jobs):
     chosen_plot = None
 
     for d in dir_cfg.dst:
-        ph = dir2ph.get(d, (0, 0))
+        ph = dir2ph.get(d, job.Phase(0, 0))
         dir_plots = plot_util.list_k32_plots(d)
         gb_free = plot_util.df_b(d) / plot_util.GB
         n_plots = len(dir_plots)

From 616ad4bbf47808134ffa23164323d2fc91a8da6f Mon Sep 17 00:00:00 2001
From: Kyle Altendorf <sda@fstab.net>
Date: Mon, 10 May 2021 21:17:33 -0400
Subject: [PATCH 84/86] set version to v0.3

---
 VERSION | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/VERSION b/VERSION
index 3b04cfb6..be586341 100644
--- a/VERSION
+++ b/VERSION
@@ -1 +1 @@
-0.2
+0.3

From 161a898acc34bfeafd52688eb0c56ebb78db4052 Mon Sep 17 00:00:00 2001
From: Bruno Lopes <lopesmcc@gmail.com>
Date: Tue, 11 May 2021 13:43:58 +0100
Subject: [PATCH 85/86] Fix phase visualizer

---
 src/plotman/reporting.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/plotman/reporting.py b/src/plotman/reporting.py
index 19f4c087..caadc463 100644
--- a/src/plotman/reporting.py
+++ b/src/plotman/reporting.py
@@ -33,7 +33,7 @@ def phases_str(phases, max_num=None):
         return first + elided + last
 
 def n_at_ph(jobs, ph):
-    return sum([1 for j in jobs if j.progress() == ph])
+    return sum([1 for j in jobs if j.progress() == job.Phase.from_tuple(ph)])
 
 def n_to_char(n):
     n_to_char_map = dict(enumerate(" .:;!"))

From 482843e5a614bff8656ee765868ebc25f263a7da Mon Sep 17 00:00:00 2001
From: Kyle Altendorf <sda@fstab.net>
Date: Tue, 11 May 2021 20:21:17 -0400
Subject: [PATCH 86/86] fixup tests

---
 src/plotman/_tests/reporting_test.py |  2 +-
 src/plotman/reporting.py             | 10 +++++-----
 2 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/src/plotman/_tests/reporting_test.py b/src/plotman/_tests/reporting_test.py
index 2013e2a9..87c8a5e2 100644
--- a/src/plotman/_tests/reporting_test.py
+++ b/src/plotman/_tests/reporting_test.py
@@ -28,7 +28,7 @@ def test_job_viz_empty():
 @patch('plotman.job.Job')
 def job_w_phase(ph, MockJob):
     j = MockJob()
-    j.progress.return_value = ph
+    j.progress.return_value = job.Phase.from_tuple(ph)
     return j
 
 def test_job_viz_positions():
diff --git a/src/plotman/reporting.py b/src/plotman/reporting.py
index caadc463..8b00a84a 100644
--- a/src/plotman/reporting.py
+++ b/src/plotman/reporting.py
@@ -33,7 +33,7 @@ def phases_str(phases, max_num=None):
         return first + elided + last
 
 def n_at_ph(jobs, ph):
-    return sum([1 for j in jobs if j.progress() == job.Phase.from_tuple(ph)])
+    return sum([1 for j in jobs if j.progress() == ph])
 
 def n_to_char(n):
     n_to_char_map = dict(enumerate(" .:;!"))
@@ -51,15 +51,15 @@ def job_viz(jobs):
     result = ''
     result += '1'
     for i in range(0, 8):
-        result += n_to_char(n_at_ph(jobs, (1, i)))
+        result += n_to_char(n_at_ph(jobs, job.Phase(1, i)))
     result += '2'
     for i in range(0, 8):
-        result += n_to_char(n_at_ph(jobs, (2, i)))
+        result += n_to_char(n_at_ph(jobs, job.Phase(2, i)))
     result += '3'
     for i in range(0, 7):
-        result += n_to_char(n_at_ph(jobs, (3, i)))
+        result += n_to_char(n_at_ph(jobs, job.Phase(3, i)))
     result += '4'
-    result += n_to_char(n_at_ph(jobs, (4, 0)))
+    result += n_to_char(n_at_ph(jobs, job.Phase(4, 0)))
     return result
 
 def status_report(jobs, width, height=None, tmp_prefix='', dst_prefix=''):