From fbca19abb6c783ab548e15b83219d950bc0bfde5 Mon Sep 17 00:00:00 2001
From: Johannes K Becker <jkbecker@bu.edu>
Date: Fri, 2 Apr 2021 09:06:55 -0400
Subject: [PATCH 01/58] local/remote switch for rsync

---
 archive.py  | 29 ++++++++++++++++++++++-------
 config.yaml |  5 +++--
 2 files changed, 25 insertions(+), 9 deletions(-)

diff --git a/archive.py b/archive.py
index 0655e549..80a53cbd 100755
--- a/archive.py
+++ b/archive.py
@@ -49,8 +49,16 @@ def compute_priority(phase, gb_free, n_plots):
 
 def get_archdir_freebytes(arch_cfg):
     archdir_freebytes = { }
-    df_cmd = ('ssh %s@%s df -BK | grep " %s/"' %
-        (arch_cfg['rsyncd_user'], arch_cfg['rsyncd_host'], arch_cfg['rsyncd_path']) )
+    arch_mode = arch_cfg.get('mode', 'remote')
+    print('> mode is', arch_mode)
+    if arch_mode == 'remote':
+        df_cmd = ('ssh %s@%s df -BK | grep " %s/"' %
+            (arch_cfg['rsyncd_user'], arch_cfg['rsyncd_host'], arch_cfg['rsyncd_path']) )
+    elif arch_mode == 'local':
+        df_cmd = ('df -BK | grep " %s/"' % arch_cfg['rsyncd_path'] )
+    else:
+        raise KeyError(f'Archive mode must be "remote" or "local" ("{arch_mode}" given). Please inspect config.yaml.')
+    print('> df_cmd is', df_cmd)
     with subprocess.Popen(df_cmd, shell=True, stdout=subprocess.PIPE) as proc:
         for line in proc.stdout.readlines():
             fields = line.split()
@@ -60,11 +68,18 @@ def get_archdir_freebytes(arch_cfg):
     return archdir_freebytes
 
 def rsync_dest(arch_cfg, arch_dir):
-    rsync_path = arch_dir.replace(arch_cfg['rsyncd_path'], arch_cfg['rsyncd_module'])
-    if rsync_path.startswith('/'):
-        rsync_path = rsync_path[1:]  # Avoid dup slashes.  TODO use path join?
-    rsync_url = 'rsync://%s@%s:12000/%s' % (
-            arch_cfg['rsyncd_user'], arch_cfg['rsyncd_host'], rsync_path)
+    arch_mode = arch_cfg.get('mode', 'remote')
+    if arch_mode == 'remote':
+        rsync_path = arch_dir.replace(arch_cfg['rsyncd_path'], arch_cfg['rsyncd_module'])
+        if rsync_path.startswith('/'):
+            rsync_path = rsync_path[1:]  # Avoid dup slashes.  TODO use path join?
+        rsync_url = 'rsync://%s@%s:12000/%s' % (
+                arch_cfg['rsyncd_user'], arch_cfg['rsyncd_host'], rsync_path)
+    elif arch_mode == 'local':
+        rsync_url = arch_cfg['rsyncd_path']
+    else:
+        raise KeyError(f'Archive mode must be "remote" or "local" ("{arch_mode}" given). Please inspect config.yaml.')
+    print('> rsync_url is', rsync_url)
     return rsync_url
 
 # TODO: maybe consolidate with similar code in job.py?
diff --git a/config.yaml b/config.yaml
index 93f2b54e..3d25da9f 100644
--- a/config.yaml
+++ b/config.yaml
@@ -38,11 +38,12 @@ directories:
         # host, and that the module is configured to match the local path.
         # See code for details.
         archive:
+                mode: remote  # remote or local
                 rsyncd_module: plots
                 rsyncd_path: /plots
                 rsyncd_bwlimit: 80000  # Bandwidth limit in KB/s
-                rsyncd_host: myfarmer
-                rsyncd_user: chia
+                rsyncd_host: myfarmer  # only required for remote mode
+                rsyncd_user: chia  # only required for remote mode
 
 
 # Plotting scheduling parameters

From a39f39a7769837c16be77ef34ad8f18c16639b78 Mon Sep 17 00:00:00 2001
From: Johannes K Becker <jkbecker@bu.edu>
Date: Fri, 2 Apr 2021 09:19:51 -0400
Subject: [PATCH 02/58] fix local rsync dest

---
 archive.py | 5 +----
 1 file changed, 1 insertion(+), 4 deletions(-)

diff --git a/archive.py b/archive.py
index 80a53cbd..d7e1eed1 100755
--- a/archive.py
+++ b/archive.py
@@ -50,7 +50,6 @@ def compute_priority(phase, gb_free, n_plots):
 def get_archdir_freebytes(arch_cfg):
     archdir_freebytes = { }
     arch_mode = arch_cfg.get('mode', 'remote')
-    print('> mode is', arch_mode)
     if arch_mode == 'remote':
         df_cmd = ('ssh %s@%s df -BK | grep " %s/"' %
             (arch_cfg['rsyncd_user'], arch_cfg['rsyncd_host'], arch_cfg['rsyncd_path']) )
@@ -58,7 +57,6 @@ def get_archdir_freebytes(arch_cfg):
         df_cmd = ('df -BK | grep " %s/"' % arch_cfg['rsyncd_path'] )
     else:
         raise KeyError(f'Archive mode must be "remote" or "local" ("{arch_mode}" given). Please inspect config.yaml.')
-    print('> df_cmd is', df_cmd)
     with subprocess.Popen(df_cmd, shell=True, stdout=subprocess.PIPE) as proc:
         for line in proc.stdout.readlines():
             fields = line.split()
@@ -76,10 +74,9 @@ def rsync_dest(arch_cfg, arch_dir):
         rsync_url = 'rsync://%s@%s:12000/%s' % (
                 arch_cfg['rsyncd_user'], arch_cfg['rsyncd_host'], rsync_path)
     elif arch_mode == 'local':
-        rsync_url = arch_cfg['rsyncd_path']
+        rsync_url = arch_dir
     else:
         raise KeyError(f'Archive mode must be "remote" or "local" ("{arch_mode}" given). Please inspect config.yaml.')
-    print('> rsync_url is', rsync_url)
     return rsync_url
 
 # TODO: maybe consolidate with similar code in job.py?

From fc7ea415fffece666dd32d4de9e419e1f5a72b78 Mon Sep 17 00:00:00 2001
From: Johannes K Becker <jkbecker@bu.edu>
Date: Fri, 2 Apr 2021 10:24:56 -0400
Subject: [PATCH 03/58] add notes to config.yaml

---
 config.yaml | 15 ++++++++++++---
 1 file changed, 12 insertions(+), 3 deletions(-)

diff --git a/config.yaml b/config.yaml
index 3d25da9f..335f6821 100644
--- a/config.yaml
+++ b/config.yaml
@@ -39,11 +39,20 @@ directories:
         # See code for details.
         archive:
                 mode: remote  # remote or local
-                rsyncd_module: plots
+                rsyncd_module: plots  # ignored in local mode
+
+                # Note for local mode:
+                # This path should be the directory *in which* your archive disks
+                # are mounted. For example, if your archive disks are
+                # /mnt/archive1 and /mnt/archive2, the path should be /mnt.
+                # Note that *all* mounted drives within that path will be
+                # considered - so if you have other things mounted as well,
+                # make sure you mount all Chia archive drives to some custom
+                # directory that only contains Chia archive mountpoints.
                 rsyncd_path: /plots
                 rsyncd_bwlimit: 80000  # Bandwidth limit in KB/s
-                rsyncd_host: myfarmer  # only required for remote mode
-                rsyncd_user: chia  # only required for remote mode
+                rsyncd_host: myfarmer  # # ignored in local mode
+                rsyncd_user: chia  # # ignored in local mode
 
 
 # Plotting scheduling parameters

From 829661cf1951a8d5ebfc4a2818a4895f10baec39 Mon Sep 17 00:00:00 2001
From: Johannes Becker <31668498+jkbecker@users.noreply.github.com>
Date: Wed, 7 Apr 2021 09:38:15 -0400
Subject: [PATCH 04/58] Update config.yaml

Co-authored-by: Kyle Altendorf <sda@fstab.net>
---
 config.yaml | 1 -
 1 file changed, 1 deletion(-)

diff --git a/config.yaml b/config.yaml
index e650b89a..7ed14ae7 100644
--- a/config.yaml
+++ b/config.yaml
@@ -87,7 +87,6 @@ directories:
                 #   index: 0
 
 
-
 # Plotting scheduling parameters
 scheduling:
         # Run a job on a particular temp dir only if the number of existing jobs

From aae3654fa0e9b5c45a5124d04bbb85dace118e94 Mon Sep 17 00:00:00 2001
From: Johannes Becker <31668498+jkbecker@users.noreply.github.com>
Date: Wed, 7 Apr 2021 09:40:03 -0400
Subject: [PATCH 05/58] Update src/plotman/archive.py

Co-authored-by: Kyle Altendorf <sda@fstab.net>
---
 src/plotman/archive.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/plotman/archive.py b/src/plotman/archive.py
index e88eb7d3..b6ba309c 100644
--- a/src/plotman/archive.py
+++ b/src/plotman/archive.py
@@ -56,7 +56,7 @@ def get_archdir_freebytes(arch_cfg):
     elif arch_mode == 'local':
         df_cmd = ('df -BK | grep " %s/"' % arch_cfg['rsyncd_path'] )
     else:
-        raise KeyError(f'Archive mode must be "remote" or "local" ("{arch_mode}" given). Please inspect config.yaml.')
+        raise KeyError(f'Archive mode must be "remote" or "local" ({arch_mode!r} given). Please inspect config.yaml.')
     with subprocess.Popen(df_cmd, shell=True, stdout=subprocess.PIPE) as proc:
         for line in proc.stdout.readlines():
             fields = line.split()

From c7d8ce2a95a188edb5356d396ce4c5edb13176dd Mon Sep 17 00:00:00 2001
From: Johannes K Becker <jkbecker@bu.edu>
Date: Wed, 28 Apr 2021 14:38:35 -0400
Subject: [PATCH 06/58] adapt config template

---
 src/plotman/resources/plotman.yaml | 27 +++++++++++++++------------
 1 file changed, 15 insertions(+), 12 deletions(-)

diff --git a/src/plotman/resources/plotman.yaml b/src/plotman/resources/plotman.yaml
index e2a6d3e0..083f0ee3 100644
--- a/src/plotman/resources/plotman.yaml
+++ b/src/plotman/resources/plotman.yaml
@@ -62,21 +62,12 @@ directories:
         # host, and that the module is configured to match the local path.
         # See code for details.
         archive:
-                mode: remote  # remote or local
-                rsyncd_module: plots  # ignored in local mode
+                rsyncd_module: plots
 
-                # Note for local mode:
-                # This path should be the directory *in which* your archive disks
-                # are mounted. For example, if your archive disks are
-                # /mnt/archive1 and /mnt/archive2, the path should be /mnt.
-                # Note that *all* mounted drives within that path will be
-                # considered - so if you have other things mounted as well,
-                # make sure you mount all Chia archive drives to some custom
-                # directory that only contains Chia archive mountpoints.
                 rsyncd_path: /plots
                 rsyncd_bwlimit: 80000  # Bandwidth limit in KB/s
-                rsyncd_host: myfarmer  # # ignored in local mode
-                rsyncd_user: chia  # # ignored in local mode
+                rsyncd_host: myfarmer
+                rsyncd_user: chia
 
                 # Optional index.  If omitted or set to 0, plotman will archive
                 # to the first archive dir with free space.  If specified,
@@ -88,6 +79,18 @@ directories:
                 # the 4 machines, or 0, 1, 0, 1.
                 #   index: 0
 
+                # Optional switch to enable local archiving.
+                # Note: rsyncd_module, rsyncd_host and rsyncd_user are ignored in local mode.
+                mode: remote  # remote or local
+                # Note for local mode:
+                # The rsyncd_path should be the directory *in which* your archive
+                # disks are mounted. For example, if your archive disks are
+                # /mnt/archive1 and /mnt/archive2, the path should be /mnt.
+                # Note that *all* mounted drives within that path will be
+                # considered - so if you have other things mounted as well,
+                # make sure you mount all Chia archive drives to some custom
+                # directory that only contains Chia archive mountpoints.
+
 
 # Plotting scheduling parameters
 scheduling:

From 68b683fd192c361eddca14a5f815a831813c85ae Mon Sep 17 00:00:00 2001
From: Johannes K Becker <jkbecker@bu.edu>
Date: Wed, 28 Apr 2021 14:39:47 -0400
Subject: [PATCH 07/58] adapt error message

---
 src/plotman/archive.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/plotman/archive.py b/src/plotman/archive.py
index aa2c3301..4b2eaae3 100644
--- a/src/plotman/archive.py
+++ b/src/plotman/archive.py
@@ -83,7 +83,7 @@ def get_archdir_freebytes(arch_cfg):
     elif arch_cfg.mode == 'local':
         df_cmd = ('df -BK | grep " %s/"' % arch_cfg.rsyncd_path )
     else:
-        raise KeyError(f'Archive mode must be "remote" or "local" ({arch_mode!r} given). Please inspect config.yaml.')
+        raise KeyError(f'Archive mode must be "remote" or "local" ({arch_cfg.mode!r} given). Please inspect plotman.yaml.')
     with subprocess.Popen(df_cmd, shell=True, stdout=subprocess.PIPE) as proc:
         for line in proc.stdout.readlines():
             fields = line.split()
@@ -105,7 +105,7 @@ def rsync_dest(arch_cfg, arch_dir):
     elif arch_cfg.mode == 'local':
         rsync_url = arch_dir
     else:
-        raise KeyError(f'Archive mode must be "remote" or "local" ("{arch_mode}" given). Please inspect config.yaml.')
+        raise KeyError(f'Archive mode must be "remote" or "local" ("{arch_cfg.mode!r}" given). Please inspect plotman.yaml.')
     return rsync_url
 
 # TODO: maybe consolidate with similar code in job.py?

From 9d5ae8a21c83abe1bb06249cc33a7b586ff4563a Mon Sep 17 00:00:00 2001
From: Johannes K Becker <jkbecker@bu.edu>
Date: Wed, 28 Apr 2021 14:41:39 -0400
Subject: [PATCH 08/58] doc fix

---
 src/plotman/resources/plotman.yaml | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/src/plotman/resources/plotman.yaml b/src/plotman/resources/plotman.yaml
index 083f0ee3..80406911 100644
--- a/src/plotman/resources/plotman.yaml
+++ b/src/plotman/resources/plotman.yaml
@@ -63,7 +63,6 @@ directories:
         # See code for details.
         archive:
                 rsyncd_module: plots
-
                 rsyncd_path: /plots
                 rsyncd_bwlimit: 80000  # Bandwidth limit in KB/s
                 rsyncd_host: myfarmer
@@ -79,7 +78,7 @@ directories:
                 # the 4 machines, or 0, 1, 0, 1.
                 #   index: 0
 
-                # Optional switch to enable local archiving.
+                # Optional switch to enable local archiving (defaults to remote if absent).
                 # Note: rsyncd_module, rsyncd_host and rsyncd_user are ignored in local mode.
                 mode: remote  # remote or local
                 # Note for local mode:

From 2d917fa32108a7bfb8a42f24e1bc61c4c78ec224 Mon Sep 17 00:00:00 2001
From: Johannes K Becker <jkbecker@bu.edu>
Date: Wed, 28 Apr 2021 14:46:37 -0400
Subject: [PATCH 09/58] whitespace, defaults

---
 src/plotman/configuration.py       | 2 +-
 src/plotman/resources/plotman.yaml | 5 +----
 2 files changed, 2 insertions(+), 5 deletions(-)

diff --git a/src/plotman/configuration.py b/src/plotman/configuration.py
index c35a8971..7692afed 100644
--- a/src/plotman/configuration.py
+++ b/src/plotman/configuration.py
@@ -46,7 +46,7 @@ class Archive:
     rsyncd_host: str
     rsyncd_user: str
     index: int = 0  # If not explicit, "index" will default to 0
-    mode: Optional[str] = None
+    mode: Optional[str] = 'remote'
 
 @dataclass
 class TmpOverrides:
diff --git a/src/plotman/resources/plotman.yaml b/src/plotman/resources/plotman.yaml
index 80406911..169f05cb 100644
--- a/src/plotman/resources/plotman.yaml
+++ b/src/plotman/resources/plotman.yaml
@@ -67,7 +67,6 @@ directories:
                 rsyncd_bwlimit: 80000  # Bandwidth limit in KB/s
                 rsyncd_host: myfarmer
                 rsyncd_user: chia
-
                 # Optional index.  If omitted or set to 0, plotman will archive
                 # to the first archive dir with free space.  If specified,
                 # plotman will skip forward up to 'index' drives (if they exist).
@@ -77,10 +76,9 @@ directories:
                 # have four plotters, you could set this to 0, 1, 2, and 3, on
                 # the 4 machines, or 0, 1, 0, 1.
                 #   index: 0
-
                 # Optional switch to enable local archiving (defaults to remote if absent).
                 # Note: rsyncd_module, rsyncd_host and rsyncd_user are ignored in local mode.
-                mode: remote  # remote or local
+                #   mode: remote  # remote or local
                 # Note for local mode:
                 # The rsyncd_path should be the directory *in which* your archive
                 # disks are mounted. For example, if your archive disks are
@@ -90,7 +88,6 @@ directories:
                 # make sure you mount all Chia archive drives to some custom
                 # directory that only contains Chia archive mountpoints.
 
-
 # Plotting scheduling parameters
 scheduling:
         # Run a job on a particular temp dir only if the number of existing jobs

From 43ead3dec41cf31c69bd8d8df9371fbdea1e032d Mon Sep 17 00:00:00 2001
From: Johannes Becker <31668498+jkbecker@users.noreply.github.com>
Date: Wed, 28 Apr 2021 17:11:08 -0400
Subject: [PATCH 10/58] Update src/plotman/configuration.py

welp, that was straightforward

Co-authored-by: Kyle Altendorf <sda@fstab.net>
---
 src/plotman/configuration.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/plotman/configuration.py b/src/plotman/configuration.py
index 7692afed..85279af5 100644
--- a/src/plotman/configuration.py
+++ b/src/plotman/configuration.py
@@ -46,7 +46,7 @@ class Archive:
     rsyncd_host: str
     rsyncd_user: str
     index: int = 0  # If not explicit, "index" will default to 0
-    mode: Optional[str] = 'remote'
+    mode: str = 'remote'
 
 @dataclass
 class TmpOverrides:

From 194dc6d2e519aa1aede9ed8701be23e22c3f4ab7 Mon Sep 17 00:00:00 2001
From: Johannes Becker <31668498+jkbecker@users.noreply.github.com>
Date: Wed, 28 Apr 2021 22:59:56 -0400
Subject: [PATCH 11/58] Update src/plotman/configuration.py

Co-authored-by: Kyle Altendorf <sda@fstab.net>
---
 src/plotman/configuration.py | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/src/plotman/configuration.py b/src/plotman/configuration.py
index 85279af5..1aa1f231 100644
--- a/src/plotman/configuration.py
+++ b/src/plotman/configuration.py
@@ -46,7 +46,12 @@ class Archive:
     rsyncd_host: str
     rsyncd_user: str
     index: int = 0  # If not explicit, "index" will default to 0
-    mode: str = 'remote'
+    mode: str = desert.field(
+        default='remote',
+        marshmallow_field=marshmallow.fields.String(
+            validate=marshmallow.validate.OneOf(choices=['remote', 'local']),
+        ),
+    )
 
 @dataclass
 class TmpOverrides:

From c5d5ad663a542b60f4a53e71adb4e0b9c679551c Mon Sep 17 00:00:00 2001
From: Johannes Becker <31668498+jkbecker@users.noreply.github.com>
Date: Wed, 28 Apr 2021 23:18:33 -0400
Subject: [PATCH 12/58] Update src/plotman/archive.py

Co-authored-by: Kyle Altendorf <sda@fstab.net>
---
 src/plotman/archive.py | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/src/plotman/archive.py b/src/plotman/archive.py
index 4b2eaae3..e896aee2 100644
--- a/src/plotman/archive.py
+++ b/src/plotman/archive.py
@@ -77,11 +77,12 @@ def compute_priority(phase, gb_free, n_plots):
 
 def get_archdir_freebytes(arch_cfg):
     archdir_freebytes = {}
+	just_df_cmd = 'df -aBK'
     if arch_cfg.mode == 'remote':
-        df_cmd = ('ssh %s@%s df -aBK | grep " %s/"' %
-            (arch_cfg.rsyncd_user, arch_cfg.rsyncd_host, arch_cfg.rsyncd_path) )
+        df_cmd = ('ssh %s@%s %s | grep " %s/"' %
+            (arch_cfg.rsyncd_user, arch_cfg.rsyncd_host, just_df_cmd, arch_cfg.rsyncd_path) )
     elif arch_cfg.mode == 'local':
-        df_cmd = ('df -BK | grep " %s/"' % arch_cfg.rsyncd_path )
+        df_cmd = '%s | grep " %s/"' % (just_df_cmd, arch_cfg.rsyncd_path)
     else:
         raise KeyError(f'Archive mode must be "remote" or "local" ({arch_cfg.mode!r} given). Please inspect plotman.yaml.')
     with subprocess.Popen(df_cmd, shell=True, stdout=subprocess.PIPE) as proc:

From 59de0c654510dd2fede672bd0698b8387e633456 Mon Sep 17 00:00:00 2001
From: Kyle Altendorf <sda@fstab.net>
Date: Thu, 29 Apr 2021 06:45:48 -0700
Subject: [PATCH 13/58] Update src/plotman/archive.py

Co-authored-by: lopesmcc <lopesmcc@gmail.com>
---
 src/plotman/archive.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/plotman/archive.py b/src/plotman/archive.py
index e896aee2..fe1df37e 100644
--- a/src/plotman/archive.py
+++ b/src/plotman/archive.py
@@ -77,7 +77,7 @@ def compute_priority(phase, gb_free, n_plots):
 
 def get_archdir_freebytes(arch_cfg):
     archdir_freebytes = {}
-	just_df_cmd = 'df -aBK'
+    just_df_cmd = 'df -aBK'
     if arch_cfg.mode == 'remote':
         df_cmd = ('ssh %s@%s %s | grep " %s/"' %
             (arch_cfg.rsyncd_user, arch_cfg.rsyncd_host, just_df_cmd, arch_cfg.rsyncd_path) )

From 11d06a718637b702559d240d01c61d11005cc3f8 Mon Sep 17 00:00:00 2001
From: Johannes K Becker <jkbecker@bu.edu>
Date: Sat, 8 May 2021 17:28:45 -0400
Subject: [PATCH 14/58] new config file format supporting local archive mode

---
 src/plotman/resources/plotman.yaml | 14 +++++---------
 1 file changed, 5 insertions(+), 9 deletions(-)

diff --git a/src/plotman/resources/plotman.yaml b/src/plotman/resources/plotman.yaml
index b642cfa3..0e4386c4 100644
--- a/src/plotman/resources/plotman.yaml
+++ b/src/plotman/resources/plotman.yaml
@@ -82,15 +82,11 @@ directories:
                 #   index: 0
                 # Optional switch to enable local archiving (defaults to remote if absent).
                 # Note: rsyncd_module, rsyncd_host and rsyncd_user are ignored in local mode.
-                #   mode: remote  # remote or local
-                # Note for local mode:
-                # The rsyncd_path should be the directory *in which* your archive
-                # disks are mounted. For example, if your archive disks are
-                # /mnt/archive1 and /mnt/archive2, the path should be /mnt.
-                # Note that *all* mounted drives within that path will be
-                # considered - so if you have other things mounted as well,
-                # make sure you mount all Chia archive drives to some custom
-                # directory that only contains Chia archive mountpoints.
+                mode: legacy  # legacy (config above) or local
+                #
+                # Local mode:
+                local:
+                        path: 'x'
 
 # Plotting scheduling parameters
 scheduling:

From 6df447e08281d3ba2a615ee0d0aba0f58698f372 Mon Sep 17 00:00:00 2001
From: Johannes K Becker <jkbecker@bu.edu>
Date: Sat, 8 May 2021 18:28:43 -0400
Subject: [PATCH 15/58] prepare for legacy vs extendable new configs

---
 src/plotman/archive.py       | 44 +++++++++++++++++++++---------------
 src/plotman/configuration.py | 16 ++++++++++++-
 2 files changed, 41 insertions(+), 19 deletions(-)

diff --git a/src/plotman/archive.py b/src/plotman/archive.py
index ae072342..cca2dc0b 100644
--- a/src/plotman/archive.py
+++ b/src/plotman/archive.py
@@ -86,13 +86,12 @@ def compute_priority(phase, gb_free, n_plots):
 
 def get_archdir_freebytes(arch_cfg):
     archdir_freebytes = {}
-    if arch_cfg.mode == 'remote':
+    if arch_cfg.mode == 'legacy':
         df_cmd = ('ssh %s@%s df -aBK | grep " %s/"' %
             (arch_cfg.rsyncd_user, arch_cfg.rsyncd_host, arch_cfg.rsyncd_path) )
-    elif arch_cfg.mode == 'local':
-        df_cmd = ('df -BK | grep " %s/"' % arch_cfg.rsyncd_path )
     else:
-        raise KeyError(f'Archive mode must be "remote" or "local" ({arch_cfg.mode!r} given). Please inspect plotman.yaml.')
+        arch_cfg_custom = getattr(arch_cfg, arch_cfg.mode)
+        df_cmd = (arch_cfg_custom.df_cmd.format(arch_cfg_custom.path))
     with subprocess.Popen(df_cmd, shell=True, stdout=subprocess.PIPE) as proc:
         for line in proc.stdout.readlines():
             fields = line.split()
@@ -105,17 +104,15 @@ def get_archdir_freebytes(arch_cfg):
     return archdir_freebytes
 
 def rsync_dest(arch_cfg, arch_dir):
-    if arch_cfg.mode == 'remote':
+    if arch_cfg.mode == 'legacy':
         rsync_path = arch_dir.replace(arch_cfg.rsyncd_path, arch_cfg.rsyncd_module)
         if rsync_path.startswith('/'):
             rsync_path = rsync_path[1:]  # Avoid dup slashes.  TODO use path join?
-        rsync_url = 'rsync://%s@%s:12000/%s' % (
+        return 'rsync://%s@%s:12000/%s' % (
                 arch_cfg.rsyncd_user, arch_cfg.rsyncd_host, rsync_path)
-    elif arch_cfg.mode == 'local':
-        rsync_url = arch_dir
     else:
-        raise KeyError(f'Archive mode must be "remote" or "local" ("{arch_cfg.mode!r}" given). Please inspect plotman.yaml.')
-    return rsync_url
+        arch_cfg_custom = getattr(arch_cfg, arch_cfg.mode)
+        return arch_cfg_custom.path
 
 # TODO: maybe consolidate with similar code in job.py?
 def get_running_archive_jobs(arch_cfg):
@@ -125,7 +122,11 @@ def get_running_archive_jobs(arch_cfg):
     dest = rsync_dest(arch_cfg, '/')
     for proc in psutil.process_iter(['pid', 'name']):
         with contextlib.suppress(psutil.NoSuchProcess):
-            if proc.name() == 'rsync':
+            if arch_cfg.mode == 'legacy':
+                proc_name = 'rsync' 
+            else:
+                proc_name = getattr(arch_cfg, arch_cfg.mode).archive_tool
+            if proc.name() == proc_name:
                 args = proc.cmdline()
                 for arg in args:
                     if arg.startswith(dest):
@@ -166,7 +167,7 @@ def archive(dir_cfg, all_jobs):
     archdir_freebytes = get_archdir_freebytes(dir_cfg.archive)
     if not archdir_freebytes:
         return(False, 'No free archive dirs found.')
-    
+
     archdir = ''
     available = [(d, space) for (d, space) in archdir_freebytes.items() if 
                  space > 1.2 * plot_util.get_k32_plotsize()]
@@ -178,10 +179,17 @@ def archive(dir_cfg, all_jobs):
         return(False, 'No archive directories found with enough free space')
     
     msg = 'Found %s with ~%d GB free' % (archdir, freespace / plot_util.GB)
-
-    bwlimit = dir_cfg.archive.rsyncd_bwlimit
-    throttle_arg = ('--bwlimit=%d' % bwlimit) if bwlimit else ''
-    cmd = ('rsync %s --no-compress --remove-source-files -P %s %s' %
-            (throttle_arg, chosen_plot, rsync_dest(dir_cfg.archive, archdir)))
-
+    if dir_cfg.archive.mode == 'legacy':
+        bwlimit = dir_cfg.archive.rsyncd_bwlimit
+        throttle_arg = ('--bwlimit=%d' % bwlimit) if bwlimit else ''
+        cmd = ('rsync %s --no-compress --remove-source-files -P %s %s' %
+                (throttle_arg, chosen_plot, rsync_dest(dir_cfg.archive, archdir)))
+    else:
+        arch_cfg_custom = getattr(dir_cfg.archive, dir_cfg.archive.mode)
+        cmd = arch_cfg_custom.archive_cmd.format(
+            arch_cfg_custom.archive_tool,
+            arch_cfg_custom.parameters,
+            chosen_plot,
+            rsync_dest(dir_cfg.archive, archdir)
+        )
     return (True, cmd)
diff --git a/src/plotman/configuration.py b/src/plotman/configuration.py
index 84c023e9..30c4c529 100644
--- a/src/plotman/configuration.py
+++ b/src/plotman/configuration.py
@@ -48,6 +48,14 @@ def get_validated_configs(config_text, config_path):
 
 # Data models used to deserializing/formatting plotman.yaml files.
 
+@attr.frozen
+class ArchiveLocal:
+    path: str
+    df_cmd: str = 'df -BK | grep " {}/"'
+    archive_tool: str = 'rsync'
+    archive_cmd: str = '{} {} {} {}'
+    parameters: str = '--bwlimit=80000 --no-compress --remove-source-files -P'
+
 @attr.frozen
 class Archive:
     rsyncd_module: str
@@ -56,7 +64,13 @@ class Archive:
     rsyncd_host: str
     rsyncd_user: str
     index: int = 0  # If not explicit, "index" will default to 0
-    mode: Optional[str] = 'remote'
+    mode: str = desert.ib(
+        default='legacy',
+        marshmallow_field=marshmallow.fields.String(
+            validate=marshmallow.validate.OneOf(choices=['legacy', 'local'])
+        ),
+    )
+    local: Optional[ArchiveLocal] = None
 
 @attr.frozen
 class TmpOverrides:

From 4c92cb1dc34a18c3e0803902e25fed1f497e4ec9 Mon Sep 17 00:00:00 2001
From: Johannes K Becker <jkbecker@bu.edu>
Date: Sat, 8 May 2021 18:32:15 -0400
Subject: [PATCH 16/58] better example value for local mode

---
 src/plotman/resources/plotman.yaml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/plotman/resources/plotman.yaml b/src/plotman/resources/plotman.yaml
index 0e4386c4..1c932df6 100644
--- a/src/plotman/resources/plotman.yaml
+++ b/src/plotman/resources/plotman.yaml
@@ -85,8 +85,8 @@ directories:
                 mode: legacy  # legacy (config above) or local
                 #
                 # Local mode:
-                local:
-                        path: 'x'
+                # local:
+                #         path: '/farm'
 
 # Plotting scheduling parameters
 scheduling:

From 57ebfe516236909972d89bcdff454c64a2007b48 Mon Sep 17 00:00:00 2001
From: Johannes K Becker <jkbecker@bu.edu>
Date: Sat, 8 May 2021 19:09:31 -0400
Subject: [PATCH 17/58] some archive tests

---
 src/plotman/_tests/archive_test.py | 69 ++++++++++++++++++++++++++++--
 1 file changed, 66 insertions(+), 3 deletions(-)

diff --git a/src/plotman/_tests/archive_test.py b/src/plotman/_tests/archive_test.py
index 9caa2abe..aa1ef88b 100755
--- a/src/plotman/_tests/archive_test.py
+++ b/src/plotman/_tests/archive_test.py
@@ -1,13 +1,13 @@
 from plotman import archive, configuration, manager
+import pytest
 
 
 def test_compute_priority():
     assert (archive.compute_priority( (3, 1), 1000, 10) >
             archive.compute_priority( (3, 6), 1000, 10) )
 
-def test_rsync_dest():
-    arch_dir = '/plotdir/012'
-    arch_cfg = configuration.Archive(
+def _archive_legacy():
+    return configuration.Archive(
         rsyncd_module='plots_mod',
         rsyncd_path='/plotdir',
         rsyncd_host='thehostname',
@@ -15,6 +15,10 @@ def test_rsync_dest():
         rsyncd_bwlimit=80000
     )
 
+def test_rsync_dest():
+    arch_dir = '/plotdir/012'
+    arch_cfg = _archive_legacy()
+
     # Normal usage
     assert ('rsync://theusername@thehostname:12000/plots_mod/012' ==
             archive.rsync_dest(arch_cfg, arch_dir))
@@ -23,3 +27,62 @@ def test_rsync_dest():
     # for matching jobs.
     assert ('rsync://theusername@thehostname:12000/' ==
             archive.rsync_dest(arch_cfg, '/'))
+
+
+def test_archive_legacy_default():
+    arch_cfg = _archive_legacy()
+    assert arch_cfg.mode == 'legacy'
+
+def _archive_badmode():
+    return configuration.Archive(
+        rsyncd_module='plots_mod',
+        rsyncd_path='/plotdir',
+        rsyncd_host='thehostname',
+        rsyncd_user='theusername',
+        rsyncd_bwlimit=80000,
+        mode='thismodedoesntexist'
+    )
+
+def test_archive_bad_mode():
+    arch_cfg = _archive_badmode()
+    assert arch_cfg.mode == 'thismodedoesntexist'
+
+
+def test_archive_bad_mode_load():
+    arch_cfg = _archive_badmode()
+    with pytest.raises(AttributeError):
+        getattr(arch_cfg, arch_cfg.mode)
+
+
+def _archive_emptymode():
+    return configuration.Archive(
+        rsyncd_module='plots_mod',
+        rsyncd_path='/plotdir',
+        rsyncd_host='thehostname',
+        rsyncd_user='theusername',
+        rsyncd_bwlimit=80000,
+        mode='local'
+    )
+
+def test_archive_local_mode_absent():
+    arch_cfg = _archive_emptymode()
+    arch_cfg_local = getattr(arch_cfg, arch_cfg.mode)
+    assert not arch_cfg_local
+
+def _archive_localmode():
+    return configuration.Archive(
+        rsyncd_module='plots_mod',
+        rsyncd_path='/plotdir',
+        rsyncd_host='thehostname',
+        rsyncd_user='theusername',
+        rsyncd_bwlimit=80000,
+        mode='local',
+        local=configuration.ArchiveLocal(
+            path='/farm'
+        )
+    )
+
+def test_archive_local_mode_load():
+    arch_cfg = _archive_localmode()
+    arch_cfg_local = getattr(arch_cfg, arch_cfg.mode)
+    assert isinstance(arch_cfg_local, configuration.ArchiveLocal)

From 160b3f905e1fce6a036bd35f51959afb8a0d1476 Mon Sep 17 00:00:00 2001
From: Johannes K Becker <jkbecker@bu.edu>
Date: Sat, 8 May 2021 19:18:42 -0400
Subject: [PATCH 18/58] rsync_dest -> arch_dest refactoring

---
 src/plotman/_tests/archive_test.py | 6 +++---
 src/plotman/archive.py             | 8 ++++----
 2 files changed, 7 insertions(+), 7 deletions(-)

diff --git a/src/plotman/_tests/archive_test.py b/src/plotman/_tests/archive_test.py
index aa1ef88b..a1d60b8e 100755
--- a/src/plotman/_tests/archive_test.py
+++ b/src/plotman/_tests/archive_test.py
@@ -15,18 +15,18 @@ def _archive_legacy():
         rsyncd_bwlimit=80000
     )
 
-def test_rsync_dest():
+def test_arch_dest():
     arch_dir = '/plotdir/012'
     arch_cfg = _archive_legacy()
 
     # Normal usage
     assert ('rsync://theusername@thehostname:12000/plots_mod/012' ==
-            archive.rsync_dest(arch_cfg, arch_dir))
+            archive.arch_dest(arch_cfg, arch_dir))
 
     # Usage for constructing just the prefix, for scanning process tables
     # for matching jobs.
     assert ('rsync://theusername@thehostname:12000/' ==
-            archive.rsync_dest(arch_cfg, '/'))
+            archive.arch_dest(arch_cfg, '/'))
 
 
 def test_archive_legacy_default():
diff --git a/src/plotman/archive.py b/src/plotman/archive.py
index cca2dc0b..3921d9b0 100644
--- a/src/plotman/archive.py
+++ b/src/plotman/archive.py
@@ -103,7 +103,7 @@ def get_archdir_freebytes(arch_cfg):
             archdir_freebytes[archdir] = freebytes
     return archdir_freebytes
 
-def rsync_dest(arch_cfg, arch_dir):
+def arch_dest(arch_cfg, arch_dir):
     if arch_cfg.mode == 'legacy':
         rsync_path = arch_dir.replace(arch_cfg.rsyncd_path, arch_cfg.rsyncd_module)
         if rsync_path.startswith('/'):
@@ -119,7 +119,7 @@ def get_running_archive_jobs(arch_cfg):
     '''Look for running rsync jobs that seem to match the pattern we use for archiving
        them.  Return a list of PIDs of matching jobs.'''
     jobs = []
-    dest = rsync_dest(arch_cfg, '/')
+    dest = arch_dest(arch_cfg, '/')
     for proc in psutil.process_iter(['pid', 'name']):
         with contextlib.suppress(psutil.NoSuchProcess):
             if arch_cfg.mode == 'legacy':
@@ -183,13 +183,13 @@ def archive(dir_cfg, all_jobs):
         bwlimit = dir_cfg.archive.rsyncd_bwlimit
         throttle_arg = ('--bwlimit=%d' % bwlimit) if bwlimit else ''
         cmd = ('rsync %s --no-compress --remove-source-files -P %s %s' %
-                (throttle_arg, chosen_plot, rsync_dest(dir_cfg.archive, archdir)))
+                (throttle_arg, chosen_plot, arch_dest(dir_cfg.archive, archdir)))
     else:
         arch_cfg_custom = getattr(dir_cfg.archive, dir_cfg.archive.mode)
         cmd = arch_cfg_custom.archive_cmd.format(
             arch_cfg_custom.archive_tool,
             arch_cfg_custom.parameters,
             chosen_plot,
-            rsync_dest(dir_cfg.archive, archdir)
+            arch_dest(dir_cfg.archive, archdir)
         )
     return (True, cmd)

From 12a82143751901ca0d11c800a80b4779b6cc00ad Mon Sep 17 00:00:00 2001
From: Kyle Altendorf <sda@fstab.net>
Date: Tue, 18 May 2021 23:55:17 -0400
Subject: [PATCH 19/58] custom archive

---
 src/plotman/archive.py       | 83 ++++++++++++++++++++++++------------
 src/plotman/configuration.py | 28 ++++++------
 2 files changed, 72 insertions(+), 39 deletions(-)

diff --git a/src/plotman/archive.py b/src/plotman/archive.py
index 3660d5e6..01b95076 100644
--- a/src/plotman/archive.py
+++ b/src/plotman/archive.py
@@ -32,9 +32,10 @@ def spawn_archive_process(dir_cfg, all_jobs):
         if not should_start:
             archiving_status = status_or_cmd
         else:
-            cmd = status_or_cmd
+            args = status_or_cmd
+            cmd = args['args']
             # TODO: do something useful with output instead of DEVNULL
-            p = subprocess.Popen(cmd,
+            p = subprocess.Popen(**args,
                     shell=True,
                     stdout=subprocess.DEVNULL,
                     stderr=subprocess.STDOUT,
@@ -90,17 +91,34 @@ def get_archdir_freebytes(arch_cfg):
     if arch_cfg.mode == 'legacy':
         df_cmd = ('ssh %s@%s df -aBK | grep " %s/"' %
             (arch_cfg.rsyncd_user, arch_cfg.rsyncd_host, posixpath.normpath(arch_cfg.rsyncd_path)) )
+        with subprocess.Popen(df_cmd, shell=True, stdout=subprocess.PIPE) as proc:
+            for line in proc.stdout.readlines():
+                fields = line.split()
+                if fields[3] == b'-':
+                    # not actually mounted
+                    continue
+                freebytes = int(fields[3][:-1]) * 1024  # Strip the final 'K'
+                archdir = (fields[5]).decode('utf-8')
+                archdir_freebytes[archdir] = freebytes
     else:
-        arch_cfg_custom = getattr(arch_cfg, arch_cfg.mode)
-        df_cmd = (arch_cfg_custom.df_cmd.format(arch_cfg_custom.path))
-    with subprocess.Popen(df_cmd, shell=True, stdout=subprocess.PIPE) as proc:
-        for line in proc.stdout.readlines():
-            fields = line.split()
-            if fields[3] == b'-':
-                # not actually mounted
-                continue
-            freebytes = int(fields[3][:-1]) * 1024  # Strip the final 'K'
-            archdir = (fields[5]).decode('utf-8')
+        extra_env = {
+            'path': arch_cfg.custom.path,
+        }
+        completed_process = subprocess.run(
+            [arch_cfg.custom.shell],
+            encoding='utf-8',
+            env={**os.environ, **extra_env},
+            input=arch_cfg.custom.disk_space,
+            stdout=subprocess.PIPE,
+            stderr=subprocess.PIPE,
+        )
+
+        print(completed_process.stdout)
+        print(completed_process.stderr)
+
+        for line in completed_process.stdout.splitlines():
+            archdir, space = line.split(':')
+            freebytes = int(space[:-1]) * 1024
             archdir_freebytes[archdir] = freebytes
     return archdir_freebytes
 
@@ -109,11 +127,13 @@ def arch_dest(arch_cfg, arch_dir):
         rsync_path = arch_dir.replace(arch_cfg.rsyncd_path, arch_cfg.rsyncd_module)
         if rsync_path.startswith('/'):
             rsync_path = rsync_path[1:]  # Avoid dup slashes.  TODO use path join?
-        return 'rsync://%s@%s:12000/%s' % (
+        rsync_url = 'rsync://%s@%s:12000/%s' % (
                 arch_cfg.rsyncd_user, arch_cfg.rsyncd_host, rsync_path)
+        return rsync_url
     else:
-        arch_cfg_custom = getattr(arch_cfg, arch_cfg.mode)
-        return arch_cfg_custom.path
+        # TODO: auaughhghh, don't do this
+        os.environ['path'] = arch_cfg.custom.path
+        return os.path.expandvars(arch_cfg.custom.transfer_detector)
 
 # TODO: maybe consolidate with similar code in job.py?
 def get_running_archive_jobs(arch_cfg):
@@ -126,7 +146,7 @@ def get_running_archive_jobs(arch_cfg):
             if arch_cfg.mode == 'legacy':
                 proc_name = 'rsync'
             else:
-                proc_name = getattr(arch_cfg, arch_cfg.mode).archive_tool
+                proc_name = arch_cfg.custom.process_name
             if proc.name() == proc_name:
                 args = proc.cmdline()
                 for arg in args:
@@ -156,8 +176,8 @@ def archive(dir_cfg, all_jobs):
             best_priority = priority
             chosen_plot = dir_plots[0]
 
-    if not chosen_plot:
-        return (False, 'No plots found')
+    # if not chosen_plot:
+    #     return (False, 'No plots found')
 
     # TODO: sanity check that archive machine is available
     # TODO: filter drives mounted RO
@@ -166,6 +186,8 @@ def archive(dir_cfg, all_jobs):
     # Pick first archive dir with sufficient space
     #
     archdir_freebytes = get_archdir_freebytes(dir_cfg.archive)
+    print(dict(sorted(archdir_freebytes.items())))
+    1/0
     if not archdir_freebytes:
         return(False, 'No free archive dirs found.')
 
@@ -180,17 +202,24 @@ def archive(dir_cfg, all_jobs):
         return(False, 'No archive directories found with enough free space')
 
     msg = 'Found %s with ~%d GB free' % (archdir, freespace / plot_util.GB)
+    dest = arch_dest(dir_cfg.archive, archdir)
     if dir_cfg.archive.mode == 'legacy':
         bwlimit = dir_cfg.archive.rsyncd_bwlimit
         throttle_arg = ('--bwlimit=%d' % bwlimit) if bwlimit else ''
         cmd = ('rsync %s --compress-level=0 --remove-source-files -P %s %s' %
-                (throttle_arg, chosen_plot, arch_dest(dir_cfg.archive, archdir)))
+                (throttle_arg, chosen_plot, dest))
+        subprocess_arguments = {'args': cmd}
     else:
-        arch_cfg_custom = getattr(dir_cfg.archive, dir_cfg.archive.mode)
-        cmd = arch_cfg_custom.archive_cmd.format(
-            arch_cfg_custom.archive_tool,
-            arch_cfg_custom.parameters,
-            chosen_plot,
-            arch_dest(dir_cfg.archive, archdir)
-        )
-    return (True, cmd)
+        custom = dir_cfg.archive.custom
+        env = {
+            'process_name': custom.process_name,
+            'source': chosen_plot,
+            'path': custom.path,
+            'destination': dest,
+        }
+        subprocess_arguments = {
+            'args': custom.transfer,
+            # 'input': custom.transfer,
+            'env': {**os.environ, **env}
+        }
+    return (True, subprocess_arguments)
diff --git a/src/plotman/configuration.py b/src/plotman/configuration.py
index f6b15fae..951c84a3 100644
--- a/src/plotman/configuration.py
+++ b/src/plotman/configuration.py
@@ -49,28 +49,32 @@ def get_validated_configs(config_text, config_path):
 # Data models used to deserializing/formatting plotman.yaml files.
 
 @attr.frozen
-class ArchiveLocal:
+class CustomArchive:
     path: str
-    df_cmd: str = 'df -BK | grep " {}/"'
-    archive_tool: str = 'rsync'
-    archive_cmd: str = '{} {} {} {}'
-    parameters: str = '--bwlimit=80000 --no-compress --remove-source-files -P'
+    # TODO: fully support or remove
+    shell: str = 'bash'
+    # disk_space: str = '''ssh chia@chia "df -BK | grep \" ${path}/\" | awk '{ print \$6 \\\":\\\" \$4 }'"'''
+    disk_space: str = """df -BK | grep " ${path}/" | awk '{gsub(/K\$/,"",$4); print $6 ":" $4*1024 }'"""
+    process_name: str = 'rsync'
+    transfer: str = '${process_name} --bwlimit=80000 --no-compress --remove-source-files -P "${source}" "${path}/${destination}"'
+    transfer_detector: str = '${path}/'
 
 @attr.frozen
 class Archive:
-    rsyncd_module: str
-    rsyncd_path: str
-    rsyncd_bwlimit: int
-    rsyncd_host: str
-    rsyncd_user: str
+    rsyncd_module: str = None
+    rsyncd_path: str = None
+    rsyncd_bwlimit: int = None
+    rsyncd_host: str = None
+    rsyncd_user: str = None
     index: int = 0  # If not explicit, "index" will default to 0
     mode: str = desert.ib(
         default='legacy',
         marshmallow_field=marshmallow.fields.String(
-            validate=marshmallow.validate.OneOf(choices=['legacy', 'local'])
+            validate=marshmallow.validate.OneOf(choices=['legacy', 'custom'])
         ),
     )
-    local: Optional[ArchiveLocal] = None
+    custom: Optional[CustomArchive] = None
+    # custom: CustomArchive = CustomArchive(path='/farm/sites')
 
 @attr.frozen
 class TmpOverrides:

From c3737eeb9824f4f07b03cfb974fc77d56e1e7990 Mon Sep 17 00:00:00 2001
From: Kyle Altendorf <sda@fstab.net>
Date: Tue, 18 May 2021 23:58:19 -0400
Subject: [PATCH 20/58] less debuggy

---
 src/plotman/archive.py | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/src/plotman/archive.py b/src/plotman/archive.py
index 01b95076..cf31b93d 100644
--- a/src/plotman/archive.py
+++ b/src/plotman/archive.py
@@ -176,8 +176,8 @@ def archive(dir_cfg, all_jobs):
             best_priority = priority
             chosen_plot = dir_plots[0]
 
-    # if not chosen_plot:
-    #     return (False, 'No plots found')
+    if not chosen_plot:
+        return (False, 'No plots found')
 
     # TODO: sanity check that archive machine is available
     # TODO: filter drives mounted RO
@@ -186,8 +186,8 @@ def archive(dir_cfg, all_jobs):
     # Pick first archive dir with sufficient space
     #
     archdir_freebytes = get_archdir_freebytes(dir_cfg.archive)
-    print(dict(sorted(archdir_freebytes.items())))
-    1/0
+    # print(dict(sorted(archdir_freebytes.items())))
+    # 1/0
     if not archdir_freebytes:
         return(False, 'No free archive dirs found.')
 

From 8978b4d04903e8e45d5220c744baed56cc03a68d Mon Sep 17 00:00:00 2001
From: Kyle Altendorf <sda@fstab.net>
Date: Wed, 19 May 2021 04:43:04 +0000
Subject: [PATCH 21/58] locally detects free space and transfers

---
 src/plotman/archive.py       | 16 +++++++++-------
 src/plotman/configuration.py |  2 +-
 2 files changed, 10 insertions(+), 8 deletions(-)

diff --git a/src/plotman/archive.py b/src/plotman/archive.py
index cf31b93d..5a0c3b74 100644
--- a/src/plotman/archive.py
+++ b/src/plotman/archive.py
@@ -113,12 +113,12 @@ def get_archdir_freebytes(arch_cfg):
             stderr=subprocess.PIPE,
         )
 
-        print(completed_process.stdout)
-        print(completed_process.stderr)
+        #print(completed_process.stdout)
+        #print(completed_process.stderr)
 
         for line in completed_process.stdout.splitlines():
             archdir, space = line.split(':')
-            freebytes = int(space[:-1]) * 1024
+            freebytes = int(space)
             archdir_freebytes[archdir] = freebytes
     return archdir_freebytes
 
@@ -131,21 +131,23 @@ def arch_dest(arch_cfg, arch_dir):
                 arch_cfg.rsyncd_user, arch_cfg.rsyncd_host, rsync_path)
         return rsync_url
     else:
-        # TODO: auaughhghh, don't do this
-        os.environ['path'] = arch_cfg.custom.path
-        return os.path.expandvars(arch_cfg.custom.transfer_detector)
+        return os.path.relpath(arch_dir, arch_cfg.custom.path)
+        #return os.path.expandvars(arch_cfg.custom.transfer_detector)
 
 # TODO: maybe consolidate with similar code in job.py?
 def get_running_archive_jobs(arch_cfg):
     '''Look for running rsync jobs that seem to match the pattern we use for archiving
        them.  Return a list of PIDs of matching jobs.'''
     jobs = []
-    dest = arch_dest(arch_cfg, '/')
     for proc in psutil.process_iter(['pid', 'name']):
         with contextlib.suppress(psutil.NoSuchProcess):
             if arch_cfg.mode == 'legacy':
+                dest = arch_dest(arch_cfg, '/')
                 proc_name = 'rsync'
             else:
+                # TODO: auaughhghh, don't do this
+                os.environ['path'] = arch_cfg.custom.path
+                dest = os.path.expandvars(arch_cfg.custom.transfer_detector)
                 proc_name = arch_cfg.custom.process_name
             if proc.name() == proc_name:
                 args = proc.cmdline()
diff --git a/src/plotman/configuration.py b/src/plotman/configuration.py
index 951c84a3..35350f47 100644
--- a/src/plotman/configuration.py
+++ b/src/plotman/configuration.py
@@ -56,7 +56,7 @@ class CustomArchive:
     # disk_space: str = '''ssh chia@chia "df -BK | grep \" ${path}/\" | awk '{ print \$6 \\\":\\\" \$4 }'"'''
     disk_space: str = """df -BK | grep " ${path}/" | awk '{gsub(/K\$/,"",$4); print $6 ":" $4*1024 }'"""
     process_name: str = 'rsync'
-    transfer: str = '${process_name} --bwlimit=80000 --no-compress --remove-source-files -P "${source}" "${path}/${destination}"'
+    transfer: str = '${process_name} --no-compress --remove-source-files -P "${source}" "${path}/${destination}"'
     transfer_detector: str = '${path}/'
 
 @attr.frozen

From 82b36843ea810ca9ed52d5ce9f3a9fdc68ef04a2 Mon Sep 17 00:00:00 2001
From: Kyle Altendorf <sda@fstab.net>
Date: Wed, 19 May 2021 04:53:40 +0000
Subject: [PATCH 22/58] rsync argument tweaks

---
 src/plotman/configuration.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/plotman/configuration.py b/src/plotman/configuration.py
index 35350f47..2f099344 100644
--- a/src/plotman/configuration.py
+++ b/src/plotman/configuration.py
@@ -56,7 +56,7 @@ class CustomArchive:
     # disk_space: str = '''ssh chia@chia "df -BK | grep \" ${path}/\" | awk '{ print \$6 \\\":\\\" \$4 }'"'''
     disk_space: str = """df -BK | grep " ${path}/" | awk '{gsub(/K\$/,"",$4); print $6 ":" $4*1024 }'"""
     process_name: str = 'rsync'
-    transfer: str = '${process_name} --no-compress --remove-source-files -P "${source}" "${path}/${destination}"'
+    transfer: str = '${process_name} --skip-compress plot --remove-source-files --inplace "${source}" "${path}/${destination}"'
     transfer_detector: str = '${path}/'
 
 @attr.frozen

From 33e276068c414a0499507c5e2c647cc30213fc69 Mon Sep 17 00:00:00 2001
From: Kyle Altendorf <sda@fstab.net>
Date: Thu, 20 May 2021 21:39:27 -0400
Subject: [PATCH 23/58] skip malformed df-alike output

---
 src/plotman/archive.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/src/plotman/archive.py b/src/plotman/archive.py
index 5a0c3b74..2af969eb 100644
--- a/src/plotman/archive.py
+++ b/src/plotman/archive.py
@@ -117,7 +117,10 @@ def get_archdir_freebytes(arch_cfg):
         #print(completed_process.stderr)
 
         for line in completed_process.stdout.splitlines():
-            archdir, space = line.split(':')
+            split = line.split(':')
+            if len(split) != 2:
+                continue
+            archdir, space = split
             freebytes = int(space)
             archdir_freebytes[archdir] = freebytes
     return archdir_freebytes

From 73fe689788d7ed4585e0f98e826fe7061d5e434f Mon Sep 17 00:00:00 2001
From: Kyle Altendorf <sda@fstab.net>
Date: Fri, 21 May 2021 22:54:54 -0400
Subject: [PATCH 24/58] switch to scripts

---
 src/plotman/archive.py       | 55 ++++++++++++++++--------------------
 src/plotman/configuration.py | 26 +++++++++++++----
 src/plotman/interactive.py   | 36 +++++++++++++++++++++++
 3 files changed, 81 insertions(+), 36 deletions(-)

diff --git a/src/plotman/archive.py b/src/plotman/archive.py
index 2af969eb..0fd48d4d 100644
--- a/src/plotman/archive.py
+++ b/src/plotman/archive.py
@@ -33,14 +33,13 @@ def spawn_archive_process(dir_cfg, all_jobs):
             archiving_status = status_or_cmd
         else:
             args = status_or_cmd
-            cmd = args['args']
             # TODO: do something useful with output instead of DEVNULL
             p = subprocess.Popen(**args,
                     shell=True,
                     stdout=subprocess.DEVNULL,
                     stderr=subprocess.STDOUT,
                     start_new_session=True)
-            log_message = 'Starting archive: ' + cmd
+            log_message = 'Starting archive: ' + args['args']
             # At least for now it seems that even if we get a new running
             # archive jobs list it doesn't contain the new rsync process.
             # My guess is that this is because the bash in the middle due to
@@ -101,21 +100,14 @@ def get_archdir_freebytes(arch_cfg):
                 archdir = (fields[5]).decode('utf-8')
                 archdir_freebytes[archdir] = freebytes
     else:
-        extra_env = {
-            'path': arch_cfg.custom.path,
-        }
         completed_process = subprocess.run(
-            [arch_cfg.custom.shell],
+            [arch_cfg.custom.disk_space_path],
             encoding='utf-8',
-            env={**os.environ, **extra_env},
-            input=arch_cfg.custom.disk_space,
+            env={**os.environ, 'path': arch_cfg.custom.path},
             stdout=subprocess.PIPE,
             stderr=subprocess.PIPE,
         )
 
-        #print(completed_process.stdout)
-        #print(completed_process.stderr)
-
         for line in completed_process.stdout.splitlines():
             split = line.split(':')
             if len(split) != 2:
@@ -135,28 +127,32 @@ def arch_dest(arch_cfg, arch_dir):
         return rsync_url
     else:
         return os.path.relpath(arch_dir, arch_cfg.custom.path)
-        #return os.path.expandvars(arch_cfg.custom.transfer_detector)
 
 # TODO: maybe consolidate with similar code in job.py?
 def get_running_archive_jobs(arch_cfg):
     '''Look for running rsync jobs that seem to match the pattern we use for archiving
        them.  Return a list of PIDs of matching jobs.'''
     jobs = []
-    for proc in psutil.process_iter(['pid', 'name']):
+    for proc in psutil.process_iter():
         with contextlib.suppress(psutil.NoSuchProcess):
-            if arch_cfg.mode == 'legacy':
-                dest = arch_dest(arch_cfg, '/')
-                proc_name = 'rsync'
-            else:
-                # TODO: auaughhghh, don't do this
-                os.environ['path'] = arch_cfg.custom.path
-                dest = os.path.expandvars(arch_cfg.custom.transfer_detector)
-                proc_name = arch_cfg.custom.process_name
-            if proc.name() == proc_name:
-                args = proc.cmdline()
-                for arg in args:
-                    if arg.startswith(dest):
-                        jobs.append(proc.pid)
+            with proc.oneshot():
+                if arch_cfg.mode == 'legacy':
+                    dest = arch_dest(arch_cfg, '/')
+                    proc_name = 'rsync'
+                else:
+                    # TODO: make a context manager
+                    try:
+                        os.environ['path'] = arch_cfg.custom.path
+                        dest = os.path.expandvars(arch_cfg.custom.transfer_process_argument_prefix)
+                    finally:
+                        # TODO: yup, this'll delete, not restore
+                        del os.environ['path']
+                    proc_name = arch_cfg.custom.transfer_process_name
+                if proc.name() == proc_name:
+                    args = proc.cmdline()
+                    for arg in args:
+                        if arg.startswith(dest):
+                            jobs.append(proc.pid)
     return jobs
 
 def archive(dir_cfg, all_jobs):
@@ -191,8 +187,6 @@ def archive(dir_cfg, all_jobs):
     # Pick first archive dir with sufficient space
     #
     archdir_freebytes = get_archdir_freebytes(dir_cfg.archive)
-    # print(dict(sorted(archdir_freebytes.items())))
-    # 1/0
     if not archdir_freebytes:
         return(False, 'No free archive dirs found.')
 
@@ -217,13 +211,14 @@ def archive(dir_cfg, all_jobs):
     else:
         custom = dir_cfg.archive.custom
         env = {
-            'process_name': custom.process_name,
+            'process_name': custom.transfer_process_name,
             'source': chosen_plot,
             'path': custom.path,
             'destination': dest,
         }
         subprocess_arguments = {
-            'args': custom.transfer,
+            'args': custom.transfer_path,
+            # 'args': custom.shell,
             # 'input': custom.transfer,
             'env': {**os.environ, **env}
         }
diff --git a/src/plotman/configuration.py b/src/plotman/configuration.py
index 2f099344..d1b90a88 100644
--- a/src/plotman/configuration.py
+++ b/src/plotman/configuration.py
@@ -1,4 +1,5 @@
 import contextlib
+import textwrap
 from typing import Dict, List, Optional
 
 import appdirs
@@ -48,16 +49,29 @@ def get_validated_configs(config_text, config_path):
 
 # Data models used to deserializing/formatting plotman.yaml files.
 
-@attr.frozen
+# TODO: bah, mutable?  bah.
+@attr.mutable
 class CustomArchive:
     path: str
     # TODO: fully support or remove
-    shell: str = 'bash'
+    # shell: str = 'bash'
     # disk_space: str = '''ssh chia@chia "df -BK | grep \" ${path}/\" | awk '{ print \$6 \\\":\\\" \$4 }'"'''
-    disk_space: str = """df -BK | grep " ${path}/" | awk '{gsub(/K\$/,"",$4); print $6 ":" $4*1024 }'"""
-    process_name: str = 'rsync'
-    transfer: str = '${process_name} --skip-compress plot --remove-source-files --inplace "${source}" "${path}/${destination}"'
-    transfer_detector: str = '${path}/'
+    disk_space_path: Optional[str] = None
+    disk_space_script: str = textwrap.dedent(
+        """
+        #!/bin/bash
+        df -BK | grep " ${path}/" | awk '{gsub(/K\$/,"",$4); print $6 ":" $4*1024 }'
+        """
+    )
+    transfer_path: Optional[str] = None
+    transfer_script: str = textwrap.dedent(
+        """
+        #!/bin/bash
+        ${process_name} --skip-compress plot --remove-source-files --inplace "${source}" "${path}/${destination}"
+        """
+    )
+    transfer_process_name: str = 'rsync'
+    transfer_process_argument_prefix: str = '${path}/'
 
 @attr.frozen
 class Archive:
diff --git a/src/plotman/interactive.py b/src/plotman/interactive.py
index c60d7236..ab73e312 100644
--- a/src/plotman/interactive.py
+++ b/src/plotman/interactive.py
@@ -3,7 +3,9 @@
 import locale
 import math
 import os
+import stat
 import subprocess
+import tempfile
 
 from plotman import archive, configuration, manager, reporting
 from plotman.job import Job
@@ -69,6 +71,40 @@ def curses_main(stdscr):
     config_text = configuration.read_configuration_text(config_path)
     cfg = configuration.get_validated_configs(config_text, config_path)
 
+    # TODO: context manager?
+    if cfg.directories.archive.mode == 'custom':
+        custom = cfg.directories.archive.custom
+
+        rwx = stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR
+
+        if custom.disk_space_path is None:
+            disk_space_script_file = tempfile.NamedTemporaryFile(
+                mode='w',
+                encoding='utf-8',
+                prefix='plotman-disk-space-script',
+                # TODO: but cleanup!
+                delete=False,
+            )
+            disk_space_script_file.write(custom.disk_space_script)
+            disk_space_script_file.flush()
+            disk_space_script_file.close()
+            custom.disk_space_path = disk_space_script_file.name
+            os.chmod(custom.disk_space_path, rwx)
+
+        if custom.transfer_path is None:
+            transfer_script_file = tempfile.NamedTemporaryFile(
+                mode='w',
+                encoding='utf-8',
+                prefix='plotman-transfer-script',
+                # TODO: but cleanup!
+                delete=False,
+            )
+            transfer_script_file.write(custom.transfer_script)
+            transfer_script_file.flush()
+            transfer_script_file.close()
+            custom.transfer_path = transfer_script_file.name
+            os.chmod(custom.transfer_path, rwx)
+
     plotting_active = True
     archiving_configured = cfg.directories.archive is not None
     archiving_active = archiving_configured

From 7480d2dde469ed6f32c29c15fb13180aff3fdc53 Mon Sep 17 00:00:00 2001
From: Kyle Altendorf <sda@fstab.net>
Date: Fri, 21 May 2021 22:58:21 -0400
Subject: [PATCH 25/58] correct default scripts

---
 src/plotman/configuration.py | 12 ++++--------
 1 file changed, 4 insertions(+), 8 deletions(-)

diff --git a/src/plotman/configuration.py b/src/plotman/configuration.py
index d1b90a88..a1b95b9f 100644
--- a/src/plotman/configuration.py
+++ b/src/plotman/configuration.py
@@ -57,19 +57,15 @@ class CustomArchive:
     # shell: str = 'bash'
     # disk_space: str = '''ssh chia@chia "df -BK | grep \" ${path}/\" | awk '{ print \$6 \\\":\\\" \$4 }'"'''
     disk_space_path: Optional[str] = None
-    disk_space_script: str = textwrap.dedent(
-        """
+    disk_space_script: str = textwrap.dedent("""\
         #!/bin/bash
         df -BK | grep " ${path}/" | awk '{gsub(/K\$/,"",$4); print $6 ":" $4*1024 }'
-        """
-    )
+    """)
     transfer_path: Optional[str] = None
-    transfer_script: str = textwrap.dedent(
-        """
+    transfer_script: str = textwrap.dedent("""\
         #!/bin/bash
         ${process_name} --skip-compress plot --remove-source-files --inplace "${source}" "${path}/${destination}"
-        """
-    )
+    """)
     transfer_process_name: str = 'rsync'
     transfer_process_argument_prefix: str = '${path}/'
 

From a35959cf9daa5a589e665c1224fbf055dc13eb8a Mon Sep 17 00:00:00 2001
From: Kyle Altendorf <sda@fstab.net>
Date: Fri, 21 May 2021 23:08:01 -0400
Subject: [PATCH 26/58] add a little config example

---
 src/plotman/resources/plotman.yaml | 12 ++++++++++++
 1 file changed, 12 insertions(+)

diff --git a/src/plotman/resources/plotman.yaml b/src/plotman/resources/plotman.yaml
index 8db6bfb1..c6ea712a 100644
--- a/src/plotman/resources/plotman.yaml
+++ b/src/plotman/resources/plotman.yaml
@@ -90,6 +90,18 @@ directories:
                 # Local mode:
                 # local:
                 #         path: '/farm'
+                custom:
+                    path: /farm/sites
+                    # The defaults use rsync for local transfers.  Below are examples
+                    # for a remote rsyncd setup.
+                    # disk_space_script: |
+                    #     #!/bin/bash
+                    #     ssh chia@chia "df -BK | grep \" ${path}/\" | awk '{ gsub(/K\$/,\"\",\$4); print \$6 \":\" \$4*1024 }'"
+                    # transfer_script: |
+                    #     #!/bin/bash
+                    #     rsync --bwlimit=80000 --skip-compress plot --remove-source-files --inplace "${source}" "rsync://chia@chia:12000/sites/${destination}"
+                    # transfer_process_name: rsync
+                    # transfer_process_argument_prefix: rsync://chia@chia:12000/sites/
 
 # Plotting scheduling parameters
 scheduling:

From 5866683744f6cd73ee9283b1ea9440b24f7ef6e9 Mon Sep 17 00:00:00 2001
From: Kyle Altendorf <sda@fstab.net>
Date: Sat, 22 May 2021 15:37:51 -0400
Subject: [PATCH 27/58] just break archiving and make it all new

---
 src/plotman/archive.py             | 108 ++++++++++-------------------
 src/plotman/configuration.py       |  86 ++++++++++++++++++-----
 src/plotman/interactive.py         |  38 +---------
 src/plotman/resources/plotman.yaml |  47 ++++---------
 4 files changed, 121 insertions(+), 158 deletions(-)

diff --git a/src/plotman/archive.py b/src/plotman/archive.py
index 0fd48d4d..31425d15 100644
--- a/src/plotman/archive.py
+++ b/src/plotman/archive.py
@@ -12,7 +12,7 @@
 import psutil
 import texttable as tt
 
-from plotman import job, manager, plot_util
+from plotman import configuration, job, manager, plot_util
 
 # TODO : write-protect and delete-protect archived plots
 
@@ -87,46 +87,27 @@ def compute_priority(phase, gb_free, n_plots):
 
 def get_archdir_freebytes(arch_cfg):
     archdir_freebytes = {}
-    if arch_cfg.mode == 'legacy':
-        df_cmd = ('ssh %s@%s df -aBK | grep " %s/"' %
-            (arch_cfg.rsyncd_user, arch_cfg.rsyncd_host, posixpath.normpath(arch_cfg.rsyncd_path)) )
-        with subprocess.Popen(df_cmd, shell=True, stdout=subprocess.PIPE) as proc:
-            for line in proc.stdout.readlines():
-                fields = line.split()
-                if fields[3] == b'-':
-                    # not actually mounted
-                    continue
-                freebytes = int(fields[3][:-1]) * 1024  # Strip the final 'K'
-                archdir = (fields[5]).decode('utf-8')
-                archdir_freebytes[archdir] = freebytes
-    else:
-        completed_process = subprocess.run(
-            [arch_cfg.custom.disk_space_path],
-            encoding='utf-8',
-            env={**os.environ, 'path': arch_cfg.custom.path},
-            stdout=subprocess.PIPE,
-            stderr=subprocess.PIPE,
-        )
-
-        for line in completed_process.stdout.splitlines():
-            split = line.split(':')
-            if len(split) != 2:
-                continue
-            archdir, space = split
-            freebytes = int(space)
-            archdir_freebytes[archdir] = freebytes
+    completed_process = subprocess.run(
+        [arch_cfg.disk_space_path],
+        encoding='utf-8',
+        env={**os.environ, **arch_cfg.environment()},
+        stdout=subprocess.PIPE,
+        stderr=subprocess.PIPE,
+    )
+
+    for line in completed_process.stdout.splitlines():
+        split = line.split(':')
+        if len(split) != 2:
+            # TODO: warning?  or something...
+            continue
+        archdir, space = split
+        freebytes = int(space)
+        archdir_freebytes[archdir] = freebytes
+
     return archdir_freebytes
 
 def arch_dest(arch_cfg, arch_dir):
-    if arch_cfg.mode == 'legacy':
-        rsync_path = arch_dir.replace(arch_cfg.rsyncd_path, arch_cfg.rsyncd_module)
-        if rsync_path.startswith('/'):
-            rsync_path = rsync_path[1:]  # Avoid dup slashes.  TODO use path join?
-        rsync_url = 'rsync://%s@%s:12000/%s' % (
-                arch_cfg.rsyncd_user, arch_cfg.rsyncd_host, rsync_path)
-        return rsync_url
-    else:
-        return os.path.relpath(arch_dir, arch_cfg.custom.path)
+    return os.path.relpath(arch_dir, arch_cfg.path)
 
 # TODO: maybe consolidate with similar code in job.py?
 def get_running_archive_jobs(arch_cfg):
@@ -136,18 +117,15 @@ def get_running_archive_jobs(arch_cfg):
     for proc in psutil.process_iter():
         with contextlib.suppress(psutil.NoSuchProcess):
             with proc.oneshot():
-                if arch_cfg.mode == 'legacy':
-                    dest = arch_dest(arch_cfg, '/')
-                    proc_name = 'rsync'
-                else:
-                    # TODO: make a context manager
-                    try:
-                        os.environ['path'] = arch_cfg.custom.path
-                        dest = os.path.expandvars(arch_cfg.custom.transfer_process_argument_prefix)
-                    finally:
-                        # TODO: yup, this'll delete, not restore
-                        del os.environ['path']
-                    proc_name = arch_cfg.custom.transfer_process_name
+                # TODO: make a context manager
+                try:
+                    os.environ.update(arch_cfg.environment())
+                    dest = os.path.expandvars(arch_cfg.transfer_process_argument_prefix)
+                finally:
+                    # TODO: yup, this'll delete, not restore
+                    for key in arch_cfg.environment():
+                        del os.environ[key]
+                proc_name = arch_cfg.transfer_process_name
                 if proc.name() == proc_name:
                     args = proc.cmdline()
                     for arg in args:
@@ -202,24 +180,14 @@ def archive(dir_cfg, all_jobs):
 
     msg = 'Found %s with ~%d GB free' % (archdir, freespace / plot_util.GB)
     dest = arch_dest(dir_cfg.archive, archdir)
-    if dir_cfg.archive.mode == 'legacy':
-        bwlimit = dir_cfg.archive.rsyncd_bwlimit
-        throttle_arg = ('--bwlimit=%d' % bwlimit) if bwlimit else ''
-        cmd = ('rsync %s --compress-level=0 --remove-source-files -P %s %s' %
-                (throttle_arg, chosen_plot, dest))
-        subprocess_arguments = {'args': cmd}
-    else:
-        custom = dir_cfg.archive.custom
-        env = {
-            'process_name': custom.transfer_process_name,
-            'source': chosen_plot,
-            'path': custom.path,
-            'destination': dest,
-        }
-        subprocess_arguments = {
-            'args': custom.transfer_path,
-            # 'args': custom.shell,
-            # 'input': custom.transfer,
-            'env': {**os.environ, **env}
-        }
+    archive = dir_cfg.archive
+    env = dir_cfg.archive.environment(
+        source=chosen_plot,
+        destination=dest,
+    )
+    subprocess_arguments = {
+        'args': archive.transfer_path,
+        'env': {**os.environ, **env}
+    }
+
     return (True, subprocess_arguments)
diff --git a/src/plotman/configuration.py b/src/plotman/configuration.py
index a1b95b9f..62701600 100644
--- a/src/plotman/configuration.py
+++ b/src/plotman/configuration.py
@@ -1,4 +1,7 @@
 import contextlib
+import os
+import stat
+import tempfile
 import textwrap
 from typing import Dict, List, Optional
 
@@ -37,6 +40,11 @@ def get_validated_configs(config_text, config_path):
     schema = desert.schema(PlotmanConfig)
     config_objects = yaml.load(config_text, Loader=yaml.SafeLoader)
 
+    version = config_objects.get('version', (0,))
+
+    if version[0] != 1:
+        raise Exception('configuration format version mismatch')
+
     try:
         loaded = schema.load(config_objects)
     except marshmallow.exceptions.ValidationError as e:
@@ -51,8 +59,11 @@ def get_validated_configs(config_text, config_path):
 
 # TODO: bah, mutable?  bah.
 @attr.mutable
-class CustomArchive:
+class Archive:
     path: str
+    index: int = 0  # If not explicit, "index" will default to 0
+    # TODO: mutable attribute...
+    env: Dict[str, str] = attr.ib(factory=dict)
     # TODO: fully support or remove
     # shell: str = 'bash'
     # disk_space: str = '''ssh chia@chia "df -BK | grep \" ${path}/\" | awk '{ print \$6 \\\":\\\" \$4 }'"'''
@@ -69,22 +80,56 @@ class CustomArchive:
     transfer_process_name: str = 'rsync'
     transfer_process_argument_prefix: str = '${path}/'
 
-@attr.frozen
-class Archive:
-    rsyncd_module: str = None
-    rsyncd_path: str = None
-    rsyncd_bwlimit: int = None
-    rsyncd_host: str = None
-    rsyncd_user: str = None
-    index: int = 0  # If not explicit, "index" will default to 0
-    mode: str = desert.ib(
-        default='legacy',
-        marshmallow_field=marshmallow.fields.String(
-            validate=marshmallow.validate.OneOf(choices=['legacy', 'custom'])
-        ),
-    )
-    custom: Optional[CustomArchive] = None
-    # custom: CustomArchive = CustomArchive(path='/farm/sites')
+    def environment(
+            self,
+            # path=None,
+            source=None,
+            process_name=None,
+            destination=None,
+    ):
+        complete = dict(self.env)
+
+        complete['path'] = self.path
+        complete['process_name'] = self.transfer_process_name
+
+        if source is not None:
+            complete['source'] = source
+
+        if destination is not None:
+            complete['destination'] = destination
+
+        return complete
+
+    def maybe_create_scripts(self):
+        rwx = stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR
+
+        if self.disk_space_path is None:
+            disk_space_script_file = tempfile.NamedTemporaryFile(
+                mode='w',
+                encoding='utf-8',
+                prefix='plotman-disk-space-script',
+                # TODO: but cleanup!
+                delete=False,
+            )
+            disk_space_script_file.write(self.disk_space_script)
+            disk_space_script_file.flush()
+            disk_space_script_file.close()
+            self.disk_space_path = disk_space_script_file.name
+            os.chmod(self.disk_space_path, rwx)
+
+        if self.transfer_path is None:
+            transfer_script_file = tempfile.NamedTemporaryFile(
+                mode='w',
+                encoding='utf-8',
+                prefix='plotman-transfer-script',
+                # TODO: but cleanup!
+                delete=False,
+            )
+            transfer_script_file.write(self.transfer_script)
+            transfer_script_file.flush()
+            transfer_script_file.close()
+            self.transfer_path = transfer_script_file.name
+            os.chmod(self.transfer_path, rwx)
 
 @attr.frozen
 class TmpOverrides:
@@ -98,6 +143,12 @@ class Directories:
     tmp2: Optional[str] = None
     tmp_overrides: Optional[Dict[str, TmpOverrides]] = None
     archive: Optional[Archive] = None
+    # archive_mode: str = desert.ib(
+    #     default='legacy',
+    #     marshmallow_field=marshmallow.fields.String(
+    #         validate=marshmallow.validate.OneOf(choices=['legacy', 'custom'])
+    #     ),
+    # )
 
     def dst_is_tmp(self):
         return self.dst is None
@@ -143,3 +194,4 @@ class PlotmanConfig:
     scheduling: Scheduling
     plotting: Plotting
     user_interface: UserInterface = attr.ib(factory=UserInterface)
+    version: List[int] = [0]
diff --git a/src/plotman/interactive.py b/src/plotman/interactive.py
index ab73e312..d03b80ff 100644
--- a/src/plotman/interactive.py
+++ b/src/plotman/interactive.py
@@ -3,9 +3,7 @@
 import locale
 import math
 import os
-import stat
 import subprocess
-import tempfile
 
 from plotman import archive, configuration, manager, reporting
 from plotman.job import Job
@@ -71,39 +69,7 @@ def curses_main(stdscr):
     config_text = configuration.read_configuration_text(config_path)
     cfg = configuration.get_validated_configs(config_text, config_path)
 
-    # TODO: context manager?
-    if cfg.directories.archive.mode == 'custom':
-        custom = cfg.directories.archive.custom
-
-        rwx = stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR
-
-        if custom.disk_space_path is None:
-            disk_space_script_file = tempfile.NamedTemporaryFile(
-                mode='w',
-                encoding='utf-8',
-                prefix='plotman-disk-space-script',
-                # TODO: but cleanup!
-                delete=False,
-            )
-            disk_space_script_file.write(custom.disk_space_script)
-            disk_space_script_file.flush()
-            disk_space_script_file.close()
-            custom.disk_space_path = disk_space_script_file.name
-            os.chmod(custom.disk_space_path, rwx)
-
-        if custom.transfer_path is None:
-            transfer_script_file = tempfile.NamedTemporaryFile(
-                mode='w',
-                encoding='utf-8',
-                prefix='plotman-transfer-script',
-                # TODO: but cleanup!
-                delete=False,
-            )
-            transfer_script_file.write(custom.transfer_script)
-            transfer_script_file.flush()
-            transfer_script_file.close()
-            custom.transfer_path = transfer_script_file.name
-            os.chmod(custom.transfer_path, rwx)
+    cfg.directories.archive.maybe_create_scripts()
 
     plotting_active = True
     archiving_configured = cfg.directories.archive is not None
@@ -207,7 +173,7 @@ def curses_main(stdscr):
         dst_dir = cfg.directories.get_dst_directories()
         dst_prefix = os.path.commonpath(dst_dir)
         if archiving_configured:
-            arch_prefix = cfg.directories.archive.rsyncd_path
+            arch_prefix = cfg.directories.archive.path
 
         n_tmpdirs = len(cfg.directories.tmp)
 
diff --git a/src/plotman/resources/plotman.yaml b/src/plotman/resources/plotman.yaml
index c6ea712a..0efa7af0 100644
--- a/src/plotman/resources/plotman.yaml
+++ b/src/plotman/resources/plotman.yaml
@@ -1,5 +1,5 @@
 # Default/example plotman.yaml configuration file
-
+version: []
 # Options for display and rendering
 user_interface:
         # Call out to the `stty` program to determine terminal size, instead of
@@ -68,40 +68,17 @@ directories:
         # allow public key login from rsyncd_user.
         # Complete example: https://github.com/ericaltendorf/plotman/wiki/Archiving
         archive:
-                rsyncd_module: plots # Define this in remote rsyncd.conf.
-                rsyncd_path: /plots # This is used via ssh. Should match path
-                                    # defined in the module referenced above.
-                rsyncd_bwlimit: 80000  # Bandwidth limit in KB/s
-                rsyncd_host: myfarmer
-                rsyncd_user: chia
-                # Optional index.  If omitted or set to 0, plotman will archive
-                # to the first archive dir with free space.  If specified,
-                # plotman will skip forward up to 'index' drives (if they exist).
-                # This can be useful to reduce io contention on a drive on the
-                # archive host if you have multiple plotters (simultaneous io
-                # can still happen at the time a drive fills up.)  E.g., if you
-                # have four plotters, you could set this to 0, 1, 2, and 3, on
-                # the 4 machines, or 0, 1, 0, 1.
-                #   index: 0
-                # Optional switch to enable local archiving (defaults to remote if absent).
-                # Note: rsyncd_module, rsyncd_host and rsyncd_user are ignored in local mode.
-                mode: legacy  # legacy (config above) or local
-                #
-                # Local mode:
-                # local:
-                #         path: '/farm'
-                custom:
-                    path: /farm/sites
-                    # The defaults use rsync for local transfers.  Below are examples
-                    # for a remote rsyncd setup.
-                    # disk_space_script: |
-                    #     #!/bin/bash
-                    #     ssh chia@chia "df -BK | grep \" ${path}/\" | awk '{ gsub(/K\$/,\"\",\$4); print \$6 \":\" \$4*1024 }'"
-                    # transfer_script: |
-                    #     #!/bin/bash
-                    #     rsync --bwlimit=80000 --skip-compress plot --remove-source-files --inplace "${source}" "rsync://chia@chia:12000/sites/${destination}"
-                    # transfer_process_name: rsync
-                    # transfer_process_argument_prefix: rsync://chia@chia:12000/sites/
+                path: /farm/sites
+                # The defaults use rsync for local transfers.  Below are examples
+                # for a remote rsyncd setup.
+                # disk_space_script: |
+                #     #!/bin/bash
+                #     ssh chia@chia "df -BK | grep \" ${path}/\" | awk '{ gsub(/K\$/,\"\",\$4); print \$6 \":\" \$4*1024 }'"
+                # transfer_script: |
+                #     #!/bin/bash
+                #     rsync --bwlimit=80000 --skip-compress plot --remove-source-files --inplace "${source}" "rsync://chia@chia:12000/sites/${destination}"
+                # transfer_process_name: rsync
+                # transfer_process_argument_prefix: rsync://chia@chia:12000/sites/
 
 # Plotting scheduling parameters
 scheduling:

From 141ba38df1475fd03bbff05304e6f9653d3cceca Mon Sep 17 00:00:00 2001
From: Kyle Altendorf <sda@fstab.net>
Date: Sat, 22 May 2021 19:29:51 -0400
Subject: [PATCH 28/58] less path

---
 src/plotman/archive.py             | 6 +-----
 src/plotman/configuration.py       | 4 ++--
 src/plotman/interactive.py         | 3 ++-
 src/plotman/resources/plotman.yaml | 1 -
 4 files changed, 5 insertions(+), 9 deletions(-)

diff --git a/src/plotman/archive.py b/src/plotman/archive.py
index 31425d15..29820311 100644
--- a/src/plotman/archive.py
+++ b/src/plotman/archive.py
@@ -106,9 +106,6 @@ def get_archdir_freebytes(arch_cfg):
 
     return archdir_freebytes
 
-def arch_dest(arch_cfg, arch_dir):
-    return os.path.relpath(arch_dir, arch_cfg.path)
-
 # TODO: maybe consolidate with similar code in job.py?
 def get_running_archive_jobs(arch_cfg):
     '''Look for running rsync jobs that seem to match the pattern we use for archiving
@@ -179,11 +176,10 @@ def archive(dir_cfg, all_jobs):
         return(False, 'No archive directories found with enough free space')
 
     msg = 'Found %s with ~%d GB free' % (archdir, freespace / plot_util.GB)
-    dest = arch_dest(dir_cfg.archive, archdir)
     archive = dir_cfg.archive
     env = dir_cfg.archive.environment(
         source=chosen_plot,
-        destination=dest,
+        destination=archdir,
     )
     subprocess_arguments = {
         'args': archive.transfer_path,
diff --git a/src/plotman/configuration.py b/src/plotman/configuration.py
index 62701600..aa936908 100644
--- a/src/plotman/configuration.py
+++ b/src/plotman/configuration.py
@@ -60,7 +60,7 @@ def get_validated_configs(config_text, config_path):
 # TODO: bah, mutable?  bah.
 @attr.mutable
 class Archive:
-    path: str
+    # path: str
     index: int = 0  # If not explicit, "index" will default to 0
     # TODO: mutable attribute...
     env: Dict[str, str] = attr.ib(factory=dict)
@@ -89,7 +89,7 @@ def environment(
     ):
         complete = dict(self.env)
 
-        complete['path'] = self.path
+        # complete['path'] = self.path
         complete['process_name'] = self.transfer_process_name
 
         if source is not None:
diff --git a/src/plotman/interactive.py b/src/plotman/interactive.py
index d03b80ff..f05c98e1 100644
--- a/src/plotman/interactive.py
+++ b/src/plotman/interactive.py
@@ -173,7 +173,8 @@ def curses_main(stdscr):
         dst_dir = cfg.directories.get_dst_directories()
         dst_prefix = os.path.commonpath(dst_dir)
         if archiving_configured:
-            arch_prefix = cfg.directories.archive.path
+            arch_prefix = ''
+            # arch_prefix = cfg.directories.archive.path
 
         n_tmpdirs = len(cfg.directories.tmp)
 
diff --git a/src/plotman/resources/plotman.yaml b/src/plotman/resources/plotman.yaml
index 0efa7af0..5a1e8d39 100644
--- a/src/plotman/resources/plotman.yaml
+++ b/src/plotman/resources/plotman.yaml
@@ -68,7 +68,6 @@ directories:
         # allow public key login from rsyncd_user.
         # Complete example: https://github.com/ericaltendorf/plotman/wiki/Archiving
         archive:
-                path: /farm/sites
                 # The defaults use rsync for local transfers.  Below are examples
                 # for a remote rsyncd setup.
                 # disk_space_script: |

From 6e291bb92ec9d542c44d8e5043f235ac3f6c5fb2 Mon Sep 17 00:00:00 2001
From: Kyle Altendorf <sda@fstab.net>
Date: Sun, 23 May 2021 13:52:54 -0400
Subject: [PATCH 29/58] fix archive prefix identification

---
 src/plotman/interactive.py | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/src/plotman/interactive.py b/src/plotman/interactive.py
index f05c98e1..0eae9bd5 100644
--- a/src/plotman/interactive.py
+++ b/src/plotman/interactive.py
@@ -173,8 +173,7 @@ def curses_main(stdscr):
         dst_dir = cfg.directories.get_dst_directories()
         dst_prefix = os.path.commonpath(dst_dir)
         if archiving_configured:
-            arch_prefix = ''
-            # arch_prefix = cfg.directories.archive.path
+            arch_prefix = os.path.commonpath(archdir_freebytes.keys())
 
         n_tmpdirs = len(cfg.directories.tmp)
 

From 3c8ead9f7425b573ea0baeb8eb0f341d9806b596 Mon Sep 17 00:00:00 2001
From: Kyle Altendorf <sda@fstab.net>
Date: Sun, 23 May 2021 14:50:48 -0400
Subject: [PATCH 30/58] add preliminary explanation and examples in config

---
 src/plotman/resources/plotman.yaml | 51 +++++++++++++++++++++++-------
 1 file changed, 40 insertions(+), 11 deletions(-)

diff --git a/src/plotman/resources/plotman.yaml b/src/plotman/resources/plotman.yaml
index 5a1e8d39..f8215840 100644
--- a/src/plotman/resources/plotman.yaml
+++ b/src/plotman/resources/plotman.yaml
@@ -59,25 +59,54 @@ directories:
                 - /mnt/dst/01
 
         # Archival configuration.  Optional; if you do not wish to run the
-        # archiving operation, comment this section out.
+        # archiving operation, comment this section out.  Almost everyone
+        # should be using the archival feature.  It is meant to distribute
+        # plots among multiple disks filling them all.  This can be done both
+        # to local and to remote disks.
+        #
+        # As of v0.4, archiving commands are highly configurable.  The basic
+        # configuration consists of a script for checking available disk space
+        # and another for actually transferring plots.  Each can be specified
+        # as either a path to an existing script or inline script contents.
+        # It is expected that most people will use existing recipes and will
+        # adjust them by specifying environment variables that will set their
+        # system specific values.  These can be provided to the scripts via
+        # the `env` key.  plotman will additionally provide `source` and
+        # `destination` environment variables to the transfer script so it
+        # knows the specifically selected items to process.  plotman also needs
+        # to be able to generally detect if a transfer process is already
+        # running.  To be able to identify externally launched transfers, the
+        # process name and an argument prefix to match must be provided.
         #
-        # Currently archival depends on an rsync daemon running on the remote
-        # host.
-        # The archival also uses ssh to connect to the remote host and check
-        # for available directories. Set up ssh keys on the remote host to
-        # allow public key login from rsyncd_user.
         # Complete example: https://github.com/ericaltendorf/plotman/wiki/Archiving
         archive:
-                # The defaults use rsync for local transfers.  Below are examples
-                # for a remote rsyncd setup.
+                # local rsync example setup
+                #
+                # env:
+                #     site_root: /farm/sites
+                # disk_space_script: |
+                #     #!/bin/bash
+                #     df -BK | grep " ${site_root}/" | awk '{ gsub(/K$/,"",$4); print $6 ":" $4*1024 }'
+                # transfer_script: |
+                #     #!/bin/bash
+                #     rsync --skip-compress plot --remove-source-files --inplace "${source}" "${destination}"
+                # transfer_process_name: rsync
+                # transfer_process_argument_prefix: ${site_root}
+
+                # remote rsync example setup
+                #
+                # env:
+                #     site_root: /farm/sites
+                #     url_root: rsync://altendky@server:12000/sites
                 # disk_space_script: |
                 #     #!/bin/bash
-                #     ssh chia@chia "df -BK | grep \" ${path}/\" | awk '{ gsub(/K\$/,\"\",\$4); print \$6 \":\" \$4*1024 }'"
+                #     ssh altendky@server "df -BK | grep \" ${site_root}/\" | awk '{ gsub(/K\$/,\"\",\$4); print \$6 \":\" \$4*1024 }'"
                 # transfer_script: |
                 #     #!/bin/bash
-                #     rsync --bwlimit=80000 --skip-compress plot --remove-source-files --inplace "${source}" "rsync://chia@chia:12000/sites/${destination}"
+                #     relative_path=$(realpath --relative-to="${site_root}" "${destination}")
+                #     rsync --bwlimit=80000 --skip-compress plot --remove-source-files --inplace "${source}" "${url_root}/${relative_path}"
                 # transfer_process_name: rsync
-                # transfer_process_argument_prefix: rsync://chia@chia:12000/sites/
+                # transfer_process_argument_prefix: ${url_root}
 
 # Plotting scheduling parameters
 scheduling:

From 0a1e96ee40ea8fa1c9ec8682c7ea0caa4fab8670 Mon Sep 17 00:00:00 2001
From: Kyle Altendorf <sda@fstab.net>
Date: Sun, 23 May 2021 16:38:27 -0400
Subject: [PATCH 31/58] misc

---
 src/plotman/_tests/archive_test.py | 97 +++++-------------------------
 src/plotman/archive.py             | 12 +---
 src/plotman/configuration.py       | 22 ++-----
 src/plotman/resources/plotman.yaml | 59 +++++++++++-------
 4 files changed, 62 insertions(+), 128 deletions(-)

diff --git a/src/plotman/_tests/archive_test.py b/src/plotman/_tests/archive_test.py
index e30116b6..398b0943 100755
--- a/src/plotman/_tests/archive_test.py
+++ b/src/plotman/_tests/archive_test.py
@@ -1,88 +1,23 @@
-from plotman import archive, configuration, job, manager
-import pytest
+from plotman import archive, job
 
 
 def test_compute_priority():
     assert (archive.compute_priority( job.Phase(major=3, minor=1), 1000, 10) >
             archive.compute_priority( job.Phase(major=3, minor=6), 1000, 10) )
 
-def _archive_legacy():
-    return configuration.Archive(
-        rsyncd_module='plots_mod',
-        rsyncd_path='/plotdir',
-        rsyncd_host='thehostname',
-        rsyncd_user='theusername',
-        rsyncd_bwlimit=80000
-    )
 
-def test_arch_dest():
-    arch_dir = '/plotdir/012'
-    arch_cfg = _archive_legacy()
-
-    # Normal usage
-    assert ('rsync://theusername@thehostname:12000/plots_mod/012' ==
-            archive.arch_dest(arch_cfg, arch_dir))
-
-    # Usage for constructing just the prefix, for scanning process tables
-    # for matching jobs.
-    assert ('rsync://theusername@thehostname:12000/' ==
-            archive.arch_dest(arch_cfg, '/'))
-
-
-def test_archive_legacy_default():
-    arch_cfg = _archive_legacy()
-    assert arch_cfg.mode == 'legacy'
-
-def _archive_badmode():
-    return configuration.Archive(
-        rsyncd_module='plots_mod',
-        rsyncd_path='/plotdir',
-        rsyncd_host='thehostname',
-        rsyncd_user='theusername',
-        rsyncd_bwlimit=80000,
-        mode='thismodedoesntexist'
-    )
-
-def test_archive_bad_mode():
-    arch_cfg = _archive_badmode()
-    assert arch_cfg.mode == 'thismodedoesntexist'
-
-
-def test_archive_bad_mode_load():
-    arch_cfg = _archive_badmode()
-    with pytest.raises(AttributeError):
-        getattr(arch_cfg, arch_cfg.mode)
-
-
-def _archive_emptymode():
-    return configuration.Archive(
-        rsyncd_module='plots_mod',
-        rsyncd_path='/plotdir',
-        rsyncd_host='thehostname',
-        rsyncd_user='theusername',
-        rsyncd_bwlimit=80000,
-        mode='local'
-    )
-
-def test_archive_local_mode_absent():
-    arch_cfg = _archive_emptymode()
-    arch_cfg_local = getattr(arch_cfg, arch_cfg.mode)
-    assert not arch_cfg_local
-
-def _archive_localmode():
-    return configuration.Archive(
-        rsyncd_module='plots_mod',
-        rsyncd_path='/plotdir',
-        rsyncd_host='thehostname',
-        rsyncd_user='theusername',
-        rsyncd_bwlimit=80000,
-        mode='local',
-        local=configuration.ArchiveLocal(
-            path='/farm'
-        )
-    )
-
-def test_archive_local_mode_load():
-    arch_cfg = _archive_localmode()
-    arch_cfg_local = getattr(arch_cfg, arch_cfg.mode)
-    assert isinstance(arch_cfg_local, configuration.ArchiveLocal)
+# @pytest.fixture(name='remote_configuration')
+# def remote_configuration_fixture():
+#     return configuration.Archive(
+#         env={},
+#         disk_space_script=textwrap.dedent("""\
+#             #!/bin/bash
+#             ssh altendky@server "df -BK | grep \\" ${site_root}/\\" | awk '{ gsub(/K\$/,\\"\\",\$4); print \$6 \":\" \$4*1024 }'"
+#         """),
+#         transfer_script=textwrap.dedent("""\
+#             relative_path=$(realpath --relative-to="${site_root}" "${destination}")
+#             "${command}" --bwlimit=80000 --skip-compress plot --remove-source-files --inplace "${source}" "${url_root}/${relative_path}"
+#         """),
+#         transfer_process_name='{command}',
+#         transfer_process_argument_prefix='{url_root}',
+#     )
diff --git a/src/plotman/archive.py b/src/plotman/archive.py
index 29820311..8fa94efb 100644
--- a/src/plotman/archive.py
+++ b/src/plotman/archive.py
@@ -114,15 +114,9 @@ def get_running_archive_jobs(arch_cfg):
     for proc in psutil.process_iter():
         with contextlib.suppress(psutil.NoSuchProcess):
             with proc.oneshot():
-                # TODO: make a context manager
-                try:
-                    os.environ.update(arch_cfg.environment())
-                    dest = os.path.expandvars(arch_cfg.transfer_process_argument_prefix)
-                finally:
-                    # TODO: yup, this'll delete, not restore
-                    for key in arch_cfg.environment():
-                        del os.environ[key]
-                proc_name = arch_cfg.transfer_process_name
+                variables = {**os.environ, **arch_cfg.environment()}
+                dest = arch_cfg.transfer_process_argument_prefix.format(**variables)
+                proc_name = arch_cfg.transfer_process_name.format(**variables)
                 if proc.name() == proc_name:
                     args = proc.cmdline()
                     for arg in args:
diff --git a/src/plotman/configuration.py b/src/plotman/configuration.py
index aa936908..239dfa1b 100644
--- a/src/plotman/configuration.py
+++ b/src/plotman/configuration.py
@@ -60,25 +60,15 @@ def get_validated_configs(config_text, config_path):
 # TODO: bah, mutable?  bah.
 @attr.mutable
 class Archive:
-    # path: str
+    transfer_process_name: str
+    transfer_process_argument_prefix: str
     index: int = 0  # If not explicit, "index" will default to 0
     # TODO: mutable attribute...
     env: Dict[str, str] = attr.ib(factory=dict)
-    # TODO: fully support or remove
-    # shell: str = 'bash'
-    # disk_space: str = '''ssh chia@chia "df -BK | grep \" ${path}/\" | awk '{ print \$6 \\\":\\\" \$4 }'"'''
     disk_space_path: Optional[str] = None
-    disk_space_script: str = textwrap.dedent("""\
-        #!/bin/bash
-        df -BK | grep " ${path}/" | awk '{gsub(/K\$/,"",$4); print $6 ":" $4*1024 }'
-    """)
+    disk_space_script: Optional[str] = None
     transfer_path: Optional[str] = None
-    transfer_script: str = textwrap.dedent("""\
-        #!/bin/bash
-        ${process_name} --skip-compress plot --remove-source-files --inplace "${source}" "${path}/${destination}"
-    """)
-    transfer_process_name: str = 'rsync'
-    transfer_process_argument_prefix: str = '${path}/'
+    transfer_script: Optional[str] = None
 
     def environment(
             self,
@@ -89,8 +79,8 @@ def environment(
     ):
         complete = dict(self.env)
 
-        # complete['path'] = self.path
-        complete['process_name'] = self.transfer_process_name
+        variables = {**os.environ, **complete}
+        complete['process_name'] = self.transfer_process_name.format(**variables)
 
         if source is not None:
             complete['source'] = source
diff --git a/src/plotman/resources/plotman.yaml b/src/plotman/resources/plotman.yaml
index f8215840..7f90ce1d 100644
--- a/src/plotman/resources/plotman.yaml
+++ b/src/plotman/resources/plotman.yaml
@@ -1,5 +1,6 @@
 # Default/example plotman.yaml configuration file
-version: []
+version: [1]
+
 # Options for display and rendering
 user_interface:
         # Call out to the `stty` program to determine terminal size, instead of
@@ -76,37 +77,51 @@ directories:
         # knows the specifically selected items to process.  plotman also needs
         # to be able to generally detect if a transfer process is already
         # running.  To be able to identify externally launched transfers, the
-        # process name and an argument prefix to match must be provided.
+        # process name and an argument prefix to match must be provided.  Note
+        # that variable substitution of environment variables including those
+        # specified in the env key can be used in both process name and process
+        # argument prefix elements but that they use the python substitution
+        # format.
         #
         # Complete example: https://github.com/ericaltendorf/plotman/wiki/Archiving
         archive:
-                # local rsync example setup
+                # -- local rsync example setup
+
+                 env:
+                         command: rsync
+                         site_root: /farm/sites
+                 disk_space_script: |
+                         #!/bin/bash
+                         df -BK | grep " ${site_root}/" | awk '{ gsub(/K$/,"",$4); print $6 ":" $4*1024 }'
+                 transfer_script: |
+                         #!/bin/bash
+                         "${command}" --skip-compress plot --remove-source-files --inplace "${source}" "${destination}"
+                 transfer_process_name: "{command}"
+                 transfer_process_argument_prefix: "{site_root}"
+
+                # -- remote rsync example setup
                 #
                 # env:
-                #     site_root: /farm/sites
+                #         command: rsync
+                #         site_root: /farm/sites
+                #         url_root: rsync://altendky@server:12000/sites
                 # disk_space_script: |
-                #     #!/bin/bash
-                #     df -BK | grep " ${site_root}/" | awk '{ gsub(/K$/,"",$4); print $6 ":" $4*1024 }'
+                #         #!/bin/bash
+                #         ssh altendky@server "df -BK | grep \" ${site_root}/\" | awk '{ gsub(/K\$/,\"\",\$4); print \$6 \":\" \$4*1024 }'"
                 # transfer_script: |
-                #     #!/bin/bash
-                #     rsync --skip-compress plot --remove-source-files --inplace "${source}" "${destination}"
-                # transfer_process_name: rsync
-                # transfer_process_argument_prefix: ${site_root}
+                #         #!/bin/bash
+                #         relative_path=$(realpath --relative-to="${site_root}" "${destination}")
+                #         "${command}" --bwlimit=80000 --skip-compress plot --remove-source-files --inplace "${source}" "${url_root}/${relative_path}"
+                # transfer_process_name: "{command}"
+                # transfer_process_argument_prefix: "{url_root}"
 
-                # remote rsync example setup
-                #
+                # -- external script example
                 # env:
-                #     site_root: /farm/sites
-                #     url_root: rsync://altendky@server:12000/sites
-                # disk_space_script: |
-                #     #!/bin/bash
-                #     ssh altendky@server "df -BK | grep \" ${site_root}/\" | awk '{ gsub(/K\$/,\"\",\$4); print \$6 \":\" \$4*1024 }'"
-                # transfer_script: |
-                #     #!/bin/bash
-                #     relative_path=$(realpath --relative-to="${site_root}" "${destination}")
-                #     rsync --bwlimit=80000 --skip-compress plot --remove-source-files --inplace "${source}" "${url_root}/${relative_path}"
+                #         some_common_value: /a/path
+                # disk_space_path: /home/me/my_disk_space_script.sh
+                # transfer_path: /home/me/my_transfer_script.sh
                 # transfer_process_name: rsync
-                # transfer_process_argument_prefix: ${url_root}
+                # transfer_process_argument_prefix: /the/destination/directory/root
 
 # Plotting scheduling parameters
 scheduling:

From f24a372b8d1e80b15b274f40c03f61c5c72bafac Mon Sep 17 00:00:00 2001
From: Kyle Altendorf <sda@fstab.net>
Date: Sun, 23 May 2021 17:11:28 -0400
Subject: [PATCH 32/58] fixup for archive subcommand as well

---
 src/plotman/interactive.py | 6 ++++--
 src/plotman/plotman.py     | 4 ++++
 2 files changed, 8 insertions(+), 2 deletions(-)

diff --git a/src/plotman/interactive.py b/src/plotman/interactive.py
index 0eae9bd5..c12d02e5 100644
--- a/src/plotman/interactive.py
+++ b/src/plotman/interactive.py
@@ -69,10 +69,12 @@ def curses_main(stdscr):
     config_text = configuration.read_configuration_text(config_path)
     cfg = configuration.get_validated_configs(config_text, config_path)
 
-    cfg.directories.archive.maybe_create_scripts()
-
     plotting_active = True
+
     archiving_configured = cfg.directories.archive is not None
+    if archiving_configured:
+        cfg.directories.archive.maybe_create_scripts()
+
     archiving_active = archiving_configured
 
     plotting_status = '<startup>'    # todo rename these msg?
diff --git a/src/plotman/plotman.py b/src/plotman/plotman.py
index 35e1616f..890499e3 100755
--- a/src/plotman/plotman.py
+++ b/src/plotman/plotman.py
@@ -137,6 +137,10 @@ def main():
     config_text = configuration.read_configuration_text(config_path)
     cfg = configuration.get_validated_configs(config_text, config_path)
 
+    archiving_configured = cfg.directories.archive is not None
+    if archiving_configured:
+        cfg.directories.archive.maybe_create_scripts()
+
     #
     # Stay alive, spawning plot jobs
     #

From 68db1f2c0d2c1626d2065ea5bf47fd69c4099d09 Mon Sep 17 00:00:00 2001
From: Kyle Altendorf <sda@fstab.net>
Date: Sun, 23 May 2021 19:11:14 -0400
Subject: [PATCH 33/58] links to wiki for configuration versions

---
 src/plotman/configuration.py       | 11 +++++++++--
 src/plotman/resources/plotman.yaml |  2 ++
 2 files changed, 11 insertions(+), 2 deletions(-)

diff --git a/src/plotman/configuration.py b/src/plotman/configuration.py
index 239dfa1b..45221ac0 100644
--- a/src/plotman/configuration.py
+++ b/src/plotman/configuration.py
@@ -42,8 +42,15 @@ def get_validated_configs(config_text, config_path):
 
     version = config_objects.get('version', (0,))
 
-    if version[0] != 1:
-        raise Exception('configuration format version mismatch')
+    expected_major_version = 1
+
+    if version[0] != expected_major_version:
+        message = textwrap.dedent(f"""\
+            Expected major version {expected_major_version}, found version {version}
+                See https://github.com/ericaltendorf/plotman/wiki/Configuration#versions
+        """)
+
+        raise Exception(message)
 
     try:
         loaded = schema.load(config_objects)
diff --git a/src/plotman/resources/plotman.yaml b/src/plotman/resources/plotman.yaml
index 7f90ce1d..a2820f1f 100644
--- a/src/plotman/resources/plotman.yaml
+++ b/src/plotman/resources/plotman.yaml
@@ -1,4 +1,6 @@
 # Default/example plotman.yaml configuration file
+
+# https://github.com/ericaltendorf/plotman/wiki/Configuration#versions
 version: [1]
 
 # Options for display and rendering

From cc25e055c3e4340c147c6aa4e607fe6c4531e85c Mon Sep 17 00:00:00 2001
From: Kyle Altendorf <sda@fstab.net>
Date: Sun, 23 May 2021 20:07:29 -0400
Subject: [PATCH 34/58] minor cleanup

---
 src/plotman/configuration.py | 8 --------
 src/plotman/interactive.py   | 1 -
 2 files changed, 9 deletions(-)

diff --git a/src/plotman/configuration.py b/src/plotman/configuration.py
index 45221ac0..452804d9 100644
--- a/src/plotman/configuration.py
+++ b/src/plotman/configuration.py
@@ -79,9 +79,7 @@ class Archive:
 
     def environment(
             self,
-            # path=None,
             source=None,
-            process_name=None,
             destination=None,
     ):
         complete = dict(self.env)
@@ -140,12 +138,6 @@ class Directories:
     tmp2: Optional[str] = None
     tmp_overrides: Optional[Dict[str, TmpOverrides]] = None
     archive: Optional[Archive] = None
-    # archive_mode: str = desert.ib(
-    #     default='legacy',
-    #     marshmallow_field=marshmallow.fields.String(
-    #         validate=marshmallow.validate.OneOf(choices=['legacy', 'custom'])
-    #     ),
-    # )
 
     def dst_is_tmp(self):
         return self.dst is None
diff --git a/src/plotman/interactive.py b/src/plotman/interactive.py
index c12d02e5..b67dba52 100644
--- a/src/plotman/interactive.py
+++ b/src/plotman/interactive.py
@@ -70,7 +70,6 @@ def curses_main(stdscr):
     cfg = configuration.get_validated_configs(config_text, config_path)
 
     plotting_active = True
-
     archiving_configured = cfg.directories.archive is not None
     if archiving_configured:
         cfg.directories.archive.maybe_create_scripts()

From de73ed42db6f4fa03df0e65c665944ac876a68a7 Mon Sep 17 00:00:00 2001
From: Kyle Altendorf <sda@fstab.net>
Date: Sun, 23 May 2021 21:16:54 -0400
Subject: [PATCH 35/58] cleanup temp script files

---
 src/plotman/configuration.py |  34 ++++--
 src/plotman/interactive.py   |  13 +-
 src/plotman/plotman.py       | 225 +++++++++++++++++------------------
 3 files changed, 135 insertions(+), 137 deletions(-)

diff --git a/src/plotman/configuration.py b/src/plotman/configuration.py
index 452804d9..fcde55fc 100644
--- a/src/plotman/configuration.py
+++ b/src/plotman/configuration.py
@@ -95,34 +95,32 @@ def environment(
 
         return complete
 
-    def maybe_create_scripts(self):
+    def maybe_create_scripts(self, temp):
         rwx = stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR
 
         if self.disk_space_path is None:
-            disk_space_script_file = tempfile.NamedTemporaryFile(
+            with tempfile.NamedTemporaryFile(
                 mode='w',
                 encoding='utf-8',
                 prefix='plotman-disk-space-script',
-                # TODO: but cleanup!
                 delete=False,
-            )
-            disk_space_script_file.write(self.disk_space_script)
-            disk_space_script_file.flush()
-            disk_space_script_file.close()
+                dir=temp,
+            ) as disk_space_script_file:
+                disk_space_script_file.write(self.disk_space_script)
+
             self.disk_space_path = disk_space_script_file.name
             os.chmod(self.disk_space_path, rwx)
 
         if self.transfer_path is None:
-            transfer_script_file = tempfile.NamedTemporaryFile(
+            with tempfile.NamedTemporaryFile(
                 mode='w',
                 encoding='utf-8',
                 prefix='plotman-transfer-script',
-                # TODO: but cleanup!
                 delete=False,
-            )
-            transfer_script_file.write(self.transfer_script)
-            transfer_script_file.flush()
-            transfer_script_file.close()
+                dir=temp,
+            ) as transfer_script_file:
+                transfer_script_file.write(self.transfer_script)
+
             self.transfer_path = transfer_script_file.name
             os.chmod(self.transfer_path, rwx)
 
@@ -184,3 +182,13 @@ class PlotmanConfig:
     plotting: Plotting
     user_interface: UserInterface = attr.ib(factory=UserInterface)
     version: List[int] = [0]
+
+    @contextlib.contextmanager
+    def setup(self):
+        prefix = f'plotman-pid_{os.getpid()}-'
+
+        with tempfile.TemporaryDirectory(prefix=prefix) as temp:
+            if self.directories.archive is not None:
+                self.directories.archive.maybe_create_scripts(temp=temp)
+
+            yield
diff --git a/src/plotman/interactive.py b/src/plotman/interactive.py
index b67dba52..341a33fe 100644
--- a/src/plotman/interactive.py
+++ b/src/plotman/interactive.py
@@ -62,18 +62,11 @@ def archiving_status_msg(configured, active, status):
     else:
         return '(not configured)'
 
-def curses_main(stdscr):
+def curses_main(stdscr, cfg):
     log = Log()
 
-    config_path = configuration.get_path()
-    config_text = configuration.read_configuration_text(config_path)
-    cfg = configuration.get_validated_configs(config_text, config_path)
-
     plotting_active = True
     archiving_configured = cfg.directories.archive is not None
-    if archiving_configured:
-        cfg.directories.archive.maybe_create_scripts()
-
     archiving_active = archiving_configured
 
     plotting_status = '<startup>'    # todo rename these msg?
@@ -329,13 +322,13 @@ def curses_main(stdscr):
             pressed_key = key
 
 
-def run_interactive():
+def run_interactive(cfg):
     locale.setlocale(locale.LC_ALL, '')
     code = locale.getpreferredencoding()
     # Then use code as the encoding for str.encode() calls.
 
     try:
-        curses.wrapper(curses_main)
+        curses.wrapper(curses_main, cfg=cfg)
     except curses.error as e:
         raise TerminalTooSmallError(
             "Your terminal may be too small, try making it bigger.",
diff --git a/src/plotman/plotman.py b/src/plotman/plotman.py
index 890499e3..9fe97fa2 100755
--- a/src/plotman/plotman.py
+++ b/src/plotman/plotman.py
@@ -137,124 +137,121 @@ def main():
     config_text = configuration.read_configuration_text(config_path)
     cfg = configuration.get_validated_configs(config_text, config_path)
 
-    archiving_configured = cfg.directories.archive is not None
-    if archiving_configured:
-        cfg.directories.archive.maybe_create_scripts()
-
-    #
-    # Stay alive, spawning plot jobs
-    #
-    if args.cmd == 'plot':
-        print('...starting plot loop')
-        while True:
-            wait_reason = manager.maybe_start_new_plot(cfg.directories, cfg.scheduling, cfg.plotting)
-
-            # TODO: report this via a channel that can be polled on demand, so we don't spam the console
-            if wait_reason:
-                print('...sleeping %d s: %s' % (cfg.scheduling.polling_time_s, wait_reason))
-
-            time.sleep(cfg.scheduling.polling_time_s)
-
-    #
-    # Analysis of completed jobs
-    #
-    elif args.cmd == 'analyze':
-
-        analyzer.analyze(args.logfile, args.clipterminals,
-                args.bytmp, args.bybitfield)
-
-    else:
-        jobs = Job.get_running_jobs(cfg.directories.log)
-
-        # Status report
-        if args.cmd == 'status':
-            result = "{0}\n\n{1}\n\nUpdated at: {2}".format(
-                reporting.status_report(jobs, get_term_width()),
-                reporting.summary(jobs),
-                datetime.datetime.today().strftime("%c"),
-            )
-            print(result)
-
-        # Directories report
-        elif args.cmd == 'dirs':
-            print(reporting.dirs_report(jobs, cfg.directories, cfg.scheduling, get_term_width()))
-
-        elif args.cmd == 'interactive':
-            interactive.run_interactive()
-
-        # Start running archival
-        elif args.cmd == 'archive':
-            print('...starting archive loop')
-            firstit = True
+    with cfg.setup():
+        #
+        # Stay alive, spawning plot jobs
+        #
+        if args.cmd == 'plot':
+            print('...starting plot loop')
             while True:
-                if not firstit:
-                    print('Sleeping 60s until next iteration...')
-                    time.sleep(60)
-                    jobs = Job.get_running_jobs(cfg.directories.log)
-                firstit = False
-
-                archiving_status, log_message = archive.spawn_archive_process(cfg.directories, jobs)
-                if log_message:
-                    print(log_message)
+                wait_reason = manager.maybe_start_new_plot(cfg.directories, cfg.scheduling, cfg.plotting)
 
+                # TODO: report this via a channel that can be polled on demand, so we don't spam the console
+                if wait_reason:
+                    print('...sleeping %d s: %s' % (cfg.scheduling.polling_time_s, wait_reason))
 
-        # Debugging: show the destination drive usage schedule
-        elif args.cmd == 'dsched':
-            for (d, ph) in manager.dstdirs_to_furthest_phase(jobs).items():
-                print('  %s : %s' % (d, str(ph)))
+                time.sleep(cfg.scheduling.polling_time_s)
 
         #
-        # Job control commands
+        # Analysis of completed jobs
         #
-        elif args.cmd in [ 'details', 'files', 'kill', 'suspend', 'resume' ]:
-            print(args)
-
-            selected = []
-
-            # TODO: clean up treatment of wildcard
-            if args.idprefix[0] == 'all':
-                selected = jobs
-            else:
-                # TODO: allow multiple idprefixes, not just take the first
-                selected = manager.select_jobs_by_partial_id(jobs, args.idprefix[0])
-                if (len(selected) == 0):
-                    print('Error: %s matched no jobs.' % args.idprefix[0])
-                elif len(selected) > 1:
-                    print('Error: "%s" matched multiple jobs:' % args.idprefix[0])
-                    for j in selected:
-                        print('  %s' % j.plot_id)
-                    selected = []
-
-            for job in selected:
-                if args.cmd == 'details':
-                    print(job.status_str_long())
-
-                elif args.cmd == 'files':
-                    temp_files = job.get_temp_files()
-                    for f in temp_files:
-                        print('  %s' % f)
-
-                elif args.cmd == 'kill':
-                    # First suspend so job doesn't create new files
-                    print('Pausing PID %d, plot id %s' % (job.proc.pid, job.plot_id))
-                    job.suspend()
-
-                    temp_files = job.get_temp_files()
-                    print('Will kill pid %d, plot id %s' % (job.proc.pid, job.plot_id))
-                    print('Will delete %d temp files' % len(temp_files))
-                    conf = input('Are you sure? ("y" to confirm): ')
-                    if (conf != 'y'):
-                        print('canceled.  If you wish to resume the job, do so manually.')
-                    else:
-                        print('killing...')
-                        job.cancel()
-                        print('cleaning up temp files...')
+        elif args.cmd == 'analyze':
+
+            analyzer.analyze(args.logfile, args.clipterminals,
+                    args.bytmp, args.bybitfield)
+
+        else:
+            jobs = Job.get_running_jobs(cfg.directories.log)
+
+            # Status report
+            if args.cmd == 'status':
+                result = "{0}\n\n{1}\n\nUpdated at: {2}".format(
+                    reporting.status_report(jobs, get_term_width()),
+                    reporting.summary(jobs),
+                    datetime.datetime.today().strftime("%c"),
+                )
+                print(result)
+
+            # Directories report
+            elif args.cmd == 'dirs':
+                print(reporting.dirs_report(jobs, cfg.directories, cfg.scheduling, get_term_width()))
+
+            elif args.cmd == 'interactive':
+                interactive.run_interactive(cfg=cfg)
+
+            # Start running archival
+            elif args.cmd == 'archive':
+                print('...starting archive loop')
+                firstit = True
+                while True:
+                    if not firstit:
+                        print('Sleeping 60s until next iteration...')
+                        time.sleep(60)
+                        jobs = Job.get_running_jobs(cfg.directories.log)
+                    firstit = False
+
+                    archiving_status, log_message = archive.spawn_archive_process(cfg.directories, jobs)
+                    if log_message:
+                        print(log_message)
+
+
+            # Debugging: show the destination drive usage schedule
+            elif args.cmd == 'dsched':
+                for (d, ph) in manager.dstdirs_to_furthest_phase(jobs).items():
+                    print('  %s : %s' % (d, str(ph)))
+
+            #
+            # Job control commands
+            #
+            elif args.cmd in [ 'details', 'files', 'kill', 'suspend', 'resume' ]:
+                print(args)
+
+                selected = []
+
+                # TODO: clean up treatment of wildcard
+                if args.idprefix[0] == 'all':
+                    selected = jobs
+                else:
+                    # TODO: allow multiple idprefixes, not just take the first
+                    selected = manager.select_jobs_by_partial_id(jobs, args.idprefix[0])
+                    if (len(selected) == 0):
+                        print('Error: %s matched no jobs.' % args.idprefix[0])
+                    elif len(selected) > 1:
+                        print('Error: "%s" matched multiple jobs:' % args.idprefix[0])
+                        for j in selected:
+                            print('  %s' % j.plot_id)
+                        selected = []
+
+                for job in selected:
+                    if args.cmd == 'details':
+                        print(job.status_str_long())
+
+                    elif args.cmd == 'files':
+                        temp_files = job.get_temp_files()
                         for f in temp_files:
-                            os.remove(f)
-
-                elif args.cmd == 'suspend':
-                    print('Suspending ' + job.plot_id)
-                    job.suspend()
-                elif args.cmd == 'resume':
-                    print('Resuming ' + job.plot_id)
-                    job.resume()
+                            print('  %s' % f)
+
+                    elif args.cmd == 'kill':
+                        # First suspend so job doesn't create new files
+                        print('Pausing PID %d, plot id %s' % (job.proc.pid, job.plot_id))
+                        job.suspend()
+
+                        temp_files = job.get_temp_files()
+                        print('Will kill pid %d, plot id %s' % (job.proc.pid, job.plot_id))
+                        print('Will delete %d temp files' % len(temp_files))
+                        conf = input('Are you sure? ("y" to confirm): ')
+                        if (conf != 'y'):
+                            print('canceled.  If you wish to resume the job, do so manually.')
+                        else:
+                            print('killing...')
+                            job.cancel()
+                            print('cleaning up temp files...')
+                            for f in temp_files:
+                                os.remove(f)
+
+                    elif args.cmd == 'suspend':
+                        print('Suspending ' + job.plot_id)
+                        job.suspend()
+                    elif args.cmd == 'resume':
+                        print('Resuming ' + job.plot_id)
+                        job.resume()

From 25195340c4c22aca72f2449e52f289acec7d06a4 Mon Sep 17 00:00:00 2001
From: Kyle Altendorf <sda@fstab.net>
Date: Wed, 26 May 2021 21:32:02 -0400
Subject: [PATCH 36/58] --preallocate --whole-file

---
 src/plotman/resources/plotman.yaml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/plotman/resources/plotman.yaml b/src/plotman/resources/plotman.yaml
index a2820f1f..94ec1a83 100644
--- a/src/plotman/resources/plotman.yaml
+++ b/src/plotman/resources/plotman.yaml
@@ -97,7 +97,7 @@ directories:
                          df -BK | grep " ${site_root}/" | awk '{ gsub(/K$/,"",$4); print $6 ":" $4*1024 }'
                  transfer_script: |
                          #!/bin/bash
-                         "${command}" --skip-compress plot --remove-source-files --inplace "${source}" "${destination}"
+                         "${command}" --inplace --preallocate --remove-source-files --skip-compress plot --whole-file "${source}" "${destination}"
                  transfer_process_name: "{command}"
                  transfer_process_argument_prefix: "{site_root}"
 
@@ -113,7 +113,7 @@ directories:
                 # transfer_script: |
                 #         #!/bin/bash
                 #         relative_path=$(realpath --relative-to="${site_root}" "${destination}")
-                #         "${command}" --bwlimit=80000 --skip-compress plot --remove-source-files --inplace "${source}" "${url_root}/${relative_path}"
+                #         "${command}" --bwlimit=80000 --inplace --preallocate --remove-source-files --skip-compress plot --whole-file "${source}" "${url_root}/${relative_path}"
                 # transfer_process_name: "{command}"
                 # transfer_process_argument_prefix: "{url_root}"
 

From 41d90337431de3507b37ab96cb2c8b45e0b43006 Mon Sep 17 00:00:00 2001
From: Kyle Altendorf <sda@fstab.net>
Date: Wed, 26 May 2021 21:51:59 -0400
Subject: [PATCH 37/58] correct indentation in example config

---
 src/plotman/resources/plotman.yaml | 22 +++++++++++-----------
 1 file changed, 11 insertions(+), 11 deletions(-)

diff --git a/src/plotman/resources/plotman.yaml b/src/plotman/resources/plotman.yaml
index 94ec1a83..56c338ad 100644
--- a/src/plotman/resources/plotman.yaml
+++ b/src/plotman/resources/plotman.yaml
@@ -89,17 +89,17 @@ directories:
         archive:
                 # -- local rsync example setup
 
-                 env:
-                         command: rsync
-                         site_root: /farm/sites
-                 disk_space_script: |
-                         #!/bin/bash
-                         df -BK | grep " ${site_root}/" | awk '{ gsub(/K$/,"",$4); print $6 ":" $4*1024 }'
-                 transfer_script: |
-                         #!/bin/bash
-                         "${command}" --inplace --preallocate --remove-source-files --skip-compress plot --whole-file "${source}" "${destination}"
-                 transfer_process_name: "{command}"
-                 transfer_process_argument_prefix: "{site_root}"
+                env:
+                        command: rsync
+                        site_root: /farm/sites
+                disk_space_script: |
+                        #!/bin/bash
+                        df -BK | grep " ${site_root}/" | awk '{ gsub(/K$/,"",$4); print $6 ":" $4*1024 }'
+                transfer_script: |
+                        #!/bin/bash
+                        "${command}" --inplace --preallocate --remove-source-files --skip-compress plot --whole-file "${source}" "${destination}"
+                transfer_process_name: "{command}"
+                transfer_process_argument_prefix: "{site_root}"
 
                 # -- remote rsync example setup
                 #

From 7ad34056f69543b59e787e2d63e2e8155b086631 Mon Sep 17 00:00:00 2001
From: Kyle Altendorf <sda@fstab.net>
Date: Wed, 26 May 2021 22:04:01 -0400
Subject: [PATCH 38/58] explain disk space script output format

---
 src/plotman/resources/plotman.yaml | 10 ++++++++++
 1 file changed, 10 insertions(+)

diff --git a/src/plotman/resources/plotman.yaml b/src/plotman/resources/plotman.yaml
index 56c338ad..dd85439c 100644
--- a/src/plotman/resources/plotman.yaml
+++ b/src/plotman/resources/plotman.yaml
@@ -92,6 +92,16 @@ directories:
                 env:
                         command: rsync
                         site_root: /farm/sites
+
+                # The disk space script must return a line for eacho directory
+                # to consider archiving to with the following form.
+                #
+                #   /some/path:1000000000000
+                #
+                # That line tells plotman that it should consider archiving
+                # plots to files at paths such as /some/path/theplotid.plot and
+                # that there is 1TB of space available for use in that
+                # directory.
                 disk_space_script: |
                         #!/bin/bash
                         df -BK | grep " ${site_root}/" | awk '{ gsub(/K$/,"",$4); print $6 ":" $4*1024 }'

From d512f6b35e9d790f302c61028923245f84068bf3 Mon Sep 17 00:00:00 2001
From: Kyle Altendorf <sda@fstab.net>
Date: Wed, 26 May 2021 23:24:58 -0400
Subject: [PATCH 39/58] lots of logging from the disk script

---
 src/plotman/archive.py     | 39 +++++++++++++++++++++++---------------
 src/plotman/interactive.py | 14 ++++++++++----
 src/plotman/plotman.py     |  4 ++--
 src/plotman/reporting.py   |  7 ++++---
 4 files changed, 40 insertions(+), 24 deletions(-)

diff --git a/src/plotman/archive.py b/src/plotman/archive.py
index 8fa94efb..1341dd7d 100644
--- a/src/plotman/archive.py
+++ b/src/plotman/archive.py
@@ -20,7 +20,7 @@ def spawn_archive_process(dir_cfg, all_jobs):
     '''Spawns a new archive process using the command created
     in the archive() function. Returns archiving status and a log message to print.'''
 
-    log_message = None
+    log_messages = []
     archiving_status = None
 
     # Look for running archive jobs.  Be robust to finding more than one
@@ -28,7 +28,8 @@ def spawn_archive_process(dir_cfg, all_jobs):
     arch_jobs = get_running_archive_jobs(dir_cfg.archive)
 
     if not arch_jobs:
-        (should_start, status_or_cmd) = archive(dir_cfg, all_jobs)
+        (should_start, status_or_cmd, archive_log_messages) = archive(dir_cfg, all_jobs)
+        log_messages.extend(archive_log_messages)
         if not should_start:
             archiving_status = status_or_cmd
         else:
@@ -39,7 +40,7 @@ def spawn_archive_process(dir_cfg, all_jobs):
                     stdout=subprocess.DEVNULL,
                     stderr=subprocess.STDOUT,
                     start_new_session=True)
-            log_message = 'Starting archive: ' + args['args']
+            log_messages.append('Starting archive: ' + args['args'])
             # At least for now it seems that even if we get a new running
             # archive jobs list it doesn't contain the new rsync process.
             # My guess is that this is because the bash in the middle due to
@@ -51,7 +52,7 @@ def spawn_archive_process(dir_cfg, all_jobs):
     if archiving_status is None:
         archiving_status = 'pid: ' + ', '.join(map(str, arch_jobs))
 
-    return archiving_status, log_message
+    return archiving_status, log_messages
 
 def compute_priority(phase, gb_free, n_plots):
     # All these values are designed around dst buffer dirs of about
@@ -86,6 +87,8 @@ def compute_priority(phase, gb_free, n_plots):
     return priority
 
 def get_archdir_freebytes(arch_cfg):
+    log_messages = []
+
     archdir_freebytes = {}
     completed_process = subprocess.run(
         [arch_cfg.disk_space_path],
@@ -95,16 +98,23 @@ def get_archdir_freebytes(arch_cfg):
         stderr=subprocess.PIPE,
     )
 
-    for line in completed_process.stdout.splitlines():
+    for line in completed_process.stdout.strip().splitlines():
+        line = line.strip()
         split = line.split(':')
         if len(split) != 2:
-            # TODO: warning?  or something...
+            log_messages.append(f'Unable to parse disk script line: {line!r}')
             continue
         archdir, space = split
         freebytes = int(space)
-        archdir_freebytes[archdir] = freebytes
+        archdir_freebytes[archdir.strip()] = freebytes
+
+    stderr = completed_process.stderr.strip()
+    if len(stderr) > 0:
+        log_messages.append('stderr from archive script:')
+        for line in stderr.splitlines():
+            log_messages.append(f'    {line}')
 
-    return archdir_freebytes
+    return archdir_freebytes, log_messages
 
 # TODO: maybe consolidate with similar code in job.py?
 def get_running_archive_jobs(arch_cfg):
@@ -130,7 +140,7 @@ def archive(dir_cfg, all_jobs):
     if we should not execute an archive job or (True, <cmd>) with the archive
     command if we should.'''
     if dir_cfg.archive is None:
-        return (False, "No 'archive' settings declared in plotman.yaml")
+        return (False, "No 'archive' settings declared in plotman.yaml", [])
 
     dir2ph = manager.dstdirs_to_furthest_phase(all_jobs)
     best_priority = -100000000
@@ -147,7 +157,7 @@ def archive(dir_cfg, all_jobs):
             chosen_plot = dir_plots[0]
 
     if not chosen_plot:
-        return (False, 'No plots found')
+        return (False, 'No plots found', [])
 
     # TODO: sanity check that archive machine is available
     # TODO: filter drives mounted RO
@@ -155,9 +165,9 @@ def archive(dir_cfg, all_jobs):
     #
     # Pick first archive dir with sufficient space
     #
-    archdir_freebytes = get_archdir_freebytes(dir_cfg.archive)
+    archdir_freebytes, log_messages = get_archdir_freebytes(dir_cfg.archive)
     if not archdir_freebytes:
-        return(False, 'No free archive dirs found.')
+        return(False, 'No free archive dirs found.', [])
 
     archdir = ''
     available = [(d, space) for (d, space) in archdir_freebytes.items() if
@@ -167,9 +177,8 @@ def archive(dir_cfg, all_jobs):
         (archdir, freespace) = sorted(available)[index]
 
     if not archdir:
-        return(False, 'No archive directories found with enough free space')
+        return(False, 'No archive directories found with enough free space', [])
 
-    msg = 'Found %s with ~%d GB free' % (archdir, freespace / plot_util.GB)
     archive = dir_cfg.archive
     env = dir_cfg.archive.environment(
         source=chosen_plot,
@@ -180,4 +189,4 @@ def archive(dir_cfg, all_jobs):
         'env': {**os.environ, **env}
     }
 
-    return (True, subprocess_arguments)
+    return (True, subprocess_arguments, log_messages)
diff --git a/src/plotman/interactive.py b/src/plotman/interactive.py
index 341a33fe..57d37572 100644
--- a/src/plotman/interactive.py
+++ b/src/plotman/interactive.py
@@ -128,11 +128,13 @@ def curses_main(stdscr, cfg):
 
             if archiving_configured:
                 if archiving_active:
-                    archiving_status, log_message = archive.spawn_archive_process(cfg.directories, jobs)
-                    if log_message:
+                    archiving_status, log_messages = archive.spawn_archive_process(cfg.directories, jobs)
+                    for log_message in log_messages:
                         log.log(log_message)
 
-                archdir_freebytes = archive.get_archdir_freebytes(cfg.directories.archive)
+                archdir_freebytes, log_messages = archive.get_archdir_freebytes(cfg.directories.archive)
+                for log_message in log_messages:
+                    log.log(log_message)
 
 
         # Get terminal size.  Recommended method is stdscr.getmaxyx(), but this
@@ -167,7 +169,11 @@ def curses_main(stdscr, cfg):
         dst_dir = cfg.directories.get_dst_directories()
         dst_prefix = os.path.commonpath(dst_dir)
         if archiving_configured:
-            arch_prefix = os.path.commonpath(archdir_freebytes.keys())
+            archive_directories = archdir_freebytes.keys()
+            if len(archive_directories) == 0:
+                arch_prefix = ''
+            else:
+                arch_prefix = os.path.commonpath(archive_directories)
 
         n_tmpdirs = len(cfg.directories.tmp)
 
diff --git a/src/plotman/plotman.py b/src/plotman/plotman.py
index 9fe97fa2..3c2d7610 100755
--- a/src/plotman/plotman.py
+++ b/src/plotman/plotman.py
@@ -190,8 +190,8 @@ def main():
                         jobs = Job.get_running_jobs(cfg.directories.log)
                     firstit = False
 
-                    archiving_status, log_message = archive.spawn_archive_process(cfg.directories, jobs)
-                    if log_message:
+                    archiving_status, log_messages = archive.spawn_archive_process(cfg.directories, jobs)
+                    for log_message in log_messages:
                         print(log_message)
 
 
diff --git a/src/plotman/reporting.py b/src/plotman/reporting.py
index b6dd69c4..d21ba01c 100644
--- a/src/plotman/reporting.py
+++ b/src/plotman/reporting.py
@@ -192,7 +192,7 @@ def dst_dir_report(jobs, dstdirs, width, prefix=''):
     return tab.draw()
 
 def arch_dir_report(archdir_freebytes, width, prefix=''):
-    cells = ['%s:%5dGB' % (abbr_path(d, prefix), int(int(space) / plot_util.GB))
+    cells = ['%s:%5dG' % (abbr_path(d, prefix), int(int(space) / plot_util.GB))
             for (d, space) in sorted(archdir_freebytes.items())]
     if not cells:
         return ''
@@ -214,9 +214,10 @@ def dirs_report(jobs, dir_cfg, sched_cfg, width):
         dst_dir_report(jobs, dst_dir, width),
     ]
     if dir_cfg.archive is not None:
+        freebytes, archive_log_messages = archive.get_archdir_freebytes(dir_cfg.archive)
         reports.extend([
             'archive dirs free space:',
-            arch_dir_report(archive.get_archdir_freebytes(dir_cfg.archive), width),
+            arch_dir_report(freebytes, width),
         ])
 
-    return '\n'.join(reports) + '\n'
+    return '\n'.join(reports) + '\n' + '\n'.join(archive_log_messages)

From 8fa5f37bcd435d01718708e4d603e48f0da35aa3 Mon Sep 17 00:00:00 2001
From: Kyle Altendorf <sda@fstab.net>
Date: Wed, 26 May 2021 23:34:47 -0400
Subject: [PATCH 40/58] fixup

---
 src/plotman/archive.py | 12 +++++++-----
 1 file changed, 7 insertions(+), 5 deletions(-)

diff --git a/src/plotman/archive.py b/src/plotman/archive.py
index 1341dd7d..3d494488 100644
--- a/src/plotman/archive.py
+++ b/src/plotman/archive.py
@@ -139,8 +139,9 @@ def archive(dir_cfg, all_jobs):
     contention on the plotting dstdir drives.  Returns either (False, <reason>)
     if we should not execute an archive job or (True, <cmd>) with the archive
     command if we should.'''
+    log_messages = []
     if dir_cfg.archive is None:
-        return (False, "No 'archive' settings declared in plotman.yaml", [])
+        return (False, "No 'archive' settings declared in plotman.yaml", log_messages)
 
     dir2ph = manager.dstdirs_to_furthest_phase(all_jobs)
     best_priority = -100000000
@@ -157,7 +158,7 @@ def archive(dir_cfg, all_jobs):
             chosen_plot = dir_plots[0]
 
     if not chosen_plot:
-        return (False, 'No plots found', [])
+        return (False, 'No plots found', log_messages)
 
     # TODO: sanity check that archive machine is available
     # TODO: filter drives mounted RO
@@ -165,9 +166,10 @@ def archive(dir_cfg, all_jobs):
     #
     # Pick first archive dir with sufficient space
     #
-    archdir_freebytes, log_messages = get_archdir_freebytes(dir_cfg.archive)
+    archdir_freebytes, freebytes_log_messages = get_archdir_freebytes(dir_cfg.archive)
+    log_messages.extend(freebytes_log_messages)
     if not archdir_freebytes:
-        return(False, 'No free archive dirs found.', [])
+        return(False, 'No free archive dirs found.', log_messages)
 
     archdir = ''
     available = [(d, space) for (d, space) in archdir_freebytes.items() if
@@ -177,7 +179,7 @@ def archive(dir_cfg, all_jobs):
         (archdir, freespace) = sorted(available)[index]
 
     if not archdir:
-        return(False, 'No archive directories found with enough free space', [])
+        return(False, 'No archive directories found with enough free space', log_messages)
 
     archive = dir_cfg.archive
     env = dir_cfg.archive.environment(

From 816f430fb8a86fee01f26b55fd90087935996511 Mon Sep 17 00:00:00 2001
From: Kyle Altendorf <sda@fstab.net>
Date: Thu, 27 May 2021 09:11:12 -0400
Subject: [PATCH 41/58] slight refactor for reporting

---
 src/plotman/reporting.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/src/plotman/reporting.py b/src/plotman/reporting.py
index d21ba01c..bf314a0e 100644
--- a/src/plotman/reporting.py
+++ b/src/plotman/reporting.py
@@ -218,6 +218,7 @@ def dirs_report(jobs, dir_cfg, sched_cfg, width):
         reports.extend([
             'archive dirs free space:',
             arch_dir_report(freebytes, width),
+            *archive_log_messages,
         ])
 
-    return '\n'.join(reports) + '\n' + '\n'.join(archive_log_messages)
+    return '\n'.join(reports) + '\n'

From 87cbdb82073faf69a43ad2a981435580447fded9 Mon Sep 17 00:00:00 2001
From: Kyle Altendorf <sda@fstab.net>
Date: Thu, 27 May 2021 13:25:01 +0000
Subject: [PATCH 42/58] less recalculation in get_running_archive_jobs()

---
 src/plotman/archive.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/src/plotman/archive.py b/src/plotman/archive.py
index 3d494488..d71c1d13 100644
--- a/src/plotman/archive.py
+++ b/src/plotman/archive.py
@@ -121,12 +121,12 @@ def get_running_archive_jobs(arch_cfg):
     '''Look for running rsync jobs that seem to match the pattern we use for archiving
        them.  Return a list of PIDs of matching jobs.'''
     jobs = []
+    variables = {**os.environ, **arch_cfg.environment()}
+    dest = arch_cfg.transfer_process_argument_prefix.format(**variables)
+    proc_name = arch_cfg.transfer_process_name.format(**variables)
     for proc in psutil.process_iter():
         with contextlib.suppress(psutil.NoSuchProcess):
             with proc.oneshot():
-                variables = {**os.environ, **arch_cfg.environment()}
-                dest = arch_cfg.transfer_process_argument_prefix.format(**variables)
-                proc_name = arch_cfg.transfer_process_name.format(**variables)
                 if proc.name() == proc_name:
                     args = proc.cmdline()
                     for arg in args:

From cf12e4c47b54fe38400aa721c45d300847dfd83f Mon Sep 17 00:00:00 2001
From: Kyle Altendorf <sda@fstab.net>
Date: Thu, 27 May 2021 22:45:54 -0400
Subject: [PATCH 43/58] hold multiple archival target definitions

---
 src/plotman/archive.py             |  27 +++---
 src/plotman/configuration.py       |  43 +++++----
 src/plotman/interactive.py         |   6 +-
 src/plotman/reporting.py           |   6 +-
 src/plotman/resources/plotman.yaml | 147 +++++++++++++++--------------
 5 files changed, 121 insertions(+), 108 deletions(-)

diff --git a/src/plotman/archive.py b/src/plotman/archive.py
index d71c1d13..d246d097 100644
--- a/src/plotman/archive.py
+++ b/src/plotman/archive.py
@@ -16,7 +16,7 @@
 
 # TODO : write-protect and delete-protect archived plots
 
-def spawn_archive_process(dir_cfg, all_jobs):
+def spawn_archive_process(dir_cfg, arch_cfg, all_jobs):
     '''Spawns a new archive process using the command created
     in the archive() function. Returns archiving status and a log message to print.'''
 
@@ -25,10 +25,10 @@ def spawn_archive_process(dir_cfg, all_jobs):
 
     # Look for running archive jobs.  Be robust to finding more than one
     # even though the scheduler should only run one at a time.
-    arch_jobs = get_running_archive_jobs(dir_cfg.archive)
+    arch_jobs = get_running_archive_jobs(arch_cfg)
 
     if not arch_jobs:
-        (should_start, status_or_cmd, archive_log_messages) = archive(dir_cfg, all_jobs)
+        (should_start, status_or_cmd, archive_log_messages) = archive(dir_cfg, arch_cfg, all_jobs)
         log_messages.extend(archive_log_messages)
         if not should_start:
             archiving_status = status_or_cmd
@@ -88,10 +88,11 @@ def compute_priority(phase, gb_free, n_plots):
 
 def get_archdir_freebytes(arch_cfg):
     log_messages = []
+    target = arch_cfg.target_definition()
 
     archdir_freebytes = {}
     completed_process = subprocess.run(
-        [arch_cfg.disk_space_path],
+        [target.disk_space_path],
         encoding='utf-8',
         env={**os.environ, **arch_cfg.environment()},
         stdout=subprocess.PIPE,
@@ -121,9 +122,10 @@ def get_running_archive_jobs(arch_cfg):
     '''Look for running rsync jobs that seem to match the pattern we use for archiving
        them.  Return a list of PIDs of matching jobs.'''
     jobs = []
+    target = arch_cfg.target_definition()
     variables = {**os.environ, **arch_cfg.environment()}
-    dest = arch_cfg.transfer_process_argument_prefix.format(**variables)
-    proc_name = arch_cfg.transfer_process_name.format(**variables)
+    dest = target.transfer_process_argument_prefix.format(**variables)
+    proc_name = target.transfer_process_name.format(**variables)
     for proc in psutil.process_iter():
         with contextlib.suppress(psutil.NoSuchProcess):
             with proc.oneshot():
@@ -134,13 +136,13 @@ def get_running_archive_jobs(arch_cfg):
                             jobs.append(proc.pid)
     return jobs
 
-def archive(dir_cfg, all_jobs):
+def archive(dir_cfg, arch_cfg, all_jobs):
     '''Configure one archive job.  Needs to know all jobs so it can avoid IO
     contention on the plotting dstdir drives.  Returns either (False, <reason>)
     if we should not execute an archive job or (True, <cmd>) with the archive
     command if we should.'''
     log_messages = []
-    if dir_cfg.archive is None:
+    if arch_cfg is None:
         return (False, "No 'archive' settings declared in plotman.yaml", log_messages)
 
     dir2ph = manager.dstdirs_to_furthest_phase(all_jobs)
@@ -166,7 +168,7 @@ def archive(dir_cfg, all_jobs):
     #
     # Pick first archive dir with sufficient space
     #
-    archdir_freebytes, freebytes_log_messages = get_archdir_freebytes(dir_cfg.archive)
+    archdir_freebytes, freebytes_log_messages = get_archdir_freebytes(arch_cfg)
     log_messages.extend(freebytes_log_messages)
     if not archdir_freebytes:
         return(False, 'No free archive dirs found.', log_messages)
@@ -175,19 +177,18 @@ def archive(dir_cfg, all_jobs):
     available = [(d, space) for (d, space) in archdir_freebytes.items() if
                  space > 1.2 * plot_util.get_k32_plotsize()]
     if len(available) > 0:
-        index = min(dir_cfg.archive.index, len(available) - 1)
+        index = min(arch_cfg.index, len(available) - 1)
         (archdir, freespace) = sorted(available)[index]
 
     if not archdir:
         return(False, 'No archive directories found with enough free space', log_messages)
 
-    archive = dir_cfg.archive
-    env = dir_cfg.archive.environment(
+    env = arch_cfg.environment(
         source=chosen_plot,
         destination=archdir,
     )
     subprocess_arguments = {
-        'args': archive.transfer_path,
+        'args': arch_cfg.target_definition().transfer_path,
         'env': {**os.environ, **env}
     }
 
diff --git a/src/plotman/configuration.py b/src/plotman/configuration.py
index fcde55fc..ac9efa55 100644
--- a/src/plotman/configuration.py
+++ b/src/plotman/configuration.py
@@ -66,26 +66,36 @@ def get_validated_configs(config_text, config_path):
 
 # TODO: bah, mutable?  bah.
 @attr.mutable
-class Archive:
+class ArchivingTarget:
     transfer_process_name: str
     transfer_process_argument_prefix: str
-    index: int = 0  # If not explicit, "index" will default to 0
-    # TODO: mutable attribute...
-    env: Dict[str, str] = attr.ib(factory=dict)
     disk_space_path: Optional[str] = None
     disk_space_script: Optional[str] = None
     transfer_path: Optional[str] = None
     transfer_script: Optional[str] = None
 
+# TODO: bah, mutable?  bah.
+@attr.mutable
+class Archiving:
+    target: str
+    # TODO: mutable attribute...
+    env: Dict[str, str]
+    index: int = 0  # If not explicit, "index" will default to 0
+    target_definitions: Dict[str, ArchivingTarget] = attr.ib(factory=dict)
+
+    def target_definition(self):
+        return self.target_definitions[self.target]
+
     def environment(
             self,
             source=None,
             destination=None,
     ):
         complete = dict(self.env)
+        target = self.target_definition()
 
         variables = {**os.environ, **complete}
-        complete['process_name'] = self.transfer_process_name.format(**variables)
+        complete['process_name'] = target.transfer_process_name.format(**variables)
 
         if source is not None:
             complete['source'] = source
@@ -97,8 +107,9 @@ def environment(
 
     def maybe_create_scripts(self, temp):
         rwx = stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR
+        target = self.target_definition()
 
-        if self.disk_space_path is None:
+        if target.disk_space_path is None:
             with tempfile.NamedTemporaryFile(
                 mode='w',
                 encoding='utf-8',
@@ -106,12 +117,12 @@ def maybe_create_scripts(self, temp):
                 delete=False,
                 dir=temp,
             ) as disk_space_script_file:
-                disk_space_script_file.write(self.disk_space_script)
+                disk_space_script_file.write(target.disk_space_script)
 
-            self.disk_space_path = disk_space_script_file.name
-            os.chmod(self.disk_space_path, rwx)
+            target.disk_space_path = disk_space_script_file.name
+            os.chmod(target.disk_space_path, rwx)
 
-        if self.transfer_path is None:
+        if target.transfer_path is None:
             with tempfile.NamedTemporaryFile(
                 mode='w',
                 encoding='utf-8',
@@ -119,10 +130,10 @@ def maybe_create_scripts(self, temp):
                 delete=False,
                 dir=temp,
             ) as transfer_script_file:
-                transfer_script_file.write(self.transfer_script)
+                transfer_script_file.write(target.transfer_script)
 
-            self.transfer_path = transfer_script_file.name
-            os.chmod(self.transfer_path, rwx)
+            target.transfer_path = transfer_script_file.name
+            os.chmod(target.transfer_path, rwx)
 
 @attr.frozen
 class TmpOverrides:
@@ -135,7 +146,6 @@ class Directories:
     dst: Optional[List[str]] = None
     tmp2: Optional[str] = None
     tmp_overrides: Optional[Dict[str, TmpOverrides]] = None
-    archive: Optional[Archive] = None
 
     def dst_is_tmp(self):
         return self.dst is None
@@ -180,6 +190,7 @@ class PlotmanConfig:
     directories: Directories
     scheduling: Scheduling
     plotting: Plotting
+    archiving: Optional[Archiving] = None
     user_interface: UserInterface = attr.ib(factory=UserInterface)
     version: List[int] = [0]
 
@@ -188,7 +199,7 @@ def setup(self):
         prefix = f'plotman-pid_{os.getpid()}-'
 
         with tempfile.TemporaryDirectory(prefix=prefix) as temp:
-            if self.directories.archive is not None:
-                self.directories.archive.maybe_create_scripts(temp=temp)
+            if self.archiving is not None:
+                self.archiving.maybe_create_scripts(temp=temp)
 
             yield
diff --git a/src/plotman/interactive.py b/src/plotman/interactive.py
index 57d37572..a85002f1 100644
--- a/src/plotman/interactive.py
+++ b/src/plotman/interactive.py
@@ -66,7 +66,7 @@ def curses_main(stdscr, cfg):
     log = Log()
 
     plotting_active = True
-    archiving_configured = cfg.directories.archive is not None
+    archiving_configured = cfg.archiving is not None
     archiving_active = archiving_configured
 
     plotting_status = '<startup>'    # todo rename these msg?
@@ -128,11 +128,11 @@ def curses_main(stdscr, cfg):
 
             if archiving_configured:
                 if archiving_active:
-                    archiving_status, log_messages = archive.spawn_archive_process(cfg.directories, jobs)
+                    archiving_status, log_messages = archive.spawn_archive_process(cfg.directories, cfg.archiving, jobs)
                     for log_message in log_messages:
                         log.log(log_message)
 
-                archdir_freebytes, log_messages = archive.get_archdir_freebytes(cfg.directories.archive)
+                archdir_freebytes, log_messages = archive.get_archdir_freebytes(cfg.archiving)
                 for log_message in log_messages:
                     log.log(log_message)
 
diff --git a/src/plotman/reporting.py b/src/plotman/reporting.py
index bf314a0e..670d82ef 100644
--- a/src/plotman/reporting.py
+++ b/src/plotman/reporting.py
@@ -207,14 +207,14 @@ def arch_dir_report(archdir_freebytes, width, prefix=''):
     return tab.draw()
 
 # TODO: remove this
-def dirs_report(jobs, dir_cfg, sched_cfg, width):
+def dirs_report(jobs, dir_cfg, arch_cfg, sched_cfg, width):
     dst_dir = dir_cfg.get_dst_directories()
     reports = [
         tmp_dir_report(jobs, dir_cfg, sched_cfg, width),
         dst_dir_report(jobs, dst_dir, width),
     ]
-    if dir_cfg.archive is not None:
-        freebytes, archive_log_messages = archive.get_archdir_freebytes(dir_cfg.archive)
+    if arch_cfg is not None:
+        freebytes, archive_log_messages = archive.get_archdir_freebytes(arch_cfg)
         reports.extend([
             'archive dirs free space:',
             arch_dir_report(freebytes, width),
diff --git a/src/plotman/resources/plotman.yaml b/src/plotman/resources/plotman.yaml
index dd85439c..b5932368 100644
--- a/src/plotman/resources/plotman.yaml
+++ b/src/plotman/resources/plotman.yaml
@@ -61,79 +61,80 @@ directories:
                 - /mnt/dst/00
                 - /mnt/dst/01
 
-        # Archival configuration.  Optional; if you do not wish to run the
-        # archiving operation, comment this section out.  Almost everyone
-        # should be using the archival feature.  It is meant to distribute
-        # plots among multiple disks filling them all.  This can be done both
-        # to local and to remote disks.
-        #
-        # As of v0.4, archiving commands are highly configurable.  The basic
-        # configuration consists of a script for checking available disk space
-        # and another for actually transferring plots.  Each can be specified
-        # as either a path to an existing script or inline script contents.
-        # It is expected that most people will use existing recipes and will
-        # adjust them by specifying environment variables that will set their
-        # system specific values.  These can be provided to the scripts via
-        # the `env` key.  plotman will additionally provide `source` and
-        # `destination` environment variables to the transfer script so it
-        # knows the specifically selected items to process.  plotman also needs
-        # to be able to generally detect if a transfer process is already
-        # running.  To be able to identify externally launched transfers, the
-        # process name and an argument prefix to match must be provided.  Note
-        # that variable substitution of environment variables including those
-        # specified in the env key can be used in both process name and process
-        # argument prefix elements but that they use the python substitution
-        # format.
-        #
-        # Complete example: https://github.com/ericaltendorf/plotman/wiki/Archiving
-        archive:
-                # -- local rsync example setup
-
-                env:
-                        command: rsync
-                        site_root: /farm/sites
-
-                # The disk space script must return a line for eacho directory
-                # to consider archiving to with the following form.
-                #
-                #   /some/path:1000000000000
-                #
-                # That line tells plotman that it should consider archiving
-                # plots to files at paths such as /some/path/theplotid.plot and
-                # that there is 1TB of space available for use in that
-                # directory.
-                disk_space_script: |
-                        #!/bin/bash
-                        df -BK | grep " ${site_root}/" | awk '{ gsub(/K$/,"",$4); print $6 ":" $4*1024 }'
-                transfer_script: |
-                        #!/bin/bash
-                        "${command}" --inplace --preallocate --remove-source-files --skip-compress plot --whole-file "${source}" "${destination}"
-                transfer_process_name: "{command}"
-                transfer_process_argument_prefix: "{site_root}"
-
-                # -- remote rsync example setup
-                #
-                # env:
-                #         command: rsync
-                #         site_root: /farm/sites
-                #         url_root: rsync://altendky@server:12000/sites
-                # disk_space_script: |
-                #         #!/bin/bash
-                #         ssh altendky@server "df -BK | grep \" ${site_root}/\" | awk '{ gsub(/K\$/,\"\",\$4); print \$6 \":\" \$4*1024 }'"
-                # transfer_script: |
-                #         #!/bin/bash
-                #         relative_path=$(realpath --relative-to="${site_root}" "${destination}")
-                #         "${command}" --bwlimit=80000 --inplace --preallocate --remove-source-files --skip-compress plot --whole-file "${source}" "${url_root}/${relative_path}"
-                # transfer_process_name: "{command}"
-                # transfer_process_argument_prefix: "{url_root}"
-
-                # -- external script example
-                # env:
-                #         some_common_value: /a/path
-                # disk_space_path: /home/me/my_disk_space_script.sh
-                # transfer_path: /home/me/my_transfer_script.sh
-                # transfer_process_name: rsync
-                # transfer_process_argument_prefix: /the/destination/directory/root
+# Archival configuration.  Optional; if you do not wish to run the
+# archiving operation, comment this section out.  Almost everyone
+# should be using the archival feature.  It is meant to distribute
+# plots among multiple disks filling them all.  This can be done both
+# to local and to remote disks.
+#
+# As of v0.4, archiving commands are highly configurable.  The basic
+# configuration consists of a script for checking available disk space
+# and another for actually transferring plots.  Each can be specified
+# as either a path to an existing script or inline script contents.
+# It is expected that most people will use existing recipes and will
+# adjust them by specifying environment variables that will set their
+# system specific values.  These can be provided to the scripts via
+# the `env` key.  plotman will additionally provide `source` and
+# `destination` environment variables to the transfer script so it
+# knows the specifically selected items to process.  plotman also needs
+# to be able to generally detect if a transfer process is already
+# running.  To be able to identify externally launched transfers, the
+# process name and an argument prefix to match must be provided.  Note
+# that variable substitution of environment variables including those
+# specified in the env key can be used in both process name and process
+# argument prefix elements but that they use the python substitution
+# format.
+#
+# Complete example: https://github.com/ericaltendorf/plotman/wiki/Archiving
+archiving:
+  target: local_rsync
+  env:
+    command: rsync
+    site_root: /farm/sites
+  target_definitions:
+    local_rsync:
+      # env:
+      #   command: rsync
+      #   site_root: /farm/sites
+
+      # The disk space script must return a line for each directory
+      # to consider archiving to with the following form.
+      #
+      #   /some/path:1000000000000
+      #
+      # That line tells plotman that it should consider archiving
+      # plots to files at paths such as /some/path/theplotid.plot and
+      # that there is 1TB of space available for use in that
+      # directory.
+      disk_space_script: |
+        #!/bin/bash
+        df -BK | grep " ${site_root}/" | awk '{ gsub(/K$/,"",$4); print $6 ":" $4*1024 }'
+      transfer_script: |
+        #!/bin/bash
+        "${command}" --preallocate --remove-source-files --skip-compress plot --whole-file "${source}" "${destination}"
+      transfer_process_name: "{command}"
+      transfer_process_argument_prefix: "{site_root}"
+    rsyncd:
+      # env:
+      #   command: rsync
+      #   site_root: /farm/sites
+      #   url_root: rsync://altendky@server:12000/sites
+      disk_space_script: |
+        #!/bin/bash
+        ssh altendky@server "df -BK | grep \" ${site_root}/\" | awk '{ gsub(/K\$/,\"\",\$4); print \$6 \":\" \$4*1024 }'"
+      transfer_script: |
+        #!/bin/bash
+        relative_path=$(realpath --relative-to="${site_root}" "${destination}")
+        "${command}" --bwlimit=80000 --preallocate --remove-source-files --skip-compress plot --whole-file "${source}" "${url_root}/${relative_path}"
+      transfer_process_name: "{command}"
+      transfer_process_argument_prefix: "{url_root}"
+    external_script:
+      # env:
+      #   some_common_value: /a/path
+      disk_space_path: /home/me/my_disk_space_script.sh
+      transfer_path: /home/me/my_transfer_script.sh
+      transfer_process_name: rsync
+      transfer_process_argument_prefix: /the/destination/directory/root
 
 # Plotting scheduling parameters
 scheduling:

From efbe6d0d3579a6d67f354e48b26613a04021f88f Mon Sep 17 00:00:00 2001
From: Kyle Altendorf <sda@fstab.net>
Date: Fri, 28 May 2021 12:49:37 -0400
Subject: [PATCH 44/58] add env with defaults and mandatories for target
 definitions

---
 src/plotman/configuration.py       | 18 ++++++++++++++++--
 src/plotman/resources/plotman.yaml | 30 +++++++++++++++++++-----------
 2 files changed, 35 insertions(+), 13 deletions(-)

diff --git a/src/plotman/configuration.py b/src/plotman/configuration.py
index ac9efa55..02324ea0 100644
--- a/src/plotman/configuration.py
+++ b/src/plotman/configuration.py
@@ -69,13 +69,15 @@ def get_validated_configs(config_text, config_path):
 class ArchivingTarget:
     transfer_process_name: str
     transfer_process_argument_prefix: str
+    # TODO: mutable attribute...
+    env: Dict[str, Optional[str]] = attr.ib(factory=dict)
     disk_space_path: Optional[str] = None
     disk_space_script: Optional[str] = None
     transfer_path: Optional[str] = None
     transfer_script: Optional[str] = None
 
 # TODO: bah, mutable?  bah.
-@attr.mutable
+@attr.frozen
 class Archiving:
     target: str
     # TODO: mutable attribute...
@@ -91,8 +93,20 @@ def environment(
             source=None,
             destination=None,
     ):
-        complete = dict(self.env)
         target = self.target_definition()
+        complete = {**target.env, **self.env}
+
+        missing_mandatory_keys = [
+            key
+            for key, value in complete.items()
+            if value is None
+        ]
+
+        if len(missing_mandatory_keys) > 0:
+            target = repr(self.target)
+            missing = ', '.join(repr(key) for key in missing_mandatory_keys)
+            message = f'Missing env options for archival target {target}: {missing}'
+            raise Exception(message)
 
         variables = {**os.environ, **complete}
         complete['process_name'] = target.transfer_process_name.format(**variables)
diff --git a/src/plotman/resources/plotman.yaml b/src/plotman/resources/plotman.yaml
index b5932368..b5ba9945 100644
--- a/src/plotman/resources/plotman.yaml
+++ b/src/plotman/resources/plotman.yaml
@@ -93,9 +93,10 @@ archiving:
     site_root: /farm/sites
   target_definitions:
     local_rsync:
-      # env:
-      #   command: rsync
-      #   site_root: /farm/sites
+      env:
+        command: rsync
+        options: --preallocate --remove-source-files --skip-compress plot --whole-file
+        site_root: null
 
       # The disk space script must return a line for each directory
       # to consider archiving to with the following form.
@@ -111,26 +112,33 @@ archiving:
         df -BK | grep " ${site_root}/" | awk '{ gsub(/K$/,"",$4); print $6 ":" $4*1024 }'
       transfer_script: |
         #!/bin/bash
-        "${command}" --preallocate --remove-source-files --skip-compress plot --whole-file "${source}" "${destination}"
+        "${command}" ${options} "${source}" "${destination}"
       transfer_process_name: "{command}"
       transfer_process_argument_prefix: "{site_root}"
     rsyncd:
-      # env:
-      #   command: rsync
-      #   site_root: /farm/sites
-      #   url_root: rsync://altendky@server:12000/sites
+      env:
+        # A value of null indicates a mandatory option
+        command: rsync
+        options: --bwlimit=80000 --preallocate --remove-source-files --skip-compress plot --whole-file
+        port: "873"
+        user: null
+        host: null
+        site_root: null
+        site: null
       disk_space_script: |
         #!/bin/bash
         ssh altendky@server "df -BK | grep \" ${site_root}/\" | awk '{ gsub(/K\$/,\"\",\$4); print \$6 \":\" \$4*1024 }'"
       transfer_script: |
         #!/bin/bash
         relative_path=$(realpath --relative-to="${site_root}" "${destination}")
-        "${command}" --bwlimit=80000 --preallocate --remove-source-files --skip-compress plot --whole-file "${source}" "${url_root}/${relative_path}"
+        url_root="rsync://${user}@${host}:${port}/${site}"
+        "${command}" ${options} "${source}" "${url_root}/${relative_path}"
       transfer_process_name: "{command}"
-      transfer_process_argument_prefix: "{url_root}"
+      transfer_process_argument_prefix: "rsync://{user}@{host}:{port}/{site}"
     external_script:
       # env:
-      #   some_common_value: /a/path
+      #   some_common_value_with_a_default: /a/path
+      #   some_mandatory option: null
       disk_space_path: /home/me/my_disk_space_script.sh
       transfer_path: /home/me/my_transfer_script.sh
       transfer_process_name: rsync

From 22ce23f7a97eb18cc67e7753063f022dad1c4c29 Mon Sep 17 00:00:00 2001
From: Kyle Altendorf <sda@fstab.net>
Date: Fri, 28 May 2021 13:18:58 -0400
Subject: [PATCH 45/58] add preset archiving target definitions

---
 src/plotman/configuration.py                  | 39 +++++++++++++-
 src/plotman/resources/plotman.yaml            | 52 -------------------
 src/plotman/resources/target_definitions.yaml | 52 +++++++++++++++++++
 3 files changed, 90 insertions(+), 53 deletions(-)
 create mode 100644 src/plotman/resources/target_definitions.yaml

diff --git a/src/plotman/configuration.py b/src/plotman/configuration.py
index 02324ea0..6b45d66f 100644
--- a/src/plotman/configuration.py
+++ b/src/plotman/configuration.py
@@ -1,4 +1,5 @@
 import contextlib
+import importlib
 import os
 import stat
 import tempfile
@@ -11,6 +12,8 @@
 import marshmallow
 import yaml
 
+from plotman import resources as plotman_resources
+
 
 class ConfigurationException(Exception):
     """Raised when plotman.yaml configuration is missing or malformed."""
@@ -59,6 +62,19 @@ def get_validated_configs(config_text, config_path):
             f"Config file at: '{config_path}' is malformed"
         ) from e
 
+    # TODO: get this IO out
+    preset_target_definitions_text = importlib.resources.read_text(
+        plotman_resources, "target_definitions.yaml",
+    )
+    preset_target_objects = yaml.safe_load(preset_target_definitions_text)
+    preset_target_schema = desert.schema(PresetTargetDefinitions)
+    preset_target_definitions = preset_target_schema.load(preset_target_objects)
+
+    loaded.archiving.target_definitions = {
+        **preset_target_definitions.target_definitions,
+        **loaded.archiving.target_definitions,
+    }
+
     return loaded
 
 
@@ -76,8 +92,29 @@ class ArchivingTarget:
     transfer_path: Optional[str] = None
     transfer_script: Optional[str] = None
 
-# TODO: bah, mutable?  bah.
+# local_rsync_archiving_target = ArchivingTarget(
+#     transfer_process_name='{site_root}',
+#     transfer_process_argument_prefix='{command}',
+#     env={
+#         'command': 'rsync',
+#         'options': '--preallocate --remove-source-files --skip-compress plot --whole-file',
+#         'site_root': None,
+#     },
+#     disk_space_script=textwrap.dedent('''\
+#         #!/bin/bash
+#         df -BK | grep " ${site_root}/" | awk '{ gsub(/K$/,"",$4); print $6 ":" $4*1024 }'
+#     '''),
+#     transfer_script=textwrap.dedent('''\
+#
+#     ''')
+# )
+
 @attr.frozen
+class PresetTargetDefinitions:
+    target_definitions: Dict[str, ArchivingTarget] = attr.ib(factory=dict)
+
+# TODO: bah, mutable?  bah.
+@attr.mutable
 class Archiving:
     target: str
     # TODO: mutable attribute...
diff --git a/src/plotman/resources/plotman.yaml b/src/plotman/resources/plotman.yaml
index b5ba9945..12024a40 100644
--- a/src/plotman/resources/plotman.yaml
+++ b/src/plotman/resources/plotman.yaml
@@ -91,58 +91,6 @@ archiving:
   env:
     command: rsync
     site_root: /farm/sites
-  target_definitions:
-    local_rsync:
-      env:
-        command: rsync
-        options: --preallocate --remove-source-files --skip-compress plot --whole-file
-        site_root: null
-
-      # The disk space script must return a line for each directory
-      # to consider archiving to with the following form.
-      #
-      #   /some/path:1000000000000
-      #
-      # That line tells plotman that it should consider archiving
-      # plots to files at paths such as /some/path/theplotid.plot and
-      # that there is 1TB of space available for use in that
-      # directory.
-      disk_space_script: |
-        #!/bin/bash
-        df -BK | grep " ${site_root}/" | awk '{ gsub(/K$/,"",$4); print $6 ":" $4*1024 }'
-      transfer_script: |
-        #!/bin/bash
-        "${command}" ${options} "${source}" "${destination}"
-      transfer_process_name: "{command}"
-      transfer_process_argument_prefix: "{site_root}"
-    rsyncd:
-      env:
-        # A value of null indicates a mandatory option
-        command: rsync
-        options: --bwlimit=80000 --preallocate --remove-source-files --skip-compress plot --whole-file
-        port: "873"
-        user: null
-        host: null
-        site_root: null
-        site: null
-      disk_space_script: |
-        #!/bin/bash
-        ssh altendky@server "df -BK | grep \" ${site_root}/\" | awk '{ gsub(/K\$/,\"\",\$4); print \$6 \":\" \$4*1024 }'"
-      transfer_script: |
-        #!/bin/bash
-        relative_path=$(realpath --relative-to="${site_root}" "${destination}")
-        url_root="rsync://${user}@${host}:${port}/${site}"
-        "${command}" ${options} "${source}" "${url_root}/${relative_path}"
-      transfer_process_name: "{command}"
-      transfer_process_argument_prefix: "rsync://{user}@{host}:{port}/{site}"
-    external_script:
-      # env:
-      #   some_common_value_with_a_default: /a/path
-      #   some_mandatory option: null
-      disk_space_path: /home/me/my_disk_space_script.sh
-      transfer_path: /home/me/my_transfer_script.sh
-      transfer_process_name: rsync
-      transfer_process_argument_prefix: /the/destination/directory/root
 
 # Plotting scheduling parameters
 scheduling:
diff --git a/src/plotman/resources/target_definitions.yaml b/src/plotman/resources/target_definitions.yaml
new file mode 100644
index 00000000..0dcb4375
--- /dev/null
+++ b/src/plotman/resources/target_definitions.yaml
@@ -0,0 +1,52 @@
+target_definitions:
+  local_rsync:
+    env:
+      command: rsync
+      options: --preallocate --remove-source-files --skip-compress plot --whole-file
+      site_root: null
+
+    # The disk space script must return a line for each directory
+    # to consider archiving to with the following form.
+    #
+    #   /some/path:1000000000000
+    #
+    # That line tells plotman that it should consider archiving
+    # plots to files at paths such as /some/path/theplotid.plot and
+    # that there is 1TB of space available for use in that
+    # directory.
+    disk_space_script: |
+      #!/bin/bash
+      df -BK | grep " ${site_root}/" | awk '{ gsub(/K$/,"",$4); print $6 ":" $4*1024 }'
+    transfer_script: |
+      #!/bin/bash
+      "${command}" ${options} "${source}" "${destination}"
+    transfer_process_name: "{command}"
+    transfer_process_argument_prefix: "{site_root}"
+  rsyncd:
+    env:
+      # A value of null indicates a mandatory option
+      command: rsync
+      options: --bwlimit=80000 --preallocate --remove-source-files --skip-compress plot --whole-file
+      port: "873"
+      user: null
+      host: null
+      site_root: null
+      site: null
+    disk_space_script: |
+      #!/bin/bash
+      ssh altendky@server "df -BK | grep \" ${site_root}/\" | awk '{ gsub(/K\$/,\"\",\$4); print \$6 \":\" \$4*1024 }'"
+    transfer_script: |
+      #!/bin/bash
+      relative_path=$(realpath --relative-to="${site_root}" "${destination}")
+      url_root="rsync://${user}@${host}:${port}/${site}"
+      "${command}" ${options} "${source}" "${url_root}/${relative_path}"
+    transfer_process_name: "{command}"
+    transfer_process_argument_prefix: "rsync://{user}@{host}:{port}/{site}"
+#  external_script:
+#    env:
+#      some_common_value_with_a_default: /a/path
+#      some_mandatory option: null
+#    disk_space_path: /home/me/my_disk_space_script.sh
+#    transfer_path: /home/me/my_transfer_script.sh
+#    transfer_process_name: rsync
+#    transfer_process_argument_prefix: /the/destination/directory/root

From 4a4a4f34d93ae51f7ca6864dd790a285daa8f5c5 Mon Sep 17 00:00:00 2001
From: Kyle Altendorf <sda@fstab.net>
Date: Sat, 29 May 2021 20:26:36 -0400
Subject: [PATCH 46/58] log archive transfer activities

---
 src/plotman/archive.py                        | 53 +++++++++++++++----
 src/plotman/configuration.py                  |  3 ++
 src/plotman/manager.py                        | 13 +++--
 src/plotman/resources/target_definitions.yaml | 10 ++--
 4 files changed, 58 insertions(+), 21 deletions(-)

diff --git a/src/plotman/archive.py b/src/plotman/archive.py
index d246d097..4e2490fc 100644
--- a/src/plotman/archive.py
+++ b/src/plotman/archive.py
@@ -9,6 +9,7 @@
 import sys
 from datetime import datetime
 
+import pendulum
 import psutil
 import texttable as tt
 
@@ -34,10 +35,36 @@ def spawn_archive_process(dir_cfg, arch_cfg, all_jobs):
             archiving_status = status_or_cmd
         else:
             args = status_or_cmd
-            # TODO: do something useful with output instead of DEVNULL
-            p = subprocess.Popen(**args,
+
+            log_file_path = dir_cfg.create_log_path(group='transfer', time=pendulum.now())
+
+            # TODO: CAMPid 09840103109429840981397487498131
+            try:
+                open_log_file = open(log_file_path, 'x')
+            except FileExistsError:
+                log_messages.append(
+                    f'Archiving log file already exists, skipping attempt to start a'
+                    f' new plot: {log_file_path!r}'
+                )
+                return (False, log_messages)
+            except FileNotFoundError as e:
+                message = (
+                    f'Unable to open log file.  Verify that the directory exists'
+                    f' and has proper write permissions: {log_file_path!r}'
+                )
+                raise Exception(message) from e
+
+            # Preferably, do not add any code between the try block above
+            # and the with block below.  IOW, this space intentionally left
+            # blank...  As is, this provides a good chance that our handle
+            # of the log file will get closed explicitly while still
+            # allowing handling of just the log file opening error.
+
+            with open_log_file:
+                # start_new_sessions to make the job independent of this controlling tty.
+                p = subprocess.Popen(**args,
                     shell=True,
-                    stdout=subprocess.DEVNULL,
+                    stdout=open_log_file,
                     stderr=subprocess.STDOUT,
                     start_new_session=True)
             log_messages.append('Starting archive: ' + args['args'])
@@ -91,13 +118,19 @@ def get_archdir_freebytes(arch_cfg):
     target = arch_cfg.target_definition()
 
     archdir_freebytes = {}
-    completed_process = subprocess.run(
-        [target.disk_space_path],
-        encoding='utf-8',
-        env={**os.environ, **arch_cfg.environment()},
-        stdout=subprocess.PIPE,
-        stderr=subprocess.PIPE,
-    )
+    timeout = 2
+    try:
+        completed_process = subprocess.run(
+            [target.disk_space_path],
+            encoding='utf-8',
+            env={**os.environ, **arch_cfg.environment()},
+            stdout=subprocess.PIPE,
+            stderr=subprocess.PIPE,
+            timeout=timeout,
+        )
+    except subprocess.TimeoutExpired:
+        log_messages.append(f'Disk space check timed out in {timeout} seconds')
+        return archdir_freebytes, log_messages
 
     for line in completed_process.stdout.strip().splitlines():
         line = line.strip()
diff --git a/src/plotman/configuration.py b/src/plotman/configuration.py
index 6b45d66f..ea25c15d 100644
--- a/src/plotman/configuration.py
+++ b/src/plotman/configuration.py
@@ -210,6 +210,9 @@ def get_dst_directories(self):
 
         return self.dst
 
+    def create_log_path(self, group, time=None):
+        timestamp = time.isoformat(timespec='microseconds').replace(':', '_')
+        return os.path.join(self.log, f'{timestamp}.{group}.log')
 
 @attr.frozen
 class Scheduling:
diff --git a/src/plotman/manager.py b/src/plotman/manager.py
index 8c418f35..83dddb75 100644
--- a/src/plotman/manager.py
+++ b/src/plotman/manager.py
@@ -110,9 +110,7 @@ def maybe_start_new_plot(dir_cfg, sched_cfg, plotting_cfg):
                 else:
                     dstdir = max(dir2ph, key=dir2ph.get)
 
-            logfile = os.path.join(
-                dir_cfg.log, pendulum.now().isoformat(timespec='microseconds').replace(':', '_') + '.log'
-            )
+            log_file_path = dir_cfg.create_log_path(group='plot', time=pendulum.now())
 
             plot_args = ['chia', 'plots', 'create',
                     '-k', str(plotting_cfg.k),
@@ -135,10 +133,11 @@ def maybe_start_new_plot(dir_cfg, sched_cfg, plotting_cfg):
             if plotting_cfg.x:
                 plot_args.append('-x')  
 
-            logmsg = ('Starting plot job: %s ; logging to %s' % (' '.join(plot_args), logfile))
+            logmsg = ('Starting plot job: %s ; logging to %s' % (' '.join(plot_args), log_file_path))
 
+            # TODO: CAMPid 09840103109429840981397487498131
             try:
-                open_log_file = open(logfile, 'x')
+                open_log_file = open(log_file_path, 'x')
             except FileExistsError:
                 # The desired log file name already exists.  Most likely another
                 # plotman process already launched a new process in response to
@@ -148,13 +147,13 @@ def maybe_start_new_plot(dir_cfg, sched_cfg, plotting_cfg):
                 # plotting process, we'll get it at the next check cycle anyways.
                 message = (
                     f'Plot log file already exists, skipping attempt to start a'
-                    f' new plot: {logfile!r}'
+                    f' new plot: {log_file_path!r}'
                 )
                 return (False, logmsg)
             except FileNotFoundError as e:
                 message = (
                     f'Unable to open log file.  Verify that the directory exists'
-                    f' and has proper write permissions: {logfile!r}'
+                    f' and has proper write permissions: {log_file_path!r}'
                 )
                 raise Exception(message) from e
 
diff --git a/src/plotman/resources/target_definitions.yaml b/src/plotman/resources/target_definitions.yaml
index 0dcb4375..515a01c0 100644
--- a/src/plotman/resources/target_definitions.yaml
+++ b/src/plotman/resources/target_definitions.yaml
@@ -27,21 +27,23 @@ target_definitions:
       # A value of null indicates a mandatory option
       command: rsync
       options: --bwlimit=80000 --preallocate --remove-source-files --skip-compress plot --whole-file
-      port: "873"
+      rsync_port: "873"
+      ssh_port: "22"
       user: null
       host: null
       site_root: null
       site: null
     disk_space_script: |
       #!/bin/bash
-      ssh altendky@server "df -BK | grep \" ${site_root}/\" | awk '{ gsub(/K\$/,\"\",\$4); print \$6 \":\" \$4*1024 }'"
+      ssh -p "${ssh_port}" "${user}@${host}" "df -BK | grep \" ${site_root}/\" | awk '{ gsub(/K\$/,\"\",\$4); print \$6 \":\" \$4*1024 }'"
     transfer_script: |
       #!/bin/bash
+      echo Launching transfer activity
       relative_path=$(realpath --relative-to="${site_root}" "${destination}")
-      url_root="rsync://${user}@${host}:${port}/${site}"
+      url_root="rsync://${user}@${host}:${rsync_port}/${site}"
       "${command}" ${options} "${source}" "${url_root}/${relative_path}"
     transfer_process_name: "{command}"
-    transfer_process_argument_prefix: "rsync://{user}@{host}:{port}/{site}"
+    transfer_process_argument_prefix: "rsync://{user}@{host}:{rsync_port}/{site}"
 #  external_script:
 #    env:
 #      some_common_value_with_a_default: /a/path

From 50fcceb397fe11e12738200212672eb8f5cb563d Mon Sep 17 00:00:00 2001
From: Kyle Altendorf <sda@fstab.net>
Date: Sun, 30 May 2021 06:35:03 -0700
Subject: [PATCH 47/58] Apply suggestions from code review

---
 src/plotman/archive.py       | 4 ++--
 src/plotman/configuration.py | 2 +-
 src/plotman/plotman.py       | 2 +-
 3 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/src/plotman/archive.py b/src/plotman/archive.py
index 4e2490fc..c022c14f 100644
--- a/src/plotman/archive.py
+++ b/src/plotman/archive.py
@@ -44,7 +44,7 @@ def spawn_archive_process(dir_cfg, arch_cfg, all_jobs):
             except FileExistsError:
                 log_messages.append(
                     f'Archiving log file already exists, skipping attempt to start a'
-                    f' new plot: {log_file_path!r}'
+                    f' new archive transfer: {log_file_path!r}'
                 )
                 return (False, log_messages)
             except FileNotFoundError as e:
@@ -67,7 +67,7 @@ def spawn_archive_process(dir_cfg, arch_cfg, all_jobs):
                     stdout=open_log_file,
                     stderr=subprocess.STDOUT,
                     start_new_session=True)
-            log_messages.append('Starting archive: ' + args['args'])
+            log_messages.append(f'Starting archive: {args["args"]} ; logging to {log_file_path}')
             # At least for now it seems that even if we get a new running
             # archive jobs list it doesn't contain the new rsync process.
             # My guess is that this is because the bash in the middle due to
diff --git a/src/plotman/configuration.py b/src/plotman/configuration.py
index ea25c15d..7dbd5ce4 100644
--- a/src/plotman/configuration.py
+++ b/src/plotman/configuration.py
@@ -210,7 +210,7 @@ def get_dst_directories(self):
 
         return self.dst
 
-    def create_log_path(self, group, time=None):
+    def create_log_path(self, group, time):
         timestamp = time.isoformat(timespec='microseconds').replace(':', '_')
         return os.path.join(self.log, f'{timestamp}.{group}.log')
 
diff --git a/src/plotman/plotman.py b/src/plotman/plotman.py
index 3c2d7610..099b4961 100755
--- a/src/plotman/plotman.py
+++ b/src/plotman/plotman.py
@@ -190,7 +190,7 @@ def main():
                         jobs = Job.get_running_jobs(cfg.directories.log)
                     firstit = False
 
-                    archiving_status, log_messages = archive.spawn_archive_process(cfg.directories, jobs)
+                    archiving_status, log_messages = archive.spawn_archive_process(cfg.directories, cfg.archiving, jobs)
                     for log_message in log_messages:
                         print(log_message)
 

From c87f6e81941be15f749effeffa355e4d18106646 Mon Sep 17 00:00:00 2001
From: Kyle Altendorf <sda@fstab.net>
Date: Sun, 30 May 2021 13:41:17 +0000
Subject: [PATCH 48/58] add set -evx to archival presets

---
 src/plotman/resources/target_definitions.yaml | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/src/plotman/resources/target_definitions.yaml b/src/plotman/resources/target_definitions.yaml
index 515a01c0..a485e546 100644
--- a/src/plotman/resources/target_definitions.yaml
+++ b/src/plotman/resources/target_definitions.yaml
@@ -16,9 +16,11 @@ target_definitions:
     # directory.
     disk_space_script: |
       #!/bin/bash
+      set -evx
       df -BK | grep " ${site_root}/" | awk '{ gsub(/K$/,"",$4); print $6 ":" $4*1024 }'
     transfer_script: |
       #!/bin/bash
+      set -evx
       "${command}" ${options} "${source}" "${destination}"
     transfer_process_name: "{command}"
     transfer_process_argument_prefix: "{site_root}"
@@ -35,9 +37,11 @@ target_definitions:
       site: null
     disk_space_script: |
       #!/bin/bash
+      set -evx
       ssh -p "${ssh_port}" "${user}@${host}" "df -BK | grep \" ${site_root}/\" | awk '{ gsub(/K\$/,\"\",\$4); print \$6 \":\" \$4*1024 }'"
     transfer_script: |
       #!/bin/bash
+      set -evx
       echo Launching transfer activity
       relative_path=$(realpath --relative-to="${site_root}" "${destination}")
       url_root="rsync://${user}@${host}:${rsync_port}/${site}"

From 11322605ffb95307a73def9440c6878dcf603449 Mon Sep 17 00:00:00 2001
From: Kyle Altendorf <sda@fstab.net>
Date: Sun, 30 May 2021 14:58:52 -0400
Subject: [PATCH 49/58] archive -> disk space in log message

---
 src/plotman/archive.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/plotman/archive.py b/src/plotman/archive.py
index c022c14f..707ea98d 100644
--- a/src/plotman/archive.py
+++ b/src/plotman/archive.py
@@ -144,7 +144,7 @@ def get_archdir_freebytes(arch_cfg):
 
     stderr = completed_process.stderr.strip()
     if len(stderr) > 0:
-        log_messages.append('stderr from archive script:')
+        log_messages.append('stderr from disk space script:')
         for line in stderr.splitlines():
             log_messages.append(f'    {line}')
 

From 7ab1523bf19ae31620641d171acf841d33d841af Mon Sep 17 00:00:00 2001
From: Kyle Altendorf <sda@fstab.net>
Date: Sun, 30 May 2021 16:07:39 -0400
Subject: [PATCH 50/58] be agnostic to trailing /s on site_root (again)

---
 src/plotman/resources/target_definitions.yaml | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/src/plotman/resources/target_definitions.yaml b/src/plotman/resources/target_definitions.yaml
index a485e546..38f86de9 100644
--- a/src/plotman/resources/target_definitions.yaml
+++ b/src/plotman/resources/target_definitions.yaml
@@ -17,7 +17,8 @@ target_definitions:
     disk_space_script: |
       #!/bin/bash
       set -evx
-      df -BK | grep " ${site_root}/" | awk '{ gsub(/K$/,"",$4); print $6 ":" $4*1024 }'
+      site_root_stripped=$(echo "${site_root}" | sed 's;/\+$;;')
+      df -BK | grep " ${site_root_stripped}/" | awk '{ gsub(/K$/,"",$4); print $6 ":" $4*1024 }'
     transfer_script: |
       #!/bin/bash
       set -evx
@@ -38,7 +39,8 @@ target_definitions:
     disk_space_script: |
       #!/bin/bash
       set -evx
-      ssh -p "${ssh_port}" "${user}@${host}" "df -BK | grep \" ${site_root}/\" | awk '{ gsub(/K\$/,\"\",\$4); print \$6 \":\" \$4*1024 }'"
+      site_root_stripped=$(echo "${site_root}" | sed 's;/\+$;;')
+      ssh -p "${ssh_port}" "${user}@${host}" "df -BK | grep \" $(echo "${site_root_stripped}" | sed 's;/\+$;;')/\" | awk '{ gsub(/K\$/,\"\",\$4); print \$6 \":\" \$4*1024 }'"
     transfer_script: |
       #!/bin/bash
       set -evx

From 2d87e20396078d6b32868b14ff0dcbd4ce15fb8a Mon Sep 17 00:00:00 2001
From: Kyle Altendorf <sda@fstab.net>
Date: Sun, 30 May 2021 19:09:31 -0400
Subject: [PATCH 51/58] add logging (for disk space script output)

---
 src/plotman/archive.py | 51 +++++++++++++++++++++++++-----------------
 src/plotman/plotman.py | 24 ++++++++++++++++++++
 2 files changed, 55 insertions(+), 20 deletions(-)

diff --git a/src/plotman/archive.py b/src/plotman/archive.py
index 707ea98d..19324622 100644
--- a/src/plotman/archive.py
+++ b/src/plotman/archive.py
@@ -1,5 +1,6 @@
 import argparse
 import contextlib
+import logging
 import math
 import os
 import posixpath
@@ -15,6 +16,9 @@
 
 from plotman import configuration, job, manager, plot_util
 
+
+logger = logging.getLogger(__name__)
+
 # TODO : write-protect and delete-protect archived plots
 
 def spawn_archive_process(dir_cfg, arch_cfg, all_jobs):
@@ -118,35 +122,42 @@ def get_archdir_freebytes(arch_cfg):
     target = arch_cfg.target_definition()
 
     archdir_freebytes = {}
-    timeout = 2
+    timeout = 5
     try:
         completed_process = subprocess.run(
             [target.disk_space_path],
-            encoding='utf-8',
             env={**os.environ, **arch_cfg.environment()},
             stdout=subprocess.PIPE,
             stderr=subprocess.PIPE,
             timeout=timeout,
         )
-    except subprocess.TimeoutExpired:
+    except subprocess.TimeoutExpired as e:
         log_messages.append(f'Disk space check timed out in {timeout} seconds')
-        return archdir_freebytes, log_messages
-
-    for line in completed_process.stdout.strip().splitlines():
-        line = line.strip()
-        split = line.split(':')
-        if len(split) != 2:
-            log_messages.append(f'Unable to parse disk script line: {line!r}')
-            continue
-        archdir, space = split
-        freebytes = int(space)
-        archdir_freebytes[archdir.strip()] = freebytes
-
-    stderr = completed_process.stderr.strip()
-    if len(stderr) > 0:
-        log_messages.append('stderr from disk space script:')
-        for line in stderr.splitlines():
-            log_messages.append(f'    {line}')
+        stdout = e.stdout.decode('utf-8', errors='ignore').strip()
+        stderr = e.stderr.decode('utf-8', errors='ignore').strip()
+    else:
+        stdout = completed_process.stdout.decode('utf-8', errors='ignore').strip()
+        stderr = completed_process.stderr.decode('utf-8', errors='ignore').strip()
+        for line in stdout.splitlines():
+            line = line.strip()
+            split = line.split(':')
+            if len(split) != 2:
+                log_messages.append(f'Unable to parse disk script line: {line!r}')
+                continue
+            archdir, space = split
+            freebytes = int(space)
+            archdir_freebytes[archdir.strip()] = freebytes
+
+    for line in log_messages:
+        logger.info(line)
+
+    logger.info('stdout from disk space script:')
+    for line in stdout.splitlines():
+        logger.info(f'    {line}')
+
+    logger.info('stderr from disk space script:')
+    for line in stderr.splitlines():
+        logger.info(f'    {line}')
 
     return archdir_freebytes, log_messages
 
diff --git a/src/plotman/plotman.py b/src/plotman/plotman.py
index 099b4961..5a27bc56 100755
--- a/src/plotman/plotman.py
+++ b/src/plotman/plotman.py
@@ -1,12 +1,17 @@
 import argparse
 import importlib
 import importlib.resources
+import logging
+import logging.handlers
 import os
 import random
 from shutil import copyfile
 import time
 import datetime
 
+import appdirs
+import pendulum
+
 # Plotman libraries
 from plotman import analyzer, archive, configuration, interactive, manager, plot_util, reporting
 from plotman import resources as plotman_resources
@@ -87,9 +92,28 @@ def get_term_width():
         columns = 120  # 80 is typically too narrow.  TODO: make a command line arg.
     return columns
 
+class Iso8601Formatter(logging.Formatter):
+    def formatTime(self, record, datefmt=None):
+        time = pendulum.from_timestamp(timestamp=record.created, tz='local')
+        return time.isoformat(timespec='microseconds', )
+
 def main():
     random.seed()
 
+    log_file_path = appdirs.user_log_dir('plotman')
+    os.makedirs(os.path.dirname(log_file_path), exist_ok=True)
+    root_logger = logging.getLogger()
+    handler = logging.handlers.RotatingFileHandler(
+        backupCount=10,
+        encoding='utf-8',
+        filename=log_file_path,
+        maxBytes=10_000_000,
+    )
+    formatter = Iso8601Formatter(fmt='%(asctime)s: %(message)s')
+    handler.setFormatter(formatter)
+    root_logger.addHandler(handler)
+    root_logger.setLevel(logging.INFO)
+
     pm_parser = PlotmanArgParser()
     args = pm_parser.parse_args()
 

From 2461304e00f41a6c84348ac83b6da7612cff944f Mon Sep 17 00:00:00 2001
From: Kyle Altendorf <sda@fstab.net>
Date: Sun, 30 May 2021 20:05:18 -0400
Subject: [PATCH 52/58] allow integers as env values to avoid user confusion
 specifying ports

---
 src/plotman/configuration.py                  | 22 +++++++++++++++++--
 src/plotman/resources/target_definitions.yaml |  4 ++--
 2 files changed, 22 insertions(+), 4 deletions(-)

diff --git a/src/plotman/configuration.py b/src/plotman/configuration.py
index 7dbd5ce4..1bb16451 100644
--- a/src/plotman/configuration.py
+++ b/src/plotman/configuration.py
@@ -77,6 +77,12 @@ def get_validated_configs(config_text, config_path):
 
     return loaded
 
+class CustomStringField(marshmallow.fields.String):
+    def _deserialize(self, value, attr, data, **kwargs):
+        if isinstance(value, int):
+            value = str(value)
+
+        return super()._deserialize(value, attr, data, **kwargs)
 
 # Data models used to deserializing/formatting plotman.yaml files.
 
@@ -86,7 +92,13 @@ class ArchivingTarget:
     transfer_process_name: str
     transfer_process_argument_prefix: str
     # TODO: mutable attribute...
-    env: Dict[str, Optional[str]] = attr.ib(factory=dict)
+    env: Dict[str, Optional[str]] = desert.ib(
+        factory=dict,
+        marshmallow_field=marshmallow.fields.Dict(
+            keys=marshmallow.fields.String(),
+            values=CustomStringField(allow_none=True),
+        ),
+    )
     disk_space_path: Optional[str] = None
     disk_space_script: Optional[str] = None
     transfer_path: Optional[str] = None
@@ -118,7 +130,13 @@ class PresetTargetDefinitions:
 class Archiving:
     target: str
     # TODO: mutable attribute...
-    env: Dict[str, str]
+    env: Dict[str, str] = desert.ib(
+        factory=dict,
+        marshmallow_field=marshmallow.fields.Dict(
+            keys=marshmallow.fields.String(),
+            values=CustomStringField(),
+        ),
+    )
     index: int = 0  # If not explicit, "index" will default to 0
     target_definitions: Dict[str, ArchivingTarget] = attr.ib(factory=dict)
 
diff --git a/src/plotman/resources/target_definitions.yaml b/src/plotman/resources/target_definitions.yaml
index 38f86de9..e70ab7f9 100644
--- a/src/plotman/resources/target_definitions.yaml
+++ b/src/plotman/resources/target_definitions.yaml
@@ -30,8 +30,8 @@ target_definitions:
       # A value of null indicates a mandatory option
       command: rsync
       options: --bwlimit=80000 --preallocate --remove-source-files --skip-compress plot --whole-file
-      rsync_port: "873"
-      ssh_port: "22"
+      rsync_port: 873
+      ssh_port: 22
       user: null
       host: null
       site_root: null

From d7aca711ef004d636b7a996f4694d4445cfbb7b7 Mon Sep 17 00:00:00 2001
From: Kyle Altendorf <sda@fstab.net>
Date: Mon, 31 May 2021 19:12:59 -0400
Subject: [PATCH 53/58] top level logging: and plots/transfers/application log
 path configuration

---
 src/plotman/_tests/configuration_test.py |  8 ++---
 src/plotman/_tests/manager_test.py       |  1 -
 src/plotman/archive.py                   |  6 ++--
 src/plotman/configuration.py             | 37 ++++++++++++++++++++----
 src/plotman/interactive.py               | 10 +++----
 src/plotman/manager.py                   |  6 ++--
 src/plotman/plotman.py                   | 33 ++++++++++-----------
 src/plotman/resources/plotman.yaml       | 15 ++++++----
 8 files changed, 71 insertions(+), 45 deletions(-)

diff --git a/src/plotman/_tests/configuration_test.py b/src/plotman/_tests/configuration_test.py
index 327c66e4..16f67b24 100644
--- a/src/plotman/_tests/configuration_test.py
+++ b/src/plotman/_tests/configuration_test.py
@@ -59,14 +59,14 @@ def test_loads_without_user_interface(config_text):
 def test_get_dst_directories_gets_dst():
     tmp = ['/tmp']
     dst = ['/dst0', '/dst1']
-    directories = configuration.Directories(log='', tmp=tmp, dst=dst)
+    directories = configuration.Directories(tmp=tmp, dst=dst)
 
     assert directories.get_dst_directories() == dst
 
 
 def test_get_dst_directories_gets_tmp():
     tmp = ['/tmp']
-    directories = configuration.Directories(log='', tmp=tmp)
+    directories = configuration.Directories(tmp=tmp)
 
     assert directories.get_dst_directories() == tmp
 
@@ -74,13 +74,13 @@ def test_get_dst_directories_gets_tmp():
 def test_dst_is_dst():
     tmp = ['/tmp']
     dst = ['/dst0', '/dst1']
-    directories = configuration.Directories(log='', tmp=tmp, dst=dst)
+    directories = configuration.Directories(tmp=tmp, dst=dst)
 
     assert not directories.dst_is_tmp()
 
 
 def test_dst_is_tmp():
     tmp = ['/tmp']
-    directories = configuration.Directories(log='', tmp=tmp)
+    directories = configuration.Directories(tmp=tmp)
 
     assert directories.dst_is_tmp()
diff --git a/src/plotman/_tests/manager_test.py b/src/plotman/_tests/manager_test.py
index 43a3f78e..7b1d2573 100755
--- a/src/plotman/_tests/manager_test.py
+++ b/src/plotman/_tests/manager_test.py
@@ -20,7 +20,6 @@ def sched_cfg():
 @pytest.fixture
 def dir_cfg():
     return configuration.Directories(
-        log="/plots/log",
         tmp=["/var/tmp", "/tmp"],
         dst=["/mnt/dst/00", "/mnt/dst/01", "/mnt/dst/03"],
         tmp_overrides={"/mnt/tmp/04": configuration.TmpOverrides(tmpdir_max_jobs=4)}
diff --git a/src/plotman/archive.py b/src/plotman/archive.py
index 19324622..4a0246cc 100644
--- a/src/plotman/archive.py
+++ b/src/plotman/archive.py
@@ -21,7 +21,7 @@
 
 # TODO : write-protect and delete-protect archived plots
 
-def spawn_archive_process(dir_cfg, arch_cfg, all_jobs):
+def spawn_archive_process(dir_cfg, arch_cfg, log_cfg, all_jobs):
     '''Spawns a new archive process using the command created
     in the archive() function. Returns archiving status and a log message to print.'''
 
@@ -40,8 +40,9 @@ def spawn_archive_process(dir_cfg, arch_cfg, all_jobs):
         else:
             args = status_or_cmd
 
-            log_file_path = dir_cfg.create_log_path(group='transfer', time=pendulum.now())
+            log_file_path = log_cfg.create_transfer_log_path(time=pendulum.now())
 
+            log_messages.append(f'Starting archive: {args["args"]} ; logging to {log_file_path}')
             # TODO: CAMPid 09840103109429840981397487498131
             try:
                 open_log_file = open(log_file_path, 'x')
@@ -71,7 +72,6 @@ def spawn_archive_process(dir_cfg, arch_cfg, all_jobs):
                     stdout=open_log_file,
                     stderr=subprocess.STDOUT,
                     start_new_session=True)
-            log_messages.append(f'Starting archive: {args["args"]} ; logging to {log_file_path}')
             # At least for now it seems that even if we get a new running
             # archive jobs list it doesn't contain the new rsync process.
             # My guess is that this is because the bash in the middle due to
diff --git a/src/plotman/configuration.py b/src/plotman/configuration.py
index caa0cc6f..93cf2826 100644
--- a/src/plotman/configuration.py
+++ b/src/plotman/configuration.py
@@ -208,9 +208,37 @@ def maybe_create_scripts(self, temp):
 class TmpOverrides:
     tmpdir_max_jobs: Optional[int] = None
 
+@attr.frozen
+class Logging:
+    plots: str = os.path.join(appdirs.user_data_dir("plotman"), 'plots')
+    transfers: str = os.path.join(appdirs.user_data_dir("plotman"), 'transfers')
+    application: str = os.path.join(appdirs.user_log_dir("plotman"), 'plotman.log')
+
+    def setup(self):
+        os.makedirs(self.plots, exist_ok=True)
+        os.makedirs(self.transfers, exist_ok=True)
+        os.makedirs(os.path.dirname(self.application), exist_ok=True)
+
+    def create_plot_log_path(self, time):
+        return self._create_log_path(
+            time=time,
+            directory=self.plots,
+            group='plot',
+        )
+
+    def create_transfer_log_path(self, time):
+        return self._create_log_path(
+            time=time,
+            directory=self.transfers,
+            group='transfer',
+        )
+
+    def _create_log_path(self, time, directory, group):
+        timestamp = time.isoformat(timespec='microseconds').replace(':', '_')
+        return os.path.join(directory, f'{timestamp}.{group}.log')
+
 @attr.frozen
 class Directories:
-    log: str
     tmp: List[str]
     dst: Optional[List[str]] = None
     tmp2: Optional[str] = None
@@ -233,10 +261,6 @@ def get_dst_directories(self):
 
         return self.dst
 
-    def create_log_path(self, group, time):
-        timestamp = time.isoformat(timespec='microseconds').replace(':', '_')
-        return os.path.join(self.log, f'{timestamp}.{group}.log')
-
 @attr.frozen
 class Scheduling:
     global_max_jobs: int
@@ -267,6 +291,7 @@ class PlotmanConfig:
     directories: Directories
     scheduling: Scheduling
     plotting: Plotting
+    logging: Logging = Logging()
     archiving: Optional[Archiving] = None
     user_interface: UserInterface = attr.ib(factory=UserInterface)
     version: List[int] = [0]
@@ -275,6 +300,8 @@ class PlotmanConfig:
     def setup(self):
         prefix = f'plotman-pid_{os.getpid()}-'
 
+        self.logging.setup()
+
         with tempfile.TemporaryDirectory(prefix=prefix) as temp:
             if self.archiving is not None:
                 self.archiving.maybe_create_scripts(temp=temp)
diff --git a/src/plotman/interactive.py b/src/plotman/interactive.py
index a85002f1..6689d0f0 100644
--- a/src/plotman/interactive.py
+++ b/src/plotman/interactive.py
@@ -81,7 +81,7 @@ def curses_main(stdscr, cfg):
     jobs_win = curses.newwin(1, 1, 1, 0)
     dirs_win = curses.newwin(1, 1, 1, 0)
 
-    jobs = Job.get_running_jobs(cfg.directories.log)
+    jobs = Job.get_running_jobs(cfg.logging.plots)
     last_refresh = None
 
     pressed_key = ''   # For debugging
@@ -103,15 +103,15 @@ def curses_main(stdscr, cfg):
             do_full_refresh = elapsed >= cfg.scheduling.polling_time_s
 
         if not do_full_refresh:
-            jobs = Job.get_running_jobs(cfg.directories.log, cached_jobs=jobs)
+            jobs = Job.get_running_jobs(cfg.logging.plots, cached_jobs=jobs)
 
         else:
             last_refresh = datetime.datetime.now()
-            jobs = Job.get_running_jobs(cfg.directories.log)
+            jobs = Job.get_running_jobs(cfg.logging.plots)
 
             if plotting_active:
                 (started, msg) = manager.maybe_start_new_plot(
-                    cfg.directories, cfg.scheduling, cfg.plotting
+                    cfg.directories, cfg.scheduling, cfg.plotting, cfg.logging
                 )
                 if (started):
                     if aging_reason is not None:
@@ -128,7 +128,7 @@ def curses_main(stdscr, cfg):
 
             if archiving_configured:
                 if archiving_active:
-                    archiving_status, log_messages = archive.spawn_archive_process(cfg.directories, cfg.archiving, jobs)
+                    archiving_status, log_messages = archive.spawn_archive_process(cfg.directories, cfg.archiving, cfg.logging, jobs)
                     for log_message in log_messages:
                         log.log(log_message)
 
diff --git a/src/plotman/manager.py b/src/plotman/manager.py
index b58f070d..7e619ed3 100644
--- a/src/plotman/manager.py
+++ b/src/plotman/manager.py
@@ -71,8 +71,8 @@ def phases_permit_new_job(phases, d, sched_cfg, dir_cfg):
 
     return True
 
-def maybe_start_new_plot(dir_cfg, sched_cfg, plotting_cfg):
-    jobs = job.Job.get_running_jobs(dir_cfg.log)
+def maybe_start_new_plot(dir_cfg, sched_cfg, plotting_cfg, log_cfg):
+    jobs = job.Job.get_running_jobs(log_cfg.plots)
 
     wait_reason = None  # If we don't start a job this iteration, this says why.
 
@@ -111,7 +111,7 @@ def maybe_start_new_plot(dir_cfg, sched_cfg, plotting_cfg):
                 else:
                     dstdir = max(dir2ph, key=dir2ph.get)
 
-            log_file_path = dir_cfg.create_log_path(group='plot', time=pendulum.now())
+            log_file_path = log_cfg.create_plot_log_path(time=pendulum.now())
 
             plot_args = ['chia', 'plots', 'create',
                     '-k', str(plotting_cfg.k),
diff --git a/src/plotman/plotman.py b/src/plotman/plotman.py
index 5a27bc56..3c9eaa39 100755
--- a/src/plotman/plotman.py
+++ b/src/plotman/plotman.py
@@ -9,7 +9,6 @@
 import time
 import datetime
 
-import appdirs
 import pendulum
 
 # Plotman libraries
@@ -100,20 +99,6 @@ def formatTime(self, record, datefmt=None):
 def main():
     random.seed()
 
-    log_file_path = appdirs.user_log_dir('plotman')
-    os.makedirs(os.path.dirname(log_file_path), exist_ok=True)
-    root_logger = logging.getLogger()
-    handler = logging.handlers.RotatingFileHandler(
-        backupCount=10,
-        encoding='utf-8',
-        filename=log_file_path,
-        maxBytes=10_000_000,
-    )
-    formatter = Iso8601Formatter(fmt='%(asctime)s: %(message)s')
-    handler.setFormatter(formatter)
-    root_logger.addHandler(handler)
-    root_logger.setLevel(logging.INFO)
-
     pm_parser = PlotmanArgParser()
     args = pm_parser.parse_args()
 
@@ -162,13 +147,25 @@ def main():
     cfg = configuration.get_validated_configs(config_text, config_path)
 
     with cfg.setup():
+        root_logger = logging.getLogger()
+        handler = logging.handlers.RotatingFileHandler(
+            backupCount=10,
+            encoding='utf-8',
+            filename=cfg.logging.application,
+            maxBytes=10_000_000,
+        )
+        formatter = Iso8601Formatter(fmt='%(asctime)s: %(message)s')
+        handler.setFormatter(formatter)
+        root_logger.addHandler(handler)
+        root_logger.setLevel(logging.INFO)
+
         #
         # Stay alive, spawning plot jobs
         #
         if args.cmd == 'plot':
             print('...starting plot loop')
             while True:
-                wait_reason = manager.maybe_start_new_plot(cfg.directories, cfg.scheduling, cfg.plotting)
+                wait_reason = manager.maybe_start_new_plot(cfg.directories, cfg.scheduling, cfg.plotting, cfg.logging)
 
                 # TODO: report this via a channel that can be polled on demand, so we don't spam the console
                 if wait_reason:
@@ -185,7 +182,7 @@ def main():
                     args.bytmp, args.bybitfield)
 
         else:
-            jobs = Job.get_running_jobs(cfg.directories.log)
+            jobs = Job.get_running_jobs(cfg.logging.plots)
 
             # Status report
             if args.cmd == 'status':
@@ -214,7 +211,7 @@ def main():
                         jobs = Job.get_running_jobs(cfg.directories.log)
                     firstit = False
 
-                    archiving_status, log_messages = archive.spawn_archive_process(cfg.directories, cfg.archiving, jobs)
+                    archiving_status, log_messages = archive.spawn_archive_process(cfg.directories, cfg.archiving, cfg.logging, jobs)
                     for log_message in log_messages:
                         print(log_message)
 
diff --git a/src/plotman/resources/plotman.yaml b/src/plotman/resources/plotman.yaml
index 12024a40..751f25a1 100644
--- a/src/plotman/resources/plotman.yaml
+++ b/src/plotman/resources/plotman.yaml
@@ -3,6 +3,15 @@
 # https://github.com/ericaltendorf/plotman/wiki/Configuration#versions
 version: [1]
 
+logging:
+        # One directory in which to store all plot job logs (the STDOUT/
+        # STDERR of all plot jobs).  In order to monitor progress, plotman
+        # reads these logs on a regular basis, so using a fast drive is
+        # recommended.
+        plots: /home/chia/chia/logs
+        # transfers:
+        # application:
+
 # Options for display and rendering
 user_interface:
         # Call out to the `stty` program to determine terminal size, instead of
@@ -14,12 +23,6 @@ user_interface:
 
 # Where to plot and log.
 directories:
-        # One directory in which to store all plot job logs (the STDOUT/
-        # STDERR of all plot jobs).  In order to monitor progress, plotman
-        # reads these logs on a regular basis, so using a fast drive is
-        # recommended.
-        log: /home/chia/chia/logs
-
         # One or more directories to use as tmp dirs for plotting.  The
         # scheduler will use all of them and distribute jobs among them.
         # It assumes that IO is independent for each one (i.e., that each

From 33b3b9772c9f197b1d30942108d2f50f3be987f0 Mon Sep 17 00:00:00 2001
From: Kyle Altendorf <sda@fstab.net>
Date: Mon, 31 May 2021 19:41:54 -0400
Subject: [PATCH 54/58] tidy

---
 src/plotman/_tests/archive_test.py | 17 -----------------
 src/plotman/configuration.py       | 17 -----------------
 2 files changed, 34 deletions(-)

diff --git a/src/plotman/_tests/archive_test.py b/src/plotman/_tests/archive_test.py
index 398b0943..62ec1570 100755
--- a/src/plotman/_tests/archive_test.py
+++ b/src/plotman/_tests/archive_test.py
@@ -4,20 +4,3 @@
 def test_compute_priority():
     assert (archive.compute_priority( job.Phase(major=3, minor=1), 1000, 10) >
             archive.compute_priority( job.Phase(major=3, minor=6), 1000, 10) )
-
-
-# @pytest.fixture(name='remote_configuration')
-# def remote_configuration_fixture():
-#     return configuration.Archive(
-#         env={},
-#         disk_space_script=textwrap.dedent("""\
-#             #!/bin/bash
-#             ssh altendky@server "df -BK | grep \\" ${site_root}/\\" | awk '{ gsub(/K\$/,\\"\\",\$4); print \$6 \":\" \$4*1024 }'"
-#         """),
-#         transfer_script=textwrap.dedent("""\
-#             relative_path=$(realpath --relative-to="${site_root}" "${destination}")
-#             "${command}" --bwlimit=80000 --skip-compress plot --remove-source-files --inplace "${source}" "${url_root}/${relative_path}"
-#         """),
-#         transfer_process_name='{command}',
-#         transfer_process_argument_prefix='{url_root}',
-#     )
diff --git a/src/plotman/configuration.py b/src/plotman/configuration.py
index 93cf2826..a68c93a9 100644
--- a/src/plotman/configuration.py
+++ b/src/plotman/configuration.py
@@ -104,23 +104,6 @@ class ArchivingTarget:
     transfer_path: Optional[str] = None
     transfer_script: Optional[str] = None
 
-# local_rsync_archiving_target = ArchivingTarget(
-#     transfer_process_name='{site_root}',
-#     transfer_process_argument_prefix='{command}',
-#     env={
-#         'command': 'rsync',
-#         'options': '--preallocate --remove-source-files --skip-compress plot --whole-file',
-#         'site_root': None,
-#     },
-#     disk_space_script=textwrap.dedent('''\
-#         #!/bin/bash
-#         df -BK | grep " ${site_root}/" | awk '{ gsub(/K$/,"",$4); print $6 ":" $4*1024 }'
-#     '''),
-#     transfer_script=textwrap.dedent('''\
-#
-#     ''')
-# )
-
 @attr.frozen
 class PresetTargetDefinitions:
     target_definitions: Dict[str, ArchivingTarget] = attr.ib(factory=dict)

From c070605bac4d7445cc6613f0b8fed4617b527b07 Mon Sep 17 00:00:00 2001
From: Kyle Altendorf <sda@fstab.net>
Date: Mon, 31 May 2021 19:57:25 -0400
Subject: [PATCH 55/58] shift some IO up a layer

---
 src/plotman/_tests/configuration_test.py | 19 +++++++++++++------
 src/plotman/configuration.py             |  6 +-----
 src/plotman/plotman.py                   |  6 +++++-
 3 files changed, 19 insertions(+), 12 deletions(-)

diff --git a/src/plotman/_tests/configuration_test.py b/src/plotman/_tests/configuration_test.py
index 16f67b24..47c2c693 100644
--- a/src/plotman/_tests/configuration_test.py
+++ b/src/plotman/_tests/configuration_test.py
@@ -13,13 +13,20 @@ def config_text_fixture():
     return importlib.resources.read_text(plotman_resources, "plotman.yaml")
 
 
-def test_get_validated_configs__default(config_text):
+@pytest.fixture(name='target_definitions_text')
+def target_definitions_text_fixture():
+    return importlib.resources.read_text(
+        plotman_resources, "target_definitions.yaml",
+    )
+
+
+def test_get_validated_configs__default(config_text, target_definitions_text):
     """Check that get_validated_configs() works with default/example plotman.yaml file."""
-    res = configuration.get_validated_configs(config_text, '')
+    res = configuration.get_validated_configs(config_text, '', target_definitions_text)
     assert isinstance(res, configuration.PlotmanConfig)
 
 
-def test_get_validated_configs__malformed(config_text):
+def test_get_validated_configs__malformed(config_text, target_definitions_text):
     """Check that get_validated_configs() raises exception with invalid plotman.yaml contents."""
     loaded_yaml = yaml.load(config_text, Loader=yaml.SafeLoader)
 
@@ -28,7 +35,7 @@ def test_get_validated_configs__malformed(config_text):
     malformed_config_text = yaml.dump(loaded_yaml, Dumper=yaml.SafeDumper)
 
     with pytest.raises(configuration.ConfigurationException) as exc_info:
-        configuration.get_validated_configs(malformed_config_text, '/the_path')
+        configuration.get_validated_configs(malformed_config_text, '/the_path', target_definitions_text)
 
     assert exc_info.value.args[0] == f"Config file at: '/the_path' is malformed"
 
@@ -44,14 +51,14 @@ def test_get_validated_configs__missing():
     )
 
 
-def test_loads_without_user_interface(config_text):
+def test_loads_without_user_interface(config_text, target_definitions_text):
     loaded_yaml = yaml.load(config_text, Loader=yaml.SafeLoader)
 
     del loaded_yaml["user_interface"]
 
     stripped_config_text = yaml.dump(loaded_yaml, Dumper=yaml.SafeDumper)
 
-    reloaded_yaml = configuration.get_validated_configs(stripped_config_text, '')
+    reloaded_yaml = configuration.get_validated_configs(stripped_config_text, '', target_definitions_text)
 
     assert reloaded_yaml.user_interface == configuration.UserInterface()
 
diff --git a/src/plotman/configuration.py b/src/plotman/configuration.py
index a68c93a9..18fedaec 100644
--- a/src/plotman/configuration.py
+++ b/src/plotman/configuration.py
@@ -35,7 +35,7 @@ def read_configuration_text(config_path):
         ) from e
 
 
-def get_validated_configs(config_text, config_path):
+def get_validated_configs(config_text, config_path, preset_target_definitions_text):
     """Return a validated instance of PlotmanConfig with data from plotman.yaml
 
     :raises ConfigurationException: Raised when plotman.yaml is either missing or malformed
@@ -62,10 +62,6 @@ def get_validated_configs(config_text, config_path):
             f"Config file at: '{config_path}' is malformed"
         ) from e
 
-    # TODO: get this IO out
-    preset_target_definitions_text = importlib.resources.read_text(
-        plotman_resources, "target_definitions.yaml",
-    )
     preset_target_objects = yaml.safe_load(preset_target_definitions_text)
     preset_target_schema = desert.schema(PresetTargetDefinitions)
     preset_target_definitions = preset_target_schema.load(preset_target_objects)
diff --git a/src/plotman/plotman.py b/src/plotman/plotman.py
index 3c9eaa39..d95aa60e 100755
--- a/src/plotman/plotman.py
+++ b/src/plotman/plotman.py
@@ -144,7 +144,11 @@ def main():
 
     config_path = configuration.get_path()
     config_text = configuration.read_configuration_text(config_path)
-    cfg = configuration.get_validated_configs(config_text, config_path)
+    preset_target_definitions_text = importlib.resources.read_text(
+        plotman_resources, "target_definitions.yaml",
+    )
+
+    cfg = configuration.get_validated_configs(config_text, config_path, preset_target_definitions_text)
 
     with cfg.setup():
         root_logger = logging.getLogger()

From 77c5ec10bb8a5c3cd597bab60f8a609cd851db20 Mon Sep 17 00:00:00 2001
From: Kyle Altendorf <sda@fstab.net>
Date: Mon, 31 May 2021 19:59:04 -0400
Subject: [PATCH 56/58] a couple more cfg.logging.plots

---
 src/plotman/interactive.py | 2 +-
 src/plotman/plotman.py     | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/plotman/interactive.py b/src/plotman/interactive.py
index 6689d0f0..d20b1b9c 100644
--- a/src/plotman/interactive.py
+++ b/src/plotman/interactive.py
@@ -119,7 +119,7 @@ def curses_main(stdscr, cfg):
                         aging_reason = None
                     log.log(msg)
                     plotting_status = '<just started job>'
-                    jobs = Job.get_running_jobs(cfg.directories.log, cached_jobs=jobs)
+                    jobs = Job.get_running_jobs(cfg.logging.plots, cached_jobs=jobs)
                 else:
                     # If a plot is delayed for any reason other than stagger, log it
                     if msg.find("stagger") < 0:
diff --git a/src/plotman/plotman.py b/src/plotman/plotman.py
index d95aa60e..cec3367c 100755
--- a/src/plotman/plotman.py
+++ b/src/plotman/plotman.py
@@ -212,7 +212,7 @@ def main():
                     if not firstit:
                         print('Sleeping 60s until next iteration...')
                         time.sleep(60)
-                        jobs = Job.get_running_jobs(cfg.directories.log)
+                        jobs = Job.get_running_jobs(cfg.logging.plots)
                     firstit = False
 
                     archiving_status, log_messages = archive.spawn_archive_process(cfg.directories, cfg.archiving, cfg.logging, jobs)

From 2e27644ed6ea6cb258819448c0c4c9f939d50626 Mon Sep 17 00:00:00 2001
From: Kyle Altendorf <sda@fstab.net>
Date: Wed, 2 Jun 2021 08:44:50 -0400
Subject: [PATCH 57/58] handle some None outputs

---
 src/plotman/archive.py | 10 ++++++++--
 1 file changed, 8 insertions(+), 2 deletions(-)

diff --git a/src/plotman/archive.py b/src/plotman/archive.py
index 4a0246cc..1376af7d 100644
--- a/src/plotman/archive.py
+++ b/src/plotman/archive.py
@@ -133,8 +133,14 @@ def get_archdir_freebytes(arch_cfg):
         )
     except subprocess.TimeoutExpired as e:
         log_messages.append(f'Disk space check timed out in {timeout} seconds')
-        stdout = e.stdout.decode('utf-8', errors='ignore').strip()
-        stderr = e.stderr.decode('utf-8', errors='ignore').strip()
+        if e.stdout is None:
+            stdout = ''
+        else:
+            stdout = e.stdout.decode('utf-8', errors='ignore').strip()
+        if e.stderr is None:
+            stderr = ''
+        else:
+            stderr = e.stderr.decode('utf-8', errors='ignore').strip()
     else:
         stdout = completed_process.stdout.decode('utf-8', errors='ignore').strip()
         stderr = completed_process.stderr.decode('utf-8', errors='ignore').strip()

From d680322ed3f28cb7f16ccbb22ea1916d376bd077 Mon Sep 17 00:00:00 2001
From: Kyle Altendorf <sda@fstab.net>
Date: Thu, 3 Jun 2021 18:32:32 -0400
Subject: [PATCH 58/58] printf for mawk to avoid scientific notation etc

---
 src/plotman/resources/target_definitions.yaml | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/src/plotman/resources/target_definitions.yaml b/src/plotman/resources/target_definitions.yaml
index e70ab7f9..b970123c 100644
--- a/src/plotman/resources/target_definitions.yaml
+++ b/src/plotman/resources/target_definitions.yaml
@@ -18,7 +18,9 @@ target_definitions:
       #!/bin/bash
       set -evx
       site_root_stripped=$(echo "${site_root}" | sed 's;/\+$;;')
-      df -BK | grep " ${site_root_stripped}/" | awk '{ gsub(/K$/,"",$4); print $6 ":" $4*1024 }'
+      # printf with %.0f used to handle mawk such as in Ubuntu Docker images
+      # otherwise it saturates and you get saturated sizes like 2147483647
+      df -BK | grep " ${site_root_stripped}/" | awk '{ gsub(/K$/,"",$4); printf "%s:%.0f\n", $6, $4*1024 }'
     transfer_script: |
       #!/bin/bash
       set -evx
@@ -40,7 +42,9 @@ target_definitions:
       #!/bin/bash
       set -evx
       site_root_stripped=$(echo "${site_root}" | sed 's;/\+$;;')
-      ssh -p "${ssh_port}" "${user}@${host}" "df -BK | grep \" $(echo "${site_root_stripped}" | sed 's;/\+$;;')/\" | awk '{ gsub(/K\$/,\"\",\$4); print \$6 \":\" \$4*1024 }'"
+      # printf with %.0f used to handle mawk such as in Ubuntu Docker images
+      # otherwise it saturates and you get saturated sizes like 2147483647
+      ssh -p "${ssh_port}" "${user}@${host}" "df -BK | grep \" $(echo "${site_root_stripped}" | sed 's;/\+$;;')/\" | awk '{ gsub(/K\$/,\"\",\$4); printf \"%s:%.0f\n\", \$6, \$4*1024 }'"
     transfer_script: |
       #!/bin/bash
       set -evx