Local archive mode (fixes #6)

src/plotman/archive.py

@@ -86,8 +86,12 @@ def compute_priority(phase, gb_free, n_plots):
 
 def get_archdir_freebytes(arch_cfg):
     archdir_freebytes = {}
-    df_cmd = ('ssh %s@%s df -aBK | grep " %s/"' %
-        (arch_cfg.rsyncd_user, arch_cfg.rsyncd_host, arch_cfg.rsyncd_path) )
+    if arch_cfg.mode == 'legacy':
+        df_cmd = ('ssh %s@%s df -aBK | grep " %s/"' %
+            (arch_cfg.rsyncd_user, arch_cfg.rsyncd_host, arch_cfg.rsyncd_path) )
+    else:
+        arch_cfg_custom = getattr(arch_cfg, arch_cfg.mode)
+        df_cmd = (arch_cfg_custom.df_cmd.format(arch_cfg_custom.path))
     with subprocess.Popen(df_cmd, shell=True, stdout=subprocess.PIPE) as proc:
         for line in proc.stdout.readlines():
             fields = line.split()
@@ -100,12 +104,15 @@ def get_archdir_freebytes(arch_cfg):
     return archdir_freebytes
 
 def rsync_dest(arch_cfg, arch_dir):
-    rsync_path = arch_dir.replace(arch_cfg.rsyncd_path, arch_cfg.rsyncd_module)
-    if rsync_path.startswith('/'):
-        rsync_path = rsync_path[1:]  # Avoid dup slashes.  TODO use path join?
-    rsync_url = 'rsync://%s@%s:12000/%s' % (
-            arch_cfg.rsyncd_user, arch_cfg.rsyncd_host, rsync_path)
-    return rsync_url
+    if arch_cfg.mode == 'legacy':
+        rsync_path = arch_dir.replace(arch_cfg.rsyncd_path, arch_cfg.rsyncd_module)
+        if rsync_path.startswith('/'):
+            rsync_path = rsync_path[1:]  # Avoid dup slashes.  TODO use path join?
+        return 'rsync://%s@%s:12000/%s' % (
+                arch_cfg.rsyncd_user, arch_cfg.rsyncd_host, rsync_path)
+    else:
+        arch_cfg_custom = getattr(arch_cfg, arch_cfg.mode)
+        return arch_cfg_custom.path
 
 # TODO: maybe consolidate with similar code in job.py?
 def get_running_archive_jobs(arch_cfg):
@@ -115,7 +122,11 @@ def get_running_archive_jobs(arch_cfg):
     dest = rsync_dest(arch_cfg, '/')
     for proc in psutil.process_iter(['pid', 'name']):
         with contextlib.suppress(psutil.NoSuchProcess):
-            if proc.name() == 'rsync':
+            if arch_cfg.mode == 'legacy':
+                proc_name = 'rsync' 
+            else:
+                proc_name = getattr(arch_cfg, arch_cfg.mode).archive_tool
+            if proc.name() == proc_name:
                 args = proc.cmdline()
                 for arg in args:
                     if arg.startswith(dest):
@@ -156,7 +167,7 @@ def archive(dir_cfg, all_jobs):
     archdir_freebytes = get_archdir_freebytes(dir_cfg.archive)
     if not archdir_freebytes:
         return(False, 'No free archive dirs found.')
-    
+
     archdir = ''
     available = [(d, space) for (d, space) in archdir_freebytes.items() if 
                  space > 1.2 * plot_util.get_k32_plotsize()]
@@ -168,10 +179,17 @@ def archive(dir_cfg, all_jobs):
         return(False, 'No archive directories found with enough free space')
 
     msg = 'Found %s with ~%d GB free' % (archdir, freespace / plot_util.GB)
-
-    bwlimit = dir_cfg.archive.rsyncd_bwlimit
-    throttle_arg = ('--bwlimit=%d' % bwlimit) if bwlimit else ''
-    cmd = ('rsync %s --no-compress --remove-source-files -P %s %s' %
-            (throttle_arg, chosen_plot, rsync_dest(dir_cfg.archive, archdir)))
-
+    if dir_cfg.archive.mode == 'legacy':
+        bwlimit = dir_cfg.archive.rsyncd_bwlimit
+        throttle_arg = ('--bwlimit=%d' % bwlimit) if bwlimit else ''
+        cmd = ('rsync %s --no-compress --remove-source-files -P %s %s' %
+                (throttle_arg, chosen_plot, rsync_dest(dir_cfg.archive, archdir)))
+    else:
+        arch_cfg_custom = getattr(dir_cfg.archive, dir_cfg.archive.mode)
+        cmd = arch_cfg_custom.archive_cmd.format(
+            arch_cfg_custom.archive_tool,
+            arch_cfg_custom.parameters,
+            chosen_plot,
+            rsync_dest(dir_cfg.archive, archdir)
+        )
     return (True, cmd)

src/plotman/configuration.py

@@ -48,6 +48,14 @@ def get_validated_configs(config_text, config_path):
 
 # Data models used to deserializing/formatting plotman.yaml files.
 
+@attr.frozen
+class ArchiveLocal:
+    path: str
+    df_cmd: str = 'df -BK | grep " {}/"'
+    archive_tool: str = 'rsync'
+    archive_cmd: str = '{} {} {} {}'
+    parameters: str = '--bwlimit=80000 --no-compress --remove-source-files -P'
+
 @attr.frozen
 class Archive:
     rsyncd_module: str
@@ -56,6 +64,13 @@ class Archive:
     rsyncd_host: str
     rsyncd_user: str
     index: int = 0  # If not explicit, "index" will default to 0
+    mode: str = desert.ib(
+        default='legacy',
+        marshmallow_field=marshmallow.fields.String(
+            validate=marshmallow.validate.OneOf(choices=['legacy', 'local'])
+        ),
+    )
+    local: Optional[ArchiveLocal] = None
 
 @attr.frozen
 class TmpOverrides:

src/plotman/resources/plotman.yaml

@@ -80,7 +80,13 @@ directories:
                 # have four plotters, you could set this to 0, 1, 2, and 3, on
                 # the 4 machines, or 0, 1, 0, 1.
                 #   index: 0
-
+                # Optional switch to enable local archiving (defaults to remote if absent).
+                # Note: rsyncd_module, rsyncd_host and rsyncd_user are ignored in local mode.
+                mode: legacy  # legacy (config above) or local
+                #
+                # Local mode:
+                # local:
+                #         path: '/farm'
 
 # Plotting scheduling parameters
 scheduling: