Merge branch 'ericaltendorf:development' into development

CHANGELOG.md

@@ -17,6 +17,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
   ([#430](https://github.com/ericaltendorf/plotman/pull/430))
 - `plotman logs` command to print and tail plot logs by their plot ID.
   ([#509](https://github.com/ericaltendorf/plotman/pull/509))
+- Support the [madMAx plotter](https://github.com/madMAx43v3r/chia-plotter).
+  See the [configuration wiki page](https://github.com/ericaltendorf/plotman/wiki/Configuration#2-v05) for help setting it up.
+  ([#797](https://github.com/ericaltendorf/plotman/pull/797))
 
 ## [0.4.1] - 2021-06-11
 ### Fixed

src/plotman/analyzer.py

@@ -58,12 +58,18 @@ def analyze(logfilenames: typing.List[str], clipterminals: bool, bytmp: bool, by
                     else:
                         sl += '-bitfield'
 
-                # Phase timing.  Sample log line:
+                # CHIA: Phase timing.  Sample log line:
                 # Time for phase 1 = 22796.7 seconds. CPU (98%) Tue Sep 29 17:57:19 2020
                 for phase in ['1', '2', '3', '4']:
                     m = re.search(r'^Time for phase ' + phase + ' = (\d+.\d+) seconds..*', line)
                     if m:
                         phase_time[phase] = float(m.group(1))
+
+                # MADMAX: Phase timing.  Sample log line: "Phase 2 took 2193.37 sec"
+                for phase in ['1', '2', '3', '4']:
+                    m = re.search(r'^Phase ' + phase + ' took (\d+.\d+) sec.*', line)
+                    if m:
+                        phase_time[phase] = float(m.group(1))
 
                 # Uniform sort.  Sample log line:
                 # Bucket 267 uniform sort. Ram: 0.920GiB, u_sort min: 0.688GiB, qs min: 0.172GiB.
@@ -82,7 +88,7 @@ def analyze(logfilenames: typing.List[str], clipterminals: bool, bytmp: bool, by
                     else:
                         print ('Warning: unrecognized sort ' + sorter)
 
-                # Job completion.  Record total time in sliced data store.
+                # CHIA: Job completion.  Record total time in sliced data store.
                 # Sample log line:
                 # Total time = 49487.1 seconds. CPU (97.26%) Wed Sep 30 01:22:10 2020
                 m = re.search(r'^Total time = (\d+.\d+) seconds.*', line)
@@ -94,13 +100,22 @@ def analyze(logfilenames: typing.List[str], clipterminals: bool, bytmp: bool, by
                         for phase in ['1', '2', '3', '4']:
                             data.setdefault(sl, {}).setdefault('phase ' + phase, []).append(phase_time[phase])
                         data.setdefault(sl, {}).setdefault('%usort', []).append(100 * n_uniform // n_sorts)
+
+                # MADMAX: Job completion.  Record total time in sliced data store.
+                # Sample log line: "Total plot creation time was 2530.76 sec"
+                m = re.search(r'^Total plot creation time was (\d+.\d+) sec.*', line)
+                if m:
+                    data.setdefault(sl, {}).setdefault('total time', []).append(float(m.group(1)))
+                    for phase in ['1', '2', '3', '4']:
+                        data.setdefault(sl, {}).setdefault('phase ' + phase, []).append(phase_time[phase])
+                    data.setdefault(sl, {}).setdefault('%usort', []).append(0)  # Not available for MADMAX
 
     # Prepare report
     tab = tt.Texttable()
     all_measures = ['%usort', 'phase 1', 'phase 2', 'phase 3', 'phase 4', 'total time']
     headings = ['Slice', 'n'] + all_measures
     tab.header(headings)
-
+    
     for sl in data.keys():
         row = [sl]
 

src/plotman/configuration.py

@@ -12,6 +12,8 @@
 # TODO: should be a desert.ib() but mypy doesn't understand it then, see below
 import desert._make
 import marshmallow
+import marshmallow.fields
+import marshmallow.validate
 import pendulum
 import yaml
 
@@ -67,6 +69,27 @@ def get_validated_configs(config_text: str, config_path: str, preset_target_defi
             f"Config file at: '{config_path}' is malformed"
         ) from e
 
+    if loaded.plotting.type == "chia" and loaded.plotting.chia is None:
+        raise ConfigurationException(
+            "chia selected as plotter but plotting: chia: was not specified in the config",
+        )
+    elif loaded.plotting.type == "madmax":
+        if loaded.plotting.madmax is None:
+            raise ConfigurationException(
+                "madmax selected as plotter but plotting: madmax: was not specified in the config",
+            )
+
+        if loaded.plotting.farmer_pk is None:
+            raise ConfigurationException(
+                "madmax selected as plotter but no plotting: farmer_pk: was specified in the config",
+            )
+
+        if loaded.plotting.pool_pk is None:
+            raise ConfigurationException(
+                "madmax selected as plotter but no plotting: pool_pk: was specified in the config",
+            )
+
+
     if loaded.archiving is not None:
         preset_target_objects = yaml.safe_load(preset_target_definitions_text)
         preset_target_schema = desert.schema(PresetTargetDefinitions)
@@ -273,17 +296,37 @@ class Scheduling:
     tmpdir_stagger_phase_minor: int
     tmpdir_stagger_phase_limit: int = 1  # If not explicit, "tmpdir_stagger_phase_limit" will default to 1
 
+@attr.frozen
+class ChiaPlotterOptions:
+    n_threads: int = 2
+    n_buckets: int = 128
+    k: Optional[int] = 32
+    e: Optional[bool] = False
+    job_buffer: Optional[int] = 3389
+    x: bool = False
+    pool_contract_address: Optional[str] = None
+
+@attr.frozen
+class MadmaxPlotterOptions:
+    n_threads: int = 4
+    n_buckets: int = 256
+
 @attr.frozen
 class Plotting:
-    k: int
-    e: bool
-    n_threads: int
-    n_buckets: int
-    job_buffer: int
     farmer_pk: Optional[str] = None
     pool_pk: Optional[str] = None
-    pool_contract_address: Optional[str] = None
-    x: bool = False
+    type: str = attr.ib(
+        default="chia",
+        metadata={
+            desert._make._DESERT_SENTINEL: {
+                'marshmallow_field': marshmallow.fields.String(
+                    validate=marshmallow.validate.OneOf(choices=["chia", "madmax"]),
+                ),
+            },
+        },
+    )
+    chia: Optional[ChiaPlotterOptions] = None
+    madmax: Optional[MadmaxPlotterOptions] = None
 
 @attr.frozen
 class UserInterface:

src/plotman/job.py

@@ -18,7 +18,7 @@
 import pendulum
 import psutil
 
-from plotman import chia
+from plotman import chia, madmax
 
 
 def job_phases_for_tmpdir(d: str, all_jobs: typing.List["Job"]) -> typing.List["Phase"]:
@@ -30,14 +30,17 @@ def job_phases_for_dstdir(d: str, all_jobs: typing.List["Job"]) -> typing.List["
     return sorted([j.progress() for j in all_jobs if j.dstdir == d])
 
 def is_plotting_cmdline(cmdline: typing.List[str]) -> bool:
-    if cmdline and 'python' in cmdline[0].lower():
+    if cmdline and 'python' in cmdline[0].lower():  # Stock Chia plotter
         cmdline = cmdline[1:]
-    return (
-        len(cmdline) >= 3
-        and 'chia' in cmdline[0]
-        and 'plots' == cmdline[1]
-        and 'create' == cmdline[2]
-    )
+        return (
+            len(cmdline) >= 3
+            and 'chia' in cmdline[0]
+            and 'plots' == cmdline[1]
+            and 'create' == cmdline[2]
+        )
+    elif cmdline and 'chia_plot' == cmdline[0].lower():  # Madmax plotter
+        return True
+    return False
 
 def parse_chia_plot_time(s: str) -> pendulum.DateTime:
     # This will grow to try ISO8601 as well for when Chia logs that way
@@ -50,14 +53,21 @@ def parse_chia_plots_create_command_line(
 ) -> "ParsedChiaPlotsCreateCommand":
     command_line = list(command_line)
     # Parse command line args
-    if 'python' in command_line[0].lower():
+    if 'python' in command_line[0].lower():  # Stock Chia plotter
         command_line = command_line[1:]
-    assert len(command_line) >= 3
-    assert 'chia' in command_line[0]
-    assert 'plots' == command_line[1]
-    assert 'create' == command_line[2]
-
-    all_command_arguments = command_line[3:]
+        assert len(command_line) >= 3
+        assert 'chia' in command_line[0]
+        assert 'plots' == command_line[1]
+        assert 'create' == command_line[2]
+        all_command_arguments = command_line[3:]
+        # TODO: We could at some point do chia version detection and pick the
+        #       associated command.  For now we'll just use the latest one we have
+        #       copied.
+        command = chia.commands.latest_command()
+    elif 'chia_plot' in command_line[0].lower():  # Madmax plotter
+        command_line = command_line[1:]
+        all_command_arguments = command_line[2:]
+        command = madmax._cli_c8121b9
 
     # nice idea, but this doesn't include -h
     # help_option_names = command.get_help_option_names(ctx=context)
@@ -69,10 +79,6 @@ def parse_chia_plots_create_command_line(
         if argument not in help_option_names
     ]
 
-    # TODO: We could at some point do chia version detection and pick the
-    #       associated command.  For now we'll just use the latest one we have
-    #       copied.
-    command = chia.commands.latest_command()
     try:
         context = command.make_context(info_name='', args=list(command_arguments))
     except click.ClickException as e:
@@ -146,6 +152,7 @@ class Job:
     jobfile: str = ''
     job_id: int = 0
     plot_id: str = '--------'
+    plotter: str = ''
     proc: psutil.Process
     k: int
     r: int
@@ -260,15 +267,24 @@ def __init__(
         #     'nobitfield': False,
         #     'exclude_final_dir': False,
         # }
-
-        self.k = self.args['size']  # type: ignore[assignment]
-        self.r = self.args['num_threads']  # type: ignore[assignment]
-        self.u = self.args['buckets']  # type: ignore[assignment]
-        self.b = self.args['buffer']  # type: ignore[assignment]
-        self.n = self.args['num']  # type: ignore[assignment]
-        self.tmpdir = self.args['tmp_dir']  # type: ignore[assignment]
-        self.tmp2dir = self.args['tmp2_dir']  # type: ignore[assignment]
-        self.dstdir = self.args['final_dir']  # type: ignore[assignment]
+        if proc.name().startswith("chia_plot"): # MADMAX
+            self.k = 32
+            self.r = self.args['threads']  # type: ignore[assignment]
+            self.u = self.args['buckets']  # type: ignore[assignment]
+            self.b = 0
+            self.n = self.args['count']  # type: ignore[assignment]
+            self.tmpdir = self.args['tmpdir']  # type: ignore[assignment]
+            self.tmp2dir = self.args['tmpdir2']  # type: ignore[assignment]
+            self.dstdir = self.args['finaldir']  # type: ignore[assignment]
+        else: # CHIA
+            self.k = self.args['size']  # type: ignore[assignment]
+            self.r = self.args['num_threads']  # type: ignore[assignment]
+            self.u = self.args['buckets']  # type: ignore[assignment]
+            self.b = self.args['buffer']  # type: ignore[assignment]
+            self.n = self.args['num']  # type: ignore[assignment]
+            self.tmpdir = self.args['tmp_dir']  # type: ignore[assignment]
+            self.tmp2dir = self.args['tmp2_dir']  # type: ignore[assignment]
+            self.dstdir = self.args['final_dir']  # type: ignore[assignment]
 
         plot_cwd: str = self.proc.cwd()
         self.tmpdir = os.path.join(plot_cwd, self.tmpdir)
@@ -311,15 +327,24 @@ def init_from_logfile(self) -> None:
                 with contextlib.suppress(UnicodeDecodeError):
                     for line in f:
                         m = re.match('^ID: ([0-9a-f]*)', line)
-                        if m:
+                        if m: # CHIA
                             self.plot_id = m.group(1)
+                            self.plotter = 'chia'
                             found_id = True
+                        else: 
+                            m = re.match("^Plot Name: plot-k(\d+)-(\d+)-(\d+)-(\d+)-(\d+)-(\d+)-(\w+)$", line)
+                            if m: # MADMAX
+                                self.plot_id = m.group(7)
+                                self.plotter = 'madmax'
+                                found_id = True
                         m = re.match(r'^Starting phase 1/4:.*\.\.\. (.*)', line)
-                        if m:
+                        if m: # CHIA
                             # Mon Nov  2 08:39:53 2020
                             self.start_time = parse_chia_plot_time(m.group(1))
                             found_log = True
                             break  # Stop reading lines in file
+                        else: # MADMAX
+                            self.start_time = pendulum.from_timestamp(os.path.getctime(self.logfile))
 
             if found_id and found_log:
                 break  # Stop trying
@@ -352,26 +377,50 @@ def set_phase_from_logfile(self) -> None:
         with open(self.logfile, 'r') as f:
             with contextlib.suppress(UnicodeDecodeError):
                 for line in f:
-                    # "Starting phase 1/4: Forward Propagation into tmp files... Sat Oct 31 11:27:04 2020"
+                    # CHIA: "Starting phase 1/4: Forward Propagation into tmp files... Sat Oct 31 11:27:04 2020"
                     m = re.match(r'^Starting phase (\d).*', line)
                     if m:
                         phase = int(m.group(1))
                         phase_subphases[phase] = 0
+
+                    # MADMAX: "[P1]" or "[P2]" or "[P3]" or "[P4]"
+                    m = re.match(r'^\[P(\d)\].*', line)
+                    if m:
+                        phase = int(m.group(1))
+                        phase_subphases[phase] = 0
 
-                    # Phase 1: "Computing table 2"
+                    # CHIA: Phase 1: "Computing table 2"
                     m = re.match(r'^Computing table (\d).*', line)
                     if m:
                         phase_subphases[1] = max(phase_subphases[1], int(m.group(1)))
+
+                    # MADMAX: Phase 1: "[P1] Table 2"
+                    m = re.match(r'^\[P1\] Table (\d).*', line)
+                    if m:
+                        phase_subphases[1] = max(phase_subphases[1], int(m.group(1)))
 
-                    # Phase 2: "Backpropagating on table 2"
+                    # CHIA: Phase 2: "Backpropagating on table 2"
                     m = re.match(r'^Backpropagating on table (\d).*', line)
                     if m:
                         phase_subphases[2] = max(phase_subphases[2], 7 - int(m.group(1)))
 
-                    # Phase 3: "Compressing tables 4 and 5"
+                    # MADMAX: Phase 2: "[P2] Table 2"
+                    m = re.match(r'^\[P2\] Table (\d).*', line)
+                    if m:
+                        phase_subphases[2] = max(phase_subphases[2], 7 - int(m.group(1)))
+
+                    # CHIA: Phase 3: "Compressing tables 4 and 5"
                     m = re.match(r'^Compressing tables (\d) and (\d).*', line)
                     if m:
                         phase_subphases[3] = max(phase_subphases[3], int(m.group(1)))
+
+                    # MADMAX: Phase 3: "[P3-1] Table 4"
+                    m = re.match(r'^\[P3\-\d\] Table (\d).*', line)
+                    if m:
+                        if 3 in phase_subphases:
+                            phase_subphases[3] = max(phase_subphases[3], int(m.group(1)))
+                        else:
+                            phase_subphases[3] = int(m.group(1))
 
                     # TODO also collect timing info: