Initial support for Madmax plotter.

CHANGELOG.md

@@ -15,6 +15,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
   ([#643](https://github.com/ericaltendorf/plotman/pull/643))
 - `plotman prometheus` command to output status for consumption by [Prometheus](https://prometheus.io/).
   ([#430](https://github.com/ericaltendorf/plotman/pull/430))
+- Initial support for MadMax plotter.
+  ([#797](https://github.com/ericaltendorf/plotman/pull/797))
 
 ## [0.4.1] - 2021-06-11
 ### Fixed

src/plotman/analyzer.py

@@ -58,12 +58,18 @@ def analyze(logfilenames: typing.List[str], clipterminals: bool, bytmp: bool, by
                     else:
                         sl += '-bitfield'
 
-                # Phase timing.  Sample log line:
+                # CHIA: Phase timing.  Sample log line:
                 # Time for phase 1 = 22796.7 seconds. CPU (98%) Tue Sep 29 17:57:19 2020
                 for phase in ['1', '2', '3', '4']:
                     m = re.search(r'^Time for phase ' + phase + ' = (\d+.\d+) seconds..*', line)
                     if m:
                         phase_time[phase] = float(m.group(1))
+
+                # MADMAX: Phase timing.  Sample log line: "Phase 2 took 2193.37 sec"
+                for phase in ['1', '2', '3', '4']:
+                    m = re.search(r'^Phase ' + phase + ' took (\d+.\d+) sec.*', line)
+                    if m:
+                        phase_time[phase] = float(m.group(1))
 
                 # Uniform sort.  Sample log line:
                 # Bucket 267 uniform sort. Ram: 0.920GiB, u_sort min: 0.688GiB, qs min: 0.172GiB.
@@ -82,7 +88,7 @@ def analyze(logfilenames: typing.List[str], clipterminals: bool, bytmp: bool, by
                     else:
                         print ('Warning: unrecognized sort ' + sorter)
 
-                # Job completion.  Record total time in sliced data store.
+                # CHIA: Job completion.  Record total time in sliced data store.
                 # Sample log line:
                 # Total time = 49487.1 seconds. CPU (97.26%) Wed Sep 30 01:22:10 2020
                 m = re.search(r'^Total time = (\d+.\d+) seconds.*', line)
@@ -94,13 +100,22 @@ def analyze(logfilenames: typing.List[str], clipterminals: bool, bytmp: bool, by
                         for phase in ['1', '2', '3', '4']:
                             data.setdefault(sl, {}).setdefault('phase ' + phase, []).append(phase_time[phase])
                         data.setdefault(sl, {}).setdefault('%usort', []).append(100 * n_uniform // n_sorts)
+
+                # MADMAX: Job completion.  Record total time in sliced data store.
+                # Sample log line: "Total plot creation time was 2530.76 sec"
+                m = re.search(r'^Total plot creation time was (\d+.\d+) sec.*', line)
+                if m:
+                    data.setdefault(sl, {}).setdefault('total time', []).append(float(m.group(1)))
+                    for phase in ['1', '2', '3', '4']:
+                        data.setdefault(sl, {}).setdefault('phase ' + phase, []).append(phase_time[phase])
+                    data.setdefault(sl, {}).setdefault('%usort', []).append(0)  # Not available for MADMAX
 
     # Prepare report
     tab = tt.Texttable()
     all_measures = ['%usort', 'phase 1', 'phase 2', 'phase 3', 'phase 4', 'total time']
     headings = ['Slice', 'n'] + all_measures
     tab.header(headings)
-
+    
     for sl in data.keys():
         row = [sl]
 

src/plotman/configuration.py

@@ -275,11 +275,12 @@ class Scheduling:
 
 @attr.frozen
 class Plotting:
-    k: int
-    e: bool
-    n_threads: int
-    n_buckets: int
-    job_buffer: int
+    type: str = "chia"
+    n_threads: int = 2
+    n_buckets: int = 128
+    k: Optional[int] = 32
+    e: Optional[bool] = False
+    job_buffer: Optional[int] = 3389
     farmer_pk: Optional[str] = None
     pool_pk: Optional[str] = None
     pool_contract_address: Optional[str] = None

src/plotman/job.py

@@ -30,14 +30,17 @@ def job_phases_for_dstdir(d: str, all_jobs: typing.List["Job"]) -> typing.List["
     return sorted([j.progress() for j in all_jobs if j.dstdir == d])
 
 def is_plotting_cmdline(cmdline: typing.List[str]) -> bool:
-    if cmdline and 'python' in cmdline[0].lower():
+    if cmdline and 'python' in cmdline[0].lower():  # Stock Chia plotter
         cmdline = cmdline[1:]
-    return (
-        len(cmdline) >= 3
-        and 'chia' in cmdline[0]
-        and 'plots' == cmdline[1]
-        and 'create' == cmdline[2]
-    )
+        return (
+            len(cmdline) >= 3
+            and 'chia' in cmdline[0]
+            and 'plots' == cmdline[1]
+            and 'create' == cmdline[2]
+        )
+    elif cmdline and 'chia_plot' == cmdline[0].lower():  # Madmax plotter
+        return True
+    return False
 
 def parse_chia_plot_time(s: str) -> pendulum.DateTime:
     # This will grow to try ISO8601 as well for when Chia logs that way
@@ -50,14 +53,16 @@ def parse_chia_plots_create_command_line(
 ) -> "ParsedChiaPlotsCreateCommand":
     command_line = list(command_line)
     # Parse command line args
-    if 'python' in command_line[0].lower():
+    if 'python' in command_line[0].lower():  # Stock Chia plotter
         command_line = command_line[1:]
-    assert len(command_line) >= 3
-    assert 'chia' in command_line[0]
-    assert 'plots' == command_line[1]
-    assert 'create' == command_line[2]
-
-    all_command_arguments = command_line[3:]
+        assert len(command_line) >= 3
+        assert 'chia' in command_line[0]
+        assert 'plots' == command_line[1]
+        assert 'create' == command_line[2]
+        all_command_arguments = command_line[3:]
+    elif 'chia_plot' in command_line[0].lower():  # Madmax plotter
+        command_line = command_line[1:]
+        all_command_arguments = command_line[2:]
 
     # nice idea, but this doesn't include -h
     # help_option_names = command.get_help_option_names(ctx=context)
@@ -146,6 +151,7 @@ class Job:
     jobfile: str = ''
     job_id: int = 0
     plot_id: str = '--------'
+    plotter: str = ''
     proc: psutil.Process
     k: int
     r: int
@@ -311,15 +317,24 @@ def init_from_logfile(self) -> None:
                 with contextlib.suppress(UnicodeDecodeError):
                     for line in f:
                         m = re.match('^ID: ([0-9a-f]*)', line)
-                        if m:
+                        if m: # CHIA
                             self.plot_id = m.group(1)
+                            self.plotter = 'chia'
                             found_id = True
+                        else: 
+                            m = re.match("^Plot Name: plot-k(\d+)-(\d+)-(\d+)-(\d+)-(\d+)-(\d+)-(\w+)$", line)
+                            if m: # MADMAX
+                                self.plot_id = m.group(7)
+                                self.plotter = 'madmax'
+                                found_id = True
                         m = re.match(r'^Starting phase 1/4:.*\.\.\. (.*)', line)
-                        if m:
+                        if m: # CHIA
                             # Mon Nov  2 08:39:53 2020
                             self.start_time = parse_chia_plot_time(m.group(1))
                             found_log = True
                             break  # Stop reading lines in file
+                        else: # MADMAX
+                            self.start_time = datetime.fromtimestamp(os.path.getctime(self.logfile))
 
             if found_id and found_log:
                 break  # Stop trying
@@ -352,26 +367,50 @@ def set_phase_from_logfile(self) -> None:
         with open(self.logfile, 'r') as f:
             with contextlib.suppress(UnicodeDecodeError):
                 for line in f:
-                    # "Starting phase 1/4: Forward Propagation into tmp files... Sat Oct 31 11:27:04 2020"
+                    # CHIA: "Starting phase 1/4: Forward Propagation into tmp files... Sat Oct 31 11:27:04 2020"
                     m = re.match(r'^Starting phase (\d).*', line)
                     if m:
                         phase = int(m.group(1))
                         phase_subphases[phase] = 0
+
+                    # MADMAX: "[P1]" or "[P2]" or "[P3]" or "[P4]"
+                    m = re.match(r'^\[P(\d)\].*', line)
+                    if m:
+                        phase = int(m.group(1))
+                        phase_subphases[phase] = 0
 
-                    # Phase 1: "Computing table 2"
+                    # CHIA: Phase 1: "Computing table 2"
                     m = re.match(r'^Computing table (\d).*', line)
                     if m:
                         phase_subphases[1] = max(phase_subphases[1], int(m.group(1)))
+
+                    # MADMAX: Phase 1: "[P1] Table 2"
+                    m = re.match(r'^\[P1\] Table (\d).*', line)
+                    if m:
+                        phase_subphases[1] = max(phase_subphases[1], int(m.group(1)))
 
-                    # Phase 2: "Backpropagating on table 2"
+                    # CHIA: Phase 2: "Backpropagating on table 2"
                     m = re.match(r'^Backpropagating on table (\d).*', line)
                     if m:
                         phase_subphases[2] = max(phase_subphases[2], 7 - int(m.group(1)))
 
-                    # Phase 3: "Compressing tables 4 and 5"
+                    # MADMAX: Phase 2: "[P2] Table 2"
+                    m = re.match(r'^\[P2\] Table (\d).*', line)
+                    if m:
+                        phase_subphases[2] = max(phase_subphases[2], 7 - int(m.group(1)))
+
+                    # CHIA: Phase 3: "Compressing tables 4 and 5"
                     m = re.match(r'^Compressing tables (\d) and (\d).*', line)
                     if m:
                         phase_subphases[3] = max(phase_subphases[3], int(m.group(1)))
+
+                    # MADMAX: Phase 3: "[P3-1] Table 4"
+                    m = re.match(r'^\[P3\-\d\] Table (\d).*', line)
+                    if m:
+                        if 3 in phase_subphases:
+                            phase_subphases[3] = max(phase_subphases[3], int(m.group(1)))
+                        else:
+                            phase_subphases[3] = int(m.group(1))
 
                     # TODO also collect timing info:
 

src/plotman/manager.py

@@ -121,29 +121,38 @@ def key(key: str) -> job.Phase:
 
             log_file_path = log_cfg.create_plot_log_path(time=pendulum.now())
 
-            plot_args: typing.List[str] = ['chia', 'plots', 'create',
+            if plotting_cfg.type == "madmax":
+                plot_args: typing.List[str] = ['chia_plot',
+                    '-n', str(1),
+                    '-r', str(plotting_cfg.n_threads),
+                    '-u', str(plotting_cfg.n_buckets),
+                    '-t', tmpdir if tmpdir.endswith('/') else (tmpdir + '/'),
+                    '-d', dstdir if dstdir.endswith('/') else (dstdir + '/') ]
+            else:
+                plot_args: typing.List[str] = ['chia', 'plots', 'create',
                     '-k', str(plotting_cfg.k),
                     '-r', str(plotting_cfg.n_threads),
                     '-u', str(plotting_cfg.n_buckets),
                     '-b', str(plotting_cfg.job_buffer),
                     '-t', tmpdir,
                     '-d', dstdir ]
-            if plotting_cfg.e:
-                plot_args.append('-e')
+                if plotting_cfg.e:
+                    plot_args.append('-e')
+                if plotting_cfg.pool_contract_address is not None:
+                    plot_args.append('-c')
+                    plot_args.append(plotting_cfg.pool_contract_address)
+                if plotting_cfg.x:
+                    plot_args.append('-x')  
             if plotting_cfg.farmer_pk is not None:
                 plot_args.append('-f')
                 plot_args.append(plotting_cfg.farmer_pk)
             if plotting_cfg.pool_pk is not None:
                 plot_args.append('-p')
                 plot_args.append(plotting_cfg.pool_pk)
-            if plotting_cfg.pool_contract_address is not None:
-                plot_args.append('-c')
-                plot_args.append(plotting_cfg.pool_contract_address)
             if dir_cfg.tmp2 is not None:
                 plot_args.append('-2')
                 plot_args.append(dir_cfg.tmp2)
-            if plotting_cfg.x:
-                plot_args.append('-x')  
+
 
             logmsg = ('Starting plot job: %s ; logging to %s' % (' '.join(plot_args), log_file_path))
 

src/plotman/reporting.py

@@ -75,7 +75,7 @@ def status_report(jobs: typing.List[job.Job], width: int, height: typing.Optiona
         n_end_rows = n_rows - n_begin_rows
 
     tab = tt.Texttable()
-    headings = ['plot id', 'k', 'tmp', 'dst', 'wall', 'phase', 'tmp',
+    headings = ['plot id', 'plotter', 'k', 'tmp', 'dst', 'wall', 'phase', 'tmp',
             'pid', 'stat', 'mem', 'user', 'sys', 'io']
     if height:
         headings.insert(0, '#')
@@ -97,6 +97,7 @@ def status_report(jobs: typing.List[job.Job], width: int, height: typing.Optiona
             try:
                 with j.proc.oneshot():
                     row = [j.plot_id[:8], # Plot ID
+                        str(j.plotter), # chia or madmax
                         str(j.k), # k size
                         abbr_path(j.tmpdir, tmp_prefix), # Temp directory
                         abbr_path(j.dstdir, dst_prefix), # Destination directory

src/plotman/resources/plotman.yaml

@@ -136,15 +136,19 @@ scheduling:
 # See documentation at
 # https://github.com/Chia-Network/chia-blockchain/wiki/CLI-Commands-Reference#create
 plotting:
-        k: 32
+        # Your public keys: farmer and pool - Required for madmax, optional for chia with mnemonic.txt
+        # farmer_pk: ...
+        # pool_pk: ...
+
+        # If you enable Chia, plot in *parallel* with higher tmpdir_max_jobs and global_max_jobs
+        type: chia           # The stock plotter: https://github.com/Chia-Network/chia-blockchain
+        k: 32                # k-size of plot, leave at 32 most of the time
         e: False             # Use -e plotting option
         n_threads: 2         # Threads per job
         n_buckets: 128       # Number of buckets to split data into
         job_buffer: 3389     # Per job memory
-        # If specified, pass through to the -f and -p options.  See CLI reference.
-        #   farmer_pk: ...
-        #   pool_pk: ...
-        # If true, Skips adding [final dir] / dst to harvester for farming. 
-        # Especially useful if you have harvesters that are running somewhere else
-        # and you are just plotting on the machine where plotman is running.
-        #   x: True
+
+        # If you enable MadMax, plot in *sequence* with very low tmpdir_max_jobs and global_max_jobs
+        #type: madmax          # Madmax plotter: https://github.com/madMAx43v3r/chia-plotter
+        #n_threads: 4          # Default is 4, crank up if you have many cores
+        #n_buckets: 256        # Default is 256