From 256f5c98fc0ecfdd99774214c5b97ff6cc2f3ffc Mon Sep 17 00:00:00 2001 From: Guy Davis Date: Thu, 17 Jun 2021 19:56:35 -0600 Subject: [PATCH 01/17] Initial support for Madmax plotter. --- src/plotman/analyzer.py | 19 +++++++- src/plotman/configuration.py | 11 +++-- src/plotman/job.py | 73 ++++++++++++++++++++++-------- src/plotman/manager.py | 25 ++++++---- src/plotman/resources/plotman.yaml | 20 ++++---- 5 files changed, 105 insertions(+), 43 deletions(-) diff --git a/src/plotman/analyzer.py b/src/plotman/analyzer.py index 67073805..dcbcdc67 100644 --- a/src/plotman/analyzer.py +++ b/src/plotman/analyzer.py @@ -57,12 +57,18 @@ def analyze(logfilenames, clipterminals, bytmp, bybitfield): else: sl += '-bitfield' - # Phase timing. Sample log line: + # CHIA: Phase timing. Sample log line: # Time for phase 1 = 22796.7 seconds. CPU (98%) Tue Sep 29 17:57:19 2020 for phase in ['1', '2', '3', '4']: m = re.search(r'^Time for phase ' + phase + ' = (\d+.\d+) seconds..*', line) if m: phase_time[phase] = float(m.group(1)) + + # MADMAX: Phase timing. Sample log line: "Phase 2 took 2193.37 sec" + for phase in ['1', '2', '3', '4']: + m = re.search(r'^Phase ' + phase + ' took (\d+.\d+) sec.*', line) + if m: + phase_time[phase] = float(m.group(1)) # Uniform sort. Sample log line: # Bucket 267 uniform sort. Ram: 0.920GiB, u_sort min: 0.688GiB, qs min: 0.172GiB. @@ -81,7 +87,7 @@ def analyze(logfilenames, clipterminals, bytmp, bybitfield): else: print ('Warning: unrecognized sort ' + sorter) - # Job completion. Record total time in sliced data store. + # CHIA: Job completion. Record total time in sliced data store. # Sample log line: # Total time = 49487.1 seconds. CPU (97.26%) Wed Sep 30 01:22:10 2020 m = re.search(r'^Total time = (\d+.\d+) seconds.*', line) @@ -93,6 +99,15 @@ def analyze(logfilenames, clipterminals, bytmp, bybitfield): for phase in ['1', '2', '3', '4']: data.setdefault(sl, {}).setdefault('phase ' + phase, []).append(phase_time[phase]) data.setdefault(sl, {}).setdefault('%usort', []).append(100 * n_uniform // n_sorts) + + # MADMAX: Job completion. Record total time in sliced data store. + # Sample log line: "Total plot creation time was 2530.76 sec" + m = re.search(r'^Total plot creation time was (\d+.\d+) sec.*', line) + if m: + data.setdefault(sl, {}).setdefault('total time', []).append(float(m.group(1))) + for phase in ['1', '2', '3', '4']: + data.setdefault(sl, {}).setdefault('phase ' + phase, []).append(phase_time[phase]) + data.setdefault(sl, {}).setdefault('%usort', []).append('') # Not available for MADMAX # Prepare report tab = tt.Texttable() diff --git a/src/plotman/configuration.py b/src/plotman/configuration.py index 220dc538..6a25c584 100644 --- a/src/plotman/configuration.py +++ b/src/plotman/configuration.py @@ -253,11 +253,12 @@ class Scheduling: @attr.frozen class Plotting: - k: int - e: bool - n_threads: int - n_buckets: int - job_buffer: int + type: str = "chia" + n_threads: int = 2 + n_buckets: int = 128 + k: Optional[int] = 32 + e: Optional[bool] = False + job_buffer: Optional[int] = 3389 farmer_pk: Optional[str] = None pool_pk: Optional[str] = None pool_contract_address: Optional[str] = None diff --git a/src/plotman/job.py b/src/plotman/job.py index e99ac07f..880e094d 100644 --- a/src/plotman/job.py +++ b/src/plotman/job.py @@ -29,14 +29,17 @@ def job_phases_for_dstdir(d, all_jobs): return sorted([j.progress() for j in all_jobs if j.dstdir == d]) def is_plotting_cmdline(cmdline): - if cmdline and 'python' in cmdline[0].lower(): + if cmdline and 'python' in cmdline[0].lower(): # Stock Chia plotter cmdline = cmdline[1:] - return ( - len(cmdline) >= 3 - and 'chia' in cmdline[0] - and 'plots' == cmdline[1] - and 'create' == cmdline[2] - ) + return ( + len(cmdline) >= 3 + and 'chia' in cmdline[0] + and 'plots' == cmdline[1] + and 'create' == cmdline[2] + ) + elif cmdline and 'chia_plot' == cmdline[0].lower(): # Madmax plotter + return True + return False def parse_chia_plot_time(s): # This will grow to try ISO8601 as well for when Chia logs that way @@ -45,14 +48,16 @@ def parse_chia_plot_time(s): def parse_chia_plots_create_command_line(command_line): command_line = list(command_line) # Parse command line args - if 'python' in command_line[0].lower(): + if 'python' in command_line[0].lower(): # Stock Chia plotter command_line = command_line[1:] - assert len(command_line) >= 3 - assert 'chia' in command_line[0] - assert 'plots' == command_line[1] - assert 'create' == command_line[2] - - all_command_arguments = command_line[3:] + assert len(command_line) >= 3 + assert 'chia' in command_line[0] + assert 'plots' == command_line[1] + assert 'create' == command_line[2] + all_command_arguments = command_line[3:] + elif 'chia_plot' in command_line[0].lower(): # Madmax plotter + command_line = command_line[1:] + all_command_arguments = command_line[2:] # nice idea, but this doesn't include -h # help_option_names = command.get_help_option_names(ctx=context) @@ -275,15 +280,22 @@ def init_from_logfile(self): with contextlib.suppress(UnicodeDecodeError): for line in f: m = re.match('^ID: ([0-9a-f]*)', line) - if m: + if m: # CHIA self.plot_id = m.group(1) found_id = True + else: + m = re.match("^Plot Name: plot-k(\d+)-(\d+)-(\d+)-(\d+)-(\d+)-(\d+)-(\w+)$", line) + if m: # MADMAX + self.plot_id = m.group(7) + found_id = True m = re.match(r'^Starting phase 1/4:.*\.\.\. (.*)', line) - if m: + if m: # CHIA # Mon Nov 2 08:39:53 2020 self.start_time = parse_chia_plot_time(m.group(1)) found_log = True break # Stop reading lines in file + else: # MADMAX + self.start_time = datetime.fromtimestamp(os.path.getctime(self.logfile)) if found_id and found_log: break # Stop trying @@ -316,26 +328,47 @@ def set_phase_from_logfile(self): with open(self.logfile, 'r') as f: with contextlib.suppress(UnicodeDecodeError): for line in f: - # "Starting phase 1/4: Forward Propagation into tmp files... Sat Oct 31 11:27:04 2020" + # CHIA: "Starting phase 1/4: Forward Propagation into tmp files... Sat Oct 31 11:27:04 2020" m = re.match(r'^Starting phase (\d).*', line) if m: phase = int(m.group(1)) phase_subphases[phase] = 0 + + # MADMAX: "[P1]" or "[P2]" or "[P3]" or "[P4]" + m = re.match(r'^\[P(\d)\].*', line) + if m: + phase = int(m.group(1)) + phase_subphases[phase] = 0 - # Phase 1: "Computing table 2" + # CHIA: Phase 1: "Computing table 2" m = re.match(r'^Computing table (\d).*', line) if m: phase_subphases[1] = max(phase_subphases[1], int(m.group(1))) + + # MADMAX: Phase 1: "[P1] Table 2" + m = re.match(r'^\[P1\] Table (\d).*', line) + if m: + phase_subphases[1] = max(phase_subphases[1], int(m.group(1))) - # Phase 2: "Backpropagating on table 2" + # CHIA: Phase 2: "Backpropagating on table 2" m = re.match(r'^Backpropagating on table (\d).*', line) if m: phase_subphases[2] = max(phase_subphases[2], 7 - int(m.group(1))) - # Phase 3: "Compressing tables 4 and 5" + # MADMAX: Phase 2: "[P2] Table 2" + m = re.match(r'^\[P2\] Table (\d).*', line) + if m: + phase_subphases[2] = max(phase_subphases[2], 7 - int(m.group(1))) + + # CHIA: Phase 3: "Compressing tables 4 and 5" m = re.match(r'^Compressing tables (\d) and (\d).*', line) if m: phase_subphases[3] = max(phase_subphases[3], int(m.group(1))) + + # MADMAX: Phase 3: "[P3-1] Table 4" + m = re.match(r'^\[P3\-\d\] Table (\d).*', line) + if m: + phase_subphases[3] = max(phase_subphases[3], int(m.group(1))) # TODO also collect timing info: diff --git a/src/plotman/manager.py b/src/plotman/manager.py index 7c5b116d..6575636b 100644 --- a/src/plotman/manager.py +++ b/src/plotman/manager.py @@ -115,29 +115,38 @@ def maybe_start_new_plot(dir_cfg, sched_cfg, plotting_cfg, log_cfg): log_file_path = log_cfg.create_plot_log_path(time=pendulum.now()) - plot_args = ['chia', 'plots', 'create', + if plotting_cfg.type == "madmax": + plot_args = ['chia_plot', + '-n', str(1), + '-r', str(plotting_cfg.n_threads), + '-u', str(plotting_cfg.n_buckets), + '-t', tmpdir if tmpdir.endswith('/') else (tmpdir + '/'), + '-d', dstdir if dstdir.endswith('/') else (dstdir + '/') ] + else: + plot_args = ['chia', 'plots', 'create', '-k', str(plotting_cfg.k), '-r', str(plotting_cfg.n_threads), '-u', str(plotting_cfg.n_buckets), '-b', str(plotting_cfg.job_buffer), '-t', tmpdir, '-d', dstdir ] - if plotting_cfg.e: - plot_args.append('-e') + if plotting_cfg.e: + plot_args.append('-e') + if plotting_cfg.pool_contract_address is not None: + plot_args.append('-c') + plot_args.append(plotting_cfg.pool_contract_address) + if plotting_cfg.x: + plot_args.append('-x') if plotting_cfg.farmer_pk is not None: plot_args.append('-f') plot_args.append(plotting_cfg.farmer_pk) if plotting_cfg.pool_pk is not None: plot_args.append('-p') plot_args.append(plotting_cfg.pool_pk) - if plotting_cfg.pool_contract_address is not None: - plot_args.append('-c') - plot_args.append(plotting_cfg.pool_contract_address) if dir_cfg.tmp2 is not None: plot_args.append('-2') plot_args.append(dir_cfg.tmp2) - if plotting_cfg.x: - plot_args.append('-x') + logmsg = ('Starting plot job: %s ; logging to %s' % (' '.join(plot_args), log_file_path)) diff --git a/src/plotman/resources/plotman.yaml b/src/plotman/resources/plotman.yaml index 09ee52ad..d4967aef 100644 --- a/src/plotman/resources/plotman.yaml +++ b/src/plotman/resources/plotman.yaml @@ -136,15 +136,19 @@ scheduling: # See documentation at # https://github.com/Chia-Network/chia-blockchain/wiki/CLI-Commands-Reference#create plotting: - k: 32 + # Your public keys: farmer and pool - Required for madmax, optional for chia with mnemonic.txt + # farmer_pk: ... + # pool_pk: ... + + # If you enable Chia, plot in *parallel* with higher tmpdir_max_jobs and global_max_jobs + type: chia # The stock plotter: https://github.com/Chia-Network/chia-blockchain + k: 32 # k-size of plot, leave at 32 most of the time e: False # Use -e plotting option n_threads: 2 # Threads per job n_buckets: 128 # Number of buckets to split data into job_buffer: 3389 # Per job memory - # If specified, pass through to the -f and -p options. See CLI reference. - # farmer_pk: ... - # pool_pk: ... - # If true, Skips adding [final dir] / dst to harvester for farming. - # Especially useful if you have harvesters that are running somewhere else - # and you are just plotting on the machine where plotman is running. - # x: True + + # If you enable MadMax, plot in *sequence* with very low tmpdir_max_jobs and global_max_jobs + #type: madmax # Madmax plotter: https://github.com/madMAx43v3r/chia-plotter + #n_threads: 4 # Default is 4, crank up if you have many cores + #n_buckets: 256 # Default is 256 From 8c1ca0e9032edfa21d92c67a9ca59e2052cf73c9 Mon Sep 17 00:00:00 2001 From: Guy Davis Date: Thu, 17 Jun 2021 20:48:15 -0600 Subject: [PATCH 02/17] Guard against index error. --- src/plotman/job.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/plotman/job.py b/src/plotman/job.py index 880e094d..a257fccf 100644 --- a/src/plotman/job.py +++ b/src/plotman/job.py @@ -368,7 +368,10 @@ def set_phase_from_logfile(self): # MADMAX: Phase 3: "[P3-1] Table 4" m = re.match(r'^\[P3\-\d\] Table (\d).*', line) if m: - phase_subphases[3] = max(phase_subphases[3], int(m.group(1))) + if 3 in phase_subphases: + phase_subphases[3] = max(phase_subphases[3], int(m.group(1))) + else: + phase_subphases[3] = int(m.group(1)) # TODO also collect timing info: From fb4a05cd866a3348d171f19ea767c21ead285602 Mon Sep 17 00:00:00 2001 From: Guy Davis Date: Fri, 18 Jun 2021 07:05:14 -0600 Subject: [PATCH 03/17] Madmax doesn't seem to output sorting info. --- src/plotman/analyzer.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/plotman/analyzer.py b/src/plotman/analyzer.py index 735e1c50..b7229fe2 100644 --- a/src/plotman/analyzer.py +++ b/src/plotman/analyzer.py @@ -108,14 +108,14 @@ def analyze(logfilenames: typing.List[str], clipterminals: bool, bytmp: bool, by data.setdefault(sl, {}).setdefault('total time', []).append(float(m.group(1))) for phase in ['1', '2', '3', '4']: data.setdefault(sl, {}).setdefault('phase ' + phase, []).append(phase_time[phase]) - data.setdefault(sl, {}).setdefault('%usort', []).append('') # Not available for MADMAX + data.setdefault(sl, {}).setdefault('%usort', []).append(0) # Not available for MADMAX # Prepare report tab = tt.Texttable() all_measures = ['%usort', 'phase 1', 'phase 2', 'phase 3', 'phase 4', 'total time'] headings = ['Slice', 'n'] + all_measures tab.header(headings) - + for sl in data.keys(): row = [sl] From 41a4ab28c3f3b721c809c48ca29cab46f841050c Mon Sep 17 00:00:00 2001 From: Guy Davis Date: Sun, 20 Jun 2021 13:43:34 -0600 Subject: [PATCH 04/17] Improvements for code review. --- CHANGELOG.md | 2 ++ src/plotman/job.py | 3 +++ src/plotman/reporting.py | 3 ++- 3 files changed, 7 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b03cf8ad..badb9e0f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,6 +15,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ([#643](https://github.com/ericaltendorf/plotman/pull/643)) - `plotman prometheus` command to output status for consumption by [Prometheus](https://prometheus.io/). ([#430](https://github.com/ericaltendorf/plotman/pull/430)) +- Initial support for MadMax plotter. + ([#797](https://github.com/ericaltendorf/plotman/pull/797)) ## [0.4.1] - 2021-06-11 ### Fixed diff --git a/src/plotman/job.py b/src/plotman/job.py index 34db4d25..0fe3fdc1 100644 --- a/src/plotman/job.py +++ b/src/plotman/job.py @@ -151,6 +151,7 @@ class Job: jobfile: str = '' job_id: int = 0 plot_id: str = '--------' + plotter: str = '' proc: psutil.Process k: int r: int @@ -318,11 +319,13 @@ def init_from_logfile(self) -> None: m = re.match('^ID: ([0-9a-f]*)', line) if m: # CHIA self.plot_id = m.group(1) + self.plotter = 'chia' found_id = True else: m = re.match("^Plot Name: plot-k(\d+)-(\d+)-(\d+)-(\d+)-(\d+)-(\d+)-(\w+)$", line) if m: # MADMAX self.plot_id = m.group(7) + self.plotter = 'madmax' found_id = True m = re.match(r'^Starting phase 1/4:.*\.\.\. (.*)', line) if m: # CHIA diff --git a/src/plotman/reporting.py b/src/plotman/reporting.py index 0a74d84d..ac759810 100644 --- a/src/plotman/reporting.py +++ b/src/plotman/reporting.py @@ -75,7 +75,7 @@ def status_report(jobs: typing.List[job.Job], width: int, height: typing.Optiona n_end_rows = n_rows - n_begin_rows tab = tt.Texttable() - headings = ['plot id', 'k', 'tmp', 'dst', 'wall', 'phase', 'tmp', + headings = ['plot id', 'plotter', 'k', 'tmp', 'dst', 'wall', 'phase', 'tmp', 'pid', 'stat', 'mem', 'user', 'sys', 'io'] if height: headings.insert(0, '#') @@ -97,6 +97,7 @@ def status_report(jobs: typing.List[job.Job], width: int, height: typing.Optiona try: with j.proc.oneshot(): row = [j.plot_id[:8], # Plot ID + str(j.plotter), # chia or madmax str(j.k), # k size abbr_path(j.tmpdir, tmp_prefix), # Temp directory abbr_path(j.dstdir, dst_prefix), # Destination directory From 169def83b4af0a8107f484dd940c1261f44ed928 Mon Sep 17 00:00:00 2001 From: Guy Davis Date: Mon, 21 Jun 2021 10:45:12 -0600 Subject: [PATCH 05/17] Update src/plotman/job.py Co-authored-by: Kyle Altendorf --- src/plotman/job.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/plotman/job.py b/src/plotman/job.py index d27cb6ee..d659c92e 100644 --- a/src/plotman/job.py +++ b/src/plotman/job.py @@ -334,7 +334,7 @@ def init_from_logfile(self) -> None: found_log = True break # Stop reading lines in file else: # MADMAX - self.start_time = datetime.fromtimestamp(os.path.getctime(self.logfile)) + self.start_time = pendulum.from_timestamp(os.path.getctime(self.logfile)) if found_id and found_log: break # Stop trying From 2761c6851a08b0f13e0d74c691c7aca04e699a97 Mon Sep 17 00:00:00 2001 From: Guy Davis Date: Mon, 21 Jun 2021 10:48:28 -0600 Subject: [PATCH 06/17] Update src/plotman/manager.py Co-authored-by: Kyle Altendorf --- src/plotman/manager.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/plotman/manager.py b/src/plotman/manager.py index a3855ba8..80d31db7 100644 --- a/src/plotman/manager.py +++ b/src/plotman/manager.py @@ -121,15 +121,16 @@ def key(key: str) -> job.Phase: log_file_path = log_cfg.create_plot_log_path(time=pendulum.now()) + plot_args: typing.List[str] if plotting_cfg.type == "madmax": - plot_args: typing.List[str] = ['chia_plot', + plot_args = ['chia_plot', '-n', str(1), '-r', str(plotting_cfg.n_threads), '-u', str(plotting_cfg.n_buckets), '-t', tmpdir if tmpdir.endswith('/') else (tmpdir + '/'), '-d', dstdir if dstdir.endswith('/') else (dstdir + '/') ] else: - plot_args: typing.List[str] = ['chia', 'plots', 'create', + plot_args = ['chia', 'plots', 'create', '-k', str(plotting_cfg.k), '-r', str(plotting_cfg.n_threads), '-u', str(plotting_cfg.n_buckets), From 8b7e7ea48a366c2bc83f0a08b828a73d43a91749 Mon Sep 17 00:00:00 2001 From: Guy Davis Date: Mon, 21 Jun 2021 11:02:21 -0600 Subject: [PATCH 07/17] Madmax CLI options mapped to Click. --- src/plotman/madmax.py | 72 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 72 insertions(+) create mode 100644 src/plotman/madmax.py diff --git a/src/plotman/madmax.py b/src/plotman/madmax.py new file mode 100644 index 00000000..79476ad7 --- /dev/null +++ b/src/plotman/madmax.py @@ -0,0 +1,72 @@ +# mypy: allow_untyped_decorators +# +# Madmax is written in C++. Below is a mapping of its CLI options to Python. +# See: https://github.com/madMAx43v3r/chia-plotter/tree/master/src +# Note: versions are git commit refs, not semantic versioning +# + +import functools +import typing + +import click +from pathlib import Path +import typing_extensions + + +class CommandProtocol(typing_extensions.Protocol): + def make_context(self, info_name: str, args: typing.List[str]) -> click.Context: + ... + + def __call__(self) -> None: + ... + + +class Commands: + def __init__(self) -> None: + self.by_version: typing.Dict[typing.Sequence[int], CommandProtocol] = {} + + def register(self, version: typing.Sequence[int]) -> typing.Callable[[CommandProtocol], None]: + if version in self.by_version: + raise Exception(f'Version already registered: {version!r}') + + return functools.partial(self._decorator, version=version) + + def _decorator(self, command: CommandProtocol, *, version: typing.Sequence[int]) -> None: + self.by_version[version] = command + + def __getitem__(self, item: typing.Sequence[int]) -> typing.Callable[[], None]: + return self.by_version[item] + + def latest_command(self) -> CommandProtocol: + return max(self.by_version.items())[1] + + +commands = Commands() +# Madmax Git on 2021-06-19 -> https://github.com/madMAx43v3r/chia-plotter/commit/c8121b987186c42c895b49818e6c13acecc51332 +@commands.register(version=("c8121b9")) +@click.command() +# https://github.com/madMAx43v3r/chia-plotter/blob/master/LICENSE +# https://github.com/madMAx43v3r/chia-plotter/blob/master/src/chia_plot.cpp#L180 +@click.option("-n", "--count", help="Number of plots to create (default = 1, -1 = infinite)", + type=int, default=1, show_default=True) +@click.option("-r", "--threads", help="Number of threads (default = 4)", + type=int, default=4, show_default=True) +@click.option("-u", "--buckets", help="Number of buckets (default = 256)", + type=int, default=256, show_default=True) +@click.option("-v", "--buckets3", help="Number of buckets for phase 3+4 (default = buckets)", + type=int, default=256) +@click.option("-t", "--tmpdir", help="Temporary directory, needs ~220 GiB (default = $PWD)", + type=click.Path(), default=Path("."), show_default=True) +@click.option("-2", "--tmpdir2", help="Temporary directory 2, needs ~110 GiB [RAM] (default = )", + type=click.Path(), default=None) +@click.option("-d", "--finaldir", help="Final directory (default = )", + type=click.Path(), default=Path("."), show_default=True) +@click.option("-p", "--poolkey", help="Pool Public Key (48 bytes)", + type=str, default=None) +@click.option("-f", "--farmerkey", help="Farmer Public Key (48 bytes)", + type=str, default=None) +@click.option("-G", "--tmptoggle", help="Alternate tmpdir/tmpdir2", + type=str, default=None) +) +def _cli_c8121b9() -> None: + pass From 4c1978b8d9efffdac772ff8c3dca0cb640100368 Mon Sep 17 00:00:00 2001 From: Kyle Altendorf Date: Mon, 21 Jun 2021 21:54:32 -0400 Subject: [PATCH 08/17] remove extra ) --- src/plotman/madmax.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/plotman/madmax.py b/src/plotman/madmax.py index 79476ad7..21a429c0 100644 --- a/src/plotman/madmax.py +++ b/src/plotman/madmax.py @@ -67,6 +67,5 @@ def latest_command(self) -> CommandProtocol: type=str, default=None) @click.option("-G", "--tmptoggle", help="Alternate tmpdir/tmpdir2", type=str, default=None) -) def _cli_c8121b9() -> None: pass From 7b1572f19c8626057d63b6d2965519bccc883212 Mon Sep 17 00:00:00 2001 From: Kyle Altendorf Date: Mon, 21 Jun 2021 22:00:51 -0400 Subject: [PATCH 09/17] use madmax cli parsing standin --- src/plotman/job.py | 11 ++++++----- src/plotman/madmax.py | 4 +++- 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/src/plotman/job.py b/src/plotman/job.py index d659c92e..01f40ffe 100644 --- a/src/plotman/job.py +++ b/src/plotman/job.py @@ -18,7 +18,7 @@ import pendulum import psutil -from plotman import chia +from plotman import chia, madmax def job_phases_for_tmpdir(d: str, all_jobs: typing.List["Job"]) -> typing.List["Phase"]: @@ -60,9 +60,14 @@ def parse_chia_plots_create_command_line( assert 'plots' == command_line[1] assert 'create' == command_line[2] all_command_arguments = command_line[3:] + # TODO: We could at some point do chia version detection and pick the + # associated command. For now we'll just use the latest one we have + # copied. + command = chia.commands.latest_command() elif 'chia_plot' in command_line[0].lower(): # Madmax plotter command_line = command_line[1:] all_command_arguments = command_line[2:] + command = madmax._cli_c8121b9 # nice idea, but this doesn't include -h # help_option_names = command.get_help_option_names(ctx=context) @@ -74,10 +79,6 @@ def parse_chia_plots_create_command_line( if argument not in help_option_names ] - # TODO: We could at some point do chia version detection and pick the - # associated command. For now we'll just use the latest one we have - # copied. - command = chia.commands.latest_command() try: context = command.make_context(info_name='', args=list(command_arguments)) except click.ClickException as e: diff --git a/src/plotman/madmax.py b/src/plotman/madmax.py index 21a429c0..c6ba0626 100644 --- a/src/plotman/madmax.py +++ b/src/plotman/madmax.py @@ -43,7 +43,9 @@ def latest_command(self) -> CommandProtocol: commands = Commands() # Madmax Git on 2021-06-19 -> https://github.com/madMAx43v3r/chia-plotter/commit/c8121b987186c42c895b49818e6c13acecc51332 -@commands.register(version=("c8121b9")) +# TODO: make Commands able to handle this. maybe configure with a list defining order? +# for now we can just access directly. +# @commands.register(version=("c8121b9")) @click.command() # https://github.com/madMAx43v3r/chia-plotter/blob/master/LICENSE # https://github.com/madMAx43v3r/chia-plotter/blob/master/src/chia_plot.cpp#L180 From 31efe1818408d8f2f39f4545c7ad3820e01a2928 Mon Sep 17 00:00:00 2001 From: Kyle Altendorf Date: Mon, 21 Jun 2021 23:05:05 -0400 Subject: [PATCH 10/17] separate chia and madmax plotter configuration sections --- src/plotman/configuration.py | 51 +++++++++++++++++++++++++++--- src/plotman/manager.py | 28 ++++++++++------ src/plotman/resources/plotman.yaml | 35 ++++++++++---------- 3 files changed, 84 insertions(+), 30 deletions(-) diff --git a/src/plotman/configuration.py b/src/plotman/configuration.py index 1c393057..29334547 100644 --- a/src/plotman/configuration.py +++ b/src/plotman/configuration.py @@ -12,6 +12,8 @@ # TODO: should be a desert.ib() but mypy doesn't understand it then, see below import desert._make import marshmallow +import marshmallow.fields +import marshmallow.validate import pendulum import yaml @@ -67,6 +69,27 @@ def get_validated_configs(config_text: str, config_path: str, preset_target_defi f"Config file at: '{config_path}' is malformed" ) from e + if loaded.plotting.type == "chia" and loaded.plotting.chia is None: + raise ConfigurationException( + "chia selected as plotter but plotting: chia: was not specified in the config", + ) + elif loaded.plotting.type == "madmax": + if loaded.plotting.madmax is None: + raise ConfigurationException( + "madmax selected as plotter but plotting: madmax: was not specified in the config", + ) + + if loaded.plotting.farmer_pk is None: + raise ConfigurationException( + "madmax selected as plotter but no plotting: farmer_pk: was specified", + ) + + if loaded.plotting.pool_pk is None: + raise ConfigurationException( + "madmax selected as plotter but no plotting: pool_pk: was specified", + ) + + if loaded.archiving is not None: preset_target_objects = yaml.safe_load(preset_target_definitions_text) preset_target_schema = desert.schema(PresetTargetDefinitions) @@ -273,18 +296,38 @@ class Scheduling: tmpdir_stagger_phase_minor: int tmpdir_stagger_phase_limit: int = 1 # If not explicit, "tmpdir_stagger_phase_limit" will default to 1 + @attr.frozen -class Plotting: - type: str = "chia" +class ChiaPlotterOptions: n_threads: int = 2 n_buckets: int = 128 k: Optional[int] = 32 e: Optional[bool] = False job_buffer: Optional[int] = 3389 + x: bool = False + pool_contract_address: Optional[str] = None + +@attr.frozen +class MadmaxPlotterOptions: + n_threads: int = 4 + n_buckets: int = 256 + +@attr.frozen +class Plotting: farmer_pk: Optional[str] = None pool_pk: Optional[str] = None - pool_contract_address: Optional[str] = None - x: bool = False + type: str = attr.ib( + default="chia", + metadata={ + desert._make._DESERT_SENTINEL: { + 'marshmallow_field': marshmallow.fields.String( + validate=marshmallow.validate.OneOf(choices=["chia", "madmax"]), + ), + }, + }, + ) + chia: Optional[ChiaPlotterOptions] = None + madmax: Optional[MadmaxPlotterOptions] = None @attr.frozen class UserInterface: diff --git a/src/plotman/manager.py b/src/plotman/manager.py index 80d31db7..4ca6068e 100644 --- a/src/plotman/manager.py +++ b/src/plotman/manager.py @@ -123,26 +123,34 @@ def key(key: str) -> job.Phase: plot_args: typing.List[str] if plotting_cfg.type == "madmax": + if plotting_cfg.madmax is None: + raise Exception( + "madmax plotter selected but not configured, report this as a plotman bug", + ) plot_args = ['chia_plot', '-n', str(1), - '-r', str(plotting_cfg.n_threads), - '-u', str(plotting_cfg.n_buckets), + '-r', str(plotting_cfg.madmax.n_threads), + '-u', str(plotting_cfg.madmax.n_buckets), '-t', tmpdir if tmpdir.endswith('/') else (tmpdir + '/'), '-d', dstdir if dstdir.endswith('/') else (dstdir + '/') ] else: + if plotting_cfg.chia is None: + raise Exception( + "chia plotter selected but not configured, report this as a plotman bug", + ) plot_args = ['chia', 'plots', 'create', - '-k', str(plotting_cfg.k), - '-r', str(plotting_cfg.n_threads), - '-u', str(plotting_cfg.n_buckets), - '-b', str(plotting_cfg.job_buffer), + '-k', str(plotting_cfg.chia.k), + '-r', str(plotting_cfg.chia.n_threads), + '-u', str(plotting_cfg.chia.n_buckets), + '-b', str(plotting_cfg.chia.job_buffer), '-t', tmpdir, '-d', dstdir ] - if plotting_cfg.e: + if plotting_cfg.chia.e: plot_args.append('-e') - if plotting_cfg.pool_contract_address is not None: + if plotting_cfg.chia.pool_contract_address is not None: plot_args.append('-c') - plot_args.append(plotting_cfg.pool_contract_address) - if plotting_cfg.x: + plot_args.append(plotting_cfg.chia.pool_contract_address) + if plotting_cfg.chia.x: plot_args.append('-x') if plotting_cfg.farmer_pk is not None: plot_args.append('-f') diff --git a/src/plotman/resources/plotman.yaml b/src/plotman/resources/plotman.yaml index 573254a0..b1b35981 100644 --- a/src/plotman/resources/plotman.yaml +++ b/src/plotman/resources/plotman.yaml @@ -132,23 +132,26 @@ scheduling: polling_time_s: 20 -# Plotting parameters. These are pass-through parameters to chia plots create. -# See documentation at -# https://github.com/Chia-Network/chia-blockchain/wiki/CLI-Commands-Reference#create +# Plotting parameters. These are pass-through parameters to the selected plotter. plotting: - # Your public keys: farmer and pool - Required for madmax, optional for chia with mnemonic.txt - # farmer_pk: ... - # pool_pk: ... - # If you enable Chia, plot in *parallel* with higher tmpdir_max_jobs and global_max_jobs - type: chia # The stock plotter: https://github.com/Chia-Network/chia-blockchain - k: 32 # k-size of plot, leave at 32 most of the time - e: False # Use -e plotting option - n_threads: 2 # Threads per job - n_buckets: 128 # Number of buckets to split data into - job_buffer: 3389 # Per job memory + type: chia + chia: + # The stock plotter: https://github.com/Chia-Network/chia-blockchain + # See documentation at + # https://github.com/Chia-Network/chia-blockchain/wiki/CLI-Commands-Reference#create + # farmer_pk: ... + # pool_pk: ... + k: 32 # k-size of plot, leave at 32 most of the time + e: False # Use -e plotting option + n_threads: 2 # Threads per job + n_buckets: 128 # Number of buckets to split data into + job_buffer: 3389 # Per job memory # If you enable MadMax, plot in *sequence* with very low tmpdir_max_jobs and global_max_jobs - #type: madmax # Madmax plotter: https://github.com/madMAx43v3r/chia-plotter - #n_threads: 4 # Default is 4, crank up if you have many cores - #n_buckets: 256 # Default is 256 + madmax: + # Madmax plotter: https://github.com/madMAx43v3r/chia-plotter + farmer_pk: abc123 + pool_pk: abc123 + n_threads: 4 # Default is 4, crank up if you have many cores + n_buckets: 256 # Default is 256 From 9d5c976912824fad6c800e7068631ad7e20f0a7d Mon Sep 17 00:00:00 2001 From: Kyle Altendorf Date: Mon, 21 Jun 2021 23:08:38 -0400 Subject: [PATCH 11/17] update configuration example --- src/plotman/configuration.py | 5 ++--- src/plotman/resources/plotman.yaml | 14 +++++++------- 2 files changed, 9 insertions(+), 10 deletions(-) diff --git a/src/plotman/configuration.py b/src/plotman/configuration.py index 29334547..6d59b936 100644 --- a/src/plotman/configuration.py +++ b/src/plotman/configuration.py @@ -81,12 +81,12 @@ def get_validated_configs(config_text: str, config_path: str, preset_target_defi if loaded.plotting.farmer_pk is None: raise ConfigurationException( - "madmax selected as plotter but no plotting: farmer_pk: was specified", + "madmax selected as plotter but no plotting: farmer_pk: was specified in the config", ) if loaded.plotting.pool_pk is None: raise ConfigurationException( - "madmax selected as plotter but no plotting: pool_pk: was specified", + "madmax selected as plotter but no plotting: pool_pk: was specified in the config", ) @@ -296,7 +296,6 @@ class Scheduling: tmpdir_stagger_phase_minor: int tmpdir_stagger_phase_limit: int = 1 # If not explicit, "tmpdir_stagger_phase_limit" will default to 1 - @attr.frozen class ChiaPlotterOptions: n_threads: int = 2 diff --git a/src/plotman/resources/plotman.yaml b/src/plotman/resources/plotman.yaml index b1b35981..49042676 100644 --- a/src/plotman/resources/plotman.yaml +++ b/src/plotman/resources/plotman.yaml @@ -132,16 +132,18 @@ scheduling: polling_time_s: 20 -# Plotting parameters. These are pass-through parameters to the selected plotter. +# Plotting parameters. These are pass-through parameters to chia plots create. +# See documentation at +# https://github.com/Chia-Network/chia-blockchain/wiki/CLI-Commands-Reference#create plotting: + # Your public keys: farmer and pool - Required for madmax, optional for chia with mnemonic.txt + # farmer_pk: ... + # pool_pk: ... + # If you enable Chia, plot in *parallel* with higher tmpdir_max_jobs and global_max_jobs type: chia chia: # The stock plotter: https://github.com/Chia-Network/chia-blockchain - # See documentation at - # https://github.com/Chia-Network/chia-blockchain/wiki/CLI-Commands-Reference#create - # farmer_pk: ... - # pool_pk: ... k: 32 # k-size of plot, leave at 32 most of the time e: False # Use -e plotting option n_threads: 2 # Threads per job @@ -151,7 +153,5 @@ plotting: # If you enable MadMax, plot in *sequence* with very low tmpdir_max_jobs and global_max_jobs madmax: # Madmax plotter: https://github.com/madMAx43v3r/chia-plotter - farmer_pk: abc123 - pool_pk: abc123 n_threads: 4 # Default is 4, crank up if you have many cores n_buckets: 256 # Default is 256 From 0dfcaf4d19a2ab03274018479cf1abc06e67effc Mon Sep 17 00:00:00 2001 From: Guy Davis Date: Tue, 22 Jun 2021 06:14:29 -0600 Subject: [PATCH 12/17] Handle two sets of args. Tmp2 trailing slash fix. --- src/plotman/job.py | 27 ++++++++++++++++++--------- src/plotman/manager.py | 10 +++++++--- 2 files changed, 25 insertions(+), 12 deletions(-) diff --git a/src/plotman/job.py b/src/plotman/job.py index 01f40ffe..4c60998c 100644 --- a/src/plotman/job.py +++ b/src/plotman/job.py @@ -267,15 +267,24 @@ def __init__( # 'nobitfield': False, # 'exclude_final_dir': False, # } - - self.k = self.args['size'] # type: ignore[assignment] - self.r = self.args['num_threads'] # type: ignore[assignment] - self.u = self.args['buckets'] # type: ignore[assignment] - self.b = self.args['buffer'] # type: ignore[assignment] - self.n = self.args['num'] # type: ignore[assignment] - self.tmpdir = self.args['tmp_dir'] # type: ignore[assignment] - self.tmp2dir = self.args['tmp2_dir'] # type: ignore[assignment] - self.dstdir = self.args['final_dir'] # type: ignore[assignment] + if proc.name().startswith("chia_plot"): # MADMAX + self.k = 32 # type: ignore[assignment] + self.r = self.args['threads'] # type: ignore[assignment] + self.u = self.args['buckets'] # type: ignore[assignment] + self.b = 0 # type: ignore[assignment] + self.n = self.args['count'] # type: ignore[assignment] + self.tmpdir = self.args['tmpdir'] # type: ignore[assignment] + self.tmp2dir = self.args['tmpdir2'] # type: ignore[assignment] + self.dstdir = self.args['finaldir'] # type: ignore[assignment] + else: # CHIA + self.k = self.args['size'] # type: ignore[assignment] + self.r = self.args['num_threads'] # type: ignore[assignment] + self.u = self.args['buckets'] # type: ignore[assignment] + self.b = self.args['buffer'] # type: ignore[assignment] + self.n = self.args['num'] # type: ignore[assignment] + self.tmpdir = self.args['tmp_dir'] # type: ignore[assignment] + self.tmp2dir = self.args['tmp2_dir'] # type: ignore[assignment] + self.dstdir = self.args['final_dir'] # type: ignore[assignment] plot_cwd: str = self.proc.cwd() self.tmpdir = os.path.join(plot_cwd, self.tmpdir) diff --git a/src/plotman/manager.py b/src/plotman/manager.py index 4ca6068e..40d07b93 100644 --- a/src/plotman/manager.py +++ b/src/plotman/manager.py @@ -133,6 +133,9 @@ def key(key: str) -> job.Phase: '-u', str(plotting_cfg.madmax.n_buckets), '-t', tmpdir if tmpdir.endswith('/') else (tmpdir + '/'), '-d', dstdir if dstdir.endswith('/') else (dstdir + '/') ] + if dir_cfg.tmp2 is not None: + plot_args.append('-2') + plot_args.append(dir_cfg.tmp2 if dir_cfg.tmp2.endswith('/') else (dir_cfg.tmp2 + '/')) else: if plotting_cfg.chia is None: raise Exception( @@ -152,15 +155,16 @@ def key(key: str) -> job.Phase: plot_args.append(plotting_cfg.chia.pool_contract_address) if plotting_cfg.chia.x: plot_args.append('-x') + if dir_cfg.tmp2 is not None: + plot_args.append('-2') + plot_args.append(dir_cfg.tmp2) if plotting_cfg.farmer_pk is not None: plot_args.append('-f') plot_args.append(plotting_cfg.farmer_pk) if plotting_cfg.pool_pk is not None: plot_args.append('-p') plot_args.append(plotting_cfg.pool_pk) - if dir_cfg.tmp2 is not None: - plot_args.append('-2') - plot_args.append(dir_cfg.tmp2) + logmsg = ('Starting plot job: %s ; logging to %s' % (' '.join(plot_args), log_file_path)) From e62f67fe90ab0a2ab643ec229cb828c7cf728579 Mon Sep 17 00:00:00 2001 From: Kyle Altendorf Date: Tue, 22 Jun 2021 18:22:32 -0700 Subject: [PATCH 13/17] Apply suggestions from code review --- src/plotman/job.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/plotman/job.py b/src/plotman/job.py index 4c60998c..3009df6b 100644 --- a/src/plotman/job.py +++ b/src/plotman/job.py @@ -268,10 +268,10 @@ def __init__( # 'exclude_final_dir': False, # } if proc.name().startswith("chia_plot"): # MADMAX - self.k = 32 # type: ignore[assignment] + self.k = 32 self.r = self.args['threads'] # type: ignore[assignment] self.u = self.args['buckets'] # type: ignore[assignment] - self.b = 0 # type: ignore[assignment] + self.b = 0 self.n = self.args['count'] # type: ignore[assignment] self.tmpdir = self.args['tmpdir'] # type: ignore[assignment] self.tmp2dir = self.args['tmpdir2'] # type: ignore[assignment] From 758073c7e6bb216dfbb6f884dfc81969026baeb8 Mon Sep 17 00:00:00 2001 From: Kyle Altendorf Date: Tue, 22 Jun 2021 21:37:41 -0400 Subject: [PATCH 14/17] Update CHANGELOG.md --- CHANGELOG.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index cd4af43b..8f15f809 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -17,7 +17,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ([#430](https://github.com/ericaltendorf/plotman/pull/430)) - `plotman logs` command to print and tail plot logs by their plot ID. ([#509](https://github.com/ericaltendorf/plotman/pull/509)) -- Initial support for MadMax plotter. +- Support the [MadMax plotter](https://github.com/madMAx43v3r/chia-plotter). + See the [configuration wiki page](https://github.com/ericaltendorf/plotman/wiki/Configuration#2-v05) for help setting it up. ([#797](https://github.com/ericaltendorf/plotman/pull/797)) ## [0.4.1] - 2021-06-11 From 53587e314010f9ff13b317135d54c2ca01ca7a8c Mon Sep 17 00:00:00 2001 From: Kyle Altendorf Date: Tue, 22 Jun 2021 22:11:08 -0400 Subject: [PATCH 15/17] Update plotman.yaml --- src/plotman/resources/plotman.yaml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/plotman/resources/plotman.yaml b/src/plotman/resources/plotman.yaml index 49042676..b33c5fdd 100644 --- a/src/plotman/resources/plotman.yaml +++ b/src/plotman/resources/plotman.yaml @@ -136,7 +136,7 @@ scheduling: # See documentation at # https://github.com/Chia-Network/chia-blockchain/wiki/CLI-Commands-Reference#create plotting: - # Your public keys: farmer and pool - Required for madmax, optional for chia with mnemonic.txt + # Your public keys: farmer and pool - Required for madMAx, optional for chia with mnemonic.txt # farmer_pk: ... # pool_pk: ... @@ -150,8 +150,8 @@ plotting: n_buckets: 128 # Number of buckets to split data into job_buffer: 3389 # Per job memory - # If you enable MadMax, plot in *sequence* with very low tmpdir_max_jobs and global_max_jobs - madmax: - # Madmax plotter: https://github.com/madMAx43v3r/chia-plotter + # If you enable madMAx, plot in *sequence* with very low tmpdir_max_jobs and global_max_jobs + madMAx: + # madMAx plotter: https://github.com/madMAx43v3r/chia-plotter n_threads: 4 # Default is 4, crank up if you have many cores n_buckets: 256 # Default is 256 From f539c73f530392050c9518ecc8a82d38fd10bed1 Mon Sep 17 00:00:00 2001 From: Kyle Altendorf Date: Tue, 22 Jun 2021 22:11:57 -0400 Subject: [PATCH 16/17] Update CHANGELOG.md --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8f15f809..4e717de8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -17,7 +17,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ([#430](https://github.com/ericaltendorf/plotman/pull/430)) - `plotman logs` command to print and tail plot logs by their plot ID. ([#509](https://github.com/ericaltendorf/plotman/pull/509)) -- Support the [MadMax plotter](https://github.com/madMAx43v3r/chia-plotter). +- Support the [madMAx plotter](https://github.com/madMAx43v3r/chia-plotter). See the [configuration wiki page](https://github.com/ericaltendorf/plotman/wiki/Configuration#2-v05) for help setting it up. ([#797](https://github.com/ericaltendorf/plotman/pull/797)) From b788def591159f5345ffe8da6dc99fc7abbd49a4 Mon Sep 17 00:00:00 2001 From: Kyle Altendorf Date: Tue, 22 Jun 2021 22:19:30 -0400 Subject: [PATCH 17/17] Update plotman.yaml --- src/plotman/resources/plotman.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/plotman/resources/plotman.yaml b/src/plotman/resources/plotman.yaml index b33c5fdd..1ccfd9c4 100644 --- a/src/plotman/resources/plotman.yaml +++ b/src/plotman/resources/plotman.yaml @@ -151,7 +151,7 @@ plotting: job_buffer: 3389 # Per job memory # If you enable madMAx, plot in *sequence* with very low tmpdir_max_jobs and global_max_jobs - madMAx: + madmax: # madMAx plotter: https://github.com/madMAx43v3r/chia-plotter n_threads: 4 # Default is 4, crank up if you have many cores n_buckets: 256 # Default is 256