diff --git a/CHANGELOG.md b/CHANGELOG.md index ab0c93c6..4e717de8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -17,6 +17,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ([#430](https://github.com/ericaltendorf/plotman/pull/430)) - `plotman logs` command to print and tail plot logs by their plot ID. ([#509](https://github.com/ericaltendorf/plotman/pull/509)) +- Support the [madMAx plotter](https://github.com/madMAx43v3r/chia-plotter). + See the [configuration wiki page](https://github.com/ericaltendorf/plotman/wiki/Configuration#2-v05) for help setting it up. + ([#797](https://github.com/ericaltendorf/plotman/pull/797)) ## [0.4.1] - 2021-06-11 ### Fixed diff --git a/src/plotman/analyzer.py b/src/plotman/analyzer.py index e210b26a..b7229fe2 100644 --- a/src/plotman/analyzer.py +++ b/src/plotman/analyzer.py @@ -58,12 +58,18 @@ def analyze(logfilenames: typing.List[str], clipterminals: bool, bytmp: bool, by else: sl += '-bitfield' - # Phase timing. Sample log line: + # CHIA: Phase timing. Sample log line: # Time for phase 1 = 22796.7 seconds. CPU (98%) Tue Sep 29 17:57:19 2020 for phase in ['1', '2', '3', '4']: m = re.search(r'^Time for phase ' + phase + ' = (\d+.\d+) seconds..*', line) if m: phase_time[phase] = float(m.group(1)) + + # MADMAX: Phase timing. Sample log line: "Phase 2 took 2193.37 sec" + for phase in ['1', '2', '3', '4']: + m = re.search(r'^Phase ' + phase + ' took (\d+.\d+) sec.*', line) + if m: + phase_time[phase] = float(m.group(1)) # Uniform sort. Sample log line: # Bucket 267 uniform sort. Ram: 0.920GiB, u_sort min: 0.688GiB, qs min: 0.172GiB. @@ -82,7 +88,7 @@ def analyze(logfilenames: typing.List[str], clipterminals: bool, bytmp: bool, by else: print ('Warning: unrecognized sort ' + sorter) - # Job completion. Record total time in sliced data store. + # CHIA: Job completion. Record total time in sliced data store. # Sample log line: # Total time = 49487.1 seconds. CPU (97.26%) Wed Sep 30 01:22:10 2020 m = re.search(r'^Total time = (\d+.\d+) seconds.*', line) @@ -94,13 +100,22 @@ def analyze(logfilenames: typing.List[str], clipterminals: bool, bytmp: bool, by for phase in ['1', '2', '3', '4']: data.setdefault(sl, {}).setdefault('phase ' + phase, []).append(phase_time[phase]) data.setdefault(sl, {}).setdefault('%usort', []).append(100 * n_uniform // n_sorts) + + # MADMAX: Job completion. Record total time in sliced data store. + # Sample log line: "Total plot creation time was 2530.76 sec" + m = re.search(r'^Total plot creation time was (\d+.\d+) sec.*', line) + if m: + data.setdefault(sl, {}).setdefault('total time', []).append(float(m.group(1))) + for phase in ['1', '2', '3', '4']: + data.setdefault(sl, {}).setdefault('phase ' + phase, []).append(phase_time[phase]) + data.setdefault(sl, {}).setdefault('%usort', []).append(0) # Not available for MADMAX # Prepare report tab = tt.Texttable() all_measures = ['%usort', 'phase 1', 'phase 2', 'phase 3', 'phase 4', 'total time'] headings = ['Slice', 'n'] + all_measures tab.header(headings) - + for sl in data.keys(): row = [sl] diff --git a/src/plotman/configuration.py b/src/plotman/configuration.py index fd768b27..6d59b936 100644 --- a/src/plotman/configuration.py +++ b/src/plotman/configuration.py @@ -12,6 +12,8 @@ # TODO: should be a desert.ib() but mypy doesn't understand it then, see below import desert._make import marshmallow +import marshmallow.fields +import marshmallow.validate import pendulum import yaml @@ -67,6 +69,27 @@ def get_validated_configs(config_text: str, config_path: str, preset_target_defi f"Config file at: '{config_path}' is malformed" ) from e + if loaded.plotting.type == "chia" and loaded.plotting.chia is None: + raise ConfigurationException( + "chia selected as plotter but plotting: chia: was not specified in the config", + ) + elif loaded.plotting.type == "madmax": + if loaded.plotting.madmax is None: + raise ConfigurationException( + "madmax selected as plotter but plotting: madmax: was not specified in the config", + ) + + if loaded.plotting.farmer_pk is None: + raise ConfigurationException( + "madmax selected as plotter but no plotting: farmer_pk: was specified in the config", + ) + + if loaded.plotting.pool_pk is None: + raise ConfigurationException( + "madmax selected as plotter but no plotting: pool_pk: was specified in the config", + ) + + if loaded.archiving is not None: preset_target_objects = yaml.safe_load(preset_target_definitions_text) preset_target_schema = desert.schema(PresetTargetDefinitions) @@ -273,17 +296,37 @@ class Scheduling: tmpdir_stagger_phase_minor: int tmpdir_stagger_phase_limit: int = 1 # If not explicit, "tmpdir_stagger_phase_limit" will default to 1 +@attr.frozen +class ChiaPlotterOptions: + n_threads: int = 2 + n_buckets: int = 128 + k: Optional[int] = 32 + e: Optional[bool] = False + job_buffer: Optional[int] = 3389 + x: bool = False + pool_contract_address: Optional[str] = None + +@attr.frozen +class MadmaxPlotterOptions: + n_threads: int = 4 + n_buckets: int = 256 + @attr.frozen class Plotting: - k: int - e: bool - n_threads: int - n_buckets: int - job_buffer: int farmer_pk: Optional[str] = None pool_pk: Optional[str] = None - pool_contract_address: Optional[str] = None - x: bool = False + type: str = attr.ib( + default="chia", + metadata={ + desert._make._DESERT_SENTINEL: { + 'marshmallow_field': marshmallow.fields.String( + validate=marshmallow.validate.OneOf(choices=["chia", "madmax"]), + ), + }, + }, + ) + chia: Optional[ChiaPlotterOptions] = None + madmax: Optional[MadmaxPlotterOptions] = None @attr.frozen class UserInterface: diff --git a/src/plotman/job.py b/src/plotman/job.py index 47c649f4..3009df6b 100644 --- a/src/plotman/job.py +++ b/src/plotman/job.py @@ -18,7 +18,7 @@ import pendulum import psutil -from plotman import chia +from plotman import chia, madmax def job_phases_for_tmpdir(d: str, all_jobs: typing.List["Job"]) -> typing.List["Phase"]: @@ -30,14 +30,17 @@ def job_phases_for_dstdir(d: str, all_jobs: typing.List["Job"]) -> typing.List[" return sorted([j.progress() for j in all_jobs if j.dstdir == d]) def is_plotting_cmdline(cmdline: typing.List[str]) -> bool: - if cmdline and 'python' in cmdline[0].lower(): + if cmdline and 'python' in cmdline[0].lower(): # Stock Chia plotter cmdline = cmdline[1:] - return ( - len(cmdline) >= 3 - and 'chia' in cmdline[0] - and 'plots' == cmdline[1] - and 'create' == cmdline[2] - ) + return ( + len(cmdline) >= 3 + and 'chia' in cmdline[0] + and 'plots' == cmdline[1] + and 'create' == cmdline[2] + ) + elif cmdline and 'chia_plot' == cmdline[0].lower(): # Madmax plotter + return True + return False def parse_chia_plot_time(s: str) -> pendulum.DateTime: # This will grow to try ISO8601 as well for when Chia logs that way @@ -50,14 +53,21 @@ def parse_chia_plots_create_command_line( ) -> "ParsedChiaPlotsCreateCommand": command_line = list(command_line) # Parse command line args - if 'python' in command_line[0].lower(): + if 'python' in command_line[0].lower(): # Stock Chia plotter command_line = command_line[1:] - assert len(command_line) >= 3 - assert 'chia' in command_line[0] - assert 'plots' == command_line[1] - assert 'create' == command_line[2] - - all_command_arguments = command_line[3:] + assert len(command_line) >= 3 + assert 'chia' in command_line[0] + assert 'plots' == command_line[1] + assert 'create' == command_line[2] + all_command_arguments = command_line[3:] + # TODO: We could at some point do chia version detection and pick the + # associated command. For now we'll just use the latest one we have + # copied. + command = chia.commands.latest_command() + elif 'chia_plot' in command_line[0].lower(): # Madmax plotter + command_line = command_line[1:] + all_command_arguments = command_line[2:] + command = madmax._cli_c8121b9 # nice idea, but this doesn't include -h # help_option_names = command.get_help_option_names(ctx=context) @@ -69,10 +79,6 @@ def parse_chia_plots_create_command_line( if argument not in help_option_names ] - # TODO: We could at some point do chia version detection and pick the - # associated command. For now we'll just use the latest one we have - # copied. - command = chia.commands.latest_command() try: context = command.make_context(info_name='', args=list(command_arguments)) except click.ClickException as e: @@ -146,6 +152,7 @@ class Job: jobfile: str = '' job_id: int = 0 plot_id: str = '--------' + plotter: str = '' proc: psutil.Process k: int r: int @@ -260,15 +267,24 @@ def __init__( # 'nobitfield': False, # 'exclude_final_dir': False, # } - - self.k = self.args['size'] # type: ignore[assignment] - self.r = self.args['num_threads'] # type: ignore[assignment] - self.u = self.args['buckets'] # type: ignore[assignment] - self.b = self.args['buffer'] # type: ignore[assignment] - self.n = self.args['num'] # type: ignore[assignment] - self.tmpdir = self.args['tmp_dir'] # type: ignore[assignment] - self.tmp2dir = self.args['tmp2_dir'] # type: ignore[assignment] - self.dstdir = self.args['final_dir'] # type: ignore[assignment] + if proc.name().startswith("chia_plot"): # MADMAX + self.k = 32 + self.r = self.args['threads'] # type: ignore[assignment] + self.u = self.args['buckets'] # type: ignore[assignment] + self.b = 0 + self.n = self.args['count'] # type: ignore[assignment] + self.tmpdir = self.args['tmpdir'] # type: ignore[assignment] + self.tmp2dir = self.args['tmpdir2'] # type: ignore[assignment] + self.dstdir = self.args['finaldir'] # type: ignore[assignment] + else: # CHIA + self.k = self.args['size'] # type: ignore[assignment] + self.r = self.args['num_threads'] # type: ignore[assignment] + self.u = self.args['buckets'] # type: ignore[assignment] + self.b = self.args['buffer'] # type: ignore[assignment] + self.n = self.args['num'] # type: ignore[assignment] + self.tmpdir = self.args['tmp_dir'] # type: ignore[assignment] + self.tmp2dir = self.args['tmp2_dir'] # type: ignore[assignment] + self.dstdir = self.args['final_dir'] # type: ignore[assignment] plot_cwd: str = self.proc.cwd() self.tmpdir = os.path.join(plot_cwd, self.tmpdir) @@ -311,15 +327,24 @@ def init_from_logfile(self) -> None: with contextlib.suppress(UnicodeDecodeError): for line in f: m = re.match('^ID: ([0-9a-f]*)', line) - if m: + if m: # CHIA self.plot_id = m.group(1) + self.plotter = 'chia' found_id = True + else: + m = re.match("^Plot Name: plot-k(\d+)-(\d+)-(\d+)-(\d+)-(\d+)-(\d+)-(\w+)$", line) + if m: # MADMAX + self.plot_id = m.group(7) + self.plotter = 'madmax' + found_id = True m = re.match(r'^Starting phase 1/4:.*\.\.\. (.*)', line) - if m: + if m: # CHIA # Mon Nov 2 08:39:53 2020 self.start_time = parse_chia_plot_time(m.group(1)) found_log = True break # Stop reading lines in file + else: # MADMAX + self.start_time = pendulum.from_timestamp(os.path.getctime(self.logfile)) if found_id and found_log: break # Stop trying @@ -352,26 +377,50 @@ def set_phase_from_logfile(self) -> None: with open(self.logfile, 'r') as f: with contextlib.suppress(UnicodeDecodeError): for line in f: - # "Starting phase 1/4: Forward Propagation into tmp files... Sat Oct 31 11:27:04 2020" + # CHIA: "Starting phase 1/4: Forward Propagation into tmp files... Sat Oct 31 11:27:04 2020" m = re.match(r'^Starting phase (\d).*', line) if m: phase = int(m.group(1)) phase_subphases[phase] = 0 + + # MADMAX: "[P1]" or "[P2]" or "[P3]" or "[P4]" + m = re.match(r'^\[P(\d)\].*', line) + if m: + phase = int(m.group(1)) + phase_subphases[phase] = 0 - # Phase 1: "Computing table 2" + # CHIA: Phase 1: "Computing table 2" m = re.match(r'^Computing table (\d).*', line) if m: phase_subphases[1] = max(phase_subphases[1], int(m.group(1))) + + # MADMAX: Phase 1: "[P1] Table 2" + m = re.match(r'^\[P1\] Table (\d).*', line) + if m: + phase_subphases[1] = max(phase_subphases[1], int(m.group(1))) - # Phase 2: "Backpropagating on table 2" + # CHIA: Phase 2: "Backpropagating on table 2" m = re.match(r'^Backpropagating on table (\d).*', line) if m: phase_subphases[2] = max(phase_subphases[2], 7 - int(m.group(1))) - # Phase 3: "Compressing tables 4 and 5" + # MADMAX: Phase 2: "[P2] Table 2" + m = re.match(r'^\[P2\] Table (\d).*', line) + if m: + phase_subphases[2] = max(phase_subphases[2], 7 - int(m.group(1))) + + # CHIA: Phase 3: "Compressing tables 4 and 5" m = re.match(r'^Compressing tables (\d) and (\d).*', line) if m: phase_subphases[3] = max(phase_subphases[3], int(m.group(1))) + + # MADMAX: Phase 3: "[P3-1] Table 4" + m = re.match(r'^\[P3\-\d\] Table (\d).*', line) + if m: + if 3 in phase_subphases: + phase_subphases[3] = max(phase_subphases[3], int(m.group(1))) + else: + phase_subphases[3] = int(m.group(1)) # TODO also collect timing info: diff --git a/src/plotman/madmax.py b/src/plotman/madmax.py new file mode 100644 index 00000000..c6ba0626 --- /dev/null +++ b/src/plotman/madmax.py @@ -0,0 +1,73 @@ +# mypy: allow_untyped_decorators +# +# Madmax is written in C++. Below is a mapping of its CLI options to Python. +# See: https://github.com/madMAx43v3r/chia-plotter/tree/master/src +# Note: versions are git commit refs, not semantic versioning +# + +import functools +import typing + +import click +from pathlib import Path +import typing_extensions + + +class CommandProtocol(typing_extensions.Protocol): + def make_context(self, info_name: str, args: typing.List[str]) -> click.Context: + ... + + def __call__(self) -> None: + ... + + +class Commands: + def __init__(self) -> None: + self.by_version: typing.Dict[typing.Sequence[int], CommandProtocol] = {} + + def register(self, version: typing.Sequence[int]) -> typing.Callable[[CommandProtocol], None]: + if version in self.by_version: + raise Exception(f'Version already registered: {version!r}') + + return functools.partial(self._decorator, version=version) + + def _decorator(self, command: CommandProtocol, *, version: typing.Sequence[int]) -> None: + self.by_version[version] = command + + def __getitem__(self, item: typing.Sequence[int]) -> typing.Callable[[], None]: + return self.by_version[item] + + def latest_command(self) -> CommandProtocol: + return max(self.by_version.items())[1] + + +commands = Commands() +# Madmax Git on 2021-06-19 -> https://github.com/madMAx43v3r/chia-plotter/commit/c8121b987186c42c895b49818e6c13acecc51332 +# TODO: make Commands able to handle this. maybe configure with a list defining order? +# for now we can just access directly. +# @commands.register(version=("c8121b9")) +@click.command() +# https://github.com/madMAx43v3r/chia-plotter/blob/master/LICENSE +# https://github.com/madMAx43v3r/chia-plotter/blob/master/src/chia_plot.cpp#L180 +@click.option("-n", "--count", help="Number of plots to create (default = 1, -1 = infinite)", + type=int, default=1, show_default=True) +@click.option("-r", "--threads", help="Number of threads (default = 4)", + type=int, default=4, show_default=True) +@click.option("-u", "--buckets", help="Number of buckets (default = 256)", + type=int, default=256, show_default=True) +@click.option("-v", "--buckets3", help="Number of buckets for phase 3+4 (default = buckets)", + type=int, default=256) +@click.option("-t", "--tmpdir", help="Temporary directory, needs ~220 GiB (default = $PWD)", + type=click.Path(), default=Path("."), show_default=True) +@click.option("-2", "--tmpdir2", help="Temporary directory 2, needs ~110 GiB [RAM] (default = )", + type=click.Path(), default=None) +@click.option("-d", "--finaldir", help="Final directory (default = )", + type=click.Path(), default=Path("."), show_default=True) +@click.option("-p", "--poolkey", help="Pool Public Key (48 bytes)", + type=str, default=None) +@click.option("-f", "--farmerkey", help="Farmer Public Key (48 bytes)", + type=str, default=None) +@click.option("-G", "--tmptoggle", help="Alternate tmpdir/tmpdir2", + type=str, default=None) +def _cli_c8121b9() -> None: + pass diff --git a/src/plotman/manager.py b/src/plotman/manager.py index 87be88f7..40d07b93 100644 --- a/src/plotman/manager.py +++ b/src/plotman/manager.py @@ -121,29 +121,51 @@ def key(key: str) -> job.Phase: log_file_path = log_cfg.create_plot_log_path(time=pendulum.now()) - plot_args: typing.List[str] = ['chia', 'plots', 'create', - '-k', str(plotting_cfg.k), - '-r', str(plotting_cfg.n_threads), - '-u', str(plotting_cfg.n_buckets), - '-b', str(plotting_cfg.job_buffer), + plot_args: typing.List[str] + if plotting_cfg.type == "madmax": + if plotting_cfg.madmax is None: + raise Exception( + "madmax plotter selected but not configured, report this as a plotman bug", + ) + plot_args = ['chia_plot', + '-n', str(1), + '-r', str(plotting_cfg.madmax.n_threads), + '-u', str(plotting_cfg.madmax.n_buckets), + '-t', tmpdir if tmpdir.endswith('/') else (tmpdir + '/'), + '-d', dstdir if dstdir.endswith('/') else (dstdir + '/') ] + if dir_cfg.tmp2 is not None: + plot_args.append('-2') + plot_args.append(dir_cfg.tmp2 if dir_cfg.tmp2.endswith('/') else (dir_cfg.tmp2 + '/')) + else: + if plotting_cfg.chia is None: + raise Exception( + "chia plotter selected but not configured, report this as a plotman bug", + ) + plot_args = ['chia', 'plots', 'create', + '-k', str(plotting_cfg.chia.k), + '-r', str(plotting_cfg.chia.n_threads), + '-u', str(plotting_cfg.chia.n_buckets), + '-b', str(plotting_cfg.chia.job_buffer), '-t', tmpdir, '-d', dstdir ] - if plotting_cfg.e: - plot_args.append('-e') + if plotting_cfg.chia.e: + plot_args.append('-e') + if plotting_cfg.chia.pool_contract_address is not None: + plot_args.append('-c') + plot_args.append(plotting_cfg.chia.pool_contract_address) + if plotting_cfg.chia.x: + plot_args.append('-x') + if dir_cfg.tmp2 is not None: + plot_args.append('-2') + plot_args.append(dir_cfg.tmp2) if plotting_cfg.farmer_pk is not None: plot_args.append('-f') plot_args.append(plotting_cfg.farmer_pk) if plotting_cfg.pool_pk is not None: plot_args.append('-p') plot_args.append(plotting_cfg.pool_pk) - if plotting_cfg.pool_contract_address is not None: - plot_args.append('-c') - plot_args.append(plotting_cfg.pool_contract_address) - if dir_cfg.tmp2 is not None: - plot_args.append('-2') - plot_args.append(dir_cfg.tmp2) - if plotting_cfg.x: - plot_args.append('-x') + + logmsg = ('Starting plot job: %s ; logging to %s' % (' '.join(plot_args), log_file_path)) diff --git a/src/plotman/reporting.py b/src/plotman/reporting.py index 0a74d84d..ac759810 100644 --- a/src/plotman/reporting.py +++ b/src/plotman/reporting.py @@ -75,7 +75,7 @@ def status_report(jobs: typing.List[job.Job], width: int, height: typing.Optiona n_end_rows = n_rows - n_begin_rows tab = tt.Texttable() - headings = ['plot id', 'k', 'tmp', 'dst', 'wall', 'phase', 'tmp', + headings = ['plot id', 'plotter', 'k', 'tmp', 'dst', 'wall', 'phase', 'tmp', 'pid', 'stat', 'mem', 'user', 'sys', 'io'] if height: headings.insert(0, '#') @@ -97,6 +97,7 @@ def status_report(jobs: typing.List[job.Job], width: int, height: typing.Optiona try: with j.proc.oneshot(): row = [j.plot_id[:8], # Plot ID + str(j.plotter), # chia or madmax str(j.k), # k size abbr_path(j.tmpdir, tmp_prefix), # Temp directory abbr_path(j.dstdir, dst_prefix), # Destination directory diff --git a/src/plotman/resources/plotman.yaml b/src/plotman/resources/plotman.yaml index 899ac07e..1ccfd9c4 100644 --- a/src/plotman/resources/plotman.yaml +++ b/src/plotman/resources/plotman.yaml @@ -136,15 +136,22 @@ scheduling: # See documentation at # https://github.com/Chia-Network/chia-blockchain/wiki/CLI-Commands-Reference#create plotting: - k: 32 - e: False # Use -e plotting option - n_threads: 2 # Threads per job - n_buckets: 128 # Number of buckets to split data into - job_buffer: 3389 # Per job memory - # If specified, pass through to the -f and -p options. See CLI reference. - # farmer_pk: ... - # pool_pk: ... - # If true, Skips adding [final dir] / dst to harvester for farming. - # Especially useful if you have harvesters that are running somewhere else - # and you are just plotting on the machine where plotman is running. - # x: True + # Your public keys: farmer and pool - Required for madMAx, optional for chia with mnemonic.txt + # farmer_pk: ... + # pool_pk: ... + + # If you enable Chia, plot in *parallel* with higher tmpdir_max_jobs and global_max_jobs + type: chia + chia: + # The stock plotter: https://github.com/Chia-Network/chia-blockchain + k: 32 # k-size of plot, leave at 32 most of the time + e: False # Use -e plotting option + n_threads: 2 # Threads per job + n_buckets: 128 # Number of buckets to split data into + job_buffer: 3389 # Per job memory + + # If you enable madMAx, plot in *sequence* with very low tmpdir_max_jobs and global_max_jobs + madmax: + # madMAx plotter: https://github.com/madMAx43v3r/chia-plotter + n_threads: 4 # Default is 4, crank up if you have many cores + n_buckets: 256 # Default is 256