diff --git a/CHANGELOG.md b/CHANGELOG.md index bd0bd942..a2d7fe14 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -18,6 +18,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ([#898](https://github.com/ericaltendorf/plotman/pull/898)) - Output same entries to plotman.log from 'plotman interactive' and ' plotman plot/archive' "daemons". ([#878](https://github.com/ericaltendorf/plotman/pull/878)) +- [BladeBit](https://github.com/harold-b/bladebit) support. + Requires BladeBit v1.1.0 for proper log monitoring. + ([#916](https://github.com/ericaltendorf/plotman/pull/916)) ## [0.5.1] - 2021-07-15 ### Fixed diff --git a/src/plotman/_tests/plotters/test_bladebit.py b/src/plotman/_tests/plotters/test_bladebit.py new file mode 100644 index 00000000..71204b7f --- /dev/null +++ b/src/plotman/_tests/plotters/test_bladebit.py @@ -0,0 +1,77 @@ +import importlib.resources + +import pendulum + +import plotman.job +import plotman.plotters.bladebit +import plotman._tests.resources + + +def test_byte_by_byte_full_load() -> None: + read_bytes = importlib.resources.read_binary( + package=plotman._tests.resources, + resource="bladebit.plot.log", + ) + + parser = plotman.plotters.bladebit.Plotter() + + for byte in (bytes([byte]) for byte in read_bytes): + parser.update(chunk=byte) + + assert parser.info == plotman.plotters.bladebit.SpecificInfo( + phase=plotman.job.Phase(major=5, minor=1), + started_at=pendulum.datetime(2021, 8, 29, 22, 22, 0, tz=None), + plot_id="1fc7b57baae24da78e3bea44d58ab51f162a3ed4d242bab2fbcc24f6577d88b3", + threads=88, + plot_size=32, + dst_dir="/mnt/tmp/01/manual-transfer/", + phase1_duration_raw=313.98, + phase2_duration_raw=44.60, + phase3_duration_raw=203.26, + phase4_duration_raw=1.11, + total_time_raw=582.91, + filename="plot-k32-2021-08-29-22-22-1fc7b57baae24da78e3bea44d58ab51f162a3ed4d242bab2fbcc24f6577d88b3.plot", + plot_name="plot-k32-2021-08-29-22-22-1fc7b57baae24da78e3bea44d58ab51f162a3ed4d242bab2fbcc24f6577d88b3", + ) + + +def test_log_phases() -> None: + # TODO: CAMPid 0978413087474699698142013249869897439887 + read_bytes = importlib.resources.read_binary( + package=plotman._tests.resources, + resource="bladebit.marked", + ) + + parser = plotman.plotters.bladebit.Plotter() + + wrong = [] + + for marked_line in read_bytes.splitlines(keepends=True): + phase_bytes, _, line_bytes = marked_line.partition(b",") + major, _, minor = phase_bytes.decode("utf-8").partition(":") + phase = plotman.job.Phase(major=int(major), minor=int(minor)) + + parser.update(chunk=line_bytes) + + if parser.info.phase != phase: # pragma: nocov + wrong.append([parser.info.phase, phase, line_bytes.decode("utf-8")]) + + assert wrong == [] + + +def test_marked_log_matches() -> None: + # TODO: CAMPid 909831931987460871349879878609830987138931700871340870 + marked_bytes = importlib.resources.read_binary( + package=plotman._tests.resources, + resource="bladebit.marked", + ) + log_bytes = importlib.resources.read_binary( + package=plotman._tests.resources, + resource="bladebit.plot.log", + ) + + for marked_line, log_line in zip( + marked_bytes.splitlines(keepends=True), log_bytes.splitlines(keepends=True) + ): + _, _, marked_just_line = marked_line.partition(b",") + assert marked_just_line == log_line diff --git a/src/plotman/_tests/plotters/test_init.py b/src/plotman/_tests/plotters/test_init.py index 52dc1c46..db18bb9e 100644 --- a/src/plotman/_tests/plotters/test_init.py +++ b/src/plotman/_tests/plotters/test_init.py @@ -10,6 +10,7 @@ import plotman.errors import plotman.job import plotman.plotters +import plotman.plotters.bladebit import plotman.plotters.chianetwork import plotman.plotters.madmax import plotman._tests.resources @@ -92,6 +93,27 @@ class CommandLineExample: cwd: str = "" +default_bladebit_arguments = dict( + sorted( + { + "threads": None, + "count": 1, + "farmer_key": None, + "pool_key": None, + "pool_contract": None, + "warm_start": False, + "plot_id": None, + "memo": None, + "show_memo": False, + "verbose": False, + "no_numa": False, + "no_cpu_affinity": False, + "out_dir": pathlib.PosixPath("."), + }.items() + ) +) + + default_chia_network_arguments = dict( sorted( { @@ -139,6 +161,129 @@ class CommandLineExample: ) +bladebit_command_line_examples: typing.List[CommandLineExample] = [ + CommandLineExample( + line=["bladebit"], + plotter=plotman.plotters.bladebit.Plotter, + parsed=plotman.job.ParsedChiaPlotsCreateCommand( + error=None, + help=False, + parameters={**default_bladebit_arguments}, + ), + ), + CommandLineExample( + line=["bladebit", "-h"], + plotter=plotman.plotters.bladebit.Plotter, + parsed=plotman.job.ParsedChiaPlotsCreateCommand( + error=None, + help=True, + parameters={**default_bladebit_arguments}, + ), + ), + CommandLineExample( + line=["bladebit", "--help"], + plotter=plotman.plotters.bladebit.Plotter, + parsed=plotman.job.ParsedChiaPlotsCreateCommand( + error=None, + help=True, + parameters={**default_bladebit_arguments}, + ), + ), + CommandLineExample( + line=["bladebit", "--invalid-option"], + plotter=plotman.plotters.bladebit.Plotter, + parsed=plotman.job.ParsedChiaPlotsCreateCommand( + error=click.NoSuchOption("--invalid-option"), + help=False, + parameters={}, + ), + ), + CommandLineExample( + line=["bladebit", "--pool-contract", "xch123abc", "--farmer-key", "abc123"], + plotter=plotman.plotters.bladebit.Plotter, + parsed=plotman.job.ParsedChiaPlotsCreateCommand( + error=None, + help=False, + parameters={ + **default_bladebit_arguments, + "pool_contract": "xch123abc", + "farmer_key": "abc123", + }, + ), + ), + CommandLineExample( + line=["here/there/bladebit"], + plotter=plotman.plotters.bladebit.Plotter, + parsed=plotman.job.ParsedChiaPlotsCreateCommand( + error=None, + help=False, + parameters={**default_bladebit_arguments}, + ), + ), + CommandLineExample( + line=[ + "bladebit", + "final/dir", + ], + cwd="/cwd", + plotter=plotman.plotters.bladebit.Plotter, + parsed=plotman.job.ParsedChiaPlotsCreateCommand( + error=None, + help=False, + parameters={ + **default_bladebit_arguments, + "out_dir": pathlib.Path("/", "cwd", "final", "dir"), + }, + ), + ), + CommandLineExample( + line=plotman.plotters.bladebit.create_command_line( + options=plotman.plotters.bladebit.Options(), + tmpdir="", + tmp2dir=None, + dstdir="/farm/dst/dir", + farmer_public_key=None, + pool_public_key=None, + pool_contract_address=None, + ), + plotter=plotman.plotters.bladebit.Plotter, + parsed=plotman.job.ParsedChiaPlotsCreateCommand( + error=None, + help=False, + parameters={ + **default_bladebit_arguments, + "verbose": True, + "out_dir": pathlib.Path("/farm/dst/dir"), + }, + ), + ), + CommandLineExample( + line=plotman.plotters.bladebit.create_command_line( + options=plotman.plotters.bladebit.Options(), + tmpdir="/farm/tmp/dir", + tmp2dir="/farm/tmp2/dir", + dstdir="/farm/dst/dir", + farmer_public_key="farmerpublickey", + pool_public_key="poolpublickey", + pool_contract_address="poolcontractaddress", + ), + plotter=plotman.plotters.bladebit.Plotter, + parsed=plotman.job.ParsedChiaPlotsCreateCommand( + error=None, + help=False, + parameters={ + **default_bladebit_arguments, + "farmer_key": "farmerpublickey", + "pool_key": "poolpublickey", + "pool_contract": "poolcontractaddress", + "verbose": True, + "out_dir": pathlib.Path("/farm/dst/dir"), + }, + ), + ), +] + + chianetwork_command_line_examples: typing.List[CommandLineExample] = [ CommandLineExample( line=["python", "chia", "plots", "create"], @@ -487,6 +632,7 @@ class CommandLineExample: command_line_examples: typing.List[CommandLineExample] = [ + *bladebit_command_line_examples, *chianetwork_command_line_examples, *madmax_command_line_examples, ] diff --git a/src/plotman/_tests/resources/bladebit.marked b/src/plotman/_tests/resources/bladebit.marked new file mode 100644 index 00000000..1c60d108 --- /dev/null +++ b/src/plotman/_tests/resources/bladebit.marked @@ -0,0 +1,129 @@ +0:0,Creating 1 plots: +0:0, Output path : /mnt/tmp/01/manual-transfer/ +0:0, Thread count : 88 +0:0, Warm start enabled : false +0:0, Farmer public key : b0a374845f4f4d6eab62fc4c5e17965d82ad7eee105818e5bd0cfcb46275a16acc4cd30955779bec841a716473416b21 +0:0, Pool contract address : xch1u8ll2ztwhseej45d6u2zp9j4mlnzhwseccr0axqws9fl2tyj5u0svdy04y +0:0, +0:0,System Memory: 348/503 GiB. +0:0,Memory required: 416 GiB. +0:0,Warning: Not enough memory available. Buffer allocation may fail. +0:1,Allocating buffers. +0:2,Generating plot 1 / 1: 1fc7b57baae24da78e3bea44d58ab51f162a3ed4d242bab2fbcc24f6577d88b3 +0:2, +1:0,Running Phase 1 +1:0,Generating F1... +1:1,Finished F1 generation in 6.93 seconds. +1:1,Sorting F1... +1:1,Finished F1 sort in 18.23 seconds. +1:2,Forward propagating to table 2... +1:2, Pairing L/R groups... +1:2, Finished pairing L/R groups in 13.2870 seconds. Created 4294962218 pairs. +1:2, Average of 236.1403 pairs per group. +1:2, Computing Fx... +1:2, Finished computing Fx in 9.0360 seconds. +1:2, Sorting entries... +1:2, Finished sorting in 33.83 seconds. +1:2,Finished forward propagating table 2 in 56.61 seconds. +1:3,Forward propagating to table 3... +1:3, Pairing L/R groups... +1:3, Finished pairing L/R groups in 10.9170 seconds. Created 4294967296 pairs. +1:3, Average of 236.1406 pairs per group. +1:3, Computing Fx... +1:3, Finished computing Fx in 8.6420 seconds. +1:3, Sorting entries... +1:3, Finished sorting in 33.37 seconds. +1:3,Finished forward propagating table 3 in 53.39 seconds. +1:4,Forward propagating to table 4... +1:4, Pairing L/R groups... +1:4, Finished pairing L/R groups in 10.9450 seconds. Created 4294947733 pairs. +1:4, Average of 236.1396 pairs per group. +1:4, Computing Fx... +1:4, Finished computing Fx in 9.3490 seconds. +1:4, Sorting entries... +1:4, Finished sorting in 32.60 seconds. +1:4,Finished forward propagating table 4 in 53.35 seconds. +1:5,Forward propagating to table 5... +1:5, Pairing L/R groups... +1:5, Finished pairing L/R groups in 10.8420 seconds. Created 4294889963 pairs. +1:5, Average of 236.1364 pairs per group. +1:5, Computing Fx... +1:5, Finished computing Fx in 9.5180 seconds. +1:5, Sorting entries... +1:5, Finished sorting in 32.60 seconds. +1:5,Finished forward propagating table 5 in 53.42 seconds. +1:6,Forward propagating to table 6... +1:6, Pairing L/R groups... +1:6, Finished pairing L/R groups in 10.9870 seconds. Created 4294907255 pairs. +1:6, Average of 236.1373 pairs per group. +1:6, Computing Fx... +1:6, Finished computing Fx in 8.5110 seconds. +1:6, Sorting entries... +1:6, Finished sorting in 31.58 seconds. +1:6,Finished forward propagating table 6 in 51.54 seconds. +1:7,Forward propagating to table 7... +1:7, Pairing L/R groups... +1:7, Finished pairing L/R groups in 11.0050 seconds. Created 4294773122 pairs. +1:7, Average of 236.1300 pairs per group. +1:7, Computing Fx... +1:7, Finished computing Fx in 9.0510 seconds. +1:7,Finished forward propagating table 7 in 20.51 seconds. +2:0,Finished Phase 1 in 313.98 seconds. +2:0,Running Phase 2 +2:1, Prunning table 6... +2:1, Finished prunning table 6 in 0.59 seconds. +2:2, Prunning table 5... +2:2, Finished prunning table 5 in 11.53 seconds. +2:3, Prunning table 4... +2:3, Finished prunning table 4 in 10.86 seconds. +2:4, Prunning table 3... +2:4, Finished prunning table 3 in 10.57 seconds. +2:5, Prunning table 2... +2:5, Finished prunning table 2 in 10.60 seconds. +3:0,Finished Phase 2 in 44.60 seconds. +3:0,Running Phase 3 +3:1, Compressing tables 1 and 2... +3:1, Finished compressing tables 1 and 2 in 31.20 seconds +3:1, Table 1 now has 3429423491 / 4294962218 entries ( 79.85% ). +3:2, Compressing tables 2 and 3... +3:2, Finished compressing tables 2 and 3 in 35.05 seconds +3:2, Table 2 now has 3439923954 / 4294967296 entries ( 80.09% ). +3:3, Compressing tables 3 and 4... +3:3, Finished compressing tables 3 and 4 in 32.41 seconds +3:3, Table 3 now has 3466101892 / 4294947733 entries ( 80.70% ). +3:4, Compressing tables 4 and 5... +3:4, Finished compressing tables 4 and 5 in 33.40 seconds +3:4, Table 4 now has 3532981230 / 4294889963 entries ( 82.26% ). +3:5, Compressing tables 5 and 6... +3:5, Finished compressing tables 5 and 6 in 34.78 seconds +3:5, Table 5 now has 3713621551 / 4294907255 entries ( 86.47% ). +3:6, Compressing tables 6 and 7... +3:6, Finished compressing tables 6 and 7 in 36.41 seconds +3:6, Table 6 now has 4294773122 / 4294773122 entries ( 100.00% ). +4:0,Finished Phase 3 in 203.26 seconds. +4:0,Running Phase 4 +4:1, Writing P7. +4:1, Finished writing P7 in 0.71 seconds. +4:2, Writing C1 table. +4:2, Finished writing C1 table in 0.00 seconds. +4:3, Writing C2 table. +4:3, Finished writing C2 table in 0.00 seconds. +4:4, Writing C3 table. +4:4, Finished writing C3 table in 0.40 seconds. +5:0,Finished Phase 4 in 1.11 seconds. +5:1,Writing final plot tables to disk +5:1, +5:1,Plot /mnt/tmp/01/manual-transfer/plot-k32-2021-08-29-22-22-1fc7b57baae24da78e3bea44d58ab51f162a3ed4d242bab2fbcc24f6577d88b3.plot finished writing to disk: +5:1, Table 1 pointer : 4096 ( 0x0000000000001000 ) +5:1, Table 2 pointer : 14839635968 ( 0x000000037482e000 ) +5:1, Table 3 pointer : 28822732800 ( 0x00000006b5f80000 ) +5:1, Table 4 pointer : 42912239616 ( 0x00000009fdc4d000 ) +5:1, Table 5 pointer : 57273606144 ( 0x0000000d55c5e000 ) +5:1, Table 6 pointer : 72369262592 ( 0x00000010d98b5000 ) +5:1, Table 7 pointer : 89827270656 ( 0x00000014ea1f6000 ) +5:1, C1 table pointer : 107543220224 ( 0x000000190a135000 ) +5:1, C2 table pointer : 107544940544 ( 0x000000190a2d9000 ) +5:1, C3 table pointer : 107544944640 ( 0x000000190a2da000 ) +5:1, +5:1,Finished writing tables to disk in 19.96 seconds. +5:1,Finished plotting in 582.91 seconds (9.72 minutes). diff --git a/src/plotman/_tests/resources/bladebit.plot.log b/src/plotman/_tests/resources/bladebit.plot.log new file mode 100644 index 00000000..5ebf5a59 --- /dev/null +++ b/src/plotman/_tests/resources/bladebit.plot.log @@ -0,0 +1,129 @@ +Creating 1 plots: + Output path : /mnt/tmp/01/manual-transfer/ + Thread count : 88 + Warm start enabled : false + Farmer public key : b0a374845f4f4d6eab62fc4c5e17965d82ad7eee105818e5bd0cfcb46275a16acc4cd30955779bec841a716473416b21 + Pool contract address : xch1u8ll2ztwhseej45d6u2zp9j4mlnzhwseccr0axqws9fl2tyj5u0svdy04y + +System Memory: 348/503 GiB. +Memory required: 416 GiB. +Warning: Not enough memory available. Buffer allocation may fail. +Allocating buffers. +Generating plot 1 / 1: 1fc7b57baae24da78e3bea44d58ab51f162a3ed4d242bab2fbcc24f6577d88b3 + +Running Phase 1 +Generating F1... +Finished F1 generation in 6.93 seconds. +Sorting F1... +Finished F1 sort in 18.23 seconds. +Forward propagating to table 2... + Pairing L/R groups... + Finished pairing L/R groups in 13.2870 seconds. Created 4294962218 pairs. + Average of 236.1403 pairs per group. + Computing Fx... + Finished computing Fx in 9.0360 seconds. + Sorting entries... + Finished sorting in 33.83 seconds. +Finished forward propagating table 2 in 56.61 seconds. +Forward propagating to table 3... + Pairing L/R groups... + Finished pairing L/R groups in 10.9170 seconds. Created 4294967296 pairs. + Average of 236.1406 pairs per group. + Computing Fx... + Finished computing Fx in 8.6420 seconds. + Sorting entries... + Finished sorting in 33.37 seconds. +Finished forward propagating table 3 in 53.39 seconds. +Forward propagating to table 4... + Pairing L/R groups... + Finished pairing L/R groups in 10.9450 seconds. Created 4294947733 pairs. + Average of 236.1396 pairs per group. + Computing Fx... + Finished computing Fx in 9.3490 seconds. + Sorting entries... + Finished sorting in 32.60 seconds. +Finished forward propagating table 4 in 53.35 seconds. +Forward propagating to table 5... + Pairing L/R groups... + Finished pairing L/R groups in 10.8420 seconds. Created 4294889963 pairs. + Average of 236.1364 pairs per group. + Computing Fx... + Finished computing Fx in 9.5180 seconds. + Sorting entries... + Finished sorting in 32.60 seconds. +Finished forward propagating table 5 in 53.42 seconds. +Forward propagating to table 6... + Pairing L/R groups... + Finished pairing L/R groups in 10.9870 seconds. Created 4294907255 pairs. + Average of 236.1373 pairs per group. + Computing Fx... + Finished computing Fx in 8.5110 seconds. + Sorting entries... + Finished sorting in 31.58 seconds. +Finished forward propagating table 6 in 51.54 seconds. +Forward propagating to table 7... + Pairing L/R groups... + Finished pairing L/R groups in 11.0050 seconds. Created 4294773122 pairs. + Average of 236.1300 pairs per group. + Computing Fx... + Finished computing Fx in 9.0510 seconds. +Finished forward propagating table 7 in 20.51 seconds. +Finished Phase 1 in 313.98 seconds. +Running Phase 2 + Prunning table 6... + Finished prunning table 6 in 0.59 seconds. + Prunning table 5... + Finished prunning table 5 in 11.53 seconds. + Prunning table 4... + Finished prunning table 4 in 10.86 seconds. + Prunning table 3... + Finished prunning table 3 in 10.57 seconds. + Prunning table 2... + Finished prunning table 2 in 10.60 seconds. +Finished Phase 2 in 44.60 seconds. +Running Phase 3 + Compressing tables 1 and 2... + Finished compressing tables 1 and 2 in 31.20 seconds + Table 1 now has 3429423491 / 4294962218 entries ( 79.85% ). + Compressing tables 2 and 3... + Finished compressing tables 2 and 3 in 35.05 seconds + Table 2 now has 3439923954 / 4294967296 entries ( 80.09% ). + Compressing tables 3 and 4... + Finished compressing tables 3 and 4 in 32.41 seconds + Table 3 now has 3466101892 / 4294947733 entries ( 80.70% ). + Compressing tables 4 and 5... + Finished compressing tables 4 and 5 in 33.40 seconds + Table 4 now has 3532981230 / 4294889963 entries ( 82.26% ). + Compressing tables 5 and 6... + Finished compressing tables 5 and 6 in 34.78 seconds + Table 5 now has 3713621551 / 4294907255 entries ( 86.47% ). + Compressing tables 6 and 7... + Finished compressing tables 6 and 7 in 36.41 seconds + Table 6 now has 4294773122 / 4294773122 entries ( 100.00% ). +Finished Phase 3 in 203.26 seconds. +Running Phase 4 + Writing P7. + Finished writing P7 in 0.71 seconds. + Writing C1 table. + Finished writing C1 table in 0.00 seconds. + Writing C2 table. + Finished writing C2 table in 0.00 seconds. + Writing C3 table. + Finished writing C3 table in 0.40 seconds. +Finished Phase 4 in 1.11 seconds. +Writing final plot tables to disk + +Plot /mnt/tmp/01/manual-transfer/plot-k32-2021-08-29-22-22-1fc7b57baae24da78e3bea44d58ab51f162a3ed4d242bab2fbcc24f6577d88b3.plot finished writing to disk: + Table 1 pointer : 4096 ( 0x0000000000001000 ) + Table 2 pointer : 14839635968 ( 0x000000037482e000 ) + Table 3 pointer : 28822732800 ( 0x00000006b5f80000 ) + Table 4 pointer : 42912239616 ( 0x00000009fdc4d000 ) + Table 5 pointer : 57273606144 ( 0x0000000d55c5e000 ) + Table 6 pointer : 72369262592 ( 0x00000010d98b5000 ) + Table 7 pointer : 89827270656 ( 0x00000014ea1f6000 ) + C1 table pointer : 107543220224 ( 0x000000190a135000 ) + C2 table pointer : 107544940544 ( 0x000000190a2d9000 ) + C3 table pointer : 107544944640 ( 0x000000190a2da000 ) + +Finished writing tables to disk in 19.96 seconds. +Finished plotting in 582.91 seconds (9.72 minutes). diff --git a/src/plotman/configuration.py b/src/plotman/configuration.py index be2b6c54..dff7d679 100644 --- a/src/plotman/configuration.py +++ b/src/plotman/configuration.py @@ -19,6 +19,7 @@ import yaml from plotman import resources as plotman_resources +import plotman.plotters.bladebit import plotman.plotters.chianetwork import plotman.plotters.madmax @@ -76,7 +77,28 @@ def get_validated_configs( f"Config file at: '{config_path}' is malformed" ) from e - if loaded.plotting.type == "chia": + if loaded.plotting.type == "bladebit": + if loaded.plotting.bladebit is None: + # TODO: fix all the `TODO: use the configured executable` so this is not + # needed. + raise ConfigurationException( + "BladeBit selected as plotter but plotting: bladebit: was not specified in the config", + ) + + if ( + loaded.plotting.pool_pk is not None + and loaded.plotting.pool_contract_address is not None + ): + raise ConfigurationException( + "BladeBit plotter accepts up to one of plotting: pool_pk: and pool_contract_address: but both are specified", + ) + + executable_name = os.path.basename(loaded.plotting.bladebit.executable) + if executable_name != "bladebit": + raise ConfigurationException( + "plotting: bladebit: executable: must refer to an executable named bladebit" + ) + elif loaded.plotting.type == "chia": if loaded.plotting.chia is None: # TODO: fix all the `TODO: use the configured executable` so this is not # needed. @@ -383,11 +405,14 @@ class Plotting: metadata={ desert._make._DESERT_SENTINEL: { "marshmallow_field": marshmallow.fields.String( - validate=marshmallow.validate.OneOf(choices=["chia", "madmax"]), + validate=marshmallow.validate.OneOf( + choices=["bladebit", "chia", "madmax"] + ), ), }, }, ) + bladebit: Optional[plotman.plotters.bladebit.Options] = None chia: Optional[plotman.plotters.chianetwork.Options] = None madmax: Optional[plotman.plotters.madmax.Options] = None @@ -445,6 +470,18 @@ def setup(self) -> Generator[None, None, None]: options=self.plotting.madmax, pool_contract_address=self.plotting.pool_contract_address, ) + elif self.plotting.type == "bladebit": + if self.plotting.bladebit is None: + message = ( + "internal plotman error, please report the full traceback and your" + + " full configuration file" + ) + raise Exception(message) + + plotman.plotters.bladebit.check_configuration( + options=self.plotting.bladebit, + pool_contract_address=self.plotting.pool_contract_address, + ) prefix = f"plotman-pid_{os.getpid()}-" diff --git a/src/plotman/manager.py b/src/plotman/manager.py index 4d91d6cf..ed28c9da 100644 --- a/src/plotman/manager.py +++ b/src/plotman/manager.py @@ -184,7 +184,21 @@ def key(key: str) -> job.Phase: log_file_path = log_cfg.create_plot_log_path(time=pendulum.now()) plot_args: typing.List[str] - if plotting_cfg.type == "madmax": + if plotting_cfg.type == "bladebit": + if plotting_cfg.bladebit is None: + raise Exception( + "bladebit plotter selected but not configured, report this as a plotman bug", + ) + plot_args = plotman.plotters.bladebit.create_command_line( + options=plotting_cfg.bladebit, + tmpdir=tmpdir, + tmp2dir=dir_cfg.tmp2, + dstdir=dstdir, + farmer_public_key=plotting_cfg.farmer_pk, + pool_public_key=plotting_cfg.pool_pk, + pool_contract_address=plotting_cfg.pool_contract_address, + ) + elif plotting_cfg.type == "madmax": if plotting_cfg.madmax is None: raise Exception( "madmax plotter selected but not configured, report this as a plotman bug", diff --git a/src/plotman/plotters/__init__.py b/src/plotman/plotters/__init__.py index e08ca658..86a56f31 100644 --- a/src/plotman/plotters/__init__.py +++ b/src/plotman/plotters/__init__.py @@ -241,10 +241,12 @@ def update(self, chunk: bytes) -> SpecificInfo: def all_plotters() -> typing.List[typing.Type[Plotter]]: # TODO: maybe avoid the import loop some other way + import plotman.plotters.bladebit import plotman.plotters.chianetwork import plotman.plotters.madmax return [ + plotman.plotters.bladebit.Plotter, plotman.plotters.chianetwork.Plotter, plotman.plotters.madmax.Plotter, ] diff --git a/src/plotman/plotters/bladebit.py b/src/plotman/plotters/bladebit.py new file mode 100644 index 00000000..65223a24 --- /dev/null +++ b/src/plotman/plotters/bladebit.py @@ -0,0 +1,542 @@ +# mypy: allow_untyped_decorators + +import collections +import os +import pathlib +import subprocess +import typing + +import attr +import click +import packaging.version +import pendulum + +import plotman.job +import plotman.plotters + + +@attr.frozen +class Options: + executable: str = "bladebit" + threads: typing.Optional[int] = None + no_numa: bool = False + + +def check_configuration( + options: Options, pool_contract_address: typing.Optional[str] +) -> None: + completed_process = subprocess.run( + args=[options.executable, "--version"], + capture_output=True, + check=True, + encoding="utf-8", + ) + version = packaging.version.Version(completed_process.stdout) + required_version = packaging.version.Version("1.1.0") + if version < required_version: + raise Exception( + f"BladeBit version {required_version} required for monitoring logs but" + f" found: {version}" + ) + + if pool_contract_address is not None: + completed_process = subprocess.run( + args=[options.executable, "--help"], + capture_output=True, + check=True, + encoding="utf-8", + ) + # TODO: report upstream + if ( + "--pool-contract" not in completed_process.stdout + and "--pool-contract" not in completed_process.stderr + ): + print(completed_process.stdout) + raise Exception( + f"found BladeBit version does not support the `--pool-contract`" + f" option for pools." + ) + + +def create_command_line( + options: Options, + tmpdir: str, + tmp2dir: typing.Optional[str], + dstdir: str, + farmer_public_key: typing.Optional[str], + pool_public_key: typing.Optional[str], + pool_contract_address: typing.Optional[str], +) -> typing.List[str]: + args = [ + options.executable, + "-v", + "-n", + "1", + ] + + if options.threads is not None: + args.append("-t") + args.append(str(options.threads)) + + if farmer_public_key is not None: + args.append("-f") + args.append(farmer_public_key) + if pool_public_key is not None: + args.append("-p") + args.append(pool_public_key) + if pool_contract_address is not None: + args.append("-c") + args.append(pool_contract_address) + + args.append(dstdir) + + return args + + +@plotman.plotters.check_SpecificInfo +@attr.frozen +class SpecificInfo: + process_id: typing.Optional[int] = None + phase: plotman.job.Phase = plotman.job.Phase(known=False) + + started_at: typing.Optional[pendulum.DateTime] = None + plot_id: str = "" + threads: int = 0 + # buffer: int = 0 + plot_size: int = 32 + dst_dir: str = "" + phase1_duration_raw: float = 0 + phase2_duration_raw: float = 0 + phase3_duration_raw: float = 0 + phase4_duration_raw: float = 0 + total_time_raw: float = 0 + # copy_time_raw: float = 0 + filename: str = "" + plot_name: str = "" + + def common(self) -> plotman.plotters.CommonInfo: + return plotman.plotters.CommonInfo( + type="bladebit", + dstdir=self.dst_dir, + phase=self.phase, + tmpdir="", + tmp2dir="", + started_at=self.started_at, + plot_id=self.plot_id, + plot_size=self.plot_size, + buckets=0, + threads=self.threads, + phase1_duration_raw=self.phase1_duration_raw, + phase2_duration_raw=self.phase2_duration_raw, + phase3_duration_raw=self.phase3_duration_raw, + phase4_duration_raw=self.phase4_duration_raw, + total_time_raw=self.total_time_raw, + filename=self.filename, + ) + + +@plotman.plotters.check_Plotter +@attr.mutable +class Plotter: + decoder: plotman.plotters.LineDecoder = attr.ib( + factory=plotman.plotters.LineDecoder + ) + info: SpecificInfo = attr.ib(factory=SpecificInfo) + parsed_command_line: typing.Optional[ + plotman.job.ParsedChiaPlotsCreateCommand + ] = None + + @classmethod + def identify_log(cls, line: str) -> bool: + return "Warm start enabled" in line + + @classmethod + def identify_process(cls, command_line: typing.List[str]) -> bool: + if len(command_line) == 0: + return False + + return "bladebit" == os.path.basename(command_line[0]).lower() + + def common_info(self) -> plotman.plotters.CommonInfo: + return self.info.common() + + def parse_command_line(self, command_line: typing.List[str], cwd: str) -> None: + # drop the bladebit + arguments = command_line[1:] + + # TODO: We could at some point do version detection and pick the + # associated command. For now we'll just use the latest one we have + # copied. + command = commands.latest_command() + + self.parsed_command_line = plotman.plotters.parse_command_line_with_click( + command=command, + arguments=arguments, + ) + + for key in ["out_dir"]: + original: os.PathLike[str] = self.parsed_command_line.parameters.get(key) # type: ignore[assignment] + if original is not None: + self.parsed_command_line.parameters[key] = pathlib.Path(cwd).joinpath( + original + ) + + def update(self, chunk: bytes) -> SpecificInfo: + new_lines = self.decoder.update(chunk=chunk) + + for line in new_lines: + if not self.info.phase.known: + self.info = attr.evolve( + self.info, phase=plotman.job.Phase(major=0, minor=0) + ) + + for pattern, handler_functions in handlers.mapping.items(): + match = pattern.search(line) + + if match is None: + continue + + for handler_function in handler_functions: + self.info = handler_function(match=match, info=self.info) + + break + + return self.info + + +handlers = plotman.plotters.RegexLineHandlers[SpecificInfo]() + + +@handlers.register(expression=r"^Running Phase (?P\d+)") +def running_phase(match: typing.Match[str], info: SpecificInfo) -> SpecificInfo: + # Running Phase 1 + major = int(match.group("phase")) + return attr.evolve(info, phase=plotman.job.Phase(major=major, minor=0)) + + +@handlers.register( + expression=r"^Finished Phase (?P\d+) in (?P[^ ]+) seconds." +) +def phase_finished(match: typing.Match[str], info: SpecificInfo) -> SpecificInfo: + # Finished Phase 1 in 313.98 seconds. + major = int(match.group("phase")) + duration = float(match.group("duration")) + duration_dict = {f"phase{major}_duration_raw": duration} + return attr.evolve( + info, phase=plotman.job.Phase(major=major + 1, minor=0), **duration_dict + ) + + +@handlers.register(expression=r"^Allocating buffers\.$") +def allocating_buffers(match: typing.Match[str], info: SpecificInfo) -> SpecificInfo: + # Allocating buffers. + return attr.evolve(info, phase=plotman.job.Phase(major=0, minor=1)) + + +@handlers.register(expression=r"^Finished F1 generation in") +def finished_f1(match: typing.Match[str], info: SpecificInfo) -> SpecificInfo: + # Finished F1 generation in 6.93 seconds. + return attr.evolve(info, phase=plotman.job.Phase(major=1, minor=1)) + + +@handlers.register(expression=r"^Forward propagating to table (?P\d+)") +def forward_propagating(match: typing.Match[str], info: SpecificInfo) -> SpecificInfo: + # Forward propagating to table 2... + minor = int(match.group("table")) + return attr.evolve(info, phase=plotman.job.Phase(major=1, minor=minor)) + + +@handlers.register(expression=r"^ *Prunn?ing table (?P
\d+)") +def pruning_table(match: typing.Match[str], info: SpecificInfo) -> SpecificInfo: + # Prunning table 6... + table = int(match.group("table")) + minor = 7 - table + return attr.evolve(info, phase=plotman.job.Phase(major=2, minor=minor)) + + +@handlers.register(expression=r"^ *Compressing tables (?P
\d+)") +def compressing_tables(match: typing.Match[str], info: SpecificInfo) -> SpecificInfo: + # Compressing tables 1 and 2... + minor = int(match.group("table")) + return attr.evolve(info, phase=plotman.job.Phase(major=3, minor=minor)) + + +@handlers.register(expression=r"^ *Writing (?P(P7|C1|C2|C3))") +def phase_4_writing(match: typing.Match[str], info: SpecificInfo) -> SpecificInfo: + # Writing P7. + minors = {"P7": 1, "C1": 2, "C2": 3, "C3": 4} + tag = match.group("tag") + minor = minors[tag] + return attr.evolve(info, phase=plotman.job.Phase(major=4, minor=minor)) + + +@handlers.register(expression=r"^Generating plot .*: (?P[^ ]+)") +def generating_plot(match: typing.Match[str], info: SpecificInfo) -> SpecificInfo: + # Generating plot 1 / 1: 1fc7b57baae24da78e3bea44d58ab51f162a3ed4d242bab2fbcc24f6577d88b3 + return attr.evolve( + info, + phase=plotman.job.Phase(major=0, minor=2), + plot_id=match.group("plot_id"), + ) + + +@handlers.register(expression=r"^Writing final plot tables to disk$") +def writing_final(match: typing.Match[str], info: SpecificInfo) -> SpecificInfo: + # Writing final plot tables to disk + return attr.evolve(info, phase=plotman.job.Phase(major=5, minor=1)) + + +@handlers.register(expression=r"^Finished plotting in (?P[^ ]+) seconds") +def total_duration(match: typing.Match[str], info: SpecificInfo) -> SpecificInfo: + # Finished plotting in 582.91 seconds (9.72 minutes). + duration = float(match.group("duration")) + return attr.evolve(info, total_time_raw=duration) + + +@handlers.register(expression=r"^ *Output path *: *(.+)") +def dst_dir(match: typing.Match[str], info: SpecificInfo) -> SpecificInfo: + # Output path : /mnt/tmp/01/manual-transfer/ + return attr.evolve(info, dst_dir=match.group(1)) + + +@handlers.register( + expression=r"^Plot .*/(?P(?Pplot-k(?P\d+)-(?P\d+)-(?P\d+)-(?P\d+)-(?P\d+)-(?P\d+)-(?P\w+)).plot) .*" +) +def plot_name_line(match: typing.Match[str], info: SpecificInfo) -> SpecificInfo: + # Plot /mnt/tmp/01/manual-transfer/plot-k32-2021-08-29-22-22-1fc7b57baae24da78e3bea44d58ab51f162a3ed4d242bab2fbcc24f6577d88b3.plot finished writing to disk: + return attr.evolve( + info, + plot_size=int(match.group("size")), + plot_name=match.group("name"), + started_at=pendulum.datetime( + year=int(match.group("year")), + month=int(match.group("month")), + day=int(match.group("day")), + hour=int(match.group("hour")), + minute=int(match.group("minute")), + tz=None, + ), + filename=match.group("filename"), + plot_id=match.group("plot_id"), + ) + + +@handlers.register(expression=r"^ *Thread count *: *(\d+)") +def threads(match: typing.Match[str], info: SpecificInfo) -> SpecificInfo: + # Thread count : 88 + return attr.evolve(info, threads=int(match.group(1))) + + +commands = plotman.plotters.core.Commands() + + +# BladeBit Git on 2021-08-29 -> https://github.com/harold-b/bladebit/commit/f3fbfff43ce493ec9e02db6f72c3b44f656ef137 +@commands.register(version=(0,)) +@click.command() +# https://github.com/harold-b/bladebit/blob/f3fbfff43ce493ec9e02db6f72c3b44f656ef137/LICENSE +# https://github.com/harold-b/bladebit/blob/f7cf06fa685c9b1811465ecd47129402bb7548a0/src/main.cpp#L75-L108 +@click.option( + "-t", + "--threads", + help=( + "Maximum number of threads to use." + " For best performance, use all available threads (default behavior)." + " Values below 2 are not recommended." + ), + type=int, + show_default=True, +) +@click.option( + "-n", + "--count", + help="Number of plots to create. Default = 1.", + type=int, + default=1, + show_default=True, +) +@click.option( + "-f", + "--farmer-key", + help="Farmer public key, specified in hexadecimal format.", + type=str, +) +@click.option( + "-p", + "--pool-key", + help=( + "Pool public key, specified in hexadecimal format." + " Either a pool public key or a pool contract address must be specified." + ), + type=str, +) +@click.option( + "-c", + "--pool-contract", + help=( + "Pool contract address, specified in hexadecimal format." + " Address where the pool reward will be sent to." + " Only used if pool public key is not specified." + ), + type=str, +) +@click.option( + "-w", + "--warm-start", + help="Touch all pages of buffer allocations before starting to plot.", + is_flag=True, + type=bool, + default=False, +) +@click.option( + "-i", + "--plot-id", + help="Specify a plot id for debugging.", + type=str, +) +@click.option( + "-v", + "--verbose", + help="Enable verbose output.", + is_flag=True, + type=bool, + default=False, +) +@click.option( + "-m", + "--no-numa", + help=( + "Disable automatic NUMA aware memory binding." + " If you set this parameter in a NUMA system you will likely get degraded performance." + ), + is_flag=True, + type=bool, + default=False, +) +@click.argument( + "out_dir", + # help=( + # "Output directory in which to output the plots." " This directory must exist." + # ), + type=click.Path(), + default=pathlib.Path("."), + # show_default=True, +) +def _cli_f3fbfff43ce493ec9e02db6f72c3b44f656ef137() -> None: + pass + + +# BladeBit Git on 2021-08-29 -> https://github.com/harold-b/bladebit/commit/b48f262336362acd6f23c5ca9a43cfd6d244cb88 +@commands.register(version=(1, 1, 0)) +@click.command() +# https://github.com/harold-b/bladebit/blob/b48f262336362acd6f23c5ca9a43cfd6d244cb88/LICENSE +# https://github.com/harold-b/bladebit/blob/b48f262336362acd6f23c5ca9a43cfd6d244cb88/src/main.cpp#L77-L119 +@click.option( + "-t", + "--threads", + help=( + "Maximum number of threads to use." + " For best performance, use all available threads (default behavior)." + " Values below 2 are not recommended." + ), + type=int, + show_default=True, +) +@click.option( + "-n", + "--count", + help="Number of plots to create. Default = 1.", + type=int, + default=1, + show_default=True, +) +@click.option( + "-f", + "--farmer-key", + help="Farmer public key, specified in hexadecimal format.", + type=str, +) +@click.option( + "-p", + "--pool-key", + help=( + "Pool public key, specified in hexadecimal format." + " Either a pool public key or a pool contract address must be specified." + ), + type=str, +) +@click.option( + "-c", + "--pool-contract", + help=( + "Pool contract address, specified in hexadecimal format." + " Address where the pool reward will be sent to." + " Only used if pool public key is not specified." + ), + type=str, +) +@click.option( + "-w", + "--warm-start", + help="Touch all pages of buffer allocations before starting to plot.", + is_flag=True, + type=bool, + default=False, +) +@click.option( + "-i", + "--plot-id", + help="Specify a plot id for debugging.", + type=str, +) +@click.option( + "--memo", + help="Specify a plot memo for debugging.", + type=str, +) +@click.option( + "--show-memo", + help="Output the memo of the next plot the be plotted.", + is_flag=True, + type=bool, + default=False, +) +@click.option( + "-v", + "--verbose", + help="Enable verbose output.", + is_flag=True, + type=bool, + default=False, +) +@click.option( + "-m", + "--no-numa", + help=( + "Disable automatic NUMA aware memory binding." + " If you set this parameter in a NUMA system you will likely get degraded performance." + ), + is_flag=True, + type=bool, + default=False, +) +@click.option( + "--no-cpu-affinity", + help=( + "Disable assigning automatic thread affinity." + " This is useful when running multiple simultaneous instances of bladebit as you can manually assign thread affinity yourself when launching bladebit." + ), + is_flag=True, + type=bool, + default=False, +) +@click.argument( + "out_dir", + # help=( + # "Output directory in which to output the plots." " This directory must exist." + # ), + type=click.Path(), + default=pathlib.Path("."), + # show_default=True, +) +def _cli_b48f262336362acd6f23c5ca9a43cfd6d244cb88() -> None: + pass