diff --git a/.coveragerc b/.coveragerc new file mode 100644 index 00000000..e6661885 --- /dev/null +++ b/.coveragerc @@ -0,0 +1,19 @@ +[paths] +source = + src + */site-packages + +[report] +precision = 1 +exclude_lines = + pragma: no cover + abc\.abstractmethod + typing\.overload + if typing.TYPE_CHECKING: + ^\s*pass\s*$ + ^\s*...\s*$ + +[run] +branch = True +source = + plotman diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index dfa08abb..f2c0f406 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -2,7 +2,9 @@ name: CI on: push: - branches: [ development ] + branches: + - main + - development tags: [ "**" ] pull_request: branches: [ "**" ] @@ -13,7 +15,8 @@ defaults: jobs: build: - name: ${{ matrix.task.name}} - ${{ matrix.os.name }} ${{ matrix.python.name }} + # Should match JOB_NAME below + name: ${{ matrix.task.name }} - ${{ matrix.os.name }} ${{ matrix.python.name }} runs-on: ${{ matrix.os.runs-on }} strategy: fail-fast: false @@ -30,6 +33,10 @@ jobs: - name: Build tox: build + env: + # Should match name above + JOB_NAME: ${{ matrix.task.name }} - ${{ matrix.os.name }} ${{ matrix.python.name }} + steps: - uses: actions/checkout@v2 with: @@ -71,7 +78,8 @@ jobs: path: dist/ test: - name: ${{ matrix.task.name}} - ${{ matrix.os.name }} ${{ matrix.python.name }} + # Should match JOB_NAME below + name: ${{ matrix.task.name }} - ${{ matrix.os.name }} ${{ matrix.python.name }} runs-on: ${{ matrix.os.runs-on }} needs: - build @@ -98,6 +106,7 @@ jobs: task: - name: Test tox: test + coverage: true include: - task: name: Check @@ -111,6 +120,8 @@ jobs: env: + # Should match name above + JOB_NAME: ${{ matrix.task.name }} - ${{ matrix.os.name }} ${{ matrix.python.name }} TOXENV: ${{ matrix.task.tox }}${{ fromJSON('["", "-"]')[matrix.python.tox != null] }}${{ matrix.python.tox }} steps: @@ -157,12 +168,116 @@ jobs: run: | tox --skip-pkg-install + - name: Coverage Processing + if: matrix.task.coverage + run: | + mkdir coverage_reports + cp .coverage "coverage_reports/.coverage.${{ env.JOB_NAME }}" + cp coverage.xml "coverage_reports/coverage.${{ env.JOB_NAME }}.xml" + + - name: Publish Coverage + if: matrix.task.coverage + uses: actions/upload-artifact@v2 + with: + name: coverage + path: coverage_reports/* + + coverage: + # Should match JOB_NAME below + name: ${{ matrix.task.name }} - ${{ matrix.os.name }} ${{ matrix.python.name }} + runs-on: ${{ matrix.os.runs-on }} + needs: + - test + strategy: + fail-fast: false + matrix: + include: + - os: + name: Linux + runs-on: ubuntu-latest + python: + name: CPython 3.8 + action: 3.8 + task: + name: Coverage + tox: check-coverage + coverage: false + download_coverage: true + + env: + # Should match name above + JOB_NAME: ${{ matrix.task.name }} - ${{ matrix.os.name }} ${{ matrix.python.name }} + TOXENV: ${{ matrix.task.tox }}${{ fromJSON('["", "-"]')[matrix.task.tox != null && matrix.python.tox != null] }}${{ matrix.python.tox }} + + steps: + - uses: actions/checkout@v2 + with: + fetch-depth: 0 + ref: ${{ github.event.pull_request.head.sha }} + + - name: Download package files + uses: actions/download-artifact@v2 + with: + name: dist + path: dist/ + + - name: Download Coverage + if: matrix.task.download_coverage + uses: actions/download-artifact@v2 + with: + name: coverage + path: coverage_reports + + - name: Set up ${{ matrix.python.name }} + uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.python.action }} + + - name: Install dependencies + run: | + python -m pip install --upgrade pip setuptools wheel + pip install tox + + - name: Prepare tox environment + run: | + tox --notest --installpkg dist/*.whl + + - name: Runner info + uses: twisted/python-info-action@v1 + + - name: Tox info + uses: twisted/python-info-action@v1 + with: + python-path: .tox/${{ env.TOXENV }}/*/python + + - name: Run tox environment + env: + BASE_REF: ${{ fromJSON(format('[{0}, {1}]', toJSON(github.event.before), toJSON(format('origin/{0}', github.base_ref))))[github.base_ref != ''] }} + run: | + tox --skip-pkg-install -- --compare-branch="${BASE_REF}" + + - name: Coverage Processing + if: always() + run: | + mkdir all_coverage_report + cp .coverage "all_coverage_report/.coverage.all" + cp coverage.xml "all_coverage_report/coverage.all.xml" + + - name: Upload Coverage + if: always() + uses: actions/upload-artifact@v2 + with: + name: coverage + path: all_coverage_report/* + all: name: All runs-on: ubuntu-latest needs: - build - test + # TODO: make this required when we have a better testing situation + # - coverage steps: - name: This shell: python diff --git a/.gitignore b/.gitignore index 82adb58b..16303851 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,4 @@ __pycache__ venv +.DS_Store +.vscode diff --git a/LICENSE-chia-blockchain b/LICENSE-chia-blockchain new file mode 100644 index 00000000..ee81ae2a --- /dev/null +++ b/LICENSE-chia-blockchain @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright 2021 Chia Network + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/MAINTENANCE.md b/MAINTENANCE.md new file mode 100644 index 00000000..b8b17c2d --- /dev/null +++ b/MAINTENANCE.md @@ -0,0 +1,19 @@ +# Maintenance + +## Overview + +This document holds guidance on maintaining aspects of plotman. + +## The `chia plots create` CLI parsing code + +In [src/plotman/chia.py](src/plotman/chia.py) there is code copied from the `chia plots create` subcommand's CLI parser definition. +When new versions of `chia-blockchain` are released, their interface code should be added to plotman. +plotman commit [1b5db4e](https://github.com/ericaltendorf/plotman/commit/1b5db4e342b9ec1f7910663a453aec3a97ba51a6) provides an example of adding a new version. + +In many cases, copying code is a poor choice. +It is believed that in this case it is appropriate since the chia code that plotman could import is not necessarily the code that is parsing the plotting process command lines anyways. +The chia command could come from another Python environment, a system package, a `.dmg`, etc. +This approach also offers future potential of using the proper version of parsing for the specific plot process being inspected. +Finally, this alleviates dealing with the dependency on the `chia-blockchain` package. +In generally, using dependencies is good. +This seems to be an exceptional case. diff --git a/MANIFEST.in b/MANIFEST.in index c6146883..5510584d 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,9 +1,10 @@ include CHANGELOG.md -include LICENSE +include LICENSE* include README.md include *.md include VERSION include tox.ini +include .coveragerc recursive-include src *.py recursive-include src/plotman/_tests/resources * recursive-include src/plotman/resources * diff --git a/README.md b/README.md index b298e7ca..b53e4244 100644 --- a/README.md +++ b/README.md @@ -181,11 +181,14 @@ To display the current location of your `plotman.yaml` file and check if it exis ## Installation -Installation for Linux: +Installation for Linux and macOS: 1. Plotman assumes that a functioning [Chia](https://github.com/Chia-Network/chia-blockchain) - installation is present on the system. Activate your `chia` environment by typing - `source /path/to/your/chia/install/activate`. + installation is present on the system. + - virtual environment (Linux, macOS): Activate your `chia` environment by typing + `source /path/to/your/chia/install/activate`. + - dmg (macOS): Follow [these instructions](https://github.com/Chia-Network/chia-blockchain/wiki/CLI-Commands-Reference#mac) + to add the `chia` binary to the `PATH` 2. Then, install Plotman using the following command: ```shell > pip install --force-reinstall git+https://github.com/ericaltendorf/plotman@main diff --git a/VERSION b/VERSION index 3b04cfb6..be586341 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.2 +0.3 diff --git a/setup.cfg b/setup.cfg index 092ae2b3..98e6f84f 100644 --- a/setup.cfg +++ b/setup.cfg @@ -37,6 +37,8 @@ package_dir= packages=find: install_requires = appdirs + attrs + click desert marshmallow pendulum @@ -52,13 +54,17 @@ console_scripts = plotman = plotman.plotman:main [options.extras_require] +coverage = + coverage + diff-cover dev = %(test)s isort -test = +test = + %(coverage)s check-manifest pytest - pytest-mock + pytest-cov pyfakefs [options.data_files] diff --git a/src/plotman/_tests/archive_test.py b/src/plotman/_tests/archive_test.py index 9caa2abe..4b98ac90 100755 --- a/src/plotman/_tests/archive_test.py +++ b/src/plotman/_tests/archive_test.py @@ -1,9 +1,9 @@ -from plotman import archive, configuration, manager +from plotman import archive, configuration, job, manager def test_compute_priority(): - assert (archive.compute_priority( (3, 1), 1000, 10) > - archive.compute_priority( (3, 6), 1000, 10) ) + assert (archive.compute_priority( job.Phase(major=3, minor=1), 1000, 10) > + archive.compute_priority( job.Phase(major=3, minor=6), 1000, 10) ) def test_rsync_dest(): arch_dir = '/plotdir/012' diff --git a/src/plotman/_tests/configuration_test.py b/src/plotman/_tests/configuration_test.py index 0f0e9d56..f0a548b6 100644 --- a/src/plotman/_tests/configuration_test.py +++ b/src/plotman/_tests/configuration_test.py @@ -8,45 +8,49 @@ from plotman import resources as plotman_resources -@pytest.fixture(name='config_path') -def config_fixture(tmp_path): - """Return direct path to plotman.yaml""" - with importlib.resources.path(plotman_resources, "plotman.yaml") as path: - yield path +@pytest.fixture(name='config_text') +def config_text_fixture(): + return importlib.resources.read_text(plotman_resources, "plotman.yaml") -def test_get_validated_configs__default(mocker, config_path): +def test_get_validated_configs__default(config_text): """Check that get_validated_configs() works with default/example plotman.yaml file.""" - mocker.patch("plotman.configuration.get_path", return_value=config_path) - res = configuration.get_validated_configs() + res = configuration.get_validated_configs(config_text, '') assert isinstance(res, configuration.PlotmanConfig) -def test_get_validated_configs__malformed(mocker, config_path): +def test_get_validated_configs__malformed(config_text): """Check that get_validated_configs() raises exception with invalid plotman.yaml contents.""" - mocker.patch("plotman.configuration.get_path", return_value=config_path) - with open(configuration.get_path(), "r") as file: - loaded_yaml = yaml.load(file, Loader=yaml.SafeLoader) + loaded_yaml = yaml.load(config_text, Loader=yaml.SafeLoader) # Purposefully malform the contents of loaded_yaml by changing tmp from List[str] --> str loaded_yaml["directories"]["tmp"] = "/mnt/tmp/00" - mocker.patch("yaml.load", return_value=loaded_yaml) + malformed_config_text = yaml.dump(loaded_yaml, Dumper=yaml.SafeDumper) with pytest.raises(configuration.ConfigurationException) as exc_info: - configuration.get_validated_configs() + configuration.get_validated_configs(malformed_config_text, '/the_path') - assert exc_info.value.args[0] == f"Config file at: '{configuration.get_path()}' is malformed" + assert exc_info.value.args[0] == f"Config file at: '/the_path' is malformed" -def test_get_validated_configs__missing(mocker, config_path): +def test_get_validated_configs__missing(): """Check that get_validated_configs() raises exception when plotman.yaml does not exist.""" - nonexistent_config = config_path.with_name("plotman2.yaml") - mocker.patch("plotman.configuration.get_path", return_value=nonexistent_config) - with pytest.raises(configuration.ConfigurationException) as exc_info: - configuration.get_validated_configs() + configuration.read_configuration_text('/invalid_path') assert exc_info.value.args[0] == ( - f"No 'plotman.yaml' file exists at expected location: '{nonexistent_config}'. To generate " + f"No 'plotman.yaml' file exists at expected location: '/invalid_path'. To generate " f"default config file, run: 'plotman config generate'" ) + + +def test_loads_without_user_interface(config_text): + loaded_yaml = yaml.load(config_text, Loader=yaml.SafeLoader) + + del loaded_yaml["user_interface"] + + stripped_config_text = yaml.dump(loaded_yaml, Dumper=yaml.SafeDumper) + + reloaded_yaml = configuration.get_validated_configs(stripped_config_text, '') + + assert reloaded_yaml.user_interface == configuration.UserInterface() diff --git a/src/plotman/_tests/job_test.py b/src/plotman/_tests/job_test.py index 04727a92..25840723 100644 --- a/src/plotman/_tests/job_test.py +++ b/src/plotman/_tests/job_test.py @@ -21,7 +21,7 @@ def update_from_logfile(self): @pytest.fixture(name='logfile_path') def logfile_fixture(tmp_path): - log_name = '2021-04-04-19:00:47.log' + log_name = '2021-04-04T19_00_47.681088-0400.log' log_contents = importlib.resources.read_binary(resources, log_name) log_file_path = tmp_path.joinpath(log_name) log_file_path.write_bytes(log_contents) @@ -54,3 +54,88 @@ def test_job_parses_time_with_non_english_locale(logfile_path, locale_name): job.Job.init_from_logfile(self=faux_job_with_logfile) assert faux_job_with_logfile.start_time == log_file_time + + +@pytest.mark.parametrize( + argnames=['arguments'], + argvalues=[ + [['-h']], + [['--help']], + [['-k', '32']], + [['-k32']], + [['-k', '32', '--help']], + ], + ids=str, +) +def test_chia_plots_create_parsing_does_not_fail(arguments): + job.parse_chia_plots_create_command_line( + command_line=['python', 'chia', 'plots', 'create', *arguments], + ) + + +@pytest.mark.parametrize( + argnames=['arguments'], + argvalues=[ + [['-h']], + [['--help']], + [['-k', '32', '--help']], + ], + ids=str, +) +def test_chia_plots_create_parsing_detects_help(arguments): + parsed = job.parse_chia_plots_create_command_line( + command_line=['python', 'chia', 'plots', 'create', *arguments], + ) + + assert parsed.help + + +@pytest.mark.parametrize( + argnames=['arguments'], + argvalues=[ + [[]], + [['-k32']], + [['-k', '32']], + ], + ids=str, +) +def test_chia_plots_create_parsing_detects_not_help(arguments): + parsed = job.parse_chia_plots_create_command_line( + command_line=['python', 'chia', 'plots', 'create', *arguments], + ) + + assert not parsed.help + + +@pytest.mark.parametrize( + argnames=['arguments'], + argvalues=[ + [[]], + [['-k32']], + [['-k', '32']], + [['--size', '32']], + ], + ids=str, +) +def test_chia_plots_create_parsing_handles_argument_forms(arguments): + parsed = job.parse_chia_plots_create_command_line( + command_line=['python', 'chia', 'plots', 'create', *arguments], + ) + + assert parsed.parameters['size'] == 32 + + +@pytest.mark.parametrize( + argnames=['arguments'], + argvalues=[ + [['--size32']], + [['--not-an-actual-option']], + ], + ids=str, +) +def test_chia_plots_create_parsing_identifies_errors(arguments): + parsed = job.parse_chia_plots_create_command_line( + command_line=['python', 'chia', 'plots', 'create', *arguments], + ) + + assert parsed.error is not None diff --git a/src/plotman/_tests/manager_test.py b/src/plotman/_tests/manager_test.py index 2f425955..cca37596 100755 --- a/src/plotman/_tests/manager_test.py +++ b/src/plotman/_tests/manager_test.py @@ -27,30 +27,37 @@ def dir_cfg(): ) def test_permit_new_job_post_milestone(sched_cfg, dir_cfg): + phases = job.Phase.list_from_tuples([ (3, 8), (4, 1) ]) assert manager.phases_permit_new_job( - [ (3, 8), (4, 1) ], '/mnt/tmp/00', sched_cfg, dir_cfg) + phases, '/mnt/tmp/00', sched_cfg, dir_cfg) def test_permit_new_job_pre_milestone(sched_cfg, dir_cfg): + phases = job.Phase.list_from_tuples([ (2, 3), (4, 1) ]) assert not manager.phases_permit_new_job( - [ (2, 3), (4, 1) ], '/mnt/tmp/00', sched_cfg, dir_cfg) + phases, '/mnt/tmp/00', sched_cfg, dir_cfg) def test_permit_new_job_too_many_jobs(sched_cfg, dir_cfg): + phases = job.Phase.list_from_tuples([ (3, 1), (3, 2), (3, 3) ]) assert not manager.phases_permit_new_job( - [ (3, 1), (3, 2), (3, 3) ], '/mnt/tmp/00', sched_cfg, dir_cfg) + phases, '/mnt/tmp/00', sched_cfg, dir_cfg) def test_permit_new_job_too_many_jobs_zerophase(sched_cfg, dir_cfg): + phases = job.Phase.list_from_tuples([ (3, 0), (3, 1), (3, 3) ]) assert not manager.phases_permit_new_job( - [ (3, 0), (3, 1), (3, 3) ], '/mnt/tmp/00', sched_cfg, dir_cfg) + phases, '/mnt/tmp/00', sched_cfg, dir_cfg) def test_permit_new_job_too_many_jobs_nonephase(sched_cfg, dir_cfg): + phases = job.Phase.list_from_tuples([ (None, None), (3, 1), (3, 3) ]) assert manager.phases_permit_new_job( - [ (None, None), (3, 1), (3, 3) ], '/mnt/tmp/00', sched_cfg, dir_cfg) + phases, '/mnt/tmp/00', sched_cfg, dir_cfg) def test_permit_new_job_override_tmp_dir(sched_cfg, dir_cfg): + phases = job.Phase.list_from_tuples([ (3, 1), (3, 2), (3, 3) ]) assert manager.phases_permit_new_job( - [ (3, 1), (3, 2), (3, 3) ], '/mnt/tmp/04', sched_cfg, dir_cfg) + phases, '/mnt/tmp/04', sched_cfg, dir_cfg) + phases = job.Phase.list_from_tuples([ (3, 1), (3, 2), (3, 3), (3, 6) ]) assert not manager.phases_permit_new_job( - [ (3, 1), (3, 2), (3, 3), (3, 6) ], '/mnt/tmp/04', sched_cfg, + phases, '/mnt/tmp/04', sched_cfg, dir_cfg) @patch('plotman.job.Job') diff --git a/src/plotman/_tests/reporting_test.py b/src/plotman/_tests/reporting_test.py index b46dd873..87c8a5e2 100644 --- a/src/plotman/_tests/reporting_test.py +++ b/src/plotman/_tests/reporting_test.py @@ -3,23 +3,24 @@ from unittest.mock import patch from plotman import reporting +from plotman import job def test_phases_str_basic(): - assert(reporting.phases_str([(1,2), (2,3), (3,4), (4,0)]) == - '1:2 2:3 3:4 4:0') + phases = job.Phase.list_from_tuples([(1,2), (2,3), (3,4), (4,0)]) + assert reporting.phases_str(phases) == '1:2 2:3 3:4 4:0' def test_phases_str_elipsis_1(): - assert(reporting.phases_str([(1,2), (2,3), (3,4), (4,0)], 3) == - '1:2 [+1] 3:4 4:0') + phases = job.Phase.list_from_tuples([(1,2), (2,3), (3,4), (4,0)]) + assert reporting.phases_str(phases, 3) == '1:2 [+1] 3:4 4:0' def test_phases_str_elipsis_2(): - assert(reporting.phases_str([(1,2), (2,3), (3,4), (4,0)], 2) == - '1:2 [+2] 4:0') + phases = job.Phase.list_from_tuples([(1,2), (2,3), (3,4), (4,0)]) + assert reporting.phases_str(phases, 2) == '1:2 [+2] 4:0' def test_phases_str_none(): - assert(reporting.phases_str([(None, None), (2, None), (3, 0)]) == - '?:? 2:? 3:0') + phases = job.Phase.list_from_tuples([(None, None), (3, 0)]) + assert reporting.phases_str(phases) == '?:? 3:0' def test_job_viz_empty(): assert(reporting.job_viz([]) == '1 2 3 4 ') @@ -27,7 +28,7 @@ def test_job_viz_empty(): @patch('plotman.job.Job') def job_w_phase(ph, MockJob): j = MockJob() - j.progress.return_value = ph + j.progress.return_value = job.Phase.from_tuple(ph) return j def test_job_viz_positions(): diff --git a/src/plotman/_tests/resources/2021-04-04-19:00:47.log b/src/plotman/_tests/resources/2021-04-04T19_00_47.681088-0400.log similarity index 100% rename from src/plotman/_tests/resources/2021-04-04-19:00:47.log rename to src/plotman/_tests/resources/2021-04-04T19_00_47.681088-0400.log diff --git a/src/plotman/_tests/resources/2021-04-04-19:00:47.notes b/src/plotman/_tests/resources/2021-04-04T19_00_47.681088-0400.notes similarity index 100% rename from src/plotman/_tests/resources/2021-04-04-19:00:47.notes rename to src/plotman/_tests/resources/2021-04-04T19_00_47.681088-0400.notes diff --git a/src/plotman/archive.py b/src/plotman/archive.py index 17a29af0..c5b389b7 100644 --- a/src/plotman/archive.py +++ b/src/plotman/archive.py @@ -11,10 +11,47 @@ import psutil import texttable as tt -from plotman import manager, plot_util +from plotman import job, manager, plot_util # TODO : write-protect and delete-protect archived plots +def spawn_archive_process(dir_cfg, all_jobs): + '''Spawns a new archive process using the command created + in the archive() function. Returns archiving status and a log message to print.''' + + log_message = None + archiving_status = None + + # Look for running archive jobs. Be robust to finding more than one + # even though the scheduler should only run one at a time. + arch_jobs = get_running_archive_jobs(dir_cfg.archive) + + if not arch_jobs: + (should_start, status_or_cmd) = archive(dir_cfg, all_jobs) + if not should_start: + archiving_status = status_or_cmd + else: + cmd = status_or_cmd + # TODO: do something useful with output instead of DEVNULL + p = subprocess.Popen(cmd, + shell=True, + stdout=subprocess.DEVNULL, + stderr=subprocess.STDOUT, + start_new_session=True) + log_message = 'Starting archive: ' + cmd + # At least for now it seems that even if we get a new running + # archive jobs list it doesn't contain the new rsync process. + # My guess is that this is because the bash in the middle due to + # shell=True is still starting up and really hasn't launched the + # new rsync process yet. So, just put a placeholder here. It + # will get filled on the next cycle. + arch_jobs.append('') + + if archiving_status is None: + archiving_status = 'pid: ' + ', '.join(map(str, arch_jobs)) + + return archiving_status, log_message + def compute_priority(phase, gb_free, n_plots): # All these values are designed around dst buffer dirs of about # ~2TB size and containing k32 plots. TODO: Generalize, and @@ -25,14 +62,14 @@ def compute_priority(phase, gb_free, n_plots): # To avoid concurrent IO, we should not touch drives that # are about to receive a new plot. If we don't know the phase, # ignore. - if (phase[0] and phase[1]): - if (phase == (3, 4)): + if (phase.known): + if (phase == job.Phase(3, 4)): priority -= 4 - elif (phase == (3, 5)): + elif (phase == job.Phase(3, 5)): priority -= 8 - elif (phase == (3, 6)): + elif (phase == job.Phase(3, 6)): priority -= 16 - elif (phase >= (3, 7)): + elif (phase >= job.Phase(3, 7)): priority -= 32 # If a drive is getting full, we should prioritize it @@ -58,7 +95,7 @@ def get_archdir_freebytes(arch_cfg): # not actually mounted continue freebytes = int(fields[3][:-1]) * 1024 # Strip the final 'K' - archdir = (fields[5]).decode('ascii') + archdir = (fields[5]).decode('utf-8') archdir_freebytes[archdir] = freebytes return archdir_freebytes @@ -98,7 +135,7 @@ def archive(dir_cfg, all_jobs): chosen_plot = None for d in dir_cfg.dst: - ph = dir2ph.get(d, (0, 0)) + ph = dir2ph.get(d, job.Phase(0, 0)) dir_plots = plot_util.list_k32_plots(d) gb_free = plot_util.df_b(d) / plot_util.GB n_plots = len(dir_plots) @@ -134,7 +171,7 @@ def archive(dir_cfg, all_jobs): bwlimit = dir_cfg.archive.rsyncd_bwlimit throttle_arg = ('--bwlimit=%d' % bwlimit) if bwlimit else '' - cmd = ('rsync %s --remove-source-files -P %s %s' % + cmd = ('rsync %s --no-compress --remove-source-files -P %s %s' % (throttle_arg, chosen_plot, rsync_dest(dir_cfg.archive, archdir))) return (True, cmd) diff --git a/src/plotman/chia.py b/src/plotman/chia.py new file mode 100644 index 00000000..a9e4f9ef --- /dev/null +++ b/src/plotman/chia.py @@ -0,0 +1,140 @@ +import functools + +import click +from pathlib import Path + + +class Commands: + def __init__(self): + self.by_version = {} + + def register(self, version): + if version in self.by_version: + raise Exception(f'Version already registered: {version!r}') + if not isinstance(version, tuple): + raise Exception(f'Version must be a tuple: {version!r}') + + return functools.partial(self._decorator, version=version) + + def _decorator(self, command, *, version): + self.by_version[version] = command + # self.by_version = dict(sorted(self.by_version.items())) + + def __getitem__(self, item): + return self.by_version[item] + + def latest_command(self): + return max(self.by_version.items())[1] + + +commands = Commands() + + +@commands.register(version=(1, 1, 2)) +@click.command() +# https://github.com/Chia-Network/chia-blockchain/blob/v1.1.2/LICENSE +# https://github.com/Chia-Network/chia-blockchain/blob/v1.1.2/chia/cmds/plots.py#L39-L83 +# start copied code +@click.option("-k", "--size", help="Plot size", type=int, default=32, show_default=True) +@click.option("--override-k", help="Force size smaller than 32", default=False, show_default=True, is_flag=True) +@click.option("-n", "--num", help="Number of plots or challenges", type=int, default=1, show_default=True) +@click.option("-b", "--buffer", help="Megabytes for sort/plot buffer", type=int, default=4608, show_default=True) +@click.option("-r", "--num_threads", help="Number of threads to use", type=int, default=2, show_default=True) +@click.option("-u", "--buckets", help="Number of buckets", type=int, default=128, show_default=True) +@click.option( + "-a", + "--alt_fingerprint", + type=int, + default=None, + help="Enter the alternative fingerprint of the key you want to use", +) +@click.option( + "-c", + "--pool_contract_address", + type=str, + default=None, + help="Address of where the pool reward will be sent to. Only used if alt_fingerprint and pool public key are None", +) +@click.option("-f", "--farmer_public_key", help="Hex farmer public key", type=str, default=None) +@click.option("-p", "--pool_public_key", help="Hex public key of pool", type=str, default=None) +@click.option( + "-t", + "--tmp_dir", + help="Temporary directory for plotting files", + type=click.Path(), + default=Path("."), + show_default=True, +) +@click.option("-2", "--tmp2_dir", help="Second temporary directory for plotting files", type=click.Path(), default=None) +@click.option( + "-d", + "--final_dir", + help="Final directory for plots (relative or absolute)", + type=click.Path(), + default=Path("."), + show_default=True, +) +@click.option("-i", "--plotid", help="PlotID in hex for reproducing plots (debugging only)", type=str, default=None) +@click.option("-m", "--memo", help="Memo in hex for reproducing plots (debugging only)", type=str, default=None) +@click.option("-e", "--nobitfield", help="Disable bitfield", default=False, is_flag=True) +@click.option( + "-x", "--exclude_final_dir", help="Skips adding [final dir] to harvester for farming", default=False, is_flag=True +) +# end copied code +def _cli(): + pass + + +@commands.register(version=(1, 1, 3)) +@click.command() +# https://github.com/Chia-Network/chia-blockchain/blob/v1.1.3/LICENSE +# https://github.com/Chia-Network/chia-blockchain/blob/v1.1.3/chia/cmds/plots.py#L39-L83 +# start copied code +@click.option("-k", "--size", help="Plot size", type=int, default=32, show_default=True) +@click.option("--override-k", help="Force size smaller than 32", default=False, show_default=True, is_flag=True) +@click.option("-n", "--num", help="Number of plots or challenges", type=int, default=1, show_default=True) +@click.option("-b", "--buffer", help="Megabytes for sort/plot buffer", type=int, default=4608, show_default=True) +@click.option("-r", "--num_threads", help="Number of threads to use", type=int, default=2, show_default=True) +@click.option("-u", "--buckets", help="Number of buckets", type=int, default=128, show_default=True) +@click.option( + "-a", + "--alt_fingerprint", + type=int, + default=None, + help="Enter the alternative fingerprint of the key you want to use", +) +@click.option( + "-c", + "--pool_contract_address", + type=str, + default=None, + help="Address of where the pool reward will be sent to. Only used if alt_fingerprint and pool public key are None", +) +@click.option("-f", "--farmer_public_key", help="Hex farmer public key", type=str, default=None) +@click.option("-p", "--pool_public_key", help="Hex public key of pool", type=str, default=None) +@click.option( + "-t", + "--tmp_dir", + help="Temporary directory for plotting files", + type=click.Path(), + default=Path("."), + show_default=True, +) +@click.option("-2", "--tmp2_dir", help="Second temporary directory for plotting files", type=click.Path(), default=None) +@click.option( + "-d", + "--final_dir", + help="Final directory for plots (relative or absolute)", + type=click.Path(), + default=Path("."), + show_default=True, +) +@click.option("-i", "--plotid", help="PlotID in hex for reproducing plots (debugging only)", type=str, default=None) +@click.option("-m", "--memo", help="Memo in hex for reproducing plots (debugging only)", type=str, default=None) +@click.option("-e", "--nobitfield", help="Disable bitfield", default=False, is_flag=True) +@click.option( + "-x", "--exclude_final_dir", help="Skips adding [final dir] to harvester for farming", default=False, is_flag=True +) +# end copied code +def _cli(): + pass diff --git a/src/plotman/configuration.py b/src/plotman/configuration.py index 44b12c43..2434456e 100644 --- a/src/plotman/configuration.py +++ b/src/plotman/configuration.py @@ -1,7 +1,8 @@ -from dataclasses import dataclass +import contextlib from typing import Dict, List, Optional import appdirs +import attr import desert import marshmallow import yaml @@ -16,29 +17,38 @@ def get_path(): return appdirs.user_config_dir("plotman") + "/plotman.yaml" -def get_validated_configs(): - """Return a validated instance of the PlotmanConfig dataclass with data from plotman.yaml +def read_configuration_text(config_path): + try: + with open(config_path, "r") as file: + return file.read() + except FileNotFoundError as e: + raise ConfigurationException( + f"No 'plotman.yaml' file exists at expected location: '{config_path}'. To generate " + f"default config file, run: 'plotman config generate'" + ) from e + + +def get_validated_configs(config_text, config_path): + """Return a validated instance of PlotmanConfig with data from plotman.yaml :raises ConfigurationException: Raised when plotman.yaml is either missing or malformed """ schema = desert.schema(PlotmanConfig) - config_file_path = get_path() + config_objects = yaml.load(config_text, Loader=yaml.SafeLoader) + try: - with open(config_file_path, "r") as file: - config_file = yaml.load(file, Loader=yaml.SafeLoader) - return schema.load(config_file) - except FileNotFoundError as e: + loaded = schema.load(config_objects) + except marshmallow.exceptions.ValidationError as e: raise ConfigurationException( - f"No 'plotman.yaml' file exists at expected location: '{config_file_path}'. To generate " - f"default config file, run: 'plotman config generate'" + f"Config file at: '{config_path}' is malformed" ) from e - except marshmallow.exceptions.ValidationError as e: - raise ConfigurationException(f"Config file at: '{config_file_path}' is malformed") from e + + return loaded # Data models used to deserializing/formatting plotman.yaml files. -@dataclass +@attr.frozen class Archive: rsyncd_module: str rsyncd_path: str @@ -47,11 +57,11 @@ class Archive: rsyncd_user: str index: int = 0 # If not explicit, "index" will default to 0 -@dataclass +@attr.frozen class TmpOverrides: tmpdir_max_jobs: Optional[int] = None -@dataclass +@attr.frozen class Directories: log: str tmp: List[str] @@ -60,7 +70,7 @@ class Directories: tmp_overrides: Optional[Dict[str, TmpOverrides]] = None archive: Optional[Archive] = None -@dataclass +@attr.frozen class Scheduling: global_max_jobs: int global_stagger_m: int @@ -70,7 +80,7 @@ class Scheduling: tmpdir_stagger_phase_minor: int tmpdir_stagger_phase_limit: int = 1 # If not explicit, "tmpdir_stagger_phase_limit" will default to 1 -@dataclass +@attr.frozen class Plotting: k: int e: bool @@ -80,13 +90,13 @@ class Plotting: farmer_pk: Optional[str] = None pool_pk: Optional[str] = None -@dataclass +@attr.frozen class UserInterface: - use_stty_size: bool + use_stty_size: bool = True -@dataclass +@attr.frozen class PlotmanConfig: - user_interface: UserInterface directories: Directories scheduling: Scheduling plotting: Plotting + user_interface: UserInterface = attr.ib(factory=UserInterface) diff --git a/src/plotman/interactive.py b/src/plotman/interactive.py index 321aa14c..5835b18d 100644 --- a/src/plotman/interactive.py +++ b/src/plotman/interactive.py @@ -4,12 +4,14 @@ import math import os import subprocess -import threading from plotman import archive, configuration, manager, reporting from plotman.job import Job +class TerminalTooSmallError(Exception): + pass + class Log: def __init__(self): self.entries = [] @@ -63,7 +65,9 @@ def archiving_status_msg(configured, active, status): def curses_main(stdscr): log = Log() - cfg = configuration.get_validated_configs() + config_path = configuration.get_path() + config_text = configuration.read_configuration_text(config_path) + cfg = configuration.get_validated_configs(config_text, config_path) plotting_active = True archiving_configured = cfg.directories.archive is not None @@ -87,6 +91,7 @@ def curses_main(stdscr): pressed_key = '' # For debugging archdir_freebytes = None + aging_reason = None while True: @@ -113,33 +118,23 @@ def curses_main(stdscr): cfg.directories, cfg.scheduling, cfg.plotting ) if (started): + if aging_reason is not None: + log.log(aging_reason) + aging_reason = None log.log(msg) plotting_status = '' jobs = Job.get_running_jobs(cfg.directories.log, cached_jobs=jobs) else: + # If a plot is delayed for any reason other than stagger, log it + if msg.find("stagger") < 0: + aging_reason = msg plotting_status = msg if archiving_configured: if archiving_active: - # Look for running archive jobs. Be robust to finding more than one - # even though the scheduler should only run one at a time. - arch_jobs = archive.get_running_archive_jobs(cfg.directories.archive) - if arch_jobs: - archiving_status = 'pid: ' + ', '.join(map(str, arch_jobs)) - else: - (should_start, status_or_cmd) = archive.archive(cfg.directories, jobs) - if not should_start: - archiving_status = status_or_cmd - else: - cmd = status_or_cmd - log.log('Starting archive: ' + cmd) - - # TODO: do something useful with output instead of DEVNULL - p = subprocess.Popen(cmd, - shell=True, - stdout=subprocess.DEVNULL, - stderr=subprocess.STDOUT, - start_new_session=True) + archiving_status, log_message = archive.spawn_archive_process(cfg.directories, jobs) + if log_message: + log.log(log_message) archdir_freebytes = archive.get_archdir_freebytes(cfg.directories.archive) @@ -178,13 +173,10 @@ def curses_main(stdscr): arch_prefix = cfg.directories.archive.rsyncd_path n_tmpdirs = len(cfg.directories.tmp) - n_tmpdirs_half = int(n_tmpdirs / 2) # Directory reports. - tmp_report_1 = reporting.tmp_dir_report( - jobs, cfg.directories, cfg.scheduling, n_cols, 0, n_tmpdirs_half, tmp_prefix) - tmp_report_2 = reporting.tmp_dir_report( - jobs, cfg.directories, cfg.scheduling, n_cols, n_tmpdirs_half, n_tmpdirs, tmp_prefix) + tmp_report = reporting.tmp_dir_report( + jobs, cfg.directories, cfg.scheduling, n_cols, 0, n_tmpdirs, tmp_prefix) dst_report = reporting.dst_dir_report( jobs, cfg.directories.dst, n_cols, dst_prefix) if archiving_configured: @@ -198,10 +190,8 @@ def curses_main(stdscr): # Layout # - tmp_h = max(len(tmp_report_1.splitlines()), - len(tmp_report_2.splitlines())) - tmp_w = len(max(tmp_report_1.splitlines() + - tmp_report_2.splitlines(), key=len)) + 1 + tmp_h = len(tmp_report.splitlines()) + tmp_w = len(max(tmp_report.splitlines(), key=len)) + 1 dst_h = len(dst_report.splitlines()) dst_w = len(max(dst_report.splitlines(), key=len)) + 1 arch_h = len(arch_report.splitlines()) + 1 @@ -273,28 +263,19 @@ def curses_main(stdscr): jobs_win.chgat(0, 0, curses.A_REVERSE) # Dirs - tmpwin_12_gutter = 3 tmpwin_dstwin_gutter = 6 maxtd_h = max([tmp_h, dst_h]) - tmpwin_1 = curses.newwin( - tmp_h, tmp_w, - dirs_pos + int((maxtd_h - tmp_h) / 2), 0) - tmpwin_1.addstr(tmp_report_1) - - tmpwin_2 = curses.newwin( + tmpwin = curses.newwin( tmp_h, tmp_w, - dirs_pos + int((maxtd_h - tmp_h) / 2), - tmp_w + tmpwin_12_gutter) - tmpwin_2.addstr(tmp_report_2) - - tmpwin_1.chgat(0, 0, curses.A_REVERSE) - tmpwin_2.chgat(0, 0, curses.A_REVERSE) + dirs_pos + int(maxtd_h - tmp_h), 0) + tmpwin.addstr(tmp_report) + tmpwin.chgat(0, 0, curses.A_REVERSE) dstwin = curses.newwin( dst_h, dst_w, - dirs_pos + int((maxtd_h - dst_h) / 2), 2 * tmp_w + tmpwin_12_gutter + tmpwin_dstwin_gutter) + dirs_pos + int((maxtd_h - dst_h) / 2), tmp_w + tmpwin_dstwin_gutter) dstwin.addstr(dst_report) dstwin.chgat(0, 0, curses.A_REVERSE) @@ -312,8 +293,7 @@ def curses_main(stdscr): stdscr.noutrefresh() header_win.noutrefresh() jobs_win.noutrefresh() - tmpwin_1.noutrefresh() - tmpwin_2.noutrefresh() + tmpwin.noutrefresh() dstwin.noutrefresh() archwin.noutrefresh() log_win.noutrefresh() @@ -350,4 +330,9 @@ def run_interactive(): code = locale.getpreferredencoding() # Then use code as the encoding for str.encode() calls. - curses.wrapper(curses_main) + try: + curses.wrapper(curses_main) + except curses.error as e: + raise TerminalTooSmallError( + "Your terminal may be too small, try making it bigger.", + ) from e diff --git a/src/plotman/job.py b/src/plotman/job.py index 6314d168..e25218a0 100644 --- a/src/plotman/job.py +++ b/src/plotman/job.py @@ -1,20 +1,24 @@ # TODO do we use all these? import argparse import contextlib +import functools import logging import os import random import re import sys -import threading import time from datetime import datetime from enum import Enum, auto from subprocess import call +import attr +import click import pendulum import psutil +from plotman import chia + def job_phases_for_tmpdir(d, all_jobs): '''Return phase 2-tuples for jobs running on tmpdir d''' @@ -25,56 +29,106 @@ def job_phases_for_dstdir(d, all_jobs): return sorted([j.progress() for j in all_jobs if j.dstdir == d]) def is_plotting_cmdline(cmdline): + if cmdline and 'python' in cmdline[0].lower(): + cmdline = cmdline[1:] return ( - len(cmdline) >= 4 - and 'python' in cmdline[0] - and cmdline[1].endswith('/chia') - and 'plots' == cmdline[2] - and 'create' == cmdline[3] + len(cmdline) >= 3 + and cmdline[0].endswith("chia") + and 'plots' == cmdline[1] + and 'create' == cmdline[2] ) -# This is a cmdline argument fix for https://github.com/ericaltendorf/plotman/issues/41 -def cmdline_argfix(cmdline): - known_keys = 'krbut2dne' - for i in cmdline: - # If the argument starts with dash and a known key and is longer than 2, - # then an argument is passed with no space between its key and value. - # This is POSIX compliant but the arg parser was tripping over it. - # In these cases, splitting that item up in separate key and value - # elements results in a `cmdline` list that is correctly formatted. - if i[0]=='-' and i[1] in known_keys and len(i)>2: - yield i[0:2] # key - yield i[2:] # value - else: - yield i - def parse_chia_plot_time(s): # This will grow to try ISO8601 as well for when Chia logs that way return pendulum.from_format(s, 'ddd MMM DD HH:mm:ss YYYY', locale='en', tz=None) +def parse_chia_plots_create_command_line(command_line): + command_line = list(command_line) + # Parse command line args + if 'python' in command_line[0].lower(): + command_line = command_line[1:] + assert len(command_line) >= 3 + assert 'chia' in command_line[0] + assert 'plots' == command_line[1] + assert 'create' == command_line[2] + + all_command_arguments = command_line[3:] + + # nice idea, but this doesn't include -h + # help_option_names = command.get_help_option_names(ctx=context) + help_option_names = {'--help', '-h'} + + command_arguments = [ + argument + for argument in all_command_arguments + if argument not in help_option_names + ] + + # TODO: We could at some point do chia version detection and pick the + # associated command. For now we'll just use the latest one we have + # copied. + command = chia.commands.latest_command() + try: + context = command.make_context(info_name='', args=list(command_arguments)) + except click.ClickException as e: + error = e + params = {} + else: + error = None + params = context.params + + return ParsedChiaPlotsCreateCommand( + error=error, + help=len(all_command_arguments) > len(command_arguments), + parameters=params, + ) + +class ParsedChiaPlotsCreateCommand: + def __init__(self, error, help, parameters): + self.error = error + self.help = help + self.parameters = parameters + +@functools.total_ordering +@attr.frozen(order=False) +class Phase: + major: int = 0 + minor: int = 0 + known: bool = True + + def __lt__(self, other): + return ( + (not self.known, self.major, self.minor) + < (not other.known, other.major, other.minor) + ) + + @classmethod + def from_tuple(cls, t): + if len(t) != 2: + raise Exception(f'phase must be created from 2-tuple: {t!r}') + + if None in t and not t[0] is t[1]: + raise Exception(f'phase can not be partially known: {t!r}') + + if t[0] is None: + return cls(known=False) + + return cls(major=t[0], minor=t[1]) + + @classmethod + def list_from_tuples(cls, l): + return [cls.from_tuple(t) for t in l] + # TODO: be more principled and explicit about what we cache vs. what we look up # dynamically from the logfile class Job: 'Represents a plotter job' - # These are constants, not updated during a run. - k = 0 - r = 0 - u = 0 - b = 0 - n = 0 # probably not used - tmpdir = '' - tmp2dir = '' - dstdir = '' logfile = '' jobfile = '' job_id = 0 plot_id = '--------' proc = None # will get a psutil.Process - help = False - - # These are dynamic, cached, and need to be udpated periodically - phase = (None, None) # Phase/subphase def get_running_jobs(logroot, cached_jobs=()): '''Return a list of running plot jobs. If a cache of preexisting jobs is provided, @@ -91,72 +145,87 @@ def get_running_jobs(logroot, cached_jobs=()): if proc.pid in cached_jobs_by_pid.keys(): jobs.append(cached_jobs_by_pid[proc.pid]) # Copy from cache else: - job = Job(proc, logroot) - if not job.help: + with proc.oneshot(): + parsed_command = parse_chia_plots_create_command_line( + command_line=proc.cmdline(), + ) + if parsed_command.error is not None: + continue + job = Job( + proc=proc, + parsed_command=parsed_command, + logroot=logroot, + ) + if job.help: + continue jobs.append(job) return jobs - - def __init__(self, proc, logroot): + + def __init__(self, proc, parsed_command, logroot): '''Initialize from an existing psutil.Process object. must know logroot in order to understand open files''' self.proc = proc - - with self.proc.oneshot(): - # Parse command line args - args = self.proc.cmdline() - assert len(args) > 4 - assert 'python' in args[0] - assert 'chia' in args[1] - assert 'plots' == args[2] - assert 'create' == args[3] - args_iter = iter(cmdline_argfix(args[4:])) - for arg in args_iter: - val = None if arg in {'-e', '--nobitfield', '-h', '--help', '--override-k'} else next(args_iter) - if arg in {'-k', '--size'}: - self.k = val - elif arg in {'-r', '--num_threads'}: - self.r = val - elif arg in {'-b', '--buffer'}: - self.b = val - elif arg in {'-u', '--buckets'}: - self.u = val - elif arg in {'-t', '--tmp_dir'}: - self.tmpdir = val - elif arg in {'-2', '--tmp2_dir'}: - self.tmp2dir = val - elif arg in {'-d', '--final_dir'}: - self.dstdir = val - elif arg in {'-n', '--num'}: - self.n = val - elif arg in {'-h', '--help'}: - self.help = True - elif arg in {'-e', '--nobitfield', '-f', '--farmer_public_key', '-p', '--pool_public_key'}: - pass - # TODO: keep track of these - elif arg == '--override-k': - pass + # These are dynamic, cached, and need to be udpated periodically + self.phase = Phase(known=False) + + self.help = parsed_command.help + self.args = parsed_command.parameters + + # an example as of 1.0.5 + # { + # 'size': 32, + # 'num_threads': 4, + # 'buckets': 128, + # 'buffer': 6000, + # 'tmp_dir': '/farm/yards/901', + # 'final_dir': '/farm/wagons/801', + # 'override_k': False, + # 'num': 1, + # 'alt_fingerprint': None, + # 'pool_contract_address': None, + # 'farmer_public_key': None, + # 'pool_public_key': None, + # 'tmp2_dir': None, + # 'plotid': None, + # 'memo': None, + # 'nobitfield': False, + # 'exclude_final_dir': False, + # } + + self.k = self.args['size'] + self.r = self.args['num_threads'] + self.u = self.args['buckets'] + self.b = self.args['buffer'] + self.n = self.args['num'] + self.tmpdir = self.args['tmp_dir'] + self.tmp2dir = self.args['tmp2_dir'] + self.dstdir = self.args['final_dir'] + + plot_cwd = self.proc.cwd() + self.tmpdir = os.path.join(plot_cwd, self.tmpdir) + if self.tmp2dir is not None: + self.tmp2dir = os.path.join(plot_cwd, self.tmp2dir) + self.dstdir = os.path.join(plot_cwd, self.dstdir) + + # Find logfile (whatever file is open under the log root). The + # file may be open more than once, e.g. for STDOUT and STDERR. + for f in self.proc.open_files(): + if logroot in f.path: + if self.logfile: + assert self.logfile == f.path else: - print('Warning: unrecognized args: %s %s' % (arg, val)) + self.logfile = f.path + break - # Find logfile (whatever file is open under the log root). The - # file may be open more than once, e.g. for STDOUT and STDERR. + if self.logfile: + # Initialize data that needs to be loaded from the logfile + self.init_from_logfile() + else: + print('Found plotting process PID {pid}, but could not find ' + 'logfile in its open files:'.format(pid = self.proc.pid)) for f in self.proc.open_files(): - if logroot in f.path: - if self.logfile: - assert self.logfile == f.path - else: - self.logfile = f.path - break - - if self.logfile: - # Initialize data that needs to be loaded from the logfile - self.init_from_logfile() - else: - print('Found plotting process PID {pid}, but could not find ' - 'logfile in its open files:'.format(pid = self.proc.pid)) - for f in self.proc.open_files(): - print(f.path) + print(f.path) @@ -247,9 +316,9 @@ def set_phase_from_logfile(self): if phase_subphases: phase = max(phase_subphases.keys()) - self.phase = (phase, phase_subphases[phase]) + self.phase = Phase(major=phase, minor=phase_subphases[phase]) else: - self.phase = (0, 0) + self.phase = Phase(major=0, minor=0) def progress(self): '''Return a 2-tuple with the job phase and subphase (by reading the logfile)''' @@ -332,7 +401,11 @@ def get_temp_files(self): # Prevent duplicate file paths by using set. temp_files = set([]) for f in self.proc.open_files(): - if self.tmpdir in f.path or self.tmp2dir in f.path or self.dstdir in f.path: + if any( + dir in f.path + for dir in [self.tmpdir, self.tmp2dir, self.dstdir] + if dir is not None + ): temp_files.add(f.path) return temp_files diff --git a/src/plotman/manager.py b/src/plotman/manager.py index 19a45d63..e0ed2851 100644 --- a/src/plotman/manager.py +++ b/src/plotman/manager.py @@ -6,10 +6,10 @@ import readline # For nice CLI import subprocess import sys -import threading import time from datetime import datetime +import pendulum import psutil # Plotman libraries @@ -37,6 +37,8 @@ def dstdirs_to_youngest_phase(all_jobs): that is emitting to that dst dir.''' result = {} for j in all_jobs: + if j.dstdir is None: + continue if not j.dstdir in result.keys() or result[j.dstdir] > j.progress(): result[j.dstdir] = j.progress() return result @@ -45,12 +47,15 @@ def phases_permit_new_job(phases, d, sched_cfg, dir_cfg): '''Scheduling logic: return True if it's OK to start a new job on a tmp dir with existing jobs in the provided phases.''' # Filter unknown-phase jobs - phases = [ph for ph in phases if ph[0] is not None and ph[1] is not None] + phases = [ph for ph in phases if ph.known] if len(phases) == 0: return True - milestone = (sched_cfg.tmpdir_stagger_phase_major, sched_cfg.tmpdir_stagger_phase_minor) + milestone = job.Phase( + major=sched_cfg.tmpdir_stagger_phase_major, + minor=sched_cfg.tmpdir_stagger_phase_minor, + ) # tmpdir_stagger_phase_limit default is 1, as declared in configuration.py if len([p for p in phases if p < milestone]) >= sched_cfg.tmpdir_stagger_phase_limit: return False @@ -77,23 +82,23 @@ def maybe_start_new_plot(dir_cfg, sched_cfg, plotting_cfg): if (youngest_job_age < global_stagger): wait_reason = 'stagger (%ds/%ds)' % (youngest_job_age, global_stagger) elif len(jobs) >= sched_cfg.global_max_jobs: - wait_reason = 'max jobs (%d)' % sched_cfg.global_max_jobs + wait_reason = 'max jobs (%d) - (%ds/%ds)' % (sched_cfg.global_max_jobs, youngest_job_age, global_stagger) else: tmp_to_all_phases = [(d, job.job_phases_for_tmpdir(d, jobs)) for d in dir_cfg.tmp] eligible = [ (d, phases) for (d, phases) in tmp_to_all_phases if phases_permit_new_job(phases, d, sched_cfg, dir_cfg) ] - rankable = [ (d, phases[0]) if phases else (d, (999, 999)) + rankable = [ (d, phases[0]) if phases else (d, job.Phase(known=False)) for (d, phases) in eligible ] if not eligible: - wait_reason = 'no eligible tempdirs' + wait_reason = 'no eligible tempdirs (%ds/%ds)' % (youngest_job_age, global_stagger) else: # Plot to oldest tmpdir. tmpdir = max(rankable, key=operator.itemgetter(1))[0] # Select the dst dir least recently selected dir2ph = { d:ph for (d, ph) in dstdirs_to_youngest_phase(jobs).items() - if d in dir_cfg.dst } + if d in dir_cfg.dst and ph is not None} unused_dirs = [d for d in dir_cfg.dst if d not in dir2ph.keys()] dstdir = '' if unused_dirs: @@ -102,7 +107,7 @@ def maybe_start_new_plot(dir_cfg, sched_cfg, plotting_cfg): dstdir = max(dir2ph, key=dir2ph.get) logfile = os.path.join( - dir_cfg.log, datetime.now().strftime('%Y-%m-%d-%H:%M:%S.log') + dir_cfg.log, pendulum.now().isoformat(timespec='microseconds').replace(':', '_') + '.log' ) plot_args = ['chia', 'plots', 'create', @@ -126,11 +131,39 @@ def maybe_start_new_plot(dir_cfg, sched_cfg, plotting_cfg): logmsg = ('Starting plot job: %s ; logging to %s' % (' '.join(plot_args), logfile)) - # start_new_sessions to make the job independent of this controlling tty. - p = subprocess.Popen(plot_args, - stdout=open(logfile, 'w'), - stderr=subprocess.STDOUT, - start_new_session=True) + try: + open_log_file = open(logfile, 'x') + except FileExistsError: + # The desired log file name already exists. Most likely another + # plotman process already launched a new process in response to + # the same scenario that triggered us. Let's at least not + # confuse things further by having two plotting processes + # logging to the same file. If we really should launch another + # plotting process, we'll get it at the next check cycle anyways. + message = ( + f'Plot log file already exists, skipping attempt to start a' + f' new plot: {logfile!r}' + ) + return (False, logmsg) + except FileNotFoundError as e: + message = ( + f'Unable to open log file. Verify that the directory exists' + f' and has proper write permissions: {logfile!r}' + ) + raise Exception(message) from e + + # Preferably, do not add any code between the try block above + # and the with block below. IOW, this space intentionally left + # blank... As is, this provides a good chance that our handle + # of the log file will get closed explicitly while still + # allowing handling of just the log file opening error. + + with open_log_file: + # start_new_sessions to make the job independent of this controlling tty. + p = subprocess.Popen(plot_args, + stdout=open_log_file, + stderr=subprocess.STDOUT, + start_new_session=True) psutil.Process(p.pid).nice(15) return (True, logmsg) diff --git a/src/plotman/plot_util.py b/src/plotman/plot_util.py index 59ef0fad..ca24ae08 100644 --- a/src/plotman/plot_util.py +++ b/src/plotman/plot_util.py @@ -51,8 +51,11 @@ def list_k32_plots(d): for plot in os.listdir(d): if re.match(r'^plot-k32-.*plot$', plot): plot = os.path.join(d, plot) - if os.stat(plot).st_size > (0.95 * get_k32_plotsize()): - plots.append(plot) + try: + if os.stat(plot).st_size > (0.95 * get_k32_plotsize()): + plots.append(plot) + except FileNotFoundError: + continue return plots diff --git a/src/plotman/plotman.py b/src/plotman/plotman.py index 77602d69..d3468735 100755 --- a/src/plotman/plotman.py +++ b/src/plotman/plotman.py @@ -132,7 +132,9 @@ def main(): print("No action requested, add 'generate' or 'path'.") return - cfg = configuration.get_validated_configs() + config_path = configuration.get_path() + config_text = configuration.read_configuration_text(config_path) + cfg = configuration.get_validated_configs(config_text, config_path) # # Stay alive, spawning plot jobs @@ -180,7 +182,11 @@ def main(): time.sleep(60) jobs = Job.get_running_jobs(cfg.directories.log) firstit = False - archive.archive(cfg.directories, jobs) + + archiving_status, log_message = archive.spawn_archive_process(cfg.directories, jobs) + if log_message: + print(log_message) + # Debugging: show the destination drive usage schedule elif args.cmd == 'dsched': @@ -202,9 +208,9 @@ def main(): # TODO: allow multiple idprefixes, not just take the first selected = manager.select_jobs_by_partial_id(jobs, args.idprefix[0]) if (len(selected) == 0): - print('Error: %s matched no jobs.' % id_spec) + print('Error: %s matched no jobs.' % args.idprefix[0]) elif len(selected) > 1: - print('Error: "%s" matched multiple jobs:' % id_spec) + print('Error: "%s" matched multiple jobs:' % args.idprefix[0]) for j in selected: print(' %s' % j.plot_id) selected = [] diff --git a/src/plotman/reporting.py b/src/plotman/reporting.py index 80af0b55..8b00a84a 100644 --- a/src/plotman/reporting.py +++ b/src/plotman/reporting.py @@ -13,10 +13,11 @@ def abbr_path(path, putative_prefix): else: return path -def phase_str(phase_pair): - (ph, subph) = phase_pair - return ((str(ph) if ph is not None else '?') + ':' - + (str(subph) if subph is not None else '?')) +def phase_str(phase): + if not phase.known: + return '?:?' + + return f'{phase.major}:{phase.minor}' def phases_str(phases, max_num=None): '''Take a list of phase-subphase pairs and return them as a compact string''' @@ -50,15 +51,15 @@ def job_viz(jobs): result = '' result += '1' for i in range(0, 8): - result += n_to_char(n_at_ph(jobs, (1, i))) + result += n_to_char(n_at_ph(jobs, job.Phase(1, i))) result += '2' for i in range(0, 8): - result += n_to_char(n_at_ph(jobs, (2, i))) + result += n_to_char(n_at_ph(jobs, job.Phase(2, i))) result += '3' for i in range(0, 7): - result += n_to_char(n_at_ph(jobs, (3, i))) + result += n_to_char(n_at_ph(jobs, job.Phase(3, i))) result += '4' - result += n_to_char(n_at_ph(jobs, (4, 0))) + result += n_to_char(n_at_ph(jobs, job.Phase(4, 0))) return result def status_report(jobs, width, height=None, tmp_prefix='', dst_prefix=''): @@ -95,21 +96,22 @@ def status_report(jobs, width, height=None, tmp_prefix='', dst_prefix=''): # Regular row else: try: - row = [j.plot_id[:8], - j.k, - abbr_path(j.tmpdir, tmp_prefix), - abbr_path(j.dstdir, dst_prefix), - plot_util.time_format(j.get_time_wall()), - phase_str(j.progress()), - plot_util.human_format(j.get_tmp_usage(), 0), - j.proc.pid, - j.get_run_status(), - plot_util.human_format(j.get_mem_usage(), 1), - plot_util.time_format(j.get_time_user()), - plot_util.time_format(j.get_time_sys()), - plot_util.time_format(j.get_time_iowait()) - ] - except psutil.NoSuchProcess: + with j.proc.oneshot(): + row = [j.plot_id[:8], + j.k, + abbr_path(j.tmpdir, tmp_prefix), + abbr_path(j.dstdir, dst_prefix), + plot_util.time_format(j.get_time_wall()), + phase_str(j.progress()), + plot_util.human_format(j.get_tmp_usage(), 0), + j.proc.pid, + j.get_run_status(), + plot_util.human_format(j.get_mem_usage(), 1), + plot_util.time_format(j.get_time_user()), + plot_util.time_format(j.get_time_sys()), + plot_util.time_format(j.get_time_iowait()) + ] + except (psutil.NoSuchProcess, psutil.AccessDenied): # In case the job has disappeared row = [j.plot_id[:8]] + (['--'] * 12) @@ -135,7 +137,7 @@ def tmp_dir_report(jobs, dir_cfg, sched_cfg, width, start_row=None, end_row=None continue phases = sorted(job.job_phases_for_tmpdir(d, jobs)) ready = manager.phases_permit_new_job(phases, d, sched_cfg, dir_cfg) - row = [abbr_path(d, prefix), 'OK' if ready else '--', phases_str(phases)] + row = [abbr_path(d, prefix), 'OK' if ready else '--', phases_str(phases, 5)] tab.add_row(row) tab.set_max_width(width) @@ -186,10 +188,14 @@ def arch_dir_report(archdir_freebytes, width, prefix=''): # TODO: remove this def dirs_report(jobs, dir_cfg, sched_cfg, width): - return ( - tmp_dir_report(jobs, dir_cfg, sched_cfg, width) + '\n' + - dst_dir_report(jobs, dir_cfg.dst, width) + '\n' + - 'archive dirs free space:\n' + - arch_dir_report(archive.get_archdir_freebytes(dir_cfg.archive), width) + '\n' - ) - + reports = [ + tmp_dir_report(jobs, dir_cfg, sched_cfg, width), + dst_dir_report(jobs, dir_cfg.dst, width), + ] + if dir_cfg.archive is not None: + reports.extend([ + 'archive dirs free space:', + arch_dir_report(archive.get_archdir_freebytes(dir_cfg.archive), width), + ]) + + return '\n'.join(reports) + '\n' diff --git a/src/plotman/resources/plotman.yaml b/src/plotman/resources/plotman.yaml index c972af60..3e672eba 100644 --- a/src/plotman/resources/plotman.yaml +++ b/src/plotman/resources/plotman.yaml @@ -59,11 +59,15 @@ directories: # archiving operation, comment this section out. # # Currently archival depends on an rsync daemon running on the remote - # host, and that the module is configured to match the local path. - # See https://github.com/ericaltendorf/plotman/wiki/Archiving for details. + # host. + # The archival also uses ssh to connect to the remote host and check + # for available directories. Set up ssh keys on the remote host to + # allow public key login from rsyncd_user. + # Complete example: https://github.com/ericaltendorf/plotman/wiki/Archiving archive: - rsyncd_module: plots - rsyncd_path: /plots + rsyncd_module: plots # Define this in remote rsyncd.conf. + rsyncd_path: /plots # This is used via ssh. Should match path + # defined in the module referenced above. rsyncd_bwlimit: 80000 # Bandwidth limit in KB/s rsyncd_host: myfarmer rsyncd_user: chia @@ -81,11 +85,14 @@ directories: # Plotting scheduling parameters scheduling: # Run a job on a particular temp dir only if the number of existing jobs - # before tmpdir_stagger_phase_major tmpdir_stagger_phase_minor + # before [tmpdir_stagger_phase_major : tmpdir_stagger_phase_minor] # is less than tmpdir_stagger_phase_limit. # Phase major corresponds to the plot phase, phase minor corresponds to # the table or table pair in sequence, phase limit corresponds to - # the number of plots allowed before [phase major, phase minor] + # the number of plots allowed before [phase major : phase minor]. + # e.g, with default settings, a new plot will start only when your plot + # reaches phase [2 : 1] on your temp drive. This setting takes precidence + # over global_stagger_m tmpdir_stagger_phase_major: 2 tmpdir_stagger_phase_minor: 1 # Optional: default is 1 @@ -112,7 +119,7 @@ plotting: e: False # Use -e plotting option n_threads: 2 # Threads per job n_buckets: 128 # Number of buckets to split data into - job_buffer: 4608 # Per job memory + job_buffer: 3389 # Per job memory # If specified, pass through to the -f and -p options. See CLI reference. # farmer_pk: ... # pool_pk: ... diff --git a/tox.ini b/tox.ini index bd7e875e..74aec0d5 100644 --- a/tox.ini +++ b/tox.ini @@ -3,15 +3,27 @@ envlist = test-py{37,38,39} [testenv] changedir = {envtmpdir} +setenv = + COVERAGE_FILE={toxinidir}/.coverage [testenv:test-py{37,38,39}] extras = test commands = - pytest --capture=no --verbose --pyargs plotman + pytest --capture=no --verbose --cov=plotman --cov-report=term-missing --cov-report=xml:{toxinidir}/coverage.xml --pyargs plotman [testenv:check] extras = test commands = check-manifest --verbose {toxinidir} + +[testenv:check-coverage] +changedir = {toxinidir} +extras = + coverage +commands = + coverage combine coverage_reports/ + coverage xml -o coverage.xml + coverage report --fail-under=35 --ignore-errors --show-missing + diff-cover --fail-under=100 {posargs:--compare-branch=development} coverage.xml