From df029028ef40df2d964bad17141693fb048907f9 Mon Sep 17 00:00:00 2001 From: Guy Davis Date: Tue, 7 Sep 2021 09:14:34 -0600 Subject: [PATCH 01/17] Disable tty usage to prevent blocking on calls from other processes --- src/plotman/plotman.py | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/src/plotman/plotman.py b/src/plotman/plotman.py index fc0fa68a..bbfb1db2 100755 --- a/src/plotman/plotman.py +++ b/src/plotman/plotman.py @@ -162,12 +162,7 @@ def parse_args(self) -> typing.Any: def get_term_width() -> int: - try: - (rows_string, columns_string) = os.popen("stty size", "r").read().split() - columns = int(columns_string) - except: - columns = 120 # 80 is typically too narrow. TODO: make a command line arg. - return columns + return 120 # Disable tty usage to prevent blocking on calls from other processes class Iso8601Formatter(logging.Formatter): From 146956874ea823dd307485ac2860ba7135e148a1 Mon Sep 17 00:00:00 2001 From: Guy Davis Date: Tue, 7 Sep 2021 10:05:31 -0600 Subject: [PATCH 02/17] Use use_stty_size setting for analzye too. --- src/plotman/analyzer.py | 3 +-- src/plotman/plotman.py | 19 ++++++++++++++----- 2 files changed, 15 insertions(+), 7 deletions(-) diff --git a/src/plotman/analyzer.py b/src/plotman/analyzer.py index a083de16..ee26ced1 100644 --- a/src/plotman/analyzer.py +++ b/src/plotman/analyzer.py @@ -10,7 +10,7 @@ def analyze( - logfilenames: typing.List[str], clipterminals: bool, bytmp: bool, bybitfield: bool + logfilenames: typing.List[str], clipterminals: bool, bytmp: bool, bybitfield: bool, columns: int ) -> None: data: typing.Dict[str, typing.Dict[str, typing.List[float]]] = {} for logfilename in logfilenames: @@ -165,7 +165,6 @@ def analyze( tab.add_row(row) - (rows, columns) = os.popen("stty size", "r").read().split() tab.set_max_width(int(columns)) s = tab.draw() print(s) diff --git a/src/plotman/plotman.py b/src/plotman/plotman.py index bbfb1db2..8c45ea2f 100755 --- a/src/plotman/plotman.py +++ b/src/plotman/plotman.py @@ -161,8 +161,17 @@ def parse_args(self) -> typing.Any: return args -def get_term_width() -> int: - return 120 # Disable tty usage to prevent blocking on calls from other processes +def get_term_width(cfg) -> int: + default_columns = 120 # 80 is typically too narrow. + if cfg.user_interface.use_stty_size: + try: + (rows_string, columns_string) = os.popen("stty size", "r").read().split() + columns = int(columns_string) + except: + columns = default_columns + else: + columns = default_columns + return columns class Iso8601Formatter(logging.Formatter): @@ -288,7 +297,7 @@ def main() -> None: elif args.cmd == "analyze": analyzer.analyze( - args.logfile, args.clipterminals, args.bytmp, args.bybitfield + args.logfile, args.clipterminals, args.bytmp, args.bybitfield, get_term_width(cfg) ) # @@ -312,7 +321,7 @@ def main() -> None: result = reporting.json_report(jobs) else: result = "{0}\n\n{1}\n\nUpdated at: {2}".format( - reporting.status_report(jobs, get_term_width()), + reporting.status_report(jobs, get_term_width(cfg)), reporting.summary(jobs), datetime.datetime.today().strftime("%c"), ) @@ -330,7 +339,7 @@ def main() -> None: cfg.directories, cfg.archiving, cfg.scheduling, - get_term_width(), + get_term_width(cfg), ) ) From 00c8dd6c78835813eeed848715a1d3d5f270a182 Mon Sep 17 00:00:00 2001 From: Guy Davis Date: Sun, 12 Sep 2021 07:25:40 -0600 Subject: [PATCH 03/17] Correct typing hopefully. --- src/plotman/plotman.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/plotman/plotman.py b/src/plotman/plotman.py index 8c45ea2f..1537d4eb 100755 --- a/src/plotman/plotman.py +++ b/src/plotman/plotman.py @@ -161,7 +161,7 @@ def parse_args(self) -> typing.Any: return args -def get_term_width(cfg) -> int: +def get_term_width(cfg: configuration.PlotmanConfig) -> int: default_columns = 120 # 80 is typically too narrow. if cfg.user_interface.use_stty_size: try: From 5e8724b87b0302ff982e713786a9424cf6aa3b2b Mon Sep 17 00:00:00 2001 From: Guy Davis Date: Sun, 12 Sep 2021 07:33:26 -0600 Subject: [PATCH 04/17] Typing attempt failed Github checks, reverting. --- src/plotman/plotman.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/plotman/plotman.py b/src/plotman/plotman.py index 1537d4eb..8c45ea2f 100755 --- a/src/plotman/plotman.py +++ b/src/plotman/plotman.py @@ -161,7 +161,7 @@ def parse_args(self) -> typing.Any: return args -def get_term_width(cfg: configuration.PlotmanConfig) -> int: +def get_term_width(cfg) -> int: default_columns = 120 # 80 is typically too narrow. if cfg.user_interface.use_stty_size: try: From e634fcbb61fb939515bf63f5f9f7d9a9deee7991 Mon Sep 17 00:00:00 2001 From: Kyle Altendorf Date: Mon, 13 Sep 2021 08:25:03 -0400 Subject: [PATCH 05/17] +dev and unreleased changelog sections --- CHANGELOG.md | 4 ++++ VERSION | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index cbc4cb13..735d135f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,10 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [unreleased] +### Fixed +### Added + ## [0.5.2] - 2021-09-12 ### Fixed - Temp files are correctly identified for cleanup. diff --git a/VERSION b/VERSION index cb0c939a..decc4e86 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.5.2 +0.5.2+dev From 9ddb29217f08d732c57aab01b39ab28ccc964d27 Mon Sep 17 00:00:00 2001 From: Kyle Altendorf Date: Mon, 13 Sep 2021 12:06:53 -0400 Subject: [PATCH 06/17] handle the case when no log file is found --- src/plotman/job.py | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/src/plotman/job.py b/src/plotman/job.py index 198ff674..1dc1444f 100644 --- a/src/plotman/job.py +++ b/src/plotman/job.py @@ -4,6 +4,7 @@ import glob import time from datetime import datetime +import sys import typing import attr @@ -105,7 +106,7 @@ class Job: plotter: "plotman.plotters.Plotter" - logfile: str = "" + logfile: typing.Optional[str] = None job_id: int = 0 proc: psutil.Process @@ -189,10 +190,11 @@ def get_running_jobs( plotter=plotter, logroot=logroot, ) - # TODO: stop reloading every time... - with open(job.logfile, "rb") as f: - r = f.read() - job.plotter.update(chunk=r) + if job.logfile is not None: + # TODO: stop reloading every time... + with open(job.logfile, "rb") as f: + r = f.read() + job.plotter.update(chunk=r) jobs.append(job) return jobs @@ -254,6 +256,10 @@ def status_str_long(self) -> str: # ) def print_logs(self, follow: bool = False) -> None: + if self.logfile is None: + print("no log file available for this plotting process", file=sys.stderr) + return + with open(self.logfile, "r") as f: if follow: line = "" From 478b8014d7782b73f7a7e5b891a78df26c24d2d9 Mon Sep 17 00:00:00 2001 From: Kyle Altendorf Date: Mon, 13 Sep 2021 12:19:14 -0400 Subject: [PATCH 07/17] Correct a changelog link --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 735d135f..917ce4e6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,7 +12,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [0.5.2] - 2021-09-12 ### Fixed - Temp files are correctly identified for cleanup. - ([#912](https://github.com/ericaltendorf/plotman/pull/913)) + ([#912](https://github.com/ericaltendorf/plotman/pull/912)) - Correct where trailing `/` on dst directories resulted in them being considered unused. ([#920](https://github.com/ericaltendorf/plotman/pull/920)) ### Added From cf81e6c6831340365aba965f5f072a73de779c75 Mon Sep 17 00:00:00 2001 From: Kyle Altendorf Date: Mon, 13 Sep 2021 12:36:38 -0400 Subject: [PATCH 08/17] changelog for 926 --- CHANGELOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 735d135f..6739c58d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [unreleased] ### Fixed +- Regression in v0.5.2 where plots lacking log files caused a traceback. + ([#926](https://github.com/ericaltendorf/plotman/pull/926)) ### Added ## [0.5.2] - 2021-09-12 From 8d47f47e1fb3dd64c76c8e77e24a35d5ef3b0a84 Mon Sep 17 00:00:00 2001 From: Kyle Altendorf Date: Mon, 13 Sep 2021 12:37:10 -0400 Subject: [PATCH 09/17] changelog for 926 --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6739c58d..9a39c7e6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,7 +7,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [unreleased] ### Fixed -- Regression in v0.5.2 where plots lacking log files caused a traceback. +- Regression in v0.5.2 where plotting processes that lack log files caused a traceback. ([#926](https://github.com/ericaltendorf/plotman/pull/926)) ### Added From 42b24040e62cfeb55e1f5ddb7cda58cc1eb3d9e7 Mon Sep 17 00:00:00 2001 From: Guy Davis Date: Tue, 14 Sep 2021 15:01:14 -0600 Subject: [PATCH 10/17] Trying to address typing and line length issues with build. --- src/plotman/analyzer.py | 6 +++++- src/plotman/plotman.py | 8 ++++++-- 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/src/plotman/analyzer.py b/src/plotman/analyzer.py index ee26ced1..25e848c0 100644 --- a/src/plotman/analyzer.py +++ b/src/plotman/analyzer.py @@ -10,7 +10,11 @@ def analyze( - logfilenames: typing.List[str], clipterminals: bool, bytmp: bool, bybitfield: bool, columns: int + logfilenames: typing.List[str], + clipterminals: bool, + bytmp: bool, + bybitfield: bool, + columns: int ) -> None: data: typing.Dict[str, typing.Dict[str, typing.List[float]]] = {} for logfilename in logfilenames: diff --git a/src/plotman/plotman.py b/src/plotman/plotman.py index 8c45ea2f..4c2b65ea 100755 --- a/src/plotman/plotman.py +++ b/src/plotman/plotman.py @@ -161,7 +161,7 @@ def parse_args(self) -> typing.Any: return args -def get_term_width(cfg) -> int: +def get_term_width(cfg: configuration.PlotmanConfig) -> int: default_columns = 120 # 80 is typically too narrow. if cfg.user_interface.use_stty_size: try: @@ -297,7 +297,11 @@ def main() -> None: elif args.cmd == "analyze": analyzer.analyze( - args.logfile, args.clipterminals, args.bytmp, args.bybitfield, get_term_width(cfg) + args.logfile, + args.clipterminals, + args.bytmp, + args.bybitfield, + get_term_width(cfg) ) # From 878710371149202226411b19dbc4c9705ea9c938 Mon Sep 17 00:00:00 2001 From: Guy Davis Date: Tue, 14 Sep 2021 15:06:32 -0600 Subject: [PATCH 11/17] Formatter wants trailing commas... --- src/plotman/analyzer.py | 2 +- src/plotman/plotman.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/plotman/analyzer.py b/src/plotman/analyzer.py index 25e848c0..a5e10245 100644 --- a/src/plotman/analyzer.py +++ b/src/plotman/analyzer.py @@ -14,7 +14,7 @@ def analyze( clipterminals: bool, bytmp: bool, bybitfield: bool, - columns: int + columns: int, ) -> None: data: typing.Dict[str, typing.Dict[str, typing.List[float]]] = {} for logfilename in logfilenames: diff --git a/src/plotman/plotman.py b/src/plotman/plotman.py index 4c2b65ea..b3bd5692 100755 --- a/src/plotman/plotman.py +++ b/src/plotman/plotman.py @@ -301,7 +301,7 @@ def main() -> None: args.clipterminals, args.bytmp, args.bybitfield, - get_term_width(cfg) + get_term_width(cfg), ) # From 836a48984a51d5cfdaf73b413c56a88763ffff69 Mon Sep 17 00:00:00 2001 From: Guy Davis Date: Tue, 14 Sep 2021 15:26:31 -0600 Subject: [PATCH 12/17] More format changes. --- src/plotman/analyzer.py | 8 ++++---- src/plotman/plotman.py | 11 +++++------ 2 files changed, 9 insertions(+), 10 deletions(-) diff --git a/src/plotman/analyzer.py b/src/plotman/analyzer.py index a5e10245..43303140 100644 --- a/src/plotman/analyzer.py +++ b/src/plotman/analyzer.py @@ -10,10 +10,10 @@ def analyze( - logfilenames: typing.List[str], - clipterminals: bool, - bytmp: bool, - bybitfield: bool, + logfilenames: typing.List[str], + clipterminals: bool, + bytmp: bool, + bybitfield: bool, columns: int, ) -> None: data: typing.Dict[str, typing.Dict[str, typing.List[float]]] = {} diff --git a/src/plotman/plotman.py b/src/plotman/plotman.py index b3bd5692..74874a39 100755 --- a/src/plotman/plotman.py +++ b/src/plotman/plotman.py @@ -236,8 +236,7 @@ def main() -> None: config_path = configuration.get_path() config_text = configuration.read_configuration_text(config_path) preset_target_definitions_text = importlib.resources.read_text( - plotman_resources, - "target_definitions.yaml", + plotman_resources, "target_definitions.yaml", ) cfg = configuration.get_validated_configs( @@ -297,10 +296,10 @@ def main() -> None: elif args.cmd == "analyze": analyzer.analyze( - args.logfile, - args.clipterminals, - args.bytmp, - args.bybitfield, + args.logfile, + args.clipterminals, + args.bytmp, + args.bybitfield, get_term_width(cfg), ) From 513eaded5deed31dc98c34cbfc2188cd2f677d7e Mon Sep 17 00:00:00 2001 From: Guy Davis Date: Tue, 14 Sep 2021 15:36:36 -0600 Subject: [PATCH 13/17] Dueling formatters... Github wins I guess. --- src/plotman/plotman.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/plotman/plotman.py b/src/plotman/plotman.py index 74874a39..e111f5a3 100755 --- a/src/plotman/plotman.py +++ b/src/plotman/plotman.py @@ -236,7 +236,8 @@ def main() -> None: config_path = configuration.get_path() config_text = configuration.read_configuration_text(config_path) preset_target_definitions_text = importlib.resources.read_text( - plotman_resources, "target_definitions.yaml", + plotman_resources, + "target_definitions.yaml", ) cfg = configuration.get_validated_configs( From f14ad0eb4097fc36ba5ee6852270e2131a5d2f02 Mon Sep 17 00:00:00 2001 From: Kyle Altendorf Date: Sat, 18 Sep 2021 21:25:47 -0400 Subject: [PATCH 14/17] Create disk spaces log path --- src/plotman/configuration.py | 8 +------- src/plotman/resources/plotman.yaml | 19 +++++++++++-------- 2 files changed, 12 insertions(+), 15 deletions(-) diff --git a/src/plotman/configuration.py b/src/plotman/configuration.py index dff7d679..c10a39ac 100644 --- a/src/plotman/configuration.py +++ b/src/plotman/configuration.py @@ -328,6 +328,7 @@ def setup(self) -> None: os.makedirs(self.plots, exist_ok=True) os.makedirs(self.transfers, exist_ok=True) os.makedirs(os.path.dirname(self.application), exist_ok=True) + os.makedirs(os.path.dirname(self.disk_spaces), exist_ok=True) def create_plot_log_path(self, time: pendulum.DateTime) -> str: return self._create_log_path( @@ -343,13 +344,6 @@ def create_transfer_log_path(self, time: pendulum.DateTime) -> str: group="transfer", ) - def create_tdisk_space_log_path(self, time: pendulum.DateTime) -> str: - return self._create_log_path( - time=time, - directory=self.disk_spaces, - group="disk_space", - ) - def _create_log_path( self, time: pendulum.DateTime, directory: str, group: str ) -> str: diff --git a/src/plotman/resources/plotman.yaml b/src/plotman/resources/plotman.yaml index c9d38a10..f3cb1c54 100644 --- a/src/plotman/resources/plotman.yaml +++ b/src/plotman/resources/plotman.yaml @@ -3,14 +3,17 @@ # https://github.com/ericaltendorf/plotman/wiki/Configuration#versions version: [2] -logging: - # One directory in which to store all plot job logs (the STDOUT/ - # STDERR of all plot jobs). In order to monitor progress, plotman - # reads these logs on a regular basis, so using a fast drive is - # recommended. - plots: /home/chia/chia/logs - # transfers: - # application: +#logging: +# # One directory in which to store all plot job logs (the STDOUT/ +# # STDERR of all plot jobs). In order to monitor progress, plotman +# # reads these logs on a regular basis, so using a fast drive is +# # recommended. +# # For Linux, these paths default to a directory under ~/.local/share/plotman/ +# plots: +# transfers: +# # For Linux, these paths default to a file at ~/.cache/plotman/log/ +# application: +# disk_spaces: # Options for display and rendering user_interface: From 034003bd9217f9a6276da4033d0c3b10259f15f1 Mon Sep 17 00:00:00 2001 From: Kyle Altendorf Date: Sat, 18 Sep 2021 21:27:49 -0400 Subject: [PATCH 15/17] changelog for 929 --- CHANGELOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index f8c1a79c..7aacc3aa 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Fixed - Regression in v0.5.2 where plotting processes that lack log files caused a traceback. ([#926](https://github.com/ericaltendorf/plotman/pull/926)) +- Create the directory for the new disk spaces log file. + ([#929](https://github.com/ericaltendorf/plotman/pull/929)) ### Added ## [0.5.2] - 2021-09-12 From cc2f1b35d175573526cb2dbf63adf57542f7fee0 Mon Sep 17 00:00:00 2001 From: Kyle Altendorf Date: Sat, 18 Sep 2021 21:48:00 -0400 Subject: [PATCH 16/17] changelog for 918 --- CHANGELOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7aacc3aa..51d405a1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ([#926](https://github.com/ericaltendorf/plotman/pull/926)) - Create the directory for the new disk spaces log file. ([#929](https://github.com/ericaltendorf/plotman/pull/929)) +- Better handle non-interactive uses that had trouble while detecting the (non-existant) terminal size. + ([#918](https://github.com/ericaltendorf/plotman/pull/918)) ### Added ## [0.5.2] - 2021-09-12 From 70e8a48353997cc1c64d230c492a6ce86915da37 Mon Sep 17 00:00:00 2001 From: Kyle Altendorf Date: Sun, 19 Sep 2021 15:27:35 -0400 Subject: [PATCH 17/17] set v0.5.3 --- CHANGELOG.md | 3 +-- VERSION | 2 +- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 51d405a1..2f76741d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,7 +5,7 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). -## [unreleased] +## [0.5.3] - 2021-09-19 ### Fixed - Regression in v0.5.2 where plotting processes that lack log files caused a traceback. ([#926](https://github.com/ericaltendorf/plotman/pull/926)) @@ -13,7 +13,6 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ([#929](https://github.com/ericaltendorf/plotman/pull/929)) - Better handle non-interactive uses that had trouble while detecting the (non-existant) terminal size. ([#918](https://github.com/ericaltendorf/plotman/pull/918)) -### Added ## [0.5.2] - 2021-09-12 ### Fixed diff --git a/VERSION b/VERSION index decc4e86..be14282b 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.5.2+dev +0.5.3