From b46efd592fddf28f89e5a720ad363a90c6c9d14e Mon Sep 17 00:00:00 2001 From: Michael Droettboom Date: Wed, 4 Sep 2024 10:44:00 -0400 Subject: [PATCH 1/3] Separate subprocess writing/parsing into their own functions And document. As suggested by @corona10 https://github.com/psf/pyperf/pull/197#discussion_r1718097565 --- pyperf/_command.py | 30 +++++++++++++++++++++--------- pyperf/_process_time.py | 17 +++++++++++++---- 2 files changed, 34 insertions(+), 13 deletions(-) diff --git a/pyperf/_command.py b/pyperf/_command.py index 592bf5c0..f4af37ff 100644 --- a/pyperf/_command.py +++ b/pyperf/_command.py @@ -8,6 +8,26 @@ from pyperf._worker import WorkerTask +def parse_subprocess_data(output): + # Parse the data send from the subprocess. + # It is three lines containing: + # - The runtime (in seconds) + # - max_rss (or -1, if not able to compute) + # - The metadata to add to the benchmark entry, as a JSON dictionary + + rss = None + metadata = {} + try: + lines = output.splitlines() + timing = float(lines[0]) + rss = int(lines[1]) + metadata = json.loads(lines[2]) + except ValueError: + raise ValueError("failed to parse script output: %r" % output) + + return timing, rss, metadata + + def bench_command(command, task, loops): path = os.path.dirname(__file__) script = os.path.join(path, '_process_time.py') @@ -23,15 +43,7 @@ def bench_command(command, task, loops): raise Exception("Command failed with exit code %s" % proc.returncode) - rss = None - metadata = {} - try: - lines = output.splitlines() - timing = float(lines[0]) - rss = int(lines[1]) - metadata = json.loads(lines[2]) - except ValueError: - raise ValueError("failed to parse script output: %r" % output) + timing, rss, metadata = parse_subprocess_data(output) if rss and rss > 0: # store the maximum diff --git a/pyperf/_process_time.py b/pyperf/_process_time.py index 199dc885..6e2de106 100644 --- a/pyperf/_process_time.py +++ b/pyperf/_process_time.py @@ -114,6 +114,18 @@ def load_hooks(metadata): return hook_managers +def write_data(dt, max_rss, metadata, out=sys.stdout): + # The data that is communicated back to the main orchestration process. + # It is three lines containing: + # - The runtime (in seconds) + # - max_rss (or -1, if not able to compute) + # - The metadata to add to the benchmark entry, as a JSON dictionary + # Write timing in seconds into stdout + print(dt, file=out) + print(max_rss or -1, file=out) + print(json.dumps(metadata), file=out) + + def main(): # Make sure that the pyperf module wasn't imported if 'pyperf' in sys.modules: @@ -162,10 +174,7 @@ def main(): for hook in hook_managers.values(): hook.teardown(metadata) - # Write timing in seconds into stdout - print(dt) - print(max_rss or -1) - print(json.dumps(metadata)) + write_data(dt, max_rss, metadata) if __name__ == "__main__": From bb2437163e36b00f6408dcd8e8b65e4ca6051a38 Mon Sep 17 00:00:00 2001 From: Michael Droettboom Date: Thu, 5 Sep 2024 09:13:11 -0400 Subject: [PATCH 2/3] Call json.dump directly --- pyperf/_process_time.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pyperf/_process_time.py b/pyperf/_process_time.py index 6e2de106..f353f757 100644 --- a/pyperf/_process_time.py +++ b/pyperf/_process_time.py @@ -115,15 +115,15 @@ def load_hooks(metadata): def write_data(dt, max_rss, metadata, out=sys.stdout): - # The data that is communicated back to the main orchestration process. + # Write the data that is communicated back to the main orchestration process. # It is three lines containing: # - The runtime (in seconds) # - max_rss (or -1, if not able to compute) # - The metadata to add to the benchmark entry, as a JSON dictionary - # Write timing in seconds into stdout print(dt, file=out) print(max_rss or -1, file=out) - print(json.dumps(metadata), file=out) + json.dump(metadata, fp=out) + print(file=out) def main(): From ac49b2068b6a4df4882cff1fdf36f3b801328e1c Mon Sep 17 00:00:00 2001 From: Michael Droettboom Date: Fri, 6 Sep 2024 11:32:25 -0400 Subject: [PATCH 3/3] Update pyperf/_command.py Co-authored-by: Victor Stinner --- pyperf/_command.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyperf/_command.py b/pyperf/_command.py index f4af37ff..cd09f3fa 100644 --- a/pyperf/_command.py +++ b/pyperf/_command.py @@ -23,7 +23,7 @@ def parse_subprocess_data(output): rss = int(lines[1]) metadata = json.loads(lines[2]) except ValueError: - raise ValueError("failed to parse script output: %r" % output) + raise ValueError("failed to parse worker output: %r" % output) return timing, rss, metadata