diff --git a/benchmarks/bm_runner.py b/benchmarks/bm_runner.py index f3efb0ea31..9b997bc87c 100644 --- a/benchmarks/bm_runner.py +++ b/benchmarks/bm_runner.py @@ -32,17 +32,26 @@ ) -def _subprocess_run_print(args, **kwargs): +def echo(echo_string: str): # Use subprocess for printing to reduce chance of printing out of sequence # with the subsequent calls. - subprocess.run(["echo", f"BM_RUNNER DEBUG: {' '.join(args)}"]) + subprocess.run(["echo", f"BM_RUNNER DEBUG: {echo_string}"]) + + +def _subprocess_runner(args, asv=False, **kwargs): + # Avoid permanent modifications if the same arguments are used more than once. + args = args.copy() + kwargs = kwargs.copy() + if asv: + args.insert(0, "asv") + kwargs["cwd"] = BENCHMARKS_DIR + echo(" ".join(args)) return subprocess.run(args, **kwargs) -def _subprocess_run_asv(args, **kwargs): - args.insert(0, "asv") - kwargs["cwd"] = BENCHMARKS_DIR - return _subprocess_run_print(args, **kwargs) +def _subprocess_runner_capture(args, **kwargs) -> str: + result = _subprocess_runner(args, capture_output=True, **kwargs) + return result.stdout.decode() def _check_requirements(package: str) -> None: @@ -65,12 +74,12 @@ def _prep_data_gen_env() -> None: python_version = "3.11" data_gen_var = "DATA_GEN_PYTHON" if data_gen_var in environ: - print("Using existing data generation environment.") + echo("Using existing data generation environment.") else: - print("Setting up the data generation environment ...") + echo("Setting up the data generation environment ...") # Get Nox to build an environment for the `tests` session, but don't # run the session. Will re-use a cached environment if appropriate. - _subprocess_run_print( + _subprocess_runner( [ "nox", f"--noxfile={root_dir / 'noxfile.py'}", @@ -86,10 +95,10 @@ def _prep_data_gen_env() -> None: ).resolve() environ[data_gen_var] = str(data_gen_python) - print("Installing Mule into data generation environment ...") + echo("Installing Mule into data generation environment ...") mule_dir = data_gen_python.parents[1] / "resources" / "mule" if not mule_dir.is_dir(): - _subprocess_run_print( + _subprocess_runner( [ "git", "clone", @@ -97,7 +106,7 @@ def _prep_data_gen_env() -> None: str(mule_dir), ] ) - _subprocess_run_print( + _subprocess_runner( [ str(data_gen_python), "-m", @@ -107,7 +116,7 @@ def _prep_data_gen_env() -> None: ] ) - print("Data generation environment ready.") + echo("Data generation environment ready.") def _setup_common() -> None: @@ -116,10 +125,10 @@ def _setup_common() -> None: _prep_data_gen_env() - print("Setting up ASV ...") - _subprocess_run_asv(["machine", "--yes"]) + echo("Setting up ASV ...") + _subprocess_runner(["machine", "--yes"], asv=True) - print("Setup complete.") + echo("Setup complete.") def _asv_compare(*commits: str, overnight_mode: bool = False) -> None: @@ -132,17 +141,15 @@ def _asv_compare(*commits: str, overnight_mode: bool = False) -> None: asv_command = ( f"compare {before} {after} --factor={COMPARE_FACTOR} --split" ) - _subprocess_run_asv(asv_command.split(" ")) + _subprocess_runner(asv_command.split(" "), asv=True) if overnight_mode: # Record performance shifts. # Run the command again but limited to only showing performance # shifts. - shifts = _subprocess_run_asv( - [*asv_command.split(" "), "--only-changed"], - capture_output=True, - text=True, - ).stdout + shifts = _subprocess_runner_capture( + [*asv_command.split(" "), "--only-changed"], asv=True + ) if shifts: # Write the shifts report to a file. # Dir is used by .github/workflows/benchmarks.yml, @@ -221,13 +228,11 @@ def func(args: argparse.Namespace) -> None: commit_range = f"{args.first_commit}^^.." asv_command = ASV_HARNESS.format(posargs=commit_range) - _subprocess_run_asv([*asv_command.split(" "), *args.asv_args]) + _subprocess_runner([*asv_command.split(" "), *args.asv_args], asv=True) # git rev-list --first-parent is the command ASV uses. git_command = f"git rev-list --first-parent {commit_range}" - commit_string = _subprocess_run_print( - git_command.split(" "), capture_output=True, text=True - ).stdout + commit_string = _subprocess_runner_capture(git_command.split(" ")) commit_list = commit_string.rstrip().split("\n") _asv_compare(*reversed(commit_list), overnight_mode=True) @@ -260,16 +265,16 @@ def func(args: argparse.Namespace) -> None: _setup_common() git_command = f"git merge-base HEAD {args.base_branch}" - merge_base = _subprocess_run_print( - git_command.split(" "), capture_output=True, text=True - ).stdout[:8] + merge_base = _subprocess_runner_capture(git_command.split(" "))[:8] with NamedTemporaryFile("w") as hashfile: hashfile.writelines([merge_base, "\n", "HEAD"]) hashfile.flush() commit_range = f"HASHFILE:{hashfile.name}" asv_command = ASV_HARNESS.format(posargs=commit_range) - _subprocess_run_asv([*asv_command.split(" "), *args.asv_args]) + _subprocess_runner( + [*asv_command.split(" "), *args.asv_args], asv=True + ) _asv_compare(merge_base, "HEAD") @@ -326,14 +331,14 @@ def csperf( asv_command = asv_command.replace(" --strict", "") # Only do a single round. asv_command = re.sub(r"rounds=\d", "rounds=1", asv_command) - _subprocess_run_asv([*asv_command.split(" "), *args.asv_args]) + _subprocess_runner([*asv_command.split(" "), *args.asv_args], asv=True) asv_command = f"publish {commit_range} --html-dir={publish_subdir}" - _subprocess_run_asv(asv_command.split(" ")) + _subprocess_runner(asv_command.split(" "), asv=True) # Print completion message. location = BENCHMARKS_DIR / ".asv" - print( + echo( f'New ASV results for "{run_type}".\n' f'See "{publish_subdir}",' f'\n or JSON files under "{location / "results"}".' @@ -380,7 +385,7 @@ def add_arguments(self) -> None: @staticmethod def func(args: argparse.Namespace) -> None: _setup_common() - _subprocess_run_asv([args.asv_sub_command, *args.asv_args]) + _subprocess_runner([args.asv_sub_command, *args.asv_args], asv=True) def main():