diff --git a/ci_watson/scripts/okify_regtests.py b/ci_watson/scripts/okify_regtests.py index f844316..430f26c 100644 --- a/ci_watson/scripts/okify_regtests.py +++ b/ci_watson/scripts/okify_regtests.py @@ -1,8 +1,9 @@ """ -Regression test okifying script. +This script "okifies" some set of regression test results (overwrites truth files such +that a set of regression test results becomes correct). Requires JFrog CLI (https://jfrog.com/getcli/) configured with credentials -that have write access to the jwst-pipeline repository. +that have write access to the specified repository (`jwst-pipeline` or `roman-pipeline`). """ import json @@ -12,6 +13,7 @@ import tempfile from argparse import ArgumentParser from contextlib import contextmanager +from enum import Enum from glob import glob from pathlib import Path @@ -19,124 +21,180 @@ import readchar from colorama import Fore -ARTIFACTORY_REPO = 'jwst-pipeline-results' -OBSERVATORIES = ["jwst", "roman"] -SPECFILE_SUFFIX = '_okify.json' -RTDATA_SUFFIX = '_rtdata.asdf' +JSON_SPEC_FILE_SUFFIX = "_okify.json" +ASDF_BREADCRUMB_FILE_SUFFIX = "_rtdata.asdf" TERMINAL_WIDTH = shutil.get_terminal_size((80, 20)).columns -def parse_args(): - parser = ArgumentParser(description='Okify regression test results') - parser.add_argument( - "observatory", - help=f"observatory (one of {', '.join(OBSERVATORIES)})" - ) - parser.add_argument( - 'build_number', - help='GitHub Actions build number for JWST builds', - metavar='build-number', - ) - parser.add_argument( - '--dry-run', action='store_true', help='pass the --dry-run flag to JFrog CLI' - ) +class Observatory(Enum): + jwst = "jwst" + roman = "roman" + + def __str__(self): + return self.value - return parser.parse_args() + @property + def runs_directory(self) -> str: + """directory on Artifactory where run results are stored""" + if self == Observatory.jwst: + return "jwst-pipeline-results/" + elif self == Observatory.roman: + return "roman-pipeline-results/regression-tests/runs/" + else: + raise NotImplementedError(f"runs directory not defined for '{self}'") -def artifactory_copy(specfile, dry_run=False): +def artifactory_copy(json_spec_file: os.PathLike, dry_run: bool = False): + """ + copy files with `jf rt cp` based on instructions in the specfile + + :param json_spec_file: JSON file indicating file transfer patterns and targets (see https://docs.jfrog-applications.jfrog.io/jfrog-applications/jfrog-cli/cli-for-jfrog-artifactory/using-file-specs) + :param dry_run: do nothing (passes `--dry-run` to JFrog CLI) + :raises CalledProcessError: if JFrog command fails + """ + jfrog_args = [] if dry_run: - jfrog_args.append('--dry-run') + jfrog_args.append("--dry-run") - args = list(['jfrog', 'rt', 'cp'] + jfrog_args + [f'--spec={specfile}']) - subprocess.run(args, check=True) + subprocess.run( + ["jfrog", "rt", "cp", *jfrog_args, f"--spec={Path(json_spec_file).absolute()}"], + check=True, + ) -def artifactory_folder_copy(specfile, dry_run=False): - """Copy a folder after removing target folder""" - jfrog_args = [] +def artifactory_folder_replace_copy(json_spec_file: os.PathLike, dry_run: bool = False): + """ + copy files with `jf rt cp` based on instructions in the specfile, deleting the destination folder first + + :param json_spec_file: JSON file indicating file transfer patterns and targets (see https://docs.jfrog-applications.jfrog.io/jfrog-applications/jfrog-cli/cli-for-jfrog-artifactory/using-file-specs) + :param dry_run: do nothing (passes `--dry-run` to JFrog CLI) + :raises CalledProcessError: if JFrog command fails + """ + + jfrog_args = ["--quiet=true"] if dry_run: - jfrog_args.append('--dry-run') + jfrog_args.append("--dry-run") # Since two different jfrog operations are required, need to read in # the spec to perform the delete. - with open(specfile) as fh: - spec = json.load(fh) - pattern = spec['files'][0]['pattern'] + '/' - target = spec['files'][0]['target'] + with open(json_spec_file) as file_handle: + spec = json.load(file_handle) + + folder_pattern = spec["files"][0]["pattern"] + "/" + folder_target = spec["files"][0]["target"] # Remove the target - folder = target + Path(pattern).stem - args = ['jfrog', 'rt', 'del', folder, '--quiet=true'] + jfrog_args - subprocess.run(args, check=True) + subprocess.run( + [ + "jfrog", + "rt", + "del", + *jfrog_args, + f"{folder_target}{Path(folder_pattern).stem}", + ], + check=True, + ) - # Copy pattern to parent of target. - args = ['jfrog', 'rt', 'cp', '--spec', specfile] + jfrog_args - subprocess.run(args, check=True) + artifactory_copy(json_spec_file, dry_run) -def artifactory_dispatch(okify_op, specfile, dry_run): - """Perform the indicated artifactory operation +def artifactory_dispatch( + json_spec_file: os.PathLike, + replace_whole_folders: bool = False, + dry_run: bool = False, +): + """ + Perform the indicated artifactory operation + + :param json_spec_file: JSON file indicating file transfer patterns and targets (see https://docs.jfrog-applications.jfrog.io/jfrog-applications/jfrog-cli/cli-for-jfrog-artifactory/using-file-specs) + :param replace_whole_folders: delete entire folders before copying + :param dry_run: do nothing (passes `--dry-run` to JFrog CLI) + :raises CalledProcessError: if JFrog command fails + """ - Parameters - ---------- - okify_op : str - The operation to perform: - - 'file_copy': Copy individual files - - 'folder_copy': Replace whole folders + if not replace_whole_folders: + artifactory_copy(json_spec_file, dry_run=dry_run) + else: + artifactory_folder_replace_copy(json_spec_file, dry_run=dry_run) - specfile : str - The full path to the jfrog spec file - dry_run : bool - True to just show what would be done +def artifactory_download_run_files( + runs_directory: os.PathLike | str, run_number: int, suffix: str +) -> list[Path]: """ - if okify_op == 'file_copy': - artifactory_copy(os.path.abspath(specfile), dry_run=dry_run) - elif okify_op == 'folder_copy': - artifactory_folder_copy(os.path.abspath(specfile), dry_run=dry_run) - else: - raise RuntimeError(f'Unknown artifactory operation: {okify_op}') + Download files with the given suffix from the given run. + + :param runs_directory: repository path where run directories are stored, i.e. `jwst-pipeline-results/` or `roman-pipeline-results/regression-tests/runs/` + :param run_number: GitHub Actions job number of regression test run + :param suffix: filename suffix to search for + :returns: list of downloaded files on the local file system + :raises CalledProcessError: if JFrog command fails + Some example searches would be: -def artifactory_get_breadcrumbs(build_number, suffix): - """Download specfiles or other breadcrumb from Artifactory associated with - a build number and return a list of their locations on the local file system + .. code-block:: shell - An example search for build 586 would be: + jfrog rt search jwst-pipeline-results/*_GITHUB_CI_*-586/*_okify.json + jfrog rt search roman-pipeline-results/*/*_okify.json --props='build.number=540;build.name=RT :: romancal' - jfrog rt search jwst-pipeline-results/*_GITHUB_CI_*-586/*_okify.json """ - # Retrieve all the okify specfiles for failed tests. - args = list( - ['jfrog', 'rt', 'dl'] - + [f"{ARTIFACTORY_REPO}/*_GITHUB_CI_*-{build_number}/*{suffix}"] - + ['--flat'] + subprocess.run( + [ + "jfrog", + "rt", + "dl", + str(Path(runs_directory) / f"*_GITHUB_CI_*-{run_number}" / f"*{suffix}"), + ], + check=True, + capture_output=True, ) - subprocess.run(args, check=True, capture_output=True) - return sorted(glob(f'*{suffix}')) + return sorted(Path.cwd().glob(f"**/*{suffix}")) -def artifactory_get_build_artifacts(build_number): - specfiles = artifactory_get_breadcrumbs(build_number, SPECFILE_SUFFIX) - asdffiles = artifactory_get_breadcrumbs(build_number, RTDATA_SUFFIX) +def artifactory_download_regtest_artifacts( + observatory: Observatory, run_number: int +) -> tuple[list[Path], list[Path]]: + """ + Download both JSON spec files and ASDF breadcrumb files from Artifactory associated with a regression test run + (via a job number), and return a list of their downloaded locations on the local file system. + + :param observatory: observatory to use + :param run_number: GitHub Actions job number of regression test run + :returns: two lists of downloaded files on the local file system; JSON specfiles, and ASDF breadcrumb files + :raises CalledProcessError: if JFrog command fails + """ + + specfiles = artifactory_download_run_files( + observatory.runs_directory, run_number, JSON_SPEC_FILE_SUFFIX + ) + asdffiles = artifactory_download_run_files( + observatory.runs_directory, run_number, ASDF_BREADCRUMB_FILE_SUFFIX + ) if len(specfiles) != len(asdffiles): - raise RuntimeError('Different number of _okify.json and _rtdata.asdf files') + raise RuntimeError("Different number of `_okify.json` and `_rtdata.asdf` files") for a, b in zip(specfiles, asdffiles): - if a.replace(SPECFILE_SUFFIX, '') != b.replace(RTDATA_SUFFIX, ''): - raise RuntimeError('The _okify.json and _rtdata.asdf files are not matched') + if str(a).replace(JSON_SPEC_FILE_SUFFIX, "") != str(b).replace( + ASDF_BREADCRUMB_FILE_SUFFIX, "" + ): + raise RuntimeError( + "The `_okify.json` and `_rtdata.asdf` files are not matched" + ) return specfiles, asdffiles @contextmanager -def pushd(newdir): +def pushd(newdir: os.PathLike): + """ + transient context that emulates `pushd` with `chdir` + """ + prevdir = os.getcwd() os.chdir(os.path.expanduser(newdir)) try: @@ -146,83 +204,120 @@ def pushd(newdir): def main(): - args = parse_args() + parser = ArgumentParser( + description='This script "okifies" a set of regression test results; that is, it overwrites truth files on Artifactory so that a set of regression test results becomes correct.' + ) + ( + parser.add_argument( + "observatory", + type=Observatory, + choices=list(Observatory), + help="Observatory to overwrite truth files for on Artifactory", + ), + ) + parser.add_argument( + "run_number", + help="GitHub Actions job number of regression test run (see https://github.com/spacetelescope/RegressionTests/actions)", + metavar="run-number", + ) + parser.add_argument( + "--dry-run", + action="store_true", + help="do nothing (passes the `--dry-run` flag to JFrog CLI)", + ) - build = args.build_number + args = parser.parse_args() + + run = args.run_number + + observatory = args.observatory # Create and chdir to a temporary directory to store specfiles with tempfile.TemporaryDirectory() as tmp_path: - print(f'Downloading test logs to {tmp_path}') + print(f"Downloading test logs to {tmp_path}") with pushd(tmp_path): # Retrieve all the okify specfiles for failed tests. - specfiles, asdffiles = artifactory_get_build_artifacts(build) - - number_failed_tests = len(specfiles) + json_spec_files, asdf_breadcrumb_files = ( + artifactory_download_regtest_artifacts(observatory, run) + ) - print(f'{number_failed_tests} failed tests to okify') + number_failed_tests = len(json_spec_files) - for i, (specfile, asdffile) in enumerate(zip(specfiles, asdffiles)): + print(f"{number_failed_tests} failed tests to okify") + for index, (json_spec_file, asdf_breadcrumb_file) in enumerate( + zip(json_spec_files, asdf_breadcrumb_files) + ): # Print traceback and OKify info for this test failure - with asdf.open(asdffile) as af: - okify_op = af.tree['okify_op'] - traceback = af.tree['traceback'] - remote_results_path = af.tree['remote_results_path'] - output = af.tree['output'] - truth_remote = af.tree['truth_remote'] + with asdf.open(asdf_breadcrumb_file) as asdf_breadcrumb: + # okify_op only useful for JWST + okify_op = ( + asdf_breadcrumb.tree["okify_op"] + if observatory == Observatory.jwst + else "file_copy" + ) + traceback = asdf_breadcrumb.tree["traceback"] + remote_results_path = Path( + asdf_breadcrumb.tree["remote_results_path"] + ) + output = Path(asdf_breadcrumb.tree["output"]) + truth_remote = asdf_breadcrumb.tree["truth_remote"] try: - test_name = af.tree['test_name'] + test_name = asdf_breadcrumb.tree["test_name"] except KeyError: - test_name = 'test_name' - - remote_results = os.path.join( - remote_results_path, os.path.basename(output) - ) - - test_number = i + 1 + test_name = "test_name" print( - f'{Fore.RED}' - + (f' {test_name} '.center(TERMINAL_WIDTH, '—')) - + f'{Fore.RESET}' + f"{Fore.RED}" + + (f" {test_name} ".center(TERMINAL_WIDTH, "—")) + + f"{Fore.RESET}" ) print(traceback) - print(f'{Fore.RED}' + ('—' * TERMINAL_WIDTH) + f'{Fore.RESET}') - print(f'{Fore.GREEN}OK: {remote_results}') - print(f'--> {truth_remote}{Fore.RESET}') + print(f"{Fore.RED}" + ("—" * TERMINAL_WIDTH) + f"{Fore.RESET}") + print(f"{Fore.GREEN}OK: {remote_results_path / output.name}") + print(f"--> {truth_remote}{Fore.RESET}") print( - f'{Fore.RED}' + f"{Fore.RED}" + ( - f'[ test {test_number} of {number_failed_tests} ]'.center( - TERMINAL_WIDTH, '—' + f"[ test {index + 1} of {number_failed_tests} ]".center( + TERMINAL_WIDTH, "—" ) ) - + f'{Fore.RESET}' + + f"{Fore.RESET}" ) # Ask if user wants to okify this test + commands = { + "o": ("okify", Fore.GREEN), + "s": ("skip", Fore.CYAN), + "q": ("quit", Fore.MAGENTA), + } while True: print( - f'{Fore.GREEN}\'o\' to okify{Fore.RESET}, ' - f'{Fore.CYAN}\'s\' to skip{Fore.RESET}, ' - f'{Fore.MAGENTA}\'q\' to quit{Fore.RESET}: ' + ", ".join( + f"{color}'{command}' to {verb}{Fore.RESET}" + for command, (verb, color) in commands.items() + ) + + ": " ) # Get the keyboard character input without pressing return result = readchar.readkey() - if result not in ['o', 's', 'q']: - print('Unrecognized command, try again') + if result not in commands: + print(f"Unrecognized command '{result}', try again") else: break - if result == 'q': + if result == "q": break - elif result == 's': + elif result == "s": pass else: artifactory_dispatch( - okify_op, os.path.abspath(specfile), dry_run=args.dry_run + json_spec_file, + replace_whole_folders=okify_op == "folder_copy", + dry_run=args.dry_run, ) - print('') + print("") -if __name__ == '__main__': +if __name__ == "__main__": main() diff --git a/pyproject.toml b/pyproject.toml index 5674229..971c241 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -16,9 +16,11 @@ classifiers = [ "Topic :: Software Development :: Libraries :: Python Modules", ] dependencies = [ + "crds", + "colorama>=0.4.1", "pytest>=6", + "readchar>=3.0", "requests", - "crds", ] dynamic = [ "version",