From 02850d4ee79771d71e2fe6087d6bbecd7b81bf24 Mon Sep 17 00:00:00 2001 From: Daniel Wheeler Date: Thu, 10 Aug 2023 17:01:37 -0400 Subject: [PATCH] feat: add convert-to-zenodo in cli Untested --- _data/python-pfhub/pfhub/scripts/cli.py | 182 ++++++++++++++++++------ 1 file changed, 135 insertions(+), 47 deletions(-) diff --git a/_data/python-pfhub/pfhub/scripts/cli.py b/_data/python-pfhub/pfhub/scripts/cli.py index 76ccfbac2..1c18d7231 100644 --- a/_data/python-pfhub/pfhub/scripts/cli.py +++ b/_data/python-pfhub/pfhub/scripts/cli.py @@ -8,14 +8,123 @@ from toolz.curried import pipe, get from toolz.curried import map as map_ -from ..convert import download_zenodo, download_meta +from ..convert import meta_to_zenodo, download_file +from ..convert import download_zenodo as download_zenodo_ +from ..convert import download_meta as download_meta_ from ..func import compact +import tempfile +import pykwalify +import pykwalify.core +import shutil + EPILOG = "See the documentation at \ https://github.com/usnistgov/pfhub/blob/master/CLI.md (under construction)" + +@click.group(epilog=EPILOG) +def cli(): + """Submit results to PFHub and manipulate PFHub data""" + + + +def output(local_filepaths): + def echo(local_filepath, newline, comma=","): + formatted_path = click.format_filename(local_filepath) + click.secho(message=f" {formatted_path}" + comma, fg="green", nl=newline) + + click.secho(message="Writing:", fg="green", nl=False) + for local_filepath in local_filepaths[:-1]: + echo(local_filepath, False) + echo(local_filepaths[-1], True, comma="") + + +@cli.command(epilog=EPILOG) +@click.argument("url", type=click_params.URL) +@click.option( + "--dest", + "-d", + help="destination directory", + default="./", + type=click.Path(exists=True), +) +def download_zenodo(url, dest): + """Download a Zenodo record + + Works with any Zenodo link + + Args: + url: the URL of either a meta.yaml or Zenodo record + dest: the destination directory + """ + record_id = get_zenodo_record_id(url, zenodo_regexs()) + sandbox = False + if record_id is None: + record_id = get_zenodo_record_id(url, zenodo_sandbox_regexs()) + sandbox = True + + if record_id is not None: + local_filepaths = download_zenodo_(record_id, sandbox=sandbox, dest=dest) + output(local_filepaths) + else: + click.secho(f"{url} does not match any expected regex for Zenodo", fg="red") + + +@cli.command(epilog=EPILOG) +@click.argument("url", type=click_params.URL) +@click.option( + "--dest", + "-d", + help="destination directory", + default="./", + type=click.Path(exists=True), +) +def download_meta(url, dest): + """Download a record from pfhub + + Download a meta.yaml from any URL + + Args: + url: the URL of either a meta.yaml or Zenodo record + dest: the destination directory + """ + + is_meta = check_meta_url(url) + if is_meta: + local_filepaths = download_meta_(url, dest=dest) + output(local_filepaths) + else: + click.secho(f"{url} is not a valid PFHub meta.yaml", fg="red") + + +@cli.command(epilog=EPILOG) +@click.argument("file_path", type=click.Path(exists=True)) +@click.option( + "--dest", + "-d", + help="destination directory", + default="./", + type=click.Path(exists=False), +) +def convert_to_zenodo(file_path, dest): + """Convert a meta.yaml into a format suitable for Zenodo submission + + Args: + file_path: the URL of the meta.yaml + dest: the destination directory + + """ + is_meta = check_meta(file_path) + if is_meta: + local_filepaths = [meta_to_zenodo(file_path)] + output(local_filepaths) + else: + click.secho(f"{file_path} is not a valid PFHub meta.yaml", fg="red") + + + def zenodo_regexs(): """Regular expression for acceptable Zenodo URLs""" return [ @@ -53,50 +162,29 @@ def get_zenodo_record_id(url, regexs): ) -@click.group(epilog=EPILOG) -def cli(): - """Submit results to PFHub and manipulate PFHub data""" - - -@cli.command(epilog=EPILOG) -@click.argument("url", type=click_params.URL) -@click.option( - "--dest", - "-d", - help="destination directory", - default="./", - type=click.Path(exists=True), -) -def download(url, dest): - """Download a PFHub record - - Download an old meta.yaml or a new PFHub record stored on Zenodo. - - Args: - url: the URL of either a meta.yaml or Zenodo record +def check_meta_url(url): + """Check that a file is a valid meta.yaml """ - - def output(local_filepaths): - def echo(local_filepath, newline, comma=","): - formatted_path = click.format_filename(local_filepath) - click.secho(message=f" {formatted_path}" + comma, fg="green", nl=newline) - - click.secho(message="Writing:", fg="green", nl=False) - for local_filepath in local_filepaths[:-1]: - echo(local_filepath, False) - echo(local_filepaths[-1], True, comma="") - - record_id = get_zenodo_record_id(url, zenodo_regexs()) - sandbox = False - if record_id is None: - record_id = get_zenodo_record_id(url, zenodo_sandbox_regexs()) - sandbox = True - - if record_id is not None: - local_filepaths = download_zenodo(record_id, sandbox=sandbox, dest=dest) - output(local_filepaths) - elif re.fullmatch(r".*/meta.yaml", url): - local_filepaths = download_meta(url, dest=dest) - output(local_filepaths) - else: - click.secho(f"{url} does not match any expected regex", fg="red") + tmpdir = tempfile.mkdtemp() + file_path = download_file(url, dest=tmpdir) + result = check_meta(file_path) + shutil.rmtree(tmpdir) + return result + + +def check_meta(path): + try: + c = pykwalify.core.Core(source_file=path, schema_files=["schema.yaml"]) + except pykwalify.errors.CoreError: + return False + try: + c.validate(raise_exception=True) + except pykwalify.errors.SchemaError: + return False + return True + + +def check_linkml(path): + """Check that a file has a valid linkml pfhub schema + """ + pass