From 33530c695b08b8beb24688344459242281db9d75 Mon Sep 17 00:00:00 2001 From: Max Wang Date: Tue, 24 Oct 2023 10:51:19 -0400 Subject: [PATCH 1/5] Add download capabilities --- .gitignore | 1 + test_harness/download.py | 54 +++++++++++++++++++++++++++++++++++++--- test_harness/main.py | 26 +++++++++++++++---- test_harness/models.py | 32 ++++++++++++++++++++++++ 4 files changed, 104 insertions(+), 9 deletions(-) diff --git a/.gitignore b/.gitignore index 85d56b1..9c5034d 100644 --- a/.gitignore +++ b/.gitignore @@ -2,3 +2,4 @@ __pycache__ *.egg-info logs/ +Translator-Tests/ \ No newline at end of file diff --git a/test_harness/download.py b/test_harness/download.py index 77732d9..68dd201 100644 --- a/test_harness/download.py +++ b/test_harness/download.py @@ -1,11 +1,57 @@ """Download tests.""" +import glob +import httpx +import io +import json import logging from pathlib import Path -from typing import List +from typing import List, Union +import zipfile -from .models import TestCase +from .models import TestCase, TestSuite -def download_tests(url: Path) -> List[TestCase]: +def download_tests(suite: Union[str, List[str]], url: Path, logger: logging.Logger) -> List[TestCase]: """Download tests from specified location.""" - raise NotImplementedError() + assert Path(url).suffix == ".zip" + logger.info(f"Downloading tests from {url}...") + # download file from internet + with httpx.Client(follow_redirects=True) as client: + tests_zip = client.get(url) + tests_zip.raise_for_status() + # we already checked if zip before download, so now unzip + with zipfile.ZipFile(io.BytesIO(tests_zip.read())) as zip_ref: + zip_ref.extractall("./Translator-Tests") + + # Find all json files in the downloaded zip + tests_paths = glob.glob("./Translator-Tests/**/*.json", recursive=True) + + all_tests = [] + suites = suite if type(suite) == list else [suite] + test_case_ids = [] + + logger.info(f"Reading in {len(tests_paths)} tests...") + + # do the reading of the tests and make a tests list + for test_path in tests_paths: + with open(test_path, "r") as f: + test_json = json.load(f) + try: + test_suite = TestSuite.parse_obj(test_json) + if test_suite.id in suites: + # if suite is selected, grab all its test cases + test_case_ids.extend(test_suite.case_ids) + except Exception as e: + # not a Test Suite + pass + try: + test_case = TestCase.parse_obj(test_json) + all_tests.append(test_case) + except Exception as e: + # not a Test Case + pass + + # only return the tests from the specified suites + tests = filter(lambda x: x in test_case_ids, all_tests) + logger.info(f"Passing along {len(tests)} tests") + return tests diff --git a/test_harness/main.py b/test_harness/main.py index 1a2f6e7..71ad462 100644 --- a/test_harness/main.py +++ b/test_harness/main.py @@ -2,6 +2,8 @@ from argparse import ArgumentParser import json from pathlib import Path +from typing import Union, List +from urllib.parse import urlparse from uuid import uuid4 from .run import run_tests @@ -11,14 +13,20 @@ setup_logger() +def url_type(arg): + url = urlparse(arg) + if all((url.scheme, url.netloc)): + return arg + raise TypeError("Invalid URL") + + def main(args): """Main Test Harness entrypoint.""" qid = str(uuid4())[:8] logger = get_logger(qid, args["log_level"]) tests = [] if "tests_url" in args: - logger.info("Downloading tests...") - tests = download_tests(args["tests_url"]) + tests = download_tests(args["suite"], args["tests_url"], logger) elif "tests" in args: tests = args["tests"] else: @@ -26,8 +34,7 @@ def main(args): "Please run this command with `-h` to see the available options." ) - logger.info("Running tests...") - report = run_tests(tests) + report = run_tests(tests, logger) if args["save_to_dashboard"]: logger.info("Saving to Testing Dashboard...") @@ -53,7 +60,16 @@ def cli(): ) download_parser.add_argument( - "tests_url", type=Path, help="URL to download in order to find the test files" + "suite", + type=Union[str, List[str]], + help="The name/id of the suite(s) to run. Once tests have been downloaded, the test cases in this suite(s) will be run.", + ) + + download_parser.add_argument( + "--tests_url", + type=url_type, + default="https://github.com/NCATSTranslator/Tests/archive/refs/heads/main.zip", + help="URL to download in order to find the test files", ) run_parser = subparsers.add_parser("run", help="Run a given set of tests") diff --git a/test_harness/models.py b/test_harness/models.py index 574db9f..73ba87e 100644 --- a/test_harness/models.py +++ b/test_harness/models.py @@ -1,5 +1,6 @@ from enum import Enum from pydantic import BaseModel +from typing import List, Optional class Type(str, Enum): @@ -37,6 +38,7 @@ class TestCase(BaseModel): output_curie: str """ + id: Optional[int] type: Type env: Env query_type: QueryType @@ -45,6 +47,36 @@ class TestCase(BaseModel): output_curie: str +class Tests(BaseModel): + """List of Test Cases.""" + + __root__: List[TestCase] + + def __len__(self): + return len(self.__root__) + + def __iter__(self): + return self.__root__.__iter__() + + def __contains__(self, v): + return self.__root__.__contains__(v) + + def __getitem__(self, i): + return self.__root__.__getitem__(i) + + +class TestSuite(BaseModel): + """ + Test Suite containing the ids of Test Cases. + + id: int + case_ids: List[int] + """ + + id: int + case_ids: List[int] + + class TestResult(BaseModel): """Output of a Test.""" From 4a00838854d6c47f74759c207c5fd0967c6a1725 Mon Sep 17 00:00:00 2001 From: Max Wang Date: Tue, 24 Oct 2023 10:53:35 -0400 Subject: [PATCH 2/5] Incorporate Tests model, tqdm, and instance of logger --- test_harness/run.py | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/test_harness/run.py b/test_harness/run.py index eec34f2..e078be4 100644 --- a/test_harness/run.py +++ b/test_harness/run.py @@ -1,22 +1,21 @@ """Run tests through the Test Runners.""" import logging -from pydantic import validate_arguments -from typing import List, Dict +from tqdm import tqdm +from typing import Dict from ui_test_runner import run_ui_test from ARS_Test_Runner.semantic_test import run_semantic_test as run_ars_test -from .models import TestCase +from .models import Tests -logger = logging.getLogger(__name__) - -@validate_arguments -def run_tests(tests: List[TestCase]) -> Dict: +def run_tests(tests: Tests, logger: logging.Logger) -> Dict: """Send tests through the Test Runners.""" + tests = Tests.parse_obj(tests) + logger.info(f"Running {len(tests)} tests...") full_report = {} # loop over all tests - for test in tests: + for test in tqdm(tests): # check if acceptance test if test.type == "acceptance": full_report[test.input_curie] = {} From 2d5b7b3110ef868b68fb92dd9d00e5c115af76c9 Mon Sep 17 00:00:00 2001 From: Max Wang Date: Tue, 24 Oct 2023 10:56:28 -0400 Subject: [PATCH 3/5] Run black formatter --- test_harness/download.py | 4 +++- test_harness/models.py | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/test_harness/download.py b/test_harness/download.py index 68dd201..e936292 100644 --- a/test_harness/download.py +++ b/test_harness/download.py @@ -11,7 +11,9 @@ from .models import TestCase, TestSuite -def download_tests(suite: Union[str, List[str]], url: Path, logger: logging.Logger) -> List[TestCase]: +def download_tests( + suite: Union[str, List[str]], url: Path, logger: logging.Logger +) -> List[TestCase]: """Download tests from specified location.""" assert Path(url).suffix == ".zip" logger.info(f"Downloading tests from {url}...") diff --git a/test_harness/models.py b/test_harness/models.py index 73ba87e..8c49bab 100644 --- a/test_harness/models.py +++ b/test_harness/models.py @@ -68,7 +68,7 @@ def __getitem__(self, i): class TestSuite(BaseModel): """ Test Suite containing the ids of Test Cases. - + id: int case_ids: List[int] """ From 70e0c0abcc9954aa97e23c873216141556e0f332 Mon Sep 17 00:00:00 2001 From: Max Wang Date: Tue, 24 Oct 2023 12:08:40 -0400 Subject: [PATCH 4/5] Fix download suite argument type --- test_harness/main.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/test_harness/main.py b/test_harness/main.py index 71ad462..5feac59 100644 --- a/test_harness/main.py +++ b/test_harness/main.py @@ -1,8 +1,6 @@ """Translator SRI Automated Test Harness.""" from argparse import ArgumentParser import json -from pathlib import Path -from typing import Union, List from urllib.parse import urlparse from uuid import uuid4 @@ -61,7 +59,7 @@ def cli(): download_parser.add_argument( "suite", - type=Union[str, List[str]], + type=str, help="The name/id of the suite(s) to run. Once tests have been downloaded, the test cases in this suite(s) will be run.", ) From ccee7e7178a60cc148fb974394575c4432157b24 Mon Sep 17 00:00:00 2001 From: Max Wang Date: Tue, 24 Oct 2023 17:33:53 -0400 Subject: [PATCH 5/5] Put the downloaded files into a temp directory that gets cleaned up --- test_harness/download.py | 62 +++++++++++++++++++++------------------- 1 file changed, 32 insertions(+), 30 deletions(-) diff --git a/test_harness/download.py b/test_harness/download.py index e936292..7088268 100644 --- a/test_harness/download.py +++ b/test_harness/download.py @@ -5,6 +5,7 @@ import json import logging from pathlib import Path +import tempfile from typing import List, Union import zipfile @@ -22,38 +23,39 @@ def download_tests( tests_zip = client.get(url) tests_zip.raise_for_status() # we already checked if zip before download, so now unzip + with tempfile.TemporaryDirectory() as tmpdir: with zipfile.ZipFile(io.BytesIO(tests_zip.read())) as zip_ref: - zip_ref.extractall("./Translator-Tests") - - # Find all json files in the downloaded zip - tests_paths = glob.glob("./Translator-Tests/**/*.json", recursive=True) - - all_tests = [] - suites = suite if type(suite) == list else [suite] - test_case_ids = [] - - logger.info(f"Reading in {len(tests_paths)} tests...") - - # do the reading of the tests and make a tests list - for test_path in tests_paths: - with open(test_path, "r") as f: - test_json = json.load(f) - try: - test_suite = TestSuite.parse_obj(test_json) - if test_suite.id in suites: - # if suite is selected, grab all its test cases - test_case_ids.extend(test_suite.case_ids) - except Exception as e: - # not a Test Suite - pass - try: - test_case = TestCase.parse_obj(test_json) - all_tests.append(test_case) - except Exception as e: - # not a Test Case - pass + zip_ref.extractall(tmpdir) + + # Find all json files in the downloaded zip + tests_paths = glob.glob(f"{tmpdir}/**/*.json", recursive=True) + + all_tests = [] + suites = suite if type(suite) == list else [suite] + test_case_ids = [] + + logger.info(f"Reading in {len(tests_paths)} tests...") + + # do the reading of the tests and make a tests list + for test_path in tests_paths: + with open(test_path, "r") as f: + test_json = json.load(f) + try: + test_suite = TestSuite.parse_obj(test_json) + if test_suite.id in suites: + # if suite is selected, grab all its test cases + test_case_ids.extend(test_suite.case_ids) + except Exception as e: + # not a Test Suite + pass + try: + test_case = TestCase.parse_obj(test_json) + all_tests.append(test_case) + except Exception as e: + # not a Test Case + pass # only return the tests from the specified suites - tests = filter(lambda x: x in test_case_ids, all_tests) + tests = list(filter(lambda x: x in test_case_ids, all_tests)) logger.info(f"Passing along {len(tests)} tests") return tests