From 32fcfd0df3dcd8d0ddebeb8be24671458489629d Mon Sep 17 00:00:00 2001 From: Ankita Sharma Date: Fri, 5 Aug 2022 22:27:56 +0530 Subject: [PATCH] Update: add tests using pytest, update GCBM API (more TODOs left) --- local/rest_api_gcbm/gcbm.py | 210 ++++++++++++------ .../rest_api_gcbm/tests/tests_simulations.py | 99 +++++++++ 2 files changed, 238 insertions(+), 71 deletions(-) create mode 100644 local/rest_api_gcbm/tests/tests_simulations.py diff --git a/local/rest_api_gcbm/gcbm.py b/local/rest_api_gcbm/gcbm.py index 36824fe1..d2117d1c 100644 --- a/local/rest_api_gcbm/gcbm.py +++ b/local/rest_api_gcbm/gcbm.py @@ -33,14 +33,19 @@ def is_category(self, path): else: return self.category in path - # Unlike list.append() in Python, this returns a bool - whether the append was successful or not + checks if the file path is of the current category - def _append(self, file_path): - if self.is_category(file_path): + def _append(self, file_path, check=True): + # Unlike list.append() in Python, this returns a bool - whether the append was successful or not + checks if the file path is of the current category + if check: + if self.is_category(file_path): + self.data.append(file_path) + return True + return False + else: self.data.append(file_path) return True - return False def _update_config(self): + json_paths = {} for file in self.data: json_config_file = GCBMList.change_extension(file, ".json") json_filepath = os.path.join(self.dirpath, json_config_file) @@ -48,10 +53,13 @@ def _update_config(self): if json_config_file.name not in self.config: self.generate_config(file, json_config_file) else: - with open(json_filepath, "r+") as _config: + # TODO: This is not required I guess + with open(json_filepath, "w+") as _config: json.dump( self.files[file], _config, indent=4 ) + json_paths[file] = json_filepath + return json_paths def _populate_config_with_hard_coded_config( self, config, hc_config, nodata @@ -74,8 +82,8 @@ def generate_config(self, file, json_config_file): json_filepath = os.path.join(self.dirpath, json_config_file) mode = "w+" - if os.path.exists(json_filepath): - mode = "r+" + # if os.path.exists(json_filepath): + # mode = "w+" # AO: disabling in favour of user defined attributes # hard_coded_path = f"hard_coded_values/{json_config_file}" @@ -89,7 +97,10 @@ def generate_config(self, file, json_config_file): with open(json_filepath, mode) as _file: if mode == "r+": - config = json.load(_file) + try: + config = json.load(_file) + except: + breakpoint() else: config = dict() @@ -134,7 +145,9 @@ def setattr(self, file, attributes): config["has_year"] = True self.files[file] = config - self._update_config() + # Output paths are returned to keep track + json_paths = self._update_config() + return json_paths @staticmethod def change_extension(file_path, new_extension): @@ -174,7 +187,14 @@ class GCBMSimulation: def __init__(self): # create a global index self.dirpath = "input/test-run" + self.config_dir_path = "templates" self.files = {} + + # Tracks the output path for the input received through `add_file` method + self.json_paths = {} + + # TODO: Once categories are changed from strings to Enums, we should find a better way to have supported categories + self.supported_categories = ["classifiers", "disturbances", "miscellaneous"] # create sub-indices of different types self.config = list() @@ -188,16 +208,16 @@ def __init__(self): self.miscellaneous = GCBMMiscellaneousList(files=self.files, config=self.config) def create_simulation_folder(self): - if not os.path.exists(self.dirpath): - os.makedirs(self.dirpath) + for dir_path in [self.dirpath, self.config_dir_path]: + if not os.path.exists(dir_path): + os.makedirs(dir_path) def create_file_index(self): - config_dir_path = "templates" assert os.path.isdir( - config_dir_path - ), f"Given config directory path: {config_dir_path} either does not exist or is not a directory." - for dirpath, _, filenames in os.walk(config_dir_path): + self.config_dir_path + ), f"Given config directory path: {self.config_dir_path} either does not exist or is not a directory." + for dirpath, _, filenames in os.walk(self.config_dir_path): for filename in filenames: # Don't read any data, but create the json file abs_filepath = os.path.abspath(os.path.join(dirpath, filename)) @@ -215,8 +235,9 @@ def create_file_index(self): sim_filepath = os.path.join(self.dirpath, filename) shutil.copy(abs_filepath, sim_filepath) + # file_path: disturbances (NOT MUST), classifiers (MUST), miscellaneous (MUST) - def add_file(self, file_path: str): + def add_file(self, file_path: str, category: str = ""): """ This function: @@ -226,25 +247,50 @@ def add_file(self, file_path: str): Parameters ========== 1. file_path (str), no default + 2. category (str), default = "", if skipped - then the categories will be deduced from the file path """ # TODO: update to accept input from Flask endpoint + # FIXME: The flask end point should do the pre-processing to be able to only pass the `file_path` as a string + # TODO: update in app.py to send valid data to add_file filename = os.path.basename(file_path) shutil.copy(file_path, os.path.join(self.dirpath, filename)) - if self.disturbances._append(filename): - self.disturbances._update_config() - return - if self.classifiers._append(filename): - self.classifiers._update_config() - return - if self.miscellaneous._append(filename): - self.miscellaneous._update_config() - return - # TODO: Add covariates here + def _disturbance(filename, check=True): + if self.disturbances._append(filename, check): + self.disturbances._update_config() + + def _classifier(filename, check=True): + if self.classifiers._append(filename, check): + self.classifiers._update_config() + + def _miscellaneous(filename, check=True): + if self.miscellaneous._append(filename, check): + self.miscellaneous._update_config() + + if category != "": + if category == "disturbances": + _disturbance(filename, check=False) + elif category == "classifiers": + _classifier(filename, check=False) + elif category == "miscellaneous": + _miscellaneous(filename, check=False) + else: + # We can also raise an error here + raise UserWarning(f"Given category {category} not supported, supported categories are: {disturbances, classifiers, miscellaneous}") + else: + print(f"Category wasn't provided, attempting to deduce it from the file path: {file_path}") + if self.disturbances._append(filename): + self.disturbances._update_config() + return + if self.classifiers._append(filename): + self.classifiers._update_config() + return + if self.miscellaneous._append(filename): + self.miscellaneous._update_config() + return + print(f"Couldn't deduce a valid from the file path {file_path}, supported categories: {self.supported_categories}") - # TODO - # self._save(file_path) def sync_config(self, file_path): def _write_to_file(file_path, data): @@ -259,97 +305,119 @@ def _write_to_file(file_path, data): # Also update the dict self.files[file_path] = data + # TODO (@ankitaS11): We can just have these as class methods later, this will reduce the redundancy in the code later def update_disturbance_config(self): - self.disturbances._update_config() + self.json_paths.update(self.disturbances._update_config()) + + + def set_disturbance_attributes(self, file_path, payload): + self.json_paths.update(self.disturbances.setattr(file_path, payload)) - def set_disturbance_attributes(self, file, payload): - self.disturbances.setattr(file, payload) def update_classifier_config(self): - self.classifiers._update_config() + self.json_paths.update(self.classifiers._update_config()) + + + def set_classifier_attributes(self, file_path, payload): + self.json_paths.update(self.classifiers.setattr(file_path, payload)) - def set_classifier_attributes(self, file, payload): - self.classifiers.setattr(file, payload) def update_miscellaneous_config(self): - self.miscellaneous._update_config() + self.json_paths.update(self.miscellaneous._update_config()) + + + def set_miscellaneous_attributes(self, file_path, payload): + self.json_paths.update(self.miscellaneous.setattr(file_path, payload)) + + + # TODO: category should be an enum instead of a string to avoid any mistakes + def set_attributes(self, category: str, file_path: str, payload: dict): + base_path = os.path.basename(file_path) + if category == "disturbances": + self.set_disturbance_attributes(base_path, payload) + elif category == "classifiers": + self.set_classifier_attributes(base_path, payload) + elif category == "miscellaneous": + self.set_miscellaneous_attributes(base_path, payload) + else: + raise UserWarning(f"Expected a valid category name out of {self.supported_categories}, but got {category}") - def set_miscellaneous_attributes(self, file, payload): - self.miscellaneous.setattr(file, payload) @staticmethod - def safe_read_json(path): + def safe_read_json(file_path): # TODO: add read method for gcbm_config.cfg and logging.conf - if ".cfg" in path: + if ".cfg" in file_path: filename = os.path.join('input/test-run', "gcbm_config.cfg") - shutil.copy(path, filename) + shutil.copy(file_path, filename) return {} - if ".conf" in path: + + if ".conf" in file_path: filename = os.path.join('input/test-run', "logging.conf") - shutil.copy(path, filename) + shutil.copy(file_path, filename) return {} # check JSON - if ".json" not in path: - raise UserWarning(f"Given path {path} not a valid json file") - return {} + if ".json" not in file_path: + raise UserWarning(f"Given path {file_path} not a valid json file") # Make sure it's a file and not a directory - if not os.path.isfile(path): + if not os.path.isfile(file_path): raise UserWarning( - f"Got a directory {path} inside the config directory path, skipping it." + f"Got a directory {file_path} inside the config directory path, skipping it." ) - return {} - with open(path, "r") as json_file: + with open(file_path, "r") as json_file: data = json.load(json_file) return data if __name__ == "__main__": sim = GCBMSimulation() - sim.add_file("tests/GCBM_New_Demo_Run/disturbances/disturbances_2011.tiff") - sim.set_disturbance_attributes("disturbances_2011.tiff", {"year": 2011, "disturbance_type": "Wildfire", "transition": 1}) + sim.add_file("tests/tiff/new_demo_run/disturbances_2011_moja.tiff") + # sim.set_disturbance_attributes("disturbances_2011_moja.tiff", {"year": 2011, "disturbance_type": "Wildfire", "transition": 1}) + sim.set_attributes(category="disturbances", file_path="disturbances_2011_moja.tiff", payload={"year": 2011, "disturbance_type": "Wildfire", "transition": 1}) - sim.add_file("tests/GCBM_New_Demo_Run/disturbances/disturbances_2012.tiff") - sim.set_disturbance_attributes("disturbances_2012.tiff", + sim.add_file("tests/tiff/new_demo_run/disturbances_2012_moja.tiff") + sim.set_disturbance_attributes("disturbances_2012_moja.tiff", {"year": 2012, "disturbance_type": "Wildfire", "transition": 1}) - sim.add_file("tests/GCBM_New_Demo_Run/disturbances/disturbances_2013.tiff") - sim.set_disturbance_attributes("disturbances_2013.tiff", + sim.add_file("tests/tiff/new_demo_run/disturbances_2013_moja.tiff") + sim.set_disturbance_attributes("disturbances_2013_moja.tiff", {"year": 2013, "disturbance_type": "Mountain pine beetle — Very severe impact", "transition": 1}) # TODO: Check how to handle multiple attributes entries (L442-451 of `app.py:master`) - # sim.set_disturbance_attributes("disturbances_2013.tiff", + # sim.set_disturbance_attributes("disturbances_2013_moja.tiff", # {"year": 2013, "disturbance_type": "Wildfire", "transition": 1}) - sim.add_file("tests/GCBM_New_Demo_Run/disturbances/disturbances_2014.tiff") - sim.set_disturbance_attributes("disturbances_2014.tiff", + sim.add_file("tests/tiff/new_demo_run/disturbances_2014_moja.tiff") + sim.set_disturbance_attributes("disturbances_2014_moja.tiff", {"year": 2014, "disturbance_type": "Mountain pine beetle — Very severe impact", "transition": 1}) - sim.add_file("tests/GCBM_New_Demo_Run/disturbances/disturbances_2015.tiff") - sim.set_disturbance_attributes("disturbances_2015.tiff", + sim.add_file("tests/tiff/new_demo_run/disturbances_2015_moja.tiff") + sim.set_disturbance_attributes("disturbances_2015_moja.tiff", {"year": 2015, "disturbance_type": "Wildfire", "transition": 1}) - sim.add_file("tests/GCBM_New_Demo_Run/disturbances/disturbances_2016.tiff") - sim.set_disturbance_attributes("disturbances_2016.tiff", + sim.add_file("tests/tiff/new_demo_run/disturbances_2016_moja.tiff") + sim.set_disturbance_attributes("disturbances_2016_moja.tiff", {"year": 2016, "disturbance_type": "Wildfire", "transition": 1}) - sim.add_file("tests/GCBM_New_Demo_Run/disturbances/disturbances_2018.tiff") - sim.set_disturbance_attributes("disturbances_2018.tiff", + sim.add_file("tests/tiff/new_demo_run/disturbances_2018_moja.tiff") + sim.set_disturbance_attributes("disturbances_2018_moja.tiff", {"year": 2018, "disturbance_type": "Wildfire", "transition": 1}) # TODO: classifiers don't have 'year' attributes - sim.add_file("tests/GCBM_New_Demo_Run/classifiers/Classifier1.tiff") - # sim.set_classifier_attributes("classifier1.tiff", + sim.add_file("tests/tiff/new_demo_run/Classifier1_moja.tiff", category="classifiers") + # sim.set_classifier_attributes("Classifier1_moja.tiff", # {"1": "TA", "2": "BP", "3": "BS", "4": "JP", "5": "WS", "6": "WB", "7": "BF", "8": "GA"}) - sim.add_file("tests/GCBM_New_Demo_Run/classifiers/Classifier2.tiff") - # sim.set_classifier_attributes("classifier1.tiff", + sim.add_file("tests/tiff/new_demo_run/Classifier2_moja.tiff", category="classifiers") + # sim.set_classifier_attributes("Classifier2_moja.tiff", # {"1": "5", "2": "6", "3": "7", "4": "8"}) - sim.add_file("tests/GCBM_New_Demo_Run/db/gcbm_input.db") - sim.add_file("tests/GCBM_New_Demo_Run/miscellaneous/initial_age.tiff") - sim.add_file("tests/GCBM_New_Demo_Run/miscellaneous/mean_annual_temperature.tiff") + sim.add_file("tests/tiff/new_demo_run/initial_age_moja.tiff", category="miscellaneous") + sim.add_file("tests/tiff/new_demo_run/mean_annual_temperature_moja.tiff", category="miscellaneous") + + # TODO: make it work + # sim.add_file("tests/reference/gcbm_new_demo_run/gcbm_input.db") diff --git a/local/rest_api_gcbm/tests/tests_simulations.py b/local/rest_api_gcbm/tests/tests_simulations.py new file mode 100644 index 00000000..b2473854 --- /dev/null +++ b/local/rest_api_gcbm/tests/tests_simulations.py @@ -0,0 +1,99 @@ +# This file tests the following GCBM simulations: +# 1. GCBM Belize with disturbances +# 2. GCBM Belize without disturbances +# 3. GCBM New Demo Run + +import sys +sys.path.append("../") + +import pytest + +from gcbm import GCBMSimulation + + +belize_dict = { + "classifiers": ["tiff/belize/LifeZone_moja.tiff"], + "miscellaneous": ["tiff/belize/mean_annual_temperature_moja.tiff"], +} + +new_demo_run_dict = { + "disturbances": ["tiff/new_demo_run/disturbances_2011_moja.tiff", "tiff/new_demo_run/disturbances_2012_moja.tiff"], + "classifiers": ["tiff/new_demo_run/Classifier1_moja.tiff"], + "miscellaneous": ["tiff/new_demo_run/mean_annual_temperature_moja.tiff"], +} + +def filter_input_dict(input_dict, if_disturbance): + filtered_dict = input_dict + if not if_disturbance: + # Ignore disturbances from the dictionary + filtered_dict = {k: v for k, v in input_dict.items() if k != "disturbances"} + + # Do sanity check on input dict + def _sanity_check(dict): + assert "classifiers" in list(dict.keys()) and "miscellaneous" in list(dict.keys()), f"classifiers and miscellaneous keys expected in the dict but got {dict.keys()} for simulation {simulation_name}" + + _sanity_check(filtered_dict) + return filtered_dict + + +@pytest.mark.parametrize( + "simulation_name, input_dict, if_disturbance", + [ + ("belize", belize_dict, True), + ("belize", belize_dict, False), + ("new_demo_run", new_demo_run_dict, True), + ("new_demo_run", new_demo_run_dict, False), + ], +) +def test_simulation_add(simulation_name, input_dict, if_disturbance): + import os + + filtered_dict = filter_input_dict(input_dict, if_disturbance) + # Test that JSON files are generated for the configs of the simulation + gcbm = GCBMSimulation() + for key, val in filtered_dict.items(): + for path in val: + gcbm.add_file(category=key, file_path=path) + expected_path = "input/test-run/" + path.split("/")[-1] + assert os.path.isfile(expected_path), f"Failed for {simulation_name}, expected {expected_path} to exist." + + +@pytest.mark.parametrize( + "simulation_name, input_dict, if_disturbance", + [ + ("belize", belize_dict, True), + ("belize", belize_dict, False), + ("new_demo_run", new_demo_run_dict, True), + ("new_demo_run", new_demo_run_dict, False), + ], +) +def test_simulation_set_attr(simulation_name, input_dict, if_disturbance): + # Do sanity check on input dict + def sanity_check(dict): + assert "classifiers" in list(dict.keys()) and "miscellaneous" in list(dict.keys()), f"classifiers and miscellaneous keys expected in the dict but got {dict.keys()} for simulation {simulation_name}" + + # Assert if the json contains the expected data (payload) + def check_data_in_json(json_path, expected_data): + import json + with open(json_path, "r") as json_file: + json_obj = json.load(json_file) + for key, val in expected_data.items(): + assert json_obj["attributes"][key] == val + + print(f"Starting test for simulation {simulation_name}") + filtered_dict = input_dict + if not if_disturbance: + # Ignore disturbances from the dictionary + filtered_dict = {k: v for k, v in input_dict.items() if k != "disturbances"} + + sanity_check(filtered_dict) + + # Test that JSON files are generated for the configs of the simulation + import os + sim = GCBMSimulation() + for key, val in filtered_dict.items(): + for path in val: + sim.add_file(category=key, file_path=path) + payload = {"year": 2011, "disturbance_type": "Wildfire", "transition": 1} + sim.set_attributes(category=key, file_path=path, payload=payload) + check_data_in_json(json_path=sim.json_paths[os.path.basename(path)], expected_data=payload)