diff --git a/.github/actions/run_tests/Dockerfile.run_geovista b/.github/actions/run_tests/Dockerfile.run_geovista
new file mode 100644
index 0000000000..c8d8d82310
--- /dev/null
+++ b/.github/actions/run_tests/Dockerfile.run_geovista
@@ -0,0 +1,15 @@
+ARG METPLUS_ENV_TAG=metplus_base.v5
+ARG METPLUS_IMG_TAG=develop
+
+FROM dtcenter/metplus-envs:${METPLUS_ENV_TAG} as env
+
+ARG METPLUS_IMG_TAG=develop
+FROM dtcenter/metplus-dev:${METPLUS_IMG_TAG}
+
+RUN mkdir -p /usr/local/conda/envs && mkdir -p /usr/local/conda/bin
+COPY --from=env /usr/local/conda/envs /usr/local/conda/envs/
+COPY --from=env /usr/local/conda/bin/conda /usr/local/conda/bin/conda
+
+# copy libGL and libEGL libraries to prevent dynamic lib errors
+COPY --from=env /lib/x86_64-linux-gnu/libEGL* /lib/x86_64-linux-gnu/
+COPY --from=env /lib/x86_64-linux-gnu/libGL* /lib/x86_64-linux-gnu/
diff --git a/.github/jobs/setup_and_run_use_cases.py b/.github/jobs/setup_and_run_use_cases.py
index 8ed341ec4a..931a79d76d 100755
--- a/.github/jobs/setup_and_run_use_cases.py
+++ b/.github/jobs/setup_and_run_use_cases.py
@@ -181,6 +181,8 @@ def _get_dockerfile_name(requirements):
return f'{dockerfile_name}_gfdl'
if 'cartopy' in str(requirements).lower():
return f'{dockerfile_name}_cartopy'
+ if 'geovista' in str(requirements).lower():
+ return f'{dockerfile_name}_geovista'
return dockerfile_name
diff --git a/.github/parm/use_case_groups.json b/.github/parm/use_case_groups.json
index 1752f76df0..041b000b2c 100644
--- a/.github/parm/use_case_groups.json
+++ b/.github/parm/use_case_groups.json
@@ -277,6 +277,6 @@
{
"category": "unstructured_grids",
"index_list": "0",
- "run": false
+ "run": true
}
]
diff --git a/internal/tests/pytests/conftest.py b/internal/tests/pytests/conftest.py
index 795a45f802..299410036f 100644
--- a/internal/tests/pytests/conftest.py
+++ b/internal/tests/pytests/conftest.py
@@ -103,7 +103,15 @@ def test_example(metplus_config):
config.logger = mock.MagicMock()
yield config
-
+
+ if config.logger.error.call_args_list:
+ err_msgs = [
+ str(msg.args[0])
+ for msg
+ in config.logger.error.call_args_list
+ if len(msg.args) != 0]
+ print("Tests raised the following errors:")
+ print("\n".join(err_msgs))
config.logger = old_logger
# don't remove output base if test fails
if request.node.rep_call.failed:
diff --git a/internal/tests/pytests/wrappers/met_db_load/test_met_db_load.py b/internal/tests/pytests/wrappers/met_db_load/test_met_db_load.py
index 6003be50e5..ba6eaf2c67 100644
--- a/internal/tests/pytests/wrappers/met_db_load/test_met_db_load.py
+++ b/internal/tests/pytests/wrappers/met_db_load/test_met_db_load.py
@@ -1,20 +1,141 @@
#!/usr/bin/env python3
+import datetime
import pytest
-
+import os
+from unittest import mock
from metplus.wrappers.met_db_load_wrapper import METDbLoadWrapper
+time_fmt = "%Y%m%d%H"
+run_times = ["2023080700", "2023080712", "2023080800"]
+
+time_info = {
+ "loop_by": "init",
+ "init": datetime.datetime(2023, 8, 7, 0, 0),
+ "now": datetime.datetime(2023, 8, 7, 0, 0),
+ "today": "20230830",
+ "instance": "",
+ "valid": "*",
+ "lead": "*",
+}
+
+xml_template = """
+
+
+ ${METPLUS_MV_HOST}
+ ${METPLUS_MV_DATABASE}
+ ${METPLUS_MV_USER}
+ ${METPLUS_MV_PASSWORD}
+
+
+ ${METPLUS_MV_VERBOSE}
+ ${METPLUS_MV_INSERT_SIZE}
+ ${METPLUS_MV_MODE_HEADER_DB_CHECK}
+ ${METPLUS_MV_DROP_INDEXES}
+ ${METPLUS_MV_APPLY_INDEXES}
+ ${METPLUS_MV_GROUP}
+ ${METPLUS_MV_LOAD_STAT}
+ ${METPLUS_MV_LOAD_MODE}
+ ${METPLUS_MV_LOAD_MTD}
+ ${METPLUS_MV_LOAD_MPR}
+
+"""
+
+xml_expected = """
+
+
+ db_host
+ db
+ user
+ big_secret
+
+
+ true
+ 128
+ true
+ false
+ true
+ group
+ true
+ true
+ true
+ true
+
+
+"""
+
+tmp_file_dict = {
+ "dir1": {"subdir1": ["file1.stat", "file2.tcst"]},
+ "dir2": ["file2.stat"],
+}
+
+
+def make_tmp_files(tmp_dir, structure=tmp_file_dict):
+ """
+ Recursive function to make a directory structure
+ and populate with empty files.
+ """
+ for key, val in structure.items():
+ this_dir = os.path.join(tmp_dir, key)
+ os.mkdir(this_dir)
+ if isinstance(val, dict):
+ make_tmp_files(os.path.join(tmp_dir, key), val)
+ elif isinstance(val, list):
+ # make empty files
+ for f in val:
+ open(os.path.join(this_dir, f), "w").close()
+
+
+# Helper class for string matching
+class MatchSubstring(str):
+ def __eq__(self, other):
+ return self in other
+
+
+def set_minimum_config_settings(config):
+ # set config variables to prevent command from running and bypass check
+ # if input files actually exist
+ config.set("config", "DO_NOT_RUN_EXE", True)
+ config.set("config", "INPUT_MUST_EXIST", False)
+
+ # set process and time config variables
+ config.set("config", "PROCESS_LIST", "METDbLoad")
+ config.set("config", "LOOP_BY", "INIT")
+ config.set("config", "INIT_TIME_FMT", time_fmt)
+ config.set("config", "INIT_BEG", run_times[0])
+ config.set("config", "INIT_END", run_times[-1])
+ config.set("config", "INIT_INCREMENT", "12H")
+ config.set("config", "LEAD_SEQ", "12H")
+ config.set("config", "LOOP_ORDER", "processes")
+
+ config.set("config", "MET_DB_LOAD_RUNTIME_FREQ", "RUN_ONCE_PER_INIT_OR_VALID")
+ config.set("config", "MET_DB_LOAD_MV_HOST", "db_host")
+ config.set("config", "MET_DB_LOAD_MV_DATABASE", "db")
+ config.set("config", "MET_DB_LOAD_MV_USER", "user")
+ config.set("config", "MET_DB_LOAD_MV_PASSWORD", "big_secret")
+ config.set("config", "MET_DB_LOAD_MV_VERBOSE", True)
+ config.set("config", "MET_DB_LOAD_MV_INSERT_SIZE", 128)
+ config.set("config", "MET_DB_LOAD_MV_MODE_HEADER_DB_CHECK", True)
+ config.set("config", "MET_DB_LOAD_MV_DROP_INDEXES", False)
+ config.set("config", "MET_DB_LOAD_MV_APPLY_INDEXES", True)
+ config.set("config", "MET_DB_LOAD_MV_GROUP", "group")
+ config.set("config", "MET_DB_LOAD_MV_LOAD_STAT", True)
+ config.set("config", "MET_DB_LOAD_MV_LOAD_MODE", True)
+ config.set("config", "MET_DB_LOAD_MV_LOAD_MTD", True)
+ config.set("config", "MET_DB_LOAD_MV_LOAD_MPR", True)
+
@pytest.mark.parametrize(
- 'filename, expected_result', [
- ('myfile.png', False),
- ('anotherfile.txt', False),
- ('goodfile.stat', True),
- ('goodfile.tcst', True),
- ('mode_goodfile.txt', True),
- ('mtd_goodfile.txt', True),
- ('monster_badfile.txt', False),
- ]
+ "filename, expected_result",
+ [
+ ("myfile.png", False),
+ ("anotherfile.txt", False),
+ ("goodfile.stat", True),
+ ("goodfile.tcst", True),
+ ("mode_goodfile.txt", True),
+ ("mtd_goodfile.txt", True),
+ ("monster_badfile.txt", False),
+ ],
)
@pytest.mark.wrapper
def test_is_loadable_file(filename, expected_result):
@@ -22,23 +143,112 @@ def test_is_loadable_file(filename, expected_result):
@pytest.mark.parametrize(
- 'filenames, expected_result', [
- (['myfile.png',
- 'anotherfile.txt'], False),
- (['myfile.png',
- 'goodfile.stat'], True),
- (['myfile.png',
- 'goodfile.tcst',
- 'anotherfile.txt'], True),
- (['myfile.png',
- 'mode_goodfile.txt'], True),
- (['myfile.png',
- 'mtd_goodfile.txt'], True),
- (['myfile.png',
- 'monster_badfile.txt'], False),
+ "filenames, expected_result",
+ [
+ (["myfile.png", "anotherfile.txt"], False),
+ (["myfile.png", "goodfile.stat"], True),
+ (["myfile.png", "goodfile.tcst", "anotherfile.txt"], True),
+ (["myfile.png", "mode_goodfile.txt"], True),
+ (["myfile.png", "mtd_goodfile.txt"], True),
+ (["myfile.png", "monster_badfile.txt"], False),
([], False),
- ]
+ ],
)
@pytest.mark.wrapper
def test_has_loadable_file(filenames, expected_result):
assert METDbLoadWrapper._has_loadable_file(filenames) == expected_result
+
+
+@pytest.mark.wrapper
+def test_METDbLoadWrapper_config(metplus_config):
+ config = metplus_config
+ set_minimum_config_settings(config)
+
+ expected = {
+ "MV_HOST": "",
+ "FIND_FILES": False,
+ "INPUT_TEMPLATE": "template.file",
+ "XML_TEMPLATE": "xml.file",
+ "MV_DATABASE": "db",
+ "MV_USER": "user",
+ "MV_PASSWORD": "big_secret",
+ "MV_VERBOSE": True,
+ "MV_INSERT_SIZE": 128,
+ }
+
+ wrapper = METDbLoadWrapper(config)
+ wrapper.logger.error.assert_any_call(MatchSubstring("Must supply an XML file"))
+
+ config.set("config", "MET_DB_LOAD_XML_FILE", "xml.file")
+ wrapper = METDbLoadWrapper(config)
+ wrapper.logger.error.assert_any_call(
+ MatchSubstring("Must supply an input template with")
+ )
+
+ config.set("config", "MET_DB_LOAD_INPUT_TEMPLATE", "template.file")
+ wrapper = METDbLoadWrapper(config)
+
+ config.set("config", "MET_DB_LOAD_MV_HOST", "")
+ wrapper = METDbLoadWrapper(config)
+ wrapper.logger.error.assert_any_call(MatchSubstring("Must set MET_DB_LOAD_MV_HOST"))
+
+ for k, v in expected.items():
+ assert wrapper.c_dict[k] == v
+
+ wrapper.c_dict["XML_TMP_FILE"] = "xml_tmp"
+ assert wrapper.get_command() == f"python3 {wrapper.app_path}.py xml_tmp"
+
+
+@pytest.mark.wrapper
+def test_METDbLoadWrapper(tmp_path_factory, metplus_config):
+ config = metplus_config
+ set_minimum_config_settings(config)
+
+ # make the temp files needed to run wrapper
+ tmp_dir = tmp_path_factory.mktemp("METdbLoad")
+ file_name = "tmplate.xml"
+ xml_file = os.path.join(tmp_dir, file_name)
+ with open(xml_file, "w") as f:
+ f.write(xml_template)
+
+ make_tmp_files(tmp_dir)
+
+ # check wrapper runs
+ config.set("config", "TMP_DIR", tmp_dir)
+ config.set("config", "MET_DB_LOAD_REMOVE_TMP_XML", False)
+ config.set("config", "MET_DB_LOAD_RUNTIME_FREQ", "RUN_ONCE_PER_INIT_OR_VALID")
+ config.set("config", "MET_DB_LOAD_XML_FILE", xml_file)
+ config.set("config", "MET_DB_LOAD_INPUT_TEMPLATE", tmp_dir)
+ wrapper = METDbLoadWrapper(config)
+ all_cmds = wrapper.run_all_times()
+
+ assert wrapper.isOK
+ assert wrapper.logger.error.assert_not_called
+ assert len(all_cmds) == 3
+
+ # check first tmp file has correct content
+ actual_xml = all_cmds[0][0].split()[-1]
+ with open(actual_xml, "r") as f:
+ assert f.read() == xml_expected
+
+ # check temp files deleted
+ config.set("config", "MET_DB_LOAD_REMOVE_TMP_XML", True)
+ wrapper = METDbLoadWrapper(config)
+ all_cmds = wrapper.run_all_times()
+ assert not os.path.exists(all_cmds[0][0].split()[-1])
+
+ # check correct return on failure
+ with mock.patch.object(wrapper, "build", return_value=False):
+ assert wrapper.run_at_time_once(time_info) == False
+
+ with mock.patch.object(wrapper, "replace_values_in_xml", return_value=False):
+ assert wrapper.run_at_time_once(time_info) == None
+
+ with mock.patch.dict(wrapper.c_dict, {"XML_TEMPLATE": False}):
+ # wrapper.c_dict['XML_TEMPLATE'] = None
+ assert wrapper.replace_values_in_xml(time_info) == False
+
+ # check handelling other times
+ time_info["lead"] = 3600
+ assert wrapper.run_at_time_once(time_info) == True
+