feature #2253 tests for system_util

internal/tests/pytests/util/system_util/test_system_util.py

@@ -1,8 +1,10 @@
 #!/usr/bin/env python3
 
+import os
 import pytest
+from unittest import mock
 
-from metplus.util.system_util import *
+import metplus.util.system_util as su
 
 @pytest.mark.parametrize(
     'filename, expected_result', [
@@ -28,7 +30,7 @@ def test_get_storm_ids(metplus_config, filename, expected_result):
                             'stat_data',
                             filename)
 
-    assert get_storms(filepath, id_only=True) == expected_result
+    assert su.get_storms(filepath, id_only=True) == expected_result
 
 
 @pytest.mark.parametrize(
@@ -57,7 +59,7 @@ def test_get_storms(metplus_config, filename, expected_result):
                             'stat_data',
                             filename)
 
-    storm_dict = get_storms(filepath)
+    storm_dict = su.get_storms(filepath)
     print(storm_dict)
     assert list(storm_dict.keys()) == expected_result
     for storm_id in expected_result[1:]:
@@ -86,7 +88,7 @@ def test_get_storms_mtd(metplus_config):
                             'mtd',
                             'fake_mtd_2d.txt')
 
-    storm_dict = get_storms(filepath, sort_column=sort_column)
+    storm_dict = su.get_storms(filepath, sort_column=sort_column)
     print(storm_dict)
     assert list(storm_dict.keys()) == expected_result
     for storm_id in expected_result[1:]:
@@ -124,7 +126,7 @@ def test_preprocess_file_stage(metplus_config, filename, ext):
     else:
         stagepath = filepath
 
-    outpath = preprocess_file(filepath, None, conf)
+    outpath = su.preprocess_file(filepath, None, conf)
     assert stagepath == outpath and os.path.exists(outpath)
 
 
@@ -155,5 +157,119 @@ def test_preprocess_file_options(metplus_config,
     if filename == 'dir':
         filename = config.getdir('METPLUS_BASE')
         expected = filename
-    result = preprocess_file(filename, data_type, config, allow_dir)
+    result = su.preprocess_file(filename, data_type, config, allow_dir)
     assert result == expected
+
+
+@pytest.mark.parametrize(
+    'input_exists,expected', [
+        (False, '/some/fake/file.bigfoot'),
+        (True, None)
+    ]
+)
+@pytest.mark.util
+def test_preprocess_file_not_exist(metplus_config, input_exists, expected):
+    config = metplus_config
+    config.set('config', 'INPUT_MUST_EXIST', input_exists)
+    result = su.preprocess_file('/some/fake/file.bigfoot', None, config)
+    assert result == expected
+
+
+@pytest.mark.util
+def test_preprocess_file_gempack(tmp_path_factory, metplus_config):
+    config = metplus_config
+
+    # setup files and paths
+    tmp_dir = tmp_path_factory.mktemp('gempak')
+    config.set('config', 'STAGING_DIR', '')
+    file_path = os.path.join(tmp_dir, 'some_file.grd')
+    open(file_path, 'a').close()
+    expected = os.path.join(tmp_dir, 'some_file.nc')
+
+    # we need to import so as to mock .build()
+    from metplus.wrappers import GempakToCFWrapper
+
+    with mock.patch.object(GempakToCFWrapper, 'build') as mock_build:
+        result = su.preprocess_file(file_path, 'GEMPAK', config)
+        mock_build.assert_called_once()
+
+    assert result == expected
+
+
+@pytest.mark.parametrize(
+    'expected, user_err, id_err', [
+    ('James(007)', None, None),
+    ('007', OSError, None),
+    ('James', None, AttributeError),
+    ('', OSError, AttributeError),
+    ]
+)
+@pytest.mark.util
+def test_get_user_info(expected, user_err, id_err):
+    with mock.patch.object(su.getpass, 'getuser', return_value='James', side_effect=user_err):
+        with mock.patch.object(su.os, 'getuid', return_value='007', side_effect=id_err):
+                actual = su.get_user_info()
+    assert actual == expected
+
+
+@pytest.mark.util
+def test_write_list_to_file(tmp_path_factory):
+    filename = tmp_path_factory.mktemp('util') / 'temp.txt'
+    output_list =['some', 'file', 'content']
+    su.write_list_to_file(filename, output_list)
+
+    with open(filename, 'r') as f:
+        actual = f.read()
+
+    assert actual == '\n'.join(output_list) + '\n'
+
+
+@pytest.mark.util
+def test_prune_empty(tmp_path_factory):
+    prune_dir = tmp_path_factory.mktemp('prune')
+
+    dir1 = prune_dir / 'empty_file_dir'
+    dir2 = prune_dir / 'not_empty_file_dir'
+    dir3 = prune_dir / 'empty_dir'
+    for d in [dir1, dir2, dir3]:
+        os.makedirs(d)
+
+    # make two files, one empty one not.
+    open(os.path.join(dir1, 'empty.txt'), 'a').close()
+    file_with_content = os.path.join(dir2, 'not_empty.txt')
+    with open(file_with_content, 'w') as f:
+        f.write('Fee fi fo fum.')
+
+    su.prune_empty(prune_dir, mock.Mock())
+
+    assert not os.path.exists(dir1)
+    assert os.path.exists(file_with_content)
+    assert not os.path.exists(dir3)
+
+
+@pytest.mark.parametrize(
+    'regex, expected', [
+        (r'\d', ['bigfoot/123.txt', 'yeti/234.txt']),
+        (r'23', ['yeti/234.txt']),
+        (r'[\s\S]+nc4', ['yeti/sasquatch.nc4']),
+        ('ha', ['bigfoot/hahaha.nc', 'sasquatch/harry.txt'])
+    ]
+)
+@pytest.mark.util
+def test_get_files(tmp_path_factory, regex, expected):
+    search_dir = tmp_path_factory.mktemp('get_files')
+
+    dirs = {
+            'bigfoot':['123.txt', 'hahaha.nc'],
+            'yeti': ['234.txt', 'sasquatch.nc4'],
+            'sasquatch': ['harry.txt', 'hendersons.nc'],
+            }
+
+    for k, v in dirs.items():
+        tmp_dir = os.path.join(search_dir, k)
+        os.makedirs(tmp_dir)
+        [open(os.path.join(tmp_dir, f), 'a').close() for f in v]
+
+    actual = su.get_files(search_dir, regex)
+    assert actual == [os.path.join(search_dir, e) for e in expected]
+

metplus/util/system_util.py

@@ -145,7 +145,8 @@ def prune_empty(output_dir, logger):
 
 def get_files(filedir, filename_regex):
     """! Get all the files (with a particular naming format) by walking
-        through the directories.
+        through the directories. Note this uses re.match and will only
+        find matches at the beginning of the file name.
 
       @param filedir The topmost directory from which the search begins.
       @param filename_regex The regular expression that defines the naming
@@ -174,7 +175,10 @@ def preprocess_file(filename, data_type, config, allow_dir=False):
     """ Decompress gzip, bzip, or zip files or convert Gempak files to NetCDF
         Args:
             @param filename: Path to file without zip extensions
+            @param data_type: str of data_type for filename
             @param config: Config object
+            @param allow_dir (optional): bool to allow 'filename' to be a
+              directory. Default is False.
         Returns:
             Path to staged unzipped file or original file if already unzipped
     """