Feature 461 make compare images configurable (#467)

* 461: scatter tests and json compare

* 461: tidy up docs and whitespace

* 461: remove accidental git add

* 461: Make CompareImages configureable by env var

* 461: fix spurious fails

* 461: fix base dir setting

* 461: add omit to coverage report

* 461: remove redundant .coveragerc

pyproject.toml

@@ -33,6 +33,11 @@ testpaths = ["test"]
 
 [tool.coverage.run]
 source = ["metplotpy/plots"]
+omit = [
+    "config.py",
+    "config-3.py",
+    ]
+relative_files = true
 
 [tool.coverage.report]
 exclude_also = [

test/bar/test_bar.py

@@ -3,7 +3,7 @@
 import pytest
 
 from metplotpy.plots.bar import bar
-# from metcalcpy.compare_images import CompareImages
+from metcalcpy.compare_images import CompareImages
 
 cwd = os.path.dirname(__file__)
 CLEANUP_FILES = ['bar.png', 'bar.points1']
@@ -71,7 +71,6 @@ def test_no_nans_in_points_file(setup, remove_files):
     remove_files(cwd, CLEANUP_FILES)
 
 
-@pytest.mark.skip("fails on linux host machines")
 def test_images_match(setup, remove_files):
     """
         Compare an expected plot with the
@@ -83,7 +82,6 @@ def test_images_match(setup, remove_files):
     remove_files(cwd, CLEANUP_FILES)
 
 
-@pytest.mark.skip("fails on linux host machines")
 def test_none_data_images_match(setup_nones):
     """
         Compare an expected plot with the
@@ -141,7 +139,6 @@ def test_point_and_plot_files_exist_default(setup_env, remove_files):
     remove_files(cwd, check_files)
 
 
-@pytest.mark.skip("fails on linux host machines")
 def test_threshold_plotting(setup_env, remove_files):
     """
       Verify that the bar plot using data with thresholds is correct.

test/box/test_box.py

@@ -1,7 +1,7 @@
 import pytest
 import os
 from metplotpy.plots.box import box
-#from metcalcpy.compare_images import CompareImages
+from metcalcpy.compare_images import CompareImages
 
 cwd = os.path.dirname(__file__)
 CLEANUP_FILES = ['box.png', 'box.points1']
@@ -28,7 +28,6 @@ def test_files_exist(setup, test_input, expected, remove_files):
     remove_files(cwd, CLEANUP_FILES)
 
 
-@pytest.mark.skip("fails on linux hosts")
 def test_images_match(setup):
     """
         Compare an expected plot with the

test/conftest.py

@@ -1,5 +1,6 @@
 import pytest
 import os
+from unittest.mock import patch
 import shutil
 import json
 import xarray as xr
@@ -10,13 +11,38 @@
 # realative file locations can be used for each test
 # file.
 # NOTE: autouse=True means this applies to ALL tests.
-# Code that updates the cwd inside a test  file is now
-# redundant and can be deleted.
+# Code that updates the cwd inside test is now redundant
+# and can be deleted.
 @pytest.fixture(autouse=True)
 def change_test_dir(request, monkeypatch):
     monkeypatch.chdir(request.fspath.dirname)
 
 
+@pytest.fixture(autouse=True)
+def patch_CompareImages(request):
+    """This fixture controls the use of CompareImages in the
+    test suite. By default, all calls to CompareImages will
+    result in the test skipping. To change this behaviour set
+    an env var METPLOTPY_COMPAREIMAGES
+    """
+    if bool(os.getenv("METPLOTPY_COMPAREIMAGES")):
+        yield
+    else:
+        class mock_CompareImages:
+            def __init__(self, img1, img2):
+                # TODO: rather than skip we could inject an alternative
+                # comparison that is more relaxed. To do this, extend
+                # this this class to generate a self.mssim value.
+                pytest.skip("CompareImages not enabled in pytest. "
+                            "To enable `export METPLOTPY_COMPAREIMAGES=$true`")
+        try:
+            with patch.object(request.module, 'CompareImages', mock_CompareImages) as mock_ci:
+                yield mock_ci
+        except AttributeError:
+            # test module doesn't import CompareImages. Do nothing.
+            yield
+
+
 def ordered(obj):
     """Recursive function to sort JSON, even lists of dicts with the same keys"""
     if isinstance(obj, dict):
@@ -26,7 +52,6 @@ def ordered(obj):
     else:
         return obj
 
-
 @pytest.fixture
 def assert_json_equal():
     def compare_json(fig, expected_json_file):
@@ -36,7 +61,6 @@ def compare_json(fig, expected_json_file):
         actual = json.loads(fig.to_json(), parse_float=str, parse_int=str)
         with open(expected_json_file) as f:
             expected = json.load(f,parse_float=str, parse_int=str)
-
         # Fail with a nice message
         if ordered(actual) == ordered(expected):
             return True
@@ -79,6 +103,7 @@ def remove_the_files(test_dir, file_list):
     return remove_the_files
 
 
+# data for netCDF file
 TEST_NC_DATA = xr.Dataset(
     {
         "precip": xr.DataArray(

test/contour/test_contour.py

@@ -1,7 +1,7 @@
 import pytest
 import os
 from metplotpy.plots.contour import contour
-#from metcalcpy.compare_images import CompareImages
+from metcalcpy.compare_images import CompareImages
 
 cwd = os.path.dirname(__file__)
 
@@ -42,7 +42,7 @@ def test_files_exist(setup, test_input, expected):
     assert os.path.isfile(test_input) == expected
     cleanup()
 
-@pytest.mark.skip("fails on linux hosts")
+
 def test_images_match(setup):
     """
         Compare an expected plots with the

test/eclv/test_eclv.py

@@ -1,7 +1,7 @@
 import pytest
 import os
 from metplotpy.plots.eclv import eclv
-#from metcalcpy.compare_images import CompareImages
+from metcalcpy.compare_images import CompareImages
 
 cwd = os.path.dirname(__file__)
 
@@ -49,7 +49,6 @@ def test_files_exist(setup, test_input, expected):
     assert os.path.isfile(test_input) == expected
     cleanup()
 
-@pytest.mark.skip("fails on linux hosts")
 def test_images_match(setup):
     """
         Compare an expected plots with the

test/ens_ss/test_ens_ss.py

@@ -1,6 +1,6 @@
 import os
 import pytest
-#from metcalcpy.compare_images import CompareImages
+from metcalcpy.compare_images import CompareImages
 from metplotpy.plots.ens_ss import ens_ss
 
 cwd = os.path.dirname(__file__)
@@ -43,7 +43,7 @@ def test_files_exist( setup, test_input, expected):
     assert os.path.isfile(test_input) == expected
     cleanup()
 
-@pytest.mark.skip("fails on linux hosts")
+
 def test_images_match(setup):
     """
         Compare an expected plot with the

test/equivalence_testing_bounds/test_equivalence_testing_bounds.py

@@ -1,7 +1,7 @@
 import pytest
 import os
 from metplotpy.plots.equivalence_testing_bounds import equivalence_testing_bounds as etb
-#from metcalcpy.compare_images import CompareImages
+from metcalcpy.compare_images import CompareImages
 
 cwd = os.path.dirname(__file__)
 CLEANUP_FILES = ['equivalence_testing_bounds.png', 'equivalence_testing_bounds.points1']
@@ -29,7 +29,6 @@ def test_files_exist(setup, test_input, expected, remove_files):
     remove_files(cwd, CLEANUP_FILES)
 
 
-@pytest.mark.skip("skimage differences causing failure")
 def test_images_match(setup, remove_files):
     '''
         Compare an expected plot with the

test/histogram/test_prob_hist.py

@@ -1,7 +1,7 @@
 import pytest
 import os
 from metplotpy.plots.histogram import prob_hist
-#from metcalcpy.compare_images import CompareImages
+from metcalcpy.compare_images import CompareImages
 
 cwd = os.path.dirname(__file__)
 
@@ -35,7 +35,7 @@ def test_files_exist(setup, test_input, expected):
     assert os.path.isfile(f"{cwd}/{test_input}") == expected
     cleanup()
 
-@pytest.mark.skip("Image comparisons fail during Github Actions checks.")
+
 def test_images_match(setup):
     """
         Compare an expected plot with the

test/histogram/test_rank_hist.py

@@ -1,7 +1,7 @@
 import pytest
 import os
 from metplotpy.plots.histogram import rank_hist
-#from metcalcpy.compare_images import CompareImages
+from metcalcpy.compare_images import CompareImages
 
 cwd = os.path.dirname(__file__)
 
@@ -36,7 +36,6 @@ def test_files_exist(setup, test_input, expected):
     cleanup()
 
 
-@pytest.mark.skip("Image comparisons fail during Github Actions checks.")
 def test_images_match(setup):
     """
         Compare an expected plot with the

test/histogram/test_rel_hist.py

@@ -1,7 +1,7 @@
 import pytest
 import os
 from metplotpy.plots.histogram import rel_hist
-#from metcalcpy.compare_images import CompareImages
+from metcalcpy.compare_images import CompareImages
 
 cwd = os.path.dirname(__file__)
 
@@ -39,7 +39,6 @@ def test_files_exist(setup, test_input, expected):
     cleanup()
 
 
-@pytest.mark.skip("Image comparisons fail in Github Actions checks.")
 def test_images_match(setup):
     """
         Compare an expected plot with the

test/histogram_2d/test_histogram_2d.py

@@ -3,8 +3,6 @@
 
 from metplotpy.plots.histogram_2d import histogram_2d as h2d
 
-# from metcalcpy.compare_images import CompareImages
-
 cwd = os.path.dirname(__file__)
 
 

test/hovmoeller/test_hovmoeller.py

@@ -2,7 +2,7 @@
 import pytest
 import metplotpy.plots.hovmoeller.hovmoeller as hov
 from metplotpy.plots import util
-#from metcalcpy.compare_images import CompareImages
+from metcalcpy.compare_images import CompareImages
 
 def dict_to_yaml(data_dict,
                 output_yaml = "test_hovmoeller.yaml"):
@@ -22,8 +22,7 @@ def cleanup(file_to_remove):
         # don't exist.  Ignore.
         pass
 
-
-@pytest.mark.skip()
+@pytest.mark.skip("Requires specific netCDF input")
 def test_default_plot_images_match():
     '''
         Compare an expected plot with the

test/line/test_line_groups_plot.py

@@ -1,7 +1,7 @@
 import pytest
 import os
 from metplotpy.plots.line import line as l
-#from metcalcpy.compare_images import CompareImages
+from metcalcpy.compare_images import CompareImages
 
 cwd = os.path.dirname(__file__)
 
@@ -47,7 +47,7 @@ def test_files_exist(setup, test_input, expected):
     assert os.path.isfile(test_input) == expected
     cleanup()
 
-@pytest.mark.skip("fails on linux hosts")
+
 def test_images_match(setup):
     '''
         Compare an expected plot with the

test/line/test_line_plot.py

@@ -5,7 +5,7 @@
 
 cwd = os.path.dirname(__file__)
 
-# from metcalcpy.compare_images import CompareImages
+from metcalcpy.compare_images import CompareImages
 
 
 @pytest.fixture
@@ -147,7 +147,6 @@ def test_no_nans_in_points_files():
         pass
 
 
-@pytest.mark.skip()
 def test_images_match(setup):
     '''
         Compare an expected plot with the
@@ -164,7 +163,6 @@ def test_images_match(setup):
     cleanup()
 
 
-@pytest.mark.skip()
 def test_new_images_match():
     '''
         Compare an expected plot with the start_at_zero option, with the
@@ -323,21 +321,20 @@ def test_fixed_var_val():
         pass
 
 
-@pytest.mark.skip("Image comparison for development only due to differences in hosts")
+@pytest.mark.skip("This test currently raises an error on line 81 of line.py. This can"
+                  " probably be fixed be revisiting fbias_fixed_vars_vals.yaml")
 def test_fixed_var_val_image_compare():
     """
         Verify that the fixed_vars_vals_input setting reproduces the
         expected plot.
 
     """
-    from metcalcpy.compare_images import CompareImages
 
     # Set up the METPLOTPY_BASE so that met_plot.py will correctly find
     # the config directory containing all the default config files.
     os.environ['METPLOTPY_BASE'] = f"{cwd}/../../"
     os.environ['TEST_DIR'] = cwd
     custom_config_filename = f"{cwd}/fbias_fixed_vars_vals.yaml"
-
     # Invoke the command to generate a line plot based on
     # the custom config file.
     l.main(custom_config_filename)
@@ -349,10 +346,7 @@ def test_fixed_var_val_image_compare():
         created_file = os.path.join(cwd, plot_file)
 
         # first verify that the output plot was created
-        if os.path.exists(created_file):
-            assert True
-        else:
-            assert False
+        assert os.path.exists(created_file)
 
         comparison = CompareImages(expected_plot, created_file)
 

test/mpr_plot/test_mpr_plot.py

@@ -1,7 +1,7 @@
 import pytest
 import os
 from metplotpy.plots.mpr_plot import mpr_plot
-#from metcalcpy.compare_images import CompareImages
+from metcalcpy.compare_images import CompareImages
 
 cwd = os.path.dirname(__file__)
 CLEANUP_FILES = ['mpr_plots.png']
@@ -29,7 +29,6 @@ def test_files_exist(setup, test_input, expected, remove_files):
     remove_files(cwd, CLEANUP_FILES)
 
 
-@pytest.mark.skip("unreliable-sometimes fails due to differences between machines.")
 def test_images_match(setup, remove_files):
     """
         Compare an expected plot with the

test/performance_diagram/test_performance_diagram.py

@@ -2,7 +2,7 @@
 import pytest
 
 from metplotpy.plots.performance_diagram import performance_diagram as pd
-#from metcalcpy.compare_images import CompareImages
+from metcalcpy.compare_images import CompareImages
 
 cwd = os.path.dirname(__file__)
 
@@ -89,7 +89,6 @@ def test_files_exist(setup_env, test_input, expected_bool, remove_files):
         pass
 
 
-@pytest.mark.skip()
 def test_images_match(setup, remove_files):
     '''
         Compare an expected plot with the