Revert "feat: use compiled flip for image comparison"

This reverts commit 0823a6b.

cato_common/images/image_comparators/flip_image_comparator.py

@@ -1,17 +1,17 @@
-import logging
 import os
+import subprocess
+import sys
 import uuid
+from pathlib import Path
 from typing import Optional
 
-import numpy
-from PIL import Image
-
 from cato_common.domain.comparison_result import ComparisonResult
 from cato_common.domain.comparison_settings import ComparisonSettings
 from cato_common.domain.result_status import ResultStatus
 
+import logging
+
 logger = logging.getLogger(__name__)
-import flip
 
 
 class FlipImageComparator:
@@ -39,55 +39,70 @@ def compare(
                 error=1,
             )
 
-        not_supported_format_error = self._verify_images_have_supported_format(
-            reference, output
-        )
-        if not_supported_format_error:
-            return ComparisonResult(
-                status=ResultStatus.FAILED,
-                message=not_supported_format_error,
-                diff_image=None,
-                error=1,
-            )
+        # check resolution
+        # compare images
 
-        # todo check resolution errors
-        try:
-            diff_image_np, mean_error, stats = flip.evaluate(
-                reference,
-                output,
-                "HDR" if reference.endswith(".exr") else "LDR",
-                parameters={},
-            )
-        except Exception as e:
-            message = str(e)
-            message = message.replace("\n", " ")
-            return ComparisonResult(
-                status=ResultStatus.FAILED,
-                message=f"FLIP: {message}",
-                diff_image=None,
-                error=0,
-            )
-        if diff_image_np.shape[0] == 0 or diff_image_np.shape[1] == 0:
-            raise ValueError("Could not read images!")
+        flip_executable = Path(__file__).parent / "nvidia_flip" / "flip.py"
+        diff_image_basename = f"diff_image_{uuid.uuid4()}"
+        args = [
+            sys.executable,
+            str(flip_executable),
+            "-r",
+            reference,
+            "-t",
+            output,
+            "--directory",
+            workdir,
+            "--basename",
+            diff_image_basename,
+            "--textfile",
+            "--start_exposure",
+            "0",
+            "--stop_exposure",
+            "1",
+        ]
+
+        status, process_output = subprocess.getstatusoutput(" ".join(args))
+        if status != 0:
+            if (
+                "Invalid image format" in process_output
+                or "cannot identify image file" in process_output
+            ):
+                raise ValueError(
+                    f"Could not read image from {output}, unsupported file format!"
+                )
+            if "Images have different resolutions" in process_output:
+                return ComparisonResult(
+                    status=ResultStatus.FAILED,
+                    message=process_output.split("\n")[-1].replace(
+                        "AssertionError: ", ""
+                    ),
+                    diff_image=None,
+                    error=0,
+                )
+            raise RuntimeError(process_output)
 
-        diff_image_path = os.path.join(workdir, f"diff_image_{uuid.uuid4()}.png")
-        diff_image = Image.fromarray(numpy.uint8(diff_image_np * 255))
-        diff_image.save(diff_image_path)
+        diff_image = os.path.join(workdir, diff_image_basename + ".png")
+        result_text_file = os.path.join(workdir, diff_image_basename + ".txt")
 
-        mean_error = round(mean_error, 6)
+        if not os.path.exists(result_text_file):
+            raise RuntimeError("result txt does not exist")
+        with open(result_text_file) as f:
+            mean_line = f.readline()
+        mean_error = float(mean_line.split(" ")[1])
 
         if mean_error > comparison_settings.threshold:
             return ComparisonResult(
                 status=ResultStatus.FAILED,
                 message=f"Images are not equal! FLIP mean error was {mean_error:.3f}, max threshold is {comparison_settings.threshold:.3f}",
-                diff_image=diff_image_path,
+                diff_image=diff_image,
                 error=mean_error,
             )
 
         return ComparisonResult(
             status=ResultStatus.SUCCESS,
             message=None,
-            diff_image=diff_image_path,
+            diff_image=diff_image,
             error=mean_error,
         )
 
@@ -99,13 +114,3 @@ def _verify_images_have_same_format(
 
         if reference_image_extension != output_image_extension:
             return f"FLIP does not support comparison of reference {reference_image_extension} to output {output_image_extension}, image need to have same format."
-
-    def _verify_images_have_supported_format(
-        self, reference: str, output: str
-    ) -> Optional[str]:
-        reference_image_extension = os.path.splitext(reference)[1]
-        output_image_extension = os.path.splitext(output)[1]
-        if reference_image_extension not in {".png", ".exr"}:
-            return f"FLIP does not support comparison of images with format {output_image_extension}. Only .png and .exr are supported."
-        if reference_image_extension not in {".png", ".exr"}:
-            return f"FLIP does not support comparison of images with format {output_image_extension}. Only .png and .exr are supported."

requirements.txt

@@ -8,6 +8,4 @@ imageio
 networkx
 tifffile
 pywavelets
-https://github.com/Latios96/flip-build-wheels/releases/download/v2024-05-29/flip-1.4-cp310-cp310-win_amd64.whl ; sys_platform == 'win32'
-https://github.com/Latios96/flip-build-wheels/releases/download/v2024-05-29/flip-1.4-cp310-cp310-linux_x86_64.whl ; sys_platform == 'linux'
 

scripts/generate_wheels.py

@@ -178,7 +178,6 @@ def find_wheel(name) -> Path:
             "scikit-image",
             "scipy",
             "pytracing",
-            "flip",
         ],
     )
 
@@ -240,7 +239,6 @@ def find_wheel(name) -> Path:
             "jsonpath-ng",
             "marshmallow-polyfield",
             "matplotlib",
-            "flip",
         ],
         include_files=[
             (

tests/unittests/cato_common/images/image_comparators/test_flip_image_comparator.py

@@ -65,7 +65,7 @@ def test_compare_image_should_fail_waith_and_without_watermark(
         status=ResultStatus.FAILED,
         message="Images are not equal! FLIP mean error was 0.102, max threshold is 0.000",
         diff_image=tmpdir.join("diff_image_c04b964d-f443-4ae9-8b43-47fe6d2422d0.png"),
-        error=0.102399,
+        error=0.102406,
     )
     assert images_are_visually_equal(
         comparison_result.diff_image,
@@ -147,27 +147,6 @@ def test_compare_image_should_fail_for_same_image_paths(
         )
 
 
-@pytest.mark.parametrize("format", [".tif", ".tiff", ".jpg", ".jpeg"])
-def test_compare_image_should_fail_when_comparing_images_that_are_not_png_or_exr(
-    tmpdir, format
-):
-    image_comparator = FlipImageComparator()
-
-    comparison_result = image_comparator.compare(
-        "reference" + format,
-        "output" + format,
-        ComparisonSettings(threshold=1, method=ComparisonMethod.FLIP),
-        str(tmpdir),
-    )
-
-    assert comparison_result == ComparisonResult(
-        status=ResultStatus.FAILED,
-        message=f"FLIP does not support comparison of images with format {format}. Only .png and .exr are supported.",
-        diff_image=None,
-        error=1,
-    )
-
-
 def test_compare_image_should_fail_when_comparing_png_to_exr(tmpdir):
     image_comparator = FlipImageComparator()
 
@@ -224,10 +203,32 @@ def test_compare_image_should_succeed_threshold_not_exceeded(
         status=ResultStatus.SUCCESS,
         message=None,
         diff_image=tmpdir.join("diff_image_c04b964d-f443-4ae9-8b43-47fe6d2422d0.png"),
-        error=0.102399,
+        error=0.102406,
     )
 
 
+@pytest.mark.parametrize(
+    "image1_name,image2_name",
+    [
+        ("unsupported-file.txt", "other-unsupported-file.txt"),
+    ],
+)
+def test_compare_image_should_fail_for_non_images(
+    image1_name, image2_name, test_resource_provider, tmpdir
+):
+    image1 = test_resource_provider.resource_by_name(image1_name)
+    image2 = test_resource_provider.resource_by_name(image2_name)
+    image_comparator = FlipImageComparator()
+
+    with pytest.raises(ValueError):
+        comparison_result = image_comparator.compare(
+            image1,
+            image2,
+            ComparisonSettings(threshold=1, method=ComparisonMethod.FLIP),
+            str(tmpdir),
+        )
+
+
 @pytest.mark.parametrize(
     "image_name,extension",
     [
@@ -282,7 +283,7 @@ def test_compare_image_should_fail_different_resolution(test_resource_provider,
 
     assert comparison_result == ComparisonResult(
         status=ResultStatus.FAILED,
-        message="FLIP: Reference and test image resolutions differ. Reference image resolution: 100x200 Test image resolution: 100x100",
+        message="Images have different resolutions! Reference image is 100x200px, output image is 100x100px",
         diff_image=None,
         error=0,
     )