diff --git a/cato_common/images/image_comparators/flip_image_comparator.py b/cato_common/images/image_comparators/flip_image_comparator.py index 8cdd5620..eeefd7ea 100644 --- a/cato_common/images/image_comparators/flip_image_comparator.py +++ b/cato_common/images/image_comparators/flip_image_comparator.py @@ -1,17 +1,17 @@ -import logging import os +import subprocess +import sys import uuid +from pathlib import Path from typing import Optional -import numpy -from PIL import Image - from cato_common.domain.comparison_result import ComparisonResult from cato_common.domain.comparison_settings import ComparisonSettings from cato_common.domain.result_status import ResultStatus +import logging + logger = logging.getLogger(__name__) -import flip class FlipImageComparator: @@ -39,55 +39,70 @@ def compare( error=1, ) - not_supported_format_error = self._verify_images_have_supported_format( - reference, output - ) - if not_supported_format_error: - return ComparisonResult( - status=ResultStatus.FAILED, - message=not_supported_format_error, - diff_image=None, - error=1, - ) + # check resolution + # compare images - # todo check resolution errors - try: - diff_image_np, mean_error, stats = flip.evaluate( - reference, - output, - "HDR" if reference.endswith(".exr") else "LDR", - parameters={}, - ) - except Exception as e: - message = str(e) - message = message.replace("\n", " ") - return ComparisonResult( - status=ResultStatus.FAILED, - message=f"FLIP: {message}", - diff_image=None, - error=0, - ) - if diff_image_np.shape[0] == 0 or diff_image_np.shape[1] == 0: - raise ValueError("Could not read images!") + flip_executable = Path(__file__).parent / "nvidia_flip" / "flip.py" + diff_image_basename = f"diff_image_{uuid.uuid4()}" + args = [ + sys.executable, + str(flip_executable), + "-r", + reference, + "-t", + output, + "--directory", + workdir, + "--basename", + diff_image_basename, + "--textfile", + "--start_exposure", + "0", + "--stop_exposure", + "1", + ] + + status, process_output = subprocess.getstatusoutput(" ".join(args)) + if status != 0: + if ( + "Invalid image format" in process_output + or "cannot identify image file" in process_output + ): + raise ValueError( + f"Could not read image from {output}, unsupported file format!" + ) + if "Images have different resolutions" in process_output: + return ComparisonResult( + status=ResultStatus.FAILED, + message=process_output.split("\n")[-1].replace( + "AssertionError: ", "" + ), + diff_image=None, + error=0, + ) + raise RuntimeError(process_output) - diff_image_path = os.path.join(workdir, f"diff_image_{uuid.uuid4()}.png") - diff_image = Image.fromarray(numpy.uint8(diff_image_np * 255)) - diff_image.save(diff_image_path) + diff_image = os.path.join(workdir, diff_image_basename + ".png") + result_text_file = os.path.join(workdir, diff_image_basename + ".txt") - mean_error = round(mean_error, 6) + if not os.path.exists(result_text_file): + raise RuntimeError("result txt does not exist") + with open(result_text_file) as f: + mean_line = f.readline() + mean_error = float(mean_line.split(" ")[1]) if mean_error > comparison_settings.threshold: return ComparisonResult( status=ResultStatus.FAILED, message=f"Images are not equal! FLIP mean error was {mean_error:.3f}, max threshold is {comparison_settings.threshold:.3f}", - diff_image=diff_image_path, + diff_image=diff_image, error=mean_error, ) return ComparisonResult( status=ResultStatus.SUCCESS, message=None, - diff_image=diff_image_path, + diff_image=diff_image, error=mean_error, ) @@ -99,13 +114,3 @@ def _verify_images_have_same_format( if reference_image_extension != output_image_extension: return f"FLIP does not support comparison of reference {reference_image_extension} to output {output_image_extension}, image need to have same format." - - def _verify_images_have_supported_format( - self, reference: str, output: str - ) -> Optional[str]: - reference_image_extension = os.path.splitext(reference)[1] - output_image_extension = os.path.splitext(output)[1] - if reference_image_extension not in {".png", ".exr"}: - return f"FLIP does not support comparison of images with format {output_image_extension}. Only .png and .exr are supported." - if reference_image_extension not in {".png", ".exr"}: - return f"FLIP does not support comparison of images with format {output_image_extension}. Only .png and .exr are supported." diff --git a/requirements.txt b/requirements.txt index 04ff96a4..4ef028a2 100644 --- a/requirements.txt +++ b/requirements.txt @@ -8,6 +8,4 @@ imageio networkx tifffile pywavelets -https://github.com/Latios96/flip-build-wheels/releases/download/v2024-05-29/flip-1.4-cp310-cp310-win_amd64.whl ; sys_platform == 'win32' -https://github.com/Latios96/flip-build-wheels/releases/download/v2024-05-29/flip-1.4-cp310-cp310-linux_x86_64.whl ; sys_platform == 'linux' diff --git a/scripts/generate_wheels.py b/scripts/generate_wheels.py index 8604224e..c5564602 100644 --- a/scripts/generate_wheels.py +++ b/scripts/generate_wheels.py @@ -178,7 +178,6 @@ def find_wheel(name) -> Path: "scikit-image", "scipy", "pytracing", - "flip", ], ) @@ -240,7 +239,6 @@ def find_wheel(name) -> Path: "jsonpath-ng", "marshmallow-polyfield", "matplotlib", - "flip", ], include_files=[ ( diff --git a/tests/resources/sphere_test_images/expected_diff_flip/exr_singlechannel_16_bit.png b/tests/resources/sphere_test_images/expected_diff_flip/exr_singlechannel_16_bit.png index 1742289c..030d2e0c 100644 Binary files a/tests/resources/sphere_test_images/expected_diff_flip/exr_singlechannel_16_bit.png and b/tests/resources/sphere_test_images/expected_diff_flip/exr_singlechannel_16_bit.png differ diff --git a/tests/resources/sphere_test_images/expected_diff_flip/exr_singlechannel_32_bit.png b/tests/resources/sphere_test_images/expected_diff_flip/exr_singlechannel_32_bit.png index 2cb16959..c278f1e7 100644 Binary files a/tests/resources/sphere_test_images/expected_diff_flip/exr_singlechannel_32_bit.png and b/tests/resources/sphere_test_images/expected_diff_flip/exr_singlechannel_32_bit.png differ diff --git a/tests/unittests/cato_common/images/image_comparators/test_flip_image_comparator.py b/tests/unittests/cato_common/images/image_comparators/test_flip_image_comparator.py index 34f33e18..0a7a33a8 100644 --- a/tests/unittests/cato_common/images/image_comparators/test_flip_image_comparator.py +++ b/tests/unittests/cato_common/images/image_comparators/test_flip_image_comparator.py @@ -65,7 +65,7 @@ def test_compare_image_should_fail_waith_and_without_watermark( status=ResultStatus.FAILED, message="Images are not equal! FLIP mean error was 0.102, max threshold is 0.000", diff_image=tmpdir.join("diff_image_c04b964d-f443-4ae9-8b43-47fe6d2422d0.png"), - error=0.102399, + error=0.102406, ) assert images_are_visually_equal( comparison_result.diff_image, @@ -147,27 +147,6 @@ def test_compare_image_should_fail_for_same_image_paths( ) -@pytest.mark.parametrize("format", [".tif", ".tiff", ".jpg", ".jpeg"]) -def test_compare_image_should_fail_when_comparing_images_that_are_not_png_or_exr( - tmpdir, format -): - image_comparator = FlipImageComparator() - - comparison_result = image_comparator.compare( - "reference" + format, - "output" + format, - ComparisonSettings(threshold=1, method=ComparisonMethod.FLIP), - str(tmpdir), - ) - - assert comparison_result == ComparisonResult( - status=ResultStatus.FAILED, - message=f"FLIP does not support comparison of images with format {format}. Only .png and .exr are supported.", - diff_image=None, - error=1, - ) - - def test_compare_image_should_fail_when_comparing_png_to_exr(tmpdir): image_comparator = FlipImageComparator() @@ -224,10 +203,32 @@ def test_compare_image_should_succeed_threshold_not_exceeded( status=ResultStatus.SUCCESS, message=None, diff_image=tmpdir.join("diff_image_c04b964d-f443-4ae9-8b43-47fe6d2422d0.png"), - error=0.102399, + error=0.102406, ) +@pytest.mark.parametrize( + "image1_name,image2_name", + [ + ("unsupported-file.txt", "other-unsupported-file.txt"), + ], +) +def test_compare_image_should_fail_for_non_images( + image1_name, image2_name, test_resource_provider, tmpdir +): + image1 = test_resource_provider.resource_by_name(image1_name) + image2 = test_resource_provider.resource_by_name(image2_name) + image_comparator = FlipImageComparator() + + with pytest.raises(ValueError): + comparison_result = image_comparator.compare( + image1, + image2, + ComparisonSettings(threshold=1, method=ComparisonMethod.FLIP), + str(tmpdir), + ) + + @pytest.mark.parametrize( "image_name,extension", [ @@ -282,7 +283,7 @@ def test_compare_image_should_fail_different_resolution(test_resource_provider, assert comparison_result == ComparisonResult( status=ResultStatus.FAILED, - message="FLIP: Reference and test image resolutions differ. Reference image resolution: 100x200 Test image resolution: 100x100", + message="Images have different resolutions! Reference image is 100x200px, output image is 100x100px", diff_image=None, error=0, )