Updated pillow, unlocked test

changelog.d/20231227_100059_boris_update_pillow.md

@@ -0,0 +1,6 @@
+### Fixed
+
+- `TIFF` images are saved as `JPEG` images with `.tif` extension in original chunks
+  (<https://github.com/opencv/cvat/pull/7100>)
+- EXIF rotated TIFF images are handled incorrectly
+  (<https://github.com/opencv/cvat/pull/7100>)

cvat-cli/requirements/base.txt

@@ -1,3 +1,3 @@
 cvat-sdk~=2.10.0
-Pillow>=10.0.1
+Pillow>=10.1.0
 setuptools>=65.5.1 # not directly required, pinned by Snyk to avoid a vulnerability

cvat-sdk/requirements/base.txt

@@ -2,7 +2,7 @@
 
 attrs >= 21.4.0
 packaging >= 21.3
-Pillow >= 10.0.1
+Pillow >= 10.1.0
 platformdirs >= 2.1.0
 tqdm >= 4.64.0
 tuspy == 0.2.5 # have it pinned, because SDK has lots of patched TUS code

cvat/apps/engine/media_extractors.py

@@ -660,12 +660,17 @@ def save_as_chunk(self, images: Iterable[tuple[Image.Image|io.IOBase|str, str, s
                 ext = os.path.splitext(path)[1].replace('.', '')
                 output = io.BytesIO()
                 if self._dimension == DimensionType.DIM_2D:
-                    if has_exif_rotation(image):
+                    # current version of Pillow applies exif rotation immediately when TIFF image opened
+                    # and it removes rotation tag after that
+                    # so, has_exif_rotation(image) will return False for TIFF images even if they were actually rotated
+                    # and original files will be added to the archive (without applied rotation)
+                    # that is why we need the second part of the condition
+                    if has_exif_rotation(image) or image.format == 'TIFF':
                         rot_image = ImageOps.exif_transpose(image)
                         try:
-                            if rot_image.format == 'TIFF':
-                            # https://pillow.readthedocs.io/en/stable/handbook/image-file-formats.html
-                            # use loseless lzw compression for tiff images
+                            if image.format == 'TIFF':
+                                # https://pillow.readthedocs.io/en/stable/handbook/image-file-formats.html
+                                # use loseless lzw compression for tiff images
                                 rot_image.save(output, format='TIFF', compression='tiff_lzw')
                             else:
                                 rot_image.save(

cvat/requirements/base.in

@@ -39,7 +39,7 @@ opencv-python-headless~=4.8
 patool==1.12
 
 pdf2image==1.14.0
-Pillow>=10.0.1
+Pillow>=10.1.0
 psutil==5.9.4
 psycopg2-binary==2.9.5
 python-ldap==3.4.3

tests/python/requirements.txt

@@ -5,7 +5,7 @@ pytest-cov==4.1.0
 requests==2.31.0
 deepdiff==5.6.0
 boto3==1.17.61
-Pillow==10.0.1
+Pillow==10.1.0
 python-dateutil==2.8.2
 pyyaml==6.0.0
 numpy==1.22.0

tests/python/rest_api/test_tasks.py

@@ -837,6 +837,7 @@ def test_can_create_task_with_exif_rotated_images(self):
                             # original is 480x640 with 90/-90 degrees rotation
                             assert frame_meta.height == 640 and frame_meta.width == 480
                             assert im.height == 640 and im.width == 480
+                            assert im.getexif().get(274, 1) == 1
 
     def test_can_create_task_with_big_images(self):
         # Checks for regressions about the issue
@@ -885,7 +886,6 @@ def test_can_create_task_with_big_images(self):
             chunk_image = chunk_zip.read(infos[0])
             assert chunk_image == image_bytes
 
-    @pytest.mark.skip(reason="need to wait new Pillow release till 15 October 2023")
     def test_can_create_task_with_exif_rotated_tif_image(self):
         task_spec = {
             "name": f"test {self._USERNAME} to create a task with exif rotated tif image",
@@ -901,37 +901,28 @@ def test_can_create_task_with_exif_rotated_tif_image(self):
             "server_files": image_files,
             "image_quality": 70,
             "segment_size": 500,
-            "use_cache": True,
+            "use_cache": False,
             "sorting_method": "natural",
         }
 
         task_id, _ = create_task(self._USERNAME, task_spec, task_data)
 
-        # check that the frame has correct width and height
-        with make_api_client(self._USERNAME) as api_client:
-            _, response = api_client.tasks_api.retrieve_data(
-                task_id, number=0, type="chunk", quality="original"
-            )
-            with zipfile.ZipFile(io.BytesIO(response.data)) as zip_file:
-                assert len(zip_file.namelist()) == 1
-                name = zip_file.namelist()[0]
-                assert name == "000000.tif"
-                with zip_file.open(name) as zipped_img:
-                    im = Image.open(zipped_img)
-                    # raw image is horizontal 100x150 with -90 degrees rotation
-                    assert im.height == 150 and im.width == 100
+        for chunk_quality in ["original", "compressed"]:
+            # check that the frame has correct width and height
+            with make_api_client(self._USERNAME) as api_client:
+                _, response = api_client.tasks_api.retrieve_data(
+                    task_id, number=0, type="chunk", quality=chunk_quality
+                )
 
-            _, response = api_client.tasks_api.retrieve_data(
-                task_id, number=0, type="chunk", quality="compressed"
-            )
-            with zipfile.ZipFile(io.BytesIO(response.data)) as zip_file:
-                assert len(zip_file.namelist()) == 1
-                name = zip_file.namelist()[0]
-                assert name == "000000.jpeg"
-                with zip_file.open(name) as zipped_img:
-                    im = Image.open(zipped_img)
-                    # raw image is horizontal 100x150 with -90 degrees rotation
-                    assert im.height == 150 and im.width == 100
+                with zipfile.ZipFile(io.BytesIO(response.data)) as zip_file:
+                    assert len(zip_file.namelist()) == 1
+                    name = zip_file.namelist()[0]
+                    assert name == "000000.tif" if chunk_quality == "original" else "000000.jpeg"
+                    with zip_file.open(name) as zipped_img:
+                        im = Image.open(zipped_img)
+                        # raw image is horizontal 100x150 with -90 degrees rotation
+                        assert im.height == 150 and im.width == 100
+                        assert im.getexif().get(274, 1) == 1
 
     def test_can_create_task_with_sorting_method_natural(self):
         task_spec = {

utils/dataset_manifest/requirements.in

@@ -1,5 +1,5 @@
 av==9.2.0  # Pinned for the whole CVAT
 natsort>=8.0.0
 opencv-python-headless>=4.4.0.42
-Pillow>=10.0.1
+Pillow>=10.1.0
 tqdm>=4.58.0

utils/dataset_manifest/requirements.txt

@@ -13,7 +13,7 @@ numpy==1.22.4
     # via opencv-python-headless
 opencv-python-headless==4.8.1.78
     # via -r utils/dataset_manifest/requirements.in
-pillow==10.0.1
+pillow==10.1.0
     # via -r utils/dataset_manifest/requirements.in
 tqdm==4.66.1
     # via -r utils/dataset_manifest/requirements.in

utils/dicom_converter/requirements.txt

@@ -1,4 +1,4 @@
 numpy==1.22.0
-Pillow==10.0.1
+Pillow==10.1.0
 pydicom==2.1.2
 tqdm==4.60.0