Map futures (#120)

* convert to map_to_futures
* tests
* tests
* check black for all + remove testdata
* black all

.gitignore

@@ -96,3 +96,4 @@ ENV/
 
 testoutput
 testdata/WT1_wkw
+testdata/tiff_mag_2_reference

setup.py

@@ -4,8 +4,16 @@
     name="wkcuber",
     packages=find_packages(exclude=("tests",)),
     use_scm_version=True,
-    setup_requires=['setuptools_scm'],
-    install_requires=["scipy", "numpy", "pillow", "pyyaml", "wkw", "cluster_tools==1.36", "natsort"],
+    setup_requires=["setuptools_scm"],
+    install_requires=[
+        "scipy",
+        "numpy",
+        "pillow",
+        "pyyaml",
+        "wkw",
+        "cluster_tools==1.36",
+        "natsort",
+    ],
     description="A cubing tool for webKnossos",
     author="Norman Rzepka",
     author_email="[email protected]",

tests/scripts/check_formatting.sh

@@ -4,4 +4,4 @@ docker run \
   --entrypoint "/bin/bash" \
   -w "/" \
   scalableminds/webknossos-cuber:${CIRCLE_BUILD_NUM} \
-  -c "black --check /app/wkcuber"
+  -c "black --check /app"

tests/scripts/downsampling.sh

@@ -17,4 +17,5 @@ docker run \
 [ $(find testoutput/tiff/color/2 -mindepth 3 -name "*.wkw" | wc -l) -eq 1 ]
 [ $(find testoutput/tiff/color/4 -mindepth 3 -name "*.wkw" | wc -l) -eq 1 ]
 [ $(find testoutput/tiff/color/8 -mindepth 3 -name "*.wkw" | wc -l) -eq 1 ]
-cmp --silent testoutput/tiff/color/2/z0/y0/x0.wkw testdata/tiff_mag_2_reference/color/2/z0/y0/x0.wkw
+cmp --silent testoutput/tiff/color/2/z0/y0/x0.wkw testdata/tiff_mag_2_reference/color/2/z0/y0/x0.wkw
+cmp --silent testoutput/tiff/color/2/header.wkw testdata/tiff_mag_2_reference/color/2/header.wkw

tests/test_downsampling.py

@@ -95,7 +95,7 @@ def downsample_test_helper(use_compress):
     )[0]
     assert np.any(source_buffer != 0)
 
-    downsample_cube_job(
+    downsample_args = (
         source_info,
         target_info,
         (2, 2, 2),
@@ -104,6 +104,7 @@ def downsample_test_helper(use_compress):
         offset,
         use_compress,
     )
+    downsample_cube_job(downsample_args)
 
     assert np.any(source_buffer != 0)
     block_type = (
@@ -155,7 +156,7 @@ def test_downsample_multi_channel():
         wkw_dataset.write(offset, source_data)
     assert np.any(source_data != 0)
 
-    downsample_cube_job(
+    downsample_args = (
         source_info,
         target_info,
         (2, 2, 2),
@@ -164,6 +165,7 @@ def test_downsample_multi_channel():
         tuple(a * WKW_CUBE_SIZE for a in offset),
         False,
     )
+    downsample_cube_job(downsample_args)
 
     channels = []
     for channel_index in range(num_channels):
@@ -184,21 +186,27 @@ def test_downsample_multi_channel():
 
     assert np.all(target_buffer == joined_buffer)
 
+
 def test_anisotropic_mag_calculation():
     mag_tests = [
-        [(10.5, 10.5, 24),      Mag(1),         Mag((2, 2, 1))],
-        [(10.5, 10.5, 21),      Mag(1),         Mag((2, 2, 1))],
-        [(10.5, 24, 10.5),      Mag(1),         Mag((2, 1, 2))],
-        [(24, 10.5, 10.5),      Mag(1),         Mag((1, 2, 2))],
-        [(10.5, 10.5, 10.5),    Mag(1),         Mag((2, 2, 2))],
-        [(10.5, 10.5, 24),      Mag((2, 2, 1)), Mag((4, 4, 1))],
-        [(10.5, 10.5, 21),      Mag((2, 2, 1)), Mag((4, 4, 2))],
-        [(10.5, 24, 10.5),      Mag((2, 1, 2)), Mag((4, 1, 4))],
-        [(24, 10.5, 10.5),      Mag((1, 2, 2)), Mag((1, 4, 4))],
-        [(10.5, 10.5, 10.5),    Mag(2),         Mag(4)],
+        [(10.5, 10.5, 24), Mag(1), Mag((2, 2, 1))],
+        [(10.5, 10.5, 21), Mag(1), Mag((2, 2, 1))],
+        [(10.5, 24, 10.5), Mag(1), Mag((2, 1, 2))],
+        [(24, 10.5, 10.5), Mag(1), Mag((1, 2, 2))],
+        [(10.5, 10.5, 10.5), Mag(1), Mag((2, 2, 2))],
+        [(10.5, 10.5, 24), Mag((2, 2, 1)), Mag((4, 4, 1))],
+        [(10.5, 10.5, 21), Mag((2, 2, 1)), Mag((4, 4, 2))],
+        [(10.5, 24, 10.5), Mag((2, 1, 2)), Mag((4, 1, 4))],
+        [(24, 10.5, 10.5), Mag((1, 2, 2)), Mag((1, 4, 4))],
+        [(10.5, 10.5, 10.5), Mag(2), Mag(4)],
     ]
     for i in range(len(mag_tests)):
-        assert mag_tests[i][2] == get_next_anisotropic_mag(mag_tests[i][1], mag_tests[i][0]), "The next anisotropic" \
-                                                                                          " Magnification of {} with " \
-                                                                                          "the size {} should be {}"\
-            .format(mag_tests[i][1], mag_tests[i][0], mag_tests[i][2])
+        assert mag_tests[i][2] == get_next_anisotropic_mag(
+            mag_tests[i][1], mag_tests[i][0]
+        ), (
+            "The next anisotropic"
+            " Magnification of {} with "
+            "the size {} should be {}".format(
+                mag_tests[i][1], mag_tests[i][0], mag_tests[i][2]
+            )
+        )

tests/test_export_wkw_as_tiff.py

@@ -9,118 +9,195 @@
 
 def test_export_tiff_stack():
     destination_path = os.path.join("testoutput", "WT1_wkw")
-    args_list = ["--source_path", os.path.join("testdata", "WT1_wkw"),
-                 "--destination_path", destination_path,
-                 "--layer_name", "color",
-                 "--name", "test_export",
-                 "--bbox", "0,0,0,100,100,5",
-                 "--mag", "1"]
+    args_list = [
+        "--source_path",
+        os.path.join("testdata", "WT1_wkw"),
+        "--destination_path",
+        destination_path,
+        "--layer_name",
+        "color",
+        "--name",
+        "test_export",
+        "--bbox",
+        "0,0,0,100,100,5",
+        "--mag",
+        "1",
+    ]
 
     bbox = {"topleft": [0, 0, 0], "size": [100, 100, 5]}
 
     run(args_list)
 
-    test_wkw_file_path = os.path.join("testdata", "WT1_wkw", "color", Mag(1).to_layer_name())
+    test_wkw_file_path = os.path.join(
+        "testdata", "WT1_wkw", "color", Mag(1).to_layer_name()
+    )
     with wkw.Dataset.open(test_wkw_file_path) as dataset:
         slice_bbox = bbox
         slice_bbox["size"] = [slice_bbox["size"][0], slice_bbox["size"][1], 1]
         for data_slice_index in range(1, bbox["size"][2] + 1):
-            slice_bbox["topleft"] = [slice_bbox["topleft"][0], slice_bbox["topleft"][1], bbox["topleft"][2] + data_slice_index]
-            tiff_path = os.path.join(destination_path, wkw_name_and_bbox_to_tiff_name("test_export", data_slice_index))
-
-            assert os.path.isfile(tiff_path), f"Expected a tiff to be written at: {tiff_path}."
+            slice_bbox["topleft"] = [
+                slice_bbox["topleft"][0],
+                slice_bbox["topleft"][1],
+                bbox["topleft"][2] + data_slice_index,
+            ]
+            tiff_path = os.path.join(
+                destination_path,
+                wkw_name_and_bbox_to_tiff_name("test_export", data_slice_index),
+            )
+
+            assert os.path.isfile(
+                tiff_path
+            ), f"Expected a tiff to be written at: {tiff_path}."
 
             test_image = np.array(Image.open(tiff_path))
             test_image.transpose((1, 0))
 
-            correct_image = dataset.read(off=slice_bbox["topleft"], shape=slice_bbox["size"])
+            correct_image = dataset.read(
+                off=slice_bbox["topleft"], shape=slice_bbox["size"]
+            )
             correct_image = np.squeeze(correct_image)
 
-            assert np.array_equal(correct_image, test_image), f"The tiff file {tiff_path} that was written is not " \
-                                                              f"equal to the original wkw_file."
+            assert np.array_equal(correct_image, test_image), (
+                f"The tiff file {tiff_path} that was written is not "
+                f"equal to the original wkw_file."
+            )
 
 
 def test_export_tiff_stack_tile_size():
     destination_path = os.path.join("testoutput", "WT1_wkw_tile_size")
-    args_list = ["--source_path", os.path.join("testdata", "WT1_wkw"),
-                 "--destination_path", destination_path,
-                 "--layer_name", "color",
-                 "--name", "test_export",
-                 "--bbox", "0,0,0,100,100,5",
-                 "--mag", "1",
-                 "--tile_size", "30,30"]
+    args_list = [
+        "--source_path",
+        os.path.join("testdata", "WT1_wkw"),
+        "--destination_path",
+        destination_path,
+        "--layer_name",
+        "color",
+        "--name",
+        "test_export",
+        "--bbox",
+        "0,0,0,100,100,5",
+        "--mag",
+        "1",
+        "--tile_size",
+        "30,30",
+    ]
 
     bbox = {"topleft": [0, 0, 0], "size": [100, 100, 5]}
 
     run(args_list)
 
     tile_bbox = {"topleft": bbox["topleft"], "size": [30, 30, 1]}
-    test_wkw_file_path = os.path.join("testdata", "WT1_wkw", "color", Mag(1).to_layer_name())
+    test_wkw_file_path = os.path.join(
+        "testdata", "WT1_wkw", "color", Mag(1).to_layer_name()
+    )
     with wkw.Dataset.open(test_wkw_file_path) as dataset:
         slice_bbox = {"topleft": bbox["topleft"], "size": bbox["size"]}
         slice_bbox["size"] = [slice_bbox["size"][0], slice_bbox["size"][1], 1]
         for data_slice_index in range(bbox["size"][2]):
 
             for y_tile_index in range(ceil(bbox["size"][1] / tile_bbox["size"][1])):
                 for x_tile_index in range(ceil(bbox["size"][0] / tile_bbox["size"][0])):
-                    tiff_path = os.path.join(destination_path, f"{data_slice_index + 1}", f"{y_tile_index + 1}", f"{x_tile_index + 1}.tiff")
+                    tiff_path = os.path.join(
+                        destination_path,
+                        f"{data_slice_index + 1}",
+                        f"{y_tile_index + 1}",
+                        f"{x_tile_index + 1}.tiff",
+                    )
 
-                    assert os.path.isfile(tiff_path), f"Expected a tiff to be written at: {tiff_path}."
+                    assert os.path.isfile(
+                        tiff_path
+                    ), f"Expected a tiff to be written at: {tiff_path}."
 
                     test_image = np.array(Image.open(tiff_path))
                     test_image.transpose((1, 0))
 
-                    correct_image = dataset.read(off=[tile_bbox["topleft"][0] + tile_bbox["size"][0] * x_tile_index,
-                                                      tile_bbox["topleft"][1] + tile_bbox["size"][1] * y_tile_index,
-                                                      tile_bbox["topleft"][2] + data_slice_index],
-                                                 shape=tile_bbox["size"])
+                    correct_image = dataset.read(
+                        off=[
+                            tile_bbox["topleft"][0]
+                            + tile_bbox["size"][0] * x_tile_index,
+                            tile_bbox["topleft"][1]
+                            + tile_bbox["size"][1] * y_tile_index,
+                            tile_bbox["topleft"][2] + data_slice_index,
+                        ],
+                        shape=tile_bbox["size"],
+                    )
 
                     correct_image = np.squeeze(correct_image)
 
-                    assert np.array_equal(correct_image, test_image), f"The tiff file {tiff_path} that was written " \
+                    assert np.array_equal(correct_image, test_image), (
+                        f"The tiff file {tiff_path} that was written "
                         f"is not equal to the original wkw_file."
+                    )
 
 
 def test_export_tiff_stack_tiles_per_dimension():
     destination_path = os.path.join("testoutput", "WT1_wkw_tiles_per_dimension")
-    args_list = ["--source_path", os.path.join("testdata", "WT1_wkw"),
-                 "--destination_path", destination_path,
-                 "--layer_name", "color",
-                 "--name", "test_export",
-                 "--bbox", "0,0,0,100,100,5",
-                 "--mag", "1",
-                 "--tiles_per_dimension", "6,6"]
+    args_list = [
+        "--source_path",
+        os.path.join("testdata", "WT1_wkw"),
+        "--destination_path",
+        destination_path,
+        "--layer_name",
+        "color",
+        "--name",
+        "test_export",
+        "--bbox",
+        "0,0,0,100,100,5",
+        "--mag",
+        "1",
+        "--tiles_per_dimension",
+        "6,6",
+    ]
 
     bbox = {"topleft": [0, 0, 0], "size": [100, 100, 5]}
 
     run(args_list)
 
     tile_bbox = {"topleft": bbox["topleft"], "size": [17, 17, 1]}
-    test_wkw_file_path = os.path.join("testdata", "WT1_wkw", "color", Mag(1).to_layer_name())
+    test_wkw_file_path = os.path.join(
+        "testdata", "WT1_wkw", "color", Mag(1).to_layer_name()
+    )
     with wkw.Dataset.open(test_wkw_file_path) as dataset:
         slice_bbox = bbox
         slice_bbox["size"] = [slice_bbox["size"][0], slice_bbox["size"][1], 1]
         for data_slice_index in range(bbox["size"][2]):
 
             for y_tile_index in range(ceil(bbox["size"][1] / tile_bbox["size"][1])):
-                for x_tile_index in range(ceil(tile_bbox["size"][0] / tile_bbox["size"][0])):
-                    tiff_path = os.path.join(destination_path, f"{data_slice_index + 1}", f"{y_tile_index + 1}", f"{x_tile_index + 1}.tiff")
-
-                    assert os.path.isfile(tiff_path), f"Expected a tiff to be written at: {tiff_path}."
+                for x_tile_index in range(
+                    ceil(tile_bbox["size"][0] / tile_bbox["size"][0])
+                ):
+                    tiff_path = os.path.join(
+                        destination_path,
+                        f"{data_slice_index + 1}",
+                        f"{y_tile_index + 1}",
+                        f"{x_tile_index + 1}.tiff",
+                    )
+
+                    assert os.path.isfile(
+                        tiff_path
+                    ), f"Expected a tiff to be written at: {tiff_path}."
 
                     test_image = np.array(Image.open(tiff_path))
                     test_image.transpose((1, 0))
 
-                    correct_image = dataset.read(off=[tile_bbox["topleft"][0] + tile_bbox["size"][0] * x_tile_index,
-                                                      tile_bbox["topleft"][1] + tile_bbox["size"][1] * y_tile_index,
-                                                      tile_bbox["topleft"][2] + data_slice_index],
-                                                 shape=tile_bbox["size"])
+                    correct_image = dataset.read(
+                        off=[
+                            tile_bbox["topleft"][0]
+                            + tile_bbox["size"][0] * x_tile_index,
+                            tile_bbox["topleft"][1]
+                            + tile_bbox["size"][1] * y_tile_index,
+                            tile_bbox["topleft"][2] + data_slice_index,
+                        ],
+                        shape=tile_bbox["size"],
+                    )
 
                     correct_image = np.squeeze(correct_image)
 
-                    assert np.array_equal(correct_image, test_image), f"The tiff file {tiff_path} that was written " \
+                    assert np.array_equal(correct_image, test_image), (
+                        f"The tiff file {tiff_path} that was written "
                         f"is not equal to the original wkw_file."
+                    )
 
 
 if __name__ == "__main__":
-    test_export_tiff_stack_tile_size()
+    test_export_tiff_stack_tile_size()