diff --git a/examples/Create ome-zarr from Single-Plane Multi-Field Acquisition.ipynb b/examples/Create ome-zarr from Single-Plane Multi-Field Acquisition.ipynb index e3cbed57..edb51712 100644 --- a/examples/Create ome-zarr from Single-Plane Multi-Field Acquisition.ipynb +++ b/examples/Create ome-zarr from Single-Plane Multi-Field Acquisition.ipynb @@ -56,7 +56,7 @@ "outputs": [], "source": [ "from faim_hcs.io.MolecularDevicesImageXpress import parse_single_plane_multi_fields\n", - "from faim_hcs.Zarr import build_zarr_scaffold, write_cyx_image_to_well, PlateLayout\n", + "from faim_hcs.Zarr import build_zarr_scaffold, write_cyx_image_to_well, PlateLayout, write_roi_table\n", "from faim_hcs.MetaSeriesUtils import get_well_image_CYX, montage_grid_image_YX\n", "from faim_hcs.UIntHistogram import UIntHistogram\n", "import shutil\n", @@ -131,8 +131,8 @@ " Projection-Mix\n", " E08\n", " s2\n", - " w2\n", - " 66923EBB-9960-4952-8955-D1721D112EE2\n", + " w1\n", + " B38C01F5-0D36-4A29-9F5A-BE62B6F7F73F\n", " .tif\n", " ../resources/Projection-Mix/2023-02-21/1334/Pr...\n", " \n", @@ -141,10 +141,10 @@ " 2023-02-21\n", " 1334\n", " Projection-Mix\n", - " E07\n", - " s2\n", - " w1\n", - " DCFD1526-D063-4F8B-9E51-F1BD2EBD9F1A\n", + " E08\n", + " s1\n", + " w2\n", + " 81928711-999D-41F6-B88C-999513D4C092\n", " .tif\n", " ../resources/Projection-Mix/2023-02-21/1334/Pr...\n", " \n", @@ -153,10 +153,10 @@ " 2023-02-21\n", " 1334\n", " Projection-Mix\n", - " E07\n", - " s1\n", - " w1\n", - " E94C24BD-45E4-450A-9919-257C714278F7\n", + " E08\n", + " s2\n", + " w3\n", + " CCE83D85-0912-429E-9F18-716A085BB5BC\n", " .tif\n", " ../resources/Projection-Mix/2023-02-21/1334/Pr...\n", " \n", @@ -166,9 +166,9 @@ " 1334\n", " Projection-Mix\n", " E07\n", - " s1\n", - " w2\n", - " B14915F6-0679-4494-82D1-F80894B32A66\n", + " s2\n", + " w3\n", + " B0A47337-5945-4B26-9F5F-4EBA468CDBA9\n", " .tif\n", " ../resources/Projection-Mix/2023-02-21/1334/Pr...\n", " \n", @@ -177,10 +177,10 @@ " 2023-02-21\n", " 1334\n", " Projection-Mix\n", - " E08\n", + " E07\n", " s1\n", - " w3\n", - " DD77D22D-07CB-4529-A1F5-DCC5473786FA\n", + " w2\n", + " B14915F6-0679-4494-82D1-F80894B32A66\n", " .tif\n", " ../resources/Projection-Mix/2023-02-21/1334/Pr...\n", " \n", @@ -189,10 +189,10 @@ " 2023-02-21\n", " 1334\n", " Projection-Mix\n", - " E08\n", + " E07\n", " s2\n", - " w3\n", - " CCE83D85-0912-429E-9F18-716A085BB5BC\n", + " w2\n", + " 607EE13F-AB5E-4E8C-BC4B-52E1118E7723\n", " .tif\n", " ../resources/Projection-Mix/2023-02-21/1334/Pr...\n", " \n", @@ -201,10 +201,10 @@ " 2023-02-21\n", " 1334\n", " Projection-Mix\n", - " E07\n", + " E08\n", " s1\n", " w3\n", - " BB87F860-FC67-4B3A-A740-A9EACF8A8F5F\n", + " DD77D22D-07CB-4529-A1F5-DCC5473786FA\n", " .tif\n", " ../resources/Projection-Mix/2023-02-21/1334/Pr...\n", " \n", @@ -226,9 +226,9 @@ " 1334\n", " Projection-Mix\n", " E07\n", - " s2\n", - " w3\n", - " B0A47337-5945-4B26-9F5F-4EBA468CDBA9\n", + " s1\n", + " w1\n", + " E94C24BD-45E4-450A-9919-257C714278F7\n", " .tif\n", " ../resources/Projection-Mix/2023-02-21/1334/Pr...\n", " \n", @@ -239,8 +239,8 @@ " Projection-Mix\n", " E07\n", " s2\n", - " w2\n", - " 607EE13F-AB5E-4E8C-BC4B-52E1118E7723\n", + " w1\n", + " DCFD1526-D063-4F8B-9E51-F1BD2EBD9F1A\n", " .tif\n", " ../resources/Projection-Mix/2023-02-21/1334/Pr...\n", " \n", @@ -250,9 +250,9 @@ " 1334\n", " Projection-Mix\n", " E08\n", - " s1\n", + " s2\n", " w2\n", - " 81928711-999D-41F6-B88C-999513D4C092\n", + " 66923EBB-9960-4952-8955-D1721D112EE2\n", " .tif\n", " ../resources/Projection-Mix/2023-02-21/1334/Pr...\n", " \n", @@ -261,10 +261,10 @@ " 2023-02-21\n", " 1334\n", " Projection-Mix\n", - " E08\n", - " s2\n", - " w1\n", - " B38C01F5-0D36-4A29-9F5A-BE62B6F7F73F\n", + " E07\n", + " s1\n", + " w3\n", + " BB87F860-FC67-4B3A-A740-A9EACF8A8F5F\n", " .tif\n", " ../resources/Projection-Mix/2023-02-21/1334/Pr...\n", " \n", @@ -274,32 +274,32 @@ ], "text/plain": [ " date acq_id name well field channel \\\n", - "0 2023-02-21 1334 Projection-Mix E08 s2 w2 \n", - "1 2023-02-21 1334 Projection-Mix E07 s2 w1 \n", - "2 2023-02-21 1334 Projection-Mix E07 s1 w1 \n", - "3 2023-02-21 1334 Projection-Mix E07 s1 w2 \n", - "4 2023-02-21 1334 Projection-Mix E08 s1 w3 \n", - "5 2023-02-21 1334 Projection-Mix E08 s2 w3 \n", - "6 2023-02-21 1334 Projection-Mix E07 s1 w3 \n", + "0 2023-02-21 1334 Projection-Mix E08 s2 w1 \n", + "1 2023-02-21 1334 Projection-Mix E08 s1 w2 \n", + "2 2023-02-21 1334 Projection-Mix E08 s2 w3 \n", + "3 2023-02-21 1334 Projection-Mix E07 s2 w3 \n", + "4 2023-02-21 1334 Projection-Mix E07 s1 w2 \n", + "5 2023-02-21 1334 Projection-Mix E07 s2 w2 \n", + "6 2023-02-21 1334 Projection-Mix E08 s1 w3 \n", "7 2023-02-21 1334 Projection-Mix E08 s1 w1 \n", - "8 2023-02-21 1334 Projection-Mix E07 s2 w3 \n", - "9 2023-02-21 1334 Projection-Mix E07 s2 w2 \n", - "10 2023-02-21 1334 Projection-Mix E08 s1 w2 \n", - "11 2023-02-21 1334 Projection-Mix E08 s2 w1 \n", + "8 2023-02-21 1334 Projection-Mix E07 s1 w1 \n", + "9 2023-02-21 1334 Projection-Mix E07 s2 w1 \n", + "10 2023-02-21 1334 Projection-Mix E08 s2 w2 \n", + "11 2023-02-21 1334 Projection-Mix E07 s1 w3 \n", "\n", " md_id ext \\\n", - "0 66923EBB-9960-4952-8955-D1721D112EE2 .tif \n", - "1 DCFD1526-D063-4F8B-9E51-F1BD2EBD9F1A .tif \n", - "2 E94C24BD-45E4-450A-9919-257C714278F7 .tif \n", - "3 B14915F6-0679-4494-82D1-F80894B32A66 .tif \n", - "4 DD77D22D-07CB-4529-A1F5-DCC5473786FA .tif \n", - "5 CCE83D85-0912-429E-9F18-716A085BB5BC .tif \n", - "6 BB87F860-FC67-4B3A-A740-A9EACF8A8F5F .tif \n", + "0 B38C01F5-0D36-4A29-9F5A-BE62B6F7F73F .tif \n", + "1 81928711-999D-41F6-B88C-999513D4C092 .tif \n", + "2 CCE83D85-0912-429E-9F18-716A085BB5BC .tif \n", + "3 B0A47337-5945-4B26-9F5F-4EBA468CDBA9 .tif \n", + "4 B14915F6-0679-4494-82D1-F80894B32A66 .tif \n", + "5 607EE13F-AB5E-4E8C-BC4B-52E1118E7723 .tif \n", + "6 DD77D22D-07CB-4529-A1F5-DCC5473786FA .tif \n", "7 17654C10-92F1-4DFD-98AA-6A01BBD77557 .tif \n", - "8 B0A47337-5945-4B26-9F5F-4EBA468CDBA9 .tif \n", - "9 607EE13F-AB5E-4E8C-BC4B-52E1118E7723 .tif \n", - "10 81928711-999D-41F6-B88C-999513D4C092 .tif \n", - "11 B38C01F5-0D36-4A29-9F5A-BE62B6F7F73F .tif \n", + "8 E94C24BD-45E4-450A-9919-257C714278F7 .tif \n", + "9 DCFD1526-D063-4F8B-9E51-F1BD2EBD9F1A .tif \n", + "10 66923EBB-9960-4952-8955-D1721D112EE2 .tif \n", + "11 BB87F860-FC67-4B3A-A740-A9EACF8A8F5F .tif \n", "\n", " path \n", "0 ../resources/Projection-Mix/2023-02-21/1334/Pr... \n", @@ -327,7 +327,7 @@ }, { "cell_type": "code", - "execution_count": 5, + "execution_count": 8, "id": "948904be", "metadata": {}, "outputs": [], @@ -347,14 +347,14 @@ }, { "cell_type": "code", - "execution_count": 6, + "execution_count": 9, "id": "37f65e50", "metadata": {}, "outputs": [ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "d39b97703a4f4214b3c1618843c8f481", + "model_id": "cca2a3f86a72410ca69527e3a9b46e26", "version_major": 2, "version_minor": 0 }, @@ -372,20 +372,26 @@ "for well in tqdm(files['well'].unique()):\n", " well_files = files[files['well'] == well]\n", " \n", - " img, hists, ch_metadata, metadta = get_well_image_CYX(\n", + " img, hists, ch_metadata, metadata, roi_tables = get_well_image_CYX(\n", " well_files=well_files,\n", " channels=channels,\n", " assemble_fn=montage_grid_image_YX,\n", " )\n", " \n", " well_group = plate[well[0]][str(int(well[1:]))][0]\n", - " write_cyx_image_to_well(img, hists, ch_metadata, metadta, well_group)" + " write_cyx_image_to_well(img, hists, ch_metadata, metadata, well_group)\n", + " \n", + " # Write all ROI tables\n", + " for roi_table in roi_tables:\n", + " write_roi_table(roi_tables[roi_table], roi_table, well_group)" ] }, { "cell_type": "markdown", "id": "7f5a8f42", - "metadata": {}, + "metadata": { + "jp-MarkdownHeadingCollapsed": true + }, "source": [ "# Inspect ome-zarr plate\n", "The data can be opened with the [ome-zarr Napari plugin](https://www.napari-hub.org/plugins/napari-ome-zarr)." @@ -408,7 +414,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.9.0" + "version": "3.9.16" } }, "nbformat": 4, diff --git a/examples/Create ome-zarr from Z-Stack Multi-Field Acquisition.ipynb b/examples/Create ome-zarr from Z-Stack Multi-Field Acquisition.ipynb index d3317af3..bd64fdde 100644 --- a/examples/Create ome-zarr from Z-Stack Multi-Field Acquisition.ipynb +++ b/examples/Create ome-zarr from Z-Stack Multi-Field Acquisition.ipynb @@ -88,7 +88,7 @@ "outputs": [], "source": [ "from faim_hcs.io.MolecularDevicesImageXpress import parse_files\n", - "from faim_hcs.Zarr import build_zarr_scaffold, write_czyx_image_to_well, write_cyx_image_to_well\n", + "from faim_hcs.Zarr import build_zarr_scaffold, write_czyx_image_to_well, write_cyx_image_to_well, write_roi_table\n", "from faim_hcs.MetaSeriesUtils import get_well_image_CYX, get_well_image_CZYX, montage_grid_image_YX\n", "from faim_hcs.UIntHistogram import UIntHistogram\n", "import shutil\n", @@ -172,9 +172,9 @@ " None\n", " Projection-Mix\n", " E08\n", - " s1\n", + " s2\n", " w1\n", - " 17654C10-92F1-4DFD-98AA-6A01BBD77557\n", + " B38C01F5-0D36-4A29-9F5A-BE62B6F7F73F\n", " .tif\n", " ../resources/Projection-Mix/2023-02-21/1334/Pr...\n", " \n", @@ -186,8 +186,8 @@ " Projection-Mix\n", " E08\n", " s1\n", - " w3\n", - " DD77D22D-07CB-4529-A1F5-DCC5473786FA\n", + " w2\n", + " 81928711-999D-41F6-B88C-999513D4C092\n", " .tif\n", " ../resources/Projection-Mix/2023-02-21/1334/Pr...\n", " \n", @@ -197,10 +197,10 @@ " 1334\n", " None\n", " Projection-Mix\n", - " E07\n", + " E08\n", " s2\n", " w3\n", - " B0A47337-5945-4B26-9F5F-4EBA468CDBA9\n", + " CCE83D85-0912-429E-9F18-716A085BB5BC\n", " .tif\n", " ../resources/Projection-Mix/2023-02-21/1334/Pr...\n", " \n", @@ -212,8 +212,8 @@ " Projection-Mix\n", " E07\n", " s2\n", - " w2\n", - " 607EE13F-AB5E-4E8C-BC4B-52E1118E7723\n", + " w3\n", + " B0A47337-5945-4B26-9F5F-4EBA468CDBA9\n", " .tif\n", " ../resources/Projection-Mix/2023-02-21/1334/Pr...\n", " \n", @@ -225,8 +225,8 @@ " Projection-Mix\n", " E07\n", " s1\n", - " w1\n", - " E94C24BD-45E4-450A-9919-257C714278F7\n", + " w2\n", + " B14915F6-0679-4494-82D1-F80894B32A66\n", " .tif\n", " ../resources/Projection-Mix/2023-02-21/1334/Pr...\n", " \n", @@ -247,12 +247,12 @@ " 91\n", " 2023-02-21\n", " 1334\n", - " 1\n", + " 9\n", " Projection-Mix\n", - " E08\n", - " s2\n", - " w2\n", - " 71F9FAE3-CF6E-43CE-84E0-2506B2C908E3\n", + " E07\n", + " s1\n", + " w1\n", + " 091EB8A5-272A-466D-B8A0-7547C6BA392B\n", " .tif\n", " ../resources/Projection-Mix/2023-02-21/1334/ZS...\n", " \n", @@ -260,12 +260,12 @@ " 92\n", " 2023-02-21\n", " 1334\n", - " 1\n", + " 9\n", " Projection-Mix\n", " E07\n", " s2\n", " w1\n", - " AC08A410-4276-4921-9FDA-9CB1249B3156\n", + " 0961945B-7AF1-4182-85E2-DCE08A54F6E6\n", " .tif\n", " ../resources/Projection-Mix/2023-02-21/1334/ZS...\n", " \n", @@ -273,12 +273,12 @@ " 93\n", " 2023-02-21\n", " 1334\n", - " 1\n", + " 9\n", " Projection-Mix\n", - " E07\n", - " s2\n", - " w4\n", - " F95A8A9F-0939-47C2-8D3E-F6E91AF0C4ED\n", + " E08\n", + " s1\n", + " w2\n", + " B8405A0A-E6F1-49F5-876D-6ECCE85CBFE0\n", " .tif\n", " ../resources/Projection-Mix/2023-02-21/1334/ZS...\n", " \n", @@ -286,12 +286,12 @@ " 94\n", " 2023-02-21\n", " 1334\n", - " 1\n", + " 9\n", " Projection-Mix\n", - " E07\n", - " s1\n", - " w4\n", - " 27CCB2E4-1BF4-45E7-8BC7-264B48EF9C4A\n", + " E08\n", + " s2\n", + " w1\n", + " E6876931-5F26-4F52-8CE6-94C2008473A8\n", " .tif\n", " ../resources/Projection-Mix/2023-02-21/1334/ZS...\n", " \n", @@ -299,12 +299,12 @@ " 95\n", " 2023-02-21\n", " 1334\n", - " 1\n", + " 9\n", " Projection-Mix\n", - " E07\n", - " s1\n", - " w1\n", - " E78EB128-BD0D-4D94-A6AD-3FF28BB1B105\n", + " E08\n", + " s2\n", + " w2\n", + " 980ECD4E-B03B-4051-A4BE-5EF45BDA5266\n", " .tif\n", " ../resources/Projection-Mix/2023-02-21/1334/ZS...\n", " \n", @@ -315,30 +315,30 @@ ], "text/plain": [ " date acq_id z name well field channel \\\n", - "0 2023-02-21 1334 None Projection-Mix E08 s1 w1 \n", - "1 2023-02-21 1334 None Projection-Mix E08 s1 w3 \n", - "2 2023-02-21 1334 None Projection-Mix E07 s2 w3 \n", - "3 2023-02-21 1334 None Projection-Mix E07 s2 w2 \n", - "4 2023-02-21 1334 None Projection-Mix E07 s1 w1 \n", + "0 2023-02-21 1334 None Projection-Mix E08 s2 w1 \n", + "1 2023-02-21 1334 None Projection-Mix E08 s1 w2 \n", + "2 2023-02-21 1334 None Projection-Mix E08 s2 w3 \n", + "3 2023-02-21 1334 None Projection-Mix E07 s2 w3 \n", + "4 2023-02-21 1334 None Projection-Mix E07 s1 w2 \n", ".. ... ... ... ... ... ... ... \n", - "91 2023-02-21 1334 1 Projection-Mix E08 s2 w2 \n", - "92 2023-02-21 1334 1 Projection-Mix E07 s2 w1 \n", - "93 2023-02-21 1334 1 Projection-Mix E07 s2 w4 \n", - "94 2023-02-21 1334 1 Projection-Mix E07 s1 w4 \n", - "95 2023-02-21 1334 1 Projection-Mix E07 s1 w1 \n", + "91 2023-02-21 1334 9 Projection-Mix E07 s1 w1 \n", + "92 2023-02-21 1334 9 Projection-Mix E07 s2 w1 \n", + "93 2023-02-21 1334 9 Projection-Mix E08 s1 w2 \n", + "94 2023-02-21 1334 9 Projection-Mix E08 s2 w1 \n", + "95 2023-02-21 1334 9 Projection-Mix E08 s2 w2 \n", "\n", " md_id ext \\\n", - "0 17654C10-92F1-4DFD-98AA-6A01BBD77557 .tif \n", - "1 DD77D22D-07CB-4529-A1F5-DCC5473786FA .tif \n", - "2 B0A47337-5945-4B26-9F5F-4EBA468CDBA9 .tif \n", - "3 607EE13F-AB5E-4E8C-BC4B-52E1118E7723 .tif \n", - "4 E94C24BD-45E4-450A-9919-257C714278F7 .tif \n", + "0 B38C01F5-0D36-4A29-9F5A-BE62B6F7F73F .tif \n", + "1 81928711-999D-41F6-B88C-999513D4C092 .tif \n", + "2 CCE83D85-0912-429E-9F18-716A085BB5BC .tif \n", + "3 B0A47337-5945-4B26-9F5F-4EBA468CDBA9 .tif \n", + "4 B14915F6-0679-4494-82D1-F80894B32A66 .tif \n", ".. ... ... \n", - "91 71F9FAE3-CF6E-43CE-84E0-2506B2C908E3 .tif \n", - "92 AC08A410-4276-4921-9FDA-9CB1249B3156 .tif \n", - "93 F95A8A9F-0939-47C2-8D3E-F6E91AF0C4ED .tif \n", - "94 27CCB2E4-1BF4-45E7-8BC7-264B48EF9C4A .tif \n", - "95 E78EB128-BD0D-4D94-A6AD-3FF28BB1B105 .tif \n", + "91 091EB8A5-272A-466D-B8A0-7547C6BA392B .tif \n", + "92 0961945B-7AF1-4182-85E2-DCE08A54F6E6 .tif \n", + "93 B8405A0A-E6F1-49F5-876D-6ECCE85CBFE0 .tif \n", + "94 E6876931-5F26-4F52-8CE6-94C2008473A8 .tif \n", + "95 980ECD4E-B03B-4051-A4BE-5EF45BDA5266 .tif \n", "\n", " path \n", "0 ../resources/Projection-Mix/2023-02-21/1334/Pr... \n", @@ -411,7 +411,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "0914e32e780345b09a98d78fef6b112f", + "model_id": "99e72a7ad9ad46648742783314b515f4", "version_major": 2, "version_minor": 0 }, @@ -435,13 +435,13 @@ " stack_files = well_files[~well_files['z'].isnull()]\n", " \n", " \n", - " projection, proj_hists, proj_ch_metadata, proj_metadata = get_well_image_CYX(\n", + " projection, proj_hists, proj_ch_metadata, proj_metadata, roi_tables_proj = get_well_image_CYX(\n", " well_files=projection_files,\n", " channels=channels,\n", " assemble_fn=montage_grid_image_YX,\n", " )\n", " \n", - " stack, stack_hist, stack_ch_metadata, stack_metadata = get_well_image_CZYX(\n", + " stack, stack_hist, stack_ch_metadata, stack_metadata, roi_tables = get_well_image_CZYX(\n", " well_files=stack_files,\n", " channels=channels,\n", " assemble_fn=montage_grid_image_YX,\n", @@ -457,7 +457,11 @@ " projections = field.create_group(\"projections\")\n", " \n", " # Write projections\n", - " write_cyx_image_to_well(projection, proj_hists, proj_ch_metadata, proj_metadata, projections, True)" + " write_cyx_image_to_well(projection, proj_hists, proj_ch_metadata, proj_metadata, projections, True)\n", + "\n", + " # Write all ROI tables\n", + " for roi_table in roi_tables:\n", + " write_roi_table(roi_tables[roi_table], roi_table, field)" ] }, { diff --git a/setup.cfg b/setup.cfg index f09988a0..04755db1 100644 --- a/setup.cfg +++ b/setup.cfg @@ -26,6 +26,8 @@ project_urls = [options] packages = find: install_requires = + anndata>=0.8.0,<=0.9.2 + fsspec<=2023.6.0 imagecodecs matplotlib numpy diff --git a/src/faim_hcs/MetaSeriesUtils.py b/src/faim_hcs/MetaSeriesUtils.py index 994394ba..c0343401 100644 --- a/src/faim_hcs/MetaSeriesUtils.py +++ b/src/faim_hcs/MetaSeriesUtils.py @@ -122,7 +122,25 @@ def _get_molecular_devices_well_bbox_2D( def montage_stage_pos_image_YX(data): - """Montage 2D fields based on stage position metadata.""" + """Montage 2D fields based on stage position metadata. + + Montages 2D fields based on stage position metadata. If the stage position + specifies overlapping images, the overlapping part is overwritten + (=> just uses the data of one image). Not well suited for regular grids, + as the stage position can show overlap, but overwriting of data at the + edge is not the intended behavior. In that case, use + `montage_grid_image_YX`. + + Also calculates ROI tables for the whole well and the field of views in + the Fractal ROI table format. We only stitch the xy planes here. + Therefore, the z starting position is always 0 and the z extent is set to + 1. This is overwritten downsteam if the 2D planes are assembled into a + 3D stack. + + :param data: list of tuples (image, metadata) + :return: img (stitched 2D np array), fov_df (dataframe with region of + interest information for the fields of view) + """ def sort_key(d): label = d[1]["stage-label"] @@ -142,6 +160,8 @@ def sort_key(d): img = np.zeros(shape, dtype=data[0][0].dtype) + fov_rois = [] + for d in data: pos_y = int( np.round(d[1]["stage-position-y"] / d[1]["spatial-calibration-y"] - min_y) @@ -152,15 +172,55 @@ def sort_key(d): img[pos_y : pos_y + d[0].shape[0], pos_x : pos_x + d[0].shape[1]] = d[0] - return img + # Create the FOV ROI table for the site in physical units + fov_rois.append( + ( + _stage_label(d[1]), + pos_y * d[1]["spatial-calibration-y"], + pos_x * d[1]["spatial-calibration-x"], + 0.0, + d[0].shape[0] * d[1]["spatial-calibration-y"], + d[0].shape[1] * d[1]["spatial-calibration-x"], + 1.0, + ) + ) + + roi_tables = create_ROI_tables(fov_rois, shape, calibration_dict=d[1]) + + return img, roi_tables def _pixel_pos(dim: str, data: dict): return np.round(data[f"stage-position-{dim}"] / data[f"spatial-calibration-{dim}"]) +def _stage_label(data: dict): + """Get the field of view (FOV) string for a given FOV dict""" + try: + return data["stage-label"].split(":")[-1][1:] + # Return an empty string if the metadata does not contain stage-label + except KeyError: + return "" + + def montage_grid_image_YX(data): - """Montage 2D fields into fixed grid, based on stage position metadata.""" + """Montage 2D fields into fixed grid, based on stage position metadata. + + Uses the stage position coordinates to decide which grid cell to put the + image in. Always writes images into a grid, thus avoiding overwriting + partially overwriting parts of images. Not well suited for arbitarily + positioned fields. In that case, use `montage_stage_pos_image_YX`. + + Also calculates ROI tables for the whole well and the field of views. + Given that Fractal ROI tables are always 3D, but we only stitch the xy + planes here, the z starting position is always 0 and the + z extent is set to 1. This is overwritten downsteam if the 2D planes are + assembled into a 3D stack. + + :param data: list of tuples of (image, metadata) + :return: img (stitched 2D np array), fov_df (dataframe with region of + interest information for the fields of view) + """ min_y = min(_pixel_pos("y", d[1]) for d in data) min_x = min(_pixel_pos("x", d[1]) for d in data) max_y = max(_pixel_pos("y", d[1]) for d in data) @@ -175,6 +235,7 @@ def montage_grid_image_YX(data): int(np.round((max_x - min_x) / step_x + 1) * step_x), ) img = np.zeros(shape, dtype=data[0][0].dtype) + fov_rois = [] for d in data: pos_x = int(np.round((_pixel_pos("x", d[1]) - min_x) / step_x)) @@ -182,8 +243,65 @@ def montage_grid_image_YX(data): img[ pos_y * step_y : (pos_y + 1) * step_y, pos_x * step_x : (pos_x + 1) * step_x ] = d[0] + # Create the FOV ROI table for the site in physical units + fov_rois.append( + ( + _stage_label(d[1]), + pos_y * step_y * d[1]["spatial-calibration-y"], + pos_x * step_x * d[1]["spatial-calibration-x"], + 0.0, + step_y * d[1]["spatial-calibration-y"], + step_x * d[1]["spatial-calibration-x"], + 1.0, + ) + ) + + roi_tables = create_ROI_tables(fov_rois, shape, calibration_dict=d[1]) + + return img, roi_tables + + +def create_ROI_tables(fov_rois, shape, calibration_dict): + columns = [ + "FieldIndex", + "x_micrometer", + "y_micrometer", + "z_micrometer", + "len_x_micrometer", + "len_y_micrometer", + "len_z_micrometer", + ] + roi_tables = {} + roi_tables["FOV_ROI_table"] = create_fov_ROI_table(fov_rois, columns) + roi_tables["well_ROI_table"] = create_well_ROI_table( + shape[1], + shape[0], + calibration_dict["spatial-calibration-x"], + calibration_dict["spatial-calibration-y"], + columns, + ) + return roi_tables + + +def create_well_ROI_table(shape_x, shape_y, pixel_size_x, pixel_size_y, columns): + well_roi = [ + "well_1", + 0.0, + 0.0, + 0.0, + shape_x * pixel_size_x, + shape_y * pixel_size_y, + 1.0, + ] + well_roi_table = pd.DataFrame(well_roi).T + well_roi_table.columns = columns + well_roi_table.set_index("FieldIndex", inplace=True) + return well_roi_table + - return img +def create_fov_ROI_table(fov_rois, columns): + roi_table = pd.DataFrame(fov_rois, columns=columns).set_index("FieldIndex") + return roi_table def verify_integrity(field_metadata: list[dict]): @@ -252,6 +370,7 @@ def get_well_image_CZYX( channel_metadata = [] px_metadata = None z_positions = [] + roi_tables = {} for ch in channels: channel_files = well_files[well_files["channel"] == ch] @@ -263,7 +382,7 @@ def get_well_image_CZYX( plane_files = channel_files[channel_files["z"] == z] if len(plane_files) > 0: - px_metadata, img, ch_meta, z_position = get_img_YX( + px_metadata, img, ch_meta, z_position, roi_tables = get_img_YX( assemble_fn=assemble_fn, files=plane_files ) @@ -294,6 +413,9 @@ def get_well_image_CZYX( z_sampling = compute_z_sampling(z_positions) px_metadata["z-scaling"] = z_sampling + max_stack_size = max([x.shape[0] for x in stacks if x is not None]) + for roi_table in roi_tables.values(): + roi_table["len_z_micrometer"] = z_sampling * (max_stack_size - 1) roll_single_plane(stacks, z_positions) @@ -308,14 +430,13 @@ def get_well_image_CZYX( "display-color": "000000", } - return czyx, channel_histograms, channel_metadata, px_metadata + return czyx, channel_histograms, channel_metadata, px_metadata, roi_tables def get_well_image_CYX( well_files: pd.DataFrame, channels: list[str], assemble_fn: Callable = montage_grid_image_YX, - include_z_position: bool = False, ) -> tuple[ArrayLike, list[UIntHistogram], list[dict], dict]: """Assemble image data for the given well-files. @@ -326,18 +447,19 @@ def get_well_image_CYX( :param well_files: all files corresponding to the well :param channels: list of required channels :param assemble_fn: creates a single image for each channel - :param include_z_position: whether to include z-position metadata - :return: CYX image, channel-histograms, channel-metadata, general-metadata + :return: CYX image, channel-histograms, channel-metadata, general-metadata, + roi-tables dictionary """ channel_imgs = {} channel_histograms = {} channel_metadata = {} px_metadata = None + roi_tables = {} for ch in channels: channel_files = well_files[well_files["channel"] == ch] if len(channel_files) > 0: - px_metadata, img, ch_metadata, z_position = get_img_YX( + px_metadata, img, ch_metadata, _, roi_tables = get_img_YX( assemble_fn, channel_files ) @@ -364,7 +486,7 @@ def get_well_image_CYX( } ) - return cyx, channel_hists, channel_meta, px_metadata + return cyx, channel_hists, channel_meta, px_metadata, roi_tables def get_img_YX(assemble_fn, files): @@ -385,7 +507,7 @@ def get_img_YX(assemble_fn, files): "spatial-calibration-units": ms_metadata["spatial-calibration-units"], "pixel-type": ms_metadata["PixelType"], } - img = assemble_fn(imgs) + img, roi_tables = assemble_fn(imgs) metadata = verify_integrity(field_metadata) zs = [z["z-position"] for z in z_positions] - return general_metadata, img, metadata, np.mean(zs) + return general_metadata, img, metadata, np.mean(zs), roi_tables diff --git a/src/faim_hcs/Zarr.py b/src/faim_hcs/Zarr.py index 81ba16b2..a228f2b4 100644 --- a/src/faim_hcs/Zarr.py +++ b/src/faim_hcs/Zarr.py @@ -4,6 +4,7 @@ from pathlib import Path from typing import Union +import anndata as ad import numpy as np import pandas as pd import zarr @@ -188,13 +189,18 @@ def _compute_chunk_size_cyx( img: ArrayLike, max_levels: int = 4, max_size: int = 2048, + lowest_res_target: int = 1024, write_empty_chunks: bool = True, + dimension_separator: str = "/", ) -> tuple[list[dict[str, list[int]]], int]: """Compute chunk-size for zarr storage. :param img: to be saved :param max_levels: max resolution pyramid levels :param max_size: chunk size maximum + :param lowest_res_target: lowest resolution target value. If the image is + smaller than this value, no more pyramid levels + will be created. :return: storage options, number of pyramid levels """ storage_options = [] @@ -210,9 +216,10 @@ def _compute_chunk_size_cyx( { "chunks": chunks.copy(), "write_empty_chunks": write_empty_chunks, + "dimension_separator": dimension_separator, } ) - if h <= max_size / 2 and w <= max_size / 2: + if h <= lowest_res_target and w <= lowest_res_target: return storage_options, i return storage_options, max_levels @@ -243,9 +250,16 @@ def write_image_to_group( axes: list[dict], group: Group, write_empty_chunks: bool = True, + **kwargs, ): + """ + Potential kwargs are `lowest_res_target`, `max_levels`, `max_size` and + `dimension_separator` that are used in `_compute_chunk_size_cyx`. + """ storage_options, max_layer = _compute_chunk_size_cyx( - img, write_empty_chunks=write_empty_chunks + img, + write_empty_chunks=write_empty_chunks, + **kwargs, ) scaler = Scaler(max_layer=max_layer) @@ -263,12 +277,18 @@ def write_image_and_metadata( general_metadata: dict, group: Group, write_empty_chunks: bool = True, + **kwargs, ): + """ + Potential kwargs are `lowest_res_target`, `max_levels`, `max_size` and + `dimension_separator` that are used in `_compute_chunk_size_cyx`. + """ write_image_to_group( img=img, axes=axes, group=group, write_empty_chunks=write_empty_chunks, + **kwargs, ) _set_multiscale_metadata(group=group, general_metadata=general_metadata, axes=axes) @@ -288,7 +308,12 @@ def write_cyx_image_to_well( general_metadata: dict, group: Group, write_empty_chunks: bool = True, + **kwargs, ): + """ + Potential kwargs are `lowest_res_target`, `max_levels`, `max_size` and + `dimension_separator` that are used in `_compute_chunk_size_cyx`. + """ if general_metadata["spatial-calibration-units"] == "um": axes = [ {"name": "c", "type": "channel"}, @@ -306,9 +331,37 @@ def write_cyx_image_to_well( general_metadata=general_metadata, group=group, write_empty_chunks=write_empty_chunks, + **kwargs, ) +def write_roi_table( + roi_table: pd.DataFrame, + table_name: str, + group: Group, +): + """Writes a roi table to an OME-Zarr image. If no table folder exists, it is created.""" + group_tables = group.require_group("tables") + + # Assign dtype explicitly, to avoid + # >> UserWarning: X converted to numpy array with dtype float64 + # when creating AnnData object + df_roi = roi_table.astype(np.float32) + + adata = ad.AnnData(X=df_roi) + adata.obs_names = roi_table.index + adata.var_names = list(map(str, roi_table.columns)) + ad._io.specs.write_elem(group_tables, table_name, adata) + update_table_metadata(group_tables, table_name) + + +def update_table_metadata(group_tables, table_name): + if "tables" not in group_tables.attrs: + group_tables.attrs["tables"] = [table_name] + elif table_name not in group_tables.attrs["tables"]: + group_tables.attrs["tables"] = group_tables.attrs["tables"] + [table_name] + + def write_czyx_image_to_well( img: ArrayLike, histograms: list[UIntHistogram], @@ -316,7 +369,12 @@ def write_czyx_image_to_well( general_metadata: dict, group: Group, write_empty_chunks: bool = True, + **kwargs, ): + """ + Potential kwargs are `lowest_res_target`, `max_levels`, `max_size` and + `dimension_separator` that are used in `_compute_chunk_size_cyx`. + """ if general_metadata["spatial-calibration-units"] == "um": axes = [ {"name": "c", "type": "channel"}, @@ -335,6 +393,7 @@ def write_czyx_image_to_well( general_metadata=general_metadata, group=group, write_empty_chunks=write_empty_chunks, + **kwargs, ) @@ -346,7 +405,7 @@ def build_omero_channel_metadata( * Color is computed from the metaseries wavelength metadata. * Label is the set to the metaseries _IllumSetting_ metadata. * Intensity scaling is obtained from the data histogram [0.01, - 0.99] quantiles. + 0.999] quantiles. :param ch_metadata: channel metadata from tiff-tags :param dtype: data type @@ -361,6 +420,12 @@ def build_omero_channel_metadata( proj_method = proj_method.replace(" ", "-") label = f"{proj_method}-Projection_{label}" + start = hist.quantile(0.01) + end = hist.quantile(0.999) + # Avoid rescaling from 0 to 0 (leads to napari display errors) + if start == end: + end = end + 1 + channels.append( { "active": True, @@ -373,8 +438,8 @@ def build_omero_channel_metadata( "window": { "min": np.iinfo(dtype).min, "max": np.iinfo(dtype).max, - "start": hist.quantile(0.01), - "end": hist.quantile(0.99), + "start": start, + "end": end, }, } ) @@ -394,7 +459,12 @@ def write_labels_to_group( parent_group: Group, write_empty_chunks: bool = True, overwrite: bool = False, + **kwargs, ): + """ + Potential kwargs are `lowest_res_target`, `max_levels`, `max_size` and + `dimension_separator` that are used in `_compute_chunk_size_cyx`. + """ try: subgroup = parent_group[f"labels/{labels_name}"] except KeyError: @@ -413,6 +483,7 @@ def write_labels_to_group( axes=axes, group=subgroup, write_empty_chunks=write_empty_chunks, + **kwargs, ) _copy_multiscales_metadata(parent_group, subgroup) diff --git a/tests/test_MetaSeriesUtils.py b/tests/test_MetaSeriesUtils.py index 82c22dc7..ed8acc56 100644 --- a/tests/test_MetaSeriesUtils.py +++ b/tests/test_MetaSeriesUtils.py @@ -9,6 +9,7 @@ from faim_hcs.io.MolecularDevicesImageXpress import parse_files from faim_hcs.MetaSeriesUtils import ( + _stage_label, get_well_image_CYX, get_well_image_CZYX, montage_grid_image_YX, @@ -23,10 +24,22 @@ def files(): return parse_files(ROOT_DIR / "resources" / "Projection-Mix") +@pytest.fixture +def roi_columns(): + return [ + "x_micrometer", + "y_micrometer", + "z_micrometer", + "len_x_micrometer", + "len_y_micrometer", + "len_z_micrometer", + ] + + def test_get_well_image_CYX(files): files2d = files[(files["z"].isnull()) & (files["channel"].isin(["w1", "w2"]))] for well in files2d["well"].unique(): - img, hists, ch_metadata, metadata = get_well_image_CYX( + img, hists, ch_metadata, metadata, roi_tables = get_well_image_CYX( files2d[files2d["well"] == well], channels=["w1", "w2"], assemble_fn=montage_stage_pos_image_YX, @@ -36,11 +49,13 @@ def test_get_well_image_CYX(files): assert "z-scaling" not in metadata for ch_meta in ch_metadata: assert "z-projection-method" in ch_meta + # TODO: Make some checks on roi_tables (from monaging with actual + # positions => different coordiantes) -def test_get_well_image_CYX_well_E07(files): +def test_get_well_image_CYX_well_E07(files, roi_columns): files2d = files[(files["z"].isnull()) & (files["channel"].isin(["w1", "w2"]))] - cyx, hists, ch_meta, metadata = get_well_image_CYX( + cyx, hists, ch_meta, metadata, roi_tables = get_well_image_CYX( well_files=files2d[files2d["well"] == "E07"], channels=["w1", "w2"] ) @@ -80,11 +95,28 @@ def test_get_well_image_CYX_well_E07(files): "spatial-calibration-y": 1.3668, } + assert list(roi_tables["well_ROI_table"].columns) == roi_columns + assert len(roi_tables["well_ROI_table"]) == 1 + target_values = [0.0, 0.0, 0.0, 1399.6032, 699.8016, 1.0] + assert ( + roi_tables["well_ROI_table"].loc["well_1"].values.flatten().tolist() + == target_values + ) + + assert list(roi_tables["FOV_ROI_table"].columns) == roi_columns + assert len(roi_tables["FOV_ROI_table"]) == 2 + target_values = [0.0, 699.8016, 0.0, 699.8016, 699.8016, 1.0] + assert ( + roi_tables["FOV_ROI_table"].loc["Site 2"].values.flatten().tolist() + == target_values + ) + def test_get_well_image_ZCYX(files): files3d = files[(~files["z"].isnull()) & (files["channel"].isin(["w1", "w2"]))] + z_len = {"E08": 45.0, "E07": 45.17999999999999} for well in files3d["well"].unique(): - img, hists, ch_metadata, metadata = get_well_image_CZYX( + img, hists, ch_metadata, metadata, roi_tables = get_well_image_CZYX( files3d[files3d["well"] == well], channels=["w1", "w2"], assemble_fn=montage_grid_image_YX, @@ -92,3 +124,40 @@ def test_get_well_image_ZCYX(files): assert img.shape == (2, 10, 512, 1024) assert len(hists) == 2 assert "z-scaling" in metadata + + roi_columns = [ + "x_micrometer", + "y_micrometer", + "z_micrometer", + "len_x_micrometer", + "len_y_micrometer", + "len_z_micrometer", + ] + assert list(roi_tables["well_ROI_table"].columns) == roi_columns + assert len(roi_tables["well_ROI_table"]) == 1 + target_values = [0.0, 0.0, 0.0, 1399.6032, 699.8016, z_len[well]] + assert ( + roi_tables["well_ROI_table"].loc["well_1"].values.flatten().tolist() + == target_values + ) + + assert list(roi_tables["FOV_ROI_table"].columns) == roi_columns + assert len(roi_tables["FOV_ROI_table"]) == 2 + target_values = [0.0, 699.8016, 0.0, 699.8016, 699.8016, z_len[well]] + assert ( + roi_tables["FOV_ROI_table"].loc["Site 2"].values.flatten().tolist() + == target_values + ) + + +test_stage_labels = [ + ({"stage-label": "E07 : Site 1"}, "Site 1"), + ({"stage-label": "E07 : Site 2"}, "Site 2"), + ({"stage-labels": "E07 : Site 2"}, ""), + ({}, ""), +] + + +@pytest.mark.parametrize("data,expected", test_stage_labels) +def test_stage_label_parser(data, expected): + assert _stage_label(data) == expected diff --git a/tests/test_Zarr.py b/tests/test_Zarr.py index e8df8852..0dc9a351 100644 --- a/tests/test_Zarr.py +++ b/tests/test_Zarr.py @@ -8,6 +8,8 @@ from os.path import exists, join from pathlib import Path +import anndata as ad + from faim_hcs.io.MolecularDevicesImageXpress import ( parse_multi_field_stacks, parse_single_plane_multi_fields, @@ -19,6 +21,7 @@ write_cyx_image_to_well, write_czyx_image_to_well, write_labels_to_group, + write_roi_table, ) ROOT_DIR = Path(__file__).parent @@ -98,13 +101,17 @@ def test_write_cyx_image_to_well(self): for well in self.files["well"].unique(): well_files = self.files[self.files["well"] == well] - img, hists, ch_metadata, metadata = get_well_image_CYX( + img, hists, ch_metadata, metadata, roi_tables = get_well_image_CYX( well_files=well_files, channels=["w1", "w2", "w3", "w4"] ) field = plate[well[0]][str(int(well[1:]))][0] write_cyx_image_to_well(img, hists, ch_metadata, metadata, field) + # Write all ROI tables + for roi_table in roi_tables: + write_roi_table(roi_tables[roi_table], roi_table, field) + e07 = plate["E"]["7"]["0"].attrs.asdict() assert ( self.zarr_root @@ -177,12 +184,76 @@ def test_write_cyx_image_to_well(self): / "0" / "C03_empty_histogram.npz" ).exists() + assert ( + self.zarr_root + / "Projection-Mix.zarr" + / "E" + / "7" + / "0" + / "tables" + / "well_ROI_table" + ).exists() + assert ( + self.zarr_root + / "Projection-Mix.zarr" + / "E" + / "7" + / "0" + / "tables" + / "FOV_ROI_table" + ).exists() assert "histograms" in e08.keys() assert "acquisition_metadata" in e08.keys() assert e08["multiscales"][0]["datasets"][0]["coordinateTransformations"][0][ "scale" ] == [1.0, 1.3668, 1.3668] + # Check ROI table content + table = ad.read_zarr( + self.zarr_root + / "Projection-Mix.zarr" + / "E" + / "7" + / "0" + / "tables" + / "well_ROI_table" + ) + df_well = table.to_df() + roi_columns = [ + "x_micrometer", + "y_micrometer", + "z_micrometer", + "len_x_micrometer", + "len_y_micrometer", + "len_z_micrometer", + ] + assert list(df_well.columns) == roi_columns + assert len(df_well) == 1 + target_values = [0.0, 0.0, 0.0, 1399.6031494140625, 699.8015747070312, 1.0] + assert df_well.loc["well_1"].values.flatten().tolist() == target_values + + table = ad.read_zarr( + self.zarr_root + / "Projection-Mix.zarr" + / "E" + / "7" + / "0" + / "tables" + / "FOV_ROI_table" + ) + df_fov = table.to_df() + assert list(df_fov.columns) == roi_columns + assert len(df_fov) == 2 + target_values = [ + 0.0, + 699.8015747070312, + 0.0, + 699.8015747070312, + 699.8015747070312, + 1.0, + ] + assert df_fov.loc["Site 2"].values.flatten().tolist() == target_values + def test_write_czyx_image_to_well(self): plate = build_zarr_scaffold( root_dir=self.zarr_root, @@ -194,13 +265,17 @@ def test_write_czyx_image_to_well(self): for well in self.files3d["well"].unique(): well_files = self.files3d[self.files3d["well"] == well] - img, hists, ch_metadata, metadata = get_well_image_CZYX( + img, hists, ch_metadata, metadata, roi_tables = get_well_image_CZYX( well_files=well_files, channels=["w1", "w2", "w3", "w4"] ) field = plate[well[0]][str(int(well[1:]))][0] write_czyx_image_to_well(img, hists, ch_metadata, metadata, field) + # Write all ROI tables + for roi_table in roi_tables: + write_roi_table(roi_tables[roi_table], roi_table, field) + e07 = plate["E"]["7"]["0"].attrs.asdict() assert ( self.zarr_root @@ -273,12 +348,83 @@ def test_write_czyx_image_to_well(self): / "0" / "C03_FITC_05_histogram.npz" ).exists() + assert ( + self.zarr_root + / "Projection-Mix.zarr" + / "E" + / "7" + / "0" + / "tables" + / "well_ROI_table" + ).exists() + assert ( + self.zarr_root + / "Projection-Mix.zarr" + / "E" + / "7" + / "0" + / "tables" + / "FOV_ROI_table" + ).exists() assert "histograms" in e08.keys() assert "acquisition_metadata" in e08.keys() assert e08["multiscales"][0]["datasets"][0]["coordinateTransformations"][0][ "scale" ] == [1.0, 5.0, 1.3668, 1.3668] + # Check ROI table content + table = ad.read_zarr( + self.zarr_root + / "Projection-Mix.zarr" + / "E" + / "7" + / "0" + / "tables" + / "well_ROI_table" + ) + df_well = table.to_df() + roi_columns = [ + "x_micrometer", + "y_micrometer", + "z_micrometer", + "len_x_micrometer", + "len_y_micrometer", + "len_z_micrometer", + ] + assert list(df_well.columns) == roi_columns + assert len(df_well) == 1 + target_values = [ + 0.0, + 0.0, + 0.0, + 1399.6031494140625, + 699.8015747070312, + 45.18000030517578, + ] + assert df_well.loc["well_1"].values.flatten().tolist() == target_values + + table = ad.read_zarr( + self.zarr_root + / "Projection-Mix.zarr" + / "E" + / "7" + / "0" + / "tables" + / "FOV_ROI_table" + ) + df_fov = table.to_df() + assert list(df_fov.columns) == roi_columns + assert len(df_fov) == 2 + target_values = [ + 0.0, + 699.8015747070312, + 0.0, + 699.8015747070312, + 699.8015747070312, + 45.18000030517578, + ] + assert df_fov.loc["Site 2"].values.flatten().tolist() == target_values + def test_write_labels(self): plate = build_zarr_scaffold( root_dir=self.zarr_root, @@ -288,7 +434,7 @@ def test_write_labels(self): barcode="test-barcode", ) well_files = self.files3d[self.files3d["well"] == "E07"] - img, hists, ch_metadata, metadata = get_well_image_CZYX( + img, hists, ch_metadata, metadata, roi_tables = get_well_image_CZYX( well_files=well_files, channels=["w1", "w2", "w3", "w4"] ) field = plate["E"]["7"][0]