From 9d6df8fdd57849ac920ac020740395726d33a659 Mon Sep 17 00:00:00 2001 From: William Moore Date: Thu, 20 Apr 2023 11:28:03 +0100 Subject: [PATCH 1/7] Add swap_filesets.py --- scripts/swap_filesets.py | 119 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 119 insertions(+) create mode 100644 scripts/swap_filesets.py diff --git a/scripts/swap_filesets.py b/scripts/swap_filesets.py new file mode 100644 index 00000000..73b12d9f --- /dev/null +++ b/scripts/swap_filesets.py @@ -0,0 +1,119 @@ +import argparse +import sys + +import omero.clients +from omero.cli import cli_login +from omero.gateway import BlitzGateway + +# Swap Images between 2 Filesets. Input is Fileset, Image, Plate or Screen +# Usage: $ python swap_filesets.py Image:1 Image:2 --report --dry-run + +def swap_fileset(conn, old_fileset, new_fileset, report = False, dryrun = False): + + if report: + print("Swapping images between Fileset %s and Fileset %s" % + (old_fileset.id, new_fileset.id)) + first_file = new_fileset.listFiles()[0] + file_path = first_file.getPath() + # remove any path after .zarr + if ".zarr" in file_path: + file_path = file_path.split(".zarr")[0] + ".zarr" + + new_images = new_fileset.copyImages() + old_images = old_fileset.copyImages() + + # link all OLD Images to the NEW Fileset + for image in old_images: + if report: + print("Moving Image: %s to Fileset: %s" % (img.id, new_fileset.id)) + if dryrun: + continue + img = image._obj + img.fileset = omero.model.FilesetI(new_fileset.id, False) + conn.getUpdateService().saveObject(img, conn.SERVICE_OPTS) + + # link all NEW Images (to be deleted) to the OLD Fileset + for image in new_images: + if report: + print("Moving Image: %s to Fileset: %s" % (img.id, old_fileset.id)) + if dryrun: + continue + img = image._obj + img.fileset = omero.model.FilesetI(old_fileset.id, False) + conn.getUpdateService().saveObject(img, conn.SERVICE_OPTS) + + # Print the HQL updates we need to update each Pixels to new Fileset file + # Set it to dataset.zarr/OME/METADATA.ome.xml since this works for Plates (Images???) + print(f"UPDATE pixels SET name = 'METADATA.ome.xml', path = '{file_path}/OME' where image in (select id from Image where fileset = {new_fileset.id});") + + +def get_object(conn, obj_string): + for dtype in ["Screen", "Plate", "Image", "Fileset"]: + if obj_string.startswith(dtype): + obj_id = int(obj_string.replace(dtype + ":", "")) + obj = conn.getObject(dtype, obj_id) + if obj is None: + print(obj_string, "not found!") + return obj + + +def get_fileset(conn, obj_string): + """obj_string is Image:123 or Fileset:123 or Plate:123""" + + obj = get_object(conn, obj_string) + if obj_string.startswith("Fileset:"): + return obj + if obj_string.startswith("Image:"): + return obj.getFileset() + if obj_string.startswith("Plate:"): + well = list(obj.listChildren())[0] + image = list(well.listChildren())[0].getImage() + return image.getFileset() + + +def main(argv): + """ + For all the Images in the old Screen, Plate, Image or Fileset, we swap the Fileset with the equivalent + Images in the new Screen, Plate, Image or Fileset (both old and new Images are updated). + Also prints an sql command(s) to update the pixels in the NEW Images only. + For Screens containing multiple Plates (Filesets), we match the Plates by Name + """ + parser = argparse.ArgumentParser() + parser.add_argument('old_object', help='Object:ID where Object is Screen, Plate, Image, Fileset') + parser.add_argument('new_object', help='Object:ID where Object is Screen, Plate, Image, Fileset') + parser.add_argument("--report", action="store_true", help="Print logs") + parser.add_argument("--dry-run", action="store_true", help="Don't save any changes") + args = parser.parse_args(argv) + old_object = args.old_object + new_object = args.new_object + + with cli_login() as cli: + conn = BlitzGateway(client_obj=cli._client) + if ":" not in old_object or ":" not in new_object: + print("Use e.g. $ python swap_fileset Plate:123 Plate:456") + # make sure we are dealing with same types: + assert (old_object.split(":")[0] == new_object.split(":")[0]) + + if "Screen" in old_object: + old_screen = get_object(conn, old_object) + new_screen = get_object(conn, new_object) + old_plates_by_name = {} + for plate in old_screen.listChildren(): + old_plates_by_name[plate.getName()] = plate + + for new_plate in new_screen.listChildren(): + name = new_plate.getName() + if name not in old_plates_by_name: + print(f"No Plate named {name} in {old_object}") + old_fileset = get_fileset(conn, f"Plate:{old_plates_by_name[name].id}") + new_fileset = get_fileset(conn, f"Plate:{new_plate.id}") + swap_fileset(conn, old_fileset, new_fileset, args.report, args.dry_run) + + else: + old_fileset = get_fileset(conn, old_object) + new_fileset = get_fileset(conn, new_object) + swap_fileset(conn, old_fileset, new_fileset, args.report, args.dry_run) + + +if __name__ == '__main__': + main(sys.argv[1:]) From 697b0c562ec8c770293d0f594a1c9b8f9af01e41 Mon Sep 17 00:00:00 2001 From: William Moore Date: Tue, 2 May 2023 10:38:15 +0100 Subject: [PATCH 2/7] swap filesets handle sparse plate --- scripts/swap_filesets.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/scripts/swap_filesets.py b/scripts/swap_filesets.py index 73b12d9f..916bcc64 100644 --- a/scripts/swap_filesets.py +++ b/scripts/swap_filesets.py @@ -66,9 +66,10 @@ def get_fileset(conn, obj_string): if obj_string.startswith("Image:"): return obj.getFileset() if obj_string.startswith("Plate:"): - well = list(obj.listChildren())[0] - image = list(well.listChildren())[0].getImage() - return image.getFileset() + for well in obj.listChildren(): + for ws in well.listChildren(): + image = ws.getImage() + return image.getFileset() def main(argv): From b8931e5021060f5b037eb6708021963347cd87c4 Mon Sep 17 00:00:00 2001 From: William Moore Date: Mon, 15 May 2023 12:39:11 +0100 Subject: [PATCH 3/7] swap_filesets.py unlinks old Images from Fileset --- scripts/swap_filesets.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/scripts/swap_filesets.py b/scripts/swap_filesets.py index 916bcc64..62181405 100644 --- a/scripts/swap_filesets.py +++ b/scripts/swap_filesets.py @@ -32,14 +32,14 @@ def swap_fileset(conn, old_fileset, new_fileset, report = False, dryrun = False) img.fileset = omero.model.FilesetI(new_fileset.id, False) conn.getUpdateService().saveObject(img, conn.SERVICE_OPTS) - # link all NEW Images (to be deleted) to the OLD Fileset + # unlink all NEW Images (to be deleted) for image in new_images: if report: - print("Moving Image: %s to Fileset: %s" % (img.id, old_fileset.id)) + print("Unlinking Image: %s" % (img.id)) if dryrun: continue img = image._obj - img.fileset = omero.model.FilesetI(old_fileset.id, False) + img.fileset = None conn.getUpdateService().saveObject(img, conn.SERVICE_OPTS) # Print the HQL updates we need to update each Pixels to new Fileset file @@ -74,8 +74,10 @@ def get_fileset(conn, obj_string): def main(argv): """ - For all the Images in the old Screen, Plate, Image or Fileset, we swap the Fileset with the equivalent - Images in the new Screen, Plate, Image or Fileset (both old and new Images are updated). + Swaps Fileset from 'Old Object' to 'New Object'. + For all the Images in the 'Old Object' (Screen, Plate, Image or Fileset), we swap the + Fileset to use the Fileset in the 'New Object'. Images in the `New Object` are left + unlinked to any Fileset, and can then be deleted. Also prints an sql command(s) to update the pixels in the NEW Images only. For Screens containing multiple Plates (Filesets), we match the Plates by Name """ From ea76076717c6f1963f4e127a02524753ccfc649b Mon Sep 17 00:00:00 2001 From: William Moore Date: Mon, 15 May 2023 14:18:52 +0100 Subject: [PATCH 4/7] swap_filesets output to sql file --- scripts/swap_filesets.py | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/scripts/swap_filesets.py b/scripts/swap_filesets.py index 62181405..766e5d56 100644 --- a/scripts/swap_filesets.py +++ b/scripts/swap_filesets.py @@ -8,7 +8,7 @@ # Swap Images between 2 Filesets. Input is Fileset, Image, Plate or Screen # Usage: $ python swap_filesets.py Image:1 Image:2 --report --dry-run -def swap_fileset(conn, old_fileset, new_fileset, report = False, dryrun = False): +def swap_fileset(conn, old_fileset, new_fileset, sql_filename, report = False, dryrun = False): if report: print("Swapping images between Fileset %s and Fileset %s" % @@ -44,7 +44,12 @@ def swap_fileset(conn, old_fileset, new_fileset, report = False, dryrun = False) # Print the HQL updates we need to update each Pixels to new Fileset file # Set it to dataset.zarr/OME/METADATA.ome.xml since this works for Plates (Images???) - print(f"UPDATE pixels SET name = 'METADATA.ome.xml', path = '{file_path}/OME' where image in (select id from Image where fileset = {new_fileset.id});") + cmd = f"UPDATE pixels SET name = 'METADATA.ome.xml', path = '{file_path}/OME' where image in (select id from Image where fileset = {new_fileset.id});" + with open(sql_filename, 'a') as f: + f.write(cmd) + f.write("\n") + if report: + print(cmd) def get_object(conn, obj_string): @@ -90,6 +95,9 @@ def main(argv): old_object = args.old_object new_object = args.new_object + sql_filename = f"fileset_swap_{old_object}.sql" + print("SQL writing to " + sql_filename) + with cli_login() as cli: conn = BlitzGateway(client_obj=cli._client) if ":" not in old_object or ":" not in new_object: @@ -110,13 +118,14 @@ def main(argv): print(f"No Plate named {name} in {old_object}") old_fileset = get_fileset(conn, f"Plate:{old_plates_by_name[name].id}") new_fileset = get_fileset(conn, f"Plate:{new_plate.id}") - swap_fileset(conn, old_fileset, new_fileset, args.report, args.dry_run) + swap_fileset(conn, old_fileset, new_fileset, sql_filename, args.report, args.dry_run) else: old_fileset = get_fileset(conn, old_object) new_fileset = get_fileset(conn, new_object) - swap_fileset(conn, old_fileset, new_fileset, args.report, args.dry_run) + swap_fileset(conn, old_fileset, new_fileset, sql_filename, args.report, args.dry_run) + print("SQL output added to " + sql_filename) if __name__ == '__main__': main(sys.argv[1:]) From ab7f75cc2112e08751ad4a3e81fdb9ecabbf33b4 Mon Sep 17 00:00:00 2001 From: William Moore Date: Mon, 15 May 2023 14:25:07 +0100 Subject: [PATCH 5/7] Fix print statements --- scripts/swap_filesets.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/swap_filesets.py b/scripts/swap_filesets.py index 766e5d56..c8ff2312 100644 --- a/scripts/swap_filesets.py +++ b/scripts/swap_filesets.py @@ -25,7 +25,7 @@ def swap_fileset(conn, old_fileset, new_fileset, sql_filename, report = False, d # link all OLD Images to the NEW Fileset for image in old_images: if report: - print("Moving Image: %s to Fileset: %s" % (img.id, new_fileset.id)) + print("Moving Image: %s to Fileset: %s" % (image.id, new_fileset.id)) if dryrun: continue img = image._obj @@ -35,7 +35,7 @@ def swap_fileset(conn, old_fileset, new_fileset, sql_filename, report = False, d # unlink all NEW Images (to be deleted) for image in new_images: if report: - print("Unlinking Image: %s" % (img.id)) + print("Unlinking Image: %s" % (image.id)) if dryrun: continue img = image._obj From cbe8bee375e323a9679efce41bff348080ec627c Mon Sep 17 00:00:00 2001 From: William Moore Date: Mon, 15 May 2023 14:38:01 +0100 Subject: [PATCH 6/7] Add sql_output as required argument --- scripts/swap_filesets.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/scripts/swap_filesets.py b/scripts/swap_filesets.py index c8ff2312..d355e968 100644 --- a/scripts/swap_filesets.py +++ b/scripts/swap_filesets.py @@ -89,13 +89,14 @@ def main(argv): parser = argparse.ArgumentParser() parser.add_argument('old_object', help='Object:ID where Object is Screen, Plate, Image, Fileset') parser.add_argument('new_object', help='Object:ID where Object is Screen, Plate, Image, Fileset') + parser.add_argument('sql_output', help='File path to output sql commands') parser.add_argument("--report", action="store_true", help="Print logs") parser.add_argument("--dry-run", action="store_true", help="Don't save any changes") args = parser.parse_args(argv) old_object = args.old_object new_object = args.new_object - sql_filename = f"fileset_swap_{old_object}.sql" + sql_filename = args.sql_output print("SQL writing to " + sql_filename) with cli_login() as cli: From c842dbf79ad8b013e0457d8b5b6b02dd4bf46803 Mon Sep 17 00:00:00 2001 From: William Moore Date: Thu, 18 May 2023 11:20:22 +0100 Subject: [PATCH 7/7] Add Dataset support to swap_filesets.py --- scripts/swap_filesets.py | 23 +++++++++++++++++++---- 1 file changed, 19 insertions(+), 4 deletions(-) diff --git a/scripts/swap_filesets.py b/scripts/swap_filesets.py index d355e968..e038f84a 100644 --- a/scripts/swap_filesets.py +++ b/scripts/swap_filesets.py @@ -53,7 +53,7 @@ def swap_fileset(conn, old_fileset, new_fileset, sql_filename, report = False, d def get_object(conn, obj_string): - for dtype in ["Screen", "Plate", "Image", "Fileset"]: + for dtype in ["Screen", "Plate", "Dataset", "Image", "Fileset"]: if obj_string.startswith(dtype): obj_id = int(obj_string.replace(dtype + ":", "")) obj = conn.getObject(dtype, obj_id) @@ -80,15 +80,15 @@ def get_fileset(conn, obj_string): def main(argv): """ Swaps Fileset from 'Old Object' to 'New Object'. - For all the Images in the 'Old Object' (Screen, Plate, Image or Fileset), we swap the + For all the Images in the 'Old Object' (Screen, Plate, Dataset, Image or Fileset), we swap the Fileset to use the Fileset in the 'New Object'. Images in the `New Object` are left unlinked to any Fileset, and can then be deleted. Also prints an sql command(s) to update the pixels in the NEW Images only. For Screens containing multiple Plates (Filesets), we match the Plates by Name """ parser = argparse.ArgumentParser() - parser.add_argument('old_object', help='Object:ID where Object is Screen, Plate, Image, Fileset') - parser.add_argument('new_object', help='Object:ID where Object is Screen, Plate, Image, Fileset') + parser.add_argument('old_object', help='Object:ID where Object is Screen, Plate, Dataset, Image, Fileset') + parser.add_argument('new_object', help='Object:ID where Object is Screen, Plate, Dataset, Image, Fileset') parser.add_argument('sql_output', help='File path to output sql commands') parser.add_argument("--report", action="store_true", help="Print logs") parser.add_argument("--dry-run", action="store_true", help="Don't save any changes") @@ -121,6 +121,21 @@ def main(argv): new_fileset = get_fileset(conn, f"Plate:{new_plate.id}") swap_fileset(conn, old_fileset, new_fileset, sql_filename, args.report, args.dry_run) + if "Dataset" in old_object: + old_dataset = get_object(conn, old_object) + new_dataset = get_object(conn, new_object) + old_images_by_name = {} + for image in old_dataset.listChildren(): + old_images_by_name[image.getName()] = image + + for new_image in new_dataset.listChildren(): + name = new_image.getName() + if name not in old_images_by_name: + print(f"No Image named {name} in {old_object}") + old_fileset = get_fileset(conn, f"Image:{old_images_by_name[name].id}") + new_fileset = get_fileset(conn, f"Image:{new_image.id}") + swap_fileset(conn, old_fileset, new_fileset, sql_filename, args.report, args.dry_run) + else: old_fileset = get_fileset(conn, old_object) new_fileset = get_fileset(conn, new_object)