From 070fd34a6debc31ddb2e1ebaf803fc9dd2906f21 Mon Sep 17 00:00:00 2001 From: bugraoezdemir <91956472+bugraoezdemir@users.noreply.github.com> Date: Thu, 15 Feb 2024 15:14:08 +0100 Subject: [PATCH 1/7] fix quoted path names --- run_conversion.py | 117 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 117 insertions(+) create mode 100644 run_conversion.py diff --git a/run_conversion.py b/run_conversion.py new file mode 100644 index 0000000..a11368a --- /dev/null +++ b/run_conversion.py @@ -0,0 +1,117 @@ +#!/usr/bin/env python +import shlex +import subprocess +import argparse +import json +import os +import sys +import re + +intlist = lambda s: [int(x) for x in re.findall(r'\b\d+\b', s)] + +if __name__ == '__main__': + homepath = os.environ.get('HOMEPATH') + temppath = os.environ.get('TEMPPATH') + parampath = os.environ.get('PARAMPATH') + defparamfile = os.path.join(parampath, 'params.json.default') + backupparamfile = os.path.join(parampath, 'params.json.backup') + paramfile = os.path.join(parampath, 'params.json') + + parser = argparse.ArgumentParser() + parser.add_argument('input_path') + parser.add_argument('output_path') + + # print("args are %s" % args) + with open(paramfile, 'rt') as f: + t_args = argparse.Namespace() + t_args.__dict__.update(json.load(f)) + args = parser.parse_args(namespace=t_args) + # cmd = ["#!/usr/bin/env bash\n"] + inps = args.input_path.replace("\ ", " ") + outs = args.output_path.replace("\ ", " ") + + cmd = [] + keys = args.__dict__.keys() + if args.output_type == 'ometiff': + cmd += ["bfconvert"] + if "noflat" in keys: + cmd += ["-noflat"] + if "series" in keys: + cmd += ["-series", '%s' % args.series] + if "timepoint" in keys: + cmd += ["-timepoint", '%s' % args.timepoint] + if "channel" in keys: + cmd += ["-channel", '%s' % args.channel] + if "z_slice" in keys: + cmd += ["-z", '%s' % args.z_slice] + if "range" in keys: + _range = intlist(args.range) + if len(_range) != 2: + raise TypeError('Range must have two integers specifying first and last indices of images.') + else: + cmd += ["-range"] + for i in _range: + cmd += ['%s' % i] + if "autoscale" in keys: + cmd += ["-autoscale"] + if "crop" in keys: + _crop = ''.join(args.crop[1:-1].split(' ')) + cmd += ["-crop", '%s' % _crop] + if "compression_tiff" in keys: + cmd += ["-compression", '%s' % args.compression_tiff] + if "resolution_scale" in keys: + cmd += ["-pyramid-scale", '%s' % args.resolution_scale] + if "resolutions_tiff" in keys: + cmd += ["-pyramid-resolutions", '%s' % args.resolutions_tiff] + # add here all params + cmd.append(f'{inps}') + cmd.append(f'{outs}') + # cmdstr = ''.join(cmd) + print(cmd) + # sys.stdout.write(cmdstr) + subprocess.run(cmd, shell = False) + elif args.output_type == 'omezarr': + cmd += ["bioformats2raw"] + if "min_xy_size" in keys: + cmd += ["--target-min-size", '%s' % args.min_xy_size] + if "resolutions_zarr" in keys: + cmd += ["--resolutions", '%s' % args.resolutions_zarr] + if "chunk_y" in keys: + cmd += ["--tile_height", '%s' % args.chunk_y] + if "chunk_x" in keys: + cmd += ["--tile_width", '%s' % args.chunk_x] + if "chunk_z" in keys: + cmd += ["--chunk_depth", '%s' % args.chunk_z] + if "downsample_type" in keys: + cmd += ["--downsample-type", '%s' % args.downsample_type] + if "compression_zarr" in keys: + cmd += ["--compression", '%s' % args.compression_zarr] + if "max_workers" in keys: + cmd += ["--max_workers", '%s' % args.max_workers] + if "no_nested" in keys: + if args.no_nested in (True, "True"): + cmd += ["--no-nested"] + elif args.no_nested in (False, "False", None, "None"): + pass + else: + raise ValueError(f"--no-nested cannot have the value {args.no_nested}") + if "drop_series" in keys: + if args.drop_series in (True, "True"): + val = '%2$d' + cmd += ["--scale-format-string", val] + elif args.drop_series in (False, "False", None, "None"): + pass + else: + raise ValueError(f"--drop_series cannot have the value {args.drop_series}") + if "overwrite" in keys: + cmd += ["--overwrite"] + cmd.append(f"{inps}") + cmd.append(f"{outs}") + # cmdstr = ''.join(cmd) + print(cmd) + # sys.stdout.write(cmdstr) + subprocess.run(cmd, shell = False) + + + + From dfacd4b0cf5d7d8ea8150066071da091ccadbb15 Mon Sep 17 00:00:00 2001 From: bugraoezdemir <91956472+bugraoezdemir@users.noreply.github.com> Date: Thu, 15 Feb 2024 15:15:10 +0100 Subject: [PATCH 2/7] Delete run_conversion.py --- run_conversion.py | 117 ---------------------------------------------- 1 file changed, 117 deletions(-) delete mode 100644 run_conversion.py diff --git a/run_conversion.py b/run_conversion.py deleted file mode 100644 index a11368a..0000000 --- a/run_conversion.py +++ /dev/null @@ -1,117 +0,0 @@ -#!/usr/bin/env python -import shlex -import subprocess -import argparse -import json -import os -import sys -import re - -intlist = lambda s: [int(x) for x in re.findall(r'\b\d+\b', s)] - -if __name__ == '__main__': - homepath = os.environ.get('HOMEPATH') - temppath = os.environ.get('TEMPPATH') - parampath = os.environ.get('PARAMPATH') - defparamfile = os.path.join(parampath, 'params.json.default') - backupparamfile = os.path.join(parampath, 'params.json.backup') - paramfile = os.path.join(parampath, 'params.json') - - parser = argparse.ArgumentParser() - parser.add_argument('input_path') - parser.add_argument('output_path') - - # print("args are %s" % args) - with open(paramfile, 'rt') as f: - t_args = argparse.Namespace() - t_args.__dict__.update(json.load(f)) - args = parser.parse_args(namespace=t_args) - # cmd = ["#!/usr/bin/env bash\n"] - inps = args.input_path.replace("\ ", " ") - outs = args.output_path.replace("\ ", " ") - - cmd = [] - keys = args.__dict__.keys() - if args.output_type == 'ometiff': - cmd += ["bfconvert"] - if "noflat" in keys: - cmd += ["-noflat"] - if "series" in keys: - cmd += ["-series", '%s' % args.series] - if "timepoint" in keys: - cmd += ["-timepoint", '%s' % args.timepoint] - if "channel" in keys: - cmd += ["-channel", '%s' % args.channel] - if "z_slice" in keys: - cmd += ["-z", '%s' % args.z_slice] - if "range" in keys: - _range = intlist(args.range) - if len(_range) != 2: - raise TypeError('Range must have two integers specifying first and last indices of images.') - else: - cmd += ["-range"] - for i in _range: - cmd += ['%s' % i] - if "autoscale" in keys: - cmd += ["-autoscale"] - if "crop" in keys: - _crop = ''.join(args.crop[1:-1].split(' ')) - cmd += ["-crop", '%s' % _crop] - if "compression_tiff" in keys: - cmd += ["-compression", '%s' % args.compression_tiff] - if "resolution_scale" in keys: - cmd += ["-pyramid-scale", '%s' % args.resolution_scale] - if "resolutions_tiff" in keys: - cmd += ["-pyramid-resolutions", '%s' % args.resolutions_tiff] - # add here all params - cmd.append(f'{inps}') - cmd.append(f'{outs}') - # cmdstr = ''.join(cmd) - print(cmd) - # sys.stdout.write(cmdstr) - subprocess.run(cmd, shell = False) - elif args.output_type == 'omezarr': - cmd += ["bioformats2raw"] - if "min_xy_size" in keys: - cmd += ["--target-min-size", '%s' % args.min_xy_size] - if "resolutions_zarr" in keys: - cmd += ["--resolutions", '%s' % args.resolutions_zarr] - if "chunk_y" in keys: - cmd += ["--tile_height", '%s' % args.chunk_y] - if "chunk_x" in keys: - cmd += ["--tile_width", '%s' % args.chunk_x] - if "chunk_z" in keys: - cmd += ["--chunk_depth", '%s' % args.chunk_z] - if "downsample_type" in keys: - cmd += ["--downsample-type", '%s' % args.downsample_type] - if "compression_zarr" in keys: - cmd += ["--compression", '%s' % args.compression_zarr] - if "max_workers" in keys: - cmd += ["--max_workers", '%s' % args.max_workers] - if "no_nested" in keys: - if args.no_nested in (True, "True"): - cmd += ["--no-nested"] - elif args.no_nested in (False, "False", None, "None"): - pass - else: - raise ValueError(f"--no-nested cannot have the value {args.no_nested}") - if "drop_series" in keys: - if args.drop_series in (True, "True"): - val = '%2$d' - cmd += ["--scale-format-string", val] - elif args.drop_series in (False, "False", None, "None"): - pass - else: - raise ValueError(f"--drop_series cannot have the value {args.drop_series}") - if "overwrite" in keys: - cmd += ["--overwrite"] - cmd.append(f"{inps}") - cmd.append(f"{outs}") - # cmdstr = ''.join(cmd) - print(cmd) - # sys.stdout.write(cmdstr) - subprocess.run(cmd, shell = False) - - - - From 89db5be71e35adc1856055a23c139e068af19197 Mon Sep 17 00:00:00 2001 From: bugraoezdemir <91956472+bugraoezdemir@users.noreply.github.com> Date: Thu, 15 Feb 2024 15:15:45 +0100 Subject: [PATCH 3/7] fix quoted path names --- bin/run_conversion.py | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/bin/run_conversion.py b/bin/run_conversion.py index a235073..a11368a 100644 --- a/bin/run_conversion.py +++ b/bin/run_conversion.py @@ -89,13 +89,24 @@ if "max_workers" in keys: cmd += ["--max_workers", '%s' % args.max_workers] if "no_nested" in keys: - cmd += ["--no-nested"] + if args.no_nested in (True, "True"): + cmd += ["--no-nested"] + elif args.no_nested in (False, "False", None, "None"): + pass + else: + raise ValueError(f"--no-nested cannot have the value {args.no_nested}") if "drop_series" in keys: - cmd += ["--scale-format-string", '%s' % "'%2$d'"] + if args.drop_series in (True, "True"): + val = '%2$d' + cmd += ["--scale-format-string", val] + elif args.drop_series in (False, "False", None, "None"): + pass + else: + raise ValueError(f"--drop_series cannot have the value {args.drop_series}") if "overwrite" in keys: cmd += ["--overwrite"] - cmd.append(f'{inps}') - cmd.append(f'{outs}') + cmd.append(f"{inps}") + cmd.append(f"{outs}") # cmdstr = ''.join(cmd) print(cmd) # sys.stdout.write(cmdstr) From 1b86c4ef471e837e07b1eda010f3e3df2e4b9a28 Mon Sep 17 00:00:00 2001 From: bugraoezdemir <91956472+bugraoezdemir@users.noreply.github.com> Date: Thu, 15 Feb 2024 18:22:55 +0100 Subject: [PATCH 4/7] fix parseCsv.py --- bin/parseCsv.py | 1 + 1 file changed, 1 insertion(+) diff --git a/bin/parseCsv.py b/bin/parseCsv.py index ecbeeec..3dc4d77 100755 --- a/bin/parseCsv.py +++ b/bin/parseCsv.py @@ -34,6 +34,7 @@ relpath = dictitem[namecol] filename = os.path.basename(relpath) pretext = os.path.dirname(relpath) + if pretext.startswith('/'): pretext = pretext[1:] newroot = os.path.join(rootpath, pretext) fnameok = True From bcb5cad6735c61bf3292ddff81f0dcda52a6c26f Mon Sep 17 00:00:00 2001 From: bugraoezdemir Date: Sat, 17 Feb 2024 19:46:12 +0100 Subject: [PATCH 5/7] fix verbosity --- modules/functions.nf | 2 +- modules/subworkflows_ometiff.nf | 2 +- modules/subworkflows_omezarr.nf | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/modules/functions.nf b/modules/functions.nf index c6cea8a..98d1407 100755 --- a/modules/functions.nf +++ b/modules/functions.nf @@ -40,7 +40,7 @@ def verify_filenames_fromPath(directory, selby, rejby) { // println((file.toString().contains(selby))) // println(!(file.toString().contains(rejby))) } - println(files) +// println(files) truth = true files.each { if (it.toString().contains(" ")) { diff --git a/modules/subworkflows_ometiff.nf b/modules/subworkflows_ometiff.nf index 9c2e71f..8ad230f 100755 --- a/modules/subworkflows_ometiff.nf +++ b/modules/subworkflows_ometiff.nf @@ -191,7 +191,7 @@ workflow Convert2OMETIFF_FromS3_CSV { // s3 &! merged && CSV } else { def fpath = file(params.in_path) - println(fpath) +// println(fpath) parsedCsv = ParseCsv( fpath.toString(), params.root_column, params.input_column, 'parsed.txt' ) // CAREFUL! ch_ = Channel.fromPath(fpath.toString()). splitCsv(header:true) diff --git a/modules/subworkflows_omezarr.nf b/modules/subworkflows_omezarr.nf index 53e78cb..186afcf 100755 --- a/modules/subworkflows_omezarr.nf +++ b/modules/subworkflows_omezarr.nf @@ -191,7 +191,7 @@ workflow Convert2OMEZARR_FromS3_CSV { // s3 &! merged && CSV } else { def fpath = file(params.in_path) - println(fpath) +// println(fpath) parsedCsv = ParseCsv( fpath.toString(), params.root_column, params.input_column, 'parsed.txt' ) // CAREFUL! ch_ = Channel.fromPath(fpath.toString()). splitCsv(header:true) From 183f9dce03fcf1fa02472a22e592dc0f8e3fbf6a Mon Sep 17 00:00:00 2001 From: bugraoezdemir Date: Mon, 19 Feb 2024 13:07:49 +0100 Subject: [PATCH 6/7] update processes and subworkflows --- bin/__pycache__/PatternHandler.cpython-39.pyc | Bin 17259 -> 17265 bytes modules/processes.nf | 62 ++++++++++++++++++ modules/subworkflows_ometiff.nf | 7 +- modules/subworkflows_omezarr.nf | 7 +- 4 files changed, 70 insertions(+), 6 deletions(-) diff --git a/bin/__pycache__/PatternHandler.cpython-39.pyc b/bin/__pycache__/PatternHandler.cpython-39.pyc index 7ad348bac29827401b299c164baf940a5303c85f..4c7e048211e14582d52fab2570b4a709d8d3402b 100644 GIT binary patch delta 49 zcmaFe#`v*~kvox>mx}=imY=<_k-LJCF=%oNqXK_wMM+vtetCRyeqLE>QOV}DjQbn` Dl2{Q} delta 43 xcmey^#`wC8kvox>mx}=i+%)EF 0 ) { @@ -167,8 +167,9 @@ workflow Convert2OMETIFF_FromLocal_CSV { // s3 &! merged && CSV ch = ch1 } output = Convert_EachFile2SeparateOMETIFF(ch) - UpdateCsv(parsedCsv, "RootOriginal", "ImageNameOriginal", "ometiff") - if (params.dest_type == "s3") { + mock = output.collect().flatten().first() + UpdateCsv(parsedCsv, "RootOriginal", "ImageNameOriginal", "ometiff", mock) + if ( params.dest_type == "s3" ) { Transfer_Local2S3Storage(output) Transfer_CSV2S3Storage(UpdateCsv.out) } diff --git a/modules/subworkflows_omezarr.nf b/modules/subworkflows_omezarr.nf index 186afcf..475014a 100755 --- a/modules/subworkflows_omezarr.nf +++ b/modules/subworkflows_omezarr.nf @@ -157,7 +157,7 @@ workflow Convert2OMEZARR_FromLocal_CSV { // s3 &! merged && CSV } else { def fpath = file(params.in_path) - parsedCsv = ParseCsv( fpath.toString(), params.root_column, params.input_column, 'parsed.txt' ) // CAREFUL! + parsedCsv = ParseCsv( fpath.toString(), params.root_column, params.input_column, 'parsed.txt' ) ch0 = Csv2Symlink1( parsedCsv, "RootOriginal", "ImageNameOriginal", 'symlinks' ).flatten() ch1 = ch0.filter { it.toString().contains(params.pattern) } if ( params.reject_pattern.size() > 0 ) { @@ -167,8 +167,9 @@ workflow Convert2OMEZARR_FromLocal_CSV { // s3 &! merged && CSV ch = ch1 } output = Convert_EachFile2SeparateOMEZARR(ch) - UpdateCsv(parsedCsv, "RootOriginal", "ImageNameOriginal", "ometiff") - if (params.dest_type == "s3") { + mock = output.collect().flatten().first() + UpdateCsv(parsedCsv, "RootOriginal", "ImageNameOriginal", "ometiff", mock) + if ( params.dest_type == "s3" ) { Transfer_Local2S3Storage(output) Transfer_CSV2S3Storage(UpdateCsv.out) } From 3fcce9429bc39dbc3420447c262e3f4ac218f8fd Mon Sep 17 00:00:00 2001 From: bugraoezdemir Date: Mon, 19 Feb 2024 14:00:02 +0100 Subject: [PATCH 7/7] update subworkflows --- modules/subworkflows_ometiff.nf | 3 ++- modules/subworkflows_omezarr.nf | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/modules/subworkflows_ometiff.nf b/modules/subworkflows_ometiff.nf index dc8c683..05ca4c4 100755 --- a/modules/subworkflows_ometiff.nf +++ b/modules/subworkflows_ometiff.nf @@ -209,7 +209,8 @@ workflow Convert2OMETIFF_FromS3_CSV { // s3 &! merged && CSV ch1f = ch1.flatMap { file(it).Name } ch = Transfer_S3Storage2Local(ch1, ch1f) output = Convert_EachFile2SeparateOMETIFF(ch) - UpdateCsv(parsedCsv, "RootOriginal", "ImageNameOriginal", "ometiff") + mock = output.collect().flatten().first() + UpdateCsv(parsedCsv, "RootOriginal", "ImageNameOriginal", "ometiff", mock) if (params.dest_type == "s3") { Transfer_Local2S3Storage(output) Transfer_CSV2S3Storage(UpdateCsv.out) diff --git a/modules/subworkflows_omezarr.nf b/modules/subworkflows_omezarr.nf index 475014a..395910c 100755 --- a/modules/subworkflows_omezarr.nf +++ b/modules/subworkflows_omezarr.nf @@ -209,7 +209,8 @@ workflow Convert2OMEZARR_FromS3_CSV { // s3 &! merged && CSV ch1f = ch1.flatMap { file(it).Name } ch = Transfer_S3Storage2Local(ch1, ch1f) output = Convert_EachFile2SeparateOMEZARR(ch) - UpdateCsv(parsedCsv, "RootOriginal", "ImageNameOriginal", "ometiff") + mock = output.collect().flatten().first() + UpdateCsv(parsedCsv, "RootOriginal", "ImageNameOriginal", "ometiff", mock) if (params.dest_type == "s3") { Transfer_Local2S3Storage(output) Transfer_CSV2S3Storage(UpdateCsv.out)