From cc45b8f87abae5114888edb479ae607d74a7215b Mon Sep 17 00:00:00 2001
From: frheault <francois.m.rheault@usherbrooke.ca>
Date: Tue, 2 May 2023 10:33:11 -0400
Subject: [PATCH 1/5] Fix missing data

---
 main.nf | 12 +++++++++++-
 1 file changed, 11 insertions(+), 1 deletion(-)

diff --git a/main.nf b/main.nf
index bad54aa..03da126 100644
--- a/main.nf
+++ b/main.nf
@@ -535,7 +535,7 @@ process Bundle_Endpoints_Metrics {
         b_metrics="$metrics"
     fi
 
-    scil_compute_endpoints_metric.py \$bundle \${b_metrics} \${bname}
+    scil_project_streamlines_to_map.py \$bundle \${bname} --in_metrics \${b_metrics} 
     cd \${bname}
     for i in *.nii.gz;
     do
@@ -782,6 +782,7 @@ process Aggregate_All_Lesion_Load {
     script:
     """
     scil_merge_json.py $jsons lesion_load.json --average_last_layer --recursive
+    scil_harmonize_json.py lesion_load.json lesion_load.json -f -v --sort_keys
     scil_convert_json_to_xlsx.py lesion_load.json lesion_load.xlsx
     """
 }
@@ -809,6 +810,7 @@ process Aggregate_All_Lesion_Load_Per_Point {
     done
     scil_merge_json.py *_avg.json lesion_load_per_point.json  \
         --recursive
+    scil_harmonize_json.py lesion_load_per_point.json lesion_load_per_point.json -f -v --sort_keys
     scil_convert_json_to_xlsx.py lesion_load_per_point.json lesion_load_per_point.xlsx \
         --stats_over_population
     """
@@ -832,6 +834,7 @@ process Aggregate_All_Endpoints_Map {
     script:
     """
     scil_merge_json.py $jsons endpoints_map.json --no_list
+    scil_harmonize_json.py endpoints_map.json endpoints_map.json -f -v --sort_keys
     scil_convert_json_to_xlsx.py endpoints_map.json endpoints_map.xlsx
     """
 }
@@ -854,6 +857,7 @@ process Aggregate_All_Endpoints_Metric_Stats {
     script:
     """
     scil_merge_json.py $jsons endpoints_metric_stats.json --no_list
+    scil_harmonize_json.py endpoints_metric_stats.json endpoints_metric_stats.json -f -v --sort_keys
     scil_convert_json_to_xlsx.py endpoints_metric_stats.json endpoints_metric_stats.xlsx
     """
 }
@@ -876,6 +880,7 @@ process Aggregate_All_Bundle_Length_Stats {
     script:
     """
     scil_merge_json.py $jsons length_stats.json --no_list
+    scil_harmonize_json.py length_stats.json length_stats.json -f -v --sort_keys
     scil_convert_json_to_xlsx.py length_stats.json length_stats.xlsx
     """
 }
@@ -898,6 +903,7 @@ process Aggregate_All_mean_std {
     script:
     """
     scil_merge_json.py $jsons mean_std.json --no_list
+    scil_harmonize_json.py mean_std.json mean_std.json -f -v --sort_keys
     scil_convert_json_to_xlsx.py mean_std.json mean_std.xlsx
     """
 }
@@ -920,6 +926,7 @@ process Aggregate_All_Volume {
     script:
     """
     scil_merge_json.py $jsons volumes.json --no_list
+    scil_harmonize_json.py volumes.json volumes.json -f -v --sort_keys
     scil_convert_json_to_xlsx.py volumes.json volumes.xlsx
     """
 }
@@ -942,6 +949,7 @@ process Aggregate_All_Streamline_Count {
     script:
     """
     scil_merge_json.py $jsons streamline_count.json --no_list
+    scil_harmonize_json.py streamline_count.json streamline_count.json -f -v --sort_keys
     scil_convert_json_to_xlsx.py streamline_count.json streamline_count.xlsx
     """
 }
@@ -964,6 +972,7 @@ process Aggregate_All_Volume_Per_Label {
     script:
     """
     scil_merge_json.py $jsons volume_per_label.json --no_list
+    scil_harmonize_json.py volume_per_label.json volume_per_label.json -f -v --sort_keys
     scil_convert_json_to_xlsx.py volume_per_label.json volume_per_label.xlsx
     """
 }
@@ -991,6 +1000,7 @@ process Aggregate_All_Mean_Std_Per_Point {
     done
     scil_merge_json.py *_avg.json mean_std_per_point.json  \
         --recursive
+    scil_harmonize_json.py mean_std_per_point.json mean_std_per_point.json -f -v --sort_keys
     scil_convert_json_to_xlsx.py mean_std_per_point.json mean_std_per_point.xlsx \
         --stats_over_population
     """

From e06ed920a15be75978fe3233897d96f94be19aa2 Mon Sep 17 00:00:00 2001
From: frheault <francois.m.rheault@usherbrooke.ca>
Date: Mon, 29 May 2023 10:15:36 -0400
Subject: [PATCH 2/5] Fix afd and labels/distances duplicated

---
 main.nf | 42 ++++++++++++++++++++----------------------
 1 file changed, 20 insertions(+), 22 deletions(-)

diff --git a/main.nf b/main.nf
index 03da126..3007633 100644
--- a/main.nf
+++ b/main.nf
@@ -478,11 +478,11 @@ process Bundle_Metrics_Stats_In_Endpoints {
         bname=\${bname/_endpoints_map_head/}
         mv \$map \${bname}_head.nii.gz
         mv \${map/_head/_tail} \${bname}_tail.nii.gz
-        if [ -f "\${bname}_afd_metric.nii.gz" ]; then
-            b_metrics="!(*afd*|*head*|*tail*).nii.gz \${bname}_afd_metric.nii.gz"
-        else
-            b_metrics="$metrics"
-        fi
+
+        mv \${bname}_afd_metric.nii.gz afd_metric.nii.gz
+        b_metrics=($metrics)
+        b_metrics="\${b_metrics[@]//*afd_metric*}"
+        b_metrics+="afd_metric.nii.gz"
 
         scil_compute_metrics_stats_in_ROI.py \${bname}_head.nii.gz $normalize_weights\
             --metrics \${b_metrics} > \${bname}_head.json
@@ -529,17 +529,16 @@ process Bundle_Endpoints_Metrics {
     fi
     bname=\${bname/_uniformized/}
     mkdir \${bname}
-    if [ -f "\${bname}_afd_metric.nii.gz" ]; then
-        b_metrics="!(*afd*).nii.gz \${bname}_afd_metric.nii.gz"
-    else
-        b_metrics="$metrics"
-    fi
+
+    mv \${bname}_afd_metric.nii.gz afd_metric.nii.gz
+    b_metrics=($metrics)
+    b_metrics="\${b_metrics[@]//*afd_metric*}"
+    b_metrics+="afd_metric.nii.gz"
 
     scil_project_streamlines_to_map.py \$bundle \${bname} --in_metrics \${b_metrics} 
     cd \${bname}
     for i in *.nii.gz;
-    do
-    mv "\$i" "${sid}__\$i";
+        do mv "\$i" "${sid}__\$i";
     done
     rename s/${sid}__${sid}__/${sid}__/ *
 
@@ -579,11 +578,11 @@ process Bundle_Mean_Std {
         bname=\${bname/_uniformized/}
         mv \$bundle \$bname.trk
 
-        if [ -f "\${bname}_afd_metric.nii.gz" ]; then
-            b_metrics="!(*afd*).nii.gz \${bname}_afd_metric.nii.gz"
-        else
-            b_metrics="$metrics"
-        fi
+        mv \${bname}_afd_metric.nii.gz afd_metric.nii.gz
+        b_metrics=($metrics)
+        b_metrics="\${b_metrics[@]//*afd_metric*}"
+        b_metrics+="afd_metric.nii.gz"
+
         scil_compute_bundle_mean_std.py $density_weighting \$bname.trk \${b_metrics} >\
             \${bname}.json
     done
@@ -714,11 +713,10 @@ process Bundle_Mean_Std_Per_Point {
         label_map=${sid}__\${bname}_labels.nii.gz
         distance_map=${sid}__\${bname}_distances.nii.gz
 
-        if [ -f "\${bname}_afd_metric.nii.gz" ]; then
-            b_metrics="!(*afd*).nii.gz \${bname}_afd_metric.nii.gz"
-        else
-            b_metrics="$metrics"
-        fi
+        mv \${bname}_afd_metric.nii.gz afd_metric.nii.gz
+        b_metrics=($metrics)
+        b_metrics="\${b_metrics[@]//*afd_metric*}"
+        b_metrics+="afd_metric.nii.gz"
 
         scil_compute_bundle_mean_std_per_point.py \$bname.trk \$label_map \
             \${b_metrics} --sort_keys $density_weighting > \$bname.json

From a62a4133c7b3cbbe54c625f2e2c9416cd91a624f Mon Sep 17 00:00:00 2001
From: frheault <francois.m.rheault@usherbrooke.ca>
Date: Mon, 29 May 2023 10:20:02 -0400
Subject: [PATCH 3/5] Extra space

---
 main.nf | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/main.nf b/main.nf
index 3007633..3fe8330 100644
--- a/main.nf
+++ b/main.nf
@@ -482,7 +482,7 @@ process Bundle_Metrics_Stats_In_Endpoints {
         mv \${bname}_afd_metric.nii.gz afd_metric.nii.gz
         b_metrics=($metrics)
         b_metrics="\${b_metrics[@]//*afd_metric*}"
-        b_metrics+="afd_metric.nii.gz"
+        b_metrics+=" afd_metric.nii.gz"
 
         scil_compute_metrics_stats_in_ROI.py \${bname}_head.nii.gz $normalize_weights\
             --metrics \${b_metrics} > \${bname}_head.json
@@ -533,7 +533,7 @@ process Bundle_Endpoints_Metrics {
     mv \${bname}_afd_metric.nii.gz afd_metric.nii.gz
     b_metrics=($metrics)
     b_metrics="\${b_metrics[@]//*afd_metric*}"
-    b_metrics+="afd_metric.nii.gz"
+    b_metrics+=" afd_metric.nii.gz"
 
     scil_project_streamlines_to_map.py \$bundle \${bname} --in_metrics \${b_metrics} 
     cd \${bname}
@@ -581,7 +581,7 @@ process Bundle_Mean_Std {
         mv \${bname}_afd_metric.nii.gz afd_metric.nii.gz
         b_metrics=($metrics)
         b_metrics="\${b_metrics[@]//*afd_metric*}"
-        b_metrics+="afd_metric.nii.gz"
+        b_metrics+=" afd_metric.nii.gz"
 
         scil_compute_bundle_mean_std.py $density_weighting \$bname.trk \${b_metrics} >\
             \${bname}.json
@@ -716,7 +716,7 @@ process Bundle_Mean_Std_Per_Point {
         mv \${bname}_afd_metric.nii.gz afd_metric.nii.gz
         b_metrics=($metrics)
         b_metrics="\${b_metrics[@]//*afd_metric*}"
-        b_metrics+="afd_metric.nii.gz"
+        b_metrics+=" afd_metric.nii.gz"
 
         scil_compute_bundle_mean_std_per_point.py \$bname.trk \$label_map \
             \${b_metrics} --sort_keys $density_weighting > \$bname.json

From 16837d7f9c643d63a7626423a4bdea829e3ba1c4 Mon Sep 17 00:00:00 2001
From: frheault <francois.m.rheault@usherbrooke.ca>
Date: Wed, 21 Jun 2023 14:10:10 -0400
Subject: [PATCH 4/5] No fodf bug fix, new labels map support

---
 USAGE           |  3 ---
 main.nf         | 48 +++++++++++++++++++++++++++++-------------------
 nextflow.config |  3 ---
 3 files changed, 29 insertions(+), 25 deletions(-)

diff --git a/USAGE b/USAGE
index 7fafa5b..c43d26c 100644
--- a/USAGE
+++ b/USAGE
@@ -68,9 +68,6 @@ OPTIONAL ARGUMENTS (current value)
 --use_provided_centroids                    Use the provided pre-computed centroids rather than using automatic computation ($use_provided_centroids)
 --bundle_suffix_to_remove                   Use to remove the suffix from RBx-Flow ($bundle_suffix_to_remove)
 
---min_streamline_count                      Minimum streamline count threshold from trimming of 'edges' in the voxel labels map ($min_streamline_count)
---min_voxel_count                           Minimum voxel count threshold from trimming of 'edges' in the voxel labels map ($min_voxel_count)
-
 --mean_std_density_weighting                Weight the mean/std by the local tract density ($mean_std_density_weighting)
 --mean_std_per_point_density_weighting      Weight the mean/std per point by the local tract density ($mean_std_per_point_density_weighting)
 --endpoints_metric_stats_normalize_weights  Normalize the weights to the [0, 1] range ($endpoints_metric_stats_normalize_weights)
diff --git a/main.nf b/main.nf
index 3fe8330..3918e12 100644
--- a/main.nf
+++ b/main.nf
@@ -257,9 +257,7 @@ process Bundle_Label_And_Distance_Maps {
 
         centroid=${sid}__\${bname}_centroid_${params.nb_points}.trk
         if [[ -f \${centroid} ]]; then
-            scil_compute_bundle_voxel_label_map.py \$bundle \${centroid} tmp_out\
-                --min_streamline_count ${params.min_streamline_count} \
-                --min_voxel_count ${params.min_voxel_count} -f
+            scil_compute_bundle_voxel_label_map.py \$bundle \${centroid} tmp_out -f
 
             mv tmp_out/labels_map.nii.gz ${sid}__\${bname}_labels.nii.gz
             mv tmp_out/distance_map.nii.gz ${sid}__\${bname}_distances.nii.gz
@@ -445,7 +443,7 @@ process Bundle_Endpoints_Map {
 metrics_for_endpoints_roi_stats
     .mix(fixel_afd_for_endpoints_roi_stats)
     .groupTuple(by: 0)
-     .map{it -> [it[0], it[1..-1].flatten()]}
+    .map{it -> [it[0], it[1..-1].flatten()]}
     .set{metrics_afd_for_endpoints_roi_stats}
 
 metrics_afd_for_endpoints_roi_stats
@@ -479,10 +477,13 @@ process Bundle_Metrics_Stats_In_Endpoints {
         mv \$map \${bname}_head.nii.gz
         mv \${map/_head/_tail} \${bname}_tail.nii.gz
 
-        mv \${bname}_afd_metric.nii.gz afd_metric.nii.gz
         b_metrics=($metrics)
-        b_metrics="\${b_metrics[@]//*afd_metric*}"
-        b_metrics+=" afd_metric.nii.gz"
+        if [[ -f \${bname}_afd_metric.nii.gz ]];
+        then
+            mv \${bname}_afd_metric.nii.gz afd_metric.nii.gz
+            b_metrics="\${b_metrics[@]//*afd_metric*}"
+            b_metrics+=" afd_metric.nii.gz"
+        fi
 
         scil_compute_metrics_stats_in_ROI.py \${bname}_head.nii.gz $normalize_weights\
             --metrics \${b_metrics} > \${bname}_head.json
@@ -498,7 +499,7 @@ process Bundle_Metrics_Stats_In_Endpoints {
 metrics_for_endpoints_metrics
     .mix(fixel_afd_for_endpoints_metrics)
     .groupTuple(by: 0)
-    .map{it -> [it[0], it[1..-1].flatten()]}
+   .map{it -> [it[0], it[1..-1].flatten()]}
     .set{metrics_afd_for_endpoints_metrics}
 
 bundles_for_endpoints_metrics
@@ -530,10 +531,13 @@ process Bundle_Endpoints_Metrics {
     bname=\${bname/_uniformized/}
     mkdir \${bname}
 
-    mv \${bname}_afd_metric.nii.gz afd_metric.nii.gz
     b_metrics=($metrics)
-    b_metrics="\${b_metrics[@]//*afd_metric*}"
-    b_metrics+=" afd_metric.nii.gz"
+    if [[ -f \${bname}_afd_metric.nii.gz ]];
+    then
+        mv \${bname}_afd_metric.nii.gz afd_metric.nii.gz
+        b_metrics="\${b_metrics[@]//*afd_metric*}"
+        b_metrics+=" afd_metric.nii.gz"
+    fi
 
     scil_project_streamlines_to_map.py \$bundle \${bname} --in_metrics \${b_metrics} 
     cd \${bname}
@@ -548,7 +552,7 @@ process Bundle_Endpoints_Metrics {
 metrics_for_mean_std
     .mix(fixel_afd_for_mean_std)
     .groupTuple(by: 0)
-     .map{it -> [it[0], it[1..-1].flatten()]}
+    .map{it -> [it[0], it[1..-1].flatten()]}
     .set{metrics_afd_for_mean_std}
 
 metrics_afd_for_mean_std
@@ -578,10 +582,13 @@ process Bundle_Mean_Std {
         bname=\${bname/_uniformized/}
         mv \$bundle \$bname.trk
 
-        mv \${bname}_afd_metric.nii.gz afd_metric.nii.gz
         b_metrics=($metrics)
-        b_metrics="\${b_metrics[@]//*afd_metric*}"
-        b_metrics+=" afd_metric.nii.gz"
+        if [[ -f \${bname}_afd_metric.nii.gz ]];
+        then
+            mv \${bname}_afd_metric.nii.gz afd_metric.nii.gz
+            b_metrics="\${b_metrics[@]//*afd_metric*}"
+            b_metrics+=" afd_metric.nii.gz"
+        fi
 
         scil_compute_bundle_mean_std.py $density_weighting \$bname.trk \${b_metrics} >\
             \${bname}.json
@@ -675,7 +682,7 @@ process Bundle_Volume_Per_Label {
 metrics_for_mean_std_per_point
     .mix(fixel_afd_for_mean_std_per_point)
     .groupTuple(by: 0)
-     .map{it -> [it[0], it[1..-1].flatten()]}
+    .map{it -> [it[0], it[1..-1].flatten()]}
     .set{metrics_afd_for_std_per_point}
 
 metrics_afd_for_std_per_point
@@ -713,10 +720,13 @@ process Bundle_Mean_Std_Per_Point {
         label_map=${sid}__\${bname}_labels.nii.gz
         distance_map=${sid}__\${bname}_distances.nii.gz
 
-        mv \${bname}_afd_metric.nii.gz afd_metric.nii.gz
         b_metrics=($metrics)
-        b_metrics="\${b_metrics[@]//*afd_metric*}"
-        b_metrics+=" afd_metric.nii.gz"
+        if [[ -f \${bname}_afd_metric.nii.gz ]];
+        then
+            mv \${bname}_afd_metric.nii.gz afd_metric.nii.gz
+            b_metrics="\${b_metrics[@]//*afd_metric*}"
+            b_metrics+=" afd_metric.nii.gz"
+        fi
 
         scil_compute_bundle_mean_std_per_point.py \$bname.trk \$label_map \
             \${b_metrics} --sort_keys $density_weighting > \$bname.json
diff --git a/nextflow.config b/nextflow.config
index 7981f2a..94e51e9 100644
--- a/nextflow.config
+++ b/nextflow.config
@@ -19,9 +19,6 @@ params {
     skip_projection_endpoints_metrics=true
     bundle_suffix_to_remove='_cleaned'
 
-    min_streamline_count=100000
-    min_voxel_count=1000000
-
     colors=["AC":"0x02d983",
             "AF_L":"0xcc0000",
             "AF_R":"0xffdf0f",

From 1f33ce64780030e15d61af9a845510692c71757d Mon Sep 17 00:00:00 2001
From: frheault <francois.m.rheault@usherbrooke.ca>
Date: Thu, 22 Jun 2023 12:01:57 -0400
Subject: [PATCH 5/5] project

---
 main.nf | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/main.nf b/main.nf
index 3918e12..72f2c7b 100644
--- a/main.nf
+++ b/main.nf
@@ -539,7 +539,7 @@ process Bundle_Endpoints_Metrics {
         b_metrics+=" afd_metric.nii.gz"
     fi
 
-    scil_project_streamlines_to_map.py \$bundle \${bname} --in_metrics \${b_metrics} 
+    scil_project_streamlines_to_map.py \$bundle \${bname} --in_metrics \${b_metrics} --from_wm
     cd \${bname}
     for i in *.nii.gz;
         do mv "\$i" "${sid}__\$i";