Skip to content

Commit

Permalink
Merge pull request #39 from AllenCell/TFE_mip
Browse files Browse the repository at this point in the history 
Background MIPs for TFE
  • Loading branch information
@chantelleleveille
chantelleleveille authored Nov 26, 2024
2 parents 24cb30a + 2dc2f9a commit d031dfe
Show file tree
Hide file tree
Showing 2 changed files with 210 additions and 81 deletions.
235 changes: 154 additions & 81 deletions nuc_morph_analysis/lib/visualization/write_data_for_colorizer.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,12 +22,16 @@
from nuc_morph_analysis.lib.preprocessing.global_dataset_filtering import (
load_dataset_with_features,
)
from nuc_morph_analysis.lib.visualization.write_mips_for_colorizer import (
save_colony_backdrop_mips,
)

from nuc_morph_analysis.lib.visualization.plotting_tools import (
get_plot_labels_for_metric,
)
from nuc_morph_analysis.lib.visualization.glossary import (
GLOSSARY,
)
)

from colorizer_data.writer import ColorizerDatasetWriter
from colorizer_data.writer import (
Expand Down Expand Up @@ -68,13 +72,6 @@ class NucMorphFeatureSpec:
# volume float In FMS manifest Volume of a single nucleus in pixels in a given frame
# height float In FMS manifest Height (in the z-direction) of the a single nucleus in pixels in a given frame
# NUC_PC1 float Needs calculated and added Value for shape mode 1 for a single nucleus in a given frame
# NUC_PC2 float Needs calculated and added Value for shape mode 2 for a single nucleus in a given frame
# NUC_PC3 float Needs calculated and added Value for shape mode 3 for a single nucleus in a given frame
# NUC_PC4 float Needs calculated and added Value for shape mode 4 for a single nucleus in a given frame
# NUC_PC5 float Needs calculated and added Value for shape mode 5 for a single nucleus in a given frame
# NUC_PC6 float Needs calculated and added Value for shape mode 6 for a single nucleus in a given frame
# NUC_PC7 float Needs calculated and added Value for shape mode 7 for a single nucleus in a given frame
# NUC_PC8 float Needs calculated and added Value for shape mode 8 for a single nucleus in a given frame


OBJECT_ID_COLUMN = "label_img"
Expand Down Expand Up @@ -103,7 +100,9 @@ class NucMorphFeatureSpec:
NucMorphFeatureSpec("density"),
NucMorphFeatureSpec("normalized_colony_depth"),
NucMorphFeatureSpec(
"termination", FeatureType.CATEGORICAL, ["Division", "Leaves FOV", "Apoptosis"]
"termination",
FeatureType.CATEGORICAL,
["Division", "Leaves FOV", "Apoptosis"],
),
NucMorphFeatureSpec("distance_from_centroid"),
NucMorphFeatureSpec("neighbor_avg_dxdt_48_volume_whole_colony"),
Expand All @@ -130,7 +129,9 @@ class NucMorphFeatureSpec:
NucMorphFeatureSpec("colony_time_at_B"),
NucMorphFeatureSpec("normalized_colony_depth"),
NucMorphFeatureSpec(
"termination", FeatureType.CATEGORICAL, ["Division", "Leaves FOV", "Apoptosis"]
"termination",
FeatureType.CATEGORICAL,
["Division", "Leaves FOV", "Apoptosis"],
),
NucMorphFeatureSpec("volume_at_A"),
NucMorphFeatureSpec("time_at_A"),
Expand Down Expand Up @@ -165,7 +166,9 @@ class NucMorphFeatureSpec:
NucMorphFeatureSpec("colony_time_at_B"),
NucMorphFeatureSpec("normalized_colony_depth"),
NucMorphFeatureSpec(
"termination", FeatureType.CATEGORICAL, ["Division", "Leaves FOV", "Apoptosis"]
"termination",
FeatureType.CATEGORICAL,
["Division", "Leaves FOV", "Apoptosis"],
),
NucMorphFeatureSpec("volume_at_A"),
NucMorphFeatureSpec("time_at_A"),
Expand Down Expand Up @@ -200,13 +203,19 @@ class NucMorphFeatureSpec:
NucMorphFeatureSpec("colony_time_at_B"),
NucMorphFeatureSpec("normalized_colony_depth"),
NucMorphFeatureSpec(
"termination", FeatureType.CATEGORICAL, ["Division", "Leaves FOV", "Apoptosis"]
"termination",
FeatureType.CATEGORICAL,
["Division", "Leaves FOV", "Apoptosis"],
),
NucMorphFeatureSpec(
"is_growth_outlier", FeatureType.CATEGORICAL, ["False", "True"]
),
NucMorphFeatureSpec("is_growth_outlier", FeatureType.CATEGORICAL, ["False", "True"]),
NucMorphFeatureSpec(
"baseline_colonies_dataset", FeatureType.CATEGORICAL, ["False", "True"]
),
NucMorphFeatureSpec("full_interphase_dataset", FeatureType.CATEGORICAL, ["False", "True"]),
NucMorphFeatureSpec(
"full_interphase_dataset", FeatureType.CATEGORICAL, ["False", "True"]
),
NucMorphFeatureSpec(
"lineage_annotated_dataset", FeatureType.CATEGORICAL, ["False", "True"]
),
Expand All @@ -229,74 +238,109 @@ class NucMorphFeatureSpec:
NucMorphFeatureSpec("SA_fold_change_fromB"),
NucMorphFeatureSpec("delta_SA_BC"),
NucMorphFeatureSpec("SA_vol_ratio"),

# mitotic and apoptotic neighbor columns
NucMorphFeatureSpec(column_name="frame_of_breakdown", type=FeatureType.CATEGORICAL, categories=["False", "True"]),
NucMorphFeatureSpec(column_name="frame_of_formation", type=FeatureType.CATEGORICAL, categories=["False", "True"]),
NucMorphFeatureSpec(column_name="number_of_frame_of_breakdown_neighbors"),
NucMorphFeatureSpec(column_name="number_of_frame_of_formation_neighbors"),
NucMorphFeatureSpec(column_name="has_mitotic_neighbor_breakdown", type=FeatureType.CATEGORICAL, categories=["False", "True"]),
NucMorphFeatureSpec(column_name="has_mitotic_neighbor_formation", type=FeatureType.CATEGORICAL, categories=["False", "True"]),
NucMorphFeatureSpec(column_name="has_mitotic_neighbor_breakdown_forward_dilated", type=FeatureType.CATEGORICAL, categories=["False", "True"]),
NucMorphFeatureSpec(column_name="has_mitotic_neighbor_formation_backward_dilated", type=FeatureType.CATEGORICAL, categories=["False", "True"]),
NucMorphFeatureSpec(column_name="has_mitotic_neighbor", type=FeatureType.CATEGORICAL, categories=["False", "True"]),
NucMorphFeatureSpec(column_name="has_mitotic_neighbor_dilated", type=FeatureType.CATEGORICAL, categories=["False", "True"]),
NucMorphFeatureSpec(column_name="frame_of_death", type=FeatureType.CATEGORICAL, categories=["False", "True"]),
NucMorphFeatureSpec(column_name="has_dying_neighbor", type=FeatureType.CATEGORICAL, categories=["False", "True"]),
NucMorphFeatureSpec(column_name="has_dying_neighbor_forward_dilated", type=FeatureType.CATEGORICAL, categories=["False", "True"]),
NucMorphFeatureSpec(column_name="number_of_frame_of_death_neighbors"),
NucMorphFeatureSpec(column_name="sum_has_mitotic_neighbor_breakdown"), # per track feature
NucMorphFeatureSpec(column_name="sum_has_mitotic_neighbor_formation"),# per track feature
NucMorphFeatureSpec(column_name="sum_has_mitotic_neighbor"),# per track feature
NucMorphFeatureSpec(column_name="sum_has_dying_neighbor"),# per track feature
NucMorphFeatureSpec(column_name="sum_number_of_frame_of_breakdown_neighbors"),# per track feature
NucMorphFeatureSpec(column_name="number_of_frame_of_death_neighbors"),# per track feature


NucMorphFeatureSpec(
"frame_of_breakdown",
type=FeatureType.CATEGORICAL,
categories=["False", "True"],
),
NucMorphFeatureSpec(
"frame_of_formation",
type=FeatureType.CATEGORICAL,
categories=["False", "True"],
),
NucMorphFeatureSpec("number_of_frame_of_breakdown_neighbors"),
NucMorphFeatureSpec("number_of_frame_of_formation_neighbors"),
NucMorphFeatureSpec(
"has_mitotic_neighbor_breakdown",
type=FeatureType.CATEGORICAL,
categories=["False", "True"],
),
NucMorphFeatureSpec(
"has_mitotic_neighbor_formation",
type=FeatureType.CATEGORICAL,
categories=["False", "True"],
),
NucMorphFeatureSpec(
"has_mitotic_neighbor_breakdown_forward_dilated",
type=FeatureType.CATEGORICAL,
categories=["False", "True"],
),
NucMorphFeatureSpec(
"has_mitotic_neighbor_formation_backward_dilated",
type=FeatureType.CATEGORICAL,
categories=["False", "True"],
),
NucMorphFeatureSpec(
"has_mitotic_neighbor",
type=FeatureType.CATEGORICAL,
categories=["False", "True"],
),
NucMorphFeatureSpec(
"has_mitotic_neighbor_dilated",
type=FeatureType.CATEGORICAL,
categories=["False", "True"],
),
NucMorphFeatureSpec(
"frame_of_death", type=FeatureType.CATEGORICAL, categories=["False", "True"]
),
NucMorphFeatureSpec(
"has_dying_neighbor",
type=FeatureType.CATEGORICAL,
categories=["False", "True"],
),
NucMorphFeatureSpec(
"has_dying_neighbor_forward_dilated",
type=FeatureType.CATEGORICAL,
categories=["False", "True"],
),
NucMorphFeatureSpec("number_of_frame_of_death_neighbors"),
NucMorphFeatureSpec("sum_has_mitotic_neighbor_breakdown"), # per track feature
NucMorphFeatureSpec("sum_has_mitotic_neighbor_formation"), # per track feature
NucMorphFeatureSpec("sum_has_mitotic_neighbor"), # per track feature
NucMorphFeatureSpec("sum_has_dying_neighbor"), # per track feature
NucMorphFeatureSpec(
"sum_number_of_frame_of_breakdown_neighbors"
), # per track feature
NucMorphFeatureSpec("number_of_frame_of_death_neighbors"), # per track feature
# new columns
# useful and likely to make it to final dataset
NucMorphFeatureSpec('2d_area_nuc_cell_ratio'),
NucMorphFeatureSpec('2d_area_nucleus'),
NucMorphFeatureSpec('2d_area_pseudo_cell'),
NucMorphFeatureSpec('2d_area_cyto'),
NucMorphFeatureSpec('2d_intensity_min_edge'),
NucMorphFeatureSpec('2d_intensity_max_edge'),
NucMorphFeatureSpec('2d_intensity_mean_edge'),

NucMorphFeatureSpec("2d_area_nuc_cell_ratio"),
NucMorphFeatureSpec("2d_area_nucleus"),
NucMorphFeatureSpec("2d_area_pseudo_cell"),
NucMorphFeatureSpec("2d_area_cyto"),
NucMorphFeatureSpec("2d_intensity_min_edge"),
NucMorphFeatureSpec("2d_intensity_max_edge"),
NucMorphFeatureSpec("2d_intensity_mean_edge"),
# potentially useful, but likely not needed in final dataset
NucMorphFeatureSpec('2d_convex_area_nucleus'),
NucMorphFeatureSpec('2d_eccentricity_nucleus'),
NucMorphFeatureSpec('2d_equivalent_diameter_nucleus'),
NucMorphFeatureSpec('2d_extent_nucleus'),
NucMorphFeatureSpec('2d_filled_area_nucleus'),
NucMorphFeatureSpec('2d_major_axis_length_nucleus'),
NucMorphFeatureSpec('2d_minor_axis_length_nucleus'),
NucMorphFeatureSpec('2d_orientation_nucleus'),
NucMorphFeatureSpec('2d_perimeter_nucleus'),
NucMorphFeatureSpec('2d_solidity_nucleus'),

NucMorphFeatureSpec("2d_convex_area_nucleus"),
NucMorphFeatureSpec("2d_eccentricity_nucleus"),
NucMorphFeatureSpec("2d_equivalent_diameter_nucleus"),
NucMorphFeatureSpec("2d_extent_nucleus"),
NucMorphFeatureSpec("2d_filled_area_nucleus"),
NucMorphFeatureSpec("2d_major_axis_length_nucleus"),
NucMorphFeatureSpec("2d_minor_axis_length_nucleus"),
NucMorphFeatureSpec("2d_orientation_nucleus"),
NucMorphFeatureSpec("2d_perimeter_nucleus"),
NucMorphFeatureSpec("2d_solidity_nucleus"),
# potentially useful, but likely not needed in final dataset
NucMorphFeatureSpec('2d_convex_area_pseudo_cell'),
NucMorphFeatureSpec('2d_eccentricity_pseudo_cell'),
NucMorphFeatureSpec('2d_equivalent_diameter_pseudo_cell'),
NucMorphFeatureSpec('2d_extent_pseudo_cell'),
NucMorphFeatureSpec('2d_filled_area_pseudo_cell'),
NucMorphFeatureSpec('2d_major_axis_length_pseudo_cell'),
NucMorphFeatureSpec('2d_minor_axis_length_pseudo_cell'),
NucMorphFeatureSpec('2d_orientation_pseudo_cell'),
NucMorphFeatureSpec('2d_perimeter_pseudo_cell'),
NucMorphFeatureSpec('2d_solidity_pseudo_cell'),

NucMorphFeatureSpec("2d_convex_area_pseudo_cell"),
NucMorphFeatureSpec("2d_eccentricity_pseudo_cell"),
NucMorphFeatureSpec("2d_equivalent_diameter_pseudo_cell"),
NucMorphFeatureSpec("2d_extent_pseudo_cell"),
NucMorphFeatureSpec("2d_filled_area_pseudo_cell"),
NucMorphFeatureSpec("2d_major_axis_length_pseudo_cell"),
NucMorphFeatureSpec("2d_minor_axis_length_pseudo_cell"),
NucMorphFeatureSpec("2d_orientation_pseudo_cell"),
NucMorphFeatureSpec("2d_perimeter_pseudo_cell"),
NucMorphFeatureSpec("2d_solidity_pseudo_cell"),
# extra
NucMorphFeatureSpec('inv_cyto_density'),
NucMorphFeatureSpec('2d_perimeter_nuc_cell_ratio'),
NucMorphFeatureSpec('2d_eccentricity_nuc_cell_ratio'),
NucMorphFeatureSpec('label_pseudo_cell'),


NucMorphFeatureSpec("inv_cyto_density"),
NucMorphFeatureSpec("2d_perimeter_nuc_cell_ratio"),
NucMorphFeatureSpec("2d_eccentricity_nuc_cell_ratio"),
NucMorphFeatureSpec("label_pseudo_cell"),
# extra old columns
NucMorphFeatureSpec('colony_depth', type=FeatureType.DISCRETE),

NucMorphFeatureSpec("colony_depth", type=FeatureType.DISCRETE),
],
}

Expand Down Expand Up @@ -336,7 +380,9 @@ def make_frame(
update_bounding_box_data(bounds_arr, seg_remapped)

time_elapsed = time.time() - start_time
logging.info("Frame {} finished in {:5.2f} seconds.".format(int(frame_number), time_elapsed))
logging.info(
"Frame {} finished in {:5.2f} seconds.".format(int(frame_number), time_elapsed)
)


def make_all_frames(
Expand Down Expand Up @@ -399,7 +445,9 @@ def make_features(
for feature in features:
if feature.column_name not in dataset.columns:
logging.warning(
"Feature '{}' not found in dataset. Skipping...".format(feature.column_name)
"Feature '{}' not found in dataset. Skipping...".format(
feature.column_name
)
)
continue

Expand All @@ -408,7 +456,7 @@ def make_features(
dataset=dataset_name,
colorizer=True,
)

# Remove parentheses from unit names, if included.
if len(unit) >= 2 and unit[0] == "(" and unit[-1] == ")":
unit = unit[1:-1]
Expand All @@ -418,12 +466,18 @@ def make_features(
# Get data and scale to use actual units
if scale_factor is not None:
data = data * scale_factor
description = GLOSSARY[feature.column_name]

description = GLOSSARY.get(feature.column_name, "")

writer.write_feature(
data,
FeatureInfo(label=label, unit=unit, type=feature.type, categories=feature.categories, description=description),
FeatureInfo(
label=label,
unit=unit,
type=feature.type,
categories=feature.categories,
description=description,
),
outliers=outliers,
)

Expand All @@ -447,6 +501,7 @@ def make_dataset(
do_frames=True,
scale=0.25,
parallel=False,
make_backdrops=True,
):
"""Make a new dataset from the given data, and write the complete dataset
files to the given output directory.
Expand Down Expand Up @@ -515,8 +570,19 @@ def make_dataset(
writer.set_frame_paths(generate_frame_paths(nframes))

make_features(full_dataset, FEATURE_COLUMNS[filter], dataset, writer)

if do_frames:
make_all_frames(grouped_frames, scale, writer, parallel)

if make_backdrops:
save_colony_backdrop_mips(
dataset, output_dir_subset + f"/{dataset}/backdrops/"
)
backdrop_paths = [f"./backdrops/{i}.png" for i in range(nframes)]
writer.add_backdrops(
"Max intensity z-projection of Lamin B1", backdrop_paths
)

writer.write_manifest(metadata=metadata)


Expand Down Expand Up @@ -558,6 +624,12 @@ def make_dataset(
"Has no effect with --noframes.",
)

parser.add_argument(
"--make_backdrops",
type=bool,
default=True,
help="True will generate backdrops (Default). False will use previously generated backdrops or display none if they have not been generated before.",
)
args = parser.parse_args()


Expand All @@ -572,6 +644,7 @@ def main():
do_frames=not args.noframes,
scale=args.scale,
parallel=args.parallel,
make_backdrops=args.make_backdrops,
)


Expand Down
Loading

0 comments on commit d031dfe

Please sign in to comment.