diff --git a/README.md b/README.md index 06bb8ba..5137b7e 100644 --- a/README.md +++ b/README.md @@ -51,7 +51,7 @@ The official documentation is available [here](https://royerlab.github.io/ultrac These additional developer documentation are available: -- Parameter [configuration schema](ultrack/config/README.md). +- Parameter [configuration schema](docs/source/configuration.rst). - Intermediate segmentation and tracking SQL database are [here](ultrack/core/README.md). ## Gurobi setup diff --git a/docs/source/conf.py b/docs/source/conf.py index a0fea6f..4dcc7ea 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -44,9 +44,11 @@ "sphinx_click.ext", "sphinx_copybutton", "sphinx.ext.autosummary", + "sphinx.ext.mathjax", "nbsphinx", "myst_parser", "sphinx_gallery.load_style", + "sphinxcontrib.autodoc_pydantic", ] source_suffix = { @@ -86,3 +88,13 @@ # Don't show class signature with the class' name. # autodoc_class_signature = "separated" + +# Pydantic auto-doc settings +autodoc_pydantic_model_show_json = False +autodoc_pydantic_model_show_config_summary = False +autodoc_pydantic_model_show_validator_summary = False +autodoc_pydantic_model_show_validator_members = False +autodoc_pydantic_model_show_field_summary = False +autodoc_pydantic_model_signature_prefix = "class" +autodoc_pydantic_field_list_validators = False +autodoc_pydantic_model_undoc_members = False diff --git a/docs/source/configuration.rst b/docs/source/configuration.rst index eb18d32..ed209ba 100644 --- a/docs/source/configuration.rst +++ b/docs/source/configuration.rst @@ -1,2 +1,31 @@ Configuration ------------- + +The configuration is at the heart of ultrack, it is used to define the parameters for each step of the pipeline and where to store the intermediate results. +The `MainConfig` is the main configuration that contains the other configurations of the individual steps plus the data configuration. + +The configurations are documented below, the parameters are ordered by importance, most important parameters are at the top of the list. Parameters which should not be changed in most of the cases are at the bottom of the list and contain a ``SPECIAL`` tag. + +.. autosummary:: + + ultrack.config.MainConfig + ultrack.config.DataConfig + ultrack.config.SegmentationConfig + ultrack.config.LinkingConfig + ultrack.config.TrackingConfig + + +.. autopydantic_model:: ultrack.config.MainConfig + :no-index: + +.. autopydantic_model:: ultrack.config.DataConfig + :no-index: + +.. autopydantic_model:: ultrack.config.SegmentationConfig + :no-index: + +.. autopydantic_model:: ultrack.config.LinkingConfig + :no-index: + +.. autopydantic_model:: ultrack.config.TrackingConfig + :no-index: diff --git a/docs/source/examples.rst b/docs/source/examples.rst index 6928af8..be31503 100644 --- a/docs/source/examples.rst +++ b/docs/source/examples.rst @@ -3,7 +3,7 @@ Examples .. include:: examples/README.rst :start-line: 2 - :end-line: 20 + :end-line: 25 Notebooks --------- diff --git a/docs/source/index.rst b/docs/source/index.rst index 1fc8152..1cebfe4 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -97,3 +97,4 @@ Documentation contents api cli + rest_api diff --git a/docs/source/rest_api.rst b/docs/source/rest_api.rst new file mode 100644 index 0000000..165fdde --- /dev/null +++ b/docs/source/rest_api.rst @@ -0,0 +1,7 @@ +REST API +-------- + +The ultrack REST API is a set of HTTP endpoints that allow you to track your data from a Ultrack server. +This is what enables the :doc:`Ultrack FIJI plugin `. + +@ilan TODO diff --git a/examples/README.rst b/examples/README.rst index 94ef354..3170303 100644 --- a/examples/README.rst +++ b/examples/README.rst @@ -1,6 +1,12 @@ Ultrack's Usage Examples ======================== +Here we provide some examples of how to use Ultrack for cell tracking. + +Some examples are provided as Jupyter notebooks with additional documentation, but we do not recommend using Jupyter notebooks for your day-to-day analysis. + +Other examples as Python scripts can be found in `here `_. + Additional packages might be required. Therefore, conda environment files are provided, which can be installed using: .. code-block:: bash @@ -25,4 +31,4 @@ To run all the examples and update the notebooks in headless mode, run: .. code-block:: bash - bash refresh_examples.sh \ No newline at end of file + bash refresh_examples.sh diff --git a/examples/flow_field_3d/tribolium_cartograph.ipynb b/examples/flow_field_3d/tribolium_cartograph.ipynb index a9c4ae9..3f17d16 100644 --- a/examples/flow_field_3d/tribolium_cartograph.ipynb +++ b/examples/flow_field_3d/tribolium_cartograph.ipynb @@ -365,7 +365,7 @@ "- `link`: Links and assign edge weights to the segmentation hypothesis, taking into consideration the previously added flow;\n", "- `solve`: Solves the tracking problem by selecting the strongly associated segmentation hypothesis.\n", "\n", - "These steps use our configuration object, `MainConfig`, which we'll set up below. Its documentation can be found [here](https://github.com/royerlab/ultrack/blob/main/ultrack/config/README.md).\n", + "These steps use our configuration object, `MainConfig`, which we'll set up below. Its documentation can be found [here](https://github.com/royerlab/ultrack/blob/main/docs/source/configuration.rst).\n", "\n", "The parameters were chosen manually by inspection.\n", "\n", diff --git a/examples/micro_sam/micro_sam_tracking.ipynb b/examples/micro_sam/micro_sam_tracking.ipynb index f34f268..3a74a26 100644 --- a/examples/micro_sam/micro_sam_tracking.ipynb +++ b/examples/micro_sam/micro_sam_tracking.ipynb @@ -190,7 +190,7 @@ "- `solve`: Solves the tracking problem by selecting the strongly connected segmentation hypothesis.\n", "\n", "Each of these steps requires its configuration, which we'll set up below.\n", - "The config documentation can be found [here](https://github.com/royerlab/ultrack/blob/main/ultrack/config/README.md)." + "The config documentation can be found [here](https://github.com/royerlab/ultrack/blob/main/docs/source/configuration.rst)." ] }, { diff --git a/examples/multi_color_ensemble/multi_color_ensemble.ipynb b/examples/multi_color_ensemble/multi_color_ensemble.ipynb index cd75d3e..9caacec 100644 --- a/examples/multi_color_ensemble/multi_color_ensemble.ipynb +++ b/examples/multi_color_ensemble/multi_color_ensemble.ipynb @@ -627,7 +627,7 @@ "source": [ "Finally, we can track the cells. Ultrack parameters are defined in the `MainConfig` class. Each step contains its own parameters in a subconfiguration (e.g., `config.linking_config`). These parameters were found by inspecting the tracking results.\n", "\n", - "The configuration documentation can be found [here](https://github.com/royerlab/ultrack/blob/main/ultrack/config/README.md)." + "The configuration documentation can be found [here](https://github.com/royerlab/ultrack/blob/main/docs/source/configuration.rst)." ] }, { diff --git a/examples/neuromast_plantseg/neuromast_plantseg.ipynb b/examples/neuromast_plantseg/neuromast_plantseg.ipynb index de5708c..7047535 100644 --- a/examples/neuromast_plantseg/neuromast_plantseg.ipynb +++ b/examples/neuromast_plantseg/neuromast_plantseg.ipynb @@ -315,7 +315,7 @@ "\n", "In this example, the parameters were optimized through inspection.\n", "\n", - "The configuration documentation can be found [here](https://github.com/royerlab/ultrack/blob/main/ultrack/config/README.md)." + "The configuration documentation can be found [here](https://github.com/royerlab/ultrack/blob/main/docs/source/configuration.rst)." ] }, { diff --git a/examples/stardist_2d/2d_tracking.ipynb b/examples/stardist_2d/2d_tracking.ipynb index 56e6e76..8fea896 100644 --- a/examples/stardist_2d/2d_tracking.ipynb +++ b/examples/stardist_2d/2d_tracking.ipynb @@ -269,7 +269,7 @@ "- `link`: Links and assign edge weights to the segmentation hypothesis;\n", "- `solve`: Solves the tracking problem by selecting the strongly connected segmentation hypothesis.\n", "\n", - "Each of these steps requires its own configuration, which we'll set up below. Its documentation can be found [here](https://github.com/royerlab/ultrack/blob/main/ultrack/config/README.md).\n", + "Each of these steps requires its own configuration, which we'll set up below. Its documentation can be found [here](https://github.com/royerlab/ultrack/blob/main/docs/source/configuration.rst).\n", "\n", "We create our configuration instance and print its default values." ] diff --git a/setup.cfg b/setup.cfg index ef4e1d5..3c5b733 100644 --- a/setup.cfg +++ b/setup.cfg @@ -61,6 +61,7 @@ docs = nbsphinx >= 0.9.3 myst-parser >= 2.0.0 sphinx-copybutton + autodoc_pydantic < 2.0.0 furo [options.entry_points] diff --git a/ultrack/config/README.md b/ultrack/config/README.md index f074d9b..e69de29 100644 --- a/ultrack/config/README.md +++ b/ultrack/config/README.md @@ -1,43 +0,0 @@ -# Configuration Schema Description - -Configurations have default values, therefore you don't need to set them all from the beginning. - - - main_config: - - - data_config: - - working_dir: Working directory for auxiliary files; - - database: Database type `sqlite` and `postgresql` supported; - - address: Postgresql database path, for example, `postgres@localhost:12345/example`; - - n_workers: Number of worker threads; - - - segmentation_config: - - anisotropy_penalization: Image graph z-axis penalization, positive values will prioritize segmenting the xy-plane first, negative will do the opposite; - - n_workers: Number of worker threads; - - min_area: Minimum segments area, regions smaller than this value are merged or removed when there is no neighboring region; - - max_area: Maximum segments area, regions larger than this value are removed; - - min_frontier: Minimum average contour value, neighboring regions with values below this are merged; - - max_noise: Upper limit of uniform distribution for additive noise on contour map; - - threshold: Threshold used to binary the cell foreground map; - - ws_hierarchy: Watershed hierarchy function from [higra](https://higra.readthedocs.io/en/stable/python/watershed_hierarchy.html) used to construct the hierarchy; - - - linking_config: - - distance_weight: Penalization weight $\gamma$ for distance between segment centroids, $w_{pq} - \gamma \|c_p - c_q\|_2$, where $c_p$ is region $p$ center of mass; - - n_workers: Number of worker threads; - - max_neighbors: Maximum number of neighbors per candidate segment; - - max_distance: Maximum distance between neighboring segments; - - - tracking_config: - - appear_weight: Penalization weight for appearing cell, should be negative; - - disappear_weight: Penalization for disappearing cell, should be negative; - - division_weight: Penalization for dividing cell, should be negative; - - dismiss_weight_guess: Threshold (<=) used to provide 0 valued hint to solver; - - include_weight_guess: Threshold (>=) used to provide 1 valued hint to solver; - - window_size: Time window size for partially solving the tracking ILP; - - overlap_size: Number of frames used to pad each size when partially solving the tracking ILP; - - solution_gap: solver solution gap; - - time_limit: solver execution time limit in seconds; - - method: solver method, (reference)[https://docs.python-mip.com/en/latest/classes.html#lp-method]; - - n_threads: Number of worker threads; - - link_function: Function used to transform the edge weights, `identity` or `power`; - - power: Expoent $\eta$ of power transform, $w_{pq}^\eta$; - - bias: Edge weights bias $b$, $w_{pq} + b$, should be negative; diff --git a/ultrack/config/config.py b/ultrack/config/config.py index a8bed2c..2249b2c 100644 --- a/ultrack/config/config.py +++ b/ultrack/config/config.py @@ -13,11 +13,30 @@ class LinkingConfig(BaseModel): + """ + Candidate cell hypotheses linking configuration + """ + + max_distance: float = 15.0 + """Maximum distance between neighboring segments """ + n_workers: int = 1 + """Number of worker threads """ + max_neighbors: int = 5 - max_distance: float = 15.0 + """Maximum number of neighbors per candidate segment """ + distance_weight: float = 0.0 + r""" + Penalization weight :math:`\gamma` for distance between segment centroids, + :math:`w_{pq} - \gamma \|c_p - c_q\|_2`, where :math:`c_p` is region :math:`p` center of mass + """ + z_score_threshold: float = 5.0 + """ + ``SPECIAL``: z-score threshold between intensity values from within + the segmentation masks of neighboring segments + """ class Config: extra = Extra.forbid @@ -27,15 +46,24 @@ class MainConfig(BaseModel): data_config: Optional[DataConfig] = Field( default_factory=DataConfig, alias="data", nullable=True ) + """ + Configuration for intermediate data storage and retrieval. + """ + segmentation_config: SegmentationConfig = Field( default_factory=SegmentationConfig, alias="segmentation" ) + """Segmentation hypotheses creation configuration """ + linking_config: LinkingConfig = Field( default_factory=LinkingConfig, alias="linking" ) + """Candidate cell hypotheses linking configuration""" + tracking_config: TrackingConfig = Field( default_factory=TrackingConfig, alias="tracking" ) + """Tracking (segmentation & linking selection) configuration """ def load_config(path: Union[str, Path]) -> MainConfig: diff --git a/ultrack/config/dataconfig.py b/ultrack/config/dataconfig.py index 3817857..281f008 100644 --- a/ultrack/config/dataconfig.py +++ b/ultrack/config/dataconfig.py @@ -16,10 +16,21 @@ class DatabaseChoices(Enum): class DataConfig(BaseModel): + """ + Configuration for intermediate data storage and retrieval. + """ + + n_workers: int = 1 + """Number of workers for parallel processing""" + working_dir: Path = Path(".") + """Working directory for auxiliary files (e.g. sqlite database, metadata)""" + database: DatabaseChoices = "sqlite" + """``SPECIAL``: Database type ``sqlite`` and ``postgresql`` supported""" + address: Optional[str] = None - n_workers: int = 1 + """``SPECIAL``: Postgresql database path, for example, ``postgres@localhost:12345/example``""" class Config: validate_assignment = True @@ -86,6 +97,7 @@ def metadata_add(self, data: Dict[str, Any]) -> None: @property def metadata(self) -> Dict[str, Any]: + """Returns metadata as dictionary.""" if not self.metadata_path.exists(): return {} diff --git a/ultrack/config/segmentationconfig.py b/ultrack/config/segmentationconfig.py index 8970d96..84b4eea 100644 --- a/ultrack/config/segmentationconfig.py +++ b/ultrack/config/segmentationconfig.py @@ -11,14 +11,44 @@ class SegmentationConfig(BaseModel): - threshold: float = 0.5 + """Segmentation hypotheses creation configuration""" + min_area: int = 100 + """ + Minimum segment number of pixels, regions smaller than this value are merged + or removed when there is no neighboring region + """ + max_area: int = 1_000_000 + """Maximum segment's number of pixels, regions larger than this value are removed """ + + n_workers: int = 1 + """Number of worker threads """ + min_frontier: float = 0.0 - anisotropy_penalization: float = 0.0 + """ + Minimum average frontier value between candidate segmentations, regions sharing an average + frontier value lower than this are merged + """ + + threshold: float = 0.5 + """Threshold used to binarize the cell foreground map""" + max_noise: float = 0.0 + """``SPECIAL``: Upper limit of uniform distribution for additive noise on contour map """ + ws_hierarchy: Callable = hg.watershed_hierarchy_by_area - n_workers: int = 1 + """ + ``SPECIAL``: Watershed hierarchy function from + `higra `_ + used to construct the hierarchy + """ + + anisotropy_penalization: float = 0.0 + """ + ``SPECIAL``: Image graph z-axis penalization, positive values will prioritize segmenting + the xy-plane first, negative will do the opposite + """ class Config: use_enum_values = True diff --git a/ultrack/config/trackingconfig.py b/ultrack/config/trackingconfig.py index d89a928..16de7fb 100644 --- a/ultrack/config/trackingconfig.py +++ b/ultrack/config/trackingconfig.py @@ -11,20 +11,54 @@ class LinkFunctionChoices(Enum): class TrackingConfig(BaseModel): + """Tracking (segmentation & linking selection) configuration""" + appear_weight: float = -0.001 + """Penalization weight for appearing cell, should be negative """ disappear_weight: float = -0.001 + """Penalization for disappearing cell, should be negative """ division_weight: float = -0.001 - dismiss_weight_guess: Optional[float] = None - include_weight_guess: Optional[float] = None + """Penalization for dividing cell, should be negative """ + + n_threads: int = -1 + """Number of worker threads """ + window_size: Optional[int] = None + """ + Time window size for partially solving the tracking ILP. + By default it solves the entire timelapse at once. + Useful for large datasets. + """ + overlap_size: int = 1 + """ + Number of frames used to shared (overlap/pad) each size when ``window_size`` is set. + This improves the tracking quality at the edges of the windows and enforce continuity of the tracks. + """ + solution_gap: float = 0.001 + """ + Solver solution gap. This will speed up the solver when finding the optimal + solution might taken a long time, but may affect the quality of the solution. + """ + time_limit: int = 36000 + """Solver execution time limit in seconds """ + method: int = 0 - n_threads: int = -1 + """``SPECIAL``: Solver method, `reference `_""" + link_function: LinkFunctionChoices = "power" + """``SPECIAL``: Function used to transform the edge weights, `identity` or `power`""" + power: float = 4 + r"""``SPECIAL``: Expoent :math:`\eta` of power transform, :math:`w_{pq}^\eta` """ + bias: float = -0.0 + """``SPECIAL``: Edge weights bias :math:`b`, :math:`w_{pq} + b`, should be negative """ + + dismiss_weight_guess: Optional[float] = None + include_weight_guess: Optional[float] = None class Config: use_enum_values = True