notebooks/Template_running_pysmFISH_pipeline.ipynb

{
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "nteract": {
          "transient": {
            "deleting": false
          }
        }
      },
      "source": [
        "# pysmFISH pipeline running template\n",
        "\n",
        "This jupyter lab notebook is used to run automated data analysis via papermill. The data will be run through the entire pipeline (full run). A copy of the run notebook will be stored in the processed experiment folder inside the notebooks subfolder. "
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 1,
      "metadata": {
        "collapsed": true,
        "execution": {
          "iopub.execute_input": "2021-08-23T11:49:47.124Z",
          "iopub.status.busy": "2021-08-23T11:49:47.117Z",
          "iopub.status.idle": "2021-08-23T11:49:57.299Z",
          "shell.execute_reply": "2021-08-23T11:49:57.372Z"
        },
        "jupyter": {
          "outputs_hidden": true,
          "source_hidden": false
        },
        "nteract": {
          "transient": {
            "deleting": false
          }
        }
      },
      "outputs": [],
      "source": [
        "from pathlib import Path\n",
        "import time\n",
        "\n",
        "from pysmFISH.pipeline import Pipeline"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "collapsed": true,
        "jupyter": {
          "outputs_hidden": true,
          "source_hidden": false
        },
        "nteract": {
          "transient": {
            "deleting": false
          }
        },
        "tags": [
          "parameters"
        ]
      },
      "outputs": [],
      "source": [
        "# THIS CELL IS TAGGED PARAMETERS\n",
        "\n",
        "# REQUIRED ARGUMENTS\n",
        "# -------------------\n",
        "\n",
        "# Path to the experiment folder\n",
        "experiment_fpath = '' \n",
        "\n",
        "# Define if it is a 'new' or 're-run' (default: new)\n",
        "run_type = 'new'\n",
        "\n",
        "# Define the parsing type. Can be: \n",
        "# original/no_parsing/reparsing_from_processing_folder/reparsing_from_storage\n",
        "# (default: original)\n",
        "parsing_type = 'original'\n",
        "\n",
        "# OPTIONAL KWARGS\n",
        "# ----------------\n",
        "\n",
        "# Path to the cold storage hard drive (default: /fish/rawdata)\n",
        "raw_data_folder_storage_path = '/fish/rawdata'\n",
        "\n",
        "# Tag to identify the zarr file with parsed images (default: img_data)\n",
        "parsed_image_tag = 'img_data'\n",
        "\n",
        "# Tag to identify the zarr file with preprocessed images (default: preprocessed_img_data)\n",
        "preprocessed_image_tag = 'preprocessed_img_data'\n",
        "\n",
        "# Path to the location where the dataset are stored (default: /fish/fish_datasets)\n",
        "dataset_folder_storage_path = '/fish/fish_datasets'\n",
        "\n",
        "# Path to the location where the dataset are stored (default: /fish/fish_results)\n",
        "results_folder_storage_path = '/fish/fish_results'\n",
        "\n",
        "# Determine if the processed images will be saved (default: True)\n",
        "save_intermediate_steps = True\n",
        "\n",
        "# Path to an existing dataset that will be used in the processing\n",
        "dataset_path = ''\n",
        "\n",
        "# Number of FOV to process in parallel (20 when running in unmanaged cluster)\n",
        "chunk_size = 20\n",
        "\n",
        "# Searching distance that define two dots as identical (default: 10)\n",
        "same_dot_radius_duplicate_dots = 5\n",
        "\n",
        "# Define the stitched counts on which the overlapping dotes will be removed \n",
        "# (default: microscope_stitched) \n",
        "stitching_selected = 'microscope_stitched'\n",
        "\n",
        "# Value to select the barcodes that are passing the \n",
        "# screening (< hamming_distance). (default: 3)\n",
        "hamming_distance = 3\n",
        "\n",
        "# Define the name of the system that will run the processing. Can be local/htcondor\n",
        "# (default htcondor). If engine == local the parameters that define the cluster\n",
        "# will be ignored\n",
        "processing_engine = 'unmanaged_cluster'\n",
        "\n",
        "# Determine if the cluster should scale depending from the processing load\n",
        "adaptive = True\n",
        "\n",
        "# Number of cores to use in htcondor (default 20)\n",
        "cores = 20\n",
        "\n",
        "# Total memory for all the cores in condor (default 200GB) or per core in local setup\n",
        "# or per process (nprocs) in the unmanaged cluster (6GB for 40 nprocs)\n",
        "memory = '6GB'\n",
        "\n",
        "# Size of the spillover disk for dask in htcondor (default 0.1GB)\n",
        "disk = '0.1GB'\n",
        "\n",
        "# Max number of jobs that the cluster can run\n",
        "maximum_jobs = 15\n",
        "\n",
        "# define the dask scheduler port. Used for the unmanaged cluster (default 23875)\n",
        "scheduler_port = 23877\n",
        "\n",
        "# define the dask dashboard port: Used for the unmanaged cluser (default 25399)\n",
        "dashboard_port = 25399\n",
        "\n",
        "# Address of the dask scheduler. Used for the unmanaged cluser. \n",
        "# 'localhost' if running of the main node (default 'localhost)\n",
        "scheduler_address = 'localhost'\n",
        "\n",
        "# Addresses of the workers (default [monod10,monod11,monod12,monod33])\n",
        "workers_addresses_list = ['monod09','monod10','monod11','monod31']\n",
        "\n",
        "# number of processes for each workers (unmanaged cluster) (default 40 for single node monod)\n",
        "nprocs = 40\n",
        "\n",
        "# number threads/process (default 1)\n",
        "nthreads = 1\n",
        "\n",
        "# Directory where to spill over on the node in htcondor (default /tmp)\n",
        "local_directory = '/tmp'\n",
        "\n",
        "# Directory where to store dask and htcondor logs\n",
        "logs_directory = ''\n",
        "\n",
        "# Save the intensity of the bits and the flipping direction\n",
        "save_bits_int = True\n",
        "\n",
        "# Start the analysis from preprocessed images\n",
        "start_from_preprocessed_imgs = False\n",
        "\n",
        "# Resume (check the *_decoded_fov_* files already present in the results folder)\n",
        "resume = False\n",
        "\n",
        "# Connect the pipeline to a previously created cluster (default False)\n",
        "# Can be: 'connect_to_client' ,'connect_to_scheduler'\n",
        "reuse_cluster = False\n",
        "\n",
        "# Already active cluster to reconnect to when you want to reuse a cluster (default None)\n",
        "active_cluster = None\n",
        "\n",
        "# Already active client to reconnect to when you want to reuse a cluster (default None)\n",
        "active_client = None\n",
        "\n",
        "# Running cluster to connect when you want reuse a cluster\n",
        "active_scheduler_address = None\n",
        "\n",
        "fresh_tissue_segmentation_engine = 'stardist'\n",
        "\n",
        "diameter_size=25\n",
        "\n",
        "min_overlapping_pixels_segmentation= 20\n",
        "\n",
        "max_expansion_radius = 25\n",
        "\n",
        "fov_alignement_mode = 'clip'\n",
        "\n",
        "remove_distinct_genes=True\n",
        "\n",
        "bead_alignment_centering_mode='scan'\n",
        "\n",
        "# Add a note if needed\n",
        "notes = 'no notes'"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "collapsed": true,
        "jupyter": {
          "outputs_hidden": true,
          "source_hidden": false
        },
        "nteract": {
          "transient": {
            "deleting": false
          }
        }
      },
      "outputs": [],
      "source": [
        "# Add a running time tag to the pipeline run name\n",
        "experiment_fpath = Path(experiment_fpath)\n",
        "date_tag = time.strftime(\"%y%m%d_%H_%M_%S\")\n",
        "pipeline_run_name = date_tag + '_' + experiment_fpath.stem"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "nteract": {
          "transient": {
            "deleting": false
          }
        },
        "tags": []
      },
      "outputs": [],
      "source": [
        "print(f\"{notes}\")"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "collapsed": true,
        "jupyter": {
          "outputs_hidden": true,
          "source_hidden": false
        },
        "nteract": {
          "transient": {
            "deleting": false
          }
        }
      },
      "outputs": [],
      "source": [
        "# Set up the pipeline run\n",
        "\n",
        "running_pipeline = Pipeline(\n",
        "                        pipeline_run_name= pipeline_run_name,\n",
        "                        experiment_fpath= experiment_fpath,\n",
        "                        run_type= run_type,\n",
        "                        parsing_type= parsing_type,\n",
        "                        processing_engine= processing_engine,\n",
        "                        cores= cores,\n",
        "                        memory= memory,\n",
        "                        disk= disk,\n",
        "                        local_directory= local_directory,\n",
        "                        chunk_size= chunk_size,\n",
        "                        raw_data_folder_storage_path= raw_data_folder_storage_path,\n",
        "                        parsed_image_tag= parsed_image_tag,\n",
        "                        preprocessed_image_tag= preprocessed_image_tag,\n",
        "                        dataset_folder_storage_path= dataset_folder_storage_path,\n",
        "                        results_folder_storage_path= results_folder_storage_path,\n",
        "                        save_intermediate_steps= save_intermediate_steps,\n",
        "                        dataset_path= dataset_path,\n",
        "                        same_dot_radius_duplicate_dots= same_dot_radius_duplicate_dots,\n",
        "                        stitching_selected= stitching_selected,\n",
        "                        hamming_distance= hamming_distance,\n",
        "                        logs_directory= logs_directory,\n",
        "                        save_bits_int= save_bits_int,\n",
        "                        start_from_preprocessed_imgs=start_from_preprocessed_imgs,\n",
        "                        scheduler_port=scheduler_port,\n",
        "                        dashboard_port=dashboard_port,\n",
        "                        scheduler_address=scheduler_address,\n",
        "                        workers_addresses_list=workers_addresses_list,\n",
        "                        nprocs=nprocs,\n",
        "                        nthreads=nthreads,\n",
        "                        reuse_cluster=reuse_cluster,\n",
        "                        active_cluster=active_cluster,\n",
        "                        active_client=active_client,\n",
        "                        active_scheduler_address=active_scheduler_address,\n",
        "                        adaptive=adaptive,\n",
        "                        maximum_jobs=maximum_jobs,\n",
        "                        resume=resume,\n",
        "                        fresh_tissue_segmentation_engine=fresh_tissue_segmentation_engine,\n",
        "                        diameter_size=diameter_size,\n",
        "                        min_overlapping_pixels_segmentation=min_overlapping_pixels_segmentation,\n",
        "                        max_expansion_radius=max_expansion_radius,\n",
        "                        fov_alignement_mode = fov_alignement_mode,\n",
        "                        remove_distinct_genes=remove_distinct_genes,\n",
        "                        bead_alignment_centering_mode=bead_alignment_centering_mode,\n",
        "                        )"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "collapsed": true,
        "jupyter": {
          "outputs_hidden": true,
          "source_hidden": false
        },
        "nteract": {
          "transient": {
            "deleting": false
          }
        }
      },
      "outputs": [],
      "source": [
        "# Full pipeline run\n",
        "\n",
        "running_pipeline.run_full()"
      ]
    }
  ],
  "metadata": {
    "kernel_info": {
      "name": "test_d"
    },
    "kernelspec": {
      "display_name": "test_d",
      "language": "python",
      "name": "test_d"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.8.5"
    },
    "nteract": {
      "version": "0.28.0"
    }
  },
  "nbformat": 4,
  "nbformat_minor": 4
}