MIPTools.def

Bootstrap: docker
From: amd64/ubuntu:20.04

##################################################################
##                        Labels Section                        ##
##################################################################
%labels
    Author Bailey Lab
    Version v0.4.0.9000

##################################################################
##                         Post Section                         ##
##################################################################
%post
    # set number of cpus to use in build
    CPU_COUNT=20

    # set build environment
    export DEBIAN_FRONTEND=noninteractive \
        CONDA_DIR=/opt/conda \
        SHELL=/bin/bash \
        LANG=en_US.UTF-8 \
        LANGUAGE=en_US.UTF-8 \
        LC_ALL=en_US.UTF-8 \
        MINICONDA_VERSION=4.8.3
    export PATH=$CONDA_DIR/bin:$PATH

    # install system packages
    apt-get update \
    && apt-get -yq dist-upgrade \
    && apt-get install -yq --no-install-recommends \
    wget \
    bzip2 \
    ca-certificates \
    sudo \
    locales \
    fonts-liberation \
    fonts-dejavu \
    git \
    build-essential \
    gcc \
    openssh-client \
    nano \
    libtbb-dev \
    libz-dev \
    libxrender1 \
    cmake \
    automake \
    autoconf \
    rsync \
    pigz \
    perl-tk \
    less \
    software-properties-common \
    libxext6 \
    libxrender1 \
    ghostscript \
    openjdk-11-jdk \
    liblzma-dev \
    libbz2-dev \
    libssl-dev \
    libcurl4-gnutls-dev \
    alien \
    unzip \
    tree \
    pandoc

    # set environment locale
    echo "en_US.UTF-8 UTF-8" >> /etc/locale.gen
    echo "LANG=en_US.UTF-8" > /etc/locale.conf
    echo "LC_ALL=en_US.UTF-8" >> /etc/environment
    echo "LANGUAGE=en_US.UTF-8" >> /etc/environment
    locale-gen en_US.UTF-8
    update-locale LANG=en_US.UTF-8

    # install bcl2fastq, if the file is there
    cd /opt/programs
    unzip bcl2fastq2*.zip || true
    alien bcl2fastq2*.rpm || true
    dpkg -i bcl2fastq2*.deb || true

    # install msa2vcf
    cd /opt/programs
    git clone https://github.com/lindenb/jvarkit.git
    cd jvarkit
    ./gradlew msa2vcf

    # install conda
    cd /tmp && \
        wget --quiet https://repo.continuum.io/miniconda/Miniconda3-py38_${MINICONDA_VERSION}-Linux-x86_64.sh && \
        echo "d63adf39f2c220950a063e0529d4ff74 *Miniconda3-py38_${MINICONDA_VERSION}-Linux-x86_64.sh" | md5sum -c - && \
        /bin/bash Miniconda3-py38_${MINICONDA_VERSION}-Linux-x86_64.sh -f -b -p $CONDA_DIR && \
        rm Miniconda3-py38_${MINICONDA_VERSION}-Linux-x86_64.sh && \
        $CONDA_DIR/bin/conda config --add channels defaults && \
        $CONDA_DIR/bin/conda config --add channels bioconda && \
        $CONDA_DIR/bin/conda config --add channels conda-forge && \
        $CONDA_DIR/bin/conda config --add channels r && \
        $CONDA_DIR/bin/conda config --system --set show_channel_urls true && \
        $CONDA_DIR/bin/conda install --quiet --yes conda="${MINICONDA_VERSION%.*}.*" && \
        conda clean --all -f -y

    # install mamba
    conda install mamba -c conda-forge

    # install conda packages using mamba
    mamba install -qy\
      "r" \
      "r-epitools" \
      "rpy2" \
      "r-irkernel" \
      "r-plotly" \
      "r-knitr" \
      "r-shiny" \
      "r-ggplot2" \
      "r-devtools" \
      "r-dplyr" \
      "r-dt" \
      "r-pkgbuild" \
      "gxx_linux-64" \
      "python" \
      "notebook" \
      "nbconvert" \
      "jupyter_contrib_nbextensions" \
      "xlrd" \
      "bcftools" \
      "samtools" \
      "vcftools" \
      "htslib" \
      "bwa" \
      "bowtie2" \
      "primer3" \
      "primer3-py" \
      "numpy" \
      "scipy" \
      "biopython" \
      "pysam" \
      "pandas" \
      "matplotlib" \
      "seaborn" \
      "scikit-learn" \
      "scandir" \
      "openpyxl" \
      "simplegeneric" \
      "matplotlib-venn" \
      "tblib" \
      "parallel" \
      "scikit-allel" \
      "bioconductor-dnacopy" \
      "basemap-data-hires" \
      "seqtk=1.3" \
      "gatk4" \
      "freebayes" \
      "lastz" \
      "plotly" \
      "texlive-core" \
      "libgfortran4"

    # install vt variant tool set
    cd /opt/programs
    git clone https://github.com/atks/vt.git
    cd vt
    git checkout 0.577
    make -j $CPU_COUNT
    scp vt /opt/bin

    # install magrittr
    Rscript -e 'install.packages("magrittr", repos = "https://cloud.r-project.org")'

    # install RealMcCoil
    Rscript -e 'devtools::install_github("OJWatson/McCOILR")'

    # install rehh
    Rscript -e 'install.packages("rehh", repos="https://cloud.r-project.org")'

    # install MIPWrangler
    cd /opt/programs
    git clone https://github.com/bailey-lab/MIPWrangler
    cd MIPWrangler
    git checkout v1.2.0
    ./install.sh $CPU_COUNT

    # install elucidator
    cd /opt/programs
    git clone https://github.com/nickjhathaway/elucidator
    cd elucidator
    git checkout develop
    ./install.sh $CPU_COUNT

    # install parasight
    scp /opt/programs/parasight_v7.6/parasight.pl /opt/bin/parasight76.pl

    # install basespace cli
    BS_PATH="https://launch.basespace.illumina.com/CLI/latest/amd64-linux/bs"
    wget $BS_PATH -O /opt/bin/bs

    # add executable flag to executables
    chmod -R +xr /usr/bin
    chmod -R +xr /opt/bin

    # create work and resources directories in /opt
    mkdir /opt/work \
        /opt/project_resources \
        /opt/species_resources \
        /opt/data \
        /opt/analysis \
        /opt/host_species \
        /opt/extras

#################################################################
##                        Files Section                        ##
#################################################################
%files
    programs /opt
    bin /opt
    src /opt
    base_resources/ /opt/resources

#################################################################
##                     Environment Section                     ##
#################################################################
%environment
    path=/opt/bin:/opt/conda/bin:/opt/programs/MIPWrangler/bin:
    path=$path/opt/programs/elucidator/bin:/opt/programs/gatk:
    path=$path$PATH
    export PATH=$path
    export XDG_RUNTIME_DIR=""
    export DEBIAN_FRONTEND=noninteractive
    export LANG=en_US.UTF-8
    export LANGUAGE="en_US.UTF-8"
    export LC_ALL="en_US.UTF-8"

#################################################################
##                         Jupyter App                         ##
#################################################################
%apprun jupyter
    # Exit if something fails or if have unset object
    set -eu

    # Port forwarding setup
    nb_port=$(shuf -i 8000-9999 -n 1)
    server_ip=$(hostname -i)
    server_user=$(whoami)@$(hostname -f)
    nb_dir=/opt

    help() {
        echo "Open an interactive Jupyter Notebook. The notebook can be used"
        echo "for post-wrangler mapping and variant calling."
        echo ""
        echo "Usage:"
        echo "  singularity run [options] --app jupyter <container>"\
        "[app_options]"
        echo ""
        echo "Options:"
        echo "  See 'singularity run'."
        echo ""
        echo "App Options:"
        echo "  -d    The port to be used to load the Jupyter Notebook."
        echo "  -h    Print the help page."
        echo "  -p    The notebook directory."
        echo ""
        echo "Examples:"
        echo "  # Set paths"
        echo "  $ resource_dir=/bin/MIPTools/base_resources"
        echo "  $ project_resources=/work/usr/DR1_project_resources"
        echo "  $ species_resources=/work/usr/pf_species_resources"
        echo "  $ wrangler_dir=/work/usr/wrangler"
        echo "  $ variant_dir=/work/usr/variant"
        echo ""
        echo "  # Run app"
        echo "  $ singularity run \\"
        echo "    -B \${resource_dir}:/opt/resources \\"
        echo "    -B \${project_resources}:/opt/project_resources \\"
        echo "    -B \${species_resources}:/opt/species_resources \\"
        echo "    -B \${wrangler_dir}:/opt/data \\"
        echo "    -B \${variant_dir}:/opt/analysis \\"
        echo "    --app jupyter <container>"
    }

    # Parse options
    while getopts "d:hp:" opt; do
        case ${opt} in
            d) nb_dir=${OPTARG} ;;
            h) help
               exit 1 ;;
            p) nb_port=${OPTARG} ;;
            *) help
               exit 1 ;;
        esac
    done

    # Save templates
    rsync /opt/resources/*.ipynb /opt/analysis --ignore-existing \
        --ignore-missing-args

    # Inform the user how to access the notebook
    echo "Use the following command if you are running this notebook from a"
    echo "remote server. Ignore if using a local computer."
    echo "ssh -f -N -L localhost:${nb_port}:${server_ip}:${nb_port}"\
    "${server_user}"

    # Setup juptyr notebook settings
    jupyter nbextension enable plotlywidget/extension
    jupyter nbextension enable toc2/main
    jupyter nbextension enable codefolding/main
    jupyter nbextension enable highlighter/highlighter
    jupyter nbextension enable keyboard_shortcut_editor/main
    jupyter nbextension enable spellchecker/main

    # Run notebook
    jupyter notebook --notebook-dir=${nb_dir} --ip=${server_ip} \
            --port=${nb_port} --no-browser

##################################################################
##                         Wrangler App                         ##
##################################################################
%apprun wrangler
    # Exit if something fails or if have unset object
    set -eu

    help() {
        echo "Run MIPWrangler on demultiplexed data."
        echo ""
        echo "Usage:"
        echo "  singularity run [options] --app wrangler <container>"\
        "[app_options]"
        echo ""
        echo "Options:"
        echo "  See 'singularity run'."
        echo ""
        echo "App Options:"
        echo "  -c    Number of available processors to use. Default: 1."
        echo "  -e    Required. A unique ID given to each sequencing run by"
        echo "        the user."
        echo "  -h    Print the help page."
        echo "  -k    Keep intermediate files generated by MIPWrangler."
        echo "  -l    Required. File providing a list of samples with "
        echo "        associated information."
        echo "  -m    Minimum capture length for stitching excluding probe"
        echo "        arms."
        echo "  -n    Starting number for MIP server. Default: 1."
        echo "  -p    Required. Probe sets to be processed."
        echo "  -s    Required. Sample sets to be processed."
        echo "  -w    Absolute path to MIPWrangler run script. "
        echo "        Default: '/opt/bin/runMIPWranglerCurrent.sh'."
        echo "  -x    Required. Additional arguments to pass to MIPWrangler"
        echo "        mipSetupAndExtractByArm. This command extracts sequences"
        echo "        and stitches paired end reads to single sequences."
        echo ""
        echo "Examples:"
        echo "  # Define variables"
        echo "  $ probe_sets='DR1,VAR4'"
        echo "  $ sample_sets='JJJ'"
        echo "  $ stitch_options='--stitchGapExtend=1,--overWriteDirs'"
        echo ""
        echo "  $ singularity run "
        echo "    -B project_resources:/opt/project_resources \\"
        echo "    -B fastq_dir:/opt/data \\"
        echo "    -B wrangler_dir:/opt/analysis \\"
        echo "    --app wrangler <container> \\"
        echo "    -e <experiment_id> -l <sample_list.file> -p \${probe_sets} \\"
        echo "    -s \${sample_sets} -x \${stitch_options}"
        echo ""
        echo "  # Run app"
        echo "  $ singularity run "
        echo "    -B project_resources:/opt/project_resources \\"
        echo "    -B fastq_dir:/opt/data \\"
        echo "    -B wrangler_dir:/opt/analysis \\"
        echo "    --app wrangler <container> \\"
        echo "    -c <cpu_count> -e <experiment_id> -l <sample_list.file> \\"
        echo "    -m <min_capture_length> -p \${probe_sets} -s \\"
        echo "    \${sample_sets} -x \${stitch_options} -k"
    }

    # Set defaults
    cluster_script="runMIPWranglerCurrent.sh"
    server_number=1
    cpu_count=1
    min_capture_length="none"
    stitch_options="none"
    keep_files=""

    # Parse options
    while getopts "c:e:hkl:m:n:p:s:w:x:" opt; do
        case ${opt} in
            c) cpu_count=${OPTARG} ;;
            e) experiment_id=${OPTARG} ;;
            h) help
               exit 1 ;;
            k) keep_files=-k ;;
            l) sample_list=${OPTARG} ;;
            m) min_capture_length=${OPTARG} ;;
            n) server_number=${OPTARG} ;;
            p) probe_sets=${OPTARG} ;;
            s) sample_sets=${OPTARG} ;;
            w) cluster_script=${OPTARG} ;;
            x) stitch_options=${OPTARG} ;;
            *) help
               exit 1 ;;
        esac
    done

    # Remove whitespace from arguments
    probe_sets=${probe_sets//[[:space:]]/}
    sample_sets=${sample_sets//[[:space:]]/}
    stitch_options=${stitch_options//[[:space:]]/}
    
    # Ensure that stitch options begin with a comma if not the default value
    # This is done as the arguments are fed in with leading dashes and the
    # python script will crash if dashes are fed in. By adding a leading comma,
    # the script will run.
    if [ ${stitch_options:0:1} != "," ] && [ ${stitch_options} != "none" ]; then
        stitch_options=",${stitch_options}"
    fi

    # Create wrangler bash scripts using python
    python /opt/src/generate_wrangler_scripts.py \
    -c ${cpu_count} -e ${experiment_id} ${keep_files} \
    -l /opt/analysis/${sample_list} -m ${min_capture_length} \
    -n ${server_number} -p ${probe_sets} -s ${sample_sets} \
    -w ${cluster_script} -x ${stitch_options}

    # Run wrangler scripts.
    # The dot space is used to let the sourced script modify the current
    # environment. If this is not needed, can just write the path.
    . /opt/analysis/wrangle.sh

##################################################################
##                    Basespace Download App                    ##
##################################################################
%apprun download
    # Exit if something fails
    set -eu

    # Set default values for paths
    output_path="/opt/analysis"
    config_path="/opt/resources/basespace.cfg"

    help() {
        echo "Download data from the Illumina BaseSpace Sequence Hub."
        echo ""
        echo "Usage:"
        echo "  singularity run [options] --app download <container>"\
        "[app_options]"
        echo ""
        echo "Options:"
        echo "  See 'singularity run'."
        echo ""
        echo "App Options:"
        echo "  -i    Required. The run ID of the data to download."
        echo "  -o    The path to the output directory."
        echo "        Default: '/opt/analysis'."
        echo "  -c    The path to the authentication credentials file."
        echo "        This file is created by 'bs auth'. For additional"
        echo "        information see the help page for that command."
        echo "        Default: '/opt/resources/basespace.cfg'."
        echo "  -h    Print the help page."
        echo ""
        echo "Examples:"
        echo "  # Set paths"
        echo "  $ resource_dir=/bin/MIPTools/base_resources"
        echo "  $ run_dir=/work/usr/example"
        echo ""
        echo "  # Run app"
        echo "  $ singularity run \\"
        echo "    -B \${resource_dir}:/opt/resources \\"
        echo "    -B \${run_dir}:/opt/analysis \\"
        echo "    --app download <container> -i <run_id>"
    }
    
    # Parse options
    while getopts "i:o:c:h" opt; do
      case "${opt}" in
        c) config_path=${OPTARG} ;;
        h) help
           exit 1 ;;
        i) run_id=${OPTARG} ;;
        o) output_path=${OPTARG} ;;
        *) help
           exit 1 ;;
      esac
    done
    
    # Ensure run_id is specified
    if [ -z ${run_id} ]; then
      echo "Argument -i must be provided"
      help >&2
      exit 1
    fi

    # Read data from config file
    # Remove whitespace from each line and export each line as a variable
    export BASESPACE_API_SERVER=$(sed "1q;d" ${config_path} | sed "s/.*=.//g")
    export BASESPACE_ACCESS_TOKEN=$(sed "2q;d" ${config_path} | sed "s/.*=.//g")

    # Download data
    bs download run --summary -i ${run_id} -o ${output_path}/${run_id}

#################################################################
##                   Superseded Download App                   ##
#################################################################
%apprun download_superseded
    # Exit if something fails or if have unset object
    set -eu

    help() {
        echo "Download data from the Illumina BaseSpace Sequence Hub."
        echo ""
        echo "Superseded Note:"
        echo "  Please note that this app has been superseded by the download" 
        echo "  app, which uses the basespace command line interface for" 
        echo "  downloading data."
        echo ""
        echo "Usage:"
        echo "  singularity run [options] --app download_superseded \\"
        echo "    <container> [app_options]"
        echo ""
        echo "Options:"
        echo "  See 'singularity run'."
        echo ""
        echo "App Options:"
        echo "  -h    Print the help page."
        echo "  -r    Required. The run ID of the data to download."
        echo ""
        echo "Additional Details:"
        echo "  An 'access_token.txt' file with a valid access token is"
        echo "  required. It must be present in the 'base_resources' directory."
        echo "  A data directory where the data will be downloaded to must be"
        echo "  mounted to '/opt/analysis'."
        echo ""
        echo "Examples:"
        echo "  # Set paths"
        echo "  $ resource_dir=/bin/MIPTools/base_resources"
        echo "  $ output_dir=/work/usr/downloaded"
        echo ""
        echo "  # Run app"
        echo "  $ singularity run \\"
        echo "    -B \${resource_dir}:/opt/resources"\
        "-B \${output_dir}:/opt/analysis \\"
        echo "    --app download_superseded <container> -r <run_id>"
    }

    while getopts "hr:" opt; do
        case ${opt} in
            h) help
               exit 1 ;;
            r) run_id=${OPTARG} ;;
            *) help
               exit 1 ;;
        esac
    done

    # Print to CLI
    echo "Downloading NextSeq run ${run_id} from BaseSpace."
    echo "Depending on the data size, this can take very long (up to 10 h)."
    echo "It is recommended to run this app in a screen (GNU screen)."
    echo "A message indicating the end of download will be printed when done."
    echo "Check nohup.out file in your output directory for the download log."

    # cd and run app
    # Use nohup to make command keep running even if get hangup signal
    cd /opt/analysis
    nohup python /opt/bin/BaseSpaceRunDownloader_v2.py \
     -r ${run_id} -a "$(cat /opt/resources/access_token.txt)"

    # Print to CLI
    echo "Download finished."

#################################################################
##                          Demux App                          ##
#################################################################
%apprun demux
    # Exit if something fails or if have unset object
    set -eu

    help() {
        echo "Demultiplex data. Generates per-sample fastq files from the raw"
        echo "sequence data consisting of bcl files."
        echo ""
        echo "Usage:"
        echo "  singularity run [options] --app demux <container> [app_options]"
        echo ""
        echo "Options:"
        echo "  See 'singularity run'."
        echo ""
        echo "App Options:"
        echo "  -h    Print the help page."
        echo "  -s    Required. Sample sheet for demultiplexing. "
        echo "        This file must be present in the directory mounted to "
        echo "        '/opt/analysis'."
        echo ""
        echo "Examples:"
        echo "  # Set paths"
        echo "  $ resource_dir=/bin/MIPTools/base_resources"
        echo "  $ bcl_dir=/work/usr/downloaded"
        echo "  $ fastq_root_dir=/work/usr/"
        echo ""
        echo "  # Run app"
        echo "  $ singularity run \\"
        echo "    -B \${resource_dir}:/opt/resources \\"
        echo "    -B \${bcl_dir}:/opt/data \\"
        echo "    -B \${fastq_root_dir}:/opt/analysis \\"
        echo "    --app demux <container> -s SampleSheet.csv"
    }

    while getopts "hs:" opt; do
        case ${opt} in
            h) help
               exit 1 ;;
            s) sample_list=${OPTARG} ;;
            *) help
               exit 1 ;;
        esac
    done

    # Define variables
    sample_sheet="/opt/analysis/${sample_list}"
    # cd to where bcl files are
    cd /opt/data

    # Create a fastq directory for saving fastqs
    mkdir -p /opt/analysis/fastq

    # Increase limit of open number of files.
    ulimit -Sn $(ulimit -Hn)

    # Run bcl2fastq
    # Use nohup to make command keep running even if get hangup signal
    nohup bcl2fastq -o /opt/analysis/fastq \
        --sample-sheet ${sample_sheet} \
        --no-lane-splitting

##################################################################
##                         Demux QC App                         ##
##################################################################
%apprun demux_qc
    # Exit if something fails or if have unset object
    set -eu

    help() {
        echo "Run quality control on demultiplexed data."
        echo ""
        echo "Usage:"
        echo "  singularity run [options] --app demux_qc <container>"\
        "[app_options]"
        echo ""
        echo "Options:"
        echo "  See 'singularity run'."
        echo ""
        echo "App Options:"
        echo "  -h    Print the help page."
        echo "  -p    Required. The sequencing platform used. Either 'miseq'"
        echo "        or 'nextseq'."
        echo ""
        echo "Examples:"
        echo "  # Set paths"
        echo "  $ resource_dir=/bin/MIPTools/base_resources"
        echo "  $ fastq_dir=/work/usr/example"
        echo ""
        echo "  # Run app"
        echo "  $ singularity run \\"
        echo "    -B \${resource_dir}:/opt/resources"\
        "-B \${fastq_dir}:/opt/analysis \\"
        echo "    --app demux_qc <container> -p 'nextseq'"
    }

    # Argument handling
    while getopts "hp:" opt; do
        case ${opt} in
            h) help
               exit 1 ;;
            p) platform=${OPTARG} ;;
            *) help
               exit 1 ;;
        esac
    done

    # Run python script
    python /opt/src/demux_qc.py -p ${platform}