tutorial/pipeline_tutorial.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "bee7eada-b75f-42e2-ba6b-e63556aaf7c1",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "import MATES\n",
    "from MATES import bam_processor\n",
    "from MATES import data_processor\n",
    "from MATES import MATES_model\n",
    "from MATES import TE_quantifier"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "4206e51e-01c5-41e7-bc8f-5fe45f03c47f",
   "metadata": {
    "jp-MarkdownHeadingCollapsed": true,
    "tags": []
   },
   "source": [
    "## Smart-seq2 Data"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "4b12236c-96a4-4e31-a760-483579703dfc",
   "metadata": {},
   "source": [
    "#### Needed file example"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "8a5d2b44-9c74-45b3-ab82-0dc53295d582",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "sample_list_file:\n",
      "SRR3934350\n",
      "SRR3934360\n",
      "SRR3934371\n",
      "SRR3934376\n",
      "SRR3934387\n",
      "\n",
      "\n",
      "bam_path_file:\n",
      "../smart_seq/STAR_Solo_subset/SRR3934350/SRR3934350_Aligned.sortedByCoord.out.bam\n",
      "../smart_seq/STAR_Solo_subset/SRR3934360/SRR3934360_Aligned.sortedByCoord.out.bam\n",
      "../smart_seq/STAR_Solo_subset/SRR3934371/SRR3934371_Aligned.sortedByCoord.out.bam\n",
      "../smart_seq/STAR_Solo_subset/SRR3934376/SRR3934376_Aligned.sortedByCoord.out.bam\n",
      "../smart_seq/STAR_Solo_subset/SRR3934387/SRR3934387_Aligned.sortedByCoord.out.bam\n"
     ]
    }
   ],
   "source": [
    "%%bash\n",
    "echo -e 'sample_list_file:'\n",
    "cat sample_list_subset.txt | head -n 5\n",
    "echo -e '\\n\\nbam_path_file:' \n",
    "cat bam_path_file_subset.txt | head -n 5"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "1119c140-d82d-4f15-b2cb-3b677e957e9d",
   "metadata": {
    "tags": []
   },
   "source": [
    "#### Get TE reference"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9c615c67-0d0d-484c-9755-da269c770fae",
   "metadata": {},
   "outputs": [],
   "source": [
    "python3 build_reference.py --species Human"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "356a1226-3106-4e5c-b859-6634373febe2",
   "metadata": {},
   "source": [
    "### Step 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "bd155f73-ea32-434e-ae4d-a949b95db4d2",
   "metadata": {},
   "outputs": [],
   "source": [
    "bam_processor.split_bam_files('Smart_seq', 20, 'sample_list_subset.txt', 'bam_path_file_subset.txt')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "611c11e7-e81c-42c5-b85c-a521190585cb",
   "metadata": {
    "tags": []
   },
   "source": [
    "### Step 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b4c875a8-70c1-45be-ab52-1992c1706f3f",
   "metadata": {},
   "outputs": [],
   "source": [
    "bam_processor.count_coverage_vec('exclusive', 'Smart_seq', 'sample_list_subset.txt')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "610986ea-1e9e-4d9c-bd9e-17903e1ee74e",
   "metadata": {
    "tags": []
   },
   "source": [
    "### Step 3"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "acad2365-da53-442f-8b9c-fc9b28cc1d44",
   "metadata": {},
   "outputs": [],
   "source": [
    "data_processor.calculate_UM_region('exclusive', 'Smart_seq', 'sample_list_subset.txt', 5, 80)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b1ec6b70-9d95-42c7-8ce3-fa98b5ac77a9",
   "metadata": {},
   "outputs": [],
   "source": [
    "data_processor.generate_training_sample('Smart_seq', 'sample_list_subset.txt', 5, 80)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a1c15d22-c2b1-443d-8703-cfa70844b61c",
   "metadata": {},
   "outputs": [],
   "source": [
    "data_processor.generate_prediction_sample('exclusive','Smart_seq','sample_list_subset.txt', 5, 80)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "2d1cd35e-b5be-4424-a897-606b15766c0f",
   "metadata": {},
   "source": [
    "### Step 4"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "7ecbcba7-76f3-4b55-b129-4b77d8a47152",
   "metadata": {},
   "outputs": [],
   "source": [
    "MATES_model.train('Smart_seq', 'sample_list_subset.txt', bin_size = 5, proportion = 80, BATCH_SIZE= 512, \n",
    "                  AE_LR = 1e-4, MLP_LR = 1e-6, AE_EPOCHS = 200, MLP_EPOCHS = 200, DEVICE = 'cuda:0')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8ff1bdc5-288f-4fda-a3d8-2ece6fa55d10",
   "metadata": {},
   "outputs": [],
   "source": [
    "MATES_model.prediction('exclusive', 'Smart_seq', 'sample_list_subset.txt', bin_size = 5, proportion = 80, \n",
    "                       AE_trained_epochs =200, MLP_trained_epochs=200, DEVICE = 'cuda:0')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ef28e076-5e89-4d6f-a632-8962379affb2",
   "metadata": {
    "tags": []
   },
   "source": [
    "### Step 5 Quantifying TEs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ba1f317a-c1ea-4c7a-a881-770a784d67cc",
   "metadata": {},
   "outputs": [],
   "source": [
    "TE_quantifier.unique_TE_MTX('exclusive', 'Smart_seq', 'sample_list_subset.txt', 20)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6ae89aa5-fc08-4efc-9895-c7c33f168c1f",
   "metadata": {},
   "outputs": [],
   "source": [
    "TE_quantifier.finalize_TE_MTX('Smart_seq')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "0ee96961",
   "metadata": {},
   "source": [
    "### Step 6 Quantifying locus-level TEs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "3ee530ea",
   "metadata": {},
   "outputs": [],
   "source": [
    "MATES_model.prediction_locus('exclusive', 'Smart_seq', 'sample_list.txt',bin_size = 5, proportion = 80, \n",
    "                       AE_trained_epochs =200, MLP_trained_epochs=200, DEVICE = 'cuda:0')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "83d97ac6",
   "metadata": {},
   "outputs": [],
   "source": [
    "TE_quantifier.quantify_locus_TE_MTX('exclusive', 'Smart_seq', 'sample_list.txt')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "2663ef69-e05c-4ccb-bf9a-bb8ff7ba3fcc",
   "metadata": {
    "tags": []
   },
   "source": [
    "## 10X Data"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "fcd5ae4a-a3d0-40ed-8d64-93c4ff985d0c",
   "metadata": {
    "tags": []
   },
   "source": [
    "#### Needed file example"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "8164bc9d-e55c-462b-b4f2-76bf6a9de341",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "sample_list_file:\n",
      "mouse_brain\n",
      "\n",
      "bam_path_file:\n",
      "/home/roxanne/scratch/MATES/mouse_brain_Aligned.sortedByCoord.out.bam\n",
      "\n",
      "bc_path_file:\n",
      "STAR_Solo/mouse_brain/mouse_brain_Solo.out/Gene/filtered/barcodes.tsv"
     ]
    }
   ],
   "source": [
    "%%bash\n",
    "echo -e 'sample_list_file:'\n",
    "cat sample_list_file.txt\n",
    "echo -e '\\n\\nbam_path_file:'\n",
    "cat bam_path_file.txt\n",
    "echo -e '\\n\\nbc_path_file:'\n",
    "cat bc_path_file.txt"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "850eabd0-1dab-4689-b51a-8d7434d4a26a",
   "metadata": {
    "jp-MarkdownHeadingCollapsed": true,
    "tags": []
   },
   "source": [
    "#### Get TE reference"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f77a1623-2c5c-4f16-af7c-28255210f583",
   "metadata": {},
   "outputs": [],
   "source": [
    "python3 build_reference.py --species Mouse"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "873b772a-45d7-476d-a477-6a108c337c51",
   "metadata": {
    "tags": []
   },
   "source": [
    "### Step 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "34803f97-75bd-4bae-8b0b-a4b51b8d3acf",
   "metadata": {},
   "outputs": [],
   "source": [
    "bam_processor.split_count_10X_data('exclusive','10X', 'sample_list_file.txt', 'bam_path_file.txt', 'bc_path_file.txt')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d307c3c6-3b81-4d10-ab35-169179487151",
   "metadata": {
    "tags": []
   },
   "source": [
    "### Step 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4bc9f925-74ab-4b17-9bf3-78202c9994ab",
   "metadata": {},
   "outputs": [],
   "source": [
    "data_processor.calculate_UM_region('exclusive', '10X', 'sample_list_file.txt', bin_size=5, proportion=80, bc_path_file='bc_path_file.txt')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "2d44ca54-f16b-420d-b637-a356c04df42c",
   "metadata": {},
   "outputs": [],
   "source": [
    "data_processor.generate_training_sample('10X', 'sample_list_file.txt', bin_size=5, proportion=80)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "218298b9-d464-43f5-bd47-ad6c668bde1b",
   "metadata": {},
   "outputs": [],
   "source": [
    "data_processor.generate_prediction_sample('exclusive','10X','sample_list_file.txt',bin_size=5, proportion=80, bc_path_file='bc_path_file.txt')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "7ccac4ca-79d1-47fd-82e4-a633985f4905",
   "metadata": {},
   "source": [
    "### Step 3"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "99b9cf45-1b1b-4413-8700-d4943a2736f8",
   "metadata": {},
   "outputs": [],
   "source": [
    "MATES_model.train('10X', 'sample_list_file.txt', bin_size = 5, proportion = 80, BATCH_SIZE= 256, \n",
    "                  AE_LR = 1e-4, MLP_LR = 1e-6, AE_EPOCHS = 200, MLP_EPOCHS = 200, DEVICE = 'cuda:0')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "411ace01-2f19-4dcb-b6be-81348e18d7db",
   "metadata": {},
   "outputs": [],
   "source": [
    "MATES_model.prediction('exclusive', '10X', 'sample_list_file.txt', bin_size = 5, proportion = 80, \n",
    "                       AE_trained_epochs =200, MLP_trained_epochs=200, DEVICE = 'cuda:0')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "911a6477-99c2-4172-a35c-1d3646088c59",
   "metadata": {
    "tags": []
   },
   "source": [
    "### Step 4 Quantifying TEs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0bc2f7f7-99dc-4561-b8a9-1a0e57c6219f",
   "metadata": {},
   "outputs": [],
   "source": [
    "TE_quantifier.unique_TE_MTX('exclusive', '10X', 'sample_list_file.txt', 20, bc_path_file='bc_path_file.txt')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a508f096-b4e5-44e2-a7c7-6bddb4ff4b73",
   "metadata": {},
   "outputs": [],
   "source": [
    "TE_quantifier.finalize_TE_MTX('10X', 'sample_list_file.txt')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "902a37a2",
   "metadata": {},
   "source": [
    "### Step 5 Quantifying locus-level TEs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ea99a239",
   "metadata": {},
   "outputs": [],
   "source": [
    "MATES_model.prediction_locus('exclusive', '10X', 'sample_list_file.txt',bin_size = 5, proportion = 80, \n",
    "                       AE_trained_epochs =200, MLP_trained_epochs=200, DEVICE = 'cuda:0')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b1a13fa1",
   "metadata": {},
   "outputs": [],
   "source": [
    "TE_quantifier.quantify_locus_TE_MTX('exclusive', '10X', 'sample_list_file.txt')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "772f2123-9a4f-4142-be42-88f2566f14b5",
   "metadata": {
    "tags": []
   },
   "source": [
    "## 10X Multi-omics Data"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "570b436c-4a2e-4d5d-b202-73bc099c48ba",
   "metadata": {},
   "source": [
    "#### Needed file example"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "f88e9a1f-6cb6-4986-a5c1-7b9f6d78a422",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "sample_list_file:\n",
      "pbmc_scRNA\n",
      "pbmc_scATAC\n",
      "\n",
      "bam_path_file:\n",
      "/home/roxanne/scratch/pbmc_granulocyte_sorted_10k/gex/STAR_Solo/S1/S1_Aligned.sortedByCoord.out.bam\n",
      "/home/roxanne/scratch/pbmc_granulocyte_sorted_10k/atac/STAR_Solo/S16/S16_Aligned.sortedByCoord.out.bam\n",
      "\n",
      "bc_path_file:\n",
      "/home/roxanne/scratch/pbmc_granulocyte_sorted_10k/gex/STAR_Solo/S1/S1_Solo.out/Gene/filtered/barcodes.tsv\n",
      "/home/roxanne/scratch/pbmc_granulocyte_sorted_10k/atac/STAR_Solo/S16/S16_Solo.out/Gene/filtered/barcodes.tsv"
     ]
    }
   ],
   "source": [
    "%%bash\n",
    "echo -e 'sample_list_file:'\n",
    "cat sample_MultiOmics.txt\n",
    "echo -e '\\n\\nbam_path_file:' \n",
    "cat bam_path_MultiOmics.txt\n",
    "echo -e '\\n\\nbc_path_file:'\n",
    "cat barcode_path_MultiOmics.txt"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "c8aacefe-99eb-4d3c-ac3e-d2ea216fc281",
   "metadata": {
    "tags": []
   },
   "source": [
    "#### Get TE reference"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "1561849a-5648-4f7a-ad63-a986f7053c7e",
   "metadata": {},
   "outputs": [],
   "source": [
    "python3 build_reference.py --species Human"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "62a0b240-2d3e-41d0-8388-a0f1e427a375",
   "metadata": {
    "tags": []
   },
   "source": [
    "### Step 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "98abebf5-b11f-4eaa-963f-ff004c247bf6",
   "metadata": {},
   "outputs": [],
   "source": [
    "bam_processor.split_count_10X_data('exclusive','10X', 'sample_MultiOmics.txt', \n",
    "                              'bam_path_MultiOmics.txt', 'barcode_path_MultiOmics.txt')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b2d201d1-9deb-4631-b158-9dcfa4409af5",
   "metadata": {
    "tags": []
   },
   "source": [
    "### Step 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0d6a41d4-b23e-44db-8175-4148712fa1a7",
   "metadata": {},
   "outputs": [],
   "source": [
    "data_processor.calculate_UM_region('exclusive', '10X', 'sample_MultiOmics.txt',bin_size=5, proportion=80, bc_path_file='barcode_path_MultiOmics.txt')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c4a86033-6102-41f2-8953-8e9052f89e5f",
   "metadata": {},
   "outputs": [],
   "source": [
    "data_processor.generate_training_sample('10X', 'sample_MultiOmics.txt', bin_size=5, proportion=80)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "01b2fc08-b283-445e-8407-63673fe46cd2",
   "metadata": {},
   "outputs": [],
   "source": [
    "data_processor.generate_prediction_sample('exclusive','10X','sample_MultiOmics.txt', bin_size=5, proportion=80, bc_path_file='barcode_path_MultiOmics.txt')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "1ea56cc1-13ad-4684-a460-ca2deda74f90",
   "metadata": {},
   "source": [
    "### Step 3"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b5d500e0-4f0e-4266-b05f-a49b934dea22",
   "metadata": {},
   "outputs": [],
   "source": [
    "MATES_model.train('10X', 'sample_MultiOmics.txt', bin_size = 5, proportion = 80, BATCH_SIZE= 4096, \n",
    "                  AE_LR = 1e-4, MLP_LR = 1e-6, AE_EPOCHS = 200, MLP_EPOCHS = 200, DEVICE = 'cuda:0')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "99ac0047-025a-46d1-8b23-941f08e1a677",
   "metadata": {},
   "outputs": [],
   "source": [
    "MATES_model.prediction('exclusive', '10X', 'sample_MultiOmics.txt', bin_size = 5, proportion = 80, \n",
    "                       AE_trained_epochs =200, MLP_trained_epochs=200, DEVICE = 'cuda:0')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "c68e95aa-2638-4d78-95b8-fa3dc90ec5e8",
   "metadata": {
    "tags": []
   },
   "source": [
    "### Step 4 Quantifying TEs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ec147a3e-981c-49f8-884b-eca757e785e7",
   "metadata": {},
   "outputs": [],
   "source": [
    "TE_quantifier.unique_TE_MTX('exclusive', '10X', 'sample_MultiOmics.txt', 20, bc_path_file='barcode_path_MultiOmics.txt')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "effb53c6-c9f0-4a02-bcb3-b6268892b9df",
   "metadata": {},
   "outputs": [],
   "source": [
    "TE_quantifier.finalize_TE_MTX('10X', 'sample_MultiOmics.txt')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "9fcf3f28",
   "metadata": {},
   "source": [
    "### Step 5 Quantifying locus-level TEs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "3da6ae05",
   "metadata": {},
   "outputs": [],
   "source": [
    "MATES_model.prediction_locus('exclusive', '10X', 'sample_MultiOmics.txt', bin_size = 5, proportion = 80, \n",
    "                       AE_trained_epochs =200, MLP_trained_epochs=200, DEVICE = 'cuda:0')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4210da02",
   "metadata": {},
   "outputs": [],
   "source": [
    "TE_quantifier.quantify_locus_TE_MTX('exclusive', '10X', 'sample_MultiOmics.txt')"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}