notebooks/bte_export_csv_files.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "b10986c1",
   "metadata": {},
   "source": [
    "## Get PFOCR Data\n",
    "\n",
    "We could get it from the API, but for now, we'll just go ahead and download the entire JSON file we gave to BTE.\n",
    "This notebook processes the PFOCR Jason file, creates a dataframe with relevant information and saves this data to Dropbox in csv files."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "60584988",
   "metadata": {},
   "outputs": [],
   "source": [
    "#load packages\n",
    "import requests\n",
    "import json\n",
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "0b527c23",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "status_code: 200\n",
      "the request has succeeded\n"
     ]
    }
   ],
   "source": [
    "# for access to the below dropbox folder\n",
    "# contact Bioinformatics core, Gladstone Institutes (bioinformatics@gladstone.ucsf.edu)\n",
    "pfocr_url = \"https://www.dropbox.com/s/bs04audl3g4frit/bte_chemicals_diseases_genes.ndjson?dl=1\"\n",
    "pfocr_request = requests.get(pfocr_url)\n",
    "print(f\"status_code: {pfocr_request.status_code}\")\n",
    "if pfocr_request.status_code != 200:\n",
    "    print(pfocr_request.text)\n",
    "else:\n",
    "    print(\"the request has succeeded\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "cd18b566",
   "metadata": {},
   "outputs": [],
   "source": [
    "#create a data frame for all the figure data from PFOCR\n",
    "#expected size of final data frame (as in Jan, 2023): 77719 rows × 4 columns\n",
    "#PFOCR has chemical, disease and gene information represented by mesh ids, mesh ids and NCBI gene ids repectively\n",
    "figures_df_data = []\n",
    "\n",
    "for line in pfocr_request.text.splitlines():\n",
    "    pfocr_result = json.loads(line)\n",
    "    figure_id = pfocr_result[\"_id\"]\n",
    "\n",
    "    chemical_curies = set()\n",
    "    disease_curies = set()\n",
    "    gene_curies = set()\n",
    "    for identifier in pfocr_result[\"associatedWith\"][\"mentions\"][\"chemicals\"][\"mesh\"]:\n",
    "        curie = \"MESH:\" + identifier\n",
    "        chemical_curies.add(curie)\n",
    "    for identifier in pfocr_result[\"associatedWith\"][\"mentions\"][\"diseases\"][\"mesh\"]:\n",
    "        curie = \"MESH:\" + identifier\n",
    "        disease_curies.add(curie)\n",
    "    for identifier in pfocr_result[\"associatedWith\"][\"mentions\"][\"genes\"][\"ncbigene\"]:\n",
    "        curie = \"NCBIGene:\" + identifier\n",
    "        gene_curies.add(curie)\n",
    "\n",
    "    figures_df_data.append({\n",
    "        \"figure_id\": figure_id,\n",
    "        \"chemical_curie_set\": chemical_curies,\n",
    "        \"disease_curie_set\": disease_curies,\n",
    "        \"gene_curie_set\": gene_curies,\n",
    "    })\n",
    "\n",
    "figures_df = pd.DataFrame.from_records(figures_df_data)\n",
    "\n",
    "#save the dataframe to Dropbox\n",
    "figures_df.to_csv('/Users/aagrawal/Dropbox (Gladstone)/pfocr2bte/figures_dataframe.csv', index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "a962153d",
   "metadata": {},
   "outputs": [],
   "source": [
    "# create figure metadata and identify unique CURIES for all PFOCR figures\n",
    "figure_metadata_records = []\n",
    "figure_curie_df_data = []\n",
    "for line in pfocr_request.text.splitlines():\n",
    "    pfocr_result = json.loads(line)\n",
    "    figure_id = pfocr_result[\"_id\"]\n",
    "\n",
    "    chemical_curies = set()\n",
    "    disease_curies = set()\n",
    "    gene_curies = set()\n",
    "    for identifier in pfocr_result[\"associatedWith\"][\"mentions\"][\"chemicals\"][\"mesh\"]:\n",
    "        curie = \"MESH:\" + identifier\n",
    "        chemical_curies.add(curie)\n",
    "        figure_curie_df_data.append({\n",
    "            \"figure_id\": figure_id,\n",
    "            \"curie\": curie,\n",
    "            \"category\": \"chemical\",\n",
    "        })\n",
    "    for identifier in pfocr_result[\"associatedWith\"][\"mentions\"][\"diseases\"][\"mesh\"]:\n",
    "        curie = \"MESH:\" + identifier\n",
    "        disease_curies.add(curie)\n",
    "        figure_curie_df_data.append({\n",
    "            \"figure_id\": figure_id,\n",
    "            \"curie\": curie,\n",
    "            \"category\": \"disease\",\n",
    "        })\n",
    "    for identifier in pfocr_result[\"associatedWith\"][\"mentions\"][\"genes\"][\"ncbigene\"]:\n",
    "        curie = \"NCBIGene:\" + identifier\n",
    "        gene_curies.add(curie)\n",
    "        figure_curie_df_data.append({\n",
    "            \"figure_id\": figure_id,\n",
    "            \"curie\": curie,\n",
    "            \"category\": \"gene\",\n",
    "        })\n",
    "\n",
    "    figure_metadata_records.append({\n",
    "        \"figure_id\": figure_id,\n",
    "        \"figure_url\": pfocr_result[\"associatedWith\"][\"figureUrl\"],\n",
    "        \"figure_title\": pfocr_result[\"associatedWith\"][\"title\"],\n",
    "    })\n",
    "\n",
    "figure_metadata_df = pd.DataFrame.from_records(figure_metadata_records, index='figure_id')\n",
    "#save the metadata dataframe to Dropbox\n",
    "figure_metadata_df.to_csv('/Users/aagrawal/Dropbox (Gladstone)/pfocr2bte/figures_metadata.csv', index=True)\n",
    "\n",
    "\n",
    "figure_curie_df = pd.DataFrame.from_records(figure_curie_df_data)\n",
    "#save the metadata dataframe to Dropbox\n",
    "figure_curie_df.to_csv('/Users/aagrawal/Dropbox (Gladstone)/pfocr2bte/figures_curie.csv', index=False)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.16"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}