04.doi-citations.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Create a DOI-to-DOI citation catalog"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import pathlib\n",
    "import re\n",
    "import urllib.parse\n",
    "\n",
    "import pandas"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Create a refined DOI to bibligraphic resource mapping"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>br_iri</th>\n",
       "      <th>type</th>\n",
       "      <th>id</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>gbr:1</td>\n",
       "      <td>doi</td>\n",
       "      <td>10.1097/igc.0000000000000609</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>gbr:1</td>\n",
       "      <td>pmcid</td>\n",
       "      <td>PMC4679344</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>gbr:1</td>\n",
       "      <td>pmid</td>\n",
       "      <td>26645990</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>gbr:1</td>\n",
       "      <td>url</td>\n",
       "      <td>http://dx.doi.org/10.1097/igc.0000000000000609</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>gbr:3</td>\n",
       "      <td>issn</td>\n",
       "      <td>1048-891X</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  br_iri   type                                              id\n",
       "0  gbr:1    doi                    10.1097/igc.0000000000000609\n",
       "1  gbr:1  pmcid                                      PMC4679344\n",
       "2  gbr:1   pmid                                        26645990\n",
       "3  gbr:1    url  http://dx.doi.org/10.1097/igc.0000000000000609\n",
       "4  gbr:3   issn                                       1048-891X"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "path = pathlib.Path('data/identifiers.tsv.xz')\n",
    "identifier_df = pandas.read_table(path, compression='xz')\n",
    "identifier_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "doi_pattern = re.compile(r'10\\.\\d{4,9}/\\S+')\n",
    "\n",
    "doi_url_pattern = re.compile(r\"https?://(?:dx\\.)?doi\\.org/(10\\..+)\")\n",
    "\n",
    "def doi_sanitize(doi):\n",
    "    \"\"\"\n",
    "    Fix common DOI formatting errors\n",
    "    \"\"\"\n",
    "    doi = doi.lower()\n",
    "    if doi.startswith('doi:'):\n",
    "        doi = doi[4:]\n",
    "    # Check for DOIs as URLs\n",
    "    match = doi_url_pattern.match(doi)\n",
    "    if match:\n",
    "        doi = match.group(1)\n",
    "        doi = urllib.parse.unquote(doi)\n",
    "    # Quality control\n",
    "    if doi_pattern.fullmatch(doi):\n",
    "        return doi\n",
    "    return None"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>br_iri</th>\n",
       "      <th>doi</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>gbr:1</td>\n",
       "      <td>10.1097/igc.0000000000000609</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>gbr:6</td>\n",
       "      <td>10.1097/01.aog.0000255980.88205.15</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>gbr:11</td>\n",
       "      <td>10.1097/00006250-200508000-00050</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>gbr:14</td>\n",
       "      <td>10.1038/nature12113</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21</th>\n",
       "      <td>gbr:18</td>\n",
       "      <td>10.1038/modpathol.3800620</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    br_iri                                 doi\n",
       "0    gbr:1        10.1097/igc.0000000000000609\n",
       "6    gbr:6  10.1097/01.aog.0000255980.88205.15\n",
       "11  gbr:11    10.1097/00006250-200508000-00050\n",
       "14  gbr:14                 10.1038/nature12113\n",
       "21  gbr:18           10.1038/modpathol.3800620"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "doi_df = (identifier_df\n",
    "    .query(\"type == 'doi' or type == 'url'\")\n",
    "    .drop('type', axis='columns')\n",
    "    .rename(columns={'id': 'doi'})\n",
    ")\n",
    "\n",
    "# Ensure all DOIs are lowercase\n",
    "doi_df.doi = doi_df.doi.map(doi_sanitize)\n",
    "doi_df = doi_df.dropna().drop_duplicates()\n",
    "doi_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'2,049 bibliographic resources have duplicate DOIs'"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "duplicate_doi_df = doi_df[doi_df.doi.duplicated(keep=False)]\n",
    "f'{len(duplicate_doi_df):,} bibliographic resources have duplicate DOIs'"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Create a DOI citation catalog"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>source</th>\n",
       "      <th>target</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>gbr:1</td>\n",
       "      <td>gbr:5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>gbr:1</td>\n",
       "      <td>gbr:6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>gbr:1</td>\n",
       "      <td>gbr:10</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  source  target\n",
       "0  gbr:1   gbr:5\n",
       "1  gbr:1   gbr:6\n",
       "2  gbr:1  gbr:10"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "path = pathlib.Path('data/citations.tsv.xz')\n",
    "citation_df = pandas.read_table(path, compression='xz')\n",
    "citation_df.head(3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>source_doi</th>\n",
       "      <th>target_doi</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>3778207</th>\n",
       "      <td>10.1002/14651858.cd002244.pub4</td>\n",
       "      <td>10.1001/archneur.1990.00530120057010</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3778223</th>\n",
       "      <td>10.1002/14651858.cd002244.pub4</td>\n",
       "      <td>10.1002/14651858.cd002244</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3778224</th>\n",
       "      <td>10.1002/14651858.cd002244.pub4</td>\n",
       "      <td>10.1002/14651858.cd002244.pub2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1663454</th>\n",
       "      <td>10.1002/14651858.cd002244.pub4</td>\n",
       "      <td>10.1002/14651858.cd002244.pub3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3778219</th>\n",
       "      <td>10.1002/14651858.cd002244.pub4</td>\n",
       "      <td>10.1007/s003810000427</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                             source_doi                            target_doi\n",
       "3778207  10.1002/14651858.cd002244.pub4  10.1001/archneur.1990.00530120057010\n",
       "3778223  10.1002/14651858.cd002244.pub4             10.1002/14651858.cd002244\n",
       "3778224  10.1002/14651858.cd002244.pub4        10.1002/14651858.cd002244.pub2\n",
       "1663454  10.1002/14651858.cd002244.pub4        10.1002/14651858.cd002244.pub3\n",
       "3778219  10.1002/14651858.cd002244.pub4                 10.1007/s003810000427"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "doi_citation_df = (doi_df\n",
    "    .rename(columns={'br_iri': 'source', 'doi': 'source_doi'})\n",
    "    .merge(citation_df)\n",
    "    .merge(doi_df.rename(columns={'br_iri': 'target', 'doi': 'target_doi'}))\n",
    "    [['source_doi', 'target_doi']]\n",
    "    .drop_duplicates()\n",
    "    .sort_values(['source_doi', 'target_doi'])\n",
    "    # Remove DOI self-cites which are likely errors\n",
    "    .query(\"source_doi != target_doi\")\n",
    ")\n",
    "doi_citation_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "7,574,387 total DOI-to-DOI citations\n",
      "203,264 DOIs with outgoing DOI citations\n",
      "3,946,611 DOIs with incoming DOI citations\n"
     ]
    }
   ],
   "source": [
    "print(f'''\n",
    "{len(doi_citation_df):,} total DOI-to-DOI citations\n",
    "{doi_citation_df.source_doi.nunique():,} DOIs with outgoing DOI citations\n",
    "{doi_citation_df.target_doi.nunique():,} DOIs with incoming DOI citations\n",
    "'''.strip())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "path = pathlib.Path('data/citations-doi.tsv.xz')\n",
    "doi_citation_df.to_csv(path, compression='xz', index=False, sep='\\t')"
   ]
  }
 ],
 "metadata": {
  "anaconda-cloud": {},
  "kernelspec": {
   "display_name": "Python [conda env:opencitations]",
   "language": "python",
   "name": "conda-env-opencitations-py"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}