MAG Co-Author Network (CADRE).ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Co-Author Network from Microsoft Academic Graph\n",
    "\n",
    "Requires an actual path to be included as `path_to_data` in the first cell below. This csv must contain columns named as referenced below.\n",
    "\n",
    "### Extracting data from CSV\n",
    "\n",
    "This first cell will fill out several dictionaries with data from the csv file:\n",
    "* **data_by_id**: article information keyed to article id.\n",
    "* **data_by_author**: author data keyed to author id"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "import os\n",
    "import csv\n",
    "\n",
    "data_by_id = {}\n",
    "data_by_author = {}\n",
    "\n",
    "path_to_data = \"sourcedata/mag_sdg.csv\"\n",
    "\n",
    "# Read in source data and iterate row by row.\n",
    "with open(path_to_data) as datafile:\n",
    "    reader = csv.DictReader(datafile) \n",
    "    # Each row contains one article author; thus each article will have as many rows as authors.\n",
    "    for row in reader:\n",
    "        year = int(row[\"year\"])\n",
    "        # Compile data only if year is in range 1999 to 2018.\n",
    "        if year < 1999 or year > 2018:\n",
    "            continue\n",
    "\n",
    "        id_ = row[\"paper_id\"]\n",
    "        author_id = row[\"author_id\"]\n",
    "        author_name = row[\"author_name\"]\n",
    "\n",
    "        # Store data about each author for lookup by name.\n",
    "        if author_id not in data_by_author:\n",
    "            data_by_author[author_id] = {\n",
    "                \"author_name\": [row[\"author_name\"]],\n",
    "                \"affiliation_id\": [row[\"affiliation_id\"]],\n",
    "                \"last_known_affiliation_id\": [row[\"last_known_affiliation_id\"]],\n",
    "                \"affiliation_name\": [row[\"affiliation_name\"]],\n",
    "                \"doi\": [row[\"doi\"]] if row[\"doi\"] != \"\" else [],\n",
    "                \"paper_id\": [row[\"paper_id\"]]\n",
    "            }\n",
    "        \n",
    "        # If author is already in the dict, append additional affiliation and article info.\n",
    "        else:\n",
    "            for key in [\"author_name\", \"affiliation_id\", \"paper_id\", \n",
    "                        \"last_known_affiliation_id\", \"doi\", \"affiliation_name\"]:\n",
    "                if row[key] not in data_by_author[author_id][key]:\n",
    "                    data_by_author[author_id][key].append(row[key])\n",
    "        \n",
    "        # Store data about each article for lookup by article ID.\n",
    "        if id_ not in data_by_id:\n",
    "            data_by_id[id_] = {\n",
    "                \"doi\": row[\"doi\"],\n",
    "                \"doc_type\": row[\"doc_type\"],\n",
    "                \"title\": row[\"original_title\"],\n",
    "                \"year\": row[\"year\"],\n",
    "                \"authors\": [\n",
    "                    {\n",
    "                        \"author_id\": row[\"author_id\"],\n",
    "                        \"name\": row[\"author_name\"],\n",
    "                        \"last_known_affiliation_id\": [row[\"last_known_affiliation_id\"]],\n",
    "                        \"affiliation_id\": [row[\"affiliation_id\"]],\n",
    "                        \"affiliation_name\": [row[\"affiliation_name\"]],\n",
    "                    }\n",
    "                ],\n",
    "                \"all_author_ids\": [row[\"author_id\"]]\n",
    "            }\n",
    "        # If article id already in dict, append additional author/affiliation data.\n",
    "        else:\n",
    "            if author_id in data_by_id[id_][\"all_author_ids\"]:\n",
    "                for a in data_by_id[id_][\"authors\"]:\n",
    "                    if a[\"author_id\"] == author_id:\n",
    "                        affiliation_id = row[\"affiliation_id\"]\n",
    "                        last_known = row[\"last_known_affiliation_id\"]\n",
    "                        normalized = row[\"affiliation_name\"]\n",
    "                        if affiliation_id not in a[\"affiliation_id\"]:\n",
    "                            a[\"affiliation_id\"].append(affiliation_id)\n",
    "                            a[\"affiliation_name\"].append(normalized)\n",
    "                            \n",
    "                        if last_known not in a[\"last_known_affiliation_id\"]:\n",
    "                            a[\"last_known_affiliation_id\"].append(last_known)\n",
    "                            \n",
    "            else:\n",
    "                new_author = {\n",
    "                    \"author_id\": row[\"author_id\"],\n",
    "                    \"name\": row[\"author_name\"],\n",
    "                    \"last_known_affiliation_id\": [row[\"last_known_affiliation_id\"]],\n",
    "                    \"affiliation_id\": [row[\"affiliation_id\"]],\n",
    "                    \"affiliation_name\": [row[\"affiliation_name\"]],\n",
    "                    }\n",
    "                data_by_id[id_][\"authors\"].append(new_author)\n",
    "                data_by_id[id_][\"all_author_ids\"].append(author_id)           "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Check values for accuracy / highlight obvious issues"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "35333"
      ]
     },
     "execution_count": 56,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(data_by_author)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'2120873293'"
      ]
     },
     "execution_count": 57,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "list(data_by_author.keys())[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'author_name': ['e ngwenya'],\n",
       " 'affiliation_id': ['129801699'],\n",
       " 'last_known_affiliation_id': ['129801699'],\n",
       " 'affiliation_name': ['university of tasmania'],\n",
       " 'doi': ['10.1007/978-1-4899-7439-6_4', '10.1007/978-1-4899-7439-6_3'],\n",
       " 'paper_id': ['420720', '1560432272']}"
      ]
     },
     "execution_count": 58,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data_by_author['2020620411']"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Many authors have multiple institutional affiliations:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1834 authors with multiple affiliations\n"
     ]
    }
   ],
   "source": [
    "count = 0\n",
    "for author, data in data_by_author.items():\n",
    "    if len(data[\"affiliation_name\"]) > 1:\n",
    "        count += 1\n",
    "        # print(author, data)\n",
    "print(count, \"authors with multiple affiliations\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Compare total number of rows to number of unique rows to roughly estimate authors per paper."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "16447 total papers, 43642 total authors\n",
      "2.6534930382440565 authors per paper\n"
     ]
    }
   ],
   "source": [
    "ids = []\n",
    "\n",
    "with open(path_to_data) as datafile:\n",
    "    \n",
    "    reader = csv.DictReader(datafile) \n",
    "    for row in reader:\n",
    "        id_ = row[\"paper_id\"]\n",
    "        ids.append(id_)\n",
    "print(len(set(ids)), \"total papers,\", len(ids), \"total authors\")\n",
    "print(len(ids) / len(set(ids)), \"authors per paper\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Building graph from author and article data"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "First, create the complete graph of all co-authorship."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "from itertools import combinations\n",
    "import networkx as nx\n",
    "\n",
    "G = nx.Graph()\n",
    "\n",
    "for id_, data in data_by_id.items():\n",
    "    \n",
    "    authors = data[\"all_author_ids\"]\n",
    "    \"\"\"\n",
    "    Iterate over all combinations of authors per paper.\n",
    "    For example if authors are 456, 789, and 123, iterate over 3 co-authorship combinations:\n",
    "        456 789\n",
    "        456 123\n",
    "        789 123\n",
    "    Add nodes and edges as needed for each combo.  \n",
    "    \"\"\"\n",
    "    for i, j in combinations(authors, 2):\n",
    "        year = data[\"year\"]\n",
    "        if G.has_edge(i, j):\n",
    "            G[i][j][\"weight\"] += 1\n",
    "            if id_ not in G[i][j][\"articles\"]:\n",
    "                G[i][j][\"articles\"].append(id_)\n",
    "            \n",
    "            if year not in G[i][j][\"years\"]:\n",
    "                G[i][j][\"years\"].append(year)\n",
    "        else:\n",
    "            G.add_edge(i, j, weight=1, articles=[id_], years=[data[\"year\"]])\n",
    "\n",
    "# Go back and iterate over all nodes created, adding affiliation/count/author data.\n",
    "for author, data in G.nodes(data=True):\n",
    "    data[\"affiliation\"] = data_by_author[author][\"affiliation_name\"][0].title()\n",
    "    data[\"papers\"] = \"<br>\\n\".join(data_by_author[author][\"doi\"])\n",
    "    data[\"name\"] = \", \".join(data_by_author[author][\"author_name\"]).title()\n",
    "    data[\"count\"] = len(data_by_author[author][\"paper_id\"])\n",
    "\n",
    "degree_dict = dict(G.degree(G.nodes()))\n",
    "nx.set_node_attributes(G, degree_dict, 'degree')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "79510"
      ]
     },
     "execution_count": 62,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(G.edges)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "29527"
      ]
     },
     "execution_count": 63,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(G.nodes)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Create subgraph containing only those edges with a `weight` greater than 1, that is, co-author pairs who published more than 1 paper in the dataset."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "def filter_edge(n1, n2):\n",
    "    \"\"\"Check if weight is larger than 1.\"\"\"\n",
    "    return G[n1][n2][\"weight\"] > 1\n",
    "\n",
    "def filter_node(n):\n",
    "    \"\"\"Filter out unconnected nodes.\"\"\"\n",
    "    return not nx.is_isolate(view, n)\n",
    "\n",
    "view = nx.subgraph_view(G, filter_edge=filter_edge)\n",
    "subview = nx.subgraph_view(view, filter_node=filter_node)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(2847, 1913)"
      ]
     },
     "execution_count": 65,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(subview.edges()), len(subview.nodes())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Compute descriptive and analytic statistics related to the density, modularity, and community structure of the graph."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "--------------\n",
      "Name: \n",
      "Type: Graph\n",
      "Number of nodes: 1913\n",
      "Number of edges: 2847\n",
      "Average degree:   2.9765\n"
     ]
    }
   ],
   "source": [
    "C_GRAPH = [subview]\n",
    "\n",
    "for g in C_GRAPH:\n",
    "    print(\"--------------\")\n",
    "    print(nx.info(g))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.0015567346956630148\n"
     ]
    }
   ],
   "source": [
    "for g in C_GRAPH:\n",
    "    print(nx.density(g))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.6756985605419137\n"
     ]
    }
   ],
   "source": [
    "from networkx import transitivity\n",
    "for g in C_GRAPH:\n",
    "    print(transitivity(g))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "517\n"
     ]
    }
   ],
   "source": [
    "components = []\n",
    "components_lens = []\n",
    "for g in C_GRAPH:\n",
    "    c = list(nx.connected_components(g))\n",
    "    components.append(c)\n",
    "    components_lens.append(len(c))\n",
    "for x in components_lens:\n",
    "    print(x)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "largest = []\n",
    "subgraphs = {}\n",
    "\n",
    "for i,c in enumerate(components):\n",
    "    # print(i)\n",
    "    largest_component = max(c, key=len)\n",
    "    subgraphs[i] = C_GRAPH[i].subgraph(largest_component)\n",
    "    # diameter = nx.diameter(subgraph)\n",
    "    # print(\"Network diameter of largest component:\", diameter)\n",
    "\n",
    "for i,g in enumerate(components):\n",
    "    print(\"Graph\", i)\n",
    "    for j,c in enumerate(g):\n",
    "        print(\"Component\", str(j) + \":\", len(c))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "----------------------\n",
      "Class 0: 95\n",
      "Class 1: 56\n",
      "Class 2: 26\n",
      "Class 3: 24\n",
      "Class 4: 21\n",
      "Class 5: 16\n",
      "Class 6: 14\n",
      "Class 7: 11\n",
      "Class 8: 7\n",
      "Class 9: 5\n",
      "Class 10: 4\n",
      "Class 11: 3\n",
      "Class 12: 3\n",
      "Class 13: 3\n"
     ]
    }
   ],
   "source": [
    "from networkx.algorithms import community\n",
    "mod_communities = []\n",
    "for s in subgraphs:\n",
    "    print(\"----------------------\")\n",
    "    cgraph = subgraphs[s]\n",
    "    communities = community.greedy_modularity_communities(cgraph)\n",
    "    mod_communities.append(communities)\n",
    "    for i,c in enumerate(communities):\n",
    "        if len(c) > 2: # Filter out modularity classes with 2 or fewer nodes\n",
    "            print('Class '+str(i)+':', len(list(c)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "for j, mc in enumerate(mod_communities):\n",
    "    \n",
    "    modularity_dict = {} # Create a blank dictionary\n",
    "    for i,c in enumerate(mc): # Loop through the list of communities, keeping track of the number for the community\n",
    "        for name in c: # Loop through each person in a community\n",
    "            modularity_dict[name] = i # Create an entry in the dictionary for the person, where the value is which group they belong to.\n",
    "\n",
    "    # Now you can add modularity information like we did the other metrics\n",
    "    nx.set_node_attributes(subgraphs[j], modularity_dict, 'modularity')\n",
    "    \n",
    "    degree_dict = dict(subgraphs[j].degree(subgraphs[j].nodes()))\n",
    "    nx.set_node_attributes(subgraphs[j], degree_dict, 'degree')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Centrality Measures"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1968854487 0.011934865232997301\n",
      "2115856384 0.00591926503462666\n",
      "2304922534 0.005856917954631874\n",
      "2107543259 0.005527923769320745\n",
      "2132072248 0.00540802094896536\n",
      "2356932058 0.005244557645621993\n",
      "2126383504 0.004198606883730748\n",
      "2168746073 0.004088366886050457\n",
      "2316804223 0.003653152340397471\n",
      "2678471448 0.0034797144541415335\n",
      "2128434403 0.0034245475754687447\n",
      "2159017802 0.003057611844222723\n",
      "2015186656 0.0028688250595112967\n",
      "2287806044 0.002843042591996567\n",
      "2250989391 0.0022606403359541434\n",
      "2140640685 0.001806692913344774\n",
      "2294583727 0.001671669633415001\n",
      "1990686718 0.0013695210945659242\n",
      "2604652503 0.0013689253538644774\n",
      "2106159574 0.0012439745789659185\n",
      "2138773381 0.0012248716391949851\n",
      "2231847126 0.0011244897980724042\n",
      "2148695815 0.0010985479145755807\n",
      "2287431464 0.0009931285819099948\n",
      "1881159122 0.0009700730938162741\n",
      "1923876551 0.0009277930676615673\n",
      "2159611613 0.0009230491896361591\n",
      "299576312 0.0008464404990867873\n",
      "1975427381 0.0007767187982370289\n",
      "1501559239 0.0007717924633644897\n",
      "2272151263 0.0006579439482185935\n",
      "2057230601 0.0006516480654043632\n",
      "2214979510 0.0006279925746666422\n",
      "2617677781 0.0006229076761055242\n",
      "2065837648 0.0006218129350227377\n",
      "2246707465 0.000548220896443846\n",
      "2031858292 0.0005141338022590104\n",
      "2112768246 0.0004883442648492427\n",
      "2043087548 0.00047218876311182694\n",
      "2060409082 0.0004678270181316392\n",
      "1434118451 0.00046663338653774993\n",
      "2144628046 0.00046635970126705334\n",
      "2164564696 0.00046635970126705334\n",
      "2442487362 0.00046635970126705334\n",
      "1988076139 0.00038598746010581035\n",
      "2282139557 0.00038598746010581035\n",
      "2151644619 0.00037561348508321465\n",
      "2025631026 0.0003362679692625898\n",
      "2003182907 0.00031273103598267974\n",
      "350541580 0.00031211590406948764\n"
     ]
    }
   ],
   "source": [
    "from operator import itemgetter\n",
    "\n",
    "cc = nx.closeness_centrality(subview)\n",
    "bc = nx.betweenness_centrality(subview)\n",
    "\n",
    "bc_sorted = sorted(bc.items(), key=itemgetter(1), reverse=True)\n",
    "cc_sorted = sorted(cc.items(), key=itemgetter(1), reverse=True)\n",
    "\n",
    "for name, value in bc_sorted[:50]:\n",
    "    print(name, value)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Output graph data as separate files containing `nodes` and `edges` for display in `flourish.studio`.\n",
    "\n",
    "Our platform for visualizing the graph, `flourish.studio` requires edge and node lists as comma-separated rows"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "import csv\n",
    "\n",
    "graph_to_output = subview\n",
    "name = \"weight_1\"\n",
    "\n",
    "nx.write_edgelist(graph_to_output, \"mag_output_authors/mag_edges_{0}.tsv\".format(name), delimiter=\"\\t\", data=[\"weight\"])\n",
    "with open(\"mag_output_authors/mag_nodes_{0}.csv\".format(name), \"w\") as f:\n",
    "    fieldnames = [\"id\", \"affiliation\", \"papers\", \"name\", \"count\", \"degree\", \"modularity\"]\n",
    "    writer = csv.DictWriter(f, fieldnames=fieldnames)\n",
    "    writer.writeheader()\n",
    "    for n, data in list(graph_to_output.nodes(data=True)):\n",
    "        row = data\n",
    "        row[\"id\"] = n\n",
    "        writer.writerow(row)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "14"
      ]
     },
     "execution_count": 43,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "max([subview[e][v][\"weight\"] for e, v in subview.edges])"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}