02b_zipcode_inference.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 84,
   "metadata": {},
   "outputs": [],
   "source": [
    "import random\n",
    "import csv\n",
    "from math import radians, cos, sin, asin, sqrt\n",
    "\n",
    "import networkx as nx\n",
    "import pickle as pkl\n",
    "import pandas as pd\n",
    "\n",
    "\n",
    "from vars import DATASET"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 85,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'boston-metro'"
      ]
     },
     "execution_count": 85,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "DATASET"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 86,
   "metadata": {},
   "outputs": [],
   "source": [
    "g = nx.read_gpickle(\"./data/01_\" + DATASET + \".gpickle\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 88,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "number of nodes in the graph:918800\n",
      "number of nodes with zipcodes:0\n",
      "ratio-zipcoded: 0.0\n"
     ]
    }
   ],
   "source": [
    "num_nodes = len(g.nodes())\n",
    "num_nodes_with_zips = len([x for x in g.nodes() if g.nodes()[x].get('zipcode')])\n",
    "\n",
    "print('number of nodes in the graph:' + str(num_nodes))\n",
    "print('number of nodes with zipcodes:' + str(num_nodes_with_zips))\n",
    "print('ratio-zipcoded: ' + str(num_nodes_with_zips/num_nodes))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# tag nodes with zipcodes, population, and total graph pop"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Known supplied zipcodes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [],
   "source": [
    "zipcode_dict = {}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 89,
   "metadata": {},
   "outputs": [],
   "source": [
    "# populate zipcode_dict with nodes for each zipcode\n",
    "with open('./data/zipcode/zipcodes_boston-metro.csv', newline='') as f:\n",
    "    csvreader = csv.reader(f, delimiter=',')\n",
    "    next(csvreader) # skip header\n",
    "    for row in csvreader:\n",
    "        node_id, zipcode = row[0], row[1]\n",
    "        \n",
    "        if zipcode_dict.get(zipcode) is not None and zipcode is not '':\n",
    "            zipcode_dict[zipcode]['ids'] = zipcode_dict[zipcode]['ids'] + [node_id]\n",
    "        elif zipcode is not '':\n",
    "            zipcode_dict[zipcode] = {}\n",
    "            zipcode_dict[zipcode]['population'] = 0\n",
    "            zipcode_dict[zipcode]['ids'] = [node_id]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 90,
   "metadata": {},
   "outputs": [],
   "source": [
    "# populate zipcode_dict with population for each zipcode\n",
    "# from https://worldpopulationreview.com/zips/massachusetts\n",
    "with open('./data/zipcode/population_by_zip_2020.csv', newline='') as f:\n",
    "    csvreader = csv.reader(f, delimiter=',')\n",
    "    next(csvreader) # skip header\n",
    "    for row in csvreader:\n",
    "        population, zipcode = int(row[3]), row[0].zfill(5) # 5 digit zipcode\n",
    "        \n",
    "        if zipcode_dict.get(zipcode):\n",
    "            zipcode_dict[zipcode]['population'] = population\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 91,
   "metadata": {},
   "outputs": [],
   "source": [
    "# compute total population visible in graph\n",
    "total_pop_dict = {}\n",
    "all_nodes_set = set(g.nodes())\n",
    "\n",
    "# populate a dictionary of zipcodes in use, and their populations\n",
    "for zipcode, inner_dict in zipcode_dict.items():\n",
    "    population = inner_dict['population']\n",
    "    nodes = set(inner_dict['ids'])\n",
    "    \n",
    "    # if the zipcode hasn't gotten its population yet, and the nodes intersect with the graph\n",
    "    if total_pop_dict.get(zipcode, True) and len(nodes.intersection(all_nodes_set)) > 0:\n",
    "        total_pop_dict[zipcode] = population\n",
    "\n",
    "# sum the values of the dictionary to get the total map population\n",
    "total_population = sum([v for k, v in total_pop_dict.items()])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 101,
   "metadata": {},
   "outputs": [],
   "source": [
    "# tag nodes with the zipcode, population, and total graph population\n",
    "# in case of nodes without zipcodes, tag them as none\n",
    "\n",
    "for node in g.nodes():\n",
    "    g.nodes()[node]['zipcode'] = None\n",
    "    g.nodes()[node]['population'] = 0\n",
    "    g.nodes()[node]['total_graph_area_pop'] = total_population\n",
    "    \n",
    "\n",
    "for zipcode, inner_dict in zipcode_dict.items():\n",
    "    population = inner_dict['population']\n",
    "    nodes = inner_dict['ids']\n",
    "    \n",
    "    for node in nodes:\n",
    "        \n",
    "        # the zipcode dictionary may have nodes not contained in\n",
    "        # the particular dataset, so make sure to only try to tag those\n",
    "        if g.nodes.get(node):\n",
    "            g.nodes()[node]['zipcode'] = zipcode\n",
    "            g.nodes()[node]['population'] = population\n",
    "            g.nodes()[node]['total_graph_area_pop'] = total_population"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 102,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "number of nodes in the graph:918800\n",
      "number of nodes with zipcodes:306361\n",
      "ratio-zipcoded: 0.33343600348280367\n"
     ]
    }
   ],
   "source": [
    "num_nodes = len(g.nodes())\n",
    "num_nodes_with_zips = len([x for x in g.nodes() if g.nodes()[x].get('zipcode')])\n",
    "\n",
    "print('number of nodes in the graph:' + str(num_nodes))\n",
    "print('number of nodes with zipcodes:' + str(num_nodes_with_zips))\n",
    "print('ratio-zipcoded: ' + str(num_nodes_with_zips/num_nodes))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# scratch"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 103,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'lat': 42.2036233,\n",
       " 'lon': -71.1058422,\n",
       " 'id': '30416737',\n",
       " 'tags': {},\n",
       " 'zipcode': None,\n",
       " 'population': 0,\n",
       " 'total_graph_area_pop': 2581490}"
      ]
     },
     "execution_count": 103,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "g.nodes()['30416737']"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## inferring zipcodes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 104,
   "metadata": {},
   "outputs": [],
   "source": [
    "def infer_zipcode_from_neighbors(node_id):\n",
    "    '''given a node_id, add a zipcode if it can be inferred.\n",
    "       return whether an inference was made, and the current zipcode'''\n",
    "    \n",
    "    # if the node has an assigned zipcode, use that.\n",
    "    if g.nodes()[node_id].get('zipcode') is not None:\n",
    "        return (False, g.nodes()[node_id].get('zipcode'))\n",
    "    \n",
    "    in_edges = g.in_edges(node_id)\n",
    "    out_edges = g.out_edges(node_id)\n",
    "    \n",
    "    in_neighbors = [x[0] for x in g.in_edges(node_id)]\n",
    "    out_neighbors = [x[1] for x in g.out_edges(node_id)]\n",
    "    \n",
    "    in_distances = [g.edges()[x]['havlen'] for x in in_edges]\n",
    "    out_distances = [g.edges()[x]['havlen'] for x in out_edges]\n",
    "    \n",
    "    in_zipcodes = [g.nodes()[x].get('zipcode', None) for x in in_neighbors]\n",
    "    out_zipcodes = [g.nodes()[x].get('zipcode', None) for x in out_neighbors]\n",
    "    \n",
    "    zip_distances = [x for x in zip(in_zipcodes, in_distances)] + [x for x in zip(out_zipcodes, out_distances)]\n",
    "    \n",
    "    # dictionary of zipcodes and their values\n",
    "    distance_dict = {}\n",
    "    for combo in zip_distances:\n",
    "        zipcode = combo[0]\n",
    "        distance = combo[1]\n",
    "        existing_distance = distance_dict.get(zipcode, 0)\n",
    "        \n",
    "        if zipcode is not None:\n",
    "            distance_dict[zipcode] = existing_distance + distance\n",
    "            \n",
    "    closest_zip = None\n",
    "    closest_distance = 1000000\n",
    "    for k, v in distance_dict.items():\n",
    "        if v < closest_distance:\n",
    "            closest_distance = v\n",
    "            closest_zip = k\n",
    "    \n",
    "    # categorize if possible\n",
    "    inference_made = False\n",
    "    if closest_zip != None:\n",
    "        g.nodes()[node_id]['zipcode'] = closest_zip\n",
    "        g.nodes()[node_id]['population'] = total_pop_dict[closest_zip]\n",
    "        g.nodes()[node_id]['total_graph_area_pop'] = total_population\n",
    "        \n",
    "        inference_made = True\n",
    "    \n",
    "    return (inference_made, closest_zip)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 105,
   "metadata": {},
   "outputs": [],
   "source": [
    "def nodes_without_zipcodes(search_list=None):\n",
    "    '''returns a list of graph nodes without zipcodes.\n",
    "       if a search list is supplied, only searches that subset of nodes'''\n",
    "    if search_list is None:\n",
    "        return [x for x in g.nodes() if not g.nodes()[x].get('zipcode')]\n",
    "    else:\n",
    "        without_zips = []\n",
    "        for x in search_list:\n",
    "            if g.nodes()[x].get('zipcode') is None:\n",
    "                without_zips.append(x)\n",
    "        return without_zips"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 106,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 3min 10s, sys: 257 ms, total: 3min 10s\n",
      "Wall time: 3min 10s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "# successively loop through nodes. If they can be categorized using neighbors,\n",
    "# categorize them.\n",
    "\n",
    "nodes_since_last_categorization = 0\n",
    "uncategorized_nodes = None\n",
    "\n",
    "while uncategorized_nodes is None or len(uncategorized_nodes) > nodes_since_last_categorization:\n",
    "    uncategorized_nodes = nodes_without_zipcodes(uncategorized_nodes)\n",
    "    \n",
    "    for node in uncategorized_nodes:\n",
    "        inference = infer_zipcode_from_neighbors(node)\n",
    "        inference_made = inference[0]\n",
    "        zipcode = inference[1]\n",
    "\n",
    "        if inference_made:\n",
    "            nodes_since_last_categorization = 0\n",
    "        else:\n",
    "            nodes_since_last_categorization += 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 107,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "6639"
      ]
     },
     "execution_count": 107,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "leftover = nodes_without_zipcodes()\n",
    "len(leftover)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Write results to disk"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 109,
   "metadata": {},
   "outputs": [],
   "source": [
    "nx.write_gpickle(g, \"./data/02_\" + DATASET + \".gpickle\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "bikepoints",
   "language": "python",
   "name": "bikepoints"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.0"
  },
  "varInspector": {
   "cols": {
    "lenName": 16,
    "lenType": 16,
    "lenVar": 40
   },
   "kernels_config": {
    "python": {
     "delete_cmd_postfix": "",
     "delete_cmd_prefix": "del ",
     "library": "var_list.py",
     "varRefreshCmd": "print(var_dic_list())"
    },
    "r": {
     "delete_cmd_postfix": ") ",
     "delete_cmd_prefix": "rm(",
     "library": "var_list.r",
     "varRefreshCmd": "cat(var_dic_list()) "
    }
   },
   "types_to_exclude": [
    "module",
    "function",
    "builtin_function_or_method",
    "instance",
    "_Feature"
   ],
   "window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}