monthly-mediterranean-data.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Update Source JSONs"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### gets raw jsons from unhcr"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import json, requests\n",
    "\n",
    "command = 'wget http://data.unhcr.org/api/stats/mediterranean/countries.json -O json_store/mediterranean_countries.json'\n",
    "!$command\n",
    "\n",
    "command = 'wget http://data.unhcr.org/api/stats/origin.json -O json_store/origin_countries.json'\n",
    "!$command\n",
    "\n",
    "command = 'wget http://data.unhcr.org/api/countries/list.json -O geo_data/countries_list.json'\n",
    "!$command                "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## gets every combination of monthly arrival data between origin and asylum countries"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#Malta is MLT not MTA, make that change \n",
    "with open('json_store/mediterranean_countries.json') as data_file:    \n",
    "    asylum_countries = json.load(data_file)\n",
    "\n",
    "with open('json_store/origin_countries.json') as data_file:\n",
    "    origin_countries = json.load(data_file)\n",
    "\n",
    "for asyl_country in asylum_countries:\n",
    "    asylum_str = asyl_country[\"country_of_asylum\"]\n",
    "    asylum_name = asyl_country[\"country_of_asylum_en\"]\n",
    "    command = \"mkdir json_store/by_country/{0}\".format(asylum_name)\n",
    "    !$command\n",
    "    command = 'wget http://data.unhcr.org/api/stats/mediterranean/monthly_arrivals_by_country.json?country={0} -O json_store/by_country/{1}/monthly{1}.json'.format(asylum_str, asylum_name)\n",
    "    !$command\n",
    "    command = 'wget http://data.unhcr.org/api/stats/mediterranean/monthly_arrivals_by_location.json?country={0} -O json_store/by_country/{1}/monthly_loc{1}.json'.format(asylum_str, asylum_name)\n",
    "    !$command\n",
    "    for origin_country in origin_countries:\n",
    "        origin_str = origin_country[\"origin\"]\n",
    "        origin_name = origin_country[\"origin_en\"]\n",
    "        result = requests.get(\"http://data.unhcr.org/api/stats/mediterranean/origin.json\", \n",
    "              params={'country_of_asylum': asylum_str, 'country_of_origin': origin_str}).json()\n",
    "        if result:\n",
    "            print result\n",
    "            with open('json_store/by_country/{0}/from{1}.json'.format(asylum_name, origin_name), 'w') as outfile:\n",
    "                json.dump(result, outfile)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## updates the deaths json, must do manually from link"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import json\n",
    "deaths = {}\n",
    "for i in range(2014,2017):\n",
    "    deaths[i] = {}\n",
    "    for j in range(1,13):\n",
    "        deaths[int(i)][int(j)] = 0\n",
    "        \n",
    "#from: http://missingmigrants.iom.int/mediterranean\n",
    "\n",
    "deaths[2014][1] = int(12)\n",
    "deaths[2014][2] = int(24)\n",
    "deaths[2014][3] = int(10)\n",
    "deaths[2014][4] = int(50)\n",
    "deaths[2014][5] = int(329)\n",
    "deaths[2014][6] = int(318)\n",
    "deaths[2014][7] = int(864)\n",
    "deaths[2014][8] = int(616)\n",
    "deaths[2014][9] = int(813)\n",
    "deaths[2014][10] = int(126)\n",
    "deaths[2014][11] = int(22)\n",
    "deaths[2014][12] = int(95)\n",
    "deaths[2015][1] = int(82)\n",
    "deaths[2015][2] = int(345)\n",
    "deaths[2015][3] = int(61)\n",
    "deaths[2015][4] = int(1244)\n",
    "deaths[2015][5] = int(95)\n",
    "deaths[2015][6] = int(11)\n",
    "deaths[2015][7] = int(230)\n",
    "deaths[2015][8] = int(686)\n",
    "deaths[2015][9] = int(268)\n",
    "deaths[2015][10] = int(432)\n",
    "deaths[2015][11] = int(106)\n",
    "deaths[2015][12] = int(210)\n",
    "deaths[2016][1] = int(369)\n",
    "deaths[2016][2] = int(54)\n",
    "deaths[2016][3] = int(325)\n",
    "deaths[2016][4] = int(630)\n",
    "deaths[2016][5] = int(1138)\n",
    "deaths[2016][6] = int(389)\n",
    "deaths[2016][7] = int(129)\n",
    "deaths[2016][8] = int(0)\n",
    "deaths[2016][9] = int(0)\n",
    "deaths[2016][10] = int(0)\n",
    "deaths[2016][11] = int(0)\n",
    "deaths[2016][12] = int(0)\n",
    "\n",
    "print deaths\n",
    "\n",
    "with open('json_store/deaths.json', 'w') as outfile:\n",
    "    json.dump(deaths, outfile)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Process Monthly Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['Greece', 'Spain', 'Italy']\n"
     ]
    }
   ],
   "source": [
    "import json\n",
    "\n",
    "from os import listdir\n",
    "from os.path import isfile, join, isdir\n",
    "\n",
    "countries_root = 'json_store/by_country/'\n",
    "\n",
    "country_folders = [f for f in listdir(countries_root) if not isdir(f)]\n",
    "print country_folders\n",
    "\n",
    "#calculates amount of people who immigrated into a country from \n",
    "#a certain place within the month \n",
    "def immigrated_in_month_from(total_year, total_from_year, total_month):\n",
    "    percent_from = total_from_year/float(total_year)\n",
    "    return int(round(total_month*percent_from, 0))\n",
    "    \n",
    "\n",
    "\n",
    "# with open('json_store/monthly_by_country.json') as data_file:    \n",
    "#     monthly_data = json.load(data_file)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Greece\n",
      "Spain\n",
      "Italy\n",
      "{'Italy': {'yearly': {2016: 56227, 2014: 170100, 2015: 153842}, 'monthly': {2016: {1: 5273, 2: 3828, 3: 9676, 4: 9149, 5: 19925, 6: 8376}, 2014: {1: 2171, 2: 3335, 3: 5459, 4: 15679, 5: 14599, 6: 22641, 7: 24031, 8: 24774, 9: 26107, 10: 15277, 11: 9295, 12: 6732}, 2015: {1: 3528, 2: 4354, 3: 2283, 4: 16063, 5: 21235, 6: 22891, 7: 23186, 8: 22609, 9: 15922, 10: 8916, 11: 3218, 12: 9637}}, 'origins': {u'Ghana': 995, u'Ethiopia': 1012, u'Egypt': 1815, u'Tunisia': 320, u'Senegal': 2565, u'Somalia': 3450, u'Guinea-Bissau': 254, u'Mali': 2614, u'Cameroon': 777, u'Sierra Leone': 217, u'Benin': 183}}, 'Spain': {'yearly': {2016: 1352, 2014: 4348, 2015: 4408}, 'monthly': {2016: {1: 447, 2: 180, 3: 276, 4: 449, 5: 0, 6: 0}, 2014: {1: 144, 2: 33, 3: 232, 4: 148, 5: 325, 6: 246, 7: 264, 8: 1705, 9: 380, 10: 341, 11: 211, 12: 319}, 2015: {1: 241, 2: 44, 3: 267, 4: 240, 5: 438, 6: 377, 7: 346, 8: 385, 9: 459, 10: 795, 11: 508, 12: 308}}, 'origins': {u'Kenya': 4, u'Cameroon': 191, u'Uganda': 5, u'Guinea-Bissau': 26, u'Mali': 31}}, 'Greece': {'yearly': {2016: 157801, 2014: 41038, 2015: 856723}, 'monthly': {2016: {1: 67415, 2: 57066, 3: 26971, 4: 3650, 5: 1721, 6: 978}, 2014: {1: 955, 2: 1001, 3: 1501, 4: 1257, 5: 1703, 6: 3198, 7: 3927, 8: 6742, 9: 7454, 10: 7432, 11: 3812, 12: 2056}, 2015: {1: 1694, 2: 2873, 3: 7874, 4: 13556, 5: 17889, 6: 31318, 7: 54899, 8: 107843, 9: 147123, 10: 211663, 11: 151249, 12: 108742}}, 'origins': {u'Afghanistan': 39502, u'Syrian Arab Rep.': 75388, u'Iraq': 23994, u'Pakistan': 6299}}}\n"
     ]
    }
   ],
   "source": [
    "all_data = {}\n",
    "\n",
    "for country_folder in country_folders:\n",
    "    country_obj = {}\n",
    "    print country_folder\n",
    "    with open('{0}{1}/monthly{1}.json'.format(countries_root, country_folder)) as data_file: \n",
    "        data_f = json.load(data_file)\n",
    "        prevYear = data_f[0][\"year\"] \n",
    "        all_months = {}\n",
    "        monthly = {}\n",
    "        yearly = {}\n",
    "        for data in data_f:\n",
    "            yearly[data[\"year\"]] = yearly.get(data[\"year\"], 0) + data[\"value\"]\n",
    "            if data[\"year\"] == prevYear:\n",
    "                all_months[data[\"month\"]] = data[\"value\"]\n",
    "            else:\n",
    "                monthly[prevYear] = all_months\n",
    "                all_months = {}\n",
    "                all_months[data[\"month\"]] = data[\"value\"]\n",
    "            prevYear = data[\"year\"]\n",
    "        monthly[prevYear] = all_months\n",
    "        all_months[data[\"month\"]] = data[\"value\"]\n",
    "        country_obj[\"yearly\"] = yearly\n",
    "        country_obj[\"monthly\"] = monthly\n",
    "        \n",
    "    origin_files = [f for f in listdir(countries_root+country_folder) if 'from' in f]\n",
    "    origins = {}\n",
    "    for origin_file in origin_files:\n",
    "        with open('{0}{1}/{2}'.format(countries_root, country_folder, origin_file)) as data_file:\n",
    "                data = json.load(data_file)[0]\n",
    "                origins[data[\"country_of_origin_en\"]] = data[\"value\"]\n",
    "    country_obj[\"origins\"] = origins\n",
    "    all_data[country_folder] = country_obj\n",
    "\n",
    "all_data.pop(\"Malta\", None)\n",
    "\n",
    "print all_data\n",
    "\n",
    "with open('json_store/all_data.json', 'w') as outfile:\n",
    "    json.dump(all_data, outfile)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'deaths': {u'2015': {u'11': 106, u'10': 432, u'12': 210, u'1': 82, u'3': 61, u'2': 345, u'5': 95, u'4': 1244, u'7': 230, u'6': 11, u'9': 268, u'8': 686}, u'2014': {u'11': 22, u'10': 126, u'12': 95, u'1': 12, u'3': 10, u'2': 24, u'5': 329, u'4': 50, u'7': 864, u'6': 318, u'9': 813, u'8': 616}, u'2016': {u'11': 0, u'10': 0, u'12': 0, u'1': 369, u'3': 325, u'2': 54, u'5': 1138, u'4': 630, u'7': 129, u'6': 389, u'9': 0, u'8': 0}}, 'Italy': {2016: {1: {'Ghana': 14, 'Tunisia': 4, 'Egypt': 25, 'Ethiopia': 14, 'Senegal': 36, 'Somalia': 48, 'Guinea-Bissau': 4, 'Mali': 36, 'Cameroon': 11, 'Sierra Leone': 3, 'Benin': 3}, 2: {'Ghana': 10, 'Tunisia': 3, 'Egypt': 18, 'Ethiopia': 10, 'Senegal': 26, 'Somalia': 35, 'Guinea-Bissau': 3, 'Mali': 26, 'Cameroon': 8, 'Sierra Leone': 2, 'Benin': 2}, 3: {'Ghana': 25, 'Tunisia': 8, 'Egypt': 46, 'Ethiopia': 26, 'Senegal': 65, 'Somalia': 88, 'Guinea-Bissau': 6, 'Mali': 67, 'Cameroon': 20, 'Sierra Leone': 6, 'Benin': 5}, 4: {'Ghana': 24, 'Tunisia': 8, 'Egypt': 44, 'Ethiopia': 24, 'Senegal': 62, 'Somalia': 83, 'Guinea-Bissau': 6, 'Mali': 63, 'Cameroon': 19, 'Sierra Leone': 5, 'Benin': 4}, 5: {'Ghana': 52, 'Tunisia': 17, 'Egypt': 95, 'Ethiopia': 53, 'Senegal': 134, 'Somalia': 181, 'Guinea-Bissau': 13, 'Mali': 137, 'Cameroon': 41, 'Sierra Leone': 11, 'Benin': 10}, 6: {'Ghana': 22, 'Tunisia': 7, 'Egypt': 40, 'Ethiopia': 22, 'Senegal': 57, 'Somalia': 76, 'Guinea-Bissau': 6, 'Mali': 58, 'Cameroon': 17, 'Sierra Leone': 5, 'Benin': 4}}, 2014: {1: {'Ghana': 6, 'Tunisia': 2, 'Egypt': 10, 'Ethiopia': 6, 'Senegal': 15, 'Somalia': 20, 'Guinea-Bissau': 1, 'Mali': 15, 'Cameroon': 4, 'Sierra Leone': 1, 'Benin': 1}, 2: {'Ghana': 9, 'Tunisia': 3, 'Egypt': 16, 'Ethiopia': 9, 'Senegal': 23, 'Somalia': 30, 'Guinea-Bissau': 2, 'Mali': 23, 'Cameroon': 7, 'Sierra Leone': 2, 'Benin': 2}, 3: {'Ghana': 14, 'Tunisia': 5, 'Egypt': 26, 'Ethiopia': 15, 'Senegal': 37, 'Somalia': 50, 'Guinea-Bissau': 4, 'Mali': 38, 'Cameroon': 11, 'Sierra Leone': 3, 'Benin': 3}, 4: {'Ghana': 41, 'Tunisia': 13, 'Egypt': 75, 'Ethiopia': 42, 'Senegal': 106, 'Somalia': 142, 'Guinea-Bissau': 10, 'Mali': 108, 'Cameroon': 32, 'Sierra Leone': 9, 'Benin': 8}, 5: {'Ghana': 38, 'Tunisia': 12, 'Egypt': 70, 'Ethiopia': 39, 'Senegal': 98, 'Somalia': 132, 'Guinea-Bissau': 10, 'Mali': 100, 'Cameroon': 30, 'Sierra Leone': 8, 'Benin': 7}, 6: {'Ghana': 59, 'Tunisia': 19, 'Egypt': 108, 'Ethiopia': 60, 'Senegal': 153, 'Somalia': 205, 'Guinea-Bissau': 15, 'Mali': 156, 'Cameroon': 46, 'Sierra Leone': 13, 'Benin': 11}, 7: {'Ghana': 63, 'Tunisia': 20, 'Egypt': 115, 'Ethiopia': 64, 'Senegal': 162, 'Somalia': 218, 'Guinea-Bissau': 16, 'Mali': 165, 'Cameroon': 49, 'Sierra Leone': 14, 'Benin': 12}, 8: {'Ghana': 65, 'Tunisia': 21, 'Egypt': 118, 'Ethiopia': 66, 'Senegal': 167, 'Somalia': 225, 'Guinea-Bissau': 17, 'Mali': 170, 'Cameroon': 51, 'Sierra Leone': 14, 'Benin': 12}, 9: {'Ghana': 68, 'Tunisia': 22, 'Egypt': 125, 'Ethiopia': 69, 'Senegal': 176, 'Somalia': 237, 'Guinea-Bissau': 17, 'Mali': 180, 'Cameroon': 53, 'Sierra Leone': 15, 'Benin': 13}, 10: {'Ghana': 40, 'Tunisia': 13, 'Egypt': 73, 'Ethiopia': 41, 'Senegal': 103, 'Somalia': 139, 'Guinea-Bissau': 10, 'Mali': 105, 'Cameroon': 31, 'Sierra Leone': 9, 'Benin': 7}, 11: {'Ghana': 24, 'Tunisia': 8, 'Egypt': 44, 'Ethiopia': 25, 'Senegal': 63, 'Somalia': 84, 'Guinea-Bissau': 6, 'Mali': 64, 'Cameroon': 19, 'Sierra Leone': 5, 'Benin': 4}, 12: {'Ghana': 18, 'Tunisia': 6, 'Egypt': 32, 'Ethiopia': 18, 'Senegal': 45, 'Somalia': 61, 'Guinea-Bissau': 4, 'Mali': 46, 'Cameroon': 14, 'Sierra Leone': 4, 'Benin': 3}}, 2015: {1: {'Ghana': 9, 'Tunisia': 3, 'Egypt': 17, 'Ethiopia': 9, 'Senegal': 24, 'Somalia': 32, 'Guinea-Bissau': 2, 'Mali': 24, 'Cameroon': 7, 'Sierra Leone': 2, 'Benin': 2}, 2: {'Ghana': 11, 'Tunisia': 4, 'Egypt': 21, 'Ethiopia': 12, 'Senegal': 29, 'Somalia': 40, 'Guinea-Bissau': 3, 'Mali': 30, 'Cameroon': 9, 'Sierra Leone': 2, 'Benin': 2}, 3: {'Ghana': 6, 'Tunisia': 2, 'Egypt': 11, 'Ethiopia': 6, 'Senegal': 15, 'Somalia': 21, 'Guinea-Bissau': 2, 'Mali': 16, 'Cameroon': 5, 'Sierra Leone': 1, 'Benin': 1}, 4: {'Ghana': 42, 'Tunisia': 14, 'Egypt': 77, 'Ethiopia': 43, 'Senegal': 108, 'Somalia': 146, 'Guinea-Bissau': 11, 'Mali': 110, 'Cameroon': 33, 'Sierra Leone': 9, 'Benin': 8}, 5: {'Ghana': 56, 'Tunisia': 18, 'Egypt': 101, 'Ethiopia': 57, 'Senegal': 143, 'Somalia': 193, 'Guinea-Bissau': 14, 'Mali': 146, 'Cameroon': 43, 'Sierra Leone': 12, 'Benin': 10}, 6: {'Ghana': 60, 'Tunisia': 19, 'Egypt': 109, 'Ethiopia': 61, 'Senegal': 154, 'Somalia': 208, 'Guinea-Bissau': 15, 'Mali': 157, 'Cameroon': 47, 'Sierra Leone': 13, 'Benin': 11}, 7: {'Ghana': 61, 'Tunisia': 20, 'Egypt': 111, 'Ethiopia': 62, 'Senegal': 156, 'Somalia': 210, 'Guinea-Bissau': 15, 'Mali': 159, 'Cameroon': 47, 'Sierra Leone': 13, 'Benin': 11}, 8: {'Ghana': 59, 'Tunisia': 19, 'Egypt': 108, 'Ethiopia': 60, 'Senegal': 153, 'Somalia': 205, 'Guinea-Bissau': 15, 'Mali': 155, 'Cameroon': 46, 'Sierra Leone': 13, 'Benin': 11}, 9: {'Ghana': 42, 'Tunisia': 13, 'Egypt': 76, 'Ethiopia': 42, 'Senegal': 107, 'Somalia': 144, 'Guinea-Bissau': 11, 'Mali': 109, 'Cameroon': 33, 'Sierra Leone': 9, 'Benin': 8}, 10: {'Ghana': 23, 'Tunisia': 8, 'Egypt': 43, 'Ethiopia': 24, 'Senegal': 60, 'Somalia': 81, 'Guinea-Bissau': 6, 'Mali': 61, 'Cameroon': 18, 'Sierra Leone': 5, 'Benin': 4}, 11: {'Ghana': 8, 'Tunisia': 3, 'Egypt': 15, 'Ethiopia': 9, 'Senegal': 22, 'Somalia': 29, 'Guinea-Bissau': 2, 'Mali': 22, 'Cameroon': 7, 'Sierra Leone': 2, 'Benin': 2}, 12: {'Ghana': 25, 'Tunisia': 8, 'Egypt': 46, 'Ethiopia': 26, 'Senegal': 65, 'Somalia': 87, 'Guinea-Bissau': 6, 'Mali': 66, 'Cameroon': 20, 'Sierra Leone': 6, 'Benin': 5}}}, 'Spain': {2016: {1: {'Kenya': 0, 'Cameroon': 8, 'Uganda': 0, 'Guinea-Bissau': 1, 'Mali': 1}, 2: {'Kenya': 0, 'Cameroon': 3, 'Uganda': 0, 'Guinea-Bissau': 0, 'Mali': 1}, 3: {'Kenya': 0, 'Cameroon': 5, 'Uganda': 0, 'Guinea-Bissau': 1, 'Mali': 1}, 4: {'Kenya': 0, 'Cameroon': 8, 'Uganda': 0, 'Guinea-Bissau': 1, 'Mali': 1}, 5: {'Kenya': 0, 'Cameroon': 0, 'Uganda': 0, 'Guinea-Bissau': 0, 'Mali': 0}, 6: {'Kenya': 0, 'Cameroon': 0, 'Uganda': 0, 'Guinea-Bissau': 0, 'Mali': 0}}, 2014: {1: {'Kenya': 0, 'Cameroon': 3, 'Uganda': 0, 'Guinea-Bissau': 0, 'Mali': 0}, 2: {'Kenya': 0, 'Cameroon': 1, 'Uganda': 0, 'Guinea-Bissau': 0, 'Mali': 0}, 3: {'Kenya': 0, 'Cameroon': 4, 'Uganda': 0, 'Guinea-Bissau': 1, 'Mali': 1}, 4: {'Kenya': 0, 'Cameroon': 3, 'Uganda': 0, 'Guinea-Bissau': 0, 'Mali': 0}, 5: {'Kenya': 0, 'Cameroon': 6, 'Uganda': 0, 'Guinea-Bissau': 1, 'Mali': 1}, 6: {'Kenya': 0, 'Cameroon': 5, 'Uganda': 0, 'Guinea-Bissau': 1, 'Mali': 1}, 7: {'Kenya': 0, 'Cameroon': 5, 'Uganda': 0, 'Guinea-Bissau': 1, 'Mali': 1}, 8: {'Kenya': 1, 'Cameroon': 32, 'Uganda': 1, 'Guinea-Bissau': 4, 'Mali': 5}, 9: {'Kenya': 0, 'Cameroon': 7, 'Uganda': 0, 'Guinea-Bissau': 1, 'Mali': 1}, 10: {'Kenya': 0, 'Cameroon': 6, 'Uganda': 0, 'Guinea-Bissau': 1, 'Mali': 1}, 11: {'Kenya': 0, 'Cameroon': 4, 'Uganda': 0, 'Guinea-Bissau': 1, 'Mali': 1}, 12: {'Kenya': 0, 'Cameroon': 6, 'Uganda': 0, 'Guinea-Bissau': 1, 'Mali': 1}}, 2015: {1: {'Kenya': 0, 'Cameroon': 5, 'Uganda': 0, 'Guinea-Bissau': 1, 'Mali': 1}, 2: {'Kenya': 0, 'Cameroon': 1, 'Uganda': 0, 'Guinea-Bissau': 0, 'Mali': 0}, 3: {'Kenya': 0, 'Cameroon': 5, 'Uganda': 0, 'Guinea-Bissau': 1, 'Mali': 1}, 4: {'Kenya': 0, 'Cameroon': 5, 'Uganda': 0, 'Guinea-Bissau': 1, 'Mali': 1}, 5: {'Kenya': 0, 'Cameroon': 8, 'Uganda': 0, 'Guinea-Bissau': 1, 'Mali': 1}, 6: {'Kenya': 0, 'Cameroon': 7, 'Uganda': 0, 'Guinea-Bissau': 1, 'Mali': 1}, 7: {'Kenya': 0, 'Cameroon': 7, 'Uganda': 0, 'Guinea-Bissau': 1, 'Mali': 1}, 8: {'Kenya': 0, 'Cameroon': 7, 'Uganda': 0, 'Guinea-Bissau': 1, 'Mali': 1}, 9: {'Kenya': 0, 'Cameroon': 9, 'Uganda': 0, 'Guinea-Bissau': 1, 'Mali': 1}, 10: {'Kenya': 0, 'Cameroon': 15, 'Uganda': 0, 'Guinea-Bissau': 2, 'Mali': 2}, 11: {'Kenya': 0, 'Cameroon': 10, 'Uganda': 0, 'Guinea-Bissau': 1, 'Mali': 2}, 12: {'Kenya': 0, 'Cameroon': 6, 'Uganda': 0, 'Guinea-Bissau': 1, 'Mali': 1}}}, 'Greece': {2016: {1: {'Afghanistan': 2523, 'Syrian Arab Rep.': 4815, 'Iraq': 1532, 'Pakistan': 402}, 2: {'Afghanistan': 2136, 'Syrian Arab Rep.': 4076, 'Iraq': 1297, 'Pakistan': 341}, 3: {'Afghanistan': 1009, 'Syrian Arab Rep.': 1926, 'Iraq': 613, 'Pakistan': 161}, 4: {'Afghanistan': 137, 'Syrian Arab Rep.': 261, 'Iraq': 83, 'Pakistan': 22}, 5: {'Afghanistan': 64, 'Syrian Arab Rep.': 123, 'Iraq': 39, 'Pakistan': 10}, 6: {'Afghanistan': 37, 'Syrian Arab Rep.': 70, 'Iraq': 22, 'Pakistan': 6}}, 2014: {1: {'Afghanistan': 36, 'Syrian Arab Rep.': 68, 'Iraq': 22, 'Pakistan': 6}, 2: {'Afghanistan': 37, 'Syrian Arab Rep.': 71, 'Iraq': 23, 'Pakistan': 6}, 3: {'Afghanistan': 56, 'Syrian Arab Rep.': 107, 'Iraq': 34, 'Pakistan': 9}, 4: {'Afghanistan': 47, 'Syrian Arab Rep.': 90, 'Iraq': 29, 'Pakistan': 8}, 5: {'Afghanistan': 64, 'Syrian Arab Rep.': 122, 'Iraq': 39, 'Pakistan': 10}, 6: {'Afghanistan': 120, 'Syrian Arab Rep.': 228, 'Iraq': 73, 'Pakistan': 19}, 7: {'Afghanistan': 147, 'Syrian Arab Rep.': 280, 'Iraq': 89, 'Pakistan': 23}, 8: {'Afghanistan': 252, 'Syrian Arab Rep.': 482, 'Iraq': 153, 'Pakistan': 40}, 9: {'Afghanistan': 279, 'Syrian Arab Rep.': 532, 'Iraq': 169, 'Pakistan': 44}, 10: {'Afghanistan': 278, 'Syrian Arab Rep.': 531, 'Iraq': 169, 'Pakistan': 44}, 11: {'Afghanistan': 143, 'Syrian Arab Rep.': 272, 'Iraq': 87, 'Pakistan': 23}, 12: {'Afghanistan': 77, 'Syrian Arab Rep.': 147, 'Iraq': 47, 'Pakistan': 12}}, 2015: {1: {'Afghanistan': 63, 'Syrian Arab Rep.': 121, 'Iraq': 39, 'Pakistan': 10}, 2: {'Afghanistan': 108, 'Syrian Arab Rep.': 205, 'Iraq': 65, 'Pakistan': 17}, 3: {'Afghanistan': 295, 'Syrian Arab Rep.': 562, 'Iraq': 179, 'Pakistan': 47}, 4: {'Afghanistan': 507, 'Syrian Arab Rep.': 968, 'Iraq': 308, 'Pakistan': 81}, 5: {'Afghanistan': 669, 'Syrian Arab Rep.': 1278, 'Iraq': 407, 'Pakistan': 107}, 6: {'Afghanistan': 1172, 'Syrian Arab Rep.': 2237, 'Iraq': 712, 'Pakistan': 187}, 7: {'Afghanistan': 2054, 'Syrian Arab Rep.': 3921, 'Iraq': 1248, 'Pakistan': 328}, 8: {'Afghanistan': 4036, 'Syrian Arab Rep.': 7702, 'Iraq': 2451, 'Pakistan': 644}, 9: {'Afghanistan': 5506, 'Syrian Arab Rep.': 10507, 'Iraq': 3344, 'Pakistan': 878}, 10: {'Afghanistan': 7921, 'Syrian Arab Rep.': 15117, 'Iraq': 4811, 'Pakistan': 1263}, 11: {'Afghanistan': 5660, 'Syrian Arab Rep.': 10802, 'Iraq': 3438, 'Pakistan': 903}, 12: {'Afghanistan': 4069, 'Syrian Arab Rep.': 7766, 'Iraq': 2472, 'Pakistan': 649}}}}\n"
     ]
    }
   ],
   "source": [
    "processed_data = {}\n",
    "with open('json_store/all_data.json') as data_file: \n",
    "    data = json.load(data_file)\n",
    "    proc_country_data = {}\n",
    "    for country_code in country_folders:\n",
    "        country_data = data[country_code]\n",
    "        origin_data = country_data[\"origins\"]\n",
    "        years_data = {}\n",
    "        for i in range(0, 3):\n",
    "            year = unicode(i + 2014)\n",
    "            year_data = {}\n",
    "            yearsum = 0;\n",
    "            if \"yearly\" in country_data:\n",
    "                for year_key, year_val in country_data[\"yearly\"].iteritems():\n",
    "                    yearsum += year_val\n",
    "                months_data = {}\n",
    "                for month_key, month_val in sorted(country_data[\"monthly\"][year].iteritems()):\n",
    "                    origin_data = {}\n",
    "                    for origin, origin_val in country_data[\"origins\"].iteritems():\n",
    "                        origin_data[str(origin)] = immigrated_in_month_from(yearsum, origin_val, month_val)\n",
    "                    months_data[int(month_key)] = origin_data\n",
    "                years_data[int(year)] = months_data\n",
    "        proc_country_data[country_code] = years_data\n",
    "processed_data = proc_country_data\n",
    "\n",
    "with open('json_store/deaths.json') as data_file:            \n",
    "    processed_data[\"deaths\"] = json.load(data_file)\n",
    "\n",
    "print processed_data\n",
    "\n",
    "with open('json_store/processed_data.json', 'w') as outfile:\n",
    "    json.dump(processed_data, outfile)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Process Geo Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Invalid: U.S. Minor Outlying Islands, , \n"
     ]
    }
   ],
   "source": [
    "import json, csv\n",
    "\n",
    "with open('geo_data/country_list.json') as data_file:    \n",
    "    countries = json.load(data_file)\n",
    "\n",
    "with open('geo_data/country_lat_long.csv', 'r') as csvfile:\n",
    "    country_lat_lon = {}\n",
    "    fieldnames = ['name', 'lat', 'lon']\n",
    "    reader = csv.DictReader(csvfile, fieldnames=fieldnames)\n",
    "    for row in reader:\n",
    "        try:\n",
    "            lat_lon = {}\n",
    "            lat_lon[\"lat\"] = float(row[\"lat\"])\n",
    "            lat_lon[\"lon\"] = float(row[\"lon\"])\n",
    "            country_lat_lon[str(row[\"name\"])] = lat_lon\n",
    "        except ValueError:\n",
    "            print \"Invalid: {0}, {1}, {2}\".format(row[\"name\"],row[\"lat\"],row[\"lon\"])\n",
    "\n",
    "def get_country_code(name):\n",
    "    for country in countries:\n",
    "        if name.lower() == country[\"name_en\"].lower():\n",
    "            return country[\"country_code\"]\n",
    "    return None\n",
    "\n",
    "def get_country_name(code):\n",
    "    for country in countries:\n",
    "        if code.lower() == country[\"country_code\"].lower():\n",
    "            return country[\"name_en\"]\n",
    "    return None\n",
    "\n",
    "def get_all_country_codes():\n",
    "    return [country[\"country_code\"] for country in countries]\n",
    "\n",
    "def get_lat_lon_by_name(name):\n",
    "    return country_lat_lon[name]\n",
    "\n",
    "def get_lat_lon_by_code(code):\n",
    "    return get_lat_lon_by_name(get_country_name(code))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## defines lat long for each country and paths"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'origin_geo': {'Ghana': {'lat': 7.946527, 'lon': -1.023194}, 'Afghanistan': {'lat': 33.93911, 'lon': 67.709953}, 'Uganda': {'lat': 1.373333, 'lon': 32.290275}, 'Tunisia': {'lat': 33.886917, 'lon': 9.537499}, 'Egypt': {'lat': 26.820553, 'lon': 30.802498}, 'Syrian Arab Rep.': {'lat': 34.802075, 'lon': 38.996815}, 'Ethiopia': {'lat': 9.145, 'lon': 40.489673}, 'Senegal': {'lat': 14.497401, 'lon': -14.452362}, 'Pakistan': {'lat': 30.375321, 'lon': 69.345116}, 'Somalia': {'lat': 5.152149, 'lon': 46.199616}, 'Guinea-Bissau': {'lat': 11.803749, 'lon': -15.180413}, 'Mali': {'lat': 17.570692, 'lon': -3.996166}, 'Kenya': {'lat': -0.023559, 'lon': 37.906193}, 'Iraq': {'lat': 33.223191, 'lon': 43.679291}, 'Cameroon': {'lat': 7.369722, 'lon': 12.354722}, 'Sierra Leone': {'lat': 8.460555, 'lon': -11.779889}, 'Benin': {'lat': 9.30769, 'lon': 2.315834}}, 'asylum_geo': {'Italy': {'lat': 41.87194, 'lon': 12.56738}, 'Spain': {'lat': 40.463667, 'lon': -3.74922}, 'Greece': {'lat': 39.074208, 'lon': 21.824312}}, 'path_geo': {'Ghana': {'Italy': [{'lat': 7.946527, 'lon': -1.023194}, {'lat': 17.607789, 'lon': 8.081666}, {'lat': 26.3351, 'lon': 17.228331}, {'lat': 41.87194, 'lon': 12.56738}]}, 'Afghanistan': {'Greece': [{'lat': 33.93911, 'lon': 67.709953}, {'lat': 32.427908, 'lon': 53.688046}, {'lat': 38.963745, 'lon': 35.243322}, {'lat': 39.074208, 'lon': 21.824312}]}, 'Uganda': {'Italy': [{'lat': 9.145, 'lon': 40.489673}, {'lat': 12.862807, 'lon': 30.217636}, {'lat': 26.3351, 'lon': 17.228331}, {'lat': 41.87194, 'lon': 12.56738}], 'Spain': [{'lat': 1.373333, 'lon': 32.290275}, {'lat': 12.862807, 'lon': 30.217636}, {'lat': 26.3351, 'lon': 17.228331}, {'lat': 28.033886, 'lon': 1.659626}, {'lat': 31.791702, 'lon': -7.09262}, {'lat': 40.463667, 'lon': -3.74922}]}, 'Pakistan': {'Italy': [{'lat': 30.375321, 'lon': 69.345116}, {'lat': 32.427908, 'lon': 53.688046}, {'lat': 38.963745, 'lon': 35.243322}, {'lat': 42.733883, 'lon': 25.48583}, {'lat': 47.516231, 'lon': 14.550072}, {'lat': 41.87194, 'lon': 12.56738}], 'Greece': [{'lat': 30.375321, 'lon': 69.345116}, {'lat': 32.427908, 'lon': 53.688046}, {'lat': 38.963745, 'lon': 35.243322}, {'lat': 39.074208, 'lon': 21.824312}]}, 'Tunisia': {'Italy': [{'lat': 33.886917, 'lon': 9.537499}, {'lat': 35.937496, 'lon': 14.375416}, {'lat': 41.87194, 'lon': 12.56738}]}, 'United Arab Emirates': {'Italy': [{'lat': 23.424076, 'lon': 53.847818}, {'lat': 23.885942, 'lon': 45.079162}, {'lat': 30.585164, 'lon': 36.238414}, {'lat': 26.820553, 'lon': 30.802498}, {'lat': 26.3351, 'lon': 17.228331}, {'lat': 41.87194, 'lon': 12.56738}]}, 'Syrian Arab Rep.': {'Greece': [{'lat': 34.802075, 'lon': 38.996815}, {'lat': 38.963745, 'lon': 35.243322}, {'lat': 39.074208, 'lon': 21.824312}]}, 'Senegal': {'Italy': [{'lat': 14.497401, 'lon': -14.452362}, {'lat': 17.607789, 'lon': 8.081666}, {'lat': 26.3351, 'lon': 17.228331}, {'lat': 41.87194, 'lon': 12.56738}], 'Spain': [{'lat': 14.497401, 'lon': -14.452362}, {'lat': 17.570692, 'lon': -3.996166}, {'lat': 28.033886, 'lon': 1.659626}, {'lat': 31.791702, 'lon': -7.09262}, {'lat': 40.463667, 'lon': -3.74922}]}, 'Ethiopia': {'Italy': [{'lat': 9.145, 'lon': 40.489673}, {'lat': 12.862807, 'lon': 30.217636}, {'lat': 26.3351, 'lon': 17.228331}, {'lat': 41.87194, 'lon': 12.56738}]}, 'Syrian Arab Republic': {'Greece': [{'lat': 34.802075, 'lon': 38.996815}, {'lat': 38.963745, 'lon': 35.243322}, {'lat': 39.074208, 'lon': 21.824312}]}, 'Mozambique': {'Spain': [{'lat': -18.665695, 'lon': 35.529562}, {'lat': -6.369028, 'lon': 34.888822}, {'lat': -0.023559, 'lon': 37.906193}, {'lat': 9.145, 'lon': 40.489673}, {'lat': 12.862807, 'lon': 30.217636}, {'lat': 26.3351, 'lon': 17.228331}, {'lat': 28.033886, 'lon': 1.659626}, {'lat': 31.791702, 'lon': -7.09262}, {'lat': 40.463667, 'lon': -3.74922}]}, 'Somalia': {'Italy': [{'lat': 5.152149, 'lon': 46.199616}, {'lat': 9.145, 'lon': 40.489673}, {'lat': 12.862807, 'lon': 30.217636}, {'lat': 26.3351, 'lon': 17.228331}, {'lat': 41.87194, 'lon': 12.56738}]}, 'Guinea-Bissau': {'Italy': [{'lat': 11.803749, 'lon': -15.180413}, {'lat': 17.570692, 'lon': -3.996166}, {'lat': 17.607789, 'lon': 8.081666}, {'lat': 26.3351, 'lon': 17.228331}, {'lat': 41.87194, 'lon': 12.56738}], 'Spain': [{'lat': 11.803749, 'lon': -15.180413}, {'lat': 17.570692, 'lon': -3.996166}, {'lat': 28.033886, 'lon': 1.659626}, {'lat': 31.791702, 'lon': -7.09262}, {'lat': 40.463667, 'lon': -3.74922}]}, 'Mali': {'Italy': [{'lat': 17.570692, 'lon': -3.996166}, {'lat': 17.607789, 'lon': 8.081666}, {'lat': 26.3351, 'lon': 17.228331}, {'lat': 41.87194, 'lon': 12.56738}], 'Spain': [{'lat': 17.570692, 'lon': -3.996166}, {'lat': 28.033886, 'lon': 1.659626}, {'lat': 31.791702, 'lon': -7.09262}, {'lat': 40.463667, 'lon': -3.74922}]}, 'Kenya': {'Spain': [{'lat': -0.023559, 'lon': 37.906193}, {'lat': 12.862807, 'lon': 30.217636}, {'lat': 26.3351, 'lon': 17.228331}, {'lat': 28.033886, 'lon': 1.659626}, {'lat': 31.791702, 'lon': -7.09262}, {'lat': 40.463667, 'lon': -3.74922}]}, 'Iraq': {'Greece': [{'lat': 33.223191, 'lon': 43.679291}, {'lat': 38.963745, 'lon': 35.243322}, {'lat': 39.074208, 'lon': 21.824312}]}, 'Cameroon': {'Italy': [{'lat': 7.369722, 'lon': 12.354722}, {'lat': 17.607789, 'lon': 8.081666}, {'lat': 26.3351, 'lon': 17.228331}, {'lat': 41.87194, 'lon': 12.56738}], 'Spain': [{'lat': 7.369722, 'lon': 12.354722}, {'lat': 17.607789, 'lon': 8.081666}, {'lat': 28.033886, 'lon': 1.659626}, {'lat': 31.791702, 'lon': -7.09262}, {'lat': 40.463667, 'lon': -3.74922}]}, 'Sierra Leone': {'Italy': [{'lat': 8.460555, 'lon': -11.779889}, {'lat': 17.570692, 'lon': -3.996166}, {'lat': 31.791702, 'lon': -7.09262}, {'lat': 40.463667, 'lon': -3.74922}]}, 'Egypt': {'Italy': [{'lat': 26.820553, 'lon': 30.802498}, {'lat': 26.3351, 'lon': 17.228331}, {'lat': 41.87194, 'lon': 12.56738}]}, 'Benin': {'Italy': [{'lat': 9.30769, 'lon': 2.315834}, {'lat': 17.607789, 'lon': 8.081666}, {'lat': 26.3351, 'lon': 17.228331}, {'lat': 41.87194, 'lon': 12.56738}]}}}\n"
     ]
    }
   ],
   "source": [
    "geo_data = {}\n",
    "path_geo = {}\n",
    "with open('json_store/all_data.json') as data_file:\n",
    "    data = json.load(data_file)\n",
    "    asyl_geo = {}\n",
    "    origin_geo = {}\n",
    "    for key in data:\n",
    "        asyl_geo[str(key)] = get_lat_lon_by_name(key)\n",
    "        for or_key in data[key][\"origins\"]:\n",
    "            origin_geo[str(or_key)] = get_lat_lon_by_name(or_key)\n",
    "            if not (key in path_geo): \n",
    "                path_geo[key] = {}\n",
    "            if not (or_key in path_geo[key]):\n",
    "                path_geo[key][or_key] = []\n",
    "geo_data['asylum_geo'] = asyl_geo\n",
    "geo_data['origin_geo'] = origin_geo\n",
    "\n",
    "path_geo = {}\n",
    "with open(\"geo_data/path_by_country.json\") as data_file:\n",
    "    data = json.load(data_file);\n",
    "    for or_key in data:        \n",
    "        path_geo[str(or_key)] = {}\n",
    "        for asyl_key in data[or_key]:\n",
    "            path_geo[or_key][str(asyl_key)] = [get_lat_lon_by_name(str(country)) for country in data[or_key][asyl_key]]\n",
    "\n",
    "geo_data[\"path_geo\"] = path_geo\n",
    "\n",
    "print geo_data\n",
    "\n",
    "with open('geo_data/geo_data.json', 'w') as outfile:\n",
    "    json.dump(geo_data, outfile)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Generate Particle Binary File"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "import array, datetime, math, random, chroma, json, calendar\n",
    "\n",
    "def get_path_through_points(points):\n",
    "    curves = []\n",
    "    currentCursor = points[0]\n",
    "    i = 0\n",
    "    for i in range(1, len(points) - 2):\n",
    "        cntrlPoint = get_control_point(points[i], points[i + 1])\n",
    "        curves.append([currentCursor, points[i], cntrlPoint]);\n",
    "        currentCursor = cntrlPoint;\n",
    "    i += 1\n",
    "    curves.append([currentCursor, points[i], points[i + 1]]);\n",
    "    return curves\n",
    "        \n",
    "def get_control_point(start, end):\n",
    "    return [(start[0] + end[0]) / 2, (start[1] + end[1]) / 2]\n",
    "\n",
    "def bezier_curve_length(curve):\n",
    "    start = {\n",
    "        'x': curve[0][0], \n",
    "        'y': curve[0][1]\n",
    "    }\n",
    "    control = {\n",
    "        'x': curve[1][0], \n",
    "        'y': curve[1][1]\n",
    "    }\n",
    "    end = {\n",
    "        'x': curve[2][0], \n",
    "        'y': curve[2][1]\n",
    "    }\n",
    "    \n",
    "    a = {'x':start['x'] - 2 * control['x'] + end['x'],\n",
    "         'y':start['y'] - 2 * control['y'] + end['y']}\n",
    "    \n",
    "    b = {'x': 2 * control['x'] - 2 * start['x'],\n",
    "         'y': 2 * control['y'] - 2 * start['y']}\n",
    "    \n",
    "    A = 4 * (a['x'] * a['x'] + a['y'] * a['y'])\n",
    "    B = 4 * (a['x'] * b['x'] + a['y'] * b['y'])\n",
    "    C = b['x'] * b['x'] + b['y'] * b['y']\n",
    "    \n",
    "    Sabc = 2 * math.sqrt(A + B + C);\n",
    "    A_2 = math.sqrt(A);\n",
    "    A_32 = 2 * A * A_2;\n",
    "    C_2 = 2 * math.sqrt(C)\n",
    "    BA = B / A_2\n",
    "    \n",
    "    return (A_32 * Sabc + A_2 * B * (Sabc - C_2) +\n",
    "        (4 * C * A - B * B) * math.log((2 * A_2 + BA + Sabc) / (BA + C_2))) / (4 * A_32);\n",
    "\n",
    "\n",
    "def LonLatToPixelXY(lon, lat):\n",
    "    x = (lon + 180) / 360 * 256\n",
    "    y = ((1 - math.log(math.tan(lat * math.pi / 180) + 1 / math.cos(lat * math.pi / 180)) / math.pi) / 2 * math.pow(2, 0)) * 256\n",
    "    return [x, y]\n",
    "\n",
    "def YearMonthDayHourToEpoch(year, month, day, hour):\n",
    "    return (datetime.datetime(int(year), int(month), int(day), int(hour)) - datetime.datetime(1970, 1, 1, 0)).total_seconds()\n",
    "\n",
    "def random_day_hour(year, month):\n",
    "    hour = random.randint(0,23)\n",
    "    if month == 2:\n",
    "        if not calendar.isleap(year):\n",
    "            day = random.randint(1, 28)\n",
    "        else:\n",
    "            day = random.randint(1, 29)\n",
    "    elif month == 4 or month == 6 or month == 9 or month == 11:\n",
    "        day = random.randint(1, 30)\n",
    "    else:\n",
    "        day = random.randint(1,31)\n",
    "    return (day, hour)\n",
    "\n",
    "def get_points_perpindicular(x1, y1, x2, y2, offset):\n",
    "    k = offset/(math.sqrt(((y1 - y2)**2) + ((x2 - x1)**2))) \n",
    "    return ([x1 + (y1-y2)*k, y1 + (x2-x1)*k], [x2 + (y1-y2)*k, y2 + (x2-x1)*k])\n",
    "\n",
    "def get_midpoint(x1, y1, x2, y2):\n",
    "    return ((x1+x2)/2, (y1+y2)/2)\n",
    "\n",
    "def get_slope(x1, y1, x2, y2):\n",
    "    return (y2-y1)/(x2-x1)\n",
    "\n",
    "def get_intersection(x1,y1,x2,y2,x3,y3,x4,y4):\n",
    "    m1 = get_slope(x1,y1,x2,y2)\n",
    "    m2 = get_slope(x3,y3,x4,y4)\n",
    "    x = (m1*x2-y2-m2*x3+y3)/(m1-m2)\n",
    "    y = m1*x-m1*x2+y2\n",
    "    return [x,y]\n",
    "    \n",
    "def quad_to_cubic(startPoint, controlPoint, endPoint):\n",
    "    cP1 = [startPoint[i] + (2/3.)*(controlPoint[i]-startPoint[i]) for i in range(2)]\n",
    "    cP2 = [endPoint[i] + (2/3.)*(controlPoint[i]-endPoint[i]) for i in range(2)]\n",
    "    return (startPoint, cP1, cP2, endPoint)\n",
    "\n",
    "def cubic_to_quad(startPoint, controlPoint1, controlPoint2, endPoint):\n",
    "    cP = [-0.25*startPoint[i] + .75*controlPoint1[i] + .75*controlPoint2[i] -0.25*endPoint[i] for i in range(2)]\n",
    "    return (startPoint, cP, endPoint)\n",
    "\n",
    "def offsetCubic(startPoint, controlPoint1, controlPoint2, endPoint, offset):\n",
    "    offStart, point1 = get_points_perpindicular(startPoint[0], startPoint[1], controlPoint1[0], controlPoint1[1], offset)\n",
    "    point4,offEnd = get_points_perpindicular(controlPoint2[0], controlPoint2[1], endPoint[0], endPoint[1], offset)\n",
    "    point2, point3 = get_points_perpindicular(controlPoint1[0], controlPoint1[1], controlPoint2[0], controlPoint2[1], offset)\n",
    "    offCP1 = get_intersection(offStart[0], offStart[1], point1[0], point1[1], point2[0], point2[1], point3[0], point3[1])\n",
    "    offCP2 = get_intersection(point2[0], point2[1], point3[0], point3[1], point4[0], point4[1], offEnd[0], offEnd[1])\n",
    "    return (offStart, offCP1, offCP2, offEnd)\n",
    "\n",
    "def get_color_range(color, leftVal, rightVal):\n",
    "    colors = []\n",
    "    leftVal = .2\n",
    "    rightVal = .9\n",
    "    hsl = color.hls\n",
    "    i = leftVal\n",
    "    while i < rightVal:\n",
    "        new_color = chroma.Color((hsl[0], hsl[1], i), 'HLS');\n",
    "        colors += new_color.rgb\n",
    "        i += .01\n",
    "    \n",
    "    i = rightVal\n",
    "    while i > leftVal:\n",
    "        new_color = chroma.Color((hsl[0], hsl[1], i), 'HLS');\n",
    "        colors += new_color.rgb\n",
    "        i -= .01\n",
    "    return colors\n",
    "\n",
    "class Particle:\n",
    "    'A particle within a given Bezier curve'\n",
    "    def __init__(self, travelTime, startPoint, controlPoint, endPoint):\n",
    "        self.travelTime = travelTime\n",
    "        self.startPoint = startPoint\n",
    "        self.controlPoint = controlPoint\n",
    "        self.endPoint = endPoint\n",
    "        \n",
    "    def addToBuff(self, startTime, endTime, offset, time_offset, isStart, isEnd, buff):\n",
    "        cubic = quad_to_cubic(self.startPoint, self.controlPoint, self.endPoint)\n",
    "        offStart, offCP1, offCP2, offEnd = offsetCubic(cubic[0], cubic[1], cubic[2], cubic[3], offset)\n",
    "        start, control, end = cubic_to_quad(offStart, offCP1, offCP2, offEnd)\n",
    "        buff += start if not isStart else self.startPoint \n",
    "        buff += end if not isEnd else self.endPoint\n",
    "        buff += control\n",
    "        buff.append(startTime)\n",
    "        buff.append(endTime)\n",
    "        buff.append(self.travelTime)\n",
    "        buff.append(time_offset)\n",
    "    \n",
    "    def __str__(self):\n",
    "        return \"{0},{1},{2},{3}\".format(self.startPoint, self.controlPoint, self.endPoint, self.travelTime)\n",
    "        \n",
    "class ParticleGenerator:\n",
    "    'Generates particles for a given month to add to the buffer'\n",
    "    def __init__(self, year, month, points, amount, time_to_complete, buff):\n",
    "        self.startTime = YearMonthDayHourToEpoch(year, month, 1, 0)\n",
    "        self.month = month\n",
    "        self.year = year\n",
    "        self.points = points\n",
    "        self.amount = amount\n",
    "        self.time_to_complete = time_to_complete\n",
    "        self.buff = buff\n",
    "    \n",
    "    def generate_particles(self):\n",
    "        curves = get_path_through_points(self.points)\n",
    "        lengths = [bezier_curve_length(curve) for curve in curves]\n",
    "        length_sum = sum(lengths)\n",
    "        time_lengths = [self.time_to_complete*(length/length_sum) for length in lengths]\n",
    "        particleset = [Particle(time_lengths[i], curves[i][0], curves[i][1], curves[i][2]) for i in range(len(curves))]\n",
    "        offsetChoices = [random.uniform(-1,1)*.065*math.sqrt(self.amount) for i in range(int(math.sqrt(self.amount)))]\n",
    "        for i in range(self.amount):\n",
    "            if i % 20 == 0:\n",
    "                offset = 0\n",
    "                time_offset = 0\n",
    "            else:\n",
    "                offset = random.choice(offsetChoices);\n",
    "                time_offset = random.uniform(-1,1)/100\n",
    "            \n",
    "            span = self.time_to_complete if not self.year==2014 or not self.month==1 else -1 \n",
    "            day, hour = random_day_hour(self.year, self.month)\n",
    "            time = YearMonthDayHourToEpoch(self.year, self.month, day, hour) - span\n",
    "            time = time if time + span >= YearMonthDayHourToEpoch(self.year, self.month, 1, 0) else time + span\n",
    "            endTime = time + sum(time_lengths)\n",
    "            starter = True\n",
    "            ender = len(particleset) == 1\n",
    "            particleset[0].addToBuff(time, endTime, offset, time_offset, starter, ender, buff)\n",
    "            starter = False\n",
    "            for j in range(1,len(particleset)):\n",
    "                ender = (j == len(particleset) - 1)\n",
    "                time += time_lengths[j-1]\n",
    "                particleset[j].addToBuff(time, endTime, offset, time_offset, starter, ender, buff)\n",
    "                \n",
    "# cubic = quad_to_cubic([1,1],[2,3],[5,5])\n",
    "# print cubic\n",
    "# offsetCubic = offset(cubic[0], cubic[1], cubic[2], cubic[3], 1)\n",
    "# print offsetCubic\n",
    "# quad = cubic_to_quad(offsetCubic[0], offsetCubic[1], offsetCubic[2], offsetCubic[3])\n",
    "# print quad\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "try:\n",
    "    processed_data\n",
    "except NameError:\n",
    "    with open('json_store/processed_data.json') as data_file:\n",
    "        processed_data = json.load(data_file)\n",
    "processed_data.pop(\"deaths\", None)\n",
    "try:\n",
    "    geo_data\n",
    "except NameError:\n",
    "    with open('geo_data/geo_data.json') as data_file:\n",
    "        geo_data = json.load(data_file)\n",
    "\n",
    "one_minute = 60\n",
    "one_hour= one_minute * 60\n",
    "one_day = one_hour * 24\n",
    "two_days = 2 * one_day\n",
    "three_days = 3 * one_day\n",
    "five_days = 5 * one_day\n",
    "six_days = 6 * one_day\n",
    "eight_days = 8 * one_day\n",
    "ten_days = 2 * five_days\n",
    "fifteen_days = five_days + ten_days\n",
    "twenty_days = 2*ten_days\n",
    "twenty_five_days = 5 * five_days\n",
    "        \n",
    "red = chroma.Color((1, 0, 0), 'RGB')\n",
    "blue = chroma.Color((0, 0, 1), 'RGB')\n",
    "green = chroma.Color((0, 1, 0), 'RGB')\n",
    "\n",
    "color = red\n",
    "total_span = fifteen_days\n",
    "\n",
    "buff = []\n",
    "\n",
    "for asyl_country in processed_data:\n",
    "    yearly = processed_data[asyl_country]\n",
    "    for year in yearly:\n",
    "        monthly = yearly[year]\n",
    "        for month in monthly:\n",
    "            for origin_country in monthly[month]:\n",
    "                amount = monthly[month][origin_country]\n",
    "                path = geo_data['path_geo'][origin_country][asyl_country]\n",
    "                path_points = [LonLatToPixelXY(lat_lon['lon'], lat_lon['lat']) for lat_lon in path]\n",
    "                ParticleGenerator(int(year), int(month), path_points, amount,total_span, buff).generate_particles()\n",
    "\n",
    "# origin = 'Syrian Arab Rep.'\n",
    "# asylum = 'Greece'\n",
    "\n",
    "# lat_lons = [LonLatToPixelXY(lat_lon['lon'], lat_lon['lat']) for lat_lon in geo_data['path_geo'][origin][asylum]] \n",
    "# amount = processed_data[asylum ][2014][1][origin]\n",
    "# ParticleGenerator(2014,1,lat_lons, amount, total_span, buff).generate_particles()\n",
    "\n",
    "# origin = 'Mali'\n",
    "# asylum = 'Italy'\n",
    "\n",
    "# lat_lons = [LonLatToPixelXY(lat_lon['lon'], lat_lon['lat']) for lat_lon in geo_data['path_geo'][origin][asylum]] \n",
    "# amount = processed_data[asylum ][2014][1][origin]\n",
    "# ParticleGenerator(2014,1,lat_lons, amount, total_span, buff).generate_particles()\n",
    "\n",
    "array.array('f', buff).tofile(open(\"./particles.bin\", 'w'))\n",
    "                "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.11"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}