exhaustive_feature_extraction_prep.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# prepare data files for exhaustive feature extraction"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "# make imports\n",
    "import pandas as pd\n",
    "\n",
    "import numpy as np\n",
    "\n",
    "from tsfresh.utilities.dataframe_functions import roll_time_series"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# files for feature time series\n",
    "cholera_case_crosstab = pd.read_csv(r'C:\\Users\\Rohil\\Documents\\iGEM\\yemen\\cholera_epi_data\\yemen_cholera_case_data_differenced.csv', dayfirst = True)\n",
    "conflict_crosstab = pd.read_csv('/Users/Rohil/Documents/iGEM/yemen/fatality_crosstab.csv')\n",
    "cholera_death_crosstab = pd.read_csv(r'C:\\Users\\Rohil\\Documents\\iGEM\\yemen\\cholera_epi_data\\yemen_cholera_death_data_differenced.csv', dayfirst = True)\n",
    "max_rainfall_crosstab = pd.read_csv(r'C:\\Users\\Rohil\\Documents\\iGEM\\yemen\\rainfall\\yemen_daily_max_rainfall_crosstab.csv', dayfirst = True)\n",
    "mean_rainfall_crosstab = pd.read_csv(r'C:\\Users\\Rohil\\Documents\\iGEM\\yemen\\rainfall\\yemen_daily_mean_rainfall_crosstab.csv', dayfirst = True)\n",
    "min_rainfall_crosstab = pd.read_csv(r'C:\\Users\\Rohil\\Documents\\iGEM\\yemen\\rainfall\\yemen_daily_min_rainfall_crosstab.csv', dayfirst = True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# data on how many people are in each governorate\n",
    "gov_pop_area_data = pd.read_excel('/Users/Rohil/Documents/iGEM/yemen/gov_area_pop_data.xlsx')\n",
    "\n",
    "# we don't have data for the entire governorate of hadramawut, only for the district of Al Mukulla\n",
    "gov_pop_area_data = gov_pop_area_data[gov_pop_area_data.iso != 'YE-HD']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "min_rainfall_crosstab.date = mean_rainfall_crosstab.date"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "# scaling values up by 10k and normalizing by population of governorate\n",
    "# essentially the val per 10k people in the governorate\n",
    "norm_cholera_case_crosstab = cholera_case_crosstab\n",
    "for index, row in gov_pop_area_data[['iso', 'population']].iterrows():\n",
    "    norm_cholera_case_crosstab[row.iso] = (norm_cholera_case_crosstab[row.iso] * 10000) / row.population"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "norm_cholera_death_crosstab = cholera_death_crosstab\n",
    "for index, row in gov_pop_area_data[['iso', 'population']].iterrows():\n",
    "    norm_cholera_death_crosstab[row.iso] = (norm_cholera_death_crosstab[row.iso] * 10000) / row.population"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "norm_conflict_crosstab = conflict_crosstab\n",
    "for index, row in gov_pop_area_data[['iso', 'population']].iterrows():\n",
    "    norm_conflict_crosstab[row.iso] = (norm_conflict_crosstab[row.iso] * 10000) / row.population"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "# flattening dataframes\n",
    "\n",
    "cases_unstacked = norm_cholera_case_crosstab.set_index('date').unstack().reset_index()\n",
    "cases_unstacked.columns = ['gov_iso', 'date', 'new_cases']\n",
    "\n",
    "deaths_unstacked = norm_cholera_death_crosstab.set_index('date').unstack().reset_index()\n",
    "deaths_unstacked.columns = ['gov_iso', 'date', 'new_deaths']\n",
    "\n",
    "fatalities_unstacked = norm_conflict_crosstab.set_index('date').unstack().reset_index()\n",
    "fatalities_unstacked.columns = ['gov_iso', 'date', 'fatalities']\n",
    "\n",
    "mean_rainfall_unstacked = mean_rainfall_crosstab.set_index('date').unstack().reset_index()\n",
    "mean_rainfall_unstacked.columns = ['gov_iso', 'date', 'mean_rainfall']\n",
    "\n",
    "min_rainfall_unstacked = min_rainfall_crosstab.set_index('date').unstack().reset_index()\n",
    "min_rainfall_unstacked.columns = ['gov_iso', 'date', 'min_rainfall']\n",
    "\n",
    "max_rainfall_unstacked = max_rainfall_crosstab.set_index('date').unstack().reset_index()\n",
    "max_rainfall_unstacked.columns = ['gov_iso', 'date', 'max_rainfall']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "# formatting dates\n",
    "cases_unstacked.date = pd.to_datetime(cases_unstacked.date, format = \"%d-%m-%y\")\n",
    "deaths_unstacked.date = pd.to_datetime(deaths_unstacked.date, format = \"%d-%m-%y\")\n",
    "mean_rainfall_unstacked.date = pd.to_datetime(mean_rainfall_unstacked.date, format = \"%d-%m-%y\")\n",
    "min_rainfall_unstacked.date = pd.to_datetime(min_rainfall_unstacked.date, format = \"%d-%m-%y\")\n",
    "max_rainfall_unstacked.date = pd.to_datetime(max_rainfall_unstacked.date, format = \"%d-%m-%y\")\n",
    "fatalities_unstacked.date = pd.to_datetime(fatalities_unstacked.date, format = \"%Y-%m-%d\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "feature_timeseries = cases_unstacked.merge(deaths_unstacked, on = ['date', 'gov_iso']).merge(fatalities_unstacked, on =['date', 'gov_iso'], how = 'left').merge(mean_rainfall_unstacked, on =['date', 'gov_iso'], how = 'left').merge(min_rainfall_unstacked, on =['date', 'gov_iso'], how = 'left').merge(max_rainfall_unstacked, on =['date', 'gov_iso'], how = 'left')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "neighboring_gov_dict = {\"YE-SA\" : [\"YE-SN\"],\n",
    "                        \"YE-AB\" : [\"YE-LA\", \"YE-SH\", \"YE-BA\"],\n",
    "                        \"YE-AD\" : [\"YE-LA\"],\n",
    "                        \"YE-DA\" : [\"YE-LA\", \"YE-TA\", \"YE-IB\", \"YE-BA\"],\n",
    "                        \"YE-BA\" : [\"YE-DH\", \"YE-IB\", \"YE-DA\", \"YE-AB\", \"YE-SH\", \"YE-MA\", \"YE-SN\", \"YE-LA\"],\n",
    "                        \"YE-HU\" : [\"YE-HJ\", \"YE-MW\", \"YE-SN\", \"YE-RA\", \"YE-DH\", \"YE-TA\", \"YE-IB\"],\n",
    "                        \"YE-JA\" : [\"YE-MA\", \"YE-SN\", \"YE-AM\", \"YE-SD\"],\n",
    "                        \"YE-MR\" : [\"YE-HD-AL\"],\n",
    "                        \"YE-MW\" : [\"YE-HU\", \"YE-HJ\", \"YE-AM\", \"YE-SN\"],\n",
    "                        \"YE-AM\" : [\"YE-HJ\", \"YE-SD\", \"YE-JA\", \"YE-SN\", \"YE-MW\"],\n",
    "                        \"YE-DH\" : [\"YE-IB\", \"YE-RA\", \"YE-SN\", \"YE-BA\", \"YE-HU\"],\n",
    "                        \"YE-HD-AL\" : [\"YE-SH\", \"YE-MR\"],\n",
    "                        \"YE-HJ\" : [\"YE-MW\", \"YE-HU\", \"YE-AM\", \"YE-SD\"],\n",
    "                        \"YE-IB\" : [\"YE-TA\", \"YE-HU\", \"YE-DH\", \"YE-BA\", \"YE-DA\"],\n",
    "                        \"YE-LA\" : [\"YE-AD\", \"YE-TA\", \"YE-DA\", \"YE-BA\", \"YE-AB\"],\n",
    "                        \"YE-MA\" : [\"YE-BA\", \"YE-SN\", \"YE-JA\", \"YE-SH\"],\n",
    "                        \"YE-RA\" : [\"YE-DH\", \"YE-HU\", \"YE-SN\"],\n",
    "                        \"YE-SD\" : [\"YE-HJ\", \"YE-AM\", \"YE-JA\"],\n",
    "                        \"YE-SN\" : [\"YE-BA\", \"YE-DH\", \"YE-RA\", \"YE-MW\", \"YE-AM\", \"YE-JA\", \"YE-MA\", \"YE-HU\"],\n",
    "                        \"YE-SH\" : [\"YE-AB\", \"YE-BA\", \"YE-MA\", \"YE-HD-AL\"],\n",
    "                        \"YE-TA\" : [\"YE-LA\", \"YE-DA\", \"YE-IB\", \"YE-HU\"]}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "#neighbor_feature_timeseries = pd.DataFrame(columns = feature_timeseries.columns)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_neighboring_vars(row, var, dataset):\n",
    "        \n",
    "    neighbor_data = dataset[(dataset.date == row.date) & (dataset.gov_iso.isin(neighboring_gov_dict[row.gov_iso]))]\n",
    "        \n",
    "    #neighbor_data['gov_iso'] = row.gov_iso + '-neighbor'\n",
    "\n",
    "    return (np.mean(neighbor_data[var]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "for var in ['new_cases', 'new_deaths', 'fatalities', 'mean_rainfall', 'min_rainfall', 'max_rainfall']:\n",
    "    feature_timeseries['neighbor_' + var] = feature_timeseries.apply(get_neighboring_vars, args = (var, feature_timeseries,), axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "#neighbor_feature_timeseries = feature_timeseries.apply(get_neighboring_vars, args = (feature_timeseries,), axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>gov_iso</th>\n",
       "      <th>date</th>\n",
       "      <th>new_cases</th>\n",
       "      <th>new_deaths</th>\n",
       "      <th>fatalities</th>\n",
       "      <th>mean_rainfall</th>\n",
       "      <th>min_rainfall</th>\n",
       "      <th>max_rainfall</th>\n",
       "      <th>neighbor_new_cases</th>\n",
       "      <th>neighbor_new_deaths</th>\n",
       "      <th>neighbor_fatalities</th>\n",
       "      <th>neighbor_mean_rainfall</th>\n",
       "      <th>neighbor_min_rainfall</th>\n",
       "      <th>neighbor_max_rainfall</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>YE-AB</td>\n",
       "      <td>2017-05-23</td>\n",
       "      <td>0.901606</td>\n",
       "      <td>0.015179</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.943220</td>\n",
       "      <td>0.002315</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.064464</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.742962</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>YE-AB</td>\n",
       "      <td>2017-05-24</td>\n",
       "      <td>0.901606</td>\n",
       "      <td>0.015179</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.943220</td>\n",
       "      <td>0.002315</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.228871</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.325468</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>YE-AB</td>\n",
       "      <td>2017-05-25</td>\n",
       "      <td>0.901606</td>\n",
       "      <td>0.015179</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.943220</td>\n",
       "      <td>0.002315</td>\n",
       "      <td>0.000039</td>\n",
       "      <td>0.032354</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.550024</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>YE-AB</td>\n",
       "      <td>2017-05-26</td>\n",
       "      <td>0.901606</td>\n",
       "      <td>0.015179</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.943220</td>\n",
       "      <td>0.002315</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.322169</td>\n",
       "      <td>0.0</td>\n",
       "      <td>3.101775</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>YE-AB</td>\n",
       "      <td>2017-05-27</td>\n",
       "      <td>0.901606</td>\n",
       "      <td>0.015179</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>1.486777</td>\n",
       "      <td>0.0</td>\n",
       "      <td>7.380904</td>\n",
       "      <td>0.943220</td>\n",
       "      <td>0.002315</td>\n",
       "      <td>0.000785</td>\n",
       "      <td>2.122804</td>\n",
       "      <td>0.0</td>\n",
       "      <td>23.119947</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>YE-AB</td>\n",
       "      <td>2017-05-28</td>\n",
       "      <td>1.123213</td>\n",
       "      <td>0.005060</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.016977</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.458392</td>\n",
       "      <td>0.974399</td>\n",
       "      <td>0.005188</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.246722</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.767995</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>YE-AB</td>\n",
       "      <td>2017-05-29</td>\n",
       "      <td>1.123213</td>\n",
       "      <td>0.005060</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>5.410453</td>\n",
       "      <td>0.0</td>\n",
       "      <td>54.309643</td>\n",
       "      <td>0.974399</td>\n",
       "      <td>0.005188</td>\n",
       "      <td>0.000239</td>\n",
       "      <td>1.580253</td>\n",
       "      <td>0.0</td>\n",
       "      <td>13.095566</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>YE-AB</td>\n",
       "      <td>2017-05-30</td>\n",
       "      <td>1.123213</td>\n",
       "      <td>0.005060</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>13.253213</td>\n",
       "      <td>0.0</td>\n",
       "      <td>59.607925</td>\n",
       "      <td>0.974399</td>\n",
       "      <td>0.005188</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.840843</td>\n",
       "      <td>0.0</td>\n",
       "      <td>3.737630</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>YE-AB</td>\n",
       "      <td>2017-05-31</td>\n",
       "      <td>1.010892</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.595513</td>\n",
       "      <td>0.001596</td>\n",
       "      <td>0.000157</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>YE-AB</td>\n",
       "      <td>2017-06-01</td>\n",
       "      <td>1.010892</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.595513</td>\n",
       "      <td>0.001596</td>\n",
       "      <td>0.000860</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>YE-AB</td>\n",
       "      <td>2017-06-02</td>\n",
       "      <td>1.010892</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.595513</td>\n",
       "      <td>0.001596</td>\n",
       "      <td>0.000785</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>YE-AB</td>\n",
       "      <td>2017-06-03</td>\n",
       "      <td>1.010892</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.595513</td>\n",
       "      <td>0.001596</td>\n",
       "      <td>0.000143</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>YE-AB</td>\n",
       "      <td>2017-06-04</td>\n",
       "      <td>1.010892</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.595513</td>\n",
       "      <td>0.001596</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>YE-AB</td>\n",
       "      <td>2017-06-05</td>\n",
       "      <td>1.269940</td>\n",
       "      <td>0.005060</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.793822</td>\n",
       "      <td>0.004928</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.046266</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.416269</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>YE-AB</td>\n",
       "      <td>2017-06-06</td>\n",
       "      <td>1.269940</td>\n",
       "      <td>0.005060</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.793822</td>\n",
       "      <td>0.004928</td>\n",
       "      <td>0.000703</td>\n",
       "      <td>0.214255</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2.696141</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>YE-AB</td>\n",
       "      <td>2017-06-07</td>\n",
       "      <td>1.269940</td>\n",
       "      <td>0.005060</td>\n",
       "      <td>0.000461</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.793822</td>\n",
       "      <td>0.004928</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>YE-AB</td>\n",
       "      <td>2017-06-08</td>\n",
       "      <td>1.563392</td>\n",
       "      <td>0.020238</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>1.299788</td>\n",
       "      <td>0.008917</td>\n",
       "      <td>0.000628</td>\n",
       "      <td>0.010307</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.175227</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>YE-AB</td>\n",
       "      <td>2017-06-09</td>\n",
       "      <td>1.563392</td>\n",
       "      <td>0.020238</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>1.299788</td>\n",
       "      <td>0.008917</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.076850</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.306444</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>YE-AB</td>\n",
       "      <td>2017-06-10</td>\n",
       "      <td>1.563392</td>\n",
       "      <td>0.020238</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>1.299788</td>\n",
       "      <td>0.008917</td>\n",
       "      <td>0.000785</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>YE-AB</td>\n",
       "      <td>2017-06-11</td>\n",
       "      <td>0.994196</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.583814</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000095</td>\n",
       "      <td>0.237297</td>\n",
       "      <td>0.0</td>\n",
       "      <td>9.942772</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20</th>\n",
       "      <td>YE-AB</td>\n",
       "      <td>2017-06-12</td>\n",
       "      <td>0.994196</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.583814</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000393</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21</th>\n",
       "      <td>YE-AB</td>\n",
       "      <td>2017-06-13</td>\n",
       "      <td>0.971428</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.800959</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22</th>\n",
       "      <td>YE-AB</td>\n",
       "      <td>2017-06-14</td>\n",
       "      <td>1.457142</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>1.763080</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23</th>\n",
       "      <td>YE-AB</td>\n",
       "      <td>2017-06-15</td>\n",
       "      <td>1.634225</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.778126</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24</th>\n",
       "      <td>YE-AB</td>\n",
       "      <td>2017-06-16</td>\n",
       "      <td>1.634225</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.778126</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000283</td>\n",
       "      <td>0.016940</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.287972</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25</th>\n",
       "      <td>YE-AB</td>\n",
       "      <td>2017-06-17</td>\n",
       "      <td>1.634225</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.778126</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000382</td>\n",
       "      <td>0.021096</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.358632</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>26</th>\n",
       "      <td>YE-AB</td>\n",
       "      <td>2017-06-18</td>\n",
       "      <td>1.517856</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>1.216932</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.001193</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27</th>\n",
       "      <td>YE-AB</td>\n",
       "      <td>2017-06-19</td>\n",
       "      <td>1.669642</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.637988</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000048</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>28</th>\n",
       "      <td>YE-AB</td>\n",
       "      <td>2017-06-20</td>\n",
       "      <td>0.015179</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.794941</td>\n",
       "      <td>0.014394</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29</th>\n",
       "      <td>YE-AB</td>\n",
       "      <td>2017-06-21</td>\n",
       "      <td>0.060714</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.686387</td>\n",
       "      <td>0.0</td>\n",
       "      <td>14.783011</td>\n",
       "      <td>1.691709</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000143</td>\n",
       "      <td>0.245619</td>\n",
       "      <td>0.0</td>\n",
       "      <td>8.472737</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5682</th>\n",
       "      <td>YE-TA</td>\n",
       "      <td>2018-01-20</td>\n",
       "      <td>0.212979</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.227256</td>\n",
       "      <td>0.000186</td>\n",
       "      <td>0.000049</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5683</th>\n",
       "      <td>YE-TA</td>\n",
       "      <td>2018-01-21</td>\n",
       "      <td>0.212979</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000096</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.227256</td>\n",
       "      <td>0.000186</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5684</th>\n",
       "      <td>YE-TA</td>\n",
       "      <td>2018-01-22</td>\n",
       "      <td>0.072770</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000135</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.127642</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5685</th>\n",
       "      <td>YE-TA</td>\n",
       "      <td>2018-01-23</td>\n",
       "      <td>0.072770</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000140</td>\n",
       "      <td>0.032569</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.455966</td>\n",
       "      <td>0.127642</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5686</th>\n",
       "      <td>YE-TA</td>\n",
       "      <td>2018-01-24</td>\n",
       "      <td>0.072770</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.046766</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.654721</td>\n",
       "      <td>0.127642</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.034575</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.277051</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5687</th>\n",
       "      <td>YE-TA</td>\n",
       "      <td>2018-01-25</td>\n",
       "      <td>0.072770</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000164</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.127642</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5688</th>\n",
       "      <td>YE-TA</td>\n",
       "      <td>2018-01-26</td>\n",
       "      <td>0.072770</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000265</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.127642</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5689</th>\n",
       "      <td>YE-TA</td>\n",
       "      <td>2018-01-27</td>\n",
       "      <td>0.072770</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000111</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.127642</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5690</th>\n",
       "      <td>YE-TA</td>\n",
       "      <td>2018-01-28</td>\n",
       "      <td>0.072770</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000116</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.127642</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000312</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5691</th>\n",
       "      <td>YE-TA</td>\n",
       "      <td>2018-01-29</td>\n",
       "      <td>0.054891</td>\n",
       "      <td>0.000314</td>\n",
       "      <td>0.000082</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.106303</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.042675</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.430132</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5692</th>\n",
       "      <td>YE-TA</td>\n",
       "      <td>2018-01-30</td>\n",
       "      <td>0.054891</td>\n",
       "      <td>0.000314</td>\n",
       "      <td>0.000159</td>\n",
       "      <td>0.176207</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2.466894</td>\n",
       "      <td>0.106303</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000629</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5693</th>\n",
       "      <td>YE-TA</td>\n",
       "      <td>2018-01-31</td>\n",
       "      <td>0.054891</td>\n",
       "      <td>0.000314</td>\n",
       "      <td>0.000154</td>\n",
       "      <td>0.166701</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.204218</td>\n",
       "      <td>0.106303</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.048712</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.423157</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5694</th>\n",
       "      <td>YE-TA</td>\n",
       "      <td>2018-02-01</td>\n",
       "      <td>0.054891</td>\n",
       "      <td>0.000314</td>\n",
       "      <td>0.000029</td>\n",
       "      <td>1.850849</td>\n",
       "      <td>0.0</td>\n",
       "      <td>8.912186</td>\n",
       "      <td>0.106303</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.290455</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2.812500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5695</th>\n",
       "      <td>YE-TA</td>\n",
       "      <td>2018-02-02</td>\n",
       "      <td>0.054891</td>\n",
       "      <td>0.000314</td>\n",
       "      <td>0.000140</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.106303</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5696</th>\n",
       "      <td>YE-TA</td>\n",
       "      <td>2018-02-03</td>\n",
       "      <td>0.054891</td>\n",
       "      <td>0.000314</td>\n",
       "      <td>0.000072</td>\n",
       "      <td>0.424286</td>\n",
       "      <td>0.0</td>\n",
       "      <td>3.150000</td>\n",
       "      <td>0.106303</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000138</td>\n",
       "      <td>2.358010</td>\n",
       "      <td>0.0</td>\n",
       "      <td>6.727500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5697</th>\n",
       "      <td>YE-TA</td>\n",
       "      <td>2018-02-04</td>\n",
       "      <td>0.054891</td>\n",
       "      <td>0.000314</td>\n",
       "      <td>0.000164</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.106303</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5698</th>\n",
       "      <td>YE-TA</td>\n",
       "      <td>2018-02-05</td>\n",
       "      <td>0.042345</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000092</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.135189</td>\n",
       "      <td>0.000186</td>\n",
       "      <td>0.001210</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5699</th>\n",
       "      <td>YE-TA</td>\n",
       "      <td>2018-02-06</td>\n",
       "      <td>0.042345</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000193</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.135189</td>\n",
       "      <td>0.000186</td>\n",
       "      <td>0.000075</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5700</th>\n",
       "      <td>YE-TA</td>\n",
       "      <td>2018-02-07</td>\n",
       "      <td>0.042345</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000067</td>\n",
       "      <td>0.643684</td>\n",
       "      <td>0.0</td>\n",
       "      <td>9.011579</td>\n",
       "      <td>0.135189</td>\n",
       "      <td>0.000186</td>\n",
       "      <td>0.000011</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5701</th>\n",
       "      <td>YE-TA</td>\n",
       "      <td>2018-02-08</td>\n",
       "      <td>0.042345</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000072</td>\n",
       "      <td>0.333395</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2.776985</td>\n",
       "      <td>0.135189</td>\n",
       "      <td>0.000186</td>\n",
       "      <td>0.000002</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5702</th>\n",
       "      <td>YE-TA</td>\n",
       "      <td>2018-02-09</td>\n",
       "      <td>0.042345</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000111</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.135189</td>\n",
       "      <td>0.000186</td>\n",
       "      <td>0.000018</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5703</th>\n",
       "      <td>YE-TA</td>\n",
       "      <td>2018-02-10</td>\n",
       "      <td>0.042345</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000058</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.135189</td>\n",
       "      <td>0.000186</td>\n",
       "      <td>0.000978</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5704</th>\n",
       "      <td>YE-TA</td>\n",
       "      <td>2018-02-11</td>\n",
       "      <td>0.042345</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000029</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.135189</td>\n",
       "      <td>0.000186</td>\n",
       "      <td>0.000450</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5705</th>\n",
       "      <td>YE-TA</td>\n",
       "      <td>2018-02-12</td>\n",
       "      <td>0.032621</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000096</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.110803</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000466</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5706</th>\n",
       "      <td>YE-TA</td>\n",
       "      <td>2018-02-13</td>\n",
       "      <td>0.032621</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000048</td>\n",
       "      <td>1.364285</td>\n",
       "      <td>0.0</td>\n",
       "      <td>9.105928</td>\n",
       "      <td>0.110803</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000084</td>\n",
       "      <td>0.325316</td>\n",
       "      <td>0.0</td>\n",
       "      <td>4.346054</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5707</th>\n",
       "      <td>YE-TA</td>\n",
       "      <td>2018-02-14</td>\n",
       "      <td>0.032621</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000188</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.110803</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000727</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5708</th>\n",
       "      <td>YE-TA</td>\n",
       "      <td>2018-02-15</td>\n",
       "      <td>0.032621</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>1.121831</td>\n",
       "      <td>0.0</td>\n",
       "      <td>8.395536</td>\n",
       "      <td>0.110803</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5709</th>\n",
       "      <td>YE-TA</td>\n",
       "      <td>2018-02-16</td>\n",
       "      <td>0.032621</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000154</td>\n",
       "      <td>2.602411</td>\n",
       "      <td>0.0</td>\n",
       "      <td>17.566046</td>\n",
       "      <td>0.110803</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000466</td>\n",
       "      <td>0.153409</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.012500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5710</th>\n",
       "      <td>YE-TA</td>\n",
       "      <td>2018-02-17</td>\n",
       "      <td>0.032621</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.110803</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000008</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5711</th>\n",
       "      <td>YE-TA</td>\n",
       "      <td>2018-02-18</td>\n",
       "      <td>0.032621</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000072</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.110803</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000177</td>\n",
       "      <td>0.269235</td>\n",
       "      <td>0.0</td>\n",
       "      <td>5.293160</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5712 rows × 14 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "     gov_iso       date  new_cases  new_deaths  fatalities  mean_rainfall  \\\n",
       "0      YE-AB 2017-05-23   0.901606    0.015179    0.000000       0.000000   \n",
       "1      YE-AB 2017-05-24   0.901606    0.015179    0.000000       0.000000   \n",
       "2      YE-AB 2017-05-25   0.901606    0.015179    0.000000       0.000000   \n",
       "3      YE-AB 2017-05-26   0.901606    0.015179    0.000000       0.000000   \n",
       "4      YE-AB 2017-05-27   0.901606    0.015179    0.000000       1.486777   \n",
       "5      YE-AB 2017-05-28   1.123213    0.005060    0.000000       0.016977   \n",
       "6      YE-AB 2017-05-29   1.123213    0.005060    0.000000       5.410453   \n",
       "7      YE-AB 2017-05-30   1.123213    0.005060    0.000000      13.253213   \n",
       "8      YE-AB 2017-05-31   1.010892    0.000000    0.000000       0.000000   \n",
       "9      YE-AB 2017-06-01   1.010892    0.000000    0.000000       0.000000   \n",
       "10     YE-AB 2017-06-02   1.010892    0.000000    0.000000       0.000000   \n",
       "11     YE-AB 2017-06-03   1.010892    0.000000    0.000000       0.000000   \n",
       "12     YE-AB 2017-06-04   1.010892    0.000000    0.000000       0.000000   \n",
       "13     YE-AB 2017-06-05   1.269940    0.005060    0.000000       0.000000   \n",
       "14     YE-AB 2017-06-06   1.269940    0.005060    0.000000       0.000000   \n",
       "15     YE-AB 2017-06-07   1.269940    0.005060    0.000461       0.000000   \n",
       "16     YE-AB 2017-06-08   1.563392    0.020238    0.000000       0.000000   \n",
       "17     YE-AB 2017-06-09   1.563392    0.020238    0.000000       0.000000   \n",
       "18     YE-AB 2017-06-10   1.563392    0.020238    0.000000       0.000000   \n",
       "19     YE-AB 2017-06-11   0.994196    0.000000    0.000000       0.000000   \n",
       "20     YE-AB 2017-06-12   0.994196    0.000000    0.000000       0.000000   \n",
       "21     YE-AB 2017-06-13   0.971428    0.000000    0.000000       0.000000   \n",
       "22     YE-AB 2017-06-14   1.457142    0.000000    0.000000       0.000000   \n",
       "23     YE-AB 2017-06-15   1.634225    0.000000    0.000000       0.000000   \n",
       "24     YE-AB 2017-06-16   1.634225    0.000000    0.000000       0.000000   \n",
       "25     YE-AB 2017-06-17   1.634225    0.000000    0.000000       0.000000   \n",
       "26     YE-AB 2017-06-18   1.517856    0.000000    0.000000       0.000000   \n",
       "27     YE-AB 2017-06-19   1.669642    0.000000    0.000000       0.000000   \n",
       "28     YE-AB 2017-06-20   0.015179    0.000000    0.000000       0.000000   \n",
       "29     YE-AB 2017-06-21   0.060714    0.000000    0.000000       0.686387   \n",
       "...      ...        ...        ...         ...         ...            ...   \n",
       "5682   YE-TA 2018-01-20   0.212979    0.000000    0.000000       0.000000   \n",
       "5683   YE-TA 2018-01-21   0.212979    0.000000    0.000096       0.000000   \n",
       "5684   YE-TA 2018-01-22   0.072770    0.000000    0.000135       0.000000   \n",
       "5685   YE-TA 2018-01-23   0.072770    0.000000    0.000140       0.032569   \n",
       "5686   YE-TA 2018-01-24   0.072770    0.000000    0.000000       0.046766   \n",
       "5687   YE-TA 2018-01-25   0.072770    0.000000    0.000164       0.000000   \n",
       "5688   YE-TA 2018-01-26   0.072770    0.000000    0.000265       0.000000   \n",
       "5689   YE-TA 2018-01-27   0.072770    0.000000    0.000111       0.000000   \n",
       "5690   YE-TA 2018-01-28   0.072770    0.000000    0.000116       0.000000   \n",
       "5691   YE-TA 2018-01-29   0.054891    0.000314    0.000082       0.000000   \n",
       "5692   YE-TA 2018-01-30   0.054891    0.000314    0.000159       0.176207   \n",
       "5693   YE-TA 2018-01-31   0.054891    0.000314    0.000154       0.166701   \n",
       "5694   YE-TA 2018-02-01   0.054891    0.000314    0.000029       1.850849   \n",
       "5695   YE-TA 2018-02-02   0.054891    0.000314    0.000140       0.000000   \n",
       "5696   YE-TA 2018-02-03   0.054891    0.000314    0.000072       0.424286   \n",
       "5697   YE-TA 2018-02-04   0.054891    0.000314    0.000164       0.000000   \n",
       "5698   YE-TA 2018-02-05   0.042345    0.000000    0.000092       0.000000   \n",
       "5699   YE-TA 2018-02-06   0.042345    0.000000    0.000193       0.000000   \n",
       "5700   YE-TA 2018-02-07   0.042345    0.000000    0.000067       0.643684   \n",
       "5701   YE-TA 2018-02-08   0.042345    0.000000    0.000072       0.333395   \n",
       "5702   YE-TA 2018-02-09   0.042345    0.000000    0.000111       0.000000   \n",
       "5703   YE-TA 2018-02-10   0.042345    0.000000    0.000058       0.000000   \n",
       "5704   YE-TA 2018-02-11   0.042345    0.000000    0.000029       0.000000   \n",
       "5705   YE-TA 2018-02-12   0.032621    0.000000    0.000096       0.000000   \n",
       "5706   YE-TA 2018-02-13   0.032621    0.000000    0.000048       1.364285   \n",
       "5707   YE-TA 2018-02-14   0.032621    0.000000    0.000188       0.000000   \n",
       "5708   YE-TA 2018-02-15   0.032621    0.000000    0.000000       1.121831   \n",
       "5709   YE-TA 2018-02-16   0.032621    0.000000    0.000154       2.602411   \n",
       "5710   YE-TA 2018-02-17   0.032621    0.000000    0.000000       0.000000   \n",
       "5711   YE-TA 2018-02-18   0.032621    0.000000    0.000072       0.000000   \n",
       "\n",
       "      min_rainfall  max_rainfall  neighbor_new_cases  neighbor_new_deaths  \\\n",
       "0              0.0      0.000000            0.943220             0.002315   \n",
       "1              0.0      0.000000            0.943220             0.002315   \n",
       "2              0.0      0.000000            0.943220             0.002315   \n",
       "3              0.0      0.000000            0.943220             0.002315   \n",
       "4              0.0      7.380904            0.943220             0.002315   \n",
       "5              0.0      0.458392            0.974399             0.005188   \n",
       "6              0.0     54.309643            0.974399             0.005188   \n",
       "7              0.0     59.607925            0.974399             0.005188   \n",
       "8              0.0      0.000000            0.595513             0.001596   \n",
       "9              0.0      0.000000            0.595513             0.001596   \n",
       "10             0.0      0.000000            0.595513             0.001596   \n",
       "11             0.0      0.000000            0.595513             0.001596   \n",
       "12             0.0      0.000000            0.595513             0.001596   \n",
       "13             0.0      0.000000            0.793822             0.004928   \n",
       "14             0.0      0.000000            0.793822             0.004928   \n",
       "15             0.0      0.000000            0.793822             0.004928   \n",
       "16             0.0      0.000000            1.299788             0.008917   \n",
       "17             0.0      0.000000            1.299788             0.008917   \n",
       "18             0.0      0.000000            1.299788             0.008917   \n",
       "19             0.0      0.000000            0.583814             0.000000   \n",
       "20             0.0      0.000000            0.583814             0.000000   \n",
       "21             0.0      0.000000            0.800959             0.000000   \n",
       "22             0.0      0.000000            1.763080             0.000000   \n",
       "23             0.0      0.000000            0.778126             0.000000   \n",
       "24             0.0      0.000000            0.778126             0.000000   \n",
       "25             0.0      0.000000            0.778126             0.000000   \n",
       "26             0.0      0.000000            1.216932             0.000000   \n",
       "27             0.0      0.000000            0.637988             0.000000   \n",
       "28             0.0      0.000000            0.794941             0.014394   \n",
       "29             0.0     14.783011            1.691709             0.000000   \n",
       "...            ...           ...                 ...                  ...   \n",
       "5682           0.0      0.000000            0.227256             0.000186   \n",
       "5683           0.0      0.000000            0.227256             0.000186   \n",
       "5684           0.0      0.000000            0.127642             0.000000   \n",
       "5685           0.0      0.455966            0.127642             0.000000   \n",
       "5686           0.0      0.654721            0.127642             0.000000   \n",
       "5687           0.0      0.000000            0.127642             0.000000   \n",
       "5688           0.0      0.000000            0.127642             0.000000   \n",
       "5689           0.0      0.000000            0.127642             0.000000   \n",
       "5690           0.0      0.000000            0.127642             0.000000   \n",
       "5691           0.0      0.000000            0.106303             0.000000   \n",
       "5692           0.0      2.466894            0.106303             0.000000   \n",
       "5693           0.0      1.204218            0.106303             0.000000   \n",
       "5694           0.0      8.912186            0.106303             0.000000   \n",
       "5695           0.0      0.000000            0.106303             0.000000   \n",
       "5696           0.0      3.150000            0.106303             0.000000   \n",
       "5697           0.0      0.000000            0.106303             0.000000   \n",
       "5698           0.0      0.000000            0.135189             0.000186   \n",
       "5699           0.0      0.000000            0.135189             0.000186   \n",
       "5700           0.0      9.011579            0.135189             0.000186   \n",
       "5701           0.0      2.776985            0.135189             0.000186   \n",
       "5702           0.0      0.000000            0.135189             0.000186   \n",
       "5703           0.0      0.000000            0.135189             0.000186   \n",
       "5704           0.0      0.000000            0.135189             0.000186   \n",
       "5705           0.0      0.000000            0.110803             0.000000   \n",
       "5706           0.0      9.105928            0.110803             0.000000   \n",
       "5707           0.0      0.000000            0.110803             0.000000   \n",
       "5708           0.0      8.395536            0.110803             0.000000   \n",
       "5709           0.0     17.566046            0.110803             0.000000   \n",
       "5710           0.0      0.000000            0.110803             0.000000   \n",
       "5711           0.0      0.000000            0.110803             0.000000   \n",
       "\n",
       "      neighbor_fatalities  neighbor_mean_rainfall  neighbor_min_rainfall  \\\n",
       "0                0.000000                0.064464                    0.0   \n",
       "1                0.000000                0.228871                    0.0   \n",
       "2                0.000039                0.032354                    0.0   \n",
       "3                0.000000                0.322169                    0.0   \n",
       "4                0.000785                2.122804                    0.0   \n",
       "5                0.000000                0.246722                    0.0   \n",
       "6                0.000239                1.580253                    0.0   \n",
       "7                0.000000                0.840843                    0.0   \n",
       "8                0.000157                0.000000                    0.0   \n",
       "9                0.000860                0.000000                    0.0   \n",
       "10               0.000785                0.000000                    0.0   \n",
       "11               0.000143                0.000000                    0.0   \n",
       "12               0.000000                0.000000                    0.0   \n",
       "13               0.000000                0.046266                    0.0   \n",
       "14               0.000703                0.214255                    0.0   \n",
       "15               0.000000                0.000000                    0.0   \n",
       "16               0.000628                0.010307                    0.0   \n",
       "17               0.000000                0.076850                    0.0   \n",
       "18               0.000785                0.000000                    0.0   \n",
       "19               0.000095                0.237297                    0.0   \n",
       "20               0.000393                0.000000                    0.0   \n",
       "21               0.000000                0.000000                    0.0   \n",
       "22               0.000000                0.000000                    0.0   \n",
       "23               0.000000                0.000000                    0.0   \n",
       "24               0.000283                0.016940                    0.0   \n",
       "25               0.000382                0.021096                    0.0   \n",
       "26               0.001193                0.000000                    0.0   \n",
       "27               0.000048                0.000000                    0.0   \n",
       "28               0.000000                0.000000                    0.0   \n",
       "29               0.000143                0.245619                    0.0   \n",
       "...                   ...                     ...                    ...   \n",
       "5682             0.000049                0.000000                    0.0   \n",
       "5683             0.000000                0.000000                    0.0   \n",
       "5684             0.000000                0.000000                    0.0   \n",
       "5685             0.000000                0.000000                    0.0   \n",
       "5686             0.000000                0.034575                    0.0   \n",
       "5687             0.000000                0.000000                    0.0   \n",
       "5688             0.000000                0.000000                    0.0   \n",
       "5689             0.000000                0.000000                    0.0   \n",
       "5690             0.000312                0.000000                    0.0   \n",
       "5691             0.000000                0.042675                    0.0   \n",
       "5692             0.000629                0.000000                    0.0   \n",
       "5693             0.000000                0.048712                    0.0   \n",
       "5694             0.000000                0.290455                    0.0   \n",
       "5695             0.000000                0.000000                    0.0   \n",
       "5696             0.000138                2.358010                    0.0   \n",
       "5697             0.000000                0.000000                    0.0   \n",
       "5698             0.001210                0.000000                    0.0   \n",
       "5699             0.000075                0.000000                    0.0   \n",
       "5700             0.000011                0.000000                    0.0   \n",
       "5701             0.000002                0.000000                    0.0   \n",
       "5702             0.000018                0.000000                    0.0   \n",
       "5703             0.000978                0.000000                    0.0   \n",
       "5704             0.000450                0.000000                    0.0   \n",
       "5705             0.000466                0.000000                    0.0   \n",
       "5706             0.000084                0.325316                    0.0   \n",
       "5707             0.000727                0.000000                    0.0   \n",
       "5708             0.000000                0.000000                    0.0   \n",
       "5709             0.000466                0.153409                    0.0   \n",
       "5710             0.000008                0.000000                    0.0   \n",
       "5711             0.000177                0.269235                    0.0   \n",
       "\n",
       "      neighbor_max_rainfall  \n",
       "0                  0.742962  \n",
       "1                  1.325468  \n",
       "2                  0.550024  \n",
       "3                  3.101775  \n",
       "4                 23.119947  \n",
       "5                  1.767995  \n",
       "6                 13.095566  \n",
       "7                  3.737630  \n",
       "8                  0.000000  \n",
       "9                  0.000000  \n",
       "10                 0.000000  \n",
       "11                 0.000000  \n",
       "12                 0.000000  \n",
       "13                 0.416269  \n",
       "14                 2.696141  \n",
       "15                 0.000000  \n",
       "16                 0.175227  \n",
       "17                 1.306444  \n",
       "18                 0.000000  \n",
       "19                 9.942772  \n",
       "20                 0.000000  \n",
       "21                 0.000000  \n",
       "22                 0.000000  \n",
       "23                 0.000000  \n",
       "24                 0.287972  \n",
       "25                 0.358632  \n",
       "26                 0.000000  \n",
       "27                 0.000000  \n",
       "28                 0.000000  \n",
       "29                 8.472737  \n",
       "...                     ...  \n",
       "5682               0.000000  \n",
       "5683               0.000000  \n",
       "5684               0.000000  \n",
       "5685               0.000000  \n",
       "5686               0.277051  \n",
       "5687               0.000000  \n",
       "5688               0.000000  \n",
       "5689               0.000000  \n",
       "5690               0.000000  \n",
       "5691               0.430132  \n",
       "5692               0.000000  \n",
       "5693               0.423157  \n",
       "5694               2.812500  \n",
       "5695               0.000000  \n",
       "5696               6.727500  \n",
       "5697               0.000000  \n",
       "5698               0.000000  \n",
       "5699               0.000000  \n",
       "5700               0.000000  \n",
       "5701               0.000000  \n",
       "5702               0.000000  \n",
       "5703               0.000000  \n",
       "5704               0.000000  \n",
       "5705               0.000000  \n",
       "5706               4.346054  \n",
       "5707               0.000000  \n",
       "5708               0.000000  \n",
       "5709               1.012500  \n",
       "5710               0.000000  \n",
       "5711               5.293160  \n",
       "\n",
       "[5712 rows x 14 columns]"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "feature_timeseries"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "# flat_neighbor_feature_timeseries = pd.DataFrame(columns=feature_timeseries.columns)\n",
    "# for row in neighbor_feature_timeseries.iteritems():\n",
    "#     flat_neighbor_feature_timeseries = flat_neighbor_feature_timeseries.append(row[1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "feature_timeseries_cols = feature_timeseries.drop(['gov_iso', 'date'], axis=1).columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "feature_timeseries['days_from'] = feature_timeseries.date - feature_timeseries.iloc[0].date\n",
    "feature_timeseries['days_from'] = feature_timeseries.days_from.dt.days"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>gov_iso</th>\n",
       "      <th>date</th>\n",
       "      <th>new_cases</th>\n",
       "      <th>new_deaths</th>\n",
       "      <th>fatalities</th>\n",
       "      <th>mean_rainfall</th>\n",
       "      <th>min_rainfall</th>\n",
       "      <th>max_rainfall</th>\n",
       "      <th>neighbor_new_cases</th>\n",
       "      <th>neighbor_new_deaths</th>\n",
       "      <th>neighbor_fatalities</th>\n",
       "      <th>neighbor_mean_rainfall</th>\n",
       "      <th>neighbor_min_rainfall</th>\n",
       "      <th>neighbor_max_rainfall</th>\n",
       "      <th>days_from</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>YE-AB</td>\n",
       "      <td>2017-05-23</td>\n",
       "      <td>0.901606</td>\n",
       "      <td>0.015179</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.94322</td>\n",
       "      <td>0.002315</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.064464</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.742962</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>YE-AB</td>\n",
       "      <td>2017-05-24</td>\n",
       "      <td>0.901606</td>\n",
       "      <td>0.015179</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.94322</td>\n",
       "      <td>0.002315</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.228871</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.325468</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>YE-AB</td>\n",
       "      <td>2017-05-25</td>\n",
       "      <td>0.901606</td>\n",
       "      <td>0.015179</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.94322</td>\n",
       "      <td>0.002315</td>\n",
       "      <td>0.000039</td>\n",
       "      <td>0.032354</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.550024</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>YE-AB</td>\n",
       "      <td>2017-05-26</td>\n",
       "      <td>0.901606</td>\n",
       "      <td>0.015179</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.94322</td>\n",
       "      <td>0.002315</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.322169</td>\n",
       "      <td>0.0</td>\n",
       "      <td>3.101775</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>YE-AB</td>\n",
       "      <td>2017-05-27</td>\n",
       "      <td>0.901606</td>\n",
       "      <td>0.015179</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.486777</td>\n",
       "      <td>0.0</td>\n",
       "      <td>7.380904</td>\n",
       "      <td>0.94322</td>\n",
       "      <td>0.002315</td>\n",
       "      <td>0.000785</td>\n",
       "      <td>2.122804</td>\n",
       "      <td>0.0</td>\n",
       "      <td>23.119947</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  gov_iso       date  new_cases  new_deaths  fatalities  mean_rainfall  \\\n",
       "0   YE-AB 2017-05-23   0.901606    0.015179         0.0       0.000000   \n",
       "1   YE-AB 2017-05-24   0.901606    0.015179         0.0       0.000000   \n",
       "2   YE-AB 2017-05-25   0.901606    0.015179         0.0       0.000000   \n",
       "3   YE-AB 2017-05-26   0.901606    0.015179         0.0       0.000000   \n",
       "4   YE-AB 2017-05-27   0.901606    0.015179         0.0       1.486777   \n",
       "\n",
       "   min_rainfall  max_rainfall  neighbor_new_cases  neighbor_new_deaths  \\\n",
       "0           0.0      0.000000             0.94322             0.002315   \n",
       "1           0.0      0.000000             0.94322             0.002315   \n",
       "2           0.0      0.000000             0.94322             0.002315   \n",
       "3           0.0      0.000000             0.94322             0.002315   \n",
       "4           0.0      7.380904             0.94322             0.002315   \n",
       "\n",
       "   neighbor_fatalities  neighbor_mean_rainfall  neighbor_min_rainfall  \\\n",
       "0             0.000000                0.064464                    0.0   \n",
       "1             0.000000                0.228871                    0.0   \n",
       "2             0.000039                0.032354                    0.0   \n",
       "3             0.000000                0.322169                    0.0   \n",
       "4             0.000785                2.122804                    0.0   \n",
       "\n",
       "   neighbor_max_rainfall  days_from  \n",
       "0               0.742962          0  \n",
       "1               1.325468          1  \n",
       "2               0.550024          2  \n",
       "3               3.101775          3  \n",
       "4              23.119947          4  "
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "feature_timeseries.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "feature_timeseries.to_csv('/Users/Rohil/Documents/iGEM/yemen/feature_engineering/exhaustive_feature_extraction_prep/unrolled_timeseries.csv', index = False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "def roll_timeseries_by_gov(timeseries, weeks_back):\n",
    "    \n",
    "    rolled_feature_timeseries = pd.DataFrame(columns=feature_timeseries.columns)\n",
    "    \n",
    "    for gov in timeseries.gov_iso.unique():\n",
    "        gov_timeseries = timeseries[timeseries.gov_iso == gov]\n",
    "        rolled_gov_timeseries = roll_time_series(gov_timeseries, column_id = 'gov_iso', column_sort = 'days_from', column_kind = None, rolling_direction = 1, max_timeshift = (weeks_back * 7))\n",
    "        rolled_gov_timeseries['id_col'] = rolled_gov_timeseries.gov_iso.map(str) + '_' + gov\n",
    "        \n",
    "        rolled_gov_timeseries.id_col = rolled_gov_timeseries.id_col.str.split('_').map(tuple)\n",
    "        \n",
    "        rolled_feature_timeseries = rolled_feature_timeseries.append(rolled_gov_timeseries)\n",
    "    \n",
    "    return (rolled_feature_timeseries)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\Rohil\\Anaconda3\\lib\\site-packages\\tsfresh\\utilities\\dataframe_functions.py:388: FutureWarning: Interpreting tuple 'by' as a list of keys, rather than a single key. Use 'by=[...]' instead of 'by=(...)'. In the future, a tuple will always mean a single key.\n",
      "  differences = df.groupby(grouper)[column_sort].apply(\n",
      "C:\\Users\\Rohil\\Anaconda3\\lib\\site-packages\\tsfresh\\utilities\\dataframe_functions.py:400: FutureWarning: Interpreting tuple 'by' as a list of keys, rather than a single key. Use 'by=[...]' instead of 'by=(...)'. In the future, a tuple will always mean a single key.\n",
      "  grouped_data = df.groupby(grouper)\n",
      "C:\\Users\\Rohil\\Anaconda3\\lib\\site-packages\\pandas\\core\\frame.py:6201: FutureWarning: Sorting because non-concatenation axis is not aligned. A future version\n",
      "of pandas will change to not sort by default.\n",
      "\n",
      "To accept the future behavior, pass 'sort=True'.\n",
      "\n",
      "To retain the current behavior and silence the warning, pass sort=False\n",
      "\n",
      "  sort=sort)\n"
     ]
    }
   ],
   "source": [
    "week_1_rolled_timeseries = roll_timeseries_by_gov(feature_timeseries, 1)\n",
    "week_2_rolled_timeseries = roll_timeseries_by_gov(feature_timeseries, 2)\n",
    "week_4_rolled_timeseries = roll_timeseries_by_gov(feature_timeseries, 4)\n",
    "week_6_rolled_timeseries = roll_timeseries_by_gov(feature_timeseries, 6)\n",
    "week_8_rolled_timeseries = roll_timeseries_by_gov(feature_timeseries, 8)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(45108, 16)"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "week_1_rolled_timeseries.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(83475, 16)"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "week_2_rolled_timeseries.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(157122, 16)"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "week_4_rolled_timeseries.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(226653, 16)"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "week_6_rolled_timeseries.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(292068, 16)"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "week_8_rolled_timeseries.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "week_1_rolled_timeseries.drop('gov_iso', axis=1).to_csv('/Users/Rohil/Documents/iGEM/yemen/feature_engineering/exhaustive_feature_extraction_prep/week_1_rolled_timeseries.csv', index = False)\n",
    "\n",
    "week_2_rolled_timeseries.drop('gov_iso', axis=1).to_csv('/Users/Rohil/Documents/iGEM/yemen/feature_engineering/exhaustive_feature_extraction_prep/week_2_rolled_timeseries.csv', index = False)\n",
    "\n",
    "week_4_rolled_timeseries.drop('gov_iso', axis=1).to_csv('/Users/Rohil/Documents/iGEM/yemen/feature_engineering/exhaustive_feature_extraction_prep/week_4_rolled_timeseries.csv', index = False)\n",
    "\n",
    "week_6_rolled_timeseries.drop('gov_iso', axis=1).to_csv('/Users/Rohil/Documents/iGEM/yemen/feature_engineering/exhaustive_feature_extraction_prep/week_6_rolled_timeseries.csv', index = False)\n",
    "\n",
    "week_8_rolled_timeseries.drop('gov_iso', axis=1).to_csv('/Users/Rohil/Documents/iGEM/yemen/feature_engineering/exhaustive_feature_extraction_prep/week_8_rolled_timeseries.csv', index = False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}