Suggested Solver - 1 - Predict returns bucket.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Suggested Solver\n",
    "\n",
    "**Note: This is only a suggested way of solving the problem. This solution is in no way complete and needs significant improvements, something we hope the competition participants will be able to do.**\n",
    "\n",
    "**Step 1.** First convert all features into objective features - this makes modeling easier. To do this, assign a numeric value to every string. For example ```'F13': {'DB1': 4, 'DB2': 2, 'DB3': 0, 'DB4': 3, 'DB5': 1}```\n",
    "\n",
    "**Step 2.** Calculate benchmark returns for every period as wi`*`Ri and break end of period TRR into three buckets: over perform index(TRR>wi`*`Ri), under perform index, +ve(TRR<wi`*`Ri, TRR>0), underperform index -ve (TRR<wi`*`Ri, TRR<0)\n",
    "Create a target variable `Y` and assign the following values: over perform index: 1 under perform index but +ve return: 0 underperform index and -ve return: -1\n",
    "\n",
    "**Step 3.** Train a model which uses the given features to predict which bucket the performance will be in, use F to predict `Y`. Train a few models to check which one is the best.\n",
    "\n",
    "**Step 4.** Define a new optimization objective function as wt`*`Ypred - transaction costs (We want as much weight as possible to be assigned to highest bucket). Run a model that first uses all the features (obviously not TRR!) to generate Ypred and then an optimizer to maximize this reward while meeting constraints.\n",
    "\n",
    "Below is a simple implementation of this model. Suggested improvements could be:\n",
    "1. Break performance into more granular buckets\n",
    "2. Use a better training mechanism to predict Y (or more feature engineering before predictions\n",
    "3. Define a better optimization objective function\n",
    "4. Use a better optimization method\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import os.path\n",
    "try:\n",
    "    from urllib2 import urlopen\n",
    "except ImportError:\n",
    "    from urllib.request import urlopen"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [],
   "source": [
    "## This calculates the reward, Do not change this function\n",
    "def getReward(wt, wt_1, ri, l, k):\n",
    "    if (ri is None) or (wt is None):\n",
    "        port_returns.append(0)\n",
    "        sharpe_ratio.append(0)\n",
    "        port_volatility.append(0)\n",
    "        reward = 0\n",
    "    else:\n",
    "        ri.fillna(0, inplace=True)\n",
    "        returns = np.dot(wt, ri)\n",
    "        port_returns.append(returns)\n",
    "        downside_returns = [x for x in port_returns if x < 0]\n",
    "        volatility = np.std(downside_returns) #np.sqrt(np.dot(weights.T, np.dot(cov_annual, weights)))\n",
    "        sharpe = np.sqrt(12)*np.mean(port_returns) / volatility if volatility!=0 else 0\n",
    "        sharpe_ratio.append(sharpe)\n",
    "        port_volatility.append(volatility)\n",
    "        if (wt_1 is None):\n",
    "            phi = 0\n",
    "        else:\n",
    "            ix = wt.index | wt_1.index\n",
    "            tn = wt.reindex(ix) - wt_1.reindex(ix)\n",
    "            tn[tn.isnull()&tn.index.isin(wt.index)] = wt\n",
    "            tn[tn.isnull()&tn.index.isin(wt_1.index)] = -wt_1\n",
    "            phi = k*tn.abs().sum()\n",
    "        reward = returns - l*volatility - phi\n",
    "#         print('returns, volatility, phi, sharpe, reward')\n",
    "#         print(returns, volatility, phi, sharpe, reward)\n",
    "    return reward\n",
    "\n",
    "## Do not change this function, this verifies if all constraints are met\n",
    "def checkConstraints(wt, wt_1, wi, Dt, St, Qt, g, U, t, T, P, delta, chi, eta):\n",
    "    violated = False\n",
    "    tol = 0.009\n",
    "    if np.abs(wt.sum()-1)>tol:\n",
    "        print(wt.sum())\n",
    "        print(\"Fully Invested Constraint Violated: Sum of weights is not 1\")\n",
    "        violated = True\n",
    "    div_constraint = np.maximum(g, 1/float(Qt.sum()))\n",
    "    if (wt.abs()-div_constraint>tol).any():\n",
    "        print(wt[(wt.abs()-div_constraint>tol)])\n",
    "        print(\"Diversification Constraint Violated: All weights are not less than parameter %.2f\"%div_constraint )\n",
    "        violated = True\n",
    "    if wt_1 is None:\n",
    "        turnover = 0\n",
    "    else:\n",
    "        ix = wt.index | wt_1.index\n",
    "        tn = wt.reindex(ix) - wt_1.reindex(ix)\n",
    "        tn[tn.isnull()&tn.index.isin(wt.index)] = wt\n",
    "        tn[tn.isnull()&tn.index.isin(wt_1.index)] = -wt_1\n",
    "        turnover = (tn).abs().sum()/2\n",
    "    turnover_list.append(turnover)\n",
    "    if (np.sum(turnover_list[-12:])>U):\n",
    "        print(\"%0.2f Turnover Constraint Violated: Turnover Limit exceeded\"%np.sum(turnover_list[-12:]))\n",
    "        violated = True\n",
    "    if (wt<t).any():\n",
    "        print(\"Shortsell Constraint Violated: all weights are not greater than parameter t\")\n",
    "        violated = True\n",
    "    if wt[wt<0].sum()<T:\n",
    "        print(\"Max Shortsell Constraint Violated: sum of all weights are not greater than parameter T\")\n",
    "        violated = True\n",
    "    if wt[wt!=0].count() < np.minimum(P, len(wt)):\n",
    "        print(\"Min number of positions Constraint Violated: count of all weights <>0 %i are not greater than parameter P%i\"%(wt[wt!=0].count(), np.minimum(P, len(wt))))\n",
    "        violated = True\n",
    "    if np.abs((wt*Dt).sum()/ (wi*Dt).sum()  - 1) - delta > tol:\n",
    "        print(\"Duration Constraint Violated: wt*Dt/ wi*Dt %.2f is greater than parameter delta\"%((wt*Dt).sum()/ (wi*Dt).sum()))\n",
    "        violated = True\n",
    "    if np.abs((wt*St).sum()/ (wi*St).sum() -1) - chi > tol:\n",
    "        print(\"Spread Constraint Violated: wt*St/ wi*St %.2f is greater than parameter chi\"%((wt*St).sum()/ (wi*St).sum()))\n",
    "        violated = True\n",
    "    if (wt*(1-Qt)).abs().sum()>tol:\n",
    "        print(\"Qualification Constraint Violated: wt*(1-qt) is not zero %.2f\"%(wt*(1-Qt)).abs().sum())\n",
    "        violated = True\n",
    "#     if returns - Rlow/ volatility <= np.sqrt(1-eta):\n",
    "#         print(\"Max Risk probability Constraint Violated: returns - Rlow/ volatility <= np.sqrt(1-eta)\")\n",
    "            \n",
    "    return violated\n",
    "\n",
    "## Do not change this function, this verifies if final constraints are met\n",
    "def checkFinalConstraints(Rmin, volmax):\n",
    "    violated = False\n",
    "    if np.sum(port_returns)<Rmin:\n",
    "        print(\"Total Return Constraint Violated: Total Return is less than Index Return\")\n",
    "        violated = True\n",
    "    if port_volatility[-1]>volmax:\n",
    "        print(\"Volatility Constraint Violated: Vol is higher than Index Vol\")\n",
    "        violated = True\n",
    "    if sharpe_ratio[-1]<Rmin/volmax:\n",
    "        print(\"Sharpe Ratio Constraint Violated: SR is less than Index SR\")\n",
    "        violated = True\n",
    "        return violated"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "def getSymbolsToTrade():\n",
    "    ################################################\n",
    "    ####   COPY FROM BELOW INTO TEMPLATE FILE   ####\n",
    "    ################################################\n",
    "    \n",
    "    return 'G1'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "24953246\n",
      "24953246\n",
      "Data Column Names:\n",
      "Index([u'AssetGroup', u'Identifier', u'F2', u'F3', u'F4', u'F5', u'q', u'F7',\n",
      "       u'F8', u'd', u'S', u'F11', u'F12', u'F13', u'F14', u'F15', u'F16',\n",
      "       u'F17', u'wI', u'F19', u'TRR', u'ERR', u'Q1', u'Q2', u'Q3', u'Q4',\n",
      "       u'Q5', u'Q6', u'Q7', u'Q8', u'Q9', u'Q10', u'Q11', u'Q12'],\n",
      "      dtype='object')\n",
      "('G1', '1996-12-31', '2018-11-30')\n",
      "           AssetGroup Identifier        F2              F3  \\\n",
      "TimeStamp                                                    \n",
      "1996-12-31         G1   57c9946c  f5251e0b          Retail   \n",
      "1996-12-31         G1   e7e41e26  68f41001         Leisure   \n",
      "1996-12-31         G1   44b6b048  43320fca  Transportation   \n",
      "1996-12-31         G1   4f59ae16  232884f1          Retail   \n",
      "1996-12-31         G1   5a58ffea  d3fb19a2  Basic Industry   \n",
      "1996-12-31         G1   960368a4  bd94d39c  Basic Industry   \n",
      "1996-12-31         G1   2cb8cd83  68f41001         Leisure   \n",
      "1996-12-31         G1   7b33849d  17978aa4  Transportation   \n",
      "1996-12-31         G1   2976bda1  09995e52          Energy   \n",
      "1996-12-31         G1   ad6a6780  17978aa4  Transportation   \n",
      "\n",
      "                                       F4  F5  q    F7   F8       d ...   Q3  \\\n",
      "TimeStamp                                                           ...        \n",
      "1996-12-31              Department Stores   2  0    A1  5.0   4.583 ...   -1   \n",
      "1996-12-31            Recreation & Travel   5  0    A2  6.0   5.508 ...    0   \n",
      "1996-12-31             Air Transportation   3  0  BBB1  8.0   6.289 ...   -4   \n",
      "1996-12-31              Department Stores  24  0  BBB1  8.0   3.240 ...    1   \n",
      "1996-12-31  Metals/Mining Excluding Steel   1  1  BBB2  9.0   9.384 ...   -4   \n",
      "1996-12-31                 Forestry/Paper   7  1    A2  6.0  11.326 ...    1   \n",
      "1996-12-31            Recreation & Travel   5  0    A2  6.0   5.310 ...    0   \n",
      "1996-12-31             Air Transportation   8  0    A2  6.0   9.149 ...    0   \n",
      "1996-12-31              Integrated Energy   6  0  BBB2  9.0   2.129 ...    0   \n",
      "1996-12-31             Air Transportation   8  0    A2  6.0   9.358 ...    0   \n",
      "\n",
      "            Q4   Q5   Q6  Q7   Q8   Q9  Q10  Q11  Q12  \n",
      "TimeStamp                                              \n",
      "1996-12-31   1  5.0  1.0   1  1.0  1.0    0   -1   -1  \n",
      "1996-12-31   1  6.0  0.0   1  2.0  0.0    0    0    0  \n",
      "1996-12-31  -4  NaN  NaN  -4  NaN  NaN   -4   -4   -4  \n",
      "1996-12-31   1  7.0  0.0   1  3.0  0.0    0    1    1  \n",
      "1996-12-31  -4  NaN  NaN  -4  NaN  NaN   -4   -4   -4  \n",
      "1996-12-31   1  6.0  0.0   1  2.0  0.0    0    1    1  \n",
      "1996-12-31   1  6.0  0.0   1  2.0  0.0    0    0    0  \n",
      "1996-12-31   0  6.0  0.0   1  2.0  0.0    0    0    0  \n",
      "1996-12-31   1  9.0  0.0   1  5.0  0.0    1    0 -100  \n",
      "1996-12-31   0  6.0  0.0   1  2.0  0.0    0    0    0  \n",
      "\n",
      "[10 rows x 34 columns]\n"
     ]
    }
   ],
   "source": [
    "index = getSymbolsToTrade()\n",
    "if not os.path.isfile('%s.csv'%index):\n",
    "    downloadFile(index)\n",
    "idx_data = pd.read_csv('%s.csv'%index, index_col='TimeStamp')\n",
    "idx_data.sort_index(axis=0, level=None, ascending=True, inplace=True)\n",
    "print(idx_data.size)\n",
    "idx_data=idx_data[~idx_data.duplicated()]\n",
    "print(idx_data.size)\n",
    "print(\"Data Column Names:\")\n",
    "print(idx_data.columns)\n",
    "print(index, idx_data.index[0], idx_data.index[-1])\n",
    "print(idx_data.head(10))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Suggested Solver\n",
    "\n",
    "Step 1. convert all features into objective features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "('AssetGroup', dtype('O'))\n",
      "('Identifier', dtype('O'))\n",
      "('F2', dtype('O'))\n",
      "('F3', dtype('O'))\n",
      "('F4', dtype('O'))\n",
      "('F7', dtype('O'))\n",
      "('F12', dtype('O'))\n",
      "('F13', dtype('O'))\n"
     ]
    }
   ],
   "source": [
    "for c in idx_data.columns:\n",
    "    if idx_data[c].dtype==object:\n",
    "        print(c, idx_data[c].dtype)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/chandinijain/Library/Python/2.7/lib/python/site-packages/ipykernel_launcher.py:8: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame\n",
      "\n",
      "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n",
      "  \n"
     ]
    }
   ],
   "source": [
    "listf = ['F3', 'F4', 'F7', 'F12', 'F13']\n",
    "feature_dict ={}\n",
    "for feature in listf:    \n",
    "    fs = idx_data[feature].unique()\n",
    "    f_dict = {}\n",
    "    count = 0\n",
    "    for f in fs:\n",
    "        idx_data[feature][idx_data[feature]==f] = count\n",
    "        f_dict[f] = count\n",
    "        count = count+1\n",
    "    feature_dict[feature] = f_dict"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'F12': {'1201-2401': 3, '201-601': 1, '2401+': 4, '601-1201': 2, '<201': 0},\n",
       " 'F13': {'DB1': 0, 'DB2': 1, 'DB3': 2, 'DB4': 3, 'DB5': 4},\n",
       " 'F3': {'Automotive': 14,\n",
       "  'Basic Industry': 3,\n",
       "  'Capital Goods': 10,\n",
       "  'Consumer Goods': 9,\n",
       "  'Energy': 4,\n",
       "  'Financial Services': 15,\n",
       "  'Healthcare': 6,\n",
       "  'Insurance': 13,\n",
       "  'Leisure': 1,\n",
       "  'Media': 7,\n",
       "  'Real Estate': 11,\n",
       "  'Retail': 0,\n",
       "  'Services': 8,\n",
       "  'Technology & Electronics': 12,\n",
       "  'Telecommunications': 5,\n",
       "  'Transportation': 2},\n",
       " 'F4': {'Advertising': 54,\n",
       "  'Aerospace/Defense': 28,\n",
       "  'Air Transportation': 2,\n",
       "  'Auto Loans': 56,\n",
       "  'Auto Parts & Equipment': 48,\n",
       "  'Automakers': 50,\n",
       "  'Beverage': 22,\n",
       "  'Building & Construction': 45,\n",
       "  'Building Materials': 42,\n",
       "  'Cable & Satellite TV': 9,\n",
       "  'Chemicals': 20,\n",
       "  'Cons/Comm/Lease Financing': 52,\n",
       "  'Department Stores': 0,\n",
       "  'Discount Stores': 19,\n",
       "  'Diversified Capital Goods': 16,\n",
       "  'Electronics': 44,\n",
       "  'Energy - Exploration & Production': 17,\n",
       "  'Environmental': 11,\n",
       "  'Food & Drug Retailers': 39,\n",
       "  'Food - Wholesale': 13,\n",
       "  'Forestry/Paper': 4,\n",
       "  'Gaming': 49,\n",
       "  'Gas Distribution': 26,\n",
       "  'Health Facilities': 7,\n",
       "  'Health Services': 37,\n",
       "  'Hotels': 46,\n",
       "  'Housing Association': 57,\n",
       "  'Integrated Energy': 5,\n",
       "  'Life Insurance': 38,\n",
       "  'Machinery': 31,\n",
       "  'Managed Care': 40,\n",
       "  'Media - Diversified': 10,\n",
       "  'Media Content': 55,\n",
       "  'Medical Products': 36,\n",
       "  'Metals/Mining Excluding Steel': 3,\n",
       "  'Oil Field Equipment & Services': 30,\n",
       "  'Oil Refining & Marketing': 32,\n",
       "  'P&C': 53,\n",
       "  'Packaging': 33,\n",
       "  'Personal & Household Products': 14,\n",
       "  'Pharmaceuticals': 35,\n",
       "  'Printing & Publishing': 8,\n",
       "  'REITs': 21,\n",
       "  'Rail': 34,\n",
       "  'RealEstate Dev & Mgt': 25,\n",
       "  'Recreation & Travel': 1,\n",
       "  'Restaurants': 43,\n",
       "  'Software/Services': 41,\n",
       "  'Specialty Retail': 18,\n",
       "  'Steel Producers/Products': 15,\n",
       "  'Support-Services': 27,\n",
       "  'Tech Hardware & Equipment': 24,\n",
       "  'Telecom - Satellite': 51,\n",
       "  'Telecom - Wireless': 47,\n",
       "  'Telecom - Wireline Integrated & Services': 6,\n",
       "  'Tobacco': 23,\n",
       "  'Transport Infrastructure/Services': 29,\n",
       "  'Trucking & Delivery': 12},\n",
       " 'F7': {'A1': 0,\n",
       "  'A2': 1,\n",
       "  'A3': 5,\n",
       "  'AA1': 6,\n",
       "  'AA2': 9,\n",
       "  'AA3': 4,\n",
       "  'AAA': 8,\n",
       "  'BBB1': 2,\n",
       "  'BBB2': 3,\n",
       "  'BBB3': 7}}"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "feature_dict"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Step 2. break end period TRR into three buckets: over perform index, under perform index, +ve, underperform index -ve\n",
    "Create a target variable 'Y' and assign the following values:\n",
    "over perform index: 1\n",
    "under perform index but +ve return: 0\n",
    "underperform index and -ve return: -1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "dates = idx_data.index.unique()\n",
    "idx_returns = pd.Series(0.0,index=dates)\n",
    "counter=0\n",
    "new_idx_data = pd.DataFrame(columns=idx_data.columns)\n",
    "while counter < len(dates):\n",
    "    date = dates[counter]\n",
    "    date_data = idx_data[idx_data.index == date]\n",
    "    \n",
    "    ## get all the identifiers for a date\n",
    "    \n",
    "    date_data.set_index( date_data['Identifier'], inplace=True)\n",
    "    date_data = date_data[~date_data.index.duplicated()]\n",
    "    \n",
    "    wi = pd.Series(date_data['wI']/100, index = date_data['Identifier'])\n",
    "    ri = pd.Series(date_data['TRR'], index = date_data['Identifier'])\n",
    "    idx_returns[date]=np.dot(wi, ri)\n",
    "    \n",
    "    date_data.set_index(date +'-'+date_data['Identifier'], inplace=True)\n",
    "    date_data.loc[date_data['TRR']<idx_returns[date], 'Y'] = 0\n",
    "    date_data.loc[date_data['TRR']<0, 'Y'] = -1\n",
    "    date_data.loc[date_data['TRR']>=idx_returns[date], 'Y'] = 1\n",
    "    new_idx_data = new_idx_data.append(date_data)\n",
    "    counter = counter+1\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Step 3. Train a model which uses the given features to predict which bucket the performance will be in. Train a few models to check which one is the best"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "X = new_idx_data.copy()\n",
    "del X['TRR']\n",
    "del X['AssetGroup']\n",
    "del X['Identifier']\n",
    "del X['Y']\n",
    "del X['F2']\n",
    "y = new_idx_data['Y'].copy()\n",
    "X['NAs'] = X.isnull().sum(axis=1)\n",
    "X = X.replace([np.inf, -np.inf], np.nan)\n",
    "X.fillna(-50, inplace=True)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " 1.0    367951\n",
      " 0.0    186095\n",
      "-1.0    179873\n",
      "Name: Y, dtype: int64\n"
     ]
    }
   ],
   "source": [
    "print(y.value_counts())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Random under-sampling:\n",
      " 1.0    186095\n",
      " 0.0    186095\n",
      "-1.0    179873\n",
      "Name: Y, dtype: int64\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<matplotlib.axes._subplots.AxesSubplot at 0x117298150>"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAY0AAAESCAYAAAABl4lHAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4xLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvAOZPmwAAGUJJREFUeJzt3XuU33Wd3/Hny0RYdxUDMmWBkA1KtAWsUbJIT1fXlSoBPRvsUYS2Ei01WqGtx7aK654FWfHI7rG6HBELS5ZQlYsgEjXIUqpr96xcwkUuKjJyKckGiOGm4oLBd//4fWb9ZZzLNzOT/CbJ83HO78z39/5+Pt/v++fgvOZ7m6SqkCSpi+cMugFJ0o7D0JAkdWZoSJI6MzQkSZ0ZGpKkzgwNSVJnhoa0HSQZSvKDJM8bdC/jSXJFkqMH3YdmN0NDO40k/ybJ2iQ/TbIhydVJfm877LeSHDTJsFOBC6vq523Ot5L8h23d23iSnJ7k86PKZwEfG0Q/2nEYGtopJPkA8Gng48A+wALgs8CyQfYFkGR3YDkw+of0dLY5d6a2NaKqbgT2SLJkpretnYehoR1ekhcCZwAnV9WXq+pnVfWLqvpqVf33Nmb3JJ9O8vft9en2w5wk70zyt6O2+Y9HD0kuTHJOkq8n+UmSG5K8pK37dpvy3XaE8/YxWnw18HhVrWtzzgReA3ymzflMq/9FkgeTPJnk5iSv6evn9CSXJ/l8kieBdyZ5XpJVSR5L8v0kH0yyrm/Ofu2U08Yk9yX5z62+FPgj4O1t/9/t6/VbwJum9I3QLsHQ0M7gXwC/AVw5wZiPAEcAi4FXAIcDf7wV+zge+CiwJzAMnAlQVa9t619RVc+vqkvHmPty4O6RN1X1EeD/Aqe0Oae0VTe1/vYCvgh8Kclv9G1nGXA5MA/4AnAasBB4MfAG4N+NDEzyHOCrwHeB/YEjgfcnOaqqvkHviOzStv9X9O3j+/T+95HGZGhoZ/Ai4MdVtXmCMf8WOKOqHqmqjfQC4B1bsY8rq+rGto8v0Pvh3tU84CeTDaqqz1fVpqraXFWfBHYHXtY35DtV9ZWq+mW7NnIc8PGqeqwdxZzdN/Z3gaGqOqOqnqmqe4Hz6YXfRH7S+pXGNOPnRaUB2ATsnWTuBMGxH/BA3/sHWq2rh/qWnwKevxVzHwNeMNmgJP8NOKn1VcAewN59Qx4cNWW/UbX+5d8B9kvyeF9tDr0jnIm8AHh8kjHahXmkoZ3Bd4CngWMnGPP39H6QjljQagA/A35zZEWS357h/m4HXjqqtsWfl27XLz5I7+hhz6qaBzwBZLw5wAZgft/7A/qWHwTuq6p5fa8XVNUx42xrxD+jd0pLGpOhoR1eVT0B/AlwTpJjk/xmkucmOTrJn7VhFwN/3J6X2LuNH7mb6bvAIUkWt2sIp29lCw/Tu64wnhuBeUn2n2DOC4DNwEZgbpI/oXekMZHLgA8n2bNt+5S+dTcCP0nyoXbBfE6SQ5P8bt/+F7ZrH/1+H7h6kv1qF2ZoaKfQrgF8gN7F7Y30ftM+BfhKG/IxYC293/rvAG5pNarqh/TuvvrfwD3AFndSdXA6sCrJ40mOG6O3Z4AL6btQDfwF8NZ259PZwDXAN4Af0jt19g/8+umo0c4A1gH3td4vp3fERVU9C7yZ3rWX+4AfA38JvLDN/VL7uinJLQAtUH7abr2VxhT/ESZp20syRO96witHHvDbBvv4j8DxVfX7U5x/BXBBVa2Z2c60MzE0pB1Ukn3pneL6DrAI+Drwmar69EAb007Nu6ekHdduwP8EDqR3x9Ml9J6Cl7YZjzQkSZ15IVyS1JmhIUnqbKe7prH33nvXwoULB92GJO1Qbr755h9X1dBk43a60Fi4cCFr164ddBuStENJ8sDkozw9JUnaCoaGJKkzQ0OS1JmhIUnqzNCQJHVmaEiSOjM0JEmdGRqSpM52uof7treFp3590C1sU/d/4k2DbmHbOv2Fk4/ZUZ3+xKA70E7IIw1JUmeGhiSpM0NDktSZoSFJ6szQkCR1ZmhIkjrzlltJO6SXr3r5oFvYpu5YfsegWxiTRxqSpM4mDY0kK5M8kuTOvtqlSW5rr/uT3NbqC5P8vG/d5/rmHJbkjiTDSc5OklbfK8m1Se5pX/ds9bRxw0luT/Kqmf/4kqSt0eVI40JgaX+hqt5eVYurajFwBfDlvtU/GllXVe/tq58LvBtY1F4j2zwVuK6qFgHXtfcAR/eNXdHmS5IGaNLQqKpvA4+Ota4dLRwHXDzRNpLsC+xRVddXVQEXAce21cuAVW151aj6RdVzPTCvbUeSNCDTvabxGuDhqrqnr3ZgkluT/E2S17Ta/sC6vjHrWg1gn6ra0JYfAvbpm/PgOHMkSQMw3bunTmDLo4wNwIKq2pTkMOArSQ7purGqqiS1tU0kWUHvFBYLFizY2umSpI6mfKSRZC7wr4FLR2pV9XRVbWrLNwM/Al4KrAfm902f32oAD4+cdmpfH2n19cAB48zZQlWdV1VLqmrJ0NDQVD+SJGkS0zk99a+AH1TVP552SjKUZE5bfjG9i9j3ttNPTyY5ol0HORG4qk1bDSxvy8tH1U9sd1EdATzRdxpLkjQAXW65vRj4DvCyJOuSnNRWHc+vXwB/LXB7uwX3cuC9VTVyEf19wF8Cw/SOQK5u9U8Ab0hyD70g+kSrrwHubePPb/MlSQM06TWNqjphnPo7x6hdQe8W3LHGrwUOHaO+CThyjHoBJ0/WnyRp+/GJcElSZ4aGJKkzQ0OS1JmhIUnqzNCQJHVmaEiSOjM0JEmdGRqSpM4MDUlSZ4aGJKkzQ0OS1JmhIUnqzNCQJHVmaEiSOjM0JEmdGRqSpM4MDUlSZ4aGJKkzQ0OS1NmkoZFkZZJHktzZVzs9yfokt7XXMX3rPpxkOMndSY7qqy9tteEkp/bVD0xyQ6tfmmS3Vt+9vR9u6xfO1IeWJE1NlyONC4GlY9Q/VVWL22sNQJKDgeOBQ9qczyaZk2QOcA5wNHAwcEIbC3BW29ZBwGPASa1+EvBYq3+qjZMkDdCkoVFV3wYe7bi9ZcAlVfV0Vd0HDAOHt9dwVd1bVc8AlwDLkgR4PXB5m78KOLZvW6va8uXAkW28JGlApnNN45Qkt7fTV3u22v7Ag31j1rXaePUXAY9X1eZR9S221dY/0cb/miQrkqxNsnbjxo3T+EiSpIlMNTTOBV4CLAY2AJ+csY6moKrOq6olVbVkaGhokK1I0k5tSqFRVQ9X1bNV9UvgfHqnnwDWAwf0DZ3fauPVNwHzkswdVd9iW239C9t4SdKATCk0kuzb9/YtwMidVauB49udTwcCi4AbgZuARe1Oqd3oXSxfXVUFfBN4a5u/HLiqb1vL2/Jbgf/TxkuSBmTuZAOSXAy8Dtg7yTrgNOB1SRYDBdwPvAegqu5KchnwPWAzcHJVPdu2cwpwDTAHWFlVd7VdfAi4JMnHgFuBC1r9AuB/JRmmdyH++Gl/WknStEwaGlV1whjlC8aojYw/EzhzjPoaYM0Y9Xv51emt/vo/AG+brD9J0vbjE+GSpM4MDUlSZ4aGJKkzQ0OS1JmhIUnqzNCQJHVmaEiSOjM0JEmdGRqSpM4MDUlSZ4aGJKkzQ0OS1JmhIUnqzNCQJHVmaEiSOjM0JEmdGRqSpM4MDUlSZ4aGJKmzSUMjycokjyS5s6/250l+kOT2JFcmmdfqC5P8PMlt7fW5vjmHJbkjyXCSs5Ok1fdKcm2Se9rXPVs9bdxw28+rZv7jS5K2RpcjjQuBpaNq1wKHVtU/B34IfLhv3Y+qanF7vbevfi7wbmBRe41s81TguqpaBFzX3gMc3Td2RZsvSRqgSUOjqr4NPDqq9tdVtbm9vR6YP9E2kuwL7FFV11dVARcBx7bVy4BVbXnVqPpF1XM9MK9tR5I0IDNxTePfA1f3vT8wya1J/ibJa1ptf2Bd35h1rQawT1VtaMsPAfv0zXlwnDlbSLIiydokazdu3DiNjyJJmsi0QiPJR4DNwBdaaQOwoKpeCXwA+GKSPbpurx2F1Nb2UVXnVdWSqloyNDS0tdMlSR3NnerEJO8E3gwc2X7YU1VPA0+35ZuT/Ah4KbCeLU9hzW81gIeT7FtVG9rpp0dafT1wwDhzJEkDMKUjjSRLgQ8Cf1hVT/XVh5LMacsvpncR+952+unJJEe0u6ZOBK5q01YDy9vy8lH1E9tdVEcAT/SdxpIkDcCkRxpJLgZeB+ydZB1wGr27pXYHrm13zl7f7pR6LXBGkl8AvwTeW1UjF9HfR+9OrOfRuwYych3kE8BlSU4CHgCOa/U1wDHAMPAU8K7pfFBJ0vRNGhpVdcIY5QvGGXsFcMU469YCh45R3wQcOUa9gJMn60+StP34RLgkqTNDQ5LUmaEhSerM0JAkdWZoSJI6MzQkSZ0ZGpKkzgwNSVJnhoYkqTNDQ5LUmaEhSerM0JAkdWZoSJI6MzQkSZ0ZGpKkzgwNSVJnhoYkqTNDQ5LUWafQSLIyySNJ7uyr7ZXk2iT3tK97tnqSnJ1kOMntSV7VN2d5G39PkuV99cOS3NHmnJ32D4+Ptw9J0mB0PdK4EFg6qnYqcF1VLQKua+8BjgYWtdcK4FzoBQBwGvBq4HDgtL4QOBd4d9+8pZPsQ5I0AJ1Co6q+DTw6qrwMWNWWVwHH9tUvqp7rgXlJ9gWOAq6tqker6jHgWmBpW7dHVV1fVQVcNGpbY+1DkjQA07mmsU9VbWjLDwH7tOX9gQf7xq1rtYnq68aoT7QPSdIAzMiF8HaEUDOxransI8mKJGuTrN24ceO2bEOSdmnTCY2H26kl2tdHWn09cEDfuPmtNlF9/hj1ifaxhao6r6qWVNWSoaGhaXwkSdJEphMaq4GRO6CWA1f11U9sd1EdATzRTjFdA7wxyZ7tAvgbgWvauieTHNHumjpx1LbG2ockaQDmdhmU5GLgdcDeSdbRuwvqE8BlSU4CHgCOa8PXAMcAw8BTwLsAqurRJH8K3NTGnVFVIxfX30fvDq3nAVe3FxPsQ5I0AJ1Co6pOGGfVkWOMLeDkcbazElg5Rn0tcOgY9U1j7UOSNBg+ES5J6szQkCR1ZmhIkjozNCRJnRkakqTODA1JUmeGhiSpM0NDktSZoSFJ6szQkCR1ZmhIkjozNCRJnRkakqTODA1JUmeGhiSpM0NDktSZoSFJ6szQkCR1ZmhIkjqbcmgkeVmS2/peTyZ5f5LTk6zvqx/TN+fDSYaT3J3kqL760lYbTnJqX/3AJDe0+qVJdpv6R5UkTdeUQ6Oq7q6qxVW1GDgMeAq4sq3+1Mi6qloDkORg4HjgEGAp8Nkkc5LMAc4BjgYOBk5oYwHOats6CHgMOGmq/UqSpm+mTk8dCfyoqh6YYMwy4JKqerqq7gOGgcPba7iq7q2qZ4BLgGVJArweuLzNXwUcO0P9SpKmYKZC43jg4r73pyS5PcnKJHu22v7Ag31j1rXaePUXAY9X1eZRdUnSgEw7NNp1hj8EvtRK5wIvARYDG4BPTncfHXpYkWRtkrUbN27c1ruTpF3WTBxpHA3cUlUPA1TVw1X1bFX9Ejif3ukngPXAAX3z5rfaePVNwLwkc0fVf01VnVdVS6pqydDQ0Ax8JEnSWGYiNE6g79RUkn371r0FuLMtrwaOT7J7kgOBRcCNwE3Aonan1G70TnWtrqoCvgm8tc1fDlw1A/1KkqZo7uRDxpfkt4A3AO/pK/9ZksVAAfePrKuqu5JcBnwP2AycXFXPtu2cAlwDzAFWVtVdbVsfAi5J8jHgVuCC6fQrSZqeaYVGVf2M3gXr/to7Jhh/JnDmGPU1wJox6vfyq9NbkqQB84lwSVJnhoYkqTNDQ5LUmaEhSerM0JAkdWZoSJI6MzQkSZ0ZGpKkzgwNSVJnhoYkqTNDQ5LUmaEhSerM0JAkdWZoSJI6MzQkSZ0ZGpKkzgwNSVJnhoYkqTNDQ5LU2bRDI8n9Se5IcluSta22V5Jrk9zTvu7Z6klydpLhJLcneVXfdpa38fckWd5XP6xtf7jNzXR7liRNzUwdafxBVS2uqiXt/anAdVW1CLiuvQc4GljUXiuAc6EXMsBpwKuBw4HTRoKmjXl337ylM9SzJGkrbavTU8uAVW15FXBsX/2i6rkemJdkX+Ao4NqqerSqHgOuBZa2dXtU1fVVVcBFfduSJG1nMxEaBfx1kpuTrGi1fapqQ1t+CNinLe8PPNg3d12rTVRfN0ZdkjQAc2dgG79XVeuT/BPg2iQ/6F9ZVZWkZmA/42phtQJgwYIF23JXkrRLm/aRRlWtb18fAa6kd03i4XZqifb1kTZ8PXBA3/T5rTZRff4Y9dE9nFdVS6pqydDQ0HQ/kiRpHNMKjSS/leQFI8vAG4E7gdXAyB1Qy4Gr2vJq4MR2F9URwBPtNNY1wBuT7NkugL8RuKatezLJEe2uqRP7tiVJ2s6me3pqH+DKdhfsXOCLVfWNJDcBlyU5CXgAOK6NXwMcAwwDTwHvAqiqR5P8KXBTG3dGVT3alt8HXAg8D7i6vSRJAzCt0Kiqe4FXjFHfBBw5Rr2Ak8fZ1kpg5Rj1tcCh0+lTkjQzfCJcktSZoSFJ6szQkCR1ZmhIkjozNCRJnRkakqTODA1JUmeGhiSpM0NDktSZoSFJ6szQkCR1ZmhIkjozNCRJnRkakqTODA1JUmeGhiSpM0NDktSZoSFJ6szQkCR1NuXQSHJAkm8m+V6Su5L8l1Y/Pcn6JLe11zF9cz6cZDjJ3UmO6qsvbbXhJKf21Q9MckOrX5pkt6n2K0mavukcaWwG/mtVHQwcAZyc5OC27lNVtbi91gC0dccDhwBLgc8mmZNkDnAOcDRwMHBC33bOats6CHgMOGka/UqSpmnKoVFVG6rqlrb8E+D7wP4TTFkGXFJVT1fVfcAwcHh7DVfVvVX1DHAJsCxJgNcDl7f5q4Bjp9qvJGn6ZuSaRpKFwCuBG1rplCS3J1mZZM9W2x94sG/aulYbr/4i4PGq2jyqPtb+VyRZm2Ttxo0bZ+ATSZLGMu3QSPJ84Arg/VX1JHAu8BJgMbAB+OR09zGZqjqvqpZU1ZKhoaFtvTtJ2mXNnc7kJM+lFxhfqKovA1TVw33rzwe+1t6uBw7omz6/1RinvgmYl2RuO9roHy9JGoDp3D0V4ALg+1X1P/rq+/YNewtwZ1teDRyfZPckBwKLgBuBm4BF7U6p3ehdLF9dVQV8E3hrm78cuGqq/UqSpm86Rxr/EngHcEeS21rtj+jd/bQYKOB+4D0AVXVXksuA79G78+rkqnoWIMkpwDXAHGBlVd3Vtvch4JIkHwNupRdSkqQBmXJoVNXfAhlj1ZoJ5pwJnDlGfc1Y86rqXnp3V0mSZgGfCJckdWZoSJI6MzQkSZ0ZGpKkzgwNSVJnhoYkqTNDQ5LUmaEhSerM0JAkdWZoSJI6MzQkSZ0ZGpKkzgwNSVJnhoYkqTNDQ5LUmaEhSerM0JAkdWZoSJI6MzQkSZ3N+tBIsjTJ3UmGk5w66H4kaVc2q0MjyRzgHOBo4GDghCQHD7YrSdp1zerQAA4Hhqvq3qp6BrgEWDbgniRplzV30A1MYn/gwb7364BXjx6UZAWwor39aZK7t0Nvg7I38OPttbOctb32tEvYrt87PprttqtdxPb9/947t/v373e6DJrtodFJVZ0HnDfoPraHJGurasmg+9DW83u3Y/P71zPbT0+tBw7oez+/1SRJAzDbQ+MmYFGSA5PsBhwPrB5wT5K0y5rVp6eqanOSU4BrgDnAyqq6a8BtDdoucRpuJ+X3bsfm9w9IVQ26B0nSDmK2n56SJM0ihoYkqTNDQ5LUmaEhbWNJ9kqy16D7kGaCoSFtA0kWJLkkyUbgBuDGJI+02sLBdqfJJJmb5D1JvpHk9va6Osl7kzx30P0NkndP7QCS7EPvT6oArK+qhwfZjyaX5DvAp4HLq+rZVpsDvA14f1UdMcj+NLEkFwOPA6vo/fki6D1cvBzYq6rePqjeBs3QmMWSLAY+B7yQXz0JP5/ef8zvq6pbBtWbJpbknqpatLXrNDsk+WFVvXRr1+0KZvXDfeJC4D1VdUN/MckRwF8BrxhEU+rk5iSfpfeb6sgf3TyA3m+qtw6sK3X1aJK3AVdU1S8BkjyH3pHiYwPtbMA80pjFJvltdbiqDtrePamb9mdvTqL3p/xHTi2uA74KXFBVTw+qN02uXXc6C3g9vwqJecA3gVOr6r7BdDZ4hsYsluRs4CXARWz52+qJwH1VdcqgepN2FUleBFBVmwbdy2xgaMxySY5my99W1wOrq2rN4LrSdCR5c1V9bdB9aGqS/HZVPTToPgbF0JC2syQfrarTBt2HpibJ16vqTYPuY1AMjR1UkhXtH5/SLJXknzL2UeL3B9eVND0+3Lfj8t/ynMWSfIjev2kf4Mb2CnBxklMH2ZumJ8nzB93DIHmksYNK8q6q+qtB96GxJfkhcEhV/WJUfTfgLp/T2HEl+X9VtWDQfQyKz2nsuD5K71kNzU6/BPYDHhhV37et0yyW5APjrQJ26SMNQ2MWS3L7eKuAfbZnL9pq7weuS3IPv7pdegFwEOCt0rPfx4E/BzaPsW6XPq3v6alZLMnDwFH8+hOoAf6uqvbb/l2pq/YE8eFseSH8ppG/RaXZK8nfAf+pqm4eY92DVXXAANqaFTzSmN2+Bjy/qm4bvSLJt7Z/O9oa7c9PXD/oPjQl7wK2eJiv7/mMJYNpaXbwSEOSOkhyS1W9atB9DNoufW5OkraCt7ljaEhSV+cPuoHZwNNTkqTOPNKQJHVmaEiSOjM0JEmdGRqSpM4MDUlSZ/8fAmbV3+R4h1cAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x1171f6650>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Class count\n",
    "count_class_0, count_class_1 , count_class_2 = y.value_counts()\n",
    "\n",
    "# Divide by class\n",
    "df_class_0 = X[y == 1]\n",
    "df_class_1 = X[y == 0]\n",
    "df_class_2 = X[y == -1]\n",
    "\n",
    "df_class_0_under = df_class_0.sample(count_class_1)\n",
    "df_test_under = pd.concat([df_class_0_under, df_class_1, df_class_2], axis=0)\n",
    "df_y_under = pd.concat([y[y.index.isin(df_class_0_under.index)], y[y == 0], y[y == -1]], axis=0)\n",
    "\n",
    "print('Random under-sampling:')\n",
    "print(df_y_under.value_counts())\n",
    "\n",
    "df_y_under.value_counts().plot(kind='bar', title='Count (target)')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.metrics import confusion_matrix \n",
    "from sklearn.model_selection import train_test_split\n",
    "# dividing X, y into train and test data \n",
    "X_train, X_test, y_train, y_test = train_test_split(df_test_under, df_y_under, random_state = 10) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.5442340018548574\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "array([[21272,  7426, 16205],\n",
       "       [ 6262, 26785, 13442],\n",
       "       [ 5317, 14251, 27056]])"
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    " # training a DescisionTreeClassifier \n",
    "from sklearn.tree import DecisionTreeClassifier \n",
    "dtree_model = DecisionTreeClassifier(max_depth = 3).fit(X_train, y_train) \n",
    "dtree_predictions = dtree_model.predict(X_test) \n",
    "\n",
    "# accuracy on X_test \n",
    "accuracy = dtree_model.score(X_test, y_test) \n",
    "print accuracy\n",
    "\n",
    "# creating a confusion matrix \n",
    "cm = confusion_matrix(y_test, dtree_predictions)\n",
    "cm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.519352828657547\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "array([[25459, 11946,  7384],\n",
       "       [ 7571, 30494,  8238],\n",
       "       [12129, 19069, 15726]])"
      ]
     },
     "execution_count": 38,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.ensemble import RandomForestClassifier\n",
    "rforest_model = RandomForestClassifier(n_estimators=100, max_depth=2,random_state=0).fit(X_train, y_train) \n",
    "rforest_predictions = rforest_model.predict(X_test) \n",
    "\n",
    "# accuracy on X_test \n",
    "accuracy = rforest_model.score(X_test, y_test) \n",
    "print accuracy\n",
    "\n",
    "# creating a confusion matrix \n",
    "cm = confusion_matrix(y_test, rforest_predictions)\n",
    "cm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([2.23716503e-01, 1.90602313e-01, 1.44858493e-01, 1.03409028e-01,\n",
       "       3.55583099e-03, 4.69643580e-02, 2.64863796e-03, 2.55140716e-02,\n",
       "       1.09587306e-02, 1.57508165e-03, 1.03123517e-03, 2.29018657e-05,\n",
       "       0.00000000e+00, 8.08035835e-04, 0.00000000e+00, 0.00000000e+00,\n",
       "       3.17135320e-05, 5.67719393e-04, 0.00000000e+00, 0.00000000e+00,\n",
       "       0.00000000e+00, 1.65762112e-03, 9.08669335e-05, 0.00000000e+00,\n",
       "       0.00000000e+00, 7.14915203e-05, 2.65634883e-02, 2.07411294e-01,\n",
       "       7.85407600e-03, 8.65087324e-05, 0.00000000e+00])"
      ]
     },
     "execution_count": 43,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "rforest_model.feature_importances_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "# # dividing X, y into train and test data \n",
    "# X_train, X_test, y_train, y_test = train_test_split(X, y, random_state = 0) \n",
    "  \n",
    "# # training a linear SVM classifier \n",
    "# from sklearn.svm import SVC \n",
    "# svm_model_linear = SVC(kernel = 'linear', C = 1).fit(X_train, y_train) \n",
    "# svm_predictions = svm_model_linear.predict(X_test) \n",
    "  \n",
    "# # model accuracy for X_test   \n",
    "# accuracy = svm_model_linear.score(X_test, y_test) \n",
    "  \n",
    "# # creating a confusion matrix \n",
    "# cm = confusion_matrix(y_test, svm_predictions) \n",
    "# cm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.45273011824715975\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "array([[20304, 10753, 13732],\n",
       "       [ 9189, 24869, 12245],\n",
       "       [14666, 14947, 17311]])"
      ]
     },
     "execution_count": 37,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# dividing X, y into train and test data \n",
    "X_train, X_test, y_train, y_test = train_test_split(df_test_under, df_y_under, random_state = 50) \n",
    "  \n",
    "# training a KNN classifier \n",
    "from sklearn.neighbors import KNeighborsClassifier \n",
    "knn = KNeighborsClassifier(n_neighbors = 7, weights='distance').fit(X_train, y_train) \n",
    "  \n",
    "# accuracy on X_test \n",
    "accuracy = knn.score(X_test, y_test) \n",
    "print accuracy \n",
    "  \n",
    "# creating a confusion matrix \n",
    "knn_predictions = knn.predict(X_test)  \n",
    "cm = confusion_matrix(y_test, knn_predictions) \n",
    "cm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.4482089033155576\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "array([[27898, 17034,    71],\n",
       "       [12388, 33827,   145],\n",
       "       [22579, 23939,   135]])"
      ]
     },
     "execution_count": 40,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# dividing X, y into train and test data \n",
    "X_train, X_test, y_train, y_test = train_test_split(df_test_under, df_y_under, random_state = 25) \n",
    "  \n",
    "# training a Naive Bayes classifier \n",
    "from sklearn.naive_bayes import GaussianNB \n",
    "gnb = GaussianNB().fit(X_train, y_train) \n",
    "gnb_predictions = gnb.predict(X_test) \n",
    "  \n",
    "# accuracy on X_test \n",
    "accuracy = gnb.score(X_test, y_test) \n",
    "print accuracy \n",
    "  \n",
    "# creating a confusion matrix \n",
    "cm = confusion_matrix(y_test, gnb_predictions) \n",
    "cm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.393861994652193\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "array([[ 197, 3201, 2826],\n",
       "       [ 161, 3124, 2515],\n",
       "       [ 351, 6134, 6548]])"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "### Taking ensemble of all three\n",
    "\n",
    "y_avg = (gnb_predictions+knn_predictions+dtree_predictions)/3\n",
    "y_avg = [1 if x>0.5 else x for x in y_avg]\n",
    "y_avg = [-1 if x<-0.5 else x for x in y_avg]\n",
    "y_avg = [0 if (x>=-0.5)and(x<=0.5) else x for x in y_avg]\n",
    "from sklearn.metrics import accuracy_score\n",
    "print(accuracy_score(y_test, y_avg))\n",
    "cm = confusion_matrix(y_test, y_avg) \n",
    "cm\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Step 4. Define a new optimization objective function as wt*bucketValue - transaction costs (We want as much weight as possible to be assigned to highest bucket). Train an optimizer to maximize this reward while meeting constraints"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [],
   "source": [
    "#     ### In this sample case, I am using scipy.optimize \n",
    "from scipy.optimize import minimize\n",
    "# import cvxpy as cp"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [],
   "source": [
    "## Here I define my objective function \n",
    "\n",
    "class CustomFeatures():\n",
    "    \n",
    "    def newFeature1(self, wt, wt_1, ri, k):\n",
    "    ################################################\n",
    "    ####   COPY FROM BELOW INTO TEMPLATE FILE   ####\n",
    "    ################################################\n",
    "#         ri = kwargs['ri']\n",
    "#         wt = kwargs['wt']\n",
    "#         wt_1 = kwargs['wt_1']\n",
    "#         l = kwargs['l']\n",
    "        ri.fillna(0, inplace=True)\n",
    "        returns = np.dot(wt, ri)\n",
    "#         volatility = np.std(port_returns) #np.sqrt(np.dot(weights.T, np.dot(cov_annual, weights)))\n",
    "        weights = pd.Series(wt, index = ri.index)\n",
    "    \n",
    "        if (wt_1 is None):\n",
    "            phi = 0\n",
    "        else:\n",
    "            ix = weights.index | wt_1.index\n",
    "            tn = weights.reindex(ix) - wt_1.reindex(ix)\n",
    "            tn[tn.isnull()&tn.index.isin(weights.index)] = weights\n",
    "            tn[tn.isnull()&tn.index.isin(wt_1.index)] = -wt_1\n",
    "            phi = k*tn.abs().sum()\n",
    "        reward = -(returns - phi)\n",
    "        return reward"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [],
   "source": [
    "def getWeights(identifiers, reward, wi, Dt, St, Qt, g, U, t, T, P, delta, chi, eta, df, trr, wt_1,**kwargs):\n",
    "    ################################################\n",
    "    ####   COPY FROM BELOW INTO TEMPLATE FILE   ####\n",
    "    ################################################\n",
    "    \n",
    "    ### I predict future returns as returns from t-1\n",
    "    ri = getPrediction(identifiers, wi, Dt, St, Qt, g, U, t, T, P, delta, chi, eta, df=df)\n",
    "    \n",
    "    div_constraint = np.maximum(g, 1/float(Qt.sum()))\n",
    "    \n",
    "    allowed = Qt[Qt==1].index   ###pre=process to assign weights only where Qt==1, this takes care of last constraint automatically\n",
    "    \n",
    "    bounds = ((t,div_constraint),) * len(allowed) #limits on weights\n",
    "    \n",
    "    cons = ({'type': 'eq', 'fun': lambda wt: 1.0 - np.sum(wt)},   ##sum of weights should be 1\n",
    "            {'type': 'ineq', 'args': (P, Qt,),'fun': lambda wt, P, Qt: np.count_nonzero(wt) - np.minimum(P, Qt.sum())}, ## minimum number of positions should be P\n",
    "            {'type': 'ineq', 'args': (delta, wi, Dt,),'fun': lambda wt, delta, wi, Dt: delta - np.abs((wt*Dt[Qt==1]).sum()/ (wi*Dt).sum() - 1)}, # duration should be within a % of index\n",
    "            {'type': 'ineq', 'args': (chi, wi, St,),'fun': lambda wt, chi, wi, St: chi - np.abs((wt*St[Qt==1]).sum()/ (wi*St).sum() - 1)}, # spread should be within a % of index\n",
    "#             {'type': 'eq', 'args': (Qt,),'fun': lambda wt, Qt: (wt*(1-Qt[Qt==1])).sum()} #only trade Q==1 assets\n",
    "           )\n",
    "    ## optimization function is defined in custom feature\n",
    "    CustomFeaturesCls = CustomFeatures()\n",
    "    weights = minimize(CustomFeaturesCls.newFeature1, wi[Qt==1], args=(wt_1, ri[Qt==1], k,), method='SLSQP', bounds=bounds,constraints=cons,\n",
    "                      options={'maxiter': 500, 'ftol': 1e-06, 'iprint': 1, 'disp': True})   \n",
    "    print(weights.success, weights.message)\n",
    "    \n",
    "    w = pd.Series(0, index = date_data['Identifier'])\n",
    "    w[qt==1] = pd.Series(weights.x , index = allowed) ## assets with q==1 are assigned weights, rest are zero\n",
    "    return w"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [],
   "source": [
    "def getKwargs():   \n",
    "    return {}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [],
   "source": [
    "## Step 3: Optional: Fill in the logic to return predictions for return on asset\n",
    "## This function takes in the same inputs as getWeights()\n",
    "\n",
    "### do not change the inputs to the function. If you want any extra inputs, specify them as **kwargs\n",
    "### you can lookup this tutorial on how to use **kwargs https://www.geeksforgeeks.org/args-kwargs-python/\n",
    "\n",
    "def getPrediction(identifiers, wi, Dt, St, Qt, g, U, t, T, P, delta, chi, eta, **kwargs):\n",
    "    ################################################\n",
    "    ####   COPY FROM BELOW INTO TEMPLATE FILE   ####\n",
    "    ################################################\n",
    "    \n",
    "    X = kwargs['df'].copy()\n",
    "    del X['AssetGroup']\n",
    "    del X['Identifier']\n",
    "    del X['F2']\n",
    "    X['NAs'] = X.isnull().sum(axis=1)\n",
    "    X = X.replace([np.inf, -np.inf], np.nan)\n",
    "    X.fillna(-50, inplace=True)\n",
    "#     y1 = dtree_model.predict(X)\n",
    "#     y2 = knn.predict(X)\n",
    "#     y3 = gnb.predict(X)\n",
    "    \n",
    "#     y_avg = (y1+y2+y3)/3\n",
    "#     y_avg = [1 if x>0.5 else x for x in y_avg]\n",
    "#     y_avg = [-1 if x<-0.5 else x for x in y_avg]\n",
    "#     y_avg = [0 if (x>=-0.5)and(x<=0.5) else x for x in y_avg]\n",
    "\n",
    "    y = rforest_model.predict(X)\n",
    "    \n",
    "    ri = pd.Series(y, index = date_data['Identifier'])\n",
    "    \n",
    "    return ri"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [],
   "source": [
    "# empty lists to store returns, volatility and weights of imaginary portfolios\n",
    "port_returns = []\n",
    "port_volatility = []\n",
    "sharpe_ratio = []\n",
    "asset_weights = []\n",
    "reward_list = []\n",
    "turnover_list = []\n",
    "idx_returns = []\n",
    "\n",
    "#port_returns ,port_volatility ,sharpe_ratio ,asset_weights ,reward_list ,turnover_list ,idx_returns \n",
    "\n",
    "#empty df to store previous period returns\n",
    "ri = None\n",
    "\n",
    "#empty dict to store values by date\n",
    "dict_metrics_by_date = {}\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "################# \n",
      "DATE 1996-12-31\n",
      "Optimization terminated successfully.    (Exit mode 0)\n",
      "            Current function value: -0.0\n",
      "            Iterations: 2\n",
      "            Function evaluations: 1179\n",
      "            Gradient evaluations: 2\n",
      "(True, 'Optimization terminated successfully.')\n",
      "{'Index Returns': 0, '12m turnover': 0, 'returns': 0, 'weights': Identifier\n",
      "57c9946c    0.000000\n",
      "e7e41e26    0.000000\n",
      "44b6b048    0.000000\n",
      "4f59ae16    0.000000\n",
      "5a58ffea    0.001813\n",
      "960368a4    0.001878\n",
      "2cb8cd83    0.000000\n",
      "7b33849d    0.000000\n",
      "2976bda1    0.000000\n",
      "ad6a6780    0.000000\n",
      "aa2cbf2d    0.001424\n",
      "64d25fbd    0.000000\n",
      "61706c88    0.001495\n",
      "fd97a74f    0.000000\n",
      "e2462d13    0.000000\n",
      "fae12e9e    0.000000\n",
      "4ccccfca    0.002032\n",
      "a8e4cca2    0.002760\n",
      "fad5bf8f    0.000000\n",
      "d97b0030    0.000000\n",
      "7ebd2fbf    0.002760\n",
      "068cbb01    0.000000\n",
      "41728933    0.000000\n",
      "291afeb6    0.000000\n",
      "9f9ba4d9    0.000000\n",
      "b71d1e84    0.002121\n",
      "1b898bb9    0.000000\n",
      "ebcdef86    0.000000\n",
      "5c77b98b    0.002046\n",
      "525c9eae    0.001605\n",
      "              ...   \n",
      "25ef5f52    0.001578\n",
      "cb02ed26    0.000000\n",
      "c0ad4a62    0.000000\n",
      "c242b203    0.000000\n",
      "9e66873f    0.000000\n",
      "36789926    0.000000\n",
      "b331c7d1    0.000000\n",
      "df4782bb    0.000000\n",
      "6e57d38b    0.000000\n",
      "0fb4ddbe    0.000000\n",
      "b4269df7    0.000000\n",
      "02832ae5    0.000000\n",
      "c1fe3ed5    0.000000\n",
      "6e8171e3    0.000000\n",
      "b969d6f7    0.000000\n",
      "78966680    0.000000\n",
      "11915ea5    0.001479\n",
      "b80e3096    0.000000\n",
      "091e61a6    0.000000\n",
      "15e9a419    0.001636\n",
      "18f2f591    0.000000\n",
      "b0eceb88    0.000000\n",
      "400b1d54    0.000000\n",
      "7fac9835    0.001515\n",
      "12b1ed06    0.000000\n",
      "a115f4fc    0.002282\n",
      "bcc788d2    0.000000\n",
      "72350552    0.000000\n",
      "17a7ddc8    0.001549\n",
      "81019580    0.001770\n",
      "Length: 1729, dtype: float64, 'Reward': 0, 'Sharpe Ratio': 0, 'volatility': 0}\n",
      "################# \n",
      "DATE 1997-01-31\n",
      "Optimization terminated successfully.    (Exit mode 0)\n",
      "            Current function value: 0.0144902167171\n",
      "            Iterations: 3\n",
      "            Function evaluations: 1808\n",
      "            Gradient evaluations: 3\n",
      "(True, 'Optimization terminated successfully.')\n",
      "{'Index Returns': 0.0518161599, '12m turnover': 0.24150361195225076, 'returns': 0.001988329676166339, 'weights': Identifier\n",
      "ed588151    0.000000e+00\n",
      "5f7e6c6e    3.402308e-18\n",
      "cae851ee    2.179161e-03\n",
      "254de02b    2.188061e-03\n",
      "64894873    0.000000e+00\n",
      "e21315f8    0.000000e+00\n",
      "60e371a7    1.878261e-03\n",
      "22d95e1e    2.167661e-03\n",
      "97ca0b09    0.000000e+00\n",
      "196315de    1.961461e-03\n",
      "2f0c7454    0.000000e+00\n",
      "4c0744df    1.872861e-03\n",
      "7f0f52e8    0.000000e+00\n",
      "afdbb4f3    2.259061e-03\n",
      "e21d3add    0.000000e+00\n",
      "7557cd46    1.599445e-17\n",
      "78977fbf    0.000000e+00\n",
      "8ec4ff8d    0.000000e+00\n",
      "eb086583    2.235361e-03\n",
      "64876756    4.404101e-16\n",
      "5e8d0626    0.000000e+00\n",
      "d9ea1ec0    0.000000e+00\n",
      "a3eb4bfd    2.916261e-03\n",
      "63467c29    0.000000e+00\n",
      "2941e978    2.011861e-03\n",
      "69199bcf    0.000000e+00\n",
      "59652b2a    0.000000e+00\n",
      "ede70026    0.000000e+00\n",
      "2cf98e4e    0.000000e+00\n",
      "1885047d    0.000000e+00\n",
      "                ...     \n",
      "64607847    2.553761e-03\n",
      "f5c3ef41    1.979931e-17\n",
      "30b777d9    2.042561e-03\n",
      "563159a9    2.290261e-03\n",
      "5100cd31    2.006361e-03\n",
      "3f3092ec    0.000000e+00\n",
      "989d3142    0.000000e+00\n",
      "cf803642    0.000000e+00\n",
      "8b9af844    0.000000e+00\n",
      "33606dbf    0.000000e+00\n",
      "eb12c698    0.000000e+00\n",
      "e28b0a1c    0.000000e+00\n",
      "b9a4e042    0.000000e+00\n",
      "6d86b436    0.000000e+00\n",
      "7ea8ccb4    1.022806e-18\n",
      "af43ab17    0.000000e+00\n",
      "d28506fa    0.000000e+00\n",
      "7f9a0dd6    0.000000e+00\n",
      "26bbd383    0.000000e+00\n",
      "eabf736a    0.000000e+00\n",
      "3ed68497    0.000000e+00\n",
      "545ad7dd    0.000000e+00\n",
      "6ffd035f    0.000000e+00\n",
      "c1e0ddf1    0.000000e+00\n",
      "0abc0e54    0.000000e+00\n",
      "82012f3a    0.000000e+00\n",
      "a4a1d0fa    0.000000e+00\n",
      "aa84cafd    0.000000e+00\n",
      "61c17860    1.881161e-03\n",
      "641f78b2    2.390261e-03\n",
      "Length: 1759, dtype: float64, 'Reward': nan, 'Sharpe Ratio': nan, 'volatility': nan}\n",
      "################# \n",
      "DATE 1997-02-28\n",
      "Optimization terminated successfully.    (Exit mode 0)\n",
      "            Current function value: 0.00733325894658\n",
      "            Iterations: 5\n",
      "            Function evaluations: 3057\n",
      "            Gradient evaluations: 5\n",
      "(True, 'Optimization terminated successfully.')\n",
      "{'Index Returns': 0.24689068530000002, '12m turnover': 0.3637245943952251, 'returns': 0.2954968070826168, 'weights': Identifier\n",
      "40831820    2.385299e-03\n",
      "d30b71ce    0.000000e+00\n",
      "efc2cf6f    2.065376e-03\n",
      "e2f40ae9    0.000000e+00\n",
      "dd85494e    0.000000e+00\n",
      "a23c1345    1.968158e-18\n",
      "e99c33be    2.012986e-03\n",
      "cab31cc2    2.018484e-03\n",
      "22258bd0    0.000000e+00\n",
      "02802843    0.000000e+00\n",
      "3b448f28    1.907489e-03\n",
      "4671e6ea    1.929544e-03\n",
      "90ed2dee    2.243122e-03\n",
      "1fa17150    0.000000e+00\n",
      "3409389f    0.000000e+00\n",
      "279dce62    0.000000e+00\n",
      "c9dcfbe6    0.000000e+00\n",
      "c652594f    3.989305e-17\n",
      "08038a4e    0.000000e+00\n",
      "61bb9f3b    0.000000e+00\n",
      "7116dca5    1.895448e-03\n",
      "6847d880    0.000000e+00\n",
      "3b1a2af7    1.941112e-03\n",
      "6a52b62f    2.031521e-03\n",
      "4d969f88    0.000000e+00\n",
      "358114ec    1.895518e-03\n",
      "768c2e23    0.000000e+00\n",
      "0795e72e    2.059384e-03\n",
      "6e3c5df4    0.000000e+00\n",
      "ee93a336    2.672345e-03\n",
      "                ...     \n",
      "9c01c1ad    0.000000e+00\n",
      "d7e039a9    0.000000e+00\n",
      "118c0985    1.985856e-03\n",
      "bf7335b4    0.000000e+00\n",
      "a6bc41a4    1.992743e-03\n",
      "9b763b6c    2.464500e-03\n",
      "4e1f56bf    0.000000e+00\n",
      "2c67f50a    0.000000e+00\n",
      "94b86359    2.025522e-03\n",
      "a3776a5b    0.000000e+00\n",
      "f255f54c    0.000000e+00\n",
      "5f967227    2.357296e-03\n",
      "51744b07    2.248553e-03\n",
      "31b4ebde    0.000000e+00\n",
      "a4a1d0fa    5.206167e-22\n",
      "705243c0    0.000000e+00\n",
      "585dcb3c    2.456801e-03\n",
      "0c5eb091    1.957143e-03\n",
      "6036780e    0.000000e+00\n",
      "25e318f5    1.880750e-03\n",
      "09eccdf9    0.000000e+00\n",
      "73dd5ea6    7.377281e-21\n",
      "7e1e3c38    1.954813e-03\n",
      "74c187e2    0.000000e+00\n",
      "8b1f07d3    0.000000e+00\n",
      "2beca7ce    0.000000e+00\n",
      "f9b78ea1    0.000000e+00\n",
      "e7cc18aa    0.000000e+00\n",
      "258f2d53    4.380574e-20\n",
      "f0cea77a    0.000000e+00\n",
      "Length: 1769, dtype: float64, 'Reward': nan, 'Sharpe Ratio': nan, 'volatility': nan}\n",
      "################# \n",
      "DATE 1997-03-31\n"
     ]
    }
   ],
   "source": [
    "### Evaluator to getweights at everytime t and calcuate reward + check if constraints are met\n",
    "\n",
    "## specifying all the constants\n",
    "counter = 0\n",
    "l = 0.1\n",
    "k = 0.03\n",
    "g = 0.04\n",
    "U = 2.5\n",
    "t= 0\n",
    "T = 0\n",
    "P = 50\n",
    "delta = 0.5\n",
    "chi = 0.15\n",
    "eta = 0.95\n",
    "\n",
    "## initializing arrays\n",
    "dates = idx_data.index.unique()\n",
    "wi = None\n",
    "\n",
    "## looping over all dates\n",
    "while counter < len(dates):\n",
    "    date = dates[counter]\n",
    "    \n",
    "    print('################# \\nDATE %s'%date)\n",
    "    \n",
    "    ## load all the data for a date\n",
    "    date_data = idx_data[idx_data.index == date]\n",
    "    \n",
    "    ## get all the identifiers for a date\n",
    "    \n",
    "    date_data.set_index( date_data['Identifier'], inplace=True)\n",
    "    date_data = date_data[~date_data.index.duplicated()]\n",
    "    cusips = date_data['Identifier']\n",
    "    \n",
    "    ## old weights\n",
    "    wt_1 = asset_weights[-1] if len(asset_weights)>0 else None\n",
    "    wt_2 = asset_weights[-2] if len(asset_weights)>1 else None\n",
    "    \n",
    "    ##old index weights\n",
    "    wi_t_1 = None if wi is None else wi.copy()\n",
    "    \n",
    "    ## calculate reward at start of time t from weights allocated at time t-1\n",
    "    reward = getReward(wt_1, wt_2, ri, l, k)\n",
    "    \n",
    "    ## calculate index return at start of time t from weights allocated at time t-1\n",
    "    if ri is None or wi_t_1 is None:\n",
    "        idx_returns.append(0)\n",
    "    else:\n",
    "        idx_returns.append(np.dot(wi_t_1, ri))\n",
    "    \n",
    "    #load specific feature info for time t\n",
    "    wi = pd.Series(date_data['wI']/100, index = date_data['Identifier'])\n",
    "    Dt = pd.Series(date_data['d'], index = date_data['Identifier'])\n",
    "    St = pd.Series(date_data['S'], index = date_data['Identifier'])\n",
    "    qt = pd.Series(date_data['q'], index = date_data['Identifier'])\n",
    "    \n",
    "    ## get new weights\n",
    "    wt = getWeights(cusips, reward, wi, Dt, St, qt, g, U, t, T, P, delta, chi, eta, \\\n",
    "                        date_data.drop(columns=['TRR']), ri, wt_1, **getKwargs()) \n",
    "\n",
    "    \n",
    "    ## store relevant info in their lists\n",
    "    asset_weights.append(wt)\n",
    "    reward_list.append(reward)\n",
    "    \n",
    "    ## verify if all constraints are met\n",
    "    if checkConstraints(wt, wt_1, wi, Dt, St, qt, g, U, t, T, P, delta, chi, eta):\n",
    "        print(\"ERROR!!!! weights don't meet contraints, exiting\")\n",
    "#         break\n",
    "    \n",
    "    dict_metrics_by_date[date] = {'returns': port_returns[-1],\n",
    "                                  'volatility': port_volatility[-1] ,\n",
    "                                  'Sharpe Ratio': sharpe_ratio[-1],\n",
    "                                  'Index Returns': idx_returns[-1],\n",
    "                                  'Reward': reward_list[-1],\n",
    "                                  '12m turnover' : np.sum(turnover_list[-12:]),\n",
    "                                  'weights' :asset_weights[-1]}\n",
    "    \n",
    "    print(dict_metrics_by_date[date])\n",
    "    \n",
    "    counter += 1\n",
    "    \n",
    "    ## Store end of month returns to calculate reward in next period\n",
    "    del ri\n",
    "    ri = pd.Series(date_data['TRR'], index = date_data['Identifier'])\n",
    "\n",
    "## Calculate returns for last period    \n",
    "if ri is not None:\n",
    "    reward = getReward(wt, wt_1, ri, l, k)\n",
    "    reward_list.append(reward)\n",
    "    idx_returns.append(np.dot(wi, ri))\n",
    "## check if contraints on total return and risk are met        \n",
    "Rmin = np.sum(idx_returns)\n",
    "volmax = np.std(idx_returns)\n",
    "if checkFinalConstraints(Rmin, volmax):\n",
    "    print(\"ERROR!!!! weights don't meet return/risk limit contraints, exiting\")\n",
    "else:\n",
    "    print(\"Portfolio Metrics:\")\n",
    "    print(\"Total Return: %.2f\"%np.sum(port_returns))\n",
    "    print(\"Standard Deviation: %.2f\"%port_volatility[-1])\n",
    "    print(\"Sharpe Ratio: %.2f\"%sharpe_ratio[-1])\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 81,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('run1','w') as data:\n",
    "    data.write(str(dict_metrics_by_date))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 78,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(0.2710803706444725, 0.8032437836394527, 1.2350242184662905)"
      ]
     },
     "execution_count": 78,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.mean(port_returns), np.std(port_returns), np.std([x for x in port_returns if x < 0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 79,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(0.589812883126556, 2.2147595417241064, 2.5638555277373474)"
      ]
     },
     "execution_count": 79,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.mean(idx_returns), np.std(idx_returns), np.std([x for x in idx_returns if x < 0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 82,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(1.1690721657712375, 0.9225253227613263)"
      ]
     },
     "execution_count": 82,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.sqrt(12)*np.mean(port_returns)/np.std(port_returns), np.sqrt(12)*np.mean(idx_returns)/np.std(idx_returns)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 83,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(0.7603494212832631, 0.7969137648215683)"
      ]
     },
     "execution_count": 83,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.sqrt(12)*np.mean(port_returns)/np.std([x for x in port_returns if x < 0]), np.sqrt(12)*np.mean(idx_returns)/np.std([x for x in idx_returns if x < 0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 87,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " 0.7685953052601675\n",
      "0.7736720808207398\n",
      "0.7752652584641955\n",
      "0.7679016904606271\n",
      "0.7673247857730527\n",
      "0.7625805380930513\n",
      "0.7616768815591277\n",
      "0.7532598105616266\n",
      "0.7532235913980986\n",
      "0.7551062498217509\n",
      "0.7741633428926226\n",
      "0.7862134726971751\n",
      "0.7812214827215072\n",
      "0.7960209010858181\n",
      "0.7999025874157809\n",
      "0.802684666500174\n",
      "0.8004423594099652\n",
      "0.817147458838857\n",
      "0.801758785249878\n",
      "0.8150078912760435\n",
      "0.822619593204505\n",
      "0.8264717101706762\n",
      "0.951934640137635\n",
      "0.8148597804851543\n",
      "0.7959227661265547\n",
      "0.7888476367758965\n",
      "0.7863910980443249\n",
      "0.7869636409128373\n",
      "0.7903986950795765\n",
      "0.8019448070622232\n",
      "0.8029626363594038\n",
      "0.8058414256980376\n",
      "0.8187981170400431\n",
      "0.7964732595985409\n",
      "0.8194685642825129\n",
      "0.8327179743102581\n",
      "0.8254660278361935\n",
      "0.8405338039145229\n",
      "0.8292202167597645\n",
      "0.839831824397471\n",
      "0.8396898895623903\n",
      "0.8538390536390533\n",
      "0.8271509844191867\n",
      "0.8407705323959702\n",
      "0.8483960542964697\n",
      "0.8495025164311225\n",
      "0.8402373587413654\n",
      "0.8509435276344715\n",
      "0.8519668575255362\n",
      "0.847618498538109\n",
      "0.8447571639552129\n",
      "0.8455425570807693\n",
      "0.8376722547484885\n",
      "0.8420252249967761\n",
      "0.8258298859432818\n",
      "0.8352044751716543\n",
      "0.8222933510892014\n",
      "0.7966972616371437\n",
      "0.7956321226674294\n",
      "0.745394244088976\n",
      "0.7661404705533752\n",
      "0.7527730401683974\n",
      "0.7473634978751087\n",
      "0.7580339991651676\n",
      "1.0258158860645576\n",
      "0.7639245758603865\n",
      "0.774483478996459\n",
      "0.8365008833879255\n",
      "0.8477194496157945\n",
      "0.9339776512842344\n",
      "0.8891447435419606\n",
      "0.9205212895038023\n",
      "0.8987028113103257\n",
      "0.9476382526198679\n",
      "0.965868977518246\n",
      "0.9015815317245923\n",
      "0.8978125895442839\n",
      "0.8976819142412833\n",
      "0.8913537210729945\n",
      "0.8563509844565096\n",
      "0.826669692111458\n",
      "0.8109587426434095\n",
      "0.8093552625957849\n",
      "0.8049941976925286\n",
      "1.0553556329764238\n",
      "0.8402162369410571\n",
      "0.8418507282457901\n",
      "0.8505153411821857\n",
      "0.8408384652030447\n",
      "0.8401997245140327\n",
      "0.8501400194885309\n",
      "0.8384641406259112\n",
      "0.8367468848199799\n",
      "0.8457566908894814\n",
      "0.8499793449095415\n",
      "0.8477663765739376\n",
      "0.8632335477381512\n",
      "0.975631515648694\n",
      "0.8697459439304993\n",
      "0.8971190310709205\n",
      "0.8952363481283262\n",
      "0.9190803246831489\n",
      "0.9094718662141232\n",
      "0.9082519949224781\n",
      "0.8985532639547554\n",
      "0.8949313739842815\n",
      "0.8979638331589707\n",
      "0.887914327150322\n",
      "0.866586493195737\n",
      "0.8576770114022518\n",
      "0.8514171510394852\n",
      "0.8532163367928378\n",
      "0.8535476375215562\n",
      "0.8427029721052925\n",
      "0.8636468475142537\n",
      "0.8634443166611636\n",
      "0.8514590909823765\n",
      "0.8622378705076066\n",
      "0.7926679231689095\n",
      "0.7887824515599915\n",
      "0.7783791626848533\n",
      "0.7461154599742839\n",
      "0.7678745401110944\n",
      "0.7607143625635855\n",
      "0.7469877051561327\n",
      "0.7660016960953684\n",
      "0.7685056240899801\n",
      "1.0308519021470928\n",
      "0.7660874632264116\n",
      "0.7480680493537946\n",
      "0.7600632419455025\n",
      "0.7626420359229764\n",
      "0.7686168847864866\n",
      "0.7771664353536953\n",
      "0.7818735365606003\n",
      "0.778160473843754\n",
      "0.7861784250075481\n",
      "0.7998349798592419\n",
      "0.7914105188474487\n",
      "0.7825386503895579\n",
      "0.7945718998957312\n",
      "0.7930724986606744\n",
      "0.7784354335675956\n",
      "0.7693111329739991\n",
      "0.7528986436279337\n",
      "0.7540341847935887\n",
      "0.7496091167862698\n",
      "0.7471160922650443\n",
      "0.7529554522757138\n",
      "0.7648557054419143\n",
      "0.7735786764208419\n",
      "0.7594305363953392\n",
      "0.7705237719608797\n",
      "0.7588530409765344\n",
      "0.7519406560808675\n",
      "1.0356322453463374\n",
      "0.7594927514533898\n",
      "0.737667007211283\n",
      "0.7642907899877318\n",
      "0.7459327608969948\n",
      "0.7384869106657929\n",
      "0.7254570903404036\n",
      "0.7108823272954947\n",
      "0.7185505339200536\n",
      "0.7206115117373627\n",
      "0.7165116234360307\n",
      "0.7303904981644684\n",
      "0.7367135527857744\n",
      "0.7279650536314709\n",
      "0.7199200932663121\n",
      "0.7337817448064354\n",
      "0.7465402880679215\n",
      "0.759937027681732\n",
      "0.7308784305567335\n",
      "0.7311714614655205\n",
      "0.741449246961347\n",
      "0.7704125777325177\n",
      "0.7659236430049294\n",
      "0.7617031381420944\n",
      "0.77088404135104\n",
      "0.7754169874320089\n",
      "0.7744529799966041\n",
      "0.7982161838023992\n",
      "0.8092647900328148\n",
      "0.8010025703079349\n",
      "0.8102075302152528\n",
      "0.8251898407538137\n",
      "0.8363782094238732\n",
      "0.8249952530975371\n",
      "0.8419367655441468\n",
      "0.8253613446053756\n",
      "0.8238763866043246\n",
      "0.8417881741997326\n",
      "0.8405633661479148\n",
      "0.844127145338812\n",
      "1.096315572341051\n",
      "0.8381490601571204\n",
      "0.8369466814042905\n",
      "0.8315435856228099\n",
      "0.8218020018844758\n",
      "0.8343853370156958\n",
      "0.8261122657419366\n",
      "0.8287121595761172\n",
      "0.8277596952465294\n",
      "1.000202043612851\n",
      "0.8556652220957582\n",
      "0.8501340941344118\n",
      "0.8521682918926684\n",
      "0.8612195616196876\n",
      "0.8613860600828981\n",
      "0.8721724159918116\n",
      "0.8741278392856018\n",
      "0.8694583299885551\n",
      "0.8642407875667237\n",
      "0.8524215163120086\n",
      "0.852045612152919\n",
      "0.8566518744351528\n",
      "0.8559366874911984\n",
      "0.8692977615169727\n",
      "0.856314908428953\n",
      "0.8561220965367449\n",
      "0.8578440722538283\n",
      "0.8822880022586996\n",
      "0.8713261814328902\n",
      "0.8707339789155569\n",
      "0.8761795389831074\n",
      "0.8728729165904698\n",
      "0.859086405222492\n",
      "0.8678758297230436\n",
      "0.8626040211989808\n",
      "0.8716107056528991\n",
      "0.8989706277305979\n",
      "0.9026008781096717\n",
      "0.8873187351638377\n",
      "1.0013143081429985\n",
      "0.8444302479131256\n",
      "0.8545613164787086\n",
      "0.8634165031823133\n",
      "0.8749075609943511\n",
      "0.844284059323938\n"
     ]
    }
   ],
   "source": [
    "counter=0\n",
    "while counter < len(dates):\n",
    "    date = dates[counter]\n",
    "    wt = dict_metrics_by_date[date]['weights']\n",
    "#     {'returns','volatility','Sharpe Ratio','Index Returns','Reward','12m turnover'}\n",
    "    if np.abs(wt.sum()-1)>tol:\n",
    "        print(wt.sum())\n",
    "    counter +=1\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}