NeuralNetworkAttempt.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n",
      "/usr/lib/python3.6/site-packages/sklearn/externals/joblib/_multiprocessing_helpers.py:28: UserWarning: [Errno 13] Permission denied.  joblib will operate in serial mode\n",
      "  warnings.warn('%s.  joblib will operate in serial mode' % (e,))\n",
      "/usr/lib/python3.6/site-packages/sklearn/cross_validation.py:41: DeprecationWarning: This module was deprecated in version 0.18 in favor of the model_selection module into which all the refactored classes and functions are moved. Also note that the interface of the new CV iterators are different from that of this module. This module will be removed in 0.20.\n",
      "  \"This module will be removed in 0.20.\", DeprecationWarning)\n"
     ]
    }
   ],
   "source": [
    "import numpy as np\n",
    "from numpy.random import seed \n",
    "seed(1) \n",
    "from tensorflow import set_random_seed \n",
    "set_random_seed(2)\n",
    "import pandas as pd\n",
    "from keras.models import Sequential\n",
    "from keras.layers import Dense, Dropout\n",
    "from keras import optimizers\n",
    "from keras.layers.normalization import BatchNormalization\n",
    "from keras.wrappers.scikit_learn import KerasRegressor\n",
    "from sklearn import preprocessing\n",
    "from sklearn.model_selection import cross_val_score\n",
    "from sklearn.model_selection import KFold\n",
    "from sklearn.preprocessing import StandardScaler\n",
    "from sklearn.pipeline import Pipeline\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.metrics import mean_squared_error\n",
    "from sklearn.cross_validation import StratifiedKFold\n",
    "from keras.layers.advanced_activations import LeakyReLU, PReLU\n",
    "from keras.layers.noise import GaussianNoise\n",
    "from keras.layers import Dropout\n",
    "\n",
    "#Stop warnings\n",
    "import warnings\n",
    "def ignore_warn(*args, **kwargs):\n",
    "    pass\n",
    "warnings.warn = ignore_warn"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The neural network model I got was the best one I've found so from just playing around with the parameters. However, it is from far being an optimal model."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def process_data():\n",
    "    '''Processes data by standardising categorical variables.'''\n",
    "    data = pd.read_csv('ATM_training.csv')\n",
    "    y = data.pop('Withdraw')\n",
    "    X_train, X_test, y_train, y_test = train_test_split(data, y, test_size=0.33, random_state=42)\n",
    "    X_train +=0.001\n",
    "    X_test +=0.001\n",
    "    scaler = StandardScaler()\n",
    "    X_tran = scaler.fit(X_train.iloc[:,0:2])\n",
    "    dat = X_tran.transform(X_train.iloc[:,0:2])\n",
    "    dat2 = X_tran.transform(X_test.iloc[:,0:2])\n",
    "\n",
    "    X_train.pop('Shops')\n",
    "    X_train.pop('ATMs')\n",
    "    X_test.pop('Shops')\n",
    "    X_test.pop('ATMs')\n",
    "    X_train += 0.01\n",
    "    X_test += 0.01\n",
    "    data1train = []\n",
    "    data2train = []\n",
    "    data1test = []\n",
    "    data2test = []\n",
    "    for i in range(len(dat)):\n",
    "        data1train.append(dat[i][0])\n",
    "        data2train.append(dat[i][1])\n",
    "    for i in range(len(dat2)):\n",
    "        data1test.append(dat2[i][0])\n",
    "        data2test.append(dat2[i][1])\n",
    "\n",
    "    X_train['Shops'] = data1train\n",
    "    X_train['ATM'] = data2train\n",
    "    X_test['Shops'] = data1test\n",
    "    X_test['ATM'] = data2test\n",
    "    return X_train, X_test, y_train, y_test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "X_train, X_test, y_train, y_test = process_data()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def eval_model(model):\n",
    "    MSE_nn = model.evaluate(X_test.as_matrix(), y_test, verbose=0)\n",
    "    print('Final MSE is {:.4f}'.format(MSE_nn))\n",
    "    return MSE_nn"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def train_model(method, epoch, batch):\n",
    "    model = method\n",
    "    model.fit(X_train.as_matrix(), y_train, nb_epoch=epoch, batch_size=batch, verbose=0)\n",
    "    score = eval_model(model)\n",
    "    return score"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "# Wide Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def baseline_model():\n",
    "    \"\"\"Basic Neural network model.\"\"\"\n",
    "    model = Sequential()\n",
    "    np.random.seed(1337) # for reproducibility\n",
    "    model.add(Dense(input_dim=6, output_dim = 24, kernel_initializer='normal', activation='relu'))\n",
    "    model.add(Dense(24, kernel_initializer='normal', activation='relu'))\n",
    "    model.add(Dense(12, kernel_initializer='normal', activation ='relu'))\n",
    "    model.add(Dense(1, kernel_initializer='normal', activation = 'relu'))\n",
    "    model.compile(loss='mean_squared_error', optimizer='adam')\n",
    "    return model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def baseline_model2():\n",
    "    \"\"\"Basic Neural network model.\"\"\"\n",
    "    model = Sequential()\n",
    "    np.random.seed(1337) # for reproducibility\n",
    "    model.add(Dense(input_dim=6, output_dim = 100, kernel_initializer='normal', activation='relu'))    \n",
    "    model.add(Dense(100, kernel_initializer='normal'))\n",
    "    model.add(PReLU(alpha_initializer='zeros', alpha_regularizer=None, alpha_constraint=None, shared_axes=None))    \n",
    "    model.add(Dense(50, kernel_initializer='normal'))\n",
    "    model.add(PReLU(alpha_initializer='zeros', alpha_regularizer=None, alpha_constraint=None, shared_axes=None))\n",
    "    model.add(Dense(25, kernel_initializer='normal'))\n",
    "    model.add(PReLU(alpha_initializer='zeros', alpha_regularizer=None, alpha_constraint=None, shared_axes=None))\n",
    "    model.add(Dense(1, kernel_initializer='normal', activation = 'relu'))\n",
    "    model.compile(loss='mean_squared_error', optimizer='adam')\n",
    "    return model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def baseline_model3():\n",
    "    \"\"\"Wide neural network model.\"\"\"\n",
    "    model = Sequential()\n",
    "    np.random.seed(1337) # for reproducibility\n",
    "    model.add(Dense(input_dim=6, output_dim = 30, kernel_initializer='normal', activation='relu'))\n",
    "    model.add(Dense(30, kernel_initializer='normal', activation = 'relu'))\n",
    "    model.add(Dense(30, kernel_initializer='normal', activation = 'relu'))\n",
    "    model.add(BatchNormalization())\n",
    "    model.add(Dense(30, kernel_initializer='normal', activation = 'relu'))\n",
    "    model.add(Dense(30, kernel_initializer='normal', activation = 'relu'))\n",
    "    model.add(BatchNormalization())\n",
    "    model.add(Dense(30, kernel_initializer='normal', activation = 'relu'))\n",
    "    model.add(Dense(1, kernel_initializer='normal', activation = 'linear'))\n",
    "    model.compile(loss='mean_squared_error', optimizer='adam')\n",
    "    return model\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def baseline_model4():\n",
    "    \"\"\"Wide neural network model.\"\"\"\n",
    "    model = Sequential()\n",
    "    np.random.seed(1337) # for reproducibility\n",
    "    model.add(Dense(input_dim=6, output_dim = 250, kernel_initializer='normal'))\n",
    "    model.add(LeakyReLU(alpha=0.1))\n",
    "    model.add(Dense(1000, kernel_initializer='normal'))\n",
    "    model.add(LeakyReLU(alpha=0.1))\n",
    "    model.add(Dense(250, kernel_initializer='normal'))\n",
    "    model.add(LeakyReLU(alpha=0.1))\n",
    "    model.add(Dense(1, kernel_initializer='normal', activation = 'linear'))\n",
    "    model.compile(loss='mean_squared_error', optimizer='adam')\n",
    "    return model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Final MSE is 0.2651\n",
      "This was for 50 epochs and 30 batch\n",
      "Final MSE is 0.2557\n",
      "This was for 50 epochs and 50 batch\n",
      "Final MSE is 0.2530\n",
      "This was for 50 epochs and 60 batch\n",
      "Final MSE is 0.2791\n",
      "This was for 50 epochs and 80 batch\n",
      "Final MSE is 0.2560\n",
      "This was for 50 epochs and 100 batch\n",
      "Final MSE is 0.2586\n",
      "This was for 75 epochs and 30 batch\n",
      "Final MSE is 0.2732\n",
      "This was for 75 epochs and 50 batch\n",
      "Final MSE is 0.2766\n",
      "This was for 75 epochs and 60 batch\n",
      "Final MSE is 0.2968\n",
      "This was for 75 epochs and 80 batch\n",
      "Final MSE is 0.2669\n",
      "This was for 75 epochs and 100 batch\n",
      "Final MSE is 0.3539\n",
      "This was for 100 epochs and 30 batch\n",
      "Final MSE is 0.2547\n",
      "This was for 100 epochs and 50 batch\n",
      "Final MSE is 0.2621\n",
      "This was for 100 epochs and 60 batch\n",
      "Final MSE is 0.2581\n",
      "This was for 100 epochs and 80 batch\n",
      "Final MSE is 0.2514\n",
      "This was for 100 epochs and 100 batch\n",
      "Final MSE is 0.2936\n",
      "This was for 150 epochs and 30 batch\n",
      "Final MSE is 0.2658\n",
      "This was for 150 epochs and 50 batch\n",
      "Final MSE is 0.2577\n",
      "This was for 150 epochs and 60 batch\n",
      "Final MSE is 0.2805\n",
      "This was for 150 epochs and 80 batch\n",
      "Final MSE is 0.2713\n",
      "This was for 150 epochs and 100 batch\n",
      "Final MSE is 0.2565\n",
      "This was for 200 epochs and 30 batch\n",
      "Final MSE is 0.2540\n",
      "This was for 200 epochs and 50 batch\n",
      "Final MSE is 0.2546\n",
      "This was for 200 epochs and 60 batch\n",
      "Final MSE is 0.2844\n",
      "This was for 200 epochs and 80 batch\n"
     ]
    }
   ],
   "source": [
    "epo = [50, 75, 100, 150, 200]\n",
    "bat = [30, 50, 60, 80, 100]\n",
    "best = 100\n",
    "ep_best, bat_best = 0, 0\n",
    "for ep in epo:\n",
    "    for ba in bat:\n",
    "        model_score = train_model(baseline_model(), ep, ba)\n",
    "        print('This was for {} epochs and {} batch'.format(ep, ba))\n",
    "        if model_score<best:\n",
    "            best = model_score\n",
    "            ep_best = ep\n",
    "            bat_best = bat_best\n",
    "print('Best model is {:.2f}, with epoch {:.2f} and batch {:.2f}'.format(best, ep_best, bat_best))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Final MSE is 0.2677\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "0.26774966872756473"
      ]
     },
     "execution_count": 45,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_model(baseline_model2(), 40, 60)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#1. Dense (init = Gaussian) -> Gaussian Noise(0.05->0.1) -> \n",
    "#PRELU or Leaky-RELU(0.1->0.3) -> BatchNormalisation -> Gaussian Dropout (0.1->0.3) -> 2. Dense and so on."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#= NAdam (lr = 0.005, momentum = 0.995)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#NAdam optimiser (nesterov Adam)\n",
    "#Batch Norm\n",
    "#PRELU\n",
    "#Gaussian Noise/Dropout"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "# Best Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Final MSE is 0.2523\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "0.25232200653756615"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "def baseline_model():\n",
    "    \"\"\"Basic Neural network model.\"\"\"\n",
    "    model = Sequential()\n",
    "    np.random.seed(1337) # for reproducibility\n",
    "    model.add(Dense(input_dim=6, output_dim = 24, kernel_initializer='normal', activation='relu'))\n",
    "    model.add(Dense(24, kernel_initializer='normal', activation='relu'))\n",
    "    model.add(Dense(12, kernel_initializer='normal', activation ='relu'))\n",
    "    model.add(Dense(1, kernel_initializer='normal', activation = 'relu'))\n",
    "    model.compile(loss='mean_squared_error', optimizer='adam')\n",
    "    return model\n",
    "    \n",
    "train_model(baseline_model(), 40, 60)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}