alex_mnist.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import sys \n",
    "sys.path.append('nngp')\n",
    "\n",
    "import os.path\n",
    "\n",
    "import numpy as np\n",
    "import tensorflow as tf\n",
    "print(tf.__version__)\n",
    "\n",
    "from sklearn.datasets import fetch_mldata\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.preprocessing import LabelBinarizer \n",
    "from sklearn.metrics import mean_squared_error\n",
    "\n",
    "from keras.models import Sequential\n",
    "from keras.layers import Dense\n",
    "from keras.optimizers import SGD\n",
    "from keras.models import load_model\n",
    "\n",
    "from nngp.nngp import NNGPKernel\n",
    "from nngp.gpr import GaussianProcessRegression\n",
    "\n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "mnist = fetch_mldata('MNIST original', data_home='mnist')\n",
    "mnist.DESCR"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "lb = LabelBinarizer()\n",
    "lb.fit(mnist.target)\n",
    "# encode target labels as zero-mean one hot encoded vector \n",
    "# with negative class = -0.1 and positive class as 0.9\n",
    "encode = lambda y: lb.transform(y) - .1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "image_train, image_test, label_train, label_test = train_test_split(\n",
    "    mnist.data, mnist.target, stratify = mnist.target, random_state = 444, test_size=.1)\n",
    "\n",
    "X_test = image_test\n",
    "y_test = encode(label_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def get_train_subset(n):\n",
    "    x, _, y, _ = train_test_split(image_train, label_train, stratify = label_train, random_state=333, train_size = n)\n",
    "    return x, encode(y)\n",
    "\n",
    "def accuracy(y, y_hat):\n",
    "    return np.mean(np.argmax(y, axis = 1) == np.argmax(y_hat, axis = 1))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def train_or_load_nn_model(nn_width, nn_depth, sample_size):\n",
    "    model_path = 'models/mnist_w%d_s%d' % (nn_width, sample_size)\n",
    "    if(os.path.isfile(model_path)):\n",
    "        model = load_model(model_path)\n",
    "    else:\n",
    "        model = Sequential()\n",
    "        model.add(Dense(nn_width, input_dim=784, kernel_initializer='normal', activation='tanh'))\n",
    "        for i in np.arange(1, nn_depth):\n",
    "            model.add(Dense(nn_width, kernel_initializer='normal', activation='tanh'))\n",
    "        model.add(Dense(10, kernel_initializer='normal'))\n",
    "        model.compile(loss='mean_squared_error', optimizer = SGD(lr=0.01, decay=1e-6, momentum=0.9, nesterov=True))\n",
    "        X_train, y_train = get_train_subset(sample_size)\n",
    "        model.fit(X_train, y_train, epochs = 100, batch_size = 256, verbose = 0)\n",
    "        model.save(model_path)\n",
    "    return model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def train_or_load_gp_model(nn_depth, sample_size):\n",
    "    grid_path = 'grids/mnist_s%d' % sample_size\n",
    "    nngp_kernel = NNGPKernel(\n",
    "        depth = nn_depth,\n",
    "        weight_var=1.79,\n",
    "        bias_var=0.83,\n",
    "        nonlin_fn= tf.tanh,\n",
    "        grid_path = grid_path,\n",
    "        use_precomputed_grid = True,\n",
    "        n_gauss=301,\n",
    "        n_var=301,\n",
    "        n_corr=301,\n",
    "        max_gauss=10,\n",
    "        max_var=100,\n",
    "        use_fixed_point_norm = False)\n",
    "    X_train, y_train = get_train_subset(sample_size)\n",
    "    return GaussianProcessRegression(X_train, y_train, kern=nngp_kernel)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "sample_sizes = np.array([1000, 3000, 5000,  10000, 20000, 50000])\n",
    "nn_depths    = np.array([8,    8,    4,     4,     2,     1])\n",
    "nn_widths = np.array([8, 16, 64, 128, 512, 1024])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "with tf.Session() as sess:\n",
    "    m = train_or_load_gp_model(nn_depth = 1, sample_size = 50000)\n",
    "    y_hat, _ = m.predict(X_test, sess)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "nn_results = np.empty((sample_sizes.shape[0], nn_widths.shape[0], 2))\n",
    "\n",
    "for i, sample_size in enumerate(sample_sizes):\n",
    "    for j, nn_width in enumerate(nn_widths):\n",
    "        m = train_or_load_nn_model(nn_width = nn_width, nn_depth = nn_depths[i], sample_size = sample_size)\n",
    "        y_hat = m.predict(X_test)\n",
    "        nn_results[i, j, 0] = accuracy(y_test, y_hat)  \n",
    "        nn_results[i, j, 1] = mean_squared_error(y_test, y_hat)\n",
    "    print('done sample size %d' % sample_size)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "nngp_results = np.empty((sample_sizes.shape[0], 2))\n",
    "\n",
    "for i, sample_size in enumerate(sample_sizes):\n",
    "    with tf.Session() as sess:\n",
    "        m = train_or_load_gp_model(nn_depth = nn_depths[i], sample_size = sample_size)\n",
    "        y_hat, _ = m.predict(X_test, sess)\n",
    "        nngp_results[i, 0] = accuracy(y_test, y_hat)  \n",
    "        nngp_results[i, 1] = mean_squared_error(y_test, y_hat)\n",
    "        print('done sample size %d' % sample_size)\n",
    "    sess.close()\n",
    "    sess = None"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "plt.figure(figsize = (15, 6))\n",
    "\n",
    "plt.subplot(121)\n",
    "\n",
    "for j, nn_width in enumerate(nn_widths):\n",
    "    plt.plot(sample_sizes, nn_results[:, j, 0], label='w%d' % nn_width)\n",
    "plt.plot(sample_sizes, nngp_results[:, 0], label='nngp')\n",
    "plt.xlabel('sample size')\n",
    "plt.legend()\n",
    "_ = plt.ylabel('accuracy')\n",
    "\n",
    "plt.subplot(122)\n",
    "\n",
    "for j, nn_width in enumerate(nn_widths):\n",
    "    plt.plot(sample_sizes, nn_results[:, j, 1], label='w%d' % nn_width)\n",
    "plt.plot(sample_sizes, nngp_results[:, 1], label='nngp')\n",
    "plt.xlabel('sample size')\n",
    "plt.legend()\n",
    "_ = plt.ylabel('mse')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python [default]",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}