mnist.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 99,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The autoreload extension is already loaded. To reload it, use:\n",
      "  %reload_ext autoreload\n"
     ]
    }
   ],
   "source": [
    "%load_ext autoreload\n",
    "%autoreload 2\n",
    "import numpy as np\n",
    "import numpy.random as rnd\n",
    "import theano\n",
    "import theano.tensor as T\n",
    "import lasagne\n",
    "import time\n",
    "import gzip\n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline\n",
    "from rnn_lib import *\n",
    "from sklearn.preprocessing import OneHotEncoder"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 100,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "%reload_ext autoreload"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "#print(theano.config)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 89,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "def load_mnist_images(filename):\n",
    "    # Read the inputs in Yann LeCun's binary format.\n",
    "    with gzip.open(filename, 'rb') as f:\n",
    "        data = np.frombuffer(f.read(), dtype=np.uint8, offset=16)\n",
    "    # The inputs are vectors now, we reshape them to monochrome 2D images,\n",
    "    # following the shape convention: (examples, channels, rows, columns)\n",
    "    data = data.reshape(-1, 28, 28)\n",
    "    # The inputs come as bytes, we convert them to float32 in range [0,1].\n",
    "    # (Actually to range [0, 255/256], for compatibility to the version\n",
    "    # provided at http://deeplearning.net/data/mnist/mnist.pkl.gz.)\n",
    "    return data\n",
    "\n",
    "def load_mnist_labels(filename):\n",
    "    # Read the labels in Yann LeCun's binary format.\n",
    "    with gzip.open(filename, 'rb') as f:\n",
    "        data = np.frombuffer(f.read(), np.uint8, offset=8)\n",
    "    # The labels are vectors of integers now, that's exactly what we want.\n",
    "    return data\n",
    "\n",
    "\n",
    "# We can now download and read the training and test set images and labels.\n",
    "X_train = load_mnist_images('train-images-idx3-ubyte.gz')\n",
    "y_train = load_mnist_labels('train-labels-idx1-ubyte.gz')\n",
    "X_test = load_mnist_images('t10k-images-idx3-ubyte.gz')\n",
    "y_test = load_mnist_labels('t10k-labels-idx1-ubyte.gz')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "((60000, 28, 28), (10000, 28, 28))"
      ]
     },
     "execution_count": 38,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_train.shape, X_test.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAsMAAABbCAYAAACWJohFAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAFS1JREFUeJzt3XeQFNXXxvEvP8wZE2AEFTBjQkEtRIJZzBgxgRkUAyZQ\njARjGUFARYFSKcGAGQETIoWxCgUElWAAFUExJ94/fJ+907MRdqZnevr5/LMwO7tz5+5MT/e5555T\nZ9myZZiZmZmZpdH/Cj0AMzMzM7NC8cmwmZmZmaWWT4bNzMzMLLV8MmxmZmZmqeWTYTMzMzNLLZ8M\nm5mZmVlq+WTYzMzMzFLLJ8NmZmZmllo+GTYzMzOz1FopzgerU6dOybe7W7ZsWZ2a3tfzEeX5iPJ8\nRHk+ojwfUZ6PKM9HeZ6TKM9H4MiwmZmZmaWWT4bNzMzMLLV8MmxmZmZmqeWTYTMzMzNLLZ8Mm5mZ\nmVlq+WTYzMzMzFLLJ8NmZmZmllqx1hm24rH77rsD0K1bNwBOPfVUAB599FEA7rnnHgDef//9AozO\nzKy43HXXXQBceOGFAEybNq3se4cddhgAc+fOjX9gZikyfvx4AOrU+a98cNu2bXPyex0ZNjMzM7PU\nKqnIcN26dQFYd911K72PIqFrrLEGAM2aNQPgggsuAOC2224D4MQTTwTg999/B6B///4AXH/99bke\ndmx22WWXsn+PGzcOgHXWWQeAZcv+a0TTuXNnADp27AjABhtsEOcQi167du0AGDlyJAD77bcfADNn\nzizYmOLUu3dvILwP/ve//66n27RpA8Drr79ekHFZvNZee20A1lprLQAOPfRQADbaaCMA7rjjDgD+\n+OOPAowutxo1agTAKaecAsC///4LwHbbbVd2n2233RZIR2S4adOmAKy88soAtG7dGoD777+/7D6a\no+o888wzAJxwwgkA/PnnnzkbZyFoTvbee28A+vbtW/a9ffbZpyBjKhV33nknEOZWq9i54siwmZmZ\nmaVWoiLDW2yxBQCrrLIKEK4Q9t13XwDWW289AI455pga/84vv/wSgLvvvhuAo446CoClS5cC8NFH\nHwHJjnjtueeeAIwePbrsNkXPFRHW89WVuSLCLVu2BELucCGv3BWB0Nieeuqp2MfQokULAKZOnRr7\nYxfS6aefDsAVV1wBlI/86HVkpUeRUf3tAVq1agXAjjvuWOHPNGzYEAj5tUn23XffAfDGG28AYdUs\nLXbYYQcgHAOOO+44IKwKbbLJJkD0mFDT44HmctCgQQD06NEDgJ9++qmWoy4Mfa5OnDgRgAULFpR9\nr0GDBuVus+ppVf7cc88F4K+//gJC7nCuODJsZmZmZqmViMiwcl0nTJgAVJ0TXFO6ilUO5M8//wyE\nXNBvvvkGgMWLFwPJyglVPvRuu+0GwIgRI4AQranIrFmzALjlllsAePzxxwGYNGkSEOapX79+eRhx\nzSgvtUmTJkC8kWFFQRo3bgzAlltuCYQdraVOz3e11VYr8EjyZ6+99gJCbqjywSFEx+Syyy4D4Ouv\nvwbC6pTea1OmTMnvYPNI+a+K0p188skArL766mX30et+/vz5QFhZUh5tp06dgJBHOmPGjHwPO29+\n+eUXIB35wBXRMf+QQw7J22OomtGDDz4IhM+dpFM0OPPfjgwvH61OKx/7rbfeAmDUqFE5fRxHhs3M\nzMwstRIRGZ43bx4AixYtAmoeGc6MzixZsgSA/fffHwi5r8OHD8/ZOIvFAw88AISKGDWhKLJ2hytH\nWtHYnXfeOYcjXDGKHkyePDn2x1ZU/ayzzgJCBDDJEa+aaN++PQDdu3eP3K7nrfqqCxcujHdgOXT8\n8ccDoY7shhtuCESj/q+99hoQqiXceuutkd+h++r72h2fBDqeDhgwAAjzoYoRFdFK0oEHHgiEqI1e\nF5pDfU0y7UVp3rx5gUdSGKo8lB0Z/vbbb4EQzdXqGZTfU6D9PZmrLWmQlpXDymifT69evYDoOckP\nP/xQ5c/qvtqX8NlnnwFhVS7XHBk2MzMzs9RKRGRYVxA9e/YEQjTqgw8+AEIlCPnwww8B6NChQ9lt\nyvtS7t9FF12UxxEXhrrKqeZn9lVpZkWMsWPHAqGusnIfNafKlVZ3l2K4ws2MPMRt6NChkf8rMlaq\nlAP78MMPA+VXYxQZTWIe5Uor/XfY22OPPQAYMmQIEHLtVTXgxhtvLPsZ5amtuuqqQMhXO+CAAyK/\n+913383XsPNGFXS6du1a5f0UmYFwbFXO8DbbbJOn0RWeXheqZlQRVZlRZDyJ74vKDBw4EICnn346\ncrt29dckB1b17NW1TxUoRL87ie+fqmRW1Sjl/RaVGTx4MBD2+Wy//fZl39MxtTJXX301EKpHaVVW\nFb5yzZFhMzMzM0utRESGRVePqiqhHczK5erSpQsQop2KBmf6+OOPATj77LPzO9gYqdpGZV3lXnzx\nRSCar6PcLVWJUORTNTV19aXcL0WblVususNxUL5y/fr1Y3vMbNmRUc11qTrttNOA8hEc5c7muvtP\nnFQtIjvar7+pcmYrqnWq72VHhFWv/JFHHsntYGOgurHZ5syZA4Sa2pl1hhURlsxubKVGq2bDhg0D\n4Lrrrit3H92mvSn33ntvHEOLxd9//w2U/5svD+WW16tXr8Lv6/1TCh0LK6OVqHfeeafAI4nPr7/+\nCoRzkZpEx3U+owpGOgfJd2TdkWEzMzMzS61ERYYlO2Lz448/Rv6v3JInnnii7Laa9kpPEvWIVy61\nopfff/89EGolK1qlWsoAzz//fORrdVRj9NJLLwVC7dE4aBdzZp3TuCgarfrC8tVXX8U+lnzL3Pl/\n5plnAuF9o4jXTTfdFP/AckQ5wMpFU7RCtXC1SlJV9yvtis6mTmtaWUkSHS+1WvbKK68AMHv2bCBU\nDahKIVdt4qLXT0WRYauYqqroNVbZMfzaa6+NbUz5pCi6zkkyVxS33nrrgoypEPRe2WmnnQCYPn06\nUHW+75prrgmEFSjl6iuS/uSTT+ZnsP/PkWEzMzMzS61ERoaz6Upd1RSUD6saqRCiHaVAO9qVG63I\nqXKoVY9XO3NzGVGtakd1vjRr1izyf+V9x0FzrMjXp59+CoS5LgWNGjUCYPTo0ZXe55577gFg4sSJ\ncQwppxR1UkRYNcZffvllIEQifvvtt8jPZeaoKUdYr39VV1Gk/JlnnsnL2OOgnNjaRDxbtWqVo9EU\nP1W1KcXVxtrIXC288sorgVBlRHWos6nykypTJJ1W0N58800gVL5Ki8033xwIKwGKlHfr1g2oeuXs\njjvuAMIeBh2X9tlnn/wMNosjw2ZmZmaWWiURGVbVCF2NqNKB6odCiGgpWnrfffcB0TqASbHrrrsC\n5TsCHXHEEUC0nnAp0u72XFIFjoMOOggIFQeyqwYoF0oRgFKg51xRl8Hx48cDoTtbkqhz2Pnnnw+E\n97oiwkceeWSFP6do1siRI8tu06qTKH/tlltuyeGIi5PyoZXTVxHlBsrbb78NFKZbZL4pIpzEz44V\noZWjzp07A9EV10yqTQ6Vz43y8RU5fuGFF4DyqzKWLOoS99RTTwFh/4lWFKs6J1FHudNPPz1y+803\n35zrYVbJkWEzMzMzS62SiAyLOiTpCkPdsyBc1eqrohyql6rKC0mg3BrlLeqqKx8R4WLMj1t//fWr\n/L7qTmt+FMnYbLPNAFhllVWAaI6bnqciFFOmTAFC3Ut1LXvvvfdq/wSKhCKj/fv3L/c9dQdSveHs\nii1JoL9zZpUMCJHOjTfeGIAzzjgDgI4dOwIhyrHWWmuV/YwiXfo6YsQIoOJa5kml3dvqEtWnTx+g\n/AoUVH5cUJ6f5vSff/7Jz2At7/Q+ePbZZ4Hc7BdRLq06k6WJOqmVAn0eagX1wQcfBMofF7SX4Kqr\nrgLCuQuEz3HlCOvzWudkDzzwQP6eQAUcGTYzMzOz1PLJsJmZmZmlVkmlSYiSuGfNmlV2m8Lz7dq1\nA6Bv375AaPmnZO1ibqagMi1qV6glWy1j5UP2ZhGVwomTUhc0hkGDBgGhVFY2bQTTsovKu6g15Cef\nfALAQw89VPYz2lipVJOFCxcCoU2oytPNmDGj1s+n0GpSSu3zzz8HwjwkkUqoqZzPRhttBMAXX3wB\nVL7JR0v9mc03GjZsCISGNmPHjs3DiOOlclfakKvXg56r3neaj8zNcNp0qdQK0fLp0UcfDYSNl/pb\nWPLoOKqvldESOVSeVqfPsIMPPhiAF198MRdDTASlYZUCNVNRS3sdS/V3V8MetaDWV23yB9h0002B\ncLzRcVoNn+LmyLCZmZmZpVZJRoZl2rRpZf/u1KkTAIcffjgQNtedc845ADRp0gSADh06xDnE5aLo\npDYGqU1qZtvp2lJDj+wC/BMmTABCInycVBpr7ty5AOy9995V3n/evHkAPP3000BoBam2jjWh1rSK\nJipSWgrUZKKqTZEVbapLGpW/00bB5557DggbN7ThVg0zhg0bBsAPP/wAwOOPP172uxS9yLwtqXT8\nUHR3zJgxke9ff/31QHjPT5o0CYhuXNX3tMlK9H7p168fUP69qA2pSVbVpuLWrVsDcO+998Y6pnzQ\n52ebNm2AsFlKpQl///33an9Hly5dAOjevXseRli8VMq11JpuHH/88UA4f1KzFB1rTzrpJAAWL14M\nwO233w6ERmiKEENYaVBUWRud58+fD4TXnY7T+ebIsJmZmZmlVklHhjPpymX48OFAyHVRjpuu6HU1\n8tprr8U7wBWgKEsuysIpIty7d28AevbsCYScWV3h/fzzz7V+rBU1YMCA2B5LueVSVX5tUijXPLuR\niGS2FJ45c2YsY4qDyuQpalkdHQsUzYAQBUzyCoFyhBX51XtclL+pQvk6Zmre1CABQpMN5QKr+Ygi\nxcoNVOOSV199FQjvYUWOpBB7EVZUVU03lCut8nTan5BkWpFbkSYIWmFMW2RYKyKZ9P7TPiXNa5Jo\nJV3PT+3oM8vYZtLfXWXSqmrbrkixoupxRYTFkWEzMzMzS62Sjgxntpc99thjAWjRogUQIsKiK/g3\n3ngjptHVXi6qSChaqCiRcoIUJTzmmGNq/RilQBVKkuyVV14BoF69epHblUud3Q4zrZSbn5kTqihg\nEnOG69atC4RW4mp/qoYhao2r56aIsPL7lP+qqhMQKvWcd955QIjmqK258vrV2EY76ceNGxcZm/ID\nGzduXKvnGCdVs1GUrCLac9CjR49YxlSsDjzwwEIPoSBUwSiTIp9ahU0inRdon4Hev5VRHnD23gKA\nE088EYju7YKwGh03R4bNzMzMLLVKKjLcrFkzALp16waE/C2ABg0aVPgzaheqvNtiajucLbveo3bJ\nX3TRRcv9uy6++GIArrnmGgDWXXddIOT4nXrqqbUbrBUdtQPNfo3ff//9QGHzwYuJdsuXCkUpFRFW\nvW1FNrVi0LJlSyC0UlYtWEXKb7jhhrLfqRzB7MiQajO/9NJLka+KAmm3ueg4lCSlUGu8Ispp1Z4C\nVQxRvenlodeQ6kynjSKoma+VbbfdFgirBaqSlCQ1/XvqfEKtlrVilJkHPGrUqByPrnYcGTYzMzOz\n1Ep0ZFjRXkUdFBFWh62qqOOYdsjms4tbrihvUV/1/O+++24gdFRbtGgRECI9nTt3BqB58+Zlv2uz\nzTYDwq5QRcMUJbT/KArftGlTYPlqFRcLRfEyO0Rlevvtt+McTtErtTzHa6+9NvJ/5RBrn4B2/G+z\nzTYV/ry+r9rBEFbUauqxxx6LfE0yVdvIrJCw9dZbR+6j1TrdN+6d8ctj3333BaBXr15AqLWvPO7q\n8kJVf/qQQw4pu00dX7M7FCrKXJMaxaVAqy4QOq5dcsklhRpObBT11p4C9URo27ZtwcZUHUeGzczM\nzCy1EhUZrl+/PhBqOGqXs3JxqqJao7feeisQcnqKOUe4Oorw6CpMlR+Ut6euehVRNFC7wLOjR/Yf\nReEri6oWM1UKad++PRBe66oNe9999wGwcOHCAoyueG211VaFHkJOLViwAAj1grWbPXOlCEIdYVXU\nUde4OXPmAMsfDS51H3/8cdm/s18zSfpc0edo9o7/yy+/HIClS5dW+fOKJO+2225lt2XXYFbd/oED\nBwLhcydNNCc6/pYi1VDu2rUrEJ7z4MGDgcJViqiJ5H3Cm5mZmZnlSFFHhpWLpO4linRVF7lR1FNd\n0yDkxK7IzthiMXnyZACmTp0KhJrJohxiRdBFOcSZNVJXpAJFmqlzzrBhwwo7kOWw3nrrAeUrqXz1\n1VdAqC5gUW+++SYQXQ1IUqQvmzrqqfqMInjK49NeA3WFK+XIVS4p2gVw+OGHF3Ak+aF8zxWh19bY\nsWOB8HmTllzhiqiigrozlkLt+myqI64I8YgRIwDo06dPwcZUU44Mm5mZmVlqFVVkeK+99gLCLuc9\n99wTCLswK6O6maqq0LdvXyB0WCoVyrdR/WTVCe3du3eF91dNQOVpzZ49O99DLDmqJmHpoY5I6rIG\nYTVKVQO+++67+Ae2gpTzOXz48MhXqx11LQWYPn06ANttt12hhrPC1HlS1TFOO+20Gv2cKmTo81cr\nKhCi5tndxdKmU6dOZf/+448/gPBaKUWqXKRul9qblQSODJuZmZlZatXJ3vWZ1werU6fKB+vfvz8Q\nIsPZdCX+3HPPAaH/t3KDlyxZkpuB1sKyZctqHEqsbj5KQVLnQ9ES5VMOGTIECNH4FRXnfChX+Ikn\nngBCPdEvvvgCqLyubJyK+fWh1wDA0KFDAXj99deBEEXLjA7mQjHPRyF4PqLyOR+qMqLX/U033QRA\nvXr1gFBdRHmhivqpWkkhLM98QPyvkcx9Olo16NixIwBz587Ny2P6PRNV0/lwZNjMzMzMUquoIsOl\nwFdlUZ6PKM9HVDHPh3Z/A4waNQoINZvHjBkDwBlnnAHkbn9CMc9HIXg+ojwfUcUeGS4Ev0aiHBk2\nMzMzM6uGI8M55quyKM9HlOcjKinzoSjxzTffDIQarDvvvDOQu9zhpMxHXDwfUZ6PKEeGy/NrJMqR\nYTMzMzOzajgynGO+KovyfER5PqI8H1GejyjPR5TnI8qR4fL8GolyZNjMzMzMrBqxRobNzMzMzIqJ\nI8NmZmZmllo+GTYzMzOz1PLJsJmZmZmllk+GzczMzCy1fDJsZmZmZqnlk2EzMzMzSy2fDJuZmZlZ\navlk2MzMzMxSyyfDZmZmZpZaPhk2MzMzs9TyybCZmZmZpZZPhs3MzMwstXwybGZmZmap5ZNhMzMz\nM0stnwybmZmZWWr5ZNjMzMzMUssnw2ZmZmaWWj4ZNjMzM7PU8smwmZmZmaWWT4bNzMzMLLV8Mmxm\nZmZmqeWTYTMzMzNLLZ8Mm5mZmVlq/R+5rShc4lHd5wAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x2b12655ccd90>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.figure(figsize=(12,3))\n",
    "for i in range(10):\n",
    "    plt.subplot(1, 10, i+1)\n",
    "    plt.imshow(X_train[i].reshape((28, 28)), cmap='gray', interpolation='nearest')\n",
    "    plt.axis('off')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 90,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "X_train = X_train.reshape((60000, 784))\n",
    "X_test = X_test.reshape((10000, 784))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 80,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# a = np.zeros((y_train.size, 10), dtype=np.int)\n",
    "# a[np.arange(y_train.size), y_train] = 1\n",
    "# y_train = a"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 81,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# a = np.zeros((y_test.size, 10), dtype=np.int)\n",
    "# a[np.arange(y_test.size), y_test] = 1\n",
    "# y_test = a"
   ]
  },
  {
   "cell_type": "raw",
   "metadata": {},
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "NUM_EPOCHS = 100\n",
    "BATCH_SIZE = 16\n",
    "N_HIDDEN = 100\n",
    "SEQ_LEN = 784\n",
    "TRAIN_SIZE = 60000\n",
    "TEST_SIZE = 10000\n",
    "GRAD_CLIP = 10"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 86,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# cross-entropy\n",
    "# ys are indices of chars, x is a matrix (? BATCH_SIZE * SEQ_LEN, VOCAB_SIZE)\n",
    "def lsmCE(x,y):\n",
    "    print(y)\n",
    "    return -T.clip(x,-20,0)[T.arange(y.shape[0]), y]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# Технические вещи\n",
    "\n",
    "# Вспомогательная функция для запаковки результата обучения \n",
    "def pack(network, inp, target, train_err, test_err,\n",
    "         train_acc, test_acc, train_fn, test_fn, norm):\n",
    "    return {'network':network,\n",
    "            'inp':inp,\n",
    "            'target':target,\n",
    "            \n",
    "            'train_err':train_err,\n",
    "            'test_err':test_err,\n",
    "            'train_acc':train_acc, \n",
    "            'test_acc':test_acc, \n",
    "            'train_fn':train_fn, \n",
    "            'test_fn':test_fn,\n",
    "            'norm':norm,\n",
    "            } "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 96,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Create a network with particular initialization for parameters\n",
    "def create_network(seq_len, dim, hidden, rnn_type):\n",
    "    print(\"Building network ...\")\n",
    "    inp = T.tensor3('input', dtype='float64')\n",
    "    # Matrix of size BATCH_SIZE * 1 to match network's output\n",
    "    target_values = T.ivector('target_output')\n",
    "    network = build_network(inp, seq_len, dim, hidden, rnn_type)\n",
    "    print(\"The network has {} params\".format(lasagne.layers.count_params(network)))\n",
    "    return (network, inp, target_values)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 103,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "def train(network, inp, target_values, Xtrain, ytrain, Xtest, ytest,\n",
    "              filename, lr, num_epochs=NUM_EPOCHS):\n",
    "    num_batches = TRAIN_SIZE // BATCH_SIZE\n",
    "    train_err=np.zeros(num_epochs)\n",
    "    train_acc=np.zeros(num_epochs)\n",
    "    test_err=np.zeros(num_epochs)\n",
    "    test_acc=np.zeros(num_epochs)\n",
    "    norm_hist=np.zeros(num_epochs)\n",
    "    \n",
    "    network_output = lasagne.layers.get_output(network)\n",
    "    cr_ent = lasagne.objectives.categorical_crossentropy(network_output, target_values).mean()\n",
    "    acc_score = lasagne.objectives.categorical_accuracy(\n",
    "        network_output, target_values).mean(dtype=theano.config.floatX) * 100\n",
    "    \n",
    "    # Retrieve all parameters from the network\n",
    "    all_params = lasagne.layers.get_all_params(network)\n",
    "    print(\"Computing updates ...\")\n",
    "    all_grads = T.grad(cr_ent, all_params)\n",
    "    scaled_grads, grad_norm = lasagne.updates.total_norm_constraint(all_grads, GRAD_CLIP, return_norm=True)\n",
    "    lr_var = theano.shared(lr)\n",
    "    updates = lasagne.updates.sgd(scaled_grads, all_params, learning_rate=lr_var)\n",
    "\n",
    "    print(\"Compiling functions ...\")    \n",
    "    train_fn = theano.function([inp, target_values], [cr_ent, acc_score, grad_norm, network_output],\n",
    "                               updates=updates, allow_input_downcast=True)\n",
    "    test_fn = theano.function([inp, target_values], [cr_ent, acc_score], allow_input_downcast=True)\n",
    "    get_output = theano.function([inp], network_output, allow_input_downcast=True)\n",
    "    \n",
    "    train_idxs = np.arange(TRAIN_SIZE)\n",
    "    np.random.shuffle(train_idxs)\n",
    "    for epoch in range(num_epochs):\n",
    "        start_time = time.time()\n",
    "        for batch in range(num_batches):\n",
    "            idx = train_idxs[BATCH_SIZE * batch : BATCH_SIZE * (batch + 1)]            \n",
    "            err, acc, norm, out = train_fn(Xtrain[idx, :, np.newaxis], ytrain[idx])\n",
    "            if np.isnan(err):\n",
    "                print(\"Error: crossentropy became nan\")\n",
    "                np.save(\"net_output\", out)\n",
    "                raise\n",
    "            elif np.isnan(norm):\n",
    "                print(\"Error: grad norm is nan\")\n",
    "                raise\n",
    "            train_err[epoch] += err\n",
    "            train_acc[epoch] += acc\n",
    "            \n",
    "            #if batch == num_batches // 4:\n",
    "            #    print(\"25%\", end=\" \")\n",
    "            #if batch == num_batches // 2:\n",
    "            #    print(\"50%\", end=\" \")\n",
    "            #if batch == num_batches // 4 * 3:\n",
    "            #    print(\"75%\")\n",
    "        \n",
    "        np.random.shuffle(train_idxs)\n",
    "            \n",
    "        train_err[epoch] /= num_batches\n",
    "        train_acc[epoch] /= num_batches\n",
    "        \n",
    "        terr, tacc = test_fn(Xtest[:, :, np.newaxis], ytest)\n",
    "        test_err[epoch] = terr\n",
    "        test_acc[epoch] = tacc\n",
    "        norm_hist[epoch] = norm.item()\n",
    "        \n",
    "        print(\"Epoch {} loss / acc test = {:.4f}, {:.4f} \\t\"\n",
    "              \"train = {:.4f}, {:.4f} norm = {:.4f} \\t time = {:.2f}s\".\n",
    "              format(epoch, test_err[epoch], test_acc[epoch], \n",
    "                     train_err[epoch], train_acc[epoch], norm.item(), time.time() - start_time))\n",
    "    \n",
    "    np.save(file=filename, arr=lasagne.layers.get_all_param_values(network))\n",
    "    return pack(network, inp, target_values, train_err, test_err, train_acc, test_acc, train_fn, test_fn, norm_hist)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Building network ...\n",
      "The network has 11310 params\n",
      "Computing updates ...\n",
      "Compiling functions ...\n"
     ]
    }
   ],
   "source": [
    "net, inp, tar = create_network(SEQ_LEN, 1, N_HIDDEN, 'nprnn')\n",
    "nprnn1 = train(net, inp, tar, X_train, y_train_, X_test, y_test_, \"mnist-data/nprnn1\", 1e-4, num_epochs=33)\n",
    "np.save(\"mnist-data/acc1\", nprnn1[\"test_acc\"])\n",
    "np.save(\"mnist-data/err1\", nprnn1[\"test_err\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# COOL DOWN LEARNING RATE AFTER 30 EPOCHS !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# net, inp, tar = create_network(build_network_np)\n",
    "# weights = np.load(\"nprnn1.npy\")\n",
    "# lasagne.layers.set_all_param_values(net, weights)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Computing updates ...\n",
      "Compiling functions ...\n",
      "25% 50% 75%\n",
      "Epoch 0 loss / acc test = 2.2970, 12.1000 \ttrain = 2.2963, 12.2983 norm = 0.3282 \t time = 544.89s\t25% 50% 75%\n",
      "Epoch 1 loss / acc test = 2.2968, 12.1000 \ttrain = 2.2961, 12.3083 norm = 0.3289 \t time = 558.65s\t25% 50% 75%\n",
      "Epoch 2 loss / acc test = 2.2967, 12.1000 \ttrain = 2.2959, 12.3133 norm = 0.3297 \t time = 562.69s\t25% 50% 75%\n",
      "Epoch 3 loss / acc test = 2.2965, 12.1000 \ttrain = 2.2956, 12.3233 norm = 0.3305 \t time = 557.25s\t25% 50% 75%\n",
      "Epoch 4 loss / acc test = 2.2963, 12.1000 \ttrain = 2.2954, 12.3300 norm = 0.3313 \t time = 559.49s\t25% 50% 75%\n",
      "Epoch 5 loss / acc test = 2.2961, 12.1100 \ttrain = 2.2951, 12.3317 norm = 0.3345 \t time = 563.65s\t25% 50% 75%\n",
      "Epoch 6 loss / acc test = 2.2959, 12.1200 \ttrain = 2.2948, 12.3333 norm = 0.3355 \t time = 557.41s\t25% 50% 75%\n",
      "Epoch 7 loss / acc test = 2.2956, 12.1200 \ttrain = 2.2945, 12.3367 norm = 0.3365 \t time = 557.89s\t25% 50% 75%\n",
      "Epoch 8 loss / acc test = 2.2954, 12.1200 \ttrain = 2.2942, 12.3383 norm = 0.3376 \t time = 562.76s\t25% 50% 75%\n",
      "Epoch 9 loss / acc test = 2.2951, 12.1300 \ttrain = 2.2939, 12.3417 norm = 0.3388 \t time = 562.46s\t25% 50% 75%\n",
      "Epoch 10 loss / acc test = 2.2948, 12.1300 \ttrain = 2.2935, 12.3433 norm = 0.3401 \t time = 559.92s\t25% 50% 75%\n",
      "Epoch 11 loss / acc test = 2.2945, 12.1300 \ttrain = 2.2931, 12.3467 norm = 0.3415 \t time = 566.43s\t25% 50% "
     ]
    }
   ],
   "source": [
    "nprnn1 = train_sgd(nprnn1[\"network\"], nprnn1[\"inp\"], nprnn1[\"target\"],\n",
    "                   X_train, y_train, X_test, y_test, \"nprnn1\", 1e-5, num_epochs=33)\n",
    "np.save(\"acc3\", nprnn1[\"test_acc\"])\n",
    "np.save(\"err3\", nprnn1[\"test_err\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Computing updates ...\n",
      "Compiling functions ...\n"
     ]
    }
   ],
   "source": [
    "nprnn1 = train_sgd(nprnn1[\"network\"], nprnn1[\"inp\"], nprnn1[\"target\"],\n",
    "                   X_train, y_train, X_test, y_test, \"nprnn1\", 1e-6, num_epochs=33)\n",
    "np.save(\"acc4\", nprnn1[\"test_acc\"])\n",
    "np.save(\"err4\", nprnn1[\"test_err\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([ 20.68,  20.68,  20.69,  20.69,  20.68,  20.69,  20.69,  20.69,\n",
       "        20.69,  20.69,  20.69,  20.71,  20.71,  20.73,  20.73,  20.75,\n",
       "        20.74,  20.75,  20.78,  20.79,  20.81,  20.8 ,  20.84,  20.84,\n",
       "        20.84,  20.85,  20.87,  20.87,  20.91,  20.92,  20.91,  20.9 ,\n",
       "        20.91])"
      ]
     },
     "execution_count": 46,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.load(\"acc4.npy\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "## IRNN"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Building network ...\n",
      "The network has 11310 params\n",
      "Computing updates ...\n",
      "Compiling functions ...\n"
     ]
    }
   ],
   "source": [
    "net, inp, tar = create_network(build_network_identity)\n",
    "irnn1 = train_sgd(net, inp, tar, X_train, y_train, X_test, y_test, \"irnn/irnn1\", 1e-8, num_epochs=10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 90,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "np.save(\"irnn/irnn-acc1\", irnn1[\"test_acc\"])\n",
    "np.save(\"irnn/irnn-err1\", irnn1[\"test_err\"])\n",
    "np.save(\"irnn/irnn-norm1\", irnn1[\"norm\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 95,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([  7.42,   8.12,   7.28,  10.03,   9.92,   9.74,  11.33,  13.4 ,\n",
       "        13.12,  12.91])"
      ]
     },
     "execution_count": 95,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.load(\"irnn/irnn-acc1.npy\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Computing updates ...\n",
      "Compiling functions ...\n"
     ]
    }
   ],
   "source": [
    "irnn1 = train_sgd(irnn1[\"network\"], irnn1[\"inp\"], irnn1[\"target\"],\n",
    "                  X_train, y_train, X_test, y_test, \"irnn/irnn1\", 1e-8, num_epochs=20)\n",
    "np.save(\"irnn/irnn-acc2\", irnn1[\"test_acc\"])\n",
    "np.save(\"irnn/irnn-err2\", irnn1[\"test_err\"])\n",
    "np.save(\"irnn/irnn-norm2\", irnn1[\"norm\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 97,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([ 12.78,  12.91,  12.94,  13.21,  14.17,  15.01,  15.22,  15.31,\n",
       "        15.3 ,  15.33,  15.37,  15.39,  15.39,  15.4 ,  15.44,  15.54,\n",
       "        15.69,  15.84,  15.96,  16.08])"
      ]
     },
     "execution_count": 97,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.load(\"irnn/irnn-acc2.npy\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Computing updates ...\n",
      "Compiling functions ...\n"
     ]
    }
   ],
   "source": [
    "irnn1 = train_sgd(irnn1[\"network\"], irnn1[\"inp\"], irnn1[\"target\"],\n",
    "                  X_train, y_train, X_test, y_test, \"irnn/irnn1\", 1e-8, num_epochs=30)\n",
    "np.save(\"irnn/irnn-acc3\", irnn1[\"test_acc\"])\n",
    "np.save(\"irnn/irnn-err3\", irnn1[\"test_err\"])\n",
    "np.save(\"irnn/irnn-norm3\", irnn1[\"norm\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Computing updates ...\n",
      "Compiling functions ...\n"
     ]
    }
   ],
   "source": [
    "irnn1 = train_sgd(irnn1[\"network\"], irnn1[\"inp\"], irnn1[\"target\"],\n",
    "                  X_train, y_train, X_test, y_test, \"irnn/irnn1\", 1e-8, num_epochs=20)\n",
    "np.save(\"irnn/irnn-acc4\", irnn1[\"test_acc\"])\n",
    "np.save(\"irnn/irnn-err4\", irnn1[\"test_err\"])\n",
    "np.save(\"irnn/irnn-norm4\", irnn1[\"norm\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 104,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([ 16.24,  16.38,  16.51,  16.63,  16.8 ,  16.9 ,  17.17,  17.31,\n",
       "        17.41,  17.62,  17.81,  17.89,  17.93,  18.08,  18.2 ,  18.25,\n",
       "        18.24,  18.13,  18.14,  18.14,  18.11,  18.12,  18.12,  18.12,\n",
       "        18.06,  18.09,  18.04,  18.02,  17.95,  17.95])"
      ]
     },
     "execution_count": 104,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.load(\"irnn/irnn-acc3.npy\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 103,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([ 17.98,  17.96,  17.9 ,  17.86,  17.9 ,  17.91,  17.87,  17.87,\n",
       "        17.85,  17.88,  17.88,  17.9 ,  17.88,  17.88,  17.87,  17.92,\n",
       "        17.93,  17.96,  17.97,  17.96])"
      ]
     },
     "execution_count": 103,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.load(\"irnn/irnn-acc4.npy\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Computing updates ...\n",
      "Compiling functions ...\n"
     ]
    }
   ],
   "source": [
    "irnn1 = train_sgd(irnn1[\"network\"], irnn1[\"inp\"], irnn1[\"target\"],\n",
    "                  X_train, y_train, X_test, y_test, \"irnn/irnn1\", 1e-8, num_epochs=20)\n",
    "np.save(\"irnn/irnn-acc5\", irnn1[\"test_acc\"])\n",
    "np.save(\"irnn/irnn-err5\", irnn1[\"test_err\"])\n",
    "np.save(\"irnn/irnn-norm5\", irnn1[\"norm\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 107,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([ 17.94,  17.98,  17.99,  18.01,  18.03,  18.05,  18.04,  18.05,\n",
       "        18.12,  18.12,  18.11,  18.13,  18.14,  18.17,  18.2 ,  18.24,\n",
       "        18.29,  18.29,  18.28,  18.29])"
      ]
     },
     "execution_count": 107,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.load(\"irnn/irnn-acc5.npy\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## LSTM"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Building network ...\n",
      "The network has 42310 params\n",
      "Computing updates ...\n",
      "Compiling functions ...\n"
     ]
    }
   ],
   "source": [
    "net, inp, tar = create_network(build_lstm)\n",
    "lstm = train_sgd(net, inp, tar, X_train, y_train, X_test, y_test, \"lstm/lstm1\", 1e-2, 1, num_epochs=10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "name 'lstm' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-4-c41288a1bc80>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m()\u001b[0m\n\u001b[1;32m----> 1\u001b[1;33m \u001b[0mlstm\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[1;31mNameError\u001b[0m: name 'lstm' is not defined"
     ]
    }
   ],
   "source": [
    "np.save(\"lstm/lstm-acc1\", lstm[\"test_acc\"])\n",
    "np.save(\"lstm/lstm-err1\", lstm[\"test_err\"])\n",
    "np.save(\"lstm/lstm-norm1\", lstm[\"norm\"])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "кернел всё время умирает :("
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## nRNN"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Building network ...\n",
      "The network has 11310 params\n"
     ]
    }
   ],
   "source": [
    "net, inp, tar = create_network(build_network_norm)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Computing updates ...\n",
      "Compiling functions ...\n",
      "25% 50% 75%\n",
      "Epoch 0 loss / acc test = 2.3008, 12.1100 \ttrain = 2.3010, 11.5000 norm = 0.2411 \t time = 553.16s\t25% 50% 75%\n",
      "Epoch 1 loss / acc test = 2.2997, 12.1300 \ttrain = 2.2999, 12.3500 norm = 0.2498 \t time = 568.38s\t25% 50% 75%\n",
      "Epoch 2 loss / acc test = 2.2981, 12.1300 \ttrain = 2.2983, 12.3583 norm = 0.2555 \t time = 554.18s\t25% 50% 75%\n",
      "Epoch 3 loss / acc test = 2.2954, 12.1300 \ttrain = 2.2957, 12.3600 norm = 0.2601 \t time = 554.17s\t25% 50% 75%\n",
      "Epoch 4 loss / acc test = 2.2902, 12.7200 \ttrain = 2.2903, 12.4083 norm = 0.2949 \t time = 570.95s\t25% 50% 75%\n",
      "Epoch 5 loss / acc test = 2.2125, 19.7500 \ttrain = 2.2692, 16.9233 norm = 3.3974 \t time = 559.90s\t25% 50% 75%\n",
      "Epoch 6 loss / acc test = 2.1204, 20.1500 \ttrain = 2.1359, 20.1450 norm = 9.1876 \t time = 561.41s\t25% 50% 75%\n",
      "Epoch 7 loss / acc test = 2.1058, 19.9700 \ttrain = 2.1021, 20.8750 norm = 11.3619 \t time = 562.18s\t25% 50% 75%\n",
      "Epoch 8 loss / acc test = 2.0925, 19.4300 \ttrain = 2.0875, 21.2767 norm = 12.5960 \t time = 559.69s\t25% 50% 75%\n",
      "Epoch 9 loss / acc test = 2.0780, 19.0000 \ttrain = 2.0726, 21.2250 norm = 12.8876 \t time = 555.80s\t"
     ]
    }
   ],
   "source": [
    "norm_rnn = train_sgd(net, inp, tar, X_train, y_train, X_test, y_test, \"nrnn/nrnn1\", 1e-4, 10, num_epochs=10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "np.save(\"nrnn/nrnn-acc1\", norm_rnn[\"test_acc\"])\n",
    "np.save(\"nrnn/nrnn-err1\", norm_rnn[\"test_err\"])\n",
    "np.save(\"nrnn/nrnn-norm1\", norm_rnn[\"norm\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Computing updates ...\n",
      "Compiling functions ...\n"
     ]
    }
   ],
   "source": [
    "norm_rnn = train_sgd(norm_rnn[\"network\"], norm_rnn[\"inp\"], norm_rnn[\"target\"],\n",
    "                     X_train, y_train, X_test, y_test, \"nrnn/nrnn1\", 1e-4, 10, num_epochs=20)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "np.save(\"nrnn/nrnn-acc2\", norm_rnn[\"test_acc\"])\n",
    "np.save(\"nrnn/nrnn-err2\", norm_rnn[\"test_err\"])\n",
    "np.save(\"nrnn/nrnn-norm2\", norm_rnn[\"norm\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Computing updates ...\n",
      "Compiling functions ...\n"
     ]
    }
   ],
   "source": [
    "norm_rnn = train_sgd(norm_rnn[\"network\"], norm_rnn[\"inp\"], norm_rnn[\"target\"],\n",
    "                     X_train, y_train, X_test, y_test, \"nrnn/nrnn1\", 1e-5, 10, num_epochs=30)\n",
    "np.save(\"nrnn/nrnn-acc3\", norm_rnn[\"test_acc\"])\n",
    "np.save(\"nrnn/nrnn-err3\", norm_rnn[\"test_err\"])\n",
    "np.save(\"nrnn/nrnn-norm3\", norm_rnn[\"norm\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Computing updates ...\n",
      "Compiling functions ...\n"
     ]
    }
   ],
   "source": [
    "norm_rnn = train_sgd(norm_rnn[\"network\"], norm_rnn[\"inp\"], norm_rnn[\"target\"],\n",
    "                     X_train, y_train, X_test, y_test, \"nrnn/nrnn1\", 1e-6, 10, num_epochs=40)\n",
    "np.save(\"nrnn/nrnn-acc4\", norm_rnn[\"test_acc\"])\n",
    "np.save(\"nrnn/nrnn-err4\", norm_rnn[\"test_err\"])\n",
    "np.save(\"nrnn/nrnn-norm4\", norm_rnn[\"norm\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 78,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([ 33.99,  33.93,  33.94,  33.86,  33.95,  34.  ,  33.95,  33.92,\n",
       "        33.92,  33.9 ,  33.89,  33.86,  33.86,  33.86,  33.86,  33.85,\n",
       "        33.85,  33.88,  33.88,  33.85,  33.87,  34.16,  34.33,  34.41,\n",
       "        34.61,  34.65,  34.78,  34.81,  34.88,  34.96])"
      ]
     },
     "execution_count": 78,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.load(\"nrnn/nrnn-acc3.npy\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Train with Adam"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 121,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([5, 0, 4, ..., 5, 6, 8])"
      ]
     },
     "execution_count": 121,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y_train_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "y_train_ = np.zeros((y_train.shape[0]), dtype=np.int)\n",
    "y_test_ = np.zeros((y_test.shape[0]), dtype=np.int)\n",
    "for i in range(y_train.shape[0]):\n",
    "    y_train_[i] = np.where(y_train[i] == 1)[0][0]\n",
    "for i in range(y_test.shape[0]):\n",
    "    y_test_[i] = np.where(y_test[i] == 1)[0][0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Building network ...\n",
      "The network has 11310 params\n",
      "Computing updates ...\n",
      "Compiling functions ...\n"
     ]
    }
   ],
   "source": [
    "net, inp, tar = create_network(build_network_norm)\n",
    "norm_rnn = train(net, inp, tar, X_train, y_train_, X_test, y_test_,\n",
    "                      lasagne.updates.adam, \"nrnn-adam/nrnn1\", 1e-3, 10, num_epochs=10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'inp': input,\n",
       " 'network': <lasagne.layers.dense.DenseLayer at 0x7f6e0acada58>,\n",
       " 'norm': array([ 12.72363662,   5.89265027,  31.86631632,   0.        ,\n",
       "          0.        ,   0.        ,   0.        ,   0.        ,\n",
       "          0.        ,   0.        ]),\n",
       " 'target': target_output,\n",
       " 'test_acc': array([ 33.59,  36.48,  35.58,  10.28,  10.28,  10.28,  10.28,  10.28,\n",
       "         10.28,  10.28]),\n",
       " 'test_err': array([  1.7779442 ,   1.66451281,   1.72241546,  17.944     ,\n",
       "         17.944     ,  17.944     ,  17.944     ,  17.944     ,\n",
       "         17.944     ,  17.944     ]),\n",
       " 'test_fn': <theano.compile.function_module.Function at 0x7f6e06e45ba8>,\n",
       " 'train_acc': array([ 28.13      ,  33.47833333,  32.42666667,  16.42666667,\n",
       "         10.44166667,  10.44166667,  10.44166667,  10.44166667,\n",
       "         10.44166667,  10.44166667]),\n",
       " 'train_err': array([  1.94751782,   1.72511917,   1.75236298,  13.39067749,\n",
       "         17.91166667,  17.91166667,  17.91166667,  17.91166667,\n",
       "         17.91166667,  17.91166667]),\n",
       " 'train_fn': <theano.compile.function_module.Function at 0x7f6e0c01eb00>}"
      ]
     },
     "execution_count": 67,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "norm_rnn"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "anaconda-cloud": {},
  "kernelspec": {
   "display_name": "Python [conda root]",
   "language": "python",
   "name": "conda-root-py"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}