Test_Keras_WGAN_4.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "+ More updates of D\n",
    "+ Implicit weight clipping of D\n",
    "+ Closer to vanilla WGAN per https://arxiv.org/pdf/1701.07875.pdf"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "%run 'Lib.ipynb'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "mkdir: cannot create directory ‘out4’: File exists\r\n"
     ]
    }
   ],
   "source": [
    "%mkdir 'out4'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n"
     ]
    }
   ],
   "source": [
    "from collections import defaultdict\n",
    "\n",
    "import keras.backend as K\n",
    "from keras.datasets import mnist\n",
    "from keras.layers import *\n",
    "from keras.models import *\n",
    "from keras.optimizers import *\n",
    "from keras.utils.generic_utils import Progbar"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "Z_SIZE = 100\n",
    "\n",
    "# batch and latent size taken from the paper\n",
    "EPOCHS = 50\n",
    "BATCH_SIZE = 100\n",
    "\n",
    "# Adam parameters suggested in https://arxiv.org/abs/1511.06434\n",
    "ADAM_LR = 0.0002\n",
    "ADAM_BETA_1 = 0.5\n",
    "\n",
    "# how many times to train D per each G(D) update\n",
    "# 5 is suggested in https://arxiv.org/pdf/1701.07875.pdf\n",
    "D_TRAIN_ITERATIONS = 5\n",
    "\n",
    "# D weight clipping per https://arxiv.org/pdf/1701.07875.pdf\n",
    "D_WEIGHTS_LOWER_BOUND = -0.01\n",
    "D_WEIGHTS_UPPER_BOUND = 0.01"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "np.random.seed(777)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "K.set_image_dim_ordering('tf')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def wasserstein_distance(y_true, y_pred):\n",
    "    return K.mean(y_true * y_pred)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def create_D():\n",
    "    input_image = Input(shape=(28, 28, 1), name='input_image')\n",
    "\n",
    "    x = Conv2D(32, (3, 3), padding='same', name='conv_1')(input_image)\n",
    "    x = LeakyReLU()(x)\n",
    "    x = MaxPool2D(pool_size=2)(x)\n",
    "    x = Dropout(0.3)(x)\n",
    "\n",
    "    x = Conv2D(64, (3, 3), padding='same', name='conv_2')(x)\n",
    "    x = MaxPool2D(pool_size=1)(x)\n",
    "    x = LeakyReLU()(x)\n",
    "    x = Dropout(0.3)(x)\n",
    "\n",
    "    x = Conv2D(128, (3, 3), padding='same', name='conv_3')(x)\n",
    "    x = MaxPool2D(pool_size=2)(x)\n",
    "    x = LeakyReLU()(x)\n",
    "    x = Dropout(0.3)(x)\n",
    "\n",
    "    x = Conv2D(256, (3, 3), padding='same', name='coonv_4')(x)\n",
    "    x = MaxPool2D(pool_size=1)(x)\n",
    "    x = LeakyReLU()(x)\n",
    "    x = Dropout(0.3)(x)\n",
    "\n",
    "    features = Flatten()(x)\n",
    "\n",
    "    output_is_fake = Dense(\n",
    "        1, activation='linear', name='output_is_fake')(features)\n",
    "    output_class = Dense(\n",
    "        10, activation='softmax', name='output_class')(features)\n",
    "\n",
    "    return Model(inputs=[input_image], outputs=[output_is_fake, output_class], name='D')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def create_G(Z_SIZE=Z_SIZE):\n",
    "    DICT_LEN = 10\n",
    "    EMBEDDING_LEN = Z_SIZE\n",
    "\n",
    "    # class#\n",
    "    input_class = Input(shape=(1, ), dtype='int32', name='input_class')\n",
    "    # encode class# to the same size as Z to use hadamard multiplication later on\n",
    "    e = Embedding(\n",
    "        DICT_LEN, EMBEDDING_LEN,\n",
    "        embeddings_initializer='glorot_uniform')(input_class)\n",
    "    embedded_class = Flatten(name='embedded_class')(e)\n",
    "\n",
    "    # latent var\n",
    "    input_z = Input(shape=(Z_SIZE, ), name='input_z')\n",
    "\n",
    "    # hadamard product\n",
    "    h = multiply([input_z, embedded_class], name='h')\n",
    "\n",
    "    # cnn part\n",
    "    x = Dense(1024)(h)\n",
    "    x = LeakyReLU()(x)\n",
    "\n",
    "    x = Dense(128 * 7 * 7)(x)\n",
    "    x = LeakyReLU()(x)\n",
    "    x = Reshape((7, 7, 128))(x)\n",
    "\n",
    "    x = UpSampling2D(size=(2, 2))(x)\n",
    "    x = Conv2D(256, (5, 5), padding='same')(x)\n",
    "    x = LeakyReLU()(x)\n",
    "\n",
    "    x = UpSampling2D(size=(2, 2))(x)\n",
    "    x = Conv2D(128, (5, 5), padding='same')(x)\n",
    "    x = LeakyReLU()(x)\n",
    "\n",
    "    x = Conv2D(1, (2, 2), padding='same', activation='tanh', name='output_generated_image')(x)\n",
    "\n",
    "    return Model(inputs=[input_z, input_class], outputs=x, name='G')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "D = create_D()\n",
    "\n",
    "# SGD without momentum, see\n",
    "# https://github.com/shekkizh/WassersteinGAN.tensorflow#observations\n",
    "D.compile(\n",
    "    optimizer=SGD(nesterov=False), \n",
    "    loss=[wasserstein_distance, 'sparse_categorical_crossentropy'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "G = create_G()\n",
    "\n",
    "G.compile(\n",
    "    optimizer=Adam(lr=ADAM_LR, beta_1=ADAM_BETA_1), loss='binary_crossentropy')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "input_z = Input(shape=(Z_SIZE, ), name='input_z_')\n",
    "input_class = Input(shape=(1, ),name='input_class_', dtype='int32')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# create combined D(G) model\n",
    "# D.trainable = False\n",
    "output_is_fake, output_class = D(G(inputs=[input_z, input_class]))\n",
    "combined = Model(inputs=[input_z, input_class], outputs=[output_is_fake, output_class])\n",
    "combined.get_layer('D').trainable = False # freeze D in generator training faze\n",
    "\n",
    "combined.compile(\n",
    "    optimizer='RMSprop',\n",
    "    loss=[wasserstein_distance, 'sparse_categorical_crossentropy']\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.image.AxesImage at 0x7f0e5c782278>"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAfoAAAH0CAYAAADVH+85AAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAAWJQAAFiUBSVIk8AAAIABJREFUeJzt3Xtw3Xd55/HPI1kX33S1YlmJE8XBTkohIQlt0mQIuQws\nKQOEJtnlj9IMA5222y4NhW13WmhN253hj51y7UKnsM1MmNnQCQOdblNgh4QkEFKGpMFLHeIEW7Zl\nW7ZkWTdbd3/3j/NTKoxk2d/nWD/50fs1oznWOec5z9c//Y4++knn/B5LKQkAAMRUU/YCAADAhUPQ\nAwAQGEEPAEBgBD0AAIER9AAABEbQAwAQGEEPAEBgBD0AAIER9AAABEbQAwAQGEEPAEBgBD0AAIER\n9AAABEbQAwAQGEEPAEBgBD0AAIGtKXsBF4KZ7ZPUJKmn5KUAAJCrW9JISulKz4OEDHpJTXV1dW2d\nnZ1tuQ8wPT2d3by2tja7VpJmZ2dLqZWkurq67NqZmZnSentqJen06dPZtWvW+J5Gnt6eWkmqr68v\nrbd3f/HU19T4fpnpeY57n6Oe/c27zScnJ7NrzczV26PM3p5tPjQ05N5fpJKD3swuk/Tnkt4mqV3S\nEUlfl/TxlNIJx0P3dHZ2tv3hH/5h9gMcPXo0u7apqSm7VpKGh4eza0dHR129Ozo6smuPHz/u6t3Z\n2Zld29XV5eo9NTWVXdvS0uLq7fnmOT4+7urt2W4TExOu3oODg656z3O0sbHR1butLfsYQiMjI67e\nmzZtyq7t6+tz9e7p6cmu9f5A7PnhzBv0nvoTJ/Kj7Gtf+5qOHz/ek/0AhdKC3syukvSMpEsk/YOk\nn0j6ZUm/L+ltZnZrSsmXHAAArHJlvhjvf6oS8h9MKd2TUvpvKaU7JX1S0tWS/nuJawMAIIRSgr44\nmn+rKi+W++szbv4zSSclvdfM1i/z0gAACKWsI/o7istvpZR+5hU9KaVRSd+TtE7Szcu9MAAAIinr\nb/RXF5d7Frn9ZVWO+HdI+vZiD2Jmzy1y0zX5SwMAII6yjuibi8vFXl4+d73v5cwAAKxyF/X76FNK\nNy50fXGkf8MyLwcAgBWnrCP6uSP25kVun7t+aBnWAgBAWGUF/UvF5Y5Fbt9eXC72N3wAAHAOygr6\nJ4rLt5rZz6zBzDZKulXSKUnPLvfCAACIpJSgTyn9VNK3VDlh/++ecfPHJa2X9HBK6eQyLw0AgFDK\nfDHef1blFLifMbO7JL0o6SZV3mO/R9KflLg2AABCKO0UuMVR/RslPaRKwH9Y0lWSPi3pZs5zDwCA\nX6lvr0spHZT0vjLXAABAZBf1++jPxsy0du3a7PqGhobs2q1bt2bXSr6xpZ4xkpJvzO2BAwdcvcfG\nxrJrN27c6OrtGZHrWbckHTp0KLvW+/9+6aWXlr7TItatW+fq7dnmkvRv//Zv2bWbN2929fZ8f/Dy\njKk9ffr00nc6iyNHjmTXetYt+Z4nnjHUkpRSKqW2WsqcXgcAAC4wgh4AgMAIegAAAiPoAQAIjKAH\nACAwgh4AgMAIegAAAiPoAQAIjKAHACAwgh4AgMAIegAAAiPoAQAIjKAHACAwgh4AgMDCjqmdnp7W\nwYMHs+s9IxG7u7uzayWpvb09u7a+vt7Vu7a2Nrt2aGjI1dszVtg7rvWSSy7Jrm1qanL19oy57erq\ncvUeGBjIrh0ZGXH17u/vd9V7nqPe3p7n2e7du129n3/++ezamhrfsZ1nf/GONV6zJj+uvP/vkydP\nZtc2NjZm13rX/erjVOVRAADAikTQAwAQGEEPAEBgBD0AAIER9AAABEbQAwAQGEEPAEBgBD0AAIER\n9AAABEbQAwAQGEEPAEBgBD0AAIER9AAABEbQAwAQGEEPAEBgYefRm5lrDnBdXV12bU9PT3at5Jut\nPjg46OrtmX+8bds2V2/PvGnPLHtJmpqayq41M1dvz366YcMGV+8f/ehH2bW1tbWu3t7tNj09nV3r\nmScv+Z5nQ0NDrt79/f3Ztd755uPj49m1nnVLvq+Z9+vt+Zq1t7dn1zKPHgAALImgBwAgMIIeAIDA\nCHoAAAIj6AEACIygBwAgMIIeAIDACHoAAAIj6AEACIygBwAgMIIeAIDACHoAAAIj6AEACIygBwAg\nsLBjamdmZnTs2LGyl5Fl8+bN2bU33HCDq/fBgweza71jRz2jID3jViXpiiuuyK7t7Ox09X722Wdd\n9R6jo6PZtd4Rmp7xnZLU1NSUXesd7+vZ1zdt2uTq7Vn7li1bXL3XrVuXXesZQy1JExMT2bWHDx92\n9fZ8b+ro6HD1rgaO6AEACIygBwAgMIIeAIDACHoAAAIj6AEACIygBwAgMIIeAIDACHoAAAIj6AEA\nCIygBwAgMIIeAIDACHoAAAIj6AEACIygBwAgMIIeAIDAws6jNzPX/GPPrO3BwcHsWkmqra3Nrm1r\na3P19vy/PbOqJd9288yqlqTnn38+u/a1r32tq/fMzEx27f79+129Paamplz1npnukvT6178+u7av\nr8/V2/M127p1q6v35ORkab3r6+uzaz3fWyTpxIkT2bXeffX06dPZtRs3bsyu9W6zVx+nKo+Swcx6\nzCwt8uF7FgIAAEnlH9EPS/rUAtePLfdCAACIqOygH0op7Sx5DQAAhMWL8QAACKzsI/oGM/t1SZdL\nOilpl6SnUkqz5S4LAIAYyg76TkkPn3HdPjN7X0rpyaWKzey5RW66xr0yAAACKPNX938n6S5Vwn69\npNdL+htJ3ZL+2cyuK29pAADEUNoRfUrp42dc9WNJv21mY5I+LGmnpHcv8Rg3LnR9caR/QxWWCQDA\nRW0lvhjvC8XlbaWuAgCAAFZi0PcXl+tLXQUAAAGsxKC/ubjcW+oqAAAIoJSgN7NfMLOfO2I3s25J\nnys+/fJyrgkAgIjKejHef5L0YTN7StJ+SaOSrpL0dkmNkh6T9D9KWhsAAGGUFfRPSLpa0vWSblXl\n7/FDkr6ryvvqH04ppZLWBgBAGKUEfXEynCVPiONRU1PjGg/o+Tmjp6cnu1aSrrjiCle9x+7du7Nr\nL7vsMlfvkZGR7FrPCEtJam1tza794Q9/6Ord0dGRXbt27VpX75MnT2bXekcij4+Pu+pnZ/NPoOn5\nekvSwMBAdq33a7Zly5bs2oaGBldvz3YbHR119fY8x6enp129L7nkkuzalpaW7FrPyPL5VuKL8QAA\nQJUQ9AAABEbQAwAQGEEPAEBgBD0AAIER9AAABEbQAwAQGEEPAEBgBD0AAIER9AAABEbQAwAQGEEP\nAEBgBD0AAIER9AAABEbQAwAQWCnz6JfD7OyshoeHs+uPHTuWXTszM5NdK0mnTp3Kru3u7nb13rZt\nW3btpZde6ur905/+NLvWM8tekrq6urJrjx8/7updV1eXXbt+/XpXb8++6pkHL0knT5501Xu2++bN\nm129PTPhN2zY4Oq9adOm7Fozc/WenJzMru3v73f1npiYyK71bDPJ9z25pib/eNr79Xp1DVV5FAAA\nsCIR9AAABEbQAwAQGEEPAEBgBD0AAIER9AAABEbQAwAQGEEPAEBgBD0AAIER9AAABEbQAwAQGEEP\nAEBgBD0AAIER9AAABBZ2TG1NTY3q6+uz61tbW129PQ4ePJhdOz4+7urt2Wavfe1rXb09Yyynp6dd\nvT3jfa+++mpXb8+o2dHRUVdvz7jWEydOuHqfPn3aVX/JJZdk13qeY5JvRG5DQ4Or90svvZRde+21\n17p679ixI7u2t7fX1duzr3rGCkvSiy++6KovG0f0AAAERtADABAYQQ8AQGAEPQAAgRH0AAAERtAD\nABAYQQ8AQGAEPQAAgRH0AAAERtADABAYQQ8AQGAEPQAAgRH0AAAERtADABAYQQ8AQGBh59FPT0+r\nr68vu35oaCi7tq6uLrtW8s2r9qxbko4dO5Zde+WVV7p6e2bKe2a6S1JbW1t2rWcuuldXV5erfmJi\nokorOX/eefQDAwPZtWNjY67etbW1pdRKUktLS3ZtfX29q3d3d3d27QsvvODqXVOTf1zq/Z7c29ub\nXdvZ2Zlda2bZtfNxRA8AQGAEPQAAgRH0AAAERtADABAYQQ8AQGAEPQAAgRH0AAAERtADABAYQQ8A\nQGAEPQAAgRH0AAAERtADABAYQQ8AQGAEPQAAgYUdU1tbW6umpqbses+I26mpqexayTdq1jO6U5Ku\nuOKK7Frv+E3PuFfv2FGPkydPuupbW1uza71jR/ft25ddOzo66urtHR3qGVu6Y8cOV+/Z2dnsWu8o\nac943wMHDrh6Hzx4MLvW+xxtbGzMru3o6HD1vvzyy7NrPWPHV9SYWjO7z8w+a2ZPm9mImSUz+/IS\nNbeY2WNmNmhm42a2y8weNDNfWgAAgFdV64j+o5KukzQmqVfSNWe7s5m9S9JXJU1I+oqkQUnvkPRJ\nSbdKur9K6wIAYFWr1t/oPyRph6QmSb9ztjuaWZOkv5U0K+n2lNL7U0r/VdIbJH1f0n1m9p4qrQsA\ngFWtKkGfUnoipfRySimdw93vk9Qh6ZGU0g/nPcaEKr8ZkJb4YQEAAJybMl51f2dx+Y0FbntK0ilJ\nt5hZ/isYAACApHKC/urics+ZN6SUZiTtU+W1A9uWc1EAAERUxtvrmovL4UVun7u+ZakHMrPnFrnp\nrC8GBABgteCEOQAABFbGEf3cEXvzIrfPXb/kWSVSSjcudH1xpH/D+S8NAIBYyjiif6m4/LlTU5nZ\nGklXSpqRtHc5FwUAQERlBP3jxeXbFrjtNknrJD2TUppcviUBABBTGUH/qKQBSe8xszfOXWlmjZL+\nsvj08yWsCwCAcKryN3ozu0fSPcWnncXlr5jZQ8W/B1JKH5GklNKImf2mKoH/HTN7RJVT4L5Tlbfe\nParKaXEBAIBTtV6M9wZJD5xx3Tb9+3vh90v6yNwNKaWvm9mbJf2JpHslNUp6RdIfSPrMOZ5hDwAA\nLKEqQZ9S2ilp53nWfE/Sr1ajPwAAWFjYefQpJdfM6E2bNrl6e3jmfDc1Nbl6T0xMZNfu3et7o4Sn\nd2dn59J3Oovx8fHsWu988UOHDmXX9vf3u3q//PLL2bX19fWu3lu2bHHVe/p7v2Z1dXXZtZ7vS5I0\nPLzYucaWtmaN71v+4cOHs2u9s9Wvvfba7NqaGt/L0Vpaljx/26I8X+9q/XKbE+YAABAYQQ8AQGAE\nPQAAgRH0AAAERtADABAYQQ8AQGAEPQAAgRH0AAAERtADABAYQQ8AQGAEPQAAgRH0AAAERtADABAY\nQQ8AQGBhx9ROTU3pwIED2fWeUZBtbW3ZtZJvlOTWrVtdvT3jWg8ePOjqfemll2bXjo2NuXp3dHRk\n105NTbl6e8Z3escSl2lwcNBV7xnve9lll7l6e77mR48edfXu7u7OrvWM15WkDRs2ZNdu377d1duz\n3Tzf1yTfc9SzrzCmFgAALImgBwAgMIIeAIDACHoAAAIj6AEACIygBwAgMIIeAIDACHoAAAIj6AEA\nCIygBwAgMIIeAIDACHoAAAIj6AEACIygBwAgMIIeAIDAws6j9/LMH/bOuvbMTm5vb3f1rq+vz671\nzmVvbW3NrvV8vSSpv78/u/bJJ5909R4bG8uuPX36tKu3Zz75zMyMq/fll1/uqp+YmMiu3b17t6v3\n8PBwdu3atWtdvWdnZ7Nrvc/Rffv2Zdd6vzd59tXm5mZXb89c+NHR0exa73NsDkf0AAAERtADABAY\nQQ8AQGAEPQAAgRH0AAAERtADABAYQQ8AQGAEPQAAgRH0AAAERtADABAYQQ8AQGAEPQAAgRH0AAAE\nRtADABBY2DG1ZuYaubp9+/bs2htvvDG7VpI6Ojqya71jS59//vns2sHBQVfvPXv2ZNc2NDS4evf2\n9mbXeraZJE1PT2fXev/fnv18w4YNrt41Nb7jjKampuzavr4+V+9LL720tN7Hjh3Lrh0YGHD19tS/\n6U1vcvVuaWnJrvWOqfXsa55t5v1+PocjegAAAiPoAQAIjKAHACAwgh4AgMAIegAAAiPoAQAIjKAH\nACAwgh4AgMAIegAAAiPoAQAIjKAHACAwgh4AgMAIegAAAiPoAQAIjKAHACCwsPPo6+vrddlll2XX\nDw8PZ9f29/dn10rS5s2bs2u984vb2tpc9R6HDx/Orh0ZGXH19syr7urqcvX2zKt++9vf7urd3t6e\nXeuZyS7559F7vuYbNmxw9fZ8zZ599llXb89zfGJiwtXb8z317rvvdvUeHx/Prm1tbXX1npmZya5d\nt25ddq33OfLq41TjQczsPjP7rJk9bWYjZpbM7MuL3Le7uH2xj0eqsSYAAFC9I/qPSrpO0pikXknX\nnEPNjyR9fYHrf1ylNQEAsOpVK+g/pErAvyLpzZKeOIeaF1JKO6vUHwAALKAqQZ9SejXYzawaDwkA\nAKqgzBfjdZnZb0lql3Rc0vdTSrtKXA8AAOGUGfRvKT5eZWbfkfRASunAuTyAmT23yE3n8hoBAADC\nK+N99Kck/YWkGyW1Fh9zf9e/XdK3zWx9CesCACCcZT+iTykdk/SnZ1z9lJm9VdJ3Jd0k6QOSPn0O\nj3XjQtcXR/o3OJcKAMBFb8WcGS+lNCPpi8Wnt5W5FgAAolgxQV+YO6Ucv7oHAKAKVlrQ31xc7i11\nFQAABLHsQW9mN5jZz/U1s7tUOfGOJC14+lwAAHB+qvJiPDO7R9I9xaedxeWvmNlDxb8HUkofKf79\nV5K2m9kzqpxNT5KulXRn8e+PpZSeqca6AABY7ar1qvs3SHrgjOu2FR+StF/SXNA/LOndkn5J0t2S\n6iQdlfT3kj6XUnq6SmsCAGDVq9YpcHdK2nmO9/2SpC9Voy8AADi7sPPoa2tr1dTUlF3vmdPtnfm8\nb9++7Nrp6WlX76mpqezarVu3unq3tLRk1x46dMjV+8CBczoZ44J6enpcvXt7e5e+0yK2bNni6n37\n7bdn13rmoktSXV2dq97T/9SpU67e3d3d2bX79+939d69e3d27ejoqKv32rVrs2uHh4ddvT3b3Luv\n7tmzJ7s2peTqXQ0r7VX3AACgigh6AAACI+gBAAiMoAcAIDCCHgCAwAh6AAACI+gBAAiMoAcAIDCC\nHgCAwAh6AAACI+gBAAiMoAcAIDCCHgCAwAh6AAACCzumVpLMLLt2cnIyu7ahoSG7VvKN0PSOoZyd\nnc2u7erqcvWura3NrvWOHfWMTD1y5Iir9+bNm7Nrm5ubXb09+4t3JHJnZ6ervqOjI7v2uuuuc/X2\nPMdPnDjh6u3ZVw8fPuzq3dbWll3r/XrX1OQfl7a2trp6b9++Pbt2165drt7VwBE9AACBEfQAAARG\n0AMAEBhBDwBAYAQ9AACBEfQAAARG0AMAEBhBDwBAYAQ9AACBEfQAAARG0AMAEBhBDwBAYAQ9AACB\nEfQAAARG0AMAEFjYefRmpjVr8v979fX12bXXX399dq1UWXuuQ4cOuXofPXo0u9YzL1qStm3bll17\nxRVXuHqPj49n13pmdEvSv/7rv2bXzs7Ounr39/dn19bW1rp6t7e3u+p37NiRXeuZZS9JKaXs2nvv\nvdfV+wc/+EF2bU9Pj6v3unXrsmtPnz7t6u3Z5r29va7eo6Oj2bWeHPJkwXwc0QMAEBhBDwBAYAQ9\nAACBEfQAAARG0AMAEBhBDwBAYAQ9AACBEfQAAARG0AMAEBhBDwBAYAQ9AACBEfQAAARG0AMAEBhB\nDwBAYGHH1M7MzGhwcDC7fsuWLdm1U1NT2bWSND09nV3rGSPpre/r63P19oyhHBsbc/Xeu3dvdu3A\nwICr98aNG7NrX3nlFVdvzwhN71ji3bt3u+qbm5uza9evX+/qfeLEiexaz34u+Z5nu3btcvVuaWnJ\nrh0eHnb1vvTSS7NrGxsbXb0946AnJyeza72jfedwRA8AQGAEPQAAgRH0AAAERtADABAYQQ8AQGAE\nPQAAgRH0AAAERtADABAYQQ8AQGAEPQAAgRH0AAAERtADABAYQQ8AQGAEPQAAgRH0AAAEFnYe/ezs\nrIaGhrLrPbO2u7u7s2u9zMxVX19fn1374osvunqXNfNZko4cOZJdu2nTJlfvtra27FrPbHJJqq2t\nza4dGRlx9R4cHHTVDwwMZNded911rt7Hjx/Prq2rq3P19uzro6Ojrt6dnZ3ZtR0dHa7ezc3N2bVX\nXXWVq/f4+Hh27Te+8Y3s2hUzj97M2s3sA2b2NTN7xczGzWzYzL5rZu83swV7mNktZvaYmQ0WNbvM\n7EEzy//OAwAAfkY1jujvl/R5SUckPSHpgKTNkn5N0hcl3W1m96eU0lyBmb1L0lclTUj6iqRBSe+Q\n9ElJtxaPCQAAnKoR9HskvVPSP6WUXv09g5n9saQfSLpXldD/anF9k6S/lTQr6faU0g+L6z8m6XFJ\n95nZe1JKj1RhbQAArGruX92nlB5PKf3j/JAvru+T9IXi09vn3XSfpA5Jj8yFfHH/CUkfLT79He+6\nAADAhX/V/XRxOTPvujuLy4VeofCUpFOSbjGzhgu5MAAAVoML9qp7M1sj6TeKT+eH+tXF5Z4za1JK\nM2a2T9IvStom6awv4zaz5xa56ZrzWy0AADFdyCP6T0h6naTHUkrfnHf93Hskhhepm7u+5UItDACA\n1eKCHNGb2QclfVjSTyS990L0kKSU0o2L9H9O0g0Xqi8AABeLqh/Rm9nvSfq0pN2S7kgpnXlWjLkj\n9sXOfjB3ff7ZbgAAgKQqB72ZPSjps5J+rErIL3TarpeKyx0L1K+RdKUqL97bW821AQCwGlUt6M3s\nj1Q54c0LqoT8sUXu+nhx+bYFbrtN0jpJz6SUfOc0BQAA1Qn64mQ3n5D0nKS7UkpnOwn1o5IGJL3H\nzN447zEaJf1l8ennq7EuAABWO/eL8czsAUl/rsqZ7p6W9MEFBqv0pJQekqSU0oiZ/aYqgf8dM3tE\nlVPgvlOVt949qsppcQEAgFM1XnV/ZXFZK+nBRe7zpKSH5j5JKX3dzN4s6U9UOUVuo6RXJP2BpM/M\nPy8+AADI5w76lNJOSTsz6r4n6Ve9/RczOzvrGqPpGdd66NCh7FrJN5pw48aNrt6e0aGNjY2u3lNT\nU9m13jG1vb29pfUeHl7slBJL2717t6u3Z3To2NiYq7f3eeIZ7+sZiSz5xr16R+R6xtxef/31rt7r\n16/PrvV+b9qwYUN2bUOD70SrJ06cyK5tamrKrvWMkZ7vQp8CFwAAlIigBwAgMIIeAIDACHoAAAIj\n6AEACIygBwAgMIIeAIDACHoAAAIj6AEACIygBwAgMIIeAIDACHoAAAIj6AEACIygBwAgMIIeAIDA\n3PPoV6rp6WkdOXIku94zI/zAgQPZtZJvRvg111zj6u2Z2+yZuyzJ9fWanp529fbMVvfO2Z6YmMiu\n9cwml3zzrj37qSS95jWvcdWfPHnSVe/R19eXXdvf3+/qPTQ0lF07OTnp6t3a2ppdu23bNlfvgYGB\n7NrR0VFX76mpqezawcHB7NqZmZns2vk4ogcAIDCCHgCAwAh6AAACI+gBAAiMoAcAIDCCHgCAwAh6\nAAACI+gBAAiMoAcAIDCCHgCAwAh6AAACI+gBAAiMoAcAIDCCHgCAwMKOqa2rq1NXV1d2fWNjY3at\ndxyjZwxlmSNTN23a5Ort+Xp5xgpLvrWbmau35+vtGTMrSS0tLdm1p06dcvXu7Ox01be1tWXXese1\n3nTTTdm1nnGrkrR3797SenvqvWOJR0ZGsmu9X+/m5ubsWs8I7ZRSdu18HNEDABAYQQ8AQGAEPQAA\ngRH0AAAERtADABAYQQ8AQGAEPQAAgRH0AAAERtADABAYQQ8AQGAEPQAAgRH0AAAERtADABAYQQ8A\nQGAEPQAAgYWdRy9Js7Oz2bWHDh3KrvXOhG9oaMiuPXjwoKu3Z5t1d3e7etfV1WXXbtmyxdXb48iR\nI676DRs2ZNd69zXPrGzv19szT16SpqamsmuPHj3q6m1m2bXe56ind0tLi6v3/v37s2uPHz/u6n3s\n2LHsWu9c9/b29uza+vr67FrP13o+jugBAAiMoAcAIDCCHgCAwAh6AAACI+gBAAiMoAcAIDCCHgCA\nwAh6AAACI+gBAAiMoAcAIDCCHgCAwAh6AAACI+gBAAiMoAcAILCwY2rNTI2Njdn1NTX5PwONj49n\n10q+MbXNzc2u3mNjY9m1vb29rt4emzdvdtVPTk5m127atMnV2zNCc926da7eJ0+ezK717KeSf1Ss\nh2ccs+Qb1+rdbjMzM9m13nGtntHE3m1+6tSp7FpPFkhSX19fdu2aNeXHrPuI3szazewDZvY1M3vF\nzMbNbNjMvmtm7zezmjPu321m6Swfj3jXBAAAKqrxo8b9kj4v6YikJyQdkLRZ0q9J+qKku83s/vTz\nP0r+SNLXF3i8H1dhTQAAQNUJ+j2S3inpn1JKp+euNLM/lvQDSfeqEvpfPaPuhZTSzir0BwAAi3D/\n6j6l9HhK6R/nh3xxfZ+kLxSf3u7tAwAAzt+FfpXAdHG50KtHuszstyS1Szou6fsppV0XeD0AAKwq\nFyzozWyNpN8oPv3GAnd5S/Exv+Y7kh5IKR24UOsCAGA1uZBH9J+Q9DpJj6WUvjnv+lOS/kKVF+Lt\nLa67VtJOSXdI+raZvSGltOT7fszsuUVuuiZ30QAARHJBTphjZh+U9GFJP5H03vm3pZSOpZT+NKX0\nfEppqPh4StJbJf2LpNdI+sCFWBcAAKtN1Y/ozez3JH1a0m5Jd6WUBs+lLqU0Y2ZflHSTpNuKx1iq\n5sZF1vCcpBvOedEAAARV1SN6M3tQ0mdVeS/8HcUr789Hf3G5vprrAgBgtapa0JvZH0n6pKQXVAn5\nYxkPc3Nxufes9wIAAOekKkFvZh9T5cV3z6ny6/qBs9z3hjNPi1tcf5ekDxWffrka6wIAYLVz/43e\nzB6Q9OeSZiU9LemDZnbm3XpSSg8V//4rSdvN7BlJc1NQrpV0Z/Hvj6WUnvGuCwAAVOfFeFcWl7WS\nHlzkPk9Keqj498OS3i3plyTdLalO0lFJfy/pcymlp6uwJgAAoCoEfXG++p3ncf8vSfqSty8AAFha\n+YNyL5BoV25AAAAIm0lEQVT6+npdfvnl2fWdnZ3Ztd7Zx62trdm1o6Ojrt4e3rnLzc3N2bXeWddr\n167NrvXOhD9x4kR2bVNTk6u3x9TUlKt+eHjYVb/AnwjP2fr1vjf2eL5mGzZscPX2fH/xfG+RpJaW\nluzaiYkJV2/P9/Pp6eml73QWhw8fzq7duHFjdm1NTXVeL39BTpgDAABWBoIeAIDACHoAAAIj6AEA\nCIygBwAgMIIeAIDACHoAAAIj6AEACIygBwAgMIIeAIDACHoAAAIj6AEACIygBwAgMIIeAIDAwo6p\nnZ2ddY3BrK2tza49ffp0dq3kG5nqHRXrGWPpHf25b9++7Nq6ujpXb882P3LkiKu3Z5sPDAy4enue\nIw0NDa7e9fX1rnrP+E/PiFvJN3J1cHDQ1dsz5nZyctLV2zOu1fv19tR7vz90dXVl13rywLufzuGI\nHgCAwAh6AAACI+gBAAiMoAcAIDCCHgCAwAh6AAACI+gBAAiMoAcAIDCCHgCAwAh6AAACI+gBAAiM\noAcAIDCCHgCAwAh6AAACs5RS2WuoOjM7Xltb2+YZ/+kZD+gdLegZkevl6V1T4/u50TNC07vNPWuf\nnZ119S5zJLJn7d6vt/d7T5nPE892837NPP9v7/PEs3bv/lLm92TPvuqpHRgY0MzMzGBKqT37QRR3\nHv3I7OysBgYGeha5/Zri8ifLtJ4I2GZ52G552G7nj22WZyVvt25JI94HCXlEvxQze06SUko3lr2W\niwXbLA/bLQ/b7fyxzfKshu3G3+gBAAiMoAcAIDCCHgCAwAh6AAACI+gBAAhsVb7qHgCA1YIjegAA\nAiPoAQAIjKAHACAwgh4AgMAIegAAAiPoAQAIjKAHACCwVRX0ZnaZmf0vMztsZpNm1mNmnzKz/MH1\nwRXbKC3y0Vf2+spiZveZ2WfN7GkzGym2x5eXqLnFzB4zs0EzGzezXWb2oJmVN1h9mZ3PdjOz7rPs\ne8nMHlnu9ZfBzNrN7ANm9jUze6XYd4bN7Ltm9n4zW/D7+Grf3853u0Xe36LOo/85ZnaVpGckXSLp\nH1SZPfzLkn5f0tvM7NaU0vESl7iSDUv61ALXjy33QlaQj0q6TpVt0Kt/n2m9IDN7l6SvSpqQ9BVJ\ng5LeIemTkm6VdP+FXOwKcl7brfAjSV9f4PofV3FdK9n9kj4v6YikJyQdkLRZ0q9J+qKku83s/jTv\n7Gfsb5Iytlsh3v6WUloVH5K+KSlJ+i9nXP9XxfVfKHuNK/FDUo+knrLXsdI+JN0habskk3R7sQ99\neZH7Nkk6JmlS0hvnXd+oyg+fSdJ7yv4/rcDt1l3c/lDZ6y55m92pSkjXnHF9pyrhlSTdO+969re8\n7RZ2f1sVv7ovjubfqkpo/fUZN/+ZpJOS3mtm65d5abhIpZSeSCm9nIrvEEu4T1KHpEdSSj+c9xgT\nqhzhStLvXIBlrjjnud0gKaX0eErpH1NKp8+4vk/SF4pPb593E/ubsrZbWKvlV/d3FJffWuCLPmpm\n31PlB4GbJX17uRd3EWgws1+XdLkqPxTtkvRUSmm23GVdNO4sLr+xwG1PSTol6RYza0gpTS7fsi4a\nXWb2W5LaJR2X9P2U0q6S17RSTBeXM/OuY39b2kLbbU64/W21BP3VxeWeRW5/WZWg3yGCfiGdkh4+\n47p9Zva+lNKTZSzoIrPo/pdSmjGzfZJ+UdI2SS8u58IuEm8pPl5lZt+R9EBK6UApK1oBzGyNpN8o\nPp0f6uxvZ3GW7TYn3P62Kn51L6m5uBxe5Pa561uWYS0Xm7+TdJcqYb9e0usl/Y0qf8/6ZzO7rryl\nXTTY//KckvQXkm6U1Fp8vFmVF1bdLunbq/zPbZ+Q9DpJj6WUvjnveva3s1tsu4Xd31ZL0CNTSunj\nxd+6jqaUTqWUfpxS+m1VXsS4VtLOcleIqFJKx1JKf5pSej6lNFR8PKXKb9/+RdJrJH2g3FWWw8w+\nKOnDqrx76L0lL+eicbbtFnl/Wy1BP/cTbPMit89dP7QMa4li7sUst5W6iosD+18VpZRmVHl7lLQK\n9z8z+z1Jn5a0W9IdKaXBM+7C/raAc9huC4qwv62WoH+puNyxyO3bi8vF/oaPn9dfXF6Uv8paZovu\nf8XfC69U5UVBe5dzURe5Vbn/mdmDkj6rynu67yheQX4m9rcznON2O5uLen9bLUH/RHH51gXOhrRR\nlRNInJL07HIv7CJ2c3G5ar5ZODxeXL5tgdtuk7RO0jOr+BXQOVbd/mdmf6TKCW9eUCWsji1yV/a3\nec5ju53NRb2/rYqgTyn9VNK3VHkB2e+ecfPHVfkp7eGU0sllXtqKZma/sNCLT8ysW9Lnik/PetpX\nSJIelTQg6T1m9sa5K82sUdJfFp9+voyFrWRmdsNCp3c1s7skfaj4dFXsf2b2MVVeRPacpLtSSgNn\nuTv7W+F8tlvk/c1Wy3krFjgF7ouSblLlPfZ7JN2SOAXuzzCznaq8cOUpSfsljUq6StLbVTnL1mOS\n3p1SmiprjWUxs3sk3VN82inpP6jy0/7TxXUDKaWPnHH/R1U5JekjqpyS9J2qvBXqUUn/cTWcROZ8\ntlvxlqbtqjxve4vbr9W/v0/8YymlueAKy8wekPSQpFlVfv280Kvpe1JKD82rWfX72/lut9D7W9mn\n5lvOD0lbVXm72BFJU6qE16cktZa9tpX4ocpbS/63Kq9QHVLlJBP9kv6vKu9DtbLXWOK22anK6TIX\n++hZoOZWVX44OiFpXNL/U+VIobbs/89K3G6S3i/p/6hyRssxVU7pekCVc7e/qez/ywraZknSd9jf\nfNst8v62ao7oAQBYjVbF3+gBAFitCHoAAAIj6AEACIygBwAgMIIeAIDACHoAAAIj6AEACIygBwAg\nMIIeAIDACHoAAAIj6AEACIygBwAgMIIeAIDACHoAAAIj6AEACIygBwAgMIIeAIDA/j9VMMcwChoI\n1gAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7f0e7b7ed2b0>"
      ]
     },
     "metadata": {
      "image/png": {
       "height": 250,
       "width": 253
      }
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# try generating image\n",
    "\n",
    "z = np.random.randn(100).reshape(1, 100)\n",
    "img = G.predict([z, np.array([8])])\n",
    "plt.imshow(img.reshape((28, 28)), cmap=plt.cm.gray)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# get our mnist data, and force it to be of shape (..., 28, 28, 1) with\n",
    "# range [-1., 1.]\n",
    "(X_train, y_train), (X_test, y_test) = mnist.load_data()\n",
    "X_train = (X_train.astype(np.float32) - 127.5) / 127.5\n",
    "X_train = np.expand_dims(X_train, axis=3)\n",
    "\n",
    "X_test = (X_test.astype(np.float32) - 127.5) / 127.5\n",
    "X_test = np.expand_dims(X_test, axis=3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.image.AxesImage at 0x7f0e5c7354a8>"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAfoAAAH0CAYAAADVH+85AAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAAWJQAAFiUBSVIk8AAAHU1JREFUeJzt3X2sZXV5L/DvU8aCEHm5WiWmWNALUtsqglYKVXlJFdpU\nQeBekgrEaKNe0GLFYlvtHam3aNJWBK9iqi0JJhcbrLZViiiggFgbMcClIi/FAU1FRS7vL4L+7h97\nTZ2O58zLXnvOPud3Pp9kZ5291n7275k1a+Z71t7rpVprAQD69DPzbgAA2HYEPQB0TNADQMcEPQB0\nTNADQMcEPQB0TNADQMcEPQB0TNADQMcEPQB0TNADQMcEPQB0TNADQMcEPQB0TNADQMcEPQB0bM28\nG9gWquqbSXZOsm7OrQDAtPZMcl9rba8xb9Jl0GcS8v9leADAqjXXj+6r6uer6q+r6t+r6tGqWldV\nZ1XVbiPfet0s+gOAOVs39g3mtkdfVc9KcnWSpyb5+yTfSPKrSX4vyRFVdXBr7Qfz6g8AejDPPfoP\nZhLyb26tHdVae3tr7bAk70vy7CT/a469AUAXqrW29INO9uZvzeQjiWe11n68wbInJflOkkry1Nba\ng1O8/zVJ9p9NtwAwN19rrR0w5g3mtUd/6DC9ZMOQT5LW2v1JvpRkxyQHLnVjANCTeX1H/+xhevMi\ny29J8rIk+yS5dLE3GfbcF7Lv9K0BQD/mtUe/yzC9d5Hl6+fvugS9AEC3VvR59It9b+E7egCYmNce\n/fo99l0WWb5+/j1L0AsAdGteQX/TMN1nkeV7D9PFvsMHALbAvIL+8mH6sqr6Tz0Mp9cdnOShJP+8\n1I0BQE/mEvSttX9LckkmF+w/eaPF70qyU5LzpzmHHgD4iXkejPc/MrkE7tlVdXiSG5O8KJNz7G9O\n8sdz7A0AujC3S+AOe/UvSHJeJgH/1iTPSvL+JAe6zj0AjDfX0+taa99K8pp59gAAPZvrbWoBgG1L\n0ANAxwQ9AHRM0ANAxwQ9AHRM0ANAxwQ9AHRM0ANAxwQ9AHRM0ANAxwQ9AHRM0ANAxwQ9AHRM0ANA\nxwQ9AHRM0ANAxwQ9AHRM0ANAxwQ9AHRM0ANAxwQ9AHRM0ANAxwQ9AHRM0ANAxwQ9AHRM0ANAxwQ9\nAHRM0ANAxwQ9AHRM0ANAxwQ9AHRM0ANAxwQ9AHRM0ANAxwQ9AHRM0ANAxwQ9AHRM0ANAxwQ9AHRM\n0ANAxwQ9AHRM0ANAxwQ9AHRM0ANAxwQ9AHRM0ANAxwQ9AHRM0ANAxwQ9AHRM0ANAxwQ9AHRM0ANA\nxwQ9AHRM0ANAxwQ9AHRM0ANAxwQ9AHRM0ANAxwQ9AHRM0ANAxwQ9AHRM0ANAx9bMuwFgZdpxxx1H\n1T//+c8fVf+a17xm6tpjjjlm1Ni77LLLqPp5qapR9Q8//PDUtaeffvqosc8555xR9avZ3Pboq2pd\nVbVFHnfOqy8A6Mm89+jvTXLWAvMfWOpGAKBH8w76e1pra+fcAwB0y8F4ANCxee/Rb19Vr07yjCQP\nJrk+yRWttR/Nty0A6MO8g373JOdvNO+bVfWa1toXN1dcVdcssmjf0Z0BQAfm+dH93yQ5PJOw3ynJ\nryT5cJI9k/xTVT1vfq0BQB/mtkffWnvXRrNuSPKGqnogyVuTrE1y9Gbe44CF5g97+vvPoE0AWNGW\n48F45w7Tl8y1CwDowHIM+u8P053m2gUAdGA5Bv2Bw/S2uXYBAB2YS9BX1S9W1U/tsVfVnkk+MDz9\n2FL2BAA9mtfBeP89yVur6ooktye5P8mzkvxWkh2SXJTkz+fUGwB0Y15Bf3mSZyd5fpKDM/k+/p4k\nV2VyXv35rbU2p94AoBtzCfrhYjibvSAOsGm77bbbqPozzjhj6tojjzxy1Nh77bXXqPoxt1wdux8x\nz/2QRx55ZOraJz7xiaPG3n777aeufcpTnjJqbKa3HA/GAwBmRNADQMcEPQB0TNADQMcEPQB0TNAD\nQMcEPQB0TNADQMcEPQB0TNADQMcEPQB0TNADQMcEPQB0TNADQMcEPQB0bC73o4ee/MIv/MKo+pNP\nPnnq2rH3hH/Oc54zqn6luuWWW0bVX3zxxVPXXnbZZaPGvuOOO6aufeELXzhq7Icffnjq2s9//vOj\nxmZ69ugBoGOCHgA6JugBoGOCHgA6JugBoGOCHgA6JugBoGOCHgA6JugBoGOCHgA6JugBoGOCHgA6\nJugBoGOCHgA65ja1kOSpT33q1LUf/OAHR4095lazrbVRY48x9lav//AP/zCq/u/+7u+mrr3++utH\njf3QQw+Nqp+Xa6+9dlT9mFsy33nnnaPGZnr26AGgY4IeADom6AGgY4IeADom6AGgY4IeADom6AGg\nY4IeADom6AGgY4IeADom6AGgY4IeADom6AGgY4IeADom6AGgY+5HD0lOOOGEqWuPOOKIGXaydR55\n5JFR9e9973unrj3zzDNHjf3YY4+Nqmfr/eEf/uGo+j/4gz+Yuvbcc88dNfbY3lcze/QA0DFBDwAd\nE/QA0DFBDwAdE/QA0DFBDwAdE/QA0DFBDwAdE/QA0DFBDwAdE/QA0DFBDwAdE/QA0DFBDwAdc5ta\nGKmq5lY/duyvfOUrU9e6zex0jjvuuFH1H//4x6euHbu9/PCHP5y69jOf+cyosZneTPboq+rYqjqn\nqq6sqvuqqlXVxzZTc1BVXVRVd1fVw1V1fVWdWlXbzaInAGB2e/TvSPK8JA8k+XaSfTf14qp6ZZJP\nJHkkyceT3J3kt5O8L8nBScb9ygsAJJndd/RvSbJPkp2TvHFTL6yqnZP8VZIfJTmktfba1trbkuyX\n5MtJjq2q42fUFwCsajMJ+tba5a21W1prbQtefmySn0tyQWvtqxu8xyOZfDKQbOaXBQBgy8zjqPvD\nhunFCyy7IslDSQ6qqu2XriUA6NM8gv7Zw/TmjRe01h5P8s1Mjh145lI2BQA9msfpdbsM03sXWb5+\n/q6be6OqumaRRZs8GBAAVgsXzAGAjs1jj379HvsuiyxfP/+ezb1Ra+2AheYPe/r7b31rANCXeezR\n3zRM99l4QVWtSbJXkseT3LaUTQFAj+YR9JcN0yMWWPaSJDsmubq19ujStQQAfZpH0F+Y5K4kx1fV\nC9bPrKodkrx7ePqhOfQFAN2ZyXf0VXVUkqOGp7sP01+rqvOGn+9qrZ2WJK21+6rqdzMJ/C9U1QWZ\nXAL3FZmcendhJpfFBQBGmtXBePslOWmjec/MT86Fvz3JaesXtNY+VVUvTfLHSY5JskOSW5P8fpKz\nt/AKewDAZswk6Ftra5Os3cqaLyX5zVmMDwAsrHrceXZ6HVvrZ3/2Z6euPfnkk0eN/Rd/8RdT1479\n9/vAAw9MXXvssceOGvtzn/vcqPoxdtxxx1H1p5xyytS17373uzf/ok3Ybrvp7+T9+OOPjxr7iCMW\nOoZ6y1x++eWjxl7FvrbYqeRbygVzAKBjgh4AOiboAaBjgh4AOiboAaBjgh4AOiboAaBjgh4AOibo\nAaBjgh4AOiboAaBjgh4AOiboAaBjgh4AOuY2tTBnp59++tS1f/ZnfzbDTrbOmFvcJslpp502qn7v\nvfeeuvbII48cNfZznvOcUfVj3H777VPXnnjiiaPGvuqqq0bVMxW3qQUAFifoAaBjgh4AOiboAaBj\ngh4AOiboAaBjgh4AOiboAaBjgh4AOiboAaBjgh4AOiboAaBjgh4AOiboAaBjgh4AOuZ+9DBnT3jC\nE6auPeuss0aN/YY3vGFU/TxV1dS1Y//fu//++6eu/frXvz5q7Fe/+tVT1952222jxmYu3I8eAFic\noAeAjgl6AOiYoAeAjgl6AOiYoAeAjgl6AOiYoAeAjgl6AOiYoAeAjgl6AOiYoAeAjgl6AOiYoAeA\njrlNLaxgBx100Kj6K6+8ckadLL0xt6m97rrrRo19+umnT117ySWXjBqbVcdtagGAxQl6AOiYoAeA\njgl6AOiYoAeAjgl6AOiYoAeAjgl6AOiYoAeAjgl6AOiYoAeAjgl6AOiYoAeAjgl6AOiYoAeAjq2Z\ndwOw2h188MFT1376058eNfaYe7rP25jejz/++FFj33TTTaPqYSnNZI++qo6tqnOq6sqquq+qWlV9\nbJHX7jksX+xxwSx6AgBmt0f/jiTPS/JAkm8n2XcLaq5L8qkF5t8wo54AYNWbVdC/JZOAvzXJS5Nc\nvgU117bW1s5ofABgATMJ+tbafwT7Sv7ODwB6M8+D8Z5eVa9P8uQkP0jy5dba9XPsBwC6M8+g/43h\n8R+q6gtJTmqt3bElb1BV1yyyaEuOEQCA7s3jPPqHkvxpkgOS7DY81n+vf0iSS6tqpzn0BQDdWfI9\n+tba95L8yUazr6iqlyW5KsmLkrwuyfu34L0OWGj+sKe//8hWAWDFWzZXxmutPZ7kI8PTl8yzFwDo\nxbIJ+sH3h6mP7gFgBpZb0B84TG+baxcA0IklD/qq2r+qfmrcqjo8kwvvJMmCl88FALbOTA7Gq6qj\nkhw1PN19mP5aVZ03/HxXa+204ee/TLJ3VV2dydX0kuS5SQ4bfn5na+3qWfQFAKvdrI663y/JSRvN\ne+bwSJLbk6wP+vOTHJ3khUmOTPKEJN9N8rdJPtBau3JGPQHAqjerS+CuTbJ2C1/70SQfncW4AMCm\nuR89JFmzZvp/Cr/zO78zauyzzjpr6tqdd9551NiPPvro1LUPPvjgqLF32223UfVjvOpVrxpVf+aZ\nZ86oE9j2lttR9wDADAl6AOiYoAeAjgl6AOiYoAeAjgl6AOiYoAeAjgl6AOiYoAeAjgl6AOiYoAeA\njgl6AOiYoAeAjgl6AOiY29RCkte//vVT15599tmjxq6qqWvH3GY2SU444YSpa2+44YZRY//rv/7r\nqPoxxt7eF1YSe/QA0DFBDwAdE/QA0DFBDwAdE/QA0DFBDwAdE/QA0DFBDwAdE/QA0DFBDwAdE/QA\n0DFBDwAdE/QA0DFBDwAdE/QA0DH3o6cLBx100Kj6M844Y0adbL3rrrtu6tqjjz561Njr1q2bunb3\n3XcfNfZ3v/vdUfVjx4fVwh49AHRM0ANAxwQ9AHRM0ANAxwQ9AHRM0ANAxwQ9AHRM0ANAxwQ9AHRM\n0ANAxwQ9AHRM0ANAxwQ9AHRM0ANAx9ymli687W1vG1W/6667zqiTrfemN71p6toxt5kda+3ataPq\nn/a0p82mEWCT7NEDQMcEPQB0TNADQMcEPQB0TNADQMcEPQB0TNADQMcEPQB0TNADQMcEPQB0TNAD\nQMcEPQB0TNADQMcEPQB0TNADQMfcj55l4ylPecrUta985Stn2MnW+ehHPzqq/qqrrppRJ1tvv/32\nm7r2xBNPHDV2VY2qv//++6euvfTSS0eNDSvJ6D36qnpyVb2uqj5ZVbdW1cNVdW9VXVVVr62qBceo\nqoOq6qKqunuoub6qTq2q7cb2BABMzGKP/rgkH0rynSSXJ7kjydOSvCrJR5IcWVXHtdba+oKqemWS\nTyR5JMnHk9yd5LeTvC/JwcN7AgAjzSLob07yiiSfaa39eP3MqvqjJP+S5JhMQv8Tw/ydk/xVkh8l\nOaS19tVh/juTXJbk2Ko6vrV2wQx6A4BVbfRH9621y1pr/7hhyA/z70xy7vD0kA0WHZvk55JcsD7k\nh9c/kuQdw9M3ju0LANj2R90/Nkwf32DeYcP04gVef0WSh5IcVFXbb8vGAGA12GZH3VfVmiTrD8vd\nMNSfPUxv3rimtfZ4VX0zyS8leWaSGzczxjWLLNp367oFgD5tyz369yT55SQXtdY+u8H8XYbpvYvU\nrZ+/67ZqDABWi22yR19Vb07y1iTfSHLCthgjSVprBywy/jVJ9t9W4wLASjHzPfqqOiXJ+5N8Pcmh\nrbW7N3rJ+j32XbKw9fPvmXVvALDazDToq+rUJOckuSGTkL9zgZfdNEz3WaB+TZK9Mjl477ZZ9gYA\nq9HMgr6qTs/kgjfXZhLy31vkpZcN0yMWWPaSJDsmubq19uisegOA1WomQT9c7OY9Sa5Jcnhr7a5N\nvPzCJHclOb6qXrDBe+yQ5N3D0w/Noi8AWO1GH4xXVSclOSOTK91dmeTNC9ysYl1r7bwkaa3dV1W/\nm0ngf6GqLsjkErivyOTUuwszuSwuADDSLI6632uYbpfk1EVe88Uk561/0lr7VFW9NMkfZ3KJ3B2S\n3Jrk95OcveF18QGA6Y0O+tba2iRrp6j7UpLfHDs+JMk8fzfcfvtxF3F80pOeNHXtr//6r48a+8Mf\n/vDUtWP/3GP/zt7+9rdPXfv5z39+1NiwkmzrS+ACAHMk6AGgY4IeADom6AGgY4IeADom6AGgY4Ie\nADom6AGgY4IeADom6AGgY4IeADom6AGgY4IeADom6AGgY4IeADpW87yP97ZSVdck2X/efbB1dt55\n56lrb7zxxlFj77777qPqx/jWt741de0ee+wxw06W1th7wr/85S+fUSewrH2ttXbAmDewRw8AHRP0\nANAxQQ8AHRP0ANAxQQ8AHRP0ANAxQQ8AHRP0ANAxQQ8AHRP0ANAxQQ8AHRP0ANAxQQ8AHRP0ANCx\nNfNuANa77777pq495ZRTRo199tlnT1379Kc/fdTYz3jGM6aunedtpr/61a+Oqn/jG984o06ATbFH\nDwAdE/QA0DFBDwAdE/QA0DFBDwAdE/QA0DFBDwAdE/QA0DFBDwAdE/QA0DFBDwAdE/QA0DFBDwAd\nE/QA0DFBDwAdcz96uvDJT35yVP1tt902de0pp5wyauwXv/jFU9fuscceo8Z+73vfO3XtmWeeOWrs\nxx57bFQ9sGXs0QNAxwQ9AHRM0ANAxwQ9AHRM0ANAxwQ9AHRM0ANAxwQ9AHRM0ANAxwQ9AHRM0ANA\nxwQ9AHRM0ANAxwQ9AHSsWmvz7mHmquqaJPvPuw8AGOlrrbUDxrzB6D36qnpyVb2uqj5ZVbdW1cNV\ndW9VXVVVr62qn9no9XtWVdvE44KxPQEAE2tm8B7HJflQku8kuTzJHUmeluRVST6S5MiqOq799EcH\n1yX51ALvd8MMegIAMpugvznJK5J8prX24/Uzq+qPkvxLkmMyCf1PbFR3bWtt7QzGBwAWMfqj+9ba\nZa21f9ww5If5dyY5d3h6yNhxAICtN4s9+k15bJg+vsCyp1fV65M8OckPkny5tXb9Nu4HAFaVbRb0\nVbUmyYnD04sXeMlvDI8Na76Q5KTW2h3bqi8AWE225R79e5L8cpKLWmuf3WD+Q0n+NJMD8W4b5j03\nydokhya5tKr2a609uLkBhtPoFrLvtE0DQE+2yXn0VfXmJO9P8o0kB7fW7t6CmjVJrkryoiSnttbe\nvwU1mwr6Hbe8YwBYlkafRz/zPfqqOiWTkP96ksO3JOSTpLX2eFV9JJOgf8nwHpurWfAP74I5ADAx\n00vgVtWpSc7J5Fz4Q4cj77fG94fpTrPsCwBWq5kFfVWdnuR9Sa7NJOS/N8XbHDhMb9vkqwCALTKT\noK+qd2Zy8N01mXxcf9cmXrv/xpfFHeYfnuQtw9OPzaIvAFjtRn9HX1UnJTkjyY+SXJnkzVW18cvW\ntdbOG37+yyR7V9XVSb49zHtuksOGn9/ZWrt6bF8AwGwOxttrmG6X5NRFXvPFJOcNP5+f5OgkL0xy\nZJInJPlukr9N8oHW2pUz6AkAiNvUAsByNv/b1AIAy5egB4COCXoA6JigB4COCXoA6JigB4COCXoA\n6JigB4COCXoA6JigB4COCXoA6JigB4COCXoA6JigB4COCXoA6JigB4COCXoA6JigB4COCXoA6Jig\nB4COCXoA6JigB4COCXoA6JigB4COCXoA6JigB4COCXoA6JigB4COCXoA6JigB4CO9Rr0e867AQCY\ngT3HvsGaGTSxHN03TNctsnzfYfqNbd9KN6yz6Vhv07Hetp51Np3lvN72zE/ybGrVWhvfygpTVdck\nSWvtgHn3slJYZ9Ox3qZjvW0962w6q2G99frRPQAQQQ8AXRP0ANAxQQ8AHRP0ANCxVXnUPQCsFvbo\nAaBjgh4AOiboAaBjgh4AOiboAaBjgh4AOiboAaBjqyroq+rnq+qvq+rfq+rRqlpXVWdV1W7z7m25\nGtZRW+Rx57z7m5eqOraqzqmqK6vqvmF9fGwzNQdV1UVVdXdVPVxV11fVqVW13VL1PW9bs96qas9N\nbHutqi5Y6v7noaqeXFWvq6pPVtWtw7Zzb1VdVVWvraoF/x9f7dvb1q63nre3Xu9H/1Oq6llJrk7y\n1CR/n8m9h381ye8lOaKqDm6t/WCOLS5n9yY5a4H5Dyx1I8vIO5I8L5N18O385J7WC6qqVyb5RJJH\nknw8yd1JfjvJ+5IcnOS4bdnsMrJV621wXZJPLTD/hhn2tZwdl+RDSb6T5PIkdyR5WpJXJflIkiOr\n6ri2wdXPbG9Jplhvg/62t9baqngk+WySluRNG83/y2H+ufPucTk+kqxLsm7efSy3R5JDk+ydpJIc\nMmxDH1vktTsn+V6SR5O8YIP5O2Tyy2dLcvy8/0zLcL3tOSw/b959z3mdHZZJSP/MRvN3zyS8WpJj\nNphve5tuvXW7va2Kj+6HvfmXZRJa/3ujxf8zyYNJTqiqnZa4NVao1trlrbVb2vA/xGYcm+TnklzQ\nWvvqBu/xSCZ7uEnyxm3Q5rKzleuNJK21y1pr/9ha+/FG8+9Mcu7w9JANFtneMtV669Zq+ej+0GF6\nyQJ/6fdX1Zcy+UXgwCSXLnVzK8D2VfXqJM/I5Jei65Nc0Vr70XzbWjEOG6YXL7DsiiQPJTmoqrZv\nrT26dG2tGE+vqtcneXKSHyT5cmvt+jn3tFw8Nkwf32Ce7W3zFlpv63W3va2WoH/2ML15keW3ZBL0\n+0TQL2T3JOdvNO+bVfWa1toX59HQCrPo9tdae7yqvpnkl5I8M8mNS9nYCvEbw+M/VNUXkpzUWrtj\nLh0tA1W1JsmJw9MNQ932tgmbWG/rdbe9rYqP7pPsMkzvXWT5+vm7LkEvK83fJDk8k7DfKcmvJPlw\nJt9n/VNVPW9+ra0Ytr/pPJTkT5MckGS34fHSTA6sOiTJpav867b3JPnlJBe11j67wXzb26Yttt66\n3d5WS9Azpdbau4bvur7bWnuotXZDa+0NmRzE+MQka+fbIb1qrX2vtfYnrbWvtdbuGR5XZPLp21eS\n/Nckr5tvl/NRVW9O8tZMzh46Yc7trBibWm89b2+rJejX/wa7yyLL18+/Zwl66cX6g1leMtcuVgbb\n3wy11h7P5PSoZBVuf1V1SpL3J/l6kkNba3dv9BLb2wK2YL0tqIftbbUE/U3DdJ9Flu89TBf7Dp+f\n9v1huiI/ylpii25/w/eFe2VyUNBtS9nUCrcqt7+qOjXJOZmc033ocAT5xmxvG9nC9bYpK3p7Wy1B\nf/kwfdkCV0N6UiYXkHgoyT8vdWMr2IHDdNX8ZzHCZcP0iAWWvSTJjkmuXsVHQE9j1W1/VXV6Jhe8\nuTaTsPreIi+1vW1gK9bbpqzo7W1VBH1r7d+SXJLJAWQnb7T4XZn8lnZ+a+3BJW5tWauqX1zo4JOq\n2jPJB4anm7zsK0mSC5PcleT4qnrB+plVtUOSdw9PPzSPxpazqtp/ocu7VtXhSd4yPF0V219VvTOT\ng8iuSXJ4a+2uTbzc9jbYmvXW8/ZWq+W6FQtcAvfGJC/K5Bz7m5Mc1FwC9z+pqrWZHLhyRZLbk9yf\n5FlJfiuTq2xdlOTo1toP59XjvFTVUUmOGp7unuTlmfy2f+Uw767W2mkbvf7CTC5JekEmlyR9RSan\nQl2Y5L+thovIbM16G05p2juTf7ffHpY/Nz85T/ydrbX1wdWtqjopyXlJfpTJx88LHU2/rrV23gY1\nq35729r11vX2Nu9L8y3lI8kemZwu9p0kP8wkvM5Kstu8e1uOj0xOLfk/mRyhek8mF5n4fpLPZXIe\nas27xzmum7WZXC5zsce6BWoOzuSXo/+X5OEk/zeTPYXt5v3nWY7rLclrk3w6kytaPpDJJV3vyOTa\n7S+e959lGa2zluQLtrdx663n7W3V7NEDwGq0Kr6jB4DVStADQMcEPQB0TNADQMcEPQB0TNADQMcE\nPQB0TNADQMcEPQB0TNADQMcEPQB0TNADQMcEPQB0TNADQMcEPQB0TNADQMcEPQB07P8DBRP/vtg3\nRjEAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7f0e7bb5e588>"
      ]
     },
     "metadata": {
      "image/png": {
       "height": 250,
       "width": 253
      }
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.imshow(X_train[777].reshape((28, 28)), cmap=plt.cm.gray)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "train_num_samples, test_num_samples = X_train.shape[0], X_test.shape[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "train_history = defaultdict(list)\n",
    "test_history = defaultdict(list)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def save_generated_samples(epoch, G):\n",
    "\n",
    "    # new batch of z's\n",
    "    zz = np.random.normal(0., 1., (100, Z_SIZE))\n",
    "\n",
    "    # new batch of classes [0..9]\n",
    "    generated_classes = np.random.randint(0, 10, 100)\n",
    "\n",
    "    generated_images = G.predict([zz, generated_classes.reshape(-1, 1)])\n",
    "\n",
    "    rr = []\n",
    "    for c in range(10):\n",
    "        rr.append(np.concatenate(generated_images[c * 10:(1 + c) * 10]).reshape(280, 28))\n",
    "    img = np.hstack(rr)\n",
    "\n",
    "    plt.imsave('out4/generated-e%d.png'%epoch, img, cmap=plt.cm.gray)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Epoch 1 / 50\n",
      "599/600 [============================>.] - ETA: 0s - D_loss: 2.3024 - D(G)_loss: 2.3013\n",
      "D: ['loss', 'output_is_fake_loss', 'output_class_loss']\n",
      "[2.302417755126953, 4.82113791804295e-05, 2.302370548248291]\n",
      "D(G): ['loss', 'D_loss', 'D_loss']\n",
      "[2.300259828567505, -0.0023456113412976265, 2.302603006362915]\n",
      "\n",
      "Epoch 2 / 50\n",
      "599/600 [============================>.] - ETA: 0s - D_loss: 2.3023 - D(G)_loss: 2.2973\n",
      "D: ['loss', 'output_is_fake_loss', 'output_class_loss']\n",
      "[2.3022072315216064, 2.6915513444691896e-05, 2.302182912826538]\n",
      "D(G): ['loss', 'D_loss', 'D_loss']\n",
      "[2.296135902404785, -0.006367475260049105, 2.302501916885376]\n",
      "\n",
      "Epoch 3 / 50\n",
      "599/600 [============================>.] - ETA: 0s - D_loss: 2.3020 - D(G)_loss: 2.2946\n",
      "D: ['loss', 'output_is_fake_loss', 'output_class_loss']\n",
      "[2.302037239074707, 1.5769151104905177e-06, 2.302034378051758]\n",
      "D(G): ['loss', 'D_loss', 'D_loss']\n",
      "[2.2937657833099365, -0.008571439422667027, 2.302337646484375]\n",
      "\n",
      "Epoch 4 / 50\n",
      "599/600 [============================>.] - ETA: 0s - D_loss: 2.3018 - D(G)_loss: 2.2928\n",
      "D: ['loss', 'output_is_fake_loss', 'output_class_loss']\n",
      "[2.30171537399292, -2.844334812834859e-05, 2.301743268966675]\n",
      "D(G): ['loss', 'D_loss', 'D_loss']\n",
      "[2.2918941974639893, -0.010087434202432632, 2.3019819259643555]\n",
      "\n",
      "Epoch 5 / 50\n",
      "599/600 [============================>.] - ETA: 0s - D_loss: 2.3014 - D(G)_loss: 2.2903\n",
      "D: ['loss', 'output_is_fake_loss', 'output_class_loss']\n",
      "[2.3011088371276855, -8.565096504753456e-05, 2.301191806793213]\n",
      "D(G): ['loss', 'D_loss', 'D_loss']\n",
      "[2.289431095123291, -0.011352297849953175, 2.30078125]\n",
      "\n",
      "Epoch 6 / 50\n",
      "599/600 [============================>.] - ETA: 0s - D_loss: 2.2977 - D(G)_loss: 2.2843\n",
      "D: ['loss', 'output_is_fake_loss', 'output_class_loss']\n",
      "[2.2909600734710693, -0.0014800827484577894, 2.2924399375915527]\n",
      "D(G): ['loss', 'D_loss', 'D_loss']\n",
      "[2.2713725566864014, -0.013645637780427933, 2.285017967224121]\n",
      "\n",
      "Epoch 7 / 50\n",
      "599/600 [============================>.] - ETA: 0s - D_loss: 1.9333 - D(G)_loss: 2.0034\n",
      "D: ['loss', 'output_is_fake_loss', 'output_class_loss']\n",
      "[1.5027530193328857, -0.017098544165492058, 1.519852876663208]\n",
      "D(G): ['loss', 'D_loss', 'D_loss']\n",
      "[1.5106059312820435, 0.4101656973361969, 1.1004412174224854]\n",
      "\n",
      "Epoch 8 / 50\n",
      "599/600 [============================>.] - ETA: 0s - D_loss: 0.6728 - D(G)_loss: 0.2936\n",
      "D: ['loss', 'output_is_fake_loss', 'output_class_loss']\n",
      "[0.603850245475769, -0.0012129582464694977, 0.6050638556480408]\n",
      "D(G): ['loss', 'D_loss', 'D_loss']\n",
      "[0.027119560167193413, -0.11236795037984848, 0.13948754966259003]\n",
      "\n",
      "Epoch 9 / 50\n",
      "599/600 [============================>.] - ETA: 0s - D_loss: 0.5229 - D(G)_loss: 0.0094\n",
      "D: ['loss', 'output_is_fake_loss', 'output_class_loss']\n",
      "[0.5096631646156311, -0.005670220125466585, 0.5153335332870483]\n",
      "D(G): ['loss', 'D_loss', 'D_loss']\n",
      "[0.00974575337022543, -0.129488006234169, 0.13923369348049164]\n",
      "\n",
      "Epoch 10 / 50\n",
      "599/600 [============================>.] - ETA: 0s - D_loss: 0.4544 - D(G)_loss: 0.0888\n",
      "D: ['loss', 'output_is_fake_loss', 'output_class_loss']\n",
      "[0.4621219336986542, -0.006605795118957758, 0.4687272608280182]\n",
      "D(G): ['loss', 'D_loss', 'D_loss']\n",
      "[0.2275625616312027, 0.10218750685453415, 0.1253754198551178]\n",
      "\n",
      "Epoch 11 / 50\n",
      "599/600 [============================>.] - ETA: 0s - D_loss: 0.3950 - D(G)_loss: 0.3957\n",
      "D: ['loss', 'output_is_fake_loss', 'output_class_loss']\n",
      "[0.41855311393737793, -0.0225729588419199, 0.4411261975765228]\n",
      "D(G): ['loss', 'D_loss', 'D_loss']\n",
      "[0.7663522362709045, 0.6413130164146423, 0.12503939867019653]\n",
      "\n",
      "Epoch 12 / 50\n",
      "599/600 [============================>.] - ETA: 0s - D_loss: 0.4693 - D(G)_loss: 1.9503\n",
      "D: ['loss', 'output_is_fake_loss', 'output_class_loss']\n",
      "[0.48176589608192444, -0.06679652631282806, 0.5485624670982361]\n",
      "D(G): ['loss', 'D_loss', 'D_loss']\n",
      "[3.353423595428467, 3.1764609813690186, 0.17696116864681244]\n",
      "\n",
      "Epoch 13 / 50\n",
      "599/600 [============================>.] - ETA: 0s - D_loss: 0.4640 - D(G)_loss: 14.9360\n",
      "D: ['loss', 'output_is_fake_loss', 'output_class_loss']\n",
      "[0.5438891649246216, -0.6140495538711548, 1.1579391956329346]\n",
      "D(G): ['loss', 'D_loss', 'D_loss']\n",
      "[24.3504638671875, 23.547189712524414, 0.8032838702201843]\n",
      "\n",
      "Epoch 14 / 50\n",
      "  4/600 [..............................] - ETA: 102s - D_loss: 0.3615 - D(G)_loss: 15.9793"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-20-6bdfba45f8e4>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m     67\u001b[0m         combined_losses = combined.train_on_batch(\n\u001b[1;32m     68\u001b[0m             \u001b[0;34m[\u001b[0m\u001b[0mzz\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mgenerated_classes\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mreshape\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m-\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;36m1\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 69\u001b[0;31m             [y_is_fake, generated_classes])\n\u001b[0m\u001b[1;32m     70\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     71\u001b[0m         \u001b[0mepoch_combined_losses\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mappend\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mcombined_losses\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/local/lib/python3.5/dist-packages/keras/engine/training.py\u001b[0m in \u001b[0;36mtrain_on_batch\u001b[0;34m(self, x, y, sample_weight, class_weight)\u001b[0m\n\u001b[1;32m   1563\u001b[0m             \u001b[0mins\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mx\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0my\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0msample_weights\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1564\u001b[0m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_make_train_function\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1565\u001b[0;31m         \u001b[0moutputs\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtrain_function\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mins\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1566\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0moutputs\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0;36m1\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1567\u001b[0m             \u001b[0;32mreturn\u001b[0m \u001b[0moutputs\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/local/lib/python3.5/dist-packages/keras/backend/tensorflow_backend.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, inputs)\u001b[0m\n\u001b[1;32m   2266\u001b[0m         updated = session.run(self.outputs + [self.updates_op],\n\u001b[1;32m   2267\u001b[0m                               \u001b[0mfeed_dict\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mfeed_dict\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 2268\u001b[0;31m                               **self.session_kwargs)\n\u001b[0m\u001b[1;32m   2269\u001b[0m         \u001b[0;32mreturn\u001b[0m \u001b[0mupdated\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0moutputs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   2270\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/local/lib/python3.5/dist-packages/tensorflow/python/client/session.py\u001b[0m in \u001b[0;36mrun\u001b[0;34m(self, fetches, feed_dict, options, run_metadata)\u001b[0m\n\u001b[1;32m    787\u001b[0m     \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    788\u001b[0m       result = self._run(None, fetches, feed_dict, options_ptr,\n\u001b[0;32m--> 789\u001b[0;31m                          run_metadata_ptr)\n\u001b[0m\u001b[1;32m    790\u001b[0m       \u001b[0;32mif\u001b[0m \u001b[0mrun_metadata\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    791\u001b[0m         \u001b[0mproto_data\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtf_session\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mTF_GetBuffer\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mrun_metadata_ptr\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/local/lib/python3.5/dist-packages/tensorflow/python/client/session.py\u001b[0m in \u001b[0;36m_run\u001b[0;34m(self, handle, fetches, feed_dict, options, run_metadata)\u001b[0m\n\u001b[1;32m    995\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mfinal_fetches\u001b[0m \u001b[0;32mor\u001b[0m \u001b[0mfinal_targets\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    996\u001b[0m       results = self._do_run(handle, final_targets, final_fetches,\n\u001b[0;32m--> 997\u001b[0;31m                              feed_dict_string, options, run_metadata)\n\u001b[0m\u001b[1;32m    998\u001b[0m     \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    999\u001b[0m       \u001b[0mresults\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/local/lib/python3.5/dist-packages/tensorflow/python/client/session.py\u001b[0m in \u001b[0;36m_do_run\u001b[0;34m(self, handle, target_list, fetch_list, feed_dict, options, run_metadata)\u001b[0m\n\u001b[1;32m   1130\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mhandle\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1131\u001b[0m       return self._do_call(_run_fn, self._session, feed_dict, fetch_list,\n\u001b[0;32m-> 1132\u001b[0;31m                            target_list, options, run_metadata)\n\u001b[0m\u001b[1;32m   1133\u001b[0m     \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1134\u001b[0m       return self._do_call(_prun_fn, self._session, handle, feed_dict,\n",
      "\u001b[0;32m/usr/local/lib/python3.5/dist-packages/tensorflow/python/client/session.py\u001b[0m in \u001b[0;36m_do_call\u001b[0;34m(self, fn, *args)\u001b[0m\n\u001b[1;32m   1137\u001b[0m   \u001b[0;32mdef\u001b[0m \u001b[0m_do_call\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfn\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1138\u001b[0m     \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1139\u001b[0;31m       \u001b[0;32mreturn\u001b[0m \u001b[0mfn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1140\u001b[0m     \u001b[0;32mexcept\u001b[0m \u001b[0merrors\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mOpError\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0me\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1141\u001b[0m       \u001b[0mmessage\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mcompat\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mas_text\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0me\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmessage\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/local/lib/python3.5/dist-packages/tensorflow/python/client/session.py\u001b[0m in \u001b[0;36m_run_fn\u001b[0;34m(session, feed_dict, fetch_list, target_list, options, run_metadata)\u001b[0m\n\u001b[1;32m   1119\u001b[0m         return tf_session.TF_Run(session, options,\n\u001b[1;32m   1120\u001b[0m                                  \u001b[0mfeed_dict\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfetch_list\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtarget_list\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1121\u001b[0;31m                                  status, run_metadata)\n\u001b[0m\u001b[1;32m   1122\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1123\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0m_prun_fn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0msession\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mhandle\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfeed_dict\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfetch_list\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "for epoch in range(EPOCHS):\n",
    "\n",
    "    print('\\nEpoch {} / {}'.format(epoch + 1, EPOCHS))\n",
    "\n",
    "    batches_num = int(train_num_samples / BATCH_SIZE)\n",
    "    progress_bar = Progbar(target=batches_num)\n",
    "\n",
    "    epoch_combined_losses = []\n",
    "    epoch_D_losses = []\n",
    "\n",
    "    for batch_i in range(batches_num):\n",
    "\n",
    "        if len(epoch_D_losses) + len(epoch_combined_losses) > 1:\n",
    "            progress_bar.update(\n",
    "                batch_i,\n",
    "                values=[\n",
    "                        ('D_loss', np.mean(np.array(epoch_D_losses), axis=0)[0]),\n",
    "                        ('D(G)_loss', np.mean(np.array(epoch_combined_losses),axis=0)[0])\n",
    "                ]\n",
    "            )\n",
    "        else:\n",
    "            progress_bar.update(batch_i)\n",
    "            \n",
    "        # 1: train D on real+generated images\n",
    "        \n",
    "        for d_it in range(D_TRAIN_ITERATIONS):\n",
    "\n",
    "            # create batches of images and labels\n",
    "            real_images = X_train[batch_i * BATCH_SIZE:(batch_i + 1) * BATCH_SIZE]\n",
    "            real_images_classes = y_train[batch_i * BATCH_SIZE:(batch_i + 1) * BATCH_SIZE]\n",
    "\n",
    "            # new batch of z's\n",
    "            zz = np.random.normal(0., 1., (BATCH_SIZE, Z_SIZE))\n",
    "\n",
    "            # new batch of classes [0..9]\n",
    "            generated_classes = np.random.randint(0, 10, BATCH_SIZE)\n",
    "\n",
    "            generated_images = G.predict([zz, generated_classes.reshape(-1, 1)])\n",
    "\n",
    "            X = np.concatenate((real_images, generated_images))\n",
    "            y_is_fake = np.array([-1] * BATCH_SIZE + [1] * BATCH_SIZE) # [is_fake=-1xBATCH_SIZE, 1xBATCH_SIZE]\n",
    "            y_classes = np.concatenate((real_images_classes, generated_classes))\n",
    "\n",
    "            # train \n",
    "            D_losses = D.train_on_batch(X, [y_is_fake, y_classes])\n",
    "            epoch_D_losses.append(D_losses)\n",
    "\n",
    "            # clip D weights\n",
    "\n",
    "            for l in D.layers:\n",
    "                weights = l.get_weights()\n",
    "                weights = [np.clip(w, D_WEIGHTS_LOWER_BOUND, D_WEIGHTS_UPPER_BOUND) for w in weights]\n",
    "                l.set_weights(weights)\n",
    "        \n",
    "        \n",
    "        # 2: train D(G) (only G is trainable)\n",
    "        \n",
    "        # xxx (2 * D_TRAIN_ITERATIONS) so G is trained for the same number of images\n",
    "        # xxx as D above (which is trained on real+generated)\n",
    "        zz = np.random.normal(0., 1.,  BATCH_SIZE, Z_SIZE)\n",
    "        generated_classes = np.random.randint(0, 10, BATCH_SIZE)\n",
    "\n",
    "        y_is_fake = -np.ones(2 * BATCH_SIZE) # all not-fake\n",
    "\n",
    "        combined_losses = combined.train_on_batch(\n",
    "            [zz, generated_classes.reshape((-1, 1))],\n",
    "            [y_is_fake, generated_classes])\n",
    "        \n",
    "        epoch_combined_losses.append(combined_losses)\n",
    "        \n",
    "    print('\\nD:', D.metrics_names)\n",
    "    print(np.mean(np.array(epoch_D_losses), axis=0).tolist())\n",
    "    \n",
    "    print('D(G):', combined.metrics_names)\n",
    "    print(np.mean(np.array(epoch_combined_losses), axis=0).tolist())\n",
    "    \n",
    "    save_generated_samples(epoch, G)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# D"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# clamp_lower = -0.1\n",
    "# clamp_upper = 0.1\n",
    "\n",
    "# # Clip discriminator weights\n",
    "# for l in D.layers:\n",
    "#     weights = l.get_weights()\n",
    "#     weights = [np.clip(w, clamp_lower, clamp_upper) for w in weights]\n",
    "#     l.set_weights(weights)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "gist": {
   "data": {
    "description": "Keras  WACGAN",
    "public": false
   },
   "id": ""
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}