A companion webpage for the ICLR2017 paper (#510)

* add tex/iclr2017.tex * prescribe edward's development version for now * add iclr2017.ipynb; revise snippets
blei-lab · Mar 7, 2017 · cbdc3df · cbdc3df
1 parent 8a722ce
commit cbdc3df
Show file tree

Hide file tree

Showing 4 changed files with 820 additions and 2 deletions.
diff --git a/docs/notebooks/iclr2017.ipynb b/docs/notebooks/iclr2017.ipynb
@@ -0,0 +1,393 @@
+{
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "This notebook is an interactive version of the [companion webpage](http://edwardlib.org/iclr2017) for the article, Deep Probabilistic Programming [(Tran et al., 2017)](https://arxiv.org/abs/1701.03757). See Edward's [API](http://edwardlib.org/api/) for details on how to interact with data, models, inference, and criticism.\n",
+    "\n",
+    "The code snippets assume the following versions.\n",
+    "```bash\n",
+    "pip install -e \"git+https://github.com/blei-lab/edward.git#egg=edward\"\n",
+    "pip install tensorflow==1.0.0  # alternatively, tensorflow-gpu==1.0.0\n",
+    "```"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {
+    "collapsed": true
+   },
+   "outputs": [],
+   "source": [
+    "import tensorflow as tf\n",
+    "from edward.models import Bernoulli, Beta\n",
+    "\n",
+    "theta = Beta(a=1.0, b=1.0)\n",
+    "x = Bernoulli(p=tf.ones(50) * theta)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {
+    "collapsed": true
+   },
+   "outputs": [],
+   "source": [
+    "import tensorflow as tf\n",
+    "from edward.models import Bernoulli, Normal\n",
+    "from keras.layers import Dense\n",
+    "\n",
+    "N = 55000  # number of data points\n",
+    "d = 50  # latent dimension\n",
+    "\n",
+    "# Probabilistic model\n",
+    "z = Normal(mu=tf.zeros([N, d]), sigma=tf.ones([N, d]))\n",
+    "h = Dense(256, activation='relu')(z)\n",
+    "x = Bernoulli(logits=Dense(28 * 28, activation=None)(h))\n",
+    "\n",
+    "# Variational model\n",
+    "qx = tf.placeholder(tf.float32, [N, 28 * 28])\n",
+    "qh = Dense(256, activation='relu')(qx)\n",
+    "qz = Normal(mu=Dense(d, activation=None)(qh),\n",
+    "            sigma=Dense(d, activation='softplus')(qh))"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {
+    "collapsed": true
+   },
+   "outputs": [],
+   "source": [
+    "import edward as ed\n",
+    "import tensorflow as tf\n",
+    "from edward.models import Normal\n",
+    "\n",
+    "H = 50  # number of hidden units\n",
+    "D = 10  # number of features\n",
+    "\n",
+    "def rnn_cell(hprev, xt):\n",
+    "  return tf.tanh(ed.dot(hprev, Wh) + ed.dot(xt, Wx) + bh)\n",
+    "\n",
+    "Wh = Normal(mu=tf.zeros([H, H]), sigma=tf.ones([H, H]))\n",
+    "Wx = Normal(mu=tf.zeros([D, H]), sigma=tf.ones([D, H]))\n",
+    "Wy = Normal(mu=tf.zeros([H, 1]), sigma=tf.ones([H, 1]))\n",
+    "bh = Normal(mu=tf.zeros(H), sigma=tf.ones(H))\n",
+    "by = Normal(mu=tf.zeros(1), sigma=tf.ones(1))\n",
+    "\n",
+    "x = tf.placeholder(tf.float32, [None, D])\n",
+    "h = tf.scan(rnn_cell, x, initializer=tf.zeros(H))\n",
+    "y = Normal(mu=tf.matmul(h, Wy) + by, sigma=1.0)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {
+    "collapsed": true
+   },
+   "outputs": [],
+   "source": [
+    "import tensorflow as tf\n",
+    "from edward.models import Categorical, Normal\n",
+    "\n",
+    "N = 10000  # number of data points\n",
+    "D = 2  # data dimension\n",
+    "K = 5  # number of clusters\n",
+    "\n",
+    "beta = Normal(mu=tf.zeros([K, D]), sigma=tf.ones([K, D]))\n",
+    "z = Categorical(logits=tf.zeros([N, K]))\n",
+    "x = Normal(mu=tf.gather(beta, z), sigma=tf.ones([N, D]))"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {
+    "collapsed": false
+   },
+   "outputs": [],
+   "source": [
+    "import edward as ed\n",
+    "import numpy as np\n",
+    "import tensorflow as tf\n",
+    "from edward.models import Categorical, Normal\n",
+    "\n",
+    "x_train = np.zeros([N, D], dtype=np.float32)\n",
+    "\n",
+    "qbeta = Normal(mu=tf.Variable(tf.zeros([K, D])),\n",
+    "               sigma=tf.exp(tf.Variable(tf.zeros([K, D]))))\n",
+    "qz = Categorical(logits=tf.Variable(tf.zeros([N, K])))\n",
+    "\n",
+    "inference = ed.VariationalInference({beta: qbeta, z: qz}, data={x: x_train})"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {
+    "collapsed": false
+   },
+   "outputs": [],
+   "source": [
+    "import edward as ed\n",
+    "import numpy as np\n",
+    "import tensorflow as tf\n",
+    "from edward.models import Empirical\n",
+    "\n",
+    "x_train = np.zeros([N, D], dtype=np.float32)\n",
+    "\n",
+    "T = 10000  # number of samples\n",
+    "qbeta = Empirical(params=tf.Variable(tf.zeros([T, K, D])))\n",
+    "qz = Empirical(params=tf.Variable(tf.zeros([T, N])))"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {
+    "collapsed": false
+   },
+   "outputs": [],
+   "source": [
+    "import edward as ed\n",
+    "import numpy as np\n",
+    "import tensorflow as tf\n",
+    "from edward.models import Normal\n",
+    "from keras.layers import Dense\n",
+    "\n",
+    "N = 55000  # number of data points\n",
+    "d = 50  # latent dimension\n",
+    "\n",
+    "def generative_network(eps):\n",
+    "  h = Dense(256, activation='relu')(eps)\n",
+    "  return Dense(28 * 28, activation=None)(h)\n",
+    "\n",
+    "def discriminative_network(x):\n",
+    "  h = Dense(28 * 28, activation='relu')(x)\n",
+    "  return Dense(h, activation=None)(1)\n",
+    "\n",
+    "# DATA\n",
+    "x_train = np.zeros([N, 28 * 28], dtype=np.float32)\n",
+    "\n",
+    "# MODEL\n",
+    "eps = Normal(mu=tf.zeros([N, d]), sigma=tf.ones([N, d]))\n",
+    "x = generative_network(eps)\n",
+    "\n",
+    "# INFERENCE\n",
+    "inference = ed.GANInference(data={x: x_train},\n",
+    "    discriminator=discriminative_network)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {
+    "collapsed": false
+   },
+   "outputs": [],
+   "source": [
+    "import edward as ed\n",
+    "import numpy as np\n",
+    "import tensorflow as tf\n",
+    "from edward.models import Categorical, PointMass\n",
+    "\n",
+    "# DATA\n",
+    "x_train = np.zeros([N, D], dtype=np.float32)\n",
+    "\n",
+    "# INFERENCE\n",
+    "qbeta = PointMass(params=tf.Variable(tf.zeros([K, D])))\n",
+    "qz = Categorical(logits=tf.Variable(tf.zeros([N, K])))\n",
+    "\n",
+    "inference_e = ed.VariationalInference({z: qz}, data={x: x_train, beta: qbeta})\n",
+    "inference_m = ed.MAP({beta: qbeta}, data={x: x_train, z: qz})\n",
+    "\n",
+    "inference_e.initialize()\n",
+    "inference_m.initialize()\n",
+    "\n",
+    "tf.initialize_all_variables().run()\n",
+    "\n",
+    "for _ in range(10000):\n",
+    "  inference_e.update()\n",
+    "  inference_m.update()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {
+    "collapsed": false
+   },
+   "outputs": [],
+   "source": [
+    "import edward as ed\n",
+    "import tensorflow as tf\n",
+    "from edward.models import Categorical, Normal\n",
+    "\n",
+    "N = 10000  # number of data points\n",
+    "M = 128  # batch size during training\n",
+    "D = 2  # data dimension\n",
+    "K = 5  # number of clusters\n",
+    "\n",
+    "# DATA\n",
+    "x_batch = tf.placeholder(tf.float32, [M, D])\n",
+    "\n",
+    "# MODEL\n",
+    "beta = Normal(mu=tf.zeros([K, D]), sigma=tf.ones([K, D]))\n",
+    "z = Categorical(logits=tf.zeros([M, K]))\n",
+    "x = Normal(mu=tf.gather(beta, z), sigma=tf.ones([M, D]))\n",
+    "\n",
+    "# INFERENCE\n",
+    "qbeta = Normal(mu=tf.Variable(tf.zeros([K, D])),\n",
+    "               sigma=tf.nn.softplus(tf.Variable(tf.zeros([K, D]))))\n",
+    "qz = Categorical(logits=tf.Variable(tf.zeros([M, D])))\n",
+    "\n",
+    "inference = ed.VariationalInference({beta: qbeta, z: qz}, data={x: x_batch})\n",
+    "inference.initialize(scale={x: float(N) / M, z: float(N) / M})"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {
+    "collapsed": false
+   },
+   "outputs": [],
+   "source": [
+    "import edward as ed\n",
+    "import numpy as np\n",
+    "import tensorflow as tf\n",
+    "from edward.models import Bernoulli, Empirical, Normal\n",
+    "\n",
+    "N = 581012  # number of data points\n",
+    "D = 54  # number of features\n",
+    "T = 100  # number of empirical samples\n",
+    "\n",
+    "# DATA\n",
+    "x_data = np.zeros([N, D], dtype=np.float32)\n",
+    "y_data = np.zeros([N], dtype=np.float32)\n",
+    "\n",
+    "# MODEL\n",
+    "x = tf.Variable(x_data, trainable=False)\n",
+    "beta = Normal(mu=tf.zeros(D), sigma=tf.ones(D))\n",
+    "y = Bernoulli(logits=ed.dot(x, beta))\n",
+    "\n",
+    "# INFERENCE\n",
+    "qbeta = Empirical(params=tf.Variable(tf.zeros([T, D])))\n",
+    "inference = ed.HMC({beta: qbeta}, data={y: y_data})\n",
+    "inference.run(step_size=0.5 / N, n_steps=10)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {
+    "collapsed": false
+   },
+   "outputs": [],
+   "source": [
+    "import tensorflow as tf\n",
+    "from edward.models import Bernoulli, Normal\n",
+    "\n",
+    "N = 1000  # number of data points\n",
+    "D = 528  # number of features\n",
+    "H = 256  # hidden layer size\n",
+    "\n",
+    "W_0 = Normal(mu=tf.zeros([D, H]), sigma=tf.ones([D, H]))\n",
+    "W_1 = Normal(mu=tf.zeros([H, 1]), sigma=tf.ones([H, 1]))\n",
+    "b_0 = Normal(mu=tf.zeros(H), sigma=tf.ones(H))\n",
+    "b_1 = Normal(mu=tf.zeros(1), sigma=tf.ones(1))\n",
+    "\n",
+    "x = tf.placeholder(tf.float32, [N, D])\n",
+    "y = Bernoulli(logits=tf.matmul(tf.nn.tanh(tf.matmul(x, W_0) + b_0), W_1) + b_1)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {
+    "collapsed": false
+   },
+   "outputs": [],
+   "source": [
+    "import tensorflow as tf\n",
+    "from edward.models import Categorical, Dirichlet\n",
+    "\n",
+    "D = 4  # number of documents\n",
+    "N = [11502, 213, 1523, 1351]  # words per doc\n",
+    "K = 10  # number of topics\n",
+    "V = 100000  # vocabulary size\n",
+    "\n",
+    "theta = Dirichlet(alpha=tf.zeros([D, K]) + 0.1)\n",
+    "phi = Dirichlet(alpha=tf.zeros([K, V]) + 0.05)\n",
+    "z = [[0] * N] * D\n",
+    "w = [[0] * N] * D\n",
+    "for d in range(D):\n",
+    "  for n in range(N[d]):\n",
+    "    z[d][n] = Categorical(pi=theta[d, :])\n",
+    "    w[d][n] = Categorical(pi=phi[z[d][n], :])"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {
+    "collapsed": true
+   },
+   "outputs": [],
+   "source": [
+    "import tensorflow as tf\n",
+    "from edward.models import Normal\n",
+    "\n",
+    "N = 10\n",
+    "M = 10\n",
+    "K = 5  # latent dimension\n",
+    "\n",
+    "U = Normal(mu=tf.zeros([M, K]), sigma=tf.ones([M, K]))\n",
+    "V = Normal(mu=tf.zeros([N, K]), sigma=tf.ones([N, K]))\n",
+    "Y = Normal(mu=tf.matmul(U, V, transpose_b=True), sigma=tf.ones([N, M]))"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {
+    "collapsed": true
+   },
+   "outputs": [],
+   "source": [
+    "import tensorflow as tf\n",
+    "from edward.models import Normal\n",
+    "\n",
+    "mu = DirichletProcess(  # see script for implementation\n",
+    "    alpha=0.1, base_cls=Normal, mu=tf.zeros(D), sigma=tf.ones(D), sample_n=N)\n",
+    "x = Normal(mu=mu, sigma=tf.ones([N, D]))"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 2",
+   "language": "python",
+   "name": "python2"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 2
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython2",
+   "version": "2.7.12"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 1
+}