Model() yield op Input internal fixes, plus add K.is_placeholder()

examples/mnist_tfrecord.py

@@ -0,0 +1,211 @@
+'''MNIST dataset with TensorFlow TFRecords.
+
+Gets to 99.25% test accuracy after 12 epochs
+(there is still a lot of margin for parameter tuning).
+'''
+import os
+import copy
+import time
+
+import numpy as np
+
+import tensorflow as tf
+from tensorflow.python.ops import data_flow_ops
+from keras import backend as K
+from keras.models import Model
+from keras.layers import Dense
+from keras.layers import Dropout
+from keras.layers import Flatten
+from keras.layers import Input
+from keras.layers import Conv2D
+from keras.layers import MaxPooling2D
+from keras.callbacks import EarlyStopping
+from keras.callbacks import TensorBoard
+from keras.objectives import categorical_crossentropy
+from keras.utils import np_utils
+from keras.utils.generic_utils import Progbar
+from keras import callbacks as cbks
+from keras import optimizers, objectives
+from keras import metrics as metrics_module
+
+from keras.datasets import mnist
+
+if K.backend() != 'tensorflow':
+    raise RuntimeError('This example can only run with the '
+                       'TensorFlow backend for the time being, '
+                       'because it requires TFRecords, which '
+                       'are not supported on other platforms.')
+
+
+def images_to_tfrecord(images, labels, filename):
+    def _int64_feature(value):
+        return tf.train.Feature(int64_list=tf.train.Int64List(value=[value]))
+
+    def _bytes_feature(value):
+        return tf.train.Feature(bytes_list=tf.train.BytesList(value=[value]))
+
+    """ Save data into TFRecord """
+    if not os.path.isfile(filename):
+        num_examples = images.shape[0]
+
+        rows = images.shape[1]
+        cols = images.shape[2]
+        depth = images.shape[3]
+
+        print('Writing', filename)
+        writer = tf.python_io.TFRecordWriter(filename)
+        for index in range(num_examples):
+            image_raw = images[index].tostring()
+            example = tf.train.Example(features=tf.train.Features(feature={
+                'height': _int64_feature(rows),
+                'width': _int64_feature(cols),
+                'depth': _int64_feature(depth),
+                'label': _int64_feature(int(labels[index])),
+                'image_raw': _bytes_feature(image_raw)}))
+            writer.write(example.SerializeToString())
+        writer.close()
+    else:
+        print('tfrecord %s already exists' % filename)
+
+
+def read_and_decode_recordinput(tf_glob, one_hot=True, classes=None, is_train=None, batch_shape=[1000, 28, 28, 1]):
+    """ Return tensor to read from TFRecord """
+    print 'Creating graph for loading %s TFRecords...' % tf_glob
+    with tf.variable_scope("TFRecords"):
+        record_input = data_flow_ops.RecordInput(tf_glob, batch_size=batch_shape[0])
+        records_op = record_input.get_yield_op()
+        records_op = tf.split(records_op, batch_shape[0], 0)
+        records_op = [tf.reshape(record, []) for record in records_op]
+        progbar = Progbar(len(records_op))
+
+        images = []
+        labels = []
+        for i, serialized_example in enumerate(records_op):
+            progbar.update(i)
+            with tf.variable_scope("parse_images", reuse=True):
+                features = tf.parse_single_example(
+                    serialized_example,
+                    features={
+                        'label': tf.FixedLenFeature([], tf.int64),
+                        'image_raw': tf.FixedLenFeature([], tf.string),
+                    })
+                img = tf.decode_raw(features['image_raw'], tf.uint8)
+                img.set_shape(batch_shape[1] * batch_shape[2])
+                img = tf.reshape(img, [1] + batch_shape[1:])
+
+                img = tf.cast(img, tf.float32) * (1. / 255) - 0.5
+
+                label = tf.cast(features['label'], tf.int32)
+                if one_hot and classes:
+                    label = tf.one_hot(label, classes)
+
+                images.append(img)
+                labels.append(label)
+
+        images = tf.parallel_stack(images, 0)
+        labels = tf.parallel_stack(labels, 0)
+        images = tf.cast(images, tf.float32)
+
+        images = tf.reshape(images, shape=batch_shape)
+
+        # StagingArea will store tensors
+        # across multiple steps to
+        # speed up execution
+        images_shape = images.get_shape()
+        labels_shape = labels.get_shape()
+        copy_stage = data_flow_ops.StagingArea(
+            [tf.float32, tf.float32],
+            shapes=[images_shape, labels_shape])
+        copy_stage_op = copy_stage.put(
+            [images, labels])
+        staged_images, staged_labels = copy_stage.get()
+
+        return images, labels
+
+
+def save_mnist_as_tfrecord():
+    (X_train, y_train), (X_test, y_test) = mnist.load_data()
+    X_train = X_train[..., np.newaxis]
+    X_test = X_test[..., np.newaxis]
+    images_to_tfrecord(images=X_train, labels=y_train, filename='train.mnist.tfrecord')
+    images_to_tfrecord(images=X_test, labels=y_test, filename='test.mnist.tfrecord')
+
+
+def cnn_layers(x_train_input):
+    x = Conv2D(32, (3, 3), activation='relu', padding='valid')(x_train_input)
+    x = Conv2D(64, (3, 3), activation='relu')(x)
+    x = MaxPooling2D(pool_size=(2, 2))(x)
+    x = Dropout(0.25)(x)
+    x = Flatten()(x)
+    x = Dense(128, activation='relu')(x)
+    x = Dropout(0.5)(x)
+    x_train_out = Dense(classes,
+                        activation='softmax',
+                        name='x_train_out')(x)
+    return x_train_out
+
+
+sess = tf.Session()
+K.set_session(sess)
+
+save_mnist_as_tfrecord()
+
+batch_size = 1000
+batch_shape = [batch_size, 28, 28, 1]
+epochs = 6000
+classes = 10
+
+x_train_batch, y_train_batch = read_and_decode_recordinput(
+    'train.mnist.tfrecord',
+    one_hot=True,
+    classes=classes,
+    is_train=True,
+    batch_shape=batch_shape)
+
+x_test_batch, y_test_batch = read_and_decode_recordinput(
+    'test.mnist.tfrecord',
+    one_hot=True,
+    classes=classes,
+    is_train=True,
+    batch_shape=batch_shape)
+
+
+x_batch_shape = x_train_batch.get_shape().as_list()
+y_batch_shape = y_train_batch.get_shape().as_list()
+
+x_train_input = Input(tensor=x_train_batch, batch_shape=x_batch_shape)
+x_train_out = cnn_layers(x_train_input)
+y_train_in_out = Input(tensor=y_train_batch, batch_shape=y_batch_shape, name='y_labels')
+cce = categorical_crossentropy(y_train_batch, x_train_out)
+train_model = Model(inputs=[x_train_input], outputs=[x_train_out])
+train_model.add_loss(cce)
+
+train_model.compile(optimizer='rmsprop',
+                    loss=None,
+                    metrics=['accuracy'])
+train_model.summary()
+
+tensorboard = TensorBoard()
+
+train_model.fit(batch_size=batch_size,
+                epochs=epochs)
+                # disabled due to Keras bug
+                # callbacks=[tensorboard])
+train_model.save_weights('saved_wt.h5')
+
+K.clear_session()
+
+# Second Session, pure Keras
+(X_train, y_train), (X_test, y_test) = mnist.load_data()
+X_train = X_train[..., np.newaxis]
+X_test = X_test[..., np.newaxis]
+x_test_inp = Input(batch_shape=(None,) + (X_test.shape[1:]))
+test_out = cnn_layers(x_test_inp)
+test_model = Model(inputs=x_test_inp, outputs=test_out)
+
+test_model.load_weights('saved_wt.h5')
+test_model.compile(optimizer='rmsprop', loss='categorical_crossentropy', metrics=['accuracy'])
+test_model.summary()
+
+loss, acc = test_model.evaluate(X_test, np_utils.to_categorical(y_test), classes)
+print('\nTest accuracy: {0}'.format(acc))

examples/variational_autoencoder.py

@@ -45,7 +45,7 @@ def sampling(args):
 # Custom loss layer
 class CustomVariationalLayer(Layer):
     def __init__(self, **kwargs):
-        self.is_placeholder = True
+        self._is_placeholder = True
         super(CustomVariationalLayer, self).__init__(**kwargs)
 
     def vae_loss(self, x, x_decoded_mean):

examples/variational_autoencoder_deconv.py

@@ -109,7 +109,7 @@ def sampling(args):
 # Custom loss layer
 class CustomVariationalLayer(Layer):
     def __init__(self, **kwargs):
-        self.is_placeholder = True
+        self._is_placeholder = True
         super(CustomVariationalLayer, self).__init__(**kwargs)
 
     def vae_loss(self, x, x_decoded_mean_squash):

keras/backend/__init__.py

@@ -10,6 +10,7 @@
 from .common import cast_to_floatx
 from .common import image_data_format
 from .common import set_image_data_format
+from .common import is_placeholder
 
 # Obtain Keras base dir path: either ~/.keras or /tmp.
 _keras_base_dir = os.path.expanduser('~')

keras/backend/cntk_backend.py

@@ -2,6 +2,7 @@
 import cntk as C
 import numpy as np
 from .common import _FLOATX, _EPSILON, image_dim_ordering, image_data_format
+from .common import is_placeholder
 from collections import defaultdict
 from contextlib import contextmanager
 import warnings
@@ -256,6 +257,7 @@ def placeholder(
         name=name)
     x._keras_shape = shape
     x._uses_learning_phase = False
+    x._is_placeholder = True
     return x
 
 

keras/backend/common.py

@@ -108,6 +108,29 @@ def cast_to_floatx(x):
     return np.asarray(x, dtype=_FLOATX)
 
 
+def is_placeholder(tensor):
+    """Returns whether a tensor is a placeholder.
+
+    # Arguments
+        tensor: A tensor instance.
+
+    # Returns
+        A boolean.
+
+    # Example
+    ```python
+        >>> from keras import backend as K
+        >>> a = K.placeholder((2, 2), sparse=False)
+        >>> print(K.is_placeholder(a))
+        True
+    ```
+    """
+    try:
+        return tensor._is_placeholder
+    except AttributeError:
+        return False
+
+
 def image_data_format():
     """Returns the default image data format convention ('channels_first' or 'channels_last').
 

keras/backend/tensorflow_backend.py

@@ -10,11 +10,13 @@
 
 import numpy as np
 import os
+from six.moves import zip_longest
 
 from .common import floatx
 from .common import _EPSILON
 from .common import image_data_format
 from ..utils.generic_utils import has_arg
+from .common import is_placeholder
 
 # Legacy functions
 from .common import set_image_dim_ordering
@@ -374,7 +376,7 @@ def is_keras_tensor(x):
     ```python
         >>> from keras import backend as K
         >>> np_var = numpy.array([1, 2])
-        >>> K.is_keras_tensor(np_var) # A numpy array is not a symbolic yensor.
+        >>> K.is_keras_tensor(np_var) # A numpy array is not a symbolic tensor.
         ValueError
         >>> k_var = tf.placeholder('float32', shape=(1,1))
         >>> K.is_keras_tensor(k_var) # A variable created directly from tensorflow/theano is not a Keras tensor.
@@ -432,6 +434,7 @@ def placeholder(shape=None, ndim=None, dtype=None, sparse=False, name=None):
         x = tf.placeholder(dtype, shape=shape, name=name)
     x._keras_shape = shape
     x._uses_learning_phase = False
+    x._is_placeholder = True
     return x
 
 
@@ -2223,9 +2226,12 @@ class Function(object):
         outputs: Output tensors to fetch.
         updates: Additional update ops to be run at function call.
         name: a name to help users identify what this function does.
+        fetches: Parameters forwarded to `tf.session.run(fetches)`.
+        feed_dict: Parameters forwarded to `tf.session.run(feed_dict)`.
     """
 
-    def __init__(self, inputs, outputs, updates=None, name=None, **session_kwargs):
+    def __init__(self, inputs, outputs, updates=None, name=None,
+                 fetches=None, feed_dict=None, **session_kwargs):
         updates = updates or []
         if not isinstance(inputs, (list, tuple)):
             raise TypeError('`inputs` to a TensorFlow backend function '
@@ -2236,8 +2242,11 @@ def __init__(self, inputs, outputs, updates=None, name=None, **session_kwargs):
         if not isinstance(updates, (list, tuple)):
             raise TypeError('`updates` in a TensorFlow backend function '
                             'should be a list or tuple.')
+        # self.inputs holds tf Tensor objects
         self.inputs = list(inputs)
         self.outputs = list(outputs)
+        self.fetches = fetches
+        self.feed_dict = feed_dict
         with tf.control_dependencies(self.outputs):
             updates_ops = []
             for update in updates:
@@ -2252,19 +2261,46 @@ def __init__(self, inputs, outputs, updates=None, name=None, **session_kwargs):
         self.session_kwargs = session_kwargs
 
     def __call__(self, inputs):
+        """Run the TensorFlow session
+
+        # Arguments
+            inputs: Data and values that will go to the feed_dict of Session.run()
+                if it is associated with a tensor, if it is None the tensor will
+                be added to the fetches parameter of Session.run().
+        """
         if not isinstance(inputs, (list, tuple)):
             raise TypeError('`inputs` should be a list or tuple.')
-        feed_dict = {}
-        for tensor, value in zip(self.inputs, inputs):
+        self.current_feed_dict = {} if self.feed_dict is None else self.feed_dict
+        self.feed_to_fetch_count = 0
+        self.current_fetches = self.outputs + [self.updates_op]
+        # self.inputs contains tf tensors, inputs contains feed_dict data.
+        for tensor, value in zip_longest(self.inputs, inputs, fillvalue=None):
+            if tensor is None and value is None:
+                continue
+            elif tensor is None and value is not None:
+                raise ValueError('A tensor containing None '
+                                 'was tied to value ' + str(value) +
+                                 'so Session.run() cannot execute, '
+                                 'please check your data and Model.')
+
             if is_sparse(tensor):
                 sparse_coo = value.tocoo()
                 indices = np.concatenate((np.expand_dims(sparse_coo.row, 1),
                                           np.expand_dims(sparse_coo.col, 1)), 1)
                 value = (indices, sparse_coo.data, sparse_coo.shape)
-            feed_dict[tensor] = value
+
+            if value is None and tensor is not None:
+                self.feed_to_fetch_count += 1
+                self.current_fetches.append(tensor)
+            else:
+                self.current_feed_dict[tensor] = value
+
+        if self.fetches is not None:
+            self.current_fetches += self.fetches
+
         session = get_session()
-        updated = session.run(self.outputs + [self.updates_op],
-                              feed_dict=feed_dict,
+        updated = session.run(fetches=self.current_fetches,
+                              feed_dict=self.current_feed_dict,
                               **self.session_kwargs)
         return updated[:len(self.outputs)]