From bd9d783a97c4759a5b0753643052994db8945ea9 Mon Sep 17 00:00:00 2001
From: Andrew Hundt <ATHundt@gmail.com>
Date: Mon, 12 Jun 2017 18:42:36 -0400
Subject: [PATCH 01/64] K.is_placeholder() added as common.py backend function

---
 examples/variational_autoencoder.py |  2 +-
 keras/backend/__init__.py           |  1 +
 keras/backend/cntk_backend.py       |  2 ++
 keras/backend/common.py             | 23 +++++++++++++++++++++++
 keras/backend/tensorflow_backend.py |  4 +++-
 keras/backend/theano_backend.py     |  1 +
 keras/engine/topology.py            | 12 +++++++++---
 7 files changed, 40 insertions(+), 5 deletions(-)

diff --git a/examples/variational_autoencoder.py b/examples/variational_autoencoder.py
index fd395b17f45..ba313f1c69b 100644
--- a/examples/variational_autoencoder.py
+++ b/examples/variational_autoencoder.py
@@ -45,7 +45,7 @@ def sampling(args):
 # Custom loss layer
 class CustomVariationalLayer(Layer):
     def __init__(self, **kwargs):
-        self.is_placeholder = True
+        self._is_placeholder = True
         super(CustomVariationalLayer, self).__init__(**kwargs)
 
     def vae_loss(self, x, x_decoded_mean):
diff --git a/keras/backend/__init__.py b/keras/backend/__init__.py
index 2e7208cf5af..00e6d52fbe8 100644
--- a/keras/backend/__init__.py
+++ b/keras/backend/__init__.py
@@ -10,6 +10,7 @@
 from .common import cast_to_floatx
 from .common import image_data_format
 from .common import set_image_data_format
+from .common import is_placeholder
 
 # Obtain Keras base dir path: either ~/.keras or /tmp.
 _keras_base_dir = os.path.expanduser('~')
diff --git a/keras/backend/cntk_backend.py b/keras/backend/cntk_backend.py
index b4161ced8e7..cfe056896a0 100644
--- a/keras/backend/cntk_backend.py
+++ b/keras/backend/cntk_backend.py
@@ -2,6 +2,7 @@
 import cntk as C
 import numpy as np
 from .common import _FLOATX, _EPSILON, image_dim_ordering, image_data_format
+from .common import is_placeholder
 from collections import defaultdict
 from contextlib import contextmanager
 import warnings
@@ -256,6 +257,7 @@ def placeholder(
         name=name)
     x._keras_shape = shape
     x._uses_learning_phase = False
+    x._is_placeholder = True
     return x
 
 
diff --git a/keras/backend/common.py b/keras/backend/common.py
index 03bf9165736..c4d98f92405 100644
--- a/keras/backend/common.py
+++ b/keras/backend/common.py
@@ -108,6 +108,29 @@ def cast_to_floatx(x):
     return np.asarray(x, dtype=_FLOATX)
 
 
+def is_placeholder(tensor):
+    """Returns whether a tensor is a placeholder.
+
+    # Arguments
+        tensor: A tensor instance.
+
+    # Returns
+        A boolean.
+
+    # Example
+    ```python
+        >>> from keras import backend as K
+        >>> a = K.placeholder((2, 2), sparse=False)
+        >>> print(K.is_placeholder(a))
+        True
+    ```
+    """
+    try:
+        return tensor._is_placeholder
+    except AttributeError:
+        return False
+
+
 def image_data_format():
     """Returns the default image data format convention ('channels_first' or 'channels_last').
 
diff --git a/keras/backend/tensorflow_backend.py b/keras/backend/tensorflow_backend.py
index ecdc0541442..cad34bdecf0 100644
--- a/keras/backend/tensorflow_backend.py
+++ b/keras/backend/tensorflow_backend.py
@@ -14,6 +14,7 @@
 from .common import floatx
 from .common import _EPSILON
 from .common import image_data_format
+from .common import is_placeholder
 
 # Legacy functions
 from .common import set_image_dim_ordering
@@ -373,7 +374,7 @@ def is_keras_tensor(x):
     ```python
         >>> from keras import backend as K
         >>> np_var = numpy.array([1, 2])
-        >>> K.is_keras_tensor(np_var) # A numpy array is not a symbolic yensor.
+        >>> K.is_keras_tensor(np_var) # A numpy array is not a symbolic tensor.
         ValueError
         >>> k_var = tf.placeholder('float32', shape=(1,1))
         >>> K.is_keras_tensor(k_var) # A variable created directly from tensorflow/theano is not a Keras tensor.
@@ -431,6 +432,7 @@ def placeholder(shape=None, ndim=None, dtype=None, sparse=False, name=None):
         x = tf.placeholder(dtype, shape=shape, name=name)
     x._keras_shape = shape
     x._uses_learning_phase = False
+    x._is_placeholder = True
     return x
 
 
diff --git a/keras/backend/theano_backend.py b/keras/backend/theano_backend.py
index f06ca0ca8be..aa73641bbf5 100644
--- a/keras/backend/theano_backend.py
+++ b/keras/backend/theano_backend.py
@@ -221,6 +221,7 @@ def placeholder(shape=None, ndim=None, dtype=None, sparse=False, name=None):
         x = T.TensorType(dtype, broadcast)(name)
     x._keras_shape = shape
     x._uses_learning_phase = False
+    x._is_placeholder = True
     return x
 
 
diff --git a/keras/engine/topology.py b/keras/engine/topology.py
index c7450351dec..b01319407cd 100644
--- a/keras/engine/topology.py
+++ b/keras/engine/topology.py
@@ -1330,13 +1330,13 @@ def __init__(self, input_shape=None, batch_size=None,
         self.dtype = dtype
 
         if input_tensor is None:
-            self.is_placeholder = True
+            self._is_placeholder = True
             input_tensor = K.placeholder(shape=batch_input_shape,
                                          dtype=dtype,
                                          sparse=self.sparse,
                                          name=self.name)
         else:
-            self.is_placeholder = False
+            self._is_placeholder = False
             input_tensor._keras_shape = batch_input_shape
         # Create an input node to add to self.outbound_node
         # and set output_tensors' _keras_history.
@@ -1535,6 +1535,8 @@ def __init__(self, inputs, outputs, name=None):
         self._output_tensor_cache = {}
         self._output_shape_cache = {}
 
+        self._input_placeholders = []
+        self._input_yield_op_tensors = []
         # User-provided arguments validation.
         for x in self.inputs:
             # Check that x is a Keras tensor.
@@ -1560,6 +1562,10 @@ def __init__(self, inputs, outputs, name=None):
                               'instantiated via `tensor = Input(shape)`.\n'
                               'The tensor that caused the issue was: ' +
                               str(x.name))
+            if K.is_placeholder(layer):
+                self._input_placeholders.append((layer, node_index, tensor_index))
+            else:
+                self._input_yield_op_tensors.append((layer, node_index, tensor_index))
         for x in self.outputs:
             if not hasattr(x, '_keras_history'):
                 cls_name = self.__class__.__name__
@@ -1621,7 +1627,7 @@ def __init__(self, inputs, outputs, name=None):
                                                    i,
                                                    layer.__class__.__name__))
             self.input_names.append(layer.name)
-            if layer.is_placeholder:
+            if K.is_placeholder(layer):
                 self._feed_input_names.append(layer.name)
                 self._feed_inputs.append(layer.input)
                 self._feed_input_shapes.append(self.inputs[i]._keras_shape)

From 0fcec00576514d750f0dee53b98d41599f160f00 Mon Sep 17 00:00:00 2001
From: Andrew Hundt <ATHundt@gmail.com>
Date: Mon, 12 Jun 2017 18:49:48 -0400
Subject: [PATCH 02/64] tensorflow_backend.py session.run(fetches, feed_dict)
 params are forwarded in Function

---
 keras/backend/tensorflow_backend.py | 22 +++++++++++++++++-----
 1 file changed, 17 insertions(+), 5 deletions(-)

diff --git a/keras/backend/tensorflow_backend.py b/keras/backend/tensorflow_backend.py
index cad34bdecf0..152dc3c0110 100644
--- a/keras/backend/tensorflow_backend.py
+++ b/keras/backend/tensorflow_backend.py
@@ -2226,9 +2226,12 @@ class Function(object):
         outputs: Output tensors to fetch.
         updates: Additional update ops to be run at function call.
         name: a name to help users identify what this function does.
+        fetches: Parameters forwarded to `tf.session.run(fetches)`.
+        feed_dict: Parameters forwarded to `tf.session.run(feed_dict)`.
     """
 
-    def __init__(self, inputs, outputs, updates=None, name=None, **session_kwargs):
+    def __init__(self, inputs, outputs, updates=None, name=None,
+                 fetches=None, feed_dict=None, **session_kwargs):
         updates = updates or []
         if not isinstance(inputs, (list, tuple)):
             raise TypeError('`inputs` to a TensorFlow backend function '
@@ -2241,6 +2244,8 @@ def __init__(self, inputs, outputs, updates=None, name=None, **session_kwargs):
                             'should be a list or tuple.')
         self.inputs = list(inputs)
         self.outputs = list(outputs)
+        self.fetches = fetches
+        self.feed_dict = feed_dict
         with tf.control_dependencies(self.outputs):
             updates_ops = []
             for update in updates:
@@ -2257,17 +2262,24 @@ def __init__(self, inputs, outputs, updates=None, name=None, **session_kwargs):
     def __call__(self, inputs):
         if not isinstance(inputs, (list, tuple)):
             raise TypeError('`inputs` should be a list or tuple.')
-        feed_dict = {}
+        if self.feed_dict is None:
+            self.feed_dict = {}
         for tensor, value in zip(self.inputs, inputs):
             if is_sparse(tensor):
                 sparse_coo = value.tocoo()
                 indices = np.concatenate((np.expand_dims(sparse_coo.row, 1),
                                           np.expand_dims(sparse_coo.col, 1)), 1)
                 value = (indices, sparse_coo.data, sparse_coo.shape)
-            feed_dict[tensor] = value
+            self.feed_dict[tensor] = value
         session = get_session()
-        updated = session.run(self.outputs + [self.updates_op],
-                              feed_dict=feed_dict,
+
+        if self.fetches is not None:
+            fetches = self.outputs + [self.updates_op] + self.fetches
+        else:
+            fetches = self.outputs + [self.updates_op]
+
+        updated = session.run(fetches,
+                              feed_dict=self.feed_dict,
                               **self.session_kwargs)
         return updated[:len(self.outputs)]
 

From bb3a76c2c02c97908f51b312004839887630a225 Mon Sep 17 00:00:00 2001
From: Andrew Hundt <ATHundt@gmail.com>
Date: Mon, 12 Jun 2017 16:46:56 -0400
Subject: [PATCH 03/64] .gitignore add .vscode developer environment

---
 .gitignore | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.gitignore b/.gitignore
index 84912fd24b0..028725bb660 100644
--- a/.gitignore
+++ b/.gitignore
@@ -19,3 +19,4 @@ examples/img/*
 
 # developer environments
 .idea
+.vscode

From 73221c97ee3827eb82b8bf456405414e9c7ddaba Mon Sep 17 00:00:00 2001
From: Andrew Hundt <ATHundt@gmail.com>
Date: Mon, 12 Jun 2017 18:59:14 -0400
Subject: [PATCH 04/64] K.is_placeholder() added as common.py backend function

---
 examples/variational_autoencoder_deconv.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/examples/variational_autoencoder_deconv.py b/examples/variational_autoencoder_deconv.py
index f5aa1224d47..3612754c494 100644
--- a/examples/variational_autoencoder_deconv.py
+++ b/examples/variational_autoencoder_deconv.py
@@ -109,7 +109,7 @@ def sampling(args):
 # Custom loss layer
 class CustomVariationalLayer(Layer):
     def __init__(self, **kwargs):
-        self.is_placeholder = True
+        self._is_placeholder = True
         super(CustomVariationalLayer, self).__init__(**kwargs)
 
     def vae_loss(self, x, x_decoded_mean_squash):

From 07057e1547339bbccb36803c14ac8467d0284dc2 Mon Sep 17 00:00:00 2001
From: Andrew Hundt <ATHundt@gmail.com>
Date: Mon, 12 Jun 2017 19:04:26 -0400
Subject: [PATCH 05/64] training.py _check_array_lengths(weights=None)

---
 keras/engine/training.py | 25 +++++++++++++------------
 1 file changed, 13 insertions(+), 12 deletions(-)

diff --git a/keras/engine/training.py b/keras/engine/training.py
index a811030ebbc..520af9cee2d 100644
--- a/keras/engine/training.py
+++ b/keras/engine/training.py
@@ -199,7 +199,7 @@ def _standardize_sample_weights(sample_weight, output_names):
                                                 'sample_weight')
 
 
-def _check_array_lengths(inputs, targets, weights):
+def _check_array_lengths(inputs, targets, weights=None):
     """Does user input validation for numpy arrays.
 
     # Arguments
@@ -212,7 +212,6 @@ def _check_array_lengths(inputs, targets, weights):
     """
     x_lengths = [x.shape[0] for x in inputs]
     y_lengths = [y.shape[0] for y in targets]
-    w_lengths = [w.shape[0] for w in weights]
     set_x = set(x_lengths)
     if len(set_x) > 1:
         raise ValueError('All input arrays (x) should have '
@@ -223,21 +222,23 @@ def _check_array_lengths(inputs, targets, weights):
         raise ValueError('All target arrays (y) should have '
                          'the same number of samples. Got array shapes: ' +
                          str([y.shape for y in targets]))
-    set_w = set(w_lengths)
-    if len(set_w) > 1:
-        raise ValueError('All sample_weight arrays should have '
-                         'the same number of samples. Got array shapes: ' +
-                         str([w.shape for w in weights]))
     if set_x and set_y and list(set_x)[0] != list(set_y)[0]:
         raise ValueError('Input arrays should have '
                          'the same number of samples as target arrays. '
                          'Found ' + str(list(set_x)[0]) + ' input samples '
                          'and ' + str(list(set_y)[0]) + ' target samples.')
-    if set_y and set_w and list(set_y)[0] != list(set_w)[0]:
-        raise ValueError('Sample_weight arrays should have '
-                         'the same number of samples as target arrays. Got ' +
-                         str(list(set_y)[0]) + ' input samples and ' +
-                         str(list(set_w)[0]) + ' target samples.')
+    if weights is not None:
+        w_lengths = [w.shape[0] for w in weights]
+        set_w = set(w_lengths)
+        if len(set_w) > 1:
+            raise ValueError('All sample_weight arrays should have '
+                             'the same number of samples. Got array shapes: ' +
+                             str([w.shape for w in weights]))
+        if set_y and set_w and list(set_y)[0] != list(set_w)[0]:
+            raise ValueError('Sample_weight arrays should have '
+                             'the same number of samples as target arrays. Got ' +
+                             str(list(set_y)[0]) + ' input samples and ' +
+                             str(list(set_w)[0]) + ' target samples.')
 
 
 def _check_loss_and_target_compatibility(targets, loss_fns, output_shapes):

From 4067e5fe72b954522960da9fad93120d909f9376 Mon Sep 17 00:00:00 2001
From: Andrew Hundt <ATHundt@gmail.com>
Date: Mon, 12 Jun 2017 19:08:21 -0400
Subject: [PATCH 06/64] training.py _weighted_mask_objective() accesses weights
 variable correctly

---
 keras/engine/training.py | 9 ++++-----
 1 file changed, 4 insertions(+), 5 deletions(-)

diff --git a/keras/engine/training.py b/keras/engine/training.py
index 520af9cee2d..4c084783454 100644
--- a/keras/engine/training.py
+++ b/keras/engine/training.py
@@ -444,13 +444,12 @@ def weighted(y_true, y_pred, weights, mask=None):
             #  to the number of unmasked samples.
             score_array /= K.mean(mask)
 
-        # reduce score_array to same ndim as weight array
-        ndim = K.ndim(score_array)
-        weight_ndim = K.ndim(weights)
-        score_array = K.mean(score_array, axis=list(range(weight_ndim, ndim)))
-
         # apply sample weighting
         if weights is not None:
+            # reduce score_array to same ndim as weight array
+            ndim = K.ndim(score_array)
+            weight_ndim = K.ndim(weights)
+            score_array = K.mean(score_array, axis=list(range(weight_ndim, ndim)))
             score_array *= weights
             score_array /= K.mean(K.cast(K.not_equal(weights, 0), K.floatx()))
         return K.mean(score_array)

From bc742c920362c9c7a6439f3987812f11eaf27b98 Mon Sep 17 00:00:00 2001
From: Andrew Hundt <ATHundt@gmail.com>
Date: Mon, 12 Jun 2017 19:23:12 -0400
Subject: [PATCH 07/64] training.py only create placeholder when tensor is a
 placeholder

---
 keras/engine/training.py | 11 +++++++----
 1 file changed, 7 insertions(+), 4 deletions(-)

diff --git a/keras/engine/training.py b/keras/engine/training.py
index 4c084783454..02ce6bbd6e3 100644
--- a/keras/engine/training.py
+++ b/keras/engine/training.py
@@ -884,10 +884,13 @@ def compile(self, optimizer, loss, metrics=None, loss_weights=None,
             else:
                 shape = self.internal_output_shapes[i]
                 name = self.output_names[i]
-                target = K.placeholder(ndim=len(shape),
-                                       name=name + '_target',
-                                       sparse=K.is_sparse(self.outputs[i]),
-                                       dtype=K.dtype(self.outputs[i]))
+                if K.is_placeholder(self.outputs[i]):
+                    target = K.placeholder(ndim=len(shape),
+                                           name=name + '_target',
+                                           sparse=K.is_sparse(self.outputs[i]),
+                                           dtype=K.dtype(self.outputs[i]))
+                else:
+                    target = self.outputs[i]
                 self.targets.append(target)
                 self._feed_targets.append(target)
 

From b2e8cec228f2acc5585344f83868075788789914 Mon Sep 17 00:00:00 2001
From: Andrew Hundt <ATHundt@gmail.com>
Date: Mon, 12 Jun 2017 23:58:48 -0400
Subject: [PATCH 08/64] tensorflow tests pass

---
 keras/backend/common.py             |   2 +-
 keras/backend/tensorflow_backend.py |  26 +++-
 keras/engine/topology.py            |   8 +-
 keras/engine/training.py            | 233 ++++++++++++++++------------
 keras/models.py                     |   2 +
 5 files changed, 164 insertions(+), 107 deletions(-)

diff --git a/keras/backend/common.py b/keras/backend/common.py
index c4d98f92405..c5d40088b75 100644
--- a/keras/backend/common.py
+++ b/keras/backend/common.py
@@ -128,7 +128,7 @@ def is_placeholder(tensor):
     try:
         return tensor._is_placeholder
     except AttributeError:
-        return False
+        return True
 
 
 def image_data_format():
diff --git a/keras/backend/tensorflow_backend.py b/keras/backend/tensorflow_backend.py
index 152dc3c0110..ac3c92e61db 100644
--- a/keras/backend/tensorflow_backend.py
+++ b/keras/backend/tensorflow_backend.py
@@ -14,7 +14,6 @@
 from .common import floatx
 from .common import _EPSILON
 from .common import image_data_format
-from .common import is_placeholder
 
 # Legacy functions
 from .common import set_image_dim_ordering
@@ -281,6 +280,31 @@ def to_dense(tensor):
 name_scope = tf.name_scope
 
 
+def is_placeholder(tensor):
+    """Returns whether a tensor is a placeholder.
+
+    # Arguments
+        tensor: A tensor instance.
+
+    # Returns
+        A boolean.
+
+    # Example
+    ```python
+        >>> from keras import backend as K
+        >>> a = K.placeholder((2, 2), sparse=False)
+        >>> print(K.is_placeholder(a))
+        True
+    ```
+    """
+    try:
+        if isinstance(tensor, tf.Variable):
+            return True
+        return tensor._is_placeholder
+    except AttributeError:
+        return True
+
+
 def variable(value, dtype=None, name=None):
     """Instantiates a variable and returns it.
 
diff --git a/keras/engine/topology.py b/keras/engine/topology.py
index b01319407cd..97b6a0a3ab8 100644
--- a/keras/engine/topology.py
+++ b/keras/engine/topology.py
@@ -113,7 +113,7 @@ def __init__(self, outbound_layer,
                  input_tensors, output_tensors,
                  input_masks, output_masks,
                  input_shapes, output_shapes,
-                 arguments=None):
+                 arguments=None, is_placeholder=False):
         # Layer instance (NOT a list).
         # this is the layer that takes a list of input tensors
         # and turns them into a list of output tensors.
@@ -157,6 +157,8 @@ def __init__(self, outbound_layer,
         # Optional keyword arguments to layer's `call`.
         self.arguments = arguments
 
+        # Indicates if the node represents a placeholder variable
+        self._is_placeholder = is_placeholder
         # Add nodes to all layers involved.
         for layer in inbound_layers:
             if layer is not None:
@@ -1342,6 +1344,7 @@ def __init__(self, input_shape=None, batch_size=None,
         # and set output_tensors' _keras_history.
         input_tensor._uses_learning_phase = False
         input_tensor._keras_history = (self, 0, 0)
+        input_tensor._is_placeholder = self._is_placeholder
         Node(self,
              inbound_layers=[],
              node_indices=[],
@@ -1351,7 +1354,8 @@ def __init__(self, input_shape=None, batch_size=None,
              input_masks=[None],
              output_masks=[None],
              input_shapes=[batch_input_shape],
-             output_shapes=[batch_input_shape])
+             output_shapes=[batch_input_shape],
+             is_placeholder=self._is_placeholder)
 
     def get_config(self):
         config = {'batch_input_shape': self.batch_input_shape,
diff --git a/keras/engine/training.py b/keras/engine/training.py
index 02ce6bbd6e3..d1ef88cdae7 100644
--- a/keras/engine/training.py
+++ b/keras/engine/training.py
@@ -211,29 +211,31 @@ def _check_array_lengths(inputs, targets, weights=None):
         ValueError: in case of incorrectly formatted data.
     """
     x_lengths = [x.shape[0] for x in inputs]
-    y_lengths = [y.shape[0] for y in targets]
     set_x = set(x_lengths)
     if len(set_x) > 1:
         raise ValueError('All input arrays (x) should have '
                          'the same number of samples. Got array shapes: ' +
                          str([x.shape for x in inputs]))
-    set_y = set(y_lengths)
-    if len(set_y) > 1:
-        raise ValueError('All target arrays (y) should have '
-                         'the same number of samples. Got array shapes: ' +
-                         str([y.shape for y in targets]))
-    if set_x and set_y and list(set_x)[0] != list(set_y)[0]:
-        raise ValueError('Input arrays should have '
-                         'the same number of samples as target arrays. '
-                         'Found ' + str(list(set_x)[0]) + ' input samples '
-                         'and ' + str(list(set_y)[0]) + ' target samples.')
-    if weights is not None:
+    if targets:
+        y_lengths = [y.shape[0] for y in targets]
+        set_y = set(y_lengths)
+        if len(set_y) > 1:
+            raise ValueError('All target arrays (y) should have '
+                             'the same number of samples. Got array shapes: ' +
+                             str([y.shape for y in targets]))
+        if set_x and set_y and list(set_x)[0] != list(set_y)[0]:
+            raise ValueError('Input arrays should have '
+                             'the same number of samples as target arrays. '
+                             'Found ' + str(list(set_x)[0]) + ' input samples '
+                             'and ' + str(list(set_y)[0]) + ' target samples.')
+    if weights:
         w_lengths = [w.shape[0] for w in weights]
         set_w = set(w_lengths)
         if len(set_w) > 1:
             raise ValueError('All sample_weight arrays should have '
                              'the same number of samples. Got array shapes: ' +
                              str([w.shape for w in weights]))
+    if targets and weights:
         if set_y and set_w and list(set_y)[0] != list(set_w)[0]:
             raise ValueError('Sample_weight arrays should have '
                              'the same number of samples as target arrays. Got ' +
@@ -718,7 +720,6 @@ def compile(self, optimizer, loss, metrics=None, loss_weights=None,
         """
         loss = loss or {}
         self.optimizer = optimizers.get(optimizer)
-        self.sample_weight_mode = sample_weight_mode
         self.loss = loss
         self.loss_weights = loss_weights
 
@@ -801,79 +802,12 @@ def compile(self, optimizer, loss, metrics=None, loss_weights=None,
                             ' - expected a list of dicts.')
 
         # Prepare sample weights.
-        sample_weights = []
-        sample_weight_modes = []
-        if isinstance(sample_weight_mode, dict):
-            for name in sample_weight_mode:
-                if name not in self.output_names:
-                    raise ValueError('Unknown entry in '
-                                     'sample_weight_mode dictionary: "' +
-                                     name + '". '
-                                     'Only expected the following keys: ' +
-                                     str(self.output_names))
-            for i, name in enumerate(self.output_names):
-                if i in skip_indices:
-                    weight = None
-                    sample_weight_modes.append(None)
-                else:
-                    if name not in sample_weight_mode:
-                        raise ValueError('Output "' + name +
-                                         '" missing from sample_weight_modes '
-                                         'dictionary')
-                    if sample_weight_mode.get(name) == 'temporal':
-                        weight = K.placeholder(ndim=2,
-                                               name=name + '_sample_weights')
-                        sample_weight_modes.append('temporal')
-                    else:
-                        weight = K.placeholder(ndim=1,
-                                               name=name + '_sample_weights')
-                        sample_weight_modes.append(None)
-                sample_weights.append(weight)
-        elif isinstance(sample_weight_mode, list):
-            if len(sample_weight_mode) != len(self.outputs):
-                raise ValueError('When passing a list as sample_weight_mode, '
-                                 'it should have one entry per model outputs. '
-                                 'The model has ' + str(len(self.outputs)) +
-                                 ' outputs, but you passed '
-                                 'sample_weight_mode=' +
-                                 str(sample_weight_mode))
-            for i in range(len(self.output_names)):
-                if i in skip_indices:
-                    weight = None
-                    sample_weight_modes.append(None)
-                else:
-                    mode = sample_weight_mode[i]
-                    name = self.output_names[i]
-                    if mode == 'temporal':
-                        weight = K.placeholder(ndim=2,
-                                               name=name + '_sample_weights')
-                        sample_weight_modes.append('temporal')
-                    else:
-                        weight = K.placeholder(ndim=1,
-                                               name=name + '_sample_weights')
-                        sample_weight_modes.append(None)
-                sample_weights.append(weight)
+        if self._input_yield_op_tensors:
+            sample_weight_mode = 'disabled'
         else:
-            for i, name in enumerate(self.output_names):
-                if i in skip_indices:
-                    sample_weight_modes.append(None)
-                    sample_weights.append(None)
-                else:
-                    if sample_weight_mode == 'temporal':
-                        sample_weights.append(
-                            K.placeholder(ndim=2,
-                                          name=name + '_sample_weights'))
-                        sample_weight_modes.append('temporal')
-                    else:
-                        sample_weights.append(
-                            K.placeholder(ndim=1,
-                                          name=name + '_sample_weights'))
-                        sample_weight_modes.append(None)
-        self.sample_weight_modes = sample_weight_modes
-        self._feed_sample_weight_modes = []
-        for i in range(len(self.outputs)):
-            if i not in skip_indices:
-                self._feed_sample_weight_modes.append(self.sample_weight_modes[i])
+            sample_weights = self._prepare_sample_weights(
+                sample_weight_mode, skip_indices)
+        self.sample_weight_mode = sample_weight_mode
 
         # Prepare targets of model.
         self.targets = []
@@ -907,7 +841,10 @@ def compile(self, optimizer, loss, metrics=None, loss_weights=None,
             y_true = self.targets[i]
             y_pred = self.outputs[i]
             weighted_loss = weighted_losses[i]
-            sample_weight = sample_weights[i]
+            if sample_weight_mode is 'disabled':
+                sample_weight = None
+            else:
+                sample_weight = sample_weights[i]
             mask = masks[i]
             loss_weight = loss_weights_list[i]
             output_loss = weighted_loss(y_true, y_pred,
@@ -978,11 +915,12 @@ def append_metric(layer_num, metric_name, metric_tensor):
 
         # Prepare gradient updates and state updates.
         self.total_loss = total_loss
-        self.sample_weights = sample_weights
-        self._feed_sample_weights = []
-        for i in range(len(self.sample_weights)):
-            if i not in skip_indices:
-                self._feed_sample_weights.append(sample_weights[i])
+        if sample_weight_mode is not 'disabled':
+            self.sample_weights = sample_weights
+            self._feed_sample_weights = []
+            for i in range(len(self.sample_weights)):
+                if i not in skip_indices:
+                    self._feed_sample_weights.append(sample_weights[i])
 
         # Functions for train, test and predict will
         # be compiled lazily when required.
@@ -1003,11 +941,90 @@ def append_metric(layer_num, metric_name, metric_tensor):
                 trainable_weights.sort(key=lambda x: x.name)
         self._collected_trainable_weights = trainable_weights
 
+    def _prepare_sample_weights(self, sample_weight_mode, skip_indices):
+        # Prepare sample weights.
+        sample_weights = []
+        sample_weight_modes = []
+        if isinstance(sample_weight_mode, dict):
+            for name in sample_weight_mode:
+                if name not in self.output_names:
+                    raise ValueError('Unknown entry in '
+                                     'sample_weight_mode dictionary: "' +
+                                     name + '". '
+                                     'Only expected the following keys: ' +
+                                     str(self.output_names))
+            for i, name in enumerate(self.output_names):
+                if i in skip_indices:
+                    weight = None
+                    sample_weight_modes.append(None)
+                else:
+                    if name not in sample_weight_mode:
+                        raise ValueError('Output "' + name +
+                                         '" missing from sample_weight_modes '
+                                         'dictionary')
+                    if sample_weight_mode.get(name) == 'temporal':
+                        weight = K.placeholder(ndim=2,
+                                               name=name + '_sample_weights')
+                        sample_weight_modes.append('temporal')
+                    else:
+                        weight = K.placeholder(ndim=1,
+                                               name=name + '_sample_weights')
+                        sample_weight_modes.append(None)
+                sample_weights.append(weight)
+        elif isinstance(sample_weight_mode, list):
+            if len(sample_weight_mode) != len(self.outputs):
+                raise ValueError('When passing a list as sample_weight_mode, '
+                                 'it should have one entry per model outputs. '
+                                 'The model has ' + str(len(self.outputs)) +
+                                 ' outputs, but you passed '
+                                 'sample_weight_mode=' +
+                                 str(sample_weight_mode))
+            for i in range(len(self.output_names)):
+                if i in skip_indices:
+                    weight = None
+                    sample_weight_modes.append(None)
+                else:
+                    mode = sample_weight_mode[i]
+                    name = self.output_names[i]
+                    if mode == 'temporal':
+                        weight = K.placeholder(ndim=2,
+                                               name=name + '_sample_weights')
+                        sample_weight_modes.append('temporal')
+                    else:
+                        weight = K.placeholder(ndim=1,
+                                               name=name + '_sample_weights')
+                        sample_weight_modes.append(None)
+                sample_weights.append(weight)
+        else:
+            for i, name in enumerate(self.output_names):
+                if i in skip_indices:
+                    sample_weight_modes.append(None)
+                    sample_weights.append(None)
+                else:
+                    if sample_weight_mode == 'temporal':
+                        sample_weights.append(
+                            K.placeholder(ndim=2,
+                                          name=name + '_sample_weights'))
+                        sample_weight_modes.append('temporal')
+                    else:
+                        sample_weights.append(
+                            K.placeholder(ndim=1,
+                                          name=name + '_sample_weights'))
+                        sample_weight_modes.append(None)
+        self.sample_weight_modes = sample_weight_modes
+        self._feed_sample_weight_modes = []
+        for i in range(len(self.outputs)):
+            if i not in skip_indices:
+                self._feed_sample_weight_modes.append(self.sample_weight_modes[i])
+        return sample_weights
+
     def _make_train_function(self):
         if not hasattr(self, 'train_function'):
             raise RuntimeError('You must compile your model before using it.')
         if self.train_function is None:
-            inputs = self._feed_inputs + self._feed_targets + self._feed_sample_weights
+            inputs = self._feed_inputs + self._feed_targets
+            if self.sample_weight_mode is not 'disabled':
+                inputs += self._feed_sample_weights
             if self.uses_learning_phase and not isinstance(K.learning_phase(), int):
                 inputs += [K.learning_phase()]
 
@@ -1027,7 +1044,9 @@ def _make_test_function(self):
         if not hasattr(self, 'test_function'):
             raise RuntimeError('You must compile your model before using it.')
         if self.test_function is None:
-            inputs = self._feed_inputs + self._feed_targets + self._feed_sample_weights
+            inputs = self._feed_inputs + self._feed_targets
+            if self.sample_weight_mode is not 'disabled':
+                inputs += self._feed_sample_weights
             if self.uses_learning_phase and not isinstance(K.learning_phase(), int):
                 inputs += [K.learning_phase()]
             # Return loss and metrics, no gradient updates.
@@ -1143,7 +1162,9 @@ def _fit_loop(self, f, ins, out_labels=None, batch_size=32,
             for batch_index, (batch_start, batch_end) in enumerate(batches):
                 batch_ids = index_array[batch_start:batch_end]
                 try:
-                    if isinstance(ins[-1], float):
+                    if not ins:
+                        ins_batch = []
+                    elif isinstance(ins[-1], float):
                         # Do not slice the training phase flag.
                         ins_batch = _slice_arrays(ins[:-1], batch_ids) + [ins[-1]]
                     else:
@@ -1312,17 +1333,23 @@ def _standardize_user_data(self, x, y,
                                     self._feed_input_shapes,
                                     check_batch_axis=False,
                                     exception_prefix='input')
-        y = _standardize_input_data(y, self._feed_output_names,
-                                    output_shapes,
-                                    check_batch_axis=False,
-                                    exception_prefix='target')
-        sample_weights = _standardize_sample_weights(sample_weight,
-                                                     self._feed_output_names)
+        if not x and y is None and self.sample_weight_mode is 'disabled':
+            y = []
+        else:
+            y = _standardize_input_data(y, self._feed_output_names,
+                                        output_shapes,
+                                        check_batch_axis=False,
+                                        exception_prefix='target')
         class_weights = _standardize_class_weights(class_weight,
                                                    self._feed_output_names)
-        sample_weights = [_standardize_weights(ref, sw, cw, mode)
-                          for (ref, sw, cw, mode)
-                          in zip(y, sample_weights, class_weights, self._feed_sample_weight_modes)]
+        if self.sample_weight_mode is not 'disabled':
+            sample_weights = _standardize_sample_weights(sample_weight,
+                                                         self._feed_output_names)
+            sample_weights = [_standardize_weights(ref, sw, cw, mode)
+                              for (ref, sw, cw, mode)
+                              in zip(y, sample_weights, class_weights, self._feed_sample_weight_modes)]
+        else:
+            sample_weights = []
         _check_array_lengths(x, y, sample_weights)
         _check_loss_and_target_compatibility(y,
                                              self._feed_loss_fns,
diff --git a/keras/models.py b/keras/models.py
index b3a8becbd11..2d826eed990 100644
--- a/keras/models.py
+++ b/keras/models.py
@@ -456,6 +456,7 @@ def add(self, layer):
                                  'For multi-output layers, '
                                  'use the functional API.')
 
+            layer.inbound_nodes[0].output_tensors[0]._is_placeholder = True
             self.outputs = [layer.inbound_nodes[0].output_tensors[0]]
             self.inputs = topology.get_source_inputs(self.outputs[0])
 
@@ -479,6 +480,7 @@ def add(self, layer):
                                 'should have a single output tensor. '
                                 'For multi-output layers, '
                                 'use the functional API.')
+            output_tensor._is_placeholder = True
             self.outputs = [output_tensor]
             # update self.inbound_nodes
             self.inbound_nodes[0].output_tensors = self.outputs

From 745f23d204f8615268ffcf5f700180e6ee49ab0c Mon Sep 17 00:00:00 2001
From: Andrew Hundt <ATHundt@gmail.com>
Date: Mon, 12 Jun 2017 23:59:02 -0400
Subject: [PATCH 09/64] mnist_tfrecord.py added

---
 examples/mnist_tfrecord.py | 209 +++++++++++++++++++++++++++++++++++++
 1 file changed, 209 insertions(+)
 create mode 100644 examples/mnist_tfrecord.py

diff --git a/examples/mnist_tfrecord.py b/examples/mnist_tfrecord.py
new file mode 100644
index 00000000000..c0f884b2ef3
--- /dev/null
+++ b/examples/mnist_tfrecord.py
@@ -0,0 +1,209 @@
+'''MNIST dataset with TensorFlow TFRecords.
+
+Gets to 99.25% test accuracy after 12 epochs
+(there is still a lot of margin for parameter tuning).
+'''
+import sys
+sys.path.append('/Users/athundt/source/keras')
+import os
+import copy
+import time
+
+import numpy as np
+
+import tensorflow as tf
+from tensorflow.python.ops import data_flow_ops
+from keras import backend as K
+from keras.models import Model
+from keras.layers import Dense, Dropout, Flatten, Input, Conv2D
+from keras.callbacks import EarlyStopping
+from keras.objectives import categorical_crossentropy
+from keras.utils import np_utils
+from keras import callbacks as cbks
+from keras import optimizers, objectives
+# from keras.engine.training import collect_metrics, weighted_objective
+from keras import metrics as metrics_module
+
+from keras.datasets import mnist
+
+if K.backend() != 'tensorflow':
+    raise RuntimeError('This example can only run with the '
+                       'TensorFlow backend for the time being, '
+                       'because it requires TFRecords, which '
+                       'are not supported on other platforms.')
+
+
+def images_to_tfrecord(images, labels, filename):
+    def _int64_feature(value):
+        return tf.train.Feature(int64_list=tf.train.Int64List(value=[value]))
+
+    def _bytes_feature(value):
+        return tf.train.Feature(bytes_list=tf.train.BytesList(value=[value]))
+
+    """ Save data into TFRecord """
+    if not os.path.isfile(filename):
+        num_examples = images.shape[0]
+
+        rows = images.shape[1]
+        cols = images.shape[2]
+        depth = images.shape[3]
+
+        print('Writing', filename)
+        writer = tf.python_io.TFRecordWriter(filename)
+        for index in range(num_examples):
+            image_raw = images[index].tostring()
+            example = tf.train.Example(features=tf.train.Features(feature={
+                'height': _int64_feature(rows),
+                'width': _int64_feature(cols),
+                'depth': _int64_feature(depth),
+                'label': _int64_feature(int(labels[index])),
+                'image_raw': _bytes_feature(image_raw)}))
+            writer.write(example.SerializeToString())
+        writer.close()
+    else:
+        print 'tfrecord already exists'
+
+
+def read_and_decode(filename, one_hot=True, classes=None, is_train=None):
+    """ Return tensor to read from TFRecord """
+    filename_queue = tf.train.string_input_producer([filename])
+    reader = tf.TFRecordReader()
+    _, serialized_example = reader.read(filename_queue)
+    features = tf.parse_single_example(serialized_example,
+                                       features={
+                                           'label': tf.FixedLenFeature([], tf.int64),
+                                           'image_raw': tf.FixedLenFeature([], tf.string),
+                                       })
+    img = tf.decode_raw(features['image_raw'], tf.uint8)
+    img.set_shape([28 * 28])
+    img = tf.reshape(img, [28, 28, 1])
+
+    img = tf.cast(img, tf.float32) * (1. / 255) - 0.5
+
+    label = tf.cast(features['label'], tf.int32)
+    if one_hot and classes:
+        label = tf.one_hot(label, classes)
+
+    x_train_batch, y_train_batch = K.tf.train.shuffle_batch(
+        [img, label],
+        batch_size=batch_size,
+        capacity=2000,
+        min_after_dequeue=1000,
+        num_threads=32)  # set the number of threads here
+
+    return x_train_batch, y_train_batch
+
+
+def read_and_decode_recordinput(tf_glob, one_hot=True, classes=None, is_train=None, batch_size=1000):
+    """ Return tensor to read from TFRecord """
+
+    record_input = data_flow_ops.RecordInput(tf_glob, batch_size=batch_size)
+    records_op = record_input.get_yield_op()
+    records_op = tf.split(records_op, batch_size, 0)
+    records_op = [tf.reshape(record, []) for record in records_op]
+
+    imgs = []
+    labels = []
+    for serialized_example in records_op:
+        features = tf.parse_single_example(serialized_example,
+                                           features={
+                                               'label': tf.FixedLenFeature([], tf.int64),
+                                               'image_raw': tf.FixedLenFeature([], tf.string),
+                                           })
+        img = tf.decode_raw(features['image_raw'], tf.uint8)
+        img.set_shape([28 * 28])
+        img = tf.reshape(img, [1, 28, 28, 1])
+
+        img = tf.cast(img, tf.float32) * (1. / 255) - 0.5
+
+        label = tf.cast(features['label'], tf.int32)
+        if one_hot and classes:
+            label = tf.one_hot(label, classes)
+
+        imgs.append(img)
+        labels.append(label)
+
+    imgs = tf.concat(imgs, 0)
+    labels = tf.concat(labels, 0)
+    return imgs, labels
+
+
+def save_mnist_as_tfrecord():
+    (X_train, y_train), (X_test, y_test) = mnist.load_data()
+    X_train = X_train[..., np.newaxis]
+    X_test = X_test[..., np.newaxis]
+    images_to_tfrecord(images=X_train, labels=y_train, filename='train.mnist.tfrecord')
+    images_to_tfrecord(images=X_test, labels=y_test, filename='test.mnist.tfrecord')
+
+
+def cnn_layers(x_train_input):
+    con1 = Conv2D(32, (3, 3), activation='relu',
+                  strides=(2, 2), padding='valid')(x_train_input)
+    con2 = Conv2D(32, (3, 3), activation='relu',
+                  strides=(2, 2))(con1)
+    fla1 = Flatten()(con2)
+    den1 = Dense(128, activation='relu')(fla1)
+    x_train_out = Dense(classes,
+                        activation='softmax',
+                        name='x_train_out')(den1)
+    return x_train_out
+
+
+def create_cnn_model(x_train_batch, y_train_batch, x_batch_shape, y_batch_shape):
+    x_train_input = Input(tensor=x_train_batch, batch_shape=x_batch_shape)
+    x_train_out = cnn_layers(x_train_input)
+    # Workaround until _is_placeholder can be deduced automatically
+    x_train_out._is_placeholder = False
+    y_train_in_out = Input(tensor=y_train_batch, batch_shape=y_batch_shape)
+    return Model(inputs=[x_train_input, y_train_in_out], outputs=[x_train_out, y_train_in_out])
+
+
+sess = tf.Session()
+K.set_session(sess)
+
+save_mnist_as_tfrecord()
+
+batch_size = 1000
+epochs = 300
+classes = 10
+
+x_train_batch, y_train_batch = read_and_decode_recordinput(
+    'train.mnist.tfrecord',
+    one_hot=True,
+    classes=classes,
+    is_train=True,
+    batch_size=batch_size)
+
+
+x_batch_shape = x_train_batch.get_shape().as_list()
+y_batch_shape = y_train_batch.get_shape().as_list()
+
+train_model = create_cnn_model(x_train_batch,
+                               y_train_batch,
+                               x_batch_shape,
+                               y_batch_shape)
+
+train_model.compile(optimizer='rmsprop',
+                    loss='categorical_crossentropy',
+                    metrics=['accuracy'])
+
+train_model.summary()
+train_model.fit(batch_size=batch_size, epochs=epochs)
+train_model.save_weights('saved_wt.h5')
+
+K.clear_session()
+
+# Second Session, pure Keras
+(X_train, y_train), (X_test, y_test) = mnist.load_data()
+X_train = X_train[..., np.newaxis]
+X_test = X_test[..., np.newaxis]
+x_test_inp = Input(batch_shape=(None,) + (X_test.shape[1:]))
+test_out = cnn_layers(x_test_inp)
+test_model = Model(inputs=x_test_inp, outputs=test_out)
+
+test_model.load_weights('saved_wt.h5')
+test_model.compile(optimizer='rmsprop', loss='categorical_crossentropy', metrics=['accuracy'])
+test_model.summary()
+
+loss, acc = test_model.evaluate(X_test, np_utils.to_categorical(y_test), classes)
+print '\nTest accuracy: {0}'.format(acc)

From 9a79ebc9f8d06cb48c3596f995d1202cd03c05b4 Mon Sep 17 00:00:00 2001
From: Andrew Hundt <ATHundt@gmail.com>
Date: Tue, 13 Jun 2017 00:10:48 -0400
Subject: [PATCH 10/64] mnist_tfrecord.py label op more clearly marked

---
 examples/mnist_tfrecord.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/examples/mnist_tfrecord.py b/examples/mnist_tfrecord.py
index c0f884b2ef3..f9a12f47e0d 100644
--- a/examples/mnist_tfrecord.py
+++ b/examples/mnist_tfrecord.py
@@ -154,7 +154,7 @@ def create_cnn_model(x_train_batch, y_train_batch, x_batch_shape, y_batch_shape)
     x_train_out = cnn_layers(x_train_input)
     # Workaround until _is_placeholder can be deduced automatically
     x_train_out._is_placeholder = False
-    y_train_in_out = Input(tensor=y_train_batch, batch_shape=y_batch_shape)
+    y_train_in_out = Input(tensor=y_train_batch, batch_shape=y_batch_shape, name='y_labels')
     return Model(inputs=[x_train_input, y_train_in_out], outputs=[x_train_out, y_train_in_out])
 
 

From b321de6d0ea49c8f83ee4e425d16bdc8eeb6b452 Mon Sep 17 00:00:00 2001
From: Andrew Hundt <ATHundt@gmail.com>
Date: Tue, 13 Jun 2017 00:58:32 -0400
Subject: [PATCH 11/64] mnist_tfrecord.py removed debug lines

---
 examples/mnist_tfrecord.py | 2 --
 1 file changed, 2 deletions(-)

diff --git a/examples/mnist_tfrecord.py b/examples/mnist_tfrecord.py
index f9a12f47e0d..049b5e7f041 100644
--- a/examples/mnist_tfrecord.py
+++ b/examples/mnist_tfrecord.py
@@ -3,8 +3,6 @@
 Gets to 99.25% test accuracy after 12 epochs
 (there is still a lot of margin for parameter tuning).
 '''
-import sys
-sys.path.append('/Users/athundt/source/keras')
 import os
 import copy
 import time

From 66251aae8d9927f3fbd3673840163fffa2c62c16 Mon Sep 17 00:00:00 2001
From: Andrew Hundt <ATHundt@gmail.com>
Date: Tue, 13 Jun 2017 14:42:25 -0400
Subject: [PATCH 12/64] training.py model.fit() fetches and feed_dict docstring
 added

---
 keras/engine/training.py | 20 ++++++++++++++++++++
 1 file changed, 20 insertions(+)

diff --git a/keras/engine/training.py b/keras/engine/training.py
index 695e80c8fb4..90759c3c12d 100644
--- a/keras/engine/training.py
+++ b/keras/engine/training.py
@@ -1443,6 +1443,26 @@ class indices (integers) to
                 sample_weight_mode="temporal" in compile().
             initial_epoch: epoch at which to start training
                 (useful for resuming a previous training run)
+            fetches: TensorFlow backend only extension point for advanced use cases
+                that require additional processing to be performed as `fit()` runs.
+                A single TensorFlow graph element, a list of graph elements, or a
+                dictionary whose values are graph elements or lists of graph elements.
+                Passes tensor ops to `tf.session.run()` and
+                returns an additional tensor tuple upon completion. This is for advanced
+                users that require auxiliary processing as fit runs. When provided,
+                additional tensor values are returned by `fit()` as follows:
+
+                    ```python
+                        history, tensors = model.fit(fetches,feed_dict)
+                    ```
+
+                See the [tf.Session.run()](https://www.tensorflow.org/api_docs/python/tf/Session)
+                `fetches` parameter for more details.
+            feed_dict: TensorFlow backend only extension point for advanced use cases
+                that require additional processing to be performed as `fit()` runs.
+                Dictionary that maps TensorFlow graph elements to values.
+                See the [tf.Session.run()](https://www.tensorflow.org/api_docs/python/tf/Session)
+                `feed_dict` parameter for more details.
 
         # Returns
             A `History` instance. Its `history` attribute contains

From 23b0ebfc1c067dd6f2c5b5063f6b8c3a53d1127a Mon Sep 17 00:00:00 2001
From: Andrew Hundt <ATHundt@gmail.com>
Date: Tue, 13 Jun 2017 14:59:15 -0400
Subject: [PATCH 13/64] Revert "training.py model.fit() fetches and feed_dict
 docstring added"

This reverts commit 66251aae8d9927f3fbd3673840163fffa2c62c16.
---
 keras/engine/training.py | 20 --------------------
 1 file changed, 20 deletions(-)

diff --git a/keras/engine/training.py b/keras/engine/training.py
index 90759c3c12d..695e80c8fb4 100644
--- a/keras/engine/training.py
+++ b/keras/engine/training.py
@@ -1443,26 +1443,6 @@ class indices (integers) to
                 sample_weight_mode="temporal" in compile().
             initial_epoch: epoch at which to start training
                 (useful for resuming a previous training run)
-            fetches: TensorFlow backend only extension point for advanced use cases
-                that require additional processing to be performed as `fit()` runs.
-                A single TensorFlow graph element, a list of graph elements, or a
-                dictionary whose values are graph elements or lists of graph elements.
-                Passes tensor ops to `tf.session.run()` and
-                returns an additional tensor tuple upon completion. This is for advanced
-                users that require auxiliary processing as fit runs. When provided,
-                additional tensor values are returned by `fit()` as follows:
-
-                    ```python
-                        history, tensors = model.fit(fetches,feed_dict)
-                    ```
-
-                See the [tf.Session.run()](https://www.tensorflow.org/api_docs/python/tf/Session)
-                `fetches` parameter for more details.
-            feed_dict: TensorFlow backend only extension point for advanced use cases
-                that require additional processing to be performed as `fit()` runs.
-                Dictionary that maps TensorFlow graph elements to values.
-                See the [tf.Session.run()](https://www.tensorflow.org/api_docs/python/tf/Session)
-                `feed_dict` parameter for more details.
 
         # Returns
             A `History` instance. Its `history` attribute contains

From 9fa27477c27a18a41e3adec5ce304abdb9fc8b73 Mon Sep 17 00:00:00 2001
From: Andrew Hundt <ATHundt@gmail.com>
Date: Wed, 14 Jun 2017 16:25:43 -0400
Subject: [PATCH 14/64] mnist_tfrecord.py remove extraneous comment

---
 examples/mnist_tfrecord.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/examples/mnist_tfrecord.py b/examples/mnist_tfrecord.py
index 049b5e7f041..ee484b85c61 100644
--- a/examples/mnist_tfrecord.py
+++ b/examples/mnist_tfrecord.py
@@ -19,7 +19,6 @@
 from keras.utils import np_utils
 from keras import callbacks as cbks
 from keras import optimizers, objectives
-# from keras.engine.training import collect_metrics, weighted_objective
 from keras import metrics as metrics_module
 
 from keras.datasets import mnist

From db16bb2e68822311201b16e2e10ed58c06eda963 Mon Sep 17 00:00:00 2001
From: Andrew Hundt <ATHundt@gmail.com>
Date: Thu, 15 Jun 2017 15:47:46 -0400
Subject: [PATCH 15/64] Container.__init__ new labels tensor parameter supports
 tensor generated labels. mnist_tfrecord.py training runs correctly

---
 examples/mnist_tfrecord.py | 107 ++++++++++++++++++++++---------------
 keras/engine/topology.py   |  10 +++-
 keras/engine/training.py   |   6 +--
 keras/legacy/interfaces.py |   3 +-
 4 files changed, 77 insertions(+), 49 deletions(-)

diff --git a/examples/mnist_tfrecord.py b/examples/mnist_tfrecord.py
index ee484b85c61..c21beb105aa 100644
--- a/examples/mnist_tfrecord.py
+++ b/examples/mnist_tfrecord.py
@@ -14,11 +14,12 @@
 from keras import backend as K
 from keras.models import Model
 from keras.layers import Dense, Dropout, Flatten, Input, Conv2D
-from keras.callbacks import EarlyStopping
+from keras.callbacks import EarlyStopping, TensorBoard
 from keras.objectives import categorical_crossentropy
 from keras.utils import np_utils
 from keras import callbacks as cbks
 from keras import optimizers, objectives
+# from keras.engine.training import collect_metrics, weighted_objective
 from keras import metrics as metrics_module
 
 from keras.datasets import mnist
@@ -58,7 +59,7 @@ def _bytes_feature(value):
             writer.write(example.SerializeToString())
         writer.close()
     else:
-        print 'tfrecord already exists'
+        print 'tfrecord %s already exists' % filename
 
 
 def read_and_decode(filename, one_hot=True, classes=None, is_train=None):
@@ -66,11 +67,12 @@ def read_and_decode(filename, one_hot=True, classes=None, is_train=None):
     filename_queue = tf.train.string_input_producer([filename])
     reader = tf.TFRecordReader()
     _, serialized_example = reader.read(filename_queue)
-    features = tf.parse_single_example(serialized_example,
-                                       features={
-                                           'label': tf.FixedLenFeature([], tf.int64),
-                                           'image_raw': tf.FixedLenFeature([], tf.string),
-                                       })
+    features = tf.parse_single_example(
+        serialized_example,
+        features={
+            'label': tf.FixedLenFeature([], tf.int64),
+            'image_raw': tf.FixedLenFeature([], tf.string),
+        })
     img = tf.decode_raw(features['image_raw'], tf.uint8)
     img.set_shape([28 * 28])
     img = tf.reshape(img, [28, 28, 1])
@@ -90,39 +92,43 @@ def read_and_decode(filename, one_hot=True, classes=None, is_train=None):
 
     return x_train_batch, y_train_batch
 
-
-def read_and_decode_recordinput(tf_glob, one_hot=True, classes=None, is_train=None, batch_size=1000):
+def read_and_decode_recordinput(tf_glob, one_hot=True, classes=None, is_train=None, batch_shape=[1000, 28, 28, 1]):
     """ Return tensor to read from TFRecord """
-
-    record_input = data_flow_ops.RecordInput(tf_glob, batch_size=batch_size)
-    records_op = record_input.get_yield_op()
-    records_op = tf.split(records_op, batch_size, 0)
-    records_op = [tf.reshape(record, []) for record in records_op]
-
-    imgs = []
-    labels = []
-    for serialized_example in records_op:
-        features = tf.parse_single_example(serialized_example,
-                                           features={
-                                               'label': tf.FixedLenFeature([], tf.int64),
-                                               'image_raw': tf.FixedLenFeature([], tf.string),
-                                           })
-        img = tf.decode_raw(features['image_raw'], tf.uint8)
-        img.set_shape([28 * 28])
-        img = tf.reshape(img, [1, 28, 28, 1])
-
-        img = tf.cast(img, tf.float32) * (1. / 255) - 0.5
-
-        label = tf.cast(features['label'], tf.int32)
-        if one_hot and classes:
-            label = tf.one_hot(label, classes)
-
-        imgs.append(img)
-        labels.append(label)
-
-    imgs = tf.concat(imgs, 0)
-    labels = tf.concat(labels, 0)
-    return imgs, labels
+    with tf.variable_scope("TFRecords"):
+        record_input = data_flow_ops.RecordInput(tf_glob, batch_size=batch_shape[0])
+        records_op = record_input.get_yield_op()
+        records_op = tf.split(records_op, batch_shape[0], 0)
+        records_op = [tf.reshape(record, []) for record in records_op]
+
+        imgs = []
+        labels = []
+        for serialized_example in records_op:
+            with tf.variable_scope("parse_images", reuse=True):
+                features = tf.parse_single_example(
+                    serialized_example,
+                    features={
+                        'label': tf.FixedLenFeature([], tf.int64),
+                        'image_raw': tf.FixedLenFeature([], tf.string),
+                    })
+                img = tf.decode_raw(features['image_raw'], tf.uint8)
+                img.set_shape(batch_shape[1] * batch_shape[2])
+                img = tf.reshape(img, [1] + batch_shape[1:])
+
+                img = tf.cast(img, tf.float32) * (1. / 255) - 0.5
+
+                label = tf.cast(features['label'], tf.int32)
+                if one_hot and classes:
+                    label = tf.one_hot(label, classes)
+
+                imgs.append(img)
+                labels.append(label)
+
+        imgs = tf.parallel_stack(imgs, 0)
+        labels = tf.parallel_stack(labels, 0)
+
+        imgs = tf.cast(imgs, tf.float32)
+        imgs = tf.reshape(imgs, shape=batch_shape)
+        return imgs, labels
 
 
 def save_mnist_as_tfrecord():
@@ -152,7 +158,7 @@ def create_cnn_model(x_train_batch, y_train_batch, x_batch_shape, y_batch_shape)
     # Workaround until _is_placeholder can be deduced automatically
     x_train_out._is_placeholder = False
     y_train_in_out = Input(tensor=y_train_batch, batch_shape=y_batch_shape, name='y_labels')
-    return Model(inputs=[x_train_input, y_train_in_out], outputs=[x_train_out, y_train_in_out])
+    return Model(inputs=[x_train_input], outputs=[x_train_out], labels=[y_train_in_out])
 
 
 sess = tf.Session()
@@ -160,8 +166,9 @@ def create_cnn_model(x_train_batch, y_train_batch, x_batch_shape, y_batch_shape)
 
 save_mnist_as_tfrecord()
 
-batch_size = 1000
-epochs = 300
+batch_size = 100
+batch_shape = [batch_size, 28, 28, 1]
+epochs = 3000
 classes = 10
 
 x_train_batch, y_train_batch = read_and_decode_recordinput(
@@ -169,7 +176,14 @@ def create_cnn_model(x_train_batch, y_train_batch, x_batch_shape, y_batch_shape)
     one_hot=True,
     classes=classes,
     is_train=True,
-    batch_size=batch_size)
+    batch_shape=batch_shape)
+
+x_test_batch, y_test_batch = read_and_decode_recordinput(
+    'test.mnist.tfrecord',
+    one_hot=True,
+    classes=classes,
+    is_train=True,
+    batch_shape=batch_shape)
 
 
 x_batch_shape = x_train_batch.get_shape().as_list()
@@ -184,8 +198,13 @@ def create_cnn_model(x_train_batch, y_train_batch, x_batch_shape, y_batch_shape)
                     loss='categorical_crossentropy',
                     metrics=['accuracy'])
 
+tensorboard = TensorBoard(write_graph=True)
+
 train_model.summary()
-train_model.fit(batch_size=batch_size, epochs=epochs)
+train_model.fit(batch_size=batch_size,
+                epochs=epochs,
+                callbacks=[tensorboard],
+                validation_data=(x_test_batch, y_test_batch))
 train_model.save_weights('saved_wt.h5')
 
 K.clear_session()
diff --git a/keras/engine/topology.py b/keras/engine/topology.py
index 97b6a0a3ab8..e01d7e6da17 100644
--- a/keras/engine/topology.py
+++ b/keras/engine/topology.py
@@ -1478,7 +1478,7 @@ class Container(Layer):
     """
 
     @interfaces.legacy_model_constructor_support
-    def __init__(self, inputs, outputs, name=None):
+    def __init__(self, inputs, outputs, labels=None, name=None):
         # Handle `name` argument.
         if not name:
             prefix = self.__class__.__name__.lower()
@@ -1495,11 +1495,19 @@ def __init__(self, inputs, outputs, name=None):
             self.inputs = list(inputs)  # Tensor or list of tensors.
         else:
             self.inputs = [inputs]
+
         if isinstance(outputs, (list, tuple)):
             self.outputs = list(outputs)
         else:
             self.outputs = [outputs]
 
+        if labels is None:
+            self.labels = [None]*len(self.outputs)
+        elif isinstance(labels, (list, tuple)):
+            self.labels = list(labels)
+        else:
+            self.labels = [labels]
+
         # Check for redundancy in inputs.
         if len(set(self.inputs)) != len(self.inputs):
             raise ValueError('The list of inputs passed to the model '
diff --git a/keras/engine/training.py b/keras/engine/training.py
index 695e80c8fb4..ccc9ef8359d 100644
--- a/keras/engine/training.py
+++ b/keras/engine/training.py
@@ -822,13 +822,13 @@ def compile(self, optimizer, loss, metrics=None, loss_weights=None,
             else:
                 shape = self.internal_output_shapes[i]
                 name = self.output_names[i]
-                if K.is_placeholder(self.outputs[i]):
+                if self.labels[i] is None:
                     target = K.placeholder(ndim=len(shape),
                                            name=name + '_target',
                                            sparse=K.is_sparse(self.outputs[i]),
                                            dtype=K.dtype(self.outputs[i]))
                 else:
-                    target = self.outputs[i]
+                    target = self.labels[i]
                 self.targets.append(target)
                 self._feed_targets.append(target)
 
@@ -1109,7 +1109,7 @@ def _fit_loop(self, f, ins, out_labels=None, batch_size=32,
             `History` object.
         """
         do_validation = False
-        if val_f and val_ins:
+        if ins and val_f and val_ins:
             do_validation = True
             if verbose:
                 print('Train on %d samples, validate on %d samples' %
diff --git a/keras/legacy/interfaces.py b/keras/legacy/interfaces.py
index be0c82f1258..73ee6b0d375 100644
--- a/keras/legacy/interfaces.py
+++ b/keras/legacy/interfaces.py
@@ -601,7 +601,8 @@ def generator_methods_args_preprocessor(args, kwargs):
 legacy_model_constructor_support = generate_legacy_interface(
     allowed_positional_args=None,
     conversions=[('input', 'inputs'),
-                 ('output', 'outputs')])
+                 ('output', 'outputs'),
+                 ('label', 'labels')])
 
 legacy_input_support = generate_legacy_interface(
     allowed_positional_args=None,

From 8ad7ec8a9c8ee59c88d90a313157dee28c6d0ec9 Mon Sep 17 00:00:00 2001
From: Andrew Hundt <ATHundt@gmail.com>
Date: Thu, 15 Jun 2017 17:36:07 -0400
Subject: [PATCH 16/64] mnist_tfrecord.py remove unused read_and_decode
 function

---
 examples/mnist_tfrecord.py | 31 -------------------------------
 1 file changed, 31 deletions(-)

diff --git a/examples/mnist_tfrecord.py b/examples/mnist_tfrecord.py
index c21beb105aa..ea452bf9cec 100644
--- a/examples/mnist_tfrecord.py
+++ b/examples/mnist_tfrecord.py
@@ -61,37 +61,6 @@ def _bytes_feature(value):
     else:
         print 'tfrecord %s already exists' % filename
 
-
-def read_and_decode(filename, one_hot=True, classes=None, is_train=None):
-    """ Return tensor to read from TFRecord """
-    filename_queue = tf.train.string_input_producer([filename])
-    reader = tf.TFRecordReader()
-    _, serialized_example = reader.read(filename_queue)
-    features = tf.parse_single_example(
-        serialized_example,
-        features={
-            'label': tf.FixedLenFeature([], tf.int64),
-            'image_raw': tf.FixedLenFeature([], tf.string),
-        })
-    img = tf.decode_raw(features['image_raw'], tf.uint8)
-    img.set_shape([28 * 28])
-    img = tf.reshape(img, [28, 28, 1])
-
-    img = tf.cast(img, tf.float32) * (1. / 255) - 0.5
-
-    label = tf.cast(features['label'], tf.int32)
-    if one_hot and classes:
-        label = tf.one_hot(label, classes)
-
-    x_train_batch, y_train_batch = K.tf.train.shuffle_batch(
-        [img, label],
-        batch_size=batch_size,
-        capacity=2000,
-        min_after_dequeue=1000,
-        num_threads=32)  # set the number of threads here
-
-    return x_train_batch, y_train_batch
-
 def read_and_decode_recordinput(tf_glob, one_hot=True, classes=None, is_train=None, batch_shape=[1000, 28, 28, 1]):
     """ Return tensor to read from TFRecord """
     with tf.variable_scope("TFRecords"):

From 48f6d5a700e726af9a6cc2634ec9306b1e1afb9f Mon Sep 17 00:00:00 2001
From: Andrew Hundt <ATHundt@gmail.com>
Date: Thu, 15 Jun 2017 17:38:24 -0400
Subject: [PATCH 17/64] mnist_tfrecord.py progbar

---
 examples/mnist_tfrecord.py | 19 ++++++++++++-------
 1 file changed, 12 insertions(+), 7 deletions(-)

diff --git a/examples/mnist_tfrecord.py b/examples/mnist_tfrecord.py
index ea452bf9cec..6a4f7473b2f 100644
--- a/examples/mnist_tfrecord.py
+++ b/examples/mnist_tfrecord.py
@@ -17,6 +17,7 @@
 from keras.callbacks import EarlyStopping, TensorBoard
 from keras.objectives import categorical_crossentropy
 from keras.utils import np_utils
+from keras.utils.generic_utils import Progbar
 from keras import callbacks as cbks
 from keras import optimizers, objectives
 # from keras.engine.training import collect_metrics, weighted_objective
@@ -63,15 +64,18 @@ def _bytes_feature(value):
 
 def read_and_decode_recordinput(tf_glob, one_hot=True, classes=None, is_train=None, batch_shape=[1000, 28, 28, 1]):
     """ Return tensor to read from TFRecord """
+    print 'Creating graph for loading TFRecords...'
     with tf.variable_scope("TFRecords"):
         record_input = data_flow_ops.RecordInput(tf_glob, batch_size=batch_shape[0])
         records_op = record_input.get_yield_op()
         records_op = tf.split(records_op, batch_shape[0], 0)
         records_op = [tf.reshape(record, []) for record in records_op]
+        progbar = Progbar(len(records_op))
 
-        imgs = []
+        images = []
         labels = []
-        for serialized_example in records_op:
+        for i, serialized_example in zip(range(len(records_op)), records_op):
+            progbar.update(i)
             with tf.variable_scope("parse_images", reuse=True):
                 features = tf.parse_single_example(
                     serialized_example,
@@ -89,15 +93,16 @@ def read_and_decode_recordinput(tf_glob, one_hot=True, classes=None, is_train=No
                 if one_hot and classes:
                     label = tf.one_hot(label, classes)
 
-                imgs.append(img)
+                images.append(img)
                 labels.append(label)
 
-        imgs = tf.parallel_stack(imgs, 0)
+        images = tf.parallel_stack(images, 0)
         labels = tf.parallel_stack(labels, 0)
+        images = tf.cast(images, tf.float32)
 
-        imgs = tf.cast(imgs, tf.float32)
-        imgs = tf.reshape(imgs, shape=batch_shape)
-        return imgs, labels
+        images = tf.reshape(images, shape=batch_shape)
+
+        return images, labels
 
 
 def save_mnist_as_tfrecord():

From e8d6ad3c4543b141e9ea7d596eb37cabef0156aa Mon Sep 17 00:00:00 2001
From: Andrew Hundt <ATHundt@gmail.com>
Date: Thu, 15 Jun 2017 17:50:07 -0400
Subject: [PATCH 18/64] mnist_tfrecord.py network is closer to mnist_cnn.py

---
 examples/mnist_tfrecord.py | 15 +++++++--------
 1 file changed, 7 insertions(+), 8 deletions(-)

diff --git a/examples/mnist_tfrecord.py b/examples/mnist_tfrecord.py
index 6a4f7473b2f..46efc4e77ca 100644
--- a/examples/mnist_tfrecord.py
+++ b/examples/mnist_tfrecord.py
@@ -13,7 +13,7 @@
 from tensorflow.python.ops import data_flow_ops
 from keras import backend as K
 from keras.models import Model
-from keras.layers import Dense, Dropout, Flatten, Input, Conv2D
+from keras.layers import Dense, Dropout, Flatten, Input, Conv2D, MaxPooling2D
 from keras.callbacks import EarlyStopping, TensorBoard
 from keras.objectives import categorical_crossentropy
 from keras.utils import np_utils
@@ -114,15 +114,14 @@ def save_mnist_as_tfrecord():
 
 
 def cnn_layers(x_train_input):
-    con1 = Conv2D(32, (3, 3), activation='relu',
-                  strides=(2, 2), padding='valid')(x_train_input)
-    con2 = Conv2D(32, (3, 3), activation='relu',
-                  strides=(2, 2))(con1)
-    fla1 = Flatten()(con2)
-    den1 = Dense(128, activation='relu')(fla1)
+    x = Conv2D(32, (3, 3), activation='relu', padding='valid')(x_train_input)
+    x = Conv2D(64, (3, 3), activation='relu')(x)
+    x = MaxPooling2D(pool_size=(2, 2))(x)
+    x = Flatten()(x)
+    x = Dense(128, activation='relu')(x)
     x_train_out = Dense(classes,
                         activation='softmax',
-                        name='x_train_out')(den1)
+                        name='x_train_out')(x)
     return x_train_out
 
 

From bc2a0f4c3f15939de5c578662f817f607217594f Mon Sep 17 00:00:00 2001
From: Andrew Hundt <ATHundt@gmail.com>
Date: Thu, 15 Jun 2017 17:52:32 -0400
Subject: [PATCH 19/64] mnist_tfrecords.py add StagingArea

---
 examples/mnist_tfrecord.py | 12 ++++++++++++
 1 file changed, 12 insertions(+)

diff --git a/examples/mnist_tfrecord.py b/examples/mnist_tfrecord.py
index 46efc4e77ca..3240acccc38 100644
--- a/examples/mnist_tfrecord.py
+++ b/examples/mnist_tfrecord.py
@@ -102,6 +102,18 @@ def read_and_decode_recordinput(tf_glob, one_hot=True, classes=None, is_train=No
 
         images = tf.reshape(images, shape=batch_shape)
 
+        # StagingArea will store tensors
+        # across multiple steps to
+        # speed up execution
+        images_shape = images.get_shape()
+        labels_shape = labels.get_shape()
+        copy_stage = data_flow_ops.StagingArea(
+            [tf.float32, tf.float32],
+            shapes=[images_shape, labels_shape])
+        copy_stage_op = copy_stage.put(
+            [images, labels])
+        staged_images, staged_labels = copy_stage.get()
+
         return images, labels
 
 

From 7ecef2983c5480bcc7496606875a0b8a8eee7ab8 Mon Sep 17 00:00:00 2001
From: Andrew Hundt <ATHundt@gmail.com>
Date: Thu, 15 Jun 2017 18:03:52 -0400
Subject: [PATCH 20/64] is_placeholder raised exceptions now return False

---
 keras/backend/common.py             | 2 +-
 keras/backend/tensorflow_backend.py | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/keras/backend/common.py b/keras/backend/common.py
index c5d40088b75..c4d98f92405 100644
--- a/keras/backend/common.py
+++ b/keras/backend/common.py
@@ -128,7 +128,7 @@ def is_placeholder(tensor):
     try:
         return tensor._is_placeholder
     except AttributeError:
-        return True
+        return False
 
 
 def image_data_format():
diff --git a/keras/backend/tensorflow_backend.py b/keras/backend/tensorflow_backend.py
index ac3c92e61db..6bc06b58dc3 100644
--- a/keras/backend/tensorflow_backend.py
+++ b/keras/backend/tensorflow_backend.py
@@ -302,7 +302,7 @@ def is_placeholder(tensor):
             return True
         return tensor._is_placeholder
     except AttributeError:
-        return True
+        return False
 
 
 def variable(value, dtype=None, name=None):

From a0639e1fd04ffbc26ff34c322a4cabe6cab02946 Mon Sep 17 00:00:00 2001
From: Andrew Hundt <ATHundt@gmail.com>
Date: Thu, 15 Jun 2017 18:09:02 -0400
Subject: [PATCH 21/64] mnist_tfrecord.py remove stray comment

---
 examples/mnist_tfrecord.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/examples/mnist_tfrecord.py b/examples/mnist_tfrecord.py
index 3240acccc38..d040b388bf0 100644
--- a/examples/mnist_tfrecord.py
+++ b/examples/mnist_tfrecord.py
@@ -20,7 +20,6 @@
 from keras.utils.generic_utils import Progbar
 from keras import callbacks as cbks
 from keras import optimizers, objectives
-# from keras.engine.training import collect_metrics, weighted_objective
 from keras import metrics as metrics_module
 
 from keras.datasets import mnist

From b7ea15f14a8a76a5656d226f6dd2da3e3ae2669f Mon Sep 17 00:00:00 2001
From: Andrew Hundt <ATHundt@gmail.com>
Date: Thu, 15 Jun 2017 21:07:00 -0400
Subject: [PATCH 22/64] training.py use append for lists instead of + operator,
 fixes crash when lists are empty and the learning phase is True

---
 keras/engine/training.py | 60 +++++++++++++++++-----------------------
 1 file changed, 26 insertions(+), 34 deletions(-)

diff --git a/keras/engine/training.py b/keras/engine/training.py
index ccc9ef8359d..d691674b4a3 100644
--- a/keras/engine/training.py
+++ b/keras/engine/training.py
@@ -1065,10 +1065,9 @@ def _make_predict_function(self):
         if not hasattr(self, 'predict_function'):
             self.predict_function = None
         if self.predict_function is None:
+            inputs = self._feed_inputs
             if self.uses_learning_phase and not isinstance(K.learning_phase(), int):
-                inputs = self._feed_inputs + [K.learning_phase()]
-            else:
-                inputs = self._feed_inputs
+                inputs.append(K.learning_phase())
             # Gets network outputs. Does not update weights.
             # Does update the network states.
             kwargs = getattr(self, '_function_kwargs', {})
@@ -1109,13 +1108,13 @@ def _fit_loop(self, f, ins, out_labels=None, batch_size=32,
             `History` object.
         """
         do_validation = False
-        if ins and val_f and val_ins:
-            do_validation = True
-            if verbose:
-                print('Train on %d samples, validate on %d samples' %
-                      (ins[0].shape[0], val_ins[0].shape[0]))
-
         if ins and hasattr(ins[0], 'shape'):
+            if val_f and val_ins:
+                do_validation = True
+                if verbose:
+                    print('Train on %d samples, validate on %d samples' %
+                          (ins[0].shape[0], val_ins[0].shape[0]))
+
             num_train_samples = ins[0].shape[0]
         else:
             # May happen if we are running `fit` without Numpy input data,
@@ -1489,10 +1488,9 @@ class indices (integers) to
                 batch_size=batch_size)
             self._make_test_function()
             val_f = self.test_function
+            val_ins = val_x + val_y + val_sample_weights
             if self.uses_learning_phase and not isinstance(K.learning_phase(), int):
-                val_ins = val_x + val_y + val_sample_weights + [0.]
-            else:
-                val_ins = val_x + val_y + val_sample_weights
+                val_ins.append(0.)
 
         elif validation_split and 0. < validation_split < 1.:
             do_validation = True
@@ -1507,20 +1505,18 @@ class indices (integers) to
                 _slice_arrays(sample_weights, split_at))
             self._make_test_function()
             val_f = self.test_function
+            val_ins = val_x + val_y + val_sample_weights
             if self.uses_learning_phase and not isinstance(K.learning_phase(), int):
-                val_ins = val_x + val_y + val_sample_weights + [0.]
-            else:
-                val_ins = val_x + val_y + val_sample_weights
+                val_ins.append(0.)
         else:
             do_validation = False
             val_f = None
             val_ins = None
 
         # Prepare input arrays and training function.
+        ins = x + y + sample_weights
         if self.uses_learning_phase and not isinstance(K.learning_phase(), int):
-            ins = x + y + sample_weights + [1.]
-        else:
-            ins = x + y + sample_weights
+            ins.append(1.)
         self._make_train_function()
         f = self.train_function
 
@@ -1574,10 +1570,10 @@ def evaluate(self, x, y, batch_size=32, verbose=1, sample_weight=None):
             check_batch_axis=False,
             batch_size=batch_size)
         # Prepare inputs, delegate logic to `_test_loop`.
+        ins = x + y + sample_weights
         if self.uses_learning_phase and not isinstance(K.learning_phase(), int):
-            ins = x + y + sample_weights + [0.]
-        else:
-            ins = x + y + sample_weights
+            ins.append(0.)
+
         self._make_test_function()
         f = self.test_function
         return self._test_loop(f, ins,
@@ -1618,10 +1614,9 @@ def predict(self, x, batch_size=32, verbose=0):
                                  'Batch size: ' + str(batch_size) + '.')
 
         # Prepare inputs, delegate logic to `_predict_loop`.
+        ins = x
         if self.uses_learning_phase and not isinstance(K.learning_phase(), int):
-            ins = x + [0.]
-        else:
-            ins = x
+            ins.append(0.)
         self._make_predict_function()
         f = self.predict_function
         return self._predict_loop(f, ins,
@@ -1668,10 +1663,9 @@ class indices (integers) to
             sample_weight=sample_weight,
             class_weight=class_weight,
             check_batch_axis=True)
+        ins = x + y + sample_weights
         if self.uses_learning_phase and not isinstance(K.learning_phase(), int):
-            ins = x + y + sample_weights + [1.]
-        else:
-            ins = x + y + sample_weights
+            ins.append(1.)
         self._make_train_function()
         outputs = self.train_function(ins)
         if len(outputs) == 1:
@@ -1710,10 +1704,9 @@ def test_on_batch(self, x, y, sample_weight=None):
             x, y,
             sample_weight=sample_weight,
             check_batch_axis=True)
+        ins = x + y + sample_weights
         if self.uses_learning_phase and not isinstance(K.learning_phase(), int):
-            ins = x + y + sample_weights + [0.]
-        else:
-            ins = x + y + sample_weights
+            ins.append(0.)
         self._make_test_function()
         outputs = self.test_function(ins)
         if len(outputs) == 1:
@@ -1731,10 +1724,9 @@ def predict_on_batch(self, x):
         """
         x = _standardize_input_data(x, self._feed_input_names,
                                     self._feed_input_shapes)
+        ins = x
         if self.uses_learning_phase and not isinstance(K.learning_phase(), int):
-            ins = x + [0.]
-        else:
-            ins = x
+            ins.append(0.)
         self._make_predict_function()
         outputs = self.predict_function(ins)
         if len(outputs) == 1:
@@ -1881,7 +1873,7 @@ def generate_arrays_from_file(path):
                 val_x, val_y, val_sample_weight)
             val_data = val_x + val_y + val_sample_weights
             if self.uses_learning_phase and not isinstance(K.learning_phase(), int):
-                val_data += [0.]
+                val_data.append(0.)
             for cbk in callbacks:
                 cbk.validation_data = val_data
         enqueuer = None

From 36a36a0cb374dadd88e11589681072f4466b1b92 Mon Sep 17 00:00:00 2001
From: Andrew Hundt <ATHundt@gmail.com>
Date: Thu, 15 Jun 2017 21:07:57 -0400
Subject: [PATCH 23/64] tensorflow_backend.py prevent Function kwargs from
 accumulating inadvertently

---
 keras/backend/tensorflow_backend.py | 16 ++++++++--------
 1 file changed, 8 insertions(+), 8 deletions(-)

diff --git a/keras/backend/tensorflow_backend.py b/keras/backend/tensorflow_backend.py
index 6bc06b58dc3..922a46dad1c 100644
--- a/keras/backend/tensorflow_backend.py
+++ b/keras/backend/tensorflow_backend.py
@@ -2286,24 +2286,24 @@ def __init__(self, inputs, outputs, updates=None, name=None,
     def __call__(self, inputs):
         if not isinstance(inputs, (list, tuple)):
             raise TypeError('`inputs` should be a list or tuple.')
+        self.current_feed_dict = self.feed_dict
         if self.feed_dict is None:
-            self.feed_dict = {}
+            self.current_feed_dict = {}
         for tensor, value in zip(self.inputs, inputs):
             if is_sparse(tensor):
                 sparse_coo = value.tocoo()
                 indices = np.concatenate((np.expand_dims(sparse_coo.row, 1),
                                           np.expand_dims(sparse_coo.col, 1)), 1)
                 value = (indices, sparse_coo.data, sparse_coo.shape)
-            self.feed_dict[tensor] = value
-        session = get_session()
+            self.current_feed_dict[tensor] = value
 
+        self.current_fetches = self.outputs + [self.updates_op]
         if self.fetches is not None:
-            fetches = self.outputs + [self.updates_op] + self.fetches
-        else:
-            fetches = self.outputs + [self.updates_op]
+            self.current_fetches += self.fetches
 
-        updated = session.run(fetches,
-                              feed_dict=self.feed_dict,
+        session = get_session()
+        updated = session.run(fetches=self.current_fetches,
+                              feed_dict=self.current_feed_dict,
                               **self.session_kwargs)
         return updated[:len(self.outputs)]
 

From 1a1183d7dbf8d23faef83091162f6f0be418b286 Mon Sep 17 00:00:00 2001
From: Andrew Hundt <ATHundt@gmail.com>
Date: Thu, 15 Jun 2017 21:30:16 -0400
Subject: [PATCH 24/64] Revert "training.py use append for lists instead of +
 operator, fixes crash when lists are empty and the learning phase is True"

This reverts commit b7ea15f14a8a76a5656d226f6dd2da3e3ae2669f.
---
 keras/engine/training.py | 60 +++++++++++++++++++++++-----------------
 1 file changed, 34 insertions(+), 26 deletions(-)

diff --git a/keras/engine/training.py b/keras/engine/training.py
index d691674b4a3..ccc9ef8359d 100644
--- a/keras/engine/training.py
+++ b/keras/engine/training.py
@@ -1065,9 +1065,10 @@ def _make_predict_function(self):
         if not hasattr(self, 'predict_function'):
             self.predict_function = None
         if self.predict_function is None:
-            inputs = self._feed_inputs
             if self.uses_learning_phase and not isinstance(K.learning_phase(), int):
-                inputs.append(K.learning_phase())
+                inputs = self._feed_inputs + [K.learning_phase()]
+            else:
+                inputs = self._feed_inputs
             # Gets network outputs. Does not update weights.
             # Does update the network states.
             kwargs = getattr(self, '_function_kwargs', {})
@@ -1108,13 +1109,13 @@ def _fit_loop(self, f, ins, out_labels=None, batch_size=32,
             `History` object.
         """
         do_validation = False
-        if ins and hasattr(ins[0], 'shape'):
-            if val_f and val_ins:
-                do_validation = True
-                if verbose:
-                    print('Train on %d samples, validate on %d samples' %
-                          (ins[0].shape[0], val_ins[0].shape[0]))
+        if ins and val_f and val_ins:
+            do_validation = True
+            if verbose:
+                print('Train on %d samples, validate on %d samples' %
+                      (ins[0].shape[0], val_ins[0].shape[0]))
 
+        if ins and hasattr(ins[0], 'shape'):
             num_train_samples = ins[0].shape[0]
         else:
             # May happen if we are running `fit` without Numpy input data,
@@ -1488,9 +1489,10 @@ class indices (integers) to
                 batch_size=batch_size)
             self._make_test_function()
             val_f = self.test_function
-            val_ins = val_x + val_y + val_sample_weights
             if self.uses_learning_phase and not isinstance(K.learning_phase(), int):
-                val_ins.append(0.)
+                val_ins = val_x + val_y + val_sample_weights + [0.]
+            else:
+                val_ins = val_x + val_y + val_sample_weights
 
         elif validation_split and 0. < validation_split < 1.:
             do_validation = True
@@ -1505,18 +1507,20 @@ class indices (integers) to
                 _slice_arrays(sample_weights, split_at))
             self._make_test_function()
             val_f = self.test_function
-            val_ins = val_x + val_y + val_sample_weights
             if self.uses_learning_phase and not isinstance(K.learning_phase(), int):
-                val_ins.append(0.)
+                val_ins = val_x + val_y + val_sample_weights + [0.]
+            else:
+                val_ins = val_x + val_y + val_sample_weights
         else:
             do_validation = False
             val_f = None
             val_ins = None
 
         # Prepare input arrays and training function.
-        ins = x + y + sample_weights
         if self.uses_learning_phase and not isinstance(K.learning_phase(), int):
-            ins.append(1.)
+            ins = x + y + sample_weights + [1.]
+        else:
+            ins = x + y + sample_weights
         self._make_train_function()
         f = self.train_function
 
@@ -1570,10 +1574,10 @@ def evaluate(self, x, y, batch_size=32, verbose=1, sample_weight=None):
             check_batch_axis=False,
             batch_size=batch_size)
         # Prepare inputs, delegate logic to `_test_loop`.
-        ins = x + y + sample_weights
         if self.uses_learning_phase and not isinstance(K.learning_phase(), int):
-            ins.append(0.)
-
+            ins = x + y + sample_weights + [0.]
+        else:
+            ins = x + y + sample_weights
         self._make_test_function()
         f = self.test_function
         return self._test_loop(f, ins,
@@ -1614,9 +1618,10 @@ def predict(self, x, batch_size=32, verbose=0):
                                  'Batch size: ' + str(batch_size) + '.')
 
         # Prepare inputs, delegate logic to `_predict_loop`.
-        ins = x
         if self.uses_learning_phase and not isinstance(K.learning_phase(), int):
-            ins.append(0.)
+            ins = x + [0.]
+        else:
+            ins = x
         self._make_predict_function()
         f = self.predict_function
         return self._predict_loop(f, ins,
@@ -1663,9 +1668,10 @@ class indices (integers) to
             sample_weight=sample_weight,
             class_weight=class_weight,
             check_batch_axis=True)
-        ins = x + y + sample_weights
         if self.uses_learning_phase and not isinstance(K.learning_phase(), int):
-            ins.append(1.)
+            ins = x + y + sample_weights + [1.]
+        else:
+            ins = x + y + sample_weights
         self._make_train_function()
         outputs = self.train_function(ins)
         if len(outputs) == 1:
@@ -1704,9 +1710,10 @@ def test_on_batch(self, x, y, sample_weight=None):
             x, y,
             sample_weight=sample_weight,
             check_batch_axis=True)
-        ins = x + y + sample_weights
         if self.uses_learning_phase and not isinstance(K.learning_phase(), int):
-            ins.append(0.)
+            ins = x + y + sample_weights + [0.]
+        else:
+            ins = x + y + sample_weights
         self._make_test_function()
         outputs = self.test_function(ins)
         if len(outputs) == 1:
@@ -1724,9 +1731,10 @@ def predict_on_batch(self, x):
         """
         x = _standardize_input_data(x, self._feed_input_names,
                                     self._feed_input_shapes)
-        ins = x
         if self.uses_learning_phase and not isinstance(K.learning_phase(), int):
-            ins.append(0.)
+            ins = x + [0.]
+        else:
+            ins = x
         self._make_predict_function()
         outputs = self.predict_function(ins)
         if len(outputs) == 1:
@@ -1873,7 +1881,7 @@ def generate_arrays_from_file(path):
                 val_x, val_y, val_sample_weight)
             val_data = val_x + val_y + val_sample_weights
             if self.uses_learning_phase and not isinstance(K.learning_phase(), int):
-                val_data.append(0.)
+                val_data += [0.]
             for cbk in callbacks:
                 cbk.validation_data = val_data
         enqueuer = None

From 9bf9b23af647e82ffa9ff117b9f87db65adec357 Mon Sep 17 00:00:00 2001
From: Andrew Hundt <ATHundt@gmail.com>
Date: Thu, 15 Jun 2017 21:32:38 -0400
Subject: [PATCH 25/64] training.py fix crash when ins does not have shape
 attribute

---
 keras/engine/training.py | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/keras/engine/training.py b/keras/engine/training.py
index ccc9ef8359d..b3f9fa62256 100644
--- a/keras/engine/training.py
+++ b/keras/engine/training.py
@@ -1109,13 +1109,13 @@ def _fit_loop(self, f, ins, out_labels=None, batch_size=32,
             `History` object.
         """
         do_validation = False
-        if ins and val_f and val_ins:
-            do_validation = True
-            if verbose:
-                print('Train on %d samples, validate on %d samples' %
-                      (ins[0].shape[0], val_ins[0].shape[0]))
 
         if ins and hasattr(ins[0], 'shape'):
+            if ins and val_f and val_ins:
+                do_validation = True
+                if verbose:
+                    print('Train on %d samples, validate on %d samples' %
+                          (ins[0].shape[0], val_ins[0].shape[0]))
             num_train_samples = ins[0].shape[0]
         else:
             # May happen if we are running `fit` without Numpy input data,

From 3d323eca24465e9756c144af3aa9578864e7f9f1 Mon Sep 17 00:00:00 2001
From: Andrew Hundt <ATHundt@gmail.com>
Date: Thu, 15 Jun 2017 21:36:45 -0400
Subject: [PATCH 26/64] training.py better verification of do_validation

---
 keras/engine/training.py | 7 +++----
 1 file changed, 3 insertions(+), 4 deletions(-)

diff --git a/keras/engine/training.py b/keras/engine/training.py
index b3f9fa62256..7e5430cf22a 100644
--- a/keras/engine/training.py
+++ b/keras/engine/training.py
@@ -1109,11 +1109,10 @@ def _fit_loop(self, f, ins, out_labels=None, batch_size=32,
             `History` object.
         """
         do_validation = False
-
+        if val_f and val_ins:
+            do_validation = True
         if ins and hasattr(ins[0], 'shape'):
-            if ins and val_f and val_ins:
-                do_validation = True
-                if verbose:
+            if val_f and val_ins and verbose:
                     print('Train on %d samples, validate on %d samples' %
                           (ins[0].shape[0], val_ins[0].shape[0]))
             num_train_samples = ins[0].shape[0]

From aa2aa7e8b62a42e22d4d64dc6c542b50f730ff28 Mon Sep 17 00:00:00 2001
From: Andrew Hundt <ATHundt@gmail.com>
Date: Thu, 15 Jun 2017 22:42:33 -0400
Subject: [PATCH 27/64] mnist_tfrecord.py remove validation data during
 model.fit() because it isn't supported

---
 examples/mnist_tfrecord.py | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/examples/mnist_tfrecord.py b/examples/mnist_tfrecord.py
index d040b388bf0..b008ac2e970 100644
--- a/examples/mnist_tfrecord.py
+++ b/examples/mnist_tfrecord.py
@@ -187,8 +187,7 @@ def create_cnn_model(x_train_batch, y_train_batch, x_batch_shape, y_batch_shape)
 train_model.summary()
 train_model.fit(batch_size=batch_size,
                 epochs=epochs,
-                callbacks=[tensorboard],
-                validation_data=(x_test_batch, y_test_batch))
+                callbacks=[tensorboard])
 train_model.save_weights('saved_wt.h5')
 
 K.clear_session()

From 4f383118e4645cafd9a0134bd08b29740e4f771d Mon Sep 17 00:00:00 2001
From: Andrew Hundt <ATHundt@gmail.com>
Date: Thu, 15 Jun 2017 22:44:10 -0400
Subject: [PATCH 28/64] pep8

---
 examples/mnist_tfrecord.py | 1 +
 keras/engine/topology.py   | 2 +-
 2 files changed, 2 insertions(+), 1 deletion(-)

diff --git a/examples/mnist_tfrecord.py b/examples/mnist_tfrecord.py
index b008ac2e970..e5f917c23ae 100644
--- a/examples/mnist_tfrecord.py
+++ b/examples/mnist_tfrecord.py
@@ -61,6 +61,7 @@ def _bytes_feature(value):
     else:
         print 'tfrecord %s already exists' % filename
 
+
 def read_and_decode_recordinput(tf_glob, one_hot=True, classes=None, is_train=None, batch_shape=[1000, 28, 28, 1]):
     """ Return tensor to read from TFRecord """
     print 'Creating graph for loading TFRecords...'
diff --git a/keras/engine/topology.py b/keras/engine/topology.py
index e01d7e6da17..14dc3c0ef53 100644
--- a/keras/engine/topology.py
+++ b/keras/engine/topology.py
@@ -1502,7 +1502,7 @@ def __init__(self, inputs, outputs, labels=None, name=None):
             self.outputs = [outputs]
 
         if labels is None:
-            self.labels = [None]*len(self.outputs)
+            self.labels = [None] * len(self.outputs)
         elif isinstance(labels, (list, tuple)):
             self.labels = list(labels)
         else:

From 773faaff367ef1912226cd0848874fcdd81101f4 Mon Sep 17 00:00:00 2001
From: Andrew Hundt <ATHundt@gmail.com>
Date: Thu, 15 Jun 2017 22:48:47 -0400
Subject: [PATCH 29/64] mnist_tfrecord.py one import per line

---
 examples/mnist_tfrecord.py | 10 ++++++++--
 1 file changed, 8 insertions(+), 2 deletions(-)

diff --git a/examples/mnist_tfrecord.py b/examples/mnist_tfrecord.py
index e5f917c23ae..506817ffc13 100644
--- a/examples/mnist_tfrecord.py
+++ b/examples/mnist_tfrecord.py
@@ -13,8 +13,14 @@
 from tensorflow.python.ops import data_flow_ops
 from keras import backend as K
 from keras.models import Model
-from keras.layers import Dense, Dropout, Flatten, Input, Conv2D, MaxPooling2D
-from keras.callbacks import EarlyStopping, TensorBoard
+from keras.layers import Dense
+from keras.layers import Dropout
+from keras.layers import Flatten
+from keras.layers import Input
+from keras.layers import Conv2D
+from keras.layers import MaxPooling2D
+from keras.callbacks import EarlyStopping
+from keras.layers import TensorBoard
 from keras.objectives import categorical_crossentropy
 from keras.utils import np_utils
 from keras.utils.generic_utils import Progbar

From 6d4b898dbc72d622f832ae58b2466125a64b7c6f Mon Sep 17 00:00:00 2001
From: Andrew Hundt <ATHundt@gmail.com>
Date: Fri, 16 Jun 2017 12:05:14 -0400
Subject: [PATCH 30/64] tensorflow_backend.py clarify Function.__call__
 according to review comments.

---
 keras/backend/tensorflow_backend.py | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/keras/backend/tensorflow_backend.py b/keras/backend/tensorflow_backend.py
index 922a46dad1c..dff41bc6105 100644
--- a/keras/backend/tensorflow_backend.py
+++ b/keras/backend/tensorflow_backend.py
@@ -2286,9 +2286,7 @@ def __init__(self, inputs, outputs, updates=None, name=None,
     def __call__(self, inputs):
         if not isinstance(inputs, (list, tuple)):
             raise TypeError('`inputs` should be a list or tuple.')
-        self.current_feed_dict = self.feed_dict
-        if self.feed_dict is None:
-            self.current_feed_dict = {}
+        self.current_feed_dict = {} if self.feed_dict is None else self.feed_dict
         for tensor, value in zip(self.inputs, inputs):
             if is_sparse(tensor):
                 sparse_coo = value.tocoo()

From 40ac8c70619bb97bad1fb3aa038b39781a1d960c Mon Sep 17 00:00:00 2001
From: Andrew Hundt <ATHundt@gmail.com>
Date: Fri, 16 Jun 2017 12:08:07 -0400
Subject: [PATCH 31/64] test_training.py initial TFRecord test in progress.
 TFRecord saving and loading works, exercise implementation in progress.

---
 tests/keras/engine/test_training.py | 268 +++++++++++++++++++++++++++-
 1 file changed, 259 insertions(+), 9 deletions(-)

diff --git a/tests/keras/engine/test_training.py b/tests/keras/engine/test_training.py
index 94bcb2ae8b7..220256b27b0 100644
--- a/tests/keras/engine/test_training.py
+++ b/tests/keras/engine/test_training.py
@@ -1,3 +1,4 @@
+import os
 import pytest
 import numpy as np
 from numpy.testing import assert_allclose
@@ -87,7 +88,8 @@ def test_model_methods():
 
     # predict_on_batch
     out = model.predict_on_batch([input_a_np, input_b_np])
-    out = model.predict_on_batch({'input_a': input_a_np, 'input_b': input_b_np})
+    out = model.predict_on_batch(
+        {'input_a': input_a_np, 'input_b': input_b_np})
 
     # predict, evaluate
     input_a_np = np.random.random((10, 3))
@@ -96,7 +98,9 @@ def test_model_methods():
     output_a_np = np.random.random((10, 4))
     output_b_np = np.random.random((10, 3))
 
-    out = model.evaluate([input_a_np, input_b_np], [output_a_np, output_b_np], batch_size=4)
+    out = model.evaluate([input_a_np, input_b_np],
+                         [output_a_np, output_b_np],
+                         batch_size=4)
     out = model.predict([input_a_np, input_b_np], batch_size=4)
 
     # with sample_weight
@@ -194,8 +198,12 @@ def mse(y_true, y_pred):
     output_a_np = np.random.random((10, 4))
     output_b_np = np.random.random((10, 3))
 
-    out = model.fit([input_a_np, input_b_np], [output_a_np, output_b_np], batch_size=4, epochs=1)
-    out = model.evaluate([input_a_np, input_b_np], [output_a_np, output_b_np], batch_size=4)
+    out = model.fit([input_a_np, input_b_np],
+                    [output_a_np, output_b_np],
+                    batch_size=4, epochs=1)
+    out = model.evaluate([input_a_np, input_b_np],
+                         [output_a_np, output_b_np],
+                         batch_size=4)
     out = model.predict([input_a_np, input_b_np], batch_size=4)
 
 
@@ -208,7 +216,8 @@ def test_sparse_input_validation_split():
     test_output = np.random.random((6, 4))
     model = Model(in1, out1)
     model.compile('rmsprop', 'mse')
-    model.fit(test_input, test_output, epochs=1, batch_size=2, validation_split=0.2)
+    model.fit(test_input, test_output, epochs=1,
+              batch_size=2, validation_split=0.2)
 
 
 @keras_test
@@ -238,15 +247,18 @@ def test_trainable_argument():
 @keras_test
 def test_check_not_failing():
     a = np.random.random((2, 1, 3))
-    _check_loss_and_target_compatibility([a], [K.categorical_crossentropy], [a.shape])
-    _check_loss_and_target_compatibility([a], [K.categorical_crossentropy], [(2, None, 3)])
+    _check_loss_and_target_compatibility(
+        [a], [K.categorical_crossentropy], [a.shape])
+    _check_loss_and_target_compatibility(
+        [a], [K.categorical_crossentropy], [(2, None, 3)])
 
 
 @keras_test
 def test_check_last_is_one():
     a = np.random.random((2, 3, 1))
     with pytest.raises(Exception) as exc:
-        _check_loss_and_target_compatibility([a], [K.categorical_crossentropy], [a.shape])
+        _check_loss_and_target_compatibility(
+            [a], [K.categorical_crossentropy], [a.shape])
 
     assert 'You are passing a target array' in str(exc)
 
@@ -255,7 +267,8 @@ def test_check_last_is_one():
 def test_check_bad_shape():
     a = np.random.random((2, 3, 5))
     with pytest.raises(Exception) as exc:
-        _check_loss_and_target_compatibility([a], [K.categorical_crossentropy], [(2, 3, 6)])
+        _check_loss_and_target_compatibility(
+            [a], [K.categorical_crossentropy], [(2, 3, 6)])
 
     assert 'targets to have the same shape' in str(exc)
 
@@ -403,6 +416,243 @@ def test_model_with_input_feed_tensor():
     assert out.shape == (10, 4)
 
 
+@pytest.mark.skipif(K.backend() != 'tensorflow', reason='Requires TF backend')
+@keras_test
+def test_model_with_input_tfrecord():
+    """We test building a model with a RecordInput as input.
+    We should be able to call fit, evaluate, predict,
+    by only passing them data for the placeholder inputs
+    in the model.
+    """
+    import tensorflow as tf
+    from tensorflow.python.lib.io import tf_record
+    from tensorflow.python.ops import data_flow_ops
+    from tensorflow.python.platform import test
+
+    """Tests for record_input_op."""
+    def to_tfrecord(images, filename):
+        # If you plan to implement a TFRecord save function see mnist_tfrecord.py
+        def _int64_feature(value):
+            return tf.train.Feature(int64_list=tf.train.Int64List(value=[value]))
+
+        def _bytes_feature(value):
+            return tf.train.Feature(bytes_list=tf.train.BytesList(value=[value]))
+
+        """ Save data into TFRecord """
+        if not os.path.isfile(filename):
+            num_examples = images.shape[0]
+            rows = images.shape[1]
+            cols = images.shape[2]
+
+            writer = tf.python_io.TFRecordWriter(filename)
+            for index in range(num_examples):
+                image_raw = images[index].tostring()
+                example = tf.train.Example(features=tf.train.Features(feature={
+                    'height': _int64_feature(rows),
+                    'width': _int64_feature(cols),
+                    'array': _bytes_feature(image_raw)}))
+                writer.write(example.SerializeToString())
+            writer.close()
+        else:
+            print 'tfrecord %s already exists' % filename
+
+    def read_and_decode_recordinput(tf_glob, batch_shape=[1]):
+        """ Return tensor to read from TFRecord """
+        # If you plan to implement TFRecord reading and decoding see mnist_tfrecord.py
+        # This implementation is for brevity, not performance.
+        print 'Creating graph for loading TFRecords...'
+        with tf.variable_scope("TFRecords"):
+            record_input = data_flow_ops.RecordInput(tf_glob, batch_size=batch_shape[0])
+            records_op = record_input.get_yield_op()
+            records_op = tf.split(records_op, batch_shape[0], 0)
+            records_op = [tf.reshape(record, []) for record in records_op]
+
+            arrays = []
+            for i, serialized_example in enumerate(records_op):
+                with tf.variable_scope("parse_array"):
+                    features = tf.parse_single_example(
+                        serialized_example,
+                        features={
+                            'height': tf.FixedLenFeature([], tf.int64),
+                            'width': tf.FixedLenFeature([], tf.int64),
+                            'array': tf.FixedLenFeature([], tf.string),
+                        })
+                    array = tf.decode_raw(features['array'], tf.float32)
+                    if len(batch_shape) == 3:
+                        height = batch_shape[1]
+                        width = batch_shape[2]
+                        # static loading, faster and more supported
+                        array.set_shape(height * width)
+                        array = tf.reshape(array, [1, height, width])
+                    else:
+                        # dynamic loading, not supported for many apis
+                        height = tf.cast(features['height'], tf.int64)
+                        width = tf.cast(features['width'], tf.int64)
+                    array = tf.reshape(array, tf.stack([height, width]))
+                    arrays.append(array)
+
+        arrays = tf.stack(arrays, 0)
+        arrays = tf.cast(arrays, tf.float32)
+        return arrays
+
+    batch_size = 10
+    input_rows = 3
+    cols = 1
+    input_a_np = np.random.random([batch_size, input_rows, cols])
+    to_tfrecord(input_a_np, 'input_a.tfrecord')
+    input_a_tf = read_and_decode_recordinput('input_a.tfrecord', input_a_np.shape)
+
+    input_b_np = np.random.random([batch_size, input_rows, cols])
+    to_tfrecord(input_a_np, 'input_b.tfrecord')
+    input_b_tf = read_and_decode_recordinput('input_b.tfrecord', input_b_np.shape)
+
+    output_a_rows = 4
+    output_a_np = np.random.random([batch_size, output_a_rows, cols])
+    to_tfrecord(input_a_np, 'output_a.tfrecord')
+    output_a_tf = read_and_decode_recordinput('output_b.tfrecord', output_a_np.shape)
+
+    output_b_rows = 3
+    output_b_np = np.random.random([batch_size, output_b_rows, cols])
+    to_tfrecord(input_a_np, 'output_b.tfrecord')
+    output_b_tf = read_and_decode_recordinput('output_b.tfrecord', output_b_np.shape)
+
+    a = Input(tensor=input_a_tf)
+    # b = Input(shape=(3,), name='input_b')
+    b = Input(tensor=input_b_tf, name='input_b')
+
+    a_2 = Dense(4, name='dense_1')(a)
+    dp = Dropout(0.5, name='dropout')
+    b_2 = dp(b)
+
+    model = Model([a, b], [a_2, b_2], labels=[output_a_tf, output_b_tf])
+    model.summary()
+
+    optimizer = 'rmsprop'
+    loss = 'mse'
+    loss_weights = [1., 0.5]
+    model.compile(optimizer, loss, metrics=['mean_squared_error'],
+                  loss_weights=loss_weights,
+                  sample_weight_mode=None)
+
+    # test train_on_batch
+    out = model.train_on_batch(input_b_np)
+    out = model.train_on_batch({'input_b': input_b_tf},
+                               [output_a_tf, output_b_tf])
+    out = model.test_on_batch({'input_b': input_b_tf},
+                              [output_a_tf, output_b_tf])
+    out = model.predict_on_batch({'input_b': input_b_tf})
+
+    # test fit
+    out = model.fit({'input_b': input_b_tf},
+                    [output_a_tf, output_b_tf], epochs=1, batch_size=10)
+    out = model.fit(input_b_tf,
+                    [output_a_tf, output_b_tf], epochs=1, batch_size=10)
+
+    # test evaluate
+    out = model.evaluate({'input_b': input_b_tf},
+                         [output_a_tf, output_b_tf], batch_size=10)
+    out = model.evaluate(input_b_tf,
+                         [output_a_tf, output_b_tf], batch_size=10)
+
+    # test predict
+    out = model.predict({'input_b': input_b_tf}, batch_size=10)
+    out = model.predict(input_b_tf, batch_size=10)
+    assert len(out) == 2
+
+    # Now test a model with a single input
+    # i.e. we don't pass any data to fit the model.
+    a = Input(tensor=tf.Variable(input_a_tf, dtype=tf.float32))
+    a_2 = Dense(4, name='dense_1')(a)
+    a_2 = Dropout(0.5, name='dropout')(a_2)
+    model = Model(a, a_2)
+    model.summary()
+
+    optimizer = 'rmsprop'
+    loss = 'mse'
+    model.compile(optimizer, loss, metrics=['mean_squared_error'])
+
+    # test train_on_batch
+    out = model.train_on_batch(None,
+                               output_a_tf)
+    out = model.train_on_batch(None,
+                               output_a_tf)
+    out = model.test_on_batch(None,
+                              output_a_tf)
+    out = model.predict_on_batch(None)
+    out = model.train_on_batch([],
+                               output_a_tf)
+    out = model.train_on_batch({},
+                               output_a_tf)
+
+    # test fit
+    out = model.fit(None,
+                    output_a_tf,
+                    epochs=1,
+                    batch_size=10)
+    out = model.fit(None,
+                    output_a_tf,
+                    epochs=1,
+                    batch_size=10)
+
+    # test evaluate
+    out = model.evaluate(None,
+                         output_a_tf,
+                         batch_size=10)
+    out = model.evaluate(None,
+                         output_a_tf,
+                         batch_size=10)
+
+    # test predict
+    out = model.predict(None, batch_size=10)
+    out = model.predict(None, batch_size=10)
+    assert out.shape == (10, 4)
+
+    # Same, without learning phase
+    # i.e. we don't pass any data to fit the model.
+    a = Input(tensor=tf.Variable(input_a_tf, dtype=tf.float32))
+    a_2 = Dense(4, name='dense_1')(a)
+    model = Model(a, a_2)
+    model.summary()
+
+    optimizer = 'rmsprop'
+    loss = 'mse'
+    model.compile(optimizer, loss, metrics=['mean_squared_error'])
+
+    # test train_on_batch
+    out = model.train_on_batch(None,
+                               output_a_tf)
+    out = model.train_on_batch(None,
+                               output_a_tf)
+    out = model.test_on_batch(None,
+                              output_a_tf)
+    out = model.predict_on_batch(None)
+    out = model.train_on_batch([],
+                               output_a_tf)
+    out = model.train_on_batch({},
+                               output_a_tf)
+
+    # test fit
+    out = model.fit(None,
+                    output_a_tf, epochs=1, batch_size=10)
+    out = model.fit(None,
+                    output_a_tf, epochs=1, batch_size=10)
+
+    # test evaluate
+    out = model.evaluate(None,
+                         output_a_tf, batch_size=10)
+    out = model.evaluate(None,
+                         output_a_tf, batch_size=10)
+
+    # test predict
+    out = model.predict(None, batch_size=10)
+    out = model.predict(None, batch_size=10)
+    assert out.shape == (10, 4)
+
+    os.remove('input_a.tfrecord')
+    os.remove('input_b.tfrecord')
+    os.remove('output_a.tfrecord')
+    os.remove('output_b.tfrecord')
+
 @keras_test
 def test_model_with_partial_loss():
     a = Input(shape=(3,), name='input_a')

From 5954c61ff5d93cda51d553468f7f302e4669bd3e Mon Sep 17 00:00:00 2001
From: Andrew Hundt <ATHundt@gmail.com>
Date: Fri, 16 Jun 2017 16:31:06 -0400
Subject: [PATCH 32/64] training.py more stringent checks of inputs targets and
 weights

---
 keras/engine/training.py | 55 +++++++++++++++++++---------------------
 1 file changed, 26 insertions(+), 29 deletions(-)

diff --git a/keras/engine/training.py b/keras/engine/training.py
index 7e5430cf22a..474b157cf33 100644
--- a/keras/engine/training.py
+++ b/keras/engine/training.py
@@ -210,37 +210,34 @@ def _check_array_lengths(inputs, targets, weights=None):
     # Raises
         ValueError: in case of incorrectly formatted data.
     """
-    x_lengths = [x.shape[0] for x in inputs]
-    set_x = set(x_lengths)
+    def set_of_lengths(x):
+        # return a set with the variation between of different shapes, with None => 0
+        return set([0]) if x is None else set([0 if y is None else y.shape[0] for y in x])
+    set_x = set_of_lengths(inputs)
+    set_y = set_of_lengths(targets)
+    set_w = set_of_lengths(weights)
     if len(set_x) > 1:
         raise ValueError('All input arrays (x) should have '
                          'the same number of samples. Got array shapes: ' +
                          str([x.shape for x in inputs]))
-    if targets:
-        y_lengths = [y.shape[0] for y in targets]
-        set_y = set(y_lengths)
-        if len(set_y) > 1:
-            raise ValueError('All target arrays (y) should have '
-                             'the same number of samples. Got array shapes: ' +
-                             str([y.shape for y in targets]))
-        if set_x and set_y and list(set_x)[0] != list(set_y)[0]:
-            raise ValueError('Input arrays should have '
-                             'the same number of samples as target arrays. '
-                             'Found ' + str(list(set_x)[0]) + ' input samples '
-                             'and ' + str(list(set_y)[0]) + ' target samples.')
-    if weights:
-        w_lengths = [w.shape[0] for w in weights]
-        set_w = set(w_lengths)
-        if len(set_w) > 1:
-            raise ValueError('All sample_weight arrays should have '
-                             'the same number of samples. Got array shapes: ' +
-                             str([w.shape for w in weights]))
-    if targets and weights:
-        if set_y and set_w and list(set_y)[0] != list(set_w)[0]:
-            raise ValueError('Sample_weight arrays should have '
-                             'the same number of samples as target arrays. Got ' +
-                             str(list(set_y)[0]) + ' input samples and ' +
-                             str(list(set_w)[0]) + ' target samples.')
+    if len(set_y) > 1:
+        raise ValueError('All target arrays (y) should have '
+                         'the same number of samples. Got array shapes: ' +
+                         str([y.shape for y in targets]))
+    if set_x and set_y and list(set_x)[0] != list(set_y)[0]:
+        raise ValueError('Input arrays should have '
+                         'the same number of samples as target arrays. '
+                         'Found ' + str(list(set_x)[0]) + ' input samples '
+                         'and ' + str(list(set_y)[0]) + ' target samples.')
+    if len(set_w) > 1:
+        raise ValueError('All sample_weight arrays should have '
+                         'the same number of samples. Got array shapes: ' +
+                         str([w.shape for w in weights]))
+    if set_y and set_w and list(set_y)[0] != list(set_w)[0]:
+        raise ValueError('Sample_weight arrays should have '
+                         'the same number of samples as target arrays. Got ' +
+                         str(list(set_y)[0]) + ' input samples and ' +
+                         str(list(set_w)[0]) + ' target samples.')
 
 
 def _check_loss_and_target_compatibility(targets, loss_fns, output_shapes):
@@ -257,11 +254,11 @@ def _check_loss_and_target_compatibility(targets, loss_fns, output_shapes):
         ValueError: if a loss function or target array
             is incompatible with an output.
     """
-    key_losses = {'mean_square_error',
+    key_losses = {'mean_squared_error',
                   'binary_crossentropy',
                   'categorical_crossentropy'}
     for y, loss, shape in zip(targets, loss_fns, output_shapes):
-        if loss is None:
+        if y is None or loss is None:
             continue
         if loss.__name__ == 'categorical_crossentropy':
             if y.shape[-1] == 1:

From 98392b2336b3a8b1a38f9f947651ed0729ac687a Mon Sep 17 00:00:00 2001
From: Andrew Hundt <ATHundt@gmail.com>
Date: Fri, 16 Jun 2017 16:36:18 -0400
Subject: [PATCH 33/64] training.py checks tensor labels in case weights aren't
 supported and raises NotImplementedError.

---
 keras/engine/training.py | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/keras/engine/training.py b/keras/engine/training.py
index 474b157cf33..2d698d2e5a5 100644
--- a/keras/engine/training.py
+++ b/keras/engine/training.py
@@ -804,6 +804,13 @@ def compile(self, optimizer, loss, metrics=None, loss_weights=None,
 
         # Prepare sample weights.
         if self._input_yield_op_tensors:
+            if not all(l is None for l in self.labels):
+                if sample_weight_mode is not None:
+                    # tensor + sample weights not yet implemented
+                    raise NotImplementedError
+                if loss_weights is not None:
+                    # tensor + loss weights not yet implemented
+                    raise NotImplementedError
             sample_weight_mode = 'disabled'
         else:
             sample_weights = self._prepare_sample_weights(

From 748c2437afb9ecb40c59eb908fe399c6b709d32a Mon Sep 17 00:00:00 2001
From: Andrew Hundt <ATHundt@gmail.com>
Date: Fri, 16 Jun 2017 16:37:55 -0400
Subject: [PATCH 34/64] training.py Model checks y more stringently

---
 keras/engine/training.py | 11 ++++-------
 1 file changed, 4 insertions(+), 7 deletions(-)

diff --git a/keras/engine/training.py b/keras/engine/training.py
index 2d698d2e5a5..0592e22d6b6 100644
--- a/keras/engine/training.py
+++ b/keras/engine/training.py
@@ -1340,13 +1340,10 @@ def _standardize_user_data(self, x, y,
                                     self._feed_input_shapes,
                                     check_batch_axis=False,
                                     exception_prefix='input')
-        if not x and y is None and self.sample_weight_mode is 'disabled':
-            y = []
-        else:
-            y = _standardize_input_data(y, self._feed_output_names,
-                                        output_shapes,
-                                        check_batch_axis=False,
-                                        exception_prefix='target')
+        y = _standardize_input_data(y, self._feed_output_names,
+                                    output_shapes,
+                                    check_batch_axis=False,
+                                    exception_prefix='target')
         class_weights = _standardize_class_weights(class_weight,
                                                    self._feed_output_names)
         if self.sample_weight_mode is not 'disabled':

From afcbaa7acf70888d5e997b111495c6100da8d4fa Mon Sep 17 00:00:00 2001
From: Andrew Hundt <ATHundt@gmail.com>
Date: Fri, 16 Jun 2017 17:17:39 -0400
Subject: [PATCH 35/64] training.py cleanup backend function calls

---
 keras/engine/training.py | 94 +++++++++++++++-------------------------
 1 file changed, 35 insertions(+), 59 deletions(-)

diff --git a/keras/engine/training.py b/keras/engine/training.py
index 0592e22d6b6..35d197fa729 100644
--- a/keras/engine/training.py
+++ b/keras/engine/training.py
@@ -1026,61 +1026,37 @@ def _prepare_sample_weights(self, sample_weight_mode, skip_indices):
                 self._feed_sample_weight_modes.append(self.sample_weight_modes[i])
         return sample_weights
 
-    def _make_train_function(self):
-        if not hasattr(self, 'train_function'):
+    def _make_function(self, function_name):
+        if not hasattr(self, function_name):
             raise RuntimeError('You must compile your model before using it.')
-        if self.train_function is None:
+        if getattr(self, function_name) is None:
             inputs = self._feed_inputs + self._feed_targets
             if self.sample_weight_mode is not 'disabled':
                 inputs += self._feed_sample_weights
             if self.uses_learning_phase and not isinstance(K.learning_phase(), int):
                 inputs += [K.learning_phase()]
 
-            training_updates = self.optimizer.get_updates(
-                self._collected_trainable_weights,
-                self.constraints,
-                self.total_loss)
-            updates = self.updates + training_updates
-            # Gets loss and metrics. Updates weights at each call.
-            self.train_function = K.function(inputs,
-                                             [self.total_loss] + self.metrics_tensors,
-                                             updates=updates,
-                                             name='train_function',
-                                             **self._function_kwargs)
-
-    def _make_test_function(self):
-        if not hasattr(self, 'test_function'):
-            raise RuntimeError('You must compile your model before using it.')
-        if self.test_function is None:
-            inputs = self._feed_inputs + self._feed_targets
-            if self.sample_weight_mode is not 'disabled':
-                inputs += self._feed_sample_weights
-            if self.uses_learning_phase and not isinstance(K.learning_phase(), int):
-                inputs += [K.learning_phase()]
-            # Return loss and metrics, no gradient updates.
-            # Does update the network states.
-            self.test_function = K.function(inputs,
-                                            [self.total_loss] + self.metrics_tensors,
-                                            updates=self.state_updates,
-                                            name='test_function',
-                                            **self._function_kwargs)
-
-    def _make_predict_function(self):
-        if not hasattr(self, 'predict_function'):
-            self.predict_function = None
-        if self.predict_function is None:
-            if self.uses_learning_phase and not isinstance(K.learning_phase(), int):
-                inputs = self._feed_inputs + [K.learning_phase()]
+            if function_name is 'train_function':
+                training_updates = self.optimizer.get_updates(
+                    self._collected_trainable_weights,
+                    self.constraints,
+                    self.total_loss)
+                updates = self.updates + training_updates
             else:
-                inputs = self._feed_inputs
-            # Gets network outputs. Does not update weights.
-            # Does update the network states.
-            kwargs = getattr(self, '_function_kwargs', {})
-            self.predict_function = K.function(inputs,
-                                               self.outputs,
-                                               updates=self.state_updates,
-                                               name='predict_function',
-                                               **kwargs)
+                updates = self.state_updates
+
+            if function_name is 'predict_function':
+                outputs = self.outputs
+                kwargs = getattr(self, '_function_kwargs', {})
+            else:
+                outputs = [self.total_loss] + self.metrics_tensors
+                kwargs = self._function_kwargs
+            # Gets loss and metrics. Updates weights at each call.
+            setattr(self, function_name, K.function(inputs,
+                                                    outputs,
+                                                    updates=updates,
+                                                    name=function_name,
+                                                    **kwargs))
 
     def _fit_loop(self, f, ins, out_labels=None, batch_size=32,
                   epochs=100, verbose=1, callbacks=None,
@@ -1487,7 +1463,7 @@ class indices (integers) to
                 sample_weight=val_sample_weight,
                 check_batch_axis=False,
                 batch_size=batch_size)
-            self._make_test_function()
+            self._make_function('test_function')
             val_f = self.test_function
             if self.uses_learning_phase and not isinstance(K.learning_phase(), int):
                 val_ins = val_x + val_y + val_sample_weights + [0.]
@@ -1505,7 +1481,7 @@ class indices (integers) to
             sample_weights, val_sample_weights = (
                 _slice_arrays(sample_weights, 0, split_at),
                 _slice_arrays(sample_weights, split_at))
-            self._make_test_function()
+            self._make_function('test_function')
             val_f = self.test_function
             if self.uses_learning_phase and not isinstance(K.learning_phase(), int):
                 val_ins = val_x + val_y + val_sample_weights + [0.]
@@ -1521,7 +1497,7 @@ class indices (integers) to
             ins = x + y + sample_weights + [1.]
         else:
             ins = x + y + sample_weights
-        self._make_train_function()
+        self._make_function('train_function')
         f = self.train_function
 
         # Prepare display labels.
@@ -1578,7 +1554,7 @@ def evaluate(self, x, y, batch_size=32, verbose=1, sample_weight=None):
             ins = x + y + sample_weights + [0.]
         else:
             ins = x + y + sample_weights
-        self._make_test_function()
+        self._make_function('test_function')
         f = self.test_function
         return self._test_loop(f, ins,
                                batch_size=batch_size,
@@ -1622,7 +1598,7 @@ def predict(self, x, batch_size=32, verbose=0):
             ins = x + [0.]
         else:
             ins = x
-        self._make_predict_function()
+        self._make_function('predict_function')
         f = self.predict_function
         return self._predict_loop(f, ins,
                                   batch_size=batch_size, verbose=verbose)
@@ -1672,7 +1648,7 @@ class indices (integers) to
             ins = x + y + sample_weights + [1.]
         else:
             ins = x + y + sample_weights
-        self._make_train_function()
+        self._make_function('train_function')
         outputs = self.train_function(ins)
         if len(outputs) == 1:
             return outputs[0]
@@ -1714,7 +1690,7 @@ def test_on_batch(self, x, y, sample_weight=None):
             ins = x + y + sample_weights + [0.]
         else:
             ins = x + y + sample_weights
-        self._make_test_function()
+        self._make_function('test_function')
         outputs = self.test_function(ins)
         if len(outputs) == 1:
             return outputs[0]
@@ -1735,7 +1711,7 @@ def predict_on_batch(self, x):
             ins = x + [0.]
         else:
             ins = x
-        self._make_predict_function()
+        self._make_function('predict_function')
         outputs = self.predict_function(ins)
         if len(outputs) == 1:
             return outputs[0]
@@ -1827,9 +1803,9 @@ def generate_arrays_from_file(path):
         epoch = initial_epoch
 
         do_validation = bool(validation_data)
-        self._make_train_function()
+        self._make_function('train_function')
         if do_validation:
-            self._make_test_function()
+            self._make_function('test_function')
 
         # python 2 has 'next', 3 has '__next__'
         # avoid any explicit version checks
@@ -2016,7 +1992,7 @@ def evaluate_generator(self, generator, steps,
             ValueError: In case the generator yields
                 data in an invalid format.
         """
-        self._make_test_function()
+        self._make_function('test_function')
 
         steps_done = 0
         wait_time = 0.01
@@ -2111,7 +2087,7 @@ def predict_generator(self, generator, steps,
             ValueError: In case the generator yields
                 data in an invalid format.
         """
-        self._make_predict_function()
+        self._make_function('predict_function')
 
         steps_done = 0
         wait_time = 0.01

From 42fbdbc3a178b1884dee8c9f9e82779c9fc2808a Mon Sep 17 00:00:00 2001
From: Andrew Hundt <ATHundt@gmail.com>
Date: Fri, 16 Jun 2017 18:43:33 -0400
Subject: [PATCH 36/64] training.py don't slice None entries

---
 keras/engine/training.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/keras/engine/training.py b/keras/engine/training.py
index 35d197fa729..5225a7a2d24 100644
--- a/keras/engine/training.py
+++ b/keras/engine/training.py
@@ -391,9 +391,9 @@ def _slice_arrays(arrays, start=None, stop=None):
             # hdf5 datasets only support list objects as indices
             if hasattr(start, 'shape'):
                 start = start.tolist()
-            return [x[start] for x in arrays]
+            return [None if x is None else x[start] for x in arrays]
         else:
-            return [x[start:stop] for x in arrays]
+            return [None if x is None else x[start:stop] for x in arrays]
     else:
         if hasattr(start, '__len__'):
             if hasattr(start, 'shape'):

From 75edea7a01152308c0c6a4815e010c6d6db4ee7b Mon Sep 17 00:00:00 2001
From: Andrew Hundt <ATHundt@gmail.com>
Date: Fri, 16 Jun 2017 18:46:07 -0400
Subject: [PATCH 37/64] topology.py always insert layer properties into feed

---
 keras/engine/topology.py | 7 +++----
 1 file changed, 3 insertions(+), 4 deletions(-)

diff --git a/keras/engine/topology.py b/keras/engine/topology.py
index 14dc3c0ef53..e7b0ad57dfa 100644
--- a/keras/engine/topology.py
+++ b/keras/engine/topology.py
@@ -1639,10 +1639,9 @@ def __init__(self, inputs, outputs, labels=None, name=None):
                                                    i,
                                                    layer.__class__.__name__))
             self.input_names.append(layer.name)
-            if K.is_placeholder(layer):
-                self._feed_input_names.append(layer.name)
-                self._feed_inputs.append(layer.input)
-                self._feed_input_shapes.append(self.inputs[i]._keras_shape)
+            self._feed_input_names.append(layer.name)
+            self._feed_inputs.append(layer.input)
+            self._feed_input_shapes.append(self.inputs[i]._keras_shape)
         for layer in self.output_layers:
             self.output_names.append(layer.name)
 

From caad65af11461dbe368c31e97630ec003da931cc Mon Sep 17 00:00:00 2001
From: Andrew Hundt <ATHundt@gmail.com>
Date: Fri, 16 Jun 2017 18:47:11 -0400
Subject: [PATCH 38/64] training.py define ins consistently

---
 keras/engine/training.py | 93 +++++++++++++++++-----------------------
 1 file changed, 40 insertions(+), 53 deletions(-)

diff --git a/keras/engine/training.py b/keras/engine/training.py
index 5225a7a2d24..7948d65dcea 100644
--- a/keras/engine/training.py
+++ b/keras/engine/training.py
@@ -1026,10 +1026,23 @@ def _prepare_sample_weights(self, sample_weight_mode, skip_indices):
                 self._feed_sample_weight_modes.append(self.sample_weight_modes[i])
         return sample_weights
 
+    def _make_ins(self, x=None, y=None, sample_weights=None, learning_phase_value=None):
+        def add_if_valid(x):
+            # Don't include None values, and in a worst case return an empty list
+            return [] if x is None else [val for val in x if val is not None]
+        # ins = add_if_valid(x) + add_if_valid(y) + add_if_valid(sample_weights)
+        ins = x + y + sample_weights
+        if learning_phase_value and self.uses_learning_phase and not isinstance(K.learning_phase(), int):
+            ins = ins + learning_phase_value
+        return ins
+
     def _make_function(self, function_name):
         if not hasattr(self, function_name):
             raise RuntimeError('You must compile your model before using it.')
-        if getattr(self, function_name) is None:
+        function = getattr(self, function_name)
+        if function is not None:
+            return function
+        else:
             inputs = self._feed_inputs + self._feed_targets
             if self.sample_weight_mode is not 'disabled':
                 inputs += self._feed_sample_weights
@@ -1052,11 +1065,11 @@ def _make_function(self, function_name):
                 outputs = [self.total_loss] + self.metrics_tensors
                 kwargs = self._function_kwargs
             # Gets loss and metrics. Updates weights at each call.
-            setattr(self, function_name, K.function(inputs,
-                                                    outputs,
-                                                    updates=updates,
-                                                    name=function_name,
-                                                    **kwargs))
+            return K.function(inputs,
+                              outputs,
+                              updates=updates,
+                              name=function_name,
+                              **kwargs)
 
     def _fit_loop(self, f, ins, out_labels=None, batch_size=32,
                   epochs=100, verbose=1, callbacks=None,
@@ -1463,12 +1476,9 @@ class indices (integers) to
                 sample_weight=val_sample_weight,
                 check_batch_axis=False,
                 batch_size=batch_size)
-            self._make_function('test_function')
+            self.test_function = self._make_function('test_function')
             val_f = self.test_function
-            if self.uses_learning_phase and not isinstance(K.learning_phase(), int):
-                val_ins = val_x + val_y + val_sample_weights + [0.]
-            else:
-                val_ins = val_x + val_y + val_sample_weights
+            val_ins = self._make_ins(val_x, val_y, val_sample_weights, [0.])
 
         elif validation_split and 0. < validation_split < 1.:
             do_validation = True
@@ -1481,23 +1491,17 @@ class indices (integers) to
             sample_weights, val_sample_weights = (
                 _slice_arrays(sample_weights, 0, split_at),
                 _slice_arrays(sample_weights, split_at))
-            self._make_function('test_function')
+            self.test_function = self._make_function('test_function')
             val_f = self.test_function
-            if self.uses_learning_phase and not isinstance(K.learning_phase(), int):
-                val_ins = val_x + val_y + val_sample_weights + [0.]
-            else:
-                val_ins = val_x + val_y + val_sample_weights
+            val_ins = self._make_ins(val_x, val_y, val_sample_weights, [0.])
         else:
             do_validation = False
             val_f = None
             val_ins = None
 
         # Prepare input arrays and training function.
-        if self.uses_learning_phase and not isinstance(K.learning_phase(), int):
-            ins = x + y + sample_weights + [1.]
-        else:
-            ins = x + y + sample_weights
-        self._make_function('train_function')
+        ins = self._make_ins(x, y, sample_weights, [1.])
+        self.train_function = self._make_function('train_function')
         f = self.train_function
 
         # Prepare display labels.
@@ -1550,11 +1554,8 @@ def evaluate(self, x, y, batch_size=32, verbose=1, sample_weight=None):
             check_batch_axis=False,
             batch_size=batch_size)
         # Prepare inputs, delegate logic to `_test_loop`.
-        if self.uses_learning_phase and not isinstance(K.learning_phase(), int):
-            ins = x + y + sample_weights + [0.]
-        else:
-            ins = x + y + sample_weights
-        self._make_function('test_function')
+        ins = self._make_ins(x, y, sample_weights, [0.])
+        self.test_function = self._make_function('test_function')
         f = self.test_function
         return self._test_loop(f, ins,
                                batch_size=batch_size,
@@ -1594,11 +1595,8 @@ def predict(self, x, batch_size=32, verbose=0):
                                  'Batch size: ' + str(batch_size) + '.')
 
         # Prepare inputs, delegate logic to `_predict_loop`.
-        if self.uses_learning_phase and not isinstance(K.learning_phase(), int):
-            ins = x + [0.]
-        else:
-            ins = x
-        self._make_function('predict_function')
+        ins = self._make_ins(x, None, None, [0.])
+        self.predict_function = self._make_function('predict_function')
         f = self.predict_function
         return self._predict_loop(f, ins,
                                   batch_size=batch_size, verbose=verbose)
@@ -1644,11 +1642,8 @@ class indices (integers) to
             sample_weight=sample_weight,
             class_weight=class_weight,
             check_batch_axis=True)
-        if self.uses_learning_phase and not isinstance(K.learning_phase(), int):
-            ins = x + y + sample_weights + [1.]
-        else:
-            ins = x + y + sample_weights
-        self._make_function('train_function')
+        ins = self._make_ins(x, y, sample_weights, [1.])
+        self.train_function = self._make_function('train_function')
         outputs = self.train_function(ins)
         if len(outputs) == 1:
             return outputs[0]
@@ -1686,11 +1681,8 @@ def test_on_batch(self, x, y, sample_weight=None):
             x, y,
             sample_weight=sample_weight,
             check_batch_axis=True)
-        if self.uses_learning_phase and not isinstance(K.learning_phase(), int):
-            ins = x + y + sample_weights + [0.]
-        else:
-            ins = x + y + sample_weights
-        self._make_function('test_function')
+        ins = self._make_ins(x, y, sample_weights, [0.])
+        self.test_function = self._make_function('test_function')
         outputs = self.test_function(ins)
         if len(outputs) == 1:
             return outputs[0]
@@ -1707,11 +1699,8 @@ def predict_on_batch(self, x):
         """
         x = _standardize_input_data(x, self._feed_input_names,
                                     self._feed_input_shapes)
-        if self.uses_learning_phase and not isinstance(K.learning_phase(), int):
-            ins = x + [0.]
-        else:
-            ins = x
-        self._make_function('predict_function')
+        ins = self._make_ins(x, None, None, [0.])
+        self.predict_function = self._make_function('predict_function')
         outputs = self.predict_function(ins)
         if len(outputs) == 1:
             return outputs[0]
@@ -1803,9 +1792,9 @@ def generate_arrays_from_file(path):
         epoch = initial_epoch
 
         do_validation = bool(validation_data)
-        self._make_function('train_function')
+        self.train_function = self._make_function('train_function')
         if do_validation:
-            self._make_function('test_function')
+            self.test_function = self._make_function('test_function')
 
         # python 2 has 'next', 3 has '__next__'
         # avoid any explicit version checks
@@ -1855,9 +1844,7 @@ def generate_arrays_from_file(path):
                                  str(validation_data))
             val_x, val_y, val_sample_weights = self._standardize_user_data(
                 val_x, val_y, val_sample_weight)
-            val_data = val_x + val_y + val_sample_weights
-            if self.uses_learning_phase and not isinstance(K.learning_phase(), int):
-                val_data += [0.]
+            val_data = self._make_ins(val_x, val_y, val_sample_weights, [0.])
             for cbk in callbacks:
                 cbk.validation_data = val_data
         enqueuer = None
@@ -1992,7 +1979,7 @@ def evaluate_generator(self, generator, steps,
             ValueError: In case the generator yields
                 data in an invalid format.
         """
-        self._make_function('test_function')
+        self.test_function = self._make_function('test_function')
 
         steps_done = 0
         wait_time = 0.01
@@ -2087,7 +2074,7 @@ def predict_generator(self, generator, steps,
             ValueError: In case the generator yields
                 data in an invalid format.
         """
-        self._make_function('predict_function')
+        self.predict_function = self._make_function('predict_function')
 
         steps_done = 0
         wait_time = 0.01

From 845dd8858ddfbe17d85aecfae0d603a2aa2ad367 Mon Sep 17 00:00:00 2001
From: Andrew Hundt <ATHundt@gmail.com>
Date: Fri, 16 Jun 2017 18:47:57 -0400
Subject: [PATCH 39/64] tensorflow_backend.py if feed_dict entry is None add it
 to fetches instead

---
 keras/backend/tensorflow_backend.py | 9 +++++++--
 1 file changed, 7 insertions(+), 2 deletions(-)

diff --git a/keras/backend/tensorflow_backend.py b/keras/backend/tensorflow_backend.py
index dff41bc6105..69bba5fe8db 100644
--- a/keras/backend/tensorflow_backend.py
+++ b/keras/backend/tensorflow_backend.py
@@ -2287,15 +2287,20 @@ def __call__(self, inputs):
         if not isinstance(inputs, (list, tuple)):
             raise TypeError('`inputs` should be a list or tuple.')
         self.current_feed_dict = {} if self.feed_dict is None else self.feed_dict
+        self.feed_to_fetch_count = 0
+        self.current_fetches = self.outputs + [self.updates_op]
         for tensor, value in zip(self.inputs, inputs):
             if is_sparse(tensor):
                 sparse_coo = value.tocoo()
                 indices = np.concatenate((np.expand_dims(sparse_coo.row, 1),
                                           np.expand_dims(sparse_coo.col, 1)), 1)
                 value = (indices, sparse_coo.data, sparse_coo.shape)
-            self.current_feed_dict[tensor] = value
+            if value is None and tensor is not None:
+                self.feed_to_fetch_count += 1
+                self.current_fetches.append(tensor)
+            else:
+                self.current_feed_dict[tensor] = value
 
-        self.current_fetches = self.outputs + [self.updates_op]
         if self.fetches is not None:
             self.current_fetches += self.fetches
 

From 39a798158068606591d8679310954c173052ebf2 Mon Sep 17 00:00:00 2001
From: Andrew Hundt <ATHundt@gmail.com>
Date: Fri, 16 Jun 2017 23:32:59 -0400
Subject: [PATCH 40/64] trainin.py ins should at least include an empty list

---
 keras/engine/training.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/keras/engine/training.py b/keras/engine/training.py
index 7948d65dcea..8d491345f92 100644
--- a/keras/engine/training.py
+++ b/keras/engine/training.py
@@ -1028,10 +1028,10 @@ def _prepare_sample_weights(self, sample_weight_mode, skip_indices):
 
     def _make_ins(self, x=None, y=None, sample_weights=None, learning_phase_value=None):
         def add_if_valid(x):
-            # Don't include None values, and in a worst case return an empty list
-            return [] if x is None else [val for val in x if val is not None]
+            # In a worst case return an empty list
+            return [None] if x is None else x
         # ins = add_if_valid(x) + add_if_valid(y) + add_if_valid(sample_weights)
-        ins = x + y + sample_weights
+        ins = add_if_valid(x) + add_if_valid(y) + add_if_valid(sample_weights)
         if learning_phase_value and self.uses_learning_phase and not isinstance(K.learning_phase(), int):
             ins = ins + learning_phase_value
         return ins

From c0d0cdaa2f431529c9cd812e23bef468b22bea24 Mon Sep 17 00:00:00 2001
From: Andrew Hundt <ATHundt@gmail.com>
Date: Fri, 16 Jun 2017 23:33:20 -0400
Subject: [PATCH 41/64] training.py first TFRecord test passes!

---
 tests/keras/engine/test_training.py | 262 ++++++++++------------------
 1 file changed, 92 insertions(+), 170 deletions(-)

diff --git a/tests/keras/engine/test_training.py b/tests/keras/engine/test_training.py
index 220256b27b0..df2f2d8cc8f 100644
--- a/tests/keras/engine/test_training.py
+++ b/tests/keras/engine/test_training.py
@@ -5,6 +5,7 @@
 import scipy.sparse as sparse
 
 from keras.layers import Dense, Dropout
+from keras.layers import Flatten
 from keras.engine.topology import Input
 from keras.engine.training import Model, _check_loss_and_target_compatibility
 from keras.models import Sequential
@@ -429,9 +430,8 @@ def test_model_with_input_tfrecord():
     from tensorflow.python.ops import data_flow_ops
     from tensorflow.python.platform import test
 
-    """Tests for record_input_op."""
-    def to_tfrecord(images, filename):
-        # If you plan to implement a TFRecord save function see mnist_tfrecord.py
+    def images_to_tfrecord(images, labels, filename):
+
         def _int64_feature(value):
             return tf.train.Feature(int64_list=tf.train.Int64List(value=[value]))
 
@@ -441,25 +441,28 @@ def _bytes_feature(value):
         """ Save data into TFRecord """
         if not os.path.isfile(filename):
             num_examples = images.shape[0]
+
             rows = images.shape[1]
             cols = images.shape[2]
+            depth = images.shape[3]
 
+            print('Writing', filename)
             writer = tf.python_io.TFRecordWriter(filename)
             for index in range(num_examples):
                 image_raw = images[index].tostring()
                 example = tf.train.Example(features=tf.train.Features(feature={
                     'height': _int64_feature(rows),
                     'width': _int64_feature(cols),
-                    'array': _bytes_feature(image_raw)}))
+                    'depth': _int64_feature(depth),
+                    'label': _int64_feature(int(labels[index])),
+                    'image_raw': _bytes_feature(image_raw)}))
                 writer.write(example.SerializeToString())
             writer.close()
         else:
             print 'tfrecord %s already exists' % filename
 
-    def read_and_decode_recordinput(tf_glob, batch_shape=[1]):
+    def read_and_decode_recordinput(tf_glob, one_hot=True, classes=None, is_train=None, batch_shape=[10, 3, 3, 1]):
         """ Return tensor to read from TFRecord """
-        # If you plan to implement TFRecord reading and decoding see mnist_tfrecord.py
-        # This implementation is for brevity, not performance.
         print 'Creating graph for loading TFRecords...'
         with tf.variable_scope("TFRecords"):
             record_input = data_flow_ops.RecordInput(tf_glob, batch_size=batch_shape[0])
@@ -467,191 +470,110 @@ def read_and_decode_recordinput(tf_glob, batch_shape=[1]):
             records_op = tf.split(records_op, batch_shape[0], 0)
             records_op = [tf.reshape(record, []) for record in records_op]
 
-            arrays = []
+            images = []
+            labels = []
             for i, serialized_example in enumerate(records_op):
-                with tf.variable_scope("parse_array"):
+                with tf.variable_scope("parse_images", reuse=True):
                     features = tf.parse_single_example(
                         serialized_example,
                         features={
-                            'height': tf.FixedLenFeature([], tf.int64),
-                            'width': tf.FixedLenFeature([], tf.int64),
-                            'array': tf.FixedLenFeature([], tf.string),
+                            'label': tf.FixedLenFeature([], tf.int64),
+                            'image_raw': tf.FixedLenFeature([], tf.string),
                         })
-                    array = tf.decode_raw(features['array'], tf.float32)
-                    if len(batch_shape) == 3:
-                        height = batch_shape[1]
-                        width = batch_shape[2]
-                        # static loading, faster and more supported
-                        array.set_shape(height * width)
-                        array = tf.reshape(array, [1, height, width])
-                    else:
-                        # dynamic loading, not supported for many apis
-                        height = tf.cast(features['height'], tf.int64)
-                        width = tf.cast(features['width'], tf.int64)
-                    array = tf.reshape(array, tf.stack([height, width]))
-                    arrays.append(array)
-
-        arrays = tf.stack(arrays, 0)
-        arrays = tf.cast(arrays, tf.float32)
-        return arrays
-
-    batch_size = 10
-    input_rows = 3
-    cols = 1
-    input_a_np = np.random.random([batch_size, input_rows, cols])
-    to_tfrecord(input_a_np, 'input_a.tfrecord')
-    input_a_tf = read_and_decode_recordinput('input_a.tfrecord', input_a_np.shape)
-
-    input_b_np = np.random.random([batch_size, input_rows, cols])
-    to_tfrecord(input_a_np, 'input_b.tfrecord')
-    input_b_tf = read_and_decode_recordinput('input_b.tfrecord', input_b_np.shape)
-
-    output_a_rows = 4
-    output_a_np = np.random.random([batch_size, output_a_rows, cols])
-    to_tfrecord(input_a_np, 'output_a.tfrecord')
-    output_a_tf = read_and_decode_recordinput('output_b.tfrecord', output_a_np.shape)
-
-    output_b_rows = 3
-    output_b_np = np.random.random([batch_size, output_b_rows, cols])
-    to_tfrecord(input_a_np, 'output_b.tfrecord')
-    output_b_tf = read_and_decode_recordinput('output_b.tfrecord', output_b_np.shape)
-
-    a = Input(tensor=input_a_tf)
-    # b = Input(shape=(3,), name='input_b')
-    b = Input(tensor=input_b_tf, name='input_b')
-
-    a_2 = Dense(4, name='dense_1')(a)
-    dp = Dropout(0.5, name='dropout')
-    b_2 = dp(b)
-
-    model = Model([a, b], [a_2, b_2], labels=[output_a_tf, output_b_tf])
-    model.summary()
-
-    optimizer = 'rmsprop'
-    loss = 'mse'
-    loss_weights = [1., 0.5]
-    model.compile(optimizer, loss, metrics=['mean_squared_error'],
-                  loss_weights=loss_weights,
-                  sample_weight_mode=None)
+                    img = tf.decode_raw(features['image_raw'], tf.uint8)
+                    img.set_shape(batch_shape[1] * batch_shape[2])
+                    img = tf.reshape(img, [1] + batch_shape[1:])
 
-    # test train_on_batch
-    out = model.train_on_batch(input_b_np)
-    out = model.train_on_batch({'input_b': input_b_tf},
-                               [output_a_tf, output_b_tf])
-    out = model.test_on_batch({'input_b': input_b_tf},
-                              [output_a_tf, output_b_tf])
-    out = model.predict_on_batch({'input_b': input_b_tf})
+                    img = tf.cast(img, tf.float32) * (1. / 255) - 0.5
 
-    # test fit
-    out = model.fit({'input_b': input_b_tf},
-                    [output_a_tf, output_b_tf], epochs=1, batch_size=10)
-    out = model.fit(input_b_tf,
-                    [output_a_tf, output_b_tf], epochs=1, batch_size=10)
+                    label = tf.cast(features['label'], tf.int32)
+                    if one_hot and classes:
+                        label = tf.one_hot(label, classes)
 
-    # test evaluate
-    out = model.evaluate({'input_b': input_b_tf},
-                         [output_a_tf, output_b_tf], batch_size=10)
-    out = model.evaluate(input_b_tf,
-                         [output_a_tf, output_b_tf], batch_size=10)
+                    images.append(img)
+                    labels.append(label)
 
-    # test predict
-    out = model.predict({'input_b': input_b_tf}, batch_size=10)
-    out = model.predict(input_b_tf, batch_size=10)
-    assert len(out) == 2
+            images = tf.parallel_stack(images, 0)
+            labels = tf.parallel_stack(labels, 0)
+            images = tf.cast(images, tf.float32)
 
-    # Now test a model with a single input
-    # i.e. we don't pass any data to fit the model.
-    a = Input(tensor=tf.Variable(input_a_tf, dtype=tf.float32))
-    a_2 = Dense(4, name='dense_1')(a)
-    a_2 = Dropout(0.5, name='dropout')(a_2)
-    model = Model(a, a_2)
-    model.summary()
+            images = tf.reshape(images, shape=batch_shape)
 
-    optimizer = 'rmsprop'
-    loss = 'mse'
-    model.compile(optimizer, loss, metrics=['mean_squared_error'])
+            return images, labels
 
-    # test train_on_batch
-    out = model.train_on_batch(None,
-                               output_a_tf)
-    out = model.train_on_batch(None,
-                               output_a_tf)
-    out = model.test_on_batch(None,
-                              output_a_tf)
-    out = model.predict_on_batch(None)
-    out = model.train_on_batch([],
-                               output_a_tf)
-    out = model.train_on_batch({},
-                               output_a_tf)
+    def replace(filename):
+        if os.path.isfile(filename):
+            print '%s already exists, replacing...' % filename
+            os.remove(filename)
 
-    # test fit
-    out = model.fit(None,
-                    output_a_tf,
-                    epochs=1,
-                    batch_size=10)
-    out = model.fit(None,
-                    output_a_tf,
-                    epochs=1,
-                    batch_size=10)
-
-    # test evaluate
-    out = model.evaluate(None,
-                         output_a_tf,
-                         batch_size=10)
-    out = model.evaluate(None,
-                         output_a_tf,
-                         batch_size=10)
-
-    # test predict
-    out = model.predict(None, batch_size=10)
-    out = model.predict(None, batch_size=10)
-    assert out.shape == (10, 4)
-
-    # Same, without learning phase
-    # i.e. we don't pass any data to fit the model.
-    a = Input(tensor=tf.Variable(input_a_tf, dtype=tf.float32))
-    a_2 = Dense(4, name='dense_1')(a)
-    model = Model(a, a_2)
+    batch_size = 10
+    input_rows = 3
+    cols = 3
+    depth = 1
+    classes = 2
+    img_batch_shape = [batch_size, input_rows, cols, depth]
+    label_batch_shape = [batch_size, classes]
+    input_a_np = np.multiply(np.random.random(img_batch_shape), batch_size)
+    input_a_np = input_a_np.astype('uint8')
+    output_a_rows = 4
+    output_a_np = np.multiply(np.random.random([batch_size]), batch_size)
+    output_a_np = output_a_np.astype('int')
+    replace('input_a.tfrecord')
+    images_to_tfrecord(input_a_np, output_a_np, 'input_a.tfrecord')
+    input_a_tf, output_b_tf = read_and_decode_recordinput(
+        'input_a.tfrecord',
+        one_hot=True,
+        classes=classes,
+        is_train=True,
+        batch_shape=img_batch_shape)
+
+    a = Input(tensor=input_a_tf, batch_shape=img_batch_shape)
+
+    a_2 = Dense(8, name='dense_1')(a)
+    dp = Dropout(0.5, name='dropout')
+    b_2 = dp(a_2)
+    f_1 = Flatten()(b_2)
+    b_3 = Dense(
+        classes,
+        activation='softmax',
+        name='dense_b3')(f_1)
+
+    y_train_in_out = Input(
+        tensor=output_b_tf,
+        batch_shape=label_batch_shape,
+        name='y_labels')
+
+    model = Model([a], [b_3], labels=[y_train_in_out])
     model.summary()
 
     optimizer = 'rmsprop'
     loss = 'mse'
-    model.compile(optimizer, loss, metrics=['mean_squared_error'])
-
-    # test train_on_batch
-    out = model.train_on_batch(None,
-                               output_a_tf)
-    out = model.train_on_batch(None,
-                               output_a_tf)
-    out = model.test_on_batch(None,
-                              output_a_tf)
-    out = model.predict_on_batch(None)
-    out = model.train_on_batch([],
-                               output_a_tf)
-    out = model.train_on_batch({},
-                               output_a_tf)
+    loss_weights = [1.]
+    try:
+        with pytest.raises(NotImplementedError) as exc:
+            model.compile(optimizer, loss, metrics=['mean_squared_error'],
+                          loss_weights=loss_weights,
+                          sample_weight_mode=None)
+    except NotImplementedError:
+        print 'Feature not implemented'
 
+    model.compile(optimizer, loss, metrics=['mean_squared_error'],
+                  sample_weight_mode=None)
+    out = model.train_on_batch(None, None)
     # test fit
     out = model.fit(None,
-                    output_a_tf, epochs=1, batch_size=10)
-    out = model.fit(None,
-                    output_a_tf, epochs=1, batch_size=10)
-
-    # test evaluate
-    out = model.evaluate(None,
-                         output_a_tf, batch_size=10)
-    out = model.evaluate(None,
-                         output_a_tf, batch_size=10)
-
-    # test predict
-    out = model.predict(None, batch_size=10)
-    out = model.predict(None, batch_size=10)
-    assert out.shape == (10, 4)
+                    None, epochs=1, batch_size=10)
+    try:
+        with pytest.raises(ValueError) as exc:
+            out = model.fit(output_a_np,
+                            [y_train_in_out],
+                            epochs=1,
+                            batch_size=10)
+    except ValueError:
+        print 'cannot mix input tensors and numpy arrays'
 
     os.remove('input_a.tfrecord')
-    os.remove('input_b.tfrecord')
-    os.remove('output_a.tfrecord')
-    os.remove('output_b.tfrecord')
+
 
 @keras_test
 def test_model_with_partial_loss():

From 6248b5454f9814b9a11a9e93078f34409a1290ef Mon Sep 17 00:00:00 2001
From: Andrew Hundt <ATHundt@gmail.com>
Date: Sat, 17 Jun 2017 04:03:21 -0400
Subject: [PATCH 42/64] predict_on_batch runs

---
 keras/engine/training.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/keras/engine/training.py b/keras/engine/training.py
index 8d491345f92..1de6c22cbd4 100644
--- a/keras/engine/training.py
+++ b/keras/engine/training.py
@@ -1033,7 +1033,7 @@ def add_if_valid(x):
         # ins = add_if_valid(x) + add_if_valid(y) + add_if_valid(sample_weights)
         ins = add_if_valid(x) + add_if_valid(y) + add_if_valid(sample_weights)
         if learning_phase_value and self.uses_learning_phase and not isinstance(K.learning_phase(), int):
-            ins = ins + learning_phase_value
+            ins = ins + add_if_valid(learning_phase_value)
         return ins
 
     def _make_function(self, function_name):
@@ -1699,7 +1699,7 @@ def predict_on_batch(self, x):
         """
         x = _standardize_input_data(x, self._feed_input_names,
                                     self._feed_input_shapes)
-        ins = self._make_ins(x, None, None, [0.])
+        ins = self._make_ins(x, None, [], [0.])
         self.predict_function = self._make_function('predict_function')
         outputs = self.predict_function(ins)
         if len(outputs) == 1:

From b8dae763daa99550e0b58c12c30d730a7edebd3d Mon Sep 17 00:00:00 2001
From: Andrew Hundt <ATHundt@gmail.com>
Date: Sat, 17 Jun 2017 04:38:05 -0400
Subject: [PATCH 43/64]        
 self._prepare_sample_weights(sample_weight_mode, skip_indices)

---
 keras/engine/training.py | 167 ++++++++++++++++++++-------------------
 1 file changed, 84 insertions(+), 83 deletions(-)

diff --git a/keras/engine/training.py b/keras/engine/training.py
index 1de6c22cbd4..d72a40fafa6 100644
--- a/keras/engine/training.py
+++ b/keras/engine/training.py
@@ -803,19 +803,7 @@ def compile(self, optimizer, loss, metrics=None, loss_weights=None,
                             ' - expected a list of dicts.')
 
         # Prepare sample weights.
-        if self._input_yield_op_tensors:
-            if not all(l is None for l in self.labels):
-                if sample_weight_mode is not None:
-                    # tensor + sample weights not yet implemented
-                    raise NotImplementedError
-                if loss_weights is not None:
-                    # tensor + loss weights not yet implemented
-                    raise NotImplementedError
-            sample_weight_mode = 'disabled'
-        else:
-            sample_weights = self._prepare_sample_weights(
-                sample_weight_mode, skip_indices)
-        self.sample_weight_mode = sample_weight_mode
+        self._prepare_sample_weights(sample_weight_mode, skip_indices)
 
         # Prepare targets of model.
         self.targets = []
@@ -951,80 +939,93 @@ def append_metric(layer_num, metric_name, metric_tensor):
 
     def _prepare_sample_weights(self, sample_weight_mode, skip_indices):
         # Prepare sample weights.
-        sample_weights = []
-        sample_weight_modes = []
-        if isinstance(sample_weight_mode, dict):
-            for name in sample_weight_mode:
-                if name not in self.output_names:
-                    raise ValueError('Unknown entry in '
-                                     'sample_weight_mode dictionary: "' +
-                                     name + '". '
-                                     'Only expected the following keys: ' +
-                                     str(self.output_names))
-            for i, name in enumerate(self.output_names):
-                if i in skip_indices:
-                    weight = None
-                    sample_weight_modes.append(None)
-                else:
-                    if name not in sample_weight_mode:
-                        raise ValueError('Output "' + name +
-                                         '" missing from sample_weight_modes '
-                                         'dictionary')
-                    if sample_weight_mode.get(name) == 'temporal':
-                        weight = K.placeholder(ndim=2,
-                                               name=name + '_sample_weights')
-                        sample_weight_modes.append('temporal')
-                    else:
-                        weight = K.placeholder(ndim=1,
-                                               name=name + '_sample_weights')
+        if self._input_yield_op_tensors:
+            if not all(l is None for l in self.labels):
+                if sample_weight_mode is not None:
+                    # tensor + sample weights not yet implemented
+                    raise NotImplementedError
+                if self.loss_weights is not None:
+                    # tensor + loss weights not yet implemented
+                    raise NotImplementedError
+            sample_weight_mode = 'disabled'
+        if sample_weight_mode is not 'disabled':
+            sample_weights = self._prepare_sample_weights(
+                sample_weight_mode, skip_indices)
+            self.sample_weight_mode = sample_weight_mode
+            sample_weights = []
+            sample_weight_modes = []
+            if isinstance(sample_weight_mode, dict):
+                for name in sample_weight_mode:
+                    if name not in self.output_names:
+                        raise ValueError('Unknown entry in '
+                                         'sample_weight_mode dictionary: "' +
+                                         name + '". '
+                                         'Only expected the following keys: ' +
+                                         str(self.output_names))
+                for i, name in enumerate(self.output_names):
+                    if i in skip_indices:
+                        weight = None
                         sample_weight_modes.append(None)
-                sample_weights.append(weight)
-        elif isinstance(sample_weight_mode, list):
-            if len(sample_weight_mode) != len(self.outputs):
-                raise ValueError('When passing a list as sample_weight_mode, '
-                                 'it should have one entry per model outputs. '
-                                 'The model has ' + str(len(self.outputs)) +
-                                 ' outputs, but you passed '
-                                 'sample_weight_mode=' +
-                                 str(sample_weight_mode))
-            for i in range(len(self.output_names)):
-                if i in skip_indices:
-                    weight = None
-                    sample_weight_modes.append(None)
-                else:
-                    mode = sample_weight_mode[i]
-                    name = self.output_names[i]
-                    if mode == 'temporal':
-                        weight = K.placeholder(ndim=2,
-                                               name=name + '_sample_weights')
-                        sample_weight_modes.append('temporal')
                     else:
-                        weight = K.placeholder(ndim=1,
-                                               name=name + '_sample_weights')
+                        if name not in sample_weight_mode:
+                            raise ValueError('Output "' + name +
+                                             '" missing from sample_weight_modes '
+                                             'dictionary')
+                        if sample_weight_mode.get(name) == 'temporal':
+                            weight = K.placeholder(ndim=2,
+                                                   name=name + '_sample_weights')
+                            sample_weight_modes.append('temporal')
+                        else:
+                            weight = K.placeholder(ndim=1,
+                                                   name=name + '_sample_weights')
+                            sample_weight_modes.append(None)
+                    sample_weights.append(weight)
+            elif isinstance(sample_weight_mode, list):
+                if len(sample_weight_mode) != len(self.outputs):
+                    raise ValueError('When passing a list as sample_weight_mode, '
+                                     'it should have one entry per model outputs. '
+                                     'The model has ' + str(len(self.outputs)) +
+                                     ' outputs, but you passed '
+                                     'sample_weight_mode=' +
+                                     str(sample_weight_mode))
+                for i in range(len(self.output_names)):
+                    if i in skip_indices:
+                        weight = None
                         sample_weight_modes.append(None)
-                sample_weights.append(weight)
-        else:
-            for i, name in enumerate(self.output_names):
-                if i in skip_indices:
-                    sample_weight_modes.append(None)
-                    sample_weights.append(None)
-                else:
-                    if sample_weight_mode == 'temporal':
-                        sample_weights.append(
-                            K.placeholder(ndim=2,
-                                          name=name + '_sample_weights'))
-                        sample_weight_modes.append('temporal')
                     else:
-                        sample_weights.append(
-                            K.placeholder(ndim=1,
-                                          name=name + '_sample_weights'))
+                        mode = sample_weight_mode[i]
+                        name = self.output_names[i]
+                        if mode == 'temporal':
+                            weight = K.placeholder(ndim=2,
+                                                   name=name + '_sample_weights')
+                            sample_weight_modes.append('temporal')
+                        else:
+                            weight = K.placeholder(ndim=1,
+                                                   name=name + '_sample_weights')
+                            sample_weight_modes.append(None)
+                    sample_weights.append(weight)
+            else:
+                for i, name in enumerate(self.output_names):
+                    if i in skip_indices:
                         sample_weight_modes.append(None)
-        self.sample_weight_modes = sample_weight_modes
-        self._feed_sample_weight_modes = []
-        for i in range(len(self.outputs)):
-            if i not in skip_indices:
-                self._feed_sample_weight_modes.append(self.sample_weight_modes[i])
-        return sample_weights
+                        sample_weights.append(None)
+                    else:
+                        if sample_weight_mode == 'temporal':
+                            sample_weights.append(
+                                K.placeholder(ndim=2,
+                                              name=name + '_sample_weights'))
+                            sample_weight_modes.append('temporal')
+                        else:
+                            sample_weights.append(
+                                K.placeholder(ndim=1,
+                                              name=name + '_sample_weights'))
+                            sample_weight_modes.append(None)
+            self.sample_weight_modes = sample_weight_modes
+            self._feed_sample_weight_modes = []
+            for i in range(len(self.outputs)):
+                if i not in skip_indices:
+                    self._feed_sample_weight_modes.append(self.sample_weight_modes[i])
+            return sample_weights
 
     def _make_ins(self, x=None, y=None, sample_weights=None, learning_phase_value=None):
         def add_if_valid(x):
@@ -1595,7 +1596,7 @@ def predict(self, x, batch_size=32, verbose=0):
                                  'Batch size: ' + str(batch_size) + '.')
 
         # Prepare inputs, delegate logic to `_predict_loop`.
-        ins = self._make_ins(x, None, None, [0.])
+        ins = self._make_ins(x, None, [], [0.])
         self.predict_function = self._make_function('predict_function')
         f = self.predict_function
         return self._predict_loop(f, ins,

From fe2916be355db7afc1e12dad50c610d7b46a217f Mon Sep 17 00:00:00 2001
From: Andrew Hundt <ATHundt@gmail.com>
Date: Sat, 17 Jun 2017 04:38:43 -0400
Subject: [PATCH 44/64] mae=>mse

---
 tests/keras/engine/test_training.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/keras/engine/test_training.py b/tests/keras/engine/test_training.py
index df2f2d8cc8f..97e6f3f2729 100644
--- a/tests/keras/engine/test_training.py
+++ b/tests/keras/engine/test_training.py
@@ -633,7 +633,7 @@ def test_model_with_external_loss():
 
     optimizer = 'rmsprop'
     loss = None
-    model.compile(optimizer, loss, metrics=['mae'])
+    model.compile(optimizer, loss, metrics=['mse'])
 
     input_a_np = np.random.random((10, 3))
 

From 4793149f124f92a926cfab2947e94fedcec51d28 Mon Sep 17 00:00:00 2001
From: Andrew Hundt <ATHundt@gmail.com>
Date: Sat, 17 Jun 2017 15:02:02 -0400
Subject: [PATCH 45/64] _prepare_sample_weights initialization & return
 changes, may be buggy

---
 keras/engine/training.py | 154 +++++++++++++++++++--------------------
 1 file changed, 77 insertions(+), 77 deletions(-)

diff --git a/keras/engine/training.py b/keras/engine/training.py
index d72a40fafa6..8d0af2a1859 100644
--- a/keras/engine/training.py
+++ b/keras/engine/training.py
@@ -803,7 +803,8 @@ def compile(self, optimizer, loss, metrics=None, loss_weights=None,
                             ' - expected a list of dicts.')
 
         # Prepare sample weights.
-        self._prepare_sample_weights(sample_weight_mode, skip_indices)
+        sample_weights, sample_weight_mode = self._prepare_sample_weights(
+            sample_weight_mode, skip_indices)
 
         # Prepare targets of model.
         self.targets = []
@@ -837,7 +838,7 @@ def compile(self, optimizer, loss, metrics=None, loss_weights=None,
             y_true = self.targets[i]
             y_pred = self.outputs[i]
             weighted_loss = weighted_losses[i]
-            if sample_weight_mode is 'disabled':
+            if self.sample_weight_mode is 'disabled':
                 sample_weight = None
             else:
                 sample_weight = sample_weights[i]
@@ -911,7 +912,7 @@ def append_metric(layer_num, metric_name, metric_tensor):
 
         # Prepare gradient updates and state updates.
         self.total_loss = total_loss
-        if sample_weight_mode is not 'disabled':
+        if self.sample_weight_mode is not 'disabled':
             self.sample_weights = sample_weights
             self._feed_sample_weights = []
             for i in range(len(self.sample_weights)):
@@ -947,85 +948,84 @@ def _prepare_sample_weights(self, sample_weight_mode, skip_indices):
                 if self.loss_weights is not None:
                     # tensor + loss weights not yet implemented
                     raise NotImplementedError
-            sample_weight_mode = 'disabled'
-        if sample_weight_mode is not 'disabled':
-            sample_weights = self._prepare_sample_weights(
-                sample_weight_mode, skip_indices)
-            self.sample_weight_mode = sample_weight_mode
-            sample_weights = []
-            sample_weight_modes = []
-            if isinstance(sample_weight_mode, dict):
-                for name in sample_weight_mode:
-                    if name not in self.output_names:
-                        raise ValueError('Unknown entry in '
-                                         'sample_weight_mode dictionary: "' +
-                                         name + '". '
-                                         'Only expected the following keys: ' +
-                                         str(self.output_names))
-                for i, name in enumerate(self.output_names):
-                    if i in skip_indices:
-                        weight = None
-                        sample_weight_modes.append(None)
+            self.sample_weight_mode = 'disabled'
+            return [], self.sample_weight_mode
+
+        self.sample_weight_mode = sample_weight_mode
+        sample_weights = []
+        sample_weight_modes = []
+        if isinstance(sample_weight_mode, dict):
+            for name in sample_weight_mode:
+                if name not in self.output_names:
+                    raise ValueError('Unknown entry in '
+                                     'sample_weight_mode dictionary: "' +
+                                     name + '". '
+                                     'Only expected the following keys: ' +
+                                     str(self.output_names))
+            for i, name in enumerate(self.output_names):
+                if i in skip_indices:
+                    weight = None
+                    sample_weight_modes.append(None)
+                else:
+                    if name not in sample_weight_mode:
+                        raise ValueError('Output "' + name +
+                                         '" missing from sample_weight_modes '
+                                         'dictionary')
+                    if sample_weight_mode.get(name) == 'temporal':
+                        weight = K.placeholder(ndim=2,
+                                               name=name + '_sample_weights')
+                        sample_weight_modes.append('temporal')
                     else:
-                        if name not in sample_weight_mode:
-                            raise ValueError('Output "' + name +
-                                             '" missing from sample_weight_modes '
-                                             'dictionary')
-                        if sample_weight_mode.get(name) == 'temporal':
-                            weight = K.placeholder(ndim=2,
-                                                   name=name + '_sample_weights')
-                            sample_weight_modes.append('temporal')
-                        else:
-                            weight = K.placeholder(ndim=1,
-                                                   name=name + '_sample_weights')
-                            sample_weight_modes.append(None)
-                    sample_weights.append(weight)
-            elif isinstance(sample_weight_mode, list):
-                if len(sample_weight_mode) != len(self.outputs):
-                    raise ValueError('When passing a list as sample_weight_mode, '
-                                     'it should have one entry per model outputs. '
-                                     'The model has ' + str(len(self.outputs)) +
-                                     ' outputs, but you passed '
-                                     'sample_weight_mode=' +
-                                     str(sample_weight_mode))
-                for i in range(len(self.output_names)):
-                    if i in skip_indices:
-                        weight = None
+                        weight = K.placeholder(ndim=1,
+                                               name=name + '_sample_weights')
                         sample_weight_modes.append(None)
+                sample_weights.append(weight)
+        elif isinstance(sample_weight_mode, list):
+            if len(sample_weight_mode) != len(self.outputs):
+                raise ValueError('When passing a list as sample_weight_mode, '
+                                 'it should have one entry per model outputs. '
+                                 'The model has ' + str(len(self.outputs)) +
+                                 ' outputs, but you passed '
+                                 'sample_weight_mode=' +
+                                 str(sample_weight_mode))
+            for i in range(len(self.output_names)):
+                if i in skip_indices:
+                    weight = None
+                    sample_weight_modes.append(None)
+                else:
+                    mode = sample_weight_mode[i]
+                    name = self.output_names[i]
+                    if mode == 'temporal':
+                        weight = K.placeholder(ndim=2,
+                                               name=name + '_sample_weights')
+                        sample_weight_modes.append('temporal')
                     else:
-                        mode = sample_weight_mode[i]
-                        name = self.output_names[i]
-                        if mode == 'temporal':
-                            weight = K.placeholder(ndim=2,
-                                                   name=name + '_sample_weights')
-                            sample_weight_modes.append('temporal')
-                        else:
-                            weight = K.placeholder(ndim=1,
-                                                   name=name + '_sample_weights')
-                            sample_weight_modes.append(None)
-                    sample_weights.append(weight)
-            else:
-                for i, name in enumerate(self.output_names):
-                    if i in skip_indices:
+                        weight = K.placeholder(ndim=1,
+                                               name=name + '_sample_weights')
                         sample_weight_modes.append(None)
-                        sample_weights.append(None)
+                sample_weights.append(weight)
+        else:
+            for i, name in enumerate(self.output_names):
+                if i in skip_indices:
+                    sample_weight_modes.append(None)
+                    sample_weights.append(None)
+                else:
+                    if sample_weight_mode == 'temporal':
+                        sample_weights.append(
+                            K.placeholder(ndim=2,
+                                          name=name + '_sample_weights'))
+                        sample_weight_modes.append('temporal')
                     else:
-                        if sample_weight_mode == 'temporal':
-                            sample_weights.append(
-                                K.placeholder(ndim=2,
-                                              name=name + '_sample_weights'))
-                            sample_weight_modes.append('temporal')
-                        else:
-                            sample_weights.append(
-                                K.placeholder(ndim=1,
-                                              name=name + '_sample_weights'))
-                            sample_weight_modes.append(None)
-            self.sample_weight_modes = sample_weight_modes
-            self._feed_sample_weight_modes = []
-            for i in range(len(self.outputs)):
-                if i not in skip_indices:
-                    self._feed_sample_weight_modes.append(self.sample_weight_modes[i])
-            return sample_weights
+                        sample_weights.append(
+                            K.placeholder(ndim=1,
+                                          name=name + '_sample_weights'))
+                        sample_weight_modes.append(None)
+        self.sample_weight_modes = sample_weight_modes
+        self._feed_sample_weight_modes = []
+        for i in range(len(self.outputs)):
+            if i not in skip_indices:
+                self._feed_sample_weight_modes.append(self.sample_weight_modes[i])
+        return sample_weights, sample_weight_mode
 
     def _make_ins(self, x=None, y=None, sample_weights=None, learning_phase_value=None):
         def add_if_valid(x):

From af382e874a452e8d907ae5684f1ed0dc12a84754 Mon Sep 17 00:00:00 2001
From: Andrew Hundt <ATHundt@gmail.com>
Date: Sat, 17 Jun 2017 15:02:34 -0400
Subject: [PATCH 46/64] _make_function implementation changes to better support
 predict_function, may be buggy

---
 keras/engine/training.py | 58 +++++++++++++++++++++++-----------------
 1 file changed, 34 insertions(+), 24 deletions(-)

diff --git a/keras/engine/training.py b/keras/engine/training.py
index 8d0af2a1859..01cf74d8625 100644
--- a/keras/engine/training.py
+++ b/keras/engine/training.py
@@ -1038,39 +1038,49 @@ def add_if_valid(x):
         return ins
 
     def _make_function(self, function_name):
-        if not hasattr(self, function_name):
+        if function_name is 'predict_function':
+            if not hasattr(self, 'predict_function'):
+                self.predict_function = None
+            if self.predict_function is None:
+                if self.uses_learning_phase and not isinstance(K.learning_phase(), int):
+                    inputs = self._feed_inputs + [K.learning_phase()]
+                else:
+                    inputs = self._feed_inputs
+                outputs = self.outputs
+        elif not hasattr(self, function_name):
             raise RuntimeError('You must compile your model before using it.')
+
         function = getattr(self, function_name)
         if function is not None:
             return function
-        else:
-            inputs = self._feed_inputs + self._feed_targets
-            if self.sample_weight_mode is not 'disabled':
+        elif function_name is not 'predict_function':
+            inputs = self._feed_inputs
+            if hasattr(self, '_feed_targets'):
+                inputs += self._feed_targets
+            if (hasattr(self, 'sample_weight_mode') and
+                    self.sample_weight_mode is not 'disabled'):
                 inputs += self._feed_sample_weights
             if self.uses_learning_phase and not isinstance(K.learning_phase(), int):
                 inputs += [K.learning_phase()]
 
-            if function_name is 'train_function':
-                training_updates = self.optimizer.get_updates(
-                    self._collected_trainable_weights,
-                    self.constraints,
-                    self.total_loss)
-                updates = self.updates + training_updates
-            else:
-                updates = self.state_updates
+            outputs = [self.total_loss] + self.metrics_tensors
 
-            if function_name is 'predict_function':
-                outputs = self.outputs
-                kwargs = getattr(self, '_function_kwargs', {})
-            else:
-                outputs = [self.total_loss] + self.metrics_tensors
-                kwargs = self._function_kwargs
-            # Gets loss and metrics. Updates weights at each call.
-            return K.function(inputs,
-                              outputs,
-                              updates=updates,
-                              name=function_name,
-                              **kwargs)
+        if function_name is 'train_function':
+            training_updates = self.optimizer.get_updates(
+                self._collected_trainable_weights,
+                self.constraints,
+                self.total_loss)
+            updates = self.updates + training_updates
+        else:
+            updates = self.state_updates
+
+        kwargs = getattr(self, '_function_kwargs', {})
+        # Gets loss and metrics. Updates weights at each call.
+        return K.function(inputs,
+                          outputs,
+                          updates=updates,
+                          name=function_name,
+                          **kwargs)
 
     def _fit_loop(self, f, ins, out_labels=None, batch_size=32,
                   epochs=100, verbose=1, callbacks=None,

From 1e6b9fbf5b6bdb2f00c7d16f93aea55122a6d0f6 Mon Sep 17 00:00:00 2001
From: Andrew Hundt <ATHundt@gmail.com>
Date: Sat, 17 Jun 2017 16:08:02 -0400
Subject: [PATCH 47/64] tensorflow_backend.py Function handles varying length
 inputs

---
 keras/backend/tensorflow_backend.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/keras/backend/tensorflow_backend.py b/keras/backend/tensorflow_backend.py
index 69bba5fe8db..653454ec896 100644
--- a/keras/backend/tensorflow_backend.py
+++ b/keras/backend/tensorflow_backend.py
@@ -10,6 +10,7 @@
 import inspect
 import numpy as np
 import os
+import itertools
 
 from .common import floatx
 from .common import _EPSILON
@@ -2289,7 +2290,7 @@ def __call__(self, inputs):
         self.current_feed_dict = {} if self.feed_dict is None else self.feed_dict
         self.feed_to_fetch_count = 0
         self.current_fetches = self.outputs + [self.updates_op]
-        for tensor, value in zip(self.inputs, inputs):
+        for tensor, value in itertools.izip_longest(self.inputs, inputs):
             if is_sparse(tensor):
                 sparse_coo = value.tocoo()
                 indices = np.concatenate((np.expand_dims(sparse_coo.row, 1),

From ea9ffbafb4598e7547c98a2abcd504e0fe518c68 Mon Sep 17 00:00:00 2001
From: Andrew Hundt <ATHundt@gmail.com>
Date: Sat, 17 Jun 2017 16:27:42 -0400
Subject: [PATCH 48/64] TFRecord predict() passes again

---
 keras/engine/training.py | 11 ++++++++---
 1 file changed, 8 insertions(+), 3 deletions(-)

diff --git a/keras/engine/training.py b/keras/engine/training.py
index 01cf74d8625..56b6bd673d6 100644
--- a/keras/engine/training.py
+++ b/keras/engine/training.py
@@ -1592,6 +1592,12 @@ def predict(self, x, batch_size=32, verbose=0):
                 or in case a stateful model receives a number of samples
                 that is not a multiple of the batch size.
         """
+
+        if self._input_yield_op_tensors and x is not None:
+            raise ValueError('Input data can only be supplied via '
+                             'the input data parameter `x` in predict() or '
+                             'tensors in the constructor of `Model()`, '
+                             'not both.')
         # Validate user data.
         x = _standardize_input_data(x, self._feed_input_names,
                                     self._feed_input_shapes,
@@ -1604,9 +1610,8 @@ def predict(self, x, batch_size=32, verbose=0):
                                  'divided by the batch size. Found: ' +
                                  str(x[0].shape[0]) + ' samples. '
                                  'Batch size: ' + str(batch_size) + '.')
-
         # Prepare inputs, delegate logic to `_predict_loop`.
-        ins = self._make_ins(x, None, [], [0.])
+        ins = self._make_ins(x, [], [], [0.])
         self.predict_function = self._make_function('predict_function')
         f = self.predict_function
         return self._predict_loop(f, ins,
@@ -1710,7 +1715,7 @@ def predict_on_batch(self, x):
         """
         x = _standardize_input_data(x, self._feed_input_names,
                                     self._feed_input_shapes)
-        ins = self._make_ins(x, None, [], [0.])
+        ins = self._make_ins(x, [], [], [0.])
         self.predict_function = self._make_function('predict_function')
         outputs = self.predict_function(ins)
         if len(outputs) == 1:

From 06b71c44c6d71e6d6760143ea40e3466b3e4fa04 Mon Sep 17 00:00:00 2001
From: Andrew Hundt <ATHundt@gmail.com>
Date: Sun, 18 Jun 2017 13:03:44 -0400
Subject: [PATCH 49/64] tensorflow_backend Function __call__ izip_longest
 explicit fillvalue

---
 keras/backend/tensorflow_backend.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/keras/backend/tensorflow_backend.py b/keras/backend/tensorflow_backend.py
index 653454ec896..7150ff886a9 100644
--- a/keras/backend/tensorflow_backend.py
+++ b/keras/backend/tensorflow_backend.py
@@ -2290,7 +2290,7 @@ def __call__(self, inputs):
         self.current_feed_dict = {} if self.feed_dict is None else self.feed_dict
         self.feed_to_fetch_count = 0
         self.current_fetches = self.outputs + [self.updates_op]
-        for tensor, value in itertools.izip_longest(self.inputs, inputs):
+        for tensor, value in itertools.izip_longest(self.inputs, inputs, fillvalue=None):
             if is_sparse(tensor):
                 sparse_coo = value.tocoo()
                 indices = np.concatenate((np.expand_dims(sparse_coo.row, 1),

From 325acc1ccaf4a3855368c64932d7f94c8d865a37 Mon Sep 17 00:00:00 2001
From: Andrew Hundt <ATHundt@gmail.com>
Date: Sun, 18 Jun 2017 14:19:57 -0400
Subject: [PATCH 50/64] _make_function call & list bugs fixed

---
 keras/engine/training.py   | 8 ++++----
 keras/models.py            | 4 ++--
 tests/test_model_saving.py | 2 +-
 3 files changed, 7 insertions(+), 7 deletions(-)

diff --git a/keras/engine/training.py b/keras/engine/training.py
index 56b6bd673d6..1c9883dfbcd 100644
--- a/keras/engine/training.py
+++ b/keras/engine/training.py
@@ -1054,14 +1054,14 @@ def _make_function(self, function_name):
         if function is not None:
             return function
         elif function_name is not 'predict_function':
-            inputs = self._feed_inputs
+            inputs = [] + self._feed_inputs
             if hasattr(self, '_feed_targets'):
-                inputs += self._feed_targets
+                inputs = inputs + self._feed_targets
             if (hasattr(self, 'sample_weight_mode') and
                     self.sample_weight_mode is not 'disabled'):
-                inputs += self._feed_sample_weights
+                inputs = inputs + self._feed_sample_weights
             if self.uses_learning_phase and not isinstance(K.learning_phase(), int):
-                inputs += [K.learning_phase()]
+                inputs = inputs + [K.learning_phase()]
 
             outputs = [self.total_loss] + self.metrics_tensors
 
diff --git a/keras/models.py b/keras/models.py
index 2d826eed990..6fb1828e072 100644
--- a/keras/models.py
+++ b/keras/models.py
@@ -280,9 +280,9 @@ def convert_custom_objects(obj):
         if 'optimizer_weights' in f:
             # Build train function (to get weight updates).
             if isinstance(model, Sequential):
-                model.model._make_train_function()
+                model.model.train_function = model.model._make_function('train_function')
             else:
-                model._make_train_function()
+                model.train_function = model._make_function('train_function')
             optimizer_weights_group = f['optimizer_weights']
             optimizer_weight_names = [n.decode('utf8') for n in
                                       optimizer_weights_group.attrs['weight_names']]
diff --git a/tests/test_model_saving.py b/tests/test_model_saving.py
index 13c65f59d00..3f6eea544b1 100644
--- a/tests/test_model_saving.py
+++ b/tests/test_model_saving.py
@@ -150,7 +150,7 @@ def test_saving_right_after_compilation():
     model.add(Dense(2, input_shape=(3,)))
     model.add(Dense(3))
     model.compile(loss='mse', optimizer='sgd', metrics=['acc'])
-    model.model._make_train_function()
+    model.model._make_function('train_function')
 
     _, fname = tempfile.mkstemp('.h5')
     save_model(model, fname)

From 974886ea1b475c62de9a7000a7225476736ed2da Mon Sep 17 00:00:00 2001
From: Andrew Hundt <ATHundt@gmail.com>
Date: Sun, 18 Jun 2017 16:38:16 -0400
Subject: [PATCH 51/64] is_placeholder and standardize user data error checks
 fixed

---
 keras/backend/tensorflow_backend.py | 26 +----------------
 keras/backend/theano_backend.py     |  1 +
 keras/engine/topology.py            |  7 +++--
 keras/engine/training.py            | 44 ++++++++++++++++-------------
 4 files changed, 31 insertions(+), 47 deletions(-)

diff --git a/keras/backend/tensorflow_backend.py b/keras/backend/tensorflow_backend.py
index 7150ff886a9..8da8cc0707c 100644
--- a/keras/backend/tensorflow_backend.py
+++ b/keras/backend/tensorflow_backend.py
@@ -15,6 +15,7 @@
 from .common import floatx
 from .common import _EPSILON
 from .common import image_data_format
+from .common import is_placeholder
 
 # Legacy functions
 from .common import set_image_dim_ordering
@@ -281,31 +282,6 @@ def to_dense(tensor):
 name_scope = tf.name_scope
 
 
-def is_placeholder(tensor):
-    """Returns whether a tensor is a placeholder.
-
-    # Arguments
-        tensor: A tensor instance.
-
-    # Returns
-        A boolean.
-
-    # Example
-    ```python
-        >>> from keras import backend as K
-        >>> a = K.placeholder((2, 2), sparse=False)
-        >>> print(K.is_placeholder(a))
-        True
-    ```
-    """
-    try:
-        if isinstance(tensor, tf.Variable):
-            return True
-        return tensor._is_placeholder
-    except AttributeError:
-        return False
-
-
 def variable(value, dtype=None, name=None):
     """Instantiates a variable and returns it.
 
diff --git a/keras/backend/theano_backend.py b/keras/backend/theano_backend.py
index aa73641bbf5..9d25fd22c7c 100644
--- a/keras/backend/theano_backend.py
+++ b/keras/backend/theano_backend.py
@@ -17,6 +17,7 @@
 import inspect
 import numpy as np
 from .common import _FLOATX, floatx, _EPSILON, image_data_format
+from .common import is_placeholder
 # Legacy functions
 from .common import set_image_dim_ordering, image_dim_ordering
 
diff --git a/keras/engine/topology.py b/keras/engine/topology.py
index e7b0ad57dfa..14dc3c0ef53 100644
--- a/keras/engine/topology.py
+++ b/keras/engine/topology.py
@@ -1639,9 +1639,10 @@ def __init__(self, inputs, outputs, labels=None, name=None):
                                                    i,
                                                    layer.__class__.__name__))
             self.input_names.append(layer.name)
-            self._feed_input_names.append(layer.name)
-            self._feed_inputs.append(layer.input)
-            self._feed_input_shapes.append(self.inputs[i]._keras_shape)
+            if K.is_placeholder(layer):
+                self._feed_input_names.append(layer.name)
+                self._feed_inputs.append(layer.input)
+                self._feed_input_shapes.append(self.inputs[i]._keras_shape)
         for layer in self.output_layers:
             self.output_names.append(layer.name)
 
diff --git a/keras/engine/training.py b/keras/engine/training.py
index 1c9883dfbcd..147f4b07898 100644
--- a/keras/engine/training.py
+++ b/keras/engine/training.py
@@ -27,7 +27,8 @@
 
 def _standardize_input_data(data, names, shapes=None,
                             check_batch_axis=True,
-                            exception_prefix=''):
+                            exception_prefix='',
+                            tensors=None):
     """Normalizes inputs and targets provided by users.
 
     Users may pass data as a list of arrays, dictionary of arrays,
@@ -50,10 +51,14 @@ def _standardize_input_data(data, names, shapes=None,
     # Raises
         ValueError: in case of improperly formatted user-provided data.
     """
+    if tensors is None:
+        tensors = []
     if not names:
         return []
     if data is None:
         return [None for _ in range(len(names))]
+    num_tensors = sum(x is not None for x in tensors)
+    expected_names = len(names) - num_tensors
     if isinstance(data, dict):
         arrays = []
         for name in names:
@@ -63,18 +68,22 @@ def _standardize_input_data(data, names, shapes=None,
                                  str(names))
             arrays.append(data[name])
     elif isinstance(data, list):
-        if len(data) != len(names):
+        if len(data) != expected_names:
             if data and hasattr(data[0], 'shape'):
+                if type(data).__module__ == np.__name__:
+                    datastr = ('the following list of ' + str(len(data)) +
+                               ' arrays: ' + str(data)[:200] +
+                               '...')
+                else:
+                    datastr = ('data with type' +
+                               type(data).__name__ + '...')
                 raise ValueError('Error when checking model ' +
                                  exception_prefix +
                                  ': the list of Numpy arrays '
                                  'that you are passing to your model '
                                  'is not the size the model expected. '
-                                 'Expected to see ' + str(len(names)) +
-                                 ' arrays but instead got '
-                                 'the following list of ' + str(len(data)) +
-                                 ' arrays: ' + str(data)[:200] +
-                                 '...')
+                                 'Expected to see ' + str(expected_names) +
+                                 ' arrays but instead got ' + datastr)
             else:
                 if len(names) == 1:
                     data = [np.asarray(data)]
@@ -85,7 +94,7 @@ def _standardize_input_data(data, names, shapes=None,
                         ': you are passing a list as '
                         'input to your model, '
                         'but the model expects '
-                        'a list of ' + str(len(names)) +
+                        'a list of ' + str(expected_names) +
                         ' Numpy arrays instead. '
                         'The list you passed was: ' +
                         str(data)[:200])
@@ -97,17 +106,17 @@ def _standardize_input_data(data, names, shapes=None,
                             ': data should be a Numpy array, '
                             'or list/dict of Numpy arrays. '
                             'Found: ' + str(data)[:200] + '...')
-        if len(names) > 1:
+        if expected_names > 1:
             # Case: model expects multiple inputs but only received
             # a single Numpy array.
-            raise ValueError('The model expects ' + str(len(names)) +
+            raise ValueError('The model expects ' + str(expected_names) +
                              exception_prefix +
                              ' arrays, but only received one array. '
                              'Found: array with shape ' + str(data.shape))
         arrays = [data]
 
     # Make arrays at least 2D.
-    for i in range(len(names)):
+    for i in range(expected_names):
         array = arrays[i]
         if len(array.shape) == 1:
             array = np.expand_dims(array, 1)
@@ -115,7 +124,7 @@ def _standardize_input_data(data, names, shapes=None,
 
     # Check shapes compatibility.
     if shapes:
-        for i in range(len(names)):
+        for i in range(expected_names):
             if shapes[i] is None:
                 continue
             array = arrays[i]
@@ -1339,11 +1348,13 @@ def _standardize_user_data(self, x, y,
         x = _standardize_input_data(x, self._feed_input_names,
                                     self._feed_input_shapes,
                                     check_batch_axis=False,
-                                    exception_prefix='input')
+                                    exception_prefix='input',
+                                    tensors=self._input_yield_op_tensors)
         y = _standardize_input_data(y, self._feed_output_names,
                                     output_shapes,
                                     check_batch_axis=False,
-                                    exception_prefix='target')
+                                    exception_prefix='target',
+                                    tensors=self.labels)
         class_weights = _standardize_class_weights(class_weight,
                                                    self._feed_output_names)
         if self.sample_weight_mode is not 'disabled':
@@ -1593,11 +1604,6 @@ def predict(self, x, batch_size=32, verbose=0):
                 that is not a multiple of the batch size.
         """
 
-        if self._input_yield_op_tensors and x is not None:
-            raise ValueError('Input data can only be supplied via '
-                             'the input data parameter `x` in predict() or '
-                             'tensors in the constructor of `Model()`, '
-                             'not both.')
         # Validate user data.
         x = _standardize_input_data(x, self._feed_input_names,
                                     self._feed_input_shapes,

From 3c7dc09644f5f3970c735ff1d80495e9c903e64a Mon Sep 17 00:00:00 2001
From: Andrew Hundt <ATHundt@gmail.com>
Date: Sun, 18 Jun 2017 16:47:30 -0400
Subject: [PATCH 52/64] topology.py add missing docstring

---
 keras/engine/topology.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/keras/engine/topology.py b/keras/engine/topology.py
index 14dc3c0ef53..d82c2f7e13e 100644
--- a/keras/engine/topology.py
+++ b/keras/engine/topology.py
@@ -97,6 +97,7 @@ class Node(object):
         output_shapes: list of output shape tuples.
         arguments: dictionary of keyword arguments that were passed to the
             `call` method of the layer at the call that created the node.
+        is_placeholder: Specifies if the Node represents a placeholder.
 
     `node_indices` and `tensor_indices` are basically fine-grained coordinates
     describing the origin of the `input_tensors`, verifying the following:

From 20077cb1c2b06d23b5f0312bbea77c8f93418fb8 Mon Sep 17 00:00:00 2001
From: Andrew Hundt <ATHundt@gmail.com>
Date: Sun, 18 Jun 2017 17:38:23 -0400
Subject: [PATCH 53/64] test_multiprocessing.py fix test which actually throws
 two exceptions

---
 tests/test_multiprocessing.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/tests/test_multiprocessing.py b/tests/test_multiprocessing.py
index b560bd4b813..06cd7280b4e 100644
--- a/tests/test_multiprocessing.py
+++ b/tests/test_multiprocessing.py
@@ -284,13 +284,13 @@ def custom_generator():
     model.add(Dense(1, input_shape=(2, )))
     model.compile(loss='mse', optimizer='adadelta')
 
-    with pytest.raises(ValueError):
+    with pytest.raises((ValueError, Exception)):
         model.predict_generator(
             custom_generator(), good_batches + 1, 1,
             workers=4, pickle_safe=True,
         )
 
-    with pytest.raises(ValueError):
+    with pytest.raises((ValueError, Exception)):
         model.predict_generator(
             custom_generator(), good_batches + 1, 1,
             pickle_safe=False,

From 9dd1ea9b8fc6189ba9475225f17ba4dbb74df95e Mon Sep 17 00:00:00 2001
From: Andrew Hundt <ATHundt@gmail.com>
Date: Sun, 18 Jun 2017 17:53:09 -0400
Subject: [PATCH 54/64] test_training.py remove extraneous try/except

---
 tests/keras/engine/test_training.py | 24 +++++++++---------------
 1 file changed, 9 insertions(+), 15 deletions(-)

diff --git a/tests/keras/engine/test_training.py b/tests/keras/engine/test_training.py
index 0daa2021ebf..29f29fd565b 100644
--- a/tests/keras/engine/test_training.py
+++ b/tests/keras/engine/test_training.py
@@ -642,13 +642,10 @@ def replace(filename):
     optimizer = 'rmsprop'
     loss = 'mse'
     loss_weights = [1.]
-    try:
-        with pytest.raises(NotImplementedError) as exc:
-            model.compile(optimizer, loss, metrics=['mean_squared_error'],
-                          loss_weights=loss_weights,
-                          sample_weight_mode=None)
-    except NotImplementedError:
-        print 'Feature not implemented'
+    with pytest.raises(NotImplementedError) as exc:
+        model.compile(optimizer, loss, metrics=['mean_squared_error'],
+                      loss_weights=loss_weights,
+                      sample_weight_mode=None)
 
     model.compile(optimizer, loss, metrics=['mean_squared_error'],
                   sample_weight_mode=None)
@@ -656,14 +653,11 @@ def replace(filename):
     # test fit
     out = model.fit(None,
                     None, epochs=1, batch_size=10)
-    try:
-        with pytest.raises(ValueError) as exc:
-            out = model.fit(output_a_np,
-                            [y_train_in_out],
-                            epochs=1,
-                            batch_size=10)
-    except ValueError:
-        print 'cannot mix input tensors and numpy arrays'
+    with pytest.raises(ValueError) as exc:
+        out = model.fit(output_a_np,
+                        [y_train_in_out],
+                        epochs=1,
+                        batch_size=10)
 
     os.remove('input_a.tfrecord')
 

From c37b82c35da6afcbf3ad32783a7ca95f74595c4a Mon Sep 17 00:00:00 2001
From: Andrew Hundt <ATHundt@gmail.com>
Date: Sun, 18 Jun 2017 18:02:05 -0400
Subject: [PATCH 55/64] test_training.py tfrecord test of predict_on_batch,
 evaluate, predict

---
 tests/keras/engine/test_training.py | 14 ++++++++++++++
 1 file changed, 14 insertions(+)

diff --git a/tests/keras/engine/test_training.py b/tests/keras/engine/test_training.py
index 29f29fd565b..4c809e1cb8a 100644
--- a/tests/keras/engine/test_training.py
+++ b/tests/keras/engine/test_training.py
@@ -649,6 +649,7 @@ def replace(filename):
 
     model.compile(optimizer, loss, metrics=['mean_squared_error'],
                   sample_weight_mode=None)
+    out = model.predict_on_batch(None)
     out = model.train_on_batch(None, None)
     # test fit
     out = model.fit(None,
@@ -659,6 +660,19 @@ def replace(filename):
                         epochs=1,
                         batch_size=10)
 
+    # test evaluate
+    out = model.evaluate(None,
+                         None,
+                         batch_size=10)
+    with pytest.raises(ValueError) as exc:
+        out = model.evaluate(input_a_np,
+                             [output_a_np,
+                              output_b_tf],
+                             batch_size=10)
+
+    # test predict
+    out = model.predict(None, batch_size=10)
+
     os.remove('input_a.tfrecord')
 
 

From 75fbfbbb4af943b46169935f6fa384cdbe9f9e67 Mon Sep 17 00:00:00 2001
From: Andrew Hundt <ATHundt@gmail.com>
Date: Sun, 18 Jun 2017 18:05:52 -0400
Subject: [PATCH 56/64] mnist_tfrecord.py _is_placeholder workaround no longer
 required.

---
 examples/mnist_tfrecord.py | 2 --
 1 file changed, 2 deletions(-)

diff --git a/examples/mnist_tfrecord.py b/examples/mnist_tfrecord.py
index 506817ffc13..f2bf556e12e 100644
--- a/examples/mnist_tfrecord.py
+++ b/examples/mnist_tfrecord.py
@@ -146,8 +146,6 @@ def cnn_layers(x_train_input):
 def create_cnn_model(x_train_batch, y_train_batch, x_batch_shape, y_batch_shape):
     x_train_input = Input(tensor=x_train_batch, batch_shape=x_batch_shape)
     x_train_out = cnn_layers(x_train_input)
-    # Workaround until _is_placeholder can be deduced automatically
-    x_train_out._is_placeholder = False
     y_train_in_out = Input(tensor=y_train_batch, batch_shape=y_batch_shape, name='y_labels')
     return Model(inputs=[x_train_input], outputs=[x_train_out], labels=[y_train_in_out])
 

From 6de4e83d44dd050779f6a1f1de57d7ec97ba2f27 Mon Sep 17 00:00:00 2001
From: Andrew Hundt <ATHundt@gmail.com>
Date: Sun, 18 Jun 2017 18:54:17 -0400
Subject: [PATCH 57/64] tensorflow_backend.py py 2+3 compatibility: from
 moves.six import zip_longest

---
 keras/backend/tensorflow_backend.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/keras/backend/tensorflow_backend.py b/keras/backend/tensorflow_backend.py
index 9c1e82b61c1..51b41f6996b 100644
--- a/keras/backend/tensorflow_backend.py
+++ b/keras/backend/tensorflow_backend.py
@@ -10,7 +10,7 @@
 import inspect
 import numpy as np
 import os
-import itertools
+from six.moves import zip_longest
 
 from .common import floatx
 from .common import _EPSILON
@@ -2264,7 +2264,7 @@ def __call__(self, inputs):
         self.current_feed_dict = {} if self.feed_dict is None else self.feed_dict
         self.feed_to_fetch_count = 0
         self.current_fetches = self.outputs + [self.updates_op]
-        for tensor, value in itertools.izip_longest(self.inputs, inputs, fillvalue=None):
+        for tensor, value in zip_longest(self.inputs, inputs, fillvalue=None):
             if is_sparse(tensor):
                 sparse_coo = value.tocoo()
                 indices = np.concatenate((np.expand_dims(sparse_coo.row, 1),

From 3d756492b25d986b051a399f6615cf5c2ee8135f Mon Sep 17 00:00:00 2001
From: Andrew Hundt <ATHundt@gmail.com>
Date: Sun, 18 Jun 2017 21:42:40 -0400
Subject: [PATCH 58/64] mnist_tfrecord.py add parens to print

---
 examples/mnist_tfrecord.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/examples/mnist_tfrecord.py b/examples/mnist_tfrecord.py
index f2bf556e12e..966d1aa54d0 100644
--- a/examples/mnist_tfrecord.py
+++ b/examples/mnist_tfrecord.py
@@ -65,7 +65,7 @@ def _bytes_feature(value):
             writer.write(example.SerializeToString())
         writer.close()
     else:
-        print 'tfrecord %s already exists' % filename
+        print('tfrecord %s already exists' % filename)
 
 
 def read_and_decode_recordinput(tf_glob, one_hot=True, classes=None, is_train=None, batch_shape=[1000, 28, 28, 1]):
@@ -210,4 +210,4 @@ def create_cnn_model(x_train_batch, y_train_batch, x_batch_shape, y_batch_shape)
 test_model.summary()
 
 loss, acc = test_model.evaluate(X_test, np_utils.to_categorical(y_test), classes)
-print '\nTest accuracy: {0}'.format(acc)
+print('\nTest accuracy: {0}'.format(acc))

From 0a1606f44b2de4528028af4794ae12b2fedf950c Mon Sep 17 00:00:00 2001
From: Andrew Hundt <ATHundt@gmail.com>
Date: Sun, 18 Jun 2017 23:07:25 -0400
Subject: [PATCH 59/64] test_training.py add parenthesis to print, plus an
 extra error check in the tensorflow_backend.py

---
 keras/backend/tensorflow_backend.py | 18 ++++++++++++++++++
 tests/keras/engine/test_training.py |  6 +++---
 2 files changed, 21 insertions(+), 3 deletions(-)

diff --git a/keras/backend/tensorflow_backend.py b/keras/backend/tensorflow_backend.py
index 51b41f6996b..4f109f67ea0 100644
--- a/keras/backend/tensorflow_backend.py
+++ b/keras/backend/tensorflow_backend.py
@@ -2241,6 +2241,7 @@ def __init__(self, inputs, outputs, updates=None, name=None,
         if not isinstance(updates, (list, tuple)):
             raise TypeError('`updates` in a TensorFlow backend function '
                             'should be a list or tuple.')
+        # self.inputs holds tf Tensor objects
         self.inputs = list(inputs)
         self.outputs = list(outputs)
         self.fetches = fetches
@@ -2259,17 +2260,34 @@ def __init__(self, inputs, outputs, updates=None, name=None,
         self.session_kwargs = session_kwargs
 
     def __call__(self, inputs):
+        """Run the TensorFlow session
+
+        # Arguments
+            inputs: Data and values that will go to the feed_dict of Session.run()
+                if it is associated with a tensor, if it is None the tensor will
+                be added to the fetches parameter of Session.run().
+        """
         if not isinstance(inputs, (list, tuple)):
             raise TypeError('`inputs` should be a list or tuple.')
         self.current_feed_dict = {} if self.feed_dict is None else self.feed_dict
         self.feed_to_fetch_count = 0
         self.current_fetches = self.outputs + [self.updates_op]
+        # self.inputs contains tf tensors, inputs contains feed_dict data.
         for tensor, value in zip_longest(self.inputs, inputs, fillvalue=None):
+            if tensor is None and value is None:
+                continue
+            elif tensor is None and value is not None:
+                raise ValueError('A tensor containing None '
+                                 'was tied to value ' + str(value) +
+                                 'so Session.run() cannot execute, '
+                                 'please check your data and Model.')
+
             if is_sparse(tensor):
                 sparse_coo = value.tocoo()
                 indices = np.concatenate((np.expand_dims(sparse_coo.row, 1),
                                           np.expand_dims(sparse_coo.col, 1)), 1)
                 value = (indices, sparse_coo.data, sparse_coo.shape)
+
             if value is None and tensor is not None:
                 self.feed_to_fetch_count += 1
                 self.current_fetches.append(tensor)
diff --git a/tests/keras/engine/test_training.py b/tests/keras/engine/test_training.py
index 4c809e1cb8a..30625e230ec 100644
--- a/tests/keras/engine/test_training.py
+++ b/tests/keras/engine/test_training.py
@@ -552,11 +552,11 @@ def _bytes_feature(value):
                 writer.write(example.SerializeToString())
             writer.close()
         else:
-            print 'tfrecord %s already exists' % filename
+            print('tfrecord %s already exists' % filename)
 
     def read_and_decode_recordinput(tf_glob, one_hot=True, classes=None, is_train=None, batch_shape=[10, 3, 3, 1]):
         """ Return tensor to read from TFRecord """
-        print 'Creating graph for loading TFRecords...'
+        print('Creating graph for loading TFRecords...')
         with tf.variable_scope("TFRecords"):
             record_input = data_flow_ops.RecordInput(tf_glob, batch_size=batch_shape[0])
             records_op = record_input.get_yield_op()
@@ -596,7 +596,7 @@ def read_and_decode_recordinput(tf_glob, one_hot=True, classes=None, is_train=No
 
     def replace(filename):
         if os.path.isfile(filename):
-            print '%s already exists, replacing...' % filename
+            print('%s already exists, replacing...' % filename)
             os.remove(filename)
 
     batch_size = 10

From b7d44a54ffee3391135abd3a3566a9c0d20a2fa8 Mon Sep 17 00:00:00 2001
From: Andrew Hundt <ATHundt@gmail.com>
Date: Mon, 19 Jun 2017 00:45:52 -0400
Subject: [PATCH 60/64] wrappers_test.py fix tolerance in def
 test_TimeDistributed_learning_phase()
 https://github.com/fchollet/keras/commit/a625fcde5cb6a1a170a51690d21b575c0e7884c1#diff-a60adf725df8a5eed11441fb09ae7fb0R98

---
 tests/keras/layers/wrappers_test.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/keras/layers/wrappers_test.py b/tests/keras/layers/wrappers_test.py
index c401040cec4..7986b843b48 100644
--- a/tests/keras/layers/wrappers_test.py
+++ b/tests/keras/layers/wrappers_test.py
@@ -97,7 +97,7 @@ def test_TimeDistributed_learning_phase():
     y = wrappers.TimeDistributed(core.Dropout(.999))(x, training=True)
     model = Model(x, y)
     y = model.predict(np.random.random((10, 3, 2)))
-    assert_allclose(0., y, atol=1e-2)
+    assert_allclose(0., y, atol=1e-1, rtol=1e-1)
 
 
 @keras_test

From 39d3ebab621cd7bdd36ed9317c13385c33202f39 Mon Sep 17 00:00:00 2001
From: Andrew Hundt <ATHundt@gmail.com>
Date: Mon, 19 Jun 2017 20:41:37 -0400
Subject: [PATCH 61/64] test_training.py Input yield op tests

---
 tests/keras/engine/test_training.py | 158 ++++++++++++++++++++++++++++
 1 file changed, 158 insertions(+)

diff --git a/tests/keras/engine/test_training.py b/tests/keras/engine/test_training.py
index f8aafd132ab..35e464032ec 100644
--- a/tests/keras/engine/test_training.py
+++ b/tests/keras/engine/test_training.py
@@ -5,6 +5,7 @@
 import scipy.sparse as sparse
 
 from keras.layers import Dense, Dropout
+from keras.layers import Flatten
 from keras.engine.topology import Input
 from keras.engine.training import Model, _check_loss_and_target_compatibility
 from keras.models import Sequential
@@ -515,6 +516,163 @@ def test_model_with_input_feed_tensor():
     assert out.shape == (10, 4)
 
 
+@pytest.mark.skipif(K.backend() != 'tensorflow', reason='Requires TF backend')
+@keras_test
+def test_model_with_input_yield_op():
+    """We test building a model with a RecordInput as input.
+    We should be able to call fit, evaluate, predict,
+    by only passing them data for the placeholder inputs
+    in the model.
+    """
+    import tensorflow as tf
+    from tensorflow.python.lib.io import tf_record
+    from tensorflow.python.ops import data_flow_ops
+    from tensorflow.python.platform import test
+
+    def images_to_tfrecord(images, labels, filename):
+
+        def _int64_feature(value):
+            return tf.train.Feature(int64_list=tf.train.Int64List(value=[value]))
+
+        def _bytes_feature(value):
+            return tf.train.Feature(bytes_list=tf.train.BytesList(value=[value]))
+
+        """ Save data into TFRecord """
+        if not os.path.isfile(filename):
+            num_examples = images.shape[0]
+
+            rows = images.shape[1]
+            cols = images.shape[2]
+            depth = images.shape[3]
+
+            print('Writing', filename)
+            writer = tf.python_io.TFRecordWriter(filename)
+            for index in range(num_examples):
+                image_raw = images[index].tostring()
+                example = tf.train.Example(features=tf.train.Features(feature={
+                    'height': _int64_feature(rows),
+                    'width': _int64_feature(cols),
+                    'depth': _int64_feature(depth),
+                    'label': _int64_feature(int(labels[index])),
+                    'image_raw': _bytes_feature(image_raw)}))
+                writer.write(example.SerializeToString())
+            writer.close()
+        else:
+            print('tfrecord %s already exists' % filename)
+
+    def read_and_decode_recordinput(tf_glob, one_hot=True, classes=None, is_train=None, batch_shape=[10, 3, 3, 1]):
+        """ Return tensor to read from TFRecord """
+        print('Creating graph for loading TFRecords...')
+        with tf.variable_scope("TFRecords"):
+            record_input = data_flow_ops.RecordInput(tf_glob, batch_size=batch_shape[0])
+            records_op = record_input.get_yield_op()
+            records_op = tf.split(records_op, batch_shape[0], 0)
+            records_op = [tf.reshape(record, []) for record in records_op]
+
+            images = []
+            labels = []
+            for i, serialized_example in enumerate(records_op):
+                with tf.variable_scope("parse_images", reuse=True):
+                    features = tf.parse_single_example(
+                        serialized_example,
+                        features={
+                            'label': tf.FixedLenFeature([], tf.int64),
+                            'image_raw': tf.FixedLenFeature([], tf.string),
+                        })
+                    img = tf.decode_raw(features['image_raw'], tf.uint8)
+                    img.set_shape(batch_shape[1] * batch_shape[2])
+                    img = tf.reshape(img, [1] + batch_shape[1:])
+
+                    img = tf.cast(img, tf.float32) * (1. / 255) - 0.5
+
+                    label = tf.cast(features['label'], tf.int32)
+                    if one_hot and classes:
+                        label = tf.one_hot(label, classes)
+
+                    images.append(img)
+                    labels.append(label)
+
+            images = tf.parallel_stack(images, 0)
+            labels = tf.parallel_stack(labels, 0)
+            images = tf.cast(images, tf.float32)
+
+            images = tf.reshape(images, shape=batch_shape)
+
+            return images, labels
+
+    def replace(filename):
+        if os.path.isfile(filename):
+            print('%s already exists, replacing...' % filename)
+            os.remove(filename)
+
+    batch_size = 10
+    input_rows = 3
+    cols = 3
+    depth = 1
+    classes = 2
+    img_batch_shape = [batch_size, input_rows, cols, depth]
+    label_batch_shape = [batch_size, classes]
+    input_a_np = np.multiply(np.random.random(img_batch_shape), batch_size)
+    input_a_np = input_a_np.astype('uint8')
+    output_a_rows = 4
+    output_a_np = np.multiply(np.random.random([batch_size]), batch_size)
+    output_a_np = output_a_np.astype('int')
+    output_b_np = np.random.random([batch_size, classes])
+    replace('input_a.tfrecord')
+    images_to_tfrecord(input_a_np, output_a_np, 'input_a.tfrecord')
+    input_a_tf, output_b_tf = read_and_decode_recordinput(
+        'input_a.tfrecord',
+        one_hot=True,
+        classes=classes,
+        is_train=True,
+        batch_shape=img_batch_shape)
+
+    a = Input(tensor=input_a_tf, batch_shape=img_batch_shape)
+
+    a_2 = Dense(8, name='dense_1')(a)
+    dp = Dropout(0.5, name='dropout')
+    b_2 = dp(a_2)
+    f_1 = Flatten()(b_2)
+    b_3 = Dense(
+        classes,
+        activation='softmax',
+        name='dense_b3')(f_1)
+
+    y_train_in_out = Input(
+        tensor=output_b_tf,
+        batch_shape=label_batch_shape,
+        name='y_labels')
+
+    model = Model([a], [b_3])
+    model.summary()
+
+    optimizer = 'rmsprop'
+    loss = 'mse'
+    loss_weights = [1., 2.]
+    with pytest.raises(ValueError) as exc:
+        model.compile(optimizer, loss, metrics=['mean_squared_error'],
+                      loss_weights=loss_weights,
+                      sample_weight_mode=None)
+
+    model.compile(optimizer, loss, metrics=['mean_squared_error'],
+                  sample_weight_mode=None)
+    out = model.predict_on_batch(output_b_np)
+    out = model.train_on_batch(None, output_b_np)
+    # test fit
+    out = model.fit(None,
+                    output_b_np, epochs=1, batch_size=10)
+
+    # test evaluate
+    out = model.evaluate(None,
+                         output_b_np,
+                         batch_size=10)
+
+    # test predict
+    out = model.predict(None, batch_size=10)
+
+    os.remove('input_a.tfrecord')
+
+
 @keras_test
 def test_model_with_partial_loss():
     a = Input(shape=(3,), name='input_a')

From ab9fe3296431b785d51cfab0ac9348c0b95ca7ea Mon Sep 17 00:00:00 2001
From: Andrew Hundt <ATHundt@gmail.com>
Date: Tue, 20 Jun 2017 17:23:34 -0400
Subject: [PATCH 62/64] mnist_tfrecord.py added with yield_op workaround
 (#7046) see
 https://github.com/fchollet/keras/pull/6928#issuecomment-309844726 for
 workaround explanation

---
 examples/mnist_tfrecord.py | 209 +++++++++++++++++++++++++++++++++++++
 1 file changed, 209 insertions(+)
 create mode 100644 examples/mnist_tfrecord.py

diff --git a/examples/mnist_tfrecord.py b/examples/mnist_tfrecord.py
new file mode 100644
index 00000000000..93d72c03831
--- /dev/null
+++ b/examples/mnist_tfrecord.py
@@ -0,0 +1,209 @@
+'''MNIST dataset with TensorFlow TFRecords.
+
+Gets to 99.25% test accuracy after 12 epochs
+(there is still a lot of margin for parameter tuning).
+'''
+import os
+import copy
+import time
+
+import numpy as np
+
+import tensorflow as tf
+from tensorflow.python.ops import data_flow_ops
+from keras import backend as K
+from keras.models import Model
+from keras.layers import Dense
+from keras.layers import Dropout
+from keras.layers import Flatten
+from keras.layers import Input
+from keras.layers import Conv2D
+from keras.layers import MaxPooling2D
+from keras.callbacks import EarlyStopping
+from keras.layers import TensorBoard
+from keras.objectives import categorical_crossentropy
+from keras.utils import np_utils
+from keras.utils.generic_utils import Progbar
+from keras import callbacks as cbks
+from keras import optimizers, objectives
+from keras import metrics as metrics_module
+
+from keras.datasets import mnist
+
+if K.backend() != 'tensorflow':
+    raise RuntimeError('This example can only run with the '
+                       'TensorFlow backend for the time being, '
+                       'because it requires TFRecords, which '
+                       'are not supported on other platforms.')
+
+
+def images_to_tfrecord(images, labels, filename):
+    def _int64_feature(value):
+        return tf.train.Feature(int64_list=tf.train.Int64List(value=[value]))
+
+    def _bytes_feature(value):
+        return tf.train.Feature(bytes_list=tf.train.BytesList(value=[value]))
+
+    """ Save data into TFRecord """
+    if not os.path.isfile(filename):
+        num_examples = images.shape[0]
+
+        rows = images.shape[1]
+        cols = images.shape[2]
+        depth = images.shape[3]
+
+        print('Writing', filename)
+        writer = tf.python_io.TFRecordWriter(filename)
+        for index in range(num_examples):
+            image_raw = images[index].tostring()
+            example = tf.train.Example(features=tf.train.Features(feature={
+                'height': _int64_feature(rows),
+                'width': _int64_feature(cols),
+                'depth': _int64_feature(depth),
+                'label': _int64_feature(int(labels[index])),
+                'image_raw': _bytes_feature(image_raw)}))
+            writer.write(example.SerializeToString())
+        writer.close()
+    else:
+        print('tfrecord %s already exists' % filename)
+
+
+def read_and_decode_recordinput(tf_glob, one_hot=True, classes=None, is_train=None, batch_shape=[1000, 28, 28, 1]):
+    """ Return tensor to read from TFRecord """
+    print 'Creating graph for loading TFRecords...'
+    with tf.variable_scope("TFRecords"):
+        record_input = data_flow_ops.RecordInput(tf_glob, batch_size=batch_shape[0])
+        records_op = record_input.get_yield_op()
+        records_op = tf.split(records_op, batch_shape[0], 0)
+        records_op = [tf.reshape(record, []) for record in records_op]
+        progbar = Progbar(len(records_op))
+
+        images = []
+        labels = []
+        for i, serialized_example in enumerate(records_op):
+            progbar.update(i)
+            with tf.variable_scope("parse_images", reuse=True):
+                features = tf.parse_single_example(
+                    serialized_example,
+                    features={
+                        'label': tf.FixedLenFeature([], tf.int64),
+                        'image_raw': tf.FixedLenFeature([], tf.string),
+                    })
+                img = tf.decode_raw(features['image_raw'], tf.uint8)
+                img.set_shape(batch_shape[1] * batch_shape[2])
+                img = tf.reshape(img, [1] + batch_shape[1:])
+
+                img = tf.cast(img, tf.float32) * (1. / 255) - 0.5
+
+                label = tf.cast(features['label'], tf.int32)
+                if one_hot and classes:
+                    label = tf.one_hot(label, classes)
+
+                images.append(img)
+                labels.append(label)
+
+        images = tf.parallel_stack(images, 0)
+        labels = tf.parallel_stack(labels, 0)
+        images = tf.cast(images, tf.float32)
+
+        images = tf.reshape(images, shape=batch_shape)
+
+        # StagingArea will store tensors
+        # across multiple steps to
+        # speed up execution
+        images_shape = images.get_shape()
+        labels_shape = labels.get_shape()
+        copy_stage = data_flow_ops.StagingArea(
+            [tf.float32, tf.float32],
+            shapes=[images_shape, labels_shape])
+        copy_stage_op = copy_stage.put(
+            [images, labels])
+        staged_images, staged_labels = copy_stage.get()
+
+        return images, labels
+
+
+def save_mnist_as_tfrecord():
+    (X_train, y_train), (X_test, y_test) = mnist.load_data()
+    X_train = X_train[..., np.newaxis]
+    X_test = X_test[..., np.newaxis]
+    images_to_tfrecord(images=X_train, labels=y_train, filename='train.mnist.tfrecord')
+    images_to_tfrecord(images=X_test, labels=y_test, filename='test.mnist.tfrecord')
+
+
+def cnn_layers(x_train_input):
+    x = Conv2D(32, (3, 3), activation='relu', padding='valid')(x_train_input)
+    x = Conv2D(64, (3, 3), activation='relu')(x)
+    x = MaxPooling2D(pool_size=(2, 2))(x)
+    x = Dropout(0.25)(x)
+    x = Flatten()(x)
+    x = Dense(128, activation='relu')(x)
+    x = Dropout(0.5)(x)
+    x_train_out = Dense(classes,
+                        activation='softmax',
+                        name='x_train_out')(x)
+    return x_train_out
+
+
+sess = tf.Session()
+K.set_session(sess)
+
+save_mnist_as_tfrecord()
+
+batch_size = 1
+batch_shape = [batch_size, 28, 28, 1]
+epochs = 60000
+classes = 10
+
+x_train_batch, y_train_batch = read_and_decode_recordinput(
+    'train.mnist.tfrecord',
+    one_hot=True,
+    classes=classes,
+    is_train=True,
+    batch_shape=batch_shape)
+
+x_test_batch, y_test_batch = read_and_decode_recordinput(
+    'test.mnist.tfrecord',
+    one_hot=True,
+    classes=classes,
+    is_train=True,
+    batch_shape=batch_shape)
+
+
+x_batch_shape = x_train_batch.get_shape().as_list()
+y_batch_shape = y_train_batch.get_shape().as_list()
+
+x_train_input = Input(tensor=x_train_batch, batch_shape=x_batch_shape)
+x_train_out = cnn_layers(x_train_input)
+y_train_in_out = Input(tensor=y_train_batch, batch_shape=y_batch_shape, name='y_labels')
+cce = categorical_crossentropy(y_train_batch, x_train_out)
+train_model = Model(inputs=[x_train_input], outputs=[x_train_out])
+train_model.add_loss(cce)
+
+train_model.compile(optimizer='rmsprop',
+                    loss=None,
+                    metrics=['accuracy'])
+
+tensorboard = TensorBoard(write_graph=True)
+
+train_model.summary()
+train_model.fit(batch_size=batch_size,
+                epochs=epochs)
+train_model.save_weights('saved_wt.h5')
+
+K.clear_session()
+
+# Second Session, pure Keras
+(X_train, y_train), (X_test, y_test) = mnist.load_data()
+X_train = X_train[..., np.newaxis]
+X_test = X_test[..., np.newaxis]
+x_test_inp = Input(batch_shape=(None,) + (X_test.shape[1:]))
+test_out = cnn_layers(x_test_inp)
+test_model = Model(inputs=x_test_inp, outputs=test_out)
+
+test_model.load_weights('saved_wt.h5')
+test_model.compile(optimizer='rmsprop', loss='categorical_crossentropy', metrics=['accuracy'])
+test_model.summary()
+
+loss, acc = test_model.evaluate(X_test, np_utils.to_categorical(y_test), classes)
+print('\nTest accuracy: {0}'.format(acc))

From e0ffad4db6464d6c36f7b964fdf231c8c1983dd5 Mon Sep 17 00:00:00 2001
From: Andrew Hundt <ATHundt@gmail.com>
Date: Tue, 20 Jun 2017 21:57:55 -0400
Subject: [PATCH 63/64] test_training.py extended yield_op tests, reduce code
 repetition cherry picked from 9fae1b2 with manual merge steps

---
 tests/keras/engine/test_training.py | 293 +++++++++++++---------------
 1 file changed, 139 insertions(+), 154 deletions(-)

diff --git a/tests/keras/engine/test_training.py b/tests/keras/engine/test_training.py
index 4095a032c01..7a0e615a788 100644
--- a/tests/keras/engine/test_training.py
+++ b/tests/keras/engine/test_training.py
@@ -5,7 +5,9 @@
 import sys
 import scipy.sparse as sparse
 
-from keras.layers import Dense, Dropout
+from keras.layers import Dense
+from keras.layers import Conv2D
+from keras.layers import Dropout
 from keras.layers import Flatten
 from keras.engine.topology import Input
 from keras.engine.training import Model, _check_loss_and_target_compatibility
@@ -29,6 +31,39 @@ def __getitem__(self, idx):
             np.random.random((self.batch_size, 3))]
 
 
+def call_model_methods(model, input_np, output_np, batch_size=10, epochs=1):
+
+    # train_on_batch
+    out = model.train_on_batch(input_np,
+                               output_np)
+    out = model.train_on_batch(input_np,
+                               output_np)
+
+    # test_on_batch
+    out = model.test_on_batch(input_np,
+                              output_np)
+
+    # predict_on_batch
+    out = model.predict_on_batch(input_np)
+
+    # fit
+    out = model.fit(input_np,
+                    output_np, epochs=1, batch_size=batch_size)
+    out = model.fit(input_np,
+                    output_np, epochs=1, batch_size=batch_size)
+
+    # evaluate
+    out = model.evaluate(input_np,
+                         output_np, batch_size=batch_size)
+    out = model.evaluate(input_np,
+                         output_np, batch_size=batch_size)
+
+    # predict
+    out = model.predict(input_np, batch_size=batch_size)
+    out = model.predict(input_np, batch_size=batch_size)
+    return out
+
+
 @keras_test
 def test_model_methods():
     a = Input(shape=(3,), name='input_a')
@@ -465,30 +500,8 @@ def test_model_with_input_feed_tensor():
                   loss_weights=loss_weights,
                   sample_weight_mode=None)
 
-    # test train_on_batch
-    out = model.train_on_batch(input_b_np,
-                               [output_a_np, output_b_np])
-    out = model.train_on_batch({'input_b': input_b_np},
-                               [output_a_np, output_b_np])
-    out = model.test_on_batch({'input_b': input_b_np},
-                              [output_a_np, output_b_np])
-    out = model.predict_on_batch({'input_b': input_b_np})
-
-    # test fit
-    out = model.fit({'input_b': input_b_np},
-                    [output_a_np, output_b_np], epochs=1, batch_size=10)
-    out = model.fit(input_b_np,
-                    [output_a_np, output_b_np], epochs=1, batch_size=10)
-
-    # test evaluate
-    out = model.evaluate({'input_b': input_b_np},
-                         [output_a_np, output_b_np], batch_size=10)
-    out = model.evaluate(input_b_np,
-                         [output_a_np, output_b_np], batch_size=10)
-
-    # test predict
-    out = model.predict({'input_b': input_b_np}, batch_size=10)
-    out = model.predict(input_b_np, batch_size=10)
+    out = call_model_methods(model, input_b_np, [output_a_np, output_b_np])
+    out = call_model_methods(model, {'input_b': input_b_np}, [output_a_np, output_b_np])
     assert len(out) == 2
 
     # Now test a model with a single input
@@ -503,34 +516,7 @@ def test_model_with_input_feed_tensor():
     loss = 'mse'
     model.compile(optimizer, loss, metrics=['mean_squared_error'])
 
-    # test train_on_batch
-    out = model.train_on_batch(None,
-                               output_a_np)
-    out = model.train_on_batch(None,
-                               output_a_np)
-    out = model.test_on_batch(None,
-                              output_a_np)
-    out = model.predict_on_batch(None)
-    out = model.train_on_batch([],
-                               output_a_np)
-    out = model.train_on_batch({},
-                               output_a_np)
-
-    # test fit
-    out = model.fit(None,
-                    output_a_np, epochs=1, batch_size=10)
-    out = model.fit(None,
-                    output_a_np, epochs=1, batch_size=10)
-
-    # test evaluate
-    out = model.evaluate(None,
-                         output_a_np, batch_size=10)
-    out = model.evaluate(None,
-                         output_a_np, batch_size=10)
-
-    # test predict
-    out = model.predict(None, batch_size=10)
-    out = model.predict(None, batch_size=10)
+    out = call_model_methods(model, None, output_a_np)
     assert out.shape == (10, 4)
 
     # Same, without learning phase
@@ -544,34 +530,14 @@ def test_model_with_input_feed_tensor():
     loss = 'mse'
     model.compile(optimizer, loss, metrics=['mean_squared_error'])
 
-    # test train_on_batch
-    out = model.train_on_batch(None,
-                               output_a_np)
-    out = model.train_on_batch(None,
-                               output_a_np)
-    out = model.test_on_batch(None,
-                              output_a_np)
-    out = model.predict_on_batch(None)
+    # train_on_batch
     out = model.train_on_batch([],
                                output_a_np)
     out = model.train_on_batch({},
                                output_a_np)
 
-    # test fit
-    out = model.fit(None,
-                    output_a_np, epochs=1, batch_size=10)
-    out = model.fit(None,
-                    output_a_np, epochs=1, batch_size=10)
-
-    # test evaluate
-    out = model.evaluate(None,
-                         output_a_np, batch_size=10)
-    out = model.evaluate(None,
-                         output_a_np, batch_size=10)
-
-    # test predict
-    out = model.predict(None, batch_size=10)
-    out = model.predict(None, batch_size=10)
+    out = call_model_methods(model, None, output_a_np)
+
     assert out.shape == (10, 4)
 
 
@@ -583,6 +549,95 @@ def test_model_with_input_yield_op():
     by only passing them data for the placeholder inputs
     in the model.
     """
+
+    batch_size = 1
+    input_rows = 20
+    cols = 20
+    depth = 1
+    classes = 2
+
+    # first batch size 1
+    img_batch_shape = [batch_size, input_rows, cols, depth]
+    input_a_np, input_a_tf, output_a_np, output_b_np, output_b_tf = create_tfrecord_data(
+        img_batch_shape, classes)
+    # tensor input, numpy output
+    input_only_tfrecord_model(input_a_tf, output_b_np, img_batch_shape, classes)
+    # tensor input, tensor output
+    input_label_tfrecord_model(input_a_tf, output_b_tf, img_batch_shape, classes)
+
+    # next batch size 3
+    batch_size = 3
+    img_batch_shape = [batch_size, input_rows, cols, depth]
+    input_a_np, input_a_tf, output_a_np, output_b_np, output_b_tf = create_tfrecord_data(
+        img_batch_shape, classes)
+    # tensor input, numpy output
+    input_only_tfrecord_model(input_a_tf, output_b_np, img_batch_shape, classes)
+    # tensor input, tensor output
+    input_label_tfrecord_model(input_a_tf, output_b_tf, img_batch_shape, classes)
+    os.remove('input_a.tfrecord')
+
+
+def cnn_layers(x_train_input, classes):
+    x = Conv2D(32, (1, 1), activation='relu', padding='valid')(x_train_input)
+    x = Dropout(0.25)(x)
+    x = Flatten()(x)
+    x = Dense(128, activation='relu')(x)
+    x = Dropout(0.5)(x)
+    x_train_out = Dense(classes,
+                        activation='softmax',
+                        name='x_train_out')(x)
+    return x_train_out
+
+
+def input_only_tfrecord_model(input_a_tf, output_b_np, img_batch_shape, classes):
+    a = Input(tensor=input_a_tf, batch_shape=img_batch_shape)
+    b = cnn_layers(a, classes)
+    model = Model([a], [b])
+    model.summary()
+
+    optimizer = 'rmsprop'
+    loss = 'mse'
+    loss_weights = [1., 2.]
+    with pytest.raises(ValueError) as exc:
+        model.compile(optimizer, loss, metrics=['mean_squared_error'],
+                      loss_weights=loss_weights,
+                      sample_weight_mode=None)
+
+    model.compile(optimizer, loss, metrics=['mean_squared_error'],
+                  sample_weight_mode=None)
+
+    call_model_methods(model, None, output_b_np, batch_size=img_batch_shape[0])
+
+
+def input_label_tfrecord_model(input_a_tf, output_b_tf, img_batch_shape, classes):
+    a = Input(tensor=input_a_tf, batch_shape=img_batch_shape)
+    b = cnn_layers(a, classes)
+    y_train_in_out = Input(
+        tensor=output_b_tf,
+        batch_shape=[img_batch_shape[0], classes],
+        name='y_labels')
+    b = cnn_layers(a, classes)
+    cce = K.categorical_crossentropy(b, y_train_in_out)
+    model = Model(inputs=[a], outputs=[b])
+    model.add_loss(cce)
+    model.summary()
+
+    optimizer = 'rmsprop'
+    loss = None
+    loss_weights = [1., 2.]
+    with pytest.raises(ValueError) as exc:
+        model.compile(optimizer, loss, metrics=['mean_squared_error'],
+                      loss_weights=loss_weights,
+                      sample_weight_mode=None)
+
+    model.compile(optimizer, loss, metrics=['mean_squared_error'],
+                  sample_weight_mode=None)
+
+    call_model_methods(model, None, None, batch_size=img_batch_shape[0])
+
+
+def create_tfrecord_data(img_batch_shape, classes):
+
     import tensorflow as tf
     from tensorflow.python.lib.io import tf_record
     from tensorflow.python.ops import data_flow_ops
@@ -621,6 +676,7 @@ def _bytes_feature(value):
 
     def read_and_decode_recordinput(tf_glob, one_hot=True, classes=None, is_train=None, batch_shape=[10, 3, 3, 1]):
         """ Return tensor to read from TFRecord """
+        import tensorflow as tf
         print('Creating graph for loading TFRecords...')
         with tf.variable_scope("TFRecords"):
             record_input = data_flow_ops.RecordInput(tf_glob, batch_size=batch_shape[0])
@@ -664,16 +720,10 @@ def replace(filename):
             print('%s already exists, replacing...' % filename)
             os.remove(filename)
 
-    batch_size = 10
-    input_rows = 3
-    cols = 3
-    depth = 1
-    classes = 2
-    img_batch_shape = [batch_size, input_rows, cols, depth]
+    [batch_size, input_rows, cols, depth] = img_batch_shape
     label_batch_shape = [batch_size, classes]
     input_a_np = np.multiply(np.random.random(img_batch_shape), batch_size)
     input_a_np = input_a_np.astype('uint8')
-    output_a_rows = 4
     output_a_np = np.multiply(np.random.random([batch_size]), batch_size)
     output_a_np = output_a_np.astype('int')
     output_b_np = np.random.random([batch_size, classes])
@@ -686,50 +736,7 @@ def replace(filename):
         is_train=True,
         batch_shape=img_batch_shape)
 
-    a = Input(tensor=input_a_tf, batch_shape=img_batch_shape)
-
-    a_2 = Dense(8, name='dense_1')(a)
-    dp = Dropout(0.5, name='dropout')
-    b_2 = dp(a_2)
-    f_1 = Flatten()(b_2)
-    b_3 = Dense(
-        classes,
-        activation='softmax',
-        name='dense_b3')(f_1)
-
-    y_train_in_out = Input(
-        tensor=output_b_tf,
-        batch_shape=label_batch_shape,
-        name='y_labels')
-
-    model = Model([a], [b_3])
-    model.summary()
-
-    optimizer = 'rmsprop'
-    loss = 'mse'
-    loss_weights = [1., 2.]
-    with pytest.raises(ValueError) as exc:
-        model.compile(optimizer, loss, metrics=['mean_squared_error'],
-                      loss_weights=loss_weights,
-                      sample_weight_mode=None)
-
-    model.compile(optimizer, loss, metrics=['mean_squared_error'],
-                  sample_weight_mode=None)
-    out = model.predict_on_batch(output_b_np)
-    out = model.train_on_batch(None, output_b_np)
-    # test fit
-    out = model.fit(None,
-                    output_b_np, epochs=1, batch_size=10)
-
-    # test evaluate
-    out = model.evaluate(None,
-                         output_b_np,
-                         batch_size=10)
-
-    # test predict
-    out = model.predict(None, batch_size=10)
-
-    os.remove('input_a.tfrecord')
+    return input_a_np, input_a_tf, output_a_np, output_b_np, output_b_tf
 
 
 @keras_test
@@ -765,6 +772,8 @@ def test_model_with_partial_loss():
     loss = {'dense_2': 'mse'}
     model.compile(optimizer, loss, metrics={'dense_1': 'mae'})
 
+    out = call_model_methods(model, input_a_np, output_a_np)
+    out = call_model_methods(model, input_a_np, [output_a_np])
     # test train_on_batch
     out = model.train_on_batch(input_a_np, output_a_np)
     out = model.test_on_batch(input_a_np, output_a_np)
@@ -794,13 +803,7 @@ def test_model_with_external_loss():
 
     input_a_np = np.random.random((10, 3))
 
-    # test train_on_batch
-    out = model.train_on_batch(input_a_np, None)
-    out = model.test_on_batch(input_a_np, None)
-    # fit
-    out = model.fit(input_a_np, None)
-    # evaluate
-    out = model.evaluate(input_a_np, None)
+    out = call_model_methods(model, input_a_np, None, batch_size=1)
 
     # No dropout, external loss.
     a = Input(shape=(3,), name='input_a')
@@ -814,13 +817,7 @@ def test_model_with_external_loss():
     loss = None
     model.compile(optimizer, loss, metrics=['mae'])
 
-    # test train_on_batch
-    out = model.train_on_batch(input_a_np, None)
-    out = model.test_on_batch(input_a_np, None)
-    # fit
-    out = model.fit(input_a_np, None)
-    # evaluate
-    out = model.evaluate(input_a_np, None)
+    out = call_model_methods(model, input_a_np, None, batch_size=10)
 
     # Test fit with no external data at all.
     if K.backend() == 'tensorflow':
@@ -836,19 +833,7 @@ def test_model_with_external_loss():
                       loss=None,
                       metrics=['mean_squared_error'])
 
-        # test train_on_batch
-        out = model.train_on_batch(None, None)
-        out = model.test_on_batch(None, None)
-        out = model.predict_on_batch(None)
-
-        # test fit
-        out = model.fit(None, None, epochs=1, batch_size=10)
-
-        # test evaluate
-        out = model.evaluate(None, None, batch_size=10)
-
-        # test predict
-        out = model.predict(None, batch_size=10)
+        out = call_model_methods(model, None, None, batch_size=10)
         assert out.shape == (10, 4)
 
 

From ad9283d08e5415981bea12e7d32790dc5e3b288c Mon Sep 17 00:00:00 2001
From: Andrew Hundt <ATHundt@gmail.com>
Date: Tue, 20 Jun 2017 21:58:22 -0400
Subject: [PATCH 64/64] generic_utils.py don't crash when dealing with batched
 data

---
 keras/utils/generic_utils.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/keras/utils/generic_utils.py b/keras/utils/generic_utils.py
index 76477d5ac38..f6c026d70ca 100644
--- a/keras/utils/generic_utils.py
+++ b/keras/utils/generic_utils.py
@@ -288,7 +288,7 @@ def update(self, current, values=None, force=False):
             for k in self.unique_values:
                 info += ' - %s:' % k
                 if isinstance(self.sum_values[k], list):
-                    avg = self.sum_values[k][0] / max(1, self.sum_values[k][1])
+                    avg = np.mean(self.sum_values[k][0] / max(1, self.sum_values[k][1]))
                     if abs(avg) > 1e-3:
                         info += ' %.4f' % avg
                     else:
@@ -311,7 +311,7 @@ def update(self, current, values=None, force=False):
                 info = '%ds' % (now - self.start)
                 for k in self.unique_values:
                     info += ' - %s:' % k
-                    avg = self.sum_values[k][0] / max(1, self.sum_values[k][1])
+                    avg = np.mean(self.sum_values[k][0] / max(1, self.sum_values[k][1]))
                     if avg > 1e-3:
                         info += ' %.4f' % avg
                     else: