From bd9d783a97c4759a5b0753643052994db8945ea9 Mon Sep 17 00:00:00 2001 From: Andrew Hundt Date: Mon, 12 Jun 2017 18:42:36 -0400 Subject: [PATCH 001/166] K.is_placeholder() added as common.py backend function --- examples/variational_autoencoder.py | 2 +- keras/backend/__init__.py | 1 + keras/backend/cntk_backend.py | 2 ++ keras/backend/common.py | 23 +++++++++++++++++++++++ keras/backend/tensorflow_backend.py | 4 +++- keras/backend/theano_backend.py | 1 + keras/engine/topology.py | 12 +++++++++--- 7 files changed, 40 insertions(+), 5 deletions(-) diff --git a/examples/variational_autoencoder.py b/examples/variational_autoencoder.py index fd395b17f45..ba313f1c69b 100644 --- a/examples/variational_autoencoder.py +++ b/examples/variational_autoencoder.py @@ -45,7 +45,7 @@ def sampling(args): # Custom loss layer class CustomVariationalLayer(Layer): def __init__(self, **kwargs): - self.is_placeholder = True + self._is_placeholder = True super(CustomVariationalLayer, self).__init__(**kwargs) def vae_loss(self, x, x_decoded_mean): diff --git a/keras/backend/__init__.py b/keras/backend/__init__.py index 2e7208cf5af..00e6d52fbe8 100644 --- a/keras/backend/__init__.py +++ b/keras/backend/__init__.py @@ -10,6 +10,7 @@ from .common import cast_to_floatx from .common import image_data_format from .common import set_image_data_format +from .common import is_placeholder # Obtain Keras base dir path: either ~/.keras or /tmp. _keras_base_dir = os.path.expanduser('~') diff --git a/keras/backend/cntk_backend.py b/keras/backend/cntk_backend.py index b4161ced8e7..cfe056896a0 100644 --- a/keras/backend/cntk_backend.py +++ b/keras/backend/cntk_backend.py @@ -2,6 +2,7 @@ import cntk as C import numpy as np from .common import _FLOATX, _EPSILON, image_dim_ordering, image_data_format +from .common import is_placeholder from collections import defaultdict from contextlib import contextmanager import warnings @@ -256,6 +257,7 @@ def placeholder( name=name) x._keras_shape = shape x._uses_learning_phase = False + x._is_placeholder = True return x diff --git a/keras/backend/common.py b/keras/backend/common.py index 03bf9165736..c4d98f92405 100644 --- a/keras/backend/common.py +++ b/keras/backend/common.py @@ -108,6 +108,29 @@ def cast_to_floatx(x): return np.asarray(x, dtype=_FLOATX) +def is_placeholder(tensor): + """Returns whether a tensor is a placeholder. + + # Arguments + tensor: A tensor instance. + + # Returns + A boolean. + + # Example + ```python + >>> from keras import backend as K + >>> a = K.placeholder((2, 2), sparse=False) + >>> print(K.is_placeholder(a)) + True + ``` + """ + try: + return tensor._is_placeholder + except AttributeError: + return False + + def image_data_format(): """Returns the default image data format convention ('channels_first' or 'channels_last'). diff --git a/keras/backend/tensorflow_backend.py b/keras/backend/tensorflow_backend.py index ecdc0541442..cad34bdecf0 100644 --- a/keras/backend/tensorflow_backend.py +++ b/keras/backend/tensorflow_backend.py @@ -14,6 +14,7 @@ from .common import floatx from .common import _EPSILON from .common import image_data_format +from .common import is_placeholder # Legacy functions from .common import set_image_dim_ordering @@ -373,7 +374,7 @@ def is_keras_tensor(x): ```python >>> from keras import backend as K >>> np_var = numpy.array([1, 2]) - >>> K.is_keras_tensor(np_var) # A numpy array is not a symbolic yensor. + >>> K.is_keras_tensor(np_var) # A numpy array is not a symbolic tensor. ValueError >>> k_var = tf.placeholder('float32', shape=(1,1)) >>> K.is_keras_tensor(k_var) # A variable created directly from tensorflow/theano is not a Keras tensor. @@ -431,6 +432,7 @@ def placeholder(shape=None, ndim=None, dtype=None, sparse=False, name=None): x = tf.placeholder(dtype, shape=shape, name=name) x._keras_shape = shape x._uses_learning_phase = False + x._is_placeholder = True return x diff --git a/keras/backend/theano_backend.py b/keras/backend/theano_backend.py index f06ca0ca8be..aa73641bbf5 100644 --- a/keras/backend/theano_backend.py +++ b/keras/backend/theano_backend.py @@ -221,6 +221,7 @@ def placeholder(shape=None, ndim=None, dtype=None, sparse=False, name=None): x = T.TensorType(dtype, broadcast)(name) x._keras_shape = shape x._uses_learning_phase = False + x._is_placeholder = True return x diff --git a/keras/engine/topology.py b/keras/engine/topology.py index c7450351dec..b01319407cd 100644 --- a/keras/engine/topology.py +++ b/keras/engine/topology.py @@ -1330,13 +1330,13 @@ def __init__(self, input_shape=None, batch_size=None, self.dtype = dtype if input_tensor is None: - self.is_placeholder = True + self._is_placeholder = True input_tensor = K.placeholder(shape=batch_input_shape, dtype=dtype, sparse=self.sparse, name=self.name) else: - self.is_placeholder = False + self._is_placeholder = False input_tensor._keras_shape = batch_input_shape # Create an input node to add to self.outbound_node # and set output_tensors' _keras_history. @@ -1535,6 +1535,8 @@ def __init__(self, inputs, outputs, name=None): self._output_tensor_cache = {} self._output_shape_cache = {} + self._input_placeholders = [] + self._input_yield_op_tensors = [] # User-provided arguments validation. for x in self.inputs: # Check that x is a Keras tensor. @@ -1560,6 +1562,10 @@ def __init__(self, inputs, outputs, name=None): 'instantiated via `tensor = Input(shape)`.\n' 'The tensor that caused the issue was: ' + str(x.name)) + if K.is_placeholder(layer): + self._input_placeholders.append((layer, node_index, tensor_index)) + else: + self._input_yield_op_tensors.append((layer, node_index, tensor_index)) for x in self.outputs: if not hasattr(x, '_keras_history'): cls_name = self.__class__.__name__ @@ -1621,7 +1627,7 @@ def __init__(self, inputs, outputs, name=None): i, layer.__class__.__name__)) self.input_names.append(layer.name) - if layer.is_placeholder: + if K.is_placeholder(layer): self._feed_input_names.append(layer.name) self._feed_inputs.append(layer.input) self._feed_input_shapes.append(self.inputs[i]._keras_shape) From 0fcec00576514d750f0dee53b98d41599f160f00 Mon Sep 17 00:00:00 2001 From: Andrew Hundt Date: Mon, 12 Jun 2017 18:49:48 -0400 Subject: [PATCH 002/166] tensorflow_backend.py session.run(fetches, feed_dict) params are forwarded in Function --- keras/backend/tensorflow_backend.py | 22 +++++++++++++++++----- 1 file changed, 17 insertions(+), 5 deletions(-) diff --git a/keras/backend/tensorflow_backend.py b/keras/backend/tensorflow_backend.py index cad34bdecf0..152dc3c0110 100644 --- a/keras/backend/tensorflow_backend.py +++ b/keras/backend/tensorflow_backend.py @@ -2226,9 +2226,12 @@ class Function(object): outputs: Output tensors to fetch. updates: Additional update ops to be run at function call. name: a name to help users identify what this function does. + fetches: Parameters forwarded to `tf.session.run(fetches)`. + feed_dict: Parameters forwarded to `tf.session.run(feed_dict)`. """ - def __init__(self, inputs, outputs, updates=None, name=None, **session_kwargs): + def __init__(self, inputs, outputs, updates=None, name=None, + fetches=None, feed_dict=None, **session_kwargs): updates = updates or [] if not isinstance(inputs, (list, tuple)): raise TypeError('`inputs` to a TensorFlow backend function ' @@ -2241,6 +2244,8 @@ def __init__(self, inputs, outputs, updates=None, name=None, **session_kwargs): 'should be a list or tuple.') self.inputs = list(inputs) self.outputs = list(outputs) + self.fetches = fetches + self.feed_dict = feed_dict with tf.control_dependencies(self.outputs): updates_ops = [] for update in updates: @@ -2257,17 +2262,24 @@ def __init__(self, inputs, outputs, updates=None, name=None, **session_kwargs): def __call__(self, inputs): if not isinstance(inputs, (list, tuple)): raise TypeError('`inputs` should be a list or tuple.') - feed_dict = {} + if self.feed_dict is None: + self.feed_dict = {} for tensor, value in zip(self.inputs, inputs): if is_sparse(tensor): sparse_coo = value.tocoo() indices = np.concatenate((np.expand_dims(sparse_coo.row, 1), np.expand_dims(sparse_coo.col, 1)), 1) value = (indices, sparse_coo.data, sparse_coo.shape) - feed_dict[tensor] = value + self.feed_dict[tensor] = value session = get_session() - updated = session.run(self.outputs + [self.updates_op], - feed_dict=feed_dict, + + if self.fetches is not None: + fetches = self.outputs + [self.updates_op] + self.fetches + else: + fetches = self.outputs + [self.updates_op] + + updated = session.run(fetches, + feed_dict=self.feed_dict, **self.session_kwargs) return updated[:len(self.outputs)] From bb3a76c2c02c97908f51b312004839887630a225 Mon Sep 17 00:00:00 2001 From: Andrew Hundt Date: Mon, 12 Jun 2017 16:46:56 -0400 Subject: [PATCH 003/166] .gitignore add .vscode developer environment --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 84912fd24b0..028725bb660 100644 --- a/.gitignore +++ b/.gitignore @@ -19,3 +19,4 @@ examples/img/* # developer environments .idea +.vscode From 73221c97ee3827eb82b8bf456405414e9c7ddaba Mon Sep 17 00:00:00 2001 From: Andrew Hundt Date: Mon, 12 Jun 2017 18:59:14 -0400 Subject: [PATCH 004/166] K.is_placeholder() added as common.py backend function --- examples/variational_autoencoder_deconv.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/variational_autoencoder_deconv.py b/examples/variational_autoencoder_deconv.py index f5aa1224d47..3612754c494 100644 --- a/examples/variational_autoencoder_deconv.py +++ b/examples/variational_autoencoder_deconv.py @@ -109,7 +109,7 @@ def sampling(args): # Custom loss layer class CustomVariationalLayer(Layer): def __init__(self, **kwargs): - self.is_placeholder = True + self._is_placeholder = True super(CustomVariationalLayer, self).__init__(**kwargs) def vae_loss(self, x, x_decoded_mean_squash): From 07057e1547339bbccb36803c14ac8467d0284dc2 Mon Sep 17 00:00:00 2001 From: Andrew Hundt Date: Mon, 12 Jun 2017 19:04:26 -0400 Subject: [PATCH 005/166] training.py _check_array_lengths(weights=None) --- keras/engine/training.py | 25 +++++++++++++------------ 1 file changed, 13 insertions(+), 12 deletions(-) diff --git a/keras/engine/training.py b/keras/engine/training.py index a811030ebbc..520af9cee2d 100644 --- a/keras/engine/training.py +++ b/keras/engine/training.py @@ -199,7 +199,7 @@ def _standardize_sample_weights(sample_weight, output_names): 'sample_weight') -def _check_array_lengths(inputs, targets, weights): +def _check_array_lengths(inputs, targets, weights=None): """Does user input validation for numpy arrays. # Arguments @@ -212,7 +212,6 @@ def _check_array_lengths(inputs, targets, weights): """ x_lengths = [x.shape[0] for x in inputs] y_lengths = [y.shape[0] for y in targets] - w_lengths = [w.shape[0] for w in weights] set_x = set(x_lengths) if len(set_x) > 1: raise ValueError('All input arrays (x) should have ' @@ -223,21 +222,23 @@ def _check_array_lengths(inputs, targets, weights): raise ValueError('All target arrays (y) should have ' 'the same number of samples. Got array shapes: ' + str([y.shape for y in targets])) - set_w = set(w_lengths) - if len(set_w) > 1: - raise ValueError('All sample_weight arrays should have ' - 'the same number of samples. Got array shapes: ' + - str([w.shape for w in weights])) if set_x and set_y and list(set_x)[0] != list(set_y)[0]: raise ValueError('Input arrays should have ' 'the same number of samples as target arrays. ' 'Found ' + str(list(set_x)[0]) + ' input samples ' 'and ' + str(list(set_y)[0]) + ' target samples.') - if set_y and set_w and list(set_y)[0] != list(set_w)[0]: - raise ValueError('Sample_weight arrays should have ' - 'the same number of samples as target arrays. Got ' + - str(list(set_y)[0]) + ' input samples and ' + - str(list(set_w)[0]) + ' target samples.') + if weights is not None: + w_lengths = [w.shape[0] for w in weights] + set_w = set(w_lengths) + if len(set_w) > 1: + raise ValueError('All sample_weight arrays should have ' + 'the same number of samples. Got array shapes: ' + + str([w.shape for w in weights])) + if set_y and set_w and list(set_y)[0] != list(set_w)[0]: + raise ValueError('Sample_weight arrays should have ' + 'the same number of samples as target arrays. Got ' + + str(list(set_y)[0]) + ' input samples and ' + + str(list(set_w)[0]) + ' target samples.') def _check_loss_and_target_compatibility(targets, loss_fns, output_shapes): From 4067e5fe72b954522960da9fad93120d909f9376 Mon Sep 17 00:00:00 2001 From: Andrew Hundt Date: Mon, 12 Jun 2017 19:08:21 -0400 Subject: [PATCH 006/166] training.py _weighted_mask_objective() accesses weights variable correctly --- keras/engine/training.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/keras/engine/training.py b/keras/engine/training.py index 520af9cee2d..4c084783454 100644 --- a/keras/engine/training.py +++ b/keras/engine/training.py @@ -444,13 +444,12 @@ def weighted(y_true, y_pred, weights, mask=None): # to the number of unmasked samples. score_array /= K.mean(mask) - # reduce score_array to same ndim as weight array - ndim = K.ndim(score_array) - weight_ndim = K.ndim(weights) - score_array = K.mean(score_array, axis=list(range(weight_ndim, ndim))) - # apply sample weighting if weights is not None: + # reduce score_array to same ndim as weight array + ndim = K.ndim(score_array) + weight_ndim = K.ndim(weights) + score_array = K.mean(score_array, axis=list(range(weight_ndim, ndim))) score_array *= weights score_array /= K.mean(K.cast(K.not_equal(weights, 0), K.floatx())) return K.mean(score_array) From bc742c920362c9c7a6439f3987812f11eaf27b98 Mon Sep 17 00:00:00 2001 From: Andrew Hundt Date: Mon, 12 Jun 2017 19:23:12 -0400 Subject: [PATCH 007/166] training.py only create placeholder when tensor is a placeholder --- keras/engine/training.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/keras/engine/training.py b/keras/engine/training.py index 4c084783454..02ce6bbd6e3 100644 --- a/keras/engine/training.py +++ b/keras/engine/training.py @@ -884,10 +884,13 @@ def compile(self, optimizer, loss, metrics=None, loss_weights=None, else: shape = self.internal_output_shapes[i] name = self.output_names[i] - target = K.placeholder(ndim=len(shape), - name=name + '_target', - sparse=K.is_sparse(self.outputs[i]), - dtype=K.dtype(self.outputs[i])) + if K.is_placeholder(self.outputs[i]): + target = K.placeholder(ndim=len(shape), + name=name + '_target', + sparse=K.is_sparse(self.outputs[i]), + dtype=K.dtype(self.outputs[i])) + else: + target = self.outputs[i] self.targets.append(target) self._feed_targets.append(target) From b2e8cec228f2acc5585344f83868075788789914 Mon Sep 17 00:00:00 2001 From: Andrew Hundt Date: Mon, 12 Jun 2017 23:58:48 -0400 Subject: [PATCH 008/166] tensorflow tests pass --- keras/backend/common.py | 2 +- keras/backend/tensorflow_backend.py | 26 +++- keras/engine/topology.py | 8 +- keras/engine/training.py | 233 ++++++++++++++++------------ keras/models.py | 2 + 5 files changed, 164 insertions(+), 107 deletions(-) diff --git a/keras/backend/common.py b/keras/backend/common.py index c4d98f92405..c5d40088b75 100644 --- a/keras/backend/common.py +++ b/keras/backend/common.py @@ -128,7 +128,7 @@ def is_placeholder(tensor): try: return tensor._is_placeholder except AttributeError: - return False + return True def image_data_format(): diff --git a/keras/backend/tensorflow_backend.py b/keras/backend/tensorflow_backend.py index 152dc3c0110..ac3c92e61db 100644 --- a/keras/backend/tensorflow_backend.py +++ b/keras/backend/tensorflow_backend.py @@ -14,7 +14,6 @@ from .common import floatx from .common import _EPSILON from .common import image_data_format -from .common import is_placeholder # Legacy functions from .common import set_image_dim_ordering @@ -281,6 +280,31 @@ def to_dense(tensor): name_scope = tf.name_scope +def is_placeholder(tensor): + """Returns whether a tensor is a placeholder. + + # Arguments + tensor: A tensor instance. + + # Returns + A boolean. + + # Example + ```python + >>> from keras import backend as K + >>> a = K.placeholder((2, 2), sparse=False) + >>> print(K.is_placeholder(a)) + True + ``` + """ + try: + if isinstance(tensor, tf.Variable): + return True + return tensor._is_placeholder + except AttributeError: + return True + + def variable(value, dtype=None, name=None): """Instantiates a variable and returns it. diff --git a/keras/engine/topology.py b/keras/engine/topology.py index b01319407cd..97b6a0a3ab8 100644 --- a/keras/engine/topology.py +++ b/keras/engine/topology.py @@ -113,7 +113,7 @@ def __init__(self, outbound_layer, input_tensors, output_tensors, input_masks, output_masks, input_shapes, output_shapes, - arguments=None): + arguments=None, is_placeholder=False): # Layer instance (NOT a list). # this is the layer that takes a list of input tensors # and turns them into a list of output tensors. @@ -157,6 +157,8 @@ def __init__(self, outbound_layer, # Optional keyword arguments to layer's `call`. self.arguments = arguments + # Indicates if the node represents a placeholder variable + self._is_placeholder = is_placeholder # Add nodes to all layers involved. for layer in inbound_layers: if layer is not None: @@ -1342,6 +1344,7 @@ def __init__(self, input_shape=None, batch_size=None, # and set output_tensors' _keras_history. input_tensor._uses_learning_phase = False input_tensor._keras_history = (self, 0, 0) + input_tensor._is_placeholder = self._is_placeholder Node(self, inbound_layers=[], node_indices=[], @@ -1351,7 +1354,8 @@ def __init__(self, input_shape=None, batch_size=None, input_masks=[None], output_masks=[None], input_shapes=[batch_input_shape], - output_shapes=[batch_input_shape]) + output_shapes=[batch_input_shape], + is_placeholder=self._is_placeholder) def get_config(self): config = {'batch_input_shape': self.batch_input_shape, diff --git a/keras/engine/training.py b/keras/engine/training.py index 02ce6bbd6e3..d1ef88cdae7 100644 --- a/keras/engine/training.py +++ b/keras/engine/training.py @@ -211,29 +211,31 @@ def _check_array_lengths(inputs, targets, weights=None): ValueError: in case of incorrectly formatted data. """ x_lengths = [x.shape[0] for x in inputs] - y_lengths = [y.shape[0] for y in targets] set_x = set(x_lengths) if len(set_x) > 1: raise ValueError('All input arrays (x) should have ' 'the same number of samples. Got array shapes: ' + str([x.shape for x in inputs])) - set_y = set(y_lengths) - if len(set_y) > 1: - raise ValueError('All target arrays (y) should have ' - 'the same number of samples. Got array shapes: ' + - str([y.shape for y in targets])) - if set_x and set_y and list(set_x)[0] != list(set_y)[0]: - raise ValueError('Input arrays should have ' - 'the same number of samples as target arrays. ' - 'Found ' + str(list(set_x)[0]) + ' input samples ' - 'and ' + str(list(set_y)[0]) + ' target samples.') - if weights is not None: + if targets: + y_lengths = [y.shape[0] for y in targets] + set_y = set(y_lengths) + if len(set_y) > 1: + raise ValueError('All target arrays (y) should have ' + 'the same number of samples. Got array shapes: ' + + str([y.shape for y in targets])) + if set_x and set_y and list(set_x)[0] != list(set_y)[0]: + raise ValueError('Input arrays should have ' + 'the same number of samples as target arrays. ' + 'Found ' + str(list(set_x)[0]) + ' input samples ' + 'and ' + str(list(set_y)[0]) + ' target samples.') + if weights: w_lengths = [w.shape[0] for w in weights] set_w = set(w_lengths) if len(set_w) > 1: raise ValueError('All sample_weight arrays should have ' 'the same number of samples. Got array shapes: ' + str([w.shape for w in weights])) + if targets and weights: if set_y and set_w and list(set_y)[0] != list(set_w)[0]: raise ValueError('Sample_weight arrays should have ' 'the same number of samples as target arrays. Got ' + @@ -718,7 +720,6 @@ def compile(self, optimizer, loss, metrics=None, loss_weights=None, """ loss = loss or {} self.optimizer = optimizers.get(optimizer) - self.sample_weight_mode = sample_weight_mode self.loss = loss self.loss_weights = loss_weights @@ -801,79 +802,12 @@ def compile(self, optimizer, loss, metrics=None, loss_weights=None, ' - expected a list of dicts.') # Prepare sample weights. - sample_weights = [] - sample_weight_modes = [] - if isinstance(sample_weight_mode, dict): - for name in sample_weight_mode: - if name not in self.output_names: - raise ValueError('Unknown entry in ' - 'sample_weight_mode dictionary: "' + - name + '". ' - 'Only expected the following keys: ' + - str(self.output_names)) - for i, name in enumerate(self.output_names): - if i in skip_indices: - weight = None - sample_weight_modes.append(None) - else: - if name not in sample_weight_mode: - raise ValueError('Output "' + name + - '" missing from sample_weight_modes ' - 'dictionary') - if sample_weight_mode.get(name) == 'temporal': - weight = K.placeholder(ndim=2, - name=name + '_sample_weights') - sample_weight_modes.append('temporal') - else: - weight = K.placeholder(ndim=1, - name=name + '_sample_weights') - sample_weight_modes.append(None) - sample_weights.append(weight) - elif isinstance(sample_weight_mode, list): - if len(sample_weight_mode) != len(self.outputs): - raise ValueError('When passing a list as sample_weight_mode, ' - 'it should have one entry per model outputs. ' - 'The model has ' + str(len(self.outputs)) + - ' outputs, but you passed ' - 'sample_weight_mode=' + - str(sample_weight_mode)) - for i in range(len(self.output_names)): - if i in skip_indices: - weight = None - sample_weight_modes.append(None) - else: - mode = sample_weight_mode[i] - name = self.output_names[i] - if mode == 'temporal': - weight = K.placeholder(ndim=2, - name=name + '_sample_weights') - sample_weight_modes.append('temporal') - else: - weight = K.placeholder(ndim=1, - name=name + '_sample_weights') - sample_weight_modes.append(None) - sample_weights.append(weight) + if self._input_yield_op_tensors: + sample_weight_mode = 'disabled' else: - for i, name in enumerate(self.output_names): - if i in skip_indices: - sample_weight_modes.append(None) - sample_weights.append(None) - else: - if sample_weight_mode == 'temporal': - sample_weights.append( - K.placeholder(ndim=2, - name=name + '_sample_weights')) - sample_weight_modes.append('temporal') - else: - sample_weights.append( - K.placeholder(ndim=1, - name=name + '_sample_weights')) - sample_weight_modes.append(None) - self.sample_weight_modes = sample_weight_modes - self._feed_sample_weight_modes = [] - for i in range(len(self.outputs)): - if i not in skip_indices: - self._feed_sample_weight_modes.append(self.sample_weight_modes[i]) + sample_weights = self._prepare_sample_weights( + sample_weight_mode, skip_indices) + self.sample_weight_mode = sample_weight_mode # Prepare targets of model. self.targets = [] @@ -907,7 +841,10 @@ def compile(self, optimizer, loss, metrics=None, loss_weights=None, y_true = self.targets[i] y_pred = self.outputs[i] weighted_loss = weighted_losses[i] - sample_weight = sample_weights[i] + if sample_weight_mode is 'disabled': + sample_weight = None + else: + sample_weight = sample_weights[i] mask = masks[i] loss_weight = loss_weights_list[i] output_loss = weighted_loss(y_true, y_pred, @@ -978,11 +915,12 @@ def append_metric(layer_num, metric_name, metric_tensor): # Prepare gradient updates and state updates. self.total_loss = total_loss - self.sample_weights = sample_weights - self._feed_sample_weights = [] - for i in range(len(self.sample_weights)): - if i not in skip_indices: - self._feed_sample_weights.append(sample_weights[i]) + if sample_weight_mode is not 'disabled': + self.sample_weights = sample_weights + self._feed_sample_weights = [] + for i in range(len(self.sample_weights)): + if i not in skip_indices: + self._feed_sample_weights.append(sample_weights[i]) # Functions for train, test and predict will # be compiled lazily when required. @@ -1003,11 +941,90 @@ def append_metric(layer_num, metric_name, metric_tensor): trainable_weights.sort(key=lambda x: x.name) self._collected_trainable_weights = trainable_weights + def _prepare_sample_weights(self, sample_weight_mode, skip_indices): + # Prepare sample weights. + sample_weights = [] + sample_weight_modes = [] + if isinstance(sample_weight_mode, dict): + for name in sample_weight_mode: + if name not in self.output_names: + raise ValueError('Unknown entry in ' + 'sample_weight_mode dictionary: "' + + name + '". ' + 'Only expected the following keys: ' + + str(self.output_names)) + for i, name in enumerate(self.output_names): + if i in skip_indices: + weight = None + sample_weight_modes.append(None) + else: + if name not in sample_weight_mode: + raise ValueError('Output "' + name + + '" missing from sample_weight_modes ' + 'dictionary') + if sample_weight_mode.get(name) == 'temporal': + weight = K.placeholder(ndim=2, + name=name + '_sample_weights') + sample_weight_modes.append('temporal') + else: + weight = K.placeholder(ndim=1, + name=name + '_sample_weights') + sample_weight_modes.append(None) + sample_weights.append(weight) + elif isinstance(sample_weight_mode, list): + if len(sample_weight_mode) != len(self.outputs): + raise ValueError('When passing a list as sample_weight_mode, ' + 'it should have one entry per model outputs. ' + 'The model has ' + str(len(self.outputs)) + + ' outputs, but you passed ' + 'sample_weight_mode=' + + str(sample_weight_mode)) + for i in range(len(self.output_names)): + if i in skip_indices: + weight = None + sample_weight_modes.append(None) + else: + mode = sample_weight_mode[i] + name = self.output_names[i] + if mode == 'temporal': + weight = K.placeholder(ndim=2, + name=name + '_sample_weights') + sample_weight_modes.append('temporal') + else: + weight = K.placeholder(ndim=1, + name=name + '_sample_weights') + sample_weight_modes.append(None) + sample_weights.append(weight) + else: + for i, name in enumerate(self.output_names): + if i in skip_indices: + sample_weight_modes.append(None) + sample_weights.append(None) + else: + if sample_weight_mode == 'temporal': + sample_weights.append( + K.placeholder(ndim=2, + name=name + '_sample_weights')) + sample_weight_modes.append('temporal') + else: + sample_weights.append( + K.placeholder(ndim=1, + name=name + '_sample_weights')) + sample_weight_modes.append(None) + self.sample_weight_modes = sample_weight_modes + self._feed_sample_weight_modes = [] + for i in range(len(self.outputs)): + if i not in skip_indices: + self._feed_sample_weight_modes.append(self.sample_weight_modes[i]) + return sample_weights + def _make_train_function(self): if not hasattr(self, 'train_function'): raise RuntimeError('You must compile your model before using it.') if self.train_function is None: - inputs = self._feed_inputs + self._feed_targets + self._feed_sample_weights + inputs = self._feed_inputs + self._feed_targets + if self.sample_weight_mode is not 'disabled': + inputs += self._feed_sample_weights if self.uses_learning_phase and not isinstance(K.learning_phase(), int): inputs += [K.learning_phase()] @@ -1027,7 +1044,9 @@ def _make_test_function(self): if not hasattr(self, 'test_function'): raise RuntimeError('You must compile your model before using it.') if self.test_function is None: - inputs = self._feed_inputs + self._feed_targets + self._feed_sample_weights + inputs = self._feed_inputs + self._feed_targets + if self.sample_weight_mode is not 'disabled': + inputs += self._feed_sample_weights if self.uses_learning_phase and not isinstance(K.learning_phase(), int): inputs += [K.learning_phase()] # Return loss and metrics, no gradient updates. @@ -1143,7 +1162,9 @@ def _fit_loop(self, f, ins, out_labels=None, batch_size=32, for batch_index, (batch_start, batch_end) in enumerate(batches): batch_ids = index_array[batch_start:batch_end] try: - if isinstance(ins[-1], float): + if not ins: + ins_batch = [] + elif isinstance(ins[-1], float): # Do not slice the training phase flag. ins_batch = _slice_arrays(ins[:-1], batch_ids) + [ins[-1]] else: @@ -1312,17 +1333,23 @@ def _standardize_user_data(self, x, y, self._feed_input_shapes, check_batch_axis=False, exception_prefix='input') - y = _standardize_input_data(y, self._feed_output_names, - output_shapes, - check_batch_axis=False, - exception_prefix='target') - sample_weights = _standardize_sample_weights(sample_weight, - self._feed_output_names) + if not x and y is None and self.sample_weight_mode is 'disabled': + y = [] + else: + y = _standardize_input_data(y, self._feed_output_names, + output_shapes, + check_batch_axis=False, + exception_prefix='target') class_weights = _standardize_class_weights(class_weight, self._feed_output_names) - sample_weights = [_standardize_weights(ref, sw, cw, mode) - for (ref, sw, cw, mode) - in zip(y, sample_weights, class_weights, self._feed_sample_weight_modes)] + if self.sample_weight_mode is not 'disabled': + sample_weights = _standardize_sample_weights(sample_weight, + self._feed_output_names) + sample_weights = [_standardize_weights(ref, sw, cw, mode) + for (ref, sw, cw, mode) + in zip(y, sample_weights, class_weights, self._feed_sample_weight_modes)] + else: + sample_weights = [] _check_array_lengths(x, y, sample_weights) _check_loss_and_target_compatibility(y, self._feed_loss_fns, diff --git a/keras/models.py b/keras/models.py index b3a8becbd11..2d826eed990 100644 --- a/keras/models.py +++ b/keras/models.py @@ -456,6 +456,7 @@ def add(self, layer): 'For multi-output layers, ' 'use the functional API.') + layer.inbound_nodes[0].output_tensors[0]._is_placeholder = True self.outputs = [layer.inbound_nodes[0].output_tensors[0]] self.inputs = topology.get_source_inputs(self.outputs[0]) @@ -479,6 +480,7 @@ def add(self, layer): 'should have a single output tensor. ' 'For multi-output layers, ' 'use the functional API.') + output_tensor._is_placeholder = True self.outputs = [output_tensor] # update self.inbound_nodes self.inbound_nodes[0].output_tensors = self.outputs From 745f23d204f8615268ffcf5f700180e6ee49ab0c Mon Sep 17 00:00:00 2001 From: Andrew Hundt Date: Mon, 12 Jun 2017 23:59:02 -0400 Subject: [PATCH 009/166] mnist_tfrecord.py added --- examples/mnist_tfrecord.py | 209 +++++++++++++++++++++++++++++++++++++ 1 file changed, 209 insertions(+) create mode 100644 examples/mnist_tfrecord.py diff --git a/examples/mnist_tfrecord.py b/examples/mnist_tfrecord.py new file mode 100644 index 00000000000..c0f884b2ef3 --- /dev/null +++ b/examples/mnist_tfrecord.py @@ -0,0 +1,209 @@ +'''MNIST dataset with TensorFlow TFRecords. + +Gets to 99.25% test accuracy after 12 epochs +(there is still a lot of margin for parameter tuning). +''' +import sys +sys.path.append('/Users/athundt/source/keras') +import os +import copy +import time + +import numpy as np + +import tensorflow as tf +from tensorflow.python.ops import data_flow_ops +from keras import backend as K +from keras.models import Model +from keras.layers import Dense, Dropout, Flatten, Input, Conv2D +from keras.callbacks import EarlyStopping +from keras.objectives import categorical_crossentropy +from keras.utils import np_utils +from keras import callbacks as cbks +from keras import optimizers, objectives +# from keras.engine.training import collect_metrics, weighted_objective +from keras import metrics as metrics_module + +from keras.datasets import mnist + +if K.backend() != 'tensorflow': + raise RuntimeError('This example can only run with the ' + 'TensorFlow backend for the time being, ' + 'because it requires TFRecords, which ' + 'are not supported on other platforms.') + + +def images_to_tfrecord(images, labels, filename): + def _int64_feature(value): + return tf.train.Feature(int64_list=tf.train.Int64List(value=[value])) + + def _bytes_feature(value): + return tf.train.Feature(bytes_list=tf.train.BytesList(value=[value])) + + """ Save data into TFRecord """ + if not os.path.isfile(filename): + num_examples = images.shape[0] + + rows = images.shape[1] + cols = images.shape[2] + depth = images.shape[3] + + print('Writing', filename) + writer = tf.python_io.TFRecordWriter(filename) + for index in range(num_examples): + image_raw = images[index].tostring() + example = tf.train.Example(features=tf.train.Features(feature={ + 'height': _int64_feature(rows), + 'width': _int64_feature(cols), + 'depth': _int64_feature(depth), + 'label': _int64_feature(int(labels[index])), + 'image_raw': _bytes_feature(image_raw)})) + writer.write(example.SerializeToString()) + writer.close() + else: + print 'tfrecord already exists' + + +def read_and_decode(filename, one_hot=True, classes=None, is_train=None): + """ Return tensor to read from TFRecord """ + filename_queue = tf.train.string_input_producer([filename]) + reader = tf.TFRecordReader() + _, serialized_example = reader.read(filename_queue) + features = tf.parse_single_example(serialized_example, + features={ + 'label': tf.FixedLenFeature([], tf.int64), + 'image_raw': tf.FixedLenFeature([], tf.string), + }) + img = tf.decode_raw(features['image_raw'], tf.uint8) + img.set_shape([28 * 28]) + img = tf.reshape(img, [28, 28, 1]) + + img = tf.cast(img, tf.float32) * (1. / 255) - 0.5 + + label = tf.cast(features['label'], tf.int32) + if one_hot and classes: + label = tf.one_hot(label, classes) + + x_train_batch, y_train_batch = K.tf.train.shuffle_batch( + [img, label], + batch_size=batch_size, + capacity=2000, + min_after_dequeue=1000, + num_threads=32) # set the number of threads here + + return x_train_batch, y_train_batch + + +def read_and_decode_recordinput(tf_glob, one_hot=True, classes=None, is_train=None, batch_size=1000): + """ Return tensor to read from TFRecord """ + + record_input = data_flow_ops.RecordInput(tf_glob, batch_size=batch_size) + records_op = record_input.get_yield_op() + records_op = tf.split(records_op, batch_size, 0) + records_op = [tf.reshape(record, []) for record in records_op] + + imgs = [] + labels = [] + for serialized_example in records_op: + features = tf.parse_single_example(serialized_example, + features={ + 'label': tf.FixedLenFeature([], tf.int64), + 'image_raw': tf.FixedLenFeature([], tf.string), + }) + img = tf.decode_raw(features['image_raw'], tf.uint8) + img.set_shape([28 * 28]) + img = tf.reshape(img, [1, 28, 28, 1]) + + img = tf.cast(img, tf.float32) * (1. / 255) - 0.5 + + label = tf.cast(features['label'], tf.int32) + if one_hot and classes: + label = tf.one_hot(label, classes) + + imgs.append(img) + labels.append(label) + + imgs = tf.concat(imgs, 0) + labels = tf.concat(labels, 0) + return imgs, labels + + +def save_mnist_as_tfrecord(): + (X_train, y_train), (X_test, y_test) = mnist.load_data() + X_train = X_train[..., np.newaxis] + X_test = X_test[..., np.newaxis] + images_to_tfrecord(images=X_train, labels=y_train, filename='train.mnist.tfrecord') + images_to_tfrecord(images=X_test, labels=y_test, filename='test.mnist.tfrecord') + + +def cnn_layers(x_train_input): + con1 = Conv2D(32, (3, 3), activation='relu', + strides=(2, 2), padding='valid')(x_train_input) + con2 = Conv2D(32, (3, 3), activation='relu', + strides=(2, 2))(con1) + fla1 = Flatten()(con2) + den1 = Dense(128, activation='relu')(fla1) + x_train_out = Dense(classes, + activation='softmax', + name='x_train_out')(den1) + return x_train_out + + +def create_cnn_model(x_train_batch, y_train_batch, x_batch_shape, y_batch_shape): + x_train_input = Input(tensor=x_train_batch, batch_shape=x_batch_shape) + x_train_out = cnn_layers(x_train_input) + # Workaround until _is_placeholder can be deduced automatically + x_train_out._is_placeholder = False + y_train_in_out = Input(tensor=y_train_batch, batch_shape=y_batch_shape) + return Model(inputs=[x_train_input, y_train_in_out], outputs=[x_train_out, y_train_in_out]) + + +sess = tf.Session() +K.set_session(sess) + +save_mnist_as_tfrecord() + +batch_size = 1000 +epochs = 300 +classes = 10 + +x_train_batch, y_train_batch = read_and_decode_recordinput( + 'train.mnist.tfrecord', + one_hot=True, + classes=classes, + is_train=True, + batch_size=batch_size) + + +x_batch_shape = x_train_batch.get_shape().as_list() +y_batch_shape = y_train_batch.get_shape().as_list() + +train_model = create_cnn_model(x_train_batch, + y_train_batch, + x_batch_shape, + y_batch_shape) + +train_model.compile(optimizer='rmsprop', + loss='categorical_crossentropy', + metrics=['accuracy']) + +train_model.summary() +train_model.fit(batch_size=batch_size, epochs=epochs) +train_model.save_weights('saved_wt.h5') + +K.clear_session() + +# Second Session, pure Keras +(X_train, y_train), (X_test, y_test) = mnist.load_data() +X_train = X_train[..., np.newaxis] +X_test = X_test[..., np.newaxis] +x_test_inp = Input(batch_shape=(None,) + (X_test.shape[1:])) +test_out = cnn_layers(x_test_inp) +test_model = Model(inputs=x_test_inp, outputs=test_out) + +test_model.load_weights('saved_wt.h5') +test_model.compile(optimizer='rmsprop', loss='categorical_crossentropy', metrics=['accuracy']) +test_model.summary() + +loss, acc = test_model.evaluate(X_test, np_utils.to_categorical(y_test), classes) +print '\nTest accuracy: {0}'.format(acc) From 9a79ebc9f8d06cb48c3596f995d1202cd03c05b4 Mon Sep 17 00:00:00 2001 From: Andrew Hundt Date: Tue, 13 Jun 2017 00:10:48 -0400 Subject: [PATCH 010/166] mnist_tfrecord.py label op more clearly marked --- examples/mnist_tfrecord.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/mnist_tfrecord.py b/examples/mnist_tfrecord.py index c0f884b2ef3..f9a12f47e0d 100644 --- a/examples/mnist_tfrecord.py +++ b/examples/mnist_tfrecord.py @@ -154,7 +154,7 @@ def create_cnn_model(x_train_batch, y_train_batch, x_batch_shape, y_batch_shape) x_train_out = cnn_layers(x_train_input) # Workaround until _is_placeholder can be deduced automatically x_train_out._is_placeholder = False - y_train_in_out = Input(tensor=y_train_batch, batch_shape=y_batch_shape) + y_train_in_out = Input(tensor=y_train_batch, batch_shape=y_batch_shape, name='y_labels') return Model(inputs=[x_train_input, y_train_in_out], outputs=[x_train_out, y_train_in_out]) From b321de6d0ea49c8f83ee4e425d16bdc8eeb6b452 Mon Sep 17 00:00:00 2001 From: Andrew Hundt Date: Tue, 13 Jun 2017 00:58:32 -0400 Subject: [PATCH 011/166] mnist_tfrecord.py removed debug lines --- examples/mnist_tfrecord.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/examples/mnist_tfrecord.py b/examples/mnist_tfrecord.py index f9a12f47e0d..049b5e7f041 100644 --- a/examples/mnist_tfrecord.py +++ b/examples/mnist_tfrecord.py @@ -3,8 +3,6 @@ Gets to 99.25% test accuracy after 12 epochs (there is still a lot of margin for parameter tuning). ''' -import sys -sys.path.append('/Users/athundt/source/keras') import os import copy import time From 66251aae8d9927f3fbd3673840163fffa2c62c16 Mon Sep 17 00:00:00 2001 From: Andrew Hundt Date: Tue, 13 Jun 2017 14:42:25 -0400 Subject: [PATCH 012/166] training.py model.fit() fetches and feed_dict docstring added --- keras/engine/training.py | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/keras/engine/training.py b/keras/engine/training.py index 695e80c8fb4..90759c3c12d 100644 --- a/keras/engine/training.py +++ b/keras/engine/training.py @@ -1443,6 +1443,26 @@ class indices (integers) to sample_weight_mode="temporal" in compile(). initial_epoch: epoch at which to start training (useful for resuming a previous training run) + fetches: TensorFlow backend only extension point for advanced use cases + that require additional processing to be performed as `fit()` runs. + A single TensorFlow graph element, a list of graph elements, or a + dictionary whose values are graph elements or lists of graph elements. + Passes tensor ops to `tf.session.run()` and + returns an additional tensor tuple upon completion. This is for advanced + users that require auxiliary processing as fit runs. When provided, + additional tensor values are returned by `fit()` as follows: + + ```python + history, tensors = model.fit(fetches,feed_dict) + ``` + + See the [tf.Session.run()](https://www.tensorflow.org/api_docs/python/tf/Session) + `fetches` parameter for more details. + feed_dict: TensorFlow backend only extension point for advanced use cases + that require additional processing to be performed as `fit()` runs. + Dictionary that maps TensorFlow graph elements to values. + See the [tf.Session.run()](https://www.tensorflow.org/api_docs/python/tf/Session) + `feed_dict` parameter for more details. # Returns A `History` instance. Its `history` attribute contains From 23b0ebfc1c067dd6f2c5b5063f6b8c3a53d1127a Mon Sep 17 00:00:00 2001 From: Andrew Hundt Date: Tue, 13 Jun 2017 14:59:15 -0400 Subject: [PATCH 013/166] Revert "training.py model.fit() fetches and feed_dict docstring added" This reverts commit 66251aae8d9927f3fbd3673840163fffa2c62c16. --- keras/engine/training.py | 20 -------------------- 1 file changed, 20 deletions(-) diff --git a/keras/engine/training.py b/keras/engine/training.py index 90759c3c12d..695e80c8fb4 100644 --- a/keras/engine/training.py +++ b/keras/engine/training.py @@ -1443,26 +1443,6 @@ class indices (integers) to sample_weight_mode="temporal" in compile(). initial_epoch: epoch at which to start training (useful for resuming a previous training run) - fetches: TensorFlow backend only extension point for advanced use cases - that require additional processing to be performed as `fit()` runs. - A single TensorFlow graph element, a list of graph elements, or a - dictionary whose values are graph elements or lists of graph elements. - Passes tensor ops to `tf.session.run()` and - returns an additional tensor tuple upon completion. This is for advanced - users that require auxiliary processing as fit runs. When provided, - additional tensor values are returned by `fit()` as follows: - - ```python - history, tensors = model.fit(fetches,feed_dict) - ``` - - See the [tf.Session.run()](https://www.tensorflow.org/api_docs/python/tf/Session) - `fetches` parameter for more details. - feed_dict: TensorFlow backend only extension point for advanced use cases - that require additional processing to be performed as `fit()` runs. - Dictionary that maps TensorFlow graph elements to values. - See the [tf.Session.run()](https://www.tensorflow.org/api_docs/python/tf/Session) - `feed_dict` parameter for more details. # Returns A `History` instance. Its `history` attribute contains From 9fa27477c27a18a41e3adec5ce304abdb9fc8b73 Mon Sep 17 00:00:00 2001 From: Andrew Hundt Date: Wed, 14 Jun 2017 16:25:43 -0400 Subject: [PATCH 014/166] mnist_tfrecord.py remove extraneous comment --- examples/mnist_tfrecord.py | 1 - 1 file changed, 1 deletion(-) diff --git a/examples/mnist_tfrecord.py b/examples/mnist_tfrecord.py index 049b5e7f041..ee484b85c61 100644 --- a/examples/mnist_tfrecord.py +++ b/examples/mnist_tfrecord.py @@ -19,7 +19,6 @@ from keras.utils import np_utils from keras import callbacks as cbks from keras import optimizers, objectives -# from keras.engine.training import collect_metrics, weighted_objective from keras import metrics as metrics_module from keras.datasets import mnist From db16bb2e68822311201b16e2e10ed58c06eda963 Mon Sep 17 00:00:00 2001 From: Andrew Hundt Date: Thu, 15 Jun 2017 15:47:46 -0400 Subject: [PATCH 015/166] Container.__init__ new labels tensor parameter supports tensor generated labels. mnist_tfrecord.py training runs correctly --- examples/mnist_tfrecord.py | 107 ++++++++++++++++++++++--------------- keras/engine/topology.py | 10 +++- keras/engine/training.py | 6 +-- keras/legacy/interfaces.py | 3 +- 4 files changed, 77 insertions(+), 49 deletions(-) diff --git a/examples/mnist_tfrecord.py b/examples/mnist_tfrecord.py index ee484b85c61..c21beb105aa 100644 --- a/examples/mnist_tfrecord.py +++ b/examples/mnist_tfrecord.py @@ -14,11 +14,12 @@ from keras import backend as K from keras.models import Model from keras.layers import Dense, Dropout, Flatten, Input, Conv2D -from keras.callbacks import EarlyStopping +from keras.callbacks import EarlyStopping, TensorBoard from keras.objectives import categorical_crossentropy from keras.utils import np_utils from keras import callbacks as cbks from keras import optimizers, objectives +# from keras.engine.training import collect_metrics, weighted_objective from keras import metrics as metrics_module from keras.datasets import mnist @@ -58,7 +59,7 @@ def _bytes_feature(value): writer.write(example.SerializeToString()) writer.close() else: - print 'tfrecord already exists' + print 'tfrecord %s already exists' % filename def read_and_decode(filename, one_hot=True, classes=None, is_train=None): @@ -66,11 +67,12 @@ def read_and_decode(filename, one_hot=True, classes=None, is_train=None): filename_queue = tf.train.string_input_producer([filename]) reader = tf.TFRecordReader() _, serialized_example = reader.read(filename_queue) - features = tf.parse_single_example(serialized_example, - features={ - 'label': tf.FixedLenFeature([], tf.int64), - 'image_raw': tf.FixedLenFeature([], tf.string), - }) + features = tf.parse_single_example( + serialized_example, + features={ + 'label': tf.FixedLenFeature([], tf.int64), + 'image_raw': tf.FixedLenFeature([], tf.string), + }) img = tf.decode_raw(features['image_raw'], tf.uint8) img.set_shape([28 * 28]) img = tf.reshape(img, [28, 28, 1]) @@ -90,39 +92,43 @@ def read_and_decode(filename, one_hot=True, classes=None, is_train=None): return x_train_batch, y_train_batch - -def read_and_decode_recordinput(tf_glob, one_hot=True, classes=None, is_train=None, batch_size=1000): +def read_and_decode_recordinput(tf_glob, one_hot=True, classes=None, is_train=None, batch_shape=[1000, 28, 28, 1]): """ Return tensor to read from TFRecord """ - - record_input = data_flow_ops.RecordInput(tf_glob, batch_size=batch_size) - records_op = record_input.get_yield_op() - records_op = tf.split(records_op, batch_size, 0) - records_op = [tf.reshape(record, []) for record in records_op] - - imgs = [] - labels = [] - for serialized_example in records_op: - features = tf.parse_single_example(serialized_example, - features={ - 'label': tf.FixedLenFeature([], tf.int64), - 'image_raw': tf.FixedLenFeature([], tf.string), - }) - img = tf.decode_raw(features['image_raw'], tf.uint8) - img.set_shape([28 * 28]) - img = tf.reshape(img, [1, 28, 28, 1]) - - img = tf.cast(img, tf.float32) * (1. / 255) - 0.5 - - label = tf.cast(features['label'], tf.int32) - if one_hot and classes: - label = tf.one_hot(label, classes) - - imgs.append(img) - labels.append(label) - - imgs = tf.concat(imgs, 0) - labels = tf.concat(labels, 0) - return imgs, labels + with tf.variable_scope("TFRecords"): + record_input = data_flow_ops.RecordInput(tf_glob, batch_size=batch_shape[0]) + records_op = record_input.get_yield_op() + records_op = tf.split(records_op, batch_shape[0], 0) + records_op = [tf.reshape(record, []) for record in records_op] + + imgs = [] + labels = [] + for serialized_example in records_op: + with tf.variable_scope("parse_images", reuse=True): + features = tf.parse_single_example( + serialized_example, + features={ + 'label': tf.FixedLenFeature([], tf.int64), + 'image_raw': tf.FixedLenFeature([], tf.string), + }) + img = tf.decode_raw(features['image_raw'], tf.uint8) + img.set_shape(batch_shape[1] * batch_shape[2]) + img = tf.reshape(img, [1] + batch_shape[1:]) + + img = tf.cast(img, tf.float32) * (1. / 255) - 0.5 + + label = tf.cast(features['label'], tf.int32) + if one_hot and classes: + label = tf.one_hot(label, classes) + + imgs.append(img) + labels.append(label) + + imgs = tf.parallel_stack(imgs, 0) + labels = tf.parallel_stack(labels, 0) + + imgs = tf.cast(imgs, tf.float32) + imgs = tf.reshape(imgs, shape=batch_shape) + return imgs, labels def save_mnist_as_tfrecord(): @@ -152,7 +158,7 @@ def create_cnn_model(x_train_batch, y_train_batch, x_batch_shape, y_batch_shape) # Workaround until _is_placeholder can be deduced automatically x_train_out._is_placeholder = False y_train_in_out = Input(tensor=y_train_batch, batch_shape=y_batch_shape, name='y_labels') - return Model(inputs=[x_train_input, y_train_in_out], outputs=[x_train_out, y_train_in_out]) + return Model(inputs=[x_train_input], outputs=[x_train_out], labels=[y_train_in_out]) sess = tf.Session() @@ -160,8 +166,9 @@ def create_cnn_model(x_train_batch, y_train_batch, x_batch_shape, y_batch_shape) save_mnist_as_tfrecord() -batch_size = 1000 -epochs = 300 +batch_size = 100 +batch_shape = [batch_size, 28, 28, 1] +epochs = 3000 classes = 10 x_train_batch, y_train_batch = read_and_decode_recordinput( @@ -169,7 +176,14 @@ def create_cnn_model(x_train_batch, y_train_batch, x_batch_shape, y_batch_shape) one_hot=True, classes=classes, is_train=True, - batch_size=batch_size) + batch_shape=batch_shape) + +x_test_batch, y_test_batch = read_and_decode_recordinput( + 'test.mnist.tfrecord', + one_hot=True, + classes=classes, + is_train=True, + batch_shape=batch_shape) x_batch_shape = x_train_batch.get_shape().as_list() @@ -184,8 +198,13 @@ def create_cnn_model(x_train_batch, y_train_batch, x_batch_shape, y_batch_shape) loss='categorical_crossentropy', metrics=['accuracy']) +tensorboard = TensorBoard(write_graph=True) + train_model.summary() -train_model.fit(batch_size=batch_size, epochs=epochs) +train_model.fit(batch_size=batch_size, + epochs=epochs, + callbacks=[tensorboard], + validation_data=(x_test_batch, y_test_batch)) train_model.save_weights('saved_wt.h5') K.clear_session() diff --git a/keras/engine/topology.py b/keras/engine/topology.py index 97b6a0a3ab8..e01d7e6da17 100644 --- a/keras/engine/topology.py +++ b/keras/engine/topology.py @@ -1478,7 +1478,7 @@ class Container(Layer): """ @interfaces.legacy_model_constructor_support - def __init__(self, inputs, outputs, name=None): + def __init__(self, inputs, outputs, labels=None, name=None): # Handle `name` argument. if not name: prefix = self.__class__.__name__.lower() @@ -1495,11 +1495,19 @@ def __init__(self, inputs, outputs, name=None): self.inputs = list(inputs) # Tensor or list of tensors. else: self.inputs = [inputs] + if isinstance(outputs, (list, tuple)): self.outputs = list(outputs) else: self.outputs = [outputs] + if labels is None: + self.labels = [None]*len(self.outputs) + elif isinstance(labels, (list, tuple)): + self.labels = list(labels) + else: + self.labels = [labels] + # Check for redundancy in inputs. if len(set(self.inputs)) != len(self.inputs): raise ValueError('The list of inputs passed to the model ' diff --git a/keras/engine/training.py b/keras/engine/training.py index 695e80c8fb4..ccc9ef8359d 100644 --- a/keras/engine/training.py +++ b/keras/engine/training.py @@ -822,13 +822,13 @@ def compile(self, optimizer, loss, metrics=None, loss_weights=None, else: shape = self.internal_output_shapes[i] name = self.output_names[i] - if K.is_placeholder(self.outputs[i]): + if self.labels[i] is None: target = K.placeholder(ndim=len(shape), name=name + '_target', sparse=K.is_sparse(self.outputs[i]), dtype=K.dtype(self.outputs[i])) else: - target = self.outputs[i] + target = self.labels[i] self.targets.append(target) self._feed_targets.append(target) @@ -1109,7 +1109,7 @@ def _fit_loop(self, f, ins, out_labels=None, batch_size=32, `History` object. """ do_validation = False - if val_f and val_ins: + if ins and val_f and val_ins: do_validation = True if verbose: print('Train on %d samples, validate on %d samples' % diff --git a/keras/legacy/interfaces.py b/keras/legacy/interfaces.py index be0c82f1258..73ee6b0d375 100644 --- a/keras/legacy/interfaces.py +++ b/keras/legacy/interfaces.py @@ -601,7 +601,8 @@ def generator_methods_args_preprocessor(args, kwargs): legacy_model_constructor_support = generate_legacy_interface( allowed_positional_args=None, conversions=[('input', 'inputs'), - ('output', 'outputs')]) + ('output', 'outputs'), + ('label', 'labels')]) legacy_input_support = generate_legacy_interface( allowed_positional_args=None, From 8ad7ec8a9c8ee59c88d90a313157dee28c6d0ec9 Mon Sep 17 00:00:00 2001 From: Andrew Hundt Date: Thu, 15 Jun 2017 17:36:07 -0400 Subject: [PATCH 016/166] mnist_tfrecord.py remove unused read_and_decode function --- examples/mnist_tfrecord.py | 31 ------------------------------- 1 file changed, 31 deletions(-) diff --git a/examples/mnist_tfrecord.py b/examples/mnist_tfrecord.py index c21beb105aa..ea452bf9cec 100644 --- a/examples/mnist_tfrecord.py +++ b/examples/mnist_tfrecord.py @@ -61,37 +61,6 @@ def _bytes_feature(value): else: print 'tfrecord %s already exists' % filename - -def read_and_decode(filename, one_hot=True, classes=None, is_train=None): - """ Return tensor to read from TFRecord """ - filename_queue = tf.train.string_input_producer([filename]) - reader = tf.TFRecordReader() - _, serialized_example = reader.read(filename_queue) - features = tf.parse_single_example( - serialized_example, - features={ - 'label': tf.FixedLenFeature([], tf.int64), - 'image_raw': tf.FixedLenFeature([], tf.string), - }) - img = tf.decode_raw(features['image_raw'], tf.uint8) - img.set_shape([28 * 28]) - img = tf.reshape(img, [28, 28, 1]) - - img = tf.cast(img, tf.float32) * (1. / 255) - 0.5 - - label = tf.cast(features['label'], tf.int32) - if one_hot and classes: - label = tf.one_hot(label, classes) - - x_train_batch, y_train_batch = K.tf.train.shuffle_batch( - [img, label], - batch_size=batch_size, - capacity=2000, - min_after_dequeue=1000, - num_threads=32) # set the number of threads here - - return x_train_batch, y_train_batch - def read_and_decode_recordinput(tf_glob, one_hot=True, classes=None, is_train=None, batch_shape=[1000, 28, 28, 1]): """ Return tensor to read from TFRecord """ with tf.variable_scope("TFRecords"): From 48f6d5a700e726af9a6cc2634ec9306b1e1afb9f Mon Sep 17 00:00:00 2001 From: Andrew Hundt Date: Thu, 15 Jun 2017 17:38:24 -0400 Subject: [PATCH 017/166] mnist_tfrecord.py progbar --- examples/mnist_tfrecord.py | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/examples/mnist_tfrecord.py b/examples/mnist_tfrecord.py index ea452bf9cec..6a4f7473b2f 100644 --- a/examples/mnist_tfrecord.py +++ b/examples/mnist_tfrecord.py @@ -17,6 +17,7 @@ from keras.callbacks import EarlyStopping, TensorBoard from keras.objectives import categorical_crossentropy from keras.utils import np_utils +from keras.utils.generic_utils import Progbar from keras import callbacks as cbks from keras import optimizers, objectives # from keras.engine.training import collect_metrics, weighted_objective @@ -63,15 +64,18 @@ def _bytes_feature(value): def read_and_decode_recordinput(tf_glob, one_hot=True, classes=None, is_train=None, batch_shape=[1000, 28, 28, 1]): """ Return tensor to read from TFRecord """ + print 'Creating graph for loading TFRecords...' with tf.variable_scope("TFRecords"): record_input = data_flow_ops.RecordInput(tf_glob, batch_size=batch_shape[0]) records_op = record_input.get_yield_op() records_op = tf.split(records_op, batch_shape[0], 0) records_op = [tf.reshape(record, []) for record in records_op] + progbar = Progbar(len(records_op)) - imgs = [] + images = [] labels = [] - for serialized_example in records_op: + for i, serialized_example in zip(range(len(records_op)), records_op): + progbar.update(i) with tf.variable_scope("parse_images", reuse=True): features = tf.parse_single_example( serialized_example, @@ -89,15 +93,16 @@ def read_and_decode_recordinput(tf_glob, one_hot=True, classes=None, is_train=No if one_hot and classes: label = tf.one_hot(label, classes) - imgs.append(img) + images.append(img) labels.append(label) - imgs = tf.parallel_stack(imgs, 0) + images = tf.parallel_stack(images, 0) labels = tf.parallel_stack(labels, 0) + images = tf.cast(images, tf.float32) - imgs = tf.cast(imgs, tf.float32) - imgs = tf.reshape(imgs, shape=batch_shape) - return imgs, labels + images = tf.reshape(images, shape=batch_shape) + + return images, labels def save_mnist_as_tfrecord(): From e8d6ad3c4543b141e9ea7d596eb37cabef0156aa Mon Sep 17 00:00:00 2001 From: Andrew Hundt Date: Thu, 15 Jun 2017 17:50:07 -0400 Subject: [PATCH 018/166] mnist_tfrecord.py network is closer to mnist_cnn.py --- examples/mnist_tfrecord.py | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/examples/mnist_tfrecord.py b/examples/mnist_tfrecord.py index 6a4f7473b2f..46efc4e77ca 100644 --- a/examples/mnist_tfrecord.py +++ b/examples/mnist_tfrecord.py @@ -13,7 +13,7 @@ from tensorflow.python.ops import data_flow_ops from keras import backend as K from keras.models import Model -from keras.layers import Dense, Dropout, Flatten, Input, Conv2D +from keras.layers import Dense, Dropout, Flatten, Input, Conv2D, MaxPooling2D from keras.callbacks import EarlyStopping, TensorBoard from keras.objectives import categorical_crossentropy from keras.utils import np_utils @@ -114,15 +114,14 @@ def save_mnist_as_tfrecord(): def cnn_layers(x_train_input): - con1 = Conv2D(32, (3, 3), activation='relu', - strides=(2, 2), padding='valid')(x_train_input) - con2 = Conv2D(32, (3, 3), activation='relu', - strides=(2, 2))(con1) - fla1 = Flatten()(con2) - den1 = Dense(128, activation='relu')(fla1) + x = Conv2D(32, (3, 3), activation='relu', padding='valid')(x_train_input) + x = Conv2D(64, (3, 3), activation='relu')(x) + x = MaxPooling2D(pool_size=(2, 2))(x) + x = Flatten()(x) + x = Dense(128, activation='relu')(x) x_train_out = Dense(classes, activation='softmax', - name='x_train_out')(den1) + name='x_train_out')(x) return x_train_out From bc2a0f4c3f15939de5c578662f817f607217594f Mon Sep 17 00:00:00 2001 From: Andrew Hundt Date: Thu, 15 Jun 2017 17:52:32 -0400 Subject: [PATCH 019/166] mnist_tfrecords.py add StagingArea --- examples/mnist_tfrecord.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/examples/mnist_tfrecord.py b/examples/mnist_tfrecord.py index 46efc4e77ca..3240acccc38 100644 --- a/examples/mnist_tfrecord.py +++ b/examples/mnist_tfrecord.py @@ -102,6 +102,18 @@ def read_and_decode_recordinput(tf_glob, one_hot=True, classes=None, is_train=No images = tf.reshape(images, shape=batch_shape) + # StagingArea will store tensors + # across multiple steps to + # speed up execution + images_shape = images.get_shape() + labels_shape = labels.get_shape() + copy_stage = data_flow_ops.StagingArea( + [tf.float32, tf.float32], + shapes=[images_shape, labels_shape]) + copy_stage_op = copy_stage.put( + [images, labels]) + staged_images, staged_labels = copy_stage.get() + return images, labels From 7ecef2983c5480bcc7496606875a0b8a8eee7ab8 Mon Sep 17 00:00:00 2001 From: Andrew Hundt Date: Thu, 15 Jun 2017 18:03:52 -0400 Subject: [PATCH 020/166] is_placeholder raised exceptions now return False --- keras/backend/common.py | 2 +- keras/backend/tensorflow_backend.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/keras/backend/common.py b/keras/backend/common.py index c5d40088b75..c4d98f92405 100644 --- a/keras/backend/common.py +++ b/keras/backend/common.py @@ -128,7 +128,7 @@ def is_placeholder(tensor): try: return tensor._is_placeholder except AttributeError: - return True + return False def image_data_format(): diff --git a/keras/backend/tensorflow_backend.py b/keras/backend/tensorflow_backend.py index ac3c92e61db..6bc06b58dc3 100644 --- a/keras/backend/tensorflow_backend.py +++ b/keras/backend/tensorflow_backend.py @@ -302,7 +302,7 @@ def is_placeholder(tensor): return True return tensor._is_placeholder except AttributeError: - return True + return False def variable(value, dtype=None, name=None): From a0639e1fd04ffbc26ff34c322a4cabe6cab02946 Mon Sep 17 00:00:00 2001 From: Andrew Hundt Date: Thu, 15 Jun 2017 18:09:02 -0400 Subject: [PATCH 021/166] mnist_tfrecord.py remove stray comment --- examples/mnist_tfrecord.py | 1 - 1 file changed, 1 deletion(-) diff --git a/examples/mnist_tfrecord.py b/examples/mnist_tfrecord.py index 3240acccc38..d040b388bf0 100644 --- a/examples/mnist_tfrecord.py +++ b/examples/mnist_tfrecord.py @@ -20,7 +20,6 @@ from keras.utils.generic_utils import Progbar from keras import callbacks as cbks from keras import optimizers, objectives -# from keras.engine.training import collect_metrics, weighted_objective from keras import metrics as metrics_module from keras.datasets import mnist From b7ea15f14a8a76a5656d226f6dd2da3e3ae2669f Mon Sep 17 00:00:00 2001 From: Andrew Hundt Date: Thu, 15 Jun 2017 21:07:00 -0400 Subject: [PATCH 022/166] training.py use append for lists instead of + operator, fixes crash when lists are empty and the learning phase is True --- keras/engine/training.py | 60 +++++++++++++++++----------------------- 1 file changed, 26 insertions(+), 34 deletions(-) diff --git a/keras/engine/training.py b/keras/engine/training.py index ccc9ef8359d..d691674b4a3 100644 --- a/keras/engine/training.py +++ b/keras/engine/training.py @@ -1065,10 +1065,9 @@ def _make_predict_function(self): if not hasattr(self, 'predict_function'): self.predict_function = None if self.predict_function is None: + inputs = self._feed_inputs if self.uses_learning_phase and not isinstance(K.learning_phase(), int): - inputs = self._feed_inputs + [K.learning_phase()] - else: - inputs = self._feed_inputs + inputs.append(K.learning_phase()) # Gets network outputs. Does not update weights. # Does update the network states. kwargs = getattr(self, '_function_kwargs', {}) @@ -1109,13 +1108,13 @@ def _fit_loop(self, f, ins, out_labels=None, batch_size=32, `History` object. """ do_validation = False - if ins and val_f and val_ins: - do_validation = True - if verbose: - print('Train on %d samples, validate on %d samples' % - (ins[0].shape[0], val_ins[0].shape[0])) - if ins and hasattr(ins[0], 'shape'): + if val_f and val_ins: + do_validation = True + if verbose: + print('Train on %d samples, validate on %d samples' % + (ins[0].shape[0], val_ins[0].shape[0])) + num_train_samples = ins[0].shape[0] else: # May happen if we are running `fit` without Numpy input data, @@ -1489,10 +1488,9 @@ class indices (integers) to batch_size=batch_size) self._make_test_function() val_f = self.test_function + val_ins = val_x + val_y + val_sample_weights if self.uses_learning_phase and not isinstance(K.learning_phase(), int): - val_ins = val_x + val_y + val_sample_weights + [0.] - else: - val_ins = val_x + val_y + val_sample_weights + val_ins.append(0.) elif validation_split and 0. < validation_split < 1.: do_validation = True @@ -1507,20 +1505,18 @@ class indices (integers) to _slice_arrays(sample_weights, split_at)) self._make_test_function() val_f = self.test_function + val_ins = val_x + val_y + val_sample_weights if self.uses_learning_phase and not isinstance(K.learning_phase(), int): - val_ins = val_x + val_y + val_sample_weights + [0.] - else: - val_ins = val_x + val_y + val_sample_weights + val_ins.append(0.) else: do_validation = False val_f = None val_ins = None # Prepare input arrays and training function. + ins = x + y + sample_weights if self.uses_learning_phase and not isinstance(K.learning_phase(), int): - ins = x + y + sample_weights + [1.] - else: - ins = x + y + sample_weights + ins.append(1.) self._make_train_function() f = self.train_function @@ -1574,10 +1570,10 @@ def evaluate(self, x, y, batch_size=32, verbose=1, sample_weight=None): check_batch_axis=False, batch_size=batch_size) # Prepare inputs, delegate logic to `_test_loop`. + ins = x + y + sample_weights if self.uses_learning_phase and not isinstance(K.learning_phase(), int): - ins = x + y + sample_weights + [0.] - else: - ins = x + y + sample_weights + ins.append(0.) + self._make_test_function() f = self.test_function return self._test_loop(f, ins, @@ -1618,10 +1614,9 @@ def predict(self, x, batch_size=32, verbose=0): 'Batch size: ' + str(batch_size) + '.') # Prepare inputs, delegate logic to `_predict_loop`. + ins = x if self.uses_learning_phase and not isinstance(K.learning_phase(), int): - ins = x + [0.] - else: - ins = x + ins.append(0.) self._make_predict_function() f = self.predict_function return self._predict_loop(f, ins, @@ -1668,10 +1663,9 @@ class indices (integers) to sample_weight=sample_weight, class_weight=class_weight, check_batch_axis=True) + ins = x + y + sample_weights if self.uses_learning_phase and not isinstance(K.learning_phase(), int): - ins = x + y + sample_weights + [1.] - else: - ins = x + y + sample_weights + ins.append(1.) self._make_train_function() outputs = self.train_function(ins) if len(outputs) == 1: @@ -1710,10 +1704,9 @@ def test_on_batch(self, x, y, sample_weight=None): x, y, sample_weight=sample_weight, check_batch_axis=True) + ins = x + y + sample_weights if self.uses_learning_phase and not isinstance(K.learning_phase(), int): - ins = x + y + sample_weights + [0.] - else: - ins = x + y + sample_weights + ins.append(0.) self._make_test_function() outputs = self.test_function(ins) if len(outputs) == 1: @@ -1731,10 +1724,9 @@ def predict_on_batch(self, x): """ x = _standardize_input_data(x, self._feed_input_names, self._feed_input_shapes) + ins = x if self.uses_learning_phase and not isinstance(K.learning_phase(), int): - ins = x + [0.] - else: - ins = x + ins.append(0.) self._make_predict_function() outputs = self.predict_function(ins) if len(outputs) == 1: @@ -1881,7 +1873,7 @@ def generate_arrays_from_file(path): val_x, val_y, val_sample_weight) val_data = val_x + val_y + val_sample_weights if self.uses_learning_phase and not isinstance(K.learning_phase(), int): - val_data += [0.] + val_data.append(0.) for cbk in callbacks: cbk.validation_data = val_data enqueuer = None From 36a36a0cb374dadd88e11589681072f4466b1b92 Mon Sep 17 00:00:00 2001 From: Andrew Hundt Date: Thu, 15 Jun 2017 21:07:57 -0400 Subject: [PATCH 023/166] tensorflow_backend.py prevent Function kwargs from accumulating inadvertently --- keras/backend/tensorflow_backend.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/keras/backend/tensorflow_backend.py b/keras/backend/tensorflow_backend.py index 6bc06b58dc3..922a46dad1c 100644 --- a/keras/backend/tensorflow_backend.py +++ b/keras/backend/tensorflow_backend.py @@ -2286,24 +2286,24 @@ def __init__(self, inputs, outputs, updates=None, name=None, def __call__(self, inputs): if not isinstance(inputs, (list, tuple)): raise TypeError('`inputs` should be a list or tuple.') + self.current_feed_dict = self.feed_dict if self.feed_dict is None: - self.feed_dict = {} + self.current_feed_dict = {} for tensor, value in zip(self.inputs, inputs): if is_sparse(tensor): sparse_coo = value.tocoo() indices = np.concatenate((np.expand_dims(sparse_coo.row, 1), np.expand_dims(sparse_coo.col, 1)), 1) value = (indices, sparse_coo.data, sparse_coo.shape) - self.feed_dict[tensor] = value - session = get_session() + self.current_feed_dict[tensor] = value + self.current_fetches = self.outputs + [self.updates_op] if self.fetches is not None: - fetches = self.outputs + [self.updates_op] + self.fetches - else: - fetches = self.outputs + [self.updates_op] + self.current_fetches += self.fetches - updated = session.run(fetches, - feed_dict=self.feed_dict, + session = get_session() + updated = session.run(fetches=self.current_fetches, + feed_dict=self.current_feed_dict, **self.session_kwargs) return updated[:len(self.outputs)] From 1a1183d7dbf8d23faef83091162f6f0be418b286 Mon Sep 17 00:00:00 2001 From: Andrew Hundt Date: Thu, 15 Jun 2017 21:30:16 -0400 Subject: [PATCH 024/166] Revert "training.py use append for lists instead of + operator, fixes crash when lists are empty and the learning phase is True" This reverts commit b7ea15f14a8a76a5656d226f6dd2da3e3ae2669f. --- keras/engine/training.py | 60 +++++++++++++++++++++++----------------- 1 file changed, 34 insertions(+), 26 deletions(-) diff --git a/keras/engine/training.py b/keras/engine/training.py index d691674b4a3..ccc9ef8359d 100644 --- a/keras/engine/training.py +++ b/keras/engine/training.py @@ -1065,9 +1065,10 @@ def _make_predict_function(self): if not hasattr(self, 'predict_function'): self.predict_function = None if self.predict_function is None: - inputs = self._feed_inputs if self.uses_learning_phase and not isinstance(K.learning_phase(), int): - inputs.append(K.learning_phase()) + inputs = self._feed_inputs + [K.learning_phase()] + else: + inputs = self._feed_inputs # Gets network outputs. Does not update weights. # Does update the network states. kwargs = getattr(self, '_function_kwargs', {}) @@ -1108,13 +1109,13 @@ def _fit_loop(self, f, ins, out_labels=None, batch_size=32, `History` object. """ do_validation = False - if ins and hasattr(ins[0], 'shape'): - if val_f and val_ins: - do_validation = True - if verbose: - print('Train on %d samples, validate on %d samples' % - (ins[0].shape[0], val_ins[0].shape[0])) + if ins and val_f and val_ins: + do_validation = True + if verbose: + print('Train on %d samples, validate on %d samples' % + (ins[0].shape[0], val_ins[0].shape[0])) + if ins and hasattr(ins[0], 'shape'): num_train_samples = ins[0].shape[0] else: # May happen if we are running `fit` without Numpy input data, @@ -1488,9 +1489,10 @@ class indices (integers) to batch_size=batch_size) self._make_test_function() val_f = self.test_function - val_ins = val_x + val_y + val_sample_weights if self.uses_learning_phase and not isinstance(K.learning_phase(), int): - val_ins.append(0.) + val_ins = val_x + val_y + val_sample_weights + [0.] + else: + val_ins = val_x + val_y + val_sample_weights elif validation_split and 0. < validation_split < 1.: do_validation = True @@ -1505,18 +1507,20 @@ class indices (integers) to _slice_arrays(sample_weights, split_at)) self._make_test_function() val_f = self.test_function - val_ins = val_x + val_y + val_sample_weights if self.uses_learning_phase and not isinstance(K.learning_phase(), int): - val_ins.append(0.) + val_ins = val_x + val_y + val_sample_weights + [0.] + else: + val_ins = val_x + val_y + val_sample_weights else: do_validation = False val_f = None val_ins = None # Prepare input arrays and training function. - ins = x + y + sample_weights if self.uses_learning_phase and not isinstance(K.learning_phase(), int): - ins.append(1.) + ins = x + y + sample_weights + [1.] + else: + ins = x + y + sample_weights self._make_train_function() f = self.train_function @@ -1570,10 +1574,10 @@ def evaluate(self, x, y, batch_size=32, verbose=1, sample_weight=None): check_batch_axis=False, batch_size=batch_size) # Prepare inputs, delegate logic to `_test_loop`. - ins = x + y + sample_weights if self.uses_learning_phase and not isinstance(K.learning_phase(), int): - ins.append(0.) - + ins = x + y + sample_weights + [0.] + else: + ins = x + y + sample_weights self._make_test_function() f = self.test_function return self._test_loop(f, ins, @@ -1614,9 +1618,10 @@ def predict(self, x, batch_size=32, verbose=0): 'Batch size: ' + str(batch_size) + '.') # Prepare inputs, delegate logic to `_predict_loop`. - ins = x if self.uses_learning_phase and not isinstance(K.learning_phase(), int): - ins.append(0.) + ins = x + [0.] + else: + ins = x self._make_predict_function() f = self.predict_function return self._predict_loop(f, ins, @@ -1663,9 +1668,10 @@ class indices (integers) to sample_weight=sample_weight, class_weight=class_weight, check_batch_axis=True) - ins = x + y + sample_weights if self.uses_learning_phase and not isinstance(K.learning_phase(), int): - ins.append(1.) + ins = x + y + sample_weights + [1.] + else: + ins = x + y + sample_weights self._make_train_function() outputs = self.train_function(ins) if len(outputs) == 1: @@ -1704,9 +1710,10 @@ def test_on_batch(self, x, y, sample_weight=None): x, y, sample_weight=sample_weight, check_batch_axis=True) - ins = x + y + sample_weights if self.uses_learning_phase and not isinstance(K.learning_phase(), int): - ins.append(0.) + ins = x + y + sample_weights + [0.] + else: + ins = x + y + sample_weights self._make_test_function() outputs = self.test_function(ins) if len(outputs) == 1: @@ -1724,9 +1731,10 @@ def predict_on_batch(self, x): """ x = _standardize_input_data(x, self._feed_input_names, self._feed_input_shapes) - ins = x if self.uses_learning_phase and not isinstance(K.learning_phase(), int): - ins.append(0.) + ins = x + [0.] + else: + ins = x self._make_predict_function() outputs = self.predict_function(ins) if len(outputs) == 1: @@ -1873,7 +1881,7 @@ def generate_arrays_from_file(path): val_x, val_y, val_sample_weight) val_data = val_x + val_y + val_sample_weights if self.uses_learning_phase and not isinstance(K.learning_phase(), int): - val_data.append(0.) + val_data += [0.] for cbk in callbacks: cbk.validation_data = val_data enqueuer = None From 9bf9b23af647e82ffa9ff117b9f87db65adec357 Mon Sep 17 00:00:00 2001 From: Andrew Hundt Date: Thu, 15 Jun 2017 21:32:38 -0400 Subject: [PATCH 025/166] training.py fix crash when ins does not have shape attribute --- keras/engine/training.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/keras/engine/training.py b/keras/engine/training.py index ccc9ef8359d..b3f9fa62256 100644 --- a/keras/engine/training.py +++ b/keras/engine/training.py @@ -1109,13 +1109,13 @@ def _fit_loop(self, f, ins, out_labels=None, batch_size=32, `History` object. """ do_validation = False - if ins and val_f and val_ins: - do_validation = True - if verbose: - print('Train on %d samples, validate on %d samples' % - (ins[0].shape[0], val_ins[0].shape[0])) if ins and hasattr(ins[0], 'shape'): + if ins and val_f and val_ins: + do_validation = True + if verbose: + print('Train on %d samples, validate on %d samples' % + (ins[0].shape[0], val_ins[0].shape[0])) num_train_samples = ins[0].shape[0] else: # May happen if we are running `fit` without Numpy input data, From 3d323eca24465e9756c144af3aa9578864e7f9f1 Mon Sep 17 00:00:00 2001 From: Andrew Hundt Date: Thu, 15 Jun 2017 21:36:45 -0400 Subject: [PATCH 026/166] training.py better verification of do_validation --- keras/engine/training.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/keras/engine/training.py b/keras/engine/training.py index b3f9fa62256..7e5430cf22a 100644 --- a/keras/engine/training.py +++ b/keras/engine/training.py @@ -1109,11 +1109,10 @@ def _fit_loop(self, f, ins, out_labels=None, batch_size=32, `History` object. """ do_validation = False - + if val_f and val_ins: + do_validation = True if ins and hasattr(ins[0], 'shape'): - if ins and val_f and val_ins: - do_validation = True - if verbose: + if val_f and val_ins and verbose: print('Train on %d samples, validate on %d samples' % (ins[0].shape[0], val_ins[0].shape[0])) num_train_samples = ins[0].shape[0] From aa2aa7e8b62a42e22d4d64dc6c542b50f730ff28 Mon Sep 17 00:00:00 2001 From: Andrew Hundt Date: Thu, 15 Jun 2017 22:42:33 -0400 Subject: [PATCH 027/166] mnist_tfrecord.py remove validation data during model.fit() because it isn't supported --- examples/mnist_tfrecord.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/examples/mnist_tfrecord.py b/examples/mnist_tfrecord.py index d040b388bf0..b008ac2e970 100644 --- a/examples/mnist_tfrecord.py +++ b/examples/mnist_tfrecord.py @@ -187,8 +187,7 @@ def create_cnn_model(x_train_batch, y_train_batch, x_batch_shape, y_batch_shape) train_model.summary() train_model.fit(batch_size=batch_size, epochs=epochs, - callbacks=[tensorboard], - validation_data=(x_test_batch, y_test_batch)) + callbacks=[tensorboard]) train_model.save_weights('saved_wt.h5') K.clear_session() From 4f383118e4645cafd9a0134bd08b29740e4f771d Mon Sep 17 00:00:00 2001 From: Andrew Hundt Date: Thu, 15 Jun 2017 22:44:10 -0400 Subject: [PATCH 028/166] pep8 --- examples/mnist_tfrecord.py | 1 + keras/engine/topology.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/examples/mnist_tfrecord.py b/examples/mnist_tfrecord.py index b008ac2e970..e5f917c23ae 100644 --- a/examples/mnist_tfrecord.py +++ b/examples/mnist_tfrecord.py @@ -61,6 +61,7 @@ def _bytes_feature(value): else: print 'tfrecord %s already exists' % filename + def read_and_decode_recordinput(tf_glob, one_hot=True, classes=None, is_train=None, batch_shape=[1000, 28, 28, 1]): """ Return tensor to read from TFRecord """ print 'Creating graph for loading TFRecords...' diff --git a/keras/engine/topology.py b/keras/engine/topology.py index e01d7e6da17..14dc3c0ef53 100644 --- a/keras/engine/topology.py +++ b/keras/engine/topology.py @@ -1502,7 +1502,7 @@ def __init__(self, inputs, outputs, labels=None, name=None): self.outputs = [outputs] if labels is None: - self.labels = [None]*len(self.outputs) + self.labels = [None] * len(self.outputs) elif isinstance(labels, (list, tuple)): self.labels = list(labels) else: From 773faaff367ef1912226cd0848874fcdd81101f4 Mon Sep 17 00:00:00 2001 From: Andrew Hundt Date: Thu, 15 Jun 2017 22:48:47 -0400 Subject: [PATCH 029/166] mnist_tfrecord.py one import per line --- examples/mnist_tfrecord.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/examples/mnist_tfrecord.py b/examples/mnist_tfrecord.py index e5f917c23ae..506817ffc13 100644 --- a/examples/mnist_tfrecord.py +++ b/examples/mnist_tfrecord.py @@ -13,8 +13,14 @@ from tensorflow.python.ops import data_flow_ops from keras import backend as K from keras.models import Model -from keras.layers import Dense, Dropout, Flatten, Input, Conv2D, MaxPooling2D -from keras.callbacks import EarlyStopping, TensorBoard +from keras.layers import Dense +from keras.layers import Dropout +from keras.layers import Flatten +from keras.layers import Input +from keras.layers import Conv2D +from keras.layers import MaxPooling2D +from keras.callbacks import EarlyStopping +from keras.layers import TensorBoard from keras.objectives import categorical_crossentropy from keras.utils import np_utils from keras.utils.generic_utils import Progbar From 6d4b898dbc72d622f832ae58b2466125a64b7c6f Mon Sep 17 00:00:00 2001 From: Andrew Hundt Date: Fri, 16 Jun 2017 12:05:14 -0400 Subject: [PATCH 030/166] tensorflow_backend.py clarify Function.__call__ according to review comments. --- keras/backend/tensorflow_backend.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/keras/backend/tensorflow_backend.py b/keras/backend/tensorflow_backend.py index 922a46dad1c..dff41bc6105 100644 --- a/keras/backend/tensorflow_backend.py +++ b/keras/backend/tensorflow_backend.py @@ -2286,9 +2286,7 @@ def __init__(self, inputs, outputs, updates=None, name=None, def __call__(self, inputs): if not isinstance(inputs, (list, tuple)): raise TypeError('`inputs` should be a list or tuple.') - self.current_feed_dict = self.feed_dict - if self.feed_dict is None: - self.current_feed_dict = {} + self.current_feed_dict = {} if self.feed_dict is None else self.feed_dict for tensor, value in zip(self.inputs, inputs): if is_sparse(tensor): sparse_coo = value.tocoo() From 40ac8c70619bb97bad1fb3aa038b39781a1d960c Mon Sep 17 00:00:00 2001 From: Andrew Hundt Date: Fri, 16 Jun 2017 12:08:07 -0400 Subject: [PATCH 031/166] test_training.py initial TFRecord test in progress. TFRecord saving and loading works, exercise implementation in progress. --- tests/keras/engine/test_training.py | 268 +++++++++++++++++++++++++++- 1 file changed, 259 insertions(+), 9 deletions(-) diff --git a/tests/keras/engine/test_training.py b/tests/keras/engine/test_training.py index 94bcb2ae8b7..220256b27b0 100644 --- a/tests/keras/engine/test_training.py +++ b/tests/keras/engine/test_training.py @@ -1,3 +1,4 @@ +import os import pytest import numpy as np from numpy.testing import assert_allclose @@ -87,7 +88,8 @@ def test_model_methods(): # predict_on_batch out = model.predict_on_batch([input_a_np, input_b_np]) - out = model.predict_on_batch({'input_a': input_a_np, 'input_b': input_b_np}) + out = model.predict_on_batch( + {'input_a': input_a_np, 'input_b': input_b_np}) # predict, evaluate input_a_np = np.random.random((10, 3)) @@ -96,7 +98,9 @@ def test_model_methods(): output_a_np = np.random.random((10, 4)) output_b_np = np.random.random((10, 3)) - out = model.evaluate([input_a_np, input_b_np], [output_a_np, output_b_np], batch_size=4) + out = model.evaluate([input_a_np, input_b_np], + [output_a_np, output_b_np], + batch_size=4) out = model.predict([input_a_np, input_b_np], batch_size=4) # with sample_weight @@ -194,8 +198,12 @@ def mse(y_true, y_pred): output_a_np = np.random.random((10, 4)) output_b_np = np.random.random((10, 3)) - out = model.fit([input_a_np, input_b_np], [output_a_np, output_b_np], batch_size=4, epochs=1) - out = model.evaluate([input_a_np, input_b_np], [output_a_np, output_b_np], batch_size=4) + out = model.fit([input_a_np, input_b_np], + [output_a_np, output_b_np], + batch_size=4, epochs=1) + out = model.evaluate([input_a_np, input_b_np], + [output_a_np, output_b_np], + batch_size=4) out = model.predict([input_a_np, input_b_np], batch_size=4) @@ -208,7 +216,8 @@ def test_sparse_input_validation_split(): test_output = np.random.random((6, 4)) model = Model(in1, out1) model.compile('rmsprop', 'mse') - model.fit(test_input, test_output, epochs=1, batch_size=2, validation_split=0.2) + model.fit(test_input, test_output, epochs=1, + batch_size=2, validation_split=0.2) @keras_test @@ -238,15 +247,18 @@ def test_trainable_argument(): @keras_test def test_check_not_failing(): a = np.random.random((2, 1, 3)) - _check_loss_and_target_compatibility([a], [K.categorical_crossentropy], [a.shape]) - _check_loss_and_target_compatibility([a], [K.categorical_crossentropy], [(2, None, 3)]) + _check_loss_and_target_compatibility( + [a], [K.categorical_crossentropy], [a.shape]) + _check_loss_and_target_compatibility( + [a], [K.categorical_crossentropy], [(2, None, 3)]) @keras_test def test_check_last_is_one(): a = np.random.random((2, 3, 1)) with pytest.raises(Exception) as exc: - _check_loss_and_target_compatibility([a], [K.categorical_crossentropy], [a.shape]) + _check_loss_and_target_compatibility( + [a], [K.categorical_crossentropy], [a.shape]) assert 'You are passing a target array' in str(exc) @@ -255,7 +267,8 @@ def test_check_last_is_one(): def test_check_bad_shape(): a = np.random.random((2, 3, 5)) with pytest.raises(Exception) as exc: - _check_loss_and_target_compatibility([a], [K.categorical_crossentropy], [(2, 3, 6)]) + _check_loss_and_target_compatibility( + [a], [K.categorical_crossentropy], [(2, 3, 6)]) assert 'targets to have the same shape' in str(exc) @@ -403,6 +416,243 @@ def test_model_with_input_feed_tensor(): assert out.shape == (10, 4) +@pytest.mark.skipif(K.backend() != 'tensorflow', reason='Requires TF backend') +@keras_test +def test_model_with_input_tfrecord(): + """We test building a model with a RecordInput as input. + We should be able to call fit, evaluate, predict, + by only passing them data for the placeholder inputs + in the model. + """ + import tensorflow as tf + from tensorflow.python.lib.io import tf_record + from tensorflow.python.ops import data_flow_ops + from tensorflow.python.platform import test + + """Tests for record_input_op.""" + def to_tfrecord(images, filename): + # If you plan to implement a TFRecord save function see mnist_tfrecord.py + def _int64_feature(value): + return tf.train.Feature(int64_list=tf.train.Int64List(value=[value])) + + def _bytes_feature(value): + return tf.train.Feature(bytes_list=tf.train.BytesList(value=[value])) + + """ Save data into TFRecord """ + if not os.path.isfile(filename): + num_examples = images.shape[0] + rows = images.shape[1] + cols = images.shape[2] + + writer = tf.python_io.TFRecordWriter(filename) + for index in range(num_examples): + image_raw = images[index].tostring() + example = tf.train.Example(features=tf.train.Features(feature={ + 'height': _int64_feature(rows), + 'width': _int64_feature(cols), + 'array': _bytes_feature(image_raw)})) + writer.write(example.SerializeToString()) + writer.close() + else: + print 'tfrecord %s already exists' % filename + + def read_and_decode_recordinput(tf_glob, batch_shape=[1]): + """ Return tensor to read from TFRecord """ + # If you plan to implement TFRecord reading and decoding see mnist_tfrecord.py + # This implementation is for brevity, not performance. + print 'Creating graph for loading TFRecords...' + with tf.variable_scope("TFRecords"): + record_input = data_flow_ops.RecordInput(tf_glob, batch_size=batch_shape[0]) + records_op = record_input.get_yield_op() + records_op = tf.split(records_op, batch_shape[0], 0) + records_op = [tf.reshape(record, []) for record in records_op] + + arrays = [] + for i, serialized_example in enumerate(records_op): + with tf.variable_scope("parse_array"): + features = tf.parse_single_example( + serialized_example, + features={ + 'height': tf.FixedLenFeature([], tf.int64), + 'width': tf.FixedLenFeature([], tf.int64), + 'array': tf.FixedLenFeature([], tf.string), + }) + array = tf.decode_raw(features['array'], tf.float32) + if len(batch_shape) == 3: + height = batch_shape[1] + width = batch_shape[2] + # static loading, faster and more supported + array.set_shape(height * width) + array = tf.reshape(array, [1, height, width]) + else: + # dynamic loading, not supported for many apis + height = tf.cast(features['height'], tf.int64) + width = tf.cast(features['width'], tf.int64) + array = tf.reshape(array, tf.stack([height, width])) + arrays.append(array) + + arrays = tf.stack(arrays, 0) + arrays = tf.cast(arrays, tf.float32) + return arrays + + batch_size = 10 + input_rows = 3 + cols = 1 + input_a_np = np.random.random([batch_size, input_rows, cols]) + to_tfrecord(input_a_np, 'input_a.tfrecord') + input_a_tf = read_and_decode_recordinput('input_a.tfrecord', input_a_np.shape) + + input_b_np = np.random.random([batch_size, input_rows, cols]) + to_tfrecord(input_a_np, 'input_b.tfrecord') + input_b_tf = read_and_decode_recordinput('input_b.tfrecord', input_b_np.shape) + + output_a_rows = 4 + output_a_np = np.random.random([batch_size, output_a_rows, cols]) + to_tfrecord(input_a_np, 'output_a.tfrecord') + output_a_tf = read_and_decode_recordinput('output_b.tfrecord', output_a_np.shape) + + output_b_rows = 3 + output_b_np = np.random.random([batch_size, output_b_rows, cols]) + to_tfrecord(input_a_np, 'output_b.tfrecord') + output_b_tf = read_and_decode_recordinput('output_b.tfrecord', output_b_np.shape) + + a = Input(tensor=input_a_tf) + # b = Input(shape=(3,), name='input_b') + b = Input(tensor=input_b_tf, name='input_b') + + a_2 = Dense(4, name='dense_1')(a) + dp = Dropout(0.5, name='dropout') + b_2 = dp(b) + + model = Model([a, b], [a_2, b_2], labels=[output_a_tf, output_b_tf]) + model.summary() + + optimizer = 'rmsprop' + loss = 'mse' + loss_weights = [1., 0.5] + model.compile(optimizer, loss, metrics=['mean_squared_error'], + loss_weights=loss_weights, + sample_weight_mode=None) + + # test train_on_batch + out = model.train_on_batch(input_b_np) + out = model.train_on_batch({'input_b': input_b_tf}, + [output_a_tf, output_b_tf]) + out = model.test_on_batch({'input_b': input_b_tf}, + [output_a_tf, output_b_tf]) + out = model.predict_on_batch({'input_b': input_b_tf}) + + # test fit + out = model.fit({'input_b': input_b_tf}, + [output_a_tf, output_b_tf], epochs=1, batch_size=10) + out = model.fit(input_b_tf, + [output_a_tf, output_b_tf], epochs=1, batch_size=10) + + # test evaluate + out = model.evaluate({'input_b': input_b_tf}, + [output_a_tf, output_b_tf], batch_size=10) + out = model.evaluate(input_b_tf, + [output_a_tf, output_b_tf], batch_size=10) + + # test predict + out = model.predict({'input_b': input_b_tf}, batch_size=10) + out = model.predict(input_b_tf, batch_size=10) + assert len(out) == 2 + + # Now test a model with a single input + # i.e. we don't pass any data to fit the model. + a = Input(tensor=tf.Variable(input_a_tf, dtype=tf.float32)) + a_2 = Dense(4, name='dense_1')(a) + a_2 = Dropout(0.5, name='dropout')(a_2) + model = Model(a, a_2) + model.summary() + + optimizer = 'rmsprop' + loss = 'mse' + model.compile(optimizer, loss, metrics=['mean_squared_error']) + + # test train_on_batch + out = model.train_on_batch(None, + output_a_tf) + out = model.train_on_batch(None, + output_a_tf) + out = model.test_on_batch(None, + output_a_tf) + out = model.predict_on_batch(None) + out = model.train_on_batch([], + output_a_tf) + out = model.train_on_batch({}, + output_a_tf) + + # test fit + out = model.fit(None, + output_a_tf, + epochs=1, + batch_size=10) + out = model.fit(None, + output_a_tf, + epochs=1, + batch_size=10) + + # test evaluate + out = model.evaluate(None, + output_a_tf, + batch_size=10) + out = model.evaluate(None, + output_a_tf, + batch_size=10) + + # test predict + out = model.predict(None, batch_size=10) + out = model.predict(None, batch_size=10) + assert out.shape == (10, 4) + + # Same, without learning phase + # i.e. we don't pass any data to fit the model. + a = Input(tensor=tf.Variable(input_a_tf, dtype=tf.float32)) + a_2 = Dense(4, name='dense_1')(a) + model = Model(a, a_2) + model.summary() + + optimizer = 'rmsprop' + loss = 'mse' + model.compile(optimizer, loss, metrics=['mean_squared_error']) + + # test train_on_batch + out = model.train_on_batch(None, + output_a_tf) + out = model.train_on_batch(None, + output_a_tf) + out = model.test_on_batch(None, + output_a_tf) + out = model.predict_on_batch(None) + out = model.train_on_batch([], + output_a_tf) + out = model.train_on_batch({}, + output_a_tf) + + # test fit + out = model.fit(None, + output_a_tf, epochs=1, batch_size=10) + out = model.fit(None, + output_a_tf, epochs=1, batch_size=10) + + # test evaluate + out = model.evaluate(None, + output_a_tf, batch_size=10) + out = model.evaluate(None, + output_a_tf, batch_size=10) + + # test predict + out = model.predict(None, batch_size=10) + out = model.predict(None, batch_size=10) + assert out.shape == (10, 4) + + os.remove('input_a.tfrecord') + os.remove('input_b.tfrecord') + os.remove('output_a.tfrecord') + os.remove('output_b.tfrecord') + @keras_test def test_model_with_partial_loss(): a = Input(shape=(3,), name='input_a') From 5954c61ff5d93cda51d553468f7f302e4669bd3e Mon Sep 17 00:00:00 2001 From: Andrew Hundt Date: Fri, 16 Jun 2017 16:31:06 -0400 Subject: [PATCH 032/166] training.py more stringent checks of inputs targets and weights --- keras/engine/training.py | 55 +++++++++++++++++++--------------------- 1 file changed, 26 insertions(+), 29 deletions(-) diff --git a/keras/engine/training.py b/keras/engine/training.py index 7e5430cf22a..474b157cf33 100644 --- a/keras/engine/training.py +++ b/keras/engine/training.py @@ -210,37 +210,34 @@ def _check_array_lengths(inputs, targets, weights=None): # Raises ValueError: in case of incorrectly formatted data. """ - x_lengths = [x.shape[0] for x in inputs] - set_x = set(x_lengths) + def set_of_lengths(x): + # return a set with the variation between of different shapes, with None => 0 + return set([0]) if x is None else set([0 if y is None else y.shape[0] for y in x]) + set_x = set_of_lengths(inputs) + set_y = set_of_lengths(targets) + set_w = set_of_lengths(weights) if len(set_x) > 1: raise ValueError('All input arrays (x) should have ' 'the same number of samples. Got array shapes: ' + str([x.shape for x in inputs])) - if targets: - y_lengths = [y.shape[0] for y in targets] - set_y = set(y_lengths) - if len(set_y) > 1: - raise ValueError('All target arrays (y) should have ' - 'the same number of samples. Got array shapes: ' + - str([y.shape for y in targets])) - if set_x and set_y and list(set_x)[0] != list(set_y)[0]: - raise ValueError('Input arrays should have ' - 'the same number of samples as target arrays. ' - 'Found ' + str(list(set_x)[0]) + ' input samples ' - 'and ' + str(list(set_y)[0]) + ' target samples.') - if weights: - w_lengths = [w.shape[0] for w in weights] - set_w = set(w_lengths) - if len(set_w) > 1: - raise ValueError('All sample_weight arrays should have ' - 'the same number of samples. Got array shapes: ' + - str([w.shape for w in weights])) - if targets and weights: - if set_y and set_w and list(set_y)[0] != list(set_w)[0]: - raise ValueError('Sample_weight arrays should have ' - 'the same number of samples as target arrays. Got ' + - str(list(set_y)[0]) + ' input samples and ' + - str(list(set_w)[0]) + ' target samples.') + if len(set_y) > 1: + raise ValueError('All target arrays (y) should have ' + 'the same number of samples. Got array shapes: ' + + str([y.shape for y in targets])) + if set_x and set_y and list(set_x)[0] != list(set_y)[0]: + raise ValueError('Input arrays should have ' + 'the same number of samples as target arrays. ' + 'Found ' + str(list(set_x)[0]) + ' input samples ' + 'and ' + str(list(set_y)[0]) + ' target samples.') + if len(set_w) > 1: + raise ValueError('All sample_weight arrays should have ' + 'the same number of samples. Got array shapes: ' + + str([w.shape for w in weights])) + if set_y and set_w and list(set_y)[0] != list(set_w)[0]: + raise ValueError('Sample_weight arrays should have ' + 'the same number of samples as target arrays. Got ' + + str(list(set_y)[0]) + ' input samples and ' + + str(list(set_w)[0]) + ' target samples.') def _check_loss_and_target_compatibility(targets, loss_fns, output_shapes): @@ -257,11 +254,11 @@ def _check_loss_and_target_compatibility(targets, loss_fns, output_shapes): ValueError: if a loss function or target array is incompatible with an output. """ - key_losses = {'mean_square_error', + key_losses = {'mean_squared_error', 'binary_crossentropy', 'categorical_crossentropy'} for y, loss, shape in zip(targets, loss_fns, output_shapes): - if loss is None: + if y is None or loss is None: continue if loss.__name__ == 'categorical_crossentropy': if y.shape[-1] == 1: From 98392b2336b3a8b1a38f9f947651ed0729ac687a Mon Sep 17 00:00:00 2001 From: Andrew Hundt Date: Fri, 16 Jun 2017 16:36:18 -0400 Subject: [PATCH 033/166] training.py checks tensor labels in case weights aren't supported and raises NotImplementedError. --- keras/engine/training.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/keras/engine/training.py b/keras/engine/training.py index 474b157cf33..2d698d2e5a5 100644 --- a/keras/engine/training.py +++ b/keras/engine/training.py @@ -804,6 +804,13 @@ def compile(self, optimizer, loss, metrics=None, loss_weights=None, # Prepare sample weights. if self._input_yield_op_tensors: + if not all(l is None for l in self.labels): + if sample_weight_mode is not None: + # tensor + sample weights not yet implemented + raise NotImplementedError + if loss_weights is not None: + # tensor + loss weights not yet implemented + raise NotImplementedError sample_weight_mode = 'disabled' else: sample_weights = self._prepare_sample_weights( From 748c2437afb9ecb40c59eb908fe399c6b709d32a Mon Sep 17 00:00:00 2001 From: Andrew Hundt Date: Fri, 16 Jun 2017 16:37:55 -0400 Subject: [PATCH 034/166] training.py Model checks y more stringently --- keras/engine/training.py | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/keras/engine/training.py b/keras/engine/training.py index 2d698d2e5a5..0592e22d6b6 100644 --- a/keras/engine/training.py +++ b/keras/engine/training.py @@ -1340,13 +1340,10 @@ def _standardize_user_data(self, x, y, self._feed_input_shapes, check_batch_axis=False, exception_prefix='input') - if not x and y is None and self.sample_weight_mode is 'disabled': - y = [] - else: - y = _standardize_input_data(y, self._feed_output_names, - output_shapes, - check_batch_axis=False, - exception_prefix='target') + y = _standardize_input_data(y, self._feed_output_names, + output_shapes, + check_batch_axis=False, + exception_prefix='target') class_weights = _standardize_class_weights(class_weight, self._feed_output_names) if self.sample_weight_mode is not 'disabled': From afcbaa7acf70888d5e997b111495c6100da8d4fa Mon Sep 17 00:00:00 2001 From: Andrew Hundt Date: Fri, 16 Jun 2017 17:17:39 -0400 Subject: [PATCH 035/166] training.py cleanup backend function calls --- keras/engine/training.py | 94 +++++++++++++++------------------------- 1 file changed, 35 insertions(+), 59 deletions(-) diff --git a/keras/engine/training.py b/keras/engine/training.py index 0592e22d6b6..35d197fa729 100644 --- a/keras/engine/training.py +++ b/keras/engine/training.py @@ -1026,61 +1026,37 @@ def _prepare_sample_weights(self, sample_weight_mode, skip_indices): self._feed_sample_weight_modes.append(self.sample_weight_modes[i]) return sample_weights - def _make_train_function(self): - if not hasattr(self, 'train_function'): + def _make_function(self, function_name): + if not hasattr(self, function_name): raise RuntimeError('You must compile your model before using it.') - if self.train_function is None: + if getattr(self, function_name) is None: inputs = self._feed_inputs + self._feed_targets if self.sample_weight_mode is not 'disabled': inputs += self._feed_sample_weights if self.uses_learning_phase and not isinstance(K.learning_phase(), int): inputs += [K.learning_phase()] - training_updates = self.optimizer.get_updates( - self._collected_trainable_weights, - self.constraints, - self.total_loss) - updates = self.updates + training_updates - # Gets loss and metrics. Updates weights at each call. - self.train_function = K.function(inputs, - [self.total_loss] + self.metrics_tensors, - updates=updates, - name='train_function', - **self._function_kwargs) - - def _make_test_function(self): - if not hasattr(self, 'test_function'): - raise RuntimeError('You must compile your model before using it.') - if self.test_function is None: - inputs = self._feed_inputs + self._feed_targets - if self.sample_weight_mode is not 'disabled': - inputs += self._feed_sample_weights - if self.uses_learning_phase and not isinstance(K.learning_phase(), int): - inputs += [K.learning_phase()] - # Return loss and metrics, no gradient updates. - # Does update the network states. - self.test_function = K.function(inputs, - [self.total_loss] + self.metrics_tensors, - updates=self.state_updates, - name='test_function', - **self._function_kwargs) - - def _make_predict_function(self): - if not hasattr(self, 'predict_function'): - self.predict_function = None - if self.predict_function is None: - if self.uses_learning_phase and not isinstance(K.learning_phase(), int): - inputs = self._feed_inputs + [K.learning_phase()] + if function_name is 'train_function': + training_updates = self.optimizer.get_updates( + self._collected_trainable_weights, + self.constraints, + self.total_loss) + updates = self.updates + training_updates else: - inputs = self._feed_inputs - # Gets network outputs. Does not update weights. - # Does update the network states. - kwargs = getattr(self, '_function_kwargs', {}) - self.predict_function = K.function(inputs, - self.outputs, - updates=self.state_updates, - name='predict_function', - **kwargs) + updates = self.state_updates + + if function_name is 'predict_function': + outputs = self.outputs + kwargs = getattr(self, '_function_kwargs', {}) + else: + outputs = [self.total_loss] + self.metrics_tensors + kwargs = self._function_kwargs + # Gets loss and metrics. Updates weights at each call. + setattr(self, function_name, K.function(inputs, + outputs, + updates=updates, + name=function_name, + **kwargs)) def _fit_loop(self, f, ins, out_labels=None, batch_size=32, epochs=100, verbose=1, callbacks=None, @@ -1487,7 +1463,7 @@ class indices (integers) to sample_weight=val_sample_weight, check_batch_axis=False, batch_size=batch_size) - self._make_test_function() + self._make_function('test_function') val_f = self.test_function if self.uses_learning_phase and not isinstance(K.learning_phase(), int): val_ins = val_x + val_y + val_sample_weights + [0.] @@ -1505,7 +1481,7 @@ class indices (integers) to sample_weights, val_sample_weights = ( _slice_arrays(sample_weights, 0, split_at), _slice_arrays(sample_weights, split_at)) - self._make_test_function() + self._make_function('test_function') val_f = self.test_function if self.uses_learning_phase and not isinstance(K.learning_phase(), int): val_ins = val_x + val_y + val_sample_weights + [0.] @@ -1521,7 +1497,7 @@ class indices (integers) to ins = x + y + sample_weights + [1.] else: ins = x + y + sample_weights - self._make_train_function() + self._make_function('train_function') f = self.train_function # Prepare display labels. @@ -1578,7 +1554,7 @@ def evaluate(self, x, y, batch_size=32, verbose=1, sample_weight=None): ins = x + y + sample_weights + [0.] else: ins = x + y + sample_weights - self._make_test_function() + self._make_function('test_function') f = self.test_function return self._test_loop(f, ins, batch_size=batch_size, @@ -1622,7 +1598,7 @@ def predict(self, x, batch_size=32, verbose=0): ins = x + [0.] else: ins = x - self._make_predict_function() + self._make_function('predict_function') f = self.predict_function return self._predict_loop(f, ins, batch_size=batch_size, verbose=verbose) @@ -1672,7 +1648,7 @@ class indices (integers) to ins = x + y + sample_weights + [1.] else: ins = x + y + sample_weights - self._make_train_function() + self._make_function('train_function') outputs = self.train_function(ins) if len(outputs) == 1: return outputs[0] @@ -1714,7 +1690,7 @@ def test_on_batch(self, x, y, sample_weight=None): ins = x + y + sample_weights + [0.] else: ins = x + y + sample_weights - self._make_test_function() + self._make_function('test_function') outputs = self.test_function(ins) if len(outputs) == 1: return outputs[0] @@ -1735,7 +1711,7 @@ def predict_on_batch(self, x): ins = x + [0.] else: ins = x - self._make_predict_function() + self._make_function('predict_function') outputs = self.predict_function(ins) if len(outputs) == 1: return outputs[0] @@ -1827,9 +1803,9 @@ def generate_arrays_from_file(path): epoch = initial_epoch do_validation = bool(validation_data) - self._make_train_function() + self._make_function('train_function') if do_validation: - self._make_test_function() + self._make_function('test_function') # python 2 has 'next', 3 has '__next__' # avoid any explicit version checks @@ -2016,7 +1992,7 @@ def evaluate_generator(self, generator, steps, ValueError: In case the generator yields data in an invalid format. """ - self._make_test_function() + self._make_function('test_function') steps_done = 0 wait_time = 0.01 @@ -2111,7 +2087,7 @@ def predict_generator(self, generator, steps, ValueError: In case the generator yields data in an invalid format. """ - self._make_predict_function() + self._make_function('predict_function') steps_done = 0 wait_time = 0.01 From 42fbdbc3a178b1884dee8c9f9e82779c9fc2808a Mon Sep 17 00:00:00 2001 From: Andrew Hundt Date: Fri, 16 Jun 2017 18:43:33 -0400 Subject: [PATCH 036/166] training.py don't slice None entries --- keras/engine/training.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/keras/engine/training.py b/keras/engine/training.py index 35d197fa729..5225a7a2d24 100644 --- a/keras/engine/training.py +++ b/keras/engine/training.py @@ -391,9 +391,9 @@ def _slice_arrays(arrays, start=None, stop=None): # hdf5 datasets only support list objects as indices if hasattr(start, 'shape'): start = start.tolist() - return [x[start] for x in arrays] + return [None if x is None else x[start] for x in arrays] else: - return [x[start:stop] for x in arrays] + return [None if x is None else x[start:stop] for x in arrays] else: if hasattr(start, '__len__'): if hasattr(start, 'shape'): From 75edea7a01152308c0c6a4815e010c6d6db4ee7b Mon Sep 17 00:00:00 2001 From: Andrew Hundt Date: Fri, 16 Jun 2017 18:46:07 -0400 Subject: [PATCH 037/166] topology.py always insert layer properties into feed --- keras/engine/topology.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/keras/engine/topology.py b/keras/engine/topology.py index 14dc3c0ef53..e7b0ad57dfa 100644 --- a/keras/engine/topology.py +++ b/keras/engine/topology.py @@ -1639,10 +1639,9 @@ def __init__(self, inputs, outputs, labels=None, name=None): i, layer.__class__.__name__)) self.input_names.append(layer.name) - if K.is_placeholder(layer): - self._feed_input_names.append(layer.name) - self._feed_inputs.append(layer.input) - self._feed_input_shapes.append(self.inputs[i]._keras_shape) + self._feed_input_names.append(layer.name) + self._feed_inputs.append(layer.input) + self._feed_input_shapes.append(self.inputs[i]._keras_shape) for layer in self.output_layers: self.output_names.append(layer.name) From caad65af11461dbe368c31e97630ec003da931cc Mon Sep 17 00:00:00 2001 From: Andrew Hundt Date: Fri, 16 Jun 2017 18:47:11 -0400 Subject: [PATCH 038/166] training.py define ins consistently --- keras/engine/training.py | 93 +++++++++++++++++----------------------- 1 file changed, 40 insertions(+), 53 deletions(-) diff --git a/keras/engine/training.py b/keras/engine/training.py index 5225a7a2d24..7948d65dcea 100644 --- a/keras/engine/training.py +++ b/keras/engine/training.py @@ -1026,10 +1026,23 @@ def _prepare_sample_weights(self, sample_weight_mode, skip_indices): self._feed_sample_weight_modes.append(self.sample_weight_modes[i]) return sample_weights + def _make_ins(self, x=None, y=None, sample_weights=None, learning_phase_value=None): + def add_if_valid(x): + # Don't include None values, and in a worst case return an empty list + return [] if x is None else [val for val in x if val is not None] + # ins = add_if_valid(x) + add_if_valid(y) + add_if_valid(sample_weights) + ins = x + y + sample_weights + if learning_phase_value and self.uses_learning_phase and not isinstance(K.learning_phase(), int): + ins = ins + learning_phase_value + return ins + def _make_function(self, function_name): if not hasattr(self, function_name): raise RuntimeError('You must compile your model before using it.') - if getattr(self, function_name) is None: + function = getattr(self, function_name) + if function is not None: + return function + else: inputs = self._feed_inputs + self._feed_targets if self.sample_weight_mode is not 'disabled': inputs += self._feed_sample_weights @@ -1052,11 +1065,11 @@ def _make_function(self, function_name): outputs = [self.total_loss] + self.metrics_tensors kwargs = self._function_kwargs # Gets loss and metrics. Updates weights at each call. - setattr(self, function_name, K.function(inputs, - outputs, - updates=updates, - name=function_name, - **kwargs)) + return K.function(inputs, + outputs, + updates=updates, + name=function_name, + **kwargs) def _fit_loop(self, f, ins, out_labels=None, batch_size=32, epochs=100, verbose=1, callbacks=None, @@ -1463,12 +1476,9 @@ class indices (integers) to sample_weight=val_sample_weight, check_batch_axis=False, batch_size=batch_size) - self._make_function('test_function') + self.test_function = self._make_function('test_function') val_f = self.test_function - if self.uses_learning_phase and not isinstance(K.learning_phase(), int): - val_ins = val_x + val_y + val_sample_weights + [0.] - else: - val_ins = val_x + val_y + val_sample_weights + val_ins = self._make_ins(val_x, val_y, val_sample_weights, [0.]) elif validation_split and 0. < validation_split < 1.: do_validation = True @@ -1481,23 +1491,17 @@ class indices (integers) to sample_weights, val_sample_weights = ( _slice_arrays(sample_weights, 0, split_at), _slice_arrays(sample_weights, split_at)) - self._make_function('test_function') + self.test_function = self._make_function('test_function') val_f = self.test_function - if self.uses_learning_phase and not isinstance(K.learning_phase(), int): - val_ins = val_x + val_y + val_sample_weights + [0.] - else: - val_ins = val_x + val_y + val_sample_weights + val_ins = self._make_ins(val_x, val_y, val_sample_weights, [0.]) else: do_validation = False val_f = None val_ins = None # Prepare input arrays and training function. - if self.uses_learning_phase and not isinstance(K.learning_phase(), int): - ins = x + y + sample_weights + [1.] - else: - ins = x + y + sample_weights - self._make_function('train_function') + ins = self._make_ins(x, y, sample_weights, [1.]) + self.train_function = self._make_function('train_function') f = self.train_function # Prepare display labels. @@ -1550,11 +1554,8 @@ def evaluate(self, x, y, batch_size=32, verbose=1, sample_weight=None): check_batch_axis=False, batch_size=batch_size) # Prepare inputs, delegate logic to `_test_loop`. - if self.uses_learning_phase and not isinstance(K.learning_phase(), int): - ins = x + y + sample_weights + [0.] - else: - ins = x + y + sample_weights - self._make_function('test_function') + ins = self._make_ins(x, y, sample_weights, [0.]) + self.test_function = self._make_function('test_function') f = self.test_function return self._test_loop(f, ins, batch_size=batch_size, @@ -1594,11 +1595,8 @@ def predict(self, x, batch_size=32, verbose=0): 'Batch size: ' + str(batch_size) + '.') # Prepare inputs, delegate logic to `_predict_loop`. - if self.uses_learning_phase and not isinstance(K.learning_phase(), int): - ins = x + [0.] - else: - ins = x - self._make_function('predict_function') + ins = self._make_ins(x, None, None, [0.]) + self.predict_function = self._make_function('predict_function') f = self.predict_function return self._predict_loop(f, ins, batch_size=batch_size, verbose=verbose) @@ -1644,11 +1642,8 @@ class indices (integers) to sample_weight=sample_weight, class_weight=class_weight, check_batch_axis=True) - if self.uses_learning_phase and not isinstance(K.learning_phase(), int): - ins = x + y + sample_weights + [1.] - else: - ins = x + y + sample_weights - self._make_function('train_function') + ins = self._make_ins(x, y, sample_weights, [1.]) + self.train_function = self._make_function('train_function') outputs = self.train_function(ins) if len(outputs) == 1: return outputs[0] @@ -1686,11 +1681,8 @@ def test_on_batch(self, x, y, sample_weight=None): x, y, sample_weight=sample_weight, check_batch_axis=True) - if self.uses_learning_phase and not isinstance(K.learning_phase(), int): - ins = x + y + sample_weights + [0.] - else: - ins = x + y + sample_weights - self._make_function('test_function') + ins = self._make_ins(x, y, sample_weights, [0.]) + self.test_function = self._make_function('test_function') outputs = self.test_function(ins) if len(outputs) == 1: return outputs[0] @@ -1707,11 +1699,8 @@ def predict_on_batch(self, x): """ x = _standardize_input_data(x, self._feed_input_names, self._feed_input_shapes) - if self.uses_learning_phase and not isinstance(K.learning_phase(), int): - ins = x + [0.] - else: - ins = x - self._make_function('predict_function') + ins = self._make_ins(x, None, None, [0.]) + self.predict_function = self._make_function('predict_function') outputs = self.predict_function(ins) if len(outputs) == 1: return outputs[0] @@ -1803,9 +1792,9 @@ def generate_arrays_from_file(path): epoch = initial_epoch do_validation = bool(validation_data) - self._make_function('train_function') + self.train_function = self._make_function('train_function') if do_validation: - self._make_function('test_function') + self.test_function = self._make_function('test_function') # python 2 has 'next', 3 has '__next__' # avoid any explicit version checks @@ -1855,9 +1844,7 @@ def generate_arrays_from_file(path): str(validation_data)) val_x, val_y, val_sample_weights = self._standardize_user_data( val_x, val_y, val_sample_weight) - val_data = val_x + val_y + val_sample_weights - if self.uses_learning_phase and not isinstance(K.learning_phase(), int): - val_data += [0.] + val_data = self._make_ins(val_x, val_y, val_sample_weights, [0.]) for cbk in callbacks: cbk.validation_data = val_data enqueuer = None @@ -1992,7 +1979,7 @@ def evaluate_generator(self, generator, steps, ValueError: In case the generator yields data in an invalid format. """ - self._make_function('test_function') + self.test_function = self._make_function('test_function') steps_done = 0 wait_time = 0.01 @@ -2087,7 +2074,7 @@ def predict_generator(self, generator, steps, ValueError: In case the generator yields data in an invalid format. """ - self._make_function('predict_function') + self.predict_function = self._make_function('predict_function') steps_done = 0 wait_time = 0.01 From 845dd8858ddfbe17d85aecfae0d603a2aa2ad367 Mon Sep 17 00:00:00 2001 From: Andrew Hundt Date: Fri, 16 Jun 2017 18:47:57 -0400 Subject: [PATCH 039/166] tensorflow_backend.py if feed_dict entry is None add it to fetches instead --- keras/backend/tensorflow_backend.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/keras/backend/tensorflow_backend.py b/keras/backend/tensorflow_backend.py index dff41bc6105..69bba5fe8db 100644 --- a/keras/backend/tensorflow_backend.py +++ b/keras/backend/tensorflow_backend.py @@ -2287,15 +2287,20 @@ def __call__(self, inputs): if not isinstance(inputs, (list, tuple)): raise TypeError('`inputs` should be a list or tuple.') self.current_feed_dict = {} if self.feed_dict is None else self.feed_dict + self.feed_to_fetch_count = 0 + self.current_fetches = self.outputs + [self.updates_op] for tensor, value in zip(self.inputs, inputs): if is_sparse(tensor): sparse_coo = value.tocoo() indices = np.concatenate((np.expand_dims(sparse_coo.row, 1), np.expand_dims(sparse_coo.col, 1)), 1) value = (indices, sparse_coo.data, sparse_coo.shape) - self.current_feed_dict[tensor] = value + if value is None and tensor is not None: + self.feed_to_fetch_count += 1 + self.current_fetches.append(tensor) + else: + self.current_feed_dict[tensor] = value - self.current_fetches = self.outputs + [self.updates_op] if self.fetches is not None: self.current_fetches += self.fetches From 39a798158068606591d8679310954c173052ebf2 Mon Sep 17 00:00:00 2001 From: Andrew Hundt Date: Fri, 16 Jun 2017 23:32:59 -0400 Subject: [PATCH 040/166] trainin.py ins should at least include an empty list --- keras/engine/training.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/keras/engine/training.py b/keras/engine/training.py index 7948d65dcea..8d491345f92 100644 --- a/keras/engine/training.py +++ b/keras/engine/training.py @@ -1028,10 +1028,10 @@ def _prepare_sample_weights(self, sample_weight_mode, skip_indices): def _make_ins(self, x=None, y=None, sample_weights=None, learning_phase_value=None): def add_if_valid(x): - # Don't include None values, and in a worst case return an empty list - return [] if x is None else [val for val in x if val is not None] + # In a worst case return an empty list + return [None] if x is None else x # ins = add_if_valid(x) + add_if_valid(y) + add_if_valid(sample_weights) - ins = x + y + sample_weights + ins = add_if_valid(x) + add_if_valid(y) + add_if_valid(sample_weights) if learning_phase_value and self.uses_learning_phase and not isinstance(K.learning_phase(), int): ins = ins + learning_phase_value return ins From c0d0cdaa2f431529c9cd812e23bef468b22bea24 Mon Sep 17 00:00:00 2001 From: Andrew Hundt Date: Fri, 16 Jun 2017 23:33:20 -0400 Subject: [PATCH 041/166] training.py first TFRecord test passes! --- tests/keras/engine/test_training.py | 262 ++++++++++------------------ 1 file changed, 92 insertions(+), 170 deletions(-) diff --git a/tests/keras/engine/test_training.py b/tests/keras/engine/test_training.py index 220256b27b0..df2f2d8cc8f 100644 --- a/tests/keras/engine/test_training.py +++ b/tests/keras/engine/test_training.py @@ -5,6 +5,7 @@ import scipy.sparse as sparse from keras.layers import Dense, Dropout +from keras.layers import Flatten from keras.engine.topology import Input from keras.engine.training import Model, _check_loss_and_target_compatibility from keras.models import Sequential @@ -429,9 +430,8 @@ def test_model_with_input_tfrecord(): from tensorflow.python.ops import data_flow_ops from tensorflow.python.platform import test - """Tests for record_input_op.""" - def to_tfrecord(images, filename): - # If you plan to implement a TFRecord save function see mnist_tfrecord.py + def images_to_tfrecord(images, labels, filename): + def _int64_feature(value): return tf.train.Feature(int64_list=tf.train.Int64List(value=[value])) @@ -441,25 +441,28 @@ def _bytes_feature(value): """ Save data into TFRecord """ if not os.path.isfile(filename): num_examples = images.shape[0] + rows = images.shape[1] cols = images.shape[2] + depth = images.shape[3] + print('Writing', filename) writer = tf.python_io.TFRecordWriter(filename) for index in range(num_examples): image_raw = images[index].tostring() example = tf.train.Example(features=tf.train.Features(feature={ 'height': _int64_feature(rows), 'width': _int64_feature(cols), - 'array': _bytes_feature(image_raw)})) + 'depth': _int64_feature(depth), + 'label': _int64_feature(int(labels[index])), + 'image_raw': _bytes_feature(image_raw)})) writer.write(example.SerializeToString()) writer.close() else: print 'tfrecord %s already exists' % filename - def read_and_decode_recordinput(tf_glob, batch_shape=[1]): + def read_and_decode_recordinput(tf_glob, one_hot=True, classes=None, is_train=None, batch_shape=[10, 3, 3, 1]): """ Return tensor to read from TFRecord """ - # If you plan to implement TFRecord reading and decoding see mnist_tfrecord.py - # This implementation is for brevity, not performance. print 'Creating graph for loading TFRecords...' with tf.variable_scope("TFRecords"): record_input = data_flow_ops.RecordInput(tf_glob, batch_size=batch_shape[0]) @@ -467,191 +470,110 @@ def read_and_decode_recordinput(tf_glob, batch_shape=[1]): records_op = tf.split(records_op, batch_shape[0], 0) records_op = [tf.reshape(record, []) for record in records_op] - arrays = [] + images = [] + labels = [] for i, serialized_example in enumerate(records_op): - with tf.variable_scope("parse_array"): + with tf.variable_scope("parse_images", reuse=True): features = tf.parse_single_example( serialized_example, features={ - 'height': tf.FixedLenFeature([], tf.int64), - 'width': tf.FixedLenFeature([], tf.int64), - 'array': tf.FixedLenFeature([], tf.string), + 'label': tf.FixedLenFeature([], tf.int64), + 'image_raw': tf.FixedLenFeature([], tf.string), }) - array = tf.decode_raw(features['array'], tf.float32) - if len(batch_shape) == 3: - height = batch_shape[1] - width = batch_shape[2] - # static loading, faster and more supported - array.set_shape(height * width) - array = tf.reshape(array, [1, height, width]) - else: - # dynamic loading, not supported for many apis - height = tf.cast(features['height'], tf.int64) - width = tf.cast(features['width'], tf.int64) - array = tf.reshape(array, tf.stack([height, width])) - arrays.append(array) - - arrays = tf.stack(arrays, 0) - arrays = tf.cast(arrays, tf.float32) - return arrays - - batch_size = 10 - input_rows = 3 - cols = 1 - input_a_np = np.random.random([batch_size, input_rows, cols]) - to_tfrecord(input_a_np, 'input_a.tfrecord') - input_a_tf = read_and_decode_recordinput('input_a.tfrecord', input_a_np.shape) - - input_b_np = np.random.random([batch_size, input_rows, cols]) - to_tfrecord(input_a_np, 'input_b.tfrecord') - input_b_tf = read_and_decode_recordinput('input_b.tfrecord', input_b_np.shape) - - output_a_rows = 4 - output_a_np = np.random.random([batch_size, output_a_rows, cols]) - to_tfrecord(input_a_np, 'output_a.tfrecord') - output_a_tf = read_and_decode_recordinput('output_b.tfrecord', output_a_np.shape) - - output_b_rows = 3 - output_b_np = np.random.random([batch_size, output_b_rows, cols]) - to_tfrecord(input_a_np, 'output_b.tfrecord') - output_b_tf = read_and_decode_recordinput('output_b.tfrecord', output_b_np.shape) - - a = Input(tensor=input_a_tf) - # b = Input(shape=(3,), name='input_b') - b = Input(tensor=input_b_tf, name='input_b') - - a_2 = Dense(4, name='dense_1')(a) - dp = Dropout(0.5, name='dropout') - b_2 = dp(b) - - model = Model([a, b], [a_2, b_2], labels=[output_a_tf, output_b_tf]) - model.summary() - - optimizer = 'rmsprop' - loss = 'mse' - loss_weights = [1., 0.5] - model.compile(optimizer, loss, metrics=['mean_squared_error'], - loss_weights=loss_weights, - sample_weight_mode=None) + img = tf.decode_raw(features['image_raw'], tf.uint8) + img.set_shape(batch_shape[1] * batch_shape[2]) + img = tf.reshape(img, [1] + batch_shape[1:]) - # test train_on_batch - out = model.train_on_batch(input_b_np) - out = model.train_on_batch({'input_b': input_b_tf}, - [output_a_tf, output_b_tf]) - out = model.test_on_batch({'input_b': input_b_tf}, - [output_a_tf, output_b_tf]) - out = model.predict_on_batch({'input_b': input_b_tf}) + img = tf.cast(img, tf.float32) * (1. / 255) - 0.5 - # test fit - out = model.fit({'input_b': input_b_tf}, - [output_a_tf, output_b_tf], epochs=1, batch_size=10) - out = model.fit(input_b_tf, - [output_a_tf, output_b_tf], epochs=1, batch_size=10) + label = tf.cast(features['label'], tf.int32) + if one_hot and classes: + label = tf.one_hot(label, classes) - # test evaluate - out = model.evaluate({'input_b': input_b_tf}, - [output_a_tf, output_b_tf], batch_size=10) - out = model.evaluate(input_b_tf, - [output_a_tf, output_b_tf], batch_size=10) + images.append(img) + labels.append(label) - # test predict - out = model.predict({'input_b': input_b_tf}, batch_size=10) - out = model.predict(input_b_tf, batch_size=10) - assert len(out) == 2 + images = tf.parallel_stack(images, 0) + labels = tf.parallel_stack(labels, 0) + images = tf.cast(images, tf.float32) - # Now test a model with a single input - # i.e. we don't pass any data to fit the model. - a = Input(tensor=tf.Variable(input_a_tf, dtype=tf.float32)) - a_2 = Dense(4, name='dense_1')(a) - a_2 = Dropout(0.5, name='dropout')(a_2) - model = Model(a, a_2) - model.summary() + images = tf.reshape(images, shape=batch_shape) - optimizer = 'rmsprop' - loss = 'mse' - model.compile(optimizer, loss, metrics=['mean_squared_error']) + return images, labels - # test train_on_batch - out = model.train_on_batch(None, - output_a_tf) - out = model.train_on_batch(None, - output_a_tf) - out = model.test_on_batch(None, - output_a_tf) - out = model.predict_on_batch(None) - out = model.train_on_batch([], - output_a_tf) - out = model.train_on_batch({}, - output_a_tf) + def replace(filename): + if os.path.isfile(filename): + print '%s already exists, replacing...' % filename + os.remove(filename) - # test fit - out = model.fit(None, - output_a_tf, - epochs=1, - batch_size=10) - out = model.fit(None, - output_a_tf, - epochs=1, - batch_size=10) - - # test evaluate - out = model.evaluate(None, - output_a_tf, - batch_size=10) - out = model.evaluate(None, - output_a_tf, - batch_size=10) - - # test predict - out = model.predict(None, batch_size=10) - out = model.predict(None, batch_size=10) - assert out.shape == (10, 4) - - # Same, without learning phase - # i.e. we don't pass any data to fit the model. - a = Input(tensor=tf.Variable(input_a_tf, dtype=tf.float32)) - a_2 = Dense(4, name='dense_1')(a) - model = Model(a, a_2) + batch_size = 10 + input_rows = 3 + cols = 3 + depth = 1 + classes = 2 + img_batch_shape = [batch_size, input_rows, cols, depth] + label_batch_shape = [batch_size, classes] + input_a_np = np.multiply(np.random.random(img_batch_shape), batch_size) + input_a_np = input_a_np.astype('uint8') + output_a_rows = 4 + output_a_np = np.multiply(np.random.random([batch_size]), batch_size) + output_a_np = output_a_np.astype('int') + replace('input_a.tfrecord') + images_to_tfrecord(input_a_np, output_a_np, 'input_a.tfrecord') + input_a_tf, output_b_tf = read_and_decode_recordinput( + 'input_a.tfrecord', + one_hot=True, + classes=classes, + is_train=True, + batch_shape=img_batch_shape) + + a = Input(tensor=input_a_tf, batch_shape=img_batch_shape) + + a_2 = Dense(8, name='dense_1')(a) + dp = Dropout(0.5, name='dropout') + b_2 = dp(a_2) + f_1 = Flatten()(b_2) + b_3 = Dense( + classes, + activation='softmax', + name='dense_b3')(f_1) + + y_train_in_out = Input( + tensor=output_b_tf, + batch_shape=label_batch_shape, + name='y_labels') + + model = Model([a], [b_3], labels=[y_train_in_out]) model.summary() optimizer = 'rmsprop' loss = 'mse' - model.compile(optimizer, loss, metrics=['mean_squared_error']) - - # test train_on_batch - out = model.train_on_batch(None, - output_a_tf) - out = model.train_on_batch(None, - output_a_tf) - out = model.test_on_batch(None, - output_a_tf) - out = model.predict_on_batch(None) - out = model.train_on_batch([], - output_a_tf) - out = model.train_on_batch({}, - output_a_tf) + loss_weights = [1.] + try: + with pytest.raises(NotImplementedError) as exc: + model.compile(optimizer, loss, metrics=['mean_squared_error'], + loss_weights=loss_weights, + sample_weight_mode=None) + except NotImplementedError: + print 'Feature not implemented' + model.compile(optimizer, loss, metrics=['mean_squared_error'], + sample_weight_mode=None) + out = model.train_on_batch(None, None) # test fit out = model.fit(None, - output_a_tf, epochs=1, batch_size=10) - out = model.fit(None, - output_a_tf, epochs=1, batch_size=10) - - # test evaluate - out = model.evaluate(None, - output_a_tf, batch_size=10) - out = model.evaluate(None, - output_a_tf, batch_size=10) - - # test predict - out = model.predict(None, batch_size=10) - out = model.predict(None, batch_size=10) - assert out.shape == (10, 4) + None, epochs=1, batch_size=10) + try: + with pytest.raises(ValueError) as exc: + out = model.fit(output_a_np, + [y_train_in_out], + epochs=1, + batch_size=10) + except ValueError: + print 'cannot mix input tensors and numpy arrays' os.remove('input_a.tfrecord') - os.remove('input_b.tfrecord') - os.remove('output_a.tfrecord') - os.remove('output_b.tfrecord') + @keras_test def test_model_with_partial_loss(): From 6248b5454f9814b9a11a9e93078f34409a1290ef Mon Sep 17 00:00:00 2001 From: Andrew Hundt Date: Sat, 17 Jun 2017 04:03:21 -0400 Subject: [PATCH 042/166] predict_on_batch runs --- keras/engine/training.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/keras/engine/training.py b/keras/engine/training.py index 8d491345f92..1de6c22cbd4 100644 --- a/keras/engine/training.py +++ b/keras/engine/training.py @@ -1033,7 +1033,7 @@ def add_if_valid(x): # ins = add_if_valid(x) + add_if_valid(y) + add_if_valid(sample_weights) ins = add_if_valid(x) + add_if_valid(y) + add_if_valid(sample_weights) if learning_phase_value and self.uses_learning_phase and not isinstance(K.learning_phase(), int): - ins = ins + learning_phase_value + ins = ins + add_if_valid(learning_phase_value) return ins def _make_function(self, function_name): @@ -1699,7 +1699,7 @@ def predict_on_batch(self, x): """ x = _standardize_input_data(x, self._feed_input_names, self._feed_input_shapes) - ins = self._make_ins(x, None, None, [0.]) + ins = self._make_ins(x, None, [], [0.]) self.predict_function = self._make_function('predict_function') outputs = self.predict_function(ins) if len(outputs) == 1: From b8dae763daa99550e0b58c12c30d730a7edebd3d Mon Sep 17 00:00:00 2001 From: Andrew Hundt Date: Sat, 17 Jun 2017 04:38:05 -0400 Subject: [PATCH 043/166] self._prepare_sample_weights(sample_weight_mode, skip_indices) --- keras/engine/training.py | 167 ++++++++++++++++++++------------------- 1 file changed, 84 insertions(+), 83 deletions(-) diff --git a/keras/engine/training.py b/keras/engine/training.py index 1de6c22cbd4..d72a40fafa6 100644 --- a/keras/engine/training.py +++ b/keras/engine/training.py @@ -803,19 +803,7 @@ def compile(self, optimizer, loss, metrics=None, loss_weights=None, ' - expected a list of dicts.') # Prepare sample weights. - if self._input_yield_op_tensors: - if not all(l is None for l in self.labels): - if sample_weight_mode is not None: - # tensor + sample weights not yet implemented - raise NotImplementedError - if loss_weights is not None: - # tensor + loss weights not yet implemented - raise NotImplementedError - sample_weight_mode = 'disabled' - else: - sample_weights = self._prepare_sample_weights( - sample_weight_mode, skip_indices) - self.sample_weight_mode = sample_weight_mode + self._prepare_sample_weights(sample_weight_mode, skip_indices) # Prepare targets of model. self.targets = [] @@ -951,80 +939,93 @@ def append_metric(layer_num, metric_name, metric_tensor): def _prepare_sample_weights(self, sample_weight_mode, skip_indices): # Prepare sample weights. - sample_weights = [] - sample_weight_modes = [] - if isinstance(sample_weight_mode, dict): - for name in sample_weight_mode: - if name not in self.output_names: - raise ValueError('Unknown entry in ' - 'sample_weight_mode dictionary: "' + - name + '". ' - 'Only expected the following keys: ' + - str(self.output_names)) - for i, name in enumerate(self.output_names): - if i in skip_indices: - weight = None - sample_weight_modes.append(None) - else: - if name not in sample_weight_mode: - raise ValueError('Output "' + name + - '" missing from sample_weight_modes ' - 'dictionary') - if sample_weight_mode.get(name) == 'temporal': - weight = K.placeholder(ndim=2, - name=name + '_sample_weights') - sample_weight_modes.append('temporal') - else: - weight = K.placeholder(ndim=1, - name=name + '_sample_weights') + if self._input_yield_op_tensors: + if not all(l is None for l in self.labels): + if sample_weight_mode is not None: + # tensor + sample weights not yet implemented + raise NotImplementedError + if self.loss_weights is not None: + # tensor + loss weights not yet implemented + raise NotImplementedError + sample_weight_mode = 'disabled' + if sample_weight_mode is not 'disabled': + sample_weights = self._prepare_sample_weights( + sample_weight_mode, skip_indices) + self.sample_weight_mode = sample_weight_mode + sample_weights = [] + sample_weight_modes = [] + if isinstance(sample_weight_mode, dict): + for name in sample_weight_mode: + if name not in self.output_names: + raise ValueError('Unknown entry in ' + 'sample_weight_mode dictionary: "' + + name + '". ' + 'Only expected the following keys: ' + + str(self.output_names)) + for i, name in enumerate(self.output_names): + if i in skip_indices: + weight = None sample_weight_modes.append(None) - sample_weights.append(weight) - elif isinstance(sample_weight_mode, list): - if len(sample_weight_mode) != len(self.outputs): - raise ValueError('When passing a list as sample_weight_mode, ' - 'it should have one entry per model outputs. ' - 'The model has ' + str(len(self.outputs)) + - ' outputs, but you passed ' - 'sample_weight_mode=' + - str(sample_weight_mode)) - for i in range(len(self.output_names)): - if i in skip_indices: - weight = None - sample_weight_modes.append(None) - else: - mode = sample_weight_mode[i] - name = self.output_names[i] - if mode == 'temporal': - weight = K.placeholder(ndim=2, - name=name + '_sample_weights') - sample_weight_modes.append('temporal') else: - weight = K.placeholder(ndim=1, - name=name + '_sample_weights') + if name not in sample_weight_mode: + raise ValueError('Output "' + name + + '" missing from sample_weight_modes ' + 'dictionary') + if sample_weight_mode.get(name) == 'temporal': + weight = K.placeholder(ndim=2, + name=name + '_sample_weights') + sample_weight_modes.append('temporal') + else: + weight = K.placeholder(ndim=1, + name=name + '_sample_weights') + sample_weight_modes.append(None) + sample_weights.append(weight) + elif isinstance(sample_weight_mode, list): + if len(sample_weight_mode) != len(self.outputs): + raise ValueError('When passing a list as sample_weight_mode, ' + 'it should have one entry per model outputs. ' + 'The model has ' + str(len(self.outputs)) + + ' outputs, but you passed ' + 'sample_weight_mode=' + + str(sample_weight_mode)) + for i in range(len(self.output_names)): + if i in skip_indices: + weight = None sample_weight_modes.append(None) - sample_weights.append(weight) - else: - for i, name in enumerate(self.output_names): - if i in skip_indices: - sample_weight_modes.append(None) - sample_weights.append(None) - else: - if sample_weight_mode == 'temporal': - sample_weights.append( - K.placeholder(ndim=2, - name=name + '_sample_weights')) - sample_weight_modes.append('temporal') else: - sample_weights.append( - K.placeholder(ndim=1, - name=name + '_sample_weights')) + mode = sample_weight_mode[i] + name = self.output_names[i] + if mode == 'temporal': + weight = K.placeholder(ndim=2, + name=name + '_sample_weights') + sample_weight_modes.append('temporal') + else: + weight = K.placeholder(ndim=1, + name=name + '_sample_weights') + sample_weight_modes.append(None) + sample_weights.append(weight) + else: + for i, name in enumerate(self.output_names): + if i in skip_indices: sample_weight_modes.append(None) - self.sample_weight_modes = sample_weight_modes - self._feed_sample_weight_modes = [] - for i in range(len(self.outputs)): - if i not in skip_indices: - self._feed_sample_weight_modes.append(self.sample_weight_modes[i]) - return sample_weights + sample_weights.append(None) + else: + if sample_weight_mode == 'temporal': + sample_weights.append( + K.placeholder(ndim=2, + name=name + '_sample_weights')) + sample_weight_modes.append('temporal') + else: + sample_weights.append( + K.placeholder(ndim=1, + name=name + '_sample_weights')) + sample_weight_modes.append(None) + self.sample_weight_modes = sample_weight_modes + self._feed_sample_weight_modes = [] + for i in range(len(self.outputs)): + if i not in skip_indices: + self._feed_sample_weight_modes.append(self.sample_weight_modes[i]) + return sample_weights def _make_ins(self, x=None, y=None, sample_weights=None, learning_phase_value=None): def add_if_valid(x): @@ -1595,7 +1596,7 @@ def predict(self, x, batch_size=32, verbose=0): 'Batch size: ' + str(batch_size) + '.') # Prepare inputs, delegate logic to `_predict_loop`. - ins = self._make_ins(x, None, None, [0.]) + ins = self._make_ins(x, None, [], [0.]) self.predict_function = self._make_function('predict_function') f = self.predict_function return self._predict_loop(f, ins, From fe2916be355db7afc1e12dad50c610d7b46a217f Mon Sep 17 00:00:00 2001 From: Andrew Hundt Date: Sat, 17 Jun 2017 04:38:43 -0400 Subject: [PATCH 044/166] mae=>mse --- tests/keras/engine/test_training.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/keras/engine/test_training.py b/tests/keras/engine/test_training.py index df2f2d8cc8f..97e6f3f2729 100644 --- a/tests/keras/engine/test_training.py +++ b/tests/keras/engine/test_training.py @@ -633,7 +633,7 @@ def test_model_with_external_loss(): optimizer = 'rmsprop' loss = None - model.compile(optimizer, loss, metrics=['mae']) + model.compile(optimizer, loss, metrics=['mse']) input_a_np = np.random.random((10, 3)) From 4793149f124f92a926cfab2947e94fedcec51d28 Mon Sep 17 00:00:00 2001 From: Andrew Hundt Date: Sat, 17 Jun 2017 15:02:02 -0400 Subject: [PATCH 045/166] _prepare_sample_weights initialization & return changes, may be buggy --- keras/engine/training.py | 154 +++++++++++++++++++-------------------- 1 file changed, 77 insertions(+), 77 deletions(-) diff --git a/keras/engine/training.py b/keras/engine/training.py index d72a40fafa6..8d0af2a1859 100644 --- a/keras/engine/training.py +++ b/keras/engine/training.py @@ -803,7 +803,8 @@ def compile(self, optimizer, loss, metrics=None, loss_weights=None, ' - expected a list of dicts.') # Prepare sample weights. - self._prepare_sample_weights(sample_weight_mode, skip_indices) + sample_weights, sample_weight_mode = self._prepare_sample_weights( + sample_weight_mode, skip_indices) # Prepare targets of model. self.targets = [] @@ -837,7 +838,7 @@ def compile(self, optimizer, loss, metrics=None, loss_weights=None, y_true = self.targets[i] y_pred = self.outputs[i] weighted_loss = weighted_losses[i] - if sample_weight_mode is 'disabled': + if self.sample_weight_mode is 'disabled': sample_weight = None else: sample_weight = sample_weights[i] @@ -911,7 +912,7 @@ def append_metric(layer_num, metric_name, metric_tensor): # Prepare gradient updates and state updates. self.total_loss = total_loss - if sample_weight_mode is not 'disabled': + if self.sample_weight_mode is not 'disabled': self.sample_weights = sample_weights self._feed_sample_weights = [] for i in range(len(self.sample_weights)): @@ -947,85 +948,84 @@ def _prepare_sample_weights(self, sample_weight_mode, skip_indices): if self.loss_weights is not None: # tensor + loss weights not yet implemented raise NotImplementedError - sample_weight_mode = 'disabled' - if sample_weight_mode is not 'disabled': - sample_weights = self._prepare_sample_weights( - sample_weight_mode, skip_indices) - self.sample_weight_mode = sample_weight_mode - sample_weights = [] - sample_weight_modes = [] - if isinstance(sample_weight_mode, dict): - for name in sample_weight_mode: - if name not in self.output_names: - raise ValueError('Unknown entry in ' - 'sample_weight_mode dictionary: "' + - name + '". ' - 'Only expected the following keys: ' + - str(self.output_names)) - for i, name in enumerate(self.output_names): - if i in skip_indices: - weight = None - sample_weight_modes.append(None) + self.sample_weight_mode = 'disabled' + return [], self.sample_weight_mode + + self.sample_weight_mode = sample_weight_mode + sample_weights = [] + sample_weight_modes = [] + if isinstance(sample_weight_mode, dict): + for name in sample_weight_mode: + if name not in self.output_names: + raise ValueError('Unknown entry in ' + 'sample_weight_mode dictionary: "' + + name + '". ' + 'Only expected the following keys: ' + + str(self.output_names)) + for i, name in enumerate(self.output_names): + if i in skip_indices: + weight = None + sample_weight_modes.append(None) + else: + if name not in sample_weight_mode: + raise ValueError('Output "' + name + + '" missing from sample_weight_modes ' + 'dictionary') + if sample_weight_mode.get(name) == 'temporal': + weight = K.placeholder(ndim=2, + name=name + '_sample_weights') + sample_weight_modes.append('temporal') else: - if name not in sample_weight_mode: - raise ValueError('Output "' + name + - '" missing from sample_weight_modes ' - 'dictionary') - if sample_weight_mode.get(name) == 'temporal': - weight = K.placeholder(ndim=2, - name=name + '_sample_weights') - sample_weight_modes.append('temporal') - else: - weight = K.placeholder(ndim=1, - name=name + '_sample_weights') - sample_weight_modes.append(None) - sample_weights.append(weight) - elif isinstance(sample_weight_mode, list): - if len(sample_weight_mode) != len(self.outputs): - raise ValueError('When passing a list as sample_weight_mode, ' - 'it should have one entry per model outputs. ' - 'The model has ' + str(len(self.outputs)) + - ' outputs, but you passed ' - 'sample_weight_mode=' + - str(sample_weight_mode)) - for i in range(len(self.output_names)): - if i in skip_indices: - weight = None + weight = K.placeholder(ndim=1, + name=name + '_sample_weights') sample_weight_modes.append(None) + sample_weights.append(weight) + elif isinstance(sample_weight_mode, list): + if len(sample_weight_mode) != len(self.outputs): + raise ValueError('When passing a list as sample_weight_mode, ' + 'it should have one entry per model outputs. ' + 'The model has ' + str(len(self.outputs)) + + ' outputs, but you passed ' + 'sample_weight_mode=' + + str(sample_weight_mode)) + for i in range(len(self.output_names)): + if i in skip_indices: + weight = None + sample_weight_modes.append(None) + else: + mode = sample_weight_mode[i] + name = self.output_names[i] + if mode == 'temporal': + weight = K.placeholder(ndim=2, + name=name + '_sample_weights') + sample_weight_modes.append('temporal') else: - mode = sample_weight_mode[i] - name = self.output_names[i] - if mode == 'temporal': - weight = K.placeholder(ndim=2, - name=name + '_sample_weights') - sample_weight_modes.append('temporal') - else: - weight = K.placeholder(ndim=1, - name=name + '_sample_weights') - sample_weight_modes.append(None) - sample_weights.append(weight) - else: - for i, name in enumerate(self.output_names): - if i in skip_indices: + weight = K.placeholder(ndim=1, + name=name + '_sample_weights') sample_weight_modes.append(None) - sample_weights.append(None) + sample_weights.append(weight) + else: + for i, name in enumerate(self.output_names): + if i in skip_indices: + sample_weight_modes.append(None) + sample_weights.append(None) + else: + if sample_weight_mode == 'temporal': + sample_weights.append( + K.placeholder(ndim=2, + name=name + '_sample_weights')) + sample_weight_modes.append('temporal') else: - if sample_weight_mode == 'temporal': - sample_weights.append( - K.placeholder(ndim=2, - name=name + '_sample_weights')) - sample_weight_modes.append('temporal') - else: - sample_weights.append( - K.placeholder(ndim=1, - name=name + '_sample_weights')) - sample_weight_modes.append(None) - self.sample_weight_modes = sample_weight_modes - self._feed_sample_weight_modes = [] - for i in range(len(self.outputs)): - if i not in skip_indices: - self._feed_sample_weight_modes.append(self.sample_weight_modes[i]) - return sample_weights + sample_weights.append( + K.placeholder(ndim=1, + name=name + '_sample_weights')) + sample_weight_modes.append(None) + self.sample_weight_modes = sample_weight_modes + self._feed_sample_weight_modes = [] + for i in range(len(self.outputs)): + if i not in skip_indices: + self._feed_sample_weight_modes.append(self.sample_weight_modes[i]) + return sample_weights, sample_weight_mode def _make_ins(self, x=None, y=None, sample_weights=None, learning_phase_value=None): def add_if_valid(x): From af382e874a452e8d907ae5684f1ed0dc12a84754 Mon Sep 17 00:00:00 2001 From: Andrew Hundt Date: Sat, 17 Jun 2017 15:02:34 -0400 Subject: [PATCH 046/166] _make_function implementation changes to better support predict_function, may be buggy --- keras/engine/training.py | 58 +++++++++++++++++++++++----------------- 1 file changed, 34 insertions(+), 24 deletions(-) diff --git a/keras/engine/training.py b/keras/engine/training.py index 8d0af2a1859..01cf74d8625 100644 --- a/keras/engine/training.py +++ b/keras/engine/training.py @@ -1038,39 +1038,49 @@ def add_if_valid(x): return ins def _make_function(self, function_name): - if not hasattr(self, function_name): + if function_name is 'predict_function': + if not hasattr(self, 'predict_function'): + self.predict_function = None + if self.predict_function is None: + if self.uses_learning_phase and not isinstance(K.learning_phase(), int): + inputs = self._feed_inputs + [K.learning_phase()] + else: + inputs = self._feed_inputs + outputs = self.outputs + elif not hasattr(self, function_name): raise RuntimeError('You must compile your model before using it.') + function = getattr(self, function_name) if function is not None: return function - else: - inputs = self._feed_inputs + self._feed_targets - if self.sample_weight_mode is not 'disabled': + elif function_name is not 'predict_function': + inputs = self._feed_inputs + if hasattr(self, '_feed_targets'): + inputs += self._feed_targets + if (hasattr(self, 'sample_weight_mode') and + self.sample_weight_mode is not 'disabled'): inputs += self._feed_sample_weights if self.uses_learning_phase and not isinstance(K.learning_phase(), int): inputs += [K.learning_phase()] - if function_name is 'train_function': - training_updates = self.optimizer.get_updates( - self._collected_trainable_weights, - self.constraints, - self.total_loss) - updates = self.updates + training_updates - else: - updates = self.state_updates + outputs = [self.total_loss] + self.metrics_tensors - if function_name is 'predict_function': - outputs = self.outputs - kwargs = getattr(self, '_function_kwargs', {}) - else: - outputs = [self.total_loss] + self.metrics_tensors - kwargs = self._function_kwargs - # Gets loss and metrics. Updates weights at each call. - return K.function(inputs, - outputs, - updates=updates, - name=function_name, - **kwargs) + if function_name is 'train_function': + training_updates = self.optimizer.get_updates( + self._collected_trainable_weights, + self.constraints, + self.total_loss) + updates = self.updates + training_updates + else: + updates = self.state_updates + + kwargs = getattr(self, '_function_kwargs', {}) + # Gets loss and metrics. Updates weights at each call. + return K.function(inputs, + outputs, + updates=updates, + name=function_name, + **kwargs) def _fit_loop(self, f, ins, out_labels=None, batch_size=32, epochs=100, verbose=1, callbacks=None, From 1e6b9fbf5b6bdb2f00c7d16f93aea55122a6d0f6 Mon Sep 17 00:00:00 2001 From: Andrew Hundt Date: Sat, 17 Jun 2017 16:08:02 -0400 Subject: [PATCH 047/166] tensorflow_backend.py Function handles varying length inputs --- keras/backend/tensorflow_backend.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/keras/backend/tensorflow_backend.py b/keras/backend/tensorflow_backend.py index 69bba5fe8db..653454ec896 100644 --- a/keras/backend/tensorflow_backend.py +++ b/keras/backend/tensorflow_backend.py @@ -10,6 +10,7 @@ import inspect import numpy as np import os +import itertools from .common import floatx from .common import _EPSILON @@ -2289,7 +2290,7 @@ def __call__(self, inputs): self.current_feed_dict = {} if self.feed_dict is None else self.feed_dict self.feed_to_fetch_count = 0 self.current_fetches = self.outputs + [self.updates_op] - for tensor, value in zip(self.inputs, inputs): + for tensor, value in itertools.izip_longest(self.inputs, inputs): if is_sparse(tensor): sparse_coo = value.tocoo() indices = np.concatenate((np.expand_dims(sparse_coo.row, 1), From ea9ffbafb4598e7547c98a2abcd504e0fe518c68 Mon Sep 17 00:00:00 2001 From: Andrew Hundt Date: Sat, 17 Jun 2017 16:27:42 -0400 Subject: [PATCH 048/166] TFRecord predict() passes again --- keras/engine/training.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/keras/engine/training.py b/keras/engine/training.py index 01cf74d8625..56b6bd673d6 100644 --- a/keras/engine/training.py +++ b/keras/engine/training.py @@ -1592,6 +1592,12 @@ def predict(self, x, batch_size=32, verbose=0): or in case a stateful model receives a number of samples that is not a multiple of the batch size. """ + + if self._input_yield_op_tensors and x is not None: + raise ValueError('Input data can only be supplied via ' + 'the input data parameter `x` in predict() or ' + 'tensors in the constructor of `Model()`, ' + 'not both.') # Validate user data. x = _standardize_input_data(x, self._feed_input_names, self._feed_input_shapes, @@ -1604,9 +1610,8 @@ def predict(self, x, batch_size=32, verbose=0): 'divided by the batch size. Found: ' + str(x[0].shape[0]) + ' samples. ' 'Batch size: ' + str(batch_size) + '.') - # Prepare inputs, delegate logic to `_predict_loop`. - ins = self._make_ins(x, None, [], [0.]) + ins = self._make_ins(x, [], [], [0.]) self.predict_function = self._make_function('predict_function') f = self.predict_function return self._predict_loop(f, ins, @@ -1710,7 +1715,7 @@ def predict_on_batch(self, x): """ x = _standardize_input_data(x, self._feed_input_names, self._feed_input_shapes) - ins = self._make_ins(x, None, [], [0.]) + ins = self._make_ins(x, [], [], [0.]) self.predict_function = self._make_function('predict_function') outputs = self.predict_function(ins) if len(outputs) == 1: From 06b71c44c6d71e6d6760143ea40e3466b3e4fa04 Mon Sep 17 00:00:00 2001 From: Andrew Hundt Date: Sun, 18 Jun 2017 13:03:44 -0400 Subject: [PATCH 049/166] tensorflow_backend Function __call__ izip_longest explicit fillvalue --- keras/backend/tensorflow_backend.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/keras/backend/tensorflow_backend.py b/keras/backend/tensorflow_backend.py index 653454ec896..7150ff886a9 100644 --- a/keras/backend/tensorflow_backend.py +++ b/keras/backend/tensorflow_backend.py @@ -2290,7 +2290,7 @@ def __call__(self, inputs): self.current_feed_dict = {} if self.feed_dict is None else self.feed_dict self.feed_to_fetch_count = 0 self.current_fetches = self.outputs + [self.updates_op] - for tensor, value in itertools.izip_longest(self.inputs, inputs): + for tensor, value in itertools.izip_longest(self.inputs, inputs, fillvalue=None): if is_sparse(tensor): sparse_coo = value.tocoo() indices = np.concatenate((np.expand_dims(sparse_coo.row, 1), From 325acc1ccaf4a3855368c64932d7f94c8d865a37 Mon Sep 17 00:00:00 2001 From: Andrew Hundt Date: Sun, 18 Jun 2017 14:19:57 -0400 Subject: [PATCH 050/166] _make_function call & list bugs fixed --- keras/engine/training.py | 8 ++++---- keras/models.py | 4 ++-- tests/test_model_saving.py | 2 +- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/keras/engine/training.py b/keras/engine/training.py index 56b6bd673d6..1c9883dfbcd 100644 --- a/keras/engine/training.py +++ b/keras/engine/training.py @@ -1054,14 +1054,14 @@ def _make_function(self, function_name): if function is not None: return function elif function_name is not 'predict_function': - inputs = self._feed_inputs + inputs = [] + self._feed_inputs if hasattr(self, '_feed_targets'): - inputs += self._feed_targets + inputs = inputs + self._feed_targets if (hasattr(self, 'sample_weight_mode') and self.sample_weight_mode is not 'disabled'): - inputs += self._feed_sample_weights + inputs = inputs + self._feed_sample_weights if self.uses_learning_phase and not isinstance(K.learning_phase(), int): - inputs += [K.learning_phase()] + inputs = inputs + [K.learning_phase()] outputs = [self.total_loss] + self.metrics_tensors diff --git a/keras/models.py b/keras/models.py index 2d826eed990..6fb1828e072 100644 --- a/keras/models.py +++ b/keras/models.py @@ -280,9 +280,9 @@ def convert_custom_objects(obj): if 'optimizer_weights' in f: # Build train function (to get weight updates). if isinstance(model, Sequential): - model.model._make_train_function() + model.model.train_function = model.model._make_function('train_function') else: - model._make_train_function() + model.train_function = model._make_function('train_function') optimizer_weights_group = f['optimizer_weights'] optimizer_weight_names = [n.decode('utf8') for n in optimizer_weights_group.attrs['weight_names']] diff --git a/tests/test_model_saving.py b/tests/test_model_saving.py index 13c65f59d00..3f6eea544b1 100644 --- a/tests/test_model_saving.py +++ b/tests/test_model_saving.py @@ -150,7 +150,7 @@ def test_saving_right_after_compilation(): model.add(Dense(2, input_shape=(3,))) model.add(Dense(3)) model.compile(loss='mse', optimizer='sgd', metrics=['acc']) - model.model._make_train_function() + model.model._make_function('train_function') _, fname = tempfile.mkstemp('.h5') save_model(model, fname) From 974886ea1b475c62de9a7000a7225476736ed2da Mon Sep 17 00:00:00 2001 From: Andrew Hundt Date: Sun, 18 Jun 2017 16:38:16 -0400 Subject: [PATCH 051/166] is_placeholder and standardize user data error checks fixed --- keras/backend/tensorflow_backend.py | 26 +---------------- keras/backend/theano_backend.py | 1 + keras/engine/topology.py | 7 +++-- keras/engine/training.py | 44 ++++++++++++++++------------- 4 files changed, 31 insertions(+), 47 deletions(-) diff --git a/keras/backend/tensorflow_backend.py b/keras/backend/tensorflow_backend.py index 7150ff886a9..8da8cc0707c 100644 --- a/keras/backend/tensorflow_backend.py +++ b/keras/backend/tensorflow_backend.py @@ -15,6 +15,7 @@ from .common import floatx from .common import _EPSILON from .common import image_data_format +from .common import is_placeholder # Legacy functions from .common import set_image_dim_ordering @@ -281,31 +282,6 @@ def to_dense(tensor): name_scope = tf.name_scope -def is_placeholder(tensor): - """Returns whether a tensor is a placeholder. - - # Arguments - tensor: A tensor instance. - - # Returns - A boolean. - - # Example - ```python - >>> from keras import backend as K - >>> a = K.placeholder((2, 2), sparse=False) - >>> print(K.is_placeholder(a)) - True - ``` - """ - try: - if isinstance(tensor, tf.Variable): - return True - return tensor._is_placeholder - except AttributeError: - return False - - def variable(value, dtype=None, name=None): """Instantiates a variable and returns it. diff --git a/keras/backend/theano_backend.py b/keras/backend/theano_backend.py index aa73641bbf5..9d25fd22c7c 100644 --- a/keras/backend/theano_backend.py +++ b/keras/backend/theano_backend.py @@ -17,6 +17,7 @@ import inspect import numpy as np from .common import _FLOATX, floatx, _EPSILON, image_data_format +from .common import is_placeholder # Legacy functions from .common import set_image_dim_ordering, image_dim_ordering diff --git a/keras/engine/topology.py b/keras/engine/topology.py index e7b0ad57dfa..14dc3c0ef53 100644 --- a/keras/engine/topology.py +++ b/keras/engine/topology.py @@ -1639,9 +1639,10 @@ def __init__(self, inputs, outputs, labels=None, name=None): i, layer.__class__.__name__)) self.input_names.append(layer.name) - self._feed_input_names.append(layer.name) - self._feed_inputs.append(layer.input) - self._feed_input_shapes.append(self.inputs[i]._keras_shape) + if K.is_placeholder(layer): + self._feed_input_names.append(layer.name) + self._feed_inputs.append(layer.input) + self._feed_input_shapes.append(self.inputs[i]._keras_shape) for layer in self.output_layers: self.output_names.append(layer.name) diff --git a/keras/engine/training.py b/keras/engine/training.py index 1c9883dfbcd..147f4b07898 100644 --- a/keras/engine/training.py +++ b/keras/engine/training.py @@ -27,7 +27,8 @@ def _standardize_input_data(data, names, shapes=None, check_batch_axis=True, - exception_prefix=''): + exception_prefix='', + tensors=None): """Normalizes inputs and targets provided by users. Users may pass data as a list of arrays, dictionary of arrays, @@ -50,10 +51,14 @@ def _standardize_input_data(data, names, shapes=None, # Raises ValueError: in case of improperly formatted user-provided data. """ + if tensors is None: + tensors = [] if not names: return [] if data is None: return [None for _ in range(len(names))] + num_tensors = sum(x is not None for x in tensors) + expected_names = len(names) - num_tensors if isinstance(data, dict): arrays = [] for name in names: @@ -63,18 +68,22 @@ def _standardize_input_data(data, names, shapes=None, str(names)) arrays.append(data[name]) elif isinstance(data, list): - if len(data) != len(names): + if len(data) != expected_names: if data and hasattr(data[0], 'shape'): + if type(data).__module__ == np.__name__: + datastr = ('the following list of ' + str(len(data)) + + ' arrays: ' + str(data)[:200] + + '...') + else: + datastr = ('data with type' + + type(data).__name__ + '...') raise ValueError('Error when checking model ' + exception_prefix + ': the list of Numpy arrays ' 'that you are passing to your model ' 'is not the size the model expected. ' - 'Expected to see ' + str(len(names)) + - ' arrays but instead got ' - 'the following list of ' + str(len(data)) + - ' arrays: ' + str(data)[:200] + - '...') + 'Expected to see ' + str(expected_names) + + ' arrays but instead got ' + datastr) else: if len(names) == 1: data = [np.asarray(data)] @@ -85,7 +94,7 @@ def _standardize_input_data(data, names, shapes=None, ': you are passing a list as ' 'input to your model, ' 'but the model expects ' - 'a list of ' + str(len(names)) + + 'a list of ' + str(expected_names) + ' Numpy arrays instead. ' 'The list you passed was: ' + str(data)[:200]) @@ -97,17 +106,17 @@ def _standardize_input_data(data, names, shapes=None, ': data should be a Numpy array, ' 'or list/dict of Numpy arrays. ' 'Found: ' + str(data)[:200] + '...') - if len(names) > 1: + if expected_names > 1: # Case: model expects multiple inputs but only received # a single Numpy array. - raise ValueError('The model expects ' + str(len(names)) + + raise ValueError('The model expects ' + str(expected_names) + exception_prefix + ' arrays, but only received one array. ' 'Found: array with shape ' + str(data.shape)) arrays = [data] # Make arrays at least 2D. - for i in range(len(names)): + for i in range(expected_names): array = arrays[i] if len(array.shape) == 1: array = np.expand_dims(array, 1) @@ -115,7 +124,7 @@ def _standardize_input_data(data, names, shapes=None, # Check shapes compatibility. if shapes: - for i in range(len(names)): + for i in range(expected_names): if shapes[i] is None: continue array = arrays[i] @@ -1339,11 +1348,13 @@ def _standardize_user_data(self, x, y, x = _standardize_input_data(x, self._feed_input_names, self._feed_input_shapes, check_batch_axis=False, - exception_prefix='input') + exception_prefix='input', + tensors=self._input_yield_op_tensors) y = _standardize_input_data(y, self._feed_output_names, output_shapes, check_batch_axis=False, - exception_prefix='target') + exception_prefix='target', + tensors=self.labels) class_weights = _standardize_class_weights(class_weight, self._feed_output_names) if self.sample_weight_mode is not 'disabled': @@ -1593,11 +1604,6 @@ def predict(self, x, batch_size=32, verbose=0): that is not a multiple of the batch size. """ - if self._input_yield_op_tensors and x is not None: - raise ValueError('Input data can only be supplied via ' - 'the input data parameter `x` in predict() or ' - 'tensors in the constructor of `Model()`, ' - 'not both.') # Validate user data. x = _standardize_input_data(x, self._feed_input_names, self._feed_input_shapes, From 3c7dc09644f5f3970c735ff1d80495e9c903e64a Mon Sep 17 00:00:00 2001 From: Andrew Hundt Date: Sun, 18 Jun 2017 16:47:30 -0400 Subject: [PATCH 052/166] topology.py add missing docstring --- keras/engine/topology.py | 1 + 1 file changed, 1 insertion(+) diff --git a/keras/engine/topology.py b/keras/engine/topology.py index 14dc3c0ef53..d82c2f7e13e 100644 --- a/keras/engine/topology.py +++ b/keras/engine/topology.py @@ -97,6 +97,7 @@ class Node(object): output_shapes: list of output shape tuples. arguments: dictionary of keyword arguments that were passed to the `call` method of the layer at the call that created the node. + is_placeholder: Specifies if the Node represents a placeholder. `node_indices` and `tensor_indices` are basically fine-grained coordinates describing the origin of the `input_tensors`, verifying the following: From 20077cb1c2b06d23b5f0312bbea77c8f93418fb8 Mon Sep 17 00:00:00 2001 From: Andrew Hundt Date: Sun, 18 Jun 2017 17:38:23 -0400 Subject: [PATCH 053/166] test_multiprocessing.py fix test which actually throws two exceptions --- tests/test_multiprocessing.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_multiprocessing.py b/tests/test_multiprocessing.py index b560bd4b813..06cd7280b4e 100644 --- a/tests/test_multiprocessing.py +++ b/tests/test_multiprocessing.py @@ -284,13 +284,13 @@ def custom_generator(): model.add(Dense(1, input_shape=(2, ))) model.compile(loss='mse', optimizer='adadelta') - with pytest.raises(ValueError): + with pytest.raises((ValueError, Exception)): model.predict_generator( custom_generator(), good_batches + 1, 1, workers=4, pickle_safe=True, ) - with pytest.raises(ValueError): + with pytest.raises((ValueError, Exception)): model.predict_generator( custom_generator(), good_batches + 1, 1, pickle_safe=False, From 9dd1ea9b8fc6189ba9475225f17ba4dbb74df95e Mon Sep 17 00:00:00 2001 From: Andrew Hundt Date: Sun, 18 Jun 2017 17:53:09 -0400 Subject: [PATCH 054/166] test_training.py remove extraneous try/except --- tests/keras/engine/test_training.py | 24 +++++++++--------------- 1 file changed, 9 insertions(+), 15 deletions(-) diff --git a/tests/keras/engine/test_training.py b/tests/keras/engine/test_training.py index 0daa2021ebf..29f29fd565b 100644 --- a/tests/keras/engine/test_training.py +++ b/tests/keras/engine/test_training.py @@ -642,13 +642,10 @@ def replace(filename): optimizer = 'rmsprop' loss = 'mse' loss_weights = [1.] - try: - with pytest.raises(NotImplementedError) as exc: - model.compile(optimizer, loss, metrics=['mean_squared_error'], - loss_weights=loss_weights, - sample_weight_mode=None) - except NotImplementedError: - print 'Feature not implemented' + with pytest.raises(NotImplementedError) as exc: + model.compile(optimizer, loss, metrics=['mean_squared_error'], + loss_weights=loss_weights, + sample_weight_mode=None) model.compile(optimizer, loss, metrics=['mean_squared_error'], sample_weight_mode=None) @@ -656,14 +653,11 @@ def replace(filename): # test fit out = model.fit(None, None, epochs=1, batch_size=10) - try: - with pytest.raises(ValueError) as exc: - out = model.fit(output_a_np, - [y_train_in_out], - epochs=1, - batch_size=10) - except ValueError: - print 'cannot mix input tensors and numpy arrays' + with pytest.raises(ValueError) as exc: + out = model.fit(output_a_np, + [y_train_in_out], + epochs=1, + batch_size=10) os.remove('input_a.tfrecord') From c37b82c35da6afcbf3ad32783a7ca95f74595c4a Mon Sep 17 00:00:00 2001 From: Andrew Hundt Date: Sun, 18 Jun 2017 18:02:05 -0400 Subject: [PATCH 055/166] test_training.py tfrecord test of predict_on_batch, evaluate, predict --- tests/keras/engine/test_training.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/tests/keras/engine/test_training.py b/tests/keras/engine/test_training.py index 29f29fd565b..4c809e1cb8a 100644 --- a/tests/keras/engine/test_training.py +++ b/tests/keras/engine/test_training.py @@ -649,6 +649,7 @@ def replace(filename): model.compile(optimizer, loss, metrics=['mean_squared_error'], sample_weight_mode=None) + out = model.predict_on_batch(None) out = model.train_on_batch(None, None) # test fit out = model.fit(None, @@ -659,6 +660,19 @@ def replace(filename): epochs=1, batch_size=10) + # test evaluate + out = model.evaluate(None, + None, + batch_size=10) + with pytest.raises(ValueError) as exc: + out = model.evaluate(input_a_np, + [output_a_np, + output_b_tf], + batch_size=10) + + # test predict + out = model.predict(None, batch_size=10) + os.remove('input_a.tfrecord') From 75fbfbbb4af943b46169935f6fa384cdbe9f9e67 Mon Sep 17 00:00:00 2001 From: Andrew Hundt Date: Sun, 18 Jun 2017 18:05:52 -0400 Subject: [PATCH 056/166] mnist_tfrecord.py _is_placeholder workaround no longer required. --- examples/mnist_tfrecord.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/examples/mnist_tfrecord.py b/examples/mnist_tfrecord.py index 506817ffc13..f2bf556e12e 100644 --- a/examples/mnist_tfrecord.py +++ b/examples/mnist_tfrecord.py @@ -146,8 +146,6 @@ def cnn_layers(x_train_input): def create_cnn_model(x_train_batch, y_train_batch, x_batch_shape, y_batch_shape): x_train_input = Input(tensor=x_train_batch, batch_shape=x_batch_shape) x_train_out = cnn_layers(x_train_input) - # Workaround until _is_placeholder can be deduced automatically - x_train_out._is_placeholder = False y_train_in_out = Input(tensor=y_train_batch, batch_shape=y_batch_shape, name='y_labels') return Model(inputs=[x_train_input], outputs=[x_train_out], labels=[y_train_in_out]) From 6de4e83d44dd050779f6a1f1de57d7ec97ba2f27 Mon Sep 17 00:00:00 2001 From: Andrew Hundt Date: Sun, 18 Jun 2017 18:54:17 -0400 Subject: [PATCH 057/166] tensorflow_backend.py py 2+3 compatibility: from moves.six import zip_longest --- keras/backend/tensorflow_backend.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/keras/backend/tensorflow_backend.py b/keras/backend/tensorflow_backend.py index 9c1e82b61c1..51b41f6996b 100644 --- a/keras/backend/tensorflow_backend.py +++ b/keras/backend/tensorflow_backend.py @@ -10,7 +10,7 @@ import inspect import numpy as np import os -import itertools +from six.moves import zip_longest from .common import floatx from .common import _EPSILON @@ -2264,7 +2264,7 @@ def __call__(self, inputs): self.current_feed_dict = {} if self.feed_dict is None else self.feed_dict self.feed_to_fetch_count = 0 self.current_fetches = self.outputs + [self.updates_op] - for tensor, value in itertools.izip_longest(self.inputs, inputs, fillvalue=None): + for tensor, value in zip_longest(self.inputs, inputs, fillvalue=None): if is_sparse(tensor): sparse_coo = value.tocoo() indices = np.concatenate((np.expand_dims(sparse_coo.row, 1), From 3d756492b25d986b051a399f6615cf5c2ee8135f Mon Sep 17 00:00:00 2001 From: Andrew Hundt Date: Sun, 18 Jun 2017 21:42:40 -0400 Subject: [PATCH 058/166] mnist_tfrecord.py add parens to print --- examples/mnist_tfrecord.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/examples/mnist_tfrecord.py b/examples/mnist_tfrecord.py index f2bf556e12e..966d1aa54d0 100644 --- a/examples/mnist_tfrecord.py +++ b/examples/mnist_tfrecord.py @@ -65,7 +65,7 @@ def _bytes_feature(value): writer.write(example.SerializeToString()) writer.close() else: - print 'tfrecord %s already exists' % filename + print('tfrecord %s already exists' % filename) def read_and_decode_recordinput(tf_glob, one_hot=True, classes=None, is_train=None, batch_shape=[1000, 28, 28, 1]): @@ -210,4 +210,4 @@ def create_cnn_model(x_train_batch, y_train_batch, x_batch_shape, y_batch_shape) test_model.summary() loss, acc = test_model.evaluate(X_test, np_utils.to_categorical(y_test), classes) -print '\nTest accuracy: {0}'.format(acc) +print('\nTest accuracy: {0}'.format(acc)) From 0a1606f44b2de4528028af4794ae12b2fedf950c Mon Sep 17 00:00:00 2001 From: Andrew Hundt Date: Sun, 18 Jun 2017 23:07:25 -0400 Subject: [PATCH 059/166] test_training.py add parenthesis to print, plus an extra error check in the tensorflow_backend.py --- keras/backend/tensorflow_backend.py | 18 ++++++++++++++++++ tests/keras/engine/test_training.py | 6 +++--- 2 files changed, 21 insertions(+), 3 deletions(-) diff --git a/keras/backend/tensorflow_backend.py b/keras/backend/tensorflow_backend.py index 51b41f6996b..4f109f67ea0 100644 --- a/keras/backend/tensorflow_backend.py +++ b/keras/backend/tensorflow_backend.py @@ -2241,6 +2241,7 @@ def __init__(self, inputs, outputs, updates=None, name=None, if not isinstance(updates, (list, tuple)): raise TypeError('`updates` in a TensorFlow backend function ' 'should be a list or tuple.') + # self.inputs holds tf Tensor objects self.inputs = list(inputs) self.outputs = list(outputs) self.fetches = fetches @@ -2259,17 +2260,34 @@ def __init__(self, inputs, outputs, updates=None, name=None, self.session_kwargs = session_kwargs def __call__(self, inputs): + """Run the TensorFlow session + + # Arguments + inputs: Data and values that will go to the feed_dict of Session.run() + if it is associated with a tensor, if it is None the tensor will + be added to the fetches parameter of Session.run(). + """ if not isinstance(inputs, (list, tuple)): raise TypeError('`inputs` should be a list or tuple.') self.current_feed_dict = {} if self.feed_dict is None else self.feed_dict self.feed_to_fetch_count = 0 self.current_fetches = self.outputs + [self.updates_op] + # self.inputs contains tf tensors, inputs contains feed_dict data. for tensor, value in zip_longest(self.inputs, inputs, fillvalue=None): + if tensor is None and value is None: + continue + elif tensor is None and value is not None: + raise ValueError('A tensor containing None ' + 'was tied to value ' + str(value) + + 'so Session.run() cannot execute, ' + 'please check your data and Model.') + if is_sparse(tensor): sparse_coo = value.tocoo() indices = np.concatenate((np.expand_dims(sparse_coo.row, 1), np.expand_dims(sparse_coo.col, 1)), 1) value = (indices, sparse_coo.data, sparse_coo.shape) + if value is None and tensor is not None: self.feed_to_fetch_count += 1 self.current_fetches.append(tensor) diff --git a/tests/keras/engine/test_training.py b/tests/keras/engine/test_training.py index 4c809e1cb8a..30625e230ec 100644 --- a/tests/keras/engine/test_training.py +++ b/tests/keras/engine/test_training.py @@ -552,11 +552,11 @@ def _bytes_feature(value): writer.write(example.SerializeToString()) writer.close() else: - print 'tfrecord %s already exists' % filename + print('tfrecord %s already exists' % filename) def read_and_decode_recordinput(tf_glob, one_hot=True, classes=None, is_train=None, batch_shape=[10, 3, 3, 1]): """ Return tensor to read from TFRecord """ - print 'Creating graph for loading TFRecords...' + print('Creating graph for loading TFRecords...') with tf.variable_scope("TFRecords"): record_input = data_flow_ops.RecordInput(tf_glob, batch_size=batch_shape[0]) records_op = record_input.get_yield_op() @@ -596,7 +596,7 @@ def read_and_decode_recordinput(tf_glob, one_hot=True, classes=None, is_train=No def replace(filename): if os.path.isfile(filename): - print '%s already exists, replacing...' % filename + print('%s already exists, replacing...' % filename) os.remove(filename) batch_size = 10 From b7d44a54ffee3391135abd3a3566a9c0d20a2fa8 Mon Sep 17 00:00:00 2001 From: Andrew Hundt Date: Mon, 19 Jun 2017 00:45:52 -0400 Subject: [PATCH 060/166] wrappers_test.py fix tolerance in def test_TimeDistributed_learning_phase() https://github.com/fchollet/keras/commit/a625fcde5cb6a1a170a51690d21b575c0e7884c1#diff-a60adf725df8a5eed11441fb09ae7fb0R98 --- tests/keras/layers/wrappers_test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/keras/layers/wrappers_test.py b/tests/keras/layers/wrappers_test.py index c401040cec4..7986b843b48 100644 --- a/tests/keras/layers/wrappers_test.py +++ b/tests/keras/layers/wrappers_test.py @@ -97,7 +97,7 @@ def test_TimeDistributed_learning_phase(): y = wrappers.TimeDistributed(core.Dropout(.999))(x, training=True) model = Model(x, y) y = model.predict(np.random.random((10, 3, 2))) - assert_allclose(0., y, atol=1e-2) + assert_allclose(0., y, atol=1e-1, rtol=1e-1) @keras_test From 39d3ebab621cd7bdd36ed9317c13385c33202f39 Mon Sep 17 00:00:00 2001 From: Andrew Hundt Date: Mon, 19 Jun 2017 20:41:37 -0400 Subject: [PATCH 061/166] test_training.py Input yield op tests --- tests/keras/engine/test_training.py | 158 ++++++++++++++++++++++++++++ 1 file changed, 158 insertions(+) diff --git a/tests/keras/engine/test_training.py b/tests/keras/engine/test_training.py index f8aafd132ab..35e464032ec 100644 --- a/tests/keras/engine/test_training.py +++ b/tests/keras/engine/test_training.py @@ -5,6 +5,7 @@ import scipy.sparse as sparse from keras.layers import Dense, Dropout +from keras.layers import Flatten from keras.engine.topology import Input from keras.engine.training import Model, _check_loss_and_target_compatibility from keras.models import Sequential @@ -515,6 +516,163 @@ def test_model_with_input_feed_tensor(): assert out.shape == (10, 4) +@pytest.mark.skipif(K.backend() != 'tensorflow', reason='Requires TF backend') +@keras_test +def test_model_with_input_yield_op(): + """We test building a model with a RecordInput as input. + We should be able to call fit, evaluate, predict, + by only passing them data for the placeholder inputs + in the model. + """ + import tensorflow as tf + from tensorflow.python.lib.io import tf_record + from tensorflow.python.ops import data_flow_ops + from tensorflow.python.platform import test + + def images_to_tfrecord(images, labels, filename): + + def _int64_feature(value): + return tf.train.Feature(int64_list=tf.train.Int64List(value=[value])) + + def _bytes_feature(value): + return tf.train.Feature(bytes_list=tf.train.BytesList(value=[value])) + + """ Save data into TFRecord """ + if not os.path.isfile(filename): + num_examples = images.shape[0] + + rows = images.shape[1] + cols = images.shape[2] + depth = images.shape[3] + + print('Writing', filename) + writer = tf.python_io.TFRecordWriter(filename) + for index in range(num_examples): + image_raw = images[index].tostring() + example = tf.train.Example(features=tf.train.Features(feature={ + 'height': _int64_feature(rows), + 'width': _int64_feature(cols), + 'depth': _int64_feature(depth), + 'label': _int64_feature(int(labels[index])), + 'image_raw': _bytes_feature(image_raw)})) + writer.write(example.SerializeToString()) + writer.close() + else: + print('tfrecord %s already exists' % filename) + + def read_and_decode_recordinput(tf_glob, one_hot=True, classes=None, is_train=None, batch_shape=[10, 3, 3, 1]): + """ Return tensor to read from TFRecord """ + print('Creating graph for loading TFRecords...') + with tf.variable_scope("TFRecords"): + record_input = data_flow_ops.RecordInput(tf_glob, batch_size=batch_shape[0]) + records_op = record_input.get_yield_op() + records_op = tf.split(records_op, batch_shape[0], 0) + records_op = [tf.reshape(record, []) for record in records_op] + + images = [] + labels = [] + for i, serialized_example in enumerate(records_op): + with tf.variable_scope("parse_images", reuse=True): + features = tf.parse_single_example( + serialized_example, + features={ + 'label': tf.FixedLenFeature([], tf.int64), + 'image_raw': tf.FixedLenFeature([], tf.string), + }) + img = tf.decode_raw(features['image_raw'], tf.uint8) + img.set_shape(batch_shape[1] * batch_shape[2]) + img = tf.reshape(img, [1] + batch_shape[1:]) + + img = tf.cast(img, tf.float32) * (1. / 255) - 0.5 + + label = tf.cast(features['label'], tf.int32) + if one_hot and classes: + label = tf.one_hot(label, classes) + + images.append(img) + labels.append(label) + + images = tf.parallel_stack(images, 0) + labels = tf.parallel_stack(labels, 0) + images = tf.cast(images, tf.float32) + + images = tf.reshape(images, shape=batch_shape) + + return images, labels + + def replace(filename): + if os.path.isfile(filename): + print('%s already exists, replacing...' % filename) + os.remove(filename) + + batch_size = 10 + input_rows = 3 + cols = 3 + depth = 1 + classes = 2 + img_batch_shape = [batch_size, input_rows, cols, depth] + label_batch_shape = [batch_size, classes] + input_a_np = np.multiply(np.random.random(img_batch_shape), batch_size) + input_a_np = input_a_np.astype('uint8') + output_a_rows = 4 + output_a_np = np.multiply(np.random.random([batch_size]), batch_size) + output_a_np = output_a_np.astype('int') + output_b_np = np.random.random([batch_size, classes]) + replace('input_a.tfrecord') + images_to_tfrecord(input_a_np, output_a_np, 'input_a.tfrecord') + input_a_tf, output_b_tf = read_and_decode_recordinput( + 'input_a.tfrecord', + one_hot=True, + classes=classes, + is_train=True, + batch_shape=img_batch_shape) + + a = Input(tensor=input_a_tf, batch_shape=img_batch_shape) + + a_2 = Dense(8, name='dense_1')(a) + dp = Dropout(0.5, name='dropout') + b_2 = dp(a_2) + f_1 = Flatten()(b_2) + b_3 = Dense( + classes, + activation='softmax', + name='dense_b3')(f_1) + + y_train_in_out = Input( + tensor=output_b_tf, + batch_shape=label_batch_shape, + name='y_labels') + + model = Model([a], [b_3]) + model.summary() + + optimizer = 'rmsprop' + loss = 'mse' + loss_weights = [1., 2.] + with pytest.raises(ValueError) as exc: + model.compile(optimizer, loss, metrics=['mean_squared_error'], + loss_weights=loss_weights, + sample_weight_mode=None) + + model.compile(optimizer, loss, metrics=['mean_squared_error'], + sample_weight_mode=None) + out = model.predict_on_batch(output_b_np) + out = model.train_on_batch(None, output_b_np) + # test fit + out = model.fit(None, + output_b_np, epochs=1, batch_size=10) + + # test evaluate + out = model.evaluate(None, + output_b_np, + batch_size=10) + + # test predict + out = model.predict(None, batch_size=10) + + os.remove('input_a.tfrecord') + + @keras_test def test_model_with_partial_loss(): a = Input(shape=(3,), name='input_a') From ab9fe3296431b785d51cfab0ac9348c0b95ca7ea Mon Sep 17 00:00:00 2001 From: Andrew Hundt Date: Tue, 20 Jun 2017 17:23:34 -0400 Subject: [PATCH 062/166] mnist_tfrecord.py added with yield_op workaround (#7046) see https://github.com/fchollet/keras/pull/6928#issuecomment-309844726 for workaround explanation --- examples/mnist_tfrecord.py | 209 +++++++++++++++++++++++++++++++++++++ 1 file changed, 209 insertions(+) create mode 100644 examples/mnist_tfrecord.py diff --git a/examples/mnist_tfrecord.py b/examples/mnist_tfrecord.py new file mode 100644 index 00000000000..93d72c03831 --- /dev/null +++ b/examples/mnist_tfrecord.py @@ -0,0 +1,209 @@ +'''MNIST dataset with TensorFlow TFRecords. + +Gets to 99.25% test accuracy after 12 epochs +(there is still a lot of margin for parameter tuning). +''' +import os +import copy +import time + +import numpy as np + +import tensorflow as tf +from tensorflow.python.ops import data_flow_ops +from keras import backend as K +from keras.models import Model +from keras.layers import Dense +from keras.layers import Dropout +from keras.layers import Flatten +from keras.layers import Input +from keras.layers import Conv2D +from keras.layers import MaxPooling2D +from keras.callbacks import EarlyStopping +from keras.layers import TensorBoard +from keras.objectives import categorical_crossentropy +from keras.utils import np_utils +from keras.utils.generic_utils import Progbar +from keras import callbacks as cbks +from keras import optimizers, objectives +from keras import metrics as metrics_module + +from keras.datasets import mnist + +if K.backend() != 'tensorflow': + raise RuntimeError('This example can only run with the ' + 'TensorFlow backend for the time being, ' + 'because it requires TFRecords, which ' + 'are not supported on other platforms.') + + +def images_to_tfrecord(images, labels, filename): + def _int64_feature(value): + return tf.train.Feature(int64_list=tf.train.Int64List(value=[value])) + + def _bytes_feature(value): + return tf.train.Feature(bytes_list=tf.train.BytesList(value=[value])) + + """ Save data into TFRecord """ + if not os.path.isfile(filename): + num_examples = images.shape[0] + + rows = images.shape[1] + cols = images.shape[2] + depth = images.shape[3] + + print('Writing', filename) + writer = tf.python_io.TFRecordWriter(filename) + for index in range(num_examples): + image_raw = images[index].tostring() + example = tf.train.Example(features=tf.train.Features(feature={ + 'height': _int64_feature(rows), + 'width': _int64_feature(cols), + 'depth': _int64_feature(depth), + 'label': _int64_feature(int(labels[index])), + 'image_raw': _bytes_feature(image_raw)})) + writer.write(example.SerializeToString()) + writer.close() + else: + print('tfrecord %s already exists' % filename) + + +def read_and_decode_recordinput(tf_glob, one_hot=True, classes=None, is_train=None, batch_shape=[1000, 28, 28, 1]): + """ Return tensor to read from TFRecord """ + print 'Creating graph for loading TFRecords...' + with tf.variable_scope("TFRecords"): + record_input = data_flow_ops.RecordInput(tf_glob, batch_size=batch_shape[0]) + records_op = record_input.get_yield_op() + records_op = tf.split(records_op, batch_shape[0], 0) + records_op = [tf.reshape(record, []) for record in records_op] + progbar = Progbar(len(records_op)) + + images = [] + labels = [] + for i, serialized_example in enumerate(records_op): + progbar.update(i) + with tf.variable_scope("parse_images", reuse=True): + features = tf.parse_single_example( + serialized_example, + features={ + 'label': tf.FixedLenFeature([], tf.int64), + 'image_raw': tf.FixedLenFeature([], tf.string), + }) + img = tf.decode_raw(features['image_raw'], tf.uint8) + img.set_shape(batch_shape[1] * batch_shape[2]) + img = tf.reshape(img, [1] + batch_shape[1:]) + + img = tf.cast(img, tf.float32) * (1. / 255) - 0.5 + + label = tf.cast(features['label'], tf.int32) + if one_hot and classes: + label = tf.one_hot(label, classes) + + images.append(img) + labels.append(label) + + images = tf.parallel_stack(images, 0) + labels = tf.parallel_stack(labels, 0) + images = tf.cast(images, tf.float32) + + images = tf.reshape(images, shape=batch_shape) + + # StagingArea will store tensors + # across multiple steps to + # speed up execution + images_shape = images.get_shape() + labels_shape = labels.get_shape() + copy_stage = data_flow_ops.StagingArea( + [tf.float32, tf.float32], + shapes=[images_shape, labels_shape]) + copy_stage_op = copy_stage.put( + [images, labels]) + staged_images, staged_labels = copy_stage.get() + + return images, labels + + +def save_mnist_as_tfrecord(): + (X_train, y_train), (X_test, y_test) = mnist.load_data() + X_train = X_train[..., np.newaxis] + X_test = X_test[..., np.newaxis] + images_to_tfrecord(images=X_train, labels=y_train, filename='train.mnist.tfrecord') + images_to_tfrecord(images=X_test, labels=y_test, filename='test.mnist.tfrecord') + + +def cnn_layers(x_train_input): + x = Conv2D(32, (3, 3), activation='relu', padding='valid')(x_train_input) + x = Conv2D(64, (3, 3), activation='relu')(x) + x = MaxPooling2D(pool_size=(2, 2))(x) + x = Dropout(0.25)(x) + x = Flatten()(x) + x = Dense(128, activation='relu')(x) + x = Dropout(0.5)(x) + x_train_out = Dense(classes, + activation='softmax', + name='x_train_out')(x) + return x_train_out + + +sess = tf.Session() +K.set_session(sess) + +save_mnist_as_tfrecord() + +batch_size = 1 +batch_shape = [batch_size, 28, 28, 1] +epochs = 60000 +classes = 10 + +x_train_batch, y_train_batch = read_and_decode_recordinput( + 'train.mnist.tfrecord', + one_hot=True, + classes=classes, + is_train=True, + batch_shape=batch_shape) + +x_test_batch, y_test_batch = read_and_decode_recordinput( + 'test.mnist.tfrecord', + one_hot=True, + classes=classes, + is_train=True, + batch_shape=batch_shape) + + +x_batch_shape = x_train_batch.get_shape().as_list() +y_batch_shape = y_train_batch.get_shape().as_list() + +x_train_input = Input(tensor=x_train_batch, batch_shape=x_batch_shape) +x_train_out = cnn_layers(x_train_input) +y_train_in_out = Input(tensor=y_train_batch, batch_shape=y_batch_shape, name='y_labels') +cce = categorical_crossentropy(y_train_batch, x_train_out) +train_model = Model(inputs=[x_train_input], outputs=[x_train_out]) +train_model.add_loss(cce) + +train_model.compile(optimizer='rmsprop', + loss=None, + metrics=['accuracy']) + +tensorboard = TensorBoard(write_graph=True) + +train_model.summary() +train_model.fit(batch_size=batch_size, + epochs=epochs) +train_model.save_weights('saved_wt.h5') + +K.clear_session() + +# Second Session, pure Keras +(X_train, y_train), (X_test, y_test) = mnist.load_data() +X_train = X_train[..., np.newaxis] +X_test = X_test[..., np.newaxis] +x_test_inp = Input(batch_shape=(None,) + (X_test.shape[1:])) +test_out = cnn_layers(x_test_inp) +test_model = Model(inputs=x_test_inp, outputs=test_out) + +test_model.load_weights('saved_wt.h5') +test_model.compile(optimizer='rmsprop', loss='categorical_crossentropy', metrics=['accuracy']) +test_model.summary() + +loss, acc = test_model.evaluate(X_test, np_utils.to_categorical(y_test), classes) +print('\nTest accuracy: {0}'.format(acc)) From 9fae1b2193f006042ed6b7e0bef5d24ee4277b9e Mon Sep 17 00:00:00 2001 From: Andrew Hundt Date: Tue, 20 Jun 2017 21:24:18 -0400 Subject: [PATCH 063/166] test_training.py extended yield_op tests, reduce code repetition --- tests/keras/engine/test_training.py | 303 +++++++++++++--------------- 1 file changed, 138 insertions(+), 165 deletions(-) diff --git a/tests/keras/engine/test_training.py b/tests/keras/engine/test_training.py index e58ad35d697..25577be7d96 100644 --- a/tests/keras/engine/test_training.py +++ b/tests/keras/engine/test_training.py @@ -5,7 +5,9 @@ import sys import scipy.sparse as sparse -from keras.layers import Dense, Dropout +from keras.layers import Dense +from keras.layers import Conv2D +from keras.layers import Dropout from keras.layers import Flatten from keras.engine.topology import Input from keras.engine.training import Model, _check_loss_and_target_compatibility @@ -29,6 +31,39 @@ def __getitem__(self, idx): np.random.random((self.batch_size, 3))] +def call_model_methods(model, input_np, output_np, batch_size=10, epochs=1): + + # train_on_batch + out = model.train_on_batch(input_np, + output_np) + out = model.train_on_batch(input_np, + output_np) + + # test_on_batch + out = model.test_on_batch(input_np, + output_np) + + # predict_on_batch + out = model.predict_on_batch(input_np) + + # fit + out = model.fit(input_np, + output_np, epochs=1, batch_size=batch_size) + out = model.fit(input_np, + output_np, epochs=1, batch_size=batch_size) + + # evaluate + out = model.evaluate(input_np, + output_np, batch_size=batch_size) + out = model.evaluate(input_np, + output_np, batch_size=batch_size) + + # predict + out = model.predict(input_np, batch_size=batch_size) + out = model.predict(input_np, batch_size=batch_size) + return out + + @keras_test def test_model_methods(): a = Input(shape=(3,), name='input_a') @@ -465,30 +500,8 @@ def test_model_with_input_feed_tensor(): loss_weights=loss_weights, sample_weight_mode=None) - # test train_on_batch - out = model.train_on_batch(input_b_np, - [output_a_np, output_b_np]) - out = model.train_on_batch({'input_b': input_b_np}, - [output_a_np, output_b_np]) - out = model.test_on_batch({'input_b': input_b_np}, - [output_a_np, output_b_np]) - out = model.predict_on_batch({'input_b': input_b_np}) - - # test fit - out = model.fit({'input_b': input_b_np}, - [output_a_np, output_b_np], epochs=1, batch_size=10) - out = model.fit(input_b_np, - [output_a_np, output_b_np], epochs=1, batch_size=10) - - # test evaluate - out = model.evaluate({'input_b': input_b_np}, - [output_a_np, output_b_np], batch_size=10) - out = model.evaluate(input_b_np, - [output_a_np, output_b_np], batch_size=10) - - # test predict - out = model.predict({'input_b': input_b_np}, batch_size=10) - out = model.predict(input_b_np, batch_size=10) + out = call_model_methods(model, input_b_np, [output_a_np, output_b_np]) + out = call_model_methods(model, {'input_b': input_b_np}, [output_a_np, output_b_np]) assert len(out) == 2 # Now test a model with a single input @@ -503,34 +516,7 @@ def test_model_with_input_feed_tensor(): loss = 'mse' model.compile(optimizer, loss, metrics=['mean_squared_error']) - # test train_on_batch - out = model.train_on_batch(None, - output_a_np) - out = model.train_on_batch(None, - output_a_np) - out = model.test_on_batch(None, - output_a_np) - out = model.predict_on_batch(None) - out = model.train_on_batch([], - output_a_np) - out = model.train_on_batch({}, - output_a_np) - - # test fit - out = model.fit(None, - output_a_np, epochs=1, batch_size=10) - out = model.fit(None, - output_a_np, epochs=1, batch_size=10) - - # test evaluate - out = model.evaluate(None, - output_a_np, batch_size=10) - out = model.evaluate(None, - output_a_np, batch_size=10) - - # test predict - out = model.predict(None, batch_size=10) - out = model.predict(None, batch_size=10) + out = call_model_methods(model, None, output_a_np) assert out.shape == (10, 4) # Same, without learning phase @@ -544,45 +530,111 @@ def test_model_with_input_feed_tensor(): loss = 'mse' model.compile(optimizer, loss, metrics=['mean_squared_error']) - # test train_on_batch - out = model.train_on_batch(None, - output_a_np) - out = model.train_on_batch(None, - output_a_np) - out = model.test_on_batch(None, - output_a_np) - out = model.predict_on_batch(None) + # train_on_batch out = model.train_on_batch([], output_a_np) out = model.train_on_batch({}, output_a_np) - # test fit - out = model.fit(None, - output_a_np, epochs=1, batch_size=10) - out = model.fit(None, - output_a_np, epochs=1, batch_size=10) - - # test evaluate - out = model.evaluate(None, - output_a_np, batch_size=10) - out = model.evaluate(None, - output_a_np, batch_size=10) - - # test predict - out = model.predict(None, batch_size=10) - out = model.predict(None, batch_size=10) + out = call_model_methods(model, None, output_a_np) + assert out.shape == (10, 4) @pytest.mark.skipif(K.backend() != 'tensorflow', reason='Requires TF backend') @keras_test -def test_model_with_input_tfrecord(): +def test_model_with_input_yield_op(): """We test building a model with a RecordInput as input. We should be able to call fit, evaluate, predict, by only passing them data for the placeholder inputs in the model. """ + + batch_size = 1 + input_rows = 20 + cols = 20 + depth = 1 + classes = 2 + + # first batch size 1 + img_batch_shape = [batch_size, input_rows, cols, depth] + input_a_np, input_a_tf, output_a_np, output_b_np, output_b_tf = create_tfrecord_data( + img_batch_shape, classes) + # tensor input, numpy output + input_only_tfrecord_model(input_a_tf, output_b_np, img_batch_shape, classes) + # tensor input, tensor output + input_label_tfrecord_model(input_a_tf, output_b_tf, img_batch_shape, classes) + + # next batch size 3 + batch_size = 3 + img_batch_shape = [batch_size, input_rows, cols, depth] + input_a_np, input_a_tf, output_a_np, output_b_np, output_b_tf = create_tfrecord_data( + img_batch_shape, classes) + # tensor input, numpy output + input_only_tfrecord_model(input_a_tf, output_b_np, img_batch_shape, classes) + # tensor input, tensor output + input_label_tfrecord_model(input_a_tf, output_b_tf, img_batch_shape, classes) + os.remove('input_a.tfrecord') + + +def cnn_layers(x_train_input, classes): + x = Conv2D(32, (1, 1), activation='relu', padding='valid')(x_train_input) + x = Dropout(0.25)(x) + x = Flatten()(x) + x = Dense(128, activation='relu')(x) + x = Dropout(0.5)(x) + x_train_out = Dense(classes, + activation='softmax', + name='x_train_out')(x) + return x_train_out + + +def input_only_tfrecord_model(input_a_tf, output_b_np, img_batch_shape, classes): + a = Input(tensor=input_a_tf, batch_shape=img_batch_shape) + b = cnn_layers(a, classes) + model = Model([a], [b]) + model.summary() + + optimizer = 'rmsprop' + loss = 'mse' + loss_weights = [1., 2.] + with pytest.raises(ValueError) as exc: + model.compile(optimizer, loss, metrics=['mean_squared_error'], + loss_weights=loss_weights, + sample_weight_mode=None) + + model.compile(optimizer, loss, metrics=['mean_squared_error'], + sample_weight_mode=None) + + call_model_methods(model, None, output_b_np, batch_size=img_batch_shape[0]) + + +def input_label_tfrecord_model(input_a_tf, output_b_tf, img_batch_shape, classes): + a = Input(tensor=input_a_tf, batch_shape=img_batch_shape) + b = cnn_layers(a, classes) + y_train_in_out = Input( + tensor=output_b_tf, + batch_shape=[img_batch_shape[1], img_batch_shape[2]], + name='y_labels') + model = Model(inputs=[a], outputs=[b], labels=[y_train_in_out]) + model.summary() + + optimizer = 'rmsprop' + loss = 'mse' + loss_weights = [1., 2.] + with pytest.raises(ValueError) as exc: + model.compile(optimizer, loss, metrics=['mean_squared_error'], + loss_weights=loss_weights, + sample_weight_mode=None) + + model.compile(optimizer, loss, metrics=['mean_squared_error'], + sample_weight_mode=None) + + call_model_methods(model, None, None, batch_size=img_batch_shape[0]) + + +def create_tfrecord_data(img_batch_shape, classes): + import tensorflow as tf from tensorflow.python.lib.io import tf_record from tensorflow.python.ops import data_flow_ops @@ -621,6 +673,7 @@ def _bytes_feature(value): def read_and_decode_recordinput(tf_glob, one_hot=True, classes=None, is_train=None, batch_shape=[10, 3, 3, 1]): """ Return tensor to read from TFRecord """ + import tensorflow as tf print('Creating graph for loading TFRecords...') with tf.variable_scope("TFRecords"): record_input = data_flow_ops.RecordInput(tf_glob, batch_size=batch_shape[0]) @@ -664,18 +717,13 @@ def replace(filename): print('%s already exists, replacing...' % filename) os.remove(filename) - batch_size = 10 - input_rows = 3 - cols = 3 - depth = 1 - classes = 2 - img_batch_shape = [batch_size, input_rows, cols, depth] + [batch_size, input_rows, cols, depth] = img_batch_shape label_batch_shape = [batch_size, classes] input_a_np = np.multiply(np.random.random(img_batch_shape), batch_size) input_a_np = input_a_np.astype('uint8') - output_a_rows = 4 output_a_np = np.multiply(np.random.random([batch_size]), batch_size) output_a_np = output_a_np.astype('int') + output_b_np = np.random.random([batch_size, classes]) replace('input_a.tfrecord') images_to_tfrecord(input_a_np, output_a_np, 'input_a.tfrecord') input_a_tf, output_b_tf = read_and_decode_recordinput( @@ -685,60 +733,7 @@ def replace(filename): is_train=True, batch_shape=img_batch_shape) - a = Input(tensor=input_a_tf, batch_shape=img_batch_shape) - - a_2 = Dense(8, name='dense_1')(a) - dp = Dropout(0.5, name='dropout') - b_2 = dp(a_2) - f_1 = Flatten()(b_2) - b_3 = Dense( - classes, - activation='softmax', - name='dense_b3')(f_1) - - y_train_in_out = Input( - tensor=output_b_tf, - batch_shape=label_batch_shape, - name='y_labels') - - model = Model([a], [b_3], labels=[y_train_in_out]) - model.summary() - - optimizer = 'rmsprop' - loss = 'mse' - loss_weights = [1.] - with pytest.raises(NotImplementedError) as exc: - model.compile(optimizer, loss, metrics=['mean_squared_error'], - loss_weights=loss_weights, - sample_weight_mode=None) - - model.compile(optimizer, loss, metrics=['mean_squared_error'], - sample_weight_mode=None) - out = model.predict_on_batch(None) - out = model.train_on_batch(None, None) - # test fit - out = model.fit(None, - None, epochs=1, batch_size=10) - with pytest.raises(ValueError) as exc: - out = model.fit(output_a_np, - [y_train_in_out], - epochs=1, - batch_size=10) - - # test evaluate - out = model.evaluate(None, - None, - batch_size=10) - with pytest.raises(ValueError) as exc: - out = model.evaluate(input_a_np, - [output_a_np, - output_b_tf], - batch_size=10) - - # test predict - out = model.predict(None, batch_size=10) - - os.remove('input_a.tfrecord') + return input_a_np, input_a_tf, output_a_np, output_b_np, output_b_tf @keras_test @@ -774,6 +769,8 @@ def test_model_with_partial_loss(): loss = {'dense_2': 'mse'} model.compile(optimizer, loss, metrics={'dense_1': 'mae'}) + out = call_model_methods(model, input_a_np, output_a_np) + out = call_model_methods(model, input_a_np, [output_a_np]) # test train_on_batch out = model.train_on_batch(input_a_np, output_a_np) out = model.test_on_batch(input_a_np, output_a_np) @@ -803,13 +800,7 @@ def test_model_with_external_loss(): input_a_np = np.random.random((10, 3)) - # test train_on_batch - out = model.train_on_batch(input_a_np, None) - out = model.test_on_batch(input_a_np, None) - # fit - out = model.fit(input_a_np, None) - # evaluate - out = model.evaluate(input_a_np, None) + out = call_model_methods(model, input_a_np, None, batch_size=1) # No dropout, external loss. a = Input(shape=(3,), name='input_a') @@ -823,13 +814,7 @@ def test_model_with_external_loss(): loss = None model.compile(optimizer, loss, metrics=['mae']) - # test train_on_batch - out = model.train_on_batch(input_a_np, None) - out = model.test_on_batch(input_a_np, None) - # fit - out = model.fit(input_a_np, None) - # evaluate - out = model.evaluate(input_a_np, None) + out = call_model_methods(model, input_a_np, None, batch_size=10) # Test fit with no external data at all. if K.backend() == 'tensorflow': @@ -845,19 +830,7 @@ def test_model_with_external_loss(): loss=None, metrics=['mean_squared_error']) - # test train_on_batch - out = model.train_on_batch(None, None) - out = model.test_on_batch(None, None) - out = model.predict_on_batch(None) - - # test fit - out = model.fit(None, None, epochs=1, batch_size=10) - - # test evaluate - out = model.evaluate(None, None, batch_size=10) - - # test predict - out = model.predict(None, batch_size=10) + out = call_model_methods(model, None, None, batch_size=10) assert out.shape == (10, 4) From e0ffad4db6464d6c36f7b964fdf231c8c1983dd5 Mon Sep 17 00:00:00 2001 From: Andrew Hundt Date: Tue, 20 Jun 2017 21:57:55 -0400 Subject: [PATCH 064/166] test_training.py extended yield_op tests, reduce code repetition cherry picked from 9fae1b2 with manual merge steps --- tests/keras/engine/test_training.py | 293 +++++++++++++--------------- 1 file changed, 139 insertions(+), 154 deletions(-) diff --git a/tests/keras/engine/test_training.py b/tests/keras/engine/test_training.py index 4095a032c01..7a0e615a788 100644 --- a/tests/keras/engine/test_training.py +++ b/tests/keras/engine/test_training.py @@ -5,7 +5,9 @@ import sys import scipy.sparse as sparse -from keras.layers import Dense, Dropout +from keras.layers import Dense +from keras.layers import Conv2D +from keras.layers import Dropout from keras.layers import Flatten from keras.engine.topology import Input from keras.engine.training import Model, _check_loss_and_target_compatibility @@ -29,6 +31,39 @@ def __getitem__(self, idx): np.random.random((self.batch_size, 3))] +def call_model_methods(model, input_np, output_np, batch_size=10, epochs=1): + + # train_on_batch + out = model.train_on_batch(input_np, + output_np) + out = model.train_on_batch(input_np, + output_np) + + # test_on_batch + out = model.test_on_batch(input_np, + output_np) + + # predict_on_batch + out = model.predict_on_batch(input_np) + + # fit + out = model.fit(input_np, + output_np, epochs=1, batch_size=batch_size) + out = model.fit(input_np, + output_np, epochs=1, batch_size=batch_size) + + # evaluate + out = model.evaluate(input_np, + output_np, batch_size=batch_size) + out = model.evaluate(input_np, + output_np, batch_size=batch_size) + + # predict + out = model.predict(input_np, batch_size=batch_size) + out = model.predict(input_np, batch_size=batch_size) + return out + + @keras_test def test_model_methods(): a = Input(shape=(3,), name='input_a') @@ -465,30 +500,8 @@ def test_model_with_input_feed_tensor(): loss_weights=loss_weights, sample_weight_mode=None) - # test train_on_batch - out = model.train_on_batch(input_b_np, - [output_a_np, output_b_np]) - out = model.train_on_batch({'input_b': input_b_np}, - [output_a_np, output_b_np]) - out = model.test_on_batch({'input_b': input_b_np}, - [output_a_np, output_b_np]) - out = model.predict_on_batch({'input_b': input_b_np}) - - # test fit - out = model.fit({'input_b': input_b_np}, - [output_a_np, output_b_np], epochs=1, batch_size=10) - out = model.fit(input_b_np, - [output_a_np, output_b_np], epochs=1, batch_size=10) - - # test evaluate - out = model.evaluate({'input_b': input_b_np}, - [output_a_np, output_b_np], batch_size=10) - out = model.evaluate(input_b_np, - [output_a_np, output_b_np], batch_size=10) - - # test predict - out = model.predict({'input_b': input_b_np}, batch_size=10) - out = model.predict(input_b_np, batch_size=10) + out = call_model_methods(model, input_b_np, [output_a_np, output_b_np]) + out = call_model_methods(model, {'input_b': input_b_np}, [output_a_np, output_b_np]) assert len(out) == 2 # Now test a model with a single input @@ -503,34 +516,7 @@ def test_model_with_input_feed_tensor(): loss = 'mse' model.compile(optimizer, loss, metrics=['mean_squared_error']) - # test train_on_batch - out = model.train_on_batch(None, - output_a_np) - out = model.train_on_batch(None, - output_a_np) - out = model.test_on_batch(None, - output_a_np) - out = model.predict_on_batch(None) - out = model.train_on_batch([], - output_a_np) - out = model.train_on_batch({}, - output_a_np) - - # test fit - out = model.fit(None, - output_a_np, epochs=1, batch_size=10) - out = model.fit(None, - output_a_np, epochs=1, batch_size=10) - - # test evaluate - out = model.evaluate(None, - output_a_np, batch_size=10) - out = model.evaluate(None, - output_a_np, batch_size=10) - - # test predict - out = model.predict(None, batch_size=10) - out = model.predict(None, batch_size=10) + out = call_model_methods(model, None, output_a_np) assert out.shape == (10, 4) # Same, without learning phase @@ -544,34 +530,14 @@ def test_model_with_input_feed_tensor(): loss = 'mse' model.compile(optimizer, loss, metrics=['mean_squared_error']) - # test train_on_batch - out = model.train_on_batch(None, - output_a_np) - out = model.train_on_batch(None, - output_a_np) - out = model.test_on_batch(None, - output_a_np) - out = model.predict_on_batch(None) + # train_on_batch out = model.train_on_batch([], output_a_np) out = model.train_on_batch({}, output_a_np) - # test fit - out = model.fit(None, - output_a_np, epochs=1, batch_size=10) - out = model.fit(None, - output_a_np, epochs=1, batch_size=10) - - # test evaluate - out = model.evaluate(None, - output_a_np, batch_size=10) - out = model.evaluate(None, - output_a_np, batch_size=10) - - # test predict - out = model.predict(None, batch_size=10) - out = model.predict(None, batch_size=10) + out = call_model_methods(model, None, output_a_np) + assert out.shape == (10, 4) @@ -583,6 +549,95 @@ def test_model_with_input_yield_op(): by only passing them data for the placeholder inputs in the model. """ + + batch_size = 1 + input_rows = 20 + cols = 20 + depth = 1 + classes = 2 + + # first batch size 1 + img_batch_shape = [batch_size, input_rows, cols, depth] + input_a_np, input_a_tf, output_a_np, output_b_np, output_b_tf = create_tfrecord_data( + img_batch_shape, classes) + # tensor input, numpy output + input_only_tfrecord_model(input_a_tf, output_b_np, img_batch_shape, classes) + # tensor input, tensor output + input_label_tfrecord_model(input_a_tf, output_b_tf, img_batch_shape, classes) + + # next batch size 3 + batch_size = 3 + img_batch_shape = [batch_size, input_rows, cols, depth] + input_a_np, input_a_tf, output_a_np, output_b_np, output_b_tf = create_tfrecord_data( + img_batch_shape, classes) + # tensor input, numpy output + input_only_tfrecord_model(input_a_tf, output_b_np, img_batch_shape, classes) + # tensor input, tensor output + input_label_tfrecord_model(input_a_tf, output_b_tf, img_batch_shape, classes) + os.remove('input_a.tfrecord') + + +def cnn_layers(x_train_input, classes): + x = Conv2D(32, (1, 1), activation='relu', padding='valid')(x_train_input) + x = Dropout(0.25)(x) + x = Flatten()(x) + x = Dense(128, activation='relu')(x) + x = Dropout(0.5)(x) + x_train_out = Dense(classes, + activation='softmax', + name='x_train_out')(x) + return x_train_out + + +def input_only_tfrecord_model(input_a_tf, output_b_np, img_batch_shape, classes): + a = Input(tensor=input_a_tf, batch_shape=img_batch_shape) + b = cnn_layers(a, classes) + model = Model([a], [b]) + model.summary() + + optimizer = 'rmsprop' + loss = 'mse' + loss_weights = [1., 2.] + with pytest.raises(ValueError) as exc: + model.compile(optimizer, loss, metrics=['mean_squared_error'], + loss_weights=loss_weights, + sample_weight_mode=None) + + model.compile(optimizer, loss, metrics=['mean_squared_error'], + sample_weight_mode=None) + + call_model_methods(model, None, output_b_np, batch_size=img_batch_shape[0]) + + +def input_label_tfrecord_model(input_a_tf, output_b_tf, img_batch_shape, classes): + a = Input(tensor=input_a_tf, batch_shape=img_batch_shape) + b = cnn_layers(a, classes) + y_train_in_out = Input( + tensor=output_b_tf, + batch_shape=[img_batch_shape[0], classes], + name='y_labels') + b = cnn_layers(a, classes) + cce = K.categorical_crossentropy(b, y_train_in_out) + model = Model(inputs=[a], outputs=[b]) + model.add_loss(cce) + model.summary() + + optimizer = 'rmsprop' + loss = None + loss_weights = [1., 2.] + with pytest.raises(ValueError) as exc: + model.compile(optimizer, loss, metrics=['mean_squared_error'], + loss_weights=loss_weights, + sample_weight_mode=None) + + model.compile(optimizer, loss, metrics=['mean_squared_error'], + sample_weight_mode=None) + + call_model_methods(model, None, None, batch_size=img_batch_shape[0]) + + +def create_tfrecord_data(img_batch_shape, classes): + import tensorflow as tf from tensorflow.python.lib.io import tf_record from tensorflow.python.ops import data_flow_ops @@ -621,6 +676,7 @@ def _bytes_feature(value): def read_and_decode_recordinput(tf_glob, one_hot=True, classes=None, is_train=None, batch_shape=[10, 3, 3, 1]): """ Return tensor to read from TFRecord """ + import tensorflow as tf print('Creating graph for loading TFRecords...') with tf.variable_scope("TFRecords"): record_input = data_flow_ops.RecordInput(tf_glob, batch_size=batch_shape[0]) @@ -664,16 +720,10 @@ def replace(filename): print('%s already exists, replacing...' % filename) os.remove(filename) - batch_size = 10 - input_rows = 3 - cols = 3 - depth = 1 - classes = 2 - img_batch_shape = [batch_size, input_rows, cols, depth] + [batch_size, input_rows, cols, depth] = img_batch_shape label_batch_shape = [batch_size, classes] input_a_np = np.multiply(np.random.random(img_batch_shape), batch_size) input_a_np = input_a_np.astype('uint8') - output_a_rows = 4 output_a_np = np.multiply(np.random.random([batch_size]), batch_size) output_a_np = output_a_np.astype('int') output_b_np = np.random.random([batch_size, classes]) @@ -686,50 +736,7 @@ def replace(filename): is_train=True, batch_shape=img_batch_shape) - a = Input(tensor=input_a_tf, batch_shape=img_batch_shape) - - a_2 = Dense(8, name='dense_1')(a) - dp = Dropout(0.5, name='dropout') - b_2 = dp(a_2) - f_1 = Flatten()(b_2) - b_3 = Dense( - classes, - activation='softmax', - name='dense_b3')(f_1) - - y_train_in_out = Input( - tensor=output_b_tf, - batch_shape=label_batch_shape, - name='y_labels') - - model = Model([a], [b_3]) - model.summary() - - optimizer = 'rmsprop' - loss = 'mse' - loss_weights = [1., 2.] - with pytest.raises(ValueError) as exc: - model.compile(optimizer, loss, metrics=['mean_squared_error'], - loss_weights=loss_weights, - sample_weight_mode=None) - - model.compile(optimizer, loss, metrics=['mean_squared_error'], - sample_weight_mode=None) - out = model.predict_on_batch(output_b_np) - out = model.train_on_batch(None, output_b_np) - # test fit - out = model.fit(None, - output_b_np, epochs=1, batch_size=10) - - # test evaluate - out = model.evaluate(None, - output_b_np, - batch_size=10) - - # test predict - out = model.predict(None, batch_size=10) - - os.remove('input_a.tfrecord') + return input_a_np, input_a_tf, output_a_np, output_b_np, output_b_tf @keras_test @@ -765,6 +772,8 @@ def test_model_with_partial_loss(): loss = {'dense_2': 'mse'} model.compile(optimizer, loss, metrics={'dense_1': 'mae'}) + out = call_model_methods(model, input_a_np, output_a_np) + out = call_model_methods(model, input_a_np, [output_a_np]) # test train_on_batch out = model.train_on_batch(input_a_np, output_a_np) out = model.test_on_batch(input_a_np, output_a_np) @@ -794,13 +803,7 @@ def test_model_with_external_loss(): input_a_np = np.random.random((10, 3)) - # test train_on_batch - out = model.train_on_batch(input_a_np, None) - out = model.test_on_batch(input_a_np, None) - # fit - out = model.fit(input_a_np, None) - # evaluate - out = model.evaluate(input_a_np, None) + out = call_model_methods(model, input_a_np, None, batch_size=1) # No dropout, external loss. a = Input(shape=(3,), name='input_a') @@ -814,13 +817,7 @@ def test_model_with_external_loss(): loss = None model.compile(optimizer, loss, metrics=['mae']) - # test train_on_batch - out = model.train_on_batch(input_a_np, None) - out = model.test_on_batch(input_a_np, None) - # fit - out = model.fit(input_a_np, None) - # evaluate - out = model.evaluate(input_a_np, None) + out = call_model_methods(model, input_a_np, None, batch_size=10) # Test fit with no external data at all. if K.backend() == 'tensorflow': @@ -836,19 +833,7 @@ def test_model_with_external_loss(): loss=None, metrics=['mean_squared_error']) - # test train_on_batch - out = model.train_on_batch(None, None) - out = model.test_on_batch(None, None) - out = model.predict_on_batch(None) - - # test fit - out = model.fit(None, None, epochs=1, batch_size=10) - - # test evaluate - out = model.evaluate(None, None, batch_size=10) - - # test predict - out = model.predict(None, batch_size=10) + out = call_model_methods(model, None, None, batch_size=10) assert out.shape == (10, 4) From ad9283d08e5415981bea12e7d32790dc5e3b288c Mon Sep 17 00:00:00 2001 From: Andrew Hundt Date: Tue, 20 Jun 2017 21:58:22 -0400 Subject: [PATCH 065/166] generic_utils.py don't crash when dealing with batched data --- keras/utils/generic_utils.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/keras/utils/generic_utils.py b/keras/utils/generic_utils.py index 76477d5ac38..f6c026d70ca 100644 --- a/keras/utils/generic_utils.py +++ b/keras/utils/generic_utils.py @@ -288,7 +288,7 @@ def update(self, current, values=None, force=False): for k in self.unique_values: info += ' - %s:' % k if isinstance(self.sum_values[k], list): - avg = self.sum_values[k][0] / max(1, self.sum_values[k][1]) + avg = np.mean(self.sum_values[k][0] / max(1, self.sum_values[k][1])) if abs(avg) > 1e-3: info += ' %.4f' % avg else: @@ -311,7 +311,7 @@ def update(self, current, values=None, force=False): info = '%ds' % (now - self.start) for k in self.unique_values: info += ' - %s:' % k - avg = self.sum_values[k][0] / max(1, self.sum_values[k][1]) + avg = np.mean(self.sum_values[k][0] / max(1, self.sum_values[k][1])) if avg > 1e-3: info += ' %.4f' % avg else: From 1a5f6b6c70fe676f750118833e84e0ac256a162b Mon Sep 17 00:00:00 2001 From: Andrew Hundt Date: Tue, 20 Jun 2017 21:58:22 -0400 Subject: [PATCH 066/166] generic_utils.py don't crash when dealing with batched data --- keras/utils/generic_utils.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/keras/utils/generic_utils.py b/keras/utils/generic_utils.py index 76477d5ac38..f6c026d70ca 100644 --- a/keras/utils/generic_utils.py +++ b/keras/utils/generic_utils.py @@ -288,7 +288,7 @@ def update(self, current, values=None, force=False): for k in self.unique_values: info += ' - %s:' % k if isinstance(self.sum_values[k], list): - avg = self.sum_values[k][0] / max(1, self.sum_values[k][1]) + avg = np.mean(self.sum_values[k][0] / max(1, self.sum_values[k][1])) if abs(avg) > 1e-3: info += ' %.4f' % avg else: @@ -311,7 +311,7 @@ def update(self, current, values=None, force=False): info = '%ds' % (now - self.start) for k in self.unique_values: info += ' - %s:' % k - avg = self.sum_values[k][0] / max(1, self.sum_values[k][1]) + avg = np.mean(self.sum_values[k][0] / max(1, self.sum_values[k][1])) if avg > 1e-3: info += ' %.4f' % avg else: From d4f5b71c6dd211a0790e2429e859b3fe2e63ca76 Mon Sep 17 00:00:00 2001 From: Andrew Hundt Date: Wed, 21 Jun 2017 00:19:55 -0400 Subject: [PATCH 067/166] tensorflow_backend Function() corner case handling and error checking --- keras/backend/tensorflow_backend.py | 45 +++++++++++++++++++++++++---- 1 file changed, 39 insertions(+), 6 deletions(-) diff --git a/keras/backend/tensorflow_backend.py b/keras/backend/tensorflow_backend.py index ec6b637207d..ba5de46dcf4 100644 --- a/keras/backend/tensorflow_backend.py +++ b/keras/backend/tensorflow_backend.py @@ -2222,9 +2222,12 @@ class Function(object): outputs: Output tensors to fetch. updates: Additional update ops to be run at function call. name: a name to help users identify what this function does. + fetches: Parameters forwarded to `tf.session.run(fetches)`. + feed_dict: Parameters forwarded to `tf.session.run(feed_dict)`. """ - def __init__(self, inputs, outputs, updates=None, name=None, **session_kwargs): + def __init__(self, inputs, outputs, updates=None, name=None, + fetches=None, feed_dict=None, **session_kwargs): updates = updates or [] if not isinstance(inputs, (list, tuple)): raise TypeError('`inputs` to a TensorFlow backend function ' @@ -2235,8 +2238,11 @@ def __init__(self, inputs, outputs, updates=None, name=None, **session_kwargs): if not isinstance(updates, (list, tuple)): raise TypeError('`updates` in a TensorFlow backend function ' 'should be a list or tuple.') + # self.inputs holds tf Tensor objects self.inputs = list(inputs) self.outputs = list(outputs) + self.fetches = fetches + self.feed_dict = feed_dict with tf.control_dependencies(self.outputs): updates_ops = [] for update in updates: @@ -2251,19 +2257,46 @@ def __init__(self, inputs, outputs, updates=None, name=None, **session_kwargs): self.session_kwargs = session_kwargs def __call__(self, inputs): + """Run the TensorFlow session + + # Arguments + inputs: Data and values that will go to the feed_dict of Session.run() + if it is associated with a tensor, if it is None the tensor will + be added to the fetches parameter of Session.run(). + """ if not isinstance(inputs, (list, tuple)): raise TypeError('`inputs` should be a list or tuple.') - feed_dict = {} - for tensor, value in zip(self.inputs, inputs): + self.current_feed_dict = {} if self.feed_dict is None else self.feed_dict + self.feed_to_fetch_count = 0 + self.current_fetches = self.outputs + [self.updates_op] + # self.inputs contains tf tensors, inputs contains feed_dict data. + for tensor, value in zip_longest(self.inputs, inputs, fillvalue=None): + if tensor is None and value is None: + continue + elif tensor is None and value is not None: + raise ValueError('A tensor containing None ' + 'was tied to value ' + str(value) + + 'so Session.run() cannot execute, ' + 'please check your data and Model.') + if is_sparse(tensor): sparse_coo = value.tocoo() indices = np.concatenate((np.expand_dims(sparse_coo.row, 1), np.expand_dims(sparse_coo.col, 1)), 1) value = (indices, sparse_coo.data, sparse_coo.shape) - feed_dict[tensor] = value + + if value is None and tensor is not None: + self.feed_to_fetch_count += 1 + self.current_fetches.append(tensor) + else: + self.current_feed_dict[tensor] = value + + if self.fetches is not None: + self.current_fetches += self.fetches + session = get_session() - updated = session.run(self.outputs + [self.updates_op], - feed_dict=feed_dict, + updated = session.run(fetches=self.current_fetches, + feed_dict=self.current_feed_dict, **self.session_kwargs) return updated[:len(self.outputs)] From de54a1c44d882d44c372bb1dfd918b886c936a76 Mon Sep 17 00:00:00 2001 From: Andrew Hundt Date: Tue, 20 Jun 2017 23:51:12 -0400 Subject: [PATCH 068/166] tensorflow_backend.py remove self.feed_to_fetch_count --- keras/backend/tensorflow_backend.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/keras/backend/tensorflow_backend.py b/keras/backend/tensorflow_backend.py index ba5de46dcf4..e359e49deef 100644 --- a/keras/backend/tensorflow_backend.py +++ b/keras/backend/tensorflow_backend.py @@ -2267,7 +2267,6 @@ def __call__(self, inputs): if not isinstance(inputs, (list, tuple)): raise TypeError('`inputs` should be a list or tuple.') self.current_feed_dict = {} if self.feed_dict is None else self.feed_dict - self.feed_to_fetch_count = 0 self.current_fetches = self.outputs + [self.updates_op] # self.inputs contains tf tensors, inputs contains feed_dict data. for tensor, value in zip_longest(self.inputs, inputs, fillvalue=None): @@ -2286,7 +2285,6 @@ def __call__(self, inputs): value = (indices, sparse_coo.data, sparse_coo.shape) if value is None and tensor is not None: - self.feed_to_fetch_count += 1 self.current_fetches.append(tensor) else: self.current_feed_dict[tensor] = value From 54241d5dde275c1fa9c37f48979e3b0799a863f9 Mon Sep 17 00:00:00 2001 From: Andrew Hundt Date: Wed, 21 Jun 2017 00:38:39 -0400 Subject: [PATCH 069/166] is_placeholder() implemented small part of #6928 and #7046 --- examples/variational_autoencoder.py | 2 +- examples/variational_autoencoder_deconv.py | 2 +- keras/backend/__init__.py | 1 + keras/backend/cntk_backend.py | 2 ++ keras/backend/common.py | 23 +++++++++++++++++++++ keras/backend/tensorflow_backend.py | 4 +++- keras/engine/topology.py | 24 +++++++++++++++++----- keras/models.py | 2 ++ 8 files changed, 52 insertions(+), 8 deletions(-) diff --git a/examples/variational_autoencoder.py b/examples/variational_autoencoder.py index fd395b17f45..ba313f1c69b 100644 --- a/examples/variational_autoencoder.py +++ b/examples/variational_autoencoder.py @@ -45,7 +45,7 @@ def sampling(args): # Custom loss layer class CustomVariationalLayer(Layer): def __init__(self, **kwargs): - self.is_placeholder = True + self._is_placeholder = True super(CustomVariationalLayer, self).__init__(**kwargs) def vae_loss(self, x, x_decoded_mean): diff --git a/examples/variational_autoencoder_deconv.py b/examples/variational_autoencoder_deconv.py index f5aa1224d47..3612754c494 100644 --- a/examples/variational_autoencoder_deconv.py +++ b/examples/variational_autoencoder_deconv.py @@ -109,7 +109,7 @@ def sampling(args): # Custom loss layer class CustomVariationalLayer(Layer): def __init__(self, **kwargs): - self.is_placeholder = True + self._is_placeholder = True super(CustomVariationalLayer, self).__init__(**kwargs) def vae_loss(self, x, x_decoded_mean_squash): diff --git a/keras/backend/__init__.py b/keras/backend/__init__.py index 2e7208cf5af..00e6d52fbe8 100644 --- a/keras/backend/__init__.py +++ b/keras/backend/__init__.py @@ -10,6 +10,7 @@ from .common import cast_to_floatx from .common import image_data_format from .common import set_image_data_format +from .common import is_placeholder # Obtain Keras base dir path: either ~/.keras or /tmp. _keras_base_dir = os.path.expanduser('~') diff --git a/keras/backend/cntk_backend.py b/keras/backend/cntk_backend.py index b4161ced8e7..cfe056896a0 100644 --- a/keras/backend/cntk_backend.py +++ b/keras/backend/cntk_backend.py @@ -2,6 +2,7 @@ import cntk as C import numpy as np from .common import _FLOATX, _EPSILON, image_dim_ordering, image_data_format +from .common import is_placeholder from collections import defaultdict from contextlib import contextmanager import warnings @@ -256,6 +257,7 @@ def placeholder( name=name) x._keras_shape = shape x._uses_learning_phase = False + x._is_placeholder = True return x diff --git a/keras/backend/common.py b/keras/backend/common.py index 97646067c9b..a147c7f8e1d 100644 --- a/keras/backend/common.py +++ b/keras/backend/common.py @@ -108,6 +108,29 @@ def cast_to_floatx(x): return np.asarray(x, dtype=_FLOATX) +def is_placeholder(tensor): + """Returns whether a tensor is a placeholder. + + # Arguments + tensor: A tensor instance. + + # Returns + A boolean. + + # Example + ```python + >>> from keras import backend as K + >>> a = K.placeholder((2, 2), sparse=False) + >>> print(K.is_placeholder(a)) + True + ``` + """ + try: + return tensor._is_placeholder + except AttributeError: + return False + + def image_data_format(): """Returns the default image data format convention ('channels_first' or 'channels_last'). diff --git a/keras/backend/tensorflow_backend.py b/keras/backend/tensorflow_backend.py index ec6b637207d..1dc5c9fb955 100644 --- a/keras/backend/tensorflow_backend.py +++ b/keras/backend/tensorflow_backend.py @@ -14,6 +14,7 @@ from .common import floatx from .common import _EPSILON from .common import image_data_format +from .common import is_placeholder # Legacy functions from .common import set_image_dim_ordering @@ -373,7 +374,7 @@ def is_keras_tensor(x): ```python >>> from keras import backend as K >>> np_var = numpy.array([1, 2]) - >>> K.is_keras_tensor(np_var) # A numpy array is not a symbolic yensor. + >>> K.is_keras_tensor(np_var) # A numpy array is not a symbolic tensor. ValueError >>> k_var = tf.placeholder('float32', shape=(1,1)) >>> K.is_keras_tensor(k_var) # A variable created directly from tensorflow/theano is not a Keras tensor. @@ -431,6 +432,7 @@ def placeholder(shape=None, ndim=None, dtype=None, sparse=False, name=None): x = tf.placeholder(dtype, shape=shape, name=name) x._keras_shape = shape x._uses_learning_phase = False + x._is_placeholder = True return x diff --git a/keras/engine/topology.py b/keras/engine/topology.py index ba5cbf707f5..61454f14330 100644 --- a/keras/engine/topology.py +++ b/keras/engine/topology.py @@ -97,6 +97,7 @@ class Node(object): output_shapes: list of output shape tuples. arguments: dictionary of keyword arguments that were passed to the `call` method of the layer at the call that created the node. + is_placeholder: Specifies if the Node represents a placeholder. `node_indices` and `tensor_indices` are basically fine-grained coordinates describing the origin of the `input_tensors`, verifying the following: @@ -113,7 +114,7 @@ def __init__(self, outbound_layer, input_tensors, output_tensors, input_masks, output_masks, input_shapes, output_shapes, - arguments=None): + arguments=None, is_placeholder=False): # Layer instance (NOT a list). # this is the layer that takes a list of input tensors # and turns them into a list of output tensors. @@ -157,6 +158,8 @@ def __init__(self, outbound_layer, # Optional keyword arguments to layer's `call`. self.arguments = arguments + # Indicates if the node represents a placeholder variable + self._is_placeholder = is_placeholder # Add nodes to all layers involved. for layer in inbound_layers: if layer is not None: @@ -1330,18 +1333,19 @@ def __init__(self, input_shape=None, batch_size=None, self.dtype = dtype if input_tensor is None: - self.is_placeholder = True + self._is_placeholder = True input_tensor = K.placeholder(shape=batch_input_shape, dtype=dtype, sparse=self.sparse, name=self.name) else: - self.is_placeholder = False + self._is_placeholder = False input_tensor._keras_shape = batch_input_shape # Create an input node to add to self.outbound_node # and set output_tensors' _keras_history. input_tensor._uses_learning_phase = False input_tensor._keras_history = (self, 0, 0) + input_tensor._is_placeholder = self._is_placeholder Node(self, inbound_layers=[], node_indices=[], @@ -1351,7 +1355,8 @@ def __init__(self, input_shape=None, batch_size=None, input_masks=[None], output_masks=[None], input_shapes=[batch_input_shape], - output_shapes=[batch_input_shape]) + output_shapes=[batch_input_shape], + is_placeholder=self._is_placeholder) def get_config(self): config = {'batch_input_shape': self.batch_input_shape, @@ -1491,11 +1496,14 @@ def __init__(self, inputs, outputs, name=None): self.inputs = list(inputs) # Tensor or list of tensors. else: self.inputs = [inputs] + if isinstance(outputs, (list, tuple)): self.outputs = list(outputs) else: self.outputs = [outputs] + self.target_configuration = [None] * len(self.outputs) + # Check for redundancy in inputs. if len(set(self.inputs)) != len(self.inputs): raise ValueError('The list of inputs passed to the model ' @@ -1535,6 +1543,8 @@ def __init__(self, inputs, outputs, name=None): self._output_tensor_cache = {} self._output_shape_cache = {} + self._input_placeholders = [] + self._input_yield_op_tensors = [] # User-provided arguments validation. for x in self.inputs: # Check that x is a Keras tensor. @@ -1560,6 +1570,10 @@ def __init__(self, inputs, outputs, name=None): 'instantiated via `tensor = Input(shape)`.\n' 'The tensor that caused the issue was: ' + str(x.name)) + if K.is_placeholder(layer): + self._input_placeholders.append((layer, node_index, tensor_index)) + else: + self._input_yield_op_tensors.append((layer, node_index, tensor_index)) for x in self.outputs: if not hasattr(x, '_keras_history'): cls_name = self.__class__.__name__ @@ -1621,7 +1635,7 @@ def __init__(self, inputs, outputs, name=None): i, layer.__class__.__name__)) self.input_names.append(layer.name) - if layer.is_placeholder: + if K.is_placeholder(layer): self._feed_input_names.append(layer.name) self._feed_inputs.append(layer.input) self._feed_input_shapes.append(self.inputs[i]._keras_shape) diff --git a/keras/models.py b/keras/models.py index 6228fc81b2a..3d9aea608c0 100644 --- a/keras/models.py +++ b/keras/models.py @@ -456,6 +456,7 @@ def add(self, layer): 'For multi-output layers, ' 'use the functional API.') + layer.inbound_nodes[0].output_tensors[0]._is_placeholder = True self.outputs = [layer.inbound_nodes[0].output_tensors[0]] self.inputs = topology.get_source_inputs(self.outputs[0]) @@ -479,6 +480,7 @@ def add(self, layer): 'should have a single output tensor. ' 'For multi-output layers, ' 'use the functional API.') + output_tensor._is_placeholder = True self.outputs = [output_tensor] # update self.inbound_nodes self.inbound_nodes[0].output_tensors = self.outputs From 800e5a1ae0ff3618a7dc2d77abc5a13cda6afc67 Mon Sep 17 00:00:00 2001 From: Andrew Hundt Date: Wed, 21 Jun 2017 00:42:14 -0400 Subject: [PATCH 070/166] import zip_longest from six.moves for #7064 --- keras/backend/tensorflow_backend.py | 1 + 1 file changed, 1 insertion(+) diff --git a/keras/backend/tensorflow_backend.py b/keras/backend/tensorflow_backend.py index e359e49deef..001110a7d2d 100644 --- a/keras/backend/tensorflow_backend.py +++ b/keras/backend/tensorflow_backend.py @@ -10,6 +10,7 @@ import inspect import numpy as np import os +from six.moves import zip_longest from .common import floatx from .common import _EPSILON From b39c97d02bb6f75712cfb9ac3f55f74ade6b0b2a Mon Sep 17 00:00:00 2001 From: Andrew Hundt Date: Wed, 21 Jun 2017 00:00:59 -0400 Subject: [PATCH 071/166] training.py _standardize_input_data() cleanup protects against corner cases --- keras/engine/training.py | 33 +++++++++++++++++++++------------ 1 file changed, 21 insertions(+), 12 deletions(-) diff --git a/keras/engine/training.py b/keras/engine/training.py index bc8ecb27b86..16eaf8f71e1 100644 --- a/keras/engine/training.py +++ b/keras/engine/training.py @@ -28,7 +28,8 @@ def _standardize_input_data(data, names, shapes=None, check_batch_axis=True, - exception_prefix=''): + exception_prefix='', + tensors=None): """Normalizes inputs and targets provided by users. Users may pass data as a list of arrays, dictionary of arrays, @@ -51,10 +52,14 @@ def _standardize_input_data(data, names, shapes=None, # Raises ValueError: in case of improperly formatted user-provided data. """ + if tensors is None: + tensors = [] if not names: return [] if data is None: return [None for _ in range(len(names))] + num_tensors = sum(x is not None for x in tensors) + expected_names = len(names) - num_tensors if isinstance(data, dict): arrays = [] for name in names: @@ -64,18 +69,22 @@ def _standardize_input_data(data, names, shapes=None, str(names)) arrays.append(data[name]) elif isinstance(data, list): - if len(data) != len(names): + if len(data) != expected_names: if data and hasattr(data[0], 'shape'): + if type(data).__module__ == np.__name__: + datastr = ('the following list of ' + str(len(data)) + + ' arrays: ' + str(data)[:200] + + '...') + else: + datastr = ('data with type' + + type(data).__name__ + '...') raise ValueError('Error when checking model ' + exception_prefix + ': the list of Numpy arrays ' 'that you are passing to your model ' 'is not the size the model expected. ' - 'Expected to see ' + str(len(names)) + - ' arrays but instead got ' - 'the following list of ' + str(len(data)) + - ' arrays: ' + str(data)[:200] + - '...') + 'Expected to see ' + str(expected_names) + + ' arrays but instead got ' + datastr) else: if len(names) == 1: data = [np.asarray(data)] @@ -86,7 +95,7 @@ def _standardize_input_data(data, names, shapes=None, ': you are passing a list as ' 'input to your model, ' 'but the model expects ' - 'a list of ' + str(len(names)) + + 'a list of ' + str(expected_names) + ' Numpy arrays instead. ' 'The list you passed was: ' + str(data)[:200]) @@ -98,17 +107,17 @@ def _standardize_input_data(data, names, shapes=None, ': data should be a Numpy array, ' 'or list/dict of Numpy arrays. ' 'Found: ' + str(data)[:200] + '...') - if len(names) > 1: + if expected_names > 1: # Case: model expects multiple inputs but only received # a single Numpy array. - raise ValueError('The model expects ' + str(len(names)) + ' ' + + raise ValueError('The model expects ' + str(expected_names) + exception_prefix + ' arrays, but only received one array. ' 'Found: array with shape ' + str(data.shape)) arrays = [data] # Make arrays at least 2D. - for i in range(len(names)): + for i in range(expected_names): array = arrays[i] if len(array.shape) == 1: array = np.expand_dims(array, 1) @@ -116,7 +125,7 @@ def _standardize_input_data(data, names, shapes=None, # Check shapes compatibility. if shapes: - for i in range(len(names)): + for i in range(expected_names): if shapes[i] is None: continue array = arrays[i] From 606f30d923d750695715677610c36e7273f44138 Mon Sep 17 00:00:00 2001 From: Andrew Hundt Date: Wed, 21 Jun 2017 01:31:10 -0400 Subject: [PATCH 072/166] test is_placeholder() --- tests/keras/engine/test_topology.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/tests/keras/engine/test_topology.py b/tests/keras/engine/test_topology.py index f3c17724c00..fd258304896 100644 --- a/tests/keras/engine/test_topology.py +++ b/tests/keras/engine/test_topology.py @@ -33,6 +33,12 @@ def test_get_losses_for(): assert dense_layer.get_losses_for(None) == [1] +@keras_test +def test_is_placeholder(): + a = Input(shape=(2,)) + assert K.is_placeholder(a) + + @keras_test def test_trainable_weights(): a = Input(shape=(2,)) From 67aa1cd77a5900138fc3649ba8e5c289a7aefaf1 Mon Sep 17 00:00:00 2001 From: Andrew Hundt Date: Tue, 20 Jun 2017 23:51:53 -0400 Subject: [PATCH 073/166] is_placeholder theano_backend.py --- keras/backend/theano_backend.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/keras/backend/theano_backend.py b/keras/backend/theano_backend.py index 17b8d85c403..192880d5dac 100644 --- a/keras/backend/theano_backend.py +++ b/keras/backend/theano_backend.py @@ -17,6 +17,7 @@ import inspect import numpy as np from .common import _FLOATX, floatx, _EPSILON, image_data_format +from .common import is_placeholder # Legacy functions from .common import set_image_dim_ordering, image_dim_ordering @@ -221,6 +222,7 @@ def placeholder(shape=None, ndim=None, dtype=None, sparse=False, name=None): x = T.TensorType(dtype, broadcast)(name) x._keras_shape = shape x._uses_learning_phase = False + x._is_placeholder = True return x From 0387a8c8d99556a397bfa5ed372c0f4faad61396 Mon Sep 17 00:00:00 2001 From: Andrew Hundt Date: Wed, 21 Jun 2017 02:23:03 -0400 Subject: [PATCH 074/166] test_training.py add test_standardize_input_data() --- tests/keras/engine/test_training.py | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/tests/keras/engine/test_training.py b/tests/keras/engine/test_training.py index e0de52246d9..5c3a832dedb 100644 --- a/tests/keras/engine/test_training.py +++ b/tests/keras/engine/test_training.py @@ -5,6 +5,7 @@ import scipy.sparse as sparse from keras.layers import Dense, Dropout +from keras.engine.training import _standardize_input_data from keras.engine.topology import Input from keras.engine.training import Model, _check_loss_and_target_compatibility from keras.models import Sequential @@ -27,6 +28,23 @@ def __getitem__(self, idx): np.random.random((self.batch_size, 3))] +@keras_test +def test_standardize_input_data(): + a_np = np.random.random((4, 3)) + a_shape = a_np.shape + a_name = 'input_a' + p = K.placeholder(a_shape) + a = Input(shape=a_shape, name='a_name') + x = _standardize_input_data( + [a_np], [a_name], [a_shape], + check_batch_axis=False, + exception_prefix='input') + x = _standardize_input_data( + [None], [a_name], [a_shape], + exception_prefix='input', + tensors=[p]) + + @keras_test def test_model_methods(): a = Input(shape=(3,), name='input_a') From 722e1317c0b3c347c21bb2a4eefd43dc5ac53593 Mon Sep 17 00:00:00 2001 From: Andrew Hundt Date: Wed, 21 Jun 2017 02:56:44 -0400 Subject: [PATCH 075/166] extra is_placeholder() tests --- tests/keras/backend/backend_test.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/keras/backend/backend_test.py b/tests/keras/backend/backend_test.py index 4e9dbedea63..4f875e07d1c 100644 --- a/tests/keras/backend/backend_test.py +++ b/tests/keras/backend/backend_test.py @@ -182,6 +182,8 @@ def test_is_keras_tensor(self): assert K.is_keras_tensor(keras_var) is True keras_placeholder = K.placeholder(shape=(2, 4, 5)) assert K.is_keras_tensor(keras_placeholder) is True + assert K.is_placeholder(keras_placeholder) is True + assert K.is_placeholder(np_var) is False def test_set_learning_phase(self): # not supported learning_phase From 38cfe25ecca82a635a86b8632a15d8bd4402af12 Mon Sep 17 00:00:00 2001 From: Andrew Hundt Date: Wed, 21 Jun 2017 03:20:07 -0400 Subject: [PATCH 076/166] Progbar() unit test --- tests/keras/utils/generic_utils_test.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/tests/keras/utils/generic_utils_test.py b/tests/keras/utils/generic_utils_test.py index 2bc71c48aec..0b4bec285b7 100644 --- a/tests/keras/utils/generic_utils_test.py +++ b/tests/keras/utils/generic_utils_test.py @@ -1,7 +1,20 @@ import pytest +import numpy as np from keras.utils.generic_utils import custom_object_scope from keras import activations from keras import regularizers +from keras.utils.test_utils import keras_test +from keras.utils.generic_utils import Progbar + + +@keras_test +def test_progbar(): + n = 2 + input_arr = np.random.random((n, n, n)) + bar = Progbar(n) + + for i, arr in enumerate(input_arr): + bar.update(i, list(arr)) def test_custom_objects_scope(): From c67cfadd8e71f03877f2e357d3a26d8f28cc5ffd Mon Sep 17 00:00:00 2001 From: Andrew Hundt Date: Wed, 21 Jun 2017 03:33:48 -0400 Subject: [PATCH 077/166] mnist_tfrecord.py added (#7061, #7072, #6928, #7046) --- examples/mnist_tfrecord.py | 209 +++++++++++++++++++++++++++++++++++++ 1 file changed, 209 insertions(+) create mode 100644 examples/mnist_tfrecord.py diff --git a/examples/mnist_tfrecord.py b/examples/mnist_tfrecord.py new file mode 100644 index 00000000000..93d72c03831 --- /dev/null +++ b/examples/mnist_tfrecord.py @@ -0,0 +1,209 @@ +'''MNIST dataset with TensorFlow TFRecords. + +Gets to 99.25% test accuracy after 12 epochs +(there is still a lot of margin for parameter tuning). +''' +import os +import copy +import time + +import numpy as np + +import tensorflow as tf +from tensorflow.python.ops import data_flow_ops +from keras import backend as K +from keras.models import Model +from keras.layers import Dense +from keras.layers import Dropout +from keras.layers import Flatten +from keras.layers import Input +from keras.layers import Conv2D +from keras.layers import MaxPooling2D +from keras.callbacks import EarlyStopping +from keras.layers import TensorBoard +from keras.objectives import categorical_crossentropy +from keras.utils import np_utils +from keras.utils.generic_utils import Progbar +from keras import callbacks as cbks +from keras import optimizers, objectives +from keras import metrics as metrics_module + +from keras.datasets import mnist + +if K.backend() != 'tensorflow': + raise RuntimeError('This example can only run with the ' + 'TensorFlow backend for the time being, ' + 'because it requires TFRecords, which ' + 'are not supported on other platforms.') + + +def images_to_tfrecord(images, labels, filename): + def _int64_feature(value): + return tf.train.Feature(int64_list=tf.train.Int64List(value=[value])) + + def _bytes_feature(value): + return tf.train.Feature(bytes_list=tf.train.BytesList(value=[value])) + + """ Save data into TFRecord """ + if not os.path.isfile(filename): + num_examples = images.shape[0] + + rows = images.shape[1] + cols = images.shape[2] + depth = images.shape[3] + + print('Writing', filename) + writer = tf.python_io.TFRecordWriter(filename) + for index in range(num_examples): + image_raw = images[index].tostring() + example = tf.train.Example(features=tf.train.Features(feature={ + 'height': _int64_feature(rows), + 'width': _int64_feature(cols), + 'depth': _int64_feature(depth), + 'label': _int64_feature(int(labels[index])), + 'image_raw': _bytes_feature(image_raw)})) + writer.write(example.SerializeToString()) + writer.close() + else: + print('tfrecord %s already exists' % filename) + + +def read_and_decode_recordinput(tf_glob, one_hot=True, classes=None, is_train=None, batch_shape=[1000, 28, 28, 1]): + """ Return tensor to read from TFRecord """ + print 'Creating graph for loading TFRecords...' + with tf.variable_scope("TFRecords"): + record_input = data_flow_ops.RecordInput(tf_glob, batch_size=batch_shape[0]) + records_op = record_input.get_yield_op() + records_op = tf.split(records_op, batch_shape[0], 0) + records_op = [tf.reshape(record, []) for record in records_op] + progbar = Progbar(len(records_op)) + + images = [] + labels = [] + for i, serialized_example in enumerate(records_op): + progbar.update(i) + with tf.variable_scope("parse_images", reuse=True): + features = tf.parse_single_example( + serialized_example, + features={ + 'label': tf.FixedLenFeature([], tf.int64), + 'image_raw': tf.FixedLenFeature([], tf.string), + }) + img = tf.decode_raw(features['image_raw'], tf.uint8) + img.set_shape(batch_shape[1] * batch_shape[2]) + img = tf.reshape(img, [1] + batch_shape[1:]) + + img = tf.cast(img, tf.float32) * (1. / 255) - 0.5 + + label = tf.cast(features['label'], tf.int32) + if one_hot and classes: + label = tf.one_hot(label, classes) + + images.append(img) + labels.append(label) + + images = tf.parallel_stack(images, 0) + labels = tf.parallel_stack(labels, 0) + images = tf.cast(images, tf.float32) + + images = tf.reshape(images, shape=batch_shape) + + # StagingArea will store tensors + # across multiple steps to + # speed up execution + images_shape = images.get_shape() + labels_shape = labels.get_shape() + copy_stage = data_flow_ops.StagingArea( + [tf.float32, tf.float32], + shapes=[images_shape, labels_shape]) + copy_stage_op = copy_stage.put( + [images, labels]) + staged_images, staged_labels = copy_stage.get() + + return images, labels + + +def save_mnist_as_tfrecord(): + (X_train, y_train), (X_test, y_test) = mnist.load_data() + X_train = X_train[..., np.newaxis] + X_test = X_test[..., np.newaxis] + images_to_tfrecord(images=X_train, labels=y_train, filename='train.mnist.tfrecord') + images_to_tfrecord(images=X_test, labels=y_test, filename='test.mnist.tfrecord') + + +def cnn_layers(x_train_input): + x = Conv2D(32, (3, 3), activation='relu', padding='valid')(x_train_input) + x = Conv2D(64, (3, 3), activation='relu')(x) + x = MaxPooling2D(pool_size=(2, 2))(x) + x = Dropout(0.25)(x) + x = Flatten()(x) + x = Dense(128, activation='relu')(x) + x = Dropout(0.5)(x) + x_train_out = Dense(classes, + activation='softmax', + name='x_train_out')(x) + return x_train_out + + +sess = tf.Session() +K.set_session(sess) + +save_mnist_as_tfrecord() + +batch_size = 1 +batch_shape = [batch_size, 28, 28, 1] +epochs = 60000 +classes = 10 + +x_train_batch, y_train_batch = read_and_decode_recordinput( + 'train.mnist.tfrecord', + one_hot=True, + classes=classes, + is_train=True, + batch_shape=batch_shape) + +x_test_batch, y_test_batch = read_and_decode_recordinput( + 'test.mnist.tfrecord', + one_hot=True, + classes=classes, + is_train=True, + batch_shape=batch_shape) + + +x_batch_shape = x_train_batch.get_shape().as_list() +y_batch_shape = y_train_batch.get_shape().as_list() + +x_train_input = Input(tensor=x_train_batch, batch_shape=x_batch_shape) +x_train_out = cnn_layers(x_train_input) +y_train_in_out = Input(tensor=y_train_batch, batch_shape=y_batch_shape, name='y_labels') +cce = categorical_crossentropy(y_train_batch, x_train_out) +train_model = Model(inputs=[x_train_input], outputs=[x_train_out]) +train_model.add_loss(cce) + +train_model.compile(optimizer='rmsprop', + loss=None, + metrics=['accuracy']) + +tensorboard = TensorBoard(write_graph=True) + +train_model.summary() +train_model.fit(batch_size=batch_size, + epochs=epochs) +train_model.save_weights('saved_wt.h5') + +K.clear_session() + +# Second Session, pure Keras +(X_train, y_train), (X_test, y_test) = mnist.load_data() +X_train = X_train[..., np.newaxis] +X_test = X_test[..., np.newaxis] +x_test_inp = Input(batch_shape=(None,) + (X_test.shape[1:])) +test_out = cnn_layers(x_test_inp) +test_model = Model(inputs=x_test_inp, outputs=test_out) + +test_model.load_weights('saved_wt.h5') +test_model.compile(optimizer='rmsprop', loss='categorical_crossentropy', metrics=['accuracy']) +test_model.summary() + +loss, acc = test_model.evaluate(X_test, np_utils.to_categorical(y_test), classes) +print('\nTest accuracy: {0}'.format(acc)) From 04cdf5f65d631eff6f9eba3caf36b10634238453 Mon Sep 17 00:00:00 2001 From: Andrew Hundt Date: Wed, 21 Jun 2017 04:10:46 -0400 Subject: [PATCH 078/166] test keras.backend.Function() input handling backend test --- tests/keras/backend/backend_test.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/tests/keras/backend/backend_test.py b/tests/keras/backend/backend_test.py index 4e9dbedea63..87bd06ebe64 100644 --- a/tests/keras/backend/backend_test.py +++ b/tests/keras/backend/backend_test.py @@ -583,7 +583,7 @@ def test_gradient(self): assert_allclose(zero_list[i], z_list[i], atol=1e-05) assert_allclose(zero_list[i + 1], zero_list[i + 1], atol=1e-05) - # cntk currently not support funciton in this way, so can't test as this + # cntk currently not support function in this way, so can't test as this def test_function(self): test_backend = [KTH, KTF] val = np.random.random((4, 2)) @@ -599,6 +599,10 @@ def test_function(self): update = x * 2 f = k.function([y], [exp], updates=[(x, update)]) f_list.append(f) + if k == KTF: + exp2 = x + k.square(y) + f = k.function([y, None, exp2], [exp, None], updates=[(x, update)]) + f([input_val])[0] function_outputs_list = [f([input_val])[0] for f in f_list] for i in range(len(function_outputs_list) - 1): From fa9ce56fcbd98f42c7415f633dea5937cd8c4ef2 Mon Sep 17 00:00:00 2001 From: Andrew Hundt Date: Wed, 21 Jun 2017 10:15:02 -0400 Subject: [PATCH 079/166] backend_test.py reverse change. --- tests/keras/backend/backend_test.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/tests/keras/backend/backend_test.py b/tests/keras/backend/backend_test.py index 87bd06ebe64..cdc48cf6d50 100644 --- a/tests/keras/backend/backend_test.py +++ b/tests/keras/backend/backend_test.py @@ -599,10 +599,6 @@ def test_function(self): update = x * 2 f = k.function([y], [exp], updates=[(x, update)]) f_list.append(f) - if k == KTF: - exp2 = x + k.square(y) - f = k.function([y, None, exp2], [exp, None], updates=[(x, update)]) - f([input_val])[0] function_outputs_list = [f([input_val])[0] for f in f_list] for i in range(len(function_outputs_list) - 1): From 8f5092d39b3161921ad004ae39aa6bd38ff2d53f Mon Sep 17 00:00:00 2001 From: Andrew Hundt Date: Wed, 21 Jun 2017 11:10:14 -0400 Subject: [PATCH 080/166] test_backend.py K.backend.function() error handling test --- tests/keras/backend/backend_test.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/tests/keras/backend/backend_test.py b/tests/keras/backend/backend_test.py index cdc48cf6d50..5b73f451eca 100644 --- a/tests/keras/backend/backend_test.py +++ b/tests/keras/backend/backend_test.py @@ -599,6 +599,17 @@ def test_function(self): update = x * 2 f = k.function([y], [exp], updates=[(x, update)]) f_list.append(f) + if k == KTF: + f = k.function([y, k.variable(1.)], [exp], updates=[(x, update)]) + f([input_val])[0] + f = k.function([y, k.variable(1.), None], [exp], updates=[(x, update)]) + f([input_val, None, None, None])[0] + with pytest.raises(ValueError): + f = K.function([y, K.variable(1.), None], [exp], updates=[(x, update)]) + f([input_val, None, input_val])[0] + with pytest.raises(ValueError): + f = K.function([y, K.variable(1.)], [exp], updates=[(x, update)]) + f([input_val, None, input_val])[0] function_outputs_list = [f([input_val])[0] for f in f_list] for i in range(len(function_outputs_list) - 1): From c7f6eb371629db04dbbd75f6d1a8f73f0ec6b86c Mon Sep 17 00:00:00 2001 From: Andrew Hundt Date: Wed, 21 Jun 2017 11:38:10 -0400 Subject: [PATCH 081/166] Fix mnist_tfrecord.py runtime errors --- examples/mnist_tfrecord.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/examples/mnist_tfrecord.py b/examples/mnist_tfrecord.py index 93d72c03831..65ab5ce5e28 100644 --- a/examples/mnist_tfrecord.py +++ b/examples/mnist_tfrecord.py @@ -20,7 +20,7 @@ from keras.layers import Conv2D from keras.layers import MaxPooling2D from keras.callbacks import EarlyStopping -from keras.layers import TensorBoard +from keras.callbacks import TensorBoard from keras.objectives import categorical_crossentropy from keras.utils import np_utils from keras.utils.generic_utils import Progbar @@ -70,7 +70,7 @@ def _bytes_feature(value): def read_and_decode_recordinput(tf_glob, one_hot=True, classes=None, is_train=None, batch_shape=[1000, 28, 28, 1]): """ Return tensor to read from TFRecord """ - print 'Creating graph for loading TFRecords...' + print 'Creating graph for loading %s TFRecords...' % tf_glob with tf.variable_scope("TFRecords"): record_input = data_flow_ops.RecordInput(tf_glob, batch_size=batch_shape[0]) records_op = record_input.get_yield_op() @@ -150,9 +150,9 @@ def cnn_layers(x_train_input): save_mnist_as_tfrecord() -batch_size = 1 +batch_size = 1000 batch_shape = [batch_size, 28, 28, 1] -epochs = 60000 +epochs = 6000 classes = 10 x_train_batch, y_train_batch = read_and_decode_recordinput( @@ -183,12 +183,14 @@ def cnn_layers(x_train_input): train_model.compile(optimizer='rmsprop', loss=None, metrics=['accuracy']) +train_model.summary() -tensorboard = TensorBoard(write_graph=True) +tensorboard = TensorBoard() -train_model.summary() train_model.fit(batch_size=batch_size, epochs=epochs) + # disabled due to Keras bug + # callbacks=[tensorboard]) train_model.save_weights('saved_wt.h5') K.clear_session() From 3a535284668e95a24966a59a5aced9dca483555f Mon Sep 17 00:00:00 2001 From: Andrew Hundt Date: Wed, 21 Jun 2017 11:44:31 -0400 Subject: [PATCH 082/166] typo fix --- keras/backend/tensorflow_backend.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/keras/backend/tensorflow_backend.py b/keras/backend/tensorflow_backend.py index 001110a7d2d..3db9f04608d 100644 --- a/keras/backend/tensorflow_backend.py +++ b/keras/backend/tensorflow_backend.py @@ -2275,8 +2275,8 @@ def __call__(self, inputs): continue elif tensor is None and value is not None: raise ValueError('A tensor containing None ' - 'was tied to value ' + str(value) + - 'so Session.run() cannot execute, ' + 'was tied to value: ' + str(value) + + ' so Session.run() cannot execute, ' 'please check your data and Model.') if is_sparse(tensor): From b1839e24eb65a78b80796ebeddad0e4b8b1b82fc Mon Sep 17 00:00:00 2001 From: Andrew Hundt Date: Wed, 21 Jun 2017 11:38:10 -0400 Subject: [PATCH 083/166] Fix mnist_tfrecord.py runtime errors --- examples/mnist_tfrecord.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/examples/mnist_tfrecord.py b/examples/mnist_tfrecord.py index 93d72c03831..65ab5ce5e28 100644 --- a/examples/mnist_tfrecord.py +++ b/examples/mnist_tfrecord.py @@ -20,7 +20,7 @@ from keras.layers import Conv2D from keras.layers import MaxPooling2D from keras.callbacks import EarlyStopping -from keras.layers import TensorBoard +from keras.callbacks import TensorBoard from keras.objectives import categorical_crossentropy from keras.utils import np_utils from keras.utils.generic_utils import Progbar @@ -70,7 +70,7 @@ def _bytes_feature(value): def read_and_decode_recordinput(tf_glob, one_hot=True, classes=None, is_train=None, batch_shape=[1000, 28, 28, 1]): """ Return tensor to read from TFRecord """ - print 'Creating graph for loading TFRecords...' + print 'Creating graph for loading %s TFRecords...' % tf_glob with tf.variable_scope("TFRecords"): record_input = data_flow_ops.RecordInput(tf_glob, batch_size=batch_shape[0]) records_op = record_input.get_yield_op() @@ -150,9 +150,9 @@ def cnn_layers(x_train_input): save_mnist_as_tfrecord() -batch_size = 1 +batch_size = 1000 batch_shape = [batch_size, 28, 28, 1] -epochs = 60000 +epochs = 6000 classes = 10 x_train_batch, y_train_batch = read_and_decode_recordinput( @@ -183,12 +183,14 @@ def cnn_layers(x_train_input): train_model.compile(optimizer='rmsprop', loss=None, metrics=['accuracy']) +train_model.summary() -tensorboard = TensorBoard(write_graph=True) +tensorboard = TensorBoard() -train_model.summary() train_model.fit(batch_size=batch_size, epochs=epochs) + # disabled due to Keras bug + # callbacks=[tensorboard]) train_model.save_weights('saved_wt.h5') K.clear_session() From 5a8fb706e7b81d26b9f00542be12f0d59e21708a Mon Sep 17 00:00:00 2001 From: Andrew Hundt Date: Wed, 21 Jun 2017 13:55:32 -0400 Subject: [PATCH 084/166] mnist_tfrecord.py fix import bug --- examples/mnist_tfrecord.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/examples/mnist_tfrecord.py b/examples/mnist_tfrecord.py index 966d1aa54d0..b947adecfec 100644 --- a/examples/mnist_tfrecord.py +++ b/examples/mnist_tfrecord.py @@ -20,7 +20,7 @@ from keras.layers import Conv2D from keras.layers import MaxPooling2D from keras.callbacks import EarlyStopping -from keras.layers import TensorBoard +from keras.callbacks import TensorBoard from keras.objectives import categorical_crossentropy from keras.utils import np_utils from keras.utils.generic_utils import Progbar @@ -187,7 +187,7 @@ def create_cnn_model(x_train_batch, y_train_batch, x_batch_shape, y_batch_shape) loss='categorical_crossentropy', metrics=['accuracy']) -tensorboard = TensorBoard(write_graph=True) +tensorboard = TensorBoard(write_graph=False) train_model.summary() train_model.fit(batch_size=batch_size, From b8085f9090aedfe619c0c3d6e25fa0ac75f1ff7e Mon Sep 17 00:00:00 2001 From: Andrew Hundt Date: Wed, 21 Jun 2017 14:27:32 -0400 Subject: [PATCH 085/166] mnist_tfrecord.py pep8 --- examples/mnist_tfrecord.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/examples/mnist_tfrecord.py b/examples/mnist_tfrecord.py index 65ab5ce5e28..45293ed0ce9 100644 --- a/examples/mnist_tfrecord.py +++ b/examples/mnist_tfrecord.py @@ -187,10 +187,10 @@ def cnn_layers(x_train_input): tensorboard = TensorBoard() +# tensorboard disabled due to Keras bug train_model.fit(batch_size=batch_size, - epochs=epochs) - # disabled due to Keras bug - # callbacks=[tensorboard]) + epochs=epochs) # callbacks=[tensorboard]) + train_model.save_weights('saved_wt.h5') K.clear_session() From 3da96e5ca4854b4222cba21e5204eedcd5427e10 Mon Sep 17 00:00:00 2001 From: Andrew Hundt Date: Wed, 21 Jun 2017 15:05:37 -0400 Subject: [PATCH 086/166] mnist_tfrecord.py add parallelism option --- examples/mnist_tfrecord.py | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/examples/mnist_tfrecord.py b/examples/mnist_tfrecord.py index 45293ed0ce9..29174fc99b6 100644 --- a/examples/mnist_tfrecord.py +++ b/examples/mnist_tfrecord.py @@ -68,11 +68,13 @@ def _bytes_feature(value): print('tfrecord %s already exists' % filename) -def read_and_decode_recordinput(tf_glob, one_hot=True, classes=None, is_train=None, batch_shape=[1000, 28, 28, 1]): +def read_and_decode_recordinput(tf_glob, one_hot=True, classes=None, is_train=None, + batch_shape=[1000, 28, 28, 1], parallelism=1): """ Return tensor to read from TFRecord """ print 'Creating graph for loading %s TFRecords...' % tf_glob with tf.variable_scope("TFRecords"): - record_input = data_flow_ops.RecordInput(tf_glob, batch_size=batch_shape[0]) + record_input = data_flow_ops.RecordInput( + tf_glob, batch_size=batch_shape[0], parallelism=parallelism) records_op = record_input.get_yield_op() records_op = tf.split(records_op, batch_shape[0], 0) records_op = [tf.reshape(record, []) for record in records_op] @@ -154,20 +156,23 @@ def cnn_layers(x_train_input): batch_shape = [batch_size, 28, 28, 1] epochs = 6000 classes = 10 +parallelism = 10 x_train_batch, y_train_batch = read_and_decode_recordinput( 'train.mnist.tfrecord', one_hot=True, classes=classes, is_train=True, - batch_shape=batch_shape) + batch_shape=batch_shape, + parallelism=parallelism) x_test_batch, y_test_batch = read_and_decode_recordinput( 'test.mnist.tfrecord', one_hot=True, classes=classes, is_train=True, - batch_shape=batch_shape) + batch_shape=batch_shape, + parallelism=parallelism) x_batch_shape = x_train_batch.get_shape().as_list() From 919fdfffed266ed97a647e2b97704098fec09070 Mon Sep 17 00:00:00 2001 From: Andrew Hundt Date: Wed, 21 Jun 2017 15:05:37 -0400 Subject: [PATCH 087/166] mnist_tfrecord.py add parallelism option --- examples/mnist_tfrecord.py | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/examples/mnist_tfrecord.py b/examples/mnist_tfrecord.py index b947adecfec..7a036b9ddad 100644 --- a/examples/mnist_tfrecord.py +++ b/examples/mnist_tfrecord.py @@ -68,11 +68,13 @@ def _bytes_feature(value): print('tfrecord %s already exists' % filename) -def read_and_decode_recordinput(tf_glob, one_hot=True, classes=None, is_train=None, batch_shape=[1000, 28, 28, 1]): +def read_and_decode_recordinput(tf_glob, one_hot=True, classes=None, is_train=None, + batch_shape=[1000, 28, 28, 1], parallelism=1): """ Return tensor to read from TFRecord """ print 'Creating graph for loading TFRecords...' with tf.variable_scope("TFRecords"): - record_input = data_flow_ops.RecordInput(tf_glob, batch_size=batch_shape[0]) + record_input = data_flow_ops.RecordInput( + tf_glob, batch_size=batch_shape[0], parallelism=parallelism) records_op = record_input.get_yield_op() records_op = tf.split(records_op, batch_shape[0], 0) records_op = [tf.reshape(record, []) for record in records_op] @@ -159,20 +161,23 @@ def create_cnn_model(x_train_batch, y_train_batch, x_batch_shape, y_batch_shape) batch_shape = [batch_size, 28, 28, 1] epochs = 3000 classes = 10 +parallelism = 10 x_train_batch, y_train_batch = read_and_decode_recordinput( 'train.mnist.tfrecord', one_hot=True, classes=classes, is_train=True, - batch_shape=batch_shape) + batch_shape=batch_shape, + parallelism=parallelism) x_test_batch, y_test_batch = read_and_decode_recordinput( 'test.mnist.tfrecord', one_hot=True, classes=classes, is_train=True, - batch_shape=batch_shape) + batch_shape=batch_shape, + parallelism=parallelism) x_batch_shape = x_train_batch.get_shape().as_list() From b8019c6ce5a5efd204d2be25a2031fe9329ca3df Mon Sep 17 00:00:00 2001 From: Andrew Hundt Date: Wed, 21 Jun 2017 16:33:55 -0400 Subject: [PATCH 088/166] reorder inputs --- tests/keras/utils/generic_utils_test.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/keras/utils/generic_utils_test.py b/tests/keras/utils/generic_utils_test.py index 858f4da08a6..acb8e28e138 100644 --- a/tests/keras/utils/generic_utils_test.py +++ b/tests/keras/utils/generic_utils_test.py @@ -3,10 +3,10 @@ import numpy as np from keras.utils.generic_utils import custom_object_scope from keras.utils.generic_utils import has_arg +from keras.utils.generic_utils import Progbar +from keras.utils.test_utils import keras_test from keras import activations from keras import regularizers -from keras.utils.test_utils import keras_test -from keras.utils.generic_utils import Progbar @keras_test From f56f1fa6305a2235a05c23b43089d3e662799504 Mon Sep 17 00:00:00 2001 From: Andrew Hundt Date: Wed, 21 Jun 2017 18:29:53 -0400 Subject: [PATCH 089/166] backend_test.py rename f to g to fix test error --- tests/keras/backend/backend_test.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/tests/keras/backend/backend_test.py b/tests/keras/backend/backend_test.py index 5b73f451eca..9aa37005769 100644 --- a/tests/keras/backend/backend_test.py +++ b/tests/keras/backend/backend_test.py @@ -600,16 +600,16 @@ def test_function(self): f = k.function([y], [exp], updates=[(x, update)]) f_list.append(f) if k == KTF: - f = k.function([y, k.variable(1.)], [exp], updates=[(x, update)]) - f([input_val])[0] - f = k.function([y, k.variable(1.), None], [exp], updates=[(x, update)]) - f([input_val, None, None, None])[0] + g = k.function([y, k.variable(1.)], [exp], updates=[(x, update)]) + g([input_val])[0] + g = k.function([y, k.variable(1.), None], [exp], updates=[(x, update)]) + g([input_val, None, None, None])[0] with pytest.raises(ValueError): - f = K.function([y, K.variable(1.), None], [exp], updates=[(x, update)]) - f([input_val, None, input_val])[0] + g = K.function([y, K.variable(1.), None], [exp], updates=[(x, update)]) + g([input_val, None, input_val])[0] with pytest.raises(ValueError): - f = K.function([y, K.variable(1.)], [exp], updates=[(x, update)]) - f([input_val, None, input_val])[0] + g = K.function([y, K.variable(1.)], [exp], updates=[(x, update)]) + g([input_val, None, input_val])[0] function_outputs_list = [f([input_val])[0] for f in f_list] for i in range(len(function_outputs_list) - 1): From 837ceeba0eb98228a478cb46140dc23d9a686b3e Mon Sep 17 00:00:00 2001 From: Andrew Hundt Date: Wed, 21 Jun 2017 18:40:16 -0400 Subject: [PATCH 090/166] mnist_tfrecord remove whitespace --- examples/mnist_tfrecord.py | 1 - 1 file changed, 1 deletion(-) diff --git a/examples/mnist_tfrecord.py b/examples/mnist_tfrecord.py index 7a036b9ddad..57323e3a6bb 100644 --- a/examples/mnist_tfrecord.py +++ b/examples/mnist_tfrecord.py @@ -179,7 +179,6 @@ def create_cnn_model(x_train_batch, y_train_batch, x_batch_shape, y_batch_shape) batch_shape=batch_shape, parallelism=parallelism) - x_batch_shape = x_train_batch.get_shape().as_list() y_batch_shape = y_train_batch.get_shape().as_list() From 750a21112e4de5c457376919cbf1df9387b9511d Mon Sep 17 00:00:00 2001 From: Andrew Hundt Date: Wed, 21 Jun 2017 18:45:26 -0400 Subject: [PATCH 091/166] mnist_tfrecord.py indentation fix --- examples/mnist_tfrecord.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/mnist_tfrecord.py b/examples/mnist_tfrecord.py index 29174fc99b6..fb84565a12a 100644 --- a/examples/mnist_tfrecord.py +++ b/examples/mnist_tfrecord.py @@ -69,7 +69,7 @@ def _bytes_feature(value): def read_and_decode_recordinput(tf_glob, one_hot=True, classes=None, is_train=None, - batch_shape=[1000, 28, 28, 1], parallelism=1): + batch_shape=[1000, 28, 28, 1], parallelism=1): """ Return tensor to read from TFRecord """ print 'Creating graph for loading %s TFRecords...' % tf_glob with tf.variable_scope("TFRecords"): From 0a8c13bfa1c776762aaf53efb1abe02ad4b4de5b Mon Sep 17 00:00:00 2001 From: Andrew Hundt Date: Wed, 21 Jun 2017 23:26:00 -0400 Subject: [PATCH 092/166] backend_test.py cleanup new TF backend test --- tests/keras/backend/backend_test.py | 27 ++++++++++++++++----------- 1 file changed, 16 insertions(+), 11 deletions(-) diff --git a/tests/keras/backend/backend_test.py b/tests/keras/backend/backend_test.py index 9aa37005769..b9b699c0721 100644 --- a/tests/keras/backend/backend_test.py +++ b/tests/keras/backend/backend_test.py @@ -599,17 +599,6 @@ def test_function(self): update = x * 2 f = k.function([y], [exp], updates=[(x, update)]) f_list.append(f) - if k == KTF: - g = k.function([y, k.variable(1.)], [exp], updates=[(x, update)]) - g([input_val])[0] - g = k.function([y, k.variable(1.), None], [exp], updates=[(x, update)]) - g([input_val, None, None, None])[0] - with pytest.raises(ValueError): - g = K.function([y, K.variable(1.), None], [exp], updates=[(x, update)]) - g([input_val, None, input_val])[0] - with pytest.raises(ValueError): - g = K.function([y, K.variable(1.)], [exp], updates=[(x, update)]) - g([input_val, None, input_val])[0] function_outputs_list = [f([input_val])[0] for f in f_list] for i in range(len(function_outputs_list) - 1): @@ -624,6 +613,22 @@ def test_function(self): assert new_val_list[i].shape == new_val_list[i + 1].shape assert_allclose(new_val_list[i], new_val_list[i + 1], atol=1e-05) + k = KTF + x = k.variable(val) + y = k.placeholder(ndim=2) + exp = k.square(x) + y + update = x * 2 + g = k.function([y, k.variable(1.)], [exp], updates=[(x, update)]) + g([input_val])[0] + g = k.function([y, k.variable(1.), None], [exp], updates=[(x, update)]) + g([input_val, None, None, None])[0] + with pytest.raises(ValueError): + g = k.function([y, k.variable(1.), None], [exp], updates=[(x, update)]) + g([input_val, None, input_val])[0] + with pytest.raises(ValueError): + g = k.function([y, k.variable(1.)], [exp], updates=[(x, update)]) + g([input_val, None, input_val])[0] + def test_rnn(self): # implement a simple RNN input_dim = 8 From 2a1cd773fdddbcd5790bfa06e46f35be4ac0beaf Mon Sep 17 00:00:00 2001 From: Andrew Hundt Date: Fri, 23 Jun 2017 00:42:36 -0400 Subject: [PATCH 093/166] lower batch size and epochs --- examples/mnist_tfrecord.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/examples/mnist_tfrecord.py b/examples/mnist_tfrecord.py index fb84565a12a..92afb98c3ef 100644 --- a/examples/mnist_tfrecord.py +++ b/examples/mnist_tfrecord.py @@ -152,9 +152,9 @@ def cnn_layers(x_train_input): save_mnist_as_tfrecord() -batch_size = 1000 +batch_size = 100 batch_shape = [batch_size, 28, 28, 1] -epochs = 6000 +epochs = 3000 classes = 10 parallelism = 10 From 0c43bbacc85f78489959433241ea94cddf86d80d Mon Sep 17 00:00:00 2001 From: Andrew Hundt Date: Wed, 21 Jun 2017 18:45:26 -0400 Subject: [PATCH 094/166] loss defaults to None in compile() --- examples/mnist_tfrecord.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/mnist_tfrecord.py b/examples/mnist_tfrecord.py index 92afb98c3ef..45194c96893 100644 --- a/examples/mnist_tfrecord.py +++ b/examples/mnist_tfrecord.py @@ -209,7 +209,7 @@ def cnn_layers(x_train_input): test_model = Model(inputs=x_test_inp, outputs=test_out) test_model.load_weights('saved_wt.h5') -test_model.compile(optimizer='rmsprop', loss='categorical_crossentropy', metrics=['accuracy']) +test_model.compile(optimizer='rmsprop', loss=None, metrics=['accuracy']) test_model.summary() loss, acc = test_model.evaluate(X_test, np_utils.to_categorical(y_test), classes) From 40d16c3581774f5967cff8a9dd0ba5578ce42650 Mon Sep 17 00:00:00 2001 From: Andrew Hundt Date: Fri, 23 Jun 2017 14:44:55 -0400 Subject: [PATCH 095/166] mnist_tfrecord.py --- examples/mnist_tfrecord.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/mnist_tfrecord.py b/examples/mnist_tfrecord.py index 45194c96893..92afb98c3ef 100644 --- a/examples/mnist_tfrecord.py +++ b/examples/mnist_tfrecord.py @@ -209,7 +209,7 @@ def cnn_layers(x_train_input): test_model = Model(inputs=x_test_inp, outputs=test_out) test_model.load_weights('saved_wt.h5') -test_model.compile(optimizer='rmsprop', loss=None, metrics=['accuracy']) +test_model.compile(optimizer='rmsprop', loss='categorical_crossentropy', metrics=['accuracy']) test_model.summary() loss, acc = test_model.evaluate(X_test, np_utils.to_categorical(y_test), classes) From 63b21aa883598eeb868ac6224df5e42e3d15dadc Mon Sep 17 00:00:00 2001 From: Andrew Hundt Date: Fri, 23 Jun 2017 21:41:04 -0400 Subject: [PATCH 096/166] try a tricky way of getting closer to the "ideal" api --- examples/mnist_tfrecord.py | 15 ++++++--------- keras/engine/training.py | 32 ++++++++++++++++++++++++++++---- 2 files changed, 34 insertions(+), 13 deletions(-) diff --git a/examples/mnist_tfrecord.py b/examples/mnist_tfrecord.py index 65ab5ce5e28..c8f9bcee593 100644 --- a/examples/mnist_tfrecord.py +++ b/examples/mnist_tfrecord.py @@ -150,9 +150,9 @@ def cnn_layers(x_train_input): save_mnist_as_tfrecord() -batch_size = 1000 +batch_size = 100 batch_shape = [batch_size, 28, 28, 1] -epochs = 6000 +epochs = 3000 classes = 10 x_train_batch, y_train_batch = read_and_decode_recordinput( @@ -173,24 +173,21 @@ def cnn_layers(x_train_input): x_batch_shape = x_train_batch.get_shape().as_list() y_batch_shape = y_train_batch.get_shape().as_list() -x_train_input = Input(tensor=x_train_batch, batch_shape=x_batch_shape) +x_train_in = Input(tensor=x_train_batch, batch_shape=x_batch_shape) x_train_out = cnn_layers(x_train_input) -y_train_in_out = Input(tensor=y_train_batch, batch_shape=y_batch_shape, name='y_labels') +y_train_in = Input(tensor=y_train_batch, batch_shape=y_batch_shape, name='y_labels') cce = categorical_crossentropy(y_train_batch, x_train_out) train_model = Model(inputs=[x_train_input], outputs=[x_train_out]) -train_model.add_loss(cce) train_model.compile(optimizer='rmsprop', - loss=None, + loss='categorical_crossentropy', metrics=['accuracy']) train_model.summary() tensorboard = TensorBoard() -train_model.fit(batch_size=batch_size, +train_model.fit(None, y_train_in, batch_size=batch_size, epochs=epochs) - # disabled due to Keras bug - # callbacks=[tensorboard]) train_model.save_weights('saved_wt.h5') K.clear_session() diff --git a/keras/engine/training.py b/keras/engine/training.py index 60b7ebfe43e..c652335dc57 100644 --- a/keras/engine/training.py +++ b/keras/engine/training.py @@ -221,8 +221,13 @@ def _check_array_lengths(inputs, targets, weights=None): ValueError: in case of incorrectly formatted data. """ def set_of_lengths(x): - # return a set with the variation between of different shapes, with None => 0 - return set([0]) if x is None else set([0 if y is None else y.shape[0] for y in x]) + # return a set with the variation between + # different shapes, with None => 0 + if x is None: + return {0} + else: + return set([0 if y is None else y.shape[0] for y in x]) + set_x = set_of_lengths(inputs) set_y = set_of_lengths(targets) set_w = set_of_lengths(weights) @@ -396,7 +401,9 @@ def _slice_arrays(arrays, start=None, stop=None): # Returns A slice of the array(s). """ - if isinstance(arrays, list): + if arrays is None: + return [None] + elif isinstance(arrays, list): if hasattr(start, '__len__'): # hdf5 datasets only support list objects as indices if hasattr(start, 'shape'): @@ -409,8 +416,10 @@ def _slice_arrays(arrays, start=None, stop=None): if hasattr(start, 'shape'): start = start.tolist() return arrays[start] - else: + elif hasattr(start, '__getitem__'): return arrays[start:stop] + else: + return [None] def _weighted_masked_objective(fn): @@ -634,6 +643,11 @@ def compile(self, optimizer, loss, metrics=None, loss_weights=None, ValueError: In case of invalid arguments for `optimizer`, `loss`, `metrics` or `sample_weight_mode`. """ + self._saved_compile_params = locals() + self._compile(locals()) + + def _compile(self, optimizer, loss, metrics=None, loss_weights=None, + sample_weight_mode=None, **kwargs): loss = loss or {} self.optimizer = optimizers.get(optimizer) self.loss = loss @@ -1377,6 +1391,16 @@ class indices (integers) to if kwargs: raise TypeError('Unrecognized keyword arguments: ' + str(kwargs)) + if (y is not None): + for i, yi in enumerate(y): + if type(x).__module__ is np.__name__: + continue + else: + self.target_configuration[i] = yi + + # assume it is some sort of tensor + self._compile(self._saved_compile_params) + # Validate user data. x, y, sample_weights = self._standardize_user_data( x, y, From b0447730d1743c67d60023e3e0932c1f0319098c Mon Sep 17 00:00:00 2001 From: Andrew Hundt Date: Fri, 23 Jun 2017 21:45:29 -0400 Subject: [PATCH 097/166] name typo --- examples/mnist_tfrecord.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/examples/mnist_tfrecord.py b/examples/mnist_tfrecord.py index c8f9bcee593..a0382fdaae7 100644 --- a/examples/mnist_tfrecord.py +++ b/examples/mnist_tfrecord.py @@ -174,10 +174,10 @@ def cnn_layers(x_train_input): y_batch_shape = y_train_batch.get_shape().as_list() x_train_in = Input(tensor=x_train_batch, batch_shape=x_batch_shape) -x_train_out = cnn_layers(x_train_input) +x_train_out = cnn_layers(x_train_in) y_train_in = Input(tensor=y_train_batch, batch_shape=y_batch_shape, name='y_labels') cce = categorical_crossentropy(y_train_batch, x_train_out) -train_model = Model(inputs=[x_train_input], outputs=[x_train_out]) +train_model = Model(inputs=[x_train_in], outputs=[x_train_out]) train_model.compile(optimizer='rmsprop', loss='categorical_crossentropy', From 7ced84d85a73a4eea01f94e57fceae4f7c4e2ac7 Mon Sep 17 00:00:00 2001 From: Andrew Hundt Date: Fri, 23 Jun 2017 21:48:25 -0400 Subject: [PATCH 098/166] set y tensor values to non-tensor values --- keras/engine/training.py | 1 + 1 file changed, 1 insertion(+) diff --git a/keras/engine/training.py b/keras/engine/training.py index c652335dc57..5797f288d9c 100644 --- a/keras/engine/training.py +++ b/keras/engine/training.py @@ -1397,6 +1397,7 @@ class indices (integers) to continue else: self.target_configuration[i] = yi + y[i] = None # assume it is some sort of tensor self._compile(self._saved_compile_params) From ea3e926e18cc0f5e0c7f431e3532b31e1c7fad6f Mon Sep 17 00:00:00 2001 From: Andrew Hundt Date: Fri, 23 Jun 2017 21:51:10 -0400 Subject: [PATCH 099/166] handle non list case --- keras/engine/training.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/keras/engine/training.py b/keras/engine/training.py index 5797f288d9c..1b1dcc5f50d 100644 --- a/keras/engine/training.py +++ b/keras/engine/training.py @@ -1391,7 +1391,10 @@ class indices (integers) to if kwargs: raise TypeError('Unrecognized keyword arguments: ' + str(kwargs)) - if (y is not None): + if is_keras_tensor(y): + self.target_configuration[0] = y + y = None + elif (y is not None): for i, yi in enumerate(y): if type(x).__module__ is np.__name__: continue From 956885f2d87ec9b4fb4a72991fa782b26d1f2d58 Mon Sep 17 00:00:00 2001 From: Andrew Hundt Date: Fri, 23 Jun 2017 21:53:35 -0400 Subject: [PATCH 100/166] handle some error cases --- keras/engine/training.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/keras/engine/training.py b/keras/engine/training.py index 1b1dcc5f50d..037cf9d666a 100644 --- a/keras/engine/training.py +++ b/keras/engine/training.py @@ -1391,9 +1391,10 @@ class indices (integers) to if kwargs: raise TypeError('Unrecognized keyword arguments: ' + str(kwargs)) - if is_keras_tensor(y): + if K.is_keras_tensor(y): self.target_configuration[0] = y y = None + self._compile(**self._saved_compile_params) elif (y is not None): for i, yi in enumerate(y): if type(x).__module__ is np.__name__: @@ -1403,7 +1404,7 @@ class indices (integers) to y[i] = None # assume it is some sort of tensor - self._compile(self._saved_compile_params) + self._compile(**self._saved_compile_params) # Validate user data. x, y, sample_weights = self._standardize_user_data( From 320cbe94b4b0f95948218feb996ee3fb9e2e0a10 Mon Sep 17 00:00:00 2001 From: Andrew Hundt Date: Fri, 23 Jun 2017 21:57:31 -0400 Subject: [PATCH 101/166] param fix --- keras/engine/training.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/keras/engine/training.py b/keras/engine/training.py index 037cf9d666a..5215f7e0e44 100644 --- a/keras/engine/training.py +++ b/keras/engine/training.py @@ -644,10 +644,10 @@ def compile(self, optimizer, loss, metrics=None, loss_weights=None, `optimizer`, `loss`, `metrics` or `sample_weight_mode`. """ self._saved_compile_params = locals() - self._compile(locals()) + self._compile(**locals()) def _compile(self, optimizer, loss, metrics=None, loss_weights=None, - sample_weight_mode=None, **kwargs): + sample_weight_mode=None, **kwargs): loss = loss or {} self.optimizer = optimizers.get(optimizer) self.loss = loss From d4ea2242a85c570881b09b52a87ea18912963885 Mon Sep 17 00:00:00 2001 From: Andrew Hundt Date: Fri, 23 Jun 2017 22:07:19 -0400 Subject: [PATCH 102/166] fix some arg stuff --- keras/engine/training.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/keras/engine/training.py b/keras/engine/training.py index 5215f7e0e44..0b551630153 100644 --- a/keras/engine/training.py +++ b/keras/engine/training.py @@ -601,8 +601,7 @@ class Model(Container): """The `Model` class adds training & evaluation routines to a `Container`. """ - def compile(self, optimizer, loss, metrics=None, loss_weights=None, - sample_weight_mode=None, **kwargs): + def compile(self, *args, **kwargs): """Configures the model for training. # Arguments @@ -643,8 +642,9 @@ def compile(self, optimizer, loss, metrics=None, loss_weights=None, ValueError: In case of invalid arguments for `optimizer`, `loss`, `metrics` or `sample_weight_mode`. """ - self._saved_compile_params = locals() - self._compile(**locals()) + self._saved_kwargs = kwargs + self._saved_args = args + self._compile(*args, **kwargs) def _compile(self, optimizer, loss, metrics=None, loss_weights=None, sample_weight_mode=None, **kwargs): @@ -1394,7 +1394,7 @@ class indices (integers) to if K.is_keras_tensor(y): self.target_configuration[0] = y y = None - self._compile(**self._saved_compile_params) + self._compile(*self._saved_args, **self._saved_kwargs) elif (y is not None): for i, yi in enumerate(y): if type(x).__module__ is np.__name__: @@ -1404,7 +1404,7 @@ class indices (integers) to y[i] = None # assume it is some sort of tensor - self._compile(**self._saved_compile_params) + self._compile(*self._saved_args, **self._saved_kwargs) # Validate user data. x, y, sample_weights = self._standardize_user_data( From 9736151133e7e3f74b7589d9ebb11a0ebc13afc1 Mon Sep 17 00:00:00 2001 From: Andrew Hundt Date: Fri, 23 Jun 2017 22:22:17 -0400 Subject: [PATCH 103/166] remove extraneous lines --- examples/mnist_tfrecord.py | 6 ------ 1 file changed, 6 deletions(-) diff --git a/examples/mnist_tfrecord.py b/examples/mnist_tfrecord.py index a0382fdaae7..fb1cb7b2a30 100644 --- a/examples/mnist_tfrecord.py +++ b/examples/mnist_tfrecord.py @@ -176,16 +176,10 @@ def cnn_layers(x_train_input): x_train_in = Input(tensor=x_train_batch, batch_shape=x_batch_shape) x_train_out = cnn_layers(x_train_in) y_train_in = Input(tensor=y_train_batch, batch_shape=y_batch_shape, name='y_labels') -cce = categorical_crossentropy(y_train_batch, x_train_out) train_model = Model(inputs=[x_train_in], outputs=[x_train_out]) - train_model.compile(optimizer='rmsprop', loss='categorical_crossentropy', metrics=['accuracy']) -train_model.summary() - -tensorboard = TensorBoard() - train_model.fit(None, y_train_in, batch_size=batch_size, epochs=epochs) train_model.save_weights('saved_wt.h5') From 4eb95e3c08df12bcabd0f8093fb0d483a0ad78f4 Mon Sep 17 00:00:00 2001 From: Andrew Hundt Date: Fri, 23 Jun 2017 23:10:15 -0400 Subject: [PATCH 104/166] Improve numpy list handling --- examples/mnist_tfrecord.py | 3 ++- keras/engine/training.py | 14 ++++++++------ 2 files changed, 10 insertions(+), 7 deletions(-) diff --git a/examples/mnist_tfrecord.py b/examples/mnist_tfrecord.py index fb1cb7b2a30..e45a5ed4942 100644 --- a/examples/mnist_tfrecord.py +++ b/examples/mnist_tfrecord.py @@ -180,7 +180,8 @@ def cnn_layers(x_train_input): train_model.compile(optimizer='rmsprop', loss='categorical_crossentropy', metrics=['accuracy']) -train_model.fit(None, y_train_in, batch_size=batch_size, +train_model.fit(None, y_train_in, + batch_size=batch_size, epochs=epochs) train_model.save_weights('saved_wt.h5') diff --git a/keras/engine/training.py b/keras/engine/training.py index 0b551630153..e503f3a2f6f 100644 --- a/keras/engine/training.py +++ b/keras/engine/training.py @@ -1395,16 +1395,18 @@ class indices (integers) to self.target_configuration[0] = y y = None self._compile(*self._saved_args, **self._saved_kwargs) - elif (y is not None): + elif y is not None: + recompile = False for i, yi in enumerate(y): - if type(x).__module__ is np.__name__: - continue - else: + if K.is_keras_tensor(yi): self.target_configuration[i] = yi y[i] = None + recompile = True + else: + self.target_configuration[i] = None - # assume it is some sort of tensor - self._compile(*self._saved_args, **self._saved_kwargs) + if recompile: + self._compile(*self._saved_args, **self._saved_kwargs) # Validate user data. x, y, sample_weights = self._standardize_user_data( From f3e25decceae0a0af4a2be107b67c89da4d1a48b Mon Sep 17 00:00:00 2001 From: Andrew Hundt Date: Fri, 23 Jun 2017 23:50:33 -0400 Subject: [PATCH 105/166] model.fit(steps_per_epoch) added --- keras/engine/training.py | 36 +++++++++++++++++++++-------- tests/keras/engine/test_training.py | 3 ++- 2 files changed, 28 insertions(+), 11 deletions(-) diff --git a/keras/engine/training.py b/keras/engine/training.py index bc8ecb27b86..8d10eeadc56 100644 --- a/keras/engine/training.py +++ b/keras/engine/training.py @@ -965,7 +965,8 @@ def _make_predict_function(self): def _fit_loop(self, f, ins, out_labels=None, batch_size=32, epochs=100, verbose=1, callbacks=None, val_f=None, val_ins=None, shuffle=True, - callback_metrics=None, initial_epoch=0): + callback_metrics=None, initial_epoch=0, + steps_per_epoch=None): """Abstract fit function for `f(ins)`. Assume that f returns a list, labeled by out_labels. @@ -988,6 +989,11 @@ def _fit_loop(self, f, ins, out_labels=None, batch_size=32, `f` and the list of display names of the outputs of `f_val`. initial_epoch: epoch at which to start training (useful for resuming a previous training run) + steps_per_epoch: Total number of steps (batches of samples) + before declaring one epoch finished and starting the + next epoch. The default `None` is equal to the number + of unique samples in your dataset divided by the batch + size, or 1 if that cannot be determined. # Returns `History` object. @@ -999,15 +1005,18 @@ def _fit_loop(self, f, ins, out_labels=None, batch_size=32, print('Train on %d samples, validate on %d samples' % (ins[0].shape[0], val_ins[0].shape[0])) - if ins and hasattr(ins[0], 'shape'): - num_train_samples = ins[0].shape[0] + if steps_per_epoch is not None: + num_train_samples = steps_per_epoch else: - # May happen if we are running `fit` without Numpy input data, - # i.e. if all inputs to the models are data tensors - # instead of placeholders. - # In that case we will run `fit` over a single batch. - num_train_samples = batch_size - verbose = 2 + if ins and hasattr(ins[0], 'shape'): + num_train_samples = ins[0].shape[0] + else: + # May happen if we are running `fit` without Numpy input data, + # i.e. if all inputs to the models are data tensors + # instead of placeholders. + # In that case we will run `fit` over a single batch. + num_train_samples = batch_size + verbose = 2 index_array = np.arange(num_train_samples) self.history = cbks.History() @@ -1269,6 +1278,7 @@ def fit(self, x=None, class_weight=None, sample_weight=None, initial_epoch=0, + steps_per_epoch=None, **kwargs): """Trains the model for a fixed number of epochs (iterations on a dataset). @@ -1319,6 +1329,11 @@ class indices (integers) to sample_weight_mode="temporal" in compile(). initial_epoch: epoch at which to start training (useful for resuming a previous training run) + steps_per_epoch: Total number of steps (batches of samples) + before declaring one epoch finished and starting the + next epoch. The default `None` is equal to the number + of unique samples in your dataset divided by the batch + size, or 1 if that cannot be determined. # Returns A `History` instance. Its `history` attribute contains @@ -1414,7 +1429,8 @@ class indices (integers) to verbose=verbose, callbacks=callbacks, val_f=val_f, val_ins=val_ins, shuffle=shuffle, callback_metrics=callback_metrics, - initial_epoch=initial_epoch) + initial_epoch=initial_epoch, + steps_per_epoch=steps_per_epoch) def evaluate(self, x, y, batch_size=32, verbose=1, sample_weight=None): """Returns the loss value & metrics values for the model in test mode. diff --git a/tests/keras/engine/test_training.py b/tests/keras/engine/test_training.py index e0de52246d9..1712827d99f 100644 --- a/tests/keras/engine/test_training.py +++ b/tests/keras/engine/test_training.py @@ -81,7 +81,8 @@ def test_model_methods(): epochs=1, batch_size=4, validation_split=0.5) out = model.fit({'input_a': input_a_np, 'input_b': input_b_np}, {'dense_1': output_a_np, 'dropout': output_b_np}, - epochs=1, batch_size=4, validation_split=0.5) + epochs=1, batch_size=4, validation_split=0.5, + steps_per_epoch=1) # test validation data out = model.fit([input_a_np, input_b_np], From 05b3ecf5c685ebcbd7645f794dcc76e6ede2d632 Mon Sep 17 00:00:00 2001 From: Andrew Hundt Date: Sat, 24 Jun 2017 00:08:04 -0400 Subject: [PATCH 106/166] mnist_tfrecord.py support steps_per_epoch --- examples/mnist_tfrecord.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/examples/mnist_tfrecord.py b/examples/mnist_tfrecord.py index e45a5ed4942..e5a33c73423 100644 --- a/examples/mnist_tfrecord.py +++ b/examples/mnist_tfrecord.py @@ -152,7 +152,8 @@ def cnn_layers(x_train_input): batch_size = 100 batch_shape = [batch_size, 28, 28, 1] -epochs = 3000 +epochs = 12 +steps_per_epoch = 10000 classes = 10 x_train_batch, y_train_batch = read_and_decode_recordinput( @@ -182,7 +183,8 @@ def cnn_layers(x_train_input): metrics=['accuracy']) train_model.fit(None, y_train_in, batch_size=batch_size, - epochs=epochs) + epochs=epochs, + steps_per_epoch=steps_per_epoch) train_model.save_weights('saved_wt.h5') K.clear_session() From e8cdd22a65536fea3766f00a567264e7715e61c2 Mon Sep 17 00:00:00 2001 From: Andrew Hundt Date: Sat, 24 Jun 2017 00:57:22 -0400 Subject: [PATCH 107/166] fix test errors --- keras/backend/cntk_backend.py | 2 +- keras/backend/tensorflow_backend.py | 9 +++++---- keras/backend/theano_backend.py | 7 ++++--- keras/engine/training.py | 11 ++++++----- 4 files changed, 16 insertions(+), 13 deletions(-) diff --git a/keras/backend/cntk_backend.py b/keras/backend/cntk_backend.py index 19827cbc577..615753ffe67 100644 --- a/keras/backend/cntk_backend.py +++ b/keras/backend/cntk_backend.py @@ -261,7 +261,7 @@ def placeholder( return x -def is_keras_tensor(x): +def is_keras_tensor(x, expect_other_types=False): return hasattr(x, '_keras_history') diff --git a/keras/backend/tensorflow_backend.py b/keras/backend/tensorflow_backend.py index 2dc03759356..e7c5e93150d 100644 --- a/keras/backend/tensorflow_backend.py +++ b/keras/backend/tensorflow_backend.py @@ -360,7 +360,7 @@ def constant(value, dtype=None, shape=None, name=None): return tf.constant(value, dtype=dtype, shape=shape, name=name) -def is_keras_tensor(x): +def is_keras_tensor(x, expect_other_types=False): """Returns whether `x` is a Keras tensor. # Arguments @@ -389,9 +389,10 @@ def is_keras_tensor(x): True ``` """ - if not isinstance(x, (tf.Tensor, - tf_variables.Variable, - tf.SparseTensor)): + if (not expect_other_types and + not isinstance(x, (tf.Tensor, + tf_variables.Variable, + tf.SparseTensor))): raise ValueError('Unexpectedly found an instance of type `' + str(type(x)) + '`. ' 'Expected a symbolic tensor instance.') return hasattr(x, '_keras_history') diff --git a/keras/backend/theano_backend.py b/keras/backend/theano_backend.py index a585bcc8abe..f44af7aaea5 100644 --- a/keras/backend/theano_backend.py +++ b/keras/backend/theano_backend.py @@ -166,7 +166,7 @@ def constant(value, dtype=None, shape=None, name=None): return const -def is_keras_tensor(x): +def is_keras_tensor(x, expect_other_types=False): """Returns whether `x` is a Keras tensor. # Arguments @@ -195,8 +195,9 @@ def is_keras_tensor(x): True ``` """ - if not isinstance(x, (T.TensorVariable, - T.sharedvar.TensorSharedVariable)): + if (not expect_other_types and + not isinstance(x, (T.TensorVariable, + T.sharedvar.TensorSharedVariable))): raise ValueError('Unexpectedly found an instance of type `' + str(type(x)) + '`. ' 'Expected a symbolic tensor instance.') return hasattr(x, '_keras_history') diff --git a/keras/engine/training.py b/keras/engine/training.py index 635265b2e7f..1327c6cbefd 100644 --- a/keras/engine/training.py +++ b/keras/engine/training.py @@ -1407,19 +1407,20 @@ class indices (integers) to if kwargs: raise TypeError('Unrecognized keyword arguments: ' + str(kwargs)) - if K.is_keras_tensor(y): - self.target_configuration[0] = y + if K.is_keras_tensor(y, expect_other_types=True): + self.target_configuration = [y] y = None self._compile(*self._saved_args, **self._saved_kwargs) elif y is not None: recompile = False + self.target_configuration = [] for i, yi in enumerate(y): - if K.is_keras_tensor(yi): - self.target_configuration[i] = yi + if K.is_keras_tensor(yi, expect_other_types=True): + self.target_configuration.append(yi) y[i] = None recompile = True else: - self.target_configuration[i] = None + self.target_configuration.append(None) if recompile: self._compile(*self._saved_args, **self._saved_kwargs) From 7d8f0b7a4b7a78f8c0c3904db6f22d5688266bfa Mon Sep 17 00:00:00 2001 From: Andrew Hundt Date: Mon, 26 Jun 2017 15:29:29 -0400 Subject: [PATCH 108/166] added _check_num_samples for cases when batch_size does not apply --- keras/engine/training.py | 74 +++++++++++++++++++--------------------- 1 file changed, 35 insertions(+), 39 deletions(-) diff --git a/keras/engine/training.py b/keras/engine/training.py index 8d10eeadc56..95a799cfa4a 100644 --- a/keras/engine/training.py +++ b/keras/engine/training.py @@ -962,6 +962,21 @@ def _make_predict_function(self): name='predict_function', **kwargs) + def _check_num_samples(self, ins, batch_size=None, steps=None, steps_name='steps'): + if steps is not None: + if batch_size is not None: + warnings.warn(steps_name + ' was specified, ' + 'so batch_size is being ignored. ' + 'Specify None for batch_size to ' + 'remove this warning.') + num_samples = steps + elif ins and hasattr(ins[0], 'shape'): + num_samples = ins[0].shape[0] + else: + raise ValueError('The input data should have shape or' + ' please specify ' + steps_name + '.') + return num_samples + def _fit_loop(self, f, ins, out_labels=None, batch_size=32, epochs=100, verbose=1, callbacks=None, val_f=None, val_ins=None, shuffle=True, @@ -991,9 +1006,7 @@ def _fit_loop(self, f, ins, out_labels=None, batch_size=32, (useful for resuming a previous training run) steps_per_epoch: Total number of steps (batches of samples) before declaring one epoch finished and starting the - next epoch. The default `None` is equal to the number - of unique samples in your dataset divided by the batch - size, or 1 if that cannot be determined. + next epoch. Ignored with the default value of `None`. # Returns `History` object. @@ -1001,22 +1014,12 @@ def _fit_loop(self, f, ins, out_labels=None, batch_size=32, do_validation = False if val_f and val_ins: do_validation = True - if verbose: + if verbose and ins and hasattr(ins[0], 'shape'): print('Train on %d samples, validate on %d samples' % (ins[0].shape[0], val_ins[0].shape[0])) - if steps_per_epoch is not None: - num_train_samples = steps_per_epoch - else: - if ins and hasattr(ins[0], 'shape'): - num_train_samples = ins[0].shape[0] - else: - # May happen if we are running `fit` without Numpy input data, - # i.e. if all inputs to the models are data tensors - # instead of placeholders. - # In that case we will run `fit` over a single batch. - num_train_samples = batch_size - verbose = 2 + num_train_samples = self._check_num_samples(ins, batch_size, + steps_per_epoch, 'steps_per_epoch') index_array = np.arange(num_train_samples) self.history = cbks.History() @@ -1098,7 +1101,7 @@ def _fit_loop(self, f, ins, out_labels=None, batch_size=32, callbacks.on_train_end() return self.history - def _predict_loop(self, f, ins, batch_size=32, verbose=0): + def _predict_loop(self, f, ins, batch_size=32, verbose=0, steps=None): """Abstract method to loop over some data in batches. # Arguments @@ -1106,21 +1109,16 @@ def _predict_loop(self, f, ins, batch_size=32, verbose=0): ins: list of tensors to be fed to `f`. batch_size: integer batch size. verbose: verbosity mode. + steps: Total number of steps (batches of samples) + before declaring _predict_loop finished. + Ignored with the default value of `None`. # Returns Array of predictions (if the model has a single output) or list of arrays of predictions (if the model has multiple outputs). """ - if ins and hasattr(ins[0], 'shape'): - samples = ins[0].shape[0] - else: - # May happen if we are running `predict` without Numpy input data, - # i.e. if all inputs to the models are data tensors - # instead of placeholders. - # In that case we will run `predict` over a single batch. - samples = batch_size - verbose = 2 + samples = self._check_num_samples(ins, batch_size, steps, 'steps') outs = [] if verbose == 1: progbar = Progbar(target=samples) @@ -1150,7 +1148,7 @@ def _predict_loop(self, f, ins, batch_size=32, verbose=0): return outs[0] return outs - def _test_loop(self, f, ins, batch_size=32, verbose=0): + def _test_loop(self, f, ins, batch_size=32, verbose=0, steps=None): """Abstract method to loop over some data in batches. # Arguments @@ -1158,6 +1156,9 @@ def _test_loop(self, f, ins, batch_size=32, verbose=0): ins: list of tensors to be fed to `f`. batch_size: integer batch size. verbose: verbosity mode. + steps: Total number of steps (batches of samples) + before declaring predictions finished. + Ignored with the default value of `None`. # Returns Scalar loss (if the model has a single output and no metrics) @@ -1165,15 +1166,7 @@ def _test_loop(self, f, ins, batch_size=32, verbose=0): and/or metrics). The attribute `model.metrics_names` will give you the display labels for the scalar outputs. """ - if ins and hasattr(ins[0], 'shape'): - samples = ins[0].shape[0] - else: - # May happen if we are running `evaluate` without Numpy input data, - # i.e. if all inputs to the models are data tensors - # instead of placeholders. - # In that case we will run `evaluate` over a single batch. - samples = batch_size - verbose = 2 + samples = self._check_num_samples(ins, batch_size, steps, 'steps') outs = [] if verbose == 1: @@ -1476,7 +1469,7 @@ def evaluate(self, x, y, batch_size=32, verbose=1, sample_weight=None): batch_size=batch_size, verbose=verbose) - def predict(self, x, batch_size=32, verbose=0): + def predict(self, x, batch_size=32, verbose=0, steps=None): """Generates output predictions for the input samples. Computation is done in batches. @@ -1486,6 +1479,9 @@ def predict(self, x, batch_size=32, verbose=0): (or list of Numpy arrays if the model has multiple outputs). batch_size: integer. verbose: verbosity mode, 0 or 1. + steps: Total number of steps (batches of samples) + before declaring predictions finished. + Ignored with the default value of `None`. # Returns Numpy array(s) of predictions. @@ -1516,8 +1512,8 @@ def predict(self, x, batch_size=32, verbose=0): ins = x self._make_predict_function() f = self.predict_function - return self._predict_loop(f, ins, - batch_size=batch_size, verbose=verbose) + return self._predict_loop(f, ins, batch_size=batch_size, + verbose=verbose, steps=steps) def train_on_batch(self, x, y, sample_weight=None, class_weight=None): From 9498dba0f4e579e492982a59f685214956d9bf26 Mon Sep 17 00:00:00 2001 From: Andrew Hundt Date: Mon, 26 Jun 2017 16:06:15 -0400 Subject: [PATCH 109/166] rename tensors param to better skip_standardizing param name. --- keras/engine/training.py | 13 ++++++++----- tests/keras/engine/test_training.py | 2 +- 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/keras/engine/training.py b/keras/engine/training.py index 6f1ecd584fe..4df0e71ed02 100644 --- a/keras/engine/training.py +++ b/keras/engine/training.py @@ -29,7 +29,7 @@ def _standardize_input_data(data, names, shapes=None, check_batch_axis=True, exception_prefix='', - tensors=None): + skip_standardizing=None): """Normalizes inputs and targets provided by users. Users may pass data as a list of arrays, dictionary of arrays, @@ -45,6 +45,9 @@ def _standardize_input_data(data, names, shapes=None, the batch axis of the arrays matches the expected value found in `shapes`. exception_prefix: String prefix used for exception formatting. + skip_standardizing: A list of items to skip standardizing, + Will be standardized if None, or elements in the list + are None will skip standardizing otherwise. # Returns List of standardized input arrays (one array per model input). @@ -52,14 +55,14 @@ def _standardize_input_data(data, names, shapes=None, # Raises ValueError: in case of improperly formatted user-provided data. """ - if tensors is None: - tensors = [] + if skip_standardizing is None: + skip_standardizing = [] if not names: return [] if data is None: return [None for _ in range(len(names))] - num_tensors = sum(x is not None for x in tensors) - expected_names = len(names) - num_tensors + num_skip_standardizing = sum(x is not None for x in skip_standardizing) + expected_names = len(names) - num_skip_standardizing if isinstance(data, dict): arrays = [] for name in names: diff --git a/tests/keras/engine/test_training.py b/tests/keras/engine/test_training.py index 170570b7d36..00c3790eb99 100644 --- a/tests/keras/engine/test_training.py +++ b/tests/keras/engine/test_training.py @@ -47,7 +47,7 @@ def test_standardize_input_data(): x = _standardize_input_data( [None], [a_name], [a_shape], exception_prefix='input', - tensors=[p]) + skip_standardizing=[p]) @keras_test From 76b1a2bb8ee3ddfa58bba88c9902faaaaf5b5b30 Mon Sep 17 00:00:00 2001 From: Andrew Hundt Date: Mon, 26 Jun 2017 16:46:59 -0400 Subject: [PATCH 110/166] mnist_tfrecord.py pep8 --- examples/mnist_tfrecord.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/examples/mnist_tfrecord.py b/examples/mnist_tfrecord.py index b85ace41f1b..7535d9e2fee 100644 --- a/examples/mnist_tfrecord.py +++ b/examples/mnist_tfrecord.py @@ -68,9 +68,12 @@ def _bytes_feature(value): print('tfrecord %s already exists' % filename) -def read_and_decode_recordinput(tf_glob, one_hot=True, classes=None, is_train=None, - batch_shape=[1000, 28, 28, 1], parallelism=1): +def read_and_decode_recordinput(tf_glob, one_hot=True, + classes=None, is_train=None, + batch_shape=None, parallelism=1): """ Return tensor to read from TFRecord """ + if batch_shape is None: + batch_shape = [1000, 28, 28, 1] print 'Creating graph for loading %s TFRecords...' % tf_glob with tf.variable_scope("TFRecords"): record_input = data_flow_ops.RecordInput( From 563c5ff2094cbc2633a4a7e2a1bf0d00af356a64 Mon Sep 17 00:00:00 2001 From: Andrew Hundt Date: Mon, 26 Jun 2017 18:07:08 -0400 Subject: [PATCH 111/166] fix test failures --- keras/engine/training.py | 30 +++++++++++++++++++++-------- tests/keras/engine/test_training.py | 2 +- 2 files changed, 23 insertions(+), 9 deletions(-) diff --git a/keras/engine/training.py b/keras/engine/training.py index 95a799cfa4a..7d418b0b665 100644 --- a/keras/engine/training.py +++ b/keras/engine/training.py @@ -1101,7 +1101,7 @@ def _fit_loop(self, f, ins, out_labels=None, batch_size=32, callbacks.on_train_end() return self.history - def _predict_loop(self, f, ins, batch_size=32, verbose=0, steps=None): + def _predict_loop(self, f, ins, batch_size=32, verbose=0): """Abstract method to loop over some data in batches. # Arguments @@ -1118,7 +1118,16 @@ def _predict_loop(self, f, ins, batch_size=32, verbose=0, steps=None): or list of arrays of predictions (if the model has multiple outputs). """ - samples = self._check_num_samples(ins, batch_size, steps, 'steps') + if ins and hasattr(ins[0], 'shape'): + samples = ins[0].shape[0] + else: + # May happen if we are running `predict` without Numpy input data, + # i.e. if all inputs to the models are data tensors + # instead of placeholders. + # In that case we will run `predict` over a single batch. + samples = batch_size + verbose = 2 + outs = [] if verbose == 1: progbar = Progbar(target=samples) @@ -1166,7 +1175,15 @@ def _test_loop(self, f, ins, batch_size=32, verbose=0, steps=None): and/or metrics). The attribute `model.metrics_names` will give you the display labels for the scalar outputs. """ - samples = self._check_num_samples(ins, batch_size, steps, 'steps') + if ins and hasattr(ins[0], 'shape'): + samples = ins[0].shape[0] + else: + # May happen if we are running `predict` without Numpy input data, + # i.e. if all inputs to the models are data tensors + # instead of placeholders. + # In that case we will run `predict` over a single batch. + samples = batch_size + verbose = 2 outs = [] if verbose == 1: @@ -1469,7 +1486,7 @@ def evaluate(self, x, y, batch_size=32, verbose=1, sample_weight=None): batch_size=batch_size, verbose=verbose) - def predict(self, x, batch_size=32, verbose=0, steps=None): + def predict(self, x, batch_size=32, verbose=0): """Generates output predictions for the input samples. Computation is done in batches. @@ -1479,9 +1496,6 @@ def predict(self, x, batch_size=32, verbose=0, steps=None): (or list of Numpy arrays if the model has multiple outputs). batch_size: integer. verbose: verbosity mode, 0 or 1. - steps: Total number of steps (batches of samples) - before declaring predictions finished. - Ignored with the default value of `None`. # Returns Numpy array(s) of predictions. @@ -1513,7 +1527,7 @@ def predict(self, x, batch_size=32, verbose=0, steps=None): self._make_predict_function() f = self.predict_function return self._predict_loop(f, ins, batch_size=batch_size, - verbose=verbose, steps=steps) + verbose=verbose) def train_on_batch(self, x, y, sample_weight=None, class_weight=None): diff --git a/tests/keras/engine/test_training.py b/tests/keras/engine/test_training.py index 1712827d99f..e48540fdcfd 100644 --- a/tests/keras/engine/test_training.py +++ b/tests/keras/engine/test_training.py @@ -672,7 +672,7 @@ def test_model_with_external_loss(): out = model.predict_on_batch(None) # test fit - out = model.fit(None, None, epochs=1, batch_size=10) + out = model.fit(None, None, epochs=1, batch_size=10, steps_per_epoch=1) # test evaluate out = model.evaluate(None, None, batch_size=10) From 0de3fc64ae99fbacfa95d4125c96de11008b717e Mon Sep 17 00:00:00 2001 From: Andrew Hundt Date: Mon, 26 Jun 2017 18:09:29 -0400 Subject: [PATCH 112/166] remove inaccurate warning --- keras/engine/training.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/keras/engine/training.py b/keras/engine/training.py index 7d418b0b665..7262876c106 100644 --- a/keras/engine/training.py +++ b/keras/engine/training.py @@ -964,11 +964,6 @@ def _make_predict_function(self): def _check_num_samples(self, ins, batch_size=None, steps=None, steps_name='steps'): if steps is not None: - if batch_size is not None: - warnings.warn(steps_name + ' was specified, ' - 'so batch_size is being ignored. ' - 'Specify None for batch_size to ' - 'remove this warning.') num_samples = steps elif ins and hasattr(ins[0], 'shape'): num_samples = ins[0].shape[0] From fb67e060fbbb65676bb03c0b8a898b3abd9a8bff Mon Sep 17 00:00:00 2001 From: Andrew Hundt Date: Mon, 26 Jun 2017 22:02:01 -0400 Subject: [PATCH 113/166] is_keras_tensor() document new argument expect_other_types --- keras/backend/tensorflow_backend.py | 3 +++ keras/backend/theano_backend.py | 3 +++ 2 files changed, 6 insertions(+) diff --git a/keras/backend/tensorflow_backend.py b/keras/backend/tensorflow_backend.py index e7c5e93150d..1c5be616540 100644 --- a/keras/backend/tensorflow_backend.py +++ b/keras/backend/tensorflow_backend.py @@ -365,6 +365,9 @@ def is_keras_tensor(x, expect_other_types=False): # Arguments x: a potential tensor. + expect_other_types: Expect types that aren't a tensor + of any kind. When True, exceptions will not be raised by + default for non tensor types, instead it will return False. # Returns A boolean: whether the argument is a Keras tensor. diff --git a/keras/backend/theano_backend.py b/keras/backend/theano_backend.py index f44af7aaea5..0bc15b4a274 100644 --- a/keras/backend/theano_backend.py +++ b/keras/backend/theano_backend.py @@ -171,6 +171,9 @@ def is_keras_tensor(x, expect_other_types=False): # Arguments x: a potential tensor. + expect_other_types: Expect types that aren't a tensor + of any kind. When True, exceptions will not be raised by + default for non tensor types, instead it will return False. # Returns A boolean: whether the argument is a Keras tensor. From b4fb0860fe48729caed0979911e91c06cdd80f68 Mon Sep 17 00:00:00 2001 From: Andrew Hundt Date: Tue, 25 Jul 2017 01:04:21 -0700 Subject: [PATCH 114/166] remove extraneous imports of is_placeholder --- keras/backend/cntk_backend.py | 1 - keras/backend/tensorflow_backend.py | 1 - keras/backend/theano_backend.py | 1 - 3 files changed, 3 deletions(-) diff --git a/keras/backend/cntk_backend.py b/keras/backend/cntk_backend.py index 9f99a46b540..bb7554f8382 100644 --- a/keras/backend/cntk_backend.py +++ b/keras/backend/cntk_backend.py @@ -2,7 +2,6 @@ import cntk as C import numpy as np from .common import _FLOATX, _EPSILON, image_dim_ordering, image_data_format -from .common import is_placeholder from collections import defaultdict from contextlib import contextmanager import warnings diff --git a/keras/backend/tensorflow_backend.py b/keras/backend/tensorflow_backend.py index 39a96f87476..f01cdcff7a3 100644 --- a/keras/backend/tensorflow_backend.py +++ b/keras/backend/tensorflow_backend.py @@ -14,7 +14,6 @@ from .common import floatx from .common import _EPSILON from .common import image_data_format -from .common import is_placeholder from ..utils.generic_utils import has_arg # Legacy functions diff --git a/keras/backend/theano_backend.py b/keras/backend/theano_backend.py index a585bcc8abe..b623b349de9 100644 --- a/keras/backend/theano_backend.py +++ b/keras/backend/theano_backend.py @@ -17,7 +17,6 @@ import numpy as np from .common import _FLOATX, floatx, _EPSILON, image_data_format -from .common import is_placeholder from ..utils.generic_utils import has_arg # Legacy functions from .common import set_image_dim_ordering, image_dim_ordering From d4f0b1c4eed4f363322e4b2939c1c54221d2b300 Mon Sep 17 00:00:00 2001 From: Andrew Hundt Date: Tue, 25 Jul 2017 02:12:38 -0700 Subject: [PATCH 115/166] _standardize_input_data rename skip_standardizing to input_tensors based on comments (#7067) --- keras/engine/training.py | 14 ++++++-------- tests/keras/engine/test_training.py | 2 +- 2 files changed, 7 insertions(+), 9 deletions(-) diff --git a/keras/engine/training.py b/keras/engine/training.py index 4df0e71ed02..42608031b4a 100644 --- a/keras/engine/training.py +++ b/keras/engine/training.py @@ -29,7 +29,7 @@ def _standardize_input_data(data, names, shapes=None, check_batch_axis=True, exception_prefix='', - skip_standardizing=None): + input_tensors=None): """Normalizes inputs and targets provided by users. Users may pass data as a list of arrays, dictionary of arrays, @@ -45,9 +45,7 @@ def _standardize_input_data(data, names, shapes=None, the batch axis of the arrays matches the expected value found in `shapes`. exception_prefix: String prefix used for exception formatting. - skip_standardizing: A list of items to skip standardizing, - Will be standardized if None, or elements in the list - are None will skip standardizing otherwise. + input_tensors: A list of input tensor based data sources. # Returns List of standardized input arrays (one array per model input). @@ -55,14 +53,14 @@ def _standardize_input_data(data, names, shapes=None, # Raises ValueError: in case of improperly formatted user-provided data. """ - if skip_standardizing is None: - skip_standardizing = [] + if input_tensors is None: + input_tensors = [] if not names: return [] if data is None: return [None for _ in range(len(names))] - num_skip_standardizing = sum(x is not None for x in skip_standardizing) - expected_names = len(names) - num_skip_standardizing + num_input_tensors = sum(x is not None for x in input_tensors) + expected_names = len(names) - num_input_tensors if isinstance(data, dict): arrays = [] for name in names: diff --git a/tests/keras/engine/test_training.py b/tests/keras/engine/test_training.py index 00c3790eb99..1d094f123ed 100644 --- a/tests/keras/engine/test_training.py +++ b/tests/keras/engine/test_training.py @@ -47,7 +47,7 @@ def test_standardize_input_data(): x = _standardize_input_data( [None], [a_name], [a_shape], exception_prefix='input', - skip_standardizing=[p]) + input_tensors=[p]) @keras_test From 4d6e3d4a0cb5ae51d846ea8b5b9bf7ad56d0085c Mon Sep 17 00:00:00 2001 From: Andrew Hundt Date: Mon, 31 Jul 2017 18:14:59 -0700 Subject: [PATCH 116/166] K.is_placeholder() becomes simple member variable is_placeholder with getattr checks --- examples/variational_autoencoder.py | 2 +- examples/variational_autoencoder_deconv.py | 2 +- keras/backend/__init__.py | 1 - keras/backend/cntk_backend.py | 2 +- keras/backend/common.py | 23 ---------------------- keras/backend/tensorflow_backend.py | 2 +- keras/backend/theano_backend.py | 2 +- keras/engine/topology.py | 14 ++++++------- keras/models.py | 4 ++-- tests/keras/backend/backend_test.py | 4 ++-- tests/keras/engine/test_topology.py | 2 +- 11 files changed, 17 insertions(+), 41 deletions(-) diff --git a/examples/variational_autoencoder.py b/examples/variational_autoencoder.py index ba313f1c69b..fd395b17f45 100644 --- a/examples/variational_autoencoder.py +++ b/examples/variational_autoencoder.py @@ -45,7 +45,7 @@ def sampling(args): # Custom loss layer class CustomVariationalLayer(Layer): def __init__(self, **kwargs): - self._is_placeholder = True + self.is_placeholder = True super(CustomVariationalLayer, self).__init__(**kwargs) def vae_loss(self, x, x_decoded_mean): diff --git a/examples/variational_autoencoder_deconv.py b/examples/variational_autoencoder_deconv.py index 3612754c494..f5aa1224d47 100644 --- a/examples/variational_autoencoder_deconv.py +++ b/examples/variational_autoencoder_deconv.py @@ -109,7 +109,7 @@ def sampling(args): # Custom loss layer class CustomVariationalLayer(Layer): def __init__(self, **kwargs): - self._is_placeholder = True + self.is_placeholder = True super(CustomVariationalLayer, self).__init__(**kwargs) def vae_loss(self, x, x_decoded_mean_squash): diff --git a/keras/backend/__init__.py b/keras/backend/__init__.py index 00e6d52fbe8..2e7208cf5af 100644 --- a/keras/backend/__init__.py +++ b/keras/backend/__init__.py @@ -10,7 +10,6 @@ from .common import cast_to_floatx from .common import image_data_format from .common import set_image_data_format -from .common import is_placeholder # Obtain Keras base dir path: either ~/.keras or /tmp. _keras_base_dir = os.path.expanduser('~') diff --git a/keras/backend/cntk_backend.py b/keras/backend/cntk_backend.py index bb7554f8382..214d08d2aad 100644 --- a/keras/backend/cntk_backend.py +++ b/keras/backend/cntk_backend.py @@ -256,7 +256,7 @@ def placeholder( name=name) x._keras_shape = shape x._uses_learning_phase = False - x._is_placeholder = True + x.is_placeholder = True return x diff --git a/keras/backend/common.py b/keras/backend/common.py index a147c7f8e1d..97646067c9b 100644 --- a/keras/backend/common.py +++ b/keras/backend/common.py @@ -108,29 +108,6 @@ def cast_to_floatx(x): return np.asarray(x, dtype=_FLOATX) -def is_placeholder(tensor): - """Returns whether a tensor is a placeholder. - - # Arguments - tensor: A tensor instance. - - # Returns - A boolean. - - # Example - ```python - >>> from keras import backend as K - >>> a = K.placeholder((2, 2), sparse=False) - >>> print(K.is_placeholder(a)) - True - ``` - """ - try: - return tensor._is_placeholder - except AttributeError: - return False - - def image_data_format(): """Returns the default image data format convention ('channels_first' or 'channels_last'). diff --git a/keras/backend/tensorflow_backend.py b/keras/backend/tensorflow_backend.py index f01cdcff7a3..bb7e5a19c89 100644 --- a/keras/backend/tensorflow_backend.py +++ b/keras/backend/tensorflow_backend.py @@ -432,7 +432,7 @@ def placeholder(shape=None, ndim=None, dtype=None, sparse=False, name=None): x = tf.placeholder(dtype, shape=shape, name=name) x._keras_shape = shape x._uses_learning_phase = False - x._is_placeholder = True + x.is_placeholder = True return x diff --git a/keras/backend/theano_backend.py b/keras/backend/theano_backend.py index b623b349de9..715059d1917 100644 --- a/keras/backend/theano_backend.py +++ b/keras/backend/theano_backend.py @@ -222,7 +222,7 @@ def placeholder(shape=None, ndim=None, dtype=None, sparse=False, name=None): x = T.TensorType(dtype, broadcast)(name) x._keras_shape = shape x._uses_learning_phase = False - x._is_placeholder = True + x.is_placeholder = True return x diff --git a/keras/engine/topology.py b/keras/engine/topology.py index 3f8d02e2ba8..01c265159d2 100644 --- a/keras/engine/topology.py +++ b/keras/engine/topology.py @@ -159,7 +159,7 @@ def __init__(self, outbound_layer, self.arguments = arguments # Indicates if the node represents a placeholder variable - self._is_placeholder = is_placeholder + self.is_placeholder = is_placeholder # Add nodes to all layers involved. for layer in inbound_layers: if layer is not None: @@ -1333,19 +1333,19 @@ def __init__(self, input_shape=None, batch_size=None, self.dtype = dtype if input_tensor is None: - self._is_placeholder = True + self.is_placeholder = True input_tensor = K.placeholder(shape=batch_input_shape, dtype=dtype, sparse=self.sparse, name=self.name) else: - self._is_placeholder = False + self.is_placeholder = False input_tensor._keras_shape = batch_input_shape # Create an input node to add to self.outbound_node # and set output_tensors' _keras_history. input_tensor._uses_learning_phase = False input_tensor._keras_history = (self, 0, 0) - input_tensor._is_placeholder = self._is_placeholder + input_tensor.is_placeholder = self.is_placeholder Node(self, inbound_layers=[], node_indices=[], @@ -1356,7 +1356,7 @@ def __init__(self, input_shape=None, batch_size=None, output_masks=[None], input_shapes=[batch_input_shape], output_shapes=[batch_input_shape], - is_placeholder=self._is_placeholder) + is_placeholder=self.is_placeholder) def get_config(self): config = {'batch_input_shape': self.batch_input_shape, @@ -1570,7 +1570,7 @@ def __init__(self, inputs, outputs, name=None): 'instantiated via `tensor = Input(shape)`.\n' 'The tensor that caused the issue was: ' + str(x.name)) - if K.is_placeholder(layer): + if getattr(layer, 'is_placeholder', False): self._input_placeholders.append((layer, node_index, tensor_index)) else: self._input_yield_op_tensors.append((layer, node_index, tensor_index)) @@ -1635,7 +1635,7 @@ def __init__(self, inputs, outputs, name=None): i, layer.__class__.__name__)) self.input_names.append(layer.name) - if K.is_placeholder(layer): + if getattr(layer, 'is_placeholder', False): self._feed_input_names.append(layer.name) self._feed_inputs.append(layer.input) self._feed_input_shapes.append(self.inputs[i]._keras_shape) diff --git a/keras/models.py b/keras/models.py index 3d9aea608c0..5406852cfdd 100644 --- a/keras/models.py +++ b/keras/models.py @@ -456,7 +456,7 @@ def add(self, layer): 'For multi-output layers, ' 'use the functional API.') - layer.inbound_nodes[0].output_tensors[0]._is_placeholder = True + layer.inbound_nodes[0].output_tensors[0].is_placeholder = True self.outputs = [layer.inbound_nodes[0].output_tensors[0]] self.inputs = topology.get_source_inputs(self.outputs[0]) @@ -480,7 +480,7 @@ def add(self, layer): 'should have a single output tensor. ' 'For multi-output layers, ' 'use the functional API.') - output_tensor._is_placeholder = True + output_tensor.is_placeholder = True self.outputs = [output_tensor] # update self.inbound_nodes self.inbound_nodes[0].output_tensors = self.outputs diff --git a/tests/keras/backend/backend_test.py b/tests/keras/backend/backend_test.py index 4f875e07d1c..c54b5fe7a83 100644 --- a/tests/keras/backend/backend_test.py +++ b/tests/keras/backend/backend_test.py @@ -182,8 +182,8 @@ def test_is_keras_tensor(self): assert K.is_keras_tensor(keras_var) is True keras_placeholder = K.placeholder(shape=(2, 4, 5)) assert K.is_keras_tensor(keras_placeholder) is True - assert K.is_placeholder(keras_placeholder) is True - assert K.is_placeholder(np_var) is False + assert getattr(keras_placeholder, 'is_placeholder', False) is True + assert getattr(np_var, 'is_placeholder', False) is False def test_set_learning_phase(self): # not supported learning_phase diff --git a/tests/keras/engine/test_topology.py b/tests/keras/engine/test_topology.py index fd258304896..bccbf182e13 100644 --- a/tests/keras/engine/test_topology.py +++ b/tests/keras/engine/test_topology.py @@ -36,7 +36,7 @@ def test_get_losses_for(): @keras_test def test_is_placeholder(): a = Input(shape=(2,)) - assert K.is_placeholder(a) + assert getattr(a, 'is_placeholder', False) @keras_test From 5b851326f369e8e00e7b6f9c53a68b24ce951461 Mon Sep 17 00:00:00 2001 From: Andrew Hundt Date: Mon, 31 Jul 2017 19:20:54 -0700 Subject: [PATCH 117/166] _standardize_input_data add missing space to exception error string --- keras/engine/training.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/keras/engine/training.py b/keras/engine/training.py index 42608031b4a..574e88d76cf 100644 --- a/keras/engine/training.py +++ b/keras/engine/training.py @@ -77,7 +77,7 @@ def _standardize_input_data(data, names, shapes=None, ' arrays: ' + str(data)[:200] + '...') else: - datastr = ('data with type' + + datastr = ('data with type ' + type(data).__name__ + '...') raise ValueError('Error when checking model ' + exception_prefix + From d60f1e209bb0a043bc419062a50dc39f945c65f6 Mon Sep 17 00:00:00 2001 From: Andrew Hundt Date: Mon, 31 Jul 2017 19:36:43 -0700 Subject: [PATCH 118/166] _standardize_input_data accepts feed inputs --- keras/engine/training.py | 23 ++++++++++++----------- 1 file changed, 12 insertions(+), 11 deletions(-) diff --git a/keras/engine/training.py b/keras/engine/training.py index 574e88d76cf..45388fc505e 100644 --- a/keras/engine/training.py +++ b/keras/engine/training.py @@ -29,7 +29,7 @@ def _standardize_input_data(data, names, shapes=None, check_batch_axis=True, exception_prefix='', - input_tensors=None): + feed_data=None): """Normalizes inputs and targets provided by users. Users may pass data as a list of arrays, dictionary of arrays, @@ -45,7 +45,7 @@ def _standardize_input_data(data, names, shapes=None, the batch axis of the arrays matches the expected value found in `shapes`. exception_prefix: String prefix used for exception formatting. - input_tensors: A list of input tensor based data sources. + feed_data: A list of placeholder feed input based data sources. # Returns List of standardized input arrays (one array per model input). @@ -53,14 +53,11 @@ def _standardize_input_data(data, names, shapes=None, # Raises ValueError: in case of improperly formatted user-provided data. """ - if input_tensors is None: - input_tensors = [] - if not names: + if not names or not feed_data: return [] if data is None: return [None for _ in range(len(names))] - num_input_tensors = sum(x is not None for x in input_tensors) - expected_names = len(names) - num_input_tensors + expected_names = sum(getattr(feed_data, 'is_placeholder', False) is True for x in feed_data) if isinstance(data, dict): arrays = [] for name in names: @@ -1236,11 +1233,13 @@ def _standardize_user_data(self, x, y, x = _standardize_input_data(x, self._feed_input_names, self._feed_input_shapes, check_batch_axis=False, - exception_prefix='input') + exception_prefix='input', + feed_data=self._feed_inputs) y = _standardize_input_data(y, self._feed_output_names, output_shapes, check_batch_axis=False, - exception_prefix='target') + exception_prefix='target', + feed_data=self._feed_outputs) sample_weights = _standardize_sample_weights(sample_weight, self._feed_output_names) class_weights = _standardize_class_weights(class_weight, @@ -1501,7 +1500,8 @@ def predict(self, x, batch_size=32, verbose=0): # Validate user data. x = _standardize_input_data(x, self._feed_input_names, self._feed_input_shapes, - check_batch_axis=False) + check_batch_axis=False, + feed_data=self._feed_inputs) if self.stateful: if x[0].shape[0] > batch_size and x[0].shape[0] % batch_size != 0: raise ValueError('In a stateful network, ' @@ -1624,7 +1624,8 @@ def predict_on_batch(self, x): Numpy array(s) of predictions. """ x = _standardize_input_data(x, self._feed_input_names, - self._feed_input_shapes) + self._feed_input_shapes, + feed_data=self.feed_inputs) if self.uses_learning_phase and not isinstance(K.learning_phase(), int): ins = x + [0.] else: From 478a6809022a5a228381a9f0970e946f524a294f Mon Sep 17 00:00:00 2001 From: Andrew Hundt Date: Mon, 31 Jul 2017 19:47:53 -0700 Subject: [PATCH 119/166] _standardize_input_data() now only standardizes data that is convertible to numpy array based on what is destined for the feed_dict --- keras/engine/training.py | 2 +- tests/keras/engine/test_training.py | 5 +++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/keras/engine/training.py b/keras/engine/training.py index 45388fc505e..08f94d5a5d3 100644 --- a/keras/engine/training.py +++ b/keras/engine/training.py @@ -57,7 +57,7 @@ def _standardize_input_data(data, names, shapes=None, return [] if data is None: return [None for _ in range(len(names))] - expected_names = sum(getattr(feed_data, 'is_placeholder', False) is True for x in feed_data) + expected_names = sum(getattr(x, 'is_placeholder', False) is True for x in feed_data) if isinstance(data, dict): arrays = [] for name in names: diff --git a/tests/keras/engine/test_training.py b/tests/keras/engine/test_training.py index 1d094f123ed..06073d0a680 100644 --- a/tests/keras/engine/test_training.py +++ b/tests/keras/engine/test_training.py @@ -43,11 +43,12 @@ def test_standardize_input_data(): x = _standardize_input_data( [a_np], [a_name], [a_shape], check_batch_axis=False, - exception_prefix='input') + exception_prefix='input', + feed_data=[p]) x = _standardize_input_data( [None], [a_name], [a_shape], exception_prefix='input', - input_tensors=[p]) + feed_data=[None]) @keras_test From 65a5bce5e9742ade7540d0278aded86524061096 Mon Sep 17 00:00:00 2001 From: Andrew Hundt Date: Mon, 31 Jul 2017 20:54:40 -0700 Subject: [PATCH 120/166] fix _feed_inputs name --- keras/engine/training.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/keras/engine/training.py b/keras/engine/training.py index 5854efe5b56..87efec9d6d2 100644 --- a/keras/engine/training.py +++ b/keras/engine/training.py @@ -1630,7 +1630,7 @@ def predict_on_batch(self, x): """ x = _standardize_input_data(x, self._feed_input_names, self._feed_input_shapes, - feed_data=self.feed_inputs) + feed_data=self._feed_inputs) if self.uses_learning_phase and not isinstance(K.learning_phase(), int): ins = x + [0.] else: From 620874f4ccae7485c752164cbe36137aa271a710 Mon Sep 17 00:00:00 2001 From: Andrew Hundt Date: Mon, 31 Jul 2017 22:06:26 -0700 Subject: [PATCH 121/166] training.py _standardize_input_data() accounts for input tensors (#7067) --- keras/engine/topology.py | 12 +++++------- keras/engine/training.py | 24 ++++++++++++++---------- tests/keras/engine/test_training.py | 4 ++-- 3 files changed, 21 insertions(+), 19 deletions(-) diff --git a/keras/engine/topology.py b/keras/engine/topology.py index bf878a15609..4afd1d7cb6c 100644 --- a/keras/engine/topology.py +++ b/keras/engine/topology.py @@ -1543,8 +1543,6 @@ def __init__(self, inputs, outputs, name=None): self._output_tensor_cache = {} self._output_shape_cache = {} - self._input_placeholders = [] - self._input_yield_op_tensors = [] # User-provided arguments validation. for x in self.inputs: # Check that x is a Keras tensor. @@ -1570,10 +1568,6 @@ def __init__(self, inputs, outputs, name=None): 'instantiated via `tensor = Input(shape)`.\n' 'The tensor that caused the issue was: ' + str(x.name)) - if getattr(layer, 'is_placeholder', False): - self._input_placeholders.append((layer, node_index, tensor_index)) - else: - self._input_yield_op_tensors.append((layer, node_index, tensor_index)) for x in self.outputs: if not hasattr(x, '_keras_history'): cls_name = self.__class__.__name__ @@ -1624,6 +1618,7 @@ def __init__(self, inputs, outputs, name=None): self._feed_input_names = [] self._feed_inputs = [] self._feed_input_shapes = [] + self._tensor_inputs = [] for i, layer in enumerate(self.input_layers): # Check that layer is an InputLayer. if not isinstance(layer, InputLayer): @@ -1635,10 +1630,13 @@ def __init__(self, inputs, outputs, name=None): i, layer.__class__.__name__)) self.input_names.append(layer.name) - if getattr(layer, 'is_placeholder', False): + if getattr(layer, 'is_placeholder', False) is True: self._feed_input_names.append(layer.name) self._feed_inputs.append(layer.input) self._feed_input_shapes.append(self.inputs[i]._keras_shape) + elif getattr(layer, 'input_tensor', False) is True: + self._tensor_inputs.append(layer.input_tensor) + for layer in self.output_layers: self.output_names.append(layer.name) diff --git a/keras/engine/training.py b/keras/engine/training.py index 87efec9d6d2..27bcc310658 100644 --- a/keras/engine/training.py +++ b/keras/engine/training.py @@ -6,6 +6,7 @@ import copy import numpy as np import six +from six.moves import map from keras.utils import Sequence from keras.utils import GeneratorEnqueuer @@ -29,7 +30,7 @@ def _standardize_input_data(data, names, shapes=None, check_batch_axis=True, exception_prefix='', - feed_data=None): + input_tensors=None): """Normalizes inputs and targets provided by users. Users may pass data as a list of arrays, dictionary of arrays, @@ -45,7 +46,7 @@ def _standardize_input_data(data, names, shapes=None, the batch axis of the arrays matches the expected value found in `shapes`. exception_prefix: String prefix used for exception formatting. - feed_data: A list of placeholder feed input based data sources. + input_tensors: A list of input tensor based data sources. # Returns List of standardized input arrays (one array per model input). @@ -53,11 +54,14 @@ def _standardize_input_data(data, names, shapes=None, # Raises ValueError: in case of improperly formatted user-provided data. """ - if not names or not feed_data: + if input_tensors is None: + input_tensors = [] + if not names: return [] if data is None: return [None for _ in range(len(names))] - expected_names = sum(getattr(x, 'is_placeholder', False) is True for x in feed_data) + num_input_tensors = sum(getattr(x, 'is_placeholder', False) is False for x in input_tensors) + expected_names = len(names) - num_input_tensors if isinstance(data, dict): arrays = [] for name in names: @@ -74,8 +78,8 @@ def _standardize_input_data(data, names, shapes=None, ' arrays: ' + str(data)[:200] + '...') else: - datastr = ('data with type ' + - type(data).__name__ + '...') + datastr = ('data with types ' + + str([str(i) for i in map(type, data)]) + '...') raise ValueError('Error when checking model ' + exception_prefix + ': the list of Numpy arrays ' @@ -1239,12 +1243,12 @@ def _standardize_user_data(self, x, y, self._feed_input_shapes, check_batch_axis=False, exception_prefix='input', - feed_data=self._feed_inputs) + input_tensors=self._tensor_inputs) y = _standardize_input_data(y, self._feed_output_names, output_shapes, check_batch_axis=False, exception_prefix='target', - feed_data=self._feed_outputs) + input_tensors=self._tensor_inputs) sample_weights = _standardize_sample_weights(sample_weight, self._feed_output_names) class_weights = _standardize_class_weights(class_weight, @@ -1506,7 +1510,7 @@ def predict(self, x, batch_size=32, verbose=0): x = _standardize_input_data(x, self._feed_input_names, self._feed_input_shapes, check_batch_axis=False, - feed_data=self._feed_inputs) + input_tensors=self._tensor_inputs) if self.stateful: if x[0].shape[0] > batch_size and x[0].shape[0] % batch_size != 0: raise ValueError('In a stateful network, ' @@ -1630,7 +1634,7 @@ def predict_on_batch(self, x): """ x = _standardize_input_data(x, self._feed_input_names, self._feed_input_shapes, - feed_data=self._feed_inputs) + input_tensors=self._tensor_inputs) if self.uses_learning_phase and not isinstance(K.learning_phase(), int): ins = x + [0.] else: diff --git a/tests/keras/engine/test_training.py b/tests/keras/engine/test_training.py index 189eb355f8e..6a21697362c 100644 --- a/tests/keras/engine/test_training.py +++ b/tests/keras/engine/test_training.py @@ -47,11 +47,11 @@ def test_standardize_input_data(): [a_np], [a_name], [a_shape], check_batch_axis=False, exception_prefix='input', - feed_data=[p]) + input_tensors=[p]) x = _standardize_input_data( [None], [a_name], [a_shape], exception_prefix='input', - feed_data=[None]) + input_tensors=[None]) @keras_test From e9b63cc3646ca2ca8cb2a44dd5026cc91193665f Mon Sep 17 00:00:00 2001 From: Andrew Hundt Date: Tue, 1 Aug 2017 00:13:08 -0700 Subject: [PATCH 122/166] improved fit(steps_per_epoch) with separate internal epoch loop in _fit_loop --- keras/engine/training.py | 113 +++++++++++++++++++++++++-------------- 1 file changed, 72 insertions(+), 41 deletions(-) diff --git a/keras/engine/training.py b/keras/engine/training.py index 9c94b6841dd..a8dd308ca99 100644 --- a/keras/engine/training.py +++ b/keras/engine/training.py @@ -977,7 +977,9 @@ def _make_predict_function(self): def _check_num_samples(self, ins, batch_size=None, steps=None, steps_name='steps'): if steps is not None: - num_samples = steps + num_samples = None + if batch_size is not None: + raise ValueError('If steps_per_epoch is set the batch_size must be None.') elif ins and hasattr(ins[0], 'shape'): num_samples = ins[0].shape[0] else: @@ -1048,6 +1050,7 @@ def _fit_loop(self, f, ins, out_labels=None, batch_size=32, callbacks.set_params({ 'batch_size': batch_size, 'epochs': epochs, + 'steps': steps_per_epoch, 'samples': num_train_samples, 'verbose': verbose, 'do_validation': do_validation, @@ -1060,49 +1063,77 @@ def _fit_loop(self, f, ins, out_labels=None, batch_size=32, for epoch in range(initial_epoch, epochs): callbacks.on_epoch_begin(epoch) - if shuffle == 'batch': - index_array = _batch_shuffle(index_array, batch_size) - elif shuffle: - np.random.shuffle(index_array) - - batches = _make_batches(num_train_samples, batch_size) epoch_logs = {} - for batch_index, (batch_start, batch_end) in enumerate(batches): - batch_ids = index_array[batch_start:batch_end] - try: - if isinstance(ins[-1], float): - # Do not slice the training phase flag. - ins_batch = _slice_arrays(ins[:-1], batch_ids) + [ins[-1]] - else: - ins_batch = _slice_arrays(ins, batch_ids) - except TypeError: - raise TypeError('TypeError while preparing batch. ' - 'If using HDF5 input data, ' - 'pass shuffle="batch".') - batch_logs = {} - batch_logs['batch'] = batch_index - batch_logs['size'] = len(batch_ids) - callbacks.on_batch_begin(batch_index, batch_logs) - outs = f(ins_batch) - if not isinstance(outs, list): - outs = [outs] - for l, o in zip(out_labels, outs): - batch_logs[l] = o + if steps_per_epoch is not None: + for step_num in range(steps_per_epoch): + batch_logs = {} + batch_logs['batch'] = step_num + batch_logs['size'] = batch_size + callbacks.on_batch_begin(step_num, batch_logs) + outs = f(ins) - callbacks.on_batch_end(batch_index, batch_logs) - if callback_model.stop_training: - break + if not isinstance(outs, list): + outs = [outs] + for l, o in zip(out_labels, outs): + batch_logs[l] = o - if batch_index == len(batches) - 1: # Last batch. - if do_validation: - val_outs = self._test_loop(val_f, val_ins, - batch_size=batch_size, - verbose=0) - if not isinstance(val_outs, list): - val_outs = [val_outs] - # Same labels assumed. - for l, o in zip(out_labels, val_outs): - epoch_logs['val_' + l] = o + callbacks.on_batch_end(step_num, batch_logs) + if callback_model.stop_training: + break + + if do_validation: + val_outs = self._test_loop(val_f, val_ins, + batch_size=batch_size, + steps_per_epoch=steps_per_epoch + verbose=0) + if not isinstance(val_outs, list): + val_outs = [val_outs] + # Same labels assumed. + for l, o in zip(out_labels, val_outs): + epoch_logs['val_' + l] = o + else: + if shuffle == 'batch': + index_array = _batch_shuffle(index_array, batch_size) + elif shuffle: + np.random.shuffle(index_array) + + batches = _make_batches(num_train_samples, batch_size) + for batch_index, (batch_start, batch_end) in enumerate(batches): + batch_ids = index_array[batch_start:batch_end] + try: + if isinstance(ins[-1], float): + # Do not slice the training phase flag. + ins_batch = _slice_arrays(ins[:-1], batch_ids) + [ins[-1]] + else: + ins_batch = _slice_arrays(ins, batch_ids) + except TypeError: + raise TypeError('TypeError while preparing batch. ' + 'If using HDF5 input data, ' + 'pass shuffle="batch".') + batch_logs = {} + batch_logs['batch'] = batch_index + batch_logs['size'] = len(batch_ids) + callbacks.on_batch_begin(batch_index, batch_logs) + outs = f(ins_batch) + if not isinstance(outs, list): + outs = [outs] + for l, o in zip(out_labels, outs): + batch_logs[l] = o + + callbacks.on_batch_end(batch_index, batch_logs) + if callback_model.stop_training: + break + + if batch_index == len(batches) - 1: # Last batch. + if do_validation: + val_outs = self._test_loop(val_f, val_ins, + batch_size=batch_size, + verbose=0) + if not isinstance(val_outs, list): + val_outs = [val_outs] + # Same labels assumed. + for l, o in zip(out_labels, val_outs): + epoch_logs['val_' + l] = o callbacks.on_epoch_end(epoch, epoch_logs) if callback_model.stop_training: break From 5bcdc36a2d6940e1d41a1f0d38861f5e94d97b09 Mon Sep 17 00:00:00 2001 From: Andrew Hundt Date: Tue, 1 Aug 2017 01:02:28 -0700 Subject: [PATCH 123/166] fit(steps_per_epoch) initial validation support --- keras/engine/training.py | 103 +++++++++++++++++----------- tests/keras/engine/test_training.py | 12 +++- 2 files changed, 72 insertions(+), 43 deletions(-) diff --git a/keras/engine/training.py b/keras/engine/training.py index a8dd308ca99..6c149139542 100644 --- a/keras/engine/training.py +++ b/keras/engine/training.py @@ -991,7 +991,7 @@ def _fit_loop(self, f, ins, out_labels=None, batch_size=32, epochs=100, verbose=1, callbacks=None, val_f=None, val_ins=None, shuffle=True, callback_metrics=None, initial_epoch=0, - steps_per_epoch=None): + steps_per_epoch=None, validation_steps=None): """Abstract fit function for `f(ins)`. Assume that f returns a list, labeled by out_labels. @@ -1024,18 +1024,27 @@ def _fit_loop(self, f, ins, out_labels=None, batch_size=32, do_validation = False if val_f and val_ins: do_validation = True + if steps_per_epoch and not validation_steps: + raise ValueError('When using `steps_per_epoch` and validation data, ' + 'you must specify a value for `validation_steps`.') if verbose and ins and hasattr(ins[0], 'shape'): print('Train on %d samples, validate on %d samples' % (ins[0].shape[0], val_ins[0].shape[0])) num_train_samples = self._check_num_samples(ins, batch_size, steps_per_epoch, 'steps_per_epoch') - index_array = np.arange(num_train_samples) + + if num_train_samples is not None: + index_array = np.arange(num_train_samples) self.history = cbks.History() callbacks = [cbks.BaseLogger()] + (callbacks or []) + [self.history] if verbose: - callbacks += [cbks.ProgbarLogger()] + if steps_per_epoch: + count_mode = 'steps' + else: + count_mode = 'samples' + callbacks += [cbks.ProgbarLogger(count_mode)] callbacks = cbks.CallbackList(callbacks) out_labels = out_labels or [] @@ -1068,7 +1077,7 @@ def _fit_loop(self, f, ins, out_labels=None, batch_size=32, for step_num in range(steps_per_epoch): batch_logs = {} batch_logs['batch'] = step_num - batch_logs['size'] = batch_size + batch_logs['size'] = 1 callbacks.on_batch_begin(step_num, batch_logs) outs = f(ins) @@ -1083,9 +1092,9 @@ def _fit_loop(self, f, ins, out_labels=None, batch_size=32, if do_validation: val_outs = self._test_loop(val_f, val_ins, - batch_size=batch_size, - steps_per_epoch=steps_per_epoch - verbose=0) + batch_size=batch_size, + steps=validation_steps, + verbose=0) if not isinstance(val_outs, list): val_outs = [val_outs] # Same labels assumed. @@ -1214,43 +1223,52 @@ def _test_loop(self, f, ins, batch_size=32, verbose=0, steps=None): and/or metrics). The attribute `model.metrics_names` will give you the display labels for the scalar outputs. """ - if ins and hasattr(ins[0], 'shape'): - samples = ins[0].shape[0] - else: - # May happen if we are running `predict` without Numpy input data, - # i.e. if all inputs to the models are data tensors - # instead of placeholders. - # In that case we will run `predict` over a single batch. - samples = batch_size - verbose = 2 - outs = [] - if verbose == 1: - progbar = Progbar(target=samples) - batches = _make_batches(samples, batch_size) - index_array = np.arange(samples) - for batch_index, (batch_start, batch_end) in enumerate(batches): - batch_ids = index_array[batch_start:batch_end] - if isinstance(ins[-1], float): - # Do not slice the training phase flag. - ins_batch = _slice_arrays(ins[:-1], batch_ids) + [ins[-1]] - else: - ins_batch = _slice_arrays(ins, batch_ids) + if batch_size is None: + if verbose == 1: + progbar = Progbar(target=steps) + for step_num in range(steps): + f(ins) + if verbose == 1: + progbar.update(step_num) - batch_outs = f(ins_batch) - if isinstance(batch_outs, list): - if batch_index == 0: - for batch_out in enumerate(batch_outs): - outs.append(0.) - for i, batch_out in enumerate(batch_outs): - outs[i] += batch_out * len(batch_ids) + else: + if ins and hasattr(ins[0], 'shape'): + samples = ins[0].shape[0] else: - if batch_index == 0: - outs.append(0.) - outs[0] += batch_outs * len(batch_ids) + # May happen if we are running `predict` without Numpy input data, + # i.e. if all inputs to the models are data tensors + # instead of placeholders. + # In that case we will run `predict` over a single batch. + samples = batch_size + verbose = 2 if verbose == 1: - progbar.update(batch_end) + progbar = Progbar(target=samples) + batches = _make_batches(samples, batch_size) + index_array = np.arange(samples) + for batch_index, (batch_start, batch_end) in enumerate(batches): + batch_ids = index_array[batch_start:batch_end] + if isinstance(ins[-1], float): + # Do not slice the training phase flag. + ins_batch = _slice_arrays(ins[:-1], batch_ids) + [ins[-1]] + else: + ins_batch = _slice_arrays(ins, batch_ids) + + batch_outs = f(ins_batch) + if isinstance(batch_outs, list): + if batch_index == 0: + for batch_out in enumerate(batch_outs): + outs.append(0.) + for i, batch_out in enumerate(batch_outs): + outs[i] += batch_out * len(batch_ids) + else: + if batch_index == 0: + outs.append(0.) + outs[0] += batch_outs * len(batch_ids) + + if verbose == 1: + progbar.update(batch_end) for i in range(len(outs)): outs[i] /= samples if len(outs) == 1: @@ -1328,6 +1346,7 @@ def fit(self, x=None, sample_weight=None, initial_epoch=0, steps_per_epoch=None, + validation_steps=None, **kwargs): """Trains the model for a fixed number of epochs (iterations on a dataset). @@ -1383,6 +1402,9 @@ class indices (integers) to next epoch. The default `None` is equal to the number of unique samples in your dataset divided by the batch size, or 1 if that cannot be determined. + validation_steps: Only relevant if `steps_per_epoch` + is specified. Total number of steps (batches of samples) + to validate before stopping. # Returns A `History` instance. Its `history` attribute contains @@ -1479,7 +1501,8 @@ class indices (integers) to val_f=val_f, val_ins=val_ins, shuffle=shuffle, callback_metrics=callback_metrics, initial_epoch=initial_epoch, - steps_per_epoch=steps_per_epoch) + steps_per_epoch=steps_per_epoch, + validation_steps=validation_steps) def evaluate(self, x, y, batch_size=32, verbose=1, sample_weight=None): """Returns the loss value & metrics values for the model in test mode. diff --git a/tests/keras/engine/test_training.py b/tests/keras/engine/test_training.py index 3ab43f08712..bc52bb6223e 100644 --- a/tests/keras/engine/test_training.py +++ b/tests/keras/engine/test_training.py @@ -141,10 +141,16 @@ def test_model_methods(): out = model.fit({'input_a': input_a_np, 'input_b': input_b_np}, [output_a_np, output_b_np], epochs=1, batch_size=4, validation_split=0.5) + + with pytest.raises(ValueError) as exc: + out = model.fit({'input_a': input_a_np, 'input_b': input_b_np}, + {'dense_1': output_a_np, 'dropout': output_b_np}, + epochs=1, batch_size=None, validation_split=0.5, + steps_per_epoch=1) out = model.fit({'input_a': input_a_np, 'input_b': input_b_np}, {'dense_1': output_a_np, 'dropout': output_b_np}, - epochs=1, batch_size=4, validation_split=0.5, - steps_per_epoch=1) + epochs=1, batch_size=None, validation_split=0.5, + steps_per_epoch=1, validation_steps=1) # test validation data out = model.fit([input_a_np, input_b_np], @@ -735,7 +741,7 @@ def test_model_with_external_loss(): out = model.predict_on_batch(None) # test fit - out = model.fit(None, None, epochs=1, batch_size=10, steps_per_epoch=1) + out = model.fit(None, None, epochs=1, batch_size=None, steps_per_epoch=1) # test evaluate out = model.evaluate(None, None, batch_size=10) From 9f343a11ecfc72722e42e14c85fba65d64683592 Mon Sep 17 00:00:00 2001 From: Andrew Hundt Date: Tue, 1 Aug 2017 01:37:06 -0700 Subject: [PATCH 124/166] training.py pep8 --- keras/engine/training.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/keras/engine/training.py b/keras/engine/training.py index 6c149139542..084ab01d6ea 100644 --- a/keras/engine/training.py +++ b/keras/engine/training.py @@ -1136,8 +1136,8 @@ def _fit_loop(self, f, ins, out_labels=None, batch_size=32, if batch_index == len(batches) - 1: # Last batch. if do_validation: val_outs = self._test_loop(val_f, val_ins, - batch_size=batch_size, - verbose=0) + batch_size=batch_size, + verbose=0) if not isinstance(val_outs, list): val_outs = [val_outs] # Same labels assumed. From d3a917e18e66738ffc27a94b34345c8fff07ac96 Mon Sep 17 00:00:00 2001 From: Andrew Hundt Date: Fri, 4 Aug 2017 19:14:27 -0400 Subject: [PATCH 125/166] mnist_tfrecord.py simpler version --- examples/mnist_tfrecord.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/examples/mnist_tfrecord.py b/examples/mnist_tfrecord.py index 8090c19fe8a..77cc28e3474 100644 --- a/examples/mnist_tfrecord.py +++ b/examples/mnist_tfrecord.py @@ -18,8 +18,6 @@ from keras.layers import Input from keras.layers import Conv2D from keras.layers import MaxPooling2D -from keras.callbacks import EarlyStopping -from keras.callbacks import TensorBoard from keras.objectives import categorical_crossentropy from keras.utils import np_utils from keras.utils.generic_utils import Progbar @@ -91,12 +89,14 @@ def cnn_layers(x_train_input): metrics=['accuracy']) train_model.summary() -tensorboard = TensorBoard() - +coord = tf.train.Coordinator() +threads = tf.train.start_queue_runners(sess, coord) train_model.fit(None, y_train_in, batch_size=None, epochs=epochs, steps_per_epoch=steps_per_epoch) train_model.save_weights('saved_wt.h5') +coord.request_stop() +coord.join(threads) K.clear_session() # Second Session, pure Keras From f6ae5ad3847b0a42c779bf988b7e8cb556f18cd6 Mon Sep 17 00:00:00 2001 From: Andrew Hundt Date: Fri, 4 Aug 2017 19:16:01 -0400 Subject: [PATCH 126/166] mnist_tfrecord.py many fewer epochs --- examples/mnist_tfrecord.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/mnist_tfrecord.py b/examples/mnist_tfrecord.py index 77cc28e3474..f558ae6dbaa 100644 --- a/examples/mnist_tfrecord.py +++ b/examples/mnist_tfrecord.py @@ -54,7 +54,7 @@ def cnn_layers(x_train_input): batch_size = 256 batch_shape = [batch_size, 28, 28, 1] steps_per_epoch = 1000 -epochs = 300 +epochs = 12 classes = 10 capacity = 10000 min_after_dequeue = 3000 From 171e9321d4dc2c43e21cd1d9fa1f1cc52c40b538 Mon Sep 17 00:00:00 2001 From: Andrew Hundt Date: Fri, 4 Aug 2017 19:49:15 -0400 Subject: [PATCH 127/166] mnist_tfrecord.py test at bottom runs --- examples/mnist_tfrecord.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/examples/mnist_tfrecord.py b/examples/mnist_tfrecord.py index f558ae6dbaa..83ee8ee048c 100644 --- a/examples/mnist_tfrecord.py +++ b/examples/mnist_tfrecord.py @@ -54,7 +54,7 @@ def cnn_layers(x_train_input): batch_size = 256 batch_shape = [batch_size, 28, 28, 1] steps_per_epoch = 1000 -epochs = 12 +epochs = 3 classes = 10 capacity = 10000 min_after_dequeue = 3000 @@ -99,10 +99,9 @@ def cnn_layers(x_train_input): coord.join(threads) K.clear_session() -# Second Session, pure Keras -(X_train, y_train), (X_test, y_test) = mnist.load_data() -X_train = X_train[..., np.newaxis] -X_test = X_test[..., np.newaxis] +# Second Session to test loading trained model without tensors +X_test = np.reshape(data.validation.images, [data.validation.images.shape[0], 28, 28, 1]) +y_test = data.validation.labels x_test_inp = Input(batch_shape=(None,) + (X_test.shape[1:])) test_out = cnn_layers(x_test_inp) test_model = Model(inputs=x_test_inp, outputs=test_out) From be423e38946c0b50168691262ab086958044b10f Mon Sep 17 00:00:00 2001 From: Andrew Hundt Date: Fri, 4 Aug 2017 20:09:06 -0400 Subject: [PATCH 128/166] mnist_tfrecord.py fix key missing lines --- examples/mnist_tfrecord.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/examples/mnist_tfrecord.py b/examples/mnist_tfrecord.py index 0229bfe6740..0995c06d286 100644 --- a/examples/mnist_tfrecord.py +++ b/examples/mnist_tfrecord.py @@ -68,6 +68,12 @@ def cnn_layers(x_train_input): enqueue_many=True, num_threads=4) +x_train_batch = tf.cast(x_train_batch, tf.float32) +x_train_batch = tf.reshape(x_train_batch, shape=batch_shape) + +y_train_batch = tf.cast(y_train_batch, tf.int32) +y_train_batch = tf.one_hot(y_train_batch, classes) + x_batch_shape = x_train_batch.get_shape().as_list() y_batch_shape = y_train_batch.get_shape().as_list() From 0b83662811b5b8096ac20c22733bb8132a529b01 Mon Sep 17 00:00:00 2001 From: Andrew Hundt Date: Fri, 4 Aug 2017 20:10:33 -0400 Subject: [PATCH 129/166] mnist_tfrecord.py add coordinator --- examples/mnist_tfrecord.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/examples/mnist_tfrecord.py b/examples/mnist_tfrecord.py index 0995c06d286..db4cf5af4fe 100644 --- a/examples/mnist_tfrecord.py +++ b/examples/mnist_tfrecord.py @@ -89,12 +89,16 @@ def cnn_layers(x_train_input): metrics=['accuracy']) train_model.summary() +coord = tf.train.Coordinator() +threads = tf.train.start_queue_runners(sess, coord) train_model.fit(batch_size=None, epochs=epochs, steps_per_epoch=steps_per_epoch) train_model.save_weights('saved_wt.h5') +coord.request_stop() +coord.join(threads) K.clear_session() # Second Session to test loading trained model without tensors From 4cd6bb31b1ae1dd1819465110a0b58f7cf5e86a4 Mon Sep 17 00:00:00 2001 From: Andrew Hundt Date: Fri, 4 Aug 2017 20:16:49 -0400 Subject: [PATCH 130/166] removed extraneous line --- examples/mnist_tfrecord.py | 1 - 1 file changed, 1 deletion(-) diff --git a/examples/mnist_tfrecord.py b/examples/mnist_tfrecord.py index db4cf5af4fe..d53728d2048 100644 --- a/examples/mnist_tfrecord.py +++ b/examples/mnist_tfrecord.py @@ -79,7 +79,6 @@ def cnn_layers(x_train_input): x_train_input = Input(tensor=x_train_batch, batch_shape=x_batch_shape) x_train_out = cnn_layers(x_train_input) -y_train_in_out = Input(tensor=y_train_batch, batch_shape=y_batch_shape, name='y_labels') cce = categorical_crossentropy(y_train_batch, x_train_out) train_model = Model(inputs=[x_train_input], outputs=[x_train_out]) train_model.add_loss(cce) From 4a15c444057f465d1c416ba745561266f5e0086c Mon Sep 17 00:00:00 2001 From: Andrew Hundt Date: Fri, 4 Aug 2017 21:56:07 -0400 Subject: [PATCH 131/166] replace is_placeholder with is_keras_placeholder to resolve conflict with cntk backend (#7067) https://github.com/fchollet/keras/pull/7067#issuecomment-319446615 --- examples/variational_autoencoder.py | 2 +- examples/variational_autoencoder_deconv.py | 2 +- keras/backend/cntk_backend.py | 2 +- keras/backend/tensorflow_backend.py | 2 +- keras/backend/theano_backend.py | 2 +- keras/engine/topology.py | 12 ++++++------ keras/engine/training.py | 2 +- keras/models.py | 4 ++-- tests/keras/backend/backend_test.py | 4 ++-- tests/keras/engine/test_topology.py | 2 +- 10 files changed, 17 insertions(+), 17 deletions(-) diff --git a/examples/variational_autoencoder.py b/examples/variational_autoencoder.py index fd395b17f45..2b29910915b 100644 --- a/examples/variational_autoencoder.py +++ b/examples/variational_autoencoder.py @@ -45,7 +45,7 @@ def sampling(args): # Custom loss layer class CustomVariationalLayer(Layer): def __init__(self, **kwargs): - self.is_placeholder = True + self.is_keras_placeholder = True super(CustomVariationalLayer, self).__init__(**kwargs) def vae_loss(self, x, x_decoded_mean): diff --git a/examples/variational_autoencoder_deconv.py b/examples/variational_autoencoder_deconv.py index dec59c4cf4e..1214f9cb34a 100644 --- a/examples/variational_autoencoder_deconv.py +++ b/examples/variational_autoencoder_deconv.py @@ -110,7 +110,7 @@ def sampling(args): # Custom loss layer class CustomVariationalLayer(Layer): def __init__(self, **kwargs): - self.is_placeholder = True + self.is_keras_placeholder = True super(CustomVariationalLayer, self).__init__(**kwargs) def vae_loss(self, x, x_decoded_mean_squash): diff --git a/keras/backend/cntk_backend.py b/keras/backend/cntk_backend.py index 21111caa1fd..cfc06191a73 100644 --- a/keras/backend/cntk_backend.py +++ b/keras/backend/cntk_backend.py @@ -251,7 +251,7 @@ def placeholder( name=name) x._keras_shape = shape x._uses_learning_phase = False - x.is_placeholder = True + x.is_keras_placeholder = True return x diff --git a/keras/backend/tensorflow_backend.py b/keras/backend/tensorflow_backend.py index 007d549acad..c08aa2b29bc 100644 --- a/keras/backend/tensorflow_backend.py +++ b/keras/backend/tensorflow_backend.py @@ -431,7 +431,7 @@ def placeholder(shape=None, ndim=None, dtype=None, sparse=False, name=None): x = tf.placeholder(dtype, shape=shape, name=name) x._keras_shape = shape x._uses_learning_phase = False - x.is_placeholder = True + x.is_keras_placeholder = True return x diff --git a/keras/backend/theano_backend.py b/keras/backend/theano_backend.py index 668c438c32e..45219858965 100644 --- a/keras/backend/theano_backend.py +++ b/keras/backend/theano_backend.py @@ -221,7 +221,7 @@ def placeholder(shape=None, ndim=None, dtype=None, sparse=False, name=None): x = T.TensorType(dtype, broadcast)(name) x._keras_shape = shape x._uses_learning_phase = False - x.is_placeholder = True + x.is_keras_placeholder = True return x diff --git a/keras/engine/topology.py b/keras/engine/topology.py index 4afd1d7cb6c..1eb40d16f7a 100644 --- a/keras/engine/topology.py +++ b/keras/engine/topology.py @@ -159,7 +159,7 @@ def __init__(self, outbound_layer, self.arguments = arguments # Indicates if the node represents a placeholder variable - self.is_placeholder = is_placeholder + self.is_keras_placeholder = is_placeholder # Add nodes to all layers involved. for layer in inbound_layers: if layer is not None: @@ -1333,19 +1333,19 @@ def __init__(self, input_shape=None, batch_size=None, self.dtype = dtype if input_tensor is None: - self.is_placeholder = True + self.is_keras_placeholder = True input_tensor = K.placeholder(shape=batch_input_shape, dtype=dtype, sparse=self.sparse, name=self.name) else: - self.is_placeholder = False + self.is_keras_placeholder = False input_tensor._keras_shape = batch_input_shape # Create an input node to add to self.outbound_node # and set output_tensors' _keras_history. input_tensor._uses_learning_phase = False input_tensor._keras_history = (self, 0, 0) - input_tensor.is_placeholder = self.is_placeholder + input_tensor.is_keras_placeholder = self.is_keras_placeholder Node(self, inbound_layers=[], node_indices=[], @@ -1356,7 +1356,7 @@ def __init__(self, input_shape=None, batch_size=None, output_masks=[None], input_shapes=[batch_input_shape], output_shapes=[batch_input_shape], - is_placeholder=self.is_placeholder) + is_placeholder=self.is_keras_placeholder) def get_config(self): config = {'batch_input_shape': self.batch_input_shape, @@ -1630,7 +1630,7 @@ def __init__(self, inputs, outputs, name=None): i, layer.__class__.__name__)) self.input_names.append(layer.name) - if getattr(layer, 'is_placeholder', False) is True: + if getattr(layer, 'is_keras_placeholder', False) is True: self._feed_input_names.append(layer.name) self._feed_inputs.append(layer.input) self._feed_input_shapes.append(self.inputs[i]._keras_shape) diff --git a/keras/engine/training.py b/keras/engine/training.py index 27bcc310658..f860fd7c3ab 100644 --- a/keras/engine/training.py +++ b/keras/engine/training.py @@ -60,7 +60,7 @@ def _standardize_input_data(data, names, shapes=None, return [] if data is None: return [None for _ in range(len(names))] - num_input_tensors = sum(getattr(x, 'is_placeholder', False) is False for x in input_tensors) + num_input_tensors = sum(getattr(x, 'is_keras_placeholder', False) is False for x in input_tensors) expected_names = len(names) - num_input_tensors if isinstance(data, dict): arrays = [] diff --git a/keras/models.py b/keras/models.py index 0e635f352ef..52bf45d59c7 100644 --- a/keras/models.py +++ b/keras/models.py @@ -451,7 +451,7 @@ def add(self, layer): 'For multi-output layers, ' 'use the functional API.') - layer.inbound_nodes[0].output_tensors[0].is_placeholder = True + layer.inbound_nodes[0].output_tensors[0].is_keras_placeholder = True self.outputs = [layer.inbound_nodes[0].output_tensors[0]] self.inputs = topology.get_source_inputs(self.outputs[0]) @@ -475,7 +475,7 @@ def add(self, layer): 'should have a single output tensor. ' 'For multi-output layers, ' 'use the functional API.') - output_tensor.is_placeholder = True + output_tensor.is_keras_placeholder = True self.outputs = [output_tensor] # update self.inbound_nodes self.inbound_nodes[0].output_tensors = self.outputs diff --git a/tests/keras/backend/backend_test.py b/tests/keras/backend/backend_test.py index eda66b1f4ad..5270f54bc52 100644 --- a/tests/keras/backend/backend_test.py +++ b/tests/keras/backend/backend_test.py @@ -159,8 +159,8 @@ def test_is_keras_tensor(self): assert k.is_keras_tensor(keras_var) is False keras_placeholder = k.placeholder(shape=(2, 4, 5)) assert k.is_keras_tensor(keras_placeholder) is False - assert getattr(keras_placeholder, 'is_placeholder', False) is True - assert getattr(np_var, 'is_placeholder', False) is False + assert getattr(keras_placeholder, 'is_keras_placeholder', False) is True + assert getattr(np_var, 'is_keras_placeholder', False) is False def test_set_learning_phase(self): # not supported learning_phase diff --git a/tests/keras/engine/test_topology.py b/tests/keras/engine/test_topology.py index b013f425a15..fb0bd4ad624 100644 --- a/tests/keras/engine/test_topology.py +++ b/tests/keras/engine/test_topology.py @@ -36,7 +36,7 @@ def test_get_losses_for(): @keras_test def test_is_placeholder(): a = Input(shape=(2,)) - assert getattr(a, 'is_placeholder', False) + assert getattr(a, 'is_keras_placeholder', False) @keras_test From 8ad38445358d188c4e62126ab4049769a44ec261 Mon Sep 17 00:00:00 2001 From: Andrew Hundt Date: Fri, 4 Aug 2017 22:06:48 -0400 Subject: [PATCH 132/166] (#6928) replace _is_placeholder with is_keras_placeholder to match (#7067) --- examples/variational_autoencoder.py | 2 +- examples/variational_autoencoder_deconv.py | 2 +- keras/backend/cntk_backend.py | 2 +- keras/backend/common.py | 2 +- keras/backend/tensorflow_backend.py | 2 +- keras/backend/theano_backend.py | 2 +- keras/engine/topology.py | 10 +++++----- keras/models.py | 4 ++-- 8 files changed, 13 insertions(+), 13 deletions(-) diff --git a/examples/variational_autoencoder.py b/examples/variational_autoencoder.py index ba313f1c69b..2b29910915b 100644 --- a/examples/variational_autoencoder.py +++ b/examples/variational_autoencoder.py @@ -45,7 +45,7 @@ def sampling(args): # Custom loss layer class CustomVariationalLayer(Layer): def __init__(self, **kwargs): - self._is_placeholder = True + self.is_keras_placeholder = True super(CustomVariationalLayer, self).__init__(**kwargs) def vae_loss(self, x, x_decoded_mean): diff --git a/examples/variational_autoencoder_deconv.py b/examples/variational_autoencoder_deconv.py index 7e1afeb26b6..1214f9cb34a 100644 --- a/examples/variational_autoencoder_deconv.py +++ b/examples/variational_autoencoder_deconv.py @@ -110,7 +110,7 @@ def sampling(args): # Custom loss layer class CustomVariationalLayer(Layer): def __init__(self, **kwargs): - self._is_placeholder = True + self.is_keras_placeholder = True super(CustomVariationalLayer, self).__init__(**kwargs) def vae_loss(self, x, x_decoded_mean_squash): diff --git a/keras/backend/cntk_backend.py b/keras/backend/cntk_backend.py index 020761147a5..7b830777dab 100644 --- a/keras/backend/cntk_backend.py +++ b/keras/backend/cntk_backend.py @@ -252,7 +252,7 @@ def placeholder( name=name) x._keras_shape = shape x._uses_learning_phase = False - x._is_placeholder = True + x.is_keras_placeholder = True return x diff --git a/keras/backend/common.py b/keras/backend/common.py index c80bc7fa403..09cf86d1039 100644 --- a/keras/backend/common.py +++ b/keras/backend/common.py @@ -124,7 +124,7 @@ def is_placeholder(tensor): ``` """ try: - return tensor._is_placeholder + return tensor.is_keras_placeholder except AttributeError: return False diff --git a/keras/backend/tensorflow_backend.py b/keras/backend/tensorflow_backend.py index 67a906a29aa..239ed3622d9 100644 --- a/keras/backend/tensorflow_backend.py +++ b/keras/backend/tensorflow_backend.py @@ -437,7 +437,7 @@ def placeholder(shape=None, ndim=None, dtype=None, sparse=False, name=None): x = tf.placeholder(dtype, shape=shape, name=name) x._keras_shape = shape x._uses_learning_phase = False - x._is_placeholder = True + x.is_keras_placeholder = True return x diff --git a/keras/backend/theano_backend.py b/keras/backend/theano_backend.py index dfa2a746391..34cdeca00f5 100644 --- a/keras/backend/theano_backend.py +++ b/keras/backend/theano_backend.py @@ -226,7 +226,7 @@ def placeholder(shape=None, ndim=None, dtype=None, sparse=False, name=None): x = T.TensorType(dtype, broadcast)(name) x._keras_shape = shape x._uses_learning_phase = False - x._is_placeholder = True + x.is_keras_placeholder = True return x diff --git a/keras/engine/topology.py b/keras/engine/topology.py index b8434a38c0a..f4000f50683 100644 --- a/keras/engine/topology.py +++ b/keras/engine/topology.py @@ -159,7 +159,7 @@ def __init__(self, outbound_layer, self.arguments = arguments # Indicates if the node represents a placeholder variable - self._is_placeholder = is_placeholder + self.is_keras_placeholder = is_placeholder # Add nodes to all layers involved. for layer in inbound_layers: if layer is not None: @@ -1333,19 +1333,19 @@ def __init__(self, input_shape=None, batch_size=None, self.dtype = dtype if input_tensor is None: - self._is_placeholder = True + self.is_keras_placeholder = True input_tensor = K.placeholder(shape=batch_input_shape, dtype=dtype, sparse=self.sparse, name=self.name) else: - self._is_placeholder = False + self.is_keras_placeholder = False input_tensor._keras_shape = batch_input_shape # Create an input node to add to self.outbound_node # and set output_tensors' _keras_history. input_tensor._uses_learning_phase = False input_tensor._keras_history = (self, 0, 0) - input_tensor._is_placeholder = self._is_placeholder + input_tensor.is_keras_placeholder = self.is_keras_placeholder Node(self, inbound_layers=[], node_indices=[], @@ -1356,7 +1356,7 @@ def __init__(self, input_shape=None, batch_size=None, output_masks=[None], input_shapes=[batch_input_shape], output_shapes=[batch_input_shape], - is_placeholder=self._is_placeholder) + is_placeholder=self.is_keras_placeholder) def get_config(self): config = {'batch_input_shape': self.batch_input_shape, diff --git a/keras/models.py b/keras/models.py index f19b982611f..d7b0f0c9b5c 100644 --- a/keras/models.py +++ b/keras/models.py @@ -451,7 +451,7 @@ def add(self, layer): 'For multi-output layers, ' 'use the functional API.') - layer.inbound_nodes[0].output_tensors[0]._is_placeholder = True + layer.inbound_nodes[0].output_tensors[0].is_keras_placeholder = True self.outputs = [layer.inbound_nodes[0].output_tensors[0]] self.inputs = topology.get_source_inputs(self.outputs[0]) @@ -475,7 +475,7 @@ def add(self, layer): 'should have a single output tensor. ' 'For multi-output layers, ' 'use the functional API.') - output_tensor._is_placeholder = True + output_tensor.is_keras_placeholder = True self.outputs = [output_tensor] # update self.inbound_nodes self.inbound_nodes[0].output_tensors = self.outputs From 6b2a9755fe817cf73c116f378a66da81946226ec Mon Sep 17 00:00:00 2001 From: Andrew Hundt Date: Sat, 5 Aug 2017 00:04:00 -0400 Subject: [PATCH 133/166] (#6928) remove is_placeholder to match (#7113) --- keras/backend/__init__.py | 1 - keras/backend/cntk_backend.py | 1 - keras/backend/common.py | 23 ----------------------- tests/keras/engine/test_topology.py | 2 +- 4 files changed, 1 insertion(+), 26 deletions(-) diff --git a/keras/backend/__init__.py b/keras/backend/__init__.py index 00e6d52fbe8..2e7208cf5af 100644 --- a/keras/backend/__init__.py +++ b/keras/backend/__init__.py @@ -10,7 +10,6 @@ from .common import cast_to_floatx from .common import image_data_format from .common import set_image_data_format -from .common import is_placeholder # Obtain Keras base dir path: either ~/.keras or /tmp. _keras_base_dir = os.path.expanduser('~') diff --git a/keras/backend/cntk_backend.py b/keras/backend/cntk_backend.py index 10e9a858bb8..9d1e0c93055 100644 --- a/keras/backend/cntk_backend.py +++ b/keras/backend/cntk_backend.py @@ -2,7 +2,6 @@ import cntk as C import numpy as np from .common import floatx, epsilon, image_dim_ordering, image_data_format -from .common import is_placeholder from collections import defaultdict from contextlib import contextmanager import warnings diff --git a/keras/backend/common.py b/keras/backend/common.py index 09cf86d1039..9970c3ea0db 100644 --- a/keras/backend/common.py +++ b/keras/backend/common.py @@ -106,29 +106,6 @@ def cast_to_floatx(x): return np.asarray(x, dtype=_FLOATX) -def is_placeholder(tensor): - """Returns whether a tensor is a placeholder. - - # Arguments - tensor: A tensor instance. - - # Returns - A boolean. - - # Example - ```python - >>> from keras import backend as K - >>> a = K.placeholder((2, 2), sparse=False) - >>> print(K.is_placeholder(a)) - True - ``` - """ - try: - return tensor.is_keras_placeholder - except AttributeError: - return False - - def image_data_format(): """Returns the default image data format convention ('channels_first' or 'channels_last'). diff --git a/tests/keras/engine/test_topology.py b/tests/keras/engine/test_topology.py index c9c88695234..112a855ca9d 100644 --- a/tests/keras/engine/test_topology.py +++ b/tests/keras/engine/test_topology.py @@ -34,7 +34,7 @@ def test_get_losses_for(): @keras_test -def test_is_placeholder(): +def test_is_keras_placeholder(): a = Input(shape=(2,)) assert getattr(a, 'is_keras_placeholder', False) From 48799e8189e5e784c9f6aaaded3df099e10a83ff Mon Sep 17 00:00:00 2001 From: Andrew Hundt Date: Mon, 7 Aug 2017 17:07:51 -0400 Subject: [PATCH 134/166] mnist_tfrecord.py and training.py clean up based on review (#7113) --- examples/mnist_tfrecord.py | 51 ++++++++++++++++---------------------- keras/engine/training.py | 23 ++++++++++++++--- 2 files changed, 41 insertions(+), 33 deletions(-) diff --git a/examples/mnist_tfrecord.py b/examples/mnist_tfrecord.py index d53728d2048..cc07c80e282 100644 --- a/examples/mnist_tfrecord.py +++ b/examples/mnist_tfrecord.py @@ -12,18 +12,10 @@ import tensorflow as tf from keras import backend as K from keras.models import Model -from keras.layers import Dense -from keras.layers import Dropout -from keras.layers import Flatten -from keras.layers import Input -from keras.layers import Conv2D -from keras.layers import MaxPooling2D -from keras.objectives import categorical_crossentropy +from keras import layers +from keras import objectives from keras.utils import np_utils -from keras.utils.generic_utils import Progbar -from keras import callbacks as cbks -from keras import optimizers, objectives -from keras import metrics as metrics_module +from keras import objectives from tensorflow.contrib.learn.python.learn.datasets.mnist import load_mnist @@ -35,16 +27,17 @@ def cnn_layers(x_train_input): - x = Conv2D(32, (3, 3), activation='relu', padding='valid')(x_train_input) - x = Conv2D(64, (3, 3), activation='relu')(x) - x = MaxPooling2D(pool_size=(2, 2))(x) - x = Dropout(0.25)(x) - x = Flatten()(x) - x = Dense(128, activation='relu')(x) - x = Dropout(0.5)(x) - x_train_out = Dense(classes, - activation='softmax', - name='x_train_out')(x) + x = layers.Conv2D(32, (3, 3), + activation='relu', padding='valid')(x_train_input) + x = layers.Conv2D(64, (3, 3), activation='relu')(x) + x = layers.MaxPooling2D(pool_size=(2, 2))(x) + x = layers.Dropout(0.25)(x) + x = layers.Flatten()(x) + x = layers.Dense(128, activation='relu')(x) + x = layers.Dropout(0.5)(x) + x_train_out = layers.Dense(classes, + activation='softmax', + name='x_train_out')(x) return x_train_out @@ -77,10 +70,11 @@ def cnn_layers(x_train_input): x_batch_shape = x_train_batch.get_shape().as_list() y_batch_shape = y_train_batch.get_shape().as_list() -x_train_input = Input(tensor=x_train_batch, batch_shape=x_batch_shape) +x_train_input = layers.Input(tensor=x_train_batch, batch_shape=x_batch_shape) x_train_out = cnn_layers(x_train_input) -cce = categorical_crossentropy(y_train_batch, x_train_out) -train_model = Model(inputs=[x_train_input], outputs=[x_train_out]) +train_model = Model(inputs=x_train_input, outputs=x_train_out) + +cce = objectives.categorical_crossentropy(y_train_batch, x_train_out) train_model.add_loss(cce) train_model.compile(optimizer='rmsprop', @@ -90,8 +84,7 @@ def cnn_layers(x_train_input): coord = tf.train.Coordinator() threads = tf.train.start_queue_runners(sess, coord) -train_model.fit(batch_size=None, - epochs=epochs, +train_model.fit(epochs=epochs, steps_per_epoch=steps_per_epoch) train_model.save_weights('saved_wt.h5') @@ -101,9 +94,9 @@ def cnn_layers(x_train_input): K.clear_session() # Second Session to test loading trained model without tensors -X_test = np.reshape(data.validation.images, [data.validation.images.shape[0], 28, 28, 1]) +x_test = np.reshape(data.validation.images, [data.validation.images.shape[0], 28, 28, 1]) y_test = data.validation.labels -x_test_inp = Input(batch_shape=(None,) + (X_test.shape[1:])) +x_test_inp = layers.Input(batch_shape=(None,) + (x_test.shape[1:])) test_out = cnn_layers(x_test_inp) test_model = Model(inputs=x_test_inp, outputs=test_out) @@ -111,5 +104,5 @@ def cnn_layers(x_train_input): test_model.compile(optimizer='rmsprop', loss='categorical_crossentropy', metrics=['accuracy']) test_model.summary() -loss, acc = test_model.evaluate(X_test, np_utils.to_categorical(y_test), classes) +loss, acc = test_model.evaluate(x_test, np_utils.to_categorical(y_test), classes) print('\nTest accuracy: {0}'.format(acc)) diff --git a/keras/engine/training.py b/keras/engine/training.py index 084ab01d6ea..bd3fd29b5a5 100644 --- a/keras/engine/training.py +++ b/keras/engine/training.py @@ -976,6 +976,16 @@ def _make_predict_function(self): **kwargs) def _check_num_samples(self, ins, batch_size=None, steps=None, steps_name='steps'): + """Determine the number of samples or steps for training and evaluation. + + # Arguments + ins: list of tensors to be fed to the keras function. + batch_size: integer batch size or None if unknown. + steps: Total number of steps (batches of samples) + before declaring _predict_loop finished. + Ignored with the default value of `None`. + steps_name: The public API's parameter name for `steps`. + """ if steps is not None: num_samples = None if batch_size is not None: @@ -1001,7 +1011,7 @@ def _fit_loop(self, f, ins, out_labels=None, batch_size=32, ins: list of tensors to be fed to `f` out_labels: list of strings, display names of the outputs of `f` - batch_size: integer batch size + batch_size: integer batch size or None if unknown. epochs: number of times to iterate over the data verbose: verbosity mode, 0, 1 or 2 callbacks: list of callbacks to be called during training @@ -1024,8 +1034,8 @@ def _fit_loop(self, f, ins, out_labels=None, batch_size=32, do_validation = False if val_f and val_ins: do_validation = True - if steps_per_epoch and not validation_steps: - raise ValueError('When using `steps_per_epoch` and validation data, ' + if steps_per_epoch and not validation_steps and not hasattr(ins[0], 'shape'): + raise ValueError('When `steps_per_epoch` validation data has no `shape` attribute, ' 'you must specify a value for `validation_steps`.') if verbose and ins and hasattr(ins[0], 'shape'): print('Train on %d samples, validate on %d samples' % @@ -1335,7 +1345,7 @@ def _get_deduped_metrics_names(self): def fit(self, x=None, y=None, - batch_size=32, + batch_size=None, epochs=1, verbose=1, callbacks=None, @@ -1362,6 +1372,8 @@ def fit(self, x=None, you can also pass a dictionary mapping output names to Numpy arrays. batch_size: integer. Number of samples per gradient update. + Defaults to 32 if using per-sample training and no batch + size is specified. epochs: integer, the number of times to iterate over the training data arrays. verbose: 0, 1, or 2. Verbosity mode. @@ -1414,6 +1426,9 @@ class indices (integers) to ValueError: In case of mismatch between the provided input data and what the model expects. """ + # backwards compatibility + if batch_size is None and steps_per_epoch is None: + batch_size = 32 # Legacy support if 'nb_epoch' in kwargs: warnings.warn('The `nb_epoch` argument in `fit` ' From 2bf969566299157baf2995487d1054a70333d3b9 Mon Sep 17 00:00:00 2001 From: Andrew Hundt Date: Mon, 7 Aug 2017 20:54:18 -0400 Subject: [PATCH 135/166] mnist_tfrecord.py extended description --- examples/mnist_tfrecord.py | 51 ++++++++++++++++++++++++++++++++------ 1 file changed, 44 insertions(+), 7 deletions(-) diff --git a/examples/mnist_tfrecord.py b/examples/mnist_tfrecord.py index cc07c80e282..4d53ce90173 100644 --- a/examples/mnist_tfrecord.py +++ b/examples/mnist_tfrecord.py @@ -1,4 +1,37 @@ -'''MNIST dataset with TensorFlow TFRecords. +'''MNIST dataset with TFRecords, the standard TensorFlow data format. + +TFRecord is a data format supported throughout TensorFlow. +This example demonstrates how to load TFRecord data using +Input Tensors. Input Tensors differ from the normal Keras +workflow because instead of fitting to data loaded into a +a numpy array, data is supplied via a special tensor that +reads data from nodes that are wired directly into model +graph with the `Input(tensor=input_tensor)` parameter. + +There are several advantages to using Input Tensors. +First, if a dataset is already in TFRecord format you +can load and train on that data directly in Keras. +Second, extended backend API capabilities such as TensorFlow +data augmentation is easy to integrate directly into your +Keras training scripts via input tensors. +Third, TensorFlow implements several data APIs for +TFRecords, some of which provide significantly faster +training performance than numpy arrays can provide because +they run via the C++ backend. Please note that this +example is tailored for brevity and clarity and not +to demonstrate performance or augmentation capabilities. + +Input Tensors also have important disadvantages. In +particular, Input Tensors are fixed at model construction +because rewiring networks is not yet supported. +For this reason, changing the data input source means +model weights must be saved and the model rebuilt +from scratch to connect the new input data. +validation cannot currently be performed as training +progresses, and must be performed after training completes. +This example demonstrates how to train with input +tensors, save the model weights, and then evaluate the +model using the numpy based Keras API. Gets to 99.25% test accuracy after 12 epochs (there is still a lot of margin for parameter tuning). @@ -40,16 +73,20 @@ def cnn_layers(x_train_input): name='x_train_out')(x) return x_train_out +sess = K.get_session() -sess = tf.Session() -K.set_session(sess) - -batch_size = 256 +batch_size = 128 batch_shape = [batch_size, 28, 28, 1] -steps_per_epoch = 1000 -epochs = 3 +steps_per_epoch = 79 +epochs = 78 classes = 10 + +# The capacity variable controls the maximum size +# to which prefetching is allowed to grow the queues. capacity = 10000 + +# min_after_dequeue is the minimum number elements in the queue +# after a dequeue, which ensures sufficient mixing of elements. min_after_dequeue = 3000 data = load_mnist() From f1af6669231b0c71dcfce350221191ccdce5d6a7 Mon Sep 17 00:00:00 2001 From: Andrew Hundt Date: Mon, 7 Aug 2017 21:40:52 -0400 Subject: [PATCH 136/166] training.py fix test error --- keras/engine/training.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/keras/engine/training.py b/keras/engine/training.py index bd3fd29b5a5..b1fdc958fab 100644 --- a/keras/engine/training.py +++ b/keras/engine/training.py @@ -1234,7 +1234,7 @@ def _test_loop(self, f, ins, batch_size=32, verbose=0, steps=None): the display labels for the scalar outputs. """ outs = [] - if batch_size is None: + if batch_size is None and steps is not None: if verbose == 1: progbar = Progbar(target=steps) for step_num in range(steps): From 351e56b8844eca961d67ea4da72b618030c9237f Mon Sep 17 00:00:00 2001 From: Andrew Hundt Date: Mon, 7 Aug 2017 22:01:29 -0400 Subject: [PATCH 137/166] mnist_tfrecord.py and training.py fixed review comments, docs, and error messages (#7113) --- examples/mnist_tfrecord.py | 29 +++++++++++++++++++++-------- keras/engine/training.py | 20 +++++++++++--------- 2 files changed, 32 insertions(+), 17 deletions(-) diff --git a/examples/mnist_tfrecord.py b/examples/mnist_tfrecord.py index 4d53ce90173..b8fc694444a 100644 --- a/examples/mnist_tfrecord.py +++ b/examples/mnist_tfrecord.py @@ -33,7 +33,7 @@ tensors, save the model weights, and then evaluate the model using the numpy based Keras API. -Gets to 99.25% test accuracy after 12 epochs +Gets to 99.1% test accuracy after 78 epochs (there is still a lot of margin for parameter tuning). ''' import os @@ -76,27 +76,38 @@ def cnn_layers(x_train_input): sess = K.get_session() batch_size = 128 -batch_shape = [batch_size, 28, 28, 1] -steps_per_epoch = 79 +batch_shape = (batch_size, 28, 28, 1) +steps_per_epoch = 469 epochs = 78 classes = 10 -# The capacity variable controls the maximum size -# to which prefetching is allowed to grow the queues. +# The capacity variable controls the maximum queue size +# allowed when prefetching data for training. capacity = 10000 # min_after_dequeue is the minimum number elements in the queue # after a dequeue, which ensures sufficient mixing of elements. min_after_dequeue = 3000 +# If `enqueue_many` is `False`, `tensors` is assumed to represent a +# single example. An input tensor with shape `[x, y, z]` will be output +# as a tensor with shape `[batch_size, x, y, z]`. +# +# If `enqueue_many` is `True`, `tensors` is assumed to represent a +# batch of examples, where the first dimension is indexed by example, +# and all members of `tensors` should have the same size in the +# first dimension. If an input tensor has shape `[*, x, y, z]`, the +# output will have shape `[batch_size, x, y, z]`. +enqueue_many = True + data = load_mnist() x_train_batch, y_train_batch = tf.train.shuffle_batch( tensors=[data.train.images, data.train.labels.astype(np.int32)], batch_size=batch_size, capacity=capacity, min_after_dequeue=min_after_dequeue, - enqueue_many=True, - num_threads=4) + enqueue_many=enqueue_many, + num_threads=8) x_train_batch = tf.cast(x_train_batch, tf.float32) x_train_batch = tf.reshape(x_train_batch, shape=batch_shape) @@ -114,6 +125,8 @@ def cnn_layers(x_train_input): cce = objectives.categorical_crossentropy(y_train_batch, x_train_out) train_model.add_loss(cce) +# Do not pass the loss directly to model.compile() +# because it is not yet supported for Input Tensors. train_model.compile(optimizer='rmsprop', loss=None, metrics=['accuracy']) @@ -131,7 +144,7 @@ def cnn_layers(x_train_input): K.clear_session() # Second Session to test loading trained model without tensors -x_test = np.reshape(data.validation.images, [data.validation.images.shape[0], 28, 28, 1]) +x_test = np.reshape(data.validation.images, (data.validation.images.shape[0], 28, 28, 1)) y_test = data.validation.labels x_test_inp = layers.Input(batch_shape=(None,) + (x_test.shape[1:])) test_out = cnn_layers(x_test_inp) diff --git a/keras/engine/training.py b/keras/engine/training.py index b1fdc958fab..5fa1af6e808 100644 --- a/keras/engine/training.py +++ b/keras/engine/training.py @@ -979,7 +979,7 @@ def _check_num_samples(self, ins, batch_size=None, steps=None, steps_name='steps """Determine the number of samples or steps for training and evaluation. # Arguments - ins: list of tensors to be fed to the keras function. + ins: list of tensors to be fed to the Keras function. batch_size: integer batch size or None if unknown. steps: Total number of steps (batches of samples) before declaring _predict_loop finished. @@ -989,15 +989,16 @@ def _check_num_samples(self, ins, batch_size=None, steps=None, steps_name='steps if steps is not None: num_samples = None if batch_size is not None: - raise ValueError('If steps_per_epoch is set the batch_size must be None.') + raise ValueError('If ' + steps_name + + ' is set the batch_size must be None.') elif ins and hasattr(ins[0], 'shape'): num_samples = ins[0].shape[0] else: - raise ValueError('The input data should have shape or' - ' please specify ' + steps_name + '.') + raise ValueError('The input data should have shape or ' + 'please specify ' + steps_name + '.') return num_samples - def _fit_loop(self, f, ins, out_labels=None, batch_size=32, + def _fit_loop(self, f, ins, out_labels=None, batch_size=None, epochs=100, verbose=1, callbacks=None, val_f=None, val_ins=None, shuffle=True, callback_metrics=None, initial_epoch=0, @@ -1372,7 +1373,7 @@ def fit(self, x=None, you can also pass a dictionary mapping output names to Numpy arrays. batch_size: integer. Number of samples per gradient update. - Defaults to 32 if using per-sample training and no batch + Defaults to 32 if training numpy arrays and no batch size is specified. epochs: integer, the number of times to iterate over the training data arrays. @@ -1411,9 +1412,10 @@ class indices (integers) to (useful for resuming a previous training run) steps_per_epoch: Total number of steps (batches of samples) before declaring one epoch finished and starting the - next epoch. The default `None` is equal to the number - of unique samples in your dataset divided by the batch - size, or 1 if that cannot be determined. + next epoch. When training with Input Tensors such as + TensorFlow data tensors, the default `None` is equal to + the number of unique samples in your dataset divided by + the batch size, or 1 if that cannot be determined. validation_steps: Only relevant if `steps_per_epoch` is specified. Total number of steps (batches of samples) to validate before stopping. From d755b8040a6de8e3a95abfd17b2e7c0ce99572b2 Mon Sep 17 00:00:00 2001 From: Andrew Hundt Date: Tue, 8 Aug 2017 01:36:45 -0400 Subject: [PATCH 138/166] training.py fix unit test error for steps_per_epoch --- examples/mnist_tfrecord.py | 4 ++-- keras/engine/training.py | 14 ++++++++++---- tests/keras/engine/test_training.py | 10 ++++------ 3 files changed, 16 insertions(+), 12 deletions(-) diff --git a/examples/mnist_tfrecord.py b/examples/mnist_tfrecord.py index b8fc694444a..342007fa96e 100644 --- a/examples/mnist_tfrecord.py +++ b/examples/mnist_tfrecord.py @@ -50,7 +50,7 @@ from keras.utils import np_utils from keras import objectives -from tensorflow.contrib.learn.python.learn.datasets.mnist import load_mnist +from tensorflow.contrib.learn.python.learn.datasets import mnist if K.backend() != 'tensorflow': raise RuntimeError('This example can only run with the ' @@ -100,7 +100,7 @@ def cnn_layers(x_train_input): # output will have shape `[batch_size, x, y, z]`. enqueue_many = True -data = load_mnist() +data = mnist.load_mnist() x_train_batch, y_train_batch = tf.train.shuffle_batch( tensors=[data.train.images, data.train.labels.astype(np.int32)], batch_size=batch_size, diff --git a/keras/engine/training.py b/keras/engine/training.py index 5fa1af6e808..1af93a4e997 100644 --- a/keras/engine/training.py +++ b/keras/engine/training.py @@ -1035,9 +1035,15 @@ def _fit_loop(self, f, ins, out_labels=None, batch_size=None, do_validation = False if val_f and val_ins: do_validation = True - if steps_per_epoch and not validation_steps and not hasattr(ins[0], 'shape'): - raise ValueError('When `steps_per_epoch` validation data has no `shape` attribute, ' - 'you must specify a value for `validation_steps`.') + if steps_per_epoch and not validation_steps: + if hasattr(ins[0], 'shape'): + validation_steps = steps_per_epoch + else: + raise ValueError('When `steps_per_epoch` validation ' + 'data has no `shape` attribute, ' + 'you must specify a value for ' + '`validation_steps`.') + if verbose and ins and hasattr(ins[0], 'shape'): print('Train on %d samples, validate on %d samples' % (ins[0].shape[0], val_ins[0].shape[0])) @@ -1216,7 +1222,7 @@ def _predict_loop(self, f, ins, batch_size=32, verbose=0): return outs[0] return outs - def _test_loop(self, f, ins, batch_size=32, verbose=0, steps=None): + def _test_loop(self, f, ins, batch_size=None, verbose=0, steps=None): """Abstract method to loop over some data in batches. # Arguments diff --git a/tests/keras/engine/test_training.py b/tests/keras/engine/test_training.py index bc52bb6223e..a17187f5a31 100644 --- a/tests/keras/engine/test_training.py +++ b/tests/keras/engine/test_training.py @@ -141,12 +141,10 @@ def test_model_methods(): out = model.fit({'input_a': input_a_np, 'input_b': input_b_np}, [output_a_np, output_b_np], epochs=1, batch_size=4, validation_split=0.5) - - with pytest.raises(ValueError) as exc: - out = model.fit({'input_a': input_a_np, 'input_b': input_b_np}, - {'dense_1': output_a_np, 'dropout': output_b_np}, - epochs=1, batch_size=None, validation_split=0.5, - steps_per_epoch=1) + out = model.fit({'input_a': input_a_np, 'input_b': input_b_np}, + {'dense_1': output_a_np, 'dropout': output_b_np}, + epochs=1, batch_size=None, validation_split=0.5, + steps_per_epoch=1) out = model.fit({'input_a': input_a_np, 'input_b': input_b_np}, {'dense_1': output_a_np, 'dropout': output_b_np}, epochs=1, batch_size=None, validation_split=0.5, From d88987da49829fc4d77adef4b8c4cb57e7bf39c9 Mon Sep 17 00:00:00 2001 From: Andrew Hundt Date: Tue, 8 Aug 2017 13:45:40 -0400 Subject: [PATCH 139/166] add Model.evaluate(steps) and correct docs according to review. (#7113) --- keras/engine/training.py | 24 +++++++++++++++++------- 1 file changed, 17 insertions(+), 7 deletions(-) diff --git a/keras/engine/training.py b/keras/engine/training.py index 1af93a4e997..67fbc3acf9a 100644 --- a/keras/engine/training.py +++ b/keras/engine/training.py @@ -1228,7 +1228,7 @@ def _test_loop(self, f, ins, batch_size=None, verbose=0, steps=None): # Arguments f: Keras function returning a list of tensors. ins: list of tensors to be fed to `f`. - batch_size: integer batch size. + batch_size: integer batch size or `None`. verbose: verbosity mode. steps: Total number of steps (batches of samples) before declaring predictions finished. @@ -1378,9 +1378,9 @@ def fit(self, x=None, If all outputs in the model are named, you can also pass a dictionary mapping output names to Numpy arrays. - batch_size: integer. Number of samples per gradient update. + batch_size: integer or `None`. Number of samples per gradient update. Defaults to 32 if training numpy arrays and no batch - size is specified. + size is specified. Defaults to `None` when `steps_per_epoch` is set. epochs: integer, the number of times to iterate over the training data arrays. verbose: 0, 1, or 2. Verbosity mode. @@ -1400,7 +1400,8 @@ def fit(self, x=None, This could be a tuple (x_val, y_val) or a tuple (x_val, y_val, val_sample_weights). shuffle: boolean, whether to shuffle the training data - before each epoch. + before each epoch. Has no effect when `steps_per_epoch` + is not `None`. class_weight: optional dictionary mapping class indices (integers) to a weight (float) to apply to the model's loss for the samples @@ -1527,7 +1528,7 @@ class indices (integers) to steps_per_epoch=steps_per_epoch, validation_steps=validation_steps) - def evaluate(self, x, y, batch_size=32, verbose=1, sample_weight=None): + def evaluate(self, x, y, batch_size=None, verbose=1, sample_weight=None, steps=None): """Returns the loss value & metrics values for the model in test mode. Computation is done in batches. @@ -1543,10 +1544,15 @@ def evaluate(self, x, y, batch_size=32, verbose=1, sample_weight=None): If all outputs in the model are named, you can also pass a dictionary mapping output names to Numpy arrays. - batch_size: integer. Number of samples per gradient update. + batch_size: integer or `None`. Number of samples per evaluation. + Defaults to 32 if evaluating numpy arrays and no batch + size is specified. Defaults to `None` when `steps_per_epoch` + is set. verbose: verbosity mode, 0 or 1. sample_weight: Array of weights to weight the contribution of different samples to the loss and metrics. + steps: number of steps to run, used for input tensors when + batch_size is not defined. # Returns Scalar test loss (if the model has a single output and no metrics) @@ -1554,6 +1560,9 @@ def evaluate(self, x, y, batch_size=32, verbose=1, sample_weight=None): and/or metrics). The attribute `model.metrics_names` will give you the display labels for the scalar outputs. """ + # backwards compatibility + if batch_size is None and steps is None: + batch_size = 32 # Validate user data. x, y, sample_weights = self._standardize_user_data( x, y, @@ -1569,7 +1578,8 @@ def evaluate(self, x, y, batch_size=32, verbose=1, sample_weight=None): f = self.test_function return self._test_loop(f, ins, batch_size=batch_size, - verbose=verbose) + verbose=verbose, + steps=steps) def predict(self, x, batch_size=32, verbose=0): """Generates output predictions for the input samples. From 3e13d6c0d82f37cb003e48c440f212a18e3347ee Mon Sep 17 00:00:00 2001 From: Andrew Hundt Date: Tue, 8 Aug 2017 15:24:58 -0400 Subject: [PATCH 140/166] Model.evaluate(steps) and Model.predict(steps) --- keras/engine/training.py | 128 +++++++++++++++++++++++++-------------- 1 file changed, 84 insertions(+), 44 deletions(-) diff --git a/keras/engine/training.py b/keras/engine/training.py index 67fbc3acf9a..5373b48090a 100644 --- a/keras/engine/training.py +++ b/keras/engine/training.py @@ -264,7 +264,7 @@ def _check_loss_and_target_compatibility(targets, loss_fns, output_shapes): 'binary_crossentropy', 'categorical_crossentropy'} for y, loss, shape in zip(targets, loss_fns, output_shapes): - if loss is None: + if loss is None or y is None: continue if loss.__name__ == 'categorical_crossentropy': if y.shape[-1] == 1: @@ -1166,7 +1166,7 @@ def _fit_loop(self, f, ins, out_labels=None, batch_size=None, callbacks.on_train_end() return self.history - def _predict_loop(self, f, ins, batch_size=32, verbose=0): + def _predict_loop(self, f, ins, batch_size=32, verbose=0, steps=None): """Abstract method to loop over some data in batches. # Arguments @@ -1183,6 +1183,9 @@ def _predict_loop(self, f, ins, batch_size=32, verbose=0): or list of arrays of predictions (if the model has multiple outputs). """ + if batch_size is None and steps is not None: + return self._step_loop(f, ins, verbose, steps) + if ins and hasattr(ins[0], 'shape'): samples = ins[0].shape[0] else: @@ -1240,58 +1243,88 @@ def _test_loop(self, f, ins, batch_size=None, verbose=0, steps=None): and/or metrics). The attribute `model.metrics_names` will give you the display labels for the scalar outputs. """ - outs = [] if batch_size is None and steps is not None: - if verbose == 1: - progbar = Progbar(target=steps) - for step_num in range(steps): - f(ins) - if verbose == 1: - progbar.update(step_num) + return self._step_loop(f, ins, verbose, steps) + if ins and hasattr(ins[0], 'shape'): + samples = ins[0].shape[0] else: - if ins and hasattr(ins[0], 'shape'): - samples = ins[0].shape[0] + # May happen if we are running `predict` without Numpy input data, + # i.e. if all inputs to the models are data tensors + # instead of placeholders. + # In that case we will run `predict` over a single batch. + samples = batch_size + verbose = 2 + + outs = [] + if verbose == 1: + progbar = Progbar(target=samples) + batches = _make_batches(samples, batch_size) + index_array = np.arange(samples) + for batch_index, (batch_start, batch_end) in enumerate(batches): + batch_ids = index_array[batch_start:batch_end] + if isinstance(ins[-1], float): + # Do not slice the training phase flag. + ins_batch = _slice_arrays(ins[:-1], batch_ids) + [ins[-1]] else: - # May happen if we are running `predict` without Numpy input data, - # i.e. if all inputs to the models are data tensors - # instead of placeholders. - # In that case we will run `predict` over a single batch. - samples = batch_size - verbose = 2 + ins_batch = _slice_arrays(ins, batch_ids) - if verbose == 1: - progbar = Progbar(target=samples) - batches = _make_batches(samples, batch_size) - index_array = np.arange(samples) - for batch_index, (batch_start, batch_end) in enumerate(batches): - batch_ids = index_array[batch_start:batch_end] - if isinstance(ins[-1], float): - # Do not slice the training phase flag. - ins_batch = _slice_arrays(ins[:-1], batch_ids) + [ins[-1]] - else: - ins_batch = _slice_arrays(ins, batch_ids) - - batch_outs = f(ins_batch) - if isinstance(batch_outs, list): - if batch_index == 0: - for batch_out in enumerate(batch_outs): - outs.append(0.) - for i, batch_out in enumerate(batch_outs): - outs[i] += batch_out * len(batch_ids) - else: - if batch_index == 0: + batch_outs = f(ins_batch) + if isinstance(batch_outs, list): + if batch_index == 0: + for batch_out in enumerate(batch_outs): outs.append(0.) - outs[0] += batch_outs * len(batch_ids) + for i, batch_out in enumerate(batch_outs): + outs[i] += batch_out * len(batch_ids) + else: + if batch_index == 0: + outs.append(0.) + outs[0] += batch_outs * len(batch_ids) - if verbose == 1: - progbar.update(batch_end) + if verbose == 1: + progbar.update(batch_end) for i in range(len(outs)): outs[i] /= samples if len(outs) == 1: return outs[0] return outs + def _step_loop(self, f, ins, verbose, steps): + """Abstract method to loop over some data in steps. + + # Arguments + f: Keras function returning a list of tensors. + ins: list of tensors to be fed to `f`. + verbose: verbosity mode. + steps: Total number of steps (batches of samples) + before declaring predictions finished. + Ignored with the default value of `None`. + + # Returns + List of outputs of the keras function + such as scalar loss or predictions, or a + single output if there is only one step. + """ + outs = [] + if verbose == 1: + progbar = Progbar(target=steps) + for step_num in range(steps): + batch_outs = f(ins) + if not isinstance(batch_outs, list): + batch_outs = [batch_outs] + if step_num == 0: + for batch_out in batch_outs: + shape = (steps,) + batch_out.shape[1:] + outs.append(np.zeros(shape, dtype=batch_out.dtype)) + + for i, batch_out in enumerate(batch_outs): + outs[step_num] = batch_out + if verbose == 1: + progbar.update(step_num) + if len(outs) == 1: + return outs[0] + return outs + def _standardize_user_data(self, x, y, sample_weight=None, class_weight=None, check_batch_axis=True, batch_size=None): @@ -1581,7 +1614,7 @@ def evaluate(self, x, y, batch_size=None, verbose=1, sample_weight=None, steps=N verbose=verbose, steps=steps) - def predict(self, x, batch_size=32, verbose=0): + def predict(self, x, batch_size=32, verbose=0, steps=None): """Generates output predictions for the input samples. Computation is done in batches. @@ -1589,8 +1622,13 @@ def predict(self, x, batch_size=32, verbose=0): # Arguments x: the input data, as a Numpy array (or list of Numpy arrays if the model has multiple outputs). - batch_size: integer. + batch_size: integer or `None`. Number of samples per prediction. + Defaults to 32 if predicting on numpy arrays and no batch + size is specified. Defaults to `None` when `steps_per_epoch` + is set. verbose: verbosity mode, 0 or 1. + steps: Total number of steps (batches of samples) to apply when + using input tensors. # Returns Numpy array(s) of predictions. @@ -1621,7 +1659,9 @@ def predict(self, x, batch_size=32, verbose=0): ins = x self._make_predict_function() f = self.predict_function - return self._predict_loop(f, ins, batch_size=batch_size, + return self._predict_loop(f, ins, + batch_size=batch_size, + steps=steps, verbose=verbose) def train_on_batch(self, x, y, From 56736f34420e5b19da3a8e6a2291aa2320df5ae3 Mon Sep 17 00:00:00 2001 From: Andrew Hundt Date: Tue, 8 Aug 2017 16:54:05 -0400 Subject: [PATCH 141/166] fix docstring comments from review (#7113) --- keras/engine/training.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/keras/engine/training.py b/keras/engine/training.py index 1af93a4e997..44141ccf4b1 100644 --- a/keras/engine/training.py +++ b/keras/engine/training.py @@ -1228,7 +1228,7 @@ def _test_loop(self, f, ins, batch_size=None, verbose=0, steps=None): # Arguments f: Keras function returning a list of tensors. ins: list of tensors to be fed to `f`. - batch_size: integer batch size. + batch_size: integer batch size or `None`. verbose: verbosity mode. steps: Total number of steps (batches of samples) before declaring predictions finished. @@ -1378,9 +1378,9 @@ def fit(self, x=None, If all outputs in the model are named, you can also pass a dictionary mapping output names to Numpy arrays. - batch_size: integer. Number of samples per gradient update. + batch_size: integer or `None`. Number of samples per gradient update. Defaults to 32 if training numpy arrays and no batch - size is specified. + size is specified. Defaults to `None` when `steps_per_epoch` is set. epochs: integer, the number of times to iterate over the training data arrays. verbose: 0, 1, or 2. Verbosity mode. @@ -1400,7 +1400,8 @@ def fit(self, x=None, This could be a tuple (x_val, y_val) or a tuple (x_val, y_val, val_sample_weights). shuffle: boolean, whether to shuffle the training data - before each epoch. + before each epoch. Has no effect when `steps_per_epoch` + is not `None`. class_weight: optional dictionary mapping class indices (integers) to a weight (float) to apply to the model's loss for the samples From e2e142f62f4439d5cf7b980468d2b537524aaa7f Mon Sep 17 00:00:00 2001 From: Andrew Hundt Date: Tue, 8 Aug 2017 17:59:48 -0400 Subject: [PATCH 142/166] adapting mnist_tfrecord.py for fit(y_train_in) --- examples/mnist_tfrecord.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/examples/mnist_tfrecord.py b/examples/mnist_tfrecord.py index 29f6bbc0091..d6f19af4bbc 100644 --- a/examples/mnist_tfrecord.py +++ b/examples/mnist_tfrecord.py @@ -113,7 +113,6 @@ def cnn_layers(x_train_input): enqueue_many=enqueue_many, num_threads=8) - x_train_batch = tf.cast(x_train_batch, tf.float32) x_train_batch = tf.reshape(x_train_batch, shape=batch_shape) @@ -125,6 +124,7 @@ def cnn_layers(x_train_input): x_train_input = layers.Input(tensor=x_train_batch, batch_shape=x_batch_shape) x_train_out = cnn_layers(x_train_input) +y_train_input = layers.Input(tensor=y_train_batch, batch_shape=y_batch_shape) train_model = Model(inputs=x_train_input, outputs=x_train_out) train_model.compile(optimizer='rmsprop', @@ -134,8 +134,9 @@ def cnn_layers(x_train_input): coord = tf.train.Coordinator() threads = tf.train.start_queue_runners(sess, coord) -train_model.fit(None, y_train_in, - epochs=epochs, steps_per_epoch=steps_per_epoch) +train_model.fit(y=y_train_input, + epochs=epochs, + steps_per_epoch=steps_per_epoch) train_model.save_weights('saved_wt.h5') coord.request_stop() From fdd9556c2e747328b16249c9d3fa675ba98349b8 Mon Sep 17 00:00:00 2001 From: Andrew Hundt Date: Tue, 8 Aug 2017 19:18:17 -0400 Subject: [PATCH 143/166] topology.py mnist_tfrecord.py bugfix --- keras/engine/topology.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/keras/engine/topology.py b/keras/engine/topology.py index 1eb40d16f7a..f0a543b7407 100644 --- a/keras/engine/topology.py +++ b/keras/engine/topology.py @@ -1634,8 +1634,8 @@ def __init__(self, inputs, outputs, name=None): self._feed_input_names.append(layer.name) self._feed_inputs.append(layer.input) self._feed_input_shapes.append(self.inputs[i]._keras_shape) - elif getattr(layer, 'input_tensor', False) is True: - self._tensor_inputs.append(layer.input_tensor) + else: + self._tensor_inputs.append((layer, node_index, tensor_index)) for layer in self.output_layers: self.output_names.append(layer.name) From 65d910314ac85c5b86e469d25038c91c8e5eb3f7 Mon Sep 17 00:00:00 2001 From: Andrew Hundt Date: Tue, 8 Aug 2017 19:33:57 -0400 Subject: [PATCH 144/166] add mnist_tfrecord_recordinput.py --- examples/mnist_tfrecord_recordinput.py | 212 +++++++++++++++++++++++++ 1 file changed, 212 insertions(+) create mode 100644 examples/mnist_tfrecord_recordinput.py diff --git a/examples/mnist_tfrecord_recordinput.py b/examples/mnist_tfrecord_recordinput.py new file mode 100644 index 00000000000..f8bc7ba45c1 --- /dev/null +++ b/examples/mnist_tfrecord_recordinput.py @@ -0,0 +1,212 @@ +'''MNIST dataset with TensorFlow TFRecords. + +Gets to 99.25% test accuracy after 12 epochs +(there is still a lot of margin for parameter tuning). +''' +import os +import copy +import time + +import numpy as np + +import tensorflow as tf +from tensorflow.python.ops import data_flow_ops +from keras import backend as K +from keras.models import Model +from keras.layers import Dense +from keras.layers import Dropout +from keras.layers import Flatten +from keras.layers import Input +from keras.layers import Conv2D +from keras.layers import MaxPooling2D +from keras.callbacks import EarlyStopping +from keras.callbacks import TensorBoard +from keras.objectives import categorical_crossentropy +from keras.utils import np_utils +from keras.utils.generic_utils import Progbar +from keras import callbacks as cbks +from keras import optimizers, objectives +from keras import metrics as metrics_module + +from keras.datasets import mnist + +if K.backend() != 'tensorflow': + raise RuntimeError('This example can only run with the ' + 'TensorFlow backend for the time being, ' + 'because it requires TFRecords, which ' + 'are not supported on other platforms.') + + +def images_to_tfrecord(images, labels, filename): + def _int64_feature(value): + return tf.train.Feature(int64_list=tf.train.Int64List(value=[value])) + + def _bytes_feature(value): + return tf.train.Feature(bytes_list=tf.train.BytesList(value=[value])) + + """ Save data into TFRecord """ + if not os.path.isfile(filename): + num_examples = images.shape[0] + + rows = images.shape[1] + cols = images.shape[2] + depth = images.shape[3] + + print('Writing', filename) + writer = tf.python_io.TFRecordWriter(filename) + for index in range(num_examples): + image_raw = images[index].tostring() + example = tf.train.Example(features=tf.train.Features(feature={ + 'height': _int64_feature(rows), + 'width': _int64_feature(cols), + 'depth': _int64_feature(depth), + 'label': _int64_feature(int(labels[index])), + 'image_raw': _bytes_feature(image_raw)})) + writer.write(example.SerializeToString()) + writer.close() + else: + print('tfrecord %s already exists' % filename) + + +def read_and_decode_recordinput(tf_glob, one_hot=True, + classes=None, is_train=None, + batch_shape=None, parallelism=1): + """ Return tensor to read from TFRecord """ + if batch_shape is None: + batch_shape = [1000, 28, 28, 1] + print 'Creating graph for loading %s TFRecords...' % tf_glob + with tf.variable_scope("TFRecords"): + record_input = data_flow_ops.RecordInput( + tf_glob, batch_size=batch_shape[0], parallelism=parallelism) + records_op = record_input.get_yield_op() + records_op = tf.split(records_op, batch_shape[0], 0) + records_op = [tf.reshape(record, []) for record in records_op] + progbar = Progbar(len(records_op)) + + images = [] + labels = [] + for i, serialized_example in enumerate(records_op): + progbar.update(i) + with tf.variable_scope("parse_images", reuse=True): + features = tf.parse_single_example( + serialized_example, + features={ + 'label': tf.FixedLenFeature([], tf.int64), + 'image_raw': tf.FixedLenFeature([], tf.string), + }) + img = tf.decode_raw(features['image_raw'], tf.uint8) + img.set_shape(batch_shape[1] * batch_shape[2]) + img = tf.reshape(img, [1] + batch_shape[1:]) + + img = tf.cast(img, tf.float32) * (1. / 255) - 0.5 + + label = tf.cast(features['label'], tf.int32) + if one_hot and classes: + label = tf.one_hot(label, classes) + + images.append(img) + labels.append(label) + + images = tf.parallel_stack(images, 0) + labels = tf.parallel_stack(labels, 0) + images = tf.cast(images, tf.float32) + + images = tf.reshape(images, shape=batch_shape) + + # StagingArea will store tensors + # across multiple steps to + # speed up execution + images_shape = images.get_shape() + labels_shape = labels.get_shape() + copy_stage = data_flow_ops.StagingArea( + [tf.float32, tf.float32], + shapes=[images_shape, labels_shape]) + copy_stage_op = copy_stage.put( + [images, labels]) + staged_images, staged_labels = copy_stage.get() + + return images, labels + + +def save_mnist_as_tfrecord(): + (X_train, y_train), (X_test, y_test) = mnist.load_data() + X_train = X_train[..., np.newaxis] + X_test = X_test[..., np.newaxis] + images_to_tfrecord(images=X_train, labels=y_train, filename='train.mnist.tfrecord') + images_to_tfrecord(images=X_test, labels=y_test, filename='test.mnist.tfrecord') + + +def cnn_layers(x_train_input): + x = Conv2D(32, (3, 3), activation='relu', padding='valid')(x_train_input) + x = Conv2D(64, (3, 3), activation='relu')(x) + x = MaxPooling2D(pool_size=(2, 2))(x) + x = Dropout(0.25)(x) + x = Flatten()(x) + x = Dense(128, activation='relu')(x) + x = Dropout(0.5)(x) + x_train_out = Dense(classes, + activation='softmax', + name='x_train_out')(x) + return x_train_out + + +sess = tf.Session() +K.set_session(sess) + +save_mnist_as_tfrecord() + +batch_size = 100 +batch_shape = [batch_size, 28, 28, 1] +epochs = 12 +steps_per_epoch = 1000 +classes = 10 +parallelism = 10 + +x_train_batch, y_train_batch = read_and_decode_recordinput( + 'train.mnist.tfrecord', + one_hot=True, + classes=classes, + is_train=True, + batch_shape=batch_shape, + parallelism=parallelism) + +x_test_batch, y_test_batch = read_and_decode_recordinput( + 'test.mnist.tfrecord', + one_hot=True, + classes=classes, + is_train=True, + batch_shape=batch_shape, + parallelism=parallelism) + +x_batch_shape = x_train_batch.get_shape().as_list() +y_batch_shape = y_train_batch.get_shape().as_list() + +x_train_in = Input(tensor=x_train_batch, batch_shape=x_batch_shape) +x_train_out = cnn_layers(x_train_in) +y_train_in = Input(tensor=y_train_batch, batch_shape=y_batch_shape, name='y_labels') +train_model = Model(inputs=[x_train_in], outputs=[x_train_out]) +train_model.compile(optimizer='rmsprop', + loss='categorical_crossentropy', + metrics=['accuracy']) +train_model.fit(None, y_train_in, + batch_size=None, + epochs=epochs, + steps_per_epoch=steps_per_epoch) +train_model.save_weights('saved_wt.h5') + +K.clear_session() + +# Second Session, pure Keras +(X_train, y_train), (X_test, y_test) = mnist.load_data() +X_train = X_train[..., np.newaxis] +X_test = X_test[..., np.newaxis] +x_test_inp = Input(batch_shape=(None,) + (X_test.shape[1:])) +test_out = cnn_layers(x_test_inp) +test_model = Model(inputs=x_test_inp, outputs=test_out) + +test_model.load_weights('saved_wt.h5') +test_model.compile(optimizer='rmsprop', loss='categorical_crossentropy', metrics=['accuracy']) +test_model.summary() + +loss, acc = test_model.evaluate(X_test, np_utils.to_categorical(y_test), classes) +print('\nTest accuracy: {0}'.format(acc)) From 6bef8c645cf0782de5df75bb0bed381fa3273189 Mon Sep 17 00:00:00 2001 From: Andrew Hundt Date: Tue, 8 Aug 2017 20:01:29 -0400 Subject: [PATCH 145/166] mnist_tfrecord_recordinput.py test with tensors --- examples/mnist_tfrecord_recordinput.py | 45 ++++++++++++++------------ 1 file changed, 24 insertions(+), 21 deletions(-) diff --git a/examples/mnist_tfrecord_recordinput.py b/examples/mnist_tfrecord_recordinput.py index f8bc7ba45c1..f2a4dd5e27d 100644 --- a/examples/mnist_tfrecord_recordinput.py +++ b/examples/mnist_tfrecord_recordinput.py @@ -129,11 +129,11 @@ def read_and_decode_recordinput(tf_glob, one_hot=True, def save_mnist_as_tfrecord(): - (X_train, y_train), (X_test, y_test) = mnist.load_data() - X_train = X_train[..., np.newaxis] - X_test = X_test[..., np.newaxis] - images_to_tfrecord(images=X_train, labels=y_train, filename='train.mnist.tfrecord') - images_to_tfrecord(images=X_test, labels=y_test, filename='test.mnist.tfrecord') + (x_train, y_train), (x_test, y_test) = mnist.load_data() + x_train = x_train[..., np.newaxis] + x_test = x_test[..., np.newaxis] + images_to_tfrecord(images=x_train, labels=y_train, filename='train.mnist.tfrecord') + images_to_tfrecord(images=x_test, labels=y_test, filename='test.mnist.tfrecord') def cnn_layers(x_train_input): @@ -170,14 +170,6 @@ def cnn_layers(x_train_input): batch_shape=batch_shape, parallelism=parallelism) -x_test_batch, y_test_batch = read_and_decode_recordinput( - 'test.mnist.tfrecord', - one_hot=True, - classes=classes, - is_train=True, - batch_shape=batch_shape, - parallelism=parallelism) - x_batch_shape = x_train_batch.get_shape().as_list() y_batch_shape = y_train_batch.get_shape().as_list() @@ -196,17 +188,28 @@ def cnn_layers(x_train_input): K.clear_session() -# Second Session, pure Keras -(X_train, y_train), (X_test, y_test) = mnist.load_data() -X_train = X_train[..., np.newaxis] -X_test = X_test[..., np.newaxis] -x_test_inp = Input(batch_shape=(None,) + (X_test.shape[1:])) -test_out = cnn_layers(x_test_inp) -test_model = Model(inputs=x_test_inp, outputs=test_out) +# Second Session, test data +x_test_batch, y_test_batch = read_and_decode_recordinput( + 'test.mnist.tfrecord', + one_hot=True, + classes=classes, + is_train=False, + batch_shape=batch_shape, + parallelism=parallelism) + +x_batch_shape = x_test_batch.get_shape().as_list() +y_batch_shape = y_test_batch.get_shape().as_list() +x_test_in = Input(tensor=x_test_batch, batch_shape=x_batch_shape) +x_test_out = cnn_layers(x_test_in) +y_test_in = Input(tensor=y_test_batch, batch_shape=y_batch_shape, name='y_labels') +test_model = Model(inputs=[x_test_in], outputs=[x_test_out]) +test_model.compile(optimizer='rmsprop', + loss='categorical_crossentropy', + metrics=['accuracy']) test_model.load_weights('saved_wt.h5') test_model.compile(optimizer='rmsprop', loss='categorical_crossentropy', metrics=['accuracy']) test_model.summary() -loss, acc = test_model.evaluate(X_test, np_utils.to_categorical(y_test), classes) +loss, acc = test_model.evaluate(None, y_test_in, classes) print('\nTest accuracy: {0}'.format(acc)) From 745e02af30509a44a34b5e0fb57b205209f890e2 Mon Sep 17 00:00:00 2001 From: Andrew Hundt Date: Tue, 8 Aug 2017 20:03:22 -0400 Subject: [PATCH 146/166] training.py create _check_for_recompile() with calls in fit() and evaluate() --- keras/engine/training.py | 41 +++++++++++++++++++++++----------------- 1 file changed, 24 insertions(+), 17 deletions(-) diff --git a/keras/engine/training.py b/keras/engine/training.py index 43e99232da5..9b8ad807804 100644 --- a/keras/engine/training.py +++ b/keras/engine/training.py @@ -1020,6 +1020,26 @@ def _make_function(self, function_name): name=function_name, **kwargs) + def _check_for_recompile(self, y): + if K.is_keras_tensor(y, expect_other_types=True): + self.target_configuration = [y] + y = None + self._compile(*self._saved_args, **self._saved_kwargs) + elif y is not None: + recompile = False + self.target_configuration = [] + for i, yi in enumerate(y): + if K.is_keras_tensor(yi, expect_other_types=True): + self.target_configuration.append(yi) + y[i] = None + recompile = True + else: + self.target_configuration.append(None) + + if recompile: + self._compile(*self._saved_args, **self._saved_kwargs) + return y + def _check_num_samples(self, ins, batch_size=None, steps=None, steps_name='steps'): """Determine the number of samples or steps for training and evaluation. @@ -1531,23 +1551,7 @@ class indices (integers) to if kwargs: raise TypeError('Unrecognized keyword arguments: ' + str(kwargs)) - if K.is_keras_tensor(y, expect_other_types=True): - self.target_configuration = [y] - y = None - self._compile(*self._saved_args, **self._saved_kwargs) - elif y is not None: - recompile = False - self.target_configuration = [] - for i, yi in enumerate(y): - if K.is_keras_tensor(yi, expect_other_types=True): - self.target_configuration.append(yi) - y[i] = None - recompile = True - else: - self.target_configuration.append(None) - - if recompile: - self._compile(*self._saved_args, **self._saved_kwargs) + y = self._check_for_recompile(y) # Validate user data. x, y, sample_weights = self._standardize_user_data( @@ -1657,6 +1661,9 @@ def evaluate(self, x, y, batch_size=None, verbose=1, sample_weight=None, steps=N # backwards compatibility if batch_size is None and steps is None: batch_size = 32 + + y = self._check_for_recompile(y) + # Validate user data. x, y, sample_weights = self._standardize_user_data( x, y, From 1ec1ce863befd3efeb95c39d1f7ff73ab61a030b Mon Sep 17 00:00:00 2001 From: Andrew Hundt Date: Tue, 8 Aug 2017 20:26:51 -0400 Subject: [PATCH 147/166] cntk_backend.py fix is_keras_tensor() --- keras/backend/cntk_backend.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/keras/backend/cntk_backend.py b/keras/backend/cntk_backend.py index 9d1e0c93055..2539e120c3d 100644 --- a/keras/backend/cntk_backend.py +++ b/keras/backend/cntk_backend.py @@ -256,10 +256,11 @@ def placeholder( def is_keras_tensor(x, expect_other_types=False): - if not isinstance(x, (C.variables.Constant, + if (not expect_other_types and + not isinstance(x, (C.variables.Constant, C.variables.Variable, C.variables.Parameter, - C.ops.functions.Function)): + C.ops.functions.Function))): raise ValueError('Unexpectedly found an instance of type `' + str(type(x)) + '`. ' 'Expected a symbolic tensor instance.') return hasattr(x, '_keras_history') From 3142a25f986c6e0bfa1d9ebb01f24f14fee89d88 Mon Sep 17 00:00:00 2001 From: Andrew Hundt Date: Tue, 8 Aug 2017 20:27:26 -0400 Subject: [PATCH 148/166] mnist_tfrecord_recordinput.py use default keras session --- examples/mnist_tfrecord_recordinput.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/examples/mnist_tfrecord_recordinput.py b/examples/mnist_tfrecord_recordinput.py index f2a4dd5e27d..ce0c253228f 100644 --- a/examples/mnist_tfrecord_recordinput.py +++ b/examples/mnist_tfrecord_recordinput.py @@ -150,8 +150,7 @@ def cnn_layers(x_train_input): return x_train_out -sess = tf.Session() -K.set_session(sess) +sess = K.get_session() save_mnist_as_tfrecord() From f9b387486b0962f14efde4195240541127d330fb Mon Sep 17 00:00:00 2001 From: Andrew Hundt Date: Tue, 8 Aug 2017 20:44:05 -0400 Subject: [PATCH 149/166] test_training.py minor variable rename for clarity --- tests/keras/engine/test_training.py | 29 +++++++++++------------------ 1 file changed, 11 insertions(+), 18 deletions(-) diff --git a/tests/keras/engine/test_training.py b/tests/keras/engine/test_training.py index c51e013833c..b750ff418d1 100644 --- a/tests/keras/engine/test_training.py +++ b/tests/keras/engine/test_training.py @@ -58,42 +58,35 @@ def test_standardize_input_data(): input_tensors=[p]) -def call_model_methods(model, input_np, output_np, batch_size=10, +def call_model_methods(model, x, y, batch_size=10, epochs=1, steps_per_epoch=None): # train_on_batch - out = model.train_on_batch(input_np, - output_np) - out = model.train_on_batch(input_np, - output_np) + out = model.train_on_batch(x, y) + out = model.train_on_batch(x, y) # test_on_batch - out = model.test_on_batch(input_np, - output_np) + out = model.test_on_batch(x, y) # predict_on_batch - out = model.predict_on_batch(input_np) + out = model.predict_on_batch(x) # fit - out = model.fit(input_np, - output_np, epochs=1, batch_size=batch_size, + out = model.fit(x, y, epochs=1, batch_size=batch_size, steps_per_epoch=steps_per_epoch) - out = model.fit(input_np, - output_np, epochs=1, batch_size=batch_size, + out = model.fit(x, y, epochs=1, batch_size=batch_size, steps_per_epoch=steps_per_epoch) # evaluate - out = model.evaluate(input_np, - output_np, batch_size=batch_size, + out = model.evaluate(x, y, batch_size=batch_size, steps=steps_per_epoch) - out = model.evaluate(input_np, - output_np, batch_size=batch_size, + out = model.evaluate(x, y, batch_size=batch_size, steps=steps_per_epoch) # predict - out = model.predict(input_np, batch_size=batch_size, + out = model.predict(x, batch_size=batch_size, steps=steps_per_epoch) - out = model.predict(input_np, batch_size=batch_size, + out = model.predict(x, batch_size=batch_size, steps=steps_per_epoch) return out From 38225bb7df39fb07eded41b0b26d0edeb27dd8c4 Mon Sep 17 00:00:00 2001 From: Andrew Hundt Date: Tue, 8 Aug 2017 20:45:40 -0400 Subject: [PATCH 150/166] pep8 --- keras/backend/cntk_backend.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/keras/backend/cntk_backend.py b/keras/backend/cntk_backend.py index 2539e120c3d..cfae484a6d2 100644 --- a/keras/backend/cntk_backend.py +++ b/keras/backend/cntk_backend.py @@ -258,9 +258,9 @@ def placeholder( def is_keras_tensor(x, expect_other_types=False): if (not expect_other_types and not isinstance(x, (C.variables.Constant, - C.variables.Variable, - C.variables.Parameter, - C.ops.functions.Function))): + C.variables.Variable, + C.variables.Parameter, + C.ops.functions.Function))): raise ValueError('Unexpectedly found an instance of type `' + str(type(x)) + '`. ' 'Expected a symbolic tensor instance.') return hasattr(x, '_keras_history') From bbf6c3e0688222c93bf615a89633a96b7424de65 Mon Sep 17 00:00:00 2001 From: Andrew Hundt Date: Tue, 8 Aug 2017 21:19:45 -0400 Subject: [PATCH 151/166] Model.train_on_batch() and Model.test_on_batch() support input tensors training.py test_training.py input tensor tests extended --- keras/engine/training.py | 2 ++ tests/keras/engine/test_training.py | 13 +++++++++---- 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/keras/engine/training.py b/keras/engine/training.py index 9b8ad807804..dfe580c7ac6 100644 --- a/keras/engine/training.py +++ b/keras/engine/training.py @@ -1763,6 +1763,7 @@ class indices (integers) to and/or metrics). The attribute `model.metrics_names` will give you the display labels for the scalar outputs. """ + y = self._check_for_recompile(y) x, y, sample_weights = self._standardize_user_data( x, y, sample_weight=sample_weight, @@ -1803,6 +1804,7 @@ def test_on_batch(self, x, y, sample_weight=None): and/or metrics). The attribute `model.metrics_names` will give you the display labels for the scalar outputs. """ + y = self._check_for_recompile(y) x, y, sample_weights = self._standardize_user_data( x, y, sample_weight=sample_weight, diff --git a/tests/keras/engine/test_training.py b/tests/keras/engine/test_training.py index b750ff418d1..14c81ee161b 100644 --- a/tests/keras/engine/test_training.py +++ b/tests/keras/engine/test_training.py @@ -713,15 +713,20 @@ def input_label_tfrecord_model(input_a_tf, output_b_tf, img_batch_shape, classes loss_weights = [1., 2.] with pytest.raises(ValueError) as exc: model.compile(optimizer, loss, metrics=['mean_squared_error'], - loss_weights=loss_weights, - sample_weight_mode=None) + loss_weights=loss_weights) - model.compile(optimizer, loss, metrics=['mean_squared_error'], - sample_weight_mode=None) + model.compile(optimizer, loss, metrics=['mean_squared_error']) call_model_methods(model, None, None, batch_size=None, steps_per_epoch=1) + model.compile(optimizer, + loss='categorical_crossentropy', + metrics=['mean_squared_error']) + + call_model_methods(model, None, y_train_in_out, + batch_size=None, steps_per_epoch=1) + def create_tfrecord_data(img_batch_shape, classes): From 91bee672740c44ac95e738de9af6bfbd55e74de1 Mon Sep 17 00:00:00 2001 From: Andrew Hundt Date: Tue, 8 Aug 2017 22:15:49 -0400 Subject: [PATCH 152/166] mnist_tfrecord*.py extended documentation --- examples/mnist_tfrecord.py | 8 ++++ examples/mnist_tfrecord_recordinput.py | 65 +++++++++++++++++--------- 2 files changed, 52 insertions(+), 21 deletions(-) diff --git a/examples/mnist_tfrecord.py b/examples/mnist_tfrecord.py index d6f19af4bbc..2956b582577 100644 --- a/examples/mnist_tfrecord.py +++ b/examples/mnist_tfrecord.py @@ -122,6 +122,11 @@ def cnn_layers(x_train_input): x_batch_shape = x_train_batch.get_shape().as_list() y_batch_shape = y_train_batch.get_shape().as_list() +# The input tensors are provided directly into the Model network +# which is fixed once it is initialized. Since the network +# is fixed, it must be reconstructed every time a new input data +# source is needed. This is substantially different from typical +# Keras numpy array inputs and is more like TensorFlow. x_train_input = layers.Input(tensor=x_train_batch, batch_shape=x_batch_shape) x_train_out = cnn_layers(x_train_input) y_train_input = layers.Input(tensor=y_train_batch, batch_shape=y_batch_shape) @@ -134,6 +139,9 @@ def cnn_layers(x_train_input): coord = tf.train.Coordinator() threads = tf.train.start_queue_runners(sess, coord) + +# The input data was created with x_train_input, +# so only the label data needs to be provided. train_model.fit(y=y_train_input, epochs=epochs, steps_per_epoch=steps_per_epoch) diff --git a/examples/mnist_tfrecord_recordinput.py b/examples/mnist_tfrecord_recordinput.py index ce0c253228f..043e2310be5 100644 --- a/examples/mnist_tfrecord_recordinput.py +++ b/examples/mnist_tfrecord_recordinput.py @@ -1,7 +1,13 @@ -'''MNIST dataset with TensorFlow TFRecords. +'''Optimized MNIST dataset with TFRecords, the standard TensorFlow data format. + +TFRecord is a data format supported throughout TensorFlow. +For a straightforward usage example see mnist_tfrecord.py. +This example demonstrates how to write and read TFRecord data using +Input Tensors in a way that is better optimized for high performance +on large datasets with Keras. Gets to 99.25% test accuracy after 12 epochs -(there is still a lot of margin for parameter tuning). +(there is still some margin for parameter tuning). ''' import os import copy @@ -38,6 +44,15 @@ def images_to_tfrecord(images, labels, filename): + """Write images and their labels to a TFRecord file. + + # Arguments + images: A numpy array or list of image data with + shape (images, rows, cols, depth). + labels: A numpy array of labels, with one for each image. + filename: Path and name for the output dataset TFRecord file. + An example is `'path/to/mnist_train.tfrecord'` + """ def _int64_feature(value): return tf.train.Feature(int64_list=tf.train.Int64List(value=[value])) @@ -69,9 +84,19 @@ def _bytes_feature(value): def read_and_decode_recordinput(tf_glob, one_hot=True, - classes=None, is_train=None, + classes=None, batch_shape=None, parallelism=1): - """ Return tensor to read from TFRecord """ + """Get a TF Tensor that supplies shuffled batches of images. + + # Arguments + tf_glob: File path for selecting one or more tfrecord files. + Examples are `'path/to/data.tfrecord'` and `'path/to/*.tfrecord'`. + one_hot: Use one hot encoding for labels, also known as categorical. + batch_shape: Specify the desired image batch shape, where the first + entry is the batch size. MNIST might be (128, 28, 28, 1). + parallelism: The number of threads to use for loading new data. + A reasonable value is the number of logical cores on your processor. + """ if batch_shape is None: batch_shape = [1000, 28, 28, 1] print 'Creating graph for loading %s TFRecords...' % tf_glob @@ -110,25 +135,14 @@ def read_and_decode_recordinput(tf_glob, one_hot=True, images = tf.parallel_stack(images, 0) labels = tf.parallel_stack(labels, 0) images = tf.cast(images, tf.float32) - images = tf.reshape(images, shape=batch_shape) - # StagingArea will store tensors - # across multiple steps to - # speed up execution - images_shape = images.get_shape() - labels_shape = labels.get_shape() - copy_stage = data_flow_ops.StagingArea( - [tf.float32, tf.float32], - shapes=[images_shape, labels_shape]) - copy_stage_op = copy_stage.put( - [images, labels]) - staged_images, staged_labels = copy_stage.get() - return images, labels def save_mnist_as_tfrecord(): + """Save one tfrecord file for each of the train and test mnist datasets. + """ (x_train, y_train), (x_test, y_test) = mnist.load_data() x_train = x_train[..., np.newaxis] x_test = x_test[..., np.newaxis] @@ -137,6 +151,8 @@ def save_mnist_as_tfrecord(): def cnn_layers(x_train_input): + """Create the CNN layers for use with either numpy inputs or tensor inputs. + """ x = Conv2D(32, (3, 3), activation='relu', padding='valid')(x_train_input) x = Conv2D(64, (3, 3), activation='relu')(x) x = MaxPooling2D(pool_size=(2, 2))(x) @@ -165,13 +181,17 @@ def cnn_layers(x_train_input): 'train.mnist.tfrecord', one_hot=True, classes=classes, - is_train=True, batch_shape=batch_shape, parallelism=parallelism) x_batch_shape = x_train_batch.get_shape().as_list() y_batch_shape = y_train_batch.get_shape().as_list() +# The input tensors are provided directly into the Model network. +# The network is fixed once it is initialized, so it must be +# reconstructed every time a new input data source is needed. +# This is substantially different from typical +# Keras numpy array inputs, and is more like TensorFlow. x_train_in = Input(tensor=x_train_batch, batch_shape=x_batch_shape) x_train_out = cnn_layers(x_train_in) y_train_in = Input(tensor=y_train_batch, batch_shape=y_batch_shape, name='y_labels') @@ -179,7 +199,10 @@ def cnn_layers(x_train_input): train_model.compile(optimizer='rmsprop', loss='categorical_crossentropy', metrics=['accuracy']) -train_model.fit(None, y_train_in, + +# The input data was created with x_train_input, +# so only the label data needs to be provided. +train_model.fit(y=y_train_in, batch_size=None, epochs=epochs, steps_per_epoch=steps_per_epoch) @@ -192,13 +215,13 @@ def cnn_layers(x_train_input): 'test.mnist.tfrecord', one_hot=True, classes=classes, - is_train=False, batch_shape=batch_shape, parallelism=parallelism) x_batch_shape = x_test_batch.get_shape().as_list() y_batch_shape = y_test_batch.get_shape().as_list() +# Create a completely new network for new input data. x_test_in = Input(tensor=x_test_batch, batch_shape=x_batch_shape) x_test_out = cnn_layers(x_test_in) y_test_in = Input(tensor=y_test_batch, batch_shape=y_batch_shape, name='y_labels') @@ -210,5 +233,5 @@ def cnn_layers(x_train_input): test_model.compile(optimizer='rmsprop', loss='categorical_crossentropy', metrics=['accuracy']) test_model.summary() -loss, acc = test_model.evaluate(None, y_test_in, classes) +loss, acc = test_model.evaluate(y=y_test_in, classes) print('\nTest accuracy: {0}'.format(acc)) From c0e6e3bb5191549a564eb3d83adcec34ae31aca0 Mon Sep 17 00:00:00 2001 From: Andrew Hundt Date: Tue, 8 Aug 2017 22:25:41 -0400 Subject: [PATCH 153/166] mnist_tfrecord*.py extended documentation --- examples/mnist_tfrecord.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/examples/mnist_tfrecord.py b/examples/mnist_tfrecord.py index 2956b582577..15d47fc531e 100644 --- a/examples/mnist_tfrecord.py +++ b/examples/mnist_tfrecord.py @@ -122,11 +122,11 @@ def cnn_layers(x_train_input): x_batch_shape = x_train_batch.get_shape().as_list() y_batch_shape = y_train_batch.get_shape().as_list() -# The input tensors are provided directly into the Model network -# which is fixed once it is initialized. Since the network -# is fixed, it must be reconstructed every time a new input data -# source is needed. This is substantially different from typical -# Keras numpy array inputs and is more like TensorFlow. +# The input tensors are provided directly into the Model network. +# The network is fixed once it is initialized, so it must be +# reconstructed every time a new input data source is needed. +# This is substantially different from typical +# Keras numpy array inputs, and is more like TensorFlow. x_train_input = layers.Input(tensor=x_train_batch, batch_shape=x_batch_shape) x_train_out = cnn_layers(x_train_input) y_train_input = layers.Input(tensor=y_train_batch, batch_shape=y_batch_shape) From 1831064298f0832015fc9298e2cd07f3ac535b26 Mon Sep 17 00:00:00 2001 From: Andrew Hundt Date: Tue, 8 Aug 2017 23:47:43 -0400 Subject: [PATCH 154/166] mnist_tfrecord_recordinput.py evaluate() bugfix --- examples/mnist_tfrecord_recordinput.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/examples/mnist_tfrecord_recordinput.py b/examples/mnist_tfrecord_recordinput.py index 043e2310be5..4e4f4437f87 100644 --- a/examples/mnist_tfrecord_recordinput.py +++ b/examples/mnist_tfrecord_recordinput.py @@ -233,5 +233,7 @@ def cnn_layers(x_train_input): test_model.compile(optimizer='rmsprop', loss='categorical_crossentropy', metrics=['accuracy']) test_model.summary() -loss, acc = test_model.evaluate(y=y_test_in, classes) -print('\nTest accuracy: {0}'.format(acc)) +# Take steps for each element of validation data. +evaluate_steps = y_batch_shape[0]/batch_size +loss, acc = test_model.evaluate(y=y_test_in, steps=evaluate_steps) +print('\nTest accuracy: {0}'.format(acc), ' loss: {0}'.format(loss)) From 86257ee33ea21849915bbc6bb9558eb96424a87d Mon Sep 17 00:00:00 2001 From: Andrew Hundt Date: Wed, 9 Aug 2017 00:11:47 -0400 Subject: [PATCH 155/166] training.py documentation of methods internal to Model --- keras/engine/training.py | 69 +++++++++++++++++++++++++++++++++++----- 1 file changed, 61 insertions(+), 8 deletions(-) diff --git a/keras/engine/training.py b/keras/engine/training.py index dfe580c7ac6..46b8c3b03b8 100644 --- a/keras/engine/training.py +++ b/keras/engine/training.py @@ -657,12 +657,20 @@ def compile(self, *args, **kwargs): ValueError: In case of invalid arguments for `optimizer`, `loss`, `metrics` or `sample_weight_mode`. """ + # `compile()` saves all arguments then calls `_compile`, + # which contains the full function implementation. + # This is to support user supplied Input Tensors. + # The saved arguments are necessary to re-create parts of + # the Model's underlying backend graph if new Input + # Tensors are provided by the user at a later time. self._saved_kwargs = kwargs self._saved_args = args self._compile(*args, **kwargs) def _compile(self, optimizer, loss, metrics=None, loss_weights=None, sample_weight_mode=None, **kwargs): + """Backend implementation of `compile()`. + """ loss = loss or {} self.optimizer = optimizers.get(optimizer) self.loss = loss @@ -877,6 +885,8 @@ def append_metric(layer_num, metric_name, metric_tensor): self._collected_trainable_weights = trainable_weights def _prepare_sample_weights(self, sample_weight_mode, skip_indices): + """Prepares and verifies sample weight member variables during compilation. + """ # Prepare sample weights. if self._tensor_inputs: if not all(l is None for l in self.target_configuration): @@ -966,16 +976,44 @@ def _prepare_sample_weights(self, sample_weight_mode, skip_indices): return sample_weights, sample_weight_mode def _make_ins(self, x=None, y=None, sample_weights=None, learning_phase_value=None): + """Create the list of inputs to the backend in a consistent manner. + + # Arguments + x: Numpy array of input data preprocessed to exclude tensors. + y: Numpy array of target data preprocessed to exclude tensors. + sample_weights: optional arrays of the same length as x, containing + weights to apply to the model's loss for each sample. + """ def add_if_valid(x): + """Ensures concatenation of inputs results in a list. + + Inputs are assembled into a list for execution, but if all + elements are `None`, adding them will result in a runtime + error instead of a list of `None` values. Therefore, + add_if_valid converts `None` values into [`None`], + while leaving valid values alone. + """ # In a worst case return an empty list return [None] if x is None else x - # ins = add_if_valid(x) + add_if_valid(y) + add_if_valid(sample_weights) ins = add_if_valid(x) + add_if_valid(y) + add_if_valid(sample_weights) if learning_phase_value and self.uses_learning_phase and not isinstance(K.learning_phase(), int): ins = ins + add_if_valid(learning_phase_value) return ins def _make_function(self, function_name): + """Create a backend function based on the current model configuration. + + Checks the state of all relevant member variables and inputs based on + the backend function type given by the `function_name` parameter, + and assemble the correct variables for this use. Includes additional + verification steps to prevent user error, such as forgetting to call + `compile()`. + + # Arguments + function_name: The name of the backend function type that is needed. + Options are `'predict_function'`, `'train_function'`, and + `'test_function'`. + """ if function_name is 'predict_function': if not hasattr(self, 'predict_function'): self.predict_function = None @@ -1021,6 +1059,21 @@ def _make_function(self, function_name): **kwargs) def _check_for_recompile(self, y): + """Calls `_compile()` if new Input Tensors were provided by the user. + + Also inserts Input Tensors into `self.target_configuration` and removing + them from `y`, so that the tensors are not incorrectly executed as input + feed values. + + # Arguments + y: List or array containing heterogeneous input tensors and numpy arrays. + + # Returns + + `y` with all entries that are input tensors replaced with `None`. + For example, `[numpy_array, tensor, tensor]` + becomes `[numpy_array, None, None]`. + """ if K.is_keras_tensor(y, expect_other_types=True): self.target_configuration = [y] y = None @@ -1028,9 +1081,9 @@ def _check_for_recompile(self, y): elif y is not None: recompile = False self.target_configuration = [] - for i, yi in enumerate(y): - if K.is_keras_tensor(yi, expect_other_types=True): - self.target_configuration.append(yi) + for i, y_i in enumerate(y): + if K.is_keras_tensor(y_i, expect_other_types=True): + self.target_configuration.append(y_i) y[i] = None recompile = True else: @@ -1626,7 +1679,7 @@ class indices (integers) to steps_per_epoch=steps_per_epoch, validation_steps=validation_steps) - def evaluate(self, x, y, batch_size=None, verbose=1, sample_weight=None, steps=None): + def evaluate(self, x=None, y=None, batch_size=None, verbose=1, sample_weight=None, steps=None): """Returns the loss value & metrics values for the model in test mode. Computation is done in batches. @@ -1679,7 +1732,7 @@ def evaluate(self, x, y, batch_size=None, verbose=1, sample_weight=None, steps=N verbose=verbose, steps=steps) - def predict(self, x, batch_size=32, verbose=0, steps=None): + def predict(self, x=None, batch_size=32, verbose=0, steps=None): """Generates output predictions for the input samples. Computation is done in batches. @@ -1727,7 +1780,7 @@ def predict(self, x, batch_size=32, verbose=0, steps=None): steps=steps, verbose=verbose) - def train_on_batch(self, x, y, + def train_on_batch(self, x=None, y=None, sample_weight=None, class_weight=None): """Runs a single gradient update on a single batch of data. @@ -1776,7 +1829,7 @@ class indices (integers) to return outputs[0] return outputs - def test_on_batch(self, x, y, sample_weight=None): + def test_on_batch(self, x=None, y=None, sample_weight=None): """Test the model on a single batch of samples. # Arguments From c62af05e74d3a0c433730d9f72aa3c12aa681235 Mon Sep 17 00:00:00 2001 From: Andrew Hundt Date: Wed, 9 Aug 2017 00:27:53 -0400 Subject: [PATCH 156/166] training.py fix _step_loop --- keras/engine/training.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/keras/engine/training.py b/keras/engine/training.py index 46b8c3b03b8..b4ea5c887c5 100644 --- a/keras/engine/training.py +++ b/keras/engine/training.py @@ -1438,7 +1438,7 @@ def _step_loop(self, f, ins, verbose, steps): outs.append(np.zeros(shape, dtype=batch_out.dtype)) for i, batch_out in enumerate(batch_outs): - outs[step_num] = batch_out + outs[i][step_num] = batch_out if verbose == 1: progbar.update(step_num) if len(outs) == 1: From f4e6e2a30964d4b8439c21c62cf3d43c7f23ab79 Mon Sep 17 00:00:00 2001 From: Andrew Hundt Date: Wed, 9 Aug 2017 00:32:22 -0400 Subject: [PATCH 157/166] mnist_tfrecord_recordinput.py fix accuracy calculation --- examples/mnist_tfrecord_recordinput.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/examples/mnist_tfrecord_recordinput.py b/examples/mnist_tfrecord_recordinput.py index 4e4f4437f87..12d104c6080 100644 --- a/examples/mnist_tfrecord_recordinput.py +++ b/examples/mnist_tfrecord_recordinput.py @@ -231,9 +231,9 @@ def cnn_layers(x_train_input): metrics=['accuracy']) test_model.load_weights('saved_wt.h5') test_model.compile(optimizer='rmsprop', loss='categorical_crossentropy', metrics=['accuracy']) -test_model.summary() # Take steps for each element of validation data. -evaluate_steps = y_batch_shape[0]/batch_size +validation_samples = 10000 +evaluate_steps = validation_samples/batch_size loss, acc = test_model.evaluate(y=y_test_in, steps=evaluate_steps) -print('\nTest accuracy: {0}'.format(acc), ' loss: {0}'.format(loss)) +print('\nTest accuracy: {0}'.format(np.mean(acc))) From d89f467aa3f738600e1ac1e30dc8b64295987334 Mon Sep 17 00:00:00 2001 From: Andrew Hundt Date: Wed, 9 Aug 2017 01:02:32 -0400 Subject: [PATCH 158/166] pep8 --- examples/mnist_tfrecord_recordinput.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/mnist_tfrecord_recordinput.py b/examples/mnist_tfrecord_recordinput.py index 12d104c6080..ea6b89d8a57 100644 --- a/examples/mnist_tfrecord_recordinput.py +++ b/examples/mnist_tfrecord_recordinput.py @@ -234,6 +234,6 @@ def cnn_layers(x_train_input): # Take steps for each element of validation data. validation_samples = 10000 -evaluate_steps = validation_samples/batch_size +evaluate_steps = validation_samples / batch_size loss, acc = test_model.evaluate(y=y_test_in, steps=evaluate_steps) print('\nTest accuracy: {0}'.format(np.mean(acc))) From b2635a1d5eb56e35c94f35a618ee069df5c22b62 Mon Sep 17 00:00:00 2001 From: Andrew Hundt Date: Wed, 9 Aug 2017 01:46:48 -0400 Subject: [PATCH 159/166] training.py _step_loop() fix test error, progbar supports more data layouts --- keras/engine/training.py | 26 +++++++++++++------------- keras/utils/generic_utils.py | 6 ++++-- 2 files changed, 17 insertions(+), 15 deletions(-) diff --git a/keras/engine/training.py b/keras/engine/training.py index b4ea5c887c5..c9f7b80e7a8 100644 --- a/keras/engine/training.py +++ b/keras/engine/training.py @@ -1425,25 +1425,25 @@ def _step_loop(self, f, ins, verbose, steps): such as scalar loss or predictions, or a single output if there is only one step. """ - outs = [] + all_outs = [] if verbose == 1: progbar = Progbar(target=steps) for step_num in range(steps): - batch_outs = f(ins) - if not isinstance(batch_outs, list): - batch_outs = [batch_outs] - if step_num == 0: - for batch_out in batch_outs: - shape = (steps,) + batch_out.shape[1:] - outs.append(np.zeros(shape, dtype=batch_out.dtype)) + outs = f(ins) + if not isinstance(outs, list): + outs = [outs] - for i, batch_out in enumerate(batch_outs): - outs[i][step_num] = batch_out + if not all_outs: + for out in outs: + all_outs.append([]) + + for i, out in enumerate(outs): + all_outs[i].append(out) if verbose == 1: progbar.update(step_num) - if len(outs) == 1: - return outs[0] - return outs + if len(all_outs) == 1: + return all_outs[0] + return all_outs def _standardize_user_data(self, x, y, sample_weight=None, class_weight=None, diff --git a/keras/utils/generic_utils.py b/keras/utils/generic_utils.py index 43274bfe65b..223ab2a1445 100644 --- a/keras/utils/generic_utils.py +++ b/keras/utils/generic_utils.py @@ -329,7 +329,8 @@ def update(self, current, values=None, force=False): for k in self.unique_values: info += ' - %s:' % k if isinstance(self.sum_values[k], list): - avg = np.mean(self.sum_values[k][0] / max(1, self.sum_values[k][1])) + avg = np.mean(np.array(self.sum_values[k][0]) / + max(1, np.array(self.sum_values[k][1]))) if abs(avg) > 1e-3: info += ' %.4f' % avg else: @@ -352,7 +353,8 @@ def update(self, current, values=None, force=False): info = '%ds' % (now - self.start) for k in self.unique_values: info += ' - %s:' % k - avg = np.mean(self.sum_values[k][0] / max(1, self.sum_values[k][1])) + avg = np.mean(np.array(self.sum_values[k][0]) / + max(1, np.array(self.sum_values[k][1]))) if avg > 1e-3: info += ' %.4f' % avg else: From a8d743de4abd1cbb44d64d7828a9dc511c05857b Mon Sep 17 00:00:00 2001 From: Andrew Hundt Date: Wed, 9 Aug 2017 02:54:35 -0400 Subject: [PATCH 160/166] tensorflow_backend.py remove unused self.feed_to_fectch_count --- keras/backend/tensorflow_backend.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/keras/backend/tensorflow_backend.py b/keras/backend/tensorflow_backend.py index b3ad6c79990..2b02e29bc57 100644 --- a/keras/backend/tensorflow_backend.py +++ b/keras/backend/tensorflow_backend.py @@ -2263,7 +2263,6 @@ def __call__(self, inputs): if not isinstance(inputs, (list, tuple)): raise TypeError('`inputs` should be a list or tuple.') self.current_feed_dict = {} if self.feed_dict is None else self.feed_dict - self.feed_to_fetch_count = 0 self.current_fetches = self.outputs + [self.updates_op] # self.inputs contains tf tensors, inputs contains feed_dict data. for tensor, value in zip_longest(self.inputs, inputs, fillvalue=None): @@ -2282,7 +2281,6 @@ def __call__(self, inputs): value = (indices, sparse_coo.data, sparse_coo.shape) if value is None and tensor is not None: - self.feed_to_fetch_count += 1 self.current_fetches.append(tensor) else: self.current_feed_dict[tensor] = value From 7e0ac461dd8b74db7bde7ee787e0ee229db9617d Mon Sep 17 00:00:00 2001 From: Andrew Hundt Date: Wed, 9 Aug 2017 04:09:48 -0400 Subject: [PATCH 161/166] mnist_tfrecord*.py clean up extraneous imports --- examples/mnist_tfrecord.py | 5 ---- examples/mnist_tfrecord_recordinput.py | 34 +++++++++----------------- 2 files changed, 11 insertions(+), 28 deletions(-) diff --git a/examples/mnist_tfrecord.py b/examples/mnist_tfrecord.py index 15d47fc531e..91e519d5b3d 100644 --- a/examples/mnist_tfrecord.py +++ b/examples/mnist_tfrecord.py @@ -48,11 +48,6 @@ from keras import layers from keras import objectives from keras.utils import np_utils -from keras import objectives -from keras.utils.generic_utils import Progbar -from keras import callbacks as cbks -from keras import optimizers, objectives -from keras import metrics as metrics_module from tensorflow.contrib.learn.python.learn.datasets import mnist diff --git a/examples/mnist_tfrecord_recordinput.py b/examples/mnist_tfrecord_recordinput.py index ea6b89d8a57..10b3dd746a0 100644 --- a/examples/mnist_tfrecord_recordinput.py +++ b/examples/mnist_tfrecord_recordinput.py @@ -19,20 +19,8 @@ from tensorflow.python.ops import data_flow_ops from keras import backend as K from keras.models import Model -from keras.layers import Dense -from keras.layers import Dropout -from keras.layers import Flatten -from keras.layers import Input -from keras.layers import Conv2D -from keras.layers import MaxPooling2D -from keras.callbacks import EarlyStopping -from keras.callbacks import TensorBoard -from keras.objectives import categorical_crossentropy -from keras.utils import np_utils +from keras import layers from keras.utils.generic_utils import Progbar -from keras import callbacks as cbks -from keras import optimizers, objectives -from keras import metrics as metrics_module from keras.datasets import mnist @@ -153,16 +141,16 @@ def save_mnist_as_tfrecord(): def cnn_layers(x_train_input): """Create the CNN layers for use with either numpy inputs or tensor inputs. """ - x = Conv2D(32, (3, 3), activation='relu', padding='valid')(x_train_input) - x = Conv2D(64, (3, 3), activation='relu')(x) - x = MaxPooling2D(pool_size=(2, 2))(x) - x = Dropout(0.25)(x) - x = Flatten()(x) - x = Dense(128, activation='relu')(x) - x = Dropout(0.5)(x) - x_train_out = Dense(classes, - activation='softmax', - name='x_train_out')(x) + x = layers.Conv2D(32, (3, 3), activation='relu', padding='valid')(x_train_input) + x = layers.Conv2D(64, (3, 3), activation='relu')(x) + x = layers.MaxPooling2D(pool_size=(2, 2))(x) + x = layers.Dropout(0.25)(x) + x = layers.Flatten()(x) + x = layers.Dense(128, activation='relu')(x) + x = layers.Dropout(0.5)(x) + x_train_out = layers.Dense(classes, + activation='softmax', + name='x_train_out')(x) return x_train_out From 4ec2e637f66a323d7a179e1fd700b2413a58190b Mon Sep 17 00:00:00 2001 From: Andrew Hundt Date: Wed, 9 Aug 2017 04:17:21 -0400 Subject: [PATCH 162/166] mnist_tfrecord_recordinput.py Input(...) -> layers.Input(...) --- examples/mnist_tfrecord_recordinput.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/examples/mnist_tfrecord_recordinput.py b/examples/mnist_tfrecord_recordinput.py index 10b3dd746a0..0972763c4ac 100644 --- a/examples/mnist_tfrecord_recordinput.py +++ b/examples/mnist_tfrecord_recordinput.py @@ -180,9 +180,9 @@ def cnn_layers(x_train_input): # reconstructed every time a new input data source is needed. # This is substantially different from typical # Keras numpy array inputs, and is more like TensorFlow. -x_train_in = Input(tensor=x_train_batch, batch_shape=x_batch_shape) +x_train_in = layers.Input(tensor=x_train_batch, batch_shape=x_batch_shape) x_train_out = cnn_layers(x_train_in) -y_train_in = Input(tensor=y_train_batch, batch_shape=y_batch_shape, name='y_labels') +y_train_in = layers.Input(tensor=y_train_batch, batch_shape=y_batch_shape, name='y_labels') train_model = Model(inputs=[x_train_in], outputs=[x_train_out]) train_model.compile(optimizer='rmsprop', loss='categorical_crossentropy', @@ -210,9 +210,9 @@ def cnn_layers(x_train_input): y_batch_shape = y_test_batch.get_shape().as_list() # Create a completely new network for new input data. -x_test_in = Input(tensor=x_test_batch, batch_shape=x_batch_shape) +x_test_in = layers.Input(tensor=x_test_batch, batch_shape=x_batch_shape) x_test_out = cnn_layers(x_test_in) -y_test_in = Input(tensor=y_test_batch, batch_shape=y_batch_shape, name='y_labels') +y_test_in = layers.Input(tensor=y_test_batch, batch_shape=y_batch_shape, name='y_labels') test_model = Model(inputs=[x_test_in], outputs=[x_test_out]) test_model.compile(optimizer='rmsprop', loss='categorical_crossentropy', From 41fdd5aa5fe080bf5fa12638b9abc9f50d835a39 Mon Sep 17 00:00:00 2001 From: Andrew Hundt Date: Fri, 11 Aug 2017 11:19:10 -0400 Subject: [PATCH 163/166] training.py improve docstrings and error case --- keras/engine/training.py | 35 +++++++++++++++++++++++++---------- 1 file changed, 25 insertions(+), 10 deletions(-) diff --git a/keras/engine/training.py b/keras/engine/training.py index 44141ccf4b1..c6a75ddb861 100644 --- a/keras/engine/training.py +++ b/keras/engine/training.py @@ -976,15 +976,30 @@ def _make_predict_function(self): **kwargs) def _check_num_samples(self, ins, batch_size=None, steps=None, steps_name='steps'): - """Determine the number of samples or steps for training and evaluation. + """Determine the number of samples provided for training and evaluation. + + The number of samples is not defined when running with `steps`, + in which case the number of samples is set to `None`. # Arguments ins: list of tensors to be fed to the Keras function. - batch_size: integer batch size or None if unknown. + batch_size: integer batch size or `None` if not defined. steps: Total number of steps (batches of samples) - before declaring _predict_loop finished. + before declaring `_predict_loop` finished. Ignored with the default value of `None`. steps_name: The public API's parameter name for `steps`. + + # Raises + ValueError when `steps` is `None` and the attribute `ins.shape` + does not exist. Also raises ValueError when `steps` is not `None` + and `batch_size` is not `None` because they are mutually + exclusive. + + # Returns + When steps is `None`, returns the number of samples to be + processed based on the size of the first dimension of the + first input numpy array. When steps is not `None` and + `batch_size` is `None`, returns `None`. """ if steps is not None: num_samples = None @@ -994,7 +1009,7 @@ def _check_num_samples(self, ins, batch_size=None, steps=None, steps_name='steps elif ins and hasattr(ins[0], 'shape'): num_samples = ins[0].shape[0] else: - raise ValueError('The input data should have shape or ' + raise ValueError('The input data should have shape, or ' 'please specify ' + steps_name + '.') return num_samples @@ -1039,10 +1054,10 @@ def _fit_loop(self, f, ins, out_labels=None, batch_size=None, if hasattr(ins[0], 'shape'): validation_steps = steps_per_epoch else: - raise ValueError('When `steps_per_epoch` validation ' - 'data has no `shape` attribute, ' - 'you must specify a value for ' - '`validation_steps`.') + raise ValueError('You must specify a value for ' + '`validation_steps` when using ' + '`steps_per_epoch` validation ' + 'without a `shape` attribute.') if verbose and ins and hasattr(ins[0], 'shape'): print('Train on %d samples, validate on %d samples' % @@ -1057,7 +1072,7 @@ def _fit_loop(self, f, ins, out_labels=None, batch_size=None, self.history = cbks.History() callbacks = [cbks.BaseLogger()] + (callbacks or []) + [self.history] if verbose: - if steps_per_epoch: + if steps_per_epoch is not None: count_mode = 'steps' else: count_mode = 'samples' @@ -1175,7 +1190,7 @@ def _predict_loop(self, f, ins, batch_size=32, verbose=0): batch_size: integer batch size. verbose: verbosity mode. steps: Total number of steps (batches of samples) - before declaring _predict_loop finished. + before declaring `_predict_loop` finished. Ignored with the default value of `None`. # Returns From 353bb1263259ce8d413b10288b781d53873b10c9 Mon Sep 17 00:00:00 2001 From: Andrew Hundt Date: Fri, 11 Aug 2017 16:25:43 -0400 Subject: [PATCH 164/166] Allows custom tensors for target based on (#5927) implemented in (#6928). --- keras/engine/training.py | 36 +++++++++++++++++++++-- tests/keras/engine/test_training.py | 45 +++++++++++++++++++++++++++++ 2 files changed, 79 insertions(+), 2 deletions(-) diff --git a/keras/engine/training.py b/keras/engine/training.py index e77f199fffa..d5c1ca8917c 100644 --- a/keras/engine/training.py +++ b/keras/engine/training.py @@ -649,13 +649,20 @@ def compile(self, *args, **kwargs): If the model has multiple outputs, you can use a different `sample_weight_mode` on each output by passing a dictionary or a list of modes. + target_tensors: Custom placeholders or input tensors provided as + a single element, list, or dictionary indexed by output node name. + Target tensors can be used for tensor based data inputs or for + targets with shape that differs from the output. When providing + a list, order of these elements must match the order of the + outputs. For every entry in which targets will be defined at a + later point by a numpy array, include an entry containing `None`. **kwargs: when using the Theano/CNTK backends, these arguments are passed into K.function. When using the TensorFlow backend, these arguments are passed into `tf.Session.run`. # Raises ValueError: In case of invalid arguments for - `optimizer`, `loss`, `metrics` or `sample_weight_mode`. + `optimizer`, `loss`, `metrics`, `sample_weight_mode`, or `target_tensors`. """ # `compile()` saves all arguments then calls `_compile`, # which contains the full function implementation. @@ -668,7 +675,7 @@ def compile(self, *args, **kwargs): self._compile(*args, **kwargs) def _compile(self, optimizer, loss, metrics=None, loss_weights=None, - sample_weight_mode=None, **kwargs): + sample_weight_mode=None, target_tensors=None, **kwargs): """Backend implementation of `compile()`. """ loss = loss or {} @@ -759,6 +766,31 @@ def _compile(self, optimizer, loss, metrics=None, loss_weights=None, sample_weight_mode, skip_indices) # Prepare targets of model. + # + # self.target_configuration has a default value + # set in `Container.__init__()`. + if target_tensors is not None: + self.target_configuration = [] + if isinstance(target_tensors, dict): + for name in target_tensors: + if name not in self.output_names: + raise ValueError('Unknown entry in target_tensors ' + 'dictionary: "' + name + '". ' + 'Only expected the following keys: ' + + str(self.output_names)) + for name in self.output_names: + self.target_configuration.append(target_tensors.get(name, 1.)) + elif isinstance(target_tensors, (list, tuple)): + self.target_configuration = list(target_tensors) + else: + self.target_configuration = [target_tensors] + + if len(self.target_configuration) != len(self.outputs): + raise ValueError('There must be a target_tensor for each ' + 'output. Found {}, expected ' + '{}'.format(len(self.target_configuration), + len(self.outputs))) + self.targets = [] self._feed_targets = [] for i in range(len(self.outputs)): diff --git a/tests/keras/engine/test_training.py b/tests/keras/engine/test_training.py index 14c81ee161b..6db292a5caa 100644 --- a/tests/keras/engine/test_training.py +++ b/tests/keras/engine/test_training.py @@ -931,5 +931,50 @@ def test_model_with_external_loss(): batch_size=None, steps_per_epoch=1) +@keras_test +def test_model_custom_target_tensors(): + a = Input(shape=(3,), name='input_a') + b = Input(shape=(3,), name='input_b') + + a_2 = Dense(4, name='dense_1')(a) + dp = Dropout(0.5, name='dropout') + b_2 = dp(b) + + y = K.placeholder([10, 4], name='y') + y1 = K.placeholder([10, 3], name='y1') + y2 = K.placeholder([7, 5], name='y2') + model = Model([a, b], [a_2, b_2]) + + optimizer = 'rmsprop' + loss = 'mse' + loss_weights = [1., 0.5] + + # test list of target tensors + with pytest.raises(ValueError): + model.compile(optimizer, loss, metrics=[], loss_weights=loss_weights, + sample_weight_mode=None, target_tensors=[y, y1, y2]) + model.compile(optimizer, loss, metrics=[], loss_weights=loss_weights, + sample_weight_mode=None, target_tensors=[y, y1]) + input_a_np = np.random.random((10, 3)) + input_b_np = np.random.random((10, 3)) + + output_a_np = np.random.random((10, 4)) + output_b_np = np.random.random((10, 3)) + + out = model.train_on_batch([input_a_np, input_b_np], + [output_a_np, output_b_np], {y: np.random.random((10, 4)), y1: np.random.random((10, 3))}) + + # test dictionary of target_tensors + with pytest.raises(ValueError): + model.compile(optimizer, loss, metrics=[], loss_weights=loss_weights, + sample_weight_mode=None, target_tensors={'does_not_exist': y2}) + # test dictionary of target_tensors + model.compile(optimizer, loss, metrics=[], loss_weights=loss_weights, + sample_weight_mode=None, target_tensors={'dense_1': y, 'dropout': y1}) + + out = model.train_on_batch([input_a_np, input_b_np], + [output_a_np, output_b_np], {y: np.random.random((10, 4)), y1: np.random.random((10, 3))}) + + if __name__ == '__main__': pytest.main([__file__]) From 8fd6fe06b9573cdc2454c43b991a86a120f4febe Mon Sep 17 00:00:00 2001 From: Andrew Hundt Date: Fri, 11 Aug 2017 18:30:13 -0400 Subject: [PATCH 165/166] test_training.py major simplification of tensor test --- tests/keras/engine/test_training.py | 140 ++++++++-------------------- 1 file changed, 41 insertions(+), 99 deletions(-) diff --git a/tests/keras/engine/test_training.py b/tests/keras/engine/test_training.py index 851748840db..a20b60cf2ab 100644 --- a/tests/keras/engine/test_training.py +++ b/tests/keras/engine/test_training.py @@ -635,32 +635,45 @@ def test_model_with_input_yield_op(): by only passing them data for the placeholder inputs in the model. """ + import tensorflow as tf batch_size = 1 input_rows = 20 cols = 20 depth = 1 classes = 2 + sess = K.get_session() # first batch size 1 img_batch_shape = [batch_size, input_rows, cols, depth] - input_a_np, input_a_tf, output_a_np, output_b_np, output_b_tf = create_tfrecord_data( + input_a_np, input_a_tf, output_a_np, output_b_np, output_b_tf = create_tensor_data( img_batch_shape, classes) + + coord = tf.train.Coordinator() + threads = tf.train.start_queue_runners(sess, coord) # tensor input, numpy output input_only_tfrecord_model(input_a_tf, output_b_np, img_batch_shape, classes) # tensor input, tensor output input_label_tfrecord_model(input_a_tf, output_b_tf, img_batch_shape, classes) + coord.request_stop() + coord.join(threads) + K.clear_session() + + sess = K.get_session() # next batch size 3 batch_size = 3 img_batch_shape = [batch_size, input_rows, cols, depth] - input_a_np, input_a_tf, output_a_np, output_b_np, output_b_tf = create_tfrecord_data( + input_a_np, input_a_tf, output_a_np, output_b_np, output_b_tf = create_tensor_data( img_batch_shape, classes) + coord = tf.train.Coordinator() + threads = tf.train.start_queue_runners(sess, coord) # tensor input, numpy output input_only_tfrecord_model(input_a_tf, output_b_np, img_batch_shape, classes) # tensor input, tensor output input_label_tfrecord_model(input_a_tf, output_b_tf, img_batch_shape, classes) - os.remove('input_a.tfrecord') + coord.request_stop() + coord.join(threads) def cnn_layers(x_train_input, classes): @@ -692,7 +705,7 @@ def input_only_tfrecord_model(input_a_tf, output_b_np, img_batch_shape, classes) model.compile(optimizer, loss, metrics=['mean_squared_error'], sample_weight_mode=None) - call_model_methods(model, None, output_b_np, batch_size=img_batch_shape[0]) + call_model_methods(model, None, output_b_np, batch_size=None, steps_per_epoch=2) def input_label_tfrecord_model(input_a_tf, output_b_tf, img_batch_shape, classes): @@ -728,105 +741,34 @@ def input_label_tfrecord_model(input_a_tf, output_b_tf, img_batch_shape, classes batch_size=None, steps_per_epoch=1) -def create_tfrecord_data(img_batch_shape, classes): - +def create_tensor_data(img_batch_shape, classes, batch_count=1): import tensorflow as tf - from tensorflow.python.lib.io import tf_record - from tensorflow.python.ops import data_flow_ops - from tensorflow.python.platform import test - - def images_to_tfrecord(images, labels, filename): - - def _int64_feature(value): - return tf.train.Feature(int64_list=tf.train.Int64List(value=[value])) - - def _bytes_feature(value): - return tf.train.Feature(bytes_list=tf.train.BytesList(value=[value])) - - """ Save data into TFRecord """ - if not os.path.isfile(filename): - num_examples = images.shape[0] - - rows = images.shape[1] - cols = images.shape[2] - depth = images.shape[3] - - print('Writing', filename) - writer = tf.python_io.TFRecordWriter(filename) - for index in range(num_examples): - image_raw = images[index].tostring() - example = tf.train.Example(features=tf.train.Features(feature={ - 'height': _int64_feature(rows), - 'width': _int64_feature(cols), - 'depth': _int64_feature(depth), - 'label': _int64_feature(int(labels[index])), - 'image_raw': _bytes_feature(image_raw)})) - writer.write(example.SerializeToString()) - writer.close() - else: - print('tfrecord %s already exists' % filename) - - def read_and_decode_recordinput(tf_glob, one_hot=True, classes=None, is_train=None, batch_shape=[10, 3, 3, 1]): - """ Return tensor to read from TFRecord """ - import tensorflow as tf - print('Creating graph for loading TFRecords...') - with tf.variable_scope("TFRecords"): - record_input = data_flow_ops.RecordInput(tf_glob, batch_size=batch_shape[0]) - records_op = record_input.get_yield_op() - records_op = tf.split(records_op, batch_shape[0], 0) - records_op = [tf.reshape(record, []) for record in records_op] - - images = [] - labels = [] - for i, serialized_example in enumerate(records_op): - with tf.variable_scope("parse_images", reuse=True): - features = tf.parse_single_example( - serialized_example, - features={ - 'label': tf.FixedLenFeature([], tf.int64), - 'image_raw': tf.FixedLenFeature([], tf.string), - }) - img = tf.decode_raw(features['image_raw'], tf.uint8) - img.set_shape(batch_shape[1] * batch_shape[2]) - img = tf.reshape(img, [1] + batch_shape[1:]) - - img = tf.cast(img, tf.float32) * (1. / 255) - 0.5 - - label = tf.cast(features['label'], tf.int32) - if one_hot and classes: - label = tf.one_hot(label, classes) - - images.append(img) - labels.append(label) - - images = tf.parallel_stack(images, 0) - labels = tf.parallel_stack(labels, 0) - images = tf.cast(images, tf.float32) - - images = tf.reshape(images, shape=batch_shape) - - return images, labels - - def replace(filename): - if os.path.isfile(filename): - print('%s already exists, replacing...' % filename) - os.remove(filename) - [batch_size, input_rows, cols, depth] = img_batch_shape - label_batch_shape = [batch_size, classes] - input_a_np = np.multiply(np.random.random(img_batch_shape), batch_size) + img_count = batch_size * batch_count + imgs_shape = [img_count, input_rows, cols, depth] + labels_shape = [img_count, classes] + input_a_np = np.multiply(np.random.random(imgs_shape), batch_size) input_a_np = input_a_np.astype('uint8') - output_a_np = np.multiply(np.random.random([batch_size]), batch_size) + output_a_np = np.multiply(np.random.random([img_count]), classes) output_a_np = output_a_np.astype('int') - output_b_np = np.random.random([batch_size, classes]) - replace('input_a.tfrecord') - images_to_tfrecord(input_a_np, output_a_np, 'input_a.tfrecord') - input_a_tf, output_b_tf = read_and_decode_recordinput( - 'input_a.tfrecord', - one_hot=True, - classes=classes, - is_train=True, - batch_shape=img_batch_shape) + output_b_np = np.random.random(labels_shape) + + from tensorflow.contrib.learn.python.learn.datasets import mnist + + input_a_tf, output_b_tf = tf.train.shuffle_batch( + tensors=[input_a_np, output_a_np.astype(np.int32)], + batch_size=batch_size, + capacity=batch_size*2, + min_after_dequeue=1, + enqueue_many=True, + num_threads=1, + name='create_tensor_data_shuffle_batch') + + input_a_tf = tf.cast(input_a_tf, tf.float32, name='input_a_tf_cast') + input_a_tf = tf.reshape(input_a_tf, shape=img_batch_shape, name='input_a_tf_reshape') + + output_b_tf = tf.cast(output_a_np, tf.int32, name='output_b_tf_cast') + output_b_tf = tf.one_hot(output_b_tf, classes, name='output_b_tf_one_hot') return input_a_np, input_a_tf, output_a_np, output_b_np, output_b_tf From 0eb2a8a81e3f0c392d17efcd56b3a689648a6721 Mon Sep 17 00:00:00 2001 From: Andrew Hundt Date: Fri, 11 Aug 2017 19:01:46 -0400 Subject: [PATCH 166/166] pep8 --- tests/keras/engine/test_training.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/keras/engine/test_training.py b/tests/keras/engine/test_training.py index a20b60cf2ab..9f906cdaabb 100644 --- a/tests/keras/engine/test_training.py +++ b/tests/keras/engine/test_training.py @@ -758,7 +758,7 @@ def create_tensor_data(img_batch_shape, classes, batch_count=1): input_a_tf, output_b_tf = tf.train.shuffle_batch( tensors=[input_a_np, output_a_np.astype(np.int32)], batch_size=batch_size, - capacity=batch_size*2, + capacity=batch_size * 2, min_after_dequeue=1, enqueue_many=True, num_threads=1,