From 9af2202aa3fd840aa0e9b11386cbeeae569f1709 Mon Sep 17 00:00:00 2001 From: Josh Fromm Date: Tue, 18 Feb 2020 10:24:22 -0800 Subject: [PATCH] [Relay][Frontend][Keras] NHWC import support. (#4899) * Basic test working * Almost all tests working. * all tests passing. * Fixed lint. * Improved Style. --- python/tvm/relay/frontend/keras.py | 148 +++++++++++++++----- tests/python/frontend/keras/test_forward.py | 47 ++++--- 2 files changed, 143 insertions(+), 52 deletions(-) diff --git a/python/tvm/relay/frontend/keras.py b/python/tvm/relay/frontend/keras.py index d21f1af124cac..caf41768ada4d 100644 --- a/python/tvm/relay/frontend/keras.py +++ b/python/tvm/relay/frontend/keras.py @@ -186,7 +186,7 @@ def _convert_merge(inexpr, keras_layer, _): assert len(inexpr) == 2, "Subtract merge takes 2 inputs." ret = _op.subtract(ret, inexpr[1]) elif merge_type in ['Add', 'Multiply', 'Maximum']: - op_map = {'Add':_op.add, 'Multiply':_op.multiply, 'Maximum':_op.maximum} + op_map = {'Add': _op.add, 'Multiply': _op.multiply, 'Maximum': _op.maximum} for i in range(1, len(inexpr)): ret = op_map[merge_type](ret, inexpr[i]) elif merge_type == 'Average': @@ -206,7 +206,7 @@ def _convert_permute(inexpr, keras_layer, _): def _convert_dense(inexpr, keras_layer, etab): weightList = keras_layer.get_weights() weight = etab.new_const(weightList[0].transpose([1, 0])) - params = {'weight':weight, 'units':weightList[0].shape[1]} + params = {'weight': weight, 'units': weightList[0].shape[1]} input_shape = keras_layer.input_shape input_dim = len(input_shape) # In case of RNN dense, input shape will be (1, 1, n) @@ -237,15 +237,28 @@ def _convert_convolution(inexpr, keras_layer, etab): is_deconv = type(keras_layer).__name__ == 'Conv2DTranspose' is_depthconv = type(keras_layer).__name__ == 'DepthwiseConv2D' weightList = keras_layer.get_weights() + weight = weightList[0] + if etab.data_layout == 'NHWC': + if is_depthconv: + kernel_layout = 'HWOI' + else: + kernel_layout = 'HWIO' + else: + kernel_layout = 'OIHW' + if is_deconv: - kernel_h, kernel_w, n_filters, in_channels = weightList[0].shape - weight = weightList[0].transpose([3, 2, 0, 1]) + kernel_h, kernel_w, n_filters, in_channels = weight.shape + if kernel_layout == 'OIHW': + weight = weight.transpose([3, 2, 0, 1]) elif is_depthconv: - kernel_h, kernel_w, in_channels, depth_mult = weightList[0].shape - weight = weightList[0].transpose([2, 3, 0, 1]) + kernel_h, kernel_w, in_channels, depth_mult = weight.shape + if kernel_layout == 'OIHW': + weight = weight.transpose([2, 3, 0, 1]) + elif etab.data_layout == 'NCHW': + kernel_h, kernel_w, in_channels, n_filters = weight.shape + weight = weight.transpose([3, 2, 0, 1]) else: - kernel_h, kernel_w, in_channels, n_filters = weightList[0].shape - weight = weightList[0].transpose([3, 2, 0, 1]) + kernel_h, kernel_w, in_channels, n_filters = weight.shape if isinstance(keras_layer.dilation_rate, (list, tuple)): dilation = [keras_layer.dilation_rate[0], keras_layer.dilation_rate[1]] else: @@ -257,7 +270,9 @@ def _convert_convolution(inexpr, keras_layer, etab): 'kernel_size': [kernel_h, kernel_w], 'strides': [stride_h, stride_w], 'dilation': dilation, - 'padding': [0, 0]} + 'padding': [0, 0], + 'data_layout': etab.data_layout, + 'kernel_layout': kernel_layout} if is_depthconv: params['channels'] = in_channels * depth_mult params['groups'] = in_channels @@ -273,9 +288,13 @@ def _convert_convolution(inexpr, keras_layer, etab): pad_l, pad_r = _get_pad_pair(in_w, dilated_kernel_w, stride_w) if pad_t == pad_b and pad_l == pad_r: params['padding'] = (pad_t, pad_l) - else: + elif etab.data_layout == 'NCHW': inexpr = _op.nn.pad(data=inexpr, pad_width=( (0, 0), (0, 0), (pad_t, pad_b), (pad_l, pad_r))) + else: + inexpr = _op.nn.pad(data=inexpr, pad_width=( + (0, 0), (pad_t, pad_b), (pad_l, pad_r), (0, 0))) + else: msg = 'Padding with {} is not supported for operator Convolution ' \ 'in frontend Keras.' @@ -284,9 +303,13 @@ def _convert_convolution(inexpr, keras_layer, etab): out = _op.nn.conv2d_transpose(data=inexpr, **params) else: out = _op.nn.conv2d(data=inexpr, **params) + if keras_layer.use_bias: bias = etab.new_const(weightList[1]) - out = _op.nn.bias_add(out, bias) + if etab.data_layout == 'NCHW': + out = _op.nn.bias_add(out, bias) + else: + out = _op.nn.bias_add(out, bias, axis=-1) # defuse activation if sys.version_info.major < 3: act_type = keras_layer.activation.func_name @@ -299,18 +322,27 @@ def _convert_convolution(inexpr, keras_layer, etab): def _convert_separable_convolution(inexpr, keras_layer, etab): _check_data_format(keras_layer) + if etab.data_layout == 'NHWC': + kernel_layout = 'HWOI' + else: + kernel_layout = 'OIHW' weightList = keras_layer.get_weights() # depthwise conv kernel_h, kernel_w, in_channels, depth_mult = weightList[0].shape stride_h, stride_w = keras_layer.strides - weight0 = weightList[0].transpose([2, 3, 0, 1]) + if kernel_layout == 'OIHW': + weight0 = weightList[0].transpose([2, 3, 0, 1]) + else: + weight0 = weightList[0] params0 = {'weight': etab.new_const(weight0), 'channels': in_channels * depth_mult, 'groups': in_channels, 'kernel_size': [kernel_h, kernel_w], 'strides': [stride_h, stride_w], 'dilation': [1, 1], - 'padding': [0, 0]} + 'padding': [0, 0], + 'data_layout': etab.data_layout, + 'kernel_layout': kernel_layout} if keras_layer.padding == 'valid': pass # we insert a separate pad operator @@ -321,27 +353,39 @@ def _convert_separable_convolution(inexpr, keras_layer, etab): pad_l, pad_r = _get_pad_pair(in_w, kernel_w, stride_w) if pad_t == pad_b and pad_l == pad_r: params0['padding'] = (pad_t, pad_l) - else: + elif etab.data_layout == 'NCHW': inexpr = _op.nn.pad(data=inexpr, pad_width=( (0, 0), (0, 0), (pad_t, pad_b), (pad_l, pad_r))) + else: + inexpr = _op.nn.pad(data=inexpr, pad_width=( + (0, 0), (pad_t, pad_b), (pad_l, pad_r), (0, 0))) + else: msg = 'Padding with {} is not supported for operator Separable ' \ 'Convolution in frontend Keras.' raise tvm.error.OpAttributeUnImplemented(msg.format(keras_layer.padding)) - depthconv = _op.nn.conv2d(data=inexpr, **params0) # pointwise conv - weight1 = weightList[1].transpose([3, 2, 0, 1]) + if kernel_layout == 'OIHW': + weight1 = weightList[1].transpose([3, 2, 0, 1]) + else: + weight1 = weightList[1] + kernel_layout = "HWIO" params1 = {'weight': etab.new_const(weight1), - 'channels': weight1.shape[0], + 'channels': weightList[1].shape[3], 'groups': 1, 'kernel_size': [1, 1], 'strides': [1, 1], - 'dilation': [1, 1]} + 'dilation': [1, 1], + 'data_layout': etab.data_layout, + 'kernel_layout': kernel_layout} out = _op.nn.conv2d(data=depthconv, **params1) if keras_layer.use_bias: bias = etab.new_const(weightList[2]) - out = _op.nn.bias_add(out, bias) + if etab.data_layout == 'NCHW': + out = _op.nn.bias_add(out, bias) + else: + out = _op.nn.bias_add(out, bias, axis=-1) # defuse activation if sys.version_info.major < 3: act_type = keras_layer.activation.func_name @@ -352,10 +396,11 @@ def _convert_separable_convolution(inexpr, keras_layer, etab): return out -def _convert_flatten(inexpr, keras_layer, _): +def _convert_flatten(inexpr, keras_layer, etab): _check_data_format(keras_layer) # NCHW -> NHWC so that dense can be correctly converted - inexpr = _op.transpose(inexpr, axes=[0, 2, 3, 1]) + if etab.data_layout == 'NCHW': + inexpr = _op.transpose(inexpr, axes=[0, 2, 3, 1]) return _op.nn.batch_flatten(inexpr) @@ -363,15 +408,19 @@ def _convert_pooling(inexpr, keras_layer, etab): _check_data_format(keras_layer) pool_type = type(keras_layer).__name__ # global pool in keras = global pool + flatten in relay + global_pool_params = {'layout': etab.data_layout} if pool_type == 'GlobalMaxPooling2D': - return _convert_flatten(_op.nn.global_max_pool2d(inexpr), keras_layer, etab) + return _convert_flatten( + _op.nn.global_max_pool2d(inexpr, **global_pool_params), keras_layer, etab) if pool_type == 'GlobalAveragePooling2D': - return _convert_flatten(_op.nn.global_avg_pool2d(inexpr), keras_layer, etab) + return _convert_flatten( + _op.nn.global_avg_pool2d(inexpr, **global_pool_params), keras_layer, etab) pool_h, pool_w = keras_layer.pool_size stride_h, stride_w = keras_layer.strides params = {'pool_size': [pool_h, pool_w], 'strides': [stride_h, stride_w], - 'padding': [0, 0]} + 'padding': [0, 0], + 'layout': etab.data_layout} if keras_layer.padding == 'valid': pass elif keras_layer.padding == 'same': @@ -392,7 +441,7 @@ def _convert_pooling(inexpr, keras_layer, etab): 'Operator {} is not supported for frontend Keras.'.format(keras_layer)) -def _convert_upsample(inexpr, keras_layer, _): +def _convert_upsample(inexpr, keras_layer, etab): _check_data_format(keras_layer) upsample_type = type(keras_layer).__name__ params = {} @@ -424,7 +473,9 @@ def _convert_upsample(inexpr, keras_layer, _): else: raise tvm.error.OpNotImplemented( 'Operator {} is not supported for frontend Keras.'.format(upsample_type)) - return _op.nn.upsampling(inexpr, **params) + params['layout'] = etab.data_layout + out = _op.nn.upsampling(inexpr, **params) + return out def _convert_cropping(inexpr, keras_layer, _): @@ -442,9 +493,15 @@ def _convert_cropping(inexpr, keras_layer, _): def _convert_batchnorm(inexpr, keras_layer, etab): + if etab.data_layout == 'NCHW' or len(keras_layer.input_shape) < 4: + axis = 1 + else: + axis = 3 + params = {'scale': False, 'center': False, - 'epsilon': keras_layer.epsilon} + 'epsilon': keras_layer.epsilon, + 'axis': axis} idx = 0 if keras_layer.scale: params['scale'] = True @@ -469,7 +526,7 @@ def _convert_batchnorm(inexpr, keras_layer, etab): return result -def _convert_padding(inexpr, keras_layer, _): +def _convert_padding(inexpr, keras_layer, etab): _check_data_format(keras_layer) padding_type = type(keras_layer).__name__ padding = keras_layer.padding @@ -495,16 +552,21 @@ def _convert_padding(inexpr, keras_layer, _): else: msg = 'Operator {} is not supported in frontend Keras.' raise tvm.error.OpNotImplemented(msg.format(padding_type)) - return _op.nn.pad(data=inexpr, - pad_width=((0, 0), (0, 0), (top, bottom), (left, right))) + if etab.data_layout == 'NCHW': + return _op.nn.pad(data=inexpr, pad_width=((0, 0), (0, 0), (top, bottom), (left, right))) + return _op.nn.pad(data=inexpr, pad_width=((0, 0), (top, bottom), (left, right), (0, 0))) -def _convert_concat(inexpr, keras_layer, _): +def _convert_concat(inexpr, keras_layer, etab): _check_data_format(keras_layer) - return _op.concatenate(_as_list(inexpr), axis=1) + if etab.data_layout == 'NHWC' or len(keras_layer.input_shape[0]) < 4: + axis = -1 + else: + axis = 1 + return _op.concatenate(_as_list(inexpr), axis=axis) -def _convert_reshape(inexpr, keras_layer, _): +def _convert_reshape(inexpr, keras_layer, etab): _check_data_format(keras_layer) inshape = keras_layer.input_shape # includes batch tshape = keras_layer.target_shape # no batch @@ -525,7 +587,10 @@ def _convert_reshape(inexpr, keras_layer, _): assert ch == tshape[-1], \ "Only supports last dimension in target shape being equal to " \ "the channel number of input tensor." - shape = (-1, ch) + tshape[:-1] + if etab.data_layout == 'NCHW': + shape = (-1, ch) + tshape[:-1] + else: + shape = (-1,) + tshape[:-1] + (ch,) return _op.reshape(inexpr, newshape=shape) @@ -740,7 +805,7 @@ def keras_op_to_relay(inexpr, keras_layer, outname, etab): etab.set_expr(name, out) -def from_keras(model, shape=None): +def from_keras(model, shape=None, layout='NCHW'): """Convert keras model to relay Function. Parameters @@ -751,6 +816,11 @@ def from_keras(model, shape=None): shape: dict of str to int list/tuple Input shapes of the model, optional + layout: str + One of 'NCHW' or 'NHWC', indicates how data should be arranged in + the output model. Default layout is 'NCHW' as it in general + performs better across TVM. + Returns ------- mod : tvm.IRModule @@ -793,6 +863,9 @@ def _convert_input_layer(keras_layer): assert isinstance(model, expected_model_class) etab = ExprTable() + # Set global data format. + assert layout in ['NCHW', 'NHWC'], "Layout must be one of 'NCHW' or NHWC" + etab.data_layout = layout for keras_layer in model.layers: if isinstance(keras_layer, input_layer_class): _convert_input_layer(keras_layer) @@ -818,7 +891,10 @@ def _convert_input_layer(keras_layer): # The one exception is InputLayer. Changing input variable names after conversion # would confuse users, so we should keep them as far as possible. Fortunately, # they are named uniquely to input_1, input_2, input_3... by default. - zip_node = zip(node.node_indices, node.tensor_indices, node.inbound_layers) + zip_node = zip( + _as_list(node.node_indices), + _as_list(node.tensor_indices), + _as_list(node.inbound_layers)) for n_idx, t_idx, inbound_layer in zip_node: if isinstance(inbound_layer, input_layer_class): expr_name = inbound_layer.name diff --git a/tests/python/frontend/keras/test_forward.py b/tests/python/frontend/keras/test_forward.py index e4df4da4e9898..f7dcb29b37aaa 100644 --- a/tests/python/frontend/keras/test_forward.py +++ b/tests/python/frontend/keras/test_forward.py @@ -21,13 +21,18 @@ from tvm.relay.testing.config import ctx_list import keras -# prevent Keras from using up all gpu memory import tensorflow as tf from tensorflow import keras as tf_keras -from keras.backend.tensorflow_backend import set_session -config = tf.ConfigProto() -config.gpu_options.per_process_gpu_memory_fraction = 0.5 -set_session(tf.Session(config=config)) +# prevent Keras from using up all gpu memory +if tf.executing_eagerly(): + gpus = tf.config.list_physical_devices('GPU') + for gpu in gpus: + tf.config.experimental.set_memory_growth(gpu, True) +else: + from keras.backend.tensorflow_backend import set_session + config = tf.ConfigProto() + config.gpu_options.per_process_gpu_memory_fraction = 0.5 + set_session(tf.Session(config=config)) def pytest_generate_tests(metafunc): @@ -52,20 +57,27 @@ def pytest_generate_tests(metafunc): using_tensorflow_keras = ("tf_keras", {"keras": tf_keras}) -def verify_keras_frontend(keras_model, need_transpose=True): +def verify_keras_frontend(keras_model, need_transpose=True, layout='NCHW'): # Keras frontend currently supports tensorflow backend only. assert(keras.backend.backend() == 'tensorflow') + if layout != 'NCHW': + need_transpose = False + in_shapes = [] for layer in keras_model._input_layers: - in_shapes.append(tuple(dim.value if dim.value is not None else 1 for dim in layer.input.shape)) + if tf.executing_eagerly(): + in_shapes.append(tuple(dim if dim is not None else 1 for dim in layer.input.shape)) + else: + in_shapes.append(tuple(dim.value if dim.value is not None else 1 for dim in layer.input.shape)) + def get_keras_output(xs, dtype='float32'): return keras_model.predict(xs) def get_tvm_output(xs, target, ctx, dtype='float32'): shape_dict = {name: x.shape for (name, x) in zip(keras_model.input_names, xs)} - mod, params = relay.frontend.from_keras(keras_model, shape_dict) + mod, params = relay.frontend.from_keras(keras_model, shape_dict, layout=layout) with relay.transform.build_config(opt_level=2): graph, lib, params = relay.build(mod, target, @@ -357,28 +369,28 @@ def test_forward_rnn(self,keras): verify_keras_frontend(keras_model, need_transpose=False) - def test_forward_vgg16(self, keras): + def test_forward_vgg16(self, keras, layout='NCHW'): keras_model = keras.applications.VGG16(include_top=True, weights='imagenet', input_shape=(224, 224, 3), classes=1000) - verify_keras_frontend(keras_model) + verify_keras_frontend(keras_model, layout=layout) - def test_forward_xception(self, keras): + def test_forward_xception(self, keras, layout='NCHW'): keras_model = keras.applications.Xception(include_top=True, weights='imagenet', input_shape=(299, 299, 3), classes=1000) - verify_keras_frontend(keras_model) + verify_keras_frontend(keras_model, layout=layout) - def test_forward_resnet50(self, keras): + def test_forward_resnet50(self, keras, layout='NCHW'): keras_model = keras.applications.ResNet50(include_top=True, weights='imagenet', input_shape=(224, 224, 3), classes=1000) - verify_keras_frontend(keras_model) + verify_keras_frontend(keras_model, layout=layout) - def test_forward_mobilenet(self, keras): + def test_forward_mobilenet(self, keras, layout='NCHW'): keras_model = keras.applications.MobileNet(include_top=True, weights='imagenet', input_shape=(224, 224, 3), classes=1000) - verify_keras_frontend(keras_model) + verify_keras_frontend(keras_model, layout=layout) if __name__ == '__main__': @@ -402,6 +414,9 @@ def test_forward_mobilenet(self, keras): sut.test_forward_reuse_layers(keras=k) sut.test_forward_rnn(keras=k) sut.test_forward_vgg16(keras=k) + sut.test_forward_vgg16(keras=k, layout='NHWC') sut.test_forward_xception(keras=k) sut.test_forward_resnet50(keras=k) + sut.test_forward_resnet50(keras=k, layout='NHWC') sut.test_forward_mobilenet(keras=k) + sut.test_forward_mobilenet(keras=k, layout='NHWC')