Keras Frontend (#273)

* vgg16 success

* remove six.PY2, use sys.version_info;
convert_activation() accepts activation type name(str, e.g. 'relu') as input;

* add convert_merge

* fix convert_batchnorm;
improve tests

* fix lint

* add numpy-style pad operator

* deal with asymmetry padding

* resnet50 success

* fix pool_convert; xception passes test

* update tvm

* fix bias error; all tests pass

* use > >, not >>

nnvm/docs/top.rst

@@ -47,6 +47,7 @@ This level enables fully connected multi-layer perceptron.
    nnvm.symbol.batch_norm
    nnvm.symbol.softmax
    nnvm.symbol.log_softmax
+   nnvm.symbol.pad
 
 
 **Level 2: Convolutions**
@@ -119,7 +120,7 @@ Detailed Definitions
 .. autofunction:: nnvm.symbol.batch_norm
 .. autofunction:: nnvm.symbol.softmax
 .. autofunction:: nnvm.symbol.log_softmax
-
+.. autofunction:: nnvm.symbol.pad
 
 .. autofunction:: nnvm.symbol.conv2d
 .. autofunction:: nnvm.symbol.conv2d_transpose

nnvm/include/nnvm/top/nn.h

@@ -101,6 +101,21 @@ struct LeakyReLUParam : public dmlc::Parameter<LeakyReLUParam> {
   }
 };
 
+
+struct PadParam : public dmlc::Parameter<PadParam> {
+  float pad_value;
+  Tuple<Tuple<int> > pad_width;
+
+  DMLC_DECLARE_PARAMETER(PadParam) {
+    DMLC_DECLARE_FIELD(pad_value).set_default(0.0)
+      .describe("The value to be padded.");
+    DMLC_DECLARE_FIELD(pad_width)
+      .describe("Number of values padded to the edges of each axis, "
+                "in the format of ((before_1, after_1), ... (before_N, after_N))");
+  }
+};
+
+
 struct Conv2DParam : public dmlc::Parameter<Conv2DParam> {
   int channels;
   TShape kernel_size;

nnvm/python/nnvm/frontend/__init__.py

@@ -3,3 +3,4 @@ from __future__ import absolute_import
 from .mxnet import from_mxnet
 from .onnx import from_onnx
 from .coreml import from_coreml
+from .keras import from_keras

nnvm/python/nnvm/frontend/common.py

@@ -135,3 +135,38 @@ class AttrConverter(object):
         if key not in attr:
             raise AttributeError("Required attribute {} not found.".format(key))
         return attr[key]
+
+
+class SymbolTable(object):
+    """Table storing symbols by names."""
+    def __init__(self):
+        self.vars = {}
+        self.params = {}
+        self.const_ctr = 1
+        self.in_padding = False
+        self.paddings = [0, 0]
+
+    def new_const(self, value):
+        name = "_param_%d" % (self.const_ctr)
+        self.const_ctr += 1
+        self.params[name] = value
+        self.vars[name] = _sym.Variable(name=name)
+        return self.vars[name]
+
+    def get_var(self, name, must_contain=True):
+        if must_contain:
+            assert name in self.vars
+        if name not in self.vars:
+            self.vars[name] = _sym.Variable(name=name)
+        return self.vars[name]
+
+    def set_var(self, name, sym):
+        assert isinstance(sym, _sym.Symbol)
+        self.vars[name] = sym
+
+    def set_padding(self, paddings):
+        self.paddings = paddings
+        self.in_padding = True
+
+    def clear_padding(self):
+        self.in_padding = False

nnvm/python/nnvm/frontend/coreml.py

@@ -4,44 +4,11 @@ from __future__ import absolute_import as _abs
 import tvm
 import numpy as np
 from .. import symbol as _sym
+from .common import SymbolTable
 
 __all__ = ['from_coreml']
 
 
-class SymbolTable(object):
-    """Table storing symbols by names."""
-    def __init__(self):
-        self.vars = {}
-        self.params = {}
-        self.const_ctr = 1
-        self.in_padding = False
-        self.paddings = [0, 0]
-
-    def new_const(self, value):
-        name = "_param_%d" % (self.const_ctr)
-        self.const_ctr += 1
-        self.params[name] = value
-        self.vars[name] = _sym.Variable(name=name)
-        return self.vars[name]
-
-    def get_var(self, name, must_contain=True):
-        if must_contain:
-            assert name in self.vars
-        if name not in self.vars:
-            self.vars[name] = _sym.Variable(name=name)
-        return self.vars[name]
-
-    def set_var(self, name, sym):
-        assert isinstance(sym, _sym.Symbol)
-        self.vars[name] = sym
-
-    def set_padding(self, paddings):
-        self.paddings = paddings
-        self.in_padding = True
-
-    def clear_padding(self):
-        self.in_padding = False
-
 def NeuralNetworkImageScaler(op, insym, symtab):
     # this changes the symbol
     biases = np.array([op.blueBias, op.greenBias, op.redBias]).reshape([3, 1, 1])

nnvm/python/nnvm/frontend/keras.py

+# pylint: disable=invalid-name, import-self
+"""Keras frontend."""
+from __future__ import absolute_import as _abs
+import sys
+import numpy as np
+import tvm
+from .. import symbol as _sym
+from .common import SymbolTable
+
+__all__ = ['from_keras']
+
+
+def _check_data_format(keras_layer):
+    if hasattr(keras_layer, ('data_format')):
+        if keras_layer.data_format != 'channels_last':
+            raise ValueError("Keras frontend currently supports data_format = channels_last only.")
+
+
+def _convert_activation(insym, keras_layer, _):
+    if isinstance(keras_layer, (str, unicode)):
+        act_type = keras_layer
+    else:
+        if sys.version_info.major < 3:
+            act_type = keras_layer.activation.func_name
+        else:
+            act_type = keras_layer.activation.__name__
+    if act_type == 'linear':
+        if isinstance(keras_layer, (str, unicode)):
+            return insym
+        return _sym.__add_scalar__(_sym.__mul_scalar__(insym, \
+            scalar=keras_layer.alpha), scalar=keras_layer.beta)
+    elif act_type == 'softmax':
+        return _sym.softmax(insym)
+    elif act_type == 'sigmoid':
+        return _sym.sigmoid(insym)
+    elif act_type == 'tanh':
+        return _sym.tanh(insym)
+    elif act_type == 'relu':
+        return _sym.relu(insym)
+    elif act_type == 'softplus':
+        return _sym.log(_sym.__add_scalar__(_sym.exp(insym), scalar=1))
+    elif act_type == 'elu':
+        raise NotImplementedError('elu not implemented')
+    elif act_type == 'relu6':
+        raise NotImplementedError('relu6 not implemented')
+    elif act_type == 'softsign':
+        raise NotImplementedError('softsign not implemented')
+    elif act_type == 'hard_sigmoid':
+        raise NotImplementedError('hard_sigmoid not implemented')
+    else:
+        raise TypeError("Unsupported activation type : {}".format(act_type))
+
+
+def _convert_advanced_activation(insym, keras_layer, _):
+    act_type = type(keras_layer).__name__
+    if act_type == 'LeakyReLU':
+        return _sym.leaky_relu(insym, alpha=keras_layer.alpha)
+    elif act_type == 'ELU':
+        raise NotImplementedError('ELU not implemented')
+    elif act_type == 'PReLU':
+        raise NotImplementedError('PReLU not implemented')
+    elif act_type == 'ThresholdedReLU':
+        raise NotImplementedError('ThresholdedReLU not implemented')
+    else:
+        raise TypeError("Unsupported advanced activation type : {}".format(act_type))
+
+
+def _convert_merge(insym, keras_layer, _):
+    merge_type = type(keras_layer).__name__
+    if merge_type == 'Add':
+        return _sym.elemwise_add(insym[0], insym[1])
+    elif merge_type == 'Subtract':
+        return _sym.elemwise_sub(insym[0], insym[1])
+    elif merge_type == 'Multiply':
+        return _sym.elemwise_mul(insym[0], insym[1])
+    elif merge_type == 'Average':
+        raise NotImplementedError('Average merge not implemented')
+    elif merge_type == 'Maximum':
+        raise NotImplementedError('Maximum merge not implemented')
+    else:
+        raise TypeError("Unsupported merge type : {}".format(merge_type))
+
+
+def _convert_dense(insym, keras_layer, symtab):
+    weightList = keras_layer.get_weights()
+    weight = symtab.new_const(weightList[0].transpose([1, 0]))
+    params = {'weight':weight, 'use_bias':False, 'units':weightList[0].shape[1]}
+    if keras_layer.use_bias:
+        params['use_bias'] = True
+        params['bias'] = symtab.new_const(weightList[1])
+    out = _sym.dense(data=insym, **params)
+    # defuse activation
+    if sys.version_info.major < 3:
+        act_type = keras_layer.activation.func_name
+    else:
+        act_type = keras_layer.activation.__name__
+    if act_type != 'linear':
+        out = _convert_activation(out, act_type, symtab)
+    return out
+
+
+def _convert_convolution(insym, keras_layer, symtab):
+    _check_data_format(keras_layer)
+    is_deconv = type(keras_layer).__name__ == 'Conv2DTranspose'
+    is_depthconv = type(keras_layer).__name__ == 'DepthwiseConv2D'
+    weightList = keras_layer.get_weights()
+    if is_deconv:
+        kernel_h, kernel_w, n_filters, in_channels = weightList[0].shape
+        weight = weightList[0].transpose([3, 2, 0, 1])
+    elif is_depthconv:
+        kernel_h, kernel_w, in_channels, depth_mult = weightList[0].shape
+        weight = weightList[0].transpose([2, 3, 0, 1])
+    else:
+        kernel_h, kernel_w, in_channels, n_filters = weightList[0].shape
+        weight = weightList[0].transpose([3, 2, 0, 1])
+    dilation = [1, 1]
+    if isinstance(keras_layer.dilation_rate, (list, tuple)):
+        dilation = [keras_layer.dilation_rate[0], keras_layer.dilation_rate[1]]
+    else:
+        dilation = [keras_layer.dilation_rate, keras_layer.dilation_rate]
+    stride_h, stride_w = keras_layer.strides
+    params = {'weight': symtab.new_const(weight),
+              'kernel_size': [kernel_h, kernel_w],
+              'strides': [stride_h, stride_w],
+              'dilation': dilation,
+              'padding': [0, 0],
+              'use_bias': False}
+    if is_depthconv:
+        params['channels'] = in_channels * depth_mult
+        params['groups'] = in_channels
+    else:
+        params['channels'] = n_filters
+    if keras_layer.use_bias:
+        params['use_bias'] = True
+        params['bias'] = symtab.new_const(weightList[1])
+    if keras_layer.padding == 'valid':
+        pass
+    # we insert a separate pad operator
+    elif keras_layer.padding == 'same':
+        in_h = keras_layer.input.shape[1].value
+        in_w = keras_layer.input.shape[2].value
+        out_h = (in_h + stride_h - 1) // stride_h
+        out_w = (in_w + stride_w - 1) // stride_w
+        pad_h = np.maximum((out_h - 1) * stride_h + kernel_h - in_h, 0)
+        pad_w = np.maximum((out_w - 1) * stride_w + kernel_w - in_w, 0)
+        pad_t = pad_h // 2
+        pad_l = pad_w // 2
+        pad_b = pad_h - pad_t
+        pad_r = pad_w - pad_l
+        insym = _sym.pad(data=insym, pad_width=((0, 0), (0, 0), (pad_t, pad_b), (pad_l, pad_r)))
+    else:
+        raise TypeError("Unsupported padding type : {}".format(keras_layer.padding))
+    if is_deconv:
+        out = _sym.conv2d_transpose(data=insym, **params)
+    else:
+        out = _sym.conv2d(data=insym, **params)
+    # defuse activation
+    if sys.version_info.major < 3:
+        act_type = keras_layer.activation.func_name
+    else:
+        act_type = keras_layer.activation.__name__
+    if act_type != 'linear':
+        out = _convert_activation(out, act_type, symtab)
+    return out
+
+
+def _convert_separable_convolution(insym, keras_layer, symtab):
+    _check_data_format(keras_layer)
+    weightList = keras_layer.get_weights()
+    # depthwise conv
+    kernel_h, kernel_w, in_channels, depth_mult = weightList[0].shape
+    stride_h, stride_w = keras_layer.strides
+    weight0 = weightList[0].transpose([2, 3, 0, 1])
+    params0 = {'weight': symtab.new_const(weight0),
+               'channels': in_channels * depth_mult,
+               'groups': in_channels,
+               'kernel_size': [kernel_h, kernel_w],
+               'strides': [stride_h, stride_w],
+               'dilation': [1, 1],
+               'padding': [0, 0],
+               'use_bias': False}
+    if keras_layer.padding == 'valid':
+        pass
+    # we insert a separate pad operator
+    elif keras_layer.padding == 'same':
+        in_h = keras_layer.input.shape[1].value
+        in_w = keras_layer.input.shape[2].value
+        out_h = (in_h + stride_h - 1) // stride_h
+        out_w = (in_w + stride_w - 1) // stride_w
+        pad_h = np.maximum((out_h - 1) * stride_h + kernel_h - in_h, 0)
+        pad_w = np.maximum((out_w - 1) * stride_w + kernel_w - in_w, 0)
+        pad_t = pad_h // 2
+        pad_l = pad_w // 2
+        pad_b = pad_h - pad_t
+        pad_r = pad_w - pad_l
+        insym = _sym.pad(data=insym, pad_width=(
+            (0, 0), (0, 0), (pad_t, pad_b), (pad_l, pad_r)))
+    else:
+        raise TypeError("Unsupported padding type : {}".format(keras_layer.padding))
+    depthconv = _sym.conv2d(data=insym, **params0)
+    # pointwise conv
+    weight1 = weightList[1].transpose([3, 2, 0, 1])
+    params1 = {'weight': symtab.new_const(weight1),
+               'channels': weight1.shape[0],
+               'groups': 1,
+               'kernel_size': [1, 1],
+               'strides': [1, 1],
+               'dilation': [1, 1],
+               'use_bias': False}
+    if keras_layer.use_bias:
+        params1['use_bias'] = True
+        params1['bias'] = symtab.new_const(weightList[2])
+    out = _sym.conv2d(data=depthconv, **params1)
+    # defuse activation
+    if sys.version_info.major < 3:
+        act_type = keras_layer.activation.func_name
+    else:
+        act_type = keras_layer.activation.__name__
+    if act_type != 'linear':
+        out = _convert_activation(out, act_type, symtab)
+    return out
+
+
+def _convert_flatten(insym, keras_layer, _):
+    _check_data_format(keras_layer)
+    # NCHW -> NHWC so that dense can be correctly converted
+    insym = _sym.transpose(insym, axes=[0, 2, 3, 1])
+    return _sym.flatten(insym)
+
+
+def _convert_pooling(insym, keras_layer, symtab):
+    _check_data_format(keras_layer)
+    pool_type = type(keras_layer).__name__
+    # global pool in keras = global pool + flatten in nnvm
+    if pool_type == 'GlobalMaxPooling2D':
+        return _convert_flatten(_sym.global_max_pool2d(insym), keras_layer, symtab)
+    elif pool_type == 'GlobalAveragePooling2D':
+        return _convert_flatten(_sym.global_avg_pool2d(insym), keras_layer, symtab)
+    else:
+        pool_h, pool_w = keras_layer.pool_size
+        stride_h, stride_w = keras_layer.strides
+        params = {'pool_size': [pool_h, pool_w],
+                  'strides': [stride_h, stride_w]}
+        if keras_layer.padding == 'valid':
+            params['padding'] = [0, 0]
+        elif keras_layer.padding == 'same':
+            in_h = keras_layer.input.shape[1].value
+            in_w = keras_layer.input.shape[2].value
+            out_h = (in_h + stride_h - 1) // stride_h
+            out_w = (in_w + stride_w - 1) // stride_w
+            pad_h = np.maximum((out_h - 1) * stride_h + pool_h - in_h, 0)
+            pad_w = np.maximum((out_w - 1) * stride_w + pool_w - in_w, 0)
+            pad_t = pad_h // 2
+            pad_l = pad_w // 2
+            pad_b = pad_h - pad_t
+            pad_r = pad_w - pad_l
+            params['padding'] = [pad_t, pad_l]
+            if pad_b > pad_t and pad_r > pad_l:
+                params['ceil_mode'] = True
+        else:
+            raise TypeError("Unsupported padding type : {}".format(keras_layer.padding))
+        if pool_type == 'MaxPooling2D':
+            return _sym.max_pool2d(insym, **params)
+        elif pool_type == 'AveragePooling2D':
+            # TODO: in keras, padded zeros are not calculated
+            return _sym.avg_pool2d(insym, **params)
+        else:
+            raise TypeError("Unsupported pooling type : {}".format(keras_layer))
+
+
+def _convert_batchnorm(insym, keras_layer, symtab):
+    params = {'scale': False,
+              'center': False,
+              'epsilon': keras_layer.epsilon}
+    idx = 0
+    if keras_layer.scale:
+        params['scale'] = True
+        gamma = keras_layer.get_weights()[idx]
+        params['gamma'] = symtab.new_const(gamma)
+        idx += 1
+    if keras_layer.center:
+        params['center'] = True
+        beta = keras_layer.get_weights()[idx]
+        params['beta'] = symtab.new_const(beta)
+        idx += 1
+    moving_mean = keras_layer.get_weights()[idx]
+    moving_var = keras_layer.get_weights()[idx + 1]
+    params['moving_mean'] = symtab.new_const(moving_mean)
+    params['moving_var'] = symtab.new_const(moving_var)
+    return _sym.batch_norm(data=insym, **params)
+
+
+def _convert_padding(insym, keras_layer, _):
+    _check_data_format(keras_layer)
+    padding_type = type(keras_layer).__name__
+    padding = keras_layer.padding
+    top = left = bottom = right = 0
+    if padding_type == 'ZeroPadding2D':
+        if isinstance(padding, int):
+            top = left = bottom = right = padding
+        elif isinstance(padding, tuple):
+            if isinstance(padding[0], int):
+                top, left = padding
+                bottom, right = padding
+            elif isinstance(padding[0], tuple):
+                top, bottom = padding[0]
+                left, right = padding[1]
+            else:
+                raise ValueError("Unrecognized padding option: {}".format(str(padding)))
+        else:
+            raise ValueError("Unrecognized padding option: {}".format(str(padding)))
+    elif padding_type == 'ZeroPadding1D':
+        raise NotImplementedError("ZeroPadding1D not implemented")
+    else:
+        raise ValueError("Unrecognized padding type: {}".format(padding_type))
+    return _sym.pad(data=insym, pad_width=((0, 0), (0, 0), (top, bottom), (left, right)))
+
+
+def _convert_concat(insym, keras_layer, _):
+    _check_data_format(keras_layer)
+    if not isinstance(insym, list):
+        insym = [insym]
+    return _sym.concatenate(*insym, axis=1)
+
+
+def _convert_reshape(insym, keras_layer, _):
+    return _sym.reshape(insym, keras_layer.shape)
+
+
+def _default_skip(insym, keras_layer, _): # pylint: disable=unused-argument
+    """Layers that can be skipped because they are train time only."""
+    return
+
+
+_convert_map = {
+    'Dense'                    : _convert_dense,
+    'Activation'               : _convert_activation,
+    'LeakyReLU'                : _convert_advanced_activation,
+    'PReLU'                    : _convert_advanced_activation,
+    'ELU'                      : _convert_advanced_activation,
+    'ThresholdedReLU'          : _convert_advanced_activation,
+
+    'AveragePooling2D'         : _convert_pooling,
+    'MaxPooling2D'             : _convert_pooling,
+    'GlobalAveragePooling2D'   : _convert_pooling,
+    'GlobalMaxPooling2D'       : _convert_pooling,
+    'Conv2D'                   : _convert_convolution,
+    'Conv2DTranspose'          : _convert_convolution,
+    'DepthwiseConv2D'          : _convert_convolution,
+    'SeparableConv2D'          : _convert_separable_convolution,
+
+    'Flatten'                  : _convert_flatten,
+    'Reshape'                  : _convert_reshape,
+    'Concatenate'              : _convert_concat,
+    'BatchNormalization'       : _convert_batchnorm,
+
+    'Add'                      : _convert_merge,
+    'Subtract'                 : _convert_merge,
+    'Multiply'                 : _convert_merge,
+    'ZeroPadding2D'            : _convert_padding,
+
+    # 'ZeroPadding1D'          : _convert_padding,
+    # 'AveragePooling1D'       : _convert_pooling,
+    # 'MaxPooling1D'           : _convert_pooling,
+    # 'GlobalAveragePooling1D' : _convert_pooling,
+    # 'GlobalMaxPooling1D'     : _convert_pooling,
+    # 'Cropping1D'             : _convert_cropping,
+    # 'Cropping2D'             : _convert_cropping,
+    # 'UpSampling1D'           : _convert_upsample,
+    # 'UpSampling2D'           : _convert_upsample,
+    # 'Conv1D'                 : _convert_convolution1d,
+
+    # 'GRU'                    : _convert_gru,
+    # 'LSTM'                   : _convert_lstm,
+    # 'SimpleRNN'              : _convert_simple_rnn,
+    # 'Bidirectional'          : _convert_bidirectional,
+    # 'TimeDistributed'        : _default_skip,
+
+    # 'Average'                : _convert_merge,
+    # 'Maximum'                : _convert_merge,
+    # 'Dot'                    : _convert_merge,
+    # 'Permute'                : _convert_permute,
+    # 'Embedding'              : _convert_embedding,
+    # 'RepeatVector'           : _convert_repeat_vector,
+
+    'InputLayer'               : _default_skip,
+    'Dropout'                  : _default_skip,
+    'SpatialDropout2D'         : _default_skip,
+    'SpatialDropout1D'         : _default_skip,
+}
+
+
+def _check_unsupported_layers(model):
+    for layer in model.layers:
+        if type(layer).__name__ not in _convert_map:
+            raise ValueError("Keras layer {} not supported.".format(type(layer).__name__))
+
+
+def keras_op_to_nnvm(insym, keras_layer, outname, symtab):
+    """Convert keras layer to nnvm symbol, and update symtab.
+
+    Parameters
+    ----------
+    insym : nnvm.symbol.Symbol or a list of it
+        The input nnvm symbol(s)
+
+    keras_layer : keras.layers
+        The keras layer to be converted
+
+    outname : str
+        Name of the output nnvm symbol
+
+    symtab : nnvm.frontend.common.SymbolTable
+        The global symbol table to be updated
+    """
+    if type(keras_layer).__name__ not in _convert_map:
+        raise NotImplementedError("{} is not supported".format((type(keras_layer).__name__)))
+    ret = _convert_map[type(keras_layer).__name__](insym, keras_layer, symtab)
+    symtab.set_var(outname, ret)
+
+
+def from_keras(model):
+    """Convert keras model to NNVM format.
+
+    Parameters
+    ----------
+    model : keras.engine.training.Model
+        The keras model to be converted
+
+    Returns
+    -------
+    sym : nnvm.Symbol
+        Compatible nnvm symbol
+
+    params : dict of str to tvm.NDArray
+        The parameter dict to be used by nnvm
+    """
+    try:
+        import keras
+    except ImportError:
+        raise ImportError('Keras must be installed')
+    assert isinstance(model, keras.engine.training.Model)
+    if keras.backend.image_data_format() != 'channels_last':
+        raise ValueError("Keras frontend currently supports data_format = channels_last only.")
+    _check_unsupported_layers(model)
+
+    symtab = SymbolTable()
+    for keras_layer in model.layers:
+        if isinstance(keras_layer, keras.engine.topology.InputLayer):
+            keras_layer.name = 'data'
+            symtab.get_var(keras_layer.name, must_contain=False)
+        else:
+            predecessors = []
+            for node in keras_layer.inbound_nodes:
+                for pred in node.inbound_layers:
+                    predecessors.append(pred.name)
+            if len(predecessors) == 1:
+                insym = symtab.get_var(predecessors[0], must_contain=True)
+            else:
+                insym = [symtab.get_var(pred, must_contain=True) for pred in predecessors]
+            keras_op_to_nnvm(insym, keras_layer, keras_layer.name, symtab)
+
+    returns = [symtab.get_var(i.name, must_contain=False) for i in model.output_layers]
+    tvmparams = {k:tvm.nd.array(np.array(v, dtype=np.float32)) for k, v in symtab.params.items()}
+    return returns[0], tvmparams

nnvm/python/nnvm/top/attr_dict.py

@@ -52,6 +52,22 @@ class AttrDict(object):
         """
         return tuple(int(x) for x in self[key][1:-1].split(",") if x)
 
+    def get_int_pair_tuple(self, key):
+        """Get tuple of integer pairs from attr dict
+
+        Parameters
+        ----------
+        key : str
+            The attr key
+
+        Returns
+        -------
+        tuple : tuple of int pairs
+            The result tuple
+        """
+        flat = [int(x.strip(' [] ')) for x in self[key][1:-1].split(",")]
+        return tuple((flat[i], flat[i+1]) for i in range(0, len(flat), 2))
+
     def get_int(self, key):
         """Get integer from attr dict
 

nnvm/python/nnvm/top/nn.py

@@ -39,6 +39,22 @@ reg.register_schedule("flatten", _fschedule_broadcast)
 reg.register_pattern("flatten", OpPattern.INJECTIVE)
 
 
+# pad
+@reg.register_compute("pad")
+def compute_pad(attrs, inputs, _):
+    """Compute definition of pad"""
+    pad_width = attrs.get_int_pair_tuple('pad_width')
+    assert len(pad_width) == len(inputs[0].shape) and \
+        len(pad_width[0]) == 2, "illegal pad_width"
+    pad_before = [x[0] for x in pad_width]
+    pad_after = [x[1] for x in pad_width]
+    pad_value = attrs.get_int('pad_value')
+    return topi.nn.pad(inputs[0], pad_before, pad_after, pad_value)
+
+reg.register_schedule("pad", _fschedule_broadcast)
+reg.register_pattern("pad", OpPattern.INJECTIVE)
+
+
 # softmax
 @reg.register_compute("softmax")
 def compute_softmax(attrs, inputs, _):
@@ -53,9 +69,9 @@ def schedule_softmax(_, outs, target):
     with tvm.target.create(target):
         return topi.generic.schedule_softmax(outs)
 
-
 reg.register_pattern("softmax", OpPattern.OPAQUE)
 
+
 # log softmax
 @reg.register_compute("log_softmax")
 def compute_log_softmax(attrs, inputs, _):
@@ -73,6 +89,7 @@ def schedule_log_softmax(_, outs, target):
 # Mark softmax as extern as we do not fuse it in call cases
 reg.register_pattern("log_softmax", OpPattern.OPAQUE)
 
+
 # dense
 @reg.register_compute("dense")
 def compute_dense(attrs, inputs, _):

nnvm/src/top/nn/nn.cc

@@ -254,5 +254,42 @@ NNVM_REGISTER_OP(leaky_relu)
 .set_attr<FInferType>("FInferType", ElemwiseType<1, 1>)
 .set_support_level(1);
 
+
+DMLC_REGISTER_PARAMETER(PadParam);
+
+inline bool PadInferShape(const nnvm::NodeAttrs& attrs,
+                          std::vector<TShape>* in_shape,
+                          std::vector<TShape>* out_shape) {
+  const PadParam& param = nnvm::get<PadParam>(attrs.parsed);
+  CHECK_EQ(in_shape->size(), 1U);
+  CHECK_EQ(out_shape->size(), 1U);
+  TShape dshape = (*in_shape)[0];
+  if (dshape.ndim() == 0) return false;
+  CHECK_EQ(param.pad_width.ndim(), dshape.ndim());
+  CHECK_EQ(param.pad_width[0].ndim(), 2U);
+  TShape oshape = dshape;
+  for (uint32_t i = 0; i < dshape.ndim(); i++) {
+    int pad_before = param.pad_width[i][0];
+    int pad_after = param.pad_width[i][1];
+    oshape[i] = dshape[i] + pad_before + pad_after;
+  }
+  NNVM_ASSIGN_OUTPUT_SHAPE(attrs, *out_shape, 0, oshape);
+  return true;
+}
+
+NNVM_REGISTER_OP(pad)
+.describe(R"code(Pad for n-D tensor.
+
+)code" NNVM_ADD_FILELINE)
+.add_argument("data", "n-D Tensor", "Input data.")
+.add_arguments(PadParam::__FIELDS__())
+.set_attr_parser(ParamParser<PadParam>)
+.set_attr<FGetAttrDict>("FGetAttrDict", ParamGetAttrDict<PadParam>)
+.set_num_outputs(1)
+.set_num_inputs(1)
+.set_attr<FInferShape>("FInferShape", PadInferShape)
+.set_attr<FInferType>("FInferType", ElemwiseType<1, 1>)
+.set_support_level(1);
+
 }  // namespace top
 }  // namespace nnvm

nnvm/tests/python/compiler/test_top_level1.py

@@ -244,6 +244,25 @@ def test_squeeze():
     verify_squeeze((1, 3, 1), axis=0)
     verify_squeeze((1, 3, 2, 5, 1), axis=-1)
 
+
+def test_pad():
+    x = sym.Variable("x")
+    y = sym.pad(x, pad_width=((0, 0), (0, 0), (0, 1), (2, 3)), pad_value=1.)
+    dtype = "float32"
+    dshape = (1, 3, 28, 28)
+    oshape = (1, 3, 29, 33)
+    shape_dict = {"x": dshape}
+    for target, ctx in ctx_list():
+        graph, lib, _ = nnvm.compiler.build(y, target, shape_dict)
+        m = graph_runtime.create(graph, lib, ctx)
+        data = tvm.nd.array(np.random.uniform(size=dshape).astype(dtype))
+        m.run(x=data)
+        out = m.get_output(0, tvm.nd.empty(oshape, dtype))
+        b_np = np.pad(data.asnumpy(), pad_width=((0, 0), (0, 0), (0, 1), (2, 3)),
+            mode='constant', constant_values=1.)
+        np.testing.assert_allclose(out.asnumpy(), b_np, rtol=1e-5)
+
+
 if __name__ == "__main__":
     test_split()
     test_concatenate()
@@ -257,3 +276,4 @@ if __name__ == "__main__":
     test_sigmoid()
     test_softmax()
     test_squeeze()
+    test_pad()

nnvm/tests/python/frontend/keras/test_forward.py

+import numpy as np
+import nnvm
+import tvm
+from tvm.contrib import graph_runtime
+from nnvm.testing.config import ctx_list
+import keras
+
+# prevent keras from using up all gpu memory
+import tensorflow as tf
+from keras.backend.tensorflow_backend import set_session
+config = tf.ConfigProto()
+config.gpu_options.per_process_gpu_memory_fraction = 0.5
+set_session(tf.Session(config=config))
+
+
+def verify_keras_frontend(keras_model):
+    in_shape = [dim.value if dim.value is not None else 1 for dim in keras_model.input_layers[0].input.shape]
+    out_shape = [dim.value if dim.value is not None else 1 for dim in keras_model.output_layers[0].output.shape]
+
+    def get_keras_output(x, dtype='float32'):
+        return keras_model.predict(x)
+
+    def get_tvm_output(x, target, ctx, input_name='data', dtype='float32'):
+        sym, params = nnvm.frontend.from_keras(keras_model)
+        shape_dict = {input_name : x.shape}
+        with nnvm.compiler.build_config(opt_level=2):
+            graph, lib, params = nnvm.compiler.build(sym, target, shape_dict, params=params)
+        m = graph_runtime.create(graph, lib, ctx)
+        m.set_input(input_name, tvm.nd.array(x.astype(dtype)))
+        m.set_input(**params)
+        m.run()
+        out = m.get_output(0, tvm.nd.empty(out_shape, dtype))
+        return out.asnumpy()
+
+    x = np.random.uniform(size=in_shape)
+    keras_out = get_keras_output(x)
+    for target, ctx in ctx_list():
+        tvm_out = get_tvm_output(x.transpose([0,3,1,2]), target, ctx)
+        np.testing.assert_allclose(keras_out, tvm_out, rtol=1e-5, atol=1e-5)
+
+
+def verify_forward_softrelu():
+    data = keras.layers.Input(shape=(32,32,3))
+    x = keras.layers.Activation('softplus')(data)
+    x = keras.layers.Concatenate()([x, x])
+    x = keras.layers.GlobalMaxPooling2D()(x)
+    keras_model = keras.models.Model(data, x)
+    verify_keras_frontend(keras_model)
+
+
+def verify_forward_leaky_relu():
+    data = keras.layers.Input(shape=(32,32,3))
+    x = keras.layers.LeakyReLU(alpha=0.3)(data)
+    x = keras.layers.Add()([x, x])
+    x = keras.layers.GlobalAveragePooling2D()(x)
+    keras_model = keras.models.Model(data, x)
+    verify_keras_frontend(keras_model)
+
+
+def verify_forward_dense():
+    data = keras.layers.Input(shape=(32,32,3))
+    x = keras.layers.MaxPooling2D(pool_size=(2,2))(data)
+    x = keras.layers.Flatten()(x)
+    x = keras.layers.Dense(10, activation='relu', kernel_initializer='uniform')(x)
+    keras_model = keras.models.Model(data, x)
+    verify_keras_frontend(keras_model)
+
+
+def verify_forward_transpose_conv():
+    data = keras.layers.Input(shape=(32,32,3))
+    x = keras.layers.Conv2D(filters=10, kernel_size=(3,3), strides=(2,2), padding='same')(data)
+    x = keras.applications.mobilenet.DepthwiseConv2D(kernel_size=(3,3), padding='same')(x)
+    x = keras.layers.Conv2DTranspose(filters=64, kernel_size=(3,3), padding='valid')(x)
+    x = keras.layers.GlobalMaxPooling2D()(x)
+    keras_model = keras.models.Model(data, x)
+    verify_keras_frontend(keras_model)
+
+
+def verify_forward_separable_conv():
+    data = keras.layers.Input(shape=(32,32,3))
+    x = keras.layers.SeparableConv2D(filters=10, kernel_size=(3,3),
+        padding='same', activation='relu')(data)
+    x = keras.layers.BatchNormalization(scale=True, center=False,
+        beta_initializer='uniform', gamma_initializer='uniform')(x)
+    x = keras.layers.GlobalAveragePooling2D()(x)
+    keras_model = keras.models.Model(data, x)
+    verify_keras_frontend(keras_model)
+
+
+def verify_forward_vgg16():
+    keras_model = keras.applications.vgg16.VGG16(include_top=True, weights='imagenet',
+        input_shape=(224,224,3), classes=1000)
+    verify_keras_frontend(keras_model)
+
+
+def verify_forward_xception():
+    keras_model = keras.applications.xception.Xception(include_top=True, weights='imagenet',
+        input_shape=(299,299,3), classes=1000)
+    verify_keras_frontend(keras_model)
+
+
+def verify_forward_resnet50():
+    keras_model = keras.applications.resnet50.ResNet50(include_top=True, weights='imagenet',
+        input_shape=(224,224,3), classes=1000)
+    verify_keras_frontend(keras_model)
+
+
+if __name__ == '__main__':
+    verify_forward_softrelu()
+    verify_forward_leaky_relu()
+    verify_forward_dense()
+    verify_forward_transpose_conv()
+    verify_forward_separable_conv()
+
+    verify_forward_vgg16()
+    verify_forward_xception()
+    verify_forward_resnet50()