[NNVM][TENSORFLOW] bug fix on bilinear and resize op integration in frontend. (#1440)

nnvm/python/nnvm/frontend/tensorflow.py

@@ -332,26 +332,14 @@ def _expand_dims():
 
 def _resize_bilinear():
     def _impl(inputs, attr, params):
-        # Change this when we have corresponding resize bilinear operation.
-        print("ResizeBilinear:Only NN (nearest neighbor) supported in symetric mode of dimensions")
-        print("Change this when we have corresponding resize bilinear operation")
-
-        # NHWC
-        input_shape = attr['_input_shapes'][inputs[0]][0]
-        in_hw = (input_shape[1], input_shape[2])
-        out_hw = params[inputs[1].list_output_names()[0]]
+        attr['size'] = attr['_output_shapes'][0][1:3]
         inputs.pop(1)
+        # NHWC
         attr['layout'] = 'NHWC'
 
-        if in_hw[0] < 0 or in_hw[1] < 0:
-            scale = 1
-        else:
-            # Considering height alone for scale
-            scale = out_hw[0] / in_hw[0]
-
-        return AttrCvt(op_name="upsampling",
-                       ignores=['Tdim', 'align_corners'],
-                       extras={'scale': scale})(inputs, attr)
+        return AttrCvt(op_name="resize",
+                       ignores=['Tdim'],
+                       extras={'method': "BILINEAR"})(inputs, attr)
     return _impl
 
 def _check_numerics():

nnvm/python/nnvm/testing/tf.py

@@ -6,7 +6,6 @@ Some helper definitions for tensorflow models.
 """
 import re
 import os.path
-import numpy as np
 
 # Tensorflow imports
 import tensorflow as tf
@@ -107,52 +106,6 @@ class NodeLookup(object):
             return ''
         return self.node_lookup[node_id]
 
-def read_normalized_tensor_from_image_file(file_name,
-                                           input_height=299,
-                                           input_width=299,
-                                           input_mean=0,
-                                           input_std=255):
-    """ Preprocessing of image
-    Parameters
-    ----------
-
-    file_name: String
-        Image filename.
-
-    input_height: int
-        model input height.
-
-    input_width: int
-        model input width
-
-    input_mean: int
-        Mean to be substracted in normalization.
-
-    input_std: int
-        Standard deviation used in normalization.
-
-    Returns
-    -------
-
-    np_array: Numpy array
-        Normalized image data as a numpy array.
-
-    """
-
-    input_name = "file_reader"
-    output_name = "normalized"
-    file_reader = tf.read_file(file_name, input_name)
-
-    image_reader = tf.image.decode_jpeg(file_reader, channels=3,
-                                        name='jpeg_reader')
-    float_caster = tf.cast(image_reader, tf.float32)
-    dims_expander = tf.expand_dims(float_caster, 0)
-    resized = tf.image.resize_bilinear(dims_expander, [input_height, input_width])
-    normalized = tf.divide(tf.subtract(resized, [input_mean]), [input_std])
-    tf.InteractiveSession()
-    np_array = normalized.eval()
-    return np_array
-
 def get_workload(model_path):
     """ Import workload from frozen protobuf
 
@@ -181,79 +134,3 @@ def get_workload(model_path):
         graph_def.ParseFromString(f.read())
         graph = tf.import_graph_def(graph_def, name='')
         return graph_def
-
-def get_workload_inception_v3():
-    """ Import Inception V3 workload from frozen protobuf
-
-    Parameters
-    ----------
-        Nothing.
-
-    Returns
-    -------
-    (normalized, graph_def) : Tuple
-        normalized is normalized input for graph testing.
-        graph_def is the tensorflow workload for Inception V3.
-    """
-
-    repo_base = 'https://github.com/dmlc/web-data/raw/master/tensorflow/models/InceptionV3/'
-    model_path = 'InceptionV3/inception_v3_2016_08_28_frozen-with_shapes.pb'
-
-    image_name = 'elephant-299.jpg'
-    image_url = os.path.join(repo_base, image_name)
-    from mxnet.gluon.utils import download
-    download(image_url, image_name)
-    normalized = read_normalized_tensor_from_image_file(os.path.join("./", image_name))
-
-    return (normalized, get_workload(model_path))
-
-def get_workload_inception_v1():
-    """ Import Inception V1 workload from frozen protobuf
-
-    Parameters
-    ----------
-        Nothing.
-
-    Returns
-    -------
-    (image_data, tvm_data, graph_def) : Tuple
-        image_data is raw encoded image data for TF input.
-        tvm_data is the decoded image data for TVM input.
-        graph_def is the tensorflow workload for Inception V1.
-
-    """
-
-    repo_base = 'https://github.com/dmlc/web-data/raw/master/tensorflow/models/InceptionV1/'
-    model_path = 'InceptionV1/classify_image_graph_def-with_shapes.pb'
-    image_name = 'elephant-299.jpg'
-    image_url = os.path.join(repo_base, image_name)
-
-    from mxnet.gluon.utils import download
-    download(image_url, image_name)
-
-    if not tf.gfile.Exists(os.path.join("./", image_name)):
-        tf.logging.fatal('File does not exist %s', image)
-    image_data = tf.gfile.FastGFile(os.path.join("./", image_name), 'rb').read()
-
-    # TVM doesn't handle decode, hence decode it.
-    from PIL import Image
-    tvm_data = Image.open(os.path.join("./", image_name)).resize((299, 299))
-    tvm_data = np.array(tvm_data)
-
-    return (image_data, tvm_data, get_workload(model_path))
-
-def get_workload_mobilenet():
-    """ Import mobilenet workload from frozen protobuf
-
-    Parameters
-    ----------
-        Nothing.
-
-    Returns
-    -------
-    graph_def: graphdef
-        graph_def is the tensorflow workload for mobilenet.
-
-    """
-
-    return get_workload("MobilenetV1/mobilenet_v1_1.0_224_frozen-with-shapes.pb")

nnvm/src/top/image/resize.cc

@@ -40,13 +40,9 @@ inline bool ResizeInferShape(const nnvm::NodeAttrs& attrs,
   dshape = ConvertLayout(dshape, param.layout, kNCHW);
 
   TShape oshape = dshape;
-  if (param.layout == "NCHW") {
   oshape[2] = param.size[0];
   oshape[3] = param.size[1];
-  } else {
-    oshape[1] = param.size[0];
-    oshape[2] = param.size[1];
-  }
+
   oshape = ConvertLayout(oshape, kNCHW, param.layout);
   NNVM_ASSIGN_OUTPUT_SHAPE(attrs, *out_shape, 0, oshape);
 

nnvm/tests/python/compiler/test_top_level2.py

@@ -300,11 +300,10 @@ def test_upsampling_bilinear():
 
 def test_resize_bilinear():
     x = sym.Variable("x")
-    scale = 2
-    y = sym.upsampling(x, scale=scale, method="BILINEAR", name="y", layout="NHWC")
+    y = sym.resize(x, size=(60, 60), method="BILINEAR", name="y", layout="NHWC")
     dtype = "float32"
     dshape = (1, 32, 32, 4)
-    oshape = (1, 32*scale, 32*scale, 4)
+    oshape = (1, 60, 60, 4)
     shape_dict = {"x": dshape}
     dtype_dict = {"x": dtype}
     for target, ctx in ctx_list():
@@ -314,7 +313,7 @@ def test_resize_bilinear():
         data = tvm.nd.array(a_np)
         m.run(x=data)
         out = m.get_output(0, tvm.nd.empty(oshape, dtype))
-        b_np = topi.testing.bilinear_resize_python(a_np, (32*scale, 32*scale), "NHWC")
+        b_np = topi.testing.bilinear_resize_python(a_np, (60, 60), "NHWC")
         np.testing.assert_allclose(out.asnumpy(), b_np, rtol=1e-5, atol=1e-5)
 
 if __name__ == "__main__":

nnvm/tests/python/frontend/tensorflow/test_forward.py

@@ -470,23 +470,64 @@ def test_forward_multi_input():
             sess.close()
 
 #######################################################################
+# Resize Bilinear
+# ---------------
+
+def _test_resize_bilinear(in_shape, to_shape, align_corners):
+    """ One iteration of resize bilinear """
+
+    data = np.random.uniform(size=in_shape).astype('float32')
+    shape_data = np.array(to_shape).astype('int32')
+
+    with tf.Graph().as_default():
+        in_data = constant_op.constant(data, shape=data.shape, dtype=data.dtype)
+        shape_data = constant_op.constant(shape_data, shape=shape_data.shape, dtype=shape_data.dtype)
+
+        # pylint: disable=unused-variable
+        resize_out = tf.image.resize_bilinear(in_data, shape_data, align_corners=align_corners)
+        # pylint: enable=unused-variable
+
+        with tf.Session() as sess:
+            graph_def = tf.graph_util.convert_variables_to_constants(
+                sess,
+                sess.graph.as_graph_def(add_shapes=True),
+                ['ResizeBilinear'],
+                )
+
+            tf_output = run_tf_graph(sess, data,
+                    'Const:0', 'ResizeBilinear:0')
+
+            tvm_output = run_tvm_graph(graph_def,
+                                       data,
+                                       "Const", tf_output.shape, data.dtype)
+
+            np.testing.assert_allclose(tf_output, tvm_output, atol=1e-3, rtol=1e-3)
+
+            sess.close()
+
+def test_forward_resize_bilinear():
+    """ Resize Bilinear """
+
+    _test_resize_bilinear((4, 16, 32, 32), [50, 50], False)
+    _test_resize_bilinear((6, 32, 64, 64), [20, 20], True)
+
+
+#######################################################################
 # Inception V3
 # ------------
 def test_forward_inception_v3():
     '''test inception V3 model'''
     with tf.Graph().as_default():
-        (data, graph_def) = nnvm.testing.tf.get_workload_inception_v3()
+        graph_def = nnvm.testing.tf.get_workload('InceptionV3/inception_v3_2016_08_28_frozen-with_shapes.pb')
         # Call the utility to import the graph definition into default graph.
         graph_def = nnvm.testing.tf.ProcessGraphDefParam(graph_def)
 
-        tvm_output = run_tvm_graph(graph_def, data, 'input', (1, 1001), 'float32')
+        data = np.random.uniform(size=(1, 299, 299, 3)).astype('float32')
+
         with tf.Session() as sess:
             tf_output = run_tf_graph(sess, data, 'input:0', 'InceptionV3/Predictions/Reshape_1:0')
-
-            top_tvm = np.squeeze(tvm_output).argsort()[-3:][::-1]
-            top_tf = np.squeeze(tf_output).argsort()[-3:][::-1]
-
-            np.testing.assert_allclose(top_tf, top_tvm, rtol=1e-5, atol=1e-5)
+            tvm_output = run_tvm_graph(graph_def, data, 'input', tf_output.shape, 'float32')
+            np.testing.assert_allclose(tf_output, tvm_output, rtol=1e-5, atol=1e-5)
 
 #######################################################################
 # Inception V1
@@ -494,16 +535,35 @@ def test_forward_inception_v3():
 def test_forward_inception_v1():
     '''test inception V1 model'''
     with tf.Graph().as_default():
-        (data, tvm_data, graph_def) = nnvm.testing.tf.get_workload_inception_v1()
+        graph_def = nnvm.testing.tf.get_workload("InceptionV1/classify_image_graph_def-with_shapes.pb")
         # Call the utility to import the graph definition into default graph.
         graph_def = nnvm.testing.tf.ProcessGraphDefParam(graph_def)
 
-        tvm_output = run_tvm_graph(graph_def, tvm_data, 'DecodeJpeg/contents', (1, 1008), 'float32')
+        # Build an image from random data.
+        from PIL import Image
+        from tvm.contrib import util
+
+        img_array = np.random.uniform(size=(1, 600, 600, 3)).astype("uint8")
+        img = Image.frombuffer('RGB', (600, 600), img_array.tostring(), 'raw', 'RGB', 0, 1)
+        temp = util.tempdir()
+        img_path = temp.relpath("tf-test.jpg")
+        img.save(img_path);
+
+        import os.path
+        if not tf.gfile.Exists(os.path.join(img_path)):
+            tf.logging.fatal('File does not exist %s', image)
+        data = tf.gfile.FastGFile(os.path.join(img_path), 'rb').read()
+
+        temp.remove()
 
+        # Extract tensorflow decoded image frame for tvm input
         with tf.Session() as sess:
-            tf_output = run_tf_graph(sess, data, 'DecodeJpeg/contents:0', 'softmax:0')
+            tvm_data = run_tf_graph(sess, data, 'DecodeJpeg/contents:0', 'DecodeJpeg:0')
 
-        np.testing.assert_allclose(tf_output, tvm_output, rtol=2e-2, atol=2e-2)
+        with tf.Session() as sess:
+            tf_output = run_tf_graph(sess, data, 'DecodeJpeg/contents:0', 'softmax:0')
+            tvm_output = run_tvm_graph(graph_def, tvm_data, 'DecodeJpeg/contents', tf_output.shape, 'float32')
+            np.testing.assert_allclose(tf_output, tvm_output, rtol=1e-5, atol=1e-5)
 
 #######################################################################
 # Mobilenet
@@ -511,7 +571,7 @@ def test_forward_inception_v1():
 def test_forward_mobilenet():
     '''test mobilenet model'''
     with tf.Graph().as_default():
-        graph_def = nnvm.testing.tf.get_workload_mobilenet()
+        graph_def = nnvm.testing.tf.get_workload("MobilenetV1/mobilenet_v1_1.0_224_frozen-with-shapes.pb")
         # Call the utility to import the graph definition into default graph.
         graph_def = nnvm.testing.tf.ProcessGraphDefParam(graph_def)
 
@@ -520,12 +580,7 @@ def test_forward_mobilenet():
 
         with tf.Session() as sess:
             tf_output = run_tf_graph(sess, data, 'input:0', out_node + ':0')
-
-            out_shape = tf_output.shape
-            tvm_output = run_tvm_graph(graph_def, data, 'input', out_shape, 'float32')
-            top_tvm = np.squeeze(tvm_output).argsort()[-10:][::-1]
-            top_tf = np.squeeze(tf_output).argsort()[-10:][::-1]
-
+            tvm_output = run_tvm_graph(graph_def, data, 'input', tf_output.shape, 'float32')
             np.testing.assert_allclose(np.squeeze(tvm_output), np.squeeze(tf_output), rtol=1e-5, atol=1e-5)
 
 #######################################################################
@@ -544,3 +599,4 @@ if __name__ == '__main__':
     test_forward_inception_v1()
     test_forward_mobilenet()
     test_forward_variable()
+    test_forward_resize_bilinear()

topi/include/topi/image/resize.h

@@ -153,7 +153,7 @@ inline Tensor resize_bilinear_nhwc(const Tensor& input,
   Expr y_ratio;
   Expr x_ratio;
 
-  if (align_corners) {
+  if (!align_corners) {
     y_ratio = make_const(Float(32), (static_cast<float>(*in_height) /
                                      static_cast<float>(*out_height)));
     x_ratio = make_const(Float(32), (static_cast<float>(*in_width) /
@@ -170,21 +170,31 @@ inline Tensor resize_bilinear_nhwc(const Tensor& input,
 
   return compute(
     out_shape, [&](const Array<Var>& indices) {
-    auto y0 = HalideIR::Internal::Cast::make(Int(32), tvm::floor(y_ratio * indices[1]));
-    auto x0 = HalideIR::Internal::Cast::make(Int(32), tvm::floor(x_ratio * indices[2]));
+    auto in_y = indices[1] * y_ratio;
+    auto yf = tvm::floor(in_y);
+    auto yc = HalideIR::Internal::Cast::make(Int(32), tvm::ceil(in_y));
 
-    auto y1 = tvm::select(((y0 + cone) > other_y), other_y, (y0 + cone));
-    auto x1 = tvm::select(((x0 + cone) > other_x), other_x, (x0 + cone));
+    auto y0 = HalideIR::Internal::Cast::make(Int(32), tvm::floor(in_y));
+    auto y1 = tvm::select((yc > other_y), other_y, yc);
+    auto y_lerp  = in_y - yf;
 
-    auto h = (y_ratio * indices[1]) - y0;
-    auto w = (x_ratio * indices[2]) - x0;;
+    auto in_x = indices[2] * x_ratio;
+    auto xf = tvm::floor(in_x);
+    auto xc = HalideIR::Internal::Cast::make(Int(32), tvm::ceil(in_x));
+
+    auto x0 = HalideIR::Internal::Cast::make(Int(32), tvm::floor(in_x));
+    auto x1 = tvm::select((xc > other_x), other_x, xc);
+    auto x_lerp  = in_x - xf;
 
     auto A = input(indices[0], y0, x0, indices[3]);
     auto B = input(indices[0], y0, x1, indices[3]);
     auto C = input(indices[0], y1, x0, indices[3]);
     auto D = input(indices[0], y1, x1, indices[3]);
 
-    return  (A*(cone-w)*(cone-h) + B*(w)*(cone-h) + C*(h)*(cone-w) + D*w*h);
+    auto top = A + (B - A) * x_lerp;
+    auto bottom = C + (D - C) * x_lerp;
+
+    return  (top + (bottom - top) * y_lerp);
     }, name, tag);
 }
 
@@ -220,7 +230,7 @@ inline Tensor resize_bilinear_nchw(const Tensor& input,
   Expr y_ratio;
   Expr x_ratio;
 
-  if (align_corners) {
+  if (!align_corners) {
     y_ratio = make_const(Float(32), (static_cast<float>(*in_height) /
                                      static_cast<float>(*out_height)));
     x_ratio = make_const(Float(32), (static_cast<float>(*in_width) /
@@ -237,21 +247,31 @@ inline Tensor resize_bilinear_nchw(const Tensor& input,
 
   return compute(
     out_shape, [&](const Array<Var>& indices) {
-    auto y0 = HalideIR::Internal::Cast::make(Int(32), tvm::floor(y_ratio * indices[2]));
-    auto x0 = HalideIR::Internal::Cast::make(Int(32), tvm::floor(x_ratio * indices[3]));
+    auto in_y = indices[2] * y_ratio;
+    auto yf = tvm::floor(in_y);
+    auto yc = HalideIR::Internal::Cast::make(Int(32), tvm::ceil(in_y));
 
-    auto y1 = tvm::select(((y0 + cone) > other_y), other_y, (y0 + cone));
-    auto x1 = tvm::select(((x0 + cone) > other_x), other_x, (x0 + cone));
+    auto y0 = HalideIR::Internal::Cast::make(Int(32), tvm::floor(in_y));
+    auto y1 = tvm::select((yc > other_y), other_y, yc);
+    auto y_lerp  = in_y - yf;
 
-    auto h = (y_ratio * indices[2]) - y0;
-    auto w = (x_ratio * indices[3]) - x0;;
+    auto in_x = indices[3] * x_ratio;
+    auto xf = tvm::floor(in_x);
+    auto xc = HalideIR::Internal::Cast::make(Int(32), tvm::ceil(in_x));
+
+    auto x0 = HalideIR::Internal::Cast::make(Int(32), tvm::floor(in_x));
+    auto x1 = tvm::select((xc > other_x), other_x, xc);
+    auto x_lerp  = in_x - xf;
 
     auto A = input(indices[0], indices[1], y0, x0);
     auto B = input(indices[0], indices[1], y0, x1);
     auto C = input(indices[0], indices[1], y1, x0);
     auto D = input(indices[0], indices[1], y1, x1);
 
-    return  ((A*(cone-w)*(cone-h)) + (B*(w)*(cone-h)) + (C*(h)*(cone-w)) + (D*w*h));
+    auto top = A + (B - A) * x_lerp;
+    auto bottom = C + (D - C) * x_lerp;
+
+    return  (top + (bottom - top) * y_lerp);
     }, name, tag);
 }
 

topi/python/topi/testing/bilinear_resize_python.py

@@ -3,32 +3,6 @@
 import math
 import numpy as np
 
-def bilinear_weights(height, width, new_h, new_w, align_corners=False):
-    """ Helper function to generate weights for bilinear scaling """
-
-    if align_corners:
-        x_ratio = np.float32(np.float32(width)/np.float32(new_w))
-        y_ratio = np.float32(np.float32(height)/np.float32(new_h))
-    else:
-        x_ratio = np.float32(np.float32(width-1)/np.float32(new_w-1))
-        y_ratio = np.float32(np.float32(height-1)/np.float32(new_h-1))
-
-    def _bilinear_interpolation(y, x):
-        x_coord = math.floor(x_ratio * x)
-        y_coord = math.floor(y_ratio * y)
-        x_diff = np.float32((x_ratio * x) - x_coord)
-        y_diff = np.float32((y_ratio * y) - y_coord)
-
-        return [y_coord, x_coord, y_diff, x_diff]
-
-    # weights to hold (srcx, srcy, x_diff, y_diff) for each out value.
-    weights = np.empty([new_h, new_w, 4], dtype='float32')
-
-    for i in range(new_h):
-        for j in range(new_w):
-            weights[i][j] = _bilinear_interpolation(i, j)
-    return weights
-
 def bilinear_resize_python(image, out_size, layout, align_corners=False):
     """ Bilinear scaling using python"""
     (new_h, new_w) = out_size
@@ -40,20 +14,32 @@ def bilinear_resize_python(image, out_size, layout, align_corners=False):
         (batch, channel, h, w) = image.shape
         scaled_image = np.ones((batch, channel, new_h, new_w))
 
-    weights = bilinear_weights(h, w, new_h, new_w, align_corners)
+    if align_corners:
+        height_scale = np.float32(h-1) / np.float32(out_size[0]-1)
+        width_scale = np.float32(w-1) / np.float32(out_size[1]-1)
+    else:
+        height_scale = np.float32(h) / np.float32(out_size[0])
+        width_scale = np.float32(w) / np.float32(out_size[1])
 
     for b in range(batch):
         for i in range(channel):
             for j in range(new_h):
                 for k in range(new_w):
-                    y0 = int(weights[j][k][0])
-                    x0 = int(weights[j][k][1])
+                    in_y = j * height_scale
+                    y0 = math.floor(in_y)
+                    y1 = min(math.ceil(in_y), h - 1)
+                    y_lerp = in_y - y0
+
+                    y0 = int(y0)
+                    y1 = int(y1)
 
-                    x1 = min((x0+1), (w-1))
-                    y1 = min((y0+1), (h-1))
+                    in_x = k * width_scale
+                    x0 = math.floor(in_x)
+                    x1 = min(math.ceil(in_x), w - 1)
+                    x_lerp = in_x - x0
 
-                    y_diff = weights[j][k][2]
-                    x_diff = weights[j][k][3]
+                    x0 = int(x0)
+                    x1 = int(x1)
 
                     if layout == 'NHWC':
                         A = image[b][y0][x0][i]
@@ -66,10 +52,10 @@ def bilinear_resize_python(image, out_size, layout, align_corners=False):
                         C = image[b][i][y1][x0]
                         D = image[b][i][y1][x1]
 
-                    pixel = np.float32((A*(1-x_diff)*(1-y_diff) +
-                                        B*(x_diff)*(1-y_diff) +
-                                        C*(y_diff)*(1-x_diff) +
-                                        D*(x_diff*y_diff)))
+                    top = A + (B - A) * x_lerp
+                    bottom = C + (D - C) * x_lerp
+
+                    pixel = np.float32(top + (bottom - top) * y_lerp)
 
                     if layout == 'NHWC':
                         scaled_image[b][j][k][i] = pixel