[FRONTEND] CoreML (#63)

* add coreml

* fix bool

* fix flatten in fullyconnected

* fix duplicate flatten

* fix syntax

* add tutorial

* fix mxnet flatten, fix tutorials

* fix flatten issue

* fix lint

nnvm/python/nnvm/frontend/__init__.py

@@ -2,3 +2,4 @@
 from __future__ import absolute_import
 from .mxnet import from_mxnet
 from .onnx import from_onnx
+from .coreml import from_coreml

nnvm/python/nnvm/frontend/mxnet.py

-# pylint: disable=invalid-name
+# pylint: disable=invalid-name, import-self
 """MXNet symbol frontend."""
 from __future__ import absolute_import as _abs
 import json
@@ -7,6 +7,14 @@ from .. import symbol as _sym
 
 __all__ = ['from_mxnet']
 
+def _get_mxnet_version():
+    try:
+        import mxnet as mx
+        version = mx.__version__
+    except ImportError:
+        version = '0.11.1'
+    return [int(x) for x in version.split('.')]
+
 def _required_attr(attr, key):
     assert isinstance(attr, dict)
     if key not in attr:
@@ -119,6 +127,9 @@ def _dense(attrs):
     op_name, new_attrs = 'dense', {}
     new_attrs['units'] = _required_attr(attrs, 'num_hidden')
     new_attrs['use_bias'] = not _parse_bool_str(attrs, 'no_bias')
+    major, minor, micro = _get_mxnet_version()
+    if major >= 0 and minor >= 11 and micro >= 1:
+        new_attrs['flatten'] = _parse_bool_str(attrs, 'flatten', 'True')
     return op_name, new_attrs
 
 def _dropout(attrs):
@@ -276,6 +287,10 @@ def _from_mxnet_impl(symbol, graph):
         childs = symbol.get_children()
         childs = [_from_mxnet_impl(c, graph) for c in _as_list(childs)]
         childs = [x for y in childs for x in _as_list(y)]  # expand group symbol
+        if new_op == _sym.dense and 'flatten' in new_attr:
+            if new_attr['flatten']:
+                childs[0] = _sym.flatten(childs[0])
+            new_attr.pop('flatten')
         node = new_op(name=name, *childs, **new_attr)
     graph[name] = node
     return node
@@ -304,7 +319,7 @@ def from_mxnet(symbol, arg_params=None, aux_params=None):
         The parameter dict to be used by nnvm
     """
     try:
-        import mxnet as mx  # pylint: disable=import-self
+        import mxnet as mx
     except ImportError as e:
         raise ImportError('{}. MXNet is required to parse symbols.'.format(e))
 
@@ -325,7 +340,7 @@ def from_mxnet(symbol, arg_params=None, aux_params=None):
         for k, v in symbol.collect_params().items():
             params[k] = tvm.nd.array(v.data().asnumpy())
     elif isinstance(symbol, mx.gluon.Block):
-        raise NotImplementedError("The dynamic Block is not supported yet.")
+        raise NotImplementedError("Only Hybrid Blocks are supported now.")
     else:
         msg = "mxnet.Symbol or gluon.HybridBlock expected, got {}".format(type(symbol))
         raise ValueError(msg)

nnvm/tests/python/frontend/coreml/model_zoo/.gitignore

+*.mlmodel
+*.jpg
+*.png

nnvm/tests/python/frontend/coreml/model_zoo/__init__.py

+import urllib
+import os
+from PIL import Image
+import numpy as np
+
+def download(url, path, overwrite=False):
+    if os.path.exists(path) and not overwrite:
+        return
+    print('Downloading {} to {}.'.format(url, path))
+    urllib.URLopener().retrieve(url, path)
+
+def get_mobilenet():
+    url = 'https://docs-assets.developer.apple.com/coreml/models/MobileNet.mlmodel'
+    dst = 'mobilenet.mlmodel'
+    real_dst = os.path.abspath(os.path.join(os.path.dirname(__file__), dst))
+    download(url, real_dst)
+    return os.path.abspath(real_dst)
+
+def get_resnet50():
+    url = 'https://docs-assets.developer.apple.com/coreml/models/Resnet50.mlmodel'
+    dst = 'resnet50.mlmodel'
+    real_dst = os.path.abspath(os.path.join(os.path.dirname(__file__), dst))
+    download(url, real_dst)
+    return os.path.abspath(real_dst)
+
+def get_cat_image():
+    url = 'https://gist.githubusercontent.com/zhreshold/bcda4716699ac97ea44f791c24310193/raw/fa7ef0e9c9a5daea686d6473a62aacd1a5885849/cat.png'
+    dst = 'cat.jpg'
+    real_dst = os.path.abspath(os.path.join(os.path.dirname(__file__), dst))
+    download(url, real_dst)
+    img = Image.open(real_dst).resize((224, 224))
+    img = np.transpose(img, (2, 0, 1))[np.newaxis, :]
+    return np.asarray(img)

nnvm/tests/python/frontend/coreml/test_forward.py

+import numpy as np
+
+import topi
+import tvm
+from tvm.contrib import graph_runtime
+import nnvm.symbol as sym
+import nnvm.compiler
+from nnvm.testing.config import ctx_list
+from nnvm import frontend
+import coremltools as cm
+import model_zoo
+
+def get_tvm_output(symbol, x, params, target, ctx,
+                   out_shape=(1000,), input_name='image', dtype='float32'):
+    shape_dict = {input_name : x.shape}
+    with nnvm.compiler.build_config(opt_level=3):
+        graph, lib, params = nnvm.compiler.build(symbol, target, shape_dict, params=params)
+    m = graph_runtime.create(graph, lib, ctx)
+    # set inputs
+    m.set_input(input_name, tvm.nd.array(x.astype(dtype)))
+    m.set_input(**params)
+    m.run()
+    # get outputs
+    out = m.get_output(0, tvm.nd.empty(out_shape, dtype))
+    return out.asnumpy()
+
+def test_model_checkonly(model_file, model_name=''):
+    model = cm.models.MLModel(model_file)
+    sym, params = nnvm.frontend.from_coreml(model)
+    x = model_zoo.get_cat_image()
+    for target, ctx in ctx_list():
+        tvm_output = get_tvm_output(sym, x, params, target, ctx)
+        print(target, ctx, model_name, 'prediction id: ', np.argmax(tvm_output.flat))
+
+def test_mobilenet_checkonly():
+    model_file = model_zoo.get_mobilenet()
+    test_model_checkonly(model_file, 'mobilenet')
+
+def test_resnet50_checkonly():
+    model_file = model_zoo.get_resnet50()
+    test_model_checkonly(model_file, 'resnet50')
+
+if __name__ == '__main__':
+    test_mobilenet_checkonly()
+    test_resnet50_checkonly()

nnvm/tutorials/from_coreml.py

+"""
+Compile CoreML Models
+=====================
+**Author**: `Joshua Z. Zhang <https://zhreshold.github.io/>`_
+
+This article is an introductory tutorial to deploy CoreML models with NNVM.
+
+For us to begin with, coremltools module is required to be installed.
+
+A quick solution is to install via pip
+```bash
+pip install -U coremltools --user
+```
+or please refer to offical site
+https://github.com/apple/coremltools
+"""
+import nnvm
+import tvm
+import coremltools as cm
+import numpy as np
+from PIL import Image
+
+def download(url, path, overwrite=False):
+    import urllib2, os
+    if os.path.exists(path) and not overwrite:
+        return
+    print('Downloading {} to {}.'.format(url, path))
+    with open(path, 'w') as f:
+        f.write(urllib2.urlopen(url).read())
+
+######################################################################
+# Load pretrained CoreML model
+# ----------------------------
+# We will download and load a pretrained mobilenet classification network
+# privided by apple in this example
+model_url = 'https://docs-assets.developer.apple.com/coreml/models/MobileNet.mlmodel'
+model_file = 'mobilenet.mlmodel'
+download(model_url, model_file)
+# now you mobilenet.mlmodel on disk
+mlmodel = cm.models.MLModel(model_file)
+# we can load the graph as NNVM compatible model
+sym, params = nnvm.frontend.from_coreml(mlmodel)
+
+######################################################################
+# Load a test image
+# ------------------
+# A single cat dominates the examples!
+from PIL import Image
+img_url = 'https://github.com/dmlc/mxnet.js/blob/master/data/cat.png?raw=true'
+download(img_url, 'cat.png')
+img = Image.open('cat.png').resize((224, 224))
+x = np.transpose(img, (2, 0, 1))[np.newaxis, :]
+
+######################################################################
+# Compile the model on NNVM
+# ---------------------------
+# We should be familiar with the process right now.
+import nnvm.compiler
+target = 'cuda'
+shape_dict = {'image': x.shape}
+graph, lib, params = nnvm.compiler.build(sym, target, shape_dict, params=params)
+
+######################################################################
+# Execute on TVM
+# -------------------
+# The process is no different from other example
+from tvm.contrib import graph_runtime
+ctx = tvm.gpu(0)
+dtype = 'float32'
+m = graph_runtime.create(graph, lib, ctx)
+# set inputs
+m.set_input('image', tvm.nd.array(x.astype(dtype)))
+m.set_input(**params)
+# execute
+m.run()
+# get outputs
+output_shape = (1000,)
+tvm_output = m.get_output(0, tvm.nd.empty(output_shape, dtype)).asnumpy()
+top1 = np.argmax(tvm_output)
+
+#####################################################################
+# Look up synset name
+# -------------------
+# Look up prdiction top 1 index in 1000 class synset.
+synset_url = ''.join(['https://gist.githubusercontent.com/zhreshold/',
+                      '4d0b62f3d01426887599d4f7ede23ee5/raw/',
+                      '596b27d23537e5a1b5751d2b0481ef172f58b539/',
+                      'imagenet1000_clsid_to_human.txt'])
+synset_name = 'synset.txt'
+download(synset_url, synset_name)
+with open(synset_name) as f:
+    synset = eval(f.read())
+print('Top-1 id', top1, 'class name', synset[top1])

nnvm/tutorials/from_onnx.py

@@ -19,19 +19,25 @@ import tvm
 import onnx
 import numpy as np
 
+def download(url, path, overwrite=False):
+    import urllib2, os
+    if os.path.exists(path) and not overwrite:
+        return
+    print('Downloading {} to {}.'.format(url, path))
+    with open(path, 'w') as f:
+        f.write(urllib2.urlopen(url).read())
+
 ######################################################################
 # Load pretrained ONNX model
 # ---------------------------------------------
 # The example super resolution model used here is exactly the same model in onnx tutorial
 # http://pytorch.org/tutorials/advanced/super_resolution_with_caffe2.html
 # we skip the pytorch model construction part, and download the saved onnx model
-import urllib2
 model_url = ''.join(['https://gist.github.com/zhreshold/',
                      'bcda4716699ac97ea44f791c24310193/raw/',
                      '41b443bf2b6cf795892d98edd28bacecd8eb0d8d/',
                      'super_resolution.onnx'])
-with open('super_resolution.onnx', 'w') as f:
-    f.write(urllib2.urlopen(model_url).read())
+download(model_url, 'super_resolution.onnx')
 # now you have super_resolution.onnx on disk
 onnx_graph = onnx.load('super_resolution.onnx')
 # we can load the graph as NNVM compatible model
@@ -43,9 +49,8 @@ sym, params = nnvm.frontend.from_onnx(onnx_graph)
 # A single cat dominates the examples!
 from PIL import Image
 img_url = 'https://github.com/dmlc/mxnet.js/blob/master/data/cat.png?raw=true'
-with open('cat.jpg', 'w') as f:
-    f.write(urllib2.urlopen(img_url).read())
-img = Image.open('cat.jpg').resize((224, 224))
+download(img_url, 'cat.png')
+img = Image.open('cat.png').resize((224, 224))
 img_ycbcr = img.convert("YCbCr")  # convert to YCbCr
 img_y, img_cb, img_cr = img_ycbcr.split()
 x = np.array(img_y)[np.newaxis, np.newaxis, :, :]