Commit aa9528b3 by Joshua Z. Zhang Committed by Tianqi Chen

[FRONTEND] CoreML (#63)

* add coreml

* fix bool

* fix flatten in fullyconnected

* fix duplicate flatten

* fix syntax

* add tutorial

* fix mxnet flatten, fix tutorials

* fix flatten issue

* fix lint
parent 30157d82
......@@ -2,3 +2,4 @@
from __future__ import absolute_import
from .mxnet import from_mxnet
from .onnx import from_onnx
from .coreml import from_coreml
# pylint: disable=invalid-name, unused-argument
"""CoreML frontend."""
from __future__ import absolute_import as _abs
import tvm
import numpy as np
from .. import symbol as _sym
__all__ = ['from_coreml']
class SymbolTable(object):
"""Table storing symbols by names."""
def __init__(self):
self.vars = {}
self.params = {}
self.const_ctr = 1
self.in_padding = False
self.paddings = [0, 0]
def new_const(self, value):
name = "_param_%d" % (self.const_ctr)
self.const_ctr += 1
self.params[name] = value
self.vars[name] = _sym.Variable(name=name)
return self.vars[name]
def get_var(self, name, must_contain=True):
if must_contain:
assert name in self.vars
if name not in self.vars:
self.vars[name] = _sym.Variable(name=name)
return self.vars[name]
def set_var(self, name, sym):
assert isinstance(sym, _sym.Symbol)
self.vars[name] = sym
def set_padding(self, paddings):
self.paddings = paddings
self.in_padding = True
def clear_padding(self):
self.in_padding = False
def NeuralNetworkImageScaler(op, insym, symtab):
# this changes the symbol
biases = np.array([op.blueBias, op.greenBias, op.redBias]).reshape([3, 1, 1])
bias = symtab.new_const(biases)
ret = _sym.__mul_scalar__(insym, scalar=op.channelScale)
ret = _sym.broadcast_add(ret, bias)
return ret
def NeuralNetworkMeanImage(op, insym, symtab):
# this changes the symbol
ret = _sym.elemwise_sub(insym, scalar=op.meanImage)
return ret
def ConvolutionLayerParams(op, insym, symtab):
"""Convolution layer params."""
weights = symtab.new_const(np.array(list(op.weights.floatValue)).reshape(
tuple([op.outputChannels, op.kernelChannels] + list(op.kernelSize))))
if op.hasBias:
biases = symtab.new_const(list(op.bias.floatValue))
dilation = list(op.dilationFactor)
if not dilation:
dilation = [1, 1]
params = {'channels':op.outputChannels,
'kernel_size':list(op.kernelSize),
'strides':list(op.stride),
'dilation': dilation,
'use_bias': op.hasBias,
'groups':op.nGroups}
if op.WhichOneof('ConvolutionPaddingType') == 'valid':
valid = op.valid
padding = [b.startEdgeSize for b in valid.paddingAmounts.borderAmounts]
padding2 = [b.endEdgeSize for b in valid.paddingAmounts.borderAmounts]
for i, j in zip(padding, padding2):
assert i == j, "Asymmetry padding not supported"
if padding:
params['padding'] = padding
elif op.WhichOneof('ConvolutionPaddingType') == 'same':
kernel = params['kernel_size']
pad_h = kernel[0] - 1
pad_w = kernel[1] - 1
pad_t = pad_h // 2
pad_l = pad_w // 2
pad_b = pad_h - pad_t
pad_r = pad_w - pad_l
assert pad_t == pad_r and pad_l == pad_b, "Asymmetry padding not supported"
params['padding'] = [pad_t, pad_l]
else:
raise NotImplementedError("Valid/Same convolution padding implemented")
if op.hasBias:
pos = [insym, weights, biases]
else:
pos = [insym, weights]
if op.isDeconvolution:
ret = _sym.conv2d_transpose(*pos, **params)
else:
ret = _sym.conv2d(*pos, **params)
# consume padding layer
if symtab.in_padding:
params['padding'] = [sum(x) for x in zip(params.get('padding', [0, 0]), symtab.paddings)]
symtab.clear_padding()
return ret
def BatchnormLayerParams(op, insym, symtab):
# this changes the symbol
if op.instanceNormalization:
raise NotImplementedError("instance normalization not implemented")
else:
params = {'gamma':symtab.new_const(list(op.gamma.floatValue)),
'beta':symtab.new_const(list(op.beta.floatValue)),
'moving_mean':symtab.new_const(list(op.mean.floatValue)),
'moving_var': symtab.new_const(list(op.variance.floatValue)),
'epsilon': op.epsilon}
return _sym.batch_norm(data=insym, **params)
def ActivationParams(op, insym, symtab):
whichActivation = op.WhichOneof('NonlinearityType')
par = getattr(op, whichActivation)
if whichActivation == 'linear':
return _sym.__add_scalar__(_sym.__mul_scalar__(insym, scalar=par.alpha), scalar=par.beta)
elif whichActivation == 'ReLU':
return _sym.relu(insym)
elif whichActivation == 'leakyReLU':
return _sym.leaky_relu(insym, alpha=par.alpha)
elif whichActivation == 'thresholdedReLU':
raise NotImplementedError('thresholdedReLU not implemented')
elif whichActivation == 'PReLU':
raise NotImplementedError('PReLU not implemented')
elif whichActivation == 'tanh':
return _sym.tanh(insym)
elif whichActivation == 'scaledTanh':
return _sym.__mul_scalar__(_sym.tanh(_sym.__mul_scalar__(
insym, scalar=par.beta)), scalar=par.alpha)
elif whichActivation == 'sigmoid':
return _sym.sigmoid(insym)
elif whichActivation == 'sigmoidHard':
raise NotImplementedError('sigmoidHard not immplemented')
elif whichActivation == 'ELU':
return _sym.__mul_scalar__(_sym.__add_scalar__(
_sym.exp(insym), scalar=-1), scalar=par.alpha)
elif whichActivation == 'softsign':
raise NotImplementedError('softsign not implemented')
elif whichActivation == 'softplus':
return _sym.log(_sym.__add_scalar__(_sym.exp(insym), scalar=1))
elif whichActivation == 'parametricSoftplus':
alpha = list(par.alpha.floatValue)
beta = list(par.alpha.floatValue)
if len(alpha) == 1:
return _sym.__mul_scalar__(_sym.log(_sym.__add_scalar__(
_sym.exp(insym), scalar=beta[0])), scalar=alpha[0])
alpha = np.array(alpha).reshape((len(alpha), 1, 1))
beta = np.array(beta).reshape((len(beta), 1, 1))
alphasym = symtab.new_const(alpha)
betasym = symtab.new_const(beta)
return _sym.broadcast_mul(_sym.log(_sym.broadcast_add(
_sym.exp(insym), betasym)), alphasym)
def ScaleLayerParams(op, insym, symtab):
"""Scale layer params."""
scale = symtab.new_const(np.array(list(op.scale.floatValue)).reshape(
tuple(list(op.shapeScale) + [1, 1])))
# scale = _sym.reshape(scale, shape=tuple(list(op.shapeScale) + [1,1]))
ret = _sym.broadcast_mul(insym, scale)
if op.hasBias:
bias = symtab.new_const(np.array(list(op.bias.floatValue)).reshape(
tuple(list(op.shapeBias) + [1, 1])))
# bias = _sym.reshape(bias, shape=tuple(list(op.shapeBias) + [1,1]))
ret = _sym.broadcast_add(ret, bias)
return ret
def PoolingLayerParams(op, insym, symtab):
if op.globalPooling:
if op.type == 0:
return _sym.global_max_pool2d(insym)
elif op.type == 1:
return _sym.global_avg_pool2d(insym)
else:
raise NotImplementedError("Only max and average pooling implemented")
else:
params = {'pool_size':list(op.kernelSize),
'strides':list(op.stride)}
if op.WhichOneof('PoolingPaddingType') == 'valid':
valid = op.valid
padding = [b.startEdgeSize for b in valid.paddingAmounts.borderAmounts]
padding2 = [b.endEdgeSize for b in valid.paddingAmounts.borderAmounts]
for i, j in zip(padding, padding2):
assert i == j
params['padding'] = padding
elif op.WhichOneof('PoolingPaddingType') == 'includeLastPixel':
# I don't know if this is correct
valid = op.includeLastPixel
padding = list(valid.paddingAmounts)
params['padding'] = padding
params['ceil_mode'] = True
else:
raise NotImplementedError("Other convolution padding not implemented")
# consume padding layer
if symtab.in_padding:
params['padding'] = [sum(x) for x in zip(
params.get('padding', [0, 0]), symtab.paddings)]
symtab.clear_padding()
if op.type == 0:
return _sym.max_pool2d(insym, **params)
elif op.type == 1:
return _sym.avg_pool2d(insym, **params)
else:
raise NotImplementedError("Only max and average pooling implemented")
def SoftmaxLayerParams(op, insym, symtab):
return _sym.softmax(_sym.flatten(insym))
def InnerProductLayerParams(op, insym, symtab):
weights = symtab.new_const(np.array(op.weights.floatValue).reshape(
(op.outputChannels, op.inputChannels)))
par = {'weight':weights, 'use_bias':False, 'units':op.outputChannels}
if op.hasBias:
bias = symtab.new_const(np.array(op.bias.floatValue))
par['bias'] = bias
par['use_bias'] = True
return _sym.dense(data=insym, **par)
def AddLayerParams(op, insyms, symtab):
if not isinstance(insyms, list):
insyms = [insyms]
ret = insyms[0]
for i in range(1, len(insyms)):
ret = _sym.elemwise_add(ret, insyms[i])
if op.alpha > 0:
ret = _sym.__add_scalar__(ret, scalar=op.alpha)
return ret
def ConcatLayerParams(op, insyms, symtab):
if not isinstance(insyms, list):
insyms = [insyms]
if op.sequenceConcat:
raise NotImplementedError("Sequence Concat not supported")
ret = insyms[0]
for i in range(1, len(insyms)):
ret = _sym.concat(ret, insyms[i], dim=1)
return ret
def FlattenLayerParams(op, insym, symtab):
if op.mode == 1:
insym = _sym.transpose(_sym.reshape(insym, shape=(0, 0, -1)), axes=(0, 2, 1))
return _sym.flatten(insym)
def PaddingLayerParams(op, insym, symtab):
"""Hacking for padding layer params."""
if op.WhichOneof('PaddingType') == 'constant':
constant = op.constant
if constant.value != 0:
raise NotImplementedError("Padding value {} not supported.".format(constant.value))
padding = [b.startEdgeSize for b in op.paddingAmounts.borderAmounts]
padding2 = [b.endEdgeSize for b in op.paddingAmounts.borderAmounts]
for i, j in zip(padding, padding2):
assert i == j
symtab.set_padding(padding)
else:
raise NotImplementedError("Only constant padding is supported now.")
return insym
_convert_map = {
'NeuralNetworkMeanImage': NeuralNetworkMeanImage,
'NeuralNetworkImageScaler': NeuralNetworkImageScaler,
'ConvolutionLayerParams':ConvolutionLayerParams,
'BatchnormLayerParams':BatchnormLayerParams,
'ActivationParams':ActivationParams,
'ScaleLayerParams':ScaleLayerParams,
'PoolingLayerParams':PoolingLayerParams,
'SoftmaxLayerParams':SoftmaxLayerParams,
'InnerProductLayerParams':InnerProductLayerParams,
'AddLayerParams':AddLayerParams,
'FlattenLayerParams':FlattenLayerParams,
'ConcatLayerParams':ConcatLayerParams,
'PaddingLayerParams':PaddingLayerParams,
}
def coreml_op_to_nnvm(op, inname, outname, symtab):
"""Convert coreml layer to nnvm layer.
Parameters
----------
coremlop: a coreml protobuf bit
prevsym: previous nnvm symbol
Returns:
-------
nnvm.sym.Symbol
Converted symbol
"""
classname = type(op).__name__
if classname not in _convert_map:
raise NotImplementedError("%s is not supported" % (classname))
if isinstance(inname, (str, unicode)):
insym = symtab.get_var(inname)
else:
insym = [symtab.get_var(i) for i in inname]
ret = _convert_map[classname](op, insym, symtab)
if outname:
symtab.set_var(outname, ret)
if classname != 'PaddingLayerParams':
assert not symtab.in_padding, "Previous padding not consumed by conv/pool"
def from_coreml(model):
"""Convert from coreml model into NNVM format.
Parameters
----------
model:
coremltools.models.MLModel of a NeuralNetworkClassifier
arg_params : dict of str to mx.NDArray
The argument parameters in mxnet
aux_params : dict of str to mx.NDArray
The auxiliary parameters in mxnet
Returns
-------
sym : nnvm.Symbol
Compatible nnvm symbol
params : dict of str to tvm.NDArray
The parameter dict to be used by nnvm
"""
try:
import coremltools as cm
except ImportError:
raise ImportError('The coremltools package must be installed')
assert isinstance(model, cm.models.MLModel)
spec = model.get_spec()
modeltype = spec.WhichOneof('Type')
assert modeltype in ['neuralNetworkClassifier', 'neuralNetwork', 'neuralNetworkRegressor']
cc = getattr(spec, modeltype)
symtab = SymbolTable()
for i in spec.description.input:
symtab.get_var(i.name, must_contain=False)
for pp in cc.preprocessing:
whichpp = pp.WhichOneof('preprocessor')
ppmethod = getattr(pp, whichpp)
# the NeuralNetworkImageScalar doesn't seem to have a featureName?
if whichpp == 'scaler':
for i in spec.description.input:
coreml_op_to_nnvm(ppmethod, i.name, i.name, symtab)
else:
coreml_op_to_nnvm(ppmethod, pp.featureName, pp.featureName, symtab)
for l in cc.layers:
layertype = l.WhichOneof('layer')
layerop = getattr(l, layertype)
assert len(l.output) == 1
if len(l.input) == 1:
coreml_op_to_nnvm(layerop, l.input[0], l.output[0], symtab)
else:
coreml_op_to_nnvm(layerop, list(l.input), l.output[0], symtab)
returns = [symtab.get_var(i.name, must_contain=False) for i in spec.description.output]
tvmparams = {k:tvm.nd.array(np.array(v, dtype=np.float32)) for k, v in symtab.params.items()}
# for now return first output
return returns[0], tvmparams
# pylint: disable=invalid-name
# pylint: disable=invalid-name, import-self
"""MXNet symbol frontend."""
from __future__ import absolute_import as _abs
import json
......@@ -7,6 +7,14 @@ from .. import symbol as _sym
__all__ = ['from_mxnet']
def _get_mxnet_version():
try:
import mxnet as mx
version = mx.__version__
except ImportError:
version = '0.11.1'
return [int(x) for x in version.split('.')]
def _required_attr(attr, key):
assert isinstance(attr, dict)
if key not in attr:
......@@ -119,6 +127,9 @@ def _dense(attrs):
op_name, new_attrs = 'dense', {}
new_attrs['units'] = _required_attr(attrs, 'num_hidden')
new_attrs['use_bias'] = not _parse_bool_str(attrs, 'no_bias')
major, minor, micro = _get_mxnet_version()
if major >= 0 and minor >= 11 and micro >= 1:
new_attrs['flatten'] = _parse_bool_str(attrs, 'flatten', 'True')
return op_name, new_attrs
def _dropout(attrs):
......@@ -276,6 +287,10 @@ def _from_mxnet_impl(symbol, graph):
childs = symbol.get_children()
childs = [_from_mxnet_impl(c, graph) for c in _as_list(childs)]
childs = [x for y in childs for x in _as_list(y)] # expand group symbol
if new_op == _sym.dense and 'flatten' in new_attr:
if new_attr['flatten']:
childs[0] = _sym.flatten(childs[0])
new_attr.pop('flatten')
node = new_op(name=name, *childs, **new_attr)
graph[name] = node
return node
......@@ -304,7 +319,7 @@ def from_mxnet(symbol, arg_params=None, aux_params=None):
The parameter dict to be used by nnvm
"""
try:
import mxnet as mx # pylint: disable=import-self
import mxnet as mx
except ImportError as e:
raise ImportError('{}. MXNet is required to parse symbols.'.format(e))
......@@ -325,7 +340,7 @@ def from_mxnet(symbol, arg_params=None, aux_params=None):
for k, v in symbol.collect_params().items():
params[k] = tvm.nd.array(v.data().asnumpy())
elif isinstance(symbol, mx.gluon.Block):
raise NotImplementedError("The dynamic Block is not supported yet.")
raise NotImplementedError("Only Hybrid Blocks are supported now.")
else:
msg = "mxnet.Symbol or gluon.HybridBlock expected, got {}".format(type(symbol))
raise ValueError(msg)
......
import urllib
import os
from PIL import Image
import numpy as np
def download(url, path, overwrite=False):
if os.path.exists(path) and not overwrite:
return
print('Downloading {} to {}.'.format(url, path))
urllib.URLopener().retrieve(url, path)
def get_mobilenet():
url = 'https://docs-assets.developer.apple.com/coreml/models/MobileNet.mlmodel'
dst = 'mobilenet.mlmodel'
real_dst = os.path.abspath(os.path.join(os.path.dirname(__file__), dst))
download(url, real_dst)
return os.path.abspath(real_dst)
def get_resnet50():
url = 'https://docs-assets.developer.apple.com/coreml/models/Resnet50.mlmodel'
dst = 'resnet50.mlmodel'
real_dst = os.path.abspath(os.path.join(os.path.dirname(__file__), dst))
download(url, real_dst)
return os.path.abspath(real_dst)
def get_cat_image():
url = 'https://gist.githubusercontent.com/zhreshold/bcda4716699ac97ea44f791c24310193/raw/fa7ef0e9c9a5daea686d6473a62aacd1a5885849/cat.png'
dst = 'cat.jpg'
real_dst = os.path.abspath(os.path.join(os.path.dirname(__file__), dst))
download(url, real_dst)
img = Image.open(real_dst).resize((224, 224))
img = np.transpose(img, (2, 0, 1))[np.newaxis, :]
return np.asarray(img)
import numpy as np
import topi
import tvm
from tvm.contrib import graph_runtime
import nnvm.symbol as sym
import nnvm.compiler
from nnvm.testing.config import ctx_list
from nnvm import frontend
import coremltools as cm
import model_zoo
def get_tvm_output(symbol, x, params, target, ctx,
out_shape=(1000,), input_name='image', dtype='float32'):
shape_dict = {input_name : x.shape}
with nnvm.compiler.build_config(opt_level=3):
graph, lib, params = nnvm.compiler.build(symbol, target, shape_dict, params=params)
m = graph_runtime.create(graph, lib, ctx)
# set inputs
m.set_input(input_name, tvm.nd.array(x.astype(dtype)))
m.set_input(**params)
m.run()
# get outputs
out = m.get_output(0, tvm.nd.empty(out_shape, dtype))
return out.asnumpy()
def test_model_checkonly(model_file, model_name=''):
model = cm.models.MLModel(model_file)
sym, params = nnvm.frontend.from_coreml(model)
x = model_zoo.get_cat_image()
for target, ctx in ctx_list():
tvm_output = get_tvm_output(sym, x, params, target, ctx)
print(target, ctx, model_name, 'prediction id: ', np.argmax(tvm_output.flat))
def test_mobilenet_checkonly():
model_file = model_zoo.get_mobilenet()
test_model_checkonly(model_file, 'mobilenet')
def test_resnet50_checkonly():
model_file = model_zoo.get_resnet50()
test_model_checkonly(model_file, 'resnet50')
if __name__ == '__main__':
test_mobilenet_checkonly()
test_resnet50_checkonly()
"""
Compile CoreML Models
=====================
**Author**: `Joshua Z. Zhang <https://zhreshold.github.io/>`_
This article is an introductory tutorial to deploy CoreML models with NNVM.
For us to begin with, coremltools module is required to be installed.
A quick solution is to install via pip
```bash
pip install -U coremltools --user
```
or please refer to offical site
https://github.com/apple/coremltools
"""
import nnvm
import tvm
import coremltools as cm
import numpy as np
from PIL import Image
def download(url, path, overwrite=False):
import urllib2, os
if os.path.exists(path) and not overwrite:
return
print('Downloading {} to {}.'.format(url, path))
with open(path, 'w') as f:
f.write(urllib2.urlopen(url).read())
######################################################################
# Load pretrained CoreML model
# ----------------------------
# We will download and load a pretrained mobilenet classification network
# privided by apple in this example
model_url = 'https://docs-assets.developer.apple.com/coreml/models/MobileNet.mlmodel'
model_file = 'mobilenet.mlmodel'
download(model_url, model_file)
# now you mobilenet.mlmodel on disk
mlmodel = cm.models.MLModel(model_file)
# we can load the graph as NNVM compatible model
sym, params = nnvm.frontend.from_coreml(mlmodel)
######################################################################
# Load a test image
# ------------------
# A single cat dominates the examples!
from PIL import Image
img_url = 'https://github.com/dmlc/mxnet.js/blob/master/data/cat.png?raw=true'
download(img_url, 'cat.png')
img = Image.open('cat.png').resize((224, 224))
x = np.transpose(img, (2, 0, 1))[np.newaxis, :]
######################################################################
# Compile the model on NNVM
# ---------------------------
# We should be familiar with the process right now.
import nnvm.compiler
target = 'cuda'
shape_dict = {'image': x.shape}
graph, lib, params = nnvm.compiler.build(sym, target, shape_dict, params=params)
######################################################################
# Execute on TVM
# -------------------
# The process is no different from other example
from tvm.contrib import graph_runtime
ctx = tvm.gpu(0)
dtype = 'float32'
m = graph_runtime.create(graph, lib, ctx)
# set inputs
m.set_input('image', tvm.nd.array(x.astype(dtype)))
m.set_input(**params)
# execute
m.run()
# get outputs
output_shape = (1000,)
tvm_output = m.get_output(0, tvm.nd.empty(output_shape, dtype)).asnumpy()
top1 = np.argmax(tvm_output)
#####################################################################
# Look up synset name
# -------------------
# Look up prdiction top 1 index in 1000 class synset.
synset_url = ''.join(['https://gist.githubusercontent.com/zhreshold/',
'4d0b62f3d01426887599d4f7ede23ee5/raw/',
'596b27d23537e5a1b5751d2b0481ef172f58b539/',
'imagenet1000_clsid_to_human.txt'])
synset_name = 'synset.txt'
download(synset_url, synset_name)
with open(synset_name) as f:
synset = eval(f.read())
print('Top-1 id', top1, 'class name', synset[top1])
......@@ -19,19 +19,25 @@ import tvm
import onnx
import numpy as np
def download(url, path, overwrite=False):
import urllib2, os
if os.path.exists(path) and not overwrite:
return
print('Downloading {} to {}.'.format(url, path))
with open(path, 'w') as f:
f.write(urllib2.urlopen(url).read())
######################################################################
# Load pretrained ONNX model
# ---------------------------------------------
# The example super resolution model used here is exactly the same model in onnx tutorial
# http://pytorch.org/tutorials/advanced/super_resolution_with_caffe2.html
# we skip the pytorch model construction part, and download the saved onnx model
import urllib2
model_url = ''.join(['https://gist.github.com/zhreshold/',
'bcda4716699ac97ea44f791c24310193/raw/',
'41b443bf2b6cf795892d98edd28bacecd8eb0d8d/',
'super_resolution.onnx'])
with open('super_resolution.onnx', 'w') as f:
f.write(urllib2.urlopen(model_url).read())
download(model_url, 'super_resolution.onnx')
# now you have super_resolution.onnx on disk
onnx_graph = onnx.load('super_resolution.onnx')
# we can load the graph as NNVM compatible model
......@@ -43,9 +49,8 @@ sym, params = nnvm.frontend.from_onnx(onnx_graph)
# A single cat dominates the examples!
from PIL import Image
img_url = 'https://github.com/dmlc/mxnet.js/blob/master/data/cat.png?raw=true'
with open('cat.jpg', 'w') as f:
f.write(urllib2.urlopen(img_url).read())
img = Image.open('cat.jpg').resize((224, 224))
download(img_url, 'cat.png')
img = Image.open('cat.png').resize((224, 224))
img_ycbcr = img.convert("YCbCr") # convert to YCbCr
img_y, img_cb, img_cr = img_ycbcr.split()
x = np.array(img_y)[np.newaxis, np.newaxis, :, :]
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment