Skip to content

T
tic
Commit c6339730 by Haichen Shen Committed by Tianqi Chen
Options

[Tutorial] Cache the test data in tutorial (#2923)

parent 8eef1565
Showing with 171 additions and 253 deletions
nnvm/tests/python/frontend/coreml/model_zoo/__init__.py
......@@ -18,7 +18,7 @@ def get_resnet50():
def get_cat_image():
url = 'https://gist.githubusercontent.com/zhreshold/bcda4716699ac97ea44f791c24310193/raw/fa7ef0e9c9a5daea686d6473a62aacd1a5885849/cat.png'
dst = 'cat.png'
real_dst = download_testdata(url, dst, module='coreml')
real_dst = download_testdata(url, dst, module='data')
img = Image.open(real_dst).resize((224, 224))
img = np.transpose(img, (2, 0, 1))[np.newaxis, :]
return np.asarray(img)
nnvm/tests/python/frontend/darknet/test_forward.py
......@@ -103,7 +103,7 @@ def test_forward(net, build_dtype='float32'):
test_image = 'dog.jpg'
img_url = 'https://github.com/siju-samuel/darknet/blob/master/data/' + test_image +'?raw=true'
img_path = download_testdata(img_url, test_image, module='darknet')
img_path = download_testdata(img_url, test_image, module='data')
img = LIB.letterbox_image(LIB.load_image_color(img_path.encode('utf-8'), 0, 0), net.w, net.h)
darknet_output = get_darknet_output(net, img)
batch_size = 1
......
nnvm/tutorials/deploy_model_on_mali_gpu.py
......@@ -15,6 +15,7 @@ import nnvm.compiler
import nnvm.testing
from tvm import rpc
from tvm.contrib import util, graph_runtime as runtime
from tvm.contrib.download import download_testdata
######################################################################
# Build TVM Runtime on Device
......@@ -81,7 +82,6 @@ from tvm.contrib import util, graph_runtime as runtime
# You can found more details about this part at tutorial :ref:`tutorial-from-mxnet`.
from mxnet.gluon.model_zoo.vision import get_model
from mxnet.gluon.utils import download
from PIL import Image
import numpy as np
......@@ -92,8 +92,9 @@ block = get_model('resnet18_v1', pretrained=True)
# In order to test our model, here we download an image of cat and
# transform its format.
img_name = 'cat.png'
download('https://github.com/dmlc/mxnet.js/blob/master/data/cat.png?raw=true', img_name)
image = Image.open(img_name).resize((224, 224))
img_path = download_testdata('https://github.com/dmlc/mxnet.js/blob/master/data/cat.png?raw=true',
img_name, module='data')
image = Image.open(img_path).resize((224, 224))
def transform_image(image):
image = np.array(image) - np.array([123., 117., 104.])
......@@ -112,9 +113,9 @@ synset_url = ''.join(['https://gist.githubusercontent.com/zhreshold/',
'596b27d23537e5a1b5751d2b0481ef172f58b539/',
'imagenet1000_clsid_to_human.txt'])
synset_name = 'synset.txt'
download(synset_url, synset_name)
with open(synset_name) as f:
synset_name = 'imagenet1000_clsid_to_human.txt'
synset_path = download_testdata(synset_url, synset_name, module='data')
with open(synset_path) as f:
synset = eval(f.read())
######################################################################
......
nnvm/tutorials/deploy_model_on_rasp.py
......@@ -14,6 +14,7 @@ import nnvm.compiler
import nnvm.testing
from tvm import rpc
from tvm.contrib import util, graph_runtime as runtime
from tvm.contrib.download import download_testdata
######################################################################
# .. _build-tvm-runtime-on-device:
......@@ -78,7 +79,6 @@ from tvm.contrib import util, graph_runtime as runtime
# You can found more details about this part at tutorial :ref:`tutorial-from-mxnet`.
from mxnet.gluon.model_zoo.vision import get_model
from mxnet.gluon.utils import download
from PIL import Image
import numpy as np
......@@ -89,8 +89,9 @@ block = get_model('resnet18_v1', pretrained=True)
# In order to test our model, here we download an image of cat and
# transform its format.
img_name = 'cat.png'
download('https://github.com/dmlc/mxnet.js/blob/master/data/cat.png?raw=true', img_name)
image = Image.open(img_name).resize((224, 224))
img_path = download_testdata('https://github.com/dmlc/mxnet.js/blob/master/data/cat.png?raw=true',
img_name, module='data')
image = Image.open(img_path).resize((224, 224))
def transform_image(image):
image = np.array(image) - np.array([123., 117., 104.])
......@@ -108,9 +109,9 @@ synset_url = ''.join(['https://gist.githubusercontent.com/zhreshold/',
'4d0b62f3d01426887599d4f7ede23ee5/raw/',
'596b27d23537e5a1b5751d2b0481ef172f58b539/',
'imagenet1000_clsid_to_human.txt'])
synset_name = 'synset.txt'
download(synset_url, synset_name)
with open(synset_name) as f:
synset_name = 'imagenet1000_clsid_to_human.txt'
synset_path = download_testdata(synset_url, synset_name, module='data')
with open(synset_path) as f:
synset = eval(f.read())
######################################################################
......
nnvm/tutorials/deploy_ssd_mxnet.py
......@@ -18,7 +18,7 @@ import numpy as np
from nnvm import compiler
from nnvm.frontend import from_mxnet
from tvm import relay
from tvm.contrib.download import download
from tvm.contrib.download import download_testdata
from tvm.contrib import graph_runtime
from mxnet.model import load_checkpoint
......@@ -65,28 +65,24 @@ inference_symbol_folder = \
inference_symbol_url = "https://gist.github.com/kevinthesun/c1904e900848df4548ce5dfb18c719c7/" \
"archive/a28c4856c827fe766aa3da0e35bad41d44f0fb26.zip"
dir = "ssd_model"
if not os.path.exists(dir):
os.makedirs(dir)
model_file_path = "%s/%s" % (dir, model_file)
test_image_path = "%s/%s" % (dir, test_image)
inference_symbol_path = "%s/inference_model.zip" % dir
download(model_url, model_file_path)
download(image_url, test_image_path)
download(inference_symbol_url, inference_symbol_path)
model_file_path = download_testdata(model_url, model_file, module=["mxnet", "ssd_model"])
inference_symbol_path = download_testdata(inference_symbol_url, "inference_model.zip",
module=["mxnet", "ssd_model"])
test_image_path = download_testdata(image_url, test_image, module="data")
model_dir = os.path.dirname(model_file_path)
zip_ref = zipfile.ZipFile(model_file_path, 'r')
zip_ref.extractall(dir)
zip_ref.extractall(model_dir)
zip_ref.close()
zip_ref = zipfile.ZipFile(inference_symbol_path)
zip_ref.extractall(dir)
zip_ref.extractall(model_dir)
zip_ref.close()
######################################################################
# Convert and compile model with NNVM or Relay for CPU.
sym = mx.sym.load("%s/%s/ssd_resnet50_inference.json" % (dir, inference_symbol_folder))
_, arg_params, aux_params = load_checkpoint("%s/%s" % (dir, model_name), 0)
sym = mx.sym.load("%s/%s/ssd_resnet50_inference.json" % (model_dir, inference_symbol_folder))
_, arg_params, aux_params = load_checkpoint("%s/%s" % (model_dir, model_name), 0)
import argparse
parser = argparse.ArgumentParser()
......
nnvm/tutorials/from_coreml.py
......@@ -21,19 +21,7 @@ import tvm
import coremltools as cm
import numpy as np
from PIL import Image
def download(url, path, overwrite=False):
import os
if os.path.isfile(path) and not overwrite:
print('File {} existed, skip.'.format(path))
return
print('Downloading from url {} to {}'.format(url, path))
try:
import urllib.request
urllib.request.urlretrieve(url, path)
except:
import urllib
urllib.urlretrieve(url, path)
from tvm.contrib.download import download_testdata
######################################################################
# Load pretrained CoreML model
......@@ -42,9 +30,9 @@ def download(url, path, overwrite=False):
# provided by apple in this example
model_url = 'https://docs-assets.developer.apple.com/coreml/models/MobileNet.mlmodel'
model_file = 'mobilenet.mlmodel'
download(model_url, model_file)
model_path = download_testdata(model_url, model_file, module='coreml')
# now you mobilenet.mlmodel on disk
mlmodel = cm.models.MLModel(model_file)
mlmodel = cm.models.MLModel(model_path)
# we can load the graph as NNVM compatible model
sym, params = nnvm.frontend.from_coreml(mlmodel)
......@@ -54,8 +42,8 @@ sym, params = nnvm.frontend.from_coreml(mlmodel)
# A single cat dominates the examples!
from PIL import Image
img_url = 'https://github.com/dmlc/mxnet.js/blob/master/data/cat.png?raw=true'
download(img_url, 'cat.png')
img = Image.open('cat.png').resize((224, 224))
img_path = download_testdata(img_url, 'cat.png', module='data')
img = Image.open(img_path).resize((224, 224))
#x = np.transpose(img, (2, 0, 1))[np.newaxis, :]
image = np.asarray(img)
image = image.transpose((2, 0, 1))
......@@ -95,8 +83,8 @@ synset_url = ''.join(['https://gist.githubusercontent.com/zhreshold/',
'4d0b62f3d01426887599d4f7ede23ee5/raw/',
'596b27d23537e5a1b5751d2b0481ef172f58b539/',
'imagenet1000_clsid_to_human.txt'])
synset_name = 'synset.txt'
download(synset_url, synset_name)
with open(synset_name) as f:
synset_name = 'imagenet1000_clsid_to_human.txt'
synset_path = download_testdata(synset_url, synset_name, module='data')
with open(synset_path) as f:
synset = eval(f.read())
print('Top-1 id', top1, 'class name', synset[top1])
nnvm/tutorials/from_darknet.py
......@@ -25,7 +25,7 @@ import tvm
import sys
from ctypes import *
from tvm.contrib.download import download
from tvm.contrib.download import download_testdata
from nnvm.testing.darknet import __darknetffi__
# Model name
......@@ -41,8 +41,8 @@ REPO_URL = 'https://github.com/siju-samuel/darknet/blob/master/'
CFG_URL = REPO_URL + 'cfg/' + CFG_NAME + '?raw=true'
WEIGHTS_URL = 'https://pjreddie.com/media/files/' + WEIGHTS_NAME
download(CFG_URL, CFG_NAME)
download(WEIGHTS_URL, WEIGHTS_NAME)
cfg_path = download_testdata(CFG_URL, CFG_NAME, module="darknet")
weights_path = download_testdata(WEIGHTS_URL, WEIGHTS_NAME, module="darknet")
# Download and Load darknet library
if sys.platform in ['linux', 'linux2']:
......@@ -55,12 +55,10 @@ else:
err = "Darknet lib is not supported on {} platform".format(sys.platform)
raise NotImplementedError(err)
download(DARKNET_URL, DARKNET_LIB)
lib_path = download_testdata(DARKNET_URL, DARKNET_LIB, module="darknet")
DARKNET_LIB = __darknetffi__.dlopen('./' + DARKNET_LIB)
cfg = "./" + str(CFG_NAME)
weights = "./" + str(WEIGHTS_NAME)
net = DARKNET_LIB.load_network(cfg.encode('utf-8'), weights.encode('utf-8'), 0)
DARKNET_LIB = __darknetffi__.dlopen(lib_path)
net = DARKNET_LIB.load_network(cfg_path.encode('utf-8'), weights_path.encode('utf-8'), 0)
dtype = 'float32'
batch_size = 1
......@@ -88,9 +86,9 @@ test_image = 'dog.jpg'
print("Loading the test image...")
img_url = 'https://github.com/siju-samuel/darknet/blob/master/data/' + \
test_image + '?raw=true'
download(img_url, test_image)
img_path = download_testdata(img_url, test_image, "data")
data = nnvm.testing.darknet.load_image(test_image, netw, neth)
data = nnvm.testing.darknet.load_image(img_path, netw, neth)
######################################################################
# Execute on TVM Runtime
# ----------------------
......@@ -150,10 +148,10 @@ coco_name = 'coco.names'
coco_url = 'https://github.com/siju-samuel/darknet/blob/master/data/' + coco_name + '?raw=true'
font_name = 'arial.ttf'
font_url = 'https://github.com/siju-samuel/darknet/blob/master/data/' + font_name + '?raw=true'
download(coco_url, coco_name)
download(font_url, font_name)
coco_path = download_testdata(coco_url, coco_name, module='data')
font_path = download_testdata(font_url, font_name, module='data')
with open(coco_name) as f:
with open(coco_path) as f:
content = f.readlines()
names = [x.strip() for x in content]
......
nnvm/tutorials/from_mxnet.py
......@@ -20,30 +20,31 @@ https://mxnet.incubator.apache.org/versions/master/install/index.html
"""
# some standard imports
import mxnet as mx
import numpy as np
import nnvm
import tvm
import numpy as np
from tvm.contrib.download import download_testdata
######################################################################
# Download Resnet18 model from Gluon Model Zoo
# ---------------------------------------------
# In this section, we download a pretrained imagenet model and classify an image.
from mxnet.gluon.model_zoo.vision import get_model
from mxnet.gluon.utils import download
from PIL import Image
from matplotlib import pyplot as plt
block = get_model('resnet18_v1', pretrained=True)
img_url = 'https://github.com/dmlc/mxnet.js/blob/master/data/cat.png?raw=true'
img_name = 'cat.png'
synset_url = ''.join(['https://gist.githubusercontent.com/zhreshold/',
'4d0b62f3d01426887599d4f7ede23ee5/raw/',
'596b27d23537e5a1b5751d2b0481ef172f58b539/',
'imagenet1000_clsid_to_human.txt'])
synset_name = 'synset.txt'
download('https://github.com/dmlc/mxnet.js/blob/master/data/cat.png?raw=true', img_name)
download(synset_url, synset_name)
with open(synset_name) as f:
synset_name = 'imagenet1000_clsid_to_human.txt'
img_path = download_testdata(img_url, img_name, module='data')
synset_path = download_testdata(synset_url, synset_name, module='data')
with open(synset_path) as f:
synset = eval(f.read())
image = Image.open(img_name).resize((224, 224))
image = Image.open(img_path).resize((224, 224))
plt.imshow(image)
plt.show()
......
nnvm/tutorials/from_mxnet_to_webgl.py
......@@ -33,6 +33,7 @@ from __future__ import print_function
import numpy as np
import tvm
from tvm.contrib.download import download_testdata
import nnvm.compiler
import nnvm.testing
......@@ -106,16 +107,14 @@ def download_synset():
print("Downloading synset...")
from mxnet import gluon
url = "https://gist.githubusercontent.com/zhreshold/" + \
"4d0b62f3d01426887599d4f7ede23ee5/raw/" + \
"596b27d23537e5a1b5751d2b0481ef172f58b539/" + \
"imagenet1000_clsid_to_human.txt"
file_name = "synset.txt"
file_name = "imagenet1000_clsid_to_human.txt"
gluon.utils.download(url, file_name)
with open(file_name) as f:
file_path = download_testdata(url, file_name, module='data')
with open(file_path) as f:
synset = eval(f.read())
print("- Synset downloaded!")
......@@ -144,14 +143,13 @@ def download_image():
print("Downloading cat image...")
from matplotlib import pyplot as plt
from mxnet import gluon
from PIL import Image
url = "https://github.com/dmlc/mxnet.js/blob/master/data/cat.png?raw=true"
img_name = "cat.png"
gluon.utils.download(url, img_name)
image = Image.open(img_name).resize((224, 224))
img_path = download_testdata(url, img_name, module='data')
image = Image.open(img_path).resize((224, 224))
print("- Cat image downloaded!")
......
nnvm/tutorials/from_onnx.py
......@@ -18,22 +18,10 @@ https://github.com/onnx/onnx
"""
import nnvm
import tvm
from tvm.contrib.download import download_testdata
import onnx
import numpy as np
def download(url, path, overwrite=False):
import os
if os.path.isfile(path) and not overwrite:
print('File {} existed, skip.'.format(path))
return
print('Downloading from url {} to {}'.format(url, path))
try:
import urllib.request
urllib.request.urlretrieve(url, path)
except:
import urllib
urllib.urlretrieve(url, path)
######################################################################
# Load pretrained ONNX model
# ---------------------------------------------
......@@ -44,9 +32,9 @@ model_url = ''.join(['https://gist.github.com/zhreshold/',
'bcda4716699ac97ea44f791c24310193/raw/',
'93672b029103648953c4e5ad3ac3aadf346a4cdc/',
'super_resolution_0.2.onnx'])
download(model_url, 'super_resolution.onnx', True)
model_path = download_testdata(model_url, 'super_resolution.onnx', module='onnx')
# now you have super_resolution.onnx on disk
onnx_model = onnx.load_model('super_resolution.onnx')
onnx_model = onnx.load_model(model_path)
# we can load the graph as NNVM compatible model
sym, params = nnvm.frontend.from_onnx(onnx_model)
......@@ -56,8 +44,8 @@ sym, params = nnvm.frontend.from_onnx(onnx_model)
# A single cat dominates the examples!
from PIL import Image
img_url = 'https://github.com/dmlc/mxnet.js/blob/master/data/cat.png?raw=true'
download(img_url, 'cat.png')
img = Image.open('cat.png').resize((224, 224))
img_path = download_testdata(img_url, 'cat.png', module='data')
img = Image.open(img_path).resize((224, 224))
img_ycbcr = img.convert("YCbCr") # convert to YCbCr
img_y, img_cb, img_cr = img_ycbcr.split()
x = np.array(img_y)[np.newaxis, np.newaxis, :, :]
......
nnvm/tutorials/from_tensorflow.py
......@@ -52,8 +52,8 @@ map_proto = 'imagenet_2012_challenge_label_map_proto.pbtxt'
map_proto_url = os.path.join(repo_base, map_proto)
# Human readable text for labels
lable_map = 'imagenet_synset_to_human_label_map.txt'
lable_map_url = os.path.join(repo_base, lable_map)
label_map = 'imagenet_synset_to_human_label_map.txt'
label_map_url = os.path.join(repo_base, label_map)
# Target settings
# Use these commented settings to build for cuda.
......@@ -70,19 +70,19 @@ ctx = tvm.cpu(0)
# Download required files
# -----------------------
# Download files listed above.
from mxnet.gluon.utils import download
from tvm.contrib.download import download_testdata
download(image_url, img_name)
download(model_url, model_name)
download(map_proto_url, map_proto)
download(lable_map_url, lable_map)
img_path = download_testdata(image_url, img_name, module='data')
model_path = download_testdata(model_url, model_name, module=['tf', 'InceptionV1'])
map_proto_path = download_testdata(map_proto_url, map_proto, module='data')
label_path = download_testdata(label_map_url, label_map, module='data')
######################################################################
# Import model
# ------------
# Creates tensorflow graph definition from protobuf file.
with tf.gfile.FastGFile(os.path.join("./", model_name), 'rb') as f:
with tf.gfile.FastGFile(model_path, 'rb') as f:
graph_def = tf.GraphDef()
graph_def.ParseFromString(f.read())
graph = tf.import_graph_def(graph_def, name='')
......@@ -103,7 +103,7 @@ with tf.gfile.FastGFile(os.path.join("./", model_name), 'rb') as f:
#
from PIL import Image
image = Image.open(img_name).resize((299, 299))
image = Image.open(img_path).resize((299, 299))
x = np.array(image)
......@@ -117,7 +117,7 @@ x = np.array(image)
# params: params converted from tensorflow params (tensor protobuf).
sym, params = nnvm.frontend.from_tensorflow(graph_def, layout=layout)
print ("Tensorflow protobuf imported as nnvm graph")
print("Tensorflow protobuf imported as nnvm graph")
######################################################################
# NNVM Compilation
# ----------------
......@@ -157,8 +157,8 @@ predictions = tvm_output.asnumpy()
predictions = np.squeeze(predictions)
# Creates node ID --> English string lookup.
node_lookup = tf_testing.NodeLookup(label_lookup_path=os.path.join("./", map_proto),
uid_lookup_path=os.path.join("./", lable_map))
node_lookup = tf_testing.NodeLookup(label_lookup_path=map_proto_path,
uid_lookup_path=label_path)
# Print top 5 predictions from TVM output.
top_k = predictions.argsort()[-5:][::-1]
......@@ -175,7 +175,7 @@ for node_id in top_k:
def create_graph():
"""Creates a graph from saved GraphDef file and returns a saver."""
# Creates graph from saved graph_def.pb.
with tf.gfile.FastGFile(model_name, 'rb') as f:
with tf.gfile.FastGFile(model_path, 'rb') as f:
graph_def = tf.GraphDef()
graph_def.ParseFromString(f.read())
graph = tf.import_graph_def(graph_def, name='')
......@@ -209,8 +209,8 @@ def run_inference_on_image(image):
predictions = np.squeeze(predictions)
# Creates node ID --> English string lookup.
node_lookup = tf_testing.NodeLookup(label_lookup_path=os.path.join("./", map_proto),
uid_lookup_path=os.path.join("./", lable_map))
node_lookup = tf_testing.NodeLookup(label_lookup_path=map_proto_path,
uid_lookup_path=label_path)
# Print top 5 predictions from tensorflow.
top_k = predictions.argsort()[-5:][::-1]
......@@ -220,4 +220,4 @@ def run_inference_on_image(image):
score = predictions[node_id]
print('%s (score = %.5f)' % (human_string, score))
run_inference_on_image (img_name)
run_inference_on_image(img_path)
nnvm/tutorials/nlp/from_darknet_rnn.py
......@@ -22,9 +22,9 @@ by the script.
"""
import random
import numpy as np
from mxnet.gluon.utils import download
import tvm
from tvm.contrib import graph_runtime
from tvm.contrib.download import download_testdata
from nnvm.testing.darknet import __darknetffi__
import nnvm
import nnvm.frontend.darknet
......@@ -49,17 +49,15 @@ REPO_URL = 'https://github.com/dmlc/web-data/blob/master/darknet/'
CFG_URL = REPO_URL + 'cfg/' + CFG_NAME + '?raw=true'
WEIGHTS_URL = REPO_URL + 'weights/' + WEIGHTS_NAME + '?raw=true'
download(CFG_URL, CFG_NAME)
download(WEIGHTS_URL, WEIGHTS_NAME)
cfg_path = download_testdata(CFG_URL, CFG_NAME, module='darknet')
weights_path = download_testdata(WEIGHTS_URL, WEIGHTS_NAME, module='darknet')
# Download and Load darknet library
DARKNET_LIB = 'libdarknet.so'
DARKNET_URL = REPO_URL + 'lib/' + DARKNET_LIB + '?raw=true'
download(DARKNET_URL, DARKNET_LIB)
DARKNET_LIB = __darknetffi__.dlopen('./' + DARKNET_LIB)
cfg = "./" + str(CFG_NAME)
weights = "./" + str(WEIGHTS_NAME)
net = DARKNET_LIB.load_network(cfg.encode('utf-8'), weights.encode('utf-8'), 0)
lib_path = download_testdata(DARKNET_URL, DARKNET_LIB, module='darknet')
DARKNET_LIB = __darknetffi__.dlopen(lib_path)
net = DARKNET_LIB.load_network(cfg_path.encode('utf-8'), weights_path.encode('utf-8'), 0)
dtype = 'float32'
batch_size = 1
......
nnvm/tutorials/nlp/keras_s2s_translate.py
......@@ -59,9 +59,9 @@ model_url = os.path.join(repo_base, model_file)
data_url = os.path.join(repo_base, data_file)
# Download files listed below.
from mxnet.gluon.utils import download
download(model_url, model_file)
download(data_url, model_file)
from tvm.contrib.download import download_testdata
model_path = download_testdata(model_url, model_file, module='keras')
data_path = download_testdata(data_url, data_file, module='data')
latent_dim = 256 # Latent dimensionality of the encoding space.
test_samples = 10000 # Number of samples used for testing.
......@@ -76,7 +76,7 @@ input_texts = []
target_texts = []
input_characters = set()
target_characters = set()
with open(data_file, 'r', encoding='utf-8') as f:
with open(data_path, 'r', encoding='utf-8') as f:
lines = f.read().split('\n')
test_samples = min(test_samples, len(lines))
max_encoder_seq_length = 0
......@@ -112,7 +112,7 @@ reverse_target_char_index = dict(
# Load Keras Model
# ----------------
# Restore the model and construct the encoder and decoder.
model = load_model(model_file)
model = load_model(model_path)
encoder_inputs = model.input[0] # input_1
encoder_outputs, state_h_enc, state_c_enc = model.layers[2].output # lstm_1
......
python/tvm/contrib/download.py
......@@ -28,6 +28,9 @@ def download(url, path, overwrite=False, size_compare=False, verbose=1, retries=
verbose: int, optional
Verbose level
retries: int, optional
Number of time to retry download, default at 3.
"""
if sys.version_info >= (3,):
import urllib.request as urllib2
......
python/tvm/relay/testing/tf.py
......@@ -331,7 +331,7 @@ def get_workload_ptb():
ptb_model_file = 'RNN/ptb/ptb_model_with_lstmblockcell.pb'
import tarfile
file_path = download_testdata(sample_url, sample_data_file, module=['tf', 'ptb_data'])
file_path = download_testdata(sample_url, sample_data_file, module=['data', 'ptb_data'])
dir_path = os.path.dirname(file_path)
t = tarfile.open(file_path, 'r')
t.extractall(dir_path)
......
tests/python/frontend/coreml/model_zoo/__init__.py
......@@ -18,7 +18,7 @@ def get_resnet50():
def get_cat_image():
url = 'https://gist.githubusercontent.com/zhreshold/bcda4716699ac97ea44f791c24310193/raw/fa7ef0e9c9a5daea686d6473a62aacd1a5885849/cat.png'
dst = 'cat.png'
real_dst = download_testdata(url, dst, module='coreml')
real_dst = download_testdata(url, dst, module='data')
img = Image.open(real_dst).resize((224, 224))
img = np.transpose(img, (2, 0, 1))[np.newaxis, :]
return np.asarray(img)
\ No newline at end of file
tutorials/frontend/deploy_model_on_rasp.py
......@@ -14,6 +14,7 @@ import tvm
import tvm.relay as relay
from tvm import rpc
from tvm.contrib import util, graph_runtime as runtime
from tvm.contrib.download import download_testdata
######################################################################
# .. _build-tvm-runtime-on-device:
......@@ -82,7 +83,6 @@ from tvm.contrib import util, graph_runtime as runtime
# You can found more details about this part at tutorial :ref:`tutorial-from-mxnet`.
from mxnet.gluon.model_zoo.vision import get_model
from mxnet.gluon.utils import download
from PIL import Image
import numpy as np
......@@ -92,9 +92,10 @@ block = get_model('resnet18_v1', pretrained=True)
######################################################################
# In order to test our model, here we download an image of cat and
# transform its format.
img_url = 'https://github.com/dmlc/mxnet.js/blob/master/data/cat.png?raw=true'
img_name = 'cat.png'
download('https://github.com/dmlc/mxnet.js/blob/master/data/cat.png?raw=true', img_name)
image = Image.open(img_name).resize((224, 224))
img_path = download_testdata(img_url, img_name, module='data')
image = Image.open(img_path).resize((224, 224))
def transform_image(image):
image = np.array(image) - np.array([123., 117., 104.])
......@@ -112,9 +113,9 @@ synset_url = ''.join(['https://gist.githubusercontent.com/zhreshold/',
'4d0b62f3d01426887599d4f7ede23ee5/raw/',
'596b27d23537e5a1b5751d2b0481ef172f58b539/',
'imagenet1000_clsid_to_human.txt'])
synset_name = 'synset.txt'
download(synset_url, synset_name)
with open(synset_name) as f:
synset_name = 'imagenet1000_clsid_to_human.txt'
synset_path = download_testdata(synset_url, synset_name, module='data')
with open(synset_path) as f:
synset = eval(f.read())
######################################################################
......
tutorials/frontend/deploy_ssd_gluoncv.py
......@@ -12,6 +12,7 @@ from matplotlib import pyplot as plt
from tvm.relay.testing.config import ctx_list
from tvm import relay
from tvm.contrib import graph_runtime
from tvm.contrib.download import download_testdata
from gluoncv import model_zoo, data, utils
......@@ -49,9 +50,9 @@ target_list = ctx_list()
######################################################################
# Download and pre-process demo image
im_fname = utils.download('https://github.com/dmlc/web-data/blob/master/' +
'gluoncv/detection/street_small.jpg?raw=true',
path='street_small.jpg')
im_fname = download_testdata('https://github.com/dmlc/web-data/blob/master/' +
'gluoncv/detection/street_small.jpg?raw=true',
'street_small.jpg', module='data')
x, img = data.transforms.presets.ssd.load_test(im_fname, short=512)
######################################################################
......
tutorials/frontend/from_caffe2.py
......@@ -19,21 +19,6 @@ A quick solution is to install via conda
or please refer to official site
https://caffe2.ai/docs/getting-started.html
"""
######################################################################
# Utils for downloading files
# ----------------------------
def download(url, path, overwrite=False):
import os
if os.path.isfile(path) and not overwrite:
print('File {} exists, skip.'.format(path))
return
print('Downloading from url {} to {}'.format(url, path))
try:
import urllib.request
urllib.request.urlretrieve(url, path)
except:
import urllib
urllib.urlretrieve(url, path)
######################################################################
# Load pretrained Caffe2 model
......@@ -52,12 +37,13 @@ resnet50 = Model('resnet50')
# Load a test image
# ------------------
# A single cat dominates the examples!
from tvm.contrib.download import download_testdata
from PIL import Image
from matplotlib import pyplot as plt
import numpy as np
img_url = 'https://github.com/dmlc/mxnet.js/blob/master/data/cat.png?raw=true'
download(img_url, 'cat.png')
img = Image.open('cat.png').resize((224, 224))
img_path = download_testdata(img_url, 'cat.png', module='data')
img = Image.open(img_path).resize((224, 224))
plt.imshow(img)
plt.show()
# input preprocess
......@@ -118,9 +104,9 @@ synset_url = ''.join(['https://gist.githubusercontent.com/zhreshold/',
'4d0b62f3d01426887599d4f7ede23ee5/raw/',
'596b27d23537e5a1b5751d2b0481ef172f58b539/',
'imagenet1000_clsid_to_human.txt'])
synset_name = 'synset.txt'
download(synset_url, synset_name)
with open(synset_name) as f:
synset_name = 'imagenet1000_clsid_to_human.txt'
synset_path = download_testdata(synset_url, synset_name, module='data')
with open(synset_path) as f:
synset = eval(f.read())
print('Relay top-1 id: {}, class name: {}'.format(top1_tvm, synset[top1_tvm]))
# confirm correctness with caffe2 output
......
tutorials/frontend/from_coreml.py
......@@ -19,23 +19,11 @@ https://github.com/apple/coremltools
"""
import tvm
import tvm.relay as relay
from tvm.contrib.download import download_testdata
import coremltools as cm
import numpy as np
from PIL import Image
def download(url, path, overwrite=False):
import os
if os.path.isfile(path) and not overwrite:
print('File {} existed, skip.'.format(path))
return
print('Downloading from url {} to {}'.format(url, path))
try:
import urllib.request
urllib.request.urlretrieve(url, path)
except:
import urllib
urllib.urlretrieve(url, path)
######################################################################
# Load pretrained CoreML model
# ----------------------------
......@@ -43,17 +31,17 @@ def download(url, path, overwrite=False):
# provided by apple in this example
model_url = 'https://docs-assets.developer.apple.com/coreml/models/MobileNet.mlmodel'
model_file = 'mobilenet.mlmodel'
download(model_url, model_file)
model_path = download_testdata(model_url, model_file, module='coreml')
# Now you have mobilenet.mlmodel on disk
mlmodel = cm.models.MLModel(model_file)
mlmodel = cm.models.MLModel(model_path)
######################################################################
# Load a test image
# ------------------
# A single cat dominates the examples!
img_url = 'https://github.com/dmlc/mxnet.js/blob/master/data/cat.png?raw=true'
download(img_url, 'cat.png')
img = Image.open('cat.png').resize((224, 224))
img_path = download_testdata(img_url, 'cat.png', module='data')
img = Image.open(img_path).resize((224, 224))
x = np.transpose(img, (2, 0, 1))[np.newaxis, :]
######################################################################
......@@ -94,8 +82,8 @@ synset_url = ''.join(['https://gist.githubusercontent.com/zhreshold/',
'4d0b62f3d01426887599d4f7ede23ee5/raw/',
'596b27d23537e5a1b5751d2b0481ef172f58b539/',
'imagenet1000_clsid_to_human.txt'])
synset_name = 'synset.txt'
download(synset_url, synset_name)
with open(synset_name) as f:
synset_name = 'imagenet1000_clsid_to_human.txt'
synset_path = download_testdata(synset_url, synset_name, module='data')
with open(synset_path) as f:
synset = eval(f.read())
print('Top-1 id', top1, 'class name', synset[top1])
tutorials/frontend/from_keras.py
......@@ -20,22 +20,10 @@ https://keras.io/#installation
"""
import tvm
import tvm.relay as relay
from tvm.contrib.download import download_testdata
import keras
import numpy as np
def download(url, path, overwrite=False):
import os
if os.path.isfile(path) and not overwrite:
print('File {} exists, skip.'.format(path))
return
print('Downloading from url {} to {}'.format(url, path))
try:
import urllib.request
urllib.request.urlretrieve(url, path)
except:
import urllib
urllib.urlretrieve(url, path)
######################################################################
# Load pretrained keras model
# ----------------------------
......@@ -43,10 +31,10 @@ def download(url, path, overwrite=False):
weights_url = ''.join(['https://github.com/fchollet/deep-learning-models/releases/',
'download/v0.2/resnet50_weights_tf_dim_ordering_tf_kernels.h5'])
weights_file = 'resnet50_weights.h5'
download(weights_url, weights_file)
weights_path = download_testdata(weights_url, weights_file, module='keras')
keras_resnet50 = keras.applications.resnet50.ResNet50(include_top=True, weights=None,
input_shape=(224, 224, 3), classes=1000)
keras_resnet50.load_weights('resnet50_weights.h5')
keras_resnet50.load_weights(weights_path)
######################################################################
# Load a test image
......@@ -56,8 +44,8 @@ from PIL import Image
from matplotlib import pyplot as plt
from keras.applications.resnet50 import preprocess_input
img_url = 'https://github.com/dmlc/mxnet.js/blob/master/data/cat.png?raw=true'
download(img_url, 'cat.png')
img = Image.open('cat.png').resize((224, 224))
img_path = download_testdata(img_url, 'cat.png', module='data')
img = Image.open(img_path).resize((224, 224))
plt.imshow(img)
plt.show()
# input preprocess
......@@ -92,9 +80,9 @@ synset_url = ''.join(['https://gist.githubusercontent.com/zhreshold/',
'4d0b62f3d01426887599d4f7ede23ee5/raw/',
'596b27d23537e5a1b5751d2b0481ef172f58b539/',
'imagenet1000_clsid_to_human.txt'])
synset_name = 'synset.txt'
download(synset_url, synset_name)
with open(synset_name) as f:
synset_name = 'imagenet1000_clsid_to_human.txt'
synset_path = download_testdata(synset_url, synset_name, module='data')
with open(synset_path) as f:
synset = eval(f.read())
print('Relay top-1 id: {}, class name: {}'.format(top1_tvm, synset[top1_tvm]))
# confirm correctness with keras output
......
tutorials/frontend/from_mxnet.py
......@@ -29,22 +29,23 @@ import numpy as np
# Download Resnet18 model from Gluon Model Zoo
# ---------------------------------------------
# In this section, we download a pretrained imagenet model and classify an image.
from tvm.contrib.download import download_testdata
from mxnet.gluon.model_zoo.vision import get_model
from mxnet.gluon.utils import download
from PIL import Image
from matplotlib import pyplot as plt
block = get_model('resnet18_v1', pretrained=True)
img_url = 'https://github.com/dmlc/mxnet.js/blob/master/data/cat.png?raw=true'
img_name = 'cat.png'
synset_url = ''.join(['https://gist.githubusercontent.com/zhreshold/',
'4d0b62f3d01426887599d4f7ede23ee5/raw/',
'596b27d23537e5a1b5751d2b0481ef172f58b539/',
'imagenet1000_clsid_to_human.txt'])
synset_name = 'synset.txt'
download('https://github.com/dmlc/mxnet.js/blob/master/data/cat.png?raw=true', img_name)
download(synset_url, synset_name)
with open(synset_name) as f:
synset_name = 'imagenet1000_clsid_to_human.txt'
img_path = download_testdata(img_url, 'cat.png', module='data')
synset_path = download_testdata(synset_url, synset_name, module='data')
with open(synset_path) as f:
synset = eval(f.read())
image = Image.open(img_name).resize((224, 224))
image = Image.open(img_path).resize((224, 224))
plt.imshow(image)
plt.show()
......
tutorials/frontend/from_onnx.py
......@@ -20,19 +20,7 @@ import onnx
import numpy as np
import tvm
import tvm.relay as relay
def download(url, path, overwrite=False):
import os
if os.path.isfile(path) and not overwrite:
print('File {} existed, skip.'.format(path))
return
print('Downloading from url {} to {}'.format(url, path))
try:
import urllib.request
urllib.request.urlretrieve(url, path)
except:
import urllib
urllib.urlretrieve(url, path)
from tvm.contrib.download import download_testdata
######################################################################
# Load pretrained ONNX model
......@@ -44,9 +32,9 @@ model_url = ''.join(['https://gist.github.com/zhreshold/',
'bcda4716699ac97ea44f791c24310193/raw/',
'93672b029103648953c4e5ad3ac3aadf346a4cdc/',
'super_resolution_0.2.onnx'])
download(model_url, 'super_resolution.onnx', False)
model_path = download_testdata(model_url, 'super_resolution.onnx', module='onnx')
# now you have super_resolution.onnx on disk
onnx_model = onnx.load('super_resolution.onnx')
onnx_model = onnx.load(model_path)
######################################################################
# Load a test image
......@@ -54,8 +42,8 @@ onnx_model = onnx.load('super_resolution.onnx')
# A single cat dominates the examples!
from PIL import Image
img_url = 'https://github.com/dmlc/mxnet.js/blob/master/data/cat.png?raw=true'
download(img_url, 'cat.png')
img = Image.open('cat.png').resize((224, 224))
img_path = download_testdata(img_url, 'cat.png', module='data')
img = Image.open(img_path).resize((224, 224))
img_ycbcr = img.convert("YCbCr") # convert to YCbCr
img_y, img_cb, img_cr = img_ycbcr.split()
x = np.array(img_y)[np.newaxis, np.newaxis, :, :]
......
tutorials/frontend/from_tensorflow.py
......@@ -43,8 +43,8 @@ map_proto = 'imagenet_2012_challenge_label_map_proto.pbtxt'
map_proto_url = os.path.join(repo_base, map_proto)
# Human readable text for labels
lable_map = 'imagenet_synset_to_human_label_map.txt'
lable_map_url = os.path.join(repo_base, lable_map)
label_map = 'imagenet_synset_to_human_label_map.txt'
label_map_url = os.path.join(repo_base, label_map)
# Target settings
# Use these commented settings to build for cuda.
......@@ -61,19 +61,19 @@ ctx = tvm.cpu(0)
# Download required files
# -----------------------
# Download files listed above.
from mxnet.gluon.utils import download
from tvm.contrib.download import download_testdata
download(image_url, img_name)
download(model_url, model_name)
download(map_proto_url, map_proto)
download(lable_map_url, lable_map)
img_path = download_testdata(image_url, img_name, module='data')
model_path = download_testdata(model_url, model_name, module=['tf', 'InceptionV1'])
map_proto_path = download_testdata(map_proto_url, map_proto, module='data')
label_path = download_testdata(label_map_url, label_map, module='data')
######################################################################
# Import model
# ------------
# Creates tensorflow graph definition from protobuf file.
with tf.gfile.FastGFile(os.path.join("./", model_name), 'rb') as f:
with tf.gfile.FastGFile(model_path, 'rb') as f:
graph_def = tf.GraphDef()
graph_def.ParseFromString(f.read())
graph = tf.import_graph_def(graph_def, name='')
......@@ -94,7 +94,7 @@ with tf.gfile.FastGFile(os.path.join("./", model_name), 'rb') as f:
#
from PIL import Image
image = Image.open(img_name).resize((299, 299))
image = Image.open(img_path).resize((299, 299))
x = np.array(image)
......@@ -110,7 +110,7 @@ shape_dict = {'DecodeJpeg/contents': x.shape}
dtype_dict = {'DecodeJpeg/contents': 'uint8'}
sym, params = relay.frontend.from_tensorflow(graph_def, layout=layout, shape=shape_dict)
print ("Tensorflow protobuf imported to relay frontend.")
print("Tensorflow protobuf imported to relay frontend.")
######################################################################
# Relay Build
# -----------
......@@ -148,8 +148,8 @@ predictions = tvm_output.asnumpy()
predictions = np.squeeze(predictions)
# Creates node ID --> English string lookup.
node_lookup = tf_testing.NodeLookup(label_lookup_path=os.path.join("./", map_proto),
uid_lookup_path=os.path.join("./", lable_map))
node_lookup = tf_testing.NodeLookup(label_lookup_path=map_proto_path,
uid_lookup_path=label_path)
# Print top 5 predictions from TVM output.
top_k = predictions.argsort()[-5:][::-1]
......@@ -166,7 +166,7 @@ for node_id in top_k:
def create_graph():
"""Creates a graph from saved GraphDef file and returns a saver."""
# Creates graph from saved graph_def.pb.
with tf.gfile.FastGFile(model_name, 'rb') as f:
with tf.gfile.FastGFile(model_path, 'rb') as f:
graph_def = tf.GraphDef()
graph_def.ParseFromString(f.read())
graph = tf.import_graph_def(graph_def, name='')
......@@ -200,8 +200,8 @@ def run_inference_on_image(image):
predictions = np.squeeze(predictions)
# Creates node ID --> English string lookup.
node_lookup = tf_testing.NodeLookup(label_lookup_path=os.path.join("./", map_proto),
uid_lookup_path=os.path.join("./", lable_map))
node_lookup = tf_testing.NodeLookup(label_lookup_path=map_proto_path,
uid_lookup_path=label_path)
# Print top 5 predictions from tensorflow.
top_k = predictions.argsort()[-5:][::-1]
......@@ -211,4 +211,4 @@ def run_inference_on_image(image):
score = predictions[node_id]
print('%s (score = %.5f)' % (human_string, score))
run_inference_on_image (img_name)
run_inference_on_image(img_path)
tutorials/frontend/from_tflite.py
......@@ -52,25 +52,14 @@ Below you can find an example for how to compile TFLite model using TVM.
######################################################################
# Utils for downloading and extracting zip files
# ---------------------------------------------
def download(url, path, overwrite=False):
import os
if os.path.isfile(path) and not overwrite:
print('File {} existed, skip.'.format(path))
return
print('Downloading from url {} to {}'.format(url, path))
try:
import urllib.request
urllib.request.urlretrieve(url, path)
except:
import urllib
urllib.urlretrieve(url, path)
import os
def extract(path):
import tarfile
if path.endswith("tgz") or path.endswith("gz"):
dir_path = os.path.dirname(path)
tar = tarfile.open(path)
tar.extractall()
tar.extractall(path=dir_path)
tar.close()
else:
raise RuntimeError('Could not decompress the file: ' + path)
......@@ -80,14 +69,17 @@ def extract(path):
# Load pretrained TFLite model
# ---------------------------------------------
# we load mobilenet V1 TFLite model provided by Google
from tvm.contrib.download import download_testdata
model_url = "http://download.tensorflow.org/models/mobilenet_v1_2018_08_02/mobilenet_v1_1.0_224.tgz"
# we download model tar file and extract, finally get mobilenet_v1_1.0_224.tflite
download(model_url, "mobilenet_v1_1.0_224.tgz", False)
extract("mobilenet_v1_1.0_224.tgz")
model_path = download_testdata(model_url, "mobilenet_v1_1.0_224.tgz", module=['tf', 'official'])
model_dir = os.path.dirname(model_path)
extract(model_path)
# now we have mobilenet_v1_1.0_224.tflite on disk and open it
tflite_model_file = "mobilenet_v1_1.0_224.tflite"
tflite_model_file = os.path.join(model_dir, "mobilenet_v1_1.0_224.tflite")
tflite_model_buf = open(tflite_model_file, "rb").read()
# get TFLite model from buffer
......@@ -103,8 +95,8 @@ from matplotlib import pyplot as plt
import numpy as np
image_url = 'https://github.com/dmlc/mxnet.js/blob/master/data/cat.png?raw=true'
download(image_url, 'cat.png')
resized_image = Image.open('cat.png').resize((224, 224))
image_path = download_testdata(image_url, 'cat.png', module='data')
resized_image = Image.open(image_path).resize((224, 224))
plt.imshow(resized_image)
plt.show()
image_data = np.asarray(resized_image).astype("float32")
......@@ -179,11 +171,11 @@ label_file_url = ''.join(['https://raw.githubusercontent.com/',
'app/src/main/assets/',
'labels_mobilenet_quant_v1_224.txt'])
label_file = "labels_mobilenet_quant_v1_224.txt"
download(label_file_url, label_file)
label_path = download_testdata(label_file_url, label_file, module='data')
# map id to 1001 classes
labels = dict()
with open(label_file) as f:
with open(label_path) as f:
for id, line in enumerate(f):
labels[id] = line
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment