[Tutorial] Cache the test data in tutorial (#2923)

nnvm/tests/python/frontend/coreml/model_zoo/__init__.py

@@ -18,7 +18,7 @@ def get_resnet50():
 def get_cat_image():
     url = 'https://gist.githubusercontent.com/zhreshold/bcda4716699ac97ea44f791c24310193/raw/fa7ef0e9c9a5daea686d6473a62aacd1a5885849/cat.png'
     dst = 'cat.png'
-    real_dst = download_testdata(url, dst, module='coreml')
+    real_dst = download_testdata(url, dst, module='data')
     img = Image.open(real_dst).resize((224, 224))
     img = np.transpose(img, (2, 0, 1))[np.newaxis, :]
     return np.asarray(img)

nnvm/tests/python/frontend/darknet/test_forward.py

@@ -103,7 +103,7 @@ def test_forward(net, build_dtype='float32'):
 
     test_image = 'dog.jpg'
     img_url = 'https://github.com/siju-samuel/darknet/blob/master/data/' + test_image   +'?raw=true'
-    img_path = download_testdata(img_url, test_image, module='darknet')
+    img_path = download_testdata(img_url, test_image, module='data')
     img = LIB.letterbox_image(LIB.load_image_color(img_path.encode('utf-8'), 0, 0), net.w, net.h)
     darknet_output = get_darknet_output(net, img)
     batch_size = 1

nnvm/tutorials/deploy_model_on_mali_gpu.py

@@ -15,6 +15,7 @@ import nnvm.compiler
 import nnvm.testing
 from tvm import rpc
 from tvm.contrib import util, graph_runtime as runtime
+from tvm.contrib.download import download_testdata
 
 ######################################################################
 # Build TVM Runtime on Device
@@ -81,7 +82,6 @@ from tvm.contrib import util, graph_runtime as runtime
 # You can found more details about this part at tutorial :ref:`tutorial-from-mxnet`.
 
 from mxnet.gluon.model_zoo.vision import get_model
-from mxnet.gluon.utils import download
 from PIL import Image
 import numpy as np
 
@@ -92,8 +92,9 @@ block = get_model('resnet18_v1', pretrained=True)
 # In order to test our model, here we download an image of cat and
 # transform its format.
 img_name = 'cat.png'
-download('https://github.com/dmlc/mxnet.js/blob/master/data/cat.png?raw=true', img_name)
-image = Image.open(img_name).resize((224, 224))
+img_path = download_testdata('https://github.com/dmlc/mxnet.js/blob/master/data/cat.png?raw=true',
+                             img_name, module='data')
+image = Image.open(img_path).resize((224, 224))
 
 def transform_image(image):
     image = np.array(image) - np.array([123., 117., 104.])
@@ -112,9 +113,9 @@ synset_url = ''.join(['https://gist.githubusercontent.com/zhreshold/',
                       '596b27d23537e5a1b5751d2b0481ef172f58b539/',
                       'imagenet1000_clsid_to_human.txt'])
 
-synset_name = 'synset.txt'
-download(synset_url, synset_name)
-with open(synset_name) as f:
+synset_name = 'imagenet1000_clsid_to_human.txt'
+synset_path = download_testdata(synset_url, synset_name, module='data')
+with open(synset_path) as f:
     synset = eval(f.read())
 
 ######################################################################

nnvm/tutorials/deploy_model_on_rasp.py

@@ -14,6 +14,7 @@ import nnvm.compiler
 import nnvm.testing
 from tvm import rpc
 from tvm.contrib import util, graph_runtime as runtime
+from tvm.contrib.download import download_testdata
 
 ######################################################################
 # .. _build-tvm-runtime-on-device:
@@ -78,7 +79,6 @@ from tvm.contrib import util, graph_runtime as runtime
 # You can found more details about this part at tutorial :ref:`tutorial-from-mxnet`.
 
 from mxnet.gluon.model_zoo.vision import get_model
-from mxnet.gluon.utils import download
 from PIL import Image
 import numpy as np
 
@@ -89,8 +89,9 @@ block = get_model('resnet18_v1', pretrained=True)
 # In order to test our model, here we download an image of cat and
 # transform its format.
 img_name = 'cat.png'
-download('https://github.com/dmlc/mxnet.js/blob/master/data/cat.png?raw=true', img_name)
-image = Image.open(img_name).resize((224, 224))
+img_path = download_testdata('https://github.com/dmlc/mxnet.js/blob/master/data/cat.png?raw=true',
+                             img_name, module='data')
+image = Image.open(img_path).resize((224, 224))
 
 def transform_image(image):
     image = np.array(image) - np.array([123., 117., 104.])
@@ -108,9 +109,9 @@ synset_url = ''.join(['https://gist.githubusercontent.com/zhreshold/',
                       '4d0b62f3d01426887599d4f7ede23ee5/raw/',
                       '596b27d23537e5a1b5751d2b0481ef172f58b539/',
                       'imagenet1000_clsid_to_human.txt'])
-synset_name = 'synset.txt'
-download(synset_url, synset_name)
-with open(synset_name) as f:
+synset_name = 'imagenet1000_clsid_to_human.txt'
+synset_path = download_testdata(synset_url, synset_name, module='data')
+with open(synset_path) as f:
     synset = eval(f.read())
 
 ######################################################################

nnvm/tutorials/deploy_ssd_mxnet.py

@@ -18,7 +18,7 @@ import numpy as np
 from nnvm import compiler
 from nnvm.frontend import from_mxnet
 from tvm import relay
-from tvm.contrib.download import download
+from tvm.contrib.download import download_testdata
 from tvm.contrib import graph_runtime
 from mxnet.model import load_checkpoint
 
@@ -65,28 +65,24 @@ inference_symbol_folder = \
 inference_symbol_url = "https://gist.github.com/kevinthesun/c1904e900848df4548ce5dfb18c719c7/" \
                        "archive/a28c4856c827fe766aa3da0e35bad41d44f0fb26.zip"
 
-dir = "ssd_model"
-if not os.path.exists(dir):
-    os.makedirs(dir)
-model_file_path = "%s/%s" % (dir, model_file)
-test_image_path = "%s/%s" % (dir, test_image)
-inference_symbol_path = "%s/inference_model.zip" % dir
-download(model_url, model_file_path)
-download(image_url, test_image_path)
-download(inference_symbol_url, inference_symbol_path)
+model_file_path = download_testdata(model_url, model_file, module=["mxnet", "ssd_model"])
+inference_symbol_path = download_testdata(inference_symbol_url, "inference_model.zip",
+                                          module=["mxnet", "ssd_model"])
+test_image_path = download_testdata(image_url, test_image, module="data")
+model_dir = os.path.dirname(model_file_path)
 
 zip_ref = zipfile.ZipFile(model_file_path, 'r')
-zip_ref.extractall(dir)
+zip_ref.extractall(model_dir)
 zip_ref.close()
 zip_ref = zipfile.ZipFile(inference_symbol_path)
-zip_ref.extractall(dir)
+zip_ref.extractall(model_dir)
 zip_ref.close()
 
 ######################################################################
 # Convert and compile model with NNVM or Relay for CPU.
 
-sym = mx.sym.load("%s/%s/ssd_resnet50_inference.json" % (dir, inference_symbol_folder))
-_, arg_params, aux_params = load_checkpoint("%s/%s" % (dir, model_name), 0)
+sym = mx.sym.load("%s/%s/ssd_resnet50_inference.json" % (model_dir, inference_symbol_folder))
+_, arg_params, aux_params = load_checkpoint("%s/%s" % (model_dir, model_name), 0)
 
 import argparse
 parser = argparse.ArgumentParser()

nnvm/tutorials/from_coreml.py

@@ -21,19 +21,7 @@ import tvm
 import coremltools as cm
 import numpy as np
 from PIL import Image
-
-def download(url, path, overwrite=False):
-    import os
-    if os.path.isfile(path) and not overwrite:
-        print('File {} existed, skip.'.format(path))
-        return
-    print('Downloading from url {} to {}'.format(url, path))
-    try:
-        import urllib.request
-        urllib.request.urlretrieve(url, path)
-    except:
-        import urllib
-        urllib.urlretrieve(url, path)
+from tvm.contrib.download import download_testdata
 
 ######################################################################
 # Load pretrained CoreML model
@@ -42,9 +30,9 @@ def download(url, path, overwrite=False):
 # provided by apple in this example
 model_url = 'https://docs-assets.developer.apple.com/coreml/models/MobileNet.mlmodel'
 model_file = 'mobilenet.mlmodel'
-download(model_url, model_file)
+model_path = download_testdata(model_url, model_file, module='coreml')
 # now you mobilenet.mlmodel on disk
-mlmodel = cm.models.MLModel(model_file)
+mlmodel = cm.models.MLModel(model_path)
 # we can load the graph as NNVM compatible model
 sym, params = nnvm.frontend.from_coreml(mlmodel)
 
@@ -54,8 +42,8 @@ sym, params = nnvm.frontend.from_coreml(mlmodel)
 # A single cat dominates the examples!
 from PIL import Image
 img_url = 'https://github.com/dmlc/mxnet.js/blob/master/data/cat.png?raw=true'
-download(img_url, 'cat.png')
-img = Image.open('cat.png').resize((224, 224))
+img_path = download_testdata(img_url, 'cat.png', module='data')
+img = Image.open(img_path).resize((224, 224))
 #x = np.transpose(img, (2, 0, 1))[np.newaxis, :]
 image = np.asarray(img)
 image = image.transpose((2, 0, 1))
@@ -95,8 +83,8 @@ synset_url = ''.join(['https://gist.githubusercontent.com/zhreshold/',
                       '4d0b62f3d01426887599d4f7ede23ee5/raw/',
                       '596b27d23537e5a1b5751d2b0481ef172f58b539/',
                       'imagenet1000_clsid_to_human.txt'])
-synset_name = 'synset.txt'
-download(synset_url, synset_name)
-with open(synset_name) as f:
+synset_name = 'imagenet1000_clsid_to_human.txt'
+synset_path = download_testdata(synset_url, synset_name, module='data')
+with open(synset_path) as f:
     synset = eval(f.read())
 print('Top-1 id', top1, 'class name', synset[top1])

nnvm/tutorials/from_darknet.py

@@ -25,7 +25,7 @@ import tvm
 import sys
 
 from ctypes import *
-from tvm.contrib.download import download
+from tvm.contrib.download import download_testdata
 from nnvm.testing.darknet import __darknetffi__
 
 # Model name
@@ -41,8 +41,8 @@ REPO_URL = 'https://github.com/siju-samuel/darknet/blob/master/'
 CFG_URL = REPO_URL + 'cfg/' + CFG_NAME + '?raw=true'
 WEIGHTS_URL = 'https://pjreddie.com/media/files/' + WEIGHTS_NAME
 
-download(CFG_URL, CFG_NAME)
-download(WEIGHTS_URL, WEIGHTS_NAME)
+cfg_path = download_testdata(CFG_URL, CFG_NAME, module="darknet")
+weights_path = download_testdata(WEIGHTS_URL, WEIGHTS_NAME, module="darknet")
 
 # Download and Load darknet library
 if sys.platform in ['linux', 'linux2']:
@@ -55,12 +55,10 @@ else:
     err = "Darknet lib is not supported on {} platform".format(sys.platform)
     raise NotImplementedError(err)
 
-download(DARKNET_URL, DARKNET_LIB)
+lib_path = download_testdata(DARKNET_URL, DARKNET_LIB, module="darknet")
 
-DARKNET_LIB = __darknetffi__.dlopen('./' + DARKNET_LIB)
-cfg = "./" + str(CFG_NAME)
-weights = "./" + str(WEIGHTS_NAME)
-net = DARKNET_LIB.load_network(cfg.encode('utf-8'), weights.encode('utf-8'), 0)
+DARKNET_LIB = __darknetffi__.dlopen(lib_path)
+net = DARKNET_LIB.load_network(cfg_path.encode('utf-8'), weights_path.encode('utf-8'), 0)
 dtype = 'float32'
 batch_size = 1
 
@@ -88,9 +86,9 @@ test_image = 'dog.jpg'
 print("Loading the test image...")
 img_url = 'https://github.com/siju-samuel/darknet/blob/master/data/' + \
           test_image + '?raw=true'
-download(img_url, test_image)
+img_path = download_testdata(img_url, test_image, "data")
 
-data = nnvm.testing.darknet.load_image(test_image, netw, neth)
+data = nnvm.testing.darknet.load_image(img_path, netw, neth)
 ######################################################################
 # Execute on TVM Runtime
 # ----------------------
@@ -150,10 +148,10 @@ coco_name = 'coco.names'
 coco_url = 'https://github.com/siju-samuel/darknet/blob/master/data/' + coco_name + '?raw=true'
 font_name = 'arial.ttf'
 font_url = 'https://github.com/siju-samuel/darknet/blob/master/data/' + font_name + '?raw=true'
-download(coco_url, coco_name)
-download(font_url, font_name)
+coco_path = download_testdata(coco_url, coco_name, module='data')
+font_path = download_testdata(font_url, font_name, module='data')
 
-with open(coco_name) as f:
+with open(coco_path) as f:
     content = f.readlines()
 
 names = [x.strip() for x in content]

nnvm/tutorials/from_mxnet.py

@@ -20,30 +20,31 @@ https://mxnet.incubator.apache.org/versions/master/install/index.html
 """
 # some standard imports
 import mxnet as mx
+import numpy as np
 import nnvm
 import tvm
-import numpy as np
+from tvm.contrib.download import download_testdata
 
 ######################################################################
 # Download Resnet18 model from Gluon Model Zoo
 # ---------------------------------------------
 # In this section, we download a pretrained imagenet model and classify an image.
 from mxnet.gluon.model_zoo.vision import get_model
-from mxnet.gluon.utils import download
 from PIL import Image
 from matplotlib import pyplot as plt
 block = get_model('resnet18_v1', pretrained=True)
+img_url = 'https://github.com/dmlc/mxnet.js/blob/master/data/cat.png?raw=true'
 img_name = 'cat.png'
 synset_url = ''.join(['https://gist.githubusercontent.com/zhreshold/',
                       '4d0b62f3d01426887599d4f7ede23ee5/raw/',
                       '596b27d23537e5a1b5751d2b0481ef172f58b539/',
                       'imagenet1000_clsid_to_human.txt'])
-synset_name = 'synset.txt'
-download('https://github.com/dmlc/mxnet.js/blob/master/data/cat.png?raw=true', img_name)
-download(synset_url, synset_name)
-with open(synset_name) as f:
+synset_name = 'imagenet1000_clsid_to_human.txt'
+img_path = download_testdata(img_url, img_name, module='data')
+synset_path = download_testdata(synset_url, synset_name, module='data')
+with open(synset_path) as f:
     synset = eval(f.read())
-image = Image.open(img_name).resize((224, 224))
+image = Image.open(img_path).resize((224, 224))
 plt.imshow(image)
 plt.show()
 

nnvm/tutorials/from_mxnet_to_webgl.py

@@ -33,6 +33,7 @@ from __future__ import print_function
 
 import numpy as np
 import tvm
+from tvm.contrib.download import download_testdata
 import nnvm.compiler
 import nnvm.testing
 
@@ -106,16 +107,14 @@ def download_synset():
 
     print("Downloading synset...")
 
-    from mxnet import gluon
-
     url = "https://gist.githubusercontent.com/zhreshold/" + \
           "4d0b62f3d01426887599d4f7ede23ee5/raw/" + \
           "596b27d23537e5a1b5751d2b0481ef172f58b539/" + \
           "imagenet1000_clsid_to_human.txt"
-    file_name = "synset.txt"
+    file_name = "imagenet1000_clsid_to_human.txt"
 
-    gluon.utils.download(url, file_name)
-    with open(file_name) as f:
+    file_path = download_testdata(url, file_name, module='data')
+    with open(file_path) as f:
         synset = eval(f.read())
 
     print("- Synset downloaded!")
@@ -144,14 +143,13 @@ def download_image():
     print("Downloading cat image...")
 
     from matplotlib import pyplot as plt
-    from mxnet import gluon
     from PIL import Image
 
     url = "https://github.com/dmlc/mxnet.js/blob/master/data/cat.png?raw=true"
     img_name = "cat.png"
 
-    gluon.utils.download(url, img_name)
-    image = Image.open(img_name).resize((224, 224))
+    img_path = download_testdata(url, img_name, module='data')
+    image = Image.open(img_path).resize((224, 224))
 
     print("- Cat image downloaded!")
 

nnvm/tutorials/from_onnx.py

@@ -18,22 +18,10 @@ https://github.com/onnx/onnx
 """
 import nnvm
 import tvm
+from tvm.contrib.download import download_testdata
 import onnx
 import numpy as np
 
-def download(url, path, overwrite=False):
-    import os
-    if os.path.isfile(path) and not overwrite:
-        print('File {} existed, skip.'.format(path))
-        return
-    print('Downloading from url {} to {}'.format(url, path))
-    try:
-        import urllib.request
-        urllib.request.urlretrieve(url, path)
-    except:
-        import urllib
-        urllib.urlretrieve(url, path)
-
 ######################################################################
 # Load pretrained ONNX model
 # ---------------------------------------------
@@ -44,9 +32,9 @@ model_url = ''.join(['https://gist.github.com/zhreshold/',
                      'bcda4716699ac97ea44f791c24310193/raw/',
                      '93672b029103648953c4e5ad3ac3aadf346a4cdc/',
                      'super_resolution_0.2.onnx'])
-download(model_url, 'super_resolution.onnx', True)
+model_path = download_testdata(model_url, 'super_resolution.onnx', module='onnx')
 # now you have super_resolution.onnx on disk
-onnx_model = onnx.load_model('super_resolution.onnx')
+onnx_model = onnx.load_model(model_path)
 # we can load the graph as NNVM compatible model
 sym, params = nnvm.frontend.from_onnx(onnx_model)
 
@@ -56,8 +44,8 @@ sym, params = nnvm.frontend.from_onnx(onnx_model)
 # A single cat dominates the examples!
 from PIL import Image
 img_url = 'https://github.com/dmlc/mxnet.js/blob/master/data/cat.png?raw=true'
-download(img_url, 'cat.png')
-img = Image.open('cat.png').resize((224, 224))
+img_path = download_testdata(img_url, 'cat.png', module='data')
+img = Image.open(img_path).resize((224, 224))
 img_ycbcr = img.convert("YCbCr")  # convert to YCbCr
 img_y, img_cb, img_cr = img_ycbcr.split()
 x = np.array(img_y)[np.newaxis, np.newaxis, :, :]

nnvm/tutorials/from_tensorflow.py

@@ -52,8 +52,8 @@ map_proto = 'imagenet_2012_challenge_label_map_proto.pbtxt'
 map_proto_url = os.path.join(repo_base, map_proto)
 
 # Human readable text for labels
-lable_map = 'imagenet_synset_to_human_label_map.txt'
-lable_map_url = os.path.join(repo_base, lable_map)
+label_map = 'imagenet_synset_to_human_label_map.txt'
+label_map_url = os.path.join(repo_base, label_map)
 
 # Target settings
 # Use these commented settings to build for cuda.
@@ -70,19 +70,19 @@ ctx = tvm.cpu(0)
 # Download required files
 # -----------------------
 # Download files listed above.
-from mxnet.gluon.utils import download
+from tvm.contrib.download import download_testdata
 
-download(image_url, img_name)
-download(model_url, model_name)
-download(map_proto_url, map_proto)
-download(lable_map_url, lable_map)
+img_path = download_testdata(image_url, img_name, module='data')
+model_path = download_testdata(model_url, model_name, module=['tf', 'InceptionV1'])
+map_proto_path = download_testdata(map_proto_url, map_proto, module='data')
+label_path = download_testdata(label_map_url, label_map, module='data')
 
 ######################################################################
 # Import model
 # ------------
 # Creates tensorflow graph definition from protobuf file.
 
-with tf.gfile.FastGFile(os.path.join("./", model_name), 'rb') as f:
+with tf.gfile.FastGFile(model_path, 'rb') as f:
     graph_def = tf.GraphDef()
     graph_def.ParseFromString(f.read())
     graph = tf.import_graph_def(graph_def, name='')
@@ -103,7 +103,7 @@ with tf.gfile.FastGFile(os.path.join("./", model_name), 'rb') as f:
 #
 
 from PIL import Image
-image = Image.open(img_name).resize((299, 299))
+image = Image.open(img_path).resize((299, 299))
 
 x = np.array(image)
 
@@ -117,7 +117,7 @@ x = np.array(image)
 #   params: params converted from tensorflow params (tensor protobuf).
 sym, params = nnvm.frontend.from_tensorflow(graph_def, layout=layout)
 
-print ("Tensorflow protobuf imported as nnvm graph")
+print("Tensorflow protobuf imported as nnvm graph")
 ######################################################################
 # NNVM Compilation
 # ----------------
@@ -157,8 +157,8 @@ predictions = tvm_output.asnumpy()
 predictions = np.squeeze(predictions)
 
 # Creates node ID --> English string lookup.
-node_lookup = tf_testing.NodeLookup(label_lookup_path=os.path.join("./", map_proto),
-                                         uid_lookup_path=os.path.join("./", lable_map))
+node_lookup = tf_testing.NodeLookup(label_lookup_path=map_proto_path,
+                                    uid_lookup_path=label_path)
 
 # Print top 5 predictions from TVM output.
 top_k = predictions.argsort()[-5:][::-1]
@@ -175,7 +175,7 @@ for node_id in top_k:
 def create_graph():
     """Creates a graph from saved GraphDef file and returns a saver."""
     # Creates graph from saved graph_def.pb.
-    with tf.gfile.FastGFile(model_name, 'rb') as f:
+    with tf.gfile.FastGFile(model_path, 'rb') as f:
         graph_def = tf.GraphDef()
         graph_def.ParseFromString(f.read())
         graph = tf.import_graph_def(graph_def, name='')
@@ -209,8 +209,8 @@ def run_inference_on_image(image):
         predictions = np.squeeze(predictions)
 
         # Creates node ID --> English string lookup.
-        node_lookup = tf_testing.NodeLookup(label_lookup_path=os.path.join("./", map_proto),
-                                                 uid_lookup_path=os.path.join("./", lable_map))
+        node_lookup = tf_testing.NodeLookup(label_lookup_path=map_proto_path,
+                                            uid_lookup_path=label_path)
 
         # Print top 5 predictions from tensorflow.
         top_k = predictions.argsort()[-5:][::-1]
@@ -220,4 +220,4 @@ def run_inference_on_image(image):
             score = predictions[node_id]
             print('%s (score = %.5f)' % (human_string, score))
 
-run_inference_on_image (img_name)
+run_inference_on_image(img_path)

nnvm/tutorials/nlp/from_darknet_rnn.py

@@ -22,9 +22,9 @@ by the script.
 """
 import random
 import numpy as np
-from mxnet.gluon.utils import download
 import tvm
 from tvm.contrib import graph_runtime
+from tvm.contrib.download import download_testdata
 from nnvm.testing.darknet import __darknetffi__
 import nnvm
 import nnvm.frontend.darknet
@@ -49,17 +49,15 @@ REPO_URL = 'https://github.com/dmlc/web-data/blob/master/darknet/'
 CFG_URL = REPO_URL + 'cfg/' + CFG_NAME + '?raw=true'
 WEIGHTS_URL = REPO_URL + 'weights/' + WEIGHTS_NAME + '?raw=true'
 
-download(CFG_URL, CFG_NAME)
-download(WEIGHTS_URL, WEIGHTS_NAME)
+cfg_path = download_testdata(CFG_URL, CFG_NAME, module='darknet')
+weights_path = download_testdata(WEIGHTS_URL, WEIGHTS_NAME, module='darknet')
 
 # Download and Load darknet library
 DARKNET_LIB = 'libdarknet.so'
 DARKNET_URL = REPO_URL + 'lib/' + DARKNET_LIB + '?raw=true'
-download(DARKNET_URL, DARKNET_LIB)
-DARKNET_LIB = __darknetffi__.dlopen('./' + DARKNET_LIB)
-cfg = "./" + str(CFG_NAME)
-weights = "./" + str(WEIGHTS_NAME)
-net = DARKNET_LIB.load_network(cfg.encode('utf-8'), weights.encode('utf-8'), 0)
+lib_path = download_testdata(DARKNET_URL, DARKNET_LIB, module='darknet')
+DARKNET_LIB = __darknetffi__.dlopen(lib_path)
+net = DARKNET_LIB.load_network(cfg_path.encode('utf-8'), weights_path.encode('utf-8'), 0)
 dtype = 'float32'
 batch_size = 1
 

nnvm/tutorials/nlp/keras_s2s_translate.py

@@ -59,9 +59,9 @@ model_url = os.path.join(repo_base, model_file)
 data_url = os.path.join(repo_base, data_file)
 
 # Download files listed below.
-from mxnet.gluon.utils import download
-download(model_url, model_file)
-download(data_url, model_file)
+from tvm.contrib.download import download_testdata
+model_path = download_testdata(model_url, model_file, module='keras')
+data_path = download_testdata(data_url, data_file, module='data')
 
 latent_dim = 256  # Latent dimensionality of the encoding space.
 test_samples = 10000  # Number of samples used for testing.
@@ -76,7 +76,7 @@ input_texts = []
 target_texts = []
 input_characters = set()
 target_characters = set()
-with open(data_file, 'r', encoding='utf-8') as f:
+with open(data_path, 'r', encoding='utf-8') as f:
     lines = f.read().split('\n')
 test_samples = min(test_samples, len(lines))
 max_encoder_seq_length = 0
@@ -112,7 +112,7 @@ reverse_target_char_index = dict(
 # Load Keras Model
 # ----------------
 # Restore the model and construct the encoder and decoder.
-model = load_model(model_file)
+model = load_model(model_path)
 encoder_inputs = model.input[0]   # input_1
 
 encoder_outputs, state_h_enc, state_c_enc = model.layers[2].output   # lstm_1

python/tvm/contrib/download.py

@@ -28,6 +28,9 @@ def download(url, path, overwrite=False, size_compare=False, verbose=1, retries=
 
     verbose: int, optional
         Verbose level
+
+    retries: int, optional
+        Number of time to retry download, default at 3.
     """
     if sys.version_info >= (3,):
         import urllib.request as urllib2

python/tvm/relay/testing/tf.py

@@ -331,7 +331,7 @@ def get_workload_ptb():
     ptb_model_file = 'RNN/ptb/ptb_model_with_lstmblockcell.pb'
 
     import tarfile
-    file_path = download_testdata(sample_url, sample_data_file, module=['tf', 'ptb_data'])
+    file_path = download_testdata(sample_url, sample_data_file, module=['data', 'ptb_data'])
     dir_path = os.path.dirname(file_path)
     t = tarfile.open(file_path, 'r')
     t.extractall(dir_path)

tests/python/frontend/coreml/model_zoo/__init__.py

@@ -18,7 +18,7 @@ def get_resnet50():
 def get_cat_image():
     url = 'https://gist.githubusercontent.com/zhreshold/bcda4716699ac97ea44f791c24310193/raw/fa7ef0e9c9a5daea686d6473a62aacd1a5885849/cat.png'
     dst = 'cat.png'
-    real_dst = download_testdata(url, dst, module='coreml')
+    real_dst = download_testdata(url, dst, module='data')
     img = Image.open(real_dst).resize((224, 224))
     img = np.transpose(img, (2, 0, 1))[np.newaxis, :]
     return np.asarray(img)
\ No newline at end of file

tutorials/frontend/deploy_model_on_rasp.py

@@ -14,6 +14,7 @@ import tvm
 import tvm.relay as relay
 from tvm import rpc
 from tvm.contrib import util, graph_runtime as runtime
+from tvm.contrib.download import download_testdata
 
 ######################################################################
 # .. _build-tvm-runtime-on-device:
@@ -82,7 +83,6 @@ from tvm.contrib import util, graph_runtime as runtime
 # You can found more details about this part at tutorial :ref:`tutorial-from-mxnet`.
 
 from mxnet.gluon.model_zoo.vision import get_model
-from mxnet.gluon.utils import download
 from PIL import Image
 import numpy as np
 
@@ -92,9 +92,10 @@ block = get_model('resnet18_v1', pretrained=True)
 ######################################################################
 # In order to test our model, here we download an image of cat and
 # transform its format.
+img_url = 'https://github.com/dmlc/mxnet.js/blob/master/data/cat.png?raw=true'
 img_name = 'cat.png'
-download('https://github.com/dmlc/mxnet.js/blob/master/data/cat.png?raw=true', img_name)
-image = Image.open(img_name).resize((224, 224))
+img_path = download_testdata(img_url, img_name, module='data')
+image = Image.open(img_path).resize((224, 224))
 
 def transform_image(image):
     image = np.array(image) - np.array([123., 117., 104.])
@@ -112,9 +113,9 @@ synset_url = ''.join(['https://gist.githubusercontent.com/zhreshold/',
                       '4d0b62f3d01426887599d4f7ede23ee5/raw/',
                       '596b27d23537e5a1b5751d2b0481ef172f58b539/',
                       'imagenet1000_clsid_to_human.txt'])
-synset_name = 'synset.txt'
-download(synset_url, synset_name)
-with open(synset_name) as f:
+synset_name = 'imagenet1000_clsid_to_human.txt'
+synset_path = download_testdata(synset_url, synset_name, module='data')
+with open(synset_path) as f:
     synset = eval(f.read())
 
 ######################################################################

tutorials/frontend/deploy_ssd_gluoncv.py

@@ -12,6 +12,7 @@ from matplotlib import pyplot as plt
 from tvm.relay.testing.config import ctx_list
 from tvm import relay
 from tvm.contrib import graph_runtime
+from tvm.contrib.download import download_testdata
 from gluoncv import model_zoo, data, utils
 
 
@@ -49,9 +50,9 @@ target_list = ctx_list()
 ######################################################################
 # Download and pre-process demo image
 
-im_fname = utils.download('https://github.com/dmlc/web-data/blob/master/' +
+im_fname = download_testdata('https://github.com/dmlc/web-data/blob/master/' +
                              'gluoncv/detection/street_small.jpg?raw=true',
-                          path='street_small.jpg')
+                             'street_small.jpg', module='data')
 x, img = data.transforms.presets.ssd.load_test(im_fname, short=512)
 
 ######################################################################

tutorials/frontend/from_caffe2.py

@@ -19,21 +19,6 @@ A quick solution is to install via conda
 or please refer to official site
 https://caffe2.ai/docs/getting-started.html
 """
-######################################################################
-# Utils for downloading files
-# ----------------------------
-def download(url, path, overwrite=False):
-    import os
-    if os.path.isfile(path) and not overwrite:
-        print('File {} exists, skip.'.format(path))
-        return
-    print('Downloading from url {} to {}'.format(url, path))
-    try:
-        import urllib.request
-        urllib.request.urlretrieve(url, path)
-    except:
-        import urllib
-        urllib.urlretrieve(url, path)
 
 ######################################################################
 # Load pretrained Caffe2 model
@@ -52,12 +37,13 @@ resnet50 = Model('resnet50')
 # Load a test image
 # ------------------
 # A single cat dominates the examples!
+from tvm.contrib.download import download_testdata
 from PIL import Image
 from matplotlib import pyplot as plt
 import numpy as np
 img_url = 'https://github.com/dmlc/mxnet.js/blob/master/data/cat.png?raw=true'
-download(img_url, 'cat.png')
-img = Image.open('cat.png').resize((224, 224))
+img_path = download_testdata(img_url, 'cat.png', module='data')
+img = Image.open(img_path).resize((224, 224))
 plt.imshow(img)
 plt.show()
 # input preprocess
@@ -118,9 +104,9 @@ synset_url = ''.join(['https://gist.githubusercontent.com/zhreshold/',
                       '4d0b62f3d01426887599d4f7ede23ee5/raw/',
                       '596b27d23537e5a1b5751d2b0481ef172f58b539/',
                       'imagenet1000_clsid_to_human.txt'])
-synset_name = 'synset.txt'
-download(synset_url, synset_name)
-with open(synset_name) as f:
+synset_name = 'imagenet1000_clsid_to_human.txt'
+synset_path = download_testdata(synset_url, synset_name, module='data')
+with open(synset_path) as f:
     synset = eval(f.read())
 print('Relay top-1 id: {}, class name: {}'.format(top1_tvm, synset[top1_tvm]))
 # confirm correctness with caffe2 output

tutorials/frontend/from_coreml.py

@@ -19,23 +19,11 @@ https://github.com/apple/coremltools
 """
 import tvm
 import tvm.relay as relay
+from tvm.contrib.download import download_testdata
 import coremltools as cm
 import numpy as np
 from PIL import Image
 
-def download(url, path, overwrite=False):
-    import os
-    if os.path.isfile(path) and not overwrite:
-        print('File {} existed, skip.'.format(path))
-        return
-    print('Downloading from url {} to {}'.format(url, path))
-    try:
-        import urllib.request
-        urllib.request.urlretrieve(url, path)
-    except:
-        import urllib
-        urllib.urlretrieve(url, path)
-
 ######################################################################
 # Load pretrained CoreML model
 # ----------------------------
@@ -43,17 +31,17 @@ def download(url, path, overwrite=False):
 # provided by apple in this example
 model_url = 'https://docs-assets.developer.apple.com/coreml/models/MobileNet.mlmodel'
 model_file = 'mobilenet.mlmodel'
-download(model_url, model_file)
+model_path = download_testdata(model_url, model_file, module='coreml')
 # Now you have mobilenet.mlmodel on disk
-mlmodel = cm.models.MLModel(model_file)
+mlmodel = cm.models.MLModel(model_path)
 
 ######################################################################
 # Load a test image
 # ------------------
 # A single cat dominates the examples!
 img_url = 'https://github.com/dmlc/mxnet.js/blob/master/data/cat.png?raw=true'
-download(img_url, 'cat.png')
-img = Image.open('cat.png').resize((224, 224))
+img_path = download_testdata(img_url, 'cat.png', module='data')
+img = Image.open(img_path).resize((224, 224))
 x = np.transpose(img, (2, 0, 1))[np.newaxis, :]
 
 ######################################################################
@@ -94,8 +82,8 @@ synset_url = ''.join(['https://gist.githubusercontent.com/zhreshold/',
                       '4d0b62f3d01426887599d4f7ede23ee5/raw/',
                       '596b27d23537e5a1b5751d2b0481ef172f58b539/',
                       'imagenet1000_clsid_to_human.txt'])
-synset_name = 'synset.txt'
-download(synset_url, synset_name)
-with open(synset_name) as f:
+synset_name = 'imagenet1000_clsid_to_human.txt'
+synset_path = download_testdata(synset_url, synset_name, module='data')
+with open(synset_path) as f:
     synset = eval(f.read())
 print('Top-1 id', top1, 'class name', synset[top1])

tutorials/frontend/from_keras.py

@@ -20,22 +20,10 @@ https://keras.io/#installation
 """
 import tvm
 import tvm.relay as relay
+from tvm.contrib.download import download_testdata
 import keras
 import numpy as np
 
-def download(url, path, overwrite=False):
-    import os
-    if os.path.isfile(path) and not overwrite:
-        print('File {} exists, skip.'.format(path))
-        return
-    print('Downloading from url {} to {}'.format(url, path))
-    try:
-        import urllib.request
-        urllib.request.urlretrieve(url, path)
-    except:
-        import urllib
-        urllib.urlretrieve(url, path)
-
 ######################################################################
 # Load pretrained keras model
 # ----------------------------
@@ -43,10 +31,10 @@ def download(url, path, overwrite=False):
 weights_url = ''.join(['https://github.com/fchollet/deep-learning-models/releases/',
                        'download/v0.2/resnet50_weights_tf_dim_ordering_tf_kernels.h5'])
 weights_file = 'resnet50_weights.h5'
-download(weights_url, weights_file)
+weights_path = download_testdata(weights_url, weights_file, module='keras')
 keras_resnet50 = keras.applications.resnet50.ResNet50(include_top=True, weights=None,
                                                       input_shape=(224, 224, 3), classes=1000)
-keras_resnet50.load_weights('resnet50_weights.h5')
+keras_resnet50.load_weights(weights_path)
 
 ######################################################################
 # Load a test image
@@ -56,8 +44,8 @@ from PIL import Image
 from matplotlib import pyplot as plt
 from keras.applications.resnet50 import preprocess_input
 img_url = 'https://github.com/dmlc/mxnet.js/blob/master/data/cat.png?raw=true'
-download(img_url, 'cat.png')
-img = Image.open('cat.png').resize((224, 224))
+img_path = download_testdata(img_url, 'cat.png', module='data')
+img = Image.open(img_path).resize((224, 224))
 plt.imshow(img)
 plt.show()
 # input preprocess
@@ -92,9 +80,9 @@ synset_url = ''.join(['https://gist.githubusercontent.com/zhreshold/',
                       '4d0b62f3d01426887599d4f7ede23ee5/raw/',
                       '596b27d23537e5a1b5751d2b0481ef172f58b539/',
                       'imagenet1000_clsid_to_human.txt'])
-synset_name = 'synset.txt'
-download(synset_url, synset_name)
-with open(synset_name) as f:
+synset_name = 'imagenet1000_clsid_to_human.txt'
+synset_path = download_testdata(synset_url, synset_name, module='data')
+with open(synset_path) as f:
     synset = eval(f.read())
 print('Relay top-1 id: {}, class name: {}'.format(top1_tvm, synset[top1_tvm]))
 # confirm correctness with keras output

tutorials/frontend/from_mxnet.py

@@ -29,22 +29,23 @@ import numpy as np
 # Download Resnet18 model from Gluon Model Zoo
 # ---------------------------------------------
 # In this section, we download a pretrained imagenet model and classify an image.
+from tvm.contrib.download import download_testdata
 from mxnet.gluon.model_zoo.vision import get_model
-from mxnet.gluon.utils import download
 from PIL import Image
 from matplotlib import pyplot as plt
 block = get_model('resnet18_v1', pretrained=True)
+img_url = 'https://github.com/dmlc/mxnet.js/blob/master/data/cat.png?raw=true'
 img_name = 'cat.png'
 synset_url = ''.join(['https://gist.githubusercontent.com/zhreshold/',
                       '4d0b62f3d01426887599d4f7ede23ee5/raw/',
                       '596b27d23537e5a1b5751d2b0481ef172f58b539/',
                       'imagenet1000_clsid_to_human.txt'])
-synset_name = 'synset.txt'
-download('https://github.com/dmlc/mxnet.js/blob/master/data/cat.png?raw=true', img_name)
-download(synset_url, synset_name)
-with open(synset_name) as f:
+synset_name = 'imagenet1000_clsid_to_human.txt'
+img_path = download_testdata(img_url, 'cat.png', module='data')
+synset_path = download_testdata(synset_url, synset_name, module='data')
+with open(synset_path) as f:
     synset = eval(f.read())
-image = Image.open(img_name).resize((224, 224))
+image = Image.open(img_path).resize((224, 224))
 plt.imshow(image)
 plt.show()
 

tutorials/frontend/from_onnx.py

@@ -20,19 +20,7 @@ import onnx
 import numpy as np
 import tvm
 import tvm.relay as relay
-
-def download(url, path, overwrite=False):
-    import os
-    if os.path.isfile(path) and not overwrite:
-        print('File {} existed, skip.'.format(path))
-        return
-    print('Downloading from url {} to {}'.format(url, path))
-    try:
-        import urllib.request
-        urllib.request.urlretrieve(url, path)
-    except:
-        import urllib
-        urllib.urlretrieve(url, path)
+from tvm.contrib.download import download_testdata
 
 ######################################################################
 # Load pretrained ONNX model
@@ -44,9 +32,9 @@ model_url = ''.join(['https://gist.github.com/zhreshold/',
                      'bcda4716699ac97ea44f791c24310193/raw/',
                      '93672b029103648953c4e5ad3ac3aadf346a4cdc/',
                      'super_resolution_0.2.onnx'])
-download(model_url, 'super_resolution.onnx', False)
+model_path = download_testdata(model_url, 'super_resolution.onnx', module='onnx')
 # now you have super_resolution.onnx on disk
-onnx_model = onnx.load('super_resolution.onnx')
+onnx_model = onnx.load(model_path)
 
 ######################################################################
 # Load a test image
@@ -54,8 +42,8 @@ onnx_model = onnx.load('super_resolution.onnx')
 # A single cat dominates the examples!
 from PIL import Image
 img_url = 'https://github.com/dmlc/mxnet.js/blob/master/data/cat.png?raw=true'
-download(img_url, 'cat.png')
-img = Image.open('cat.png').resize((224, 224))
+img_path = download_testdata(img_url, 'cat.png', module='data')
+img = Image.open(img_path).resize((224, 224))
 img_ycbcr = img.convert("YCbCr")  # convert to YCbCr
 img_y, img_cb, img_cr = img_ycbcr.split()
 x = np.array(img_y)[np.newaxis, np.newaxis, :, :]

tutorials/frontend/from_tensorflow.py

@@ -43,8 +43,8 @@ map_proto = 'imagenet_2012_challenge_label_map_proto.pbtxt'
 map_proto_url = os.path.join(repo_base, map_proto)
 
 # Human readable text for labels
-lable_map = 'imagenet_synset_to_human_label_map.txt'
-lable_map_url = os.path.join(repo_base, lable_map)
+label_map = 'imagenet_synset_to_human_label_map.txt'
+label_map_url = os.path.join(repo_base, label_map)
 
 # Target settings
 # Use these commented settings to build for cuda.
@@ -61,19 +61,19 @@ ctx = tvm.cpu(0)
 # Download required files
 # -----------------------
 # Download files listed above.
-from mxnet.gluon.utils import download
+from tvm.contrib.download import download_testdata
 
-download(image_url, img_name)
-download(model_url, model_name)
-download(map_proto_url, map_proto)
-download(lable_map_url, lable_map)
+img_path = download_testdata(image_url, img_name, module='data')
+model_path = download_testdata(model_url, model_name, module=['tf', 'InceptionV1'])
+map_proto_path = download_testdata(map_proto_url, map_proto, module='data')
+label_path = download_testdata(label_map_url, label_map, module='data')
 
 ######################################################################
 # Import model
 # ------------
 # Creates tensorflow graph definition from protobuf file.
 
-with tf.gfile.FastGFile(os.path.join("./", model_name), 'rb') as f:
+with tf.gfile.FastGFile(model_path, 'rb') as f:
     graph_def = tf.GraphDef()
     graph_def.ParseFromString(f.read())
     graph = tf.import_graph_def(graph_def, name='')
@@ -94,7 +94,7 @@ with tf.gfile.FastGFile(os.path.join("./", model_name), 'rb') as f:
 #
 
 from PIL import Image
-image = Image.open(img_name).resize((299, 299))
+image = Image.open(img_path).resize((299, 299))
 
 x = np.array(image)
 
@@ -110,7 +110,7 @@ shape_dict = {'DecodeJpeg/contents': x.shape}
 dtype_dict = {'DecodeJpeg/contents': 'uint8'}
 sym, params = relay.frontend.from_tensorflow(graph_def, layout=layout, shape=shape_dict)
 
-print ("Tensorflow protobuf imported to relay frontend.")
+print("Tensorflow protobuf imported to relay frontend.")
 ######################################################################
 # Relay Build
 # -----------
@@ -148,8 +148,8 @@ predictions = tvm_output.asnumpy()
 predictions = np.squeeze(predictions)
 
 # Creates node ID --> English string lookup.
-node_lookup = tf_testing.NodeLookup(label_lookup_path=os.path.join("./", map_proto),
-                                         uid_lookup_path=os.path.join("./", lable_map))
+node_lookup = tf_testing.NodeLookup(label_lookup_path=map_proto_path,
+                                    uid_lookup_path=label_path)
 
 # Print top 5 predictions from TVM output.
 top_k = predictions.argsort()[-5:][::-1]
@@ -166,7 +166,7 @@ for node_id in top_k:
 def create_graph():
     """Creates a graph from saved GraphDef file and returns a saver."""
     # Creates graph from saved graph_def.pb.
-    with tf.gfile.FastGFile(model_name, 'rb') as f:
+    with tf.gfile.FastGFile(model_path, 'rb') as f:
         graph_def = tf.GraphDef()
         graph_def.ParseFromString(f.read())
         graph = tf.import_graph_def(graph_def, name='')
@@ -200,8 +200,8 @@ def run_inference_on_image(image):
         predictions = np.squeeze(predictions)
 
         # Creates node ID --> English string lookup.
-        node_lookup = tf_testing.NodeLookup(label_lookup_path=os.path.join("./", map_proto),
-                                                 uid_lookup_path=os.path.join("./", lable_map))
+        node_lookup = tf_testing.NodeLookup(label_lookup_path=map_proto_path,
+                                            uid_lookup_path=label_path)
 
         # Print top 5 predictions from tensorflow.
         top_k = predictions.argsort()[-5:][::-1]
@@ -211,4 +211,4 @@ def run_inference_on_image(image):
             score = predictions[node_id]
             print('%s (score = %.5f)' % (human_string, score))
 
-run_inference_on_image (img_name)
+run_inference_on_image(img_path)

tutorials/frontend/from_tflite.py

@@ -52,25 +52,14 @@ Below you can find an example for how to compile TFLite model using TVM.
 ######################################################################
 # Utils for downloading and extracting zip files
 # ---------------------------------------------
-
-def download(url, path, overwrite=False):
-    import os
-    if os.path.isfile(path) and not overwrite:
-        print('File {} existed, skip.'.format(path))
-        return
-    print('Downloading from url {} to {}'.format(url, path))
-    try:
-        import urllib.request
-        urllib.request.urlretrieve(url, path)
-    except:
-        import urllib
-        urllib.urlretrieve(url, path)
+import os
 
 def extract(path):
     import tarfile
     if path.endswith("tgz") or path.endswith("gz"):
+        dir_path = os.path.dirname(path)
         tar = tarfile.open(path)
-        tar.extractall()
+        tar.extractall(path=dir_path)
         tar.close()
     else:
         raise RuntimeError('Could not decompress the file: ' + path)
@@ -80,14 +69,17 @@ def extract(path):
 # Load pretrained TFLite model
 # ---------------------------------------------
 # we load mobilenet V1 TFLite model provided by Google
+from tvm.contrib.download import download_testdata
+
 model_url = "http://download.tensorflow.org/models/mobilenet_v1_2018_08_02/mobilenet_v1_1.0_224.tgz"
 
 # we download model tar file and extract, finally get mobilenet_v1_1.0_224.tflite
-download(model_url, "mobilenet_v1_1.0_224.tgz", False)
-extract("mobilenet_v1_1.0_224.tgz")
+model_path = download_testdata(model_url, "mobilenet_v1_1.0_224.tgz", module=['tf', 'official'])
+model_dir = os.path.dirname(model_path)
+extract(model_path)
 
 # now we have mobilenet_v1_1.0_224.tflite on disk and open it
-tflite_model_file = "mobilenet_v1_1.0_224.tflite"
+tflite_model_file = os.path.join(model_dir, "mobilenet_v1_1.0_224.tflite")
 tflite_model_buf = open(tflite_model_file, "rb").read()
 
 # get TFLite model from buffer
@@ -103,8 +95,8 @@ from matplotlib import pyplot as plt
 import numpy as np
 
 image_url = 'https://github.com/dmlc/mxnet.js/blob/master/data/cat.png?raw=true'
-download(image_url, 'cat.png')
-resized_image = Image.open('cat.png').resize((224, 224))
+image_path = download_testdata(image_url, 'cat.png', module='data')
+resized_image = Image.open(image_path).resize((224, 224))
 plt.imshow(resized_image)
 plt.show()
 image_data = np.asarray(resized_image).astype("float32")
@@ -179,11 +171,11 @@ label_file_url = ''.join(['https://raw.githubusercontent.com/',
                           'app/src/main/assets/',
                           'labels_mobilenet_quant_v1_224.txt'])
 label_file = "labels_mobilenet_quant_v1_224.txt"
-download(label_file_url, label_file)
+label_path = download_testdata(label_file_url, label_file, module='data')
 
 # map id to 1001 classes
 labels = dict()
-with open(label_file) as f:
+with open(label_path) as f:
     for id, line in enumerate(f):
         labels[id] = line