test_forward.py 18.1 KB
Newer Older
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements.  See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership.  The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License.  You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied.  See the License for the
# specific language governing permissions and limitations
# under the License.
17 18 19 20 21 22 23 24
"""
Compile Darknet Models
=====================
This article is a test script to test darknet models with NNVM.
All the required models and libraries will be downloaded from the internet
by the script.
"""
import numpy as np
25 26
import tvm
from tvm.contrib import graph_runtime
27
from tvm.contrib.download import download_testdata
28
download_testdata.__test__ = False
29
from nnvm import frontend
Siju committed
30 31
from tvm.relay.testing.darknet import LAYERTYPE
from tvm.relay.testing.darknet import __darknetffi__
32 33
import nnvm.compiler

34
DARKNET_LIB = 'libdarknet2.0.so'
35 36
DARKNETLIB_URL = 'https://github.com/siju-samuel/darknet/blob/master/lib/' \
                                    + DARKNET_LIB + '?raw=true'
37
LIB = __darknetffi__.dlopen(download_testdata(DARKNETLIB_URL, DARKNET_LIB, module='darknet'))
38

39 40 41 42
DARKNET_TEST_IMAGE_NAME = 'dog.jpg'
DARKNET_TEST_IMAGE_URL = 'https://github.com/siju-samuel/darknet/blob/master/data/' + DARKNET_TEST_IMAGE_NAME +'?raw=true'
DARKNET_TEST_IMAGE_PATH = download_testdata(DARKNET_TEST_IMAGE_URL, DARKNET_TEST_IMAGE_NAME, module='data')

43 44 45 46 47 48 49 50 51 52
def _read_memory_buffer(shape, data, dtype='float32'):
    length = 1
    for x in shape:
        length *= x
    data_np = np.zeros(length, dtype=dtype)
    for i in range(length):
        data_np[i] = data[i]
    return data_np.reshape(shape)

def _get_tvm_output(net, data, build_dtype='float32'):
53 54 55 56 57 58
    '''Compute TVM output'''
    dtype = 'float32'
    sym, params = frontend.darknet.from_darknet(net, dtype)

    target = 'llvm'
    shape_dict = {'data': data.shape}
59 60
    graph, library, params = nnvm.compiler.build(sym, target, shape_dict,
                                                 build_dtype, params=params)
61 62 63 64 65 66 67 68
    # Execute on TVM
    ctx = tvm.cpu(0)
    m = graph_runtime.create(graph, library, ctx)
    # set inputs
    m.set_input('data', tvm.nd.array(data.astype(dtype)))
    m.set_input(**params)
    m.run()
    # get outputs
69 70 71
    tvm_out = []
    for i in range(m.get_num_outputs()):
        tvm_out.append(m.get_output(i).asnumpy())
72 73
    return tvm_out

74 75 76 77 78 79
def _load_net(cfg_url, cfg_name, weights_url, weights_name):
    cfg_path = download_testdata(cfg_url, cfg_name, module='darknet')
    weights_path = download_testdata(weights_url, weights_name, module='darknet')
    net = LIB.load_network(cfg_path.encode('utf-8'), weights_path.encode('utf-8'), 0)
    return net

80
def verify_darknet_frontend(net, build_dtype='float32'):
81 82
    '''Test network with given input image on both darknet and tvm'''
    def get_darknet_output(net, img):
83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104
        LIB.network_predict_image(net, img)
        out = []
        for i in range(net.n):
            layer = net.layers[i]
            if layer.type == LAYERTYPE.REGION:
                attributes = np.array([layer.n, layer.out_c, layer.out_h,
                                       layer.out_w, layer.classes,
                                       layer.coords, layer.background],
                                      dtype=np.int32)
                out.insert(0, attributes)
                out.insert(0, _read_memory_buffer((layer.n*2, ), layer.biases))
                layer_outshape = (layer.batch, layer.out_c,
                                  layer.out_h, layer.out_w)
                out.insert(0, _read_memory_buffer(layer_outshape, layer.output))
            elif layer.type == LAYERTYPE.YOLO:
                attributes = np.array([layer.n, layer.out_c, layer.out_h,
                                       layer.out_w, layer.classes,
                                       layer.total],
                                      dtype=np.int32)
                out.insert(0, attributes)
                out.insert(0, _read_memory_buffer((layer.total*2, ), layer.biases))
                out.insert(0, _read_memory_buffer((layer.n, ), layer.mask, dtype='int32'))
105
                layer_outshape = (layer.batch, layer.out_c,
106 107 108 109 110 111 112 113 114 115 116 117 118
                                  layer.out_h, layer.out_w)
                out.insert(0, _read_memory_buffer(layer_outshape, layer.output))
            elif i == net.n-1:
                if layer.type == LAYERTYPE.CONNECTED:
                    darknet_outshape = (layer.batch, layer.out_c)
                elif layer.type in [LAYERTYPE.SOFTMAX]:
                    darknet_outshape = (layer.batch, layer.outputs)
                else:
                    darknet_outshape = (layer.batch, layer.out_c,
                                        layer.out_h, layer.out_w)
                out.insert(0, _read_memory_buffer(darknet_outshape, layer.output))
        return out

119
    dtype = 'float32'
120

121
    img = LIB.letterbox_image(LIB.load_image_color(DARKNET_TEST_IMAGE_PATH.encode('utf-8'), 0, 0), net.w, net.h)
122
    darknet_output = get_darknet_output(net, img)
123 124 125 126 127 128 129 130 131
    batch_size = 1
    data = np.empty([batch_size, img.c, img.h, img.w], dtype)
    i = 0
    for c in range(img.c):
        for h in range(img.h):
            for k in range(img.w):
                data[0][c][h][k] = img.data[i]
                i = i + 1

132 133
    tvm_out = _get_tvm_output(net, data, build_dtype)
    for tvm_outs, darknet_out in zip(tvm_out, darknet_output):
134
        tvm.testing.assert_allclose(darknet_out, tvm_outs, rtol=1e-3, atol=1e-3)
135

136
def verify_rnn_forward(net):
137 138 139 140 141 142 143 144
    '''Test network with given input data on both darknet and tvm'''
    def get_darknet_network_predict(net, data):
        return LIB.network_predict(net, data)
    from cffi import FFI
    ffi = FFI()
    np_arr = np.zeros([1, net.inputs], dtype='float32')
    np_arr[0, 84] = 1
    cffi_arr = ffi.cast('float*', np_arr.ctypes.data)
145
    tvm_out = _get_tvm_output(net, np_arr)[0]
146 147 148 149
    darknet_output = get_darknet_network_predict(net, cffi_arr)
    darknet_out = np.zeros(net.outputs, dtype='float32')
    for i in range(net.outputs):
        darknet_out[i] = darknet_output[i]
150 151 152
    last_layer = net.layers[net.n-1]
    darknet_outshape = (last_layer.batch, last_layer.outputs)
    darknet_out = darknet_out.reshape(darknet_outshape)
153
    tvm.testing.assert_allclose(darknet_out, tvm_out, rtol=1e-4, atol=1e-4)
154

155 156 157 158 159 160 161
def test_forward_extraction():
    '''test extraction model'''
    model_name = 'extraction'
    cfg_name = model_name + '.cfg'
    weights_name = model_name + '.weights'
    cfg_url = 'https://github.com/pjreddie/darknet/blob/master/cfg/' + cfg_name + '?raw=true'
    weights_url = 'http://pjreddie.com/media/files/' + weights_name + '?raw=true'
162
    net = _load_net(cfg_url, cfg_name, weights_url, weights_name)
163
    verify_darknet_frontend(net)
164 165 166 167 168 169 170 171 172
    LIB.free_network(net)

def test_forward_alexnet():
    '''test alexnet model'''
    model_name = 'alexnet'
    cfg_name = model_name + '.cfg'
    weights_name = model_name + '.weights'
    cfg_url = 'https://github.com/pjreddie/darknet/blob/master/cfg/' + cfg_name + '?raw=true'
    weights_url = 'http://pjreddie.com/media/files/' + weights_name + '?raw=true'
173
    net = _load_net(cfg_url, cfg_name, weights_url, weights_name)
174
    verify_darknet_frontend(net)
175 176 177 178 179 180 181 182 183
    LIB.free_network(net)

def test_forward_resnet50():
    '''test resnet50 model'''
    model_name = 'resnet50'
    cfg_name = model_name + '.cfg'
    weights_name = model_name + '.weights'
    cfg_url = 'https://github.com/pjreddie/darknet/blob/master/cfg/' + cfg_name + '?raw=true'
    weights_url = 'http://pjreddie.com/media/files/' + weights_name + '?raw=true'
184
    net = _load_net(cfg_url, cfg_name, weights_url, weights_name)
185
    verify_darknet_frontend(net)
186 187
    LIB.free_network(net)

188 189
def test_forward_yolov2():
    '''test yolov2 model'''
190 191 192 193 194
    model_name = 'yolov2'
    cfg_name = model_name + '.cfg'
    weights_name = model_name + '.weights'
    cfg_url = 'https://github.com/pjreddie/darknet/blob/master/cfg/' + cfg_name + '?raw=true'
    weights_url = 'http://pjreddie.com/media/files/' + weights_name + '?raw=true'
195
    net = _load_net(cfg_url, cfg_name, weights_url, weights_name)
196
    build_dtype = {}
197
    verify_darknet_frontend(net, build_dtype)
198 199 200 201 202 203 204 205 206
    LIB.free_network(net)

def test_forward_yolov3():
    '''test yolov3 model'''
    model_name = 'yolov3'
    cfg_name = model_name + '.cfg'
    weights_name = model_name + '.weights'
    cfg_url = 'https://github.com/pjreddie/darknet/blob/master/cfg/' + cfg_name + '?raw=true'
    weights_url = 'http://pjreddie.com/media/files/' + weights_name + '?raw=true'
207
    net = _load_net(cfg_url, cfg_name, weights_url, weights_name)
208
    build_dtype = {}
209
    verify_darknet_frontend(net, build_dtype)
210 211 212 213 214 215 216 217 218
    LIB.free_network(net)

def test_forward_convolutional():
    '''test convolutional layer'''
    net = LIB.make_network(1)
    layer = LIB.make_convolutional_layer(1, 224, 224, 3, 32, 1, 3, 2, 0, 1, 0, 0, 0, 0)
    net.layers[0] = layer
    net.w = net.h = 224
    LIB.resize_network(net, 224, 224)
219
    verify_darknet_frontend(net)
220 221 222 223 224 225 226 227 228
    LIB.free_network(net)

def test_forward_dense():
    '''test fully connected layer'''
    net = LIB.make_network(1)
    layer = LIB.make_connected_layer(1, 75, 20, 1, 0, 0)
    net.layers[0] = layer
    net.w = net.h = 5
    LIB.resize_network(net, 5, 5)
229
    verify_darknet_frontend(net)
230 231 232 233 234 235 236 237 238 239 240 241 242
    LIB.free_network(net)

def test_forward_dense_batchnorm():
    '''test fully connected layer with batchnorm'''
    net = LIB.make_network(1)
    layer = LIB.make_connected_layer(1, 12, 2, 1, 1, 0)
    for i in range(5):
        layer.rolling_mean[i] = np.random.rand(1)
        layer.rolling_variance[i] = np.random.rand(1)
        layer.scales[i] = np.random.rand(1)
    net.layers[0] = layer
    net.w = net.h = 2
    LIB.resize_network(net, 2, 2)
243
    verify_darknet_frontend(net)
244 245 246 247 248 249 250 251 252
    LIB.free_network(net)

def test_forward_maxpooling():
    '''test maxpooling layer'''
    net = LIB.make_network(1)
    layer = LIB.make_maxpool_layer(1, 224, 224, 3, 2, 2, 0)
    net.layers[0] = layer
    net.w = net.h = 224
    LIB.resize_network(net, 224, 224)
253
    verify_darknet_frontend(net)
254 255 256 257 258 259 260 261 262
    LIB.free_network(net)

def test_forward_avgpooling():
    '''test avgerage pooling layer'''
    net = LIB.make_network(1)
    layer = LIB.make_avgpool_layer(1, 224, 224, 3)
    net.layers[0] = layer
    net.w = net.h = 224
    LIB.resize_network(net, 224, 224)
263
    verify_darknet_frontend(net)
264 265 266 267 268 269 270 271 272 273 274 275
    LIB.free_network(net)

def test_forward_batch_norm():
    '''test batch normalization layer'''
    net = LIB.make_network(1)
    layer = LIB.make_convolutional_layer(1, 224, 224, 3, 32, 1, 3, 2, 0, 1, 1, 0, 0, 0)
    for i in range(32):
        layer.rolling_mean[i] = np.random.rand(1)
        layer.rolling_variance[i] = np.random.rand(1)
    net.layers[0] = layer
    net.w = net.h = 224
    LIB.resize_network(net, 224, 224)
276
    verify_darknet_frontend(net)
277 278 279 280 281 282 283 284 285
    LIB.free_network(net)

def test_forward_shortcut():
    '''test shortcut layer'''
    net = LIB.make_network(3)
    layer_1 = LIB.make_convolutional_layer(1, 224, 224, 3, 32, 1, 3, 2, 0, 1, 0, 0, 0, 0)
    layer_2 = LIB.make_convolutional_layer(1, 111, 111, 32, 32, 1, 1, 1, 0, 1, 0, 0, 0, 0)
    layer_3 = LIB.make_shortcut_layer(1, 0, 111, 111, 32, 111, 111, 32)
    layer_3.activation = 1
286 287
    layer_3.alpha = 1
    layer_3.beta = 1
288 289 290 291 292
    net.layers[0] = layer_1
    net.layers[1] = layer_2
    net.layers[2] = layer_3
    net.w = net.h = 224
    LIB.resize_network(net, 224, 224)
293
    verify_darknet_frontend(net)
294 295 296 297 298 299 300 301 302 303 304
    LIB.free_network(net)

def test_forward_reorg():
    '''test reorg layer'''
    net = LIB.make_network(2)
    layer_1 = LIB.make_convolutional_layer(1, 222, 222, 3, 32, 1, 3, 2, 0, 1, 0, 0, 0, 0)
    layer_2 = LIB.make_reorg_layer(1, 110, 110, 32, 2, 0, 0, 0)
    net.layers[0] = layer_1
    net.layers[1] = layer_2
    net.w = net.h = 222
    LIB.resize_network(net, 222, 222)
305
    verify_darknet_frontend(net)
306 307 308 309 310
    LIB.free_network(net)

def test_forward_region():
    '''test region layer'''
    net = LIB.make_network(2)
311 312
    layer_1 = LIB.make_convolutional_layer(1, 19, 19, 3, 425, 1, 1, 1, 0, 1, 0, 0, 0, 0)
    layer_2 = LIB.make_region_layer(1, 19, 19, 5, 80, 4)
313 314 315
    layer_2.softmax = 1
    net.layers[0] = layer_1
    net.layers[1] = layer_2
316 317
    net.w = net.h = 19
    LIB.resize_network(net, 19, 19)
318
    build_dtype = {}
319
    verify_darknet_frontend(net, build_dtype)
320 321
    LIB.free_network(net)

322 323 324 325
def test_forward_yolo_op():
    '''test yolo layer'''
    net = LIB.make_network(2)
    layer_1 = LIB.make_convolutional_layer(1, 224, 224, 3, 14, 1, 3, 2, 0, 1, 0, 0, 0, 0)
326
    layer_2 = LIB.make_yolo_layer(1, 111, 111, 2, 9, __darknetffi__.NULL, 2)
327 328 329 330
    net.layers[0] = layer_1
    net.layers[1] = layer_2
    net.w = net.h = 224
    LIB.resize_network(net, 224, 224)
331
    build_dtype = {}
332
    verify_darknet_frontend(net, build_dtype)
333 334 335 336 337 338 339 340 341 342
    LIB.free_network(net)

def test_forward_upsample():
    '''test upsample layer'''
    net = LIB.make_network(1)
    layer = LIB.make_upsample_layer(1, 19, 19, 3, 3)
    layer.scale = 1
    net.layers[0] = layer
    net.w = net.h = 19
    LIB.resize_network(net, 19, 19)
343
    verify_darknet_frontend(net)
344 345
    LIB.free_network(net)

346 347 348 349 350 351 352 353 354 355
def test_forward_l2normalize():
    '''test l2 normalization layer'''
    net = LIB.make_network(1)
    layer = LIB.make_l2norm_layer(1, 224*224*3)
    layer.c = layer.out_c = 3
    layer.h = layer.out_h = 224
    layer.w = layer.out_w = 224
    net.layers[0] = layer
    net.w = net.h = 224
    LIB.resize_network(net, 224, 224)
356
    verify_darknet_frontend(net)
357 358
    LIB.free_network(net)

359 360 361 362 363 364 365 366
def test_forward_elu():
    '''test elu activation layer'''
    net = LIB.make_network(1)
    layer_1 = LIB.make_convolutional_layer(1, 224, 224, 3, 32, 1, 3, 2, 0, 1, 0, 0, 0, 0)
    layer_1.activation = 8
    net.layers[0] = layer_1
    net.w = net.h = 224
    LIB.resize_network(net, 224, 224)
367
    verify_darknet_frontend(net)
368 369 370 371 372 373
    LIB.free_network(net)

def test_forward_softmax():
    '''test softmax layer'''
    net = LIB.make_network(1)
    layer_1 = LIB.make_softmax_layer(1, 75, 1)
374
    layer_1.temperature = 1
375 376 377
    net.layers[0] = layer_1
    net.w = net.h = 5
    LIB.resize_network(net, net.w, net.h)
378
    verify_darknet_frontend(net)
379 380 381 382 383 384
    LIB.free_network(net)

def test_forward_softmax_temperature():
    '''test softmax layer'''
    net = LIB.make_network(1)
    layer_1 = LIB.make_softmax_layer(1, 75, 1)
385
    layer_1.temperature = 0.8
386 387 388
    net.layers[0] = layer_1
    net.w = net.h = 5
    LIB.resize_network(net, net.w, net.h)
389
    verify_darknet_frontend(net)
390 391
    LIB.free_network(net)

392
def test_forward_rnn():
393
    '''test RNN layer'''
394 395 396 397 398 399 400 401 402 403 404 405 406 407
    net = LIB.make_network(1)
    batch = 1
    inputs = 256
    outputs = 256
    steps = 1
    activation = 1
    batch_normalize = 0
    adam = 0
    layer_1 = LIB.make_rnn_layer(batch, inputs, outputs, steps, activation, batch_normalize, adam)
    net.layers[0] = layer_1
    net.inputs = inputs
    net.outputs = outputs
    net.w = net.h = 0
    LIB.resize_network(net, net.w, net.h)
408
    verify_rnn_forward(net)
409 410
    LIB.free_network(net)

411
def _test_forward_crnn():
412
    '''test CRNN layer'''
413 414 415 416 417 418 419 420 421 422 423 424 425 426 427 428 429 430 431 432
    net = LIB.make_network(1)
    batch = 1
    c = 3
    h = 224
    w = 224
    hidden_filters = c
    output_filters = c
    steps = 1
    activation = 0
    batch_normalize = 0
    inputs = 256
    outputs = 256
    layer_1 = LIB.make_crnn_layer(batch, h, w, c, hidden_filters, output_filters,
                                  steps, activation, batch_normalize)
    net.layers[0] = layer_1
    net.inputs = inputs
    net.outputs = output_filters * h * w
    net.w = w
    net.h = h
    LIB.resize_network(net, net.w, net.h)
433
    verify_darknet_frontend(net)
434 435
    LIB.free_network(net)

436 437 438 439 440 441 442 443 444 445 446 447 448 449 450
def test_forward_lstm():
    '''test LSTM layer'''
    net = LIB.make_network(1)
    batch = 1
    inputs = 256
    outputs = 256
    steps = 1
    batch_normalize = 0
    adam = 0
    layer_1 = LIB.make_lstm_layer(batch, inputs, outputs, steps, batch_normalize, adam)
    net.layers[0] = layer_1
    net.inputs = inputs
    net.outputs = outputs
    net.w = net.h = 0
    LIB.resize_network(net, net.w, net.h)
451
    verify_rnn_forward(net)
452 453 454 455 456 457 458 459 460 461 462 463 464 465 466 467 468
    LIB.free_network(net)

def test_forward_gru():
    '''test GRU layer'''
    net = LIB.make_network(1)
    batch = 1
    inputs = 256
    outputs = 256
    steps = 1
    batch_normalize = 0
    adam = 0
    layer_1 = LIB.make_gru_layer(batch, inputs, outputs, steps, batch_normalize, adam)
    net.layers[0] = layer_1
    net.inputs = inputs
    net.outputs = outputs
    net.w = net.h = 0
    LIB.resize_network(net, net.w, net.h)
469
    verify_rnn_forward(net)
470 471
    LIB.free_network(net)

472 473 474 475 476 477 478 479 480 481 482 483 484 485 486 487 488 489 490 491 492 493 494
def test_forward_activation_logistic():
    '''test logistic activation layer'''
    net = LIB.make_network(1)
    batch = 1
    h = 224
    w = 224
    c = 3
    n = 32
    groups = 1
    size = 3
    stride = 2
    padding = 0
    activation = 0
    batch_normalize = 0
    binary = 0
    xnor = 0
    adam = 0
    layer_1 = LIB.make_convolutional_layer(batch, h, w, c, n, groups, size, stride, padding,
                                           activation, batch_normalize, binary, xnor, adam)
    net.layers[0] = layer_1
    net.w = w
    net.h = h
    LIB.resize_network(net, net.w, net.h)
495
    verify_darknet_frontend(net)
496 497
    LIB.free_network(net)

498 499 500 501
if __name__ == '__main__':
    test_forward_resnet50()
    test_forward_alexnet()
    test_forward_extraction()
502 503
    test_forward_yolov2()
    test_forward_yolov3()
504 505 506 507 508 509 510 511 512
    test_forward_convolutional()
    test_forward_maxpooling()
    test_forward_avgpooling()
    test_forward_batch_norm()
    test_forward_shortcut()
    test_forward_dense()
    test_forward_dense_batchnorm()
    test_forward_softmax()
    test_forward_softmax_temperature()
513
    test_forward_rnn()
514 515
    test_forward_reorg()
    test_forward_region()
516 517
    test_forward_yolo_op()
    test_forward_upsample()
518
    test_forward_l2normalize()
519
    test_forward_elu()
520
    test_forward_rnn()
521 522
# FIXME: Skip CRNN test since it causes segfault in libdarknet2.0.so
#    _test_forward_crnn()
523 524 525
    test_forward_lstm()
    test_forward_gru()
    test_forward_activation_logistic()