test_top_level1.py 18.2 KB
Newer Older
Yuwei HU committed
1 2
import numpy as np
import tvm
3
from tvm.contrib import graph_runtime
4
import topi.testing
Yuwei HU committed
5 6
import nnvm.symbol as sym
import nnvm.compiler
7
from nnvm.testing.config import ctx_list
8
from nnvm.testing.check_computation import check_function
Yuwei HU committed
9

10 11
def test_check_function():
    # test the testing function
12

13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163
    x = sym.Variable("x")
    y = sym.Variable("y")

    # different styles of returning gradients from the backward function
    check_function(x + 2*y, lambda x, y: x + 2*y,
                   lambda x, y, head_grads: [head_grads, 2*head_grads],
                   shape={'x': (1, 2), y: (1, 2)}, dtype='float32')
    check_function(x + 2*y, lambda x, y: x + 2*y,
                   lambda x, y, head_grads: (head_grads, 2*head_grads),
                   shape={'x': (1, 2), y: (1, 2)}, dtype='float32')
    check_function(x + 2*y, lambda x, y: x + 2*y,
                   lambda x, y, head_grads: {'x': head_grads, 'y': 2*head_grads},
                   shape={'x': (1, 2), y: (1, 2)}, dtype='float32')
    check_function(x + 2*y, lambda x, y: x + 2*y,
                   lambda x, y, head_grads: {'y': 2*head_grads},
                   shape={'x': (1, 2), y: (1, 2)}, dtype='float32')
    check_function(x + 2*y, lambda x, y: x + 2*y,
                   lambda x, y, head_grads: [2*head_grads],
                   grad_input_vars=[y],
                   shape={'x': (1, 2), y: (1, 2)}, dtype='float32')
    check_function(x + 2*y, lambda x, y: x + 2*y,
                   lambda x, y, head_grads: 2*head_grads,
                   grad_input_vars=[y],
                   shape={'x': (1, 2), y: (1, 2)}, dtype='float32')
    check_function(x + 2*y, lambda x, y: x + 2*y,
                   lambda x, y, head_grads: 2*head_grads,
                   grad_input_vars=[y],
                   shape={'x': (1, 2), y: (1, 2)}, dtype='float64')

    # test just numerical gradients
    # different styles of shape and dtype passing
    check_function(x + 2*y, shape={'x': (1, 2), y: (1, 2)},
                   numerical_grads=True)
    check_function(x + 2*y, shape={'x': (1, 2), y: (1, 2)}, dtype='float32',
                   numerical_grads=True)
    check_function(x + 2*y, shape={'x': (1, 2), y: (1, 2)}, dtype={x: 'float32', 'y': 'float32'},
                   numerical_grads=True)
    check_function(x + 2*y, shape=(1, 2), dtype='float32',
                   numerical_grads=True)

    # specifying variable attributes on variable creation
    # (in this case type codes must be used)
    x = sym.Variable("x", dtype=0, shape=(1, 2))
    check_function(x + 2*y, shape={y: (1, 2)}, dtype={'y': 'float32'}, numerical_grads=True)
    y = sym.Variable("y", dtype=0, shape=(1, 2))

    # shape overriding
    def _fwd1(x, y):
        assert x.shape == (1, 1)
        assert y.shape == (1, 2)
        return x + 2*y
    check_function(x + 2*y, _fwd1, shape={x: (1, 1)})

    # in_range
    def _fwd2(x, y):
        assert x.shape == (100,)
        assert (x <= 0.9).all()
        assert (x >= 0.8).all()
        return x + 2*y
    check_function(x + 2*y, _fwd2, shape=(100,), in_range=(0.8, 0.9), numerical_grads=False)
    check_function(x + 2*y, _fwd2, shape=(100,), in_range={'x': (0.8, 0.9)}, numerical_grads=False)
    check_function(x + 2*y, backward=lambda x, y, head_grads: [1.0, 2.0],
                   in_range={'head_grads_0': (1.0, 1.0)})
    # explicit passing of values
    check_function(x + 2*y, backward=lambda x, y, head_grads: [1.0, 2.0],
                   values={'head_grads_0': np.full((1, 2), 1.0)})

    # check that the function reports errors
    def _check_function_must_fail(*args, **kwargs):
        error = AssertionError
        if 'error' in kwargs:
            error = kwargs['error']
            del kwargs['error']
        try:
            check_function(*args, quiet=True, **kwargs)
        except error:
            pass
        else:
            raise AssertionError("check_function didn't raise an exception")

    _check_function_must_fail(x + 2*y, error=ValueError)
    _check_function_must_fail(x + 2*y, lambda x, y: x + y)
    _check_function_must_fail(x + 2*y, backward=lambda x, y, head_grads: [1.0, 2.0])
    _check_function_must_fail(sym.block_grad(x + 2*y), numerical_grads=True)
    _check_function_must_fail(x*x, numerical_grads=True,
                              numerical_grads_params={'atol': 0.0, 'rtol': 0.0})

    # different styles of returning results from the forward function
    check_function(x + 2*y, lambda x, y: [x + 2*y], numerical_grads=False)
    _check_function_must_fail(x + 2*y, lambda x, y: [x + 2*y, x], numerical_grads=False,
                              error=ValueError)
    _check_function_must_fail(x + 2*y, lambda x, y: [], numerical_grads=False,
                              error=ValueError)

    # multiple outputs
    z = sym.Group([2*x + y, x + 2*y])
    check_function(z, lambda x, y: [2*x + y, x + 2*y])
    check_function(z, lambda x, y: (2*x + y, x + 2*y))
    check_function(z, backward=lambda x, y, head_grads: [2*head_grads[0] + head_grads[1],
                                                         head_grads[0] + 2*head_grads[1]])
    _check_function_must_fail(z, backward=lambda x, y, head_grads: [2*head_grads[0],
                                                                    2*head_grads[1]])
    check_function(z, backward=lambda x, y, head_grads: [head_grads[1], 2*head_grads[1]],
                   in_range={'head_grads_0': (0, 0)})
    check_function(z, numerical_grads=True)

    z = sym.Group([sym.block_grad(2*x + y), x + 2*y])
    check_function(z, lambda x, y: [2*x + y, x + 2*y], numerical_grads=False)
    _check_function_must_fail(z, lambda x, y: [2*x + y, x + 2*y])
    _check_function_must_fail(z, numerical_grads=True)

    z = sym.Group([2*x + y, sym.block_grad(x + 2*y)])
    _check_function_must_fail(z, numerical_grads=True)

    z = sym.Group([2*x + y, x + 2*y, x, y, sym.sum(x)])
    check_function(z, lambda x, y: [2*x + y, x + 2*y, x, y, np.sum(x)])

    # passing additional parameters to forward and backward
    def _fwd3(x, p):
        assert p == 'v'
        return x + 1
    def _bwd3(x, p, head_grads):
        assert p == 'v'
        return head_grads
    check_function(x + 1, _fwd3, _bwd3, additional_params={'p': 'v'})

    # implicitly created variables and shape/dtype inference for inputs
    x = sym.Variable("x", shape=(2, 3), dtype=0)
    b = sym.Variable("b")
    y = sym.dense(data=x, bias=b, units=4)
    # Don't check gradients on cuda because is doesn't yet support ewise after reduce
    check_function(y, exclude_targets={'cuda'}, numerical_grads=True)
    check_function(y, shape={'x': (3, 4)}, exclude_targets={'cuda'}, numerical_grads=True)
    check_function(y, dtype={'x': 'float64'}, exclude_targets={'cuda'}, numerical_grads=True)

    x = sym.Variable("x")
    b = sym.Variable("b")
    w = sym.Variable("w")
    y = sym.dense(data=x, bias=b, weight=w, units=4)
    def _fwd_dense(x, w, b):
        return np.dot(x, w.T) + b
    check_function(y, _fwd_dense, shape={'x': (1,2)}, dtype={'x': 'float32'}, numerical_grads=False)
    check_function(y, _fwd_dense, shape={'x': (1,2)}, dtype={'w': 'float64'}, numerical_grads=False)
    _check_function_must_fail(y, _fwd_dense, shape={'x': (1,2)},
                              dtype={'w': 'float64', 'b': 'float32'},
                              numerical_grads=False,
                              error=nnvm._base.NNVMError)
    # fails because no shape
    _check_function_must_fail(y, _fwd_dense, numerical_grads=False, error=ValueError)
    # ok because type is float32 by default
    check_function(y, _fwd_dense, shape={'x': (1,2)}, numerical_grads=False)
164

165 166
def test_relu():
    x = sym.Variable("x")
167 168 169 170 171 172
    y = sym.relu(sym.leaky_relu(x, alpha=0.3) - 0.2)

    def forward(x):
        x = (x < 0) * x * 0.3 + (x > 0) * x - 0.2
        return (x > 0) * x

Yao Wang committed
173 174 175 176 177
    def backward(head_grads, x):
        sub = (x < 0) * x * 0.3 + (x > 0) * x - 0.2
        return [(sub > 0).astype("float") * \
                ((x > 0).astype("float") + 0.3 * (x < 0).astype("float")) * head_grads]

178 179
    shape = {'x': (1, 3, 32, 32)}
    check_function(y, forward, backward, shape=shape)
180

181 182 183 184 185 186 187 188
def test_prelu_nchw():
    x = sym.Variable("x")
    a = sym.Variable("a")
    y = sym.prelu(data=x, alpha=a)

    def forward(x, a):
        return (x < 0) * (x * a.reshape(3, 1, 1)) + (x>=0) * x

189 190
    shape = {'x': (1, 3, 32, 32), 'a': (3,)}
    check_function(y, forward, shape=shape)
191 192 193 194 195 196 197 198 199

def test_prelu_nhwc():
    x = sym.Variable("x")
    a = sym.Variable("a")
    y = sym.prelu(data=x, alpha=a, axis=3)

    def forward(x, a):
        return (x < 0) * (x * a.reshape(1, 1, 3)) + (x>=0) * x

200 201
    shape = {'x': (1, 32, 32, 3), 'a': (3,)}
    check_function(y, forward, shape=shape)
202 203 204 205 206 207 208 209 210

def test_sym_scalar_pow():
    scalar = 3
    x = sym.Variable("x")
    y = x**scalar

    def forward(x):
        return x**scalar

Yao Wang committed
211 212
    def backward(head_grads, x):
        return [scalar * x**(scalar -  1) * head_grads]
213

214 215
    shape = {'x': (1, 3, 32, 32)}
    check_function(y, forward, backward, shape=shape)
216 217 218 219 220 221 222 223 224 225


def test_scalar_sym_pow():
    scalar = 3
    x = sym.Variable("x")
    y = scalar**x

    def forward(x):
        return scalar**x

Yao Wang committed
226 227
    def backward(head_grads, x):
        return [np.log(scalar) * scalar**x * head_grads]
228

229 230
    shape = {'x': (1, 3, 32, 32)}
    check_function(y, forward, backward, shape=shape)
231 232 233 234 235


def test_exp():
    x = sym.Variable("x")
    y = sym.exp(x)
236 237 238 239

    def forward(x):
        return np.exp(x)

Yao Wang committed
240 241
    def backward(head_grads, x):
        return [np.exp(x) * head_grads]
242

243 244
    shape = {'x': (1, 3, 32, 32)}
    check_function(y, forward, backward, shape=shape)
245 246 247 248 249


def test_log():
    x = sym.Variable("x")
    y = sym.log(x)
250 251 252 253

    def forward(x):
        return np.log(x)

Yao Wang committed
254 255
    def backward(head_grads, x):
        return [1. / x * head_grads]
256

257 258
    shape = {'x': (1, 3, 32, 32)}
    check_function(y, forward, backward, in_range=(0.002, 2.0), shape=shape)
259 260 261 262 263


def test_tanh():
    x = sym.Variable("x")
    y = sym.tanh(x)
264 265 266 267

    def forward(x):
        return np.sinh(x) / np.cosh(x)

Yao Wang committed
268
    def backward(head_grads, x):
269
        y_np = forward(x)
Yao Wang committed
270
        return [(1 - y_np**2) * head_grads]
271

272 273
    shape = {'x': (1, 3, 32, 32)}
    check_function(y, forward, backward, shape=shape)
274 275 276 277 278


def test_sigmoid():
    x = sym.Variable("x")
    y = sym.sigmoid(x)
279 280 281 282

    def forward(x):
        return 1.0 / (1.0 + np.exp(-x))

Yao Wang committed
283
    def backward(head_grads, x):
284
        y_np = forward(x)
Yao Wang committed
285
        return [y_np *(1 - y_np) * head_grads]
286

287 288
    shape = {'x': (1, 3, 32, 32)}
    check_function(y, forward, backward, shape=shape)
289 290


Yuwei HU committed
291 292 293
def test_softmax():
    x = sym.Variable("x")
    y = sym.softmax(x)
294 295 296 297

    def forward(x):
        return topi.testing.softmax_python(x)

Yao Wang committed
298 299 300 301 302
    def backward(head_grads, x):
        y = topi.testing.softmax_python(x)
        grad = y * (head_grads - np.sum(y * head_grads, axis=1, keepdims=True))
        return [grad]

303 304 305 306
    check_function(y, forward, backward,
                   shape={'x': (10, 1000)}, numerical_grads=False)
    check_function(y, forward, backward,
                   shape={'x': (2, 10)})
307 308


309 310 311
def test_log_softmax():
    x = sym.Variable("x")
    y = sym.log_softmax(x)
312 313 314 315

    def forward(x):
        return topi.testing.log_softmax_python(x)

Yao Wang committed
316 317
    def backward(head_grads, x):
        y = topi.testing.log_softmax_python(x)
318
        grad = head_grads - np.exp(y) * np.sum(head_grads, axis=1, keepdims=True)
Yao Wang committed
319 320
        return [grad]

321 322 323 324
    check_function(y, forward, backward,
                   shape={'x': (10, 1000)}, numerical_grads=False)
    check_function(y, forward, backward,
                   shape={'x': (2, 10)})
325 326


327
def test_dense():
Yao Wang committed
328 329 330 331
    x = sym.Variable("x", shape=(10, 100))
    w = sym.Variable("dense_weight", shape=(3, 100))
    b = sym.Variable("dense_bias", shape=(3,))
    y = sym.dense(x, w, b, use_bias=True, units=3, name="dense")
332
    y = sym.flatten(y)
333 334 335

    def forward(x, dense_weight, dense_bias):
        return np.dot(x, dense_weight.T) + dense_bias
336 337 338 339 340 341 342 343 344 345
    shape = {
        'x': (10, 100),
        'w': (3, 100),
        'b': (3,)
    }
    # Don't check gradients on cuda because is doesn't yet support ewise after reduce
    check_function(y, forward, shape=shape,
                   exclude_targets={'cuda'}, numerical_grads=True)
    check_function(y, forward, shape=shape,
                   only_targets={'cuda'}, numerical_grads=False)
346 347 348 349 350 351 352 353 354 355 356 357


def test_batchnorm():
    x = sym.Variable("x")
    beta = sym.Variable("beta")
    gamma = sym.Variable("gamma")
    moving_var = sym.Variable("moving_var")
    moving_mean = sym.Variable("moving_mean")
    eps = 1e-5
    y = sym.batch_norm(
        x, gamma, beta, moving_mean, moving_var, epsilon=eps)

358 359 360
    def forward(x, gamma, beta, moving_mean, moving_var):
        return (x - moving_mean) / np.sqrt(moving_var + eps) * gamma + beta

361 362 363 364 365 366 367
    shape = {
        'x': (10, 20),
        'gamma': (20,),
        'beta': (20,),
        'moving_mean': (20,),
        'moving_var': (20,)
    }
368

369
    check_function(y, forward, in_range=(0.001, 1.0), shape=shape)
Yuwei HU committed
370 371


372
def verify_concatenate(ishape, axis):
373
    x = [sym.Variable("x%d" % i, shape=ishape[i]) for i in range(len(ishape))]
374
    y = sym.concatenate(*x, axis=axis) + 1
375 376 377 378 379

    def forward(**kwargs):
        return np.concatenate(list(kwargs.values()), axis=axis) + 1

    check_function(y, forward)
380

381

382 383 384 385 386 387
def test_concatenate():
    verify_concatenate([(2, 3, 4), (1, 3, 4)], axis=0)
    verify_concatenate([(2, 4), (2, 7)], axis=1)


def verify_split(ishape, indices_or_sections, axis):
388
    x = sym.Variable("x", shape=ishape)
389
    y = sym.split(x, indices_or_sections=indices_or_sections, axis=axis)
390 391 392 393 394

    def forward(x):
        return np.split(x, indices_or_sections, axis=axis)

    check_function(y, forward)
395

396

397 398 399 400 401
def test_split():
    verify_split((2, 3), 2, axis=0)
    verify_split((5, 3), [3], axis=0)
    verify_split((5, 9, 3), [3, 4], axis=1)

402 403
def verify_strided_slice(ishape, begin, end, strideinp=None):
    stride = strideinp if strideinp else [1, 1, 1]
404
    x = sym.Variable("x", shape=ishape)
405 406 407 408
    if strideinp:
        y = sym.strided_slice(x, begin = begin, end = end, stride = stride) + 1
    else:
        y = sym.strided_slice(x, begin = begin, end = end) + 1
409

410 411 412 413
    for i in range(len(begin), 3):
        begin.append(0)
    for i in range(len(end), 3):
        end.append(ishape[i])
414 415

    def test_forward(x):
416 417 418
        return x[begin[0]:end[0]:stride[0],
                    begin[1]:end[1]:stride[1], begin[2]:end[2]:stride[2]] + 1

419
    check_function(y, test_forward)
420 421 422 423 424 425 426 427 428 429 430

def test_strided_slice():
    verify_strided_slice((3, 4, 3), [0, 0, 0], [4, -5, 4], [1, -1, 2])
    verify_strided_slice((3, 4, 3), [1, 1, 0], [4, 4, 3], [2, 1, 1])
    verify_strided_slice((3, 4, 3), [1, -1, 0], [4, -5, 3], [2, -1, 1])
    verify_strided_slice((3, 4, 3), [1, 0, 0], [2, 2, 3], [1, 1, 2])
    verify_strided_slice((3, 4, 3), [1, -1, 0], [2, -3, 3], [1, -1, 1])
    verify_strided_slice((3, 4, 3), [1, 1, 0], [4, 4, 3])
    verify_strided_slice((3, 4, 3), [1, 1, 0], [4, 1000, 3])
    verify_strided_slice((3, 4, 3), [1, 1, 0], [4, 4])
    verify_strided_slice((3, 4, 3), [1, 1], [4, 4, 3])
Xingjian Shi committed
431

432 433 434 435
def verify_take(src_shape, indices_src, axis=None):
    src_dtype = "float32"
    indices_dtype = "int32"
    indices_src = np.array(indices_src, dtype=indices_dtype)
436 437
    a = sym.Variable("a", shape=src_shape)
    indices = sym.Variable("indices", shape=indices_src.shape)
438
    y = sym.take(a, indices, axis=axis)
439 440 441 442 443 444 445 446 447

    def forward(a, indices):
        return np.take(a, indices=indices, axis=axis)

    a_src = np.arange(np.prod(src_shape), dtype=src_dtype).reshape(src_shape)

    check_function(y, forward,
                   dtype={'a': src_dtype, 'indices': indices_dtype},
                   values={'a': a_src, 'indices': indices_src})
448 449 450 451 452 453 454 455 456 457 458 459

def test_take():
    verify_take((4,), [1])
    verify_take((4,), [[0,1,2,3]])
    verify_take((3,3,3), [[11,25]])
    verify_take((4,), [[0,1],[2,3]])
    verify_take((4,), [1], 0)
    verify_take((2,2), [[[1,0],[0,1]]], 0)
    verify_take((2,2), [[[1,0],[0,1]]], 1)
    verify_take((4,3,5,6), [[2,1,0,0]], -2)


460
def verify_squeeze(shape, axis):
Xingjian Shi committed
461
    x = sym.Variable("x")
462
    if axis is not None:
Xingjian Shi committed
463 464 465 466
        y = sym.squeeze(x, axis=axis)
    else:
        y = sym.squeeze(x)
    y = y + 1
467 468 469 470

    def forward(x):
        return np.squeeze(x, axis=axis) + 1

Yao Wang committed
471 472 473
    def backward(head_grads, x):
        return [np.reshape(head_grads, x.shape)]

474
    check_function(y, forward, backward, shape=shape)
475

Xingjian Shi committed
476 477 478 479 480 481

def test_squeeze():
    verify_squeeze((1, 3, 2, 5), None)
    verify_squeeze((1, 3, 1), axis=0)
    verify_squeeze((1, 3, 2, 5, 1), axis=-1)

Yuwei Hu committed
482 483 484 485

def test_pad():
    x = sym.Variable("x")
    y = sym.pad(x, pad_width=((0, 0), (0, 0), (0, 1), (2, 3)), pad_value=1.)
486 487 488 489 490 491

    def forward(x):
        return np.pad(x,
                      pad_width=((0, 0), (0, 0), (0, 1), (2, 3)),
                      mode='constant', constant_values=1.)

492 493
    shape = {'x': (1, 3, 28, 28)}
    check_function(y, forward, shape=shape)
Yuwei Hu committed
494

495
def verify_lrn(ishape, size, axis, bias, alpha, beta):
496
    x = sym.Variable("x", shape=ishape)
497 498
    y = sym.lrn(x, size=size, axis=axis, bias=bias, alpha=alpha, beta=beta)

499 500 501 502 503 504 505 506
    def forward1(x):
        return topi.testing.lrn_python(x, size, axis, bias, alpha, beta)

    check_function(y, forward1)

    def forward2(x):
        y = forward1(x)
        return (y > 0)*y
507 508

    #Checking LRN op followed by elementwise op relu
509
    check_function(sym.relu(y), forward2, in_range={'x': (-10.0, 10.0)})
510 511

def verify_l2_normalize(ishape, eps, axis):
512
    x = sym.Variable("x", shape=ishape)
513 514
    y = sym.l2_normalize(x, eps=eps, axis=axis)

515 516 517 518 519 520 521 522
    def forward1(x):
        return topi.testing.l2_normalize_python(x, eps, axis)

    check_function(y, forward1)

    def forward2(x):
        y = forward1(x)
        return (y > 0)*y
523 524

    #Checking L2 normalization op followed by elementwise op relu
525
    check_function(sym.relu(y), forward2, in_range={'x': (-10.0, 10.0)})
526 527 528 529 530 531 532 533

def test_lrn():
    verify_lrn((1, 3, 20, 20), 3, 1, 1.0, 1.0, 0.5)
    verify_lrn((1, 3, 20, 20), 3, 1, 2.0, 1.0, 0.75)

def test_l2_normalize():
    verify_l2_normalize((1, 3, 20, 20), 0.001, (1,))
    verify_l2_normalize((1, 3, 20, 20), 0.001, (1, 2))
Yuwei Hu committed
534

Yuwei HU committed
535
if __name__ == "__main__":
536
    test_check_function()
537 538
    test_split()
    test_concatenate()
539
    test_log_softmax()
540 541
    test_batchnorm()
    test_dense()
542
    test_relu()
543 544
    test_prelu_nchw()
    test_prelu_nhwc()
545 546
    test_sym_scalar_pow()
    test_scalar_sym_pow()
547 548 549 550
    test_exp()
    test_log()
    test_tanh()
    test_sigmoid()
Yuwei HU committed
551
    test_softmax()
Xingjian Shi committed
552
    test_squeeze()
Yuwei Hu committed
553
    test_pad()
554
    test_take()
555 556
    test_lrn()
    test_l2_normalize()
557
    test_strided_slice()