Skip to content

T
tic
Commit 64d3393e by 雾雨魔理沙 Committed by Tianqi Chen
Options

add relu (#1849)

parent 0f053c82
Showing with 120 additions and 59 deletions
docs/langref/relay_op.rst
......@@ -37,7 +37,7 @@ This level enables fully connected multi-layer perceptron.
tvm.relay.mod
tvm.relay.tanh
tvm.relay.sigmoid
tvm.relay.nn.relu
**Level 2: Convolutions**
......@@ -75,6 +75,7 @@ This level enables additional math and transform operators.
tvm.relay.negative
**Level 4: Broadcast and Reductions**
.. autosummary::
......@@ -92,6 +93,7 @@ This level enables additional math and transform operators.
tvm.relay.minimum
tvm.relay.pow
**Level 5: Vision/Image Operators**
.. autosummary::
......@@ -116,6 +118,7 @@ Level 1 Definitions
.. autofunction:: tvm.relay.concatenate
.. autofunction:: tvm.relay.nn.softmax
.. autofunction:: tvm.relay.nn.log_softmax
.. autofunction:: tvm.relay.nn.relu
Level 2 Definitions
......@@ -141,6 +144,12 @@ Level 3 Definitions
.. autofunction:: tvm.relay.copy
.. autofunction:: tvm.relay.transpose
Level 3 Definitions
-------------------
.. autofunction:: tvm.relay.zeros_like
.. autofunction:: tvm.relay.ones_like
Level 4 Definitions
-------------------
.. autofunction:: tvm.relay.right_shift
......
python/tvm/relay/op/nn/nn.py
......@@ -364,3 +364,22 @@ def batch_flatten(data):
The Flattened result.
"""
return _make.batch_flatten(data)
def relu(data):
"""Rectified linear unit.
.. math::
out = max(x, 0)
Parameters
----------
data : relay.Expr
The input data
Returns
-------
result : relay.Expr
The computed result.
"""
return _make.relu(data)
src/relay/op/nn/nn.cc
......@@ -9,6 +9,7 @@
#include <tvm/relay/attrs/image.h>
#include <vector>
#include "../type_relations.h"
#include "../op_common.h"
#include "layout.h"
namespace tvm {
......@@ -132,5 +133,15 @@ Example::
.set_support_level(2)
.add_type_rel("BatchFlatten", BatchFlattenRel);
RELAY_REGISTER_UNARY_OP("relay.op.nn._make.", "relu")
.describe(R"code(Returns the relu input array, computed element-wise.
.. math::
max(x, 0)
)code" TVM_ADD_FILELINE)
.set_support_level(1)
.add_type_rel("Identity", IdentityRel);
} // namespace relay
} // namespace tvm
src/relay/op/op_common.h
......@@ -24,6 +24,52 @@ std::vector<T> AsVector(const Array<T> &array) {
return result;
}
/*! Quick helper macro
* - Expose a positional make function to construct the node.
* - Register op to the registry.
*
* We make the decision to always only expose positional argument.
* We will do rewrapping in the frontend to support language
* sugars such as keyword arguments and default value.
*
* \param Prefix the prefix of the registry, for example, "relay.op._make.".
*
* \param OpName the name of registry.
*/
#define RELAY_REGISTER_UNARY_OP(Prefix, OpName) \
TVM_REGISTER_API(Prefix OpName) \
.set_body_typed<Expr(Expr)>([](Expr data) { \
static const Op& op = Op::Get(OpName); \
return CallNode::make(op, {data}, Attrs(), {}); \
}); \
RELAY_REGISTER_OP(OpName) \
.set_num_inputs(1) \
.add_argument("data", "Tensor", "The input tensor.")
/*! Quick helper macro
* - Expose a positional make function to construct the node.
* - Register op to the registry.
*
* We make the decision to always only expose positional argument.
* We will do rewrapping in the frontend to support language
* sugars such as keyword arguments and default value.
*
* \param Prefix the prefix of the registry, for example, "relay.op._make.".
*
* \param OpName the name of registry.
*/
#define RELAY_REGISTER_BINARY_OP(Prefix, OpName) \
TVM_REGISTER_API(Prefix OpName) \
.set_body_typed<Expr(Expr, Expr)>([](Expr lhs, Expr rhs) { \
static const Op& op = Op::Get(OpName); \
return CallNode::make(op, {lhs, rhs}, Attrs(), {}); \
}); \
RELAY_REGISTER_OP(OpName) \
.set_num_inputs(2) \
.add_argument("lhs", "Tensor", "The left hand side tensor.") \
.add_argument("rhs", "Tensor", "The right hand side tensor.") \
.add_type_rel("Broadcast", BroadcastRel)
} // namespace relay
} // namespace tvm
......
src/relay/op/tensor/binary.cc
......@@ -6,61 +6,51 @@
#include <tvm/relay/expr.h>
#include <tvm/relay/op.h>
#include "../type_relations.h"
#include "../op_common.h"
namespace tvm {
namespace relay {
#define RELAY_REGISTER_BINARY_OP(OpName) \
TVM_REGISTER_API("relay.op._make." OpName) \
.set_body_typed<Expr(Expr, Expr)>([](Expr lhs, Expr rhs) { \
static const Op& op = Op::Get(OpName); \
return CallNode::make(op, {lhs, rhs}, Attrs(), {}); \
}); \
RELAY_REGISTER_OP(OpName) \
.set_num_inputs(2) \
.add_argument("lhs", "Tensor", "The left hand side tensor.") \
.add_argument("rhs", "Tensor", "The right hand side tensor.") \
.add_type_rel("Broadcast", BroadcastRel)
RELAY_REGISTER_BINARY_OP("add")
// Addition
RELAY_REGISTER_BINARY_OP("relay.op._make.", "add")
.describe("Elementwise add with with broadcasting")
.set_support_level(1);
// Subtraction
RELAY_REGISTER_BINARY_OP("subtract")
RELAY_REGISTER_BINARY_OP("relay.op._make.", "subtract")
.describe("Elementwise substract with broadcasting")
.set_support_level(1);
// Right shift
RELAY_REGISTER_BINARY_OP("right_shift")
RELAY_REGISTER_BINARY_OP("relay.op._make.", "right_shift")
.describe("Elementwise right shift with broadcasting")
.set_support_level(4);
RELAY_REGISTER_BINARY_OP("left_shift")
RELAY_REGISTER_BINARY_OP("relay.op._make.", "left_shift")
.describe("Elementwise left shift with broadcasting")
.set_support_level(4);
RELAY_REGISTER_BINARY_OP("maximum")
RELAY_REGISTER_BINARY_OP("relay.op._make.", "maximum")
.describe("Elementwise maximum of two tensors with broadcasting")
.set_support_level(4);
RELAY_REGISTER_BINARY_OP("minimum")
RELAY_REGISTER_BINARY_OP("relay.op._make.", "minimum")
.describe("Elementwise minimum of two tensors with broadcasting")
.set_support_level(4);
RELAY_REGISTER_BINARY_OP("divide")
RELAY_REGISTER_BINARY_OP("relay.op._make.", "divide")
.describe("Elementwise divide with broadcasting")
.set_support_level(1);
RELAY_REGISTER_BINARY_OP("multiply")
RELAY_REGISTER_BINARY_OP("relay.op._make.", "multiply")
.describe("Elementwise multiply with broadcasting")
.set_support_level(1);
RELAY_REGISTER_BINARY_OP("pow")
RELAY_REGISTER_BINARY_OP("relay.op._make.", "pow")
.describe("Elementwise power with broadcasting")
.set_support_level(4);
RELAY_REGISTER_BINARY_OP("mod")
RELAY_REGISTER_BINARY_OP("relay.op._make.", "mod")
.describe("Elementwise mod with broadcasting")
.set_support_level(1);
......@@ -68,7 +58,7 @@ RELAY_REGISTER_BINARY_OP("mod")
#define RELAY_REGISTER_CMP_OP(OpName) \
TVM_REGISTER_API("relay.op._make." OpName) \
.set_body_typed<Expr(Expr, Expr)>([](Expr lhs, Expr rhs) { \
static const Op& op = Op::Get(OpName); \
static const Op& op = Op::Get(OpName); \
return CallNode::make(op, {lhs, rhs}, Attrs(), {}); \
}); \
RELAY_REGISTER_OP(OpName) \
......
src/relay/op/tensor/unary.cc
......@@ -6,30 +6,14 @@
#include <tvm/relay/expr.h>
#include <tvm/relay/op.h>
#include "../type_relations.h"
#include "../op_common.h"
namespace tvm {
namespace relay {
// Quick helper macro
// - Expose a positional make function to construct the node.
// - Register op to the registry.
//
// We make the decision to always only expose positional argument.
// We will do rewrapping in the frontend to support language
// sugars such as keyword arguments and default value.
//
#define RELAY_REGISTER_UNARY_OP(OpName) \
TVM_REGISTER_API("relay.op._make." OpName) \
.set_body_typed<Expr(Expr)>([](Expr data) { \
static const Op& op = Op::Get(OpName); \
return CallNode::make(op, {data}, Attrs(), {}); \
}); \
RELAY_REGISTER_OP(OpName) \
.set_num_inputs(1) \
.add_argument("data", "Tensor", "The input tensor.")
RELAY_REGISTER_UNARY_OP("log")
.describe(R"code(Returns the log of input array, computed element-wise.
RELAY_REGISTER_UNARY_OP("relay.op._make.", "log")
.describe(R"code(Returns the log input array, computed element-wise.
.. math::
log(x)
......@@ -38,8 +22,8 @@ RELAY_REGISTER_UNARY_OP("log")
.set_support_level(1)
.add_type_rel("Identity", IdentityRel);
RELAY_REGISTER_UNARY_OP("exp")
.describe(R"code(Returns the exp of input array, computed element-wise.
RELAY_REGISTER_UNARY_OP("relay.op._make.", "exp")
.describe(R"code(Returns the exp input array, computed element-wise.
.. math::
\exp(x)
......@@ -49,7 +33,7 @@ RELAY_REGISTER_UNARY_OP("exp")
.add_type_rel("Identity", IdentityRel);
RELAY_REGISTER_UNARY_OP("sqrt")
RELAY_REGISTER_UNARY_OP("relay.op._make.", "sqrt")
.describe(R"code(Returns the sqrt input array, computed element-wise.
.. math::
......@@ -59,19 +43,19 @@ RELAY_REGISTER_UNARY_OP("sqrt")
.set_support_level(1)
.add_type_rel("Identity", IdentityRel);
RELAY_REGISTER_UNARY_OP("zeros_like")
RELAY_REGISTER_UNARY_OP("relay.op._make.", "zeros_like")
.describe(R"code(Returns an array of zeros, with same type and shape as the input.
)code" TVM_ADD_FILELINE)
.set_support_level(1)
.add_type_rel("Identity", IdentityRel);
RELAY_REGISTER_UNARY_OP("ones_like")
RELAY_REGISTER_UNARY_OP("relay.op._make.", "ones_like")
.describe(R"code(Returns an array of ones, with same type and shape as the input.
)code" TVM_ADD_FILELINE)
.set_support_level(1)
.add_type_rel("Identity", IdentityRel);
RELAY_REGISTER_UNARY_OP("sigmoid")
RELAY_REGISTER_UNARY_OP("relay.op._make.", "sigmoid")
.describe(R"code(Returns the sigmoid input array, computed element-wise.
.. math::
......@@ -81,7 +65,7 @@ RELAY_REGISTER_UNARY_OP("sigmoid")
.set_support_level(1)
.add_type_rel("Identity", IdentityRel);
RELAY_REGISTER_UNARY_OP("copy")
RELAY_REGISTER_UNARY_OP("relay.op._make.", "copy")
.describe(R"code(Copy a tensor.
)code" TVM_ADD_FILELINE)
.set_support_level(3)
......@@ -118,13 +102,14 @@ RELAY_REGISTER_OP("clip")
.set_support_level(3)
.add_type_rel("Clip", IdentityRel);
RELAY_REGISTER_UNARY_OP("floor")
RELAY_REGISTER_UNARY_OP("relay.op._make.", "floor")
.describe(R"code(Returns the floor of input array, computed element-wise.
)code" TVM_ADD_FILELINE)
.set_support_level(3)
.add_type_rel("Identity", IdentityRel);
RELAY_REGISTER_UNARY_OP("ceil")
RELAY_REGISTER_UNARY_OP("relay.op._make.", "ceil")
.describe(R"code(Returns the ceil of input array, computed element-wise.
.. math::
......@@ -134,7 +119,7 @@ RELAY_REGISTER_UNARY_OP("ceil")
.set_support_level(3)
.add_type_rel("Identity", IdentityRel);
RELAY_REGISTER_UNARY_OP("trunc")
RELAY_REGISTER_UNARY_OP("relay.op._make.", "trunc")
.describe(R"code(Returns the trunc of input array, computed element-wise.
.. math::
......@@ -144,7 +129,7 @@ RELAY_REGISTER_UNARY_OP("trunc")
.set_support_level(3)
.add_type_rel("Identity", IdentityRel);
RELAY_REGISTER_UNARY_OP("round")
RELAY_REGISTER_UNARY_OP("relay.op._make.", "round")
.describe(R"code(Returns the round of input array, computed element-wise.
.. math::
......@@ -154,7 +139,7 @@ RELAY_REGISTER_UNARY_OP("round")
.set_support_level(3)
.add_type_rel("Identity", IdentityRel);
RELAY_REGISTER_UNARY_OP("abs")
RELAY_REGISTER_UNARY_OP("relay.op._make.", "abs")
.describe(R"code(Returns the abs of input array, computed element-wise.
.. math::
......@@ -164,7 +149,7 @@ RELAY_REGISTER_UNARY_OP("abs")
.set_support_level(3)
.add_type_rel("Identity", IdentityRel);
RELAY_REGISTER_UNARY_OP("tanh")
RELAY_REGISTER_UNARY_OP("relay.op._make.", "tanh")
.describe(R"code(Returns the tanh of input array, computed element-wise.
.. math::
......@@ -174,7 +159,7 @@ RELAY_REGISTER_UNARY_OP("tanh")
.set_support_level(1)
.add_type_rel("Identity", IdentityRel);
RELAY_REGISTER_UNARY_OP("negative")
RELAY_REGISTER_UNARY_OP("relay.op._make.", "negative")
.describe(R"code(Returns the numeric negative of input array, computed element-wise.
.. math::
......
tests/python/relay/test_op_level1.py
......@@ -70,7 +70,8 @@ def test_unary_op():
for op in [relay.exp,
relay.log,
relay.sqrt,
relay.sigmoid]:
relay.sigmoid,
relay.nn.relu]:
ib = relay.ir_builder.IRBuilder()
x = ib.param("x", relay.TensorType((10, 4), "int32"))
with ib.function(x) as func:
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment