Skip to content

Commit

Permalink
add relu (apache#1849)
Browse files Browse the repository at this point in the history
  • Loading branch information
@MarisaKirisame @tqchen
MarisaKirisame authored and tqchen committed Oct 9, 2018
1 parent b3c03ff commit 6ffdd28
Show file tree
Hide file tree
Showing 7 changed files with 120 additions and 59 deletions.
11 changes: 10 additions & 1 deletion docs/langref/relay_op.rst
View file
Original file line number Diff line number Diff line change
Expand Up @@ -37,7 +37,7 @@ This level enables fully connected multi-layer perceptron.
tvm.relay.mod
tvm.relay.tanh
tvm.relay.sigmoid

tvm.relay.nn.relu

**Level 2: Convolutions**

Expand Down Expand Up @@ -75,6 +75,7 @@ This level enables additional math and transform operators.
tvm.relay.negative



**Level 4: Broadcast and Reductions**

.. autosummary::
Expand All @@ -92,6 +93,7 @@ This level enables additional math and transform operators.
tvm.relay.minimum
tvm.relay.pow


**Level 5: Vision/Image Operators**

.. autosummary::
Expand All @@ -116,6 +118,7 @@ Level 1 Definitions
.. autofunction:: tvm.relay.concatenate
.. autofunction:: tvm.relay.nn.softmax
.. autofunction:: tvm.relay.nn.log_softmax
.. autofunction:: tvm.relay.nn.relu


Level 2 Definitions
Expand All @@ -141,6 +144,12 @@ Level 3 Definitions
.. autofunction:: tvm.relay.copy
.. autofunction:: tvm.relay.transpose

Level 3 Definitions
-------------------
.. autofunction:: tvm.relay.zeros_like
.. autofunction:: tvm.relay.ones_like


Level 4 Definitions
-------------------
.. autofunction:: tvm.relay.right_shift
Expand Down
19 changes: 19 additions & 0 deletions python/tvm/relay/op/nn/nn.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -364,3 +364,22 @@ def batch_flatten(data):
The Flattened result.
"""
return _make.batch_flatten(data)


def relu(data):
"""Rectified linear unit.
.. math::
out = max(x, 0)
Parameters
----------
data : relay.Expr
The input data
Returns
-------
result : relay.Expr
The computed result.
"""
return _make.relu(data)
11 changes: 11 additions & 0 deletions src/relay/op/nn/nn.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@
#include <tvm/relay/attrs/image.h>
#include <vector>
#include "../type_relations.h"
#include "../op_common.h"
#include "layout.h"

namespace tvm {
Expand Down Expand Up @@ -132,5 +133,15 @@ Example::
.set_support_level(2)
.add_type_rel("BatchFlatten", BatchFlattenRel);

RELAY_REGISTER_UNARY_OP("relay.op.nn._make.", "relu")
.describe(R"code(Returns the relu input array, computed element-wise.
.. math::
max(x, 0)
)code" TVM_ADD_FILELINE)
.set_support_level(1)
.add_type_rel("Identity", IdentityRel);

} // namespace relay
} // namespace tvm
46 changes: 46 additions & 0 deletions src/relay/op/op_common.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,52 @@ std::vector<T> AsVector(const Array<T> &array) {
return result;
}

/*! Quick helper macro
* - Expose a positional make function to construct the node.
* - Register op to the registry.
*
* We make the decision to always only expose positional argument.
* We will do rewrapping in the frontend to support language
* sugars such as keyword arguments and default value.
*
* \param Prefix the prefix of the registry, for example, "relay.op._make.".
*
* \param OpName the name of registry.
*/
#define RELAY_REGISTER_UNARY_OP(Prefix, OpName) \
TVM_REGISTER_API(Prefix OpName) \
.set_body_typed<Expr(Expr)>([](Expr data) { \
static const Op& op = Op::Get(OpName); \
return CallNode::make(op, {data}, Attrs(), {}); \
}); \
RELAY_REGISTER_OP(OpName) \
.set_num_inputs(1) \
.add_argument("data", "Tensor", "The input tensor.")

/*! Quick helper macro
* - Expose a positional make function to construct the node.
* - Register op to the registry.
*
* We make the decision to always only expose positional argument.
* We will do rewrapping in the frontend to support language
* sugars such as keyword arguments and default value.
*
* \param Prefix the prefix of the registry, for example, "relay.op._make.".
*
* \param OpName the name of registry.
*/
#define RELAY_REGISTER_BINARY_OP(Prefix, OpName) \
TVM_REGISTER_API(Prefix OpName) \
.set_body_typed<Expr(Expr, Expr)>([](Expr lhs, Expr rhs) { \
static const Op& op = Op::Get(OpName); \
return CallNode::make(op, {lhs, rhs}, Attrs(), {}); \
}); \
RELAY_REGISTER_OP(OpName) \
.set_num_inputs(2) \
.add_argument("lhs", "Tensor", "The left hand side tensor.") \
.add_argument("rhs", "Tensor", "The right hand side tensor.") \
.add_type_rel("Broadcast", BroadcastRel)

} // namespace relay
} // namespace tvm

Expand Down
36 changes: 13 additions & 23 deletions src/relay/op/tensor/binary.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,69 +6,59 @@
#include <tvm/relay/expr.h>
#include <tvm/relay/op.h>
#include "../type_relations.h"
#include "../op_common.h"

namespace tvm {
namespace relay {

#define RELAY_REGISTER_BINARY_OP(OpName) \
TVM_REGISTER_API("relay.op._make." OpName) \
.set_body_typed<Expr(Expr, Expr)>([](Expr lhs, Expr rhs) { \
static const Op& op = Op::Get(OpName); \
return CallNode::make(op, {lhs, rhs}, Attrs(), {}); \
}); \
RELAY_REGISTER_OP(OpName) \
.set_num_inputs(2) \
.add_argument("lhs", "Tensor", "The left hand side tensor.") \
.add_argument("rhs", "Tensor", "The right hand side tensor.") \
.add_type_rel("Broadcast", BroadcastRel)

RELAY_REGISTER_BINARY_OP("add")
// Addition
RELAY_REGISTER_BINARY_OP("relay.op._make.", "add")
.describe("Elementwise add with with broadcasting")
.set_support_level(1);

// Subtraction
RELAY_REGISTER_BINARY_OP("subtract")
RELAY_REGISTER_BINARY_OP("relay.op._make.", "subtract")
.describe("Elementwise substract with broadcasting")
.set_support_level(1);

// Right shift
RELAY_REGISTER_BINARY_OP("right_shift")
RELAY_REGISTER_BINARY_OP("relay.op._make.", "right_shift")
.describe("Elementwise right shift with broadcasting")
.set_support_level(4);

RELAY_REGISTER_BINARY_OP("left_shift")
RELAY_REGISTER_BINARY_OP("relay.op._make.", "left_shift")
.describe("Elementwise left shift with broadcasting")
.set_support_level(4);

RELAY_REGISTER_BINARY_OP("maximum")
RELAY_REGISTER_BINARY_OP("relay.op._make.", "maximum")
.describe("Elementwise maximum of two tensors with broadcasting")
.set_support_level(4);

RELAY_REGISTER_BINARY_OP("minimum")
RELAY_REGISTER_BINARY_OP("relay.op._make.", "minimum")
.describe("Elementwise minimum of two tensors with broadcasting")
.set_support_level(4);

RELAY_REGISTER_BINARY_OP("divide")
RELAY_REGISTER_BINARY_OP("relay.op._make.", "divide")
.describe("Elementwise divide with broadcasting")
.set_support_level(1);

RELAY_REGISTER_BINARY_OP("multiply")
RELAY_REGISTER_BINARY_OP("relay.op._make.", "multiply")
.describe("Elementwise multiply with broadcasting")
.set_support_level(1);

RELAY_REGISTER_BINARY_OP("pow")
RELAY_REGISTER_BINARY_OP("relay.op._make.", "pow")
.describe("Elementwise power with broadcasting")
.set_support_level(4);

RELAY_REGISTER_BINARY_OP("mod")
RELAY_REGISTER_BINARY_OP("relay.op._make.", "mod")
.describe("Elementwise mod with broadcasting")
.set_support_level(1);

// Comparisons
#define RELAY_REGISTER_CMP_OP(OpName) \
TVM_REGISTER_API("relay.op._make." OpName) \
.set_body_typed<Expr(Expr, Expr)>([](Expr lhs, Expr rhs) { \
static const Op& op = Op::Get(OpName); \
static const Op& op = Op::Get(OpName); \
return CallNode::make(op, {lhs, rhs}, Attrs(), {}); \
}); \
RELAY_REGISTER_OP(OpName) \
Expand Down
53 changes: 19 additions & 34 deletions src/relay/op/tensor/unary.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,30 +6,14 @@
#include <tvm/relay/expr.h>
#include <tvm/relay/op.h>
#include "../type_relations.h"
#include "../op_common.h"

namespace tvm {
namespace relay {

// Quick helper macro
// - Expose a positional make function to construct the node.
// - Register op to the registry.
//
// We make the decision to always only expose positional argument.
// We will do rewrapping in the frontend to support language
// sugars such as keyword arguments and default value.
//
#define RELAY_REGISTER_UNARY_OP(OpName) \
TVM_REGISTER_API("relay.op._make." OpName) \
.set_body_typed<Expr(Expr)>([](Expr data) { \
static const Op& op = Op::Get(OpName); \
return CallNode::make(op, {data}, Attrs(), {}); \
}); \
RELAY_REGISTER_OP(OpName) \
.set_num_inputs(1) \
.add_argument("data", "Tensor", "The input tensor.")

RELAY_REGISTER_UNARY_OP("log")
.describe(R"code(Returns the log of input array, computed element-wise.

RELAY_REGISTER_UNARY_OP("relay.op._make.", "log")
.describe(R"code(Returns the log input array, computed element-wise.
.. math::
log(x)
Expand All @@ -38,8 +22,8 @@ RELAY_REGISTER_UNARY_OP("log")
.set_support_level(1)
.add_type_rel("Identity", IdentityRel);

RELAY_REGISTER_UNARY_OP("exp")
.describe(R"code(Returns the exp of input array, computed element-wise.
RELAY_REGISTER_UNARY_OP("relay.op._make.", "exp")
.describe(R"code(Returns the exp input array, computed element-wise.
.. math::
\exp(x)
Expand All @@ -49,7 +33,7 @@ RELAY_REGISTER_UNARY_OP("exp")
.add_type_rel("Identity", IdentityRel);


RELAY_REGISTER_UNARY_OP("sqrt")
RELAY_REGISTER_UNARY_OP("relay.op._make.", "sqrt")
.describe(R"code(Returns the sqrt input array, computed element-wise.
.. math::
Expand All @@ -59,19 +43,19 @@ RELAY_REGISTER_UNARY_OP("sqrt")
.set_support_level(1)
.add_type_rel("Identity", IdentityRel);

RELAY_REGISTER_UNARY_OP("zeros_like")
RELAY_REGISTER_UNARY_OP("relay.op._make.", "zeros_like")
.describe(R"code(Returns an array of zeros, with same type and shape as the input.
)code" TVM_ADD_FILELINE)
.set_support_level(1)
.add_type_rel("Identity", IdentityRel);

RELAY_REGISTER_UNARY_OP("ones_like")
RELAY_REGISTER_UNARY_OP("relay.op._make.", "ones_like")
.describe(R"code(Returns an array of ones, with same type and shape as the input.
)code" TVM_ADD_FILELINE)
.set_support_level(1)
.add_type_rel("Identity", IdentityRel);

RELAY_REGISTER_UNARY_OP("sigmoid")
RELAY_REGISTER_UNARY_OP("relay.op._make.", "sigmoid")
.describe(R"code(Returns the sigmoid input array, computed element-wise.
.. math::
Expand All @@ -81,7 +65,7 @@ RELAY_REGISTER_UNARY_OP("sigmoid")
.set_support_level(1)
.add_type_rel("Identity", IdentityRel);

RELAY_REGISTER_UNARY_OP("copy")
RELAY_REGISTER_UNARY_OP("relay.op._make.", "copy")
.describe(R"code(Copy a tensor.
)code" TVM_ADD_FILELINE)
.set_support_level(3)
Expand Down Expand Up @@ -118,13 +102,14 @@ RELAY_REGISTER_OP("clip")
.set_support_level(3)
.add_type_rel("Clip", IdentityRel);

RELAY_REGISTER_UNARY_OP("floor")

RELAY_REGISTER_UNARY_OP("relay.op._make.", "floor")
.describe(R"code(Returns the floor of input array, computed element-wise.
)code" TVM_ADD_FILELINE)
.set_support_level(3)
.add_type_rel("Identity", IdentityRel);

RELAY_REGISTER_UNARY_OP("ceil")
RELAY_REGISTER_UNARY_OP("relay.op._make.", "ceil")
.describe(R"code(Returns the ceil of input array, computed element-wise.
.. math::
Expand All @@ -134,7 +119,7 @@ RELAY_REGISTER_UNARY_OP("ceil")
.set_support_level(3)
.add_type_rel("Identity", IdentityRel);

RELAY_REGISTER_UNARY_OP("trunc")
RELAY_REGISTER_UNARY_OP("relay.op._make.", "trunc")
.describe(R"code(Returns the trunc of input array, computed element-wise.
.. math::
Expand All @@ -144,7 +129,7 @@ RELAY_REGISTER_UNARY_OP("trunc")
.set_support_level(3)
.add_type_rel("Identity", IdentityRel);

RELAY_REGISTER_UNARY_OP("round")
RELAY_REGISTER_UNARY_OP("relay.op._make.", "round")
.describe(R"code(Returns the round of input array, computed element-wise.
.. math::
Expand All @@ -154,7 +139,7 @@ RELAY_REGISTER_UNARY_OP("round")
.set_support_level(3)
.add_type_rel("Identity", IdentityRel);

RELAY_REGISTER_UNARY_OP("abs")
RELAY_REGISTER_UNARY_OP("relay.op._make.", "abs")
.describe(R"code(Returns the abs of input array, computed element-wise.
.. math::
Expand All @@ -164,7 +149,7 @@ RELAY_REGISTER_UNARY_OP("abs")
.set_support_level(3)
.add_type_rel("Identity", IdentityRel);

RELAY_REGISTER_UNARY_OP("tanh")
RELAY_REGISTER_UNARY_OP("relay.op._make.", "tanh")
.describe(R"code(Returns the tanh of input array, computed element-wise.
.. math::
Expand All @@ -174,7 +159,7 @@ RELAY_REGISTER_UNARY_OP("tanh")
.set_support_level(1)
.add_type_rel("Identity", IdentityRel);

RELAY_REGISTER_UNARY_OP("negative")
RELAY_REGISTER_UNARY_OP("relay.op._make.", "negative")
.describe(R"code(Returns the numeric negative of input array, computed element-wise.
.. math::
Expand Down
3 changes: 2 additions & 1 deletion tests/python/relay/test_op_level1.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -70,7 +70,8 @@ def test_unary_op():
for op in [relay.exp,
relay.log,
relay.sqrt,
relay.sigmoid]:
relay.sigmoid,
relay.nn.relu]:
ib = relay.ir_builder.IRBuilder()
x = ib.param("x", relay.TensorType((10, 4), "int32"))
with ib.function(x) as func:
Expand Down

0 comments on commit 6ffdd28

Please sign in to comment.