Skip to content
This repository has been archived by the owner on Nov 17, 2023. It is now read-only.

[WIP] Fix precision issue of test case test_rnnrelu_bidirectional and test_rnnrelu_sym #11445

Closed
wants to merge 2 commits into from
Closed

[WIP] Fix precision issue of test case test_rnnrelu_bidirectional and test_rnnrelu_sym #11445

Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
85442a0
fix test case bug
Jun 28, 2018
6b895a1
adjust tolerance only for relu
Jun 28, 2018
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
56 changes: 28 additions & 28 deletions tests/python/unittest/test_operator.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,7 @@
from common import setup_module, with_seed, teardown
import unittest

def check_rnn_consistency(cell1, cell2, T, N, I, H, grad_req):
def check_rnn_consistency(cell1, cell2, T, N, I, H, grad_req, rtol, atol):
dshape = (N, T, I)
data = mx.sym.Variable('data')

Expand All @@ -51,18 +51,18 @@ def check_rnn_consistency(cell1, cell2, T, N, I, H, grad_req):
# check inference
mod1.forward(batch, is_train=False)
mod2.forward(batch, is_train=False)
assert_allclose(mod1.get_outputs()[0].asnumpy(), mod2.get_outputs()[0].asnumpy(), rtol=1e-2, atol=1e-4)
assert_allclose(mod1.get_outputs()[0].asnumpy(), mod2.get_outputs()[0].asnumpy(), rtol=rtol, atol=atol)

# check training
mod1.forward(batch, is_train=True)
mod2.forward(batch, is_train=True)
assert_allclose(mod1.get_outputs()[0].asnumpy(), mod2.get_outputs()[0].asnumpy(), rtol=1e-2, atol=1e-4)
assert_allclose(mod1.get_outputs()[0].asnumpy(), mod2.get_outputs()[0].asnumpy(), rtol=rtol, atol=atol)

dy = mx.random.uniform(shape=mod1.get_outputs()[0].shape)
mod1.backward(out_grads=[dy])
mod2.backward(out_grads=[dy])
if grad_req != 'null':
assert_allclose(mod1.get_input_grads()[0].asnumpy(), mod2.get_input_grads()[0].asnumpy(), rtol=1e-2, atol=1e-4)
assert_allclose(mod1.get_input_grads()[0].asnumpy(), mod2.get_input_grads()[0].asnumpy(), rtol=rtol, atol=atol)
else:
assert(mod1.get_input_grads()[0] == None)
assert(mod2.get_input_grads()[0] == None)
Expand All @@ -78,9 +78,9 @@ def test_lstm_sym():
stack.add(mx.rnn.LSTMCell(H, prefix='l1_'))
stack.add(mx.rnn.LSTMCell(H, prefix='l2_'))

check_rnn_consistency(fused, stack, T, N, I, H, 'write')
check_rnn_consistency(fused, stack, T, N, I, H, 'add')
check_rnn_consistency(fused, stack, T, N, I, H, 'null')
check_rnn_consistency(fused, stack, T, N, I, H, 'write', 1e-2, 1e-4)
check_rnn_consistency(fused, stack, T, N, I, H, 'add', 1e-2, 1e-4)
check_rnn_consistency(fused, stack, T, N, I, H, 'null', 1e-2, 1e-4)

@with_seed()
def test_lstm_bidirectional():
Expand All @@ -98,9 +98,9 @@ def test_lstm_bidirectional():
mx.rnn.LSTMCell(H, prefix='r1_'),
output_prefix='bi_lstm_1_'))

check_rnn_consistency(fused, stack, T, N, I, H, 'write')
check_rnn_consistency(fused, stack, T, N, I, H, 'add')
check_rnn_consistency(fused, stack, T, N, I, H, 'null')
check_rnn_consistency(fused, stack, T, N, I, H, 'write', 1e-2, 1e-4)
check_rnn_consistency(fused, stack, T, N, I, H, 'add', 1e-2, 1e-4)
check_rnn_consistency(fused, stack, T, N, I, H, 'null', 1e-2, 1e-4)

@with_seed()
def test_gru_sym():
Expand All @@ -111,9 +111,9 @@ def test_gru_sym():
stack.add(mx.rnn.GRUCell(H, prefix='l1_'))
stack.add(mx.rnn.GRUCell(H, prefix='l2_'))

check_rnn_consistency(fused, stack, T, N, I, H, 'write')
check_rnn_consistency(fused, stack, T, N, I, H, 'add')
check_rnn_consistency(fused, stack, T, N, I, H, 'null')
check_rnn_consistency(fused, stack, T, N, I, H, 'write', 1e-2, 1e-4)
check_rnn_consistency(fused, stack, T, N, I, H, 'add', 1e-2, 1e-4)
check_rnn_consistency(fused, stack, T, N, I, H, 'null', 1e-2, 1e-4)

@with_seed()
def test_gru_bidirectional():
Expand All @@ -133,9 +133,9 @@ def test_gru_bidirectional():
mx.rnn.GRUCell(H, prefix='r1_'),
output_prefix='bi_gru_1_'))

check_rnn_consistency(fused, stack, T, N, I, H, 'write')
check_rnn_consistency(fused, stack, T, N, I, H, 'add')
check_rnn_consistency(fused, stack, T, N, I, H, 'null')
check_rnn_consistency(fused, stack, T, N, I, H, 'write', 1e-2, 1e-4)
check_rnn_consistency(fused, stack, T, N, I, H, 'add', 1e-2, 1e-4)
check_rnn_consistency(fused, stack, T, N, I, H, 'null', 1e-2, 1e-4)

@with_seed()
def test_rnntanh_sym():
Expand All @@ -147,9 +147,9 @@ def test_rnntanh_sym():
stack.add(mx.rnn.RNNCell(H, activation='tanh', prefix='l1_'))
stack.add(mx.rnn.RNNCell(H, activation='tanh', prefix='l2_'))

check_rnn_consistency(fused, stack, T, N, I, H, 'write')
check_rnn_consistency(fused, stack, T, N, I, H, 'add')
check_rnn_consistency(fused, stack, T, N, I, H, 'null')
check_rnn_consistency(fused, stack, T, N, I, H, 'write', 1e-2, 1e-4)
check_rnn_consistency(fused, stack, T, N, I, H, 'add', 1e-2, 1e-4)
check_rnn_consistency(fused, stack, T, N, I, H, 'null', 1e-2, 1e-4)

@with_seed()
def test_rnntanh_bidirectional():
Expand All @@ -168,9 +168,9 @@ def test_rnntanh_bidirectional():
mx.rnn.RNNCell(H, activation='tanh', prefix='r1_'),
output_prefix='bi_rnntanh_1_'))

check_rnn_consistency(fused, stack, T, N, I, H, 'write')
check_rnn_consistency(fused, stack, T, N, I, H, 'add')
check_rnn_consistency(fused, stack, T, N, I, H, 'null')
check_rnn_consistency(fused, stack, T, N, I, H, 'write', 1e-2, 1e-4)
check_rnn_consistency(fused, stack, T, N, I, H, 'add', 1e-2, 1e-4)
check_rnn_consistency(fused, stack, T, N, I, H, 'null', 1e-2, 1e-4)

@with_seed()
def test_rnnrelu_sym():
Expand All @@ -182,9 +182,9 @@ def test_rnnrelu_sym():
stack.add(mx.rnn.RNNCell(H, activation='relu', prefix='l1_'))
stack.add(mx.rnn.RNNCell(H, activation='relu', prefix='l2_'))

check_rnn_consistency(fused, stack, T, N, I, H, 'write')
check_rnn_consistency(fused, stack, T, N, I, H, 'add')
check_rnn_consistency(fused, stack, T, N, I, H, 'null')
check_rnn_consistency(fused, stack, T, N, I, H, 'write', 1e-2, 1e-2)
check_rnn_consistency(fused, stack, T, N, I, H, 'add', 1e-2, 1e-2)
check_rnn_consistency(fused, stack, T, N, I, H, 'null', 1e-2, 1e-2)

@with_seed()
def test_rnnrelu_bidirectional():
Expand All @@ -203,9 +203,9 @@ def test_rnnrelu_bidirectional():
mx.rnn.RNNCell(H, activation='relu', prefix='r1_'),
output_prefix='bi_rnnrelu_1_'))

check_rnn_consistency(fused, stack, T, N, I, H, 'write')
check_rnn_consistency(fused, stack, T, N, I, H, 'add')
check_rnn_consistency(fused, stack, T, N, I, H, 'null')
check_rnn_consistency(fused, stack, T, N, I, H, 'write', 1e-2, 1e-2)
check_rnn_consistency(fused, stack, T, N, I, H, 'add', 1e-2, 1e-2)
check_rnn_consistency(fused, stack, T, N, I, H, 'null', 1e-2, 1e-2)

@with_seed()
def test_lstm_dropout():
Expand Down