Skip to content

Commit

Permalink
Parallel execution of TensorFlow Layer 1 Python tests (#22173)
Browse files Browse the repository at this point in the history 
* Update TF Layer 1 tests

* Enable parallel execution of tests

* Fix typo

* Fix typo

* Enable parallel tensorflow tests for windows

---------

Co-authored-by: Roman Kazantsev <[email protected]>
  • Loading branch information
@rajatkrishna @rkazants
rajatkrishna and rkazants authored Jan 18, 2024
1 parent 45f6285 commit 1a8f72f
Show file tree
Hide file tree
Showing 8 changed files with 129 additions and 96 deletions.
2 changes: 1 addition & 1 deletion .github/workflows/job_python_unit_tests.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -249,7 +249,7 @@ jobs:
run: |
# requires 'unit_tests' from 'mo'
export PYTHONPATH=${INSTALL_TEST_DIR}/mo
python3 -m pytest ${LAYER_TESTS_INSTALL_DIR}/tensorflow_tests/ --use_new_frontend -m precommit_tf_fe --junitxml=${INSTALL_TEST_DIR}/TEST-tf_fe.xml
python3 -m pytest ${LAYER_TESTS_INSTALL_DIR}/tensorflow_tests/ --use_new_frontend -m precommit_tf_fe -n logical --junitxml=${INSTALL_TEST_DIR}/TEST-tf_fe.xml
env:
TEST_DEVICE: CPU
TEST_PRECISION: FP16
Expand Down
2 changes: 1 addition & 1 deletion .github/workflows/windows.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -473,7 +473,7 @@ jobs:
run: |
:: requires 'unit_tests' from 'tools/mo'
set PYTHONPATH=${{ env.INSTALL_TEST_DIR }}\mo;%PYTHONPATH%
python3 -m pytest ${{ env.LAYER_TESTS_INSTALL_DIR }}/tensorflow_tests/ --use_new_frontend -m precommit_tf_fe --junitxml=${{ env.INSTALL_TEST_DIR }}/TEST-tf_fe.xml
python3 -m pytest ${{ env.LAYER_TESTS_INSTALL_DIR }}/tensorflow_tests/ --use_new_frontend -n logical -m precommit_tf_fe --junitxml=${{ env.INSTALL_TEST_DIR }}/TEST-tf_fe.xml
env:
TEST_DEVICE: CPU
TEST_PRECISION: FP16
Expand Down
21 changes: 13 additions & 8 deletions tests/layer_tests/tensorflow_tests/test_tf_ArgMinMax.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,11 @@
# Documentation: https://www.tensorflow.org/api_docs/python/tf/raw_ops/ArgMin
# https://www.tensorflow.org/api_docs/python/tf/raw_ops/ArgMax

OPS = {
'tf.raw_ops.ArgMax': tf.raw_ops.ArgMax,
'tf.raw_ops.ArgMin': tf.raw_ops.ArgMin
}

class TestArgMinMax(CommonTFLayerTest):
def _prepare_input(self, inputs_info):
assert 'input' in inputs_info
Expand Down Expand Up @@ -41,24 +46,24 @@ def create_argmin_max_net(self, input_shape, dimension, input_type, output_type,
return tf_net, ref_net

test_data = [
dict(input_shape=[20], dimension=0),
dict(input_shape=[20, 30], dimension=1),
dict(input_shape=[2, 30, 3, 4], dimension=2),
[[20], 0],
[[20, 30], 1],
[[2, 30, 3, 4], 2],
]

@pytest.mark.parametrize("params", test_data)
@pytest.mark.parametrize("input_shape, dimension", test_data)
@pytest.mark.parametrize("input_type", [np.float32, np.int32])
@pytest.mark.parametrize("output_type", [tf.int32, tf.int64])
@pytest.mark.parametrize("op_type", [tf.raw_ops.ArgMax, tf.raw_ops.ArgMin])
@pytest.mark.parametrize("op_type", ['tf.raw_ops.ArgMax', 'tf.raw_ops.ArgMin'])
@pytest.mark.precommit_tf_fe
@pytest.mark.nightly
@pytest.mark.xfail(condition=platform.system() == 'Linux' and platform.machine() in ['arm', 'armv7l',
'aarch64',
'arm64', 'ARM64'],
reason='Ticket - 126314')
def test_argmin_max_net(self, params, input_type, output_type, op_type, ie_device, precision, ir_version, temp_dir,
use_new_frontend):
def test_argmin_max_net(self, input_shape, dimension, input_type, output_type, op_type, ie_device, precision, ir_version, temp_dir, use_new_frontend):
params = dict(input_shape=input_shape, dimension=dimension)
self._test(*self.create_argmin_max_net(**params, input_type=input_type,
output_type=output_type, op_type=op_type),
output_type=output_type, op_type=OPS[op_type]),
ie_device, precision, ir_version, temp_dir=temp_dir,
use_new_frontend=use_new_frontend)
14 changes: 10 additions & 4 deletions tests/layer_tests/tensorflow_tests/test_tf_CheckNumerics.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,11 @@
from common.tf_layer_test_class import CommonTFLayerTest


OPS = {
"tf.raw_ops.CheckNumerics": tf.raw_ops.CheckNumerics,
"tf.raw_ops.CheckNumericsV2": tf.raw_ops.CheckNumericsV2
}

class TestCheckNumerics(CommonTFLayerTest):
def _prepare_input(self, inputs_info):
assert 'x' in inputs_info
Expand All @@ -33,15 +38,16 @@ def create_check_numerics_net(self, input_shape, input_type, op):
return tf_net, None

test_data_basic = [
dict(input_shape=[2, 6], input_type=np.float32, op=tf.raw_ops.CheckNumerics),
dict(input_shape=[3, 4, 5], input_type=np.float32, op=tf.raw_ops.CheckNumericsV2),
[[2, 6], np.float32, 'tf.raw_ops.CheckNumerics'],
[[3, 4, 5], np.float32, 'tf.raw_ops.CheckNumericsV2'],
]

@pytest.mark.parametrize("params", test_data_basic)
@pytest.mark.parametrize("input_shape, input_type, op", test_data_basic)
@pytest.mark.precommit_tf_fe
@pytest.mark.nightly
def test_check_numerics_basic(self, params, ie_device, precision, ir_version, temp_dir,
def test_check_numerics_basic(self, input_shape, input_type, op, ie_device, precision, ir_version, temp_dir,
use_new_frontend):
params = dict(input_shape=input_shape, input_type=input_type, op=OPS[op])
self._test(*self.create_check_numerics_net(**params),
ie_device, precision, ir_version, temp_dir=temp_dir,
use_new_frontend=use_new_frontend)
88 changes: 52 additions & 36 deletions tests/layer_tests/tensorflow_tests/test_tf_ComplexFFT.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,21 @@
from common.tf_layer_test_class import CommonTFLayerTest


OPS = {
'tf.raw_ops.IRFFT': tf.raw_ops.IRFFT,
'tf.raw_ops.IRFFT2D': tf.raw_ops.IRFFT2D,
'tf.raw_ops.IRFFT3D': tf.raw_ops.IRFFT3D,
'tf.raw_ops.FFT': tf.raw_ops.FFT,
'tf.raw_ops.FFT2D': tf.raw_ops.FFT2D,
'tf.raw_ops.FFT3D': tf.raw_ops.FFT3D,
'tf.raw_ops.IFFT': tf.raw_ops.IFFT,
'tf.raw_ops.IFFT2D': tf.raw_ops.IFFT2D,
'tf.raw_ops.IFFT3D': tf.raw_ops.IFFT3D,
'tf.raw_ops.RFFT': tf.raw_ops.RFFT,
'tf.raw_ops.RFFT2D': tf.raw_ops.RFFT2D,
'tf.raw_ops.RFFT3D': tf.raw_ops.RFFT3D
}

class TestComplexFFT(CommonTFLayerTest):
def _prepare_input(self, inputs_info):
rng = np.random.default_rng()
Expand Down Expand Up @@ -41,30 +56,31 @@ def create_complex_fft_net(self, input_shape, shift_roll, axis_roll, fft_op):
return tf_net, None

test_data_basic = [
dict(input_shape=[1, 50, 2], shift_roll=[10, 1], axis_roll=[-2, -1]),
dict(input_shape=[4, 20, 3], shift_roll=[2, 10], axis_roll=[0, 1]),
dict(input_shape=[1, 50, 50, 2], shift_roll=[10, 20], axis_roll=[-2, -1]),
dict(input_shape=[4, 20, 30, 3], shift_roll=[2, 10], axis_roll=[0, 1]),
dict(input_shape=[1, 50, 50, 30, 2], shift_roll=[10, 20, 4], axis_roll=[-3, -2, -1]),
dict(input_shape=[4, 20, 30, 10, 3], shift_roll=[2, 10], axis_roll=[1, 2]),
[[1, 50, 2], [10, 1], [-2, -1]],
[[4, 20, 3], [2, 10], [0, 1]],
[[1, 50, 50, 2], [10, 20], [-2, -1]],
[[4, 20, 30, 3], [2, 10], [0, 1]],
[[1, 50, 50, 30, 2], [10, 20, 4], [-3, -2, -1]],
[[4, 20, 30, 10, 3], [2, 10], [1, 2]],
]

@pytest.mark.parametrize("fft_op", [
tf.raw_ops.FFT, tf.raw_ops.FFT2D, tf.raw_ops.FFT3D,
tf.raw_ops.IFFT, tf.raw_ops.IFFT2D, tf.raw_ops.IFFT3D
"tf.raw_ops.FFT", "tf.raw_ops.FFT2D", "tf.raw_ops.FFT3D",
"tf.raw_ops.IFFT", "tf.raw_ops.IFFT2D", "tf.raw_ops.IFFT3D"
])
@pytest.mark.parametrize("params", test_data_basic)
@pytest.mark.parametrize("input_shape, shift_roll, axis_roll", test_data_basic)
@pytest.mark.precommit_tf_fe
@pytest.mark.nightly
@pytest.mark.xfail(condition=platform.system() == 'Linux' and platform.machine() in ['arm', 'armv7l',
'aarch64',
'arm64', 'ARM64'],
reason='Ticket - 126314')
def test_complex_fft_basic(self, params, fft_op,
def test_complex_fft_basic(self, input_shape, shift_roll, axis_roll, fft_op,
ie_device, precision, ir_version, temp_dir,
use_new_frontend):
params = dict(input_shape=input_shape, shift_roll=shift_roll, axis_roll=axis_roll)
self._test(
*self.create_complex_fft_net(**params, fft_op=fft_op),
*self.create_complex_fft_net(**params, fft_op=OPS[fft_op]),
ie_device, precision, ir_version, temp_dir=temp_dir,
use_new_frontend=use_new_frontend, custom_eps=1e-2)

Expand Down Expand Up @@ -95,20 +111,19 @@ def create_complex_abs_net(self, input_shape):
return tf_net, None

test_data_basic = [
dict(input_shape=[]),
dict(input_shape=[2]),
dict(input_shape=[1, 3]),
dict(input_shape=[2, 3, 4]),
dict(input_shape=[3, 4, 5, 6]),
[],
[2],
[1, 3],
[2, 3, 4],
[3, 4, 5, 6],
]

@pytest.mark.parametrize("params", test_data_basic)
@pytest.mark.parametrize("input_shape", test_data_basic)
@pytest.mark.precommit_tf_fe
@pytest.mark.nightly
def test_complex_abs_basic(self, params, ie_device, precision, ir_version, temp_dir,
def test_complex_abs_basic(self, input_shape, ie_device, precision, ir_version, temp_dir,
use_new_frontend):
self._test(
*self.create_complex_abs_net(**params),
*self.create_complex_abs_net(input_shape),
ie_device, precision, ir_version, temp_dir=temp_dir,
use_new_frontend=use_new_frontend)

Expand Down Expand Up @@ -138,18 +153,19 @@ def create_complex_rfft_net(self, input_shape, fft_length, rfft_op):
return tf_net, None

test_data_basic = [
dict(input_shape=[1, 3, 20], fft_length=[10], rfft_op=tf.raw_ops.RFFT),
dict(input_shape=[1, 3, 20], fft_length=[20], rfft_op=tf.raw_ops.RFFT),
dict(input_shape=[1, 3, 20, 40], fft_length=[20, 10], rfft_op=tf.raw_ops.RFFT2D),
dict(input_shape=[1, 3, 20, 40], fft_length=[10, 40], rfft_op=tf.raw_ops.RFFT2D),
dict(input_shape=[1, 2, 10, 20, 5], fft_length=[2, 5, 3], rfft_op=tf.raw_ops.RFFT3D),
[[1, 3, 20], [10], 'tf.raw_ops.RFFT'],
[[1, 3, 20], [20], 'tf.raw_ops.RFFT'],
[[1, 3, 20, 40], [20, 10], 'tf.raw_ops.RFFT2D'],
[[1, 3, 20, 40], [10, 40], 'tf.raw_ops.RFFT2D'],
[[1, 2, 10, 20, 5], [2, 5, 3], 'tf.raw_ops.RFFT3D']
]

@pytest.mark.parametrize("params", test_data_basic)
@pytest.mark.parametrize("input_shape, fft_length, rfft_op", test_data_basic)
@pytest.mark.precommit_tf_fe
@pytest.mark.nightly
def test_complex_rfft_basic(self, params, ie_device, precision, ir_version, temp_dir,
def test_complex_rfft_basic(self, input_shape, fft_length, rfft_op, ie_device, precision, ir_version, temp_dir,
use_new_frontend):
params = dict(input_shape=input_shape, fft_length=fft_length, rfft_op=OPS[rfft_op])
self._test(
*self.create_complex_rfft_net(**params),
ie_device, precision, ir_version, temp_dir=temp_dir,
Expand Down Expand Up @@ -183,19 +199,19 @@ def create_complex_irfft_net(self, input_shape, fft_length, irfft_op):
return tf_net, None

test_data_basic = [
dict(input_shape=[1, 3, 20], fft_length=[10], irfft_op=tf.raw_ops.IRFFT),
dict(input_shape=[1, 3, 20], fft_length=[20], irfft_op=tf.raw_ops.IRFFT),
dict(input_shape=[1, 3, 20, 40], fft_length=[20, 10], irfft_op=tf.raw_ops.IRFFT2D),
dict(input_shape=[1, 3, 20, 40], fft_length=[10, 40], irfft_op=tf.raw_ops.IRFFT2D),
pytest.param(dict(input_shape=[1, 10, 20, 30, 5], fft_length=[2, 3, 4], irfft_op=tf.raw_ops.IRFFT3D),
marks=pytest.mark.xfail(reason="accuracy-issue-124452"))
[[1, 3, 20], [10], 'tf.raw_ops.IRFFT'],
[[1, 3, 20], [20], 'tf.raw_ops.IRFFT'],
[[1, 3, 20, 40], [20, 10], 'tf.raw_ops.IRFFT2D'],
[[1, 3, 20, 40], [10, 40], 'tf.raw_ops.IRFFT2D'],
pytest.param([1, 10, 20, 30, 5], [2, 3, 4], 'tf.raw_ops.IRFFT3D',
marks=pytest.mark.xfail(reason="accuracy-issue-124452"))
]

@pytest.mark.parametrize("params", test_data_basic)
@pytest.mark.parametrize("input_shape, fft_length, irfft_op", test_data_basic)
@pytest.mark.precommit_tf_fe
@pytest.mark.nightly
def test_complex_irfft_basic(self, params, ie_device, precision, ir_version, temp_dir,
def test_complex_irfft_basic(self, input_shape, fft_length, irfft_op, ie_device, precision, ir_version, temp_dir,
use_new_frontend):
params = dict(input_shape=input_shape, fft_length=fft_length, irfft_op=OPS[irfft_op])
self._test(
*self.create_complex_irfft_net(**params),
ie_device, precision, ir_version, temp_dir=temp_dir,
Expand Down
34 changes: 19 additions & 15 deletions tests/layer_tests/tensorflow_tests/test_tf_FakeQuantWithMinMaxVars.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,12 @@
from common.tf_layer_test_class import CommonTFLayerTest


OPS = {
'tf.raw_ops.FakeQuantWithMinMaxVarsPerChannel': tf.raw_ops.FakeQuantWithMinMaxVarsPerChannel,
'tf.raw_ops.FakeQuantWithMinMaxVars': tf.raw_ops.FakeQuantWithMinMaxVars,
'tf.raw_ops.FakeQuantWithMinMaxArgs': tf.raw_ops.FakeQuantWithMinMaxArgs
}

class TestFakeQuantWithMinMaxVars(CommonTFLayerTest):
def _prepare_input(self, inputs_info):
# generate elements so that the input tensor may contain repeating elements
Expand All @@ -32,38 +38,36 @@ def create_fake_quant_with_min_max_vars_net(self, inputs_shape, min_value, max_v

test_basic = [
# test FakeQuantWithMinMaxVars
dict(inputs_shape=[2, 6, 4], min_value=-3, max_value=4, num_bits=None, narrow_range=None),
dict(inputs_shape=[3, 2, 1, 5], min_value=-4, max_value=5, num_bits=14, narrow_range=True),
dict(inputs_shape=[3, 2, 4], min_value=2, max_value=4, num_bits=10, narrow_range=False),
dict(inputs_shape=[1, 2, 3], min_value=-6, max_value=-3, num_bits=8, narrow_range=True),
[[2, 6, 4], -3, 4, None, None],
[[3, 2, 1, 5], -4, 5, 14, True],
[[3, 2, 4], 2, 4, 10, False],
[[1, 2, 3], -6, -3, 8, True],
]

@pytest.mark.parametrize("params", test_basic)
@pytest.mark.parametrize("inputs_shape, min_value, max_value, num_bits, narrow_range", test_basic)
@pytest.mark.parametrize("fake_quant_op", [
tf.raw_ops.FakeQuantWithMinMaxVars, tf.raw_ops.FakeQuantWithMinMaxArgs
'tf.raw_ops.FakeQuantWithMinMaxVars', 'tf.raw_ops.FakeQuantWithMinMaxArgs'
])
@pytest.mark.precommit_tf_fe
@pytest.mark.nightly
@pytest.mark.xfail(condition=platform.system() == 'Darwin' and platform.machine() == 'arm64',
reason='Ticket - 122716')
def test_fake_quant_with_min_max_vars_basic(self, params, fake_quant_op, ie_device, precision, ir_version, temp_dir,
use_new_frontend):
self._test(*self.create_fake_quant_with_min_max_vars_net(**params, fake_quant_op=fake_quant_op),
def test_fake_quant_with_min_max_vars_basic(self, inputs_shape, min_value, max_value, num_bits, narrow_range, fake_quant_op, ie_device, precision, ir_version, temp_dir, use_new_frontend):
params = dict(inputs_shape=inputs_shape, min_value=min_value, max_value=max_value, num_bits=num_bits, narrow_range=narrow_range)
self._test(*self.create_fake_quant_with_min_max_vars_net(**params, fake_quant_op=OPS[fake_quant_op]),
ie_device, precision, ir_version, temp_dir=temp_dir,
use_new_frontend=use_new_frontend)

test_per_channel_basic = [
dict(inputs_shape=[2, 6, 4], min_value=[-4, -3, -5, -8], max_value=[4, 7, 9, 5], num_bits=None,
narrow_range=None,
fake_quant_op=tf.raw_ops.FakeQuantWithMinMaxVarsPerChannel),
[[2, 6, 4], [-4, -3, -5, -8], [4, 7, 9, 5], None, None, 'tf.raw_ops.FakeQuantWithMinMaxVarsPerChannel'],
]

@pytest.mark.parametrize("params", test_per_channel_basic)
@pytest.mark.parametrize("inputs_shape, min_value, max_value, num_bits, narrow_range, fake_quant_op", test_per_channel_basic)
@pytest.mark.precommit_tf_fe
@pytest.mark.nightly
@pytest.mark.xfail("104822")
def test_fake_quant_with_min_max_vars_per_channel_basic(self, params, ie_device, precision, ir_version, temp_dir,
use_new_frontend):
def test_fake_quant_with_min_max_vars_per_channel_basic(self, inputs_shape, min_value, max_value, num_bits, narrow_range, fake_quant_op, ie_device, precision, ir_version, temp_dir, use_new_frontend):
params=dict(inputs_shape=inputs_shape, min_value=min_value, max_value=max_value, num_bits=num_bits, narrow_range=narrow_range, fake_quant_op=OPS[fake_quant_op])
self._test(*self.create_fake_quant_with_min_max_vars_net(**params),
ie_device, precision, ir_version, temp_dir=temp_dir,
use_new_frontend=use_new_frontend)
20 changes: 14 additions & 6 deletions tests/layer_tests/tensorflow_tests/test_tf_Identity.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,13 @@
from common.tf_layer_test_class import CommonTFLayerTest


OPS = {
'tf.raw_ops.Identity': tf.raw_ops.Identity,
'tf.raw_ops.PreventGradient': tf.raw_ops.PreventGradient,
'tf.raw_ops.Snapshot': tf.raw_ops.Snapshot,
'tf.raw_ops.StopGradient': tf.raw_ops.StopGradient,
}

class TestIdentity(CommonTFLayerTest):
def create_identity_net(self, input_shape, identity_op):
tf.compat.v1.reset_default_graph()
Expand All @@ -22,17 +29,18 @@ def create_identity_net(self, input_shape, identity_op):
return tf_net, None

test_data_basic = [
dict(input_shape=[2], identity_op=tf.raw_ops.Identity),
dict(input_shape=[2, 3], identity_op=tf.raw_ops.PreventGradient),
dict(input_shape=[], identity_op=tf.raw_ops.Snapshot),
dict(input_shape=[1, 2, 3], identity_op=tf.raw_ops.StopGradient)
[[2], 'tf.raw_ops.Identity'],
[[2, 3], 'tf.raw_ops.PreventGradient'],
[[], 'tf.raw_ops.Snapshot'],
[[1, 2, 3], 'tf.raw_ops.StopGradient']
]

@pytest.mark.parametrize("params", test_data_basic)
@pytest.mark.parametrize("input_shape, identity_op", test_data_basic)
@pytest.mark.precommit_tf_fe
@pytest.mark.nightly
def test_identity_basic(self, params, ie_device, precision, ir_version, temp_dir,
def test_identity_basic(self, input_shape, identity_op, ie_device, precision, ir_version, temp_dir,
use_new_frontend):
params = dict(input_shape=input_shape, identity_op=OPS[identity_op])
self._test(*self.create_identity_net(**params),
ie_device, precision, ir_version, temp_dir=temp_dir,
use_new_frontend=use_new_frontend)
Loading

0 comments on commit 1a8f72f

Please sign in to comment.