From ab64489d9b0e3e0c334bf9a21f6842305c69cc59 Mon Sep 17 00:00:00 2001 From: Evgeny Kotov Date: Wed, 27 Mar 2024 07:02:01 +0100 Subject: [PATCH] Implement performance tests in nightly validation to compare read_model and ovc.convert with OVVP output (#23303) ### Details: - add nightly performance tests with skip list - [restored PR](https://github.com/openvinotoolkit/openvino/pull/21353) - [restored PR #2](https://github.com/openvinotoolkit/openvino/pull/22325) - added OVVP support - extend skip list with the models that have unstable results ### Tickets: - 126282 --- .../models_hub_common/constants.py | 6 +- .../test_performance_model.py | 32 ++- .../models_hub_common/utils.py | 22 ++ .../performance_tests/conftest.py | 35 +++ .../performance_tests/nightly_models.skip | 264 ++++++++++++++++++ .../test_tf_hub_performance_model.py | 59 +++- 6 files changed, 391 insertions(+), 27 deletions(-) create mode 100644 tests/model_hub_tests/performance_tests/nightly_models.skip diff --git a/tests/model_hub_tests/models_hub_common/constants.py b/tests/model_hub_tests/models_hub_common/constants.py index dfde0e7ecd625e..d46c858912d85c 100644 --- a/tests/model_hub_tests/models_hub_common/constants.py +++ b/tests/model_hub_tests/models_hub_common/constants.py @@ -10,11 +10,13 @@ loading and heating and includes measurement only one of 2 models - got through convert and read_model. Both "converted" and "read_model" modes will be 2 * runtime_measure_duration ''' -runtime_measure_duration = os.environ.get('RUNTIME_MEASURE_DURATION', '60') +precommit_runtime_measure_duration = os.environ.get('PRECOMMIT_RUNTIME_MEASURE_DURATION', '60') +nightly_runtime_measure_duration = os.environ.get('NIGHTLY_RUNTIME_MEASURE_DURATION', '15') ''' @brief Time in seconds of heating before measurement ''' -runtime_heat_duration = os.environ.get('RUNTIME_HEAT_DURATION', '5') +precommit_runtime_heat_duration = os.environ.get('PRECOMMIT_RUNTIME_HEAT_DURATION', '5') +nigtly_runtime_heat_duration = os.environ.get('NIGHTLY_RUNTIME_HEAT_DURATION', '5') tf_hub_cache_dir = os.environ.get('TFHUB_CACHE_DIR', os.path.join(tempfile.gettempdir(), "tfhub_modules")) diff --git a/tests/model_hub_tests/models_hub_common/test_performance_model.py b/tests/model_hub_tests/models_hub_common/test_performance_model.py index 90704aa0d6939f..7cbbaec63278e8 100644 --- a/tests/model_hub_tests/models_hub_common/test_performance_model.py +++ b/tests/model_hub_tests/models_hub_common/test_performance_model.py @@ -61,6 +61,9 @@ def __init__(self): self.infer_time_ratio = 0.0 self.error_message = '' self.status = None + self.model_name = '' + self.model_link = '' + self.ie_device = '' class TestModelPerformance: @@ -120,14 +123,12 @@ def get_read_model(self, model_path: str): core = ov.Core() return core.read_model(model=model_path) - def heat_hardware(self, ov_model, inputs) -> None: - _, heat_n_repeats, _ = utils.measure(utils.nano_secs(const.runtime_heat_duration), ov_model, (inputs,)) + def heat_hardware(self, ov_model, inputs, conf) -> None: + _, heat_n_repeats, _ = utils.measure(conf.runtime_heat_duration, ov_model, (inputs,)) print('heat done in {} repeats'.format(heat_n_repeats)) - def measure_inference(self, ov_model, inputs) -> ModelResults: - time_slices, infer_n_repeats, real_runtime = utils.measure(utils.nano_secs(const.runtime_measure_duration), - ov_model, - (inputs,)) + def measure_inference(self, ov_model, inputs, conf) -> ModelResults: + time_slices, infer_n_repeats, real_runtime = utils.measure(conf.runtime_measure_duration, ov_model, (inputs,)) print('measurement done in {} repeats'.format(infer_n_repeats)) infer_throughput = float(infer_n_repeats * (10 ** 9)) / real_runtime infer_mean_time_ns = np.mean(time_slices) @@ -141,16 +142,19 @@ def measure_inference(self, ov_model, inputs) -> ModelResults: results.infer_variance = infer_variance return results - def infer_model(self, ov_model, inputs) -> ModelResults: - self.heat_hardware(ov_model, inputs) - return self.measure_inference(ov_model, inputs) + def infer_model(self, ov_model, inputs, conf) -> ModelResults: + self.heat_hardware(ov_model, inputs, conf) + return self.measure_inference(ov_model, inputs, conf) def compile_model(self, model, ie_device): core = ov.Core() return core.compile_model(model, ie_device) - def __run(self, model_name, model_link, ie_device): + def __run(self, model_name, model_link, ie_device, conf): results = Results() + results.model_name = model_name + results.model_link = model_link + results.ie_device = ie_device results.status = None try: results.status = Status.LOAD_MODEL @@ -168,11 +172,11 @@ def __run(self, model_name, model_link, ie_device): results.status = Status.INFER_CONVERTED_MODEL results.converted_model_results = utils.call_with_timer('Infer converted model', self.infer_model, - (converted_model, inputs)) + (converted_model, inputs, conf)) results.status = Status.INFER_READ_MODEL results.read_model_results = utils.call_with_timer('Infer read model', self.infer_model, - (read_model, inputs)) + (read_model, inputs, conf)) infer_time_ratio = (results.converted_model_results.infer_mean_time / results.read_model_results.infer_mean_time) @@ -196,10 +200,10 @@ def __run(self, model_name, model_link, ie_device): ex_type=ex_type.__name__, ex_value=ex_value) return results - def run(self, model_name, model_link, ie_device): + def run(self, model_name, model_link, ie_device, conf): self.result = Results() t0 = time.time() - self.result = multiprocessing_run(self.__run, [model_name, model_link, ie_device], model_name, + self.result = multiprocessing_run(self.__run, [model_name, model_link, ie_device, conf], model_name, self.infer_timeout) t1 = time.time() utils.print_stat('test run time {} secs', (t1 - t0)) diff --git a/tests/model_hub_tests/models_hub_common/utils.py b/tests/model_hub_tests/models_hub_common/utils.py index 403464eaae1350..9c5a02207b5729 100644 --- a/tests/model_hub_tests/models_hub_common/utils.py +++ b/tests/model_hub_tests/models_hub_common/utils.py @@ -32,6 +32,28 @@ def get_models_list(file_name: str): return models +def get_skipped_model_links(filename: str): + links = set() + if not os.path.exists(filename): + return links + with open(filename) as f: + for model_info in f: + model_info = model_info.strip() + model_name, model_link = model_info.split(',') + links.add(model_link) + return links + + +def get_models_list_not_skipped(model_list_file: str, skip_list_file: str): + skipped_links = get_skipped_model_links(skip_list_file) + not_skipped_models = [] + for model_name, model_link, mark, reason in get_models_list(model_list_file): + if model_link in skipped_links: + continue + not_skipped_models.append((model_name, model_link, mark, reason)) + return not_skipped_models + + def compare_two_tensors(ov_res, fw_res, eps): is_ok = True if not np.allclose(ov_res, fw_res, atol=eps, rtol=eps, equal_nan=True): diff --git a/tests/model_hub_tests/performance_tests/conftest.py b/tests/model_hub_tests/performance_tests/conftest.py index a8567ed8d196db..8f0bc2a99e6b63 100644 --- a/tests/model_hub_tests/performance_tests/conftest.py +++ b/tests/model_hub_tests/performance_tests/conftest.py @@ -7,6 +7,39 @@ from models_hub_common.utils import get_params from py.xml import html from models_hub_common.utils import round_num +import json +import requests + + +def pytest_sessionstart(session): + print('pytest_sessionstart') + session.results = [] + + +def pytest_sessionfinish(session, exitstatus): + print('pytest_sessionfinish') + dump_data = [] + for item in session.results: + dump_item = dict() + dump_item['version'] = '1' + dump_item['status'] = str(item.status) + dump_item['ie_device'] = item.ie_device + dump_item['model_name'] = item.model_name + dump_item['model_link'] = item.model_link + dump_item['time_infer_conv'] = str(item.converted_model_results.infer_mean_time) + dump_item['time_infer_read'] = str(item.read_model_results.infer_mean_time) + dump_item['infer_time_ratio'] = str(item.infer_time_ratio) + dump_data.append(dump_item) + data = json.dumps(dump_data) + url = 'https://openvino-validation-portal.toolbox.iotg.sclab.intel.com/api/v1/Seeder_Tf_Hub_Compile_Read/seed' + headers = { + 'accept': 'application/json', + 'Content-Type': 'application/json' + } + try: + requests.post(url, headers=headers, data=data) + except: + print('error while sending data to OVVP') def pytest_generate_tests(metafunc): @@ -22,6 +55,8 @@ def pytest_runtest_makereport(item, call): if getattr(item.obj.__self__, 'result', None) is not None: results = item.obj.__self__.result report._results = results + if report.when == 'call': + item.session.results.append(results) @pytest.mark.optionalhook diff --git a/tests/model_hub_tests/performance_tests/nightly_models.skip b/tests/model_hub_tests/performance_tests/nightly_models.skip new file mode 100644 index 00000000000000..fba49f49d95ac0 --- /dev/null +++ b/tests/model_hub_tests/performance_tests/nightly_models.skip @@ -0,0 +1,264 @@ +universal-sentence-encoder,https://www.kaggle.com/models/google/universal-sentence-encoder/frameworks/tensorFlow2/variations/universal-sentence-encoder/versions/2 +universal-sentence-encoder-multilingual,https://www.kaggle.com/models/google/universal-sentence-encoder/frameworks/tensorFlow2/variations/multilingual/versions/2 +universal-sentence-encoder-large,https://www.kaggle.com/models/google/universal-sentence-encoder/frameworks/tensorFlow2/variations/large/versions/2 +bert_en_uncased_preprocess,https://www.kaggle.com/models/tensorflow/bert/frameworks/tensorFlow2/variations/en-uncased-preprocess/versions/3 +tf2-preview/mobilenet_v2/feature_vector,https://www.kaggle.com/models/google/mobilenet-v2/frameworks/tensorFlow2/variations/tf2-preview-feature-vector/versions/4 +nnlm-en-dim128,https://www.kaggle.com/models/google/nnlm/frameworks/tensorFlow2/variations/en-dim128/versions/1 +elmo,https://www.kaggle.com/models/google/elmo/frameworks/tensorFlow1/variations/elmo/versions/3 +universal-sentence-encoder-multilingual-large,https://www.kaggle.com/models/google/universal-sentence-encoder/frameworks/tensorFlow2/variations/multilingual-large/versions/2 +spice,https://www.kaggle.com/models/google/spice/frameworks/tensorFlow1/variations/spice/versions/2 +tf2-preview/nnlm-en-dim128,https://www.kaggle.com/models/google/nnlm/frameworks/tensorFlow2/variations/tf2-preview-en-dim128/versions/1 +universal-sentence-encoder-lite,https://www.kaggle.com/models/google/universal-sentence-encoder/frameworks/tensorFlow1/variations/lite/versions/2 +tf2-preview/mobilenet_v2/classification,https://www.kaggle.com/models/google/mobilenet-v2/frameworks/tensorFlow2/variations/tf2-preview-classification/versions/4 +i3d-kinetics-400,https://www.kaggle.com/models/deepmind/i3d-kinetics/frameworks/tensorFlow1/variations/400/versions/1 +tf2-preview/gnews-swivel-20dim,https://www.kaggle.com/models/google/gnews-swivel/frameworks/tensorFlow2/variations/tf2-preview-20dim/versions/1 +faster_rcnn/openimages_v4/inception_resnet_v2,https://www.kaggle.com/models/google/faster-rcnn-inception-resnet-v2/frameworks/tensorFlow1/variations/faster-rcnn-openimages-v4-inception-resnet-v2/versions/1 +openimages_v4/ssd/mobilenet_v2,https://www.kaggle.com/models/google/mobilenet-v2/frameworks/tensorFlow1/variations/openimages-v4-ssd-mobilenet-v2/versions/1 +universal-sentence-encoder-xling-many,https://www.kaggle.com/models/google/universal-sentence-encoder/frameworks/tensorFlow1/variations/xling-many/versions/1 +nnlm-en-dim128-with-normalization,https://www.kaggle.com/models/google/nnlm/frameworks/tensorFlow2/variations/en-dim128-with-normalization/versions/1 +nnlm-en-dim50,https://www.kaggle.com/models/google/nnlm/frameworks/tensorFlow2/variations/en-dim50/versions/1 +universal-sentence-encoder-cmlm/multilingual-preprocess,https://www.kaggle.com/models/google/universal-sentence-encoder/frameworks/tensorFlow2/variations/cmlm-multilingual-preprocess/versions/2 +vggish,https://www.kaggle.com/models/google/vggish/frameworks/tensorFlow2/variations/vggish/versions/1 +tf2-preview/inception_v3/feature_vector,https://www.kaggle.com/models/google/inception-v3/frameworks/tensorFlow2/variations/tf2-preview-feature-vector/versions/4 +efficientdet/lite0/detection,https://www.kaggle.com/models/tensorflow/efficientdet/frameworks/tensorFlow2/variations/lite0-detection/versions/1 +nnlm-ja-dim128,https://www.kaggle.com/models/google/nnlm/frameworks/tensorFlow2/variations/ja-dim128/versions/1 +universal-sentence-encoder-multilingual-qa,https://www.kaggle.com/models/google/universal-sentence-encoder/frameworks/tensorFlow2/variations/multilingual-qa/versions/2 +nnlm-ja-dim128-with-normalization,https://www.kaggle.com/models/google/nnlm/frameworks/tensorFlow2/variations/ja-dim128-with-normalization/versions/1 +nnlm-en-dim50-with-normalization,https://www.kaggle.com/models/google/nnlm/frameworks/tensorFlow2/variations/en-dim50-with-normalization/versions/1 +universal-sentence-encoder-qa,https://www.kaggle.com/models/google/universal-sentence-encoder/frameworks/tensorFlow2/variations/qa/versions/2 +biggan-deep-256,https://www.kaggle.com/models/deepmind/biggan/frameworks/tensorFlow1/variations/deep-256/versions/1 +efficientdet/lite2/detection,https://www.kaggle.com/models/tensorflow/efficientdet/frameworks/tensorFlow2/variations/lite2-detection/versions/1 +delf,https://www.kaggle.com/models/google/delf/frameworks/tensorFlow1/variations/delf/versions/1 +tf2-preview/gnews-swivel-20dim-with-oov,https://www.kaggle.com/models/google/gnews-swivel/frameworks/tensorFlow2/variations/tf2-preview-20dim-with-oov/versions/1 +tf2-preview/nnlm-es-dim50-with-normalization,https://www.kaggle.com/models/google/nnlm/frameworks/tensorFlow2/variations/tf2-preview-es-dim50-with-normalization/versions/1 +enformer,https://www.kaggle.com/models/deepmind/enformer/frameworks/tensorFlow2/variations/enformer/versions/1 +albert_en_base,https://www.kaggle.com/models/tensorflow/albert/frameworks/tensorFlow2/variations/en-base/versions/3 +nonsemantic-speech-benchmark/trill-distilled,https://www.kaggle.com/models/google/nonsemantic-speech-benchmark/frameworks/tensorFlow2/variations/trill-distilled/versions/3 +biggan-256,https://www.kaggle.com/models/deepmind/biggan/frameworks/tensorFlow1/variations/256/versions/2 +nnlm-de-dim50-with-normalization,https://www.kaggle.com/models/google/nnlm/frameworks/tensorFlow2/variations/de-dim50-with-normalization/versions/1 +bert_multi_cased_preprocess,https://www.kaggle.com/models/tensorflow/bert/frameworks/tensorFlow2/variations/multi-cased-preprocess/versions/3 +tfgan/eval/inception,https://www.kaggle.com/models/tensorflow/inception/frameworks/tensorFlow1/variations/tfgan-eval-inception/versions/1 +Wiki-words-250,https://www.kaggle.com/models/google/wiki-words/frameworks/tensorFlow2/variations/250/versions/1 +tfgan/eval/mnist/logits,https://www.kaggle.com/models/tensorflow/mnist/frameworks/tensorFlow1/variations/logits/versions/1 +albert_en_preprocess,https://www.kaggle.com/models/tensorflow/albert/frameworks/tensorFlow2/variations/en-preprocess/versions/3 +object_detection/mobile_object_localizer_v1,https://www.kaggle.com/models/google/mobile-object-localizer-v1/frameworks/tensorFlow1/variations/object-detection-mobile-object-localizer-v1/versions/1 +efficientdet/lite3/detection,https://www.kaggle.com/models/tensorflow/efficientdet/frameworks/tensorFlow2/variations/lite3-detection/versions/1 +tf2-preview/inception_v3/classification,https://www.kaggle.com/models/google/inception-v3/frameworks/tensorFlow2/variations/tf2-preview-classification/versions/4 +random-nnlm-en-dim128,https://www.kaggle.com/models/google/random-nnlm-en/frameworks/tensorFlow1/variations/dim128/versions/1 +biggan-deep-512,https://www.kaggle.com/models/deepmind/biggan/frameworks/tensorFlow1/variations/deep-512/versions/1 +tweening_conv3d_bair,https://www.kaggle.com/models/google/tweening-conv3d/frameworks/tensorFlow1/variations/bair/versions/1 +bert_en_cased_preprocess,https://www.kaggle.com/models/tensorflow/bert/frameworks/tensorFlow2/variations/en-cased-preprocess/versions/3 +albert_en_xxlarge,https://www.kaggle.com/models/tensorflow/albert/frameworks/tensorFlow2/variations/en-xxlarge/versions/3 +MuRIL_preprocess,https://www.kaggle.com/models/google/muril/frameworks/tensorFlow2/variations/preprocess/versions/1 +efficientdet/lite3x/detection,https://www.kaggle.com/models/tensorflow/efficientdet/frameworks/tensorFlow2/variations/lite3x-detection/versions/1 +midas/v2,https://www.kaggle.com/models/intel/midas/frameworks/tensorFlow1/variations/v2/versions/2 +nnlm-ja-dim50,https://www.kaggle.com/models/google/nnlm/frameworks/tensorFlow2/variations/ja-dim50/versions/1 +electra_small,https://www.kaggle.com/models/google/electra/frameworks/tensorFlow2/variations/small/versions/2 +nonsemantic-speech-benchmark/trill,https://www.kaggle.com/models/google/nonsemantic-speech-benchmark/frameworks/tensorFlow2/variations/trill/versions/3 +bigbigan-resnet50,https://www.kaggle.com/models/deepmind/bigbigan/frameworks/tensorFlow1/variations/resnet50/versions/1 +Wiki-words-250-with-normalization,https://www.kaggle.com/models/google/wiki-words/frameworks/tensorFlow2/variations/250-with-normalization/versions/1 +sentence-t5/st5-base,https://www.kaggle.com/models/google/sentence-t5/frameworks/tensorFlow2/variations/st5-base/versions/1 +bert_zh_preprocess,https://www.kaggle.com/models/tensorflow/bert/frameworks/tensorFlow2/variations/zh-preprocess/versions/3 +cropnet/classifier/cassava_disease_V1,https://www.kaggle.com/models/google/cropnet/frameworks/tensorFlow2/variations/classifier-cassava-disease-v1/versions/1 +midas/v2_1_small,https://www.kaggle.com/models/intel/midas/frameworks/tensorFlow1/variations/v2-1-small/versions/1 +i3d-kinetics-600,https://www.kaggle.com/models/deepmind/i3d-kinetics/frameworks/tensorFlow1/variations/600/versions/1 +nnlm-de-dim50,https://www.kaggle.com/models/google/nnlm/frameworks/tensorFlow2/variations/de-dim50/versions/1 +musiq/ava,https://www.kaggle.com/models/google/musiq/frameworks/tensorFlow2/variations/ava/versions/1 +nnlm-es-dim128,https://www.kaggle.com/models/google/nnlm/frameworks/tensorFlow2/variations/es-dim128/versions/1 +mil-nce/s3d,https://www.kaggle.com/models/deepmind/mil-nce/frameworks/tensorFlow1/variations/s3d/versions/1 +Wiki-words-500,https://www.kaggle.com/models/google/wiki-words/frameworks/tensorFlow2/variations/500/versions/1 +biggan-deep-128,https://www.kaggle.com/models/deepmind/biggan/frameworks/tensorFlow1/variations/deep-128/versions/1 +nnlm-es-dim128-with-normalization,https://www.kaggle.com/models/google/nnlm/frameworks/tensorFlow2/variations/es-dim128-with-normalization/versions/1 +vit_s16_fe,https://www.kaggle.com/models/spsayakpaul/vision-transformer/frameworks/tensorFlow2/variations/vit-s16-fe/versions/1 +zh_segmentation,https://www.kaggle.com/models/google/zh-segmentation/frameworks/tensorFlow1/variations/zh-segmentation/versions/1 +wiki40b-lm-es,https://www.kaggle.com/models/google/wiki40b-lm/frameworks/tensorFlow1/variations/es/versions/1 +nnlm-es-dim50-with-normalization,https://www.kaggle.com/models/google/nnlm/frameworks/tensorFlow2/variations/es-dim50-with-normalization/versions/1 +cord-19/swivel-128d,https://www.kaggle.com/models/tensorflow/cord-19-swivel-128d/frameworks/tensorFlow2/variations/cord-19-swivel-128d/versions/2 +centernet/resnet101v1_fpn_512x512,https://www.kaggle.com/models/tensorflow/centernet-resnet/frameworks/tensorFlow2/variations/101v1-fpn-512x512/versions/1 +movinet/a0/stream/kinetics-600/classification,https://www.kaggle.com/models/google/movinet/frameworks/tensorFlow2/variations/a0-stream-kinetics-600-classification/versions/2 +boundless/quarter,https://www.kaggle.com/models/google/boundless/frameworks/tensorFlow1/variations/quarter/versions/1 +gtr/gtr-base,https://www.kaggle.com/models/google/gtr/frameworks/tensorFlow2/variations/gtr-base/versions/1 +LAReQA/mBERT_En_En,https://www.kaggle.com/models/google/lareqa/frameworks/tensorFlow1/variations/mbert-en-en/versions/1 +small_bert/bert_uncased_L-2_H-128_A-2,https://www.kaggle.com/models/google/bert/frameworks/tensorFlow1/variations/bert-uncased-l-2-h-128-a-2/versions/2 +distilbert_en_uncased_preprocess,https://www.kaggle.com/models/jeongukjae/distilbert/frameworks/tensorFlow2/variations/en-uncased-preprocess/versions/2 +movinet/a2/stream/kinetics-600/classification,https://www.kaggle.com/models/google/movinet/frameworks/tensorFlow2/variations/a2-stream-kinetics-600-classification/versions/2 +nnlm-zh-dim50,https://www.kaggle.com/models/google/nnlm/frameworks/tensorFlow2/variations/zh-dim50/versions/1 +nnlm-id-dim128-with-normalization,https://www.kaggle.com/models/google/nnlm/frameworks/tensorFlow2/variations/id-dim128-with-normalization/versions/1 +humpback_whale,https://www.kaggle.com/models/google/humpback-whale/frameworks/tensorFlow2/variations/humpback-whale/versions/1 +nnlm-id-dim50,https://www.kaggle.com/models/google/nnlm/frameworks/tensorFlow2/variations/id-dim50/versions/1 +nonsemantic-speech-benchmark/frill,https://www.kaggle.com/models/google/nonsemantic-speech-benchmark/frameworks/tensorFlow2/variations/frill/versions/1 +vit_s16_classification,https://www.kaggle.com/models/spsayakpaul/vision-transformer/frameworks/tensorFlow2/variations/vit-s16-classification/versions/1 +roberta_en_cased_preprocess,https://www.kaggle.com/models/kaggle/roberta/frameworks/tensorFlow2/variations/en-cased-preprocess/versions/1 +tf2-preview/nnlm-ko-dim128-with-normalization,https://www.kaggle.com/models/google/nnlm/frameworks/tensorFlow2/variations/tf2-preview-ko-dim128-with-normalization/versions/1 +nnlm-id-dim128,https://www.kaggle.com/models/google/nnlm/frameworks/tensorFlow2/variations/id-dim128/versions/1 +distilbert_multi_cased_preprocess,https://www.kaggle.com/models/jeongukjae/distilbert/frameworks/tensorFlow2/variations/multi-cased-preprocess/versions/2 +nnlm-de-dim128,https://www.kaggle.com/models/google/nnlm/frameworks/tensorFlow2/variations/de-dim128/versions/1 +bertseq2seq/roberta24_gigaword,https://www.kaggle.com/models/google/bertseq2seq/frameworks/tensorFlow1/variations/roberta24-gigaword/versions/1 +vit_b8_fe,https://www.kaggle.com/models/spsayakpaul/vision-transformer/frameworks/tensorFlow2/variations/vit-b8-fe/versions/1 +bertseq2seq/roberta24_cnndm,https://www.kaggle.com/models/google/bertseq2seq/frameworks/tensorFlow1/variations/roberta24-cnndm/versions/1 +universal-sentence-encoder-xling/en-de,https://www.kaggle.com/models/google/universal-sentence-encoder/frameworks/tensorFlow1/variations/xling-en-de/versions/1 +smaller_LaBSE_15lang_preprocess,https://www.kaggle.com/models/kaggle/smaller-labse-15lang/frameworks/tensorFlow2/variations/preprocess/versions/1 +vgg19-block5-conv2-unpooling-decoder,https://www.kaggle.com/models/emilutz/vgg19-unpooling-encoder-decoder/frameworks/tensorFlow2/variations/vgg19-block5-conv2-unpooling-decoder/versions/1 +tf2-preview/nnlm-en-dim50-with-normalization,https://www.kaggle.com/models/google/nnlm/frameworks/tensorFlow2/variations/tf2-preview-en-dim50-with-normalization/versions/1 +cropnet/feature_vector/concat,https://www.kaggle.com/models/google/cropnet/frameworks/tensorFlow2/variations/feature-vector-concat/versions/1 +nnlm-zh-dim128,https://www.kaggle.com/models/google/nnlm/frameworks/tensorFlow2/variations/zh-dim128/versions/1 +tf2-preview/nnlm-ja-dim128-with-normalization,https://www.kaggle.com/models/google/nnlm/frameworks/tensorFlow2/variations/tf2-preview-ja-dim128-with-normalization/versions/1 +LAReQA/mBERT_X_Y,https://www.kaggle.com/models/google/lareqa/frameworks/tensorFlow1/variations/mbert-x-y/versions/1 +nnlm-es-dim50,https://www.kaggle.com/models/google/nnlm/frameworks/tensorFlow2/variations/es-dim50/versions/1 +nnlm-zh-dim128-with-normalization,https://www.kaggle.com/models/google/nnlm/frameworks/tensorFlow2/variations/zh-dim128-with-normalization/versions/1 +sentence-t5/st5-large,https://www.kaggle.com/models/google/sentence-t5/frameworks/tensorFlow2/variations/st5-large/versions/1 +mmv/s3d,https://www.kaggle.com/models/deepmind/mmv/frameworks/tensorFlow1/variations/s3d/versions/1 +ulmfit/en/sp35k_uncased,https://www.kaggle.com/models/edrone/ulmfit/frameworks/tensorFlow2/variations/en-sp35k-uncased/versions/1 +bertseq2seq/roberta24_wikisplit,https://www.kaggle.com/models/google/bertseq2seq/frameworks/tensorFlow1/variations/roberta24-wikisplit/versions/1 +mmv/tsm-resnet50x2,https://www.kaggle.com/models/deepmind/mmv/frameworks/tensorFlow1/variations/tsm-resnet50x2/versions/1 +cropnet/feature_vector/imagenet,https://www.kaggle.com/models/google/cropnet/frameworks/tensorFlow2/variations/feature-vector-imagenet/versions/1 +nnlm-ko-dim50-with-normalization,https://www.kaggle.com/models/google/nnlm/frameworks/tensorFlow2/variations/ko-dim50-with-normalization/versions/1 +nnlm-zh-dim50-with-normalization,https://www.kaggle.com/models/google/nnlm/frameworks/tensorFlow2/variations/zh-dim50-with-normalization/versions/1 +tf2-preview/nnlm-de-dim128-with-normalization,https://www.kaggle.com/models/google/nnlm/frameworks/tensorFlow2/variations/tf2-preview-de-dim128-with-normalization/versions/1 +wiki40b-lm-zh-cn,https://www.kaggle.com/models/google/wiki40b-lm/frameworks/tensorFlow1/variations/zh-cn/versions/1 +HRNet/coco-hrnetv2-w48,https://www.kaggle.com/models/google/hrnet/frameworks/tensorFlow2/variations/hrnet-coco-hrnetv2-w48/versions/1 +german-mbmelgan,https://www.kaggle.com/models/myusufs/german-mbmelgan/frameworks/tensorFlow2/variations/german-mbmelgan/versions/1 +wiki40b-lm-fr,https://www.kaggle.com/models/google/wiki40b-lm/frameworks/tensorFlow1/variations/fr/versions/1 +german-tacotron2,https://www.kaggle.com/models/myusufs/german-tacotron2/frameworks/tensorFlow2/variations/german-tacotron2/versions/1 +gtr/gtr-large,https://www.kaggle.com/models/google/gtr/frameworks/tensorFlow2/variations/gtr-large/versions/1 +trillsson1,https://www.kaggle.com/models/google/trillsson/frameworks/tensorFlow2/variations/1/versions/1 +tf2-preview/nnlm-de-dim50-with-normalization,https://www.kaggle.com/models/google/nnlm/frameworks/tensorFlow2/variations/tf2-preview-de-dim50-with-normalization/versions/1 +vit_b8_classification,https://www.kaggle.com/models/spsayakpaul/vision-transformer/frameworks/tensorFlow2/variations/vit-b8-classification/versions/1 +universal-sentence-encoder-xling/en-es,https://www.kaggle.com/models/google/universal-sentence-encoder/frameworks/tensorFlow1/variations/xling-en-es/versions/1 +mil-nce/i3d,https://www.kaggle.com/models/deepmind/mil-nce/frameworks/tensorFlow1/variations/i3d/versions/1 +vit_l16_fe,https://www.kaggle.com/models/spsayakpaul/vision-transformer/frameworks/tensorFlow2/variations/vit-l16-fe/versions/1 +vit_r50_l32_fe,https://www.kaggle.com/models/spsayakpaul/vision-transformer/frameworks/tensorFlow2/variations/vit-r50-l32-fe/versions/1 +imagenet/efficientnet_v2_imagenet21k_ft1k_b2/feature_vector,https://www.kaggle.com/models/google/efficientnet-v2/frameworks/tensorFlow2/variations/imagenet21k-ft1k-b2-feature-vector/versions/1 +tf2-preview/nnlm-ko-dim50,https://www.kaggle.com/models/google/nnlm/frameworks/tensorFlow2/variations/tf2-preview-ko-dim50/versions/1 +wav2vec2-xlsr-53,https://www.kaggle.com/models/kaggle/wav2vec2/frameworks/tensorFlow2/variations/xlsr-53/versions/1 +movinet/a5/stream/kinetics-600/classification,https://www.kaggle.com/models/google/movinet/frameworks/tensorFlow2/variations/a5-stream-kinetics-600-classification/versions/2 +vit_r26_s32_lightaug_fe,https://www.kaggle.com/models/spsayakpaul/vision-transformer/frameworks/tensorFlow2/variations/vit-r26-s32-lightaug-fe/versions/1 +mmt/architecture_image-q-24,https://www.kaggle.com/models/deepmind/mmt/frameworks/tensorFlow1/variations/architecture-image-q-24/versions/1 +wiki40b-lm-nl,https://www.kaggle.com/models/google/wiki40b-lm/frameworks/tensorFlow1/variations/nl/versions/1 +tf2-preview/nnlm-zh-dim50,https://www.kaggle.com/models/google/nnlm/frameworks/tensorFlow2/variations/tf2-preview-zh-dim50/versions/1 +compare_gan/model_9_celebahq128_resnet19,https://www.kaggle.com/models/google/compare-gan/frameworks/tensorFlow1/variations/model-9-celebahq128-resnet19/versions/1 +tf2-preview/nnlm-zh-dim128,https://www.kaggle.com/models/google/nnlm/frameworks/tensorFlow2/variations/tf2-preview-zh-dim128/versions/1 +tweening_conv3d_kth,https://www.kaggle.com/models/google/tweening-conv3d/frameworks/tensorFlow1/variations/kth/versions/1 +LAReQA/mBERT_X_X_mono,https://www.kaggle.com/models/google/lareqa/frameworks/tensorFlow1/variations/mbert-x-x-mono/versions/1 +tf2-preview/nnlm-ja-dim50,https://www.kaggle.com/models/google/nnlm/frameworks/tensorFlow2/variations/tf2-preview-ja-dim50/versions/1 +nnlm-ko-dim50,https://www.kaggle.com/models/google/nnlm/frameworks/tensorFlow2/variations/ko-dim50/versions/1 +compare_gan/model_12_cifar10_resnet_cifar,https://www.kaggle.com/models/google/compare-gan/frameworks/tensorFlow1/variations/model-12-cifar10-resnet-cifar/versions/1 +spiral/default-fluid-gansn-celebahq64-gen-19steps,https://www.kaggle.com/models/deepmind/spiral/frameworks/tensorFlow1/variations/default-fluid-gansn-celebahq64-gen-19steps/versions/1 +compare_gan/s3gan_5_128x128,https://www.kaggle.com/models/google/compare-gan/frameworks/tensorFlow1/variations/s3gan-5-128x128/versions/1 +spiral/default-wgangp-celebahq64-gen-19steps/agent5,https://www.kaggle.com/models/deepmind/spiral/frameworks/tensorFlow1/variations/default-wgangp-celebahq64-gen-19steps-agent5/versions/1 +image_augmentation/crop_rotate_color,https://www.kaggle.com/models/google/image-augmentation/frameworks/tensorFlow1/variations/crop-rotate-color/versions/1 +tf2-preview/nnlm-id-dim50-with-normalization,https://www.kaggle.com/models/google/nnlm/frameworks/tensorFlow2/variations/tf2-preview-id-dim50-with-normalization/versions/1 +tf2-preview/nnlm-es-dim128,https://www.kaggle.com/models/google/nnlm/frameworks/tensorFlow2/variations/tf2-preview-es-dim128/versions/1 +wiki40b-lm-sr,https://www.kaggle.com/models/google/wiki40b-lm/frameworks/tensorFlow1/variations/sr/versions/1 +tf2-preview/nnlm-es-dim50,https://www.kaggle.com/models/google/nnlm/frameworks/tensorFlow2/variations/tf2-preview-es-dim50/versions/1 +wiki40b-lm-ko,https://www.kaggle.com/models/google/wiki40b-lm/frameworks/tensorFlow1/variations/ko/versions/1 +mmt/baseline_baseline,https://www.kaggle.com/models/deepmind/mmt/frameworks/tensorFlow1/variations/baseline-baseline/versions/1 +boundless/three_quarter,https://www.kaggle.com/models/google/boundless/frameworks/tensorFlow1/variations/three-quarter/versions/1 +musiq/koniq-10k,https://www.kaggle.com/models/google/musiq/frameworks/tensorFlow2/variations/koniq-10k/versions/1 +relative-patch-location,https://www.kaggle.com/models/vtab/relative-patch-location/frameworks/tensorFlow1/variations/relative-patch-location/versions/1 +ulmfit/en/sp35k_cased,https://www.kaggle.com/models/edrone/ulmfit/frameworks/tensorFlow2/variations/en-sp35k-cased/versions/1 +sentence-t5/st5-3b,https://www.kaggle.com/models/google/sentence-t5/frameworks/tensorFlow2/variations/st5-3b/versions/1 +bertseq2seq/bert24_en_de,https://www.kaggle.com/models/google/bertseq2seq/frameworks/tensorFlow1/variations/bert24-en-de/versions/1 +vit_b32_fe,https://www.kaggle.com/models/spsayakpaul/vision-transformer/frameworks/tensorFlow2/variations/vit-b32-fe/versions/1 +vit_b32_classification,https://www.kaggle.com/models/spsayakpaul/vision-transformer/frameworks/tensorFlow2/variations/vit-b32-classification/versions/1 +wiki40b-lm-ca,https://www.kaggle.com/models/google/wiki40b-lm/frameworks/tensorFlow1/variations/ca/versions/1 +spiral/default-wgangp-celebahq64-gen-19steps/agent3,https://www.kaggle.com/models/deepmind/spiral/frameworks/tensorFlow1/variations/default-wgangp-celebahq64-gen-19steps-agent3/versions/1 +ulmfit/pl/sp50k_uncased,https://www.kaggle.com/models/edrone/ulmfit/frameworks/tensorFlow2/variations/pl-sp50k-uncased/versions/1 +mixer_b16_i1k_fe,https://www.kaggle.com/models/spsayakpaul/mlp-mixer/frameworks/tensorFlow2/variations/mixer-b16-i1k-fe/versions/1 +mmt/loss_single-modality-contrastive1024,https://www.kaggle.com/models/deepmind/mmt/frameworks/tensorFlow1/variations/loss-single-modality-contrastive1024/versions/1 +wiki40b-lm-tr,https://www.kaggle.com/models/google/wiki40b-lm/frameworks/tensorFlow1/variations/tr/versions/1 +mixer_b16_sam_fe,https://www.kaggle.com/models/spsayakpaul/mlp-mixer/frameworks/tensorFlow2/variations/mixer-b16-sam-fe/versions/1 +logit_reconstruction/robust,https://www.kaggle.com/models/google/logit-reconstruction/frameworks/tensorFlow1/variations/robust/versions/1 +mmt/baseline_baseline-no-bert-transfer,https://www.kaggle.com/models/deepmind/mmt/frameworks/tensorFlow1/variations/baseline-baseline-no-bert-transfer/versions/1 +nnlm-ko-dim128,https://www.kaggle.com/models/google/nnlm/frameworks/tensorFlow2/variations/ko-dim128/versions/1 +vit_l16_classification,https://www.kaggle.com/models/spsayakpaul/vision-transformer/frameworks/tensorFlow2/variations/vit-l16-classification/versions/1 +wiki40b-lm-lt,https://www.kaggle.com/models/google/wiki40b-lm/frameworks/tensorFlow1/variations/lt/versions/1 +wiki40b-lm-pl,https://www.kaggle.com/models/google/wiki40b-lm/frameworks/tensorFlow1/variations/pl/versions/1 +spiral/default-wgangp-celebahq64-gen-19steps/agent8,https://www.kaggle.com/models/deepmind/spiral/frameworks/tensorFlow1/variations/default-wgangp-celebahq64-gen-19steps-agent8/versions/1 +HRNet/ade20k-hrnetv2-w48,https://www.kaggle.com/models/google/hrnet/frameworks/tensorFlow2/variations/hrnet-ade20k-hrnetv2-w48/versions/1 +mmt/architecture_image-q-12,https://www.kaggle.com/models/deepmind/mmt/frameworks/tensorFlow1/variations/architecture-image-q-12/versions/1 +spiral/default-wgangp-celebahq64-gen-19steps/agent9,https://www.kaggle.com/models/deepmind/spiral/frameworks/tensorFlow1/variations/default-wgangp-celebahq64-gen-19steps-agent9/versions/1 +answer_equivalence/bem,https://www.kaggle.com/models/google/bert/frameworks/tensorFlow2/variations/answer-equivalence-bem/versions/1 +wiki40b-lm-id,https://www.kaggle.com/models/google/wiki40b-lm/frameworks/tensorFlow1/variations/id/versions/1 +distilbert_en_cased_preprocess,https://www.kaggle.com/models/jeongukjae/distilbert/frameworks/tensorFlow2/variations/en-cased-preprocess/versions/2 +wiki40b-lm-ar,https://www.kaggle.com/models/google/wiki40b-lm/frameworks/tensorFlow1/variations/ar/versions/1 +compare_gan/model_2_celebahq128_resnet19,https://www.kaggle.com/models/google/compare-gan/frameworks/tensorFlow1/variations/model-2-celebahq128-resnet19/versions/1 +vgg19-block4-conv2-unpooling-decoder,https://www.kaggle.com/models/emilutz/vgg19-unpooling-encoder-decoder/frameworks/tensorFlow2/variations/vgg19-block4-conv2-unpooling-decoder/versions/1 +compare_gan/ssgan_128x128,https://www.kaggle.com/models/google/compare-gan/frameworks/tensorFlow1/variations/ssgan-128x128/versions/1 +spiral/default-wgangp-celebahq64-gen-19steps/agent4,https://www.kaggle.com/models/deepmind/spiral/frameworks/tensorFlow1/variations/default-wgangp-celebahq64-gen-19steps-agent4/versions/1 +image_augmentation/crop_color,https://www.kaggle.com/models/google/image-augmentation/frameworks/tensorFlow1/variations/crop-color/versions/1 +wiki40b-lm-fa,https://www.kaggle.com/models/google/wiki40b-lm/frameworks/tensorFlow1/variations/fa/versions/1 +vit_r26_s32_medaug_classification,https://www.kaggle.com/models/spsayakpaul/vision-transformer/frameworks/tensorFlow2/variations/vit-r26-s32-medaug-classification/versions/1 +compare_gan/model_10_lsun_bedroom_resnet19,https://www.kaggle.com/models/google/compare-gan/frameworks/tensorFlow1/variations/model-10-lsun-bedroom-resnet19/versions/1 +mmt/data_cc,https://www.kaggle.com/models/deepmind/mmt/frameworks/tensorFlow1/variations/data-cc/versions/1 +spiral/default-wgangp-celebahq64-gen-19steps/agent1,https://www.kaggle.com/models/deepmind/spiral/frameworks/tensorFlow1/variations/default-wgangp-celebahq64-gen-19steps-agent1/versions/1 +image_augmentation/nas_imagenet,https://www.kaggle.com/models/google/image-augmentation/frameworks/tensorFlow1/variations/nas-imagenet/versions/1 +mixer_b16_i21k_fe,https://www.kaggle.com/models/spsayakpaul/mlp-mixer/frameworks/tensorFlow2/variations/mixer-b16-i21k-fe/versions/1 +nonsemantic-speech-benchmark/trillsson1,https://www.kaggle.com/models/google/trillsson/frameworks/tensorFlow2/variations/nonsemantic-speech-benchmark-trillsson1/versions/1 +trillsson5,https://www.kaggle.com/models/google/trillsson/frameworks/tensorFlow2/variations/5/versions/1 +logit_reconstruction/inceptionv3,https://www.kaggle.com/models/google/logit-reconstruction/frameworks/tensorFlow1/variations/inceptionv3/versions/1 +spiral/default-wgangp-celebahq64-gen-19steps/agent6,https://www.kaggle.com/models/deepmind/spiral/frameworks/tensorFlow1/variations/default-wgangp-celebahq64-gen-19steps-agent6/versions/1 +tf2-preview/nnlm-id-dim128,https://www.kaggle.com/models/google/nnlm/frameworks/tensorFlow2/variations/tf2-preview-id-dim128/versions/1 +LEALLA/LEALLA-large,https://www.kaggle.com/models/google/lealla/frameworks/tensorFlow2/variations/lealla-large/versions/1 +compare_gan/model_13_cifar10_resnet_cifar,https://www.kaggle.com/models/google/compare-gan/frameworks/tensorFlow1/variations/model-13-cifar10-resnet-cifar/versions/1 +vit_b16_classification,https://www.kaggle.com/models/spsayakpaul/vision-transformer/frameworks/tensorFlow2/variations/vit-b16-classification/versions/1 +compare_gan/model_11_cifar10_resnet_cifar,https://www.kaggle.com/models/google/compare-gan/frameworks/tensorFlow1/variations/model-11-cifar10-resnet-cifar/versions/1 +wiki40b-lm-fi,https://www.kaggle.com/models/google/wiki40b-lm/frameworks/tensorFlow1/variations/fi/versions/1 +vit_r26_s32_medaug_fe,https://www.kaggle.com/models/spsayakpaul/vision-transformer/frameworks/tensorFlow2/variations/vit-r26-s32-medaug-fe/versions/1 +image_augmentation/flipx_crop_rotate_color,https://www.kaggle.com/models/google/image-augmentation/frameworks/tensorFlow1/variations/flipx-crop-rotate-color/versions/1 +circularnet_3,https://www.kaggle.com/models/google/circularnet/frameworks/tensorFlow2/variations/3/versions/1 +random-nnlm-en-dim50,https://www.kaggle.com/models/google/random-nnlm-en/frameworks/tensorFlow1/variations/dim50/versions/1 +mixer_b16_i21k_classification,https://www.kaggle.com/models/spsayakpaul/mlp-mixer/frameworks/tensorFlow2/variations/mixer-b16-i21k-classification/versions/1 +mixer_b16_sam_classification,https://www.kaggle.com/models/spsayakpaul/mlp-mixer/frameworks/tensorFlow2/variations/mixer-b16-sam-classification/versions/1 +wav2vec2-robust,https://www.kaggle.com/models/kaggle/wav2vec2/frameworks/tensorFlow2/variations/robust/versions/1 +wiki40b-lm-lv,https://www.kaggle.com/models/google/wiki40b-lm/frameworks/tensorFlow1/variations/lv/versions/1 +sentence-t5/st5-11b,https://www.kaggle.com/models/google/sentence-t5/frameworks/tensorFlow2/variations/st5-11b/versions/1 +gtr/gtr-xxl,https://www.kaggle.com/models/google/gtr/frameworks/tensorFlow2/variations/gtr-xxl/versions/1 +mmt/data_mscoco,https://www.kaggle.com/models/deepmind/mmt/frameworks/tensorFlow1/variations/data-mscoco/versions/1 +vit_r26_s32_lightaug_classification,https://www.kaggle.com/models/spsayakpaul/vision-transformer/frameworks/tensorFlow2/variations/vit-r26-s32-lightaug-classification/versions/1 +compare_gan/s3gan_10_256x256,https://www.kaggle.com/models/google/compare-gan/frameworks/tensorFlow1/variations/s3gan-10-256x256/versions/1 +mixer_b16_i1k_classification,https://www.kaggle.com/models/spsayakpaul/mlp-mixer/frameworks/tensorFlow2/variations/mixer-b16-i1k-classification/versions/1 +HRNet/msegpcontext-hrnetv2-w48,https://www.kaggle.com/models/google/hrnet/frameworks/tensorFlow2/variations/hrnet-msegpcontext-hrnetv2-w48/versions/1 +mmt/baseline-ft_baseline-no-bert-transfer,https://www.kaggle.com/models/deepmind/mmt/frameworks/tensorFlow1/variations/baseline-ft-baseline-no-bert-transfer/versions/1 +wiki40b-lm-el,https://www.kaggle.com/models/google/wiki40b-lm/frameworks/tensorFlow1/variations/el/versions/1 +wiki40b-lm-ro,https://www.kaggle.com/models/google/wiki40b-lm/frameworks/tensorFlow1/variations/ro/versions/1 +wiki40b-lm-ms,https://www.kaggle.com/models/google/wiki40b-lm/frameworks/tensorFlow1/variations/ms/versions/1 +wiki40b-lm-th,https://www.kaggle.com/models/google/wiki40b-lm/frameworks/tensorFlow1/variations/th/versions/1 +mmt/data_sbu,https://www.kaggle.com/models/deepmind/mmt/frameworks/tensorFlow1/variations/data-sbu/versions/1 +mmt/data_combined-instance,https://www.kaggle.com/models/deepmind/mmt/frameworks/tensorFlow1/variations/data-combined-instance/versions/1 +vila/image,https://www.kaggle.com/models/google/vila/frameworks/tensorFlow2/variations/image/versions/1 +mmt/architecture_vilbert-4block,https://www.kaggle.com/models/deepmind/mmt/frameworks/tensorFlow1/variations/architecture-vilbert-4block/versions/1 +mmt/architecture-ft_language-q-12,https://www.kaggle.com/models/deepmind/mmt/frameworks/tensorFlow1/variations/architecture-ft-language-q-12/versions/1 +mmt/architecture_single-modality,https://www.kaggle.com/models/deepmind/mmt/frameworks/tensorFlow1/variations/architecture-single-modality/versions/1 +compare_gan/model_4_lsun_bedroom_resnet19,https://www.kaggle.com/models/google/compare-gan/frameworks/tensorFlow1/variations/model-4-lsun-bedroom-resnet19/versions/1 +movinet/a3/stream/kinetics-600/classification,https://www.kaggle.com/models/google/movinet/frameworks/tensorFlow2/variations/a3-stream-kinetics-600-classification/versions/2 +mmt/architecture_vilbert-2block,https://www.kaggle.com/models/deepmind/mmt/frameworks/tensorFlow1/variations/architecture-vilbert-2block/versions/1 +jigsaw,https://www.kaggle.com/models/vtab/jigsaw/frameworks/tensorFlow1/variations/jigsaw/versions/1 +tf2-preview/nnlm-zh-dim50-with-normalization,https://www.kaggle.com/models/google/nnlm/frameworks/tensorFlow2/variations/tf2-preview-zh-dim50-with-normalization/versions/1 +wiki40b-lm-de,https://www.kaggle.com/models/google/wiki40b-lm/frameworks/tensorFlow1/variations/de/versions/1 +mixer_b32_sam_fe,https://www.kaggle.com/models/spsayakpaul/mlp-mixer/frameworks/tensorFlow2/variations/mixer-b32-sam-fe/versions/1 +mmt/data-ft_sbu,https://www.kaggle.com/models/deepmind/mmt/frameworks/tensorFlow1/variations/data-ft-sbu/versions/1 +mmt/data_cc-with-bert,https://www.kaggle.com/models/deepmind/mmt/frameworks/tensorFlow1/variations/data-cc-with-bert/versions/1 +tf2-preview/nnlm-id-dim50,https://www.kaggle.com/models/google/nnlm/frameworks/tensorFlow2/variations/tf2-preview-id-dim50/versions/1 +vgg19-block2-conv2-unpooling-decoder,https://www.kaggle.com/models/emilutz/vgg19-unpooling-encoder-decoder/frameworks/tensorFlow2/variations/vgg19-block2-conv2-unpooling-decoder/versions/1 +Wiki-words-500-with-normalization,https://www.kaggle.com/models/google/wiki-words/frameworks/tensorFlow2/variations/500-with-normalization/versions/1 +wiki40b-lm-zh-tw,https://www.kaggle.com/models/google/wiki40b-lm/frameworks/tensorFlow1/variations/zh-tw/versions/1 +centernet/hourglass_512x512,https://www.kaggle.com/models/tensorflow/centernet-hourglass/frameworks/tensorFlow2/variations/512x512/versions/1 +centernet/hourglass_1024x1024,https://www.kaggle.com/models/tensorflow/centernet-hourglass/frameworks/tensorFlow2/variations/1024x1024/versions/1 +centernet/hourglass_512x512_kpts,https://www.kaggle.com/models/tensorflow/centernet-hourglass/frameworks/tensorFlow2/variations/512x512-kpts/versions/1 +bit/s-r152x4,https://www.kaggle.com/models/google/bit/frameworks/tensorFlow2/variations/s-r152x4/versions/1 +small_bert/bert_uncased_L-8_H-128_A-2,https://www.kaggle.com/models/google/bert/frameworks/tensorFlow1/variations/bert-uncased-l-8-h-128-a-2/versions/2 +centernet/resnet50v2_512x512_kpts,https://www.kaggle.com/models/tensorflow/centernet-resnet/frameworks/tensorFlow2/variations/50v2-512x512-kpts/versions/1 +centernet/resnet50v1_fpn_512x512,https://www.kaggle.com/models/tensorflow/centernet-resnet/frameworks/tensorFlow2/variations/50v1-fpn-512x512/versions/1 +centernet/resnet50v1_fpn_512x512_kpts,https://www.kaggle.com/models/tensorflow/centernet-resnet/frameworks/tensorFlow2/variations/50v1-fpn-512x512-kpts/versions/1 +biggan-512,https://www.kaggle.com/models/deepmind/biggan/frameworks/tensorFlow1/variations/512/versions/2 +tiny_video_net/tvn1,https://www.kaggle.com/models/google/tiny-video-net/frameworks/tensorFlow1/variations/tvn1/versions/1 +cond-biggan,https://www.kaggle.com/models/vtab/cond-biggan/frameworks/tensorFlow1/variations/cond-biggan/versions/1 +supcon/resnet_v1_50/imagenet/classification,https://www.kaggle.com/models/google/resnet-v1/frameworks/tensorFlow1/variations/50-imagenet-classification/versions/1 +unsupervised-adversarial-training/cifar10/wrn_106,https://www.kaggle.com/models/deepmind/wrn-106/frameworks/tensorFlow1/variations/unsupervised-adversarial-training-cifar10-wrn-106/versions/1 +supcon/resnet_v1_101/imagenet/classification,https://www.kaggle.com/models/google/resnet-v1/frameworks/tensorFlow1/variations/101-imagenet-classification/versions/1 +uncond-biggan,https://www.kaggle.com/models/vtab/uncond-biggan/frameworks/tensorFlow1/variations/uncond-biggan/versions/1 +tiny_video_net/tvn3,https://www.kaggle.com/models/google/tiny-video-net/frameworks/tensorFlow1/variations/tvn3/versions/1 +supcon/resnet_v1_200/imagenet/classification,https://www.kaggle.com/models/google/resnet-v1/frameworks/tensorFlow1/variations/200-imagenet-classification/versions/1 +llr-pretrain-adv/latents,https://www.kaggle.com/models/deepmind/llr-pretrain-adv/frameworks/tensorFlow1/variations/latents/versions/1 +tiny_video_net/mobile_1,https://www.kaggle.com/models/google/tiny-video-net/frameworks/tensorFlow1/variations/mobile-1/versions/1 diff --git a/tests/model_hub_tests/performance_tests/test_tf_hub_performance_model.py b/tests/model_hub_tests/performance_tests/test_tf_hub_performance_model.py index 08b4c785a2019b..a0d560c3a10b18 100644 --- a/tests/model_hub_tests/performance_tests/test_tf_hub_performance_model.py +++ b/tests/model_hub_tests/performance_tests/test_tf_hub_performance_model.py @@ -5,21 +5,39 @@ import gc import os import shutil +from collections import namedtuple +from enum import Enum import pytest import tensorflow_hub as hub -from models_hub_common.constants import no_clean_cache_dir -from models_hub_common.constants import tf_hub_cache_dir from models_hub_common.test_performance_model import TestModelPerformance -from models_hub_common.utils import get_models_list -from models_hub_common.utils import cleanup_dir +import models_hub_common.utils as utils +import models_hub_common.constants as const + + +Conf = namedtuple("Conf", "runtime_measure_duration runtime_heat_duration") + + +class TestType(Enum): + PRECOMMIT = 0 + NIGHTLY = 1 + + +def get_tests_conf(test_type: TestType) -> Conf: + options = {TestType.NIGHTLY: + Conf(utils.nano_secs(const.nightly_runtime_measure_duration), + utils.nano_secs(const.nigtly_runtime_heat_duration)), + TestType.PRECOMMIT: + Conf(utils.nano_secs(const.precommit_runtime_measure_duration), + utils.nano_secs(const.precommit_runtime_heat_duration))} + return options[test_type] def clean_cache(): - if not os.path.exists(tf_hub_cache_dir): + if not os.path.exists(const.tf_hub_cache_dir): return - for file_name in os.listdir(tf_hub_cache_dir): - file_path = os.path.join(tf_hub_cache_dir, file_name) + for file_name in os.listdir(const.tf_hub_cache_dir): + file_path = os.path.join(const.tf_hub_cache_dir, file_name) try: if os.path.isfile(file_path) or os.path.islink(file_path): os.unlink(file_path) @@ -29,22 +47,41 @@ def clean_cache(): pass +def get_nightly_config_path(config_name): + dir_path = os.path.join(os.path.dirname(os.path.dirname(__file__)), "tensorflow", "model_lists") + return os.path.join(dir_path, config_name) + + +def get_local_config_path(config_name): + return os.path.join(os.path.dirname(__file__), config_name) + + class TestTFPerformanceModel(TestModelPerformance): def load_model(self, model_name, model_link): hub.load(model_link) return hub.resolve(model_link) def teardown_method(self): - if not no_clean_cache_dir: - cleanup_dir(tf_hub_cache_dir) + if not const.no_clean_cache_dir: + utils.cleanup_dir(const.tf_hub_cache_dir) # deallocate memory after each test case gc.collect() @pytest.mark.parametrize("model_name,model_link,mark,reason", - get_models_list(os.path.join(os.path.dirname(__file__), "precommit_models"))) + utils.get_models_list(get_local_config_path("precommit_models"))) @pytest.mark.precommit def test_convert_model_precommit(self, model_name, model_link, mark, reason, ie_device): assert mark is None or mark == 'skip', "Incorrect test case: {}, {}".format(model_name, model_link) if mark == 'skip': pytest.skip(reason) - self.run(model_name, model_link, ie_device) + self.run(model_name, model_link, ie_device, get_tests_conf(TestType.PRECOMMIT)) + + @pytest.mark.parametrize("model_name,model_link,mark,reason", + utils.get_models_list_not_skipped(get_nightly_config_path("nightly_tf_hub"), + get_local_config_path("nightly_models.skip"))) + @pytest.mark.nightly + def test_convert_model_all_models(self, model_name, model_link, mark, reason, ie_device): + assert mark is None or mark == 'skip', "Incorrect test case: {}, {}".format(model_name, model_link) + if mark == 'skip': + pytest.skip(reason) + self.run(model_name, model_link, ie_device, get_tests_conf(TestType.NIGHTLY))