From a819605986287ac13b881448bf5a1b9752de4d2f Mon Sep 17 00:00:00 2001 From: Anastasiia Pnevskaia Date: Mon, 26 Feb 2024 13:41:09 +0100 Subject: [PATCH] [TF FE] Test TF Hugging Face models (#22970) ### Details: - TF Hugging face tests ### Tickets: - 132904 --------- Co-authored-by: Roman Kazantsev --- .github/CODEOWNERS | 2 +- .github/labeler.yml | 4 +- .../workflows/job_pytorch_models_tests.yml | 8 +- ...ts.yml => job_tensorflow_models_tests.yml} | 8 +- .github/workflows/linux.yml | 8 +- .github/workflows/linux_arm64.yml | 8 +- .../src/openvino/frontend/tensorflow/utils.py | 27 +- .../models_hub_common/constants.py | 4 +- .../{tf_hub_tests => pytorch}/conftest.py | 0 .../detectron2_models | 0 .../detectron2_precommit | 0 .../hf_transformers_models | 0 .../{torch_tests => pytorch}/requirements.txt | 0 .../requirements_secondary.txt | 0 .../scripts/process_op_report.py | 0 .../{torch_tests => pytorch}/test_aliked.py | 0 .../test_detectron2.py | 0 .../{torch_tests => pytorch}/test_edsr.py | 0 .../{torch_tests => pytorch}/test_gfpgan.py | 0 .../test_hf_transformers.py | 0 .../test_speech-transformer.py | 0 .../{torch_tests => pytorch}/test_timm.py | 0 .../test_torchbench.py | 0 .../test_torchvision_models.py | 0 .../{torch_tests => pytorch}/test_tpsmm.py | 0 .../{torch_tests => pytorch}/timm_models | 0 .../{torch_tests => pytorch}/torch_utils.py | 0 .../torchbench_models | 0 .../torchvision_models | 0 .../{torch_tests => tensorflow}/conftest.py | 2 +- .../tensorflow/model_lists/nightly_hf | 449 ++++++++++++++++++ .../model_lists/nightly_tf_hub} | 0 .../model_lists/precommit} | 8 +- .../requirements.txt | 2 + .../test_tf_convert_model.py} | 53 ++- .../test_tf_hub_api_notebooks.py | 2 +- .../{tf_hub_tests => tensorflow}/utils.py | 0 37 files changed, 541 insertions(+), 44 deletions(-) rename .github/workflows/{job_tensorflow_hub_models_tests.yml => job_tensorflow_models_tests.yml} (94%) rename tests/model_hub_tests/{tf_hub_tests => pytorch}/conftest.py (100%) rename tests/model_hub_tests/{torch_tests => pytorch}/detectron2_models (100%) rename tests/model_hub_tests/{torch_tests => pytorch}/detectron2_precommit (100%) rename tests/model_hub_tests/{torch_tests => pytorch}/hf_transformers_models (100%) rename tests/model_hub_tests/{torch_tests => pytorch}/requirements.txt (100%) rename tests/model_hub_tests/{torch_tests => pytorch}/requirements_secondary.txt (100%) rename tests/model_hub_tests/{torch_tests => pytorch}/scripts/process_op_report.py (100%) rename tests/model_hub_tests/{torch_tests => pytorch}/test_aliked.py (100%) rename tests/model_hub_tests/{torch_tests => pytorch}/test_detectron2.py (100%) rename tests/model_hub_tests/{torch_tests => pytorch}/test_edsr.py (100%) rename tests/model_hub_tests/{torch_tests => pytorch}/test_gfpgan.py (100%) rename tests/model_hub_tests/{torch_tests => pytorch}/test_hf_transformers.py (100%) rename tests/model_hub_tests/{torch_tests => pytorch}/test_speech-transformer.py (100%) rename tests/model_hub_tests/{torch_tests => pytorch}/test_timm.py (100%) rename tests/model_hub_tests/{torch_tests => pytorch}/test_torchbench.py (100%) rename tests/model_hub_tests/{torch_tests => pytorch}/test_torchvision_models.py (100%) rename tests/model_hub_tests/{torch_tests => pytorch}/test_tpsmm.py (100%) rename tests/model_hub_tests/{torch_tests => pytorch}/timm_models (100%) rename tests/model_hub_tests/{torch_tests => pytorch}/torch_utils.py (100%) rename tests/model_hub_tests/{torch_tests => pytorch}/torchbench_models (100%) rename tests/model_hub_tests/{torch_tests => pytorch}/torchvision_models (100%) rename tests/model_hub_tests/{torch_tests => tensorflow}/conftest.py (97%) create mode 100644 tests/model_hub_tests/tensorflow/model_lists/nightly_hf rename tests/model_hub_tests/{tf_hub_tests/nightly_models => tensorflow/model_lists/nightly_tf_hub} (100%) rename tests/model_hub_tests/{tf_hub_tests/precommit_models => tensorflow/model_lists/precommit} (92%) rename tests/model_hub_tests/{tf_hub_tests => tensorflow}/requirements.txt (71%) rename tests/model_hub_tests/{tf_hub_tests/test_tf_hub_convert_model.py => tensorflow/test_tf_convert_model.py} (77%) rename tests/model_hub_tests/{tf_hub_tests => tensorflow}/test_tf_hub_api_notebooks.py (98%) rename tests/model_hub_tests/{tf_hub_tests => tensorflow}/utils.py (100%) diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index f513c829cbd642..24204b0b8b2568 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -93,7 +93,7 @@ /tests/layer_tests/tensorflow_tests @openvinotoolkit/openvino-tf-frontend-maintainers /tests/layer_tests/jax_tests @openvinotoolkit/openvino-tf-frontend-maintainers /tests/model_hub_tests @openvinotoolkit/openvino-tf-frontend-maintainers -/tests/model_hub_tests/torch_tests @openvinotoolkit/openvino-pytorch-frontend-maintainers +/tests/model_hub_tests/pytorch @openvinotoolkit/openvino-pytorch-frontend-maintainers # Tools: /tools/ @openvinotoolkit/openvino-tools-maintainers diff --git a/.github/labeler.yml b/.github/labeler.yml index 0605e7d879fa9c..4cf3cbbecf1f59 100644 --- a/.github/labeler.yml +++ b/.github/labeler.yml @@ -142,7 +142,7 @@ - 'tests/layer_tests/tensorflow2_keras_tests/**/*' - 'tests/layer_tests/jax_tests/**/*' - any: ['tests/model_hub_tests/**', - '!tests/model_hub_tests/torch_tests/**/*'] + '!tests/model_hub_tests/pytorch/**/*'] 'category: TFL FE': - 'src/frontends/tensorflow_lite/**/*' @@ -156,7 +156,7 @@ - 'tests/layer_tests/py_frontend_tests/test_torch_decoder.py' - 'tests/layer_tests/py_frontend_tests/test_torch_frontend.py' - any: ['tests/model_hub_tests/**', - '!tests/model_hub_tests/tf_hub_tests/**/*'] + '!tests/model_hub_tests/tensorflow/**/*'] 'category: tools': - any: ['tools/**', diff --git a/.github/workflows/job_pytorch_models_tests.yml b/.github/workflows/job_pytorch_models_tests.yml index 1128be9918350d..f43e39c9ce2cab 100644 --- a/.github/workflows/job_pytorch_models_tests.yml +++ b/.github/workflows/job_pytorch_models_tests.yml @@ -111,8 +111,8 @@ jobs: - name: Install PyTorch tests requirements run: | - python3 -m pip install -r ${MODEL_HUB_TESTS_INSTALL_DIR}/torch_tests/requirements.txt - python3 -m pip install -r ${MODEL_HUB_TESTS_INSTALL_DIR}/torch_tests/requirements_secondary.txt + python3 -m pip install -r ${MODEL_HUB_TESTS_INSTALL_DIR}/pytorch/requirements.txt + python3 -m pip install -r ${MODEL_HUB_TESTS_INSTALL_DIR}/pytorch/requirements_secondary.txt echo "Available storage:" df -h env: @@ -121,7 +121,7 @@ jobs: - name: PyTorch Models Tests run: | export PYTHONPATH=${MODEL_HUB_TESTS_INSTALL_DIR}:$PYTHONPATH - python3 -m pytest ${MODEL_HUB_TESTS_INSTALL_DIR}/torch_tests -m ${TYPE} --html=${INSTALL_TEST_DIR}/TEST-torch_model_tests.html --self-contained-html -v + python3 -m pytest ${MODEL_HUB_TESTS_INSTALL_DIR}/pytorch -m ${TYPE} --html=${INSTALL_TEST_DIR}/TEST-torch_model_tests.html --self-contained-html -v env: TYPE: ${{ inputs.event == 'schedule' && 'nightly' || 'precommit'}} TEST_DEVICE: CPU @@ -131,7 +131,7 @@ jobs: - name: Reformat unsupported ops file if: '!cancelled()' run: | - python3 ${MODEL_HUB_TESTS_INSTALL_DIR}/torch_tests/scripts/process_op_report.py ${INSTALL_TEST_DIR}/TEST-torch_unsupported_ops.log + python3 ${MODEL_HUB_TESTS_INSTALL_DIR}/pytorch/scripts/process_op_report.py ${INSTALL_TEST_DIR}/TEST-torch_unsupported_ops.log - name: Available storage after tests run: | diff --git a/.github/workflows/job_tensorflow_hub_models_tests.yml b/.github/workflows/job_tensorflow_models_tests.yml similarity index 94% rename from .github/workflows/job_tensorflow_hub_models_tests.yml rename to .github/workflows/job_tensorflow_models_tests.yml index 787acfe621154a..31b2ea442d3811 100644 --- a/.github/workflows/job_tensorflow_hub_models_tests.yml +++ b/.github/workflows/job_tensorflow_models_tests.yml @@ -1,4 +1,4 @@ -name: TensorFlow Hub Models tests +name: TensorFlow Models tests on: workflow_call: @@ -109,12 +109,12 @@ jobs: python3 -m pip install ${INSTALL_DIR}/openvino_tokenizers-* - name: Install TF Hub tests requirements - run: python3 -m pip install -r ${MODEL_HUB_TESTS_INSTALL_DIR}/tf_hub_tests/requirements.txt + run: python3 -m pip install -r ${MODEL_HUB_TESTS_INSTALL_DIR}/tensorflow/requirements.txt - - name: TensorFlow Hub Tests - TF FE + - name: TensorFlow Models Tests - TF FE run: | export PYTHONPATH=${MODEL_HUB_TESTS_INSTALL_DIR}:$PYTHONPATH - python3 -m pytest ${MODEL_HUB_TESTS_INSTALL_DIR}/tf_hub_tests/ -m ${TYPE} --html=${INSTALL_TEST_DIR}/TEST-tf_hub_tf_fe.html --self-contained-html -v + python3 -m pytest ${MODEL_HUB_TESTS_INSTALL_DIR}/tensorflow/ -m ${TYPE} --html=${INSTALL_TEST_DIR}/TEST-tf_hub_tf_fe.html --self-contained-html -v env: TYPE: ${{ inputs.event == 'schedule' && 'nightly' || 'precommit'}} TEST_DEVICE: CPU diff --git a/.github/workflows/linux.yml b/.github/workflows/linux.yml index 1f1915588a5fa6..e13f626d3f8b0b 100644 --- a/.github/workflows/linux.yml +++ b/.github/workflows/linux.yml @@ -501,12 +501,12 @@ jobs: # image: openvinogithubactions.azurecr.io/dockerhub/ubuntu:20.04 # volumes: # - /mount:/mount - TensorFlow_Hub_Models_Tests: - name: TensorFlow Hub Models tests + TensorFlow_Models_Tests: + name: TensorFlow Models tests if: fromJSON(needs.smart_ci.outputs.affected_components).TF_FE.test || fromJSON(needs.smart_ci.outputs.affected_components).TFL_FE.test needs: [ Build, Smart_CI, Openvino_tokenizers ] - uses: ./.github/workflows/job_tensorflow_hub_models_tests.yml + uses: ./.github/workflows/job_tensorflow_models_tests.yml with: runner: ${{ github.event_name == 'schedule' && 'ubuntu-20.04-16-cores' || 'ubuntu-20.04-8-cores' }} event: ${{ github.event_name }} @@ -755,7 +755,7 @@ jobs: Overall_Status: name: ci/gha_overall_status needs: [Smart_CI, Build, Debian_Packages, Samples, Conformance, ONNX_Runtime, CXX_Unit_Tests, Python_Unit_Tests, - CPU_Functional_Tests, TensorFlow_Hub_Models_Tests, PyTorch_Models_Tests, NVIDIA_Plugin, Openvino_tokenizers] + CPU_Functional_Tests, TensorFlow_Models_Tests, PyTorch_Models_Tests, NVIDIA_Plugin, Openvino_tokenizers] if: ${{ always() }} runs-on: ubuntu-latest steps: diff --git a/.github/workflows/linux_arm64.yml b/.github/workflows/linux_arm64.yml index 7f6ec8a70f7590..2274295a753096 100644 --- a/.github/workflows/linux_arm64.yml +++ b/.github/workflows/linux_arm64.yml @@ -361,13 +361,13 @@ jobs: runner: 'aks-linux-16-cores-arm' image: 'openvinogithubactions.azurecr.io/dockerhub/ubuntu:20.04' - TensorFlow_Hub_Models_Tests: - name: TensorFlow Hub Models tests + TensorFlow_Models_Tests: + name: TensorFlow Models tests if: ${{ 'false' }} # TODO: Enable once the dependencies are ready for arm (no tensorflow-text available for arm from PyPI) # if: fromJSON(needs.smart_ci.outputs.affected_components).TF_FE.test || # fromJSON(needs.smart_ci.outputs.affected_components).TFL_FE.test needs: [ Build, Smart_CI ] - uses: ./.github/workflows/job_tensorflow_hub_models_tests.yml + uses: ./.github/workflows/job_tensorflow_models_tests.yml with: runner: 'aks-linux-16-cores-arm' container: '{"image": "openvinogithubactions.azurecr.io/dockerhub/ubuntu:20.04"}' @@ -387,7 +387,7 @@ jobs: Overall_Status: name: ci/gha_overall_status_linux_arm64 needs: [Smart_CI, Build, Debian_Packages, Samples, ONNX_Runtime, CXX_Unit_Tests, Python_Unit_Tests, CPU_Functional_Tests, - TensorFlow_Hub_Models_Tests, PyTorch_Models_Tests] + TensorFlow_Models_Tests, PyTorch_Models_Tests] if: ${{ always() }} runs-on: ubuntu-latest steps: diff --git a/src/bindings/python/src/openvino/frontend/tensorflow/utils.py b/src/bindings/python/src/openvino/frontend/tensorflow/utils.py index 036c4c6c9a8ae7..982afb377f2884 100644 --- a/src/bindings/python/src/openvino/frontend/tensorflow/utils.py +++ b/src/bindings/python/src/openvino/frontend/tensorflow/utils.py @@ -151,13 +151,28 @@ def get_concrete_func(tf_function, example_input, input_needs_packing, error_mes return concrete_func +def get_signature_from_input(keras_model): + if not hasattr(keras_model, 'input') or getattr(keras_model, 'input') is None: + return None + return getattr(keras_model, 'input') + + +def get_signature_from_input_signature(keras_model): + if not hasattr(keras_model, 'input_signature') or getattr(keras_model, 'input_signature') is None: + return None + return getattr(keras_model, 'input_signature') + + def create_generic_function_from_keras_model(keras_model): import tensorflow as tf - assert isinstance(keras_model, tf.keras.Model), \ - "[TensorFlow Frontend] internal error: the input model must be of Keras model type" - if not hasattr(keras_model, 'input') or getattr(keras_model, 'input') is None: + assert isinstance(keras_model, (tf.keras.Model, tf.Module)), \ + "[TensorFlow Frontend] internal error: the input model must be of tf.keras.Model or tf.Module model type" + + keras_input_signature = get_signature_from_input(keras_model) + if keras_input_signature is None: + keras_input_signature = get_signature_from_input_signature(keras_model) + if keras_input_signature is None: return None - keras_input_signature = getattr(keras_model, 'input') tf_input_signature = None wrapper_function = None if isinstance(keras_input_signature, dict): @@ -218,7 +233,7 @@ def trace_tf_model(model, input_shapes, input_types, example_input): elif isinstance(model, tf.types.experimental.GenericFunction): tf_function = model input_needs_packing = False - elif isinstance(model, tf.keras.Model): + elif isinstance(model, (tf.keras.Model, tf.Module)): tf_function = create_generic_function_from_keras_model(model) if tf_function is not None: input_needs_packing = False @@ -380,7 +395,7 @@ def extract_model_graph(argv): argv["input_model"] = model.graph return True if Version(env_setup["tensorflow"]) >= parse("2.6.0") and isinstance(model, (tf.types.experimental.GenericFunction, - tf.types.experimental.ConcreteFunction)): + tf.types.experimental.ConcreteFunction)): return True if isinstance(model, tf.train.Checkpoint): if isinstance(model.root, tf.keras.Model): diff --git a/tests/model_hub_tests/models_hub_common/constants.py b/tests/model_hub_tests/models_hub_common/constants.py index cf2b92637d59e4..96b70c54bb50cd 100644 --- a/tests/model_hub_tests/models_hub_common/constants.py +++ b/tests/model_hub_tests/models_hub_common/constants.py @@ -16,10 +16,12 @@ ''' runtime_heat_duration = os.environ.get('RUNTIME_HEAT_DURATION', '5') - tf_hub_cache_dir = os.environ.get('TFHUB_CACHE_DIR', os.path.join(tempfile.gettempdir(), "tfhub_modules")) +hf_cache_dir = os.environ.get('HF_HUB_CACHE', + os.path.join(tempfile.gettempdir(), "hugging_face")) os.environ['TFHUB_CACHE_DIR'] = tf_hub_cache_dir +os.environ['HF_HUB_CACHE'] = hf_cache_dir no_clean_cache_dir = False hf_hub_cache_dir = tempfile.gettempdir() diff --git a/tests/model_hub_tests/tf_hub_tests/conftest.py b/tests/model_hub_tests/pytorch/conftest.py similarity index 100% rename from tests/model_hub_tests/tf_hub_tests/conftest.py rename to tests/model_hub_tests/pytorch/conftest.py diff --git a/tests/model_hub_tests/torch_tests/detectron2_models b/tests/model_hub_tests/pytorch/detectron2_models similarity index 100% rename from tests/model_hub_tests/torch_tests/detectron2_models rename to tests/model_hub_tests/pytorch/detectron2_models diff --git a/tests/model_hub_tests/torch_tests/detectron2_precommit b/tests/model_hub_tests/pytorch/detectron2_precommit similarity index 100% rename from tests/model_hub_tests/torch_tests/detectron2_precommit rename to tests/model_hub_tests/pytorch/detectron2_precommit diff --git a/tests/model_hub_tests/torch_tests/hf_transformers_models b/tests/model_hub_tests/pytorch/hf_transformers_models similarity index 100% rename from tests/model_hub_tests/torch_tests/hf_transformers_models rename to tests/model_hub_tests/pytorch/hf_transformers_models diff --git a/tests/model_hub_tests/torch_tests/requirements.txt b/tests/model_hub_tests/pytorch/requirements.txt similarity index 100% rename from tests/model_hub_tests/torch_tests/requirements.txt rename to tests/model_hub_tests/pytorch/requirements.txt diff --git a/tests/model_hub_tests/torch_tests/requirements_secondary.txt b/tests/model_hub_tests/pytorch/requirements_secondary.txt similarity index 100% rename from tests/model_hub_tests/torch_tests/requirements_secondary.txt rename to tests/model_hub_tests/pytorch/requirements_secondary.txt diff --git a/tests/model_hub_tests/torch_tests/scripts/process_op_report.py b/tests/model_hub_tests/pytorch/scripts/process_op_report.py similarity index 100% rename from tests/model_hub_tests/torch_tests/scripts/process_op_report.py rename to tests/model_hub_tests/pytorch/scripts/process_op_report.py diff --git a/tests/model_hub_tests/torch_tests/test_aliked.py b/tests/model_hub_tests/pytorch/test_aliked.py similarity index 100% rename from tests/model_hub_tests/torch_tests/test_aliked.py rename to tests/model_hub_tests/pytorch/test_aliked.py diff --git a/tests/model_hub_tests/torch_tests/test_detectron2.py b/tests/model_hub_tests/pytorch/test_detectron2.py similarity index 100% rename from tests/model_hub_tests/torch_tests/test_detectron2.py rename to tests/model_hub_tests/pytorch/test_detectron2.py diff --git a/tests/model_hub_tests/torch_tests/test_edsr.py b/tests/model_hub_tests/pytorch/test_edsr.py similarity index 100% rename from tests/model_hub_tests/torch_tests/test_edsr.py rename to tests/model_hub_tests/pytorch/test_edsr.py diff --git a/tests/model_hub_tests/torch_tests/test_gfpgan.py b/tests/model_hub_tests/pytorch/test_gfpgan.py similarity index 100% rename from tests/model_hub_tests/torch_tests/test_gfpgan.py rename to tests/model_hub_tests/pytorch/test_gfpgan.py diff --git a/tests/model_hub_tests/torch_tests/test_hf_transformers.py b/tests/model_hub_tests/pytorch/test_hf_transformers.py similarity index 100% rename from tests/model_hub_tests/torch_tests/test_hf_transformers.py rename to tests/model_hub_tests/pytorch/test_hf_transformers.py diff --git a/tests/model_hub_tests/torch_tests/test_speech-transformer.py b/tests/model_hub_tests/pytorch/test_speech-transformer.py similarity index 100% rename from tests/model_hub_tests/torch_tests/test_speech-transformer.py rename to tests/model_hub_tests/pytorch/test_speech-transformer.py diff --git a/tests/model_hub_tests/torch_tests/test_timm.py b/tests/model_hub_tests/pytorch/test_timm.py similarity index 100% rename from tests/model_hub_tests/torch_tests/test_timm.py rename to tests/model_hub_tests/pytorch/test_timm.py diff --git a/tests/model_hub_tests/torch_tests/test_torchbench.py b/tests/model_hub_tests/pytorch/test_torchbench.py similarity index 100% rename from tests/model_hub_tests/torch_tests/test_torchbench.py rename to tests/model_hub_tests/pytorch/test_torchbench.py diff --git a/tests/model_hub_tests/torch_tests/test_torchvision_models.py b/tests/model_hub_tests/pytorch/test_torchvision_models.py similarity index 100% rename from tests/model_hub_tests/torch_tests/test_torchvision_models.py rename to tests/model_hub_tests/pytorch/test_torchvision_models.py diff --git a/tests/model_hub_tests/torch_tests/test_tpsmm.py b/tests/model_hub_tests/pytorch/test_tpsmm.py similarity index 100% rename from tests/model_hub_tests/torch_tests/test_tpsmm.py rename to tests/model_hub_tests/pytorch/test_tpsmm.py diff --git a/tests/model_hub_tests/torch_tests/timm_models b/tests/model_hub_tests/pytorch/timm_models similarity index 100% rename from tests/model_hub_tests/torch_tests/timm_models rename to tests/model_hub_tests/pytorch/timm_models diff --git a/tests/model_hub_tests/torch_tests/torch_utils.py b/tests/model_hub_tests/pytorch/torch_utils.py similarity index 100% rename from tests/model_hub_tests/torch_tests/torch_utils.py rename to tests/model_hub_tests/pytorch/torch_utils.py diff --git a/tests/model_hub_tests/torch_tests/torchbench_models b/tests/model_hub_tests/pytorch/torchbench_models similarity index 100% rename from tests/model_hub_tests/torch_tests/torchbench_models rename to tests/model_hub_tests/pytorch/torchbench_models diff --git a/tests/model_hub_tests/torch_tests/torchvision_models b/tests/model_hub_tests/pytorch/torchvision_models similarity index 100% rename from tests/model_hub_tests/torch_tests/torchvision_models rename to tests/model_hub_tests/pytorch/torchvision_models diff --git a/tests/model_hub_tests/torch_tests/conftest.py b/tests/model_hub_tests/tensorflow/conftest.py similarity index 97% rename from tests/model_hub_tests/torch_tests/conftest.py rename to tests/model_hub_tests/tensorflow/conftest.py index 3f39a5be8c2485..ffd4e563a7031a 100644 --- a/tests/model_hub_tests/torch_tests/conftest.py +++ b/tests/model_hub_tests/tensorflow/conftest.py @@ -9,4 +9,4 @@ def pytest_generate_tests(metafunc): test_gen_attrs_names = list(inspect.signature(get_params).parameters) params = get_params() - metafunc.parametrize(test_gen_attrs_names, params, scope="function") + metafunc.parametrize(test_gen_attrs_names, params, scope="function") \ No newline at end of file diff --git a/tests/model_hub_tests/tensorflow/model_lists/nightly_hf b/tests/model_hub_tests/tensorflow/model_lists/nightly_hf new file mode 100644 index 00000000000000..536a0c39413acc --- /dev/null +++ b/tests/model_hub_tests/tensorflow/model_lists/nightly_hf @@ -0,0 +1,449 @@ +albert/albert-base-v2,hf_transformers +albert/albert-large-v2,hf_transformers +albert/albert-xxlarge-v2,hf_transformers +google-bert/bert-base-cased,hf_transformers +google-bert/bert-base-chinese,hf_transformers +google-bert/bert-base-german-cased,hf_transformers +google-bert/bert-base-multilingual-cased,hf_transformers +google-bert/bert-base-multilingual-uncased,hf_transformers +google-bert/bert-base-uncased,hf_transformers +google-bert/bert-large-cased,hf_transformers +google-bert/bert-large-uncased-whole-word-masking-finetuned-squad,hf_transformers +google-bert/bert-large-uncased-whole-word-masking,hf_transformers +google-bert/bert-large-uncased,hf_transformers +almanach/camembert-base,hf_transformers +distilbert/distilbert-base-cased-distilled-squad,hf_transformers +distilbert/distilbert-base-cased,hf_transformers +distilbert/distilbert-base-multilingual-cased,hf_transformers +distilbert/distilbert-base-uncased-distilled-squad,hf_transformers +distilbert/distilbert-base-uncased-finetuned-sst-2-english,hf_transformers +distilbert/distilbert-base-uncased,hf_transformers +distilbert/distilgpt2,hf_transformers +distilbert/distilroberta-base,hf_transformers +openai-community/gpt2-large,hf_transformers +openai-community/gpt2-medium,hf_transformers +openai-community/gpt2-xl,hf_transformers +openai-community/gpt2,hf_transformers +openai-community/openai-gpt,hf_transformers +openai-community/roberta-base-openai-detector,hf_transformers +FacebookAI/roberta-base,hf_transformers +FacebookAI/roberta-large-mnli,hf_transformers +FacebookAI/roberta-large,hf_transformers +google-t5/t5-11b,hf_transformers +google-t5/t5-3b,hf_transformers +google-t5/t5-base,hf_transformers +google-t5/t5-large,hf_transformers +google-t5/t5-small,hf_transformers +FacebookAI/xlm-roberta-base,hf_transformers +FacebookAI/xlm-roberta-large,hf_transformers +xlnet/xlnet-base-cased,hf_transformers +xlnet/xlnet-large-cased,hf_transformers +AndrewChar/model-QA-5-epoch-RU,hf_transformers +CAMeL-Lab/bert-base-arabic-camelbert-da-sentiment,hf_transformers +CAMeL-Lab/bert-base-arabic-camelbert-da,hf_transformers +Davlan/bert-base-multilingual-cased-ner-hrl,hf_transformers +Davlan/distilbert-base-multilingual-cased-ner-hrl,hf_transformers +Davlan/xlm-roberta-large-ner-hrl,hf_transformers +DeepESP/gpt2-spanish,hf_transformers +EleutherAI/gpt-j-6b,hf_transformers +GroNLP/bert-base-dutch-cased,hf_transformers +Helsinki-NLP/opus-mt-ar-en,hf_transformers +Helsinki-NLP/opus-mt-de-en,hf_transformers +Helsinki-NLP/opus-mt-en-ar,hf_transformers +Helsinki-NLP/opus-mt-en-de,hf_transformers +Helsinki-NLP/opus-mt-en-es,hf_transformers +Helsinki-NLP/opus-mt-en-fr,hf_transformers +Helsinki-NLP/opus-mt-en-hi,hf_transformers +Helsinki-NLP/opus-mt-en-mul,hf_transformers +Helsinki-NLP/opus-mt-en-ru,hf_transformers +Helsinki-NLP/opus-mt-en-zh,hf_transformers +Helsinki-NLP/opus-mt-es-en,hf_transformers +Helsinki-NLP/opus-mt-fr-en,hf_transformers +Helsinki-NLP/opus-mt-ja-en,hf_transformers +Helsinki-NLP/opus-mt-ko-en,hf_transformers +Helsinki-NLP/opus-mt-mul-en,hf_transformers +Helsinki-NLP/opus-mt-pl-en,hf_transformers +Helsinki-NLP/opus-mt-ru-en,hf_transformers +Helsinki-NLP/opus-mt-tr-en,hf_transformers +Helsinki-NLP/opus-mt-zh-en,hf_transformers +HooshvareLab/bert-base-parsbert-uncased,hf_transformers +Jean-Baptiste/roberta-large-ner-english,hf_transformers +KETI-AIR/ke-t5-base,hf_transformers +M-CLIP/M-BERT-Base-ViT-B,hf_transformers +Michau/t5-base-en-generate-headline,hf_transformers +MilaNLProc/feel-it-italian-emotion,hf_transformers +MilaNLProc/feel-it-italian-sentiment,hf_transformers +NbAiLab/nb-bert-base,hf_transformers +NbAiLab/nb-bert-large,hf_transformers +ProsusAI/finbert,hf_transformers +Rostlab/prot_bert_bfd,hf_transformers +SZTAKI-HLT/hubert-base-cc,hf_transformers +Sentdex/GPyT,hf_transformers +Seznam/small-e-czech,hf_transformers +s-nlp/russian_toxicity_classifier,hf_transformers +apanc/russian-inappropriate-messages,hf_transformers +apanc/russian-sensitive-topics,hf_transformers +T-Systems-onsite/cross-en-de-roberta-sentence-transformer,hf_transformers +TsinghuaAI/CPM-Generate,hf_transformers +UBC-NLP/AraT5-base,hf_transformers +UBC-NLP/MARBERT,hf_transformers +Vamsi/T5_Paraphrase_Paws,hf_transformers +alan-turing-institute/mt5-large-finetuned-mnli-xtreme-xnli,hf_transformers +allegro/herbert-base-cased,hf_transformers +allenai/led-base-16384,hf_transformers +allenai/led-large-16384-arxiv,hf_transformers +allenai/led-large-16384,hf_transformers +allenai/longformer-base-4096,hf_transformers +allenai/longformer-large-4096,hf_transformers +allenai/macaw-large,hf_transformers +allenai/specter,hf_transformers +alvaroalon2/biobert_chemical_ner,hf_transformers +amazon/bort,hf_transformers +amberoad/bert-multilingual-passage-reranking-msmarco,hf_transformers +anferico/bert-for-patents,hf_transformers +arpanghoshal/EmoRoBERTa,hf_transformers +asafaya/bert-base-arabic,hf_transformers +asi/gpt-fr-cased-base,hf_transformers +aubmindlab/aragpt2-base,hf_transformers +aubmindlab/bert-base-arabert,hf_transformers +aubmindlab/bert-base-arabertv02,hf_transformers +aubmindlab/bert-base-arabertv2,hf_transformers +bhadresh-savani/bert-base-uncased-emotion,hf_transformers +bhadresh-savani/distilbert-base-uncased-emotion,hf_transformers +bolbolzaban/gpt2-persian,hf_transformers +cahya/bert-base-indonesian-522M,hf_transformers +cambridgeltl/SapBERT-from-PubMedBERT-fulltext,hf_transformers +cardiffnlp/twitter-roberta-base-emoji,hf_transformers +cardiffnlp/twitter-roberta-base-emotion,hf_transformers +cardiffnlp/twitter-roberta-base-hate,hf_transformers +cardiffnlp/twitter-roberta-base-irony,hf_transformers +cardiffnlp/twitter-roberta-base-offensive,hf_transformers +cardiffnlp/twitter-roberta-base-sentiment,hf_transformers +cardiffnlp/twitter-roberta-base,hf_transformers +cardiffnlp/twitter-xlm-roberta-base-sentiment,hf_transformers +cardiffnlp/twitter-xlm-roberta-base,hf_transformers +chkla/roberta-argument,hf_transformers +tohoku-nlp/bert-base-japanese-v2,hf_transformers +tohoku-nlp/bert-base-japanese-whole-word-masking,hf_transformers +tohoku-nlp/bert-base-japanese,hf_transformers +clips/mfaq,hf_sentence-transformers +cmarkea/distilcamembert-base-ner,hf_transformers +cmarkea/distilcamembert-base-sentiment,hf_transformers +cmarkea/distilcamembert-base,hf_transformers +cointegrated/LaBSE-en-ru,hf_transformers +d4data/bias-detection-model,hf_transformers +dangvantuan/sentence-camembert-large,hf_transformers +datificate/gpt2-small-spanish,hf_transformers +dbmdz/bert-base-german-cased,hf_transformers +dbmdz/bert-base-german-uncased,hf_transformers +dbmdz/bert-base-italian-cased,hf_transformers +dbmdz/bert-base-italian-xxl-cased,hf_transformers +dbmdz/bert-base-turkish-128k-cased,hf_transformers +dbmdz/bert-base-turkish-128k-uncased,hf_transformers +dbmdz/bert-base-turkish-cased,hf_transformers +dbmdz/bert-base-turkish-uncased,hf_transformers +dbmdz/bert-large-cased-finetuned-conll03-english,hf_transformers +dbmdz/german-gpt2,hf_transformers +dccuchile/bert-base-spanish-wwm-cased,hf_transformers +dccuchile/bert-base-spanish-wwm-uncased,hf_transformers +deep-learning-analytics/GrammarCorrector,hf_transformers +deepset/bert-large-uncased-whole-word-masking-squad2,hf_transformers +deepset/gbert-base,hf_transformers +deepset/gbert-large,hf_transformers +deepset/gelectra-base-germanquad,hf_transformers +deepset/gelectra-large-germanquad,hf_transformers +deepset/gelectra-large,hf_transformers +deepset/roberta-base-squad2,hf_transformers +dslim/bert-base-NER-uncased,hf_transformers +dslim/bert-base-NER,hf_transformers +dslim/bert-large-NER,hf_transformers +emilyalsentzer/Bio_ClinicalBERT,hf_transformers +AgentPublic/camembert-base-squadFR-fquad-piaf,hf_transformers +facebook/bart-base,hf_transformers +facebook/bart-large-cnn,hf_transformers +facebook/bart-large-xsum,hf_transformers +facebook/bart-large,hf_transformers +facebook/blenderbot-400M-distill,hf_transformers +facebook/blenderbot_small-90M,hf_transformers +facebook/convnext-large-224,hf_transformers +facebook/convnext-tiny-224,hf_transformers +facebook/deit-base-distilled-patch16-224,hf_transformers +facebook/dino-vitb16,hf_transformers +facebook/dpr-ctx_encoder-single-nq-base,hf_transformers +facebook/dpr-question_encoder-single-nq-base,hf_transformers +facebook/hubert-base-ls960,hf_transformers +facebook/hubert-large-ll60k,hf_transformers +facebook/hubert-large-ls960-ft,hf_transformers +facebook/mbart-large-50-many-to-many-mmt,hf_transformers +facebook/mbart-large-50-many-to-one-mmt,hf_transformers +facebook/mbart-large-50-one-to-many-mmt,hf_transformers +facebook/mbart-large-50,hf_transformers +facebook/mbart-large-cc25,hf_transformers +facebook/rag-sequence-nq,hf_transformers +facebook/rag-token-nq,hf_transformers +facebook/s2t-small-librispeech-asr,hf_transformers +facebook/vit-mae-base,hf_transformers +facebook/wav2vec2-base-960h,hf_transformers +facebook/wav2vec2-large-960h-lv60-self,hf_transformers +facebook/xglm-1.7B,hf_transformers +facebook/xglm-564M,hf_transformers +finiteautomata/bertweet-base-sentiment-analysis,hf_transformers +flax-community/t5-recipe-generation,hf_transformers +flexudy/t5-base-multi-sentence-doctor,hf_transformers +flexudy/t5-small-wav2vec2-grammar-fixer,hf_transformers +gerulata/slovakbert,hf_transformers +google/byt5-base,hf_transformers +google/byt5-small,hf_transformers +google/byt5-xxl,hf_transformers +google/electra-base-discriminator,hf_transformers +google/electra-small-discriminator,hf_transformers +google/mobilebert-uncased,hf_transformers +google/mt5-base,hf_transformers +google/mt5-large,hf_transformers +google/mt5-small,hf_transformers +google/mt5-xl,hf_transformers +google/mt5-xxl,hf_transformers +google/muril-base-cased,hf_transformers +google/pegasus-large,hf_transformers +google/pegasus-xsum,hf_transformers +google/rembert,hf_transformers +google/t5-11b-ssm-tqa,hf_transformers +google/t5-base-lm-adapt,hf_transformers +google/t5-efficient-tiny,hf_transformers +google/t5-efficient-xxl,hf_transformers +google/t5-v1_1-base,hf_transformers +google/t5-v1_1-large,hf_transformers +google/t5-v1_1-small,hf_transformers +google/t5-v1_1-xl,hf_transformers +google/t5-v1_1-xxl,hf_transformers +google/t5-xl-lm-adapt,hf_transformers +google/tapas-base-finetuned-wtq,hf_transformers +google/tapas-large-finetuned-wtq,hf_transformers +google/vit-base-patch16-224-in21k,hf_transformers +google/vit-base-patch16-224,hf_transformers +google/vit-base-patch16-384,hf_transformers +google/vit-base-patch32-224-in21k,hf_transformers +google/vit-base-patch32-384,hf_transformers +google/vit-large-patch16-224-in21k,hf_transformers +google/vit-large-patch16-224,hf_transformers +gsarti/it5-base,hf_transformers +hatmimoha/arabic-ner,hf_transformers +hfl/chinese-bert-wwm-ext,hf_transformers +hfl/chinese-bert-wwm,hf_transformers +hfl/chinese-electra-180g-small-discriminator,hf_transformers +hfl/chinese-macbert-base,hf_transformers +hfl/chinese-macbert-large,hf_transformers +hfl/chinese-pert-base,hf_transformers +hfl/chinese-roberta-wwm-ext-large,hf_transformers +hfl/chinese-roberta-wwm-ext,hf_transformers +hfl/chinese-xlnet-base,hf_transformers +hfl/rbt3,hf_transformers +huggingface/CodeBERTa-language-id,hf_transformers +huggingface/CodeBERTa-small-v1,hf_transformers +human-centered-summarization/financial-summarization-pegasus,hf_transformers +iarfmoose/t5-base-question-generator,hf_transformers +indigo-ai/BERTino,hf_transformers +indobenchmark/indobert-base-p1,hf_transformers +j-hartmann/emotion-english-distilroberta-base,hf_transformers +jhgan/ko-sroberta-multitask,hf_sentence-transformers +joeddav/distilbert-base-uncased-go-emotions-student,hf_transformers +joeddav/xlm-roberta-large-xnli,hf_transformers +jplu/tf-xlm-r-ner-40-lang,hf_transformers +knkarthick/MEETING-SUMMARY-BART-LARGE-XSUM-SAMSUM-DIALOGSUM-AMI,hf_transformers +knkarthick/MEETING_SUMMARY,hf_transformers +kykim/bert-kor-base,hf_transformers +m-newhauser/distilbert-political-tweets,hf_transformers +m3hrdadfi/wav2vec2-large-xlsr-persian-v3,hf_transformers +microsoft/CodeGPT-small-java-adaptedGPT2,hf_transformers +microsoft/CodeGPT-small-java,hf_transformers +microsoft/CodeGPT-small-py,hf_transformers +microsoft/DialoGPT-large,hf_transformers +microsoft/DialoGPT-medium,hf_transformers +microsoft/DialoGPT-small,hf_transformers +microsoft/MiniLM-L12-H384-uncased,hf_transformers +microsoft/Multilingual-MiniLM-L12-H384,hf_transformers +microsoft/codebert-base-mlm,hf_transformers +microsoft/codebert-base,hf_transformers +microsoft/deberta-base,hf_transformers +microsoft/deberta-large,hf_transformers +microsoft/deberta-v2-xlarge,hf_transformers +microsoft/deberta-v2-xxlarge,hf_transformers +microsoft/deberta-v3-base,hf_transformers +microsoft/deberta-v3-large,hf_transformers +microsoft/deberta-v3-small,hf_transformers +microsoft/deberta-v3-xsmall,hf_transformers +microsoft/deberta-xlarge-mnli,hf_transformers +microsoft/graphcodebert-base,hf_transformers +microsoft/layoutlm-base-uncased,hf_transformers +microsoft/mdeberta-v3-base,hf_transformers +microsoft/mpnet-base,hf_transformers +microsoft/swin-base-patch4-window7-224-in22k,hf_transformers +microsoft/swin-tiny-patch4-window7-224,hf_transformers +microsoft/xtremedistil-l12-h384-uncased,hf_transformers +microsoft/xtremedistil-l6-h256-uncased,hf_transformers +microsoft/xtremedistil-l6-h384-uncased,hf_transformers +ml6team/bert-base-uncased-city-country-ner,hf_transformers +monsoon-nlp/hindi-bert,hf_transformers +mrm8488/longformer-base-4096-finetuned-squadv2,hf_transformers +mrm8488/t5-base-finetuned-question-generation-ap,hf_transformers +mymusise/gpt2-medium-chinese,hf_transformers +neuralmind/bert-base-portuguese-cased,hf_transformers +nlpaueb/bert-base-greek-uncased-v1,hf_transformers +nlpaueb/bert-base-uncased-contracts,hf_transformers +nlpaueb/legal-bert-base-uncased,hf_transformers +nlpaueb/legal-bert-small-uncased,hf_transformers +nlpaueb/sec-bert-base,hf_transformers +nlpaueb/sec-bert-shape,hf_transformers +nlptown/bert-base-multilingual-uncased-sentiment,hf_transformers +nvidia/mit-b0,hf_transformers +nvidia/segformer-b0-finetuned-ade-512-512,hf_transformers +nvidia/segformer-b5-finetuned-ade-640-640,hf_transformers +nvidia/segformer-b5-finetuned-cityscapes-1024-1024,hf_transformers +oliverguhr/fullstop-punctuation-multilang-large,hf_transformers +oliverguhr/german-sentiment-bert,hf_transformers +onlplab/alephbert-base,hf_transformers +openai/clip-vit-base-patch32,hf_transformers +openai/clip-vit-large-patch14,hf_transformers +papluca/xlm-roberta-base-language-detection,hf_transformers +patrickvonplaten/led-large-16384-pubmed,hf_transformers +pdelobelle/robbert-v2-dutch-base,hf_transformers +pierreguillou/bert-base-cased-squad-v1.1-portuguese,hf_transformers +pierreguillou/bert-large-cased-squad-v1.1-portuguese,hf_transformers +pierreguillou/gpt2-small-portuguese,hf_transformers +pucpr/biobertpt-all,hf_transformers +rasa/LaBSE,hf_transformers +razent/SciFive-large-Pubmed_PMC,hf_transformers +rinna/japanese-gpt2-medium,hf_transformers +rinna/japanese-gpt2-small,hf_transformers +rinna/japanese-gpt2-xsmall,hf_transformers +rinna/japanese-roberta-base,hf_transformers +rsvp-ai/bertserini-bert-base-squad,hf_transformers +sagorsarker/bangla-bert-base,hf_transformers +salti/bert-base-multilingual-cased-finetuned-squad,hf_transformers +sampathkethineedi/industry-classification,hf_transformers +samrawal/bert-base-uncased_clinical-ner,hf_transformers +ai-forever/sbert_large_mt_nlu_ru,hf_transformers +sentence-transformers/LaBSE,hf_sentence-transformers +sentence-transformers/all-MiniLM-L6-v2,hf_sentence-transformers +sentence-transformers/bert-base-nli-mean-tokens,hf_sentence-transformers +sentence-transformers/clip-ViT-B-32-multilingual-v1,hf_sentence-transformers +sentence-transformers/distiluse-base-multilingual-cased-v1,hf_sentence-transformers +sentence-transformers/distiluse-base-multilingual-cased-v2,hf_sentence-transformers +sentence-transformers/distiluse-base-multilingual-cased,hf_sentence-transformers +sentence-transformers/msmarco-MiniLM-L-12-v3,hf_sentence-transformers +sentence-transformers/msmarco-MiniLM-L-6-v3,hf_sentence-transformers +sentence-transformers/msmarco-bert-base-dot-v5,hf_sentence-transformers +sentence-transformers/msmarco-distilbert-base-tas-b,hf_sentence-transformers +sentence-transformers/multi-qa-MiniLM-L6-cos-v1,hf_sentence-transformers +sentence-transformers/multi-qa-MiniLM-L6-dot-v1,hf_sentence-transformers +sentence-transformers/paraphrase-MiniLM-L3-v2,hf_sentence-transformers +sentence-transformers/paraphrase-MiniLM-L6-v2,hf_sentence-transformers +sentence-transformers/paraphrase-mpnet-base-v2,hf_sentence-transformers +sentence-transformers/paraphrase-multilingual-MiniLM-L12-v2,hf_sentence-transformers +sentence-transformers/paraphrase-multilingual-mpnet-base-v2,hf_sentence-transformers +sentence-transformers/paraphrase-xlm-r-multilingual-v1,hf_sentence-transformers +sentence-transformers/stsb-xlm-r-multilingual,hf_sentence-transformers +setu4993/LaBSE,hf_transformers +setu4993/smaller-LaBSE,hf_transformers +siebert/sentiment-roberta-large-english,hf_transformers +smanjil/German-MedBERT,hf_transformers +sshleifer/tiny-gpt2,hf_transformers +typeform/distilbert-base-uncased-mnli,hf_transformers +uer/albert-base-chinese-cluecorpussmall,hf_transformers +uer/bart-base-chinese-cluecorpussmall,hf_transformers +uer/gpt2-chinese-ancient,hf_transformers +uer/gpt2-chinese-cluecorpussmall,hf_transformers +uer/gpt2-chinese-lyric,hf_transformers +uer/gpt2-chinese-poem,hf_transformers +uer/gpt2-distil-chinese-cluecorpussmall,hf_transformers +uer/roberta-base-chinese-extractive-qa,hf_transformers +uer/roberta-base-finetuned-chinanews-chinese,hf_transformers +uer/roberta-base-finetuned-cluener2020-chinese,hf_transformers +uer/roberta-base-finetuned-dianping-chinese,hf_transformers +uer/roberta-base-finetuned-jd-binary-chinese,hf_transformers +uer/t5-base-chinese-cluecorpussmall,hf_transformers +uer/t5-small-chinese-cluecorpussmall,hf_transformers +uer/t5-v1_1-base-chinese-cluecorpussmall,hf_transformers +ugaray96/biobert_ncbi_disease_ner,hf_transformers +unicamp-dl/ptt5-base-portuguese-vocab,hf_transformers +valhalla/longformer-base-4096-finetuned-squadv1,hf_transformers +vinai/bertweet-base,hf_transformers +vinai/phobert-base,hf_transformers +w11wo/indonesian-roberta-base-sentiment-classifier,hf_transformers +ydshieh/vit-gpt2-coco-en,hf_transformers +yiyanghkust/finbert-tone,hf_transformers +casehold/legalbert,hf_transformers +allenai/PRIMERA,hf_transformers +cardiffnlp/twitter-roberta-base-sentiment-latest,hf_transformers +microsoft/resnet-18,hf_transformers +microsoft/resnet-50,hf_transformers +MaRiOrOsSi/t5-base-finetuned-question-answering,hf_transformers +Helsinki-NLP/opus-mt-tc-big-en-ar,hf_transformers +Helsinki-NLP/opus-mt-tc-big-en-pt,hf_transformers +Helsinki-NLP/opus-mt-tc-big-en-tr,hf_transformers +Helsinki-NLP/opus-mt-tc-big-ar-en,hf_transformers +sentence-transformers/use-cmlm-multilingual,hf_sentence-transformers +microsoft/layoutlmv3-base,hf_transformers +microsoft/layoutlmv3-large,hf_transformers +openai/clip-vit-large-patch14-336,hf_transformers +hfl/chinese-pert-base-mrc,hf_transformers +facebook/opt-125m,hf_transformers +facebook/opt-350m,hf_transformers +facebook/opt-1.3b,hf_transformers +facebook/opt-2.7b,hf_transformers +facebook/opt-6.7b,hf_transformers +facebook/opt-13b,hf_transformers +facebook/opt-30b,hf_transformers +eleldar/language-detection,hf_transformers +apple/mobilevit-small,hf_transformers +M-CLIP/XLM-Roberta-Large-Vit-B-16Plus,hf_transformers +cardiffnlp/tweet-topic-21-multi,hf_transformers +KB/bert-base-swedish-cased,hf_transformers +facebook/mgenre-wiki,hf_transformers +facebook/opt-66b,hf_transformers +jonaskoenig/topic_classification_04,hf_transformers +impira/layoutlm-document-qa,hf_transformers +abeja/gpt2-large-japanese,hf_transformers +openai/whisper-tiny,hf_transformers +openai/whisper-base,hf_transformers +openai/whisper-small,hf_transformers +openai/whisper-medium,hf_transformers +openai/whisper-large,hf_transformers +openai/whisper-tiny.en,hf_transformers +openai/whisper-base.en,hf_transformers +openai/whisper-small.en,hf_transformers +openai/whisper-medium.en,hf_transformers +facebook/esm2_t33_650M_UR50D,hf_transformers +VietAI/envit5-translation,hf_transformers +facebook/esm2_t36_3B_UR50D,hf_transformers +facebook/esm1b_t33_650M_UR50S,hf_transformers +google/flan-t5-small,hf_transformers +google/flan-t5-base,hf_transformers +google/flan-t5-large,hf_transformers +google/flan-t5-xl,hf_transformers +google/flan-t5-xxl,hf_transformers +hfl/chinese-lert-base,hf_transformers +hfl/chinese-lert-large,hf_transformers +openai/whisper-large-v2,hf_transformers +Salesforce/blip-image-captioning-base,hf_transformers +Salesforce/blip-vqa-base,hf_transformers +Salesforce/blip-image-captioning-large,hf_transformers +Salesforce/blip-vqa-capfilt-large,hf_transformers +jordiclive/flan-t5-3b-summarizer,hf_transformers +merve/chatgpt-prompts-bart-long,hf_transformers +hyperonym/xlm-roberta-longformer-base-16384,hf_transformers +almanach/camembert-bio-base,hf_transformers +cardiffnlp/twitter-roberta-base-emotion-multilabel-latest,hf_transformers +merve/chatgpt-prompt-generator-v12,hf_transformers +Yueh-Huan/news-category-classification-distilbert,hf_transformers +Kaludi/chatgpt-gpt4-prompts-bart-large-cnn-samsum,hf_transformers +cardiffnlp/twitter-roberta-base-hate-latest,hf_transformers +InstaDeepAI/nucleotide-transformer-2.5b-multi-species,hf_transformers +facebook/sam-vit-huge,hf_transformers +facebook/sam-vit-base,hf_transformers +facebook/sam-vit-large,hf_transformers +Blaxzter/LaBSE-sentence-embeddings,hf_transformers +tohoku-nlp/bert-base-japanese-v3,hf_transformers +intfloat/e5-small-v2,hf_sentence-transformers +NbAiLab/nb-whisper-small-beta,hf_transformers +SnypzZz/Llama2-13b-Language-translate,hf_transformers \ No newline at end of file diff --git a/tests/model_hub_tests/tf_hub_tests/nightly_models b/tests/model_hub_tests/tensorflow/model_lists/nightly_tf_hub similarity index 100% rename from tests/model_hub_tests/tf_hub_tests/nightly_models rename to tests/model_hub_tests/tensorflow/model_lists/nightly_tf_hub diff --git a/tests/model_hub_tests/tf_hub_tests/precommit_models b/tests/model_hub_tests/tensorflow/model_lists/precommit similarity index 92% rename from tests/model_hub_tests/tf_hub_tests/precommit_models rename to tests/model_hub_tests/tensorflow/model_lists/precommit index d3d9fbe232b15a..9ff4eccdc3e6ee 100644 --- a/tests/model_hub_tests/tf_hub_tests/precommit_models +++ b/tests/model_hub_tests/tensorflow/model_lists/precommit @@ -25,4 +25,10 @@ planet/vision/classifier/planet_v2,https://www.kaggle.com/models/google/planet-v # TF1 models in .pb format i3d-rgb,https://storage.openvinotoolkit.org/repositories/open_model_zoo/public/2022.1/i3d-rgb-tf/rgb.frozen.pb # Model with SentencePiece tokenizer, use openvino-tokenizers package -universal-sentence-encoder-multilingual,https://www.kaggle.com/models/google/universal-sentence-encoder/frameworks/tensorFlow2/variations/multilingual/versions/2 \ No newline at end of file +universal-sentence-encoder-multilingual,https://www.kaggle.com/models/google/universal-sentence-encoder/frameworks/tensorFlow2/variations/multilingual/versions/2 +# Hugging Face Models +microsoft/deberta-v3-small,hf_transformers +facebook/convnext-tiny-224,hf_transformers +google/vit-base-patch16-224,hf_transformers +sentence-transformers/paraphrase-mpnet-base-v2,hf_transformers +facebook/sam-vit-base,hf_transformers \ No newline at end of file diff --git a/tests/model_hub_tests/tf_hub_tests/requirements.txt b/tests/model_hub_tests/tensorflow/requirements.txt similarity index 71% rename from tests/model_hub_tests/tf_hub_tests/requirements.txt rename to tests/model_hub_tests/tensorflow/requirements.txt index e793935ddd6047..5f3fcef7870967 100644 --- a/tests/model_hub_tests/tf_hub_tests/requirements.txt +++ b/tests/model_hub_tests/tensorflow/requirements.txt @@ -5,3 +5,5 @@ pytest-html tensorflow tensorflow-hub tensorflow-text +transformers +tf-sentence-transformers \ No newline at end of file diff --git a/tests/model_hub_tests/tf_hub_tests/test_tf_hub_convert_model.py b/tests/model_hub_tests/tensorflow/test_tf_convert_model.py similarity index 77% rename from tests/model_hub_tests/tf_hub_tests/test_tf_hub_convert_model.py rename to tests/model_hub_tests/tensorflow/test_tf_convert_model.py index 8345ad8981ac3b..1537a0425fb461 100644 --- a/tests/model_hub_tests/tf_hub_tests/test_tf_hub_convert_model.py +++ b/tests/model_hub_tests/tensorflow/test_tf_convert_model.py @@ -13,17 +13,30 @@ import tensorflow_hub as hub # noinspection PyUnresolvedReferences import tensorflow_text # do not delete, needed for text models -from models_hub_common.constants import tf_hub_cache_dir + +from models_hub_common.constants import tf_hub_cache_dir, hf_cache_dir from models_hub_common.test_convert_model import TestConvertModel from models_hub_common.utils import get_models_list -from tf_hub_tests.utils import type_map, load_graph, get_input_signature, get_output_signature +from utils import type_map, load_graph, get_input_signature, get_output_signature + + +def is_hf_link(link: str): + return link.startswith("hf_") class TestTFHubConvertModel(TestConvertModel): def setup_class(self): self.model_dir = tempfile.TemporaryDirectory() - def load_model(self, model_name, model_link): + def load_model(self, model_name, model_link: str): + if is_hf_link(model_link): + library_type = model_link[3:] + if library_type == "transformers": + from transformers import TFAutoModel + return TFAutoModel.from_pretrained(model_name) + elif library_type == "sentence-transformers": + from tf_sentence_transformers import SentenceTransformer + return SentenceTransformer.from_pretrained(model_name) if 'storage.openvinotoolkit.org' in model_link: # this models is from public OpenVINO storage subprocess.check_call(["wget", "-nv", model_link], cwd=self.model_dir.name) @@ -53,12 +66,14 @@ def load_model(self, model_name, model_link): def get_inputs_info(self, model_obj): if type(model_obj) is tf_v1.Graph: input_signature = get_input_signature(model_obj) + elif hasattr(model_obj, "input_signature") and model_obj.input_signature is not None: + input_signature = model_obj.input_signature else: assert len(model_obj.structured_input_signature) > 1, "incorrect model or test issue" input_signature = model_obj.structured_input_signature[1].items() inputs_info = [] - for input_name, input_info in input_signature: + for input_name, input_info in (input_signature.items() if isinstance(input_signature, dict) else input_signature): input_shape = [] try: if input_info.shape.as_list() == [None, None, None, 3] and input_info.dtype == tf.float32: @@ -115,22 +130,30 @@ def infer_fw_model(self, model_obj, inputs): return output_dict + def clean_dir(self, dir_name: str): + if os.path.exists(dir_name): + for file_name in os.listdir(dir_name): + file_path = os.path.join(dir_name, file_name) + try: + if os.path.isfile(file_path) or os.path.islink(file_path): + os.unlink(file_path) + elif os.path.isdir(file_path): + shutil.rmtree(file_path) + except Exception as e: + pass + def teardown_method(self): # remove all downloaded files for TF Hub models - for file_name in os.listdir(tf_hub_cache_dir): - file_path = os.path.join(tf_hub_cache_dir, file_name) - try: - if os.path.isfile(file_path) or os.path.islink(file_path): - os.unlink(file_path) - elif os.path.isdir(file_path): - shutil.rmtree(file_path) - except Exception as e: - pass + self.clean_dir(tf_hub_cache_dir) + + # remove all downloaded files for HF models + self.clean_dir(hf_cache_dir) + # deallocate memory after each test case gc.collect() @pytest.mark.parametrize("model_name,model_link,mark,reason", - get_models_list(os.path.join(os.path.dirname(__file__), "precommit_models"))) + get_models_list(os.path.join(os.path.dirname(__file__), "model_lists", "precommit"))) @pytest.mark.precommit def test_convert_model_precommit(self, model_name, model_link, mark, reason, ie_device): assert mark is None or mark == 'skip' or mark == 'xfail', \ @@ -142,7 +165,7 @@ def test_convert_model_precommit(self, model_name, model_link, mark, reason, ie_ self.run(model_name, model_link, ie_device) @pytest.mark.parametrize("model_name,model_link,mark,reason", - get_models_list(os.path.join(os.path.dirname(__file__), "nightly_models"))) + get_models_list(os.path.join(os.path.dirname(__file__), "model_lists", "nightly_tf_hub"))) @pytest.mark.nightly def test_convert_model_all_models(self, model_name, model_link, mark, reason, ie_device): assert mark is None or mark == 'skip' or mark == 'xfail', \ diff --git a/tests/model_hub_tests/tf_hub_tests/test_tf_hub_api_notebooks.py b/tests/model_hub_tests/tensorflow/test_tf_hub_api_notebooks.py similarity index 98% rename from tests/model_hub_tests/tf_hub_tests/test_tf_hub_api_notebooks.py rename to tests/model_hub_tests/tensorflow/test_tf_hub_api_notebooks.py index 80b31329ed38d3..3657b5b5416949 100644 --- a/tests/model_hub_tests/tf_hub_tests/test_tf_hub_api_notebooks.py +++ b/tests/model_hub_tests/tensorflow/test_tf_hub_api_notebooks.py @@ -5,7 +5,7 @@ import tensorflow as tf import tensorflow_hub as hub from models_hub_common.test_convert_model import TestConvertModel -from tf_hub_tests.utils import get_input_info +from utils import get_input_info class TestTFHubApiNotebooks(TestConvertModel): diff --git a/tests/model_hub_tests/tf_hub_tests/utils.py b/tests/model_hub_tests/tensorflow/utils.py similarity index 100% rename from tests/model_hub_tests/tf_hub_tests/utils.py rename to tests/model_hub_tests/tensorflow/utils.py