diff --git a/.github/workflows/job_pytorch_models_tests.yml b/.github/workflows/job_pytorch_models_tests.yml index c39ec81467eb75..c740cd89079ec2 100644 --- a/.github/workflows/job_pytorch_models_tests.yml +++ b/.github/workflows/job_pytorch_models_tests.yml @@ -12,8 +12,8 @@ on: type: string required: false default: '{"image": null}' - event: - description: 'Event that triggered the workflow. E.g., "schedule" for nightly runs' + model_scope: + description: 'Scope of models for testing.' type: string required: true @@ -22,7 +22,7 @@ permissions: read-all jobs: PyTorch_Models_Tests: name: PyTorch Models tests - timeout-minutes: ${{ inputs.event == 'schedule' && 400 || 30 }} + timeout-minutes: ${{ inputs.model_scope == 'precommit' && 30 || 400 }} runs-on: ${{ inputs.runner }} container: ${{ fromJSON(inputs.container) }} defaults: @@ -113,7 +113,15 @@ jobs: python3 -m pip install ${INSTALL_DIR}/tools/openvino-* python3 -m pip install ${INSTALL_DIR}/openvino_tokenizers-* - - name: Install PyTorch tests requirements + - name: Install PyTorch tests requirements for precommit + if: ${{ inputs.model_scope == 'precommit' }} + run: | + python3 -m pip install -r ${MODEL_HUB_TESTS_INSTALL_DIR}/pytorch/requirements.txt + env: + CPLUS_INCLUDE_PATH: ${{ env.Python_ROOT_DIR }}/include/python${{ env.PYTHON_VERSION }} + + - name: Install PyTorch tests requirements for nightly + if: ${{ inputs.model_scope != 'precommit' }} run: | python3 -m pip install -r ${MODEL_HUB_TESTS_INSTALL_DIR}/pytorch/requirements.txt python3 -m pip install -r ${MODEL_HUB_TESTS_INSTALL_DIR}/pytorch/requirements_secondary.txt @@ -123,48 +131,46 @@ jobs: CPLUS_INCLUDE_PATH: ${{ env.Python_ROOT_DIR }}/include/python${{ env.PYTHON_VERSION }} - name: PyTorch Models Tests Timm and Torchvision + if: ${{ inputs.model_scope == 'precommit' || inputs.model_scope == 'nightly_scope1' }} run: | export PYTHONPATH=${MODEL_HUB_TESTS_INSTALL_DIR}:$PYTHONPATH python3 -m pytest ${MODEL_HUB_TESTS_INSTALL_DIR}/pytorch/ -m ${TYPE} --html=${INSTALL_TEST_DIR}/TEST-torch_model_timm_tv_tests.html --self-contained-html -v -n 4 -k "TestTimmConvertModel or TestTorchHubConvertModel" env: - TYPE: ${{ inputs.event == 'schedule' && 'nightly' || 'precommit'}} + TYPE: ${{ inputs.model_scope == 'precommit' && 'precommit' || 'nightly' }} TEST_DEVICE: CPU OP_REPORT_FILE: ${{ env.INSTALL_TEST_DIR }}/TEST-torch_unsupported_ops.log - name: PyTorch Models Tests Not Timm or Torchvision - if: always() + if: ${{ inputs.model_scope == 'precommit' || inputs.model_scope == 'nightly_scope2' }} run: | export PYTHONPATH=${MODEL_HUB_TESTS_INSTALL_DIR}:$PYTHONPATH - python3 -m pytest ${MODEL_HUB_TESTS_INSTALL_DIR}/pytorch -m ${TYPE} --html=${INSTALL_TEST_DIR}/TEST-torch_model_tests.html --self-contained-html -v -k "not (TestTimmConvertModel or TestTorchHubConvertModel or test_pa_precommit)" + python3 -m pytest ${MODEL_HUB_TESTS_INSTALL_DIR}/pytorch -m ${TYPE} --html=${INSTALL_TEST_DIR}/TEST-torch_model_tests.html --self-contained-html -v -k "not (TestTimmConvertModel or TestTorchHubConvertModel)" env: - TYPE: ${{ inputs.event == 'schedule' && 'nightly' || 'precommit'}} + TYPE: ${{ inputs.model_scope == 'precommit' && 'precommit' || 'nightly' }} TEST_DEVICE: CPU USE_SYSTEM_CACHE: False OP_REPORT_FILE: ${{ env.INSTALL_TEST_DIR }}/TEST-torch_unsupported_ops.log - name: PagedAttention Test - if: always() + if: ${{ inputs.model_scope == 'precommit' }} run: | export PYTHONPATH=${MODEL_HUB_TESTS_INSTALL_DIR}:$PYTHONPATH - python3 -m pytest ${MODEL_HUB_TESTS_INSTALL_DIR}/pytorch/test_pa_transformation.py -m ${TYPE} --html=${INSTALL_TEST_DIR}/TEST-torch_pagedattention_tests.html --self-contained-html -v --tb=short -n 4 + python3 -m pytest ${MODEL_HUB_TESTS_INSTALL_DIR}/transformation_tests/test_pa_transformation.py -m precommit --html=${INSTALL_TEST_DIR}/TEST-torch_pagedattention_tests.html --self-contained-html -v --tb=short -n 4 env: - TYPE: ${{ inputs.event == 'schedule' && 'nightly' || 'precommit'}} TEST_DEVICE: CPU USE_SYSTEM_CACHE: False - OP_REPORT_FILE: ${{ env.INSTALL_TEST_DIR }}/TEST-torch_unsupported_ops.log - name: StatefulToStateless Test - if: always() + if: ${{ inputs.model_scope == 'precommit' }} run: | export PYTHONPATH=${MODEL_HUB_TESTS_INSTALL_DIR}:$PYTHONPATH - python3 -m pytest ${MODEL_HUB_TESTS_INSTALL_DIR}/pytorch/test_stateful_to_stateless_transformation.py -m ${TYPE} --html=${INSTALL_TEST_DIR}/TEST-torch_stateful_to_stateless_tests.html --self-contained-html -v --tb=short + python3 -m pytest ${MODEL_HUB_TESTS_INSTALL_DIR}/transformation_tests/test_stateful_to_stateless_transformation.py -m precommit --html=${INSTALL_TEST_DIR}/TEST-torch_stateful_to_stateless_tests.html --self-contained-html -v --tb=short env: - TYPE: ${{ inputs.event == 'schedule' && 'nightly' || 'precommit'}} TEST_DEVICE: CPU USE_SYSTEM_CACHE: False - name: Reformat unsupported ops file - if: '!cancelled()' + if: ${{ inputs.model_scope != 'precommit' && !cancelled()}} run: | python3 ${MODEL_HUB_TESTS_INSTALL_DIR}/pytorch/scripts/process_op_report.py ${INSTALL_TEST_DIR}/TEST-torch_unsupported_ops.log diff --git a/.github/workflows/linux.yml b/.github/workflows/linux.yml index e75cc2a1867e55..5b6e8336d864d1 100644 --- a/.github/workflows/linux.yml +++ b/.github/workflows/linux.yml @@ -557,8 +557,26 @@ jobs: needs: [ Build, Smart_CI, Openvino_tokenizers ] uses: ./.github/workflows/job_pytorch_models_tests.yml with: - runner: ${{ github.event_name == 'schedule' && 'ubuntu-20.04-16-cores' || 'ubuntu-20.04-8-cores' }} - event: ${{ github.event_name }} + runner: 'ubuntu-20.04-8-cores' + model_scope: 'precommit' + + PyTorch_Models_Tests_Nightly_Scope1: + name: PyTorch Models Nightly Scope1 tests + if: ${{ github.event_name == 'schedule' }} + needs: [ Build, Smart_CI, Openvino_tokenizers ] + uses: ./.github/workflows/job_pytorch_models_tests.yml + with: + runner: 'ubuntu-20.04-16-cores' + model_scope: 'nightly_scope1' + + PyTorch_Models_Tests_Nightly_Scope2: + name: PyTorch Models Nightly Scope2 tests + if: ${{ github.event_name == 'schedule' }} + needs: [ Build, Smart_CI, Openvino_tokenizers ] + uses: ./.github/workflows/job_pytorch_models_tests.yml + with: + runner: 'ubuntu-20.04-16-cores' + model_scope: 'nightly_scope2' NVIDIA_Plugin: name: NVIDIA plugin diff --git a/.github/workflows/linux_arm64.yml b/.github/workflows/linux_arm64.yml index 9156fa711e752a..825a4b076d631d 100644 --- a/.github/workflows/linux_arm64.yml +++ b/.github/workflows/linux_arm64.yml @@ -401,7 +401,7 @@ jobs: with: runner: 'aks-linux-16-cores-arm' container: '{"image": "${{ fromJSON(needs.docker.outputs.images).ov_test.ubuntu_20_04_arm64 }}"}' - event: ${{ github.event_name }} + model_scope: 'precommit' Overall_Status: name: ci/gha_overall_status_linux_arm64 diff --git a/tests/model_hub_tests/pytorch/hf_transformers_models b/tests/model_hub_tests/pytorch/hf_transformers_models index bf76bc90e5a806..5da9db39095810 100644 --- a/tests/model_hub_tests/pytorch/hf_transformers_models +++ b/tests/model_hub_tests/pytorch/hf_transformers_models @@ -296,7 +296,7 @@ openai-gpt,openai-gpt OpenAssistant/oasst-rm-2-pythia-6.9b-epoch-1,gpt_neox_reward_model,skip,Load problem openmmlab/upernet-convnext-small,upernet openMUSE/clip-vit-large-patch14-text-enc,clip_text_model,skip,Load problem -OpenVINO/opt-125m-gptq,opt +katuni4ka/opt-125m-gptq,opt PatrickHaller/ngme-llama-264M,ngme,skip,Load problem patrickvonplaten/bert2gpt2-cnn_dailymail-fp16,encoder_decoder,skip,Load problem paulhindemith/test-zeroshot,test-zeroshot,skip,Load problem diff --git a/tests/model_hub_tests/transformation_tests/conftest.py b/tests/model_hub_tests/transformation_tests/conftest.py new file mode 100644 index 00000000000000..64efa7cf7f5506 --- /dev/null +++ b/tests/model_hub_tests/transformation_tests/conftest.py @@ -0,0 +1,12 @@ +# Copyright (C) 2018-2024 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +import inspect + +from models_hub_common.utils import get_params + + +def pytest_generate_tests(metafunc): + test_gen_attrs_names = list(inspect.signature(get_params).parameters) + params = get_params() + metafunc.parametrize(test_gen_attrs_names, params, scope="function") diff --git a/tests/model_hub_tests/pytorch/models/hf-tiny-random-models-precommit b/tests/model_hub_tests/transformation_tests/models/hf-tiny-random-models-precommit similarity index 100% rename from tests/model_hub_tests/pytorch/models/hf-tiny-random-models-precommit rename to tests/model_hub_tests/transformation_tests/models/hf-tiny-random-models-precommit diff --git a/tests/model_hub_tests/pytorch/models/tiny-set-stateful-models-precommit b/tests/model_hub_tests/transformation_tests/models/tiny-set-stateful-models-precommit similarity index 100% rename from tests/model_hub_tests/pytorch/models/tiny-set-stateful-models-precommit rename to tests/model_hub_tests/transformation_tests/models/tiny-set-stateful-models-precommit diff --git a/tests/model_hub_tests/pytorch/test_pa_transformation.py b/tests/model_hub_tests/transformation_tests/test_pa_transformation.py similarity index 100% rename from tests/model_hub_tests/pytorch/test_pa_transformation.py rename to tests/model_hub_tests/transformation_tests/test_pa_transformation.py diff --git a/tests/model_hub_tests/pytorch/test_stateful_to_stateless_transformation.py b/tests/model_hub_tests/transformation_tests/test_stateful_to_stateless_transformation.py similarity index 100% rename from tests/model_hub_tests/pytorch/test_stateful_to_stateless_transformation.py rename to tests/model_hub_tests/transformation_tests/test_stateful_to_stateless_transformation.py