Merge branch 'master' into e2e_oss

.github/actions/smart-ci/action.yml

@@ -13,6 +13,9 @@ inputs:
   commit_sha:
     description: "GitHub commit hash. Used if no PR number is set"
     required: false
+  ref_name:
+    description: "GitHub ref name"
+    required: false
   component_pattern:
     description: "Pattern to extract component name from PR label. If not set, any label is considered a component name"
     required: false
@@ -88,6 +91,7 @@ runs:
           python ${{ github.action_path }}/smart_ci.py \
           $([[ -n "${{ inputs.pr }}" ]] && echo '--pr ${{ inputs.pr }}' || echo '-s ${{ inputs.commit_sha }}') \
           -r ${{ inputs.repository }} \
+          -f "${{ inputs.ref_name }}" \
           -p "${{ inputs.component_pattern }}" \
           -c "${{ inputs.components_config }}" \
           -m "${{ inputs.components_config_schema }}" \

.github/actions/smart-ci/smart_ci.py

@@ -109,11 +109,26 @@ def get_changed_component_names(pr, all_possible_components: set, component_patt
     return components
 
 
+def get_changeset(gh_api, pr, target_branch, commit_sha):
+    """Returns changeset either from PR or commit"""
+    if pr:
+        return gh_api.pulls.list_files(pr)
+    if target_branch:
+        target_branch_head_commit = gh_api.repos.get_branch(target_branch).commit.sha
+        # In merge-queue branch all commits between head of target branch and head of current branch (commit_sha)
+        # contain changes added to queue earlier to be validated together. Getting all of them + changes from
+        # commit_sha below
+        changed_files = gh_api.repos.compare_commits(f'{target_branch_head_commit}...{commit_sha}').get('files', [])
+        return changed_files
+    raise ValueError(f'Either "pr" or "target_branch" parameter must be non-empty')
+
+
 def parse_args():
     parser = argparse.ArgumentParser(description='Returns product components changed in a given PR or commit')
     parser.add_argument('--pr', type=int, required=False, help='PR number. If not set, --commit is used')
     parser.add_argument('-s', '--commit-sha', required=False, help='Commit SHA. If not set, --pr is used')
     parser.add_argument('-r', '--repo', help='GitHub repository')
+    parser.add_argument('-f', '--ref_name', required=False, help='GitHub ref name')
     parser.add_argument('-p', '--pattern', default=None, help='Pattern to extract component name from PR label. '
                                                               'If not set, any label is considered a component name')
     parser.add_argument('-c', '--components-config', default='.github/components.yml',
@@ -172,18 +187,27 @@ def main():
         component_name = component_name_from_label(label, args.pattern)
         all_possible_components.add(component_name if component_name else label)
 
-    no_match_files_changed = False
+    run_full_scope = False
     # For now, we don't want to apply smart ci rules for post-commits
     is_postcommit = not pr
-    if is_postcommit:
+
+    merge_queue_prefix = 'gh-readonly-queue/'
+    is_merge_queue = args.ref_name.startswith(merge_queue_prefix)
+    merge_queue_target_branch = re.findall(f'^{merge_queue_prefix}(.*)/', args.ref_name)[0] if is_merge_queue else None
+
+    if is_merge_queue:
+        logger.info(f"The run is a merge-queue run, executing full validation scope for all components, if "
+                    f"not all queued changes match patterns in 'skip-when-only-listed-files-changed'")
+        run_full_scope = True
+    elif is_postcommit:
         logger.info(f"The run is a post-commit run, executing full validation scope for all components")
+        run_full_scope = True
     else:
         no_match_files_changed = 'no-match-files' in [label.name for label in pr.labels]
         if no_match_files_changed:
             logger.info(f"There are changed files that don't match any pattern in labeler config, "
                         f"executing full validation scope for all components")
-
-    run_full_scope = is_postcommit or no_match_files_changed
+            run_full_scope = True
 
     # In post-commits - validate all components regardless of changeset
     # In pre-commits - validate only changed components with their dependencies
@@ -197,15 +221,15 @@ def main():
     affected_components = cfg.get_affected_components(changed_component_names)
 
     skip_workflow = False
-    if args.pr and not run_full_scope:
+    if is_merge_queue or (args.pr and not run_full_scope):
         if args.skip_when_only_listed_labels_set:
             excepted_labels = set(args.skip_when_only_listed_labels_set.split(','))
             excepted_labels_only = changed_component_names - excepted_labels == set()
             skip_workflow = excepted_labels_only
 
         if not skip_workflow and args.skip_when_only_listed_files_changed:
             # To avoid spending extra API requests running step below only if necessary
-            changed_files = gh_api.pulls.list_files(args.pr)
+            changed_files = get_changeset(gh_api, args.pr, merge_queue_target_branch, args.commit_sha)
             patterns = set(args.skip_when_only_listed_files_changed.split(','))
 
             matched_files_only = all(any(fnmatch(f.filename, pattern) for pattern in patterns) for f in changed_files)

.github/workflows/android_arm64.yml

@@ -31,6 +31,7 @@ jobs:
           repository: ${{ github.repository }}
           pr: ${{ github.event.number }}
           commit_sha: ${{ github.sha }}
+          ref_name: ${{ github.ref_name }}
           component_pattern: "category: (.*)"
           repo_token: ${{ secrets.GITHUB_TOKEN }}
           skip_when_only_listed_labels_set: 'docs'
@@ -53,6 +54,8 @@ jobs:
       CMAKE_GENERATOR: 'Ninja'
       CMAKE_CXX_COMPILER_LAUNCHER: sccache
       CMAKE_C_COMPILER_LAUNCHER: sccache
+      SCCACHE_IGNORE_SERVER_IO_ERROR: 1
+      SCCACHE_SERVER_PORT: 35555
       OPENVINO_REPO: '/__w/openvino/openvino/openvino'
       VCPKG_ROOT: '/__w/openvino/openvino/vcpkg'
       BUILD_DIR: '/__w/openvino/openvino/build'
@@ -125,7 +128,7 @@ jobs:
       - name: Install sccache
         uses: mozilla-actions/[email protected]
         with:
-          version: "v0.5.4"
+          version: "v0.7.5"
 
       #
       # Build

.github/workflows/build_doc.yml

@@ -54,7 +54,7 @@ jobs:
 
       - name: Cache documentation
         id: cache_sphinx_docs
-        uses: actions/cache@v3
+        uses: actions/cache@v4
         with:
           path: build/docs/_build/.doctrees
           key: sphinx-docs-cache

.github/workflows/coverity.yml

@@ -36,6 +36,8 @@ jobs:
       CMAKE_GENERATOR: 'Ninja Multi-Config'
       CMAKE_CXX_COMPILER_LAUNCHER: sccache
       CMAKE_C_COMPILER_LAUNCHER: sccache
+      SCCACHE_IGNORE_SERVER_IO_ERROR: 1
+      SCCACHE_SERVER_PORT: 35555
       GITHUB_WORKSPACE: '/__w/openvino/openvino'
       OPENVINO_REPO: /__w/openvino/openvino/openvino
       OPENVINO_CONTRIB_REPO: /__w/openvino/openvino/openvino_contrib
@@ -77,7 +79,7 @@ jobs:
       - name: Install sccache
         uses: mozilla-actions/[email protected]
         with:
-          version: "v0.5.4"
+          version: "v0.7.5"
 
       - name: Setup Python ${{ env.PYTHON_VERSION }}
         uses: ./openvino/.github/actions/setup_python

.github/workflows/dependency_review.yml

@@ -12,6 +12,6 @@ jobs:
         uses: actions/checkout@v4
 
       - name: Dependency Review
-        uses: actions/dependency-review-action@v3
+        uses: actions/dependency-review-action@v4
         with:
           config-file: './.github/dependency_review.yml'

.github/workflows/fedora.yml

@@ -31,6 +31,7 @@ jobs:
           repository: ${{ github.repository }}
           pr: ${{ github.event.number }}
           commit_sha: ${{ github.sha }}
+          ref_name: ${{ github.ref_name }}
           component_pattern: "category: (.*)"
           repo_token: ${{ secrets.GITHUB_TOKEN }}
           skip_when_only_listed_labels_set: 'docs'
@@ -53,6 +54,8 @@ jobs:
       CMAKE_GENERATOR: 'Ninja'
       CMAKE_CXX_COMPILER_LAUNCHER: sccache
       CMAKE_C_COMPILER_LAUNCHER: sccache
+      SCCACHE_IGNORE_SERVER_IO_ERROR: 1
+      SCCACHE_SERVER_PORT: 35555
       GITHUB_WORKSPACE: '/__w/openvino/openvino'
       OPENVINO_REPO: /__w/openvino/openvino/openvino
       INSTALL_DIR: /__w/openvino/openvino/openvino_install
@@ -87,7 +90,7 @@ jobs:
       - name: Install sccache
         uses: mozilla-actions/[email protected]
         with:
-          version: "v0.5.4"
+          version: "v0.7.5"
 
       - name: Install python dependencies
         run: |

.github/workflows/job_cpu_functional_tests.yml

@@ -83,7 +83,7 @@ jobs:
         run: python3 -m pip install -r ${INSTALL_TEST_DIR}/functional_test_utils/layer_tests_summary/requirements.txt
 
       - name: Restore tests execution time
-        uses: actions/cache/restore@v3
+        uses: actions/cache/restore@v4
         with:
           path: ${{ env.PARALLEL_TEST_CACHE }}
           key: ${{ runner.os }}-${{ runner.arch }}-tests-functional-cpu-stamp-${{ github.sha }}
@@ -98,10 +98,10 @@ jobs:
           fi
           
           python3 ${PARALLEL_TEST_SCRIPT} -e ${INSTALL_TEST_DIR}/ov_cpu_func_tests -c ${PARALLEL_TEST_CACHE} -w ${INSTALL_TEST_DIR} -s suite -rf 0 -- --gtest_print_time=1 --gtest_filter=*smoke*
-        timeout-minutes: 20
+        timeout-minutes: 25
 
       - name: Save tests execution time
-        uses: actions/cache/save@v3
+        uses: actions/cache/save@v4
         if: github.ref_name == 'master'
         with:
           path: ${{ env.PARALLEL_TEST_CACHE }}

.github/workflows/job_onnx_models_tests.yml

@@ -103,4 +103,4 @@ jobs:
           python3 -m pip install pytest-xdist[psutil] pytest-forked
 
       - name: ONNX Models Tests
-        run: python3 -m pytest --backend="CPU" --model_zoo_dir="${MODELS_SHARE_PATH}" ${INSTALL_TEST_DIR}/onnx/tests/tests_python/test_zoo_models.py -v -n 12 --forked -k 'not _cuda' --model_zoo_xfail
+        run: python3 -m pytest --backend="CPU" --model_zoo_dir="${MODELS_SHARE_PATH}" ${INSTALL_TEST_DIR}/onnx/tests/tests_python/test_zoo_models.py -v -n auto --forked -k 'not _cuda' --model_zoo_xfail

.github/workflows/job_onnx_runtime.yml

@@ -33,6 +33,8 @@ jobs:
       CMAKE_GENERATOR: 'Ninja Multi-Config'
       CMAKE_CXX_COMPILER_LAUNCHER: sccache
       CMAKE_C_COMPILER_LAUNCHER: sccache
+      SCCACHE_IGNORE_SERVER_IO_ERROR: 1
+      SCCACHE_SERVER_PORT: 35555
       SCCACHE_AZURE_KEY_PREFIX: ${{ inputs.sccache-azure-key-prefix }}
       ONNX_RUNTIME_REPO: ${{ github.workspace }}/onnxruntime
       ONNX_RUNTIME_UTILS: ${{ github.workspace }}/install/onnxruntime
@@ -80,7 +82,10 @@ jobs:
           popd
 
       - name: Install OpenVINO dependencies
-        run: ${INSTALL_DIR}/install_dependencies/install_openvino_dependencies.sh -c=core -c=dev -y
+        run: |
+          ${INSTALL_DIR}/install_dependencies/install_openvino_dependencies.sh -c=core -c=dev -y
+          # since we are on Ubuntu 22.04, but compiled OpenVINO on Ubuntu 20.04, we need to install `libtbb2`
+          apt-get install --assume-yes --no-install-recommends libtbb2
 
       - name: Clone ONNX Runtime
         run: |
@@ -97,7 +102,7 @@ jobs:
       - name: Install sccache
         uses: mozilla-actions/[email protected]
         with:
-          version: "v0.5.4"
+          version: "v0.7.5"
 
       - name: Build Lin ONNX Runtime
         run: |
@@ -139,13 +144,6 @@ jobs:
           ./onnxruntime_global_thread_pools_test
         working-directory: ${{ env.ONNX_RUNTIME_BUILD_DIR }}/RelWithDebInfo/RelWithDebInfo
 
-#      Test removed in onnxruntime 1.16.2 
-#      - name: Run onnxruntime_api_tests_without_env
-#        run: |
-#          source ${INSTALL_DIR}/setupvars.sh
-#          ./onnxruntime_api_tests_without_env
-#        working-directory: ${{ env.ONNX_RUNTIME_BUILD_DIR }}/RelWithDebInfo/RelWithDebInfo
-
       - name: Run pytorch-converted tests
         run: |
           source ${INSTALL_DIR}/setupvars.sh

.github/workflows/job_python_unit_tests.yml

@@ -249,7 +249,7 @@ jobs:
         run: |
           # requires 'unit_tests' from 'mo'
           export PYTHONPATH=${INSTALL_TEST_DIR}/mo
-          python3 -m pytest ${LAYER_TESTS_INSTALL_DIR}/tensorflow_tests/ --use_new_frontend -m precommit_tf_fe --junitxml=${INSTALL_TEST_DIR}/TEST-tf_fe.xml
+          python3 -m pytest ${LAYER_TESTS_INSTALL_DIR}/tensorflow_tests/ --use_new_frontend -m precommit_tf_fe -n logical --junitxml=${INSTALL_TEST_DIR}/TEST-tf_fe.xml
         env:
           TEST_DEVICE: CPU
           TEST_PRECISION: FP16

.github/workflows/linux.yml

@@ -38,6 +38,7 @@ jobs:
           repository: ${{ github.repository }}
           pr: ${{ github.event.number }}
           commit_sha: ${{ github.sha }}
+          ref_name: ${{ github.ref_name }}
           component_pattern: "category: (.*)"
           repo_token: ${{ secrets.GITHUB_TOKEN }}
           skip_when_only_listed_labels_set: 'docs'
@@ -61,6 +62,8 @@ jobs:
       CMAKE_GENERATOR: 'Ninja Multi-Config'
       CMAKE_CXX_COMPILER_LAUNCHER: sccache
       CMAKE_C_COMPILER_LAUNCHER: sccache
+      SCCACHE_IGNORE_SERVER_IO_ERROR: 1
+      SCCACHE_SERVER_PORT: 35555
       GITHUB_WORKSPACE: '/__w/openvino/openvino'
       OPENVINO_REPO: /__w/openvino/openvino/openvino
       OPENVINO_CONTRIB_REPO: /__w/openvino/openvino/openvino_contrib
@@ -113,7 +116,7 @@ jobs:
       - name: Install sccache
         uses: mozilla-actions/[email protected]
         with:
-          version: "v0.5.4"
+          version: "v0.7.5"
 
       - name: Setup Python ${{ env.PYTHON_VERSION }}
         uses: ./openvino/.github/actions/setup_python
@@ -406,14 +409,16 @@ jobs:
 
   ONNX_Runtime:
     name: ONNX Runtime Integration
-    if: fromJSON(needs.smart_ci.outputs.affected_components).ONNX_RT ||
-        fromJSON(needs.smart_ci.outputs.affected_components).ONNX_FE
+    # Enable back once https://github.com/microsoft/onnxruntime/pull/19184 is merged
+    if: ${{ 'false' }}
+    # if: fromJSON(needs.smart_ci.outputs.affected_components).ONNX_RT ||
+    #     fromJSON(needs.smart_ci.outputs.affected_components).ONNX_FE
     needs: [ Build, Smart_CI ]
     uses: ./.github/workflows/job_onnx_runtime.yml
     with:
       runner: 'aks-linux-16-cores-32gb'
-      container: '{"image": "openvinogithubactions.azurecr.io/dockerhub/ubuntu:20.04", "volumes": ["/mount:/mount"], "options": "-e SCCACHE_AZURE_BLOB_CONTAINER -e SCCACHE_AZURE_CONNECTION_STRING"}'
-      sccache-azure-key-prefix: 'ubuntu20_x86_64_onnxruntime'
+      container: '{"image": "openvinogithubactions.azurecr.io/dockerhub/ubuntu:22.04", "volumes": ["/mount:/mount"], "options": "-e SCCACHE_AZURE_BLOB_CONTAINER -e SCCACHE_AZURE_CONNECTION_STRING"}'
+      sccache-azure-key-prefix: 'ubuntu22_x86_64_onnxruntime'
 
   ONNX_Models:
     name: ONNX Models Tests
@@ -573,6 +578,8 @@ jobs:
       CMAKE_CUDA_COMPILER_LAUNCHER: sccache
       CMAKE_CXX_COMPILER_LAUNCHER: sccache
       CMAKE_C_COMPILER_LAUNCHER: sccache
+      SCCACHE_IGNORE_SERVER_IO_ERROR: 1
+      SCCACHE_SERVER_PORT: 35555
       INSTALL_DIR: /__w/openvino/openvino/install
       OPENVINO_DEVELOPER_PACKAGE: /__w/openvino/openvino/install/developer_package
       OPENVINO_REPO: /__w/openvino/openvino/openvino
@@ -635,7 +642,7 @@ jobs:
       - name: Install sccache
         uses: mozilla-actions/[email protected]
         with:
-          version: "v0.5.4"
+          version: "v0.7.5"
 
       - name: Install CUDA
         run: |

.github/workflows/linux_arm64.yml

@@ -35,6 +35,7 @@ jobs:
           repository: ${{ github.repository }}
           pr: ${{ github.event.number }}
           commit_sha: ${{ github.sha }}
+          ref_name: ${{ github.ref_name }}
           component_pattern: "category: (.*)"
           repo_token: ${{ secrets.GITHUB_TOKEN }}
           skip_when_only_listed_labels_set: 'docs'
@@ -63,6 +64,8 @@ jobs:
       CMAKE_GENERATOR: 'Ninja Multi-Config'
       CMAKE_CXX_COMPILER_LAUNCHER: sccache
       CMAKE_C_COMPILER_LAUNCHER: sccache
+      SCCACHE_IGNORE_SERVER_IO_ERROR: 1
+      SCCACHE_SERVER_PORT: 35555
       GITHUB_WORKSPACE: '/__w/openvino/openvino'
       OPENVINO_REPO: /__w/openvino/openvino/openvino
       OPENVINO_CONTRIB_REPO: /__w/openvino/openvino/openvino_contrib
@@ -113,7 +116,7 @@ jobs:
       - name: Install sccache
         uses: mozilla-actions/[email protected]
         with:
-          version: "v0.5.4"
+          version: "v0.7.5"
 
       - name: Setup Python ${{ env.PYTHON_VERSION }}
         uses: ./openvino/.github/actions/setup_python
@@ -307,14 +310,16 @@ jobs:
 
   ONNX_Runtime:
     name: ONNX Runtime Integration
-    if: fromJSON(needs.smart_ci.outputs.affected_components).ONNX_RT ||
-        fromJSON(needs.smart_ci.outputs.affected_components).ONNX_FE
+    # Enable back once https://github.com/microsoft/onnxruntime/pull/19184 is merged
+    if: ${{ 'false' }}
+    # if: fromJSON(needs.smart_ci.outputs.affected_components).ONNX_RT ||
+    #     fromJSON(needs.smart_ci.outputs.affected_components).ONNX_FE
     needs: [ Build, Smart_CI ]
     uses: ./.github/workflows/job_onnx_runtime.yml
     with:
       runner: 'aks-linux-16-cores-arm'
-      container: '{"image": "openvinogithubactions.azurecr.io/dockerhub/ubuntu:20.04", "volumes": ["/mount:/mount"], "options": "-e SCCACHE_AZURE_BLOB_CONTAINER -e SCCACHE_AZURE_CONNECTION_STRING"}'
-      sccache-azure-key-prefix: 'ubuntu20_aarch64_onnxruntime'
+      container: '{"image": "openvinogithubactions.azurecr.io/dockerhub/ubuntu:22.04", "volumes": ["/mount:/mount"], "options": "-e SCCACHE_AZURE_BLOB_CONTAINER -e SCCACHE_AZURE_CONNECTION_STRING"}'
+      sccache-azure-key-prefix: 'ubuntu22_aarch64_onnxruntime'
 
   CXX_Unit_Tests:
     name: C++ unit tests