Merge branch 'e2e_oss' of https://github.com/rnugmanx/openvino into e…

…2e_oss

.github/actions/setup_python/action.yml

@@ -35,13 +35,13 @@ runs:
 
     - if: ${{ runner.os == 'Linux' && runner.arch == 'ARM64' }}
       name: Setup Python ${{ inputs.version }}
-      uses: akashchi/deadsnakes-action@f01521a69eee61eaca3a34440bea3ce838317846
+      uses: akashchi/deadsnakes-action@92417281055a5878a0450f240a5b95883eb2d7e2
       with:
         python-version: ${{ inputs.version }}
 
     - if: ${{ runner.os == 'macOS' || runner.os == 'Windows' || (runner.os == 'Linux' && runner.arch != 'ARM64') }}
       name: Setup Python ${{ inputs.version }}
-      uses: actions/setup-python@v4
+      uses: actions/setup-python@v5
       with:
         python-version: ${{ inputs.version }}
       env:

.github/actions/smart-ci/action.yml

@@ -13,6 +13,9 @@ inputs:
   commit_sha:
     description: "GitHub commit hash. Used if no PR number is set"
     required: false
+  ref_name:
+    description: "GitHub ref name"
+    required: false
   component_pattern:
     description: "Pattern to extract component name from PR label. If not set, any label is considered a component name"
     required: false
@@ -88,6 +91,7 @@ runs:
           python ${{ github.action_path }}/smart_ci.py \
           $([[ -n "${{ inputs.pr }}" ]] && echo '--pr ${{ inputs.pr }}' || echo '-s ${{ inputs.commit_sha }}') \
           -r ${{ inputs.repository }} \
+          -f "${{ inputs.ref_name }}" \
           -p "${{ inputs.component_pattern }}" \
           -c "${{ inputs.components_config }}" \
           -m "${{ inputs.components_config_schema }}" \

.github/actions/smart-ci/smart_ci.py

@@ -109,11 +109,26 @@ def get_changed_component_names(pr, all_possible_components: set, component_patt
     return components
 
 
+def get_changeset(gh_api, pr, target_branch, commit_sha):
+    """Returns changeset either from PR or commit"""
+    if pr:
+        return gh_api.pulls.list_files(pr)
+    if target_branch:
+        target_branch_head_commit = gh_api.repos.get_branch(target_branch).commit.sha
+        # In merge-queue branch all commits between head of target branch and head of current branch (commit_sha)
+        # contain changes added to queue earlier to be validated together. Getting all of them + changes from
+        # commit_sha below
+        changed_files = gh_api.repos.compare_commits(f'{target_branch_head_commit}...{commit_sha}').get('files', [])
+        return changed_files
+    raise ValueError(f'Either "pr" or "target_branch" parameter must be non-empty')
+
+
 def parse_args():
     parser = argparse.ArgumentParser(description='Returns product components changed in a given PR or commit')
     parser.add_argument('--pr', type=int, required=False, help='PR number. If not set, --commit is used')
     parser.add_argument('-s', '--commit-sha', required=False, help='Commit SHA. If not set, --pr is used')
     parser.add_argument('-r', '--repo', help='GitHub repository')
+    parser.add_argument('-f', '--ref_name', required=False, help='GitHub ref name')
     parser.add_argument('-p', '--pattern', default=None, help='Pattern to extract component name from PR label. '
                                                               'If not set, any label is considered a component name')
     parser.add_argument('-c', '--components-config', default='.github/components.yml',
@@ -172,18 +187,27 @@ def main():
         component_name = component_name_from_label(label, args.pattern)
         all_possible_components.add(component_name if component_name else label)
 
-    no_match_files_changed = False
+    run_full_scope = False
     # For now, we don't want to apply smart ci rules for post-commits
     is_postcommit = not pr
-    if is_postcommit:
+
+    merge_queue_prefix = 'gh-readonly-queue/'
+    is_merge_queue = args.ref_name.startswith(merge_queue_prefix)
+    merge_queue_target_branch = re.findall(f'^{merge_queue_prefix}(.*)/', args.ref_name)[0] if is_merge_queue else None
+
+    if is_merge_queue:
+        logger.info(f"The run is a merge-queue run, executing full validation scope for all components, if "
+                    f"not all queued changes match patterns in 'skip-when-only-listed-files-changed'")
+        run_full_scope = True
+    elif is_postcommit:
         logger.info(f"The run is a post-commit run, executing full validation scope for all components")
+        run_full_scope = True
     else:
         no_match_files_changed = 'no-match-files' in [label.name for label in pr.labels]
         if no_match_files_changed:
             logger.info(f"There are changed files that don't match any pattern in labeler config, "
                         f"executing full validation scope for all components")
-
-    run_full_scope = is_postcommit or no_match_files_changed
+            run_full_scope = True
 
     # In post-commits - validate all components regardless of changeset
     # In pre-commits - validate only changed components with their dependencies
@@ -197,15 +221,15 @@ def main():
     affected_components = cfg.get_affected_components(changed_component_names)
 
     skip_workflow = False
-    if args.pr and not run_full_scope:
+    if is_merge_queue or (args.pr and not run_full_scope):
         if args.skip_when_only_listed_labels_set:
             excepted_labels = set(args.skip_when_only_listed_labels_set.split(','))
             excepted_labels_only = changed_component_names - excepted_labels == set()
             skip_workflow = excepted_labels_only
 
         if not skip_workflow and args.skip_when_only_listed_files_changed:
             # To avoid spending extra API requests running step below only if necessary
-            changed_files = gh_api.pulls.list_files(args.pr)
+            changed_files = get_changeset(gh_api, args.pr, merge_queue_target_branch, args.commit_sha)
             patterns = set(args.skip_when_only_listed_files_changed.split(','))
 
             matched_files_only = all(any(fnmatch(f.filename, pattern) for pattern in patterns) for f in changed_files)

.github/components.yml

@@ -82,6 +82,7 @@ TEMPLATE:
     - C_API
     - Python_API
     - NVIDIA
+    - TOKENIZERS
   build:
     - IR_FE
 
@@ -195,6 +196,7 @@ IE_Tests:
     - TEMPLATE
     - AUTO
     - NVIDIA
+    - TOKENIZERS
   build:
     - IR_FE
 
@@ -225,3 +227,7 @@ NVIDIA:
 ONNX_RT:
   revalidate: []
   build: []
+
+TOKENIZERS:
+  revalidate: []
+  build: []

.github/workflows/android_arm64.yml

@@ -2,6 +2,7 @@ name: Android ARM64 with vcpkg
 on:
   workflow_dispatch:
   pull_request:
+  merge_group:
   push:
     branches:
       - master
@@ -31,6 +32,7 @@ jobs:
           repository: ${{ github.repository }}
           pr: ${{ github.event.number }}
           commit_sha: ${{ github.sha }}
+          ref_name: ${{ github.ref_name }}
           component_pattern: "category: (.*)"
           repo_token: ${{ secrets.GITHUB_TOKEN }}
           skip_when_only_listed_labels_set: 'docs'
@@ -53,6 +55,10 @@ jobs:
       CMAKE_GENERATOR: 'Ninja'
       CMAKE_CXX_COMPILER_LAUNCHER: sccache
       CMAKE_C_COMPILER_LAUNCHER: sccache
+      SCCACHE_IGNORE_SERVER_IO_ERROR: 1
+      SCCACHE_SERVER_PORT: 35555
+      SCCACHE_ERROR_LOG: /__w/openvino/sccache_log.txt
+      SCCACHE_LOG: warn
       OPENVINO_REPO: '/__w/openvino/openvino/openvino'
       VCPKG_ROOT: '/__w/openvino/openvino/vcpkg'
       BUILD_DIR: '/__w/openvino/openvino/build'
@@ -123,9 +129,9 @@ jobs:
           echo "yes" | ./cmdline-tools/bin/sdkmanager --sdk_root=${ANDROID_TOOLS} --install "ndk-bundle" "platform-tools" "platforms;android-${{ env.ANDROID_SDK_VERSION }}"
 
       - name: Install sccache
-        uses: mozilla-actions/[email protected].3
+        uses: mozilla-actions/[email protected].4
         with:
-          version: "v0.5.4"
+          version: "v0.7.5"
 
       #
       # Build
@@ -171,6 +177,17 @@ jobs:
       - name: Show ccache stats
         run: ${SCCACHE_PATH} --show-stats
 
+      #
+      # Upload build logs
+      #
+      - name: Upload build logs
+        uses: actions/upload-artifact@v3
+        if: always()
+        with:
+          name: build_logs
+          path: ${{ env.SCCACHE_ERROR_LOG }}
+          if-no-files-found: 'ignore'
+
   Overall_Status:
     name: ci/gha_overall_status_android
     needs: [Smart_CI, Build]

.github/workflows/build_doc.yml

@@ -1,6 +1,7 @@
 name: Documentation
 on:
   pull_request:
+  merge_group:
 
 env:
   DOXY_VER: '1.9.6'
@@ -54,7 +55,7 @@ jobs:
 
       - name: Cache documentation
         id: cache_sphinx_docs
-        uses: actions/cache@v3
+        uses: actions/cache@v4
         with:
           path: build/docs/_build/.doctrees
           key: sphinx-docs-cache

.github/workflows/coverity.yml

@@ -36,6 +36,8 @@ jobs:
       CMAKE_GENERATOR: 'Ninja Multi-Config'
       CMAKE_CXX_COMPILER_LAUNCHER: sccache
       CMAKE_C_COMPILER_LAUNCHER: sccache
+      SCCACHE_IGNORE_SERVER_IO_ERROR: 1
+      SCCACHE_SERVER_PORT: 35555
       GITHUB_WORKSPACE: '/__w/openvino/openvino'
       OPENVINO_REPO: /__w/openvino/openvino/openvino
       OPENVINO_CONTRIB_REPO: /__w/openvino/openvino/openvino_contrib
@@ -75,9 +77,9 @@ jobs:
           apt install --assume-yes --no-install-recommends default-jdk
 
       - name: Install sccache
-        uses: mozilla-actions/[email protected].3
+        uses: mozilla-actions/[email protected].4
         with:
-          version: "v0.5.4"
+          version: "v0.7.5"
 
       - name: Setup Python ${{ env.PYTHON_VERSION }}
         uses: ./openvino/.github/actions/setup_python

.github/workflows/dependency_review.yml

@@ -12,6 +12,6 @@ jobs:
         uses: actions/checkout@v4
 
       - name: Dependency Review
-        uses: actions/dependency-review-action@v3
+        uses: actions/dependency-review-action@v4
         with:
           config-file: './.github/dependency_review.yml'

.github/workflows/fedora.yml

@@ -2,6 +2,7 @@ name: Fedora (RHEL), Python 3.9
 on:
   workflow_dispatch:
   pull_request:
+  merge_group:
   push:
     branches:
       - master
@@ -31,6 +32,7 @@ jobs:
           repository: ${{ github.repository }}
           pr: ${{ github.event.number }}
           commit_sha: ${{ github.sha }}
+          ref_name: ${{ github.ref_name }}
           component_pattern: "category: (.*)"
           repo_token: ${{ secrets.GITHUB_TOKEN }}
           skip_when_only_listed_labels_set: 'docs'
@@ -53,6 +55,10 @@ jobs:
       CMAKE_GENERATOR: 'Ninja'
       CMAKE_CXX_COMPILER_LAUNCHER: sccache
       CMAKE_C_COMPILER_LAUNCHER: sccache
+      SCCACHE_IGNORE_SERVER_IO_ERROR: 1
+      SCCACHE_SERVER_PORT: 35555
+      SCCACHE_ERROR_LOG: /__w/openvino/sccache_log.txt
+      SCCACHE_LOG: warn
       GITHUB_WORKSPACE: '/__w/openvino/openvino'
       OPENVINO_REPO: /__w/openvino/openvino/openvino
       INSTALL_DIR: /__w/openvino/openvino/openvino_install
@@ -85,9 +91,9 @@ jobs:
         run: bash ${OPENVINO_REPO}/install_build_dependencies.sh
 
       - name: Install sccache
-        uses: mozilla-actions/[email protected].3
+        uses: mozilla-actions/[email protected].4
         with:
-          version: "v0.5.4"
+          version: "v0.7.5"
 
       - name: Install python dependencies
         run: |
@@ -160,8 +166,15 @@ jobs:
           cmake --build ${BUILD_DIR} --parallel --target package --verbose
 
       #
-      # Upload build artifacts
+      # Upload build artifacts and logs
       #
+      - name: Upload build logs
+        uses: actions/upload-artifact@v3
+        if: always()
+        with:
+          name: build_logs
+          path: ${{ env.SCCACHE_ERROR_LOG }}
+          if-no-files-found: 'ignore'
 
       - name: Upload openvino package
         if: ${{ always() }}
@@ -234,10 +247,10 @@ jobs:
           python3 /usr/share/openvino/samples/python/hello_query_device/hello_query_device.py
           python3 -c 'from openvino import Core; Core().get_property("CPU", "AVAILABLE_DEVICES")'
           python3 -c 'from openvino import Core; Core().get_property("GPU", "AVAILABLE_DEVICES")'
-          python3 -c 'from openvino import Core; Core().get_property("AUTO", "SUPPORTED_METRICS")'
-          python3 -c 'from openvino import Core; Core().get_property("MULTI", "SUPPORTED_METRICS")'
-          python3 -c 'from openvino import Core; Core().get_property("HETERO", "SUPPORTED_METRICS")'
-          python3 -c 'from openvino import Core; Core().get_property("BATCH", "SUPPORTED_METRICS")'
+          python3 -c 'from openvino import Core; Core().get_property("AUTO", "SUPPORTED_PROPERTIES")'
+          python3 -c 'from openvino import Core; Core().get_property("MULTI", "SUPPORTED_PROPERTIES")'
+          python3 -c 'from openvino import Core; Core().get_property("HETERO", "SUPPORTED_PROPERTIES")'
+          python3 -c 'from openvino import Core; Core().get_property("BATCH", "SUPPORTED_PROPERTIES")'
           python3 -c 'from openvino.frontend import FrontEndManager; assert len(FrontEndManager().get_available_front_ends()) == 6'
           benchmark_app --help
           ovc --help

.github/workflows/job_cpu_functional_tests.yml

@@ -29,7 +29,11 @@ jobs:
       INSTALL_TEST_DIR: ${{ github.workspace }}/install/tests
       PARALLEL_TEST_SCRIPT: ${{ github.workspace }}/install/tests/functional_test_utils/layer_tests_summary/run_parallel.py
       PARALLEL_TEST_CACHE: ${{ github.workspace }}/install/tests/test_cache.lst
+    if: ${{ github.event_name != 'merge_group' }}
     steps:
+      - name: Set apt retries
+        run: echo 'Acquire::Retries "10";' > /etc/apt/apt.conf.d/80-retries
+
       - name: Download OpenVINO package
         uses: actions/download-artifact@v3
         with:
@@ -55,7 +59,7 @@ jobs:
           pushd $INSTALL_DIR
             tar -xzf openvino_package.tar.gz -C $INSTALL_DIR
           popd
-          
+
           pushd $INSTALL_TEST_DIR
             tar -xzf openvino_tests.tar.gz -C $INSTALL_DIR
           popd
@@ -83,7 +87,7 @@ jobs:
         run: python3 -m pip install -r ${INSTALL_TEST_DIR}/functional_test_utils/layer_tests_summary/requirements.txt
 
       - name: Restore tests execution time
-        uses: actions/cache/restore@v3
+        uses: actions/cache/restore@v4
         with:
           path: ${{ env.PARALLEL_TEST_CACHE }}
           key: ${{ runner.os }}-${{ runner.arch }}-tests-functional-cpu-stamp-${{ github.sha }}
@@ -96,20 +100,20 @@ jobs:
           if [[ -f "${INSTALL_DIR}/setupvars.sh" ]]; then
             source ${INSTALL_DIR}/setupvars.sh
           fi
-          
+
           python3 ${PARALLEL_TEST_SCRIPT} -e ${INSTALL_TEST_DIR}/ov_cpu_func_tests -c ${PARALLEL_TEST_CACHE} -w ${INSTALL_TEST_DIR} -s suite -rf 0 -- --gtest_print_time=1 --gtest_filter=*smoke*
-        timeout-minutes: 20
+        timeout-minutes: 25
 
       - name: Save tests execution time
-        uses: actions/cache/save@v3
+        uses: actions/cache/save@v4
         if: github.ref_name == 'master'
         with:
           path: ${{ env.PARALLEL_TEST_CACHE }}
           key: ${{ runner.os }}-${{ runner.arch }}-tests-functional-cpu-stamp-${{ github.sha }}
 
       - name: Upload Test Results
         uses: actions/upload-artifact@v3
-        if: ${{ !cancelled() }}
+        if: always()
         with:
           name: test-results-functional-cpu
           path: |

.github/workflows/job_cxx_unit_tests.yml

@@ -31,7 +31,11 @@ jobs:
       DEBIAN_FRONTEND: noninteractive # to prevent apt-get from waiting user input
       INSTALL_DIR: ${{ github.workspace }}/install
       INSTALL_TEST_DIR: ${{ github.workspace }}/install/tests
+    if: ${{ github.event_name != 'merge_group' }}
     steps:
+      - name: Set apt retries
+        run: echo 'Acquire::Retries "10";' > /etc/apt/apt.conf.d/80-retries
+
       - name: Download OpenVINO package
         uses: actions/download-artifact@v3
         with:

.github/workflows/job_debian_packages.yml

@@ -26,6 +26,8 @@ jobs:
       DEBIAN_FRONTEND: noninteractive # to prevent apt-get from waiting user input
       DEBIAN_PACKAGES_DIR: ${{ github.workspace }}/packages
     steps:
+      - name: Set apt retries
+        run: echo 'Acquire::Retries "10";' > /etc/apt/apt.conf.d/80-retries
 
       - name: Download OpenVINO debian packages
         uses: actions/download-artifact@v3
@@ -74,10 +76,10 @@ jobs:
             python3 -c 'from openvino import Core; Core().get_property("GPU", "AVAILABLE_DEVICES")'
           fi
           
-          python3 -c 'from openvino import Core; Core().get_property("AUTO", "SUPPORTED_METRICS")'
-          python3 -c 'from openvino import Core; Core().get_property("MULTI", "SUPPORTED_METRICS")'
-          python3 -c 'from openvino import Core; Core().get_property("HETERO", "SUPPORTED_METRICS")'
-          python3 -c 'from openvino import Core; Core().get_property("BATCH", "SUPPORTED_METRICS")'
+          python3 -c 'from openvino import Core; Core().get_property("AUTO", "SUPPORTED_PROPERTIES")'
+          python3 -c 'from openvino import Core; Core().get_property("MULTI", "SUPPORTED_PROPERTIES")'
+          python3 -c 'from openvino import Core; Core().get_property("HETERO", "SUPPORTED_PROPERTIES")'
+          python3 -c 'from openvino import Core; Core().get_property("BATCH", "SUPPORTED_PROPERTIES")'
           python3 -c 'from openvino.frontend import FrontEndManager; assert len(FrontEndManager().get_available_front_ends()) == 6'
           benchmark_app --help
           ovc --help