Merge branch 'master' into e2e_oss

.github/actions/setup_python/action.yml

@@ -35,13 +35,13 @@ runs:
 
     - if: ${{ runner.os == 'Linux' && runner.arch == 'ARM64' }}
       name: Setup Python ${{ inputs.version }}
-      uses: akashchi/deadsnakes-action@f01521a69eee61eaca3a34440bea3ce838317846
+      uses: akashchi/deadsnakes-action@92417281055a5878a0450f240a5b95883eb2d7e2
       with:
         python-version: ${{ inputs.version }}
 
     - if: ${{ runner.os == 'macOS' || runner.os == 'Windows' || (runner.os == 'Linux' && runner.arch != 'ARM64') }}
       name: Setup Python ${{ inputs.version }}
-      uses: actions/setup-python@v4
+      uses: actions/setup-python@v5
       with:
         python-version: ${{ inputs.version }}
       env:

.github/components.yml

@@ -82,6 +82,7 @@ TEMPLATE:
     - C_API
     - Python_API
     - NVIDIA
+    - TOKENIZERS
   build:
     - IR_FE
 
@@ -195,6 +196,7 @@ IE_Tests:
     - TEMPLATE
     - AUTO
     - NVIDIA
+    - TOKENIZERS
   build:
     - IR_FE
 
@@ -225,3 +227,7 @@ NVIDIA:
 ONNX_RT:
   revalidate: []
   build: []
+
+TOKENIZERS:
+  revalidate: []
+  build: []

.github/workflows/android_arm64.yml

@@ -129,7 +129,7 @@ jobs:
           echo "yes" | ./cmdline-tools/bin/sdkmanager --sdk_root=${ANDROID_TOOLS} --install "ndk-bundle" "platform-tools" "platforms;android-${{ env.ANDROID_SDK_VERSION }}"
 
       - name: Install sccache
-        uses: mozilla-actions/[email protected].3
+        uses: mozilla-actions/[email protected].4
         with:
           version: "v0.7.5"
 

.github/workflows/coverity.yml

@@ -77,7 +77,7 @@ jobs:
           apt install --assume-yes --no-install-recommends default-jdk
 
       - name: Install sccache
-        uses: mozilla-actions/[email protected].3
+        uses: mozilla-actions/[email protected].4
         with:
           version: "v0.7.5"
 

.github/workflows/fedora.yml

@@ -91,7 +91,7 @@ jobs:
         run: bash ${OPENVINO_REPO}/install_build_dependencies.sh
 
       - name: Install sccache
-        uses: mozilla-actions/[email protected].3
+        uses: mozilla-actions/[email protected].4
         with:
           version: "v0.7.5"
 
@@ -247,10 +247,10 @@ jobs:
           python3 /usr/share/openvino/samples/python/hello_query_device/hello_query_device.py
           python3 -c 'from openvino import Core; Core().get_property("CPU", "AVAILABLE_DEVICES")'
           python3 -c 'from openvino import Core; Core().get_property("GPU", "AVAILABLE_DEVICES")'
-          python3 -c 'from openvino import Core; Core().get_property("AUTO", "SUPPORTED_METRICS")'
-          python3 -c 'from openvino import Core; Core().get_property("MULTI", "SUPPORTED_METRICS")'
-          python3 -c 'from openvino import Core; Core().get_property("HETERO", "SUPPORTED_METRICS")'
-          python3 -c 'from openvino import Core; Core().get_property("BATCH", "SUPPORTED_METRICS")'
+          python3 -c 'from openvino import Core; Core().get_property("AUTO", "SUPPORTED_PROPERTIES")'
+          python3 -c 'from openvino import Core; Core().get_property("MULTI", "SUPPORTED_PROPERTIES")'
+          python3 -c 'from openvino import Core; Core().get_property("HETERO", "SUPPORTED_PROPERTIES")'
+          python3 -c 'from openvino import Core; Core().get_property("BATCH", "SUPPORTED_PROPERTIES")'
           python3 -c 'from openvino.frontend import FrontEndManager; assert len(FrontEndManager().get_available_front_ends()) == 6'
           benchmark_app --help
           ovc --help

.github/workflows/job_cpu_functional_tests.yml

@@ -31,6 +31,9 @@ jobs:
       PARALLEL_TEST_CACHE: ${{ github.workspace }}/install/tests/test_cache.lst
     if: ${{ github.event_name != 'merge_group' }}
     steps:
+      - name: Set apt retries
+        run: echo 'Acquire::Retries "10";' > /etc/apt/apt.conf.d/80-retries
+
       - name: Download OpenVINO package
         uses: actions/download-artifact@v3
         with:

.github/workflows/job_cxx_unit_tests.yml

@@ -33,6 +33,9 @@ jobs:
       INSTALL_TEST_DIR: ${{ github.workspace }}/install/tests
     if: ${{ github.event_name != 'merge_group' }}
     steps:
+      - name: Set apt retries
+        run: echo 'Acquire::Retries "10";' > /etc/apt/apt.conf.d/80-retries
+
       - name: Download OpenVINO package
         uses: actions/download-artifact@v3
         with:

.github/workflows/job_debian_packages.yml

@@ -26,6 +26,8 @@ jobs:
       DEBIAN_FRONTEND: noninteractive # to prevent apt-get from waiting user input
       DEBIAN_PACKAGES_DIR: ${{ github.workspace }}/packages
     steps:
+      - name: Set apt retries
+        run: echo 'Acquire::Retries "10";' > /etc/apt/apt.conf.d/80-retries
 
       - name: Download OpenVINO debian packages
         uses: actions/download-artifact@v3
@@ -74,10 +76,10 @@ jobs:
             python3 -c 'from openvino import Core; Core().get_property("GPU", "AVAILABLE_DEVICES")'
           fi
           
-          python3 -c 'from openvino import Core; Core().get_property("AUTO", "SUPPORTED_METRICS")'
-          python3 -c 'from openvino import Core; Core().get_property("MULTI", "SUPPORTED_METRICS")'
-          python3 -c 'from openvino import Core; Core().get_property("HETERO", "SUPPORTED_METRICS")'
-          python3 -c 'from openvino import Core; Core().get_property("BATCH", "SUPPORTED_METRICS")'
+          python3 -c 'from openvino import Core; Core().get_property("AUTO", "SUPPORTED_PROPERTIES")'
+          python3 -c 'from openvino import Core; Core().get_property("MULTI", "SUPPORTED_PROPERTIES")'
+          python3 -c 'from openvino import Core; Core().get_property("HETERO", "SUPPORTED_PROPERTIES")'
+          python3 -c 'from openvino import Core; Core().get_property("BATCH", "SUPPORTED_PROPERTIES")'
           python3 -c 'from openvino.frontend import FrontEndManager; assert len(FrontEndManager().get_available_front_ends()) == 6'
           benchmark_app --help
           ovc --help

.github/workflows/job_onnx_models_tests.yml

@@ -32,6 +32,9 @@ jobs:
       ONNX_MODEL_ZOO_SHA: "d58213534f2a4d1c4b19ba62b3bb5f544353256e"
     if: ${{ github.event_name != 'merge_group' }}
     steps:
+      - name: Set apt retries
+        run: echo 'Acquire::Retries "10";' > /etc/apt/apt.conf.d/80-retries
+
       - name: Download OpenVINO package
         uses: actions/download-artifact@v3
         with:

.github/workflows/job_onnx_runtime.yml

@@ -40,6 +40,9 @@ jobs:
       ONNX_RUNTIME_UTILS: ${{ github.workspace }}/install/onnxruntime
       ONNX_RUNTIME_BUILD_DIR: ${{ github.workspace }}/onnxruntime/build
     steps:
+      - name: Set apt retries
+        run: echo 'Acquire::Retries "10";' > /etc/apt/apt.conf.d/80-retries
+
       - name: Download OpenVINO package
         uses: actions/download-artifact@v3
         with:
@@ -100,7 +103,7 @@ jobs:
         run: bash ${OPENVINO_REPO}/install_build_dependencies.sh
 
       - name: Install sccache
-        uses: mozilla-actions/[email protected].3
+        uses: mozilla-actions/[email protected].4
         with:
           version: "v0.7.5"
 

.github/workflows/job_python_unit_tests.yml

@@ -37,6 +37,9 @@ jobs:
       INSTALL_TEST_DIR: ${{ github.workspace }}/install/tests
       LAYER_TESTS_INSTALL_DIR: ${{ github.workspace }}/install/tests/layer_tests
     steps:
+      - name: Set apt retries
+        run: echo 'Acquire::Retries "10";' > /etc/apt/apt.conf.d/80-retries
+
       - name: Download OpenVINO package
         uses: actions/download-artifact@v3
         with:
@@ -216,6 +219,15 @@ jobs:
           TEST_DEVICE: CPU
           TEST_PRECISION: FP32
 
+      - name: PyTorch torch.export Layer Tests
+        if: ${{ fromJSON(inputs.affected-components).PyTorch_FE.test && runner.arch != 'ARM64' }} # Ticket: 126287
+        run: |
+          python3 -m pytest ${LAYER_TESTS_INSTALL_DIR}/pytorch_tests -m precommit_torch_export --junitxml=${INSTALL_TEST_DIR}/TEST-pytorch.xml
+        env:
+          TEST_DEVICE: CPU
+          TEST_PRECISION: FP32
+          PYTORCH_TRACING_MODE: EXPORT
+
       - name: PyTorch torch.compile TORCHFX Layer Tests
         if: ${{ fromJSON(inputs.affected-components).PyTorch_FE.test && runner.os != 'macOS' }}
         run: |

.github/workflows/job_pytorch_models_tests.yml

@@ -33,11 +33,18 @@ jobs:
       INSTALL_TEST_DIR: ${{ github.workspace }}/install/tests
       MODEL_HUB_TESTS_INSTALL_DIR: ${{ github.workspace }}/install/tests/model_hub_tests
     steps:
-
       - name: Check sudo
         if: ${{ runner.os == 'Linux' }}
         run: if [ "$(id -u)" -eq 0 ]; then apt update && apt --assume-yes install sudo; fi
 
+      - name: Set apt retries
+        run: |
+          if [ "$(id -u)" -eq 0 ]; then
+            echo 'Acquire::Retries "10";' > /etc/apt/apt.conf.d/80-retries
+          else
+            sudo sh -c "echo 'Acquire::Retries \"10\";' >> /etc/apt/apt.conf.d/80-retries"
+          fi
+
       - name: Download OpenVINO package
         uses: actions/download-artifact@v3
         with:

.github/workflows/job_samples_tests.yml

@@ -31,6 +31,9 @@ jobs:
       INSTALL_TEST_DIR: ${{ github.workspace }}/install/tests
       BUILD_DIR: ${{ github.workspace }}/build
     steps:
+      - name: Set apt retries
+        run: echo 'Acquire::Retries "10";' > /etc/apt/apt.conf.d/80-retries
+
       - name: Download OpenVINO package
         uses: actions/download-artifact@v3
         with:

.github/workflows/job_tensorflow_hub_models_tests.yml

@@ -33,11 +33,18 @@ jobs:
       INSTALL_TEST_DIR: ${{ github.workspace }}/install/tests
       MODEL_HUB_TESTS_INSTALL_DIR: ${{ github.workspace }}/install/tests/model_hub_tests
     steps:
-
       - name: Check sudo
         if: ${{ runner.os == 'Linux' }}
         run: if [ "$(id -u)" -eq 0 ]; then apt update && apt --assume-yes install sudo; fi
 
+      - name: Set apt retries
+        run: |
+          if [ "$(id -u)" -eq 0 ]; then
+            echo 'Acquire::Retries "10";' > /etc/apt/apt.conf.d/80-retries
+          else
+            sudo sh -c "echo 'Acquire::Retries \"10\";' >> /etc/apt/apt.conf.d/80-retries"
+          fi
+
       - name: Download OpenVINO package
         uses: actions/download-artifact@v3
         with:

.github/workflows/linux.yml

@@ -80,6 +80,9 @@ jobs:
     if: "!needs.smart_ci.outputs.skip_workflow"
 
     steps:
+      - name: Set apt retries
+        run: echo 'Acquire::Retries "10";' > /etc/apt/apt.conf.d/80-retries
+
       - name: Install git
         run: |
           apt-get update
@@ -117,7 +120,7 @@ jobs:
           apt install --assume-yes --no-install-recommends default-jdk
 
       - name: Install sccache
-        uses: mozilla-actions/[email protected].3
+        uses: mozilla-actions/[email protected].4
         with:
           version: "v0.7.5"
 
@@ -602,6 +605,9 @@ jobs:
     if: fromJSON(needs.smart_ci.outputs.affected_components).NVIDIA
 
     steps:
+      - name: Set apt retries
+        run: echo 'Acquire::Retries "10";' > /etc/apt/apt.conf.d/80-retries
+
       - name: Fetch install_build_dependencies.sh
         uses: actions/checkout@v4
         with:
@@ -652,7 +658,7 @@ jobs:
           apt -y --no-install-recommends install software-properties-common curl
 
       - name: Install sccache
-        uses: mozilla-actions/[email protected].3
+        uses: mozilla-actions/[email protected].4
         with:
           version: "v0.7.5"
 
@@ -691,6 +697,98 @@ jobs:
       - name: Show ccache stats
         run: ${SCCACHE_PATH} --show-stats
 
+  Openvino_tokenizers:
+    name: OpenVINO tokenizers extension
+    needs: [ Build, Smart_CI ]
+    timeout-minutes: 25
+    defaults:
+      run:
+        shell: bash
+    runs-on: aks-linux-4-cores-16gb
+    container:
+      image: openvinogithubactions.azurecr.io/dockerhub/ubuntu:20.04
+      volumes:
+        - /mount:/mount
+    env:
+      INSTALL_DIR: /__w/openvino/openvino/install
+      OPENVINO_CONTRIB_REPO: /__w/openvino/openvino/openvino_contrib
+      OPENVINO_TOKENIZERS_REPO: /__w/openvino/openvino/openvino_contrib/modules/custom_operations
+      EXTENSION_BUILD_DIR: /__w/openvino/openvino/openvino_tokenizers
+    if: fromJSON(needs.smart_ci.outputs.affected_components).TOKENIZERS
+
+    steps:
+      - name: checkout action
+        uses: actions/checkout@v4
+        with:
+          sparse-checkout: |
+            .github/actions/setup_python
+            install_build_dependencies.sh
+
+      - name: Setup Python ${{ env.PYTHON_VERSION }}
+        uses: ./.github/actions/setup_python
+        with:
+          version: ${{ env.PYTHON_VERSION }}
+          pip-cache-path: ${{ env.PIP_CACHE_PATH }}
+          should-setup-pip-paths: 'true'
+          self-hosted-runner: 'true'
+          show-cache-info: 'false'
+
+      - name: Clone OpenVINO Contrib
+        uses: actions/checkout@v4
+        with:
+          repository: 'openvinotoolkit/openvino_contrib'
+          path: ${{ env.OPENVINO_CONTRIB_REPO }}
+          ref: 'master'
+
+      - name: Download OpenVINO package
+        uses: actions/download-artifact@v3
+        with:
+          name: openvino_package
+          path: ${{ env.INSTALL_DIR }}
+
+      - name: Extract OpenVINO packages
+        run: |
+          pushd ${INSTALL_DIR}
+            tar -xzf openvino_package.tar.gz -C ${INSTALL_DIR}
+          popd
+
+      #
+      # Dependencies
+      #
+
+      - name: Install build dependencies
+        run: ./install_build_dependencies.sh
+
+      - name: Install python dependencies
+        run: |
+          # wheel packaging
+          python3 -m pip install -r ${OPENVINO_TOKENIZERS_REPO}/requirements-build.txt
+
+      #
+      # Build
+      #
+
+      - name: Build tokenizers wheel
+        run: |
+          source ${INSTALL_DIR}/setupvars.sh
+          python -m build --wheel --outdir ${EXTENSION_BUILD_DIR} ${OPENVINO_TOKENIZERS_REPO}
+        env:
+          CMAKE_ARGS: '-DBUILD_FAST_TOKENIZERS=OFF'
+          CMAKE_BUILD_PARALLEL_LEVEL: '4'
+          CMAKE_GENERATOR: 'Unix Makefiles'
+
+      #
+      # Upload build artifacts
+      #
+
+      - name: Upload openvino tokenizers wheel
+        if: ${{ always() }}
+        uses: actions/upload-artifact@v3
+        with:
+          name: openvino_tokenizers_wheel
+          path: ${{ env.EXTENSION_BUILD_DIR }}/*.whl
+          if-no-files-found: 'error'
+
   GPU_Stub:
     needs: [ Build, Smart_CI ]
     runs-on: ubuntu-latest
@@ -705,7 +803,7 @@ jobs:
   Overall_Status:
     name: ci/gha_overall_status
     needs: [Smart_CI, Build, Debian_Packages, Samples, Conformance, ONNX_Runtime, CXX_Unit_Tests, Python_Unit_Tests,
-            CPU_Functional_Tests, TensorFlow_Hub_Models_Tests, PyTorch_Models_Tests, NVIDIA_Plugin]
+            CPU_Functional_Tests, TensorFlow_Hub_Models_Tests, PyTorch_Models_Tests, NVIDIA_Plugin, Openvino_tokenizers]
     if: ${{ always() }}
     runs-on: ubuntu-latest
     steps:

.github/workflows/linux_arm64.yml

@@ -82,6 +82,9 @@ jobs:
     if: "!needs.smart_ci.outputs.skip_workflow"
 
     steps:
+      - name: Set apt retries
+        run: echo 'Acquire::Retries "10";' > /etc/apt/apt.conf.d/80-retries
+
       - name: Install git
         run: apt-get update && apt-get install --assume-yes --no-install-recommends git ca-certificates
 
@@ -117,7 +120,7 @@ jobs:
           apt install --assume-yes --no-install-recommends default-jdk
 
       - name: Install sccache
-        uses: mozilla-actions/[email protected].3
+        uses: mozilla-actions/[email protected].4
         with:
           version: "v0.7.5"