Merge remote-tracking branch 'upstream/master' into ci/gha/add-saniti…

…zer-checks
akashchi · Feb 6, 2024 · 0e65932 · 0e65932
2 parents e13b649 + 39c35a2
commit 0e65932
Show file tree

Hide file tree

Showing 1,115 changed files with 9,429 additions and 29,256 deletions.
diff --git a/.github/components.yml b/.github/components.yml
@@ -82,7 +82,6 @@ TEMPLATE:
     - C_API
     - Python_API
     - NVIDIA
-    - TOKENIZERS
   build:
     - IR_FE
 
@@ -130,6 +129,7 @@ TF_FE:
   build:
     - CPU
     - Python_API
+    - TOKENIZERS
 
 TFL_FE:
   revalidate:
@@ -144,6 +144,7 @@ PyTorch_FE:
   build:
     - CPU
     - Python_API
+    - TOKENIZERS
 
 C_API:
   build:
@@ -196,7 +197,6 @@ IE_Tests:
     - TEMPLATE
     - AUTO
     - NVIDIA
-    - TOKENIZERS
   build:
     - IR_FE
 

diff --git a/.github/workflows/android_arm64.yml b/.github/workflows/android_arm64.yml
@@ -5,7 +5,7 @@ on:
   merge_group:
   push:
     branches:
-      - master
+      # - master
       - 'releases/**'
 
 concurrency:

diff --git a/.github/workflows/fedora.yml b/.github/workflows/fedora.yml
@@ -5,7 +5,7 @@ on:
   merge_group:
   push:
     branches:
-      - master
+      # - master
       - 'releases/**'
 
 concurrency:

diff --git a/.github/workflows/job_cpu_functional_tests.yml b/.github/workflows/job_cpu_functional_tests.yml
@@ -29,7 +29,6 @@ jobs:
       INSTALL_TEST_DIR: ${{ github.workspace }}/install/tests
       PARALLEL_TEST_SCRIPT: ${{ github.workspace }}/install/tests/functional_test_utils/layer_tests_summary/run_parallel.py
       PARALLEL_TEST_CACHE: ${{ github.workspace }}/install/tests/test_cache.lst
-    if: ${{ github.event_name != 'merge_group' }}
     steps:
       - name: Set apt retries
         run: echo 'Acquire::Retries "10";' > /etc/apt/apt.conf.d/80-retries

diff --git a/.github/workflows/job_cxx_unit_tests.yml b/.github/workflows/job_cxx_unit_tests.yml
@@ -31,7 +31,6 @@ jobs:
       DEBIAN_FRONTEND: noninteractive # to prevent apt-get from waiting user input
       INSTALL_DIR: ${{ github.workspace }}/install
       INSTALL_TEST_DIR: ${{ github.workspace }}/install/tests
-    if: ${{ github.event_name != 'merge_group' }}
     steps:
       - name: Set apt retries
         run: echo 'Acquire::Retries "10";' > /etc/apt/apt.conf.d/80-retries

diff --git a/.github/workflows/job_pytorch_models_tests.yml b/.github/workflows/job_pytorch_models_tests.yml
@@ -51,6 +51,12 @@ jobs:
           name: openvino_package
           path: ${{ env.INSTALL_DIR }}
 
+      - name: Download OpenVINO tokenizers extension
+        uses: actions/download-artifact@v3
+        with:
+          name: openvino_tokenizers_wheel
+          path: ${{ env.INSTALL_DIR }}
+
       - name: Download OpenVINO tests package
         uses: actions/download-artifact@v3
         with:
@@ -98,7 +104,9 @@ jobs:
           self-hosted-runner: ${{ contains(inputs.runner, 'aks') }}
 
       - name: Install OpenVINO Python wheels
-        run: python3 -m pip install ${INSTALL_DIR}/tools/openvino-*
+        run: |
+          python3 -m pip install ${INSTALL_DIR}/tools/openvino-*
+          python3 -m pip install ${INSTALL_DIR}/openvino_tokenizers-*
 
       - name: Install PyTorch tests requirements
         run: |

diff --git a/.github/workflows/job_tensorflow_hub_models_tests.yml b/.github/workflows/job_tensorflow_hub_models_tests.yml
@@ -51,6 +51,12 @@ jobs:
           name: openvino_package
           path: ${{ env.INSTALL_DIR }}
 
+      - name: Download OpenVINO tokenizers extension
+        uses: actions/download-artifact@v3
+        with:
+          name: openvino_tokenizers_wheel
+          path: ${{ env.INSTALL_DIR }}
+
       - name: Download OpenVINO tests package
         uses: actions/download-artifact@v3
         with:
@@ -97,7 +103,9 @@ jobs:
           self-hosted-runner: ${{ contains(inputs.runner, 'aks') }}
 
       - name: Install OpenVINO Python wheels
-        run: python3 -m pip install ${INSTALL_DIR}/tools/openvino-*
+        run: |
+          python3 -m pip install ${INSTALL_DIR}/tools/openvino-*
+          python3 -m pip install ${INSTALL_DIR}/openvino_tokenizers-*
 
       - name: Install TF Hub tests requirements
         run: python3 -m pip install -r ${MODEL_HUB_TESTS_INSTALL_DIR}/tf_hub_tests/requirements.txt

diff --git a/.github/workflows/linux.yml b/.github/workflows/linux.yml
@@ -480,7 +480,7 @@ jobs:
     name: TensorFlow Hub Models tests
     if: fromJSON(needs.smart_ci.outputs.affected_components).TF_FE.test ||
         fromJSON(needs.smart_ci.outputs.affected_components).TFL_FE.test
-    needs: [ Build, Smart_CI ]
+    needs: [ Build, Smart_CI, Openvino_tokenizers ]
     uses: ./.github/workflows/job_tensorflow_hub_models_tests.yml
     with:
       runner: ${{ github.event_name == 'schedule' && 'ubuntu-20.04-16-cores' || 'ubuntu-20.04-8-cores' }}
@@ -494,7 +494,7 @@ jobs:
   PyTorch_Models_Tests:
     name: PyTorch Models tests
     if: fromJSON(needs.smart_ci.outputs.affected_components).PyTorch_FE.test
-    needs: [ Build, Smart_CI ]
+    needs: [ Build, Smart_CI, Openvino_tokenizers ]
     uses: ./.github/workflows/job_pytorch_models_tests.yml
     with:
       runner: ${{ github.event_name == 'schedule' && 'ubuntu-20.04-16-cores' || 'ubuntu-20.04-8-cores' }}

diff --git a/.github/workflows/linux_arm64.yml b/.github/workflows/linux_arm64.yml
@@ -5,7 +5,7 @@ on:
   merge_group:
   push:
     branches:
-      - master
+      # - master
       - 'releases/**'
 
 concurrency:

diff --git a/.github/workflows/linux_riscv.yml b/.github/workflows/linux_riscv.yml
@@ -60,8 +60,8 @@ jobs:
       OPENVINO_REPO: /__w/openvino/openvino/openvino
       OPENVINO_BUILD_DIR: /__w/openvino/openvino/openvino_build
       INSTALL_DIR: /__w/openvino/openvino/openvino_install
-      CONAN_USER_HOME: /mount/caches/ccache/ubuntu22_riscv64_master_release/.conan
-      CCACHE_DIR: /mount/caches/ccache/ubuntu22_riscv64_master_release
+      CONAN_USER_HOME: /mount/caches/ccache/ubuntu22_riscv64_master/.conan
+      CCACHE_DIR: /mount/caches/ccache/ubuntu22_riscv64_master
       CCACHE_TEMPDIR: /__w/openvino/openvino/ccache_temp
       CCACHE_MAXSIZE: 50G
     if: ${{ !needs.smart_ci.outputs.skip_workflow && github.event_name != 'merge_group' }}
@@ -220,7 +220,7 @@ jobs:
       - name: Check status of all jobs
         if: >-
           ${{
-            contains(needs.*.result, 'failure') || 
+            contains(needs.*.result, 'failure') ||
             contains(needs.*.result, 'cancelled')
           }}
         run: exit 1
diff --git a/.github/workflows/webassembly.yml b/.github/workflows/webassembly.yml
@@ -5,7 +5,7 @@ on:
   merge_group:
   push:
     branches:
-      - master
+      # - master
       - 'releases/**'
 
 concurrency:

diff --git a/.github/workflows/windows.yml b/.github/workflows/windows.yml
@@ -252,8 +252,7 @@ jobs:
           if-no-files-found: 'error'
 
       - name: Upload openvino js package
-        if: ${{ 'false' }} # 128689
-        # if: fromJSON(needs.smart_ci.outputs.affected_components).JS_API
+        if: fromJSON(needs.smart_ci.outputs.affected_components).JS_API
         uses: actions/upload-artifact@v3
         with:
           name: openvino_js_package
@@ -364,8 +363,7 @@ jobs:
     env:
       OPENVINO_JS_DIR: "${{ github.workspace }}\\openvino\\src\\bindings\\js"
       OPENVINO_JS_LIBS_DIR: "${{ github.workspace }}\\openvino\\src\\bindings\\js\\node\\bin"
-    if: ${{ 'false' }} # 128689
-    # if: fromJSON(needs.smart_ci.outputs.affected_components).JS_API
+    if: fromJSON(needs.smart_ci.outputs.affected_components).JS_API
 
     steps:
       - name: Fetch OpenVINO JS sources

diff --git a/...ion/openvino_ir/operation_sets/operations_specifications/activation/Swish_4.rst b/...ion/openvino_ir/operation_sets/operations_specifications/activation/Swish_4.rst
@@ -5,7 +5,7 @@ Swish
 
 
 .. meta::
-  :description: Learn about Swish-4 - an element-wise, activation operation, which 
+  :description: Learn about Swish-4 - an element-wise, activation operation, which
                 can be performed on a single tensor in OpenVINO.
 
 **Versioned name**: *Swish-4*
@@ -55,7 +55,7 @@ Example: Second input ``beta`` provided
                 <dim>256</dim>
                 <dim>56</dim>
             </port>
-            <port id="1">  < !-- beta value: 2.0 -->
+            <port id="1">  <!-- beta value: 2.0 -->
             </port>
         </input>
         <output>

diff --git a/...on/openvino_ir/operation_sets/operations_specifications/arithmetic/CumSum_3.rst b/...on/openvino_ir/operation_sets/operations_specifications/arithmetic/CumSum_3.rst
@@ -5,7 +5,7 @@ CumSum
 
 
 .. meta::
-  :description: Learn about CumSum-3 - an element-wise, arithmetic operation, which 
+  :description: Learn about CumSum-3 - an element-wise, arithmetic operation, which
                 can be performed on a single tensor in OpenVINO.
 
 **Versioned name**: *CumSum-3*
@@ -24,7 +24,7 @@ To perform the summation in the opposite direction of the axis, set reverse attr
 
   * **Description**: If the attribute is set to ``true``, then exclusive sums are returned, the ``j-th`` element is not included in the ``j-th`` sum. Otherwise, the inclusive sum of the first ``j`` elements for the ``j-th`` element is calculated.
   * **Range of values**:
-    
+
     * ``false`` - include the top element
     * ``true`` - do not include the top element
   * **Type**: ``boolean``
@@ -35,7 +35,7 @@ To perform the summation in the opposite direction of the axis, set reverse attr
 
   * **Description**: If set to ``true`` will perform the sums in reverse direction.
   * **Range of values**:
-    
+
     * ``false`` - do not perform sums in reverse direction
     * ``true`` - perform sums in reverse direction
   * **Type**: ``boolean``
@@ -63,16 +63,16 @@ To perform the summation in the opposite direction of the axis, set reverse attr
 
 .. code-block:: xml
    :force:
-   
+
    <layer ... type="CumSum" exclusive="0" reverse="0">
        <input>
-           <port id="0">     < !-- input value is: [1., 2., 3., 4., 5.] -->
+           <port id="0">     <!-- input value is: [1., 2., 3., 4., 5.] -->
                <dim>5</dim>
            </port>
-           <port id="1"/>     < !-- axis value is: 0 -->
+           <port id="1"/>     <!-- axis value is: 0 -->
        </input>
        <output>
-           <port id="2">     < !-- output value is: [1., 3., 6., 10., 15.] -->
+           <port id="2">     <!-- output value is: [1., 3., 6., 10., 15.] -->
                <dim>5</dim>
            </port>
        </output>
@@ -82,16 +82,16 @@ To perform the summation in the opposite direction of the axis, set reverse attr
 
 .. code-block:: xml
    :force:
-   
+
    <layer ... type="CumSum" exclusive="1" reverse="0">
        <input>
-           <port id="0">     < !-- input value is: [1., 2., 3., 4., 5.] -->
+           <port id="0">     <!-- input value is: [1., 2., 3., 4., 5.] -->
                <dim>5</dim>
            </port>
-           <port id="1"/>     < !-- axis value is: 0 -->
+           <port id="1"/>     <!-- axis value is: 0 -->
        </input>
        <output>
-           <port id="2">     < !-- output value is: [0., 1., 3., 6., 10.] -->
+           <port id="2">     <!-- output value is: [0., 1., 3., 6., 10.] -->
                <dim>5</dim>
            </port>
        </output>
@@ -101,16 +101,16 @@ To perform the summation in the opposite direction of the axis, set reverse attr
 
 .. code-block:: xml
    :force:
-   
+
    <layer ... type="CumSum" exclusive="0" reverse="1">
        <input>
-           <port id="0">     < !-- input value is: [1., 2., 3., 4., 5.] -->
+           <port id="0">     <!-- input value is: [1., 2., 3., 4., 5.] -->
                <dim>5</dim>
            </port>
-           <port id="1"/>     < !-- axis value is: 0 -->
+           <port id="1"/>     <!-- axis value is: 0 -->
        </input>
        <output>
-           <port id="2">     < !-- output value is: [15., 14., 12., 9., 5.] -->
+           <port id="2">     <!-- output value is: [15., 14., 12., 9., 5.] -->
                <dim>5</dim>
            </port>
        </output>
@@ -120,7 +120,7 @@ To perform the summation in the opposite direction of the axis, set reverse attr
 
 .. code-block:: xml
    :force:
-   
+
    <layer ... type="CumSum" exclusive="1" reverse="1">
        <input>
            <port id="0">     < -- input value is: [1., 2., 3., 4., 5.] -->

diff --git a/...tion/openvino_ir/operation_sets/operations_specifications/arithmetic/Sqrt_1.rst b/...tion/openvino_ir/operation_sets/operations_specifications/arithmetic/Sqrt_1.rst
@@ -5,7 +5,7 @@ Sqrt
 
 
 .. meta::
-  :description: Learn about Sqrt-1 - an element-wise, arithmetic operation, which 
+  :description: Learn about Sqrt-1 - an element-wise, arithmetic operation, which
                 can be performed on a single tensor in OpenVINO.
 
 **Versioned name**: *Sqrt-1*
@@ -48,12 +48,12 @@ Sqrt
     <layer ... type="Sqrt">
         <input>
             <port id="0">
-                <dim>4</dim> < !-- float input values: [4.0, 7.0, 9.0, 10.0] -->
+                <dim>4</dim> <!-- float input values: [4.0, 7.0, 9.0, 10.0] -->
             </port>
         </input>
         <output>
             <port id="1">
-                <dim>4</dim> < !-- float output values: [2.0, 2.6457512, 3.0, 3.1622777] -->
+                <dim>4</dim> <!-- float output values: [2.0, 2.6457512, 3.0, 3.1622777] -->
             </port>
         </output>
     </layer>
@@ -66,12 +66,12 @@ Sqrt
     <layer ... type="Sqrt">
         <input>
             <port id="0">
-                <dim>4</dim> < !-- int input values: [4, 7, 9, 10] -->
+                <dim>4</dim> <!-- int input values: [4, 7, 9, 10] -->
             </port>
         </input>
         <output>
             <port id="1">
-                <dim>4</dim> < !-- int output values: [2, 3, 3, 3] -->
+                <dim>4</dim> <!-- int output values: [2, 3, 3, 3] -->
             </port>
         </output>
     </layer>

diff --git a/...openvino_ir/operation_sets/operations_specifications/comparison/IsFinite_10.rst b/...openvino_ir/operation_sets/operations_specifications/comparison/IsFinite_10.rst
@@ -5,7 +5,7 @@ IsFinite
 
 
 .. meta::
-  :description: Learn about IsFinite-10 - an element-wise, comparison operation, which 
+  :description: Learn about IsFinite-10 - an element-wise, comparison operation, which
                 can be performed on a single tensor in OpenVINO.
 
 **Versioned name**: *IsFinite-10*
@@ -64,12 +64,12 @@ IsFinite
    <layer ... type="IsFinite">
        <input>
            <port id="0" precision="FP32">
-               <dim>4</dim> < !-- Input value is: [NaN, 2.1, 3.7, Inf] -->
+               <dim>4</dim> <!-- Input value is: [NaN, 2.1, 3.7, Inf] -->
            </port>
        </input>
        <output>
            <port id="1" precision="BOOL">
-               <dim>4</dim> < !-- Output value is: [False, True, True, False] -->
+               <dim>4</dim> <!-- Output value is: [False, True, True, False] -->
            </port>
        </output>
    </layer>