[CI] Prebuild openvino packages (#69)

* 2023.3.0 branch * DENABLE_PYTHON_PACKAGING * enaled openvino wheel build * compatibility reqs * set 2023.3.0.1 ver * setupvars.bat * use cmd for win * fixed cmd syntax * exlude_from_all for FetchContent * fixed typo * reverted fast_tokenizers fetch * Renamed back to user_ov_extensions for 2023.3 (#31) * download_ov_package * use install dir * remove root path * changed condition * changed job condition * fixed condition * upload artifact * strip tarball * set test condition * use pip openvino version * use openvino py from the package * test build * fail download * check tarball * continue-on-error * check status step * use outcome * cleanup * fixed extract * test build branch * added mac * test conditions * win pipeline * fixed test archive command * strip dirs * changed links to nightly * setupvars * fixed test conditions * test openvino builds * cleanup * fixed install subdir --------- Co-authored-by: Artur Paniukov <[email protected]> Co-authored-by: Ilya Lavrenov <[email protected]>
openvinotoolkit · Mar 13, 2024 · 3cbd339 · 3cbd339
1 parent 4db4de6
commit 3cbd339
Show file tree

Hide file tree

Showing 3 changed files with 173 additions and 28 deletions.
diff --git a/.github/workflows/linux.yml b/.github/workflows/linux.yml
@@ -14,9 +14,42 @@ concurrency:
 
 env:
   PYTHON_VERSION: '3.11'
+  OV_BRANCH: 'master'
+  OV_TARBALL: ''
 
 jobs:
+  openvino_download:
+    outputs:
+      status: ${{ steps.openvino_download.outcome }}
+    timeout-minutes: 10
+    defaults:
+      run:
+        shell: bash
+    runs-on: ubuntu-20.04
+
+    steps:
+      - name: Download OpenVINO build
+        id: openvino_download
+        run: |
+          wget ${{ env.OV_TARBALL}} --progress=bar:force:noscroll -O openvino_package.tar.gz
+          tar -tvf openvino_package.tar.gz
+        continue-on-error: true
+
+      #
+      # Upload to artifacts
+      #
+
+      - name: Upload openvino package
+        if: steps.openvino_download.outcome == 'success'
+        uses: actions/upload-artifact@v4
+        with:
+          name: openvino_package
+          path: openvino_package.tar.gz
+          if-no-files-found: 'error'
+
   openvino_build:
+    needs: [openvino_download]
+    if: needs.openvino_download.outputs.status != 'success'
     timeout-minutes: 150
     defaults:
       run:
@@ -47,7 +80,7 @@ jobs:
           repository: 'openvinotoolkit/openvino'
           path: ${{ env.OPENVINO_REPO }}
           submodules: 'true'
-          ref: 'master'
+          ref: ${{ env.OV_BRANCH}}
 
       #
       # Dependencies
@@ -107,6 +140,8 @@ jobs:
             -DCMAKE_COMPILE_WARNING_AS_ERROR=ON \
             -DCMAKE_CXX_COMPILER_LAUNCHER=ccache \
             -DCMAKE_C_COMPILER_LAUNCHER=ccache \
+            -DENABLE_PYTHON=ON \
+            -DENABLE_WHEEL=ON \
             -S ${OPENVINO_REPO} \
             -B ${BUILD_DIR}
 
@@ -121,8 +156,8 @@ jobs:
 
       - name: Cmake install - OpenVINO
         run: |
-          cmake -DCMAKE_INSTALL_PREFIX=${INSTALL_DIR} -P ${BUILD_DIR}/cmake_install.cmake
-          cmake -DCMAKE_INSTALL_PREFIX=${INSTALL_DIR} -DCOMPONENT=python_wheels -P ${BUILD_DIR}/cmake_install.cmake
+          cmake -DCMAKE_INSTALL_PREFIX=${INSTALL_DIR}/openvino_package -P ${BUILD_DIR}/cmake_install.cmake
+          cmake -DCMAKE_INSTALL_PREFIX=${INSTALL_DIR}/openvino_package -DCOMPONENT=python_wheels -P ${BUILD_DIR}/cmake_install.cmake
 
       - name: Pack Artifacts
         run: |
@@ -144,7 +179,10 @@ jobs:
 
   openvino_tokenizers:
     name: OpenVINO tokenizers extension
-    needs: [ openvino_build ]
+    needs: [ openvino_download, openvino_build ]
+    if: |
+      always() &&
+      (needs.openvino_download.outputs.status == 'success' || needs.openvino_build.result == 'success')
     timeout-minutes: 25
     defaults:
       run:
@@ -168,7 +206,7 @@ jobs:
         with:
           repository: 'openvinotoolkit/openvino'
           path: ${{ env.OPENVINO_REPO }}
-          ref: 'master'
+          ref: ${{ env.OV_BRANCH }}
           sparse-checkout: |
             install_build_dependencies.sh
 
@@ -187,7 +225,7 @@ jobs:
       - name: Extract OpenVINO packages
         run: |
           pushd ${INSTALL_DIR}
-            tar -xzf openvino_package.tar.gz -C ${INSTALL_DIR}
+            tar -xzf openvino_package.tar.gz -C ${INSTALL_DIR} --strip-components=1
           popd
 
       #
@@ -230,7 +268,8 @@ jobs:
 
   openvino_tokenizers_tests:
     name: OpenVINO tokenizers tests
-    needs: [ openvino_tokenizers ]
+    needs: [ openvino_download, openvino_build, openvino_tokenizers]
+    if: always() && needs.openvino_tokenizers.result == 'success'
     timeout-minutes: 25
     defaults:
       run:
@@ -267,13 +306,14 @@ jobs:
           name: openvino_package
           path: ${{ env.INSTALL_DIR }}
 
-      - name: Extract OpenVINO packages
+      - name: Extract OpenVINO build package
         run: |
           pushd ${INSTALL_DIR}
-            tar -xzf openvino_package.tar.gz -C ${INSTALL_DIR}
+            tar -xzf openvino_package.tar.gz -C ${INSTALL_DIR} --strip-components=1
           popd
 
       - name: Install OpenVINO Python wheel
+        if: needs.openvino_build.result == 'success'
         run: |
           # Find and install wheel
           pushd ${INSTALL_DIR}/tools
@@ -289,7 +329,15 @@ jobs:
             python3 -m pip install $wheel_name[dev]
           popd
 
-      - name: Tokenizers regression tests
+      - name: Tokenizers regression tests (using openvino python modules)
+        if: needs.openvino_download.outputs.status == 'success'
+        run: |
+          source ${INSTALL_DIR}/setupvars.sh
+          python3 -m pytest tokenizers_test.py
+        working-directory: ${{ env.OPENVINO_TOKENIZERS_REPO }}/tests
+
+      - name: Tokenizers regression tests (using openvino python wheel)
+        if: needs.openvino_build.result == 'success'
         run: |
           python3 -m pytest tokenizers_test.py
         working-directory: ${{ env.OPENVINO_TOKENIZERS_REPO }}/tests

diff --git a/.github/workflows/mac.yml b/.github/workflows/mac.yml
@@ -15,9 +15,42 @@ concurrency:
 env:
   PYTHON_VERSION: '3.11'
   MACOSX_DEPLOYMENT_TARGET: '10.12'
+  OV_BRANCH: 'master'
+  OV_TARBALL: ''
 
 jobs:
+  openvino_download:
+    outputs:
+      status: ${{ steps.openvino_download.outcome }}
+    timeout-minutes: 10
+    defaults:
+      run:
+        shell: bash
+    runs-on: ubuntu-20.04
+
+    steps:
+      - name: Download OpenVINO build
+        id: openvino_download
+        run: |
+          wget ${{ env.OV_TARBALL}} --progress=bar:force:noscroll -O openvino_package.tar.gz
+          tar -tvf openvino_package.tar.gz
+        continue-on-error: true
+
+      #
+      # Upload to artifacts
+      #
+
+      - name: Upload openvino package
+        if: steps.openvino_download.outcome == 'success'
+        uses: actions/upload-artifact@v4
+        with:
+          name: openvino_package
+          path: openvino_package.tar.gz
+          if-no-files-found: 'error'
+
   openvino_build:
+    needs: [openvino_download]
+    if: needs.openvino_download.outputs.status != 'success'
     timeout-minutes: 150
     defaults:
       run:
@@ -39,7 +72,7 @@ jobs:
           repository: 'openvinotoolkit/openvino'
           path: ${{ env.OPENVINO_REPO }}
           submodules: 'true'
-          ref: 'master'
+          ref: ${{ env.OV_BRANCH }}
 
       #
       # Dependencies
@@ -98,6 +131,8 @@ jobs:
             -DCMAKE_COMPILE_WARNING_AS_ERROR=ON \
             -DCMAKE_CXX_COMPILER_LAUNCHER=ccache \
             -DCMAKE_C_COMPILER_LAUNCHER=ccache \
+            -DENABLE_PYTHON=ON \
+            -DENABLE_WHEEL=ON \
             -S ${OPENVINO_REPO} \
             -B ${BUILD_DIR}
 
@@ -112,8 +147,8 @@ jobs:
 
       - name: Cmake install - OpenVINO
         run: |
-          cmake -DCMAKE_INSTALL_PREFIX=${INSTALL_DIR} -P ${BUILD_DIR}/cmake_install.cmake
-          cmake -DCMAKE_INSTALL_PREFIX=${INSTALL_DIR} -DCOMPONENT=python_wheels -P ${BUILD_DIR}/cmake_install.cmake
+          cmake -DCMAKE_INSTALL_PREFIX=${INSTALL_DIR}/openvino_package -P ${BUILD_DIR}/cmake_install.cmake
+          cmake -DCMAKE_INSTALL_PREFIX=${INSTALL_DIR}/openvino_package -DCOMPONENT=python_wheels -P ${BUILD_DIR}/cmake_install.cmake
 
       - name: Pack Artifacts
         run: |
@@ -135,7 +170,10 @@ jobs:
 
   openvino_tokenizers:
     name: OpenVINO tokenizers extension
-    needs: [ openvino_build ]
+    needs: [ openvino_download, openvino_build ]
+    if: |
+      always() &&
+      (needs.openvino_download.outputs.status == 'success' || needs.openvino_build.result == 'success')
     timeout-minutes: 25
     defaults:
       run:
@@ -159,7 +197,7 @@ jobs:
         with:
           repository: 'openvinotoolkit/openvino'
           path: ${{ env.OPENVINO_REPO }}
-          ref: 'master'
+          ref: ${{ env.OV_BRANCH }}
           sparse-checkout: |
             install_build_dependencies.sh
 
@@ -178,7 +216,7 @@ jobs:
       - name: Extract OpenVINO packages
         run: |
           pushd ${INSTALL_DIR}
-            tar -xzf openvino_package.tar.gz -C ${INSTALL_DIR}
+            tar -xzf openvino_package.tar.gz -C ${INSTALL_DIR} --strip-components=1
           popd
 
       #
@@ -220,7 +258,8 @@ jobs:
 
   openvino_tokenizers_tests:
     name: OpenVINO tokenizers tests
-    needs: [ openvino_tokenizers ]
+    needs: [ openvino_download, openvino_build, openvino_tokenizers]
+    if: always() && needs.openvino_tokenizers.result == 'success'
     timeout-minutes: 25
     defaults:
       run:
@@ -260,10 +299,11 @@ jobs:
       - name: Extract OpenVINO packages
         run: |
           pushd ${INSTALL_DIR}
-            tar -xzf openvino_package.tar.gz -C ${INSTALL_DIR}
+            tar -xzf openvino_package.tar.gz -C ${INSTALL_DIR} --strip-components=1
           popd
 
       - name: Install OpenVINO Python wheel
+        if: needs.openvino_build.result == 'success'
         run: |
           # Find and install wheel
           pushd ${INSTALL_DIR}/tools
@@ -279,7 +319,15 @@ jobs:
             python3 -m pip install $wheel_name[dev]
           popd
 
-      - name: Tokenizers regression tests
+      - name: Tokenizers regression tests (using openvino python modules)
+        if: needs.openvino_download.outputs.status == 'success'
+        run: |
+          source ${INSTALL_DIR}/setupvars.sh
+          python3 -m pytest tokenizers_test.py
+        working-directory: ${{ env.OPENVINO_TOKENIZERS_REPO }}/tests
+
+      - name: Tokenizers regression tests (using openvino python wheel)
+        if: needs.openvino_build.result == 'success'
         run: |
           python3 -m pytest tokenizers_test.py
         working-directory: ${{ env.OPENVINO_TOKENIZERS_REPO }}/tests