From da5c8b65d446895ed22ed0d3f7d3c9d8785b1195 Mon Sep 17 00:00:00 2001 From: Mikhail Ryzhov Date: Wed, 13 Mar 2024 13:54:07 +0100 Subject: [PATCH 1/2] [CI] Prebuild openvino packages (#69) * 2023.3.0 branch * DENABLE_PYTHON_PACKAGING * enaled openvino wheel build * compatibility reqs * set 2023.3.0.1 ver * setupvars.bat * use cmd for win * fixed cmd syntax * exlude_from_all for FetchContent * fixed typo * reverted fast_tokenizers fetch * Renamed back to user_ov_extensions for 2023.3 (#31) * download_ov_package * use install dir * remove root path * changed condition * changed job condition * fixed condition * upload artifact * strip tarball * set test condition * use pip openvino version * use openvino py from the package * test build * fail download * check tarball * continue-on-error * check status step * use outcome * cleanup * fixed extract * test build branch * added mac * test conditions * win pipeline * fixed test archive command * strip dirs * changed links to nightly * setupvars * fixed test conditions * test openvino builds * cleanup * fixed install subdir --------- Co-authored-by: Artur Paniukov Co-authored-by: Ilya Lavrenov --- .github/workflows/linux.yml | 66 +++++++++++++++++++++++++++++------ .github/workflows/mac.yml | 64 ++++++++++++++++++++++++++++----- .github/workflows/windows.yml | 66 ++++++++++++++++++++++++++++++----- 3 files changed, 168 insertions(+), 28 deletions(-) diff --git a/.github/workflows/linux.yml b/.github/workflows/linux.yml index 12f8e4643..cb61fe7b2 100644 --- a/.github/workflows/linux.yml +++ b/.github/workflows/linux.yml @@ -14,9 +14,42 @@ concurrency: env: PYTHON_VERSION: '3.11' + OV_BRANCH: 'releases/2023/3' + OV_TARBALL: 'https://storage.openvinotoolkit.org/repositories/openvino/packages/2023.3/linux/l_openvino_toolkit_ubuntu20_2023.3.0.13775.ceeafaf64f3_x86_64.tgz' jobs: + openvino_download: + outputs: + status: ${{ steps.openvino_download.outcome }} + timeout-minutes: 10 + defaults: + run: + shell: bash + runs-on: ubuntu-20.04 + + steps: + - name: Download OpenVINO build + id: openvino_download + run: | + wget ${{ env.OV_TARBALL}} --progress=bar:force:noscroll -O openvino_package.tar.gz + tar -tvf openvino_package.tar.gz + continue-on-error: true + + # + # Upload to artifacts + # + + - name: Upload openvino package + if: steps.openvino_download.outcome == 'success' + uses: actions/upload-artifact@v4 + with: + name: openvino_package + path: openvino_package.tar.gz + if-no-files-found: 'error' + openvino_build: + needs: [openvino_download] + if: needs.openvino_download.outputs.status != 'success' timeout-minutes: 150 defaults: run: @@ -47,7 +80,7 @@ jobs: repository: 'openvinotoolkit/openvino' path: ${{ env.OPENVINO_REPO }} submodules: 'true' - ref: 'releases/2023/3' + ref: ${{ env.OV_BRANCH}} # # Dependencies @@ -124,8 +157,8 @@ jobs: - name: Cmake install - OpenVINO run: | - cmake -DCMAKE_INSTALL_PREFIX=${INSTALL_DIR} -P ${BUILD_DIR}/cmake_install.cmake - cmake -DCMAKE_INSTALL_PREFIX=${INSTALL_DIR} -DCOMPONENT=python_wheels -P ${BUILD_DIR}/cmake_install.cmake + cmake -DCMAKE_INSTALL_PREFIX=${INSTALL_DIR}/openvino_package -P ${BUILD_DIR}/cmake_install.cmake + cmake -DCMAKE_INSTALL_PREFIX=${INSTALL_DIR}/openvino_package -DCOMPONENT=python_wheels -P ${BUILD_DIR}/cmake_install.cmake - name: Pack Artifacts run: | @@ -147,7 +180,10 @@ jobs: openvino_tokenizers: name: OpenVINO tokenizers extension - needs: [ openvino_build ] + needs: [ openvino_download, openvino_build ] + if: | + always() && + (needs.openvino_download.outputs.status == 'success' || needs.openvino_build.result == 'success') timeout-minutes: 25 defaults: run: @@ -171,7 +207,7 @@ jobs: with: repository: 'openvinotoolkit/openvino' path: ${{ env.OPENVINO_REPO }} - ref: 'releases/2023/3' + ref: ${{ env.OV_BRANCH }} sparse-checkout: | install_build_dependencies.sh @@ -190,7 +226,7 @@ jobs: - name: Extract OpenVINO packages run: | pushd ${INSTALL_DIR} - tar -xzf openvino_package.tar.gz -C ${INSTALL_DIR} + tar -xzf openvino_package.tar.gz -C ${INSTALL_DIR} --strip-components=1 popd # @@ -233,7 +269,8 @@ jobs: openvino_tokenizers_tests: name: OpenVINO tokenizers tests - needs: [ openvino_tokenizers ] + needs: [ openvino_download, openvino_build, openvino_tokenizers] + if: always() && needs.openvino_tokenizers.result == 'success' timeout-minutes: 25 defaults: run: @@ -270,13 +307,14 @@ jobs: name: openvino_package path: ${{ env.INSTALL_DIR }} - - name: Extract OpenVINO packages + - name: Extract OpenVINO build package run: | pushd ${INSTALL_DIR} - tar -xzf openvino_package.tar.gz -C ${INSTALL_DIR} + tar -xzf openvino_package.tar.gz -C ${INSTALL_DIR} --strip-components=1 popd - name: Install OpenVINO Python wheel + if: needs.openvino_build.result == 'success' run: | # Find and install wheel pushd ${INSTALL_DIR}/tools @@ -292,7 +330,15 @@ jobs: python3 -m pip install $wheel_name[dev] popd - - name: Tokenizers regression tests + - name: Tokenizers regression tests (using openvino python modules) + if: needs.openvino_download.outputs.status == 'success' + run: | + source ${INSTALL_DIR}/setupvars.sh + python3 -m pytest tokenizers_test.py + working-directory: ${{ env.OPENVINO_TOKENIZERS_REPO }}/tests + + - name: Tokenizers regression tests (using openvino python wheel) + if: needs.openvino_build.result == 'success' run: | python3 -m pytest tokenizers_test.py working-directory: ${{ env.OPENVINO_TOKENIZERS_REPO }}/tests diff --git a/.github/workflows/mac.yml b/.github/workflows/mac.yml index a9ddecef5..e893bacc9 100644 --- a/.github/workflows/mac.yml +++ b/.github/workflows/mac.yml @@ -15,9 +15,42 @@ concurrency: env: PYTHON_VERSION: '3.11' MACOSX_DEPLOYMENT_TARGET: '10.12' + OV_BRANCH: 'releases/2023/3' + OV_TARBALL: 'https://storage.openvinotoolkit.org/repositories/openvino/packages/2023.3/macos/m_openvino_toolkit_macos_10_15_2023.3.0.13775.ceeafaf64f3_x86_64.tgz' jobs: + openvino_download: + outputs: + status: ${{ steps.openvino_download.outcome }} + timeout-minutes: 10 + defaults: + run: + shell: bash + runs-on: ubuntu-20.04 + + steps: + - name: Download OpenVINO build + id: openvino_download + run: | + wget ${{ env.OV_TARBALL}} --progress=bar:force:noscroll -O openvino_package.tar.gz + tar -tvf openvino_package.tar.gz + continue-on-error: true + + # + # Upload to artifacts + # + + - name: Upload openvino package + if: steps.openvino_download.outcome == 'success' + uses: actions/upload-artifact@v4 + with: + name: openvino_package + path: openvino_package.tar.gz + if-no-files-found: 'error' + openvino_build: + needs: [openvino_download] + if: needs.openvino_download.outputs.status != 'success' timeout-minutes: 150 defaults: run: @@ -39,7 +72,7 @@ jobs: repository: 'openvinotoolkit/openvino' path: ${{ env.OPENVINO_REPO }} submodules: 'true' - ref: 'releases/2023/3' + ref: ${{ env.OV_BRANCH }} # # Dependencies @@ -115,8 +148,8 @@ jobs: - name: Cmake install - OpenVINO run: | - cmake -DCMAKE_INSTALL_PREFIX=${INSTALL_DIR} -P ${BUILD_DIR}/cmake_install.cmake - cmake -DCMAKE_INSTALL_PREFIX=${INSTALL_DIR} -DCOMPONENT=python_wheels -P ${BUILD_DIR}/cmake_install.cmake + cmake -DCMAKE_INSTALL_PREFIX=${INSTALL_DIR}/openvino_package -P ${BUILD_DIR}/cmake_install.cmake + cmake -DCMAKE_INSTALL_PREFIX=${INSTALL_DIR}/openvino_package -DCOMPONENT=python_wheels -P ${BUILD_DIR}/cmake_install.cmake - name: Pack Artifacts run: | @@ -138,7 +171,10 @@ jobs: openvino_tokenizers: name: OpenVINO tokenizers extension - needs: [ openvino_build ] + needs: [ openvino_download, openvino_build ] + if: | + always() && + (needs.openvino_download.outputs.status == 'success' || needs.openvino_build.result == 'success') timeout-minutes: 25 defaults: run: @@ -162,7 +198,7 @@ jobs: with: repository: 'openvinotoolkit/openvino' path: ${{ env.OPENVINO_REPO }} - ref: 'releases/2023/3' + ref: ${{ env.OV_BRANCH }} sparse-checkout: | install_build_dependencies.sh @@ -181,7 +217,7 @@ jobs: - name: Extract OpenVINO packages run: | pushd ${INSTALL_DIR} - tar -xzf openvino_package.tar.gz -C ${INSTALL_DIR} + tar -xzf openvino_package.tar.gz -C ${INSTALL_DIR} --strip-components=1 popd # @@ -223,7 +259,8 @@ jobs: openvino_tokenizers_tests: name: OpenVINO tokenizers tests - needs: [ openvino_tokenizers ] + needs: [ openvino_download, openvino_build, openvino_tokenizers] + if: always() && needs.openvino_tokenizers.result == 'success' timeout-minutes: 25 defaults: run: @@ -263,10 +300,11 @@ jobs: - name: Extract OpenVINO packages run: | pushd ${INSTALL_DIR} - tar -xzf openvino_package.tar.gz -C ${INSTALL_DIR} + tar -xzf openvino_package.tar.gz -C ${INSTALL_DIR} --strip-components=1 popd - name: Install OpenVINO Python wheel + if: needs.openvino_build.result == 'success' run: | # Find and install wheel pushd ${INSTALL_DIR}/tools @@ -282,7 +320,15 @@ jobs: python3 -m pip install $wheel_name[dev] popd - - name: Tokenizers regression tests + - name: Tokenizers regression tests (using openvino python modules) + if: needs.openvino_download.outputs.status == 'success' + run: | + source ${INSTALL_DIR}/setupvars.sh + python3 -m pytest tokenizers_test.py + working-directory: ${{ env.OPENVINO_TOKENIZERS_REPO }}/tests + + - name: Tokenizers regression tests (using openvino python wheel) + if: needs.openvino_build.result == 'success' run: | python3 -m pytest tokenizers_test.py working-directory: ${{ env.OPENVINO_TOKENIZERS_REPO }}/tests diff --git a/.github/workflows/windows.yml b/.github/workflows/windows.yml index e3e75bfbd..ae9a771d3 100644 --- a/.github/workflows/windows.yml +++ b/.github/workflows/windows.yml @@ -14,9 +14,42 @@ concurrency: env: PYTHON_VERSION: '3.11' + OV_BRANCH: 'releases/2023/3' + OV_TARBALL: 'https://storage.openvinotoolkit.org/repositories/openvino/packages/2023.3/windows/w_openvino_toolkit_windows_2023.3.0.13775.ceeafaf64f3_x86_64.zip' jobs: + openvino_download: + outputs: + status: ${{ steps.openvino_download.outcome }} + timeout-minutes: 10 + defaults: + run: + shell: bash + runs-on: ubuntu-20.04 + + steps: + - name: Download OpenVINO build + id: openvino_download + run: | + wget ${{ env.OV_TARBALL}} --progress=bar:force:noscroll -O openvino_package.zip + unzip -l openvino_package.zip + continue-on-error: true + + # + # Upload to artifacts + # + + - name: Upload openvino package + if: steps.openvino_download.outcome == 'success' + uses: actions/upload-artifact@v4 + with: + name: openvino_package + path: openvino_package.zip + if-no-files-found: 'error' + openvino_build: + needs: [openvino_download] + if: needs.openvino_download.outputs.status != 'success' timeout-minutes: 150 defaults: run: @@ -41,7 +74,7 @@ jobs: repository: 'openvinotoolkit/openvino' path: ${{ env.OPENVINO_REPO }} submodules: 'true' - ref: 'releases/2023/3' + ref: ${{ env.OV_BRANCH }} # # Dependencies @@ -119,8 +152,8 @@ jobs: - name: Cmake install - OpenVINO run: | - cmake -DCMAKE_INSTALL_PREFIX=${{ env.INSTALL_DIR }} -P ${{ env.BUILD_DIR }}/cmake_install.cmake - cmake -DCMAKE_INSTALL_PREFIX=${{ env.INSTALL_DIR }} -DCOMPONENT=python_wheels -P ${{ env.BUILD_DIR }}/cmake_install.cmake + cmake -DCMAKE_INSTALL_PREFIX=${{ env.INSTALL_DIR }}/ov_package -P ${{ env.BUILD_DIR }}/cmake_install.cmake + cmake -DCMAKE_INSTALL_PREFIX=${{ env.INSTALL_DIR }}/ov_package -DCOMPONENT=python_wheels -P ${{ env.BUILD_DIR }}/cmake_install.cmake - name: Pack Artifacts run: | @@ -146,7 +179,10 @@ jobs: openvino_tokenizers: name: OpenVINO tokenizers extension - needs: [ openvino_build ] + needs: [ openvino_download, openvino_build ] + if: | + always() && + (needs.openvino_download.outputs.status == 'success' || needs.openvino_build.result == 'success') timeout-minutes: 25 defaults: run: @@ -170,7 +206,7 @@ jobs: with: repository: 'openvinotoolkit/openvino' path: ${{ env.OPENVINO_REPO }} - ref: 'releases/2023/3' + ref: ${{ env.OV_BRANCH }} sparse-checkout: | install_build_dependencies.sh @@ -189,7 +225,8 @@ jobs: - name: Extract OpenVINO packages run: | pushd ${{ env.INSTALL_DIR }} - Expand-Archive openvino_package.zip -DestinationPath "${{ env.INSTALL_DIR }}" + Expand-Archive openvino_package.zip -DestinationPath ./tmp + mv ./tmp/*/* . popd # @@ -229,7 +266,8 @@ jobs: openvino_tokenizers_tests: name: OpenVINO tokenizers tests - needs: [ openvino_tokenizers ] + needs: [ openvino_download, openvino_build, openvino_tokenizers] + if: always() && needs.openvino_tokenizers.result == 'success' timeout-minutes: 25 defaults: run: @@ -270,10 +308,12 @@ jobs: - name: Extract OpenVINO packages run: | pushd ${{ env.INSTALL_DIR }} - Expand-Archive openvino_package.zip -DestinationPath "${{ env.INSTALL_DIR }}" + Expand-Archive openvino_package.zip -DestinationPath ./tmp + mv ./tmp/*/* . popd - name: Install OpenVINO Python wheel + if: needs.openvino_build.result == 'success' run: | # Find and install wheel $ovCoreWheelPath=Get-ChildItem -Path "${{ env.INSTALL_DIR }}\\tools" -Filter openvino-*.whl | % { $_.FullName } @@ -285,7 +325,15 @@ jobs: $ovCoreWheelPath=Get-ChildItem -Path "${{ env.INSTALL_DIR }}\\ov_tokenizers" -Filter openvino_tokenizers*.whl | % { $_.FullName } python3 -m pip install "$ovCoreWheelPath[all]" - - name: Tokenizers regression tests + - name: Tokenizers regression tests (using openvino python modules) + if: needs.openvino_download.outputs.status == 'success' run: | + . "${{ env.INSTALL_DIR }}/setupvars.ps1" python3 -m pytest tokenizers_test.py working-directory: ${{ env.OPENVINO_TOKENIZERS_REPO }}/tests + + - name: Tokenizers regression tests + if: needs.openvino_build.result == 'success' + run: | + python3 -m pytest tokenizers_test.py + working-directory: ${{ env.OPENVINO_TOKENIZERS_REPO }}/tests \ No newline at end of file From b381ea71a1826a0b64a2c39d03c36c1d580c3630 Mon Sep 17 00:00:00 2001 From: Mikhail Ryzhov Date: Wed, 13 Mar 2024 14:15:37 +0100 Subject: [PATCH 2/2] use setupvars.bat --- .github/workflows/windows.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/windows.yml b/.github/workflows/windows.yml index ae9a771d3..d49d3b4fa 100644 --- a/.github/workflows/windows.yml +++ b/.github/workflows/windows.yml @@ -327,8 +327,9 @@ jobs: - name: Tokenizers regression tests (using openvino python modules) if: needs.openvino_download.outputs.status == 'success' + shell: cmd run: | - . "${{ env.INSTALL_DIR }}/setupvars.ps1" + call "${{ env.INSTALL_DIR }}\\setupvars.bat" python3 -m pytest tokenizers_test.py working-directory: ${{ env.OPENVINO_TOKENIZERS_REPO }}/tests