From 29cc40df5b4b24bc3b2ec3b550391a20fc844d88 Mon Sep 17 00:00:00 2001 From: Mikhail Ryzhov Date: Wed, 7 Feb 2024 14:49:41 +0100 Subject: [PATCH 1/7] mac --- .github/workflows/mac.yml | 300 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 300 insertions(+) create mode 100644 .github/workflows/mac.yml diff --git a/.github/workflows/mac.yml b/.github/workflows/mac.yml new file mode 100644 index 000000000..f7eefd33d --- /dev/null +++ b/.github/workflows/mac.yml @@ -0,0 +1,300 @@ +name: macOS (12, Python 3.11) +on: + workflow_dispatch: + pull_request: + push: + branches: + - master + - 'releases/**' + +concurrency: + # github.ref is not unique in post-commit + group: ${{ github.event_name == 'push' && github.run_id || github.ref }}-mac + cancel-in-progress: true + +env: + PYTHON_VERSION: '3.11' + +jobs: + openvino_build: + timeout-minutes: 150 + defaults: + run: + shell: bash + runs-on: 'macos-13-6core' + env: + DEBIAN_FRONTEND: noninteractive # to prevent apt-get from waiting user input + CMAKE_BUILD_TYPE: 'Release' + CMAKE_GENERATOR: 'Ninja Multi-Config' + CMAKE_CXX_COMPILER_LAUNCHER: ccache + CMAKE_C_COMPILER_LAUNCHER: ccache + OPENVINO_REPO: ${{ github.workspace }}/openvino + INSTALL_DIR: ${{ github.workspace }}/openvino/install + BUILD_DIR: ${{ github.workspace }}/openvino/build + + steps: + - name: Set apt retries + run: echo 'Acquire::Retries "10";' | sudo tee -a /etc/apt/apt.conf.d/80-retries > /dev/null + + - name: Install git + run: | + sudo apt-get update + sudo apt-get install --assume-yes --no-install-recommends git ca-certificates + + - name: Clone OpenVINO + uses: actions/checkout@v4 + with: + repository: 'openvinotoolkit/openvino' + path: ${{ env.OPENVINO_REPO }} + submodules: 'true' + ref: 'master' + + # + # Dependencies + # + + - name: Install build dependencies + run: | + sudo -E ${OPENVINO_REPO}/install_build_dependencies.sh + + - name: Setup Python ${{ env.PYTHON_VERSION }} + uses: actions/setup-python@v5 + with: + python-version: ${{ env.PYTHON_VERSION }} + cache: 'pip' + + - name: Install python dependencies + run: | + # For Python API: build and wheel packaging + python3 -m pip install -r ${OPENVINO_REPO}/src/bindings/python/wheel/requirements-dev.txt + + # + # Build + # + + - name: Setup ccache + uses: hendrikmuhs/ccache-action@v1.2 + with: + max-size: "2000M" + # Should save cache only if run in the master branch of the base repo + # github.ref_name is 'ref/PR_#' in case of the PR, and 'branch_name' when executed on push + save: ${{ github.ref_name == 'master' && 'true' || 'false' }} + verbose: 2 + key: linux-ubuntu + restore-keys: | + linux-ubuntu + + - name: CMake configure - OpenVINO + run: | + cmake \ + -G "${{ env.CMAKE_GENERATOR }}" \ + -DENABLE_CPPLINT=OFF \ + -DENABLE_NCC_STYLE=OFF \ + -DENABLE_TESTS=OFF \ + -DENABLE_STRICT_DEPENDENCIES=OFF \ + -DENABLE_SYSTEM_TBB=ON \ + -DENABLE_SYSTEM_OPENCL=ON \ + -DCMAKE_VERBOSE_MAKEFILE=ON \ + -DCPACK_GENERATOR=TGZ \ + -DENABLE_JS=OFF \ + -DENABLE_SAMPLES=OFF \ + -DENABLE_OV_ONNX_FRONTEND=OFF \ + -DENABLE_OV_PADDLE_FRONTEND=OFF \ + -DENABLE_OV_PYTORCH_FRONTEND=OFF \ + -DENABLE_OV_TF_FRONTEND=OFF \ + -DENABLE_OV_TF_LITE_FRONTEND=OFF \ + -DENABLE_INTEL_GPU=OFF \ + -DCMAKE_COMPILE_WARNING_AS_ERROR=ON \ + -DCMAKE_CXX_COMPILER_LAUNCHER=ccache \ + -DCMAKE_C_COMPILER_LAUNCHER=ccache \ + -S ${OPENVINO_REPO} \ + -B ${BUILD_DIR} + + - name: Clean ccache stats + run: ccache --zero-stats --show-config + + - name: Cmake build - OpenVINO + run: cmake --build ${BUILD_DIR} --parallel --config ${{ env.CMAKE_BUILD_TYPE }} + + - name: Show ccache stats + run: ccache --show-stats + + - name: Cmake install - OpenVINO + run: | + cmake -DCMAKE_INSTALL_PREFIX=${INSTALL_DIR} -P ${BUILD_DIR}/cmake_install.cmake + cmake -DCMAKE_INSTALL_PREFIX=${INSTALL_DIR} -DCOMPONENT=python_wheels -P ${BUILD_DIR}/cmake_install.cmake + + - name: Pack Artifacts + run: | + pushd ${INSTALL_DIR} + tar -czvf ${BUILD_DIR}/openvino_package.tar.gz * + popd + + # + # Upload build artifacts and logs + # + + - name: Upload openvino package + if: ${{ always() }} + uses: actions/upload-artifact@v3 + with: + name: openvino_package + path: ${{ env.BUILD_DIR }}/openvino_package.tar.gz + if-no-files-found: 'error' + + openvino_tokenizers: + name: OpenVINO tokenizers extension + needs: [ openvino_build ] + timeout-minutes: 25 + defaults: + run: + shell: bash + runs-on: macos-13 + + env: + OPENVINO_REPO: ${{ github.workspace }}/openvino + INSTALL_DIR: ${{ github.workspace }}/openvino/install + OPENVINO_TOKENIZERS_REPO: ${{ github.workspace }}/openvino_tokenizers + BUILD_DIR: ${{ github.workspace }}/openvino_tokenizers/build + + steps: + - name: Clone Openvino tokenizers + uses: actions/checkout@v4 + with: + path: ${{ env.OPENVINO_TOKENIZERS_REPO }} + + - name: Clone Openvino + uses: actions/checkout@v4 + with: + repository: 'openvinotoolkit/openvino' + path: ${{ env.OPENVINO_REPO }} + ref: 'master' + sparse-checkout: | + install_build_dependencies.sh + + - name: Setup Python ${{ env.PYTHON_VERSION }} + uses: actions/setup-python@v5 + with: + python-version: ${{ env.PYTHON_VERSION }} + cache: 'pip' + + - name: Download OpenVINO package + uses: actions/download-artifact@v3 + with: + name: openvino_package + path: ${{ env.INSTALL_DIR }} + + - name: Extract OpenVINO packages + run: | + pushd ${INSTALL_DIR} + tar -xzf openvino_package.tar.gz -C ${INSTALL_DIR} + popd + + # + # Dependencies + # + + - name: Install build dependencies + run: sudo ${{ env.OPENVINO_REPO }}/install_build_dependencies.sh + + - name: Install python dependencies + run: | + # wheel packaging + python3 -m pip install -r ${OPENVINO_TOKENIZERS_REPO}/requirements-build.txt + + # + # Build + # + + - name: Build tokenizers wheel + run: | + source ${INSTALL_DIR}/setupvars.sh + python -m build --wheel --outdir ${BUILD_DIR} ${OPENVINO_TOKENIZERS_REPO} + env: + CMAKE_ARGS: '-DBUILD_FAST_TOKENIZERS=OFF' + CMAKE_BUILD_PARALLEL_LEVEL: '4' + CMAKE_GENERATOR: 'Unix Makefiles' + SKBUILD_WHEEL_BUILD_TAG: ${{github.run_attempt}} + + # + # Upload build artifacts + # + + - name: Upload openvino tokenizers wheel + if: ${{ always() }} + uses: actions/upload-artifact@v3 + with: + name: openvino_tokenizers_wheel + path: ${{ env.BUILD_DIR }}/*.whl + if-no-files-found: 'error' + + openvino_tokenizers_tests: + name: OpenVINO tokenizers tests + needs: [ openvino_tokenizers ] + timeout-minutes: 25 + defaults: + run: + shell: bash + runs-on: macos-13 + + env: + OPENVINO_REPO: ${{ github.workspace }}/openvino + INSTALL_DIR: ${{ github.workspace }}/openvino/install + OPENVINO_TOKENIZERS_REPO: ${{ github.workspace }}/openvino_tokenizers + BUILD_DIR: ${{ github.workspace }}/openvino_tokenizers/build + + steps: + - name: Clone Openvino tokenizers sources and tests + uses: actions/checkout@v4 + with: + path: ${{ env.OPENVINO_TOKENIZERS_REPO }} + + - name: Setup Python ${{ env.PYTHON_VERSION }} + uses: actions/setup-python@v5 + with: + python-version: ${{ env.PYTHON_VERSION }} + cache: 'pip' + + - name: Download tokenizers package + uses: actions/download-artifact@v3 + with: + name: openvino_tokenizers_wheel + path: ${{ env.INSTALL_DIR }}/ov_tokenizers + + - name: Download OpenVINO package + uses: actions/download-artifact@v3 + with: + name: openvino_package + path: ${{ env.INSTALL_DIR }} + + - name: Extract OpenVINO packages + run: | + pushd ${INSTALL_DIR} + tar -xzf openvino_package.tar.gz -C ${INSTALL_DIR} + popd + + - name: Install OpenVINO Python wheel + run: | + # Find and install wheel + pushd ${INSTALL_DIR}/tools + wheel_name=$(find . -name 'openvino-*.whl') + python3 -m pip install $wheel_name + popd + + - name: Install OpenVINO tokenizers wheel + run: | + # Find and install wheel + pushd ${INSTALL_DIR}/ov_tokenizers + wheel_name=$(find . -name 'openvino_tokenizers*.whl') + python3 -m pip install $wheel_name[dev] + popd + + - name: Tokenizers Bandit tests + run: | + bandit -c pyproject.toml -r python + working-directory: ${{ env.OPENVINO_TOKENIZERS_REPO }} + + - name: Tokenizers regression tests + run: | + python3 -m pytest tokenizers_test.py + working-directory: ${{ env.OPENVINO_TOKENIZERS_REPO }}/tests \ No newline at end of file From 3ac5daa50791a9869eee828103e9db3703492bbf Mon Sep 17 00:00:00 2001 From: Mikhail Ryzhov Date: Wed, 7 Feb 2024 14:56:08 +0100 Subject: [PATCH 2/7] macos-13-6core --- .github/workflows/mac.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/mac.yml b/.github/workflows/mac.yml index f7eefd33d..e9e3c2ccc 100644 --- a/.github/workflows/mac.yml +++ b/.github/workflows/mac.yml @@ -21,7 +21,7 @@ jobs: defaults: run: shell: bash - runs-on: 'macos-13-6core' + runs-on: 'macos-13-6-core' env: DEBIAN_FRONTEND: noninteractive # to prevent apt-get from waiting user input CMAKE_BUILD_TYPE: 'Release' From a8163416daccca15ff2dea2aa1c066ffc9768c80 Mon Sep 17 00:00:00 2001 From: Mikhail Ryzhov Date: Wed, 7 Feb 2024 14:57:31 +0100 Subject: [PATCH 3/7] 'macos-13-6-cores' --- .github/workflows/mac.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/mac.yml b/.github/workflows/mac.yml index e9e3c2ccc..c1cad6609 100644 --- a/.github/workflows/mac.yml +++ b/.github/workflows/mac.yml @@ -21,7 +21,7 @@ jobs: defaults: run: shell: bash - runs-on: 'macos-13-6-core' + runs-on: 'macos-13-6-cores' env: DEBIAN_FRONTEND: noninteractive # to prevent apt-get from waiting user input CMAKE_BUILD_TYPE: 'Release' From 6cd791cf3ba12d48f138a3a5caf0b52566b752b5 Mon Sep 17 00:00:00 2001 From: Mikhail Ryzhov Date: Wed, 7 Feb 2024 15:09:36 +0100 Subject: [PATCH 4/7] macos-13-large --- .github/workflows/mac.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/mac.yml b/.github/workflows/mac.yml index c1cad6609..72d3afba7 100644 --- a/.github/workflows/mac.yml +++ b/.github/workflows/mac.yml @@ -21,7 +21,7 @@ jobs: defaults: run: shell: bash - runs-on: 'macos-13-6-cores' + runs-on: 'macos-13-large' env: DEBIAN_FRONTEND: noninteractive # to prevent apt-get from waiting user input CMAKE_BUILD_TYPE: 'Release' From f305cac158d615add8dff9c40e18185fc9f3d3a2 Mon Sep 17 00:00:00 2001 From: Mikhail Ryzhov Date: Wed, 7 Feb 2024 15:13:17 +0100 Subject: [PATCH 5/7] deps --- .github/workflows/mac.yml | 13 ++----------- 1 file changed, 2 insertions(+), 11 deletions(-) diff --git a/.github/workflows/mac.yml b/.github/workflows/mac.yml index 72d3afba7..97c00ee39 100644 --- a/.github/workflows/mac.yml +++ b/.github/workflows/mac.yml @@ -33,14 +33,6 @@ jobs: BUILD_DIR: ${{ github.workspace }}/openvino/build steps: - - name: Set apt retries - run: echo 'Acquire::Retries "10";' | sudo tee -a /etc/apt/apt.conf.d/80-retries > /dev/null - - - name: Install git - run: | - sudo apt-get update - sudo apt-get install --assume-yes --no-install-recommends git ca-certificates - - name: Clone OpenVINO uses: actions/checkout@v4 with: @@ -54,8 +46,7 @@ jobs: # - name: Install build dependencies - run: | - sudo -E ${OPENVINO_REPO}/install_build_dependencies.sh + run: brew install coreutils ninja - name: Setup Python ${{ env.PYTHON_VERSION }} uses: actions/setup-python@v5 @@ -195,7 +186,7 @@ jobs: # - name: Install build dependencies - run: sudo ${{ env.OPENVINO_REPO }}/install_build_dependencies.sh + run: brew install coreutils ninja - name: Install python dependencies run: | From 1c7cce25ad1ffaa5607e0fff170acf5c2162e84b Mon Sep 17 00:00:00 2001 From: Mikhail Ryzhov Date: Wed, 7 Feb 2024 16:03:33 +0100 Subject: [PATCH 6/7] changed mac compiler --- .github/workflows/mac.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/mac.yml b/.github/workflows/mac.yml index 97c00ee39..f589602df 100644 --- a/.github/workflows/mac.yml +++ b/.github/workflows/mac.yml @@ -204,7 +204,6 @@ jobs: env: CMAKE_ARGS: '-DBUILD_FAST_TOKENIZERS=OFF' CMAKE_BUILD_PARALLEL_LEVEL: '4' - CMAKE_GENERATOR: 'Unix Makefiles' SKBUILD_WHEEL_BUILD_TAG: ${{github.run_attempt}} # From e167c46f3701e90eb36e29d2216f395080204a65 Mon Sep 17 00:00:00 2001 From: Mikhail Ryzhov Date: Wed, 7 Feb 2024 16:34:53 +0100 Subject: [PATCH 7/7] Update .github/workflows/mac.yml Co-authored-by: Artur Paniukov --- .github/workflows/mac.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/mac.yml b/.github/workflows/mac.yml index f589602df..445eea7a4 100644 --- a/.github/workflows/mac.yml +++ b/.github/workflows/mac.yml @@ -287,4 +287,5 @@ jobs: - name: Tokenizers regression tests run: | python3 -m pytest tokenizers_test.py - working-directory: ${{ env.OPENVINO_TOKENIZERS_REPO }}/tests \ No newline at end of file + working-directory: ${{ env.OPENVINO_TOKENIZERS_REPO }}/tests + \ No newline at end of file