Skip to content

Commit

Permalink
Merge branch 'master' into pfn_build_log
Browse files Browse the repository at this point in the history
  • Loading branch information
ShaojieZhuIntel authored Oct 15, 2024
2 parents 321ae93 + fe2f67b commit 3163c79
Show file tree
Hide file tree
Showing 91 changed files with 1,576 additions and 662 deletions.
9 changes: 7 additions & 2 deletions .github/actions/openvino_provider/action.yml
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,11 @@ inputs:
description: 'Branch of OpenVINO to take the revision from if no specific hash was provided.
Taken from github context by default'
required: false
event_name:
description: 'Even name from which artifacts were generated. "push" by default; overwrite it with "pull_request"
if revision/branch is from PR'
default: "push"
required: false
ov_artifact_name:
description: "Name under which to upload provided OpenVINO build artifacts, set automatically by default"
required: false
Expand Down Expand Up @@ -107,7 +112,7 @@ runs:
with:
platform: ${{ inputs.platform }}_${{ inputs.arch }}
storage_root: ${{ inputs.commit_share_path }}
event_name: "commit"
event_name: ${{ inputs.event_name }}
trigger_repo_sha: ${{ env.OV_REVISION }}
branch_name: ${{ inputs.branch_name }}
to_restore: ${{ inputs.commit_packages_to_provide }}
Expand Down Expand Up @@ -137,7 +142,7 @@ runs:
artifacts_path=${{ steps.openvino_commit_download.outputs.artifacts_path }}
cd $artifacts_path
version=$(yq eval '.components.dldt.custom_params.wheel_product_version' manifest.yml)
wheel_path=${{ inputs.install_dir && '$artifacts_path/tools' || './tools' }}
wheel_path=${{ inputs.install_dir && '$artifacts_path/wheels' || './wheels' }}
default_find_links_cmd="--find-links=$wheel_path"
find_links_cmd=$([[ -n "$PIP_FIND_LINKS" ]] && echo "" || echo "$default_find_links_cmd")
echo "ov_version=$version" >> $GITHUB_OUTPUT
Expand Down
4 changes: 2 additions & 2 deletions .github/actions/restore_artifacts/restore_artifacts.py
Original file line number Diff line number Diff line change
Expand Up @@ -33,10 +33,10 @@ def include_filter(include_list: set | list):
"""
Returns input for shutil.copytree ignore - to copy only files from include list
"""
def _filter(_, files: list):
def _filter(root, files: list):
if not include_list:
return []
return [f for f in files if f not in include_list]
return [f for f in files if f not in include_list and Path(root).name not in include_list]

return _filter

Expand Down
2 changes: 2 additions & 0 deletions .github/labeler.yml
Original file line number Diff line number Diff line change
Expand Up @@ -150,6 +150,7 @@
- 'tests/layer_tests/tensorflow_tests/**/*'
- 'tests/layer_tests/tensorflow2_keras_tests/**/*'
- 'tests/layer_tests/jax_tests/**/*'
- 'tests/requirements_tensorflow'
- any: ['tests/model_hub_tests/**',
'!tests/model_hub_tests/pytorch/**/*',
'!tests/model_hub_tests/jax/**/*']
Expand All @@ -165,6 +166,7 @@
- 'src/bindings/python/src/openvino/frontend/pytorch/**/*'
- 'tests/layer_tests/py_frontend_tests/test_torch_decoder.py'
- 'tests/layer_tests/py_frontend_tests/test_torch_frontend.py'
- 'tests/requirements_pytorch'
- any: ['tests/model_hub_tests/**',
'!tests/model_hub_tests/tensorflow/**/*',
'!tests/model_hub_tests/jax/**/*']
Expand Down
61 changes: 39 additions & 22 deletions .github/workflows/job_build_linux.yml
Original file line number Diff line number Diff line change
Expand Up @@ -82,6 +82,7 @@ jobs:
INSTALL_DIR: /__w/openvino/openvino/openvino_install
INSTALL_DIR_JS: /__w/openvino/openvino/openvino_install/js
INSTALL_TEST_DIR: /__w/openvino/openvino/tests_install
INSTALL_WHEELS_DIR: /__w/openvino/openvino/install/wheels
DEVELOPER_PACKAGE_DIR: /__w/openvino/openvino/developer_package_install
BUILD_DIR: /__w/openvino/openvino/openvino_build
SCCACHE_AZURE_KEY_PREFIX: ${{ inputs.os }}_${{ inputs.arch }}_Release
Expand Down Expand Up @@ -174,25 +175,24 @@ jobs:

- name: Cmake install - OpenVINO
run: |
cmake -DCMAKE_INSTALL_PREFIX=${INSTALL_DIR} -P ${BUILD_DIR}/cmake_install.cmake
cmake -DCMAKE_INSTALL_PREFIX=${INSTALL_TEST_DIR} -DCOMPONENT=tests -P ${BUILD_DIR}/cmake_install.cmake
cmake -DCMAKE_INSTALL_PREFIX=${DEVELOPER_PACKAGE_DIR} -DCOMPONENT=developer_package -P ${BUILD_DIR}/cmake_install.cmake
cmake -DCMAKE_INSTALL_PREFIX=${INSTALL_DIR} -DCOMPONENT=python_wheels -P ${BUILD_DIR}/cmake_install.cmake
- name: Pack Artifacts
run: |
pushd ${INSTALL_DIR}
tar -I pigz -cvf ${BUILD_DIR}/openvino_package.tar.gz *
popd
pushd ${DEVELOPER_PACKAGE_DIR}
tar -I pigz -cvf ${BUILD_DIR}/openvino_developer_package.tar.gz *
popd
pushd ${INSTALL_TEST_DIR}
tar -I pigz -cvf ${BUILD_DIR}/openvino_tests.tar.gz *
popd
cmake --install . --config ${{ env.CMAKE_BUILD_TYPE }} --prefix ${INSTALL_DIR}
cmake --install . --config ${{ env.CMAKE_BUILD_TYPE }} --prefix ${INSTALL_WHEELS_DIR} --component python_wheels
cmake --install . --config ${{ env.CMAKE_BUILD_TYPE }} --prefix ${INSTALL_TEST_DIR} --component tests
cmake --install . --config ${{ env.CMAKE_BUILD_TYPE }} --prefix ${DEVELOPER_PACKAGE_DIR} --component developer_package
working-directory: ${{ env.BUILD_DIR }}

- name: Pack openvino_package
run: tar -I pigz -cvf ${BUILD_DIR}/openvino_package.tar.gz *
working-directory: ${{ env.INSTALL_DIR }}

- name: Pack openvino_developer_package
run: tar -I pigz -cvf ${BUILD_DIR}/openvino_developer_package.tar.gz *
working-directory: ${{ env.DEVELOPER_PACKAGE_DIR }}

- name: Pack openvino_tests
run: tar -I pigz -cvf ${BUILD_DIR}/openvino_tests.tar.gz *
working-directory: ${{ env.INSTALL_TEST_DIR }}

- name: Build Debian packages
if: ${{ inputs.build-debian-packages }}
run: |
Expand All @@ -205,6 +205,7 @@ jobs:
-UTBB* \
-DENABLE_SYSTEM_TBB=ON \
-DENABLE_PYTHON_PACKAGING=ON \
-DENABLE_WHEEL=OFF \
-DENABLE_TESTS=OFF \
-DPython3_EXECUTABLE=$python_exec \
-DCPACK_GENERATOR=DEB \
Expand All @@ -217,16 +218,20 @@ jobs:
cmake \
-DCUSTOM_OPERATIONS="calculate_grid;complex_mul;fft;grid_sample;sparse_conv;sparse_conv_transpose" \
-DOPENVINO_EXTRA_MODULES="${OPENVINO_CONTRIB_REPO}/modules/java_api;${OPENVINO_CONTRIB_REPO}/modules/custom_operations" \
-DENABLE_WHEEL=OFF \
-S ${OPENVINO_REPO} \
-B ${BUILD_DIR}
cmake --build ${BUILD_DIR} --parallel --config ${{ env.CMAKE_BUILD_TYPE }}
- name: CMake configure, build and install - OpenVINO JS API
if: ${{ fromJSON(inputs.affected-components).JS_API && inputs.build-js }}
run: |
cmake -UTBB* -DCPACK_GENERATOR=NPM -DENABLE_SYSTEM_TBB=OFF -S ${OPENVINO_REPO} -B ${BUILD_DIR}
cmake -UTBB* -S ${OPENVINO_REPO} -B ${BUILD_DIR} \
-DCPACK_GENERATOR=NPM \
-DENABLE_SYSTEM_TBB=OFF \
-DENABLE_WHEEL=OFF
cmake --build ${BUILD_DIR} --parallel
cmake -DCMAKE_INSTALL_PREFIX=${INSTALL_DIR_JS} -P ${BUILD_DIR}/cmake_install.cmake
cmake --install ${BUILD_DIR} --prefix ${INSTALL_DIR_JS}
- name: Build RPM packages
if: ${{ inputs.build-rpm-packages }}
Expand All @@ -235,6 +240,7 @@ jobs:
-DCPACK_GENERATOR=RPM \
-DENABLE_SYSTEM_TBB=ON \
-DENABLE_PYTHON_PACKAGING=ON \
-DENABLE_WHEEL=OFF \
-DENABLE_TESTS=OFF \
${BUILD_DIR}
cmake --build ${BUILD_DIR} --parallel --target package --verbose
Expand All @@ -257,7 +263,15 @@ jobs:
name: openvino_package
path: ${{ env.BUILD_DIR }}/openvino_package.tar.gz
if-no-files-found: 'error'


- name: Upload openvino wheels
if: ${{ inputs.os != 'debian_10' && inputs.arch != 'arm' }}
uses: actions/upload-artifact@50769540e7f4bd5e21e526ee35c689e35e0d6874 # v4.4.0
with:
name: openvino_wheels
path: ${{ env.INSTALL_WHEELS_DIR }}/wheels/*.whl
if-no-files-found: 'error'

- name: Upload openvino js package
if: ${{ fromJSON(inputs.affected-components).JS_API && inputs.build-js }}
uses: actions/upload-artifact@50769540e7f4bd5e21e526ee35c689e35e0d6874 # v4.4.0
Expand Down Expand Up @@ -317,5 +331,8 @@ jobs:
${{ env.BUILD_DIR }}/openvino_tests.tar.gz
${{ env.BUILD_DIR }}/deb
${{ env.MANIFEST_PATH }}
${{ env.STORE_WHEELS == 'true' && format('{0}/wheels', env.BUILD_DIR) || '' }}
storage_dir: ${{ env.PRODUCT_TYPE }}
storage_root: ${{ env.ARTIFACTS_SHARE }}
env:
STORE_WHEELS: ${{ inputs.os != 'debian_10' && inputs.arch != 'arm' }}
25 changes: 19 additions & 6 deletions .github/workflows/job_build_windows.yml
Original file line number Diff line number Diff line change
Expand Up @@ -44,6 +44,7 @@ jobs:
INSTALL_DIR: "${{ github.workspace }}\\openvino_install"
INSTALL_DIR_JS: "${{ github.workspace }}\\openvino_install\\js"
INSTALL_TEST_DIR: "${{ github.workspace }}\\tests_install"
INSTALL_WHEELS_DIR: "${{ github.workspace }}\\install\\wheels"
BUILD_DIR: "${{ github.workspace }}\\openvino_build"
ARTIFACTS_SHARE: "C:\\mount\\build-artifacts"
MANIFEST_PATH: "${{ github.workspace }}\\manifest.yml"
Expand Down Expand Up @@ -179,13 +180,14 @@ jobs:

- name: Cmake install - OpenVINO
run: |
cmake -DCMAKE_INSTALL_PREFIX=${{ env.INSTALL_DIR }} -P ${{ env.BUILD_DIR }}/cmake_install.cmake
cmake -DCMAKE_INSTALL_PREFIX=${{ env.INSTALL_TEST_DIR }} -DCOMPONENT=tests -P ${{ env.BUILD_DIR }}/cmake_install.cmake
cmake -DCMAKE_INSTALL_PREFIX=${{ env.INSTALL_DIR }} -DCOMPONENT=python_wheels -P ${{ env.BUILD_DIR }}/cmake_install.cmake
cmake --install . --config ${{ env.CMAKE_BUILD_TYPE }} --prefix ${{ env.INSTALL_DIR }}
cmake --install . --config ${{ env.CMAKE_BUILD_TYPE }} --prefix ${{ env.INSTALL_WHEELS_DIR }} --component python_wheels
cmake --install . --config ${{ env.CMAKE_BUILD_TYPE }} --prefix ${{ env.INSTALL_TEST_DIR }} --component tests
working-directory: ${{ env.BUILD_DIR }}

- name: Pack Artifacts
run: |
$file=Get-ChildItem -Path "${{ env.INSTALL_DIR }}"
$file = Get-ChildItem -Path "${{ env.INSTALL_DIR }}"
$compress = @{
Path = $file
CompressionLevel = "Optimal"
Expand All @@ -204,9 +206,12 @@ jobs:
- name: CMake configure, build and install - OpenVINO JS API
if: ${{ fromJSON(inputs.affected-components).JS_API }}
run: |
cmake -DCPACK_GENERATOR=NPM -DENABLE_SYSTEM_TBB=OFF -UTBB* -S ${{ env.OPENVINO_REPO }} -B ${{ env.BUILD_DIR }}
cmake -UTBB* -S ${{ env.OPENVINO_REPO }} -B ${{ env.BUILD_DIR }} `
-DCPACK_GENERATOR=NPM `
-DENABLE_SYSTEM_TBB=OFF `
-DENABLE_WHEEL=OFF
cmake --build ${{ env.BUILD_DIR }} --parallel
cmake -DCMAKE_INSTALL_PREFIX=${{ env.INSTALL_DIR_JS }} -P ${{ env.BUILD_DIR }}/cmake_install.cmake
cmake --install ${{ env.BUILD_DIR }} --config ${{ env.CMAKE_BUILD_TYPE }} --prefix ${{ env.INSTALL_DIR_JS }}
#
# Upload build artifacts and logs
Expand All @@ -218,6 +223,13 @@ jobs:
name: openvino_package
path: ${{ env.BUILD_DIR }}/openvino_package.zip
if-no-files-found: 'error'

- name: Upload openvino wheels
uses: actions/upload-artifact@50769540e7f4bd5e21e526ee35c689e35e0d6874 # v4.4.0
with:
name: openvino_wheels
path: ${{ env.BUILD_DIR }}/wheels/*.whl
if-no-files-found: 'error'

- name: Upload openvino tests package
uses: actions/upload-artifact@50769540e7f4bd5e21e526ee35c689e35e0d6874 # v4.4.0
Expand All @@ -242,6 +254,7 @@ jobs:
artifacts: |
${{ env.BUILD_DIR }}/openvino_package.zip
${{ env.BUILD_DIR }}/openvino_tests.zip
${{ env.INSTALL_WHEELS_DIR }}/wheels
${{ env.MANIFEST_PATH }}
storage_dir: ${{ env.PRODUCT_TYPE }}
storage_root: ${{ env.ARTIFACTS_SHARE }}
37 changes: 15 additions & 22 deletions .github/workflows/job_jax_models_tests.yml
Original file line number Diff line number Diff line change
Expand Up @@ -33,25 +33,22 @@ jobs:
OPENVINO_REPO: ${{ github.workspace }}/openvino
INSTALL_DIR: ${{ github.workspace }}/install
INSTALL_TEST_DIR: ${{ github.workspace }}/install/tests
INSTALL_WHEELS_DIR: ${{ github.workspace }}/install/wheels
MODEL_HUB_TESTS_INSTALL_DIR: ${{ github.workspace }}/install/tests/model_hub_tests
steps:
- name: Download OpenVINO package
- name: Download OpenVINO artifacts (tarballs)
uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8
with:
name: openvino_package
pattern: openvino_[tests]*
path: ${{ env.INSTALL_DIR }}

- name: Download OpenVINO tokenizers extension
merge-multiple: true

- name: Download OpenVINO artifacts (wheels)
uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8
with:
name: openvino_tokenizers_wheel
path: ${{ env.INSTALL_DIR }}

- name: Download OpenVINO tests package
uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8
with:
name: openvino_tests
path: ${{ env.INSTALL_TEST_DIR }}
pattern: openvino_[tokenizers_wheel|wheels]*
path: ${{ env.INSTALL_WHEELS_DIR }}
merge-multiple: true

# Needed as ${{ github.workspace }} is not working correctly when using Docker
- name: Setup Variables
Expand All @@ -60,16 +57,11 @@ jobs:
echo "INSTALL_DIR=$GITHUB_WORKSPACE/install" >> "$GITHUB_ENV"
echo "INSTALL_TEST_DIR=$GITHUB_WORKSPACE/install/tests" >> "$GITHUB_ENV"
echo "MODEL_HUB_TESTS_INSTALL_DIR=$GITHUB_WORKSPACE/install/tests/model_hub_tests" >> "$GITHUB_ENV"
- name: Extract OpenVINO packages
- name: Extract OpenVINO packages and tests
run: |
pushd ${INSTALL_DIR}
tar -I pigz -xf openvino_package.tar.gz -C ${INSTALL_DIR}
popd
pushd ${INSTALL_TEST_DIR}
tar -I pigz -xf openvino_tests.tar.gz -C ${INSTALL_DIR}
popd
working-directory: ${{ env.INSTALL_DIR }}

- name: Fetch setup_python action
uses: actions/checkout@d632683dd7b4114ad314bca15554477dd762a938 # v4.2.0
Expand All @@ -90,8 +82,9 @@ jobs:
run: |
# To enable pytest parallel features
python3 -m pip install pytest-xdist[psutil]
python3 -m pip install ${INSTALL_DIR}/tools/openvino-*
python3 -m pip install ${INSTALL_DIR}/openvino_tokenizers-*
python3 -m pip install ./openvino-*
python3 -m pip install ./openvino_tokenizers-*
working-directory: ${{ env.INSTALL_WHEELS_DIR }}

- name: Install JAX tests requirements for precommit
run: |
Expand Down
35 changes: 16 additions & 19 deletions .github/workflows/job_onnx_models_tests.yml
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,7 @@ jobs:
DEBIAN_FRONTEND: noninteractive # to prevent apt-get from waiting user input
OPENVINO_REPO: ${{ github.workspace }}/openvino
INSTALL_DIR: ${{ github.workspace }}/install
INSTALL_WHEELS_DIR: ${{ github.workspace }}/install/wheels
INSTALL_TEST_DIR: ${{ github.workspace }}/install/tests
ONNX_MODELS_PATH: ${{ github.workspace }}/onnx_test_models
# instead of using static MODELS_SHARE_PATH
Expand All @@ -37,18 +38,20 @@ jobs:
ONNX_MODEL_ZOO_SHA: "5faef4c33eba0395177850e1e31c4a6a9e634c82"
if: ${{ github.event_name != 'merge_group' }}
steps:
- name: Download OpenVINO package
- name: Download OpenVINO artifacts (tests)
uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8
with:
name: openvino_package
pattern: openvino_[tests]*
path: ${{ env.INSTALL_DIR }}

- name: Download OpenVINO tests package
merge-multiple: true

- name: Download OpenVINO artifacts (wheels)
uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8
with:
name: openvino_tests
path: ${{ env.INSTALL_TEST_DIR }}

pattern: openvino_[wheels]*
path: ${{ env.INSTALL_WHEELS_DIR }}
merge-multiple: true

# Needed as ${{ github.workspace }} is not working correctly when using Docker
- name: Setup Variables
run: |
Expand All @@ -59,15 +62,10 @@ jobs:
echo $MODELS_SHARE_PATH
echo "LOGS_FOLDER=$GITHUB_WORKSPACE/onnx_models_tests_logs" >> "$GITHUB_ENV"
- name: Extract OpenVINO packages
- name: Extract OpenVINO packages and tests
run: |
pushd ${INSTALL_DIR}
tar -I pigz -xf openvino_package.tar.gz -C ${INSTALL_DIR}
popd
pushd ${INSTALL_TEST_DIR}
tar -I pigz -xf openvino_tests.tar.gz -C ${INSTALL_DIR}
popd
working-directory: ${{ env.INSTALL_DIR }}

# Issue 148922
# Can be a possible root cause for the bug
Expand All @@ -87,15 +85,14 @@ jobs:
- name: Install OpenVINO Python wheels
run: |
# Install the core OV wheel
python3 -m pip install ${INSTALL_DIR}/tools/openvino-*.whl
python3 -m pip install ./openvino-*.whl
extras_to_install="onnx"
# Find and install OV dev wheel
pushd ${INSTALL_DIR}/tools
ov_dev_wheel_name=$(find . -name 'openvino_dev*.whl')
python3 -m pip install $ov_dev_wheel_name[$extras_to_install]
popd
ov_dev_wheel_name=$(find . -name 'openvino_dev*.whl')
python3 -m pip install $ov_dev_wheel_name[$extras_to_install]
working-directory: ${{ env.INSTALL_WHEELS_DIR }}

- name: Install Python tests dependencies
run: |
Expand Down
Loading

0 comments on commit 3163c79

Please sign in to comment.