Skip to content

Commit

Permalink
Merge branch 'master' into nashez/jit_less_emitter
Browse files Browse the repository at this point in the history
  • Loading branch information
mlukasze authored Dec 10, 2024
2 parents 073d0ed + 67f2537 commit 73381db
Show file tree
Hide file tree
Showing 1,055 changed files with 29,486 additions and 25,386 deletions.
6 changes: 3 additions & 3 deletions .github/actions/cache/package-lock.json

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

48 changes: 48 additions & 0 deletions .github/actions/install_ov_wheels/action.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,48 @@
name: 'Find and install OpenVINO Python wheels'
description: 'Finds the OpenVINO Python wheels suitable for the "python3" executable and installs them'
inputs:
wheels-dir-path:
description: 'Path to the directory in which wheels are located'
required: true
wheels-to-install:
description: 'List of wheel names to install in the form of "openvino openvino_tokenizers"'
runs:
using: 'composite'
steps:
- name: Install OpenVINO Python wheels (Windows)
shell: pwsh
if: runner.os == 'Windows'
run: |
# Get the Python version
$pyVersion = python3 -c "import sys; print(f'{sys.version_info.major}{sys.version_info.minor}')"
foreach ($wheel in $("${{ inputs.wheels-to-install }}" -split ' ')) {
# Search for the python-specific wheel version and install it if exists
$wheelPath = Get-ChildItem -Path ${{ inputs.wheels-dir-path }} -Filter "$wheel-*cp$pyVersion*.whl" | Select-Object -First 1
if ($wheelPath) {
python3 -m pip install $wheelPath.FullName
} else {
# If the python-specific version does not exist, install by name only
$wheelPathByName = Get-ChildItem -Path ${{ inputs.wheels-dir-path }} -Filter "$wheel-*.whl" | Select-Object -First 1
python3 -m pip install $wheelPathByName.FullName
}
}
- name: Install OpenVINO Python wheels (Linux and macOS)
shell: bash
if: runner.os != 'Windows'
run: |
py_version=$(python3 -c "import sys; print(f'{sys.version_info.major}{sys.version_info.minor}')")
for wheel in ${{ inputs.wheels-to-install }}; do
echo "Installing the ${wheel} wheel"
# Search for the python-specific wheel version and install it if exists
wheel_path=$(find ${{ inputs.wheels-dir-path }} -name "$wheel-*cp$py_version*.whl")
echo "Wheel path: ${wheel_path}"
if [ -n "${wheel_path}" ]; then
python3 -m pip install $wheel_path
else
# If the python-specific version does not exist, install by name only
python3 -m pip install ${{ inputs.wheels-dir-path }}/$wheel-*.whl
fi
done
4 changes: 2 additions & 2 deletions .github/actions/openvino_provider/action.yml
Original file line number Diff line number Diff line change
Expand Up @@ -177,7 +177,7 @@ runs:
else
ov_package_url=$(curl -s ${{ inputs.nightly_package_source }} | jq -r '.${{ inputs.platform }}_${{ inputs.arch }}')
fi
cd ${{ inputs.install_dir || env.GITHUB_WORKSPACE }}
cd ${{ inputs.install_dir || github.workspace }}
package_basename=$(basename $ov_package_url)
wget $ov_package_url --progress=bar:force:noscroll -O $package_basename
package_folder=${package_basename%.*}
Expand All @@ -196,7 +196,7 @@ runs:
uses: actions/upload-artifact@50769540e7f4bd5e21e526ee35c689e35e0d6874 # v4.4.0
with:
name: ${{ steps.openvino_s3_download.outputs.ov_artifact_name }}
path: ${{ steps.openvino_s3_download.outputs.ov_package_path }}
path: ${{ github.workspace }}/${{ steps.openvino_s3_download.outputs.ov_package_path }}
if-no-files-found: 'error'

- name: Get wheel
Expand Down
4 changes: 4 additions & 0 deletions .github/actions/openvino_provider/get_s3_package.py
Original file line number Diff line number Diff line change
Expand Up @@ -54,6 +54,10 @@ def main(product, version_pattern, platform, arch, folder):
matching_files = filter_files_by_criteria(all_files, product, version_pattern, platform, arch, folder)
if matching_files:
logger.info(f"Matching packages: {sorted(matching_files)}")
if len(matching_files) > 1:
custom_release_build_pattern = fr".*/{version_pattern}/(linux_|windows_|macos_).*/.*"
# Exclude custom release builds, if any, from matches
matching_files = [file for file in matching_files if not re.search(custom_release_build_pattern, file)]
package_url = f"https://storage.openvinotoolkit.org{sorted(matching_files)[-1]}"
logger.info(f"Returning package URL: {package_url}")
action_utils.set_github_output("package_url", package_url)
Expand Down
8 changes: 5 additions & 3 deletions .github/workflows/debian_10_arm.yml
Original file line number Diff line number Diff line change
Expand Up @@ -49,7 +49,7 @@ jobs:
Docker:
needs: Smart_CI
if: "!needs.smart_ci.outputs.skip_workflow"
runs-on: aks-linux-16-cores-arm-docker-build
runs-on: aks-linux-4-cores-8gb-arm-docker-build
container:
image: openvinogithubactions.azurecr.io/docker_build:0.2
volumes:
Expand All @@ -75,7 +75,7 @@ jobs:
if: "!needs.smart_ci.outputs.skip_workflow"
uses: ./.github/workflows/job_build_linux.yml
with:
runner: 'aks-linux-16-cores-arm'
runner: 'aks-linux-16-cores-32gb-arm'
container: '{"image": "${{ fromJSON(needs.docker.outputs.images).ov_build.debian_10_arm }}", "volumes": ["/mount:/mount"], "options": "-e SCCACHE_AZURE_BLOB_CONTAINER -e SCCACHE_AZURE_CONNECTION_STRING"}'
affected-components: ${{ needs.smart_ci.outputs.affected_components }}
event-name: ${{ github.event_name }}
Expand Down Expand Up @@ -104,7 +104,7 @@ jobs:
needs: [ Docker, Build, Smart_CI ]
uses: ./.github/workflows/job_cxx_unit_tests.yml
with:
runner: 'aks-linux-16-cores-arm'
runner: 'aks-linux-8-cores-16gb-arm'
image: ${{ fromJSON(needs.docker.outputs.images).ov_test.debian_10_arm }}
affected-components: ${{ needs.smart_ci.outputs.affected_components }}
os: 'debian_10'
Expand All @@ -116,6 +116,8 @@ jobs:
needs: [ Docker, Build, Smart_CI ]
uses: ./.github/workflows/job_cpu_functional_tests.yml
with:
# Additional investigation needed why CPU functional tests are failing on v6 VM size's version,
# so leave it as it is for now
runner: 'aks-linux-16-cores-arm'
image: ${{ fromJSON(needs.docker.outputs.images).ov_test.debian_10_arm }}
python-version: '3.7'
Expand Down
58 changes: 55 additions & 3 deletions .github/workflows/job_build_windows.yml
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,11 @@ on:
description: 'A string of options passed to CMake'
type: string
required: true
build-additional-python-wheels:
description: 'Whether to build additional, i.e., non-system Python wheels. Should have Python 3.9-3.12 installed'
type: boolean
required: false
default: false

permissions: read-all

Expand Down Expand Up @@ -157,8 +162,7 @@ jobs:
run: echo SSL_CERT_FILE=$(python3 -m certifi) >> $env:GITHUB_ENV

- name: CMake configure
run: |
cmake -S ${{ env.OPENVINO_REPO }} -B ${{ env.BUILD_DIR }} ${{ inputs.cmake-options }}
run: cmake -S ${{ env.OPENVINO_REPO }} -B ${{ env.BUILD_DIR }} ${{ inputs.cmake-options }}

- name: Clean ccache stats
run: '& ccache --zero-stats'
Expand All @@ -176,6 +180,54 @@ jobs:
cmake --install . --config ${{ env.CMAKE_BUILD_TYPE }} --prefix ${{ env.INSTALL_TEST_DIR }} --component tests
working-directory: ${{ env.BUILD_DIR }}

# Setup additional Python versions for wheels building
- name: Setup Python 3.9
if: ${{ inputs.build-additional-python-wheels }}
uses: ./openvino/.github/actions/setup_python
with:
version: '3.9'
pip-cache-path: ${{ env.PIP_CACHE_PATH }}
should-setup-pip-paths: 'true'
self-hosted-runner: 'true'

# Setup additional Python versions for wheels building
- name: Setup Python 3.10
if: ${{ inputs.build-additional-python-wheels }}
uses: ./openvino/.github/actions/setup_python
with:
version: '3.10'
pip-cache-path: ${{ env.PIP_CACHE_PATH }}
should-setup-pip-paths: 'true'
self-hosted-runner: 'true'

# Setup additional Python versions for wheels building
- name: Setup Python 3.12
if: ${{ inputs.build-additional-python-wheels }}
uses: ./openvino/.github/actions/setup_python
with:
version: '3.12'
pip-cache-path: ${{ env.PIP_CACHE_PATH }}
should-setup-pip-paths: 'true'
self-hosted-runner: 'true'

- name: Build additional Python wheels
if: ${{ inputs.build-additional-python-wheels }}
run: |
$pyVersions = '3.9', '3.10', '3.12'
foreach ($pyVersion in $pyVersions) {
$pyBuildDir = "${{ github.workspace }}/py$pyVersion"
New-Item -ItemType Directory -Path "$pyBuildDir" -Force
$pythonCommand = "py -$pyVersion -c `"import sys; print(f'{sys.executable}')`""
$pythonExecutablePath = & cmd /c $pythonCommand
& $pythonExecutablePath -m pip install -r ${{ env.OPENVINO_REPO }}/src/bindings/python/wheel/requirements-dev.txt
cmake -DPython3_EXECUTABLE="$pythonExecutablePath" -DOpenVINODeveloperPackage_DIR=${{ env.BUILD_DIR }} -S ${{ env.OPENVINO_REPO }}/src/bindings/python -B "$pyBuildDir"
cmake --build "$pyBuildDir" --parallel --config ${{ env.CMAKE_BUILD_TYPE }}
cmake --install "$pyBuildDir" --config ${{ env.CMAKE_BUILD_TYPE }} --prefix ${{ env.INSTALL_WHEELS_DIR }} --component python_wheels
}
- name: Pack Artifacts
run: |
$file = Get-ChildItem -Path "${{ env.INSTALL_DIR }}"
Expand Down Expand Up @@ -220,7 +272,7 @@ jobs:
uses: actions/upload-artifact@b4b15b8c7c6ac21ea08fcf65892d2ee8f75cf882 # v4.4.3
with:
name: openvino_wheels
path: ${{ env.BUILD_DIR }}/wheels/*.whl
path: ${{ env.INSTALL_WHEELS_DIR }}/wheels/*.whl
if-no-files-found: 'error'

- name: Upload openvino tests package
Expand Down
142 changes: 142 additions & 0 deletions .github/workflows/job_python_api_tests.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,142 @@
name: Python API tests

on:
workflow_call:
inputs:
runner:
description: 'Machine on which the tests would run'
type: string
required: true
container:
description: 'JSON to be converted to the value of the "container" configuration for the job'
type: string
required: false
default: '{"image": null}'
python-version:
description: 'Python version to setup. E.g., "3.11"'
type: string
required: true

permissions: read-all

env:
PIP_CACHE_PATH: /mount/caches/pip/linux

jobs:
Python_Unit_Tests:
name: Python API tests
timeout-minutes: 30
runs-on: ${{ inputs.runner }}
container: ${{ fromJSON(inputs.container) }}
defaults:
run:
shell: bash
env:
DEBIAN_FRONTEND: noninteractive # to prevent apt-get from waiting user input
OPENVINO_REPO: ${{ github.workspace }}/openvino
INSTALL_DIR: ${{ github.workspace }}/install
INSTALL_TEST_DIR: ${{ github.workspace }}/install/openvino_tests
INSTALL_WHEELS_DIR: ${{ github.workspace }}/install/openvino_wheels
steps:
- name: Download OpenVINO artifacts (tarballs and wheels)
uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8
with:
pattern: openvino_@(wheels|tests)
path: ${{ env.INSTALL_DIR }}

# Needed as ${{ github.workspace }} is not working correctly when using Docker
- name: Setup Variables
run: |
echo "OPENVINO_REPO=$GITHUB_WORKSPACE/openvino" >> "$GITHUB_ENV"
echo "INSTALL_DIR=$GITHUB_WORKSPACE/install" >> "$GITHUB_ENV"
echo "INSTALL_TEST_DIR=$GITHUB_WORKSPACE/install/openvino_tests" >> "$GITHUB_ENV"
echo "INSTALL_WHEELS_DIR=$GITHUB_WORKSPACE/install/openvino_wheels" >> "$GITHUB_ENV"
- name: Install OpenVINO dependencies (mac)
if: runner.os == 'macOS'
run: brew install pigz

- name: Extract OpenVINO packages
run: pigz -dc openvino_tests.tar.gz | tar -xf - -C ${INSTALL_TEST_DIR}
working-directory: ${{ env.INSTALL_TEST_DIR }}

- name: Fetch setup_python and install wheels actions
uses: actions/checkout@d632683dd7b4114ad314bca15554477dd762a938 # v4.2.0
with:
sparse-checkout: |
.github/actions/setup_python/action.yml
.github/actions/install_ov_wheels/action.yml
sparse-checkout-cone-mode: false
path: 'action_root'

- name: Setup Python ${{ inputs.python-version }}
uses: ./action_root/.github/actions/setup_python
with:
version: ${{ inputs.python-version }}
pip-cache-path: ${{ runner.os == 'Linux' && env.PIP_CACHE_PATH || '' }}
should-setup-pip-paths: ${{ runner.os == 'Linux' }}
self-hosted-runner: ${{ runner.os == 'Linux' }}

#
# Tests
#
- name: Install OpenVINO Python wheels
uses: ./action_root/.github/actions/install_ov_wheels
with:
wheels-dir-path: ${{ env.INSTALL_WHEELS_DIR }}
wheels-to-install: 'openvino'

- name: Install Python API tests dependencies
run: python3 -m pip install -r ${INSTALL_TEST_DIR}/tests/bindings/python/requirements_test.txt

#
# Tests
#

- name: Python API Tests
run: |
# for 'template' extension
export LD_LIBRARY_PATH=${INSTALL_TEST_DIR}/tests/:$LD_LIBRARY_PATH
python3 -m pytest -sv ${INSTALL_TEST_DIR}/tests/pyopenvino \
--junitxml=${INSTALL_TEST_DIR}/TEST-Pyngraph.xml \
--ignore=${INSTALL_TEST_DIR}/tests/pyopenvino/tests/test_utils/test_utils.py
- name: Python API Tests -- numpy>=2.0.0
run: |
python3 -m pip uninstall -y numpy
python3 -m pip install "numpy~=2.0.0"
python3 -m pip install -r ${INSTALL_TEST_DIR}/tests/bindings/python/requirements_test.txt
# for 'template' extension
export LD_LIBRARY_PATH=${INSTALL_TEST_DIR}/tests/:$LD_LIBRARY_PATH
python3 -m pytest -sv ${INSTALL_TEST_DIR}/tests/pyopenvino \
--junitxml=${INSTALL_TEST_DIR}/TEST-Pyngraph_new_numpy.xml \
--ignore=${INSTALL_TEST_DIR}/tests/pyopenvino/tests/test_utils/test_utils.py
- name: Clone API snippets
if: runner.os != 'macOS'
uses: actions/checkout@d632683dd7b4114ad314bca15554477dd762a938 # v4.2.0
with:
sparse-checkout: docs/articles_en/assets/snippets
path: ${{ env.OPENVINO_REPO }}
submodules: 'false'

- name: Docs Python snippets
if: runner.os != 'macOS'
run: |
# torch, onnx
python3 -m pip install -r ${INSTALL_TEST_DIR}/tests/python/preprocess/torchvision/requirements.txt -r ${INSTALL_TEST_DIR}/tests/requirements_onnx
# to find 'snippets' module in docs
export PYTHONPATH=${OPENVINO_REPO}/docs/articles_en/assets
# for 'template' extension
export LD_LIBRARY_PATH=${INSTALL_TEST_DIR}/tests/:$LD_LIBRARY_PATH
python3 ${OPENVINO_REPO}/docs/articles_en/assets/snippets/main.py
- name: Upload Test Results
uses: actions/upload-artifact@50769540e7f4bd5e21e526ee35c689e35e0d6874 # v4.4.0
if: ${{ !cancelled() }}
with:
name: test-results-python-api-${{ inputs.python-version }}
path: |
${{ env.INSTALL_TEST_DIR }}/TEST*.html
${{ env.INSTALL_TEST_DIR }}/TEST*.xml
if-no-files-found: 'warn'
Loading

0 comments on commit 73381db

Please sign in to comment.