Skip to content

Commit

Permalink
Merge branch 'master' into cecilia/sdpa_softmax_opt
Browse files Browse the repository at this point in the history
  • Loading branch information
peterchen-intel authored Dec 16, 2024
2 parents e1af9ed + 357eb54 commit cd350c6
Show file tree
Hide file tree
Showing 764 changed files with 33,367 additions and 24,470 deletions.
2 changes: 1 addition & 1 deletion .github/dockerfiles/docker_tag
Original file line number Diff line number Diff line change
@@ -1 +1 @@
pr-27430
pr-27597
76 changes: 76 additions & 0 deletions .github/dockerfiles/ov_test/debian_10_py310/Dockerfile
Original file line number Diff line number Diff line change
@@ -0,0 +1,76 @@
ARG REGISTRY="docker.io"
FROM ${REGISTRY}/library/debian:10.13

USER root

# APT configuration
RUN echo 'Acquire::Retries "10";' > /etc/apt/apt.conf && \
echo 'APT::Get::Assume-Yes "true";' >> /etc/apt/apt.conf && \
echo 'APT::Get::Fix-Broken "true";' >> /etc/apt/apt.conf && \
echo 'APT::Get::no-install-recommends "true";' >> /etc/apt/apt.conf

ENV DEBIAN_FRONTEND="noninteractive" \
TZ="Europe/London"

RUN apt-get update && \
apt-get install \
git \
libc6-dev \
# parallel gzip
pigz \
# Python
python3 \
python3-pip \
python3-dev \
python3-venv \
python3-distutils \
# To build Python 3.10 from source
build-essential \
libffi-dev \
libgdbm-dev \
libc6-dev \
libssl-dev \
zlib1g-dev \
libbz2-dev \
libreadline-dev \
libsqlite3-dev \
libncurses5-dev \
libncursesw5-dev \
xz-utils \
tk-dev \
libxml2-dev \
libxmlsec1-dev \
liblzma-dev \
wget \
curl \
&& \
rm -rf /var/lib/apt/lists/*

# Install openvino dependencies
ADD scripts/install_dependencies/install_openvino_dependencies.sh /install_openvino_dependencies.sh
RUN chmod +x /install_openvino_dependencies.sh && \
/install_openvino_dependencies.sh && \
rm -rf /var/lib/apt/lists/*

# Setup Python 3.10
RUN wget https://www.python.org/ftp/python/3.10.9/Python-3.10.9.tar.xz

RUN tar -xf Python-3.10.9.tar.xz && \
cd Python-3.10.9 && \
./configure --enable-optimizations && \
make -j 8 && \
make altinstall

# Setup pip
ENV PIP_VERSION="24.0"
RUN curl https://bootstrap.pypa.io/get-pip.py -o get-pip.py && \
python3.10 get-pip.py --no-cache-dir pip==${PIP_VERSION} && \
rm -f get-pip.py

# Use Python 3.10 as default instead of Python 3.7
# Using venv here 'cause other methods to switch the default Python on Ubuntu 20 break both system and wheels build
RUN python3.10 -m venv venv
ENV PATH="/venv/bin:$PATH"

ENV PIP_CACHE_DIR=/mount/caches/pip/linux/${PIP_VERSION}
ENV PIP_INSTALL_PATH=/venv/lib/python3.10/site-packages
8 changes: 7 additions & 1 deletion .github/dockerfiles/ov_test/fedora_33/Dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,13 @@ USER root
RUN yum update -y && yum install -y \
git \
curl \
python3
python3 \
findutils \
ocl-icd \
ocl-icd-devel \
# parallel gzip
pigz \
xz

# Install Node
ENV NODE_VERSION=21.7.3
Expand Down
52 changes: 52 additions & 0 deletions .github/dockerfiles/ov_test/ubuntu_20_04_x64_py313/Dockerfile
Original file line number Diff line number Diff line change
@@ -0,0 +1,52 @@
ARG REGISTRY="docker.io"
FROM ${REGISTRY}/library/ubuntu:20.04

USER root

# APT configuration
RUN echo 'Acquire::Retries "10";' > /etc/apt/apt.conf && \
echo 'APT::Get::Assume-Yes "true";' >> /etc/apt/apt.conf && \
echo 'APT::Get::Fix-Broken "true";' >> /etc/apt/apt.conf && \
echo 'APT::Get::no-install-recommends "true";' >> /etc/apt/apt.conf

ENV DEBIAN_FRONTEND="noninteractive" \
TZ="Europe/London"

RUN apt-get update && \
apt-get install software-properties-common && \
add-apt-repository --yes --no-update ppa:git-core/ppa && \
add-apt-repository --yes --no-update ppa:deadsnakes/ppa && \
apt-get update && \
apt-get install \
curl \
git \
gpg-agent \
tzdata \
# parallel gzip
pigz \
# Python
python3.13-dev \
python3.13-venv \
&& \
rm -rf /var/lib/apt/lists/*

# Install openvino dependencies
ADD scripts/install_dependencies/install_openvino_dependencies.sh /install_openvino_dependencies.sh
RUN chmod +x /install_openvino_dependencies.sh && \
/install_openvino_dependencies.sh && \
rm -rf /var/lib/apt/lists/*

# Setup pip
ENV PIP_VERSION="24.0"
RUN curl https://bootstrap.pypa.io/get-pip.py -o get-pip.py && \
python3 get-pip.py --no-cache-dir pip==${PIP_VERSION} && \
python3.13 get-pip.py --no-cache-dir pip==${PIP_VERSION} && \
rm -f get-pip.py

# Use Python 3.13 as default instead of Python 3.8
# Using venv here 'cause other methods to switch the default Python on Ubuntu 20 break both system and wheels build
RUN python3.13 -m venv venv
ENV PATH="/venv/bin:$PATH"

ENV PIP_CACHE_DIR=/mount/caches/pip/linux/${PIP_VERSION}
ENV PIP_INSTALL_PATH=/venv/lib/python3.13/site-packages
Original file line number Diff line number Diff line change
@@ -1,40 +1,38 @@
name: Export workflow metrics (BETA)
name: Export workflow metrics

on:
workflow_run:
workflows:
- Android ARM64 with vcpkg
- Android x64
- Documentation
- Cleanup PIP caches
- Code snippets
- Code Style
- Code coverage
- Coverity (Ubuntu 20.04, Python 3.11)
- Debian 10 ARM
- Fedora 29 (RHEL 8.4), Python 3.9
- Linux (Ubuntu 20.04, Python 3.9)
- Linux (Ubuntu 22.04, Python 3.11)
- Linux (Ubuntu 24.04, Python 3.12)
- Linux ARM64 (Ubuntu 20.04, Python 3.11)
- Linux Static CC (Ubuntu 22.04, Python 3.11, Clang)
- Linux RISC-V with Conan (Ubuntu 22.04, Python 3.10)
- macOS (Python 3.11)
- macOS ARM64 (Python 3.11)
- MO
- Python API Checks
- Webassembly
- Windows (VS 2019, Python 3.11, Release)
- Windows (VS 2019, Python 3.11, Debug)
- Windows Conditional Compilation (VS 2022, Python 3.11)
- Rerun Workflow with Known Errors
- "Android ARM64 with vcpkg"
- "Android x64"
- "Cleanup caches"
- "Coverity (Ubuntu 20.04, Python 3.11)"
- "Debian 10 ARM"
- "Fedora 29 (RHEL 8.4), Python 3.9"
- "Linux (Ubuntu 20.04, Python 3.9)"
- "Linux (Ubuntu 22.04, Python 3.11)"
- "Linux (Ubuntu 24.04, Python 3.12)"
- "Linux ARM64 (Ubuntu 20.04, Python 3.11)"
- "Linux Static CC (Ubuntu 22.04, Python 3.11, Clang)"
- "Linux RISC-V with Conan (Ubuntu 22.04, Python 3.10)"
- "Linux (Ubuntu 22.04, Python 3.11, Intel DPC\\+\\+ Compiler)"
- "Linux CPU Plugin Snippets with LIBXSMM (Ubuntu 20.04)"
- "Linux Sanitizers (Ubuntu 20.04, Python 3.9)"
- "macOS (Python 3.11)"
- "macOS ARM64 (Python 3.11)"
- "Manylinux 2014"
- "Webassembly"
- "Windows (VS 2019, Python 3.11, Release)"
- "Windows (VS 2019, Python 3.11, Debug)"
- "Windows Conditional Compilation (VS 2022, Python 3.11)"
- "Rerun Workflow with Known Errors"
types:
- completed

permissions: read-all

jobs:
otel-export-trace:
export-workflow-metrics:
name: Export finished workflow metrics
runs-on: aks-linux-2-cores-8gb
if: ${{ github.repository_owner == 'openvinotoolkit' }}
Expand Down
4 changes: 2 additions & 2 deletions .github/workflows/job_python_api_tests.yml
Original file line number Diff line number Diff line change
Expand Up @@ -101,10 +101,10 @@ jobs:
--junitxml=${INSTALL_TEST_DIR}/TEST-Pyngraph.xml \
--ignore=${INSTALL_TEST_DIR}/tests/pyopenvino/tests/test_utils/test_utils.py
- name: Python API Tests -- numpy>=2.0.0
- name: Python API Tests -- numpy<2.0.0
run: |
python3 -m pip uninstall -y numpy
python3 -m pip install "numpy~=2.0.0"
python3 -m pip install "numpy~=1.26.0"
python3 -m pip install -r ${INSTALL_TEST_DIR}/tests/bindings/python/requirements_test.txt
# for 'template' extension
export LD_LIBRARY_PATH=${INSTALL_TEST_DIR}/tests/:$LD_LIBRARY_PATH
Expand Down
55 changes: 50 additions & 5 deletions .github/workflows/manylinux_2014.yml
Original file line number Diff line number Diff line change
Expand Up @@ -69,6 +69,11 @@ jobs:
images: |
ov_build/ubuntu_22_04_x64_docker
ov_build/manylinux2014_x86_64
ov_test/ubuntu_20_04_x64_py313
ov_test/ubuntu_22_04_x64
ov_test/ubuntu_24_04_x64
ov_test/fedora_33
ov_test/debian_10_py310
registry: 'openvinogithubactions.azurecr.io'
dockerfiles_root_dir: '.github/dockerfiles'
changed_components: ${{ needs.smart_ci.outputs.changed_components }}
Expand All @@ -92,6 +97,7 @@ jobs:
OPENVINO_REPO: ${{ github.workspace }}/src
INSTALL_DIR: ${{ github.workspace }}/install/openvino
INSTALL_WHEELS_DIR: ${{ github.workspace }}/install/wheels
INSTALL_TEST_DIR: ${{ github.workspace }}/install/tests
BUILD_DIR: ${{ github.workspace }}/build
DOCKER_CONFIG: "/mount/.docker"
CMAKE_CXX_COMPILER_LAUNCHER: sccache
Expand Down Expand Up @@ -135,6 +141,7 @@ jobs:
-v ${{ env.OPENVINO_REPO }}:/work/src \
-v ov_build_cache:/work/build \
-v ${{ env.INSTALL_DIR }}:/work/install \
-v ${{ env.INSTALL_TEST_DIR }}:/work/api_tests \
-e SCCACHE_AZURE_BLOB_CONTAINER \
-e SCCACHE_AZURE_CONNECTION_STRING \
-e SCCACHE_SERVER_PORT \
Expand All @@ -148,16 +155,18 @@ jobs:
-w /work/src \
${{ fromJSON(needs.docker.outputs.images).ov_build.manylinux2014_x86_64 }} \
/bin/bash -c "
cmake -DENABLE_CPPLINT=OFF -DENABLE_NCC_STYLE=OFF -DCMAKE_VERBOSE_MAKEFILE=ON -DENABLE_PYTHON=OFF -DENABLE_WHEEL=OFF -S /work/src -B /work/build &&
python3.12 -m pip install -r /work/src/src/bindings/python/wheel/requirements-dev.txt
cmake -DPython3_EXECUTABLE=/usr/local/bin/python3.12 -DENABLE_CPPLINT=OFF -DENABLE_NCC_STYLE=OFF -DENABLE_TESTS=ON -DCMAKE_VERBOSE_MAKEFILE=ON -DENABLE_OV_TF_FRONTEND=OFF -DENABLE_OV_TF_LITE_FRONTEND=OFF -DENABLE_OV_PADDLE_FRONTEND=OFF -DENABLE_OV_PYTORCH_FRONTEND=ON -DENABLE_OV_JAX_FRONTEND=OFF -DENABLE_OV_ONNX_FRONTEND=ON -DENABLE_PYTHON=ON -DENABLE_WHEEL=ON -S /work/src -B /work/build &&
cmake --build /work/build --parallel $(nproc) --config ${{ env.CMAKE_BUILD_TYPE }} &&
cmake --install /work/build --config ${{ env.CMAKE_BUILD_TYPE }} --prefix /work/install
cmake --install /work/build --config ${{ env.CMAKE_BUILD_TYPE }} --prefix /work/api_tests --component tests
"
- name: Pack Artifacts
run: mkdir -p ${{ env.BUILD_DIR }} && tar -cvf - * | pigz > ${{ env.BUILD_DIR }}/openvino_package.tar.gz
working-directory: ${{ env.INSTALL_DIR }}

- name: Build Python API(Python 3.9-3.13)
- name: Build Python API (Python 3.9-3.13)
run: |
SUPPORTED_PYTHON_VERSIONS=("39" "310" "311" "312" "313")
for PY_VER in "${SUPPORTED_PYTHON_VERSIONS[@]}"; do
Expand Down Expand Up @@ -190,6 +199,10 @@ jobs:
"
done
- name: Pack openvino_tests
run: tar -cvf - * | pigz > ${{ env.BUILD_DIR }}/openvino_tests.tar.gz
working-directory: ${{ env.INSTALL_TEST_DIR }}

#
# Upload build artifacts
#
Expand All @@ -208,7 +221,15 @@ jobs:
name: openvino_wheels
path: ${{ env.INSTALL_WHEELS_DIR }}/wheels/*.whl
if-no-files-found: 'error'


- name: Upload openvino tests package
if: ${{ always() }}
uses: actions/upload-artifact@b4b15b8c7c6ac21ea08fcf65892d2ee8f75cf882 # v4.4.3
with:
name: openvino_tests
path: ${{ env.BUILD_DIR }}/openvino_tests.tar.gz
if-no-files-found: 'error'

- name: Store artifacts to a shared drive
id: store_artifacts
if: ${{ always() }}
Expand All @@ -220,10 +241,34 @@ jobs:
${{ env.INSTALL_WHEELS_DIR }}/wheels
storage_dir: ${{ env.PRODUCT_TYPE }}
storage_root: ${{ env.ARTIFACTS_SHARE }}


Python_API_Tests:
name: Python API tests
needs: [ Docker, Build, Smart_CI ]
uses: ./.github/workflows/job_python_api_tests.yml
strategy:
fail-fast: false
matrix:
include:
- python-version: "3.9"
image: ${{ fromJSON(needs.docker.outputs.images).ov_test.fedora_33 }}
- python-version: "3.10"
image: ${{ fromJSON(needs.docker.outputs.images).ov_test.debian_10_py310 }}
- python-version: "3.11"
image: ${{ fromJSON(needs.docker.outputs.images).ov_test.ubuntu_22_04_x64 }}
- python-version: "3.12"
image: ${{ fromJSON(needs.docker.outputs.images).ov_test.ubuntu_24_04_x64 }}
- python-version: "3.13"
image: ${{ fromJSON(needs.docker.outputs.images).ov_test.ubuntu_20_04_x64_py313 }}
with:
runner: 'aks-linux-4-cores-16gb'
container: '{"image": "${{ matrix.image }}", "volumes": ["/mount:/mount"]}'
python-version: ${{ matrix.python-version }}
if: fromJSON(needs.smart_ci.outputs.affected_components).Python_API.test

Overall_Status:
name: ci/gha_overall_status_manylinux2014
needs: [Smart_CI, Build]
needs: [Smart_CI, Build, Python_API_Tests]
if: ${{ always() }}
runs-on: ubuntu-latest
steps:
Expand Down
Loading

0 comments on commit cd350c6

Please sign in to comment.