Skip to content

Commit

Permalink
[CI] [GHA] Introduce Debian 10 ARM (#26156)
Browse files Browse the repository at this point in the history
### Tickets:
 - *148717*
  • Loading branch information
akashchi authored Aug 29, 2024
1 parent c78fef6 commit fcd1138
Show file tree
Hide file tree
Showing 8 changed files with 276 additions and 18 deletions.
2 changes: 1 addition & 1 deletion .github/dockerfiles/docker_tag
Original file line number Diff line number Diff line change
@@ -1 +1 @@
pr-26067
pr-26156
89 changes: 89 additions & 0 deletions .github/dockerfiles/ov_build/debian_10_arm/Dockerfile
Original file line number Diff line number Diff line change
@@ -0,0 +1,89 @@
FROM openvinogithubactions.azurecr.io/dockerio/library/debian:10.13

USER root

# APT configuration
RUN echo 'Acquire::Retries "10";' > /etc/apt/apt.conf && \
echo 'APT::Get::Assume-Yes "true";' >> /etc/apt/apt.conf && \
echo 'APT::Get::Fix-Broken "true";' >> /etc/apt/apt.conf && \
echo 'APT::Get::no-install-recommends "true";' >> /etc/apt/apt.conf

ENV DEBIAN_FRONTEND="noninteractive" \
TZ="Europe/London"

RUN apt-get update && \
apt-get install \
software-properties-common \
curl \
git \
gpg-agent \
tzdata \
# Pythons \
python3 \
python3-pip \
python3-dev \
python3-venv \
python3-distutils \
libhdf5-dev \
# For building Python from source
build-essential \
libffi-dev \
libgdbm-dev \
libc6-dev \
libssl-dev \
zlib1g-dev \
libbz2-dev \
libreadline-dev \
libsqlite3-dev \
libncurses5-dev \
libncursesw5-dev \
xz-utils \
tk-dev \
libxml2-dev \
libxmlsec1-dev \
liblzma-dev \
wget \
# Compilers
gcc-arm-linux-gnueabihf \
g++-arm-linux-gnueabihf \
&& \
rm -rf /var/lib/apt/lists/*

# Install build dependencies
ADD install_build_dependencies.sh /install_build_dependencies.sh
RUN chmod +x /install_build_dependencies.sh && \
/install_build_dependencies.sh && \
rm -rf /var/lib/apt/lists/*

# Install sscache
ARG SCCACHE_VERSION="v0.7.5"
ENV SCCACHE_HOME="/opt/sccache" \
SCCACHE_PATH="/opt/sccache/sccache"

RUN mkdir ${SCCACHE_HOME} && cd ${SCCACHE_HOME} && \
SCCACHE_ARCHIVE="sccache-${SCCACHE_VERSION}-aarch64-unknown-linux-musl.tar.gz" && \
curl -SLO https://github.com/mozilla/sccache/releases/download/${SCCACHE_VERSION}/${SCCACHE_ARCHIVE} && \
tar -xzf ${SCCACHE_ARCHIVE} --strip-components=1 && rm ${SCCACHE_ARCHIVE}

ENV PATH="$SCCACHE_HOME:$PATH"

# Setup Python
RUN wget https://www.python.org/ftp/python/3.11.9/Python-3.11.9.tar.xz

RUN tar -xf Python-3.11.9.tar.xz && \
cd Python-3.11.9 && \
./configure --enable-optimizations && \
make -j 8 && \
make altinstall

# Setup pip
ENV PIP_VERSION="24.0"
RUN python3 -m pip install --upgrade pip==24.0
RUN python3.11 -m pip install --upgrade pip==24.0

# Use Python 3.11 as default
# Using venv here because other methods to switch the default Python break both system and wheels build
RUN python3.11 -m venv venv
ENV PATH="/venv/bin:$SCCACHE_HOME:$PATH"

ENV PIP_CACHE_DIR=/mount/caches/pip/linux/${PIP_VERSION}
111 changes: 111 additions & 0 deletions .github/workflows/debian_10_arm.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,111 @@
name: Debian 10 ARM
on:
workflow_dispatch:
pull_request:
merge_group:
push:
branches:
- master
- 'releases/**'

concurrency:
# github.ref is not unique in post-commit
group: ${{ github.event_name == 'push' && github.run_id || github.ref }}-debian-10-arm
cancel-in-progress: true

permissions: read-all

jobs:
Smart_CI:
runs-on: ubuntu-latest
outputs:
affected_components: "${{ steps.smart_ci.outputs.affected_components }}"
changed_components: "${{ steps.smart_ci.outputs.changed_components }}"
skip_workflow: "${{ steps.smart_ci.outputs.skip_workflow }}"
steps:
- name: checkout action
uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
with:
sparse-checkout: .github/actions/smart-ci

- name: Get affected components
id: smart_ci
uses: ./.github/actions/smart-ci
with:
repository: ${{ github.repository }}
pr: ${{ github.event.number }}
commit_sha: ${{ github.sha }}
ref_name: ${{ github.ref_name }}
component_pattern: "category: (.*)"
repo_token: ${{ secrets.GITHUB_TOKEN }}
skip_when_only_listed_labels_set: 'docs'
skip_when_only_listed_files_changed: '*.md,*.rst,*.png,*.jpg,*.svg,*/layer_tests_summary/*,*/conformance/*'

- name: Show affected components
run: |
echo "${{ toJSON(steps.smart_ci.outputs.affected_components) }}"
shell: bash

Docker:
needs: Smart_CI
runs-on: aks-linux-16-cores-arm-docker-build
container:
image: openvinogithubactions.azurecr.io/docker_build:0.2
volumes:
- /mount:/mount
outputs:
images: "${{ steps.handle_docker.outputs.images }}"
steps:
- name: Checkout
uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7

- uses: ./.github/actions/handle_docker
id: handle_docker
with:
images: |
ov_build/debian_10_arm
registry: 'openvinogithubactions.azurecr.io'
dockerfiles_root_dir: '.github/dockerfiles'
changed_components: ${{ needs.smart_ci.outputs.changed_components }}

Build:
needs: [Docker, Smart_CI]
if: "!needs.smart_ci.outputs.skip_workflow"
uses: ./.github/workflows/job_build_linux.yml
with:
runner: 'aks-linux-16-cores-arm'
container: '{"image": "${{ fromJSON(needs.docker.outputs.images).ov_build.debian_10_arm }}", "volumes": ["/mount:/mount"], "options": "-e SCCACHE_AZURE_BLOB_CONTAINER -e SCCACHE_AZURE_CONNECTION_STRING"}'
affected-components: ${{ needs.smart_ci.outputs.affected_components }}
event-name: ${{ github.event_name }}
os: 'debian_10'
arch: 'arm'
build-js: false
build-debian-packages: false
build-contrib: false
cmake-options: |-
-DCMAKE_TOOLCHAIN_FILE=${OPENVINO_REPO}/cmake/arm.toolchain.cmake \
-DTHREADS_PTHREAD_ARG="-pthread" \
-DCMAKE_COMPILE_WARNING_AS_ERROR=OFF \
-DCMAKE_BUILD_TYPE=Release \
-DENABLE_STRICT_DEPENDENCIES=OFF \
-DENABLE_CPPLINT=OFF \
-DENABLE_NCC_STYLE=OFF \
-DCMAKE_VERBOSE_MAKEFILE=ON \
-DENABLE_CONFORMANCE_PGQL=ON \
-DENABLE_LTO=ON \
-DENABLE_TESTS=ON \
-DENABLE_PYTHON=OFF
Overall_Status:
name: ci/gha_overall_status_debian_10_arm
needs: [Smart_CI, Build]
if: ${{ always() }}
runs-on: ubuntu-latest
steps:
- name: Check status of all jobs
if: >-
${{
contains(needs.*.result, 'failure') ||
contains(needs.*.result, 'cancelled')
}}
run: exit 1
49 changes: 32 additions & 17 deletions .github/workflows/job_build_linux.yml
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,30 @@ on:
description: 'OS that is used for building in the form of "ubuntu_20_04"'
type: string
required: true
arch:
description: 'Target architecture'
type: string
default: 'x86_64'
required: false
cmake-options:
description: 'A string of options passed to CMake'
type: string
required: true
build-js:
description: 'Whether to build OpenVINO JS Bindings'
type: boolean
required: false
default: true
build-debian-packages:
description: 'Whether to build Debian packages'
type: boolean
required: false
default: true
build-contrib:
description: 'Whether to build OpenVINO Contrib'
type: boolean
required: false
default: true

permissions: read-all

Expand All @@ -40,7 +64,6 @@ jobs:
env:
DEBIAN_FRONTEND: noninteractive # to prevent apt-get from waiting user input
CMAKE_BUILD_TYPE: 'Release'
CMAKE_GENERATOR: 'Ninja Multi-Config'
CMAKE_CXX_COMPILER_LAUNCHER: sccache
CMAKE_C_COMPILER_LAUNCHER: sccache
SCCACHE_IGNORE_SERVER_IO_ERROR: 1
Expand All @@ -55,11 +78,11 @@ jobs:
INSTALL_TEST_DIR: /__w/openvino/openvino/tests_install
DEVELOPER_PACKAGE_DIR: /__w/openvino/openvino/developer_package_install
BUILD_DIR: /__w/openvino/openvino/openvino_build
SCCACHE_AZURE_KEY_PREFIX: ${{ inputs.os }}_x86_64_Release
SCCACHE_AZURE_KEY_PREFIX: ${{ inputs.os }}_${{ inputs.arch }}_Release
ONNX_RUNTIME_UTILS: /__w/openvino/openvino/openvino/src/frontends/onnx/tests/ci_utils/onnxruntime
ARTIFACTS_SHARE: "/mount/build-artifacts"
MANIFEST_PATH: '/__w/openvino/openvino/manifest.yml'
PRODUCT_TYPE: public_linux_${{ inputs.os }}_release
PRODUCT_TYPE: public_linux_${{ inputs.os }}_${{ inputs.arch }}_release
steps:
- name: Clone OpenVINO
uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
Expand Down Expand Up @@ -131,17 +154,7 @@ jobs:
- name: CMake configure - OpenVINO
run: |
cmake \
-G "${{ env.CMAKE_GENERATOR }}" \
-DENABLE_CPPLINT=OFF \
-DENABLE_NCC_STYLE=OFF \
-DENABLE_TESTS=ON \
-DENABLE_STRICT_DEPENDENCIES=OFF \
-DENABLE_SYSTEM_OPENCL=ON \
-DCMAKE_VERBOSE_MAKEFILE=ON \
-DCPACK_GENERATOR=TGZ \
-DCMAKE_COMPILE_WARNING_AS_ERROR=ON \
-DCMAKE_CXX_COMPILER_LAUNCHER=${{ env.CMAKE_CXX_COMPILER_LAUNCHER }} \
-DCMAKE_C_COMPILER_LAUNCHER=${{ env.CMAKE_C_COMPILER_LAUNCHER }} \
${{ inputs.cmake-options }} \
-S ${OPENVINO_REPO} \
-B ${BUILD_DIR}
Expand Down Expand Up @@ -182,6 +195,7 @@ jobs:
popd
- name: Build Debian packages
if: ${{ inputs.build-debian-packages }}
run: |
# Ubuntu 24 does not allow using the system Python directly so
# we have to use Python from the virtual environment created in Docker
Expand All @@ -199,6 +213,7 @@ jobs:
cmake --build ${BUILD_DIR} --parallel --config ${{ env.CMAKE_BUILD_TYPE }} --target package
- name: Cmake & Build - OpenVINO Contrib
if: ${{ inputs.build-contrib }}
run: |
cmake \
-DCUSTOM_OPERATIONS="calculate_grid;complex_mul;fft;grid_sample;sparse_conv;sparse_conv_transpose" \
Expand All @@ -208,7 +223,7 @@ jobs:
cmake --build ${BUILD_DIR} --parallel --config ${{ env.CMAKE_BUILD_TYPE }}
- name: CMake configure, build and install - OpenVINO JS API
if: ${{ fromJSON(inputs.affected-components).JS_API }}
if: ${{ fromJSON(inputs.affected-components).JS_API && inputs.build-js }}
run: |
cmake -UTBB* -DCPACK_GENERATOR=NPM -DENABLE_SYSTEM_TBB=OFF -S ${OPENVINO_REPO} -B ${BUILD_DIR}
cmake --build ${BUILD_DIR} --parallel
Expand All @@ -234,7 +249,7 @@ jobs:
if-no-files-found: 'error'

- name: Upload openvino js package
if: ${{ fromJSON(inputs.affected-components).JS_API }}
if: ${{ fromJSON(inputs.affected-components).JS_API && inputs.build-js }}
uses: actions/upload-artifact@0b2256b8c012f0828dc542b3febcab082c67f72b # v4.3.4
with:
name: openvino_js_package
Expand All @@ -250,7 +265,7 @@ jobs:
if-no-files-found: 'error'

- name: Upload openvino debian packages
if: ${{ always() }}
if: ${{ inputs.build-debian-packages }}
uses: actions/upload-artifact@0b2256b8c012f0828dc542b3febcab082c67f72b # v4.3.4
with:
name: openvino_debian_packages
Expand Down
10 changes: 10 additions & 0 deletions .github/workflows/ubuntu_20.yml
Original file line number Diff line number Diff line change
Expand Up @@ -85,6 +85,16 @@ jobs:
affected-components: ${{ needs.smart_ci.outputs.affected_components }}
event-name: ${{ github.event_name }}
os: 'ubuntu_20_04'
cmake-options: |-
-G "Ninja Multi-Config" \
-DENABLE_CPPLINT=OFF \
-DENABLE_NCC_STYLE=OFF \
-DENABLE_TESTS=ON \
-DENABLE_STRICT_DEPENDENCIES=OFF \
-DENABLE_SYSTEM_OPENCL=ON \
-DCMAKE_VERBOSE_MAKEFILE=ON \
-DCPACK_GENERATOR=TGZ \
-DCMAKE_COMPILE_WARNING_AS_ERROR=ON
Debian_Packages:
name: Debian Packages
Expand Down
10 changes: 10 additions & 0 deletions .github/workflows/ubuntu_22.yml
Original file line number Diff line number Diff line change
Expand Up @@ -87,6 +87,16 @@ jobs:
affected-components: ${{ needs.smart_ci.outputs.affected_components }}
event-name: ${{ github.event_name }}
os: 'ubuntu_22_04'
cmake-options: |-
-G "Ninja Multi-Config" \
-DENABLE_CPPLINT=OFF \
-DENABLE_NCC_STYLE=OFF \
-DENABLE_TESTS=ON \
-DENABLE_STRICT_DEPENDENCIES=OFF \
-DENABLE_SYSTEM_OPENCL=ON \
-DCMAKE_VERBOSE_MAKEFILE=ON \
-DCPACK_GENERATOR=TGZ \
-DCMAKE_COMPILE_WARNING_AS_ERROR=ON
Debian_Packages:
name: Debian Packages
Expand Down
13 changes: 13 additions & 0 deletions .github/workflows/ubuntu_22_dpcpp.yml
Original file line number Diff line number Diff line change
Expand Up @@ -74,6 +74,19 @@ jobs:
affected-components: ${{ needs.smart_ci.outputs.affected_components }}
event-name: ${{ github.event_name }}
os: 'ubuntu_22_04_dpcpp'
build-js: false
build-debian-packages: false
build-contrib: false
cmake-options: |-
-G "Ninja Multi-Config" \
-DENABLE_CPPLINT=OFF \
-DENABLE_NCC_STYLE=OFF \
-DENABLE_TESTS=ON \
-DENABLE_STRICT_DEPENDENCIES=OFF \
-DENABLE_SYSTEM_OPENCL=ON \
-DCMAKE_VERBOSE_MAKEFILE=ON \
-DCPACK_GENERATOR=TGZ \
-DCMAKE_COMPILE_WARNING_AS_ERROR=ON
Overall_Status:
name: ci/gha_overall_status_ubuntu_22.04_dpcpp
Expand Down
10 changes: 10 additions & 0 deletions .github/workflows/ubuntu_24.yml
Original file line number Diff line number Diff line change
Expand Up @@ -83,6 +83,16 @@ jobs:
affected-components: ${{ needs.smart_ci.outputs.affected_components }}
event-name: ${{ github.event_name }}
os: 'ubuntu_24_04'
cmake-options: |-
-G "Ninja Multi-Config" \
-DENABLE_CPPLINT=OFF \
-DENABLE_NCC_STYLE=OFF \
-DENABLE_TESTS=ON \
-DENABLE_STRICT_DEPENDENCIES=OFF \
-DENABLE_SYSTEM_OPENCL=ON \
-DCMAKE_VERBOSE_MAKEFILE=ON \
-DCPACK_GENERATOR=TGZ \
-DCMAKE_COMPILE_WARNING_AS_ERROR=ON
Debian_Packages:
name: Debian Packages
Expand Down

0 comments on commit fcd1138

Please sign in to comment.