Skip to content

Commit

Permalink
WIP remove arm64 docker
Browse files Browse the repository at this point in the history
  • Loading branch information
@folmos-at-orange
folmos-at-orange committed Feb 6, 2024
1 parent 56344c8 commit 753be4a
Show file tree
Hide file tree
Showing 3 changed files with 77 additions and 131 deletions.
182 changes: 64 additions & 118 deletions .github/workflows/build-conda-package.yml
View file
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
---
name: Build Conda Packages
name: Conda
env:
DEFAULT_KHIOPS_REVISION: dev
DEFAULT_SAMPLES_REVISION: main
Expand All @@ -8,48 +8,52 @@ on:
inputs:
khiops-revision:
default: dev
description: khiops repo tag/branch/hash
description: khiops repo revision
samples-revision:
default: main
description: khiops-samples repo tag/branch/hash
description: khiops-samples repo revision
push:
tags: ['*']
tags: [v*]
pull_request:
paths: [.github/workflows/build-conda-package.yml]
defaults:
run:
shell: bash -el {0}
concurrency:
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
cancel-in-progress: true
jobs:
build-conda-package:
build-packages:
name: Build Packages
strategy:
fail-fast: false
matrix:
# Use the oldest supported Mac OS and Ubuntu versions for GLIBC compatibility
os: [ubuntu-20.04, windows-latest, macos-11]
runs-on: ${{ matrix.os }}
outputs:
pkg-version: ${{ steps.get-pkg-version.pkg-version }}
steps:
# We move KHIOPS_REVISION to the environment so that we can use
# them in both push and workflow_dispatch events
- name: Move KHIOPS_REVISION to the environment (push event)
if: github.event_name == 'push'
run: echo "KHIOPS_REVISION=${DEFAULT_KHIOPS_REVISION}" >> "$GITHUB_ENV"
- name: Move KHIOPS_REVISION to the environment (workflow_dispatch event)
if: github.event_name == 'workflow_dispatch'
run: echo "KHIOPS_REVISION=${{ inputs.khiops-revision }}" >> "$GITHUB_ENV"
- name: Checkout Sources
uses: actions/checkout@v3
uses: actions/checkout@v4
with:
# Checkout the full repository to have the tags so versioneer works properly
# See issue https://github.com/actions/checkout/issues/701
fetch-depth: 0
# We move KHIOPS_REVISION to the environment so that we can use
# them in both push and workflow_dispatch events
- name: Move KHIOPS_REVISION to the environment (push/PR)
if: github.event_name == 'push' || github.event_name == 'pull_request'
run: echo "KHIOPS_REVISION=${DEFAULT_KHIOPS_REVISION}" >> "$GITHUB_ENV"
- name: Move KHIOPS_REVISION to the environment (workflow_dispatch)
if: github.event_name == 'workflow_dispatch'
run: echo "KHIOPS_REVISION=${{ inputs.khiops-revision }}" >> "$GITHUB_ENV"
- name: Install Miniconda
if: success() || failure()
uses: conda-incubator/setup-miniconda@v3
with:
activate-environment: khiops-python-env
- name: Install Dependency Requirements for Building Conda Packages
run: conda install conda-build conda-verify
run: conda install conda-build=3.27.0 conda-verify
# We need MacOS SDK 10.10 to build on Big Sur
- name: Install Mac OS SDK 10.10
if: runner.os == 'macOS'
Expand All @@ -59,35 +63,50 @@ jobs:
- name: Build Khiops Conda Package (Windows)
if: runner.os == 'Windows'
run: |
mkdir build
conda build --output-folder build packaging/conda
# In Linux/macOS we need the conda-forge channel to install mpi 3.4.3
mkdir khiops-conda
conda build --output-folder khiops-conda ./packaging/conda
dir khiops-conda
# In Linux/macOS we need the conda-forge channel to install their pinned versions
- name: Build Khiops Conda Package (Linux/macOS)
if: runner.os != 'Windows'
run: |
mkdir build
conda build --channel conda-forge --output-folder build packaging/conda
conda build --version
mkdir khiops-conda
conda build --debug --channel conda-forge --output-folder khiops-conda ./packaging/conda
ls khiops-conda
- name: Extract Conda Package Version and Rename
id: get-pkg-version
if: success() || failure()
run: |
PKG_VERSION=$(\
conda search --override-channels --channel ./khiops-conda/ khiops \
| awk '!/#|channels/ {print $2}' \
| sort -u \
)
echo "pkg-version=$PKG_VERSION" >> "$GITHUB_OUTPUT"
- name: Upload Khiops Conda Package
uses: actions/upload-artifact@v3
uses: actions/upload-artifact@v4
with:
name: khiops-conda-package-no-osx-arm64
path: build/*/khiops*.tar.bz2
name: khiops-${ steps.get-pkg-version.pkg-version }}-conda
path: ./khiops-conda
retention-days: 7
# Test Conda package on brand new environments
test-conda-package:
needs: build-conda-package
test-packages:
needs: build-packages
strategy:
fail-fast: false
matrix:
os:
- ubuntu-20.04
#- ubuntu-20.04
- ubuntu-22.04
- windows-2019
- windows-2022
- macos-11
- macos-12
- macos-13
python-version: ['3.8', '3.9', '3.10', '3.11']
#- windows-2019
#- windows-2022
#- macos-11
#- macos-12
#- macos-13
# DDD
# python-version: ['3.8', '3.9', '3.10', '3.11']
python-version: ['3.11']
runs-on: ${{ matrix.os }}
env:
KHIOPS_SAMPLES_DIR: ./khiops-samples-repo
Expand All @@ -101,35 +120,30 @@ jobs:
if: github.event_name == 'workflow_dispatch'
run: echo "SAMPLES_REVISION=${{ inputs.samples-revision }}" >> "$GITHUB_ENV"
- name: Checkout Khiops samples
uses: actions/checkout@v3
uses: actions/checkout@v4
with:
repository: khiopsml/khiops-samples
ref: ${{ env.SAMPLES_REVISION }}
token: ${{ secrets.GITHUB_TOKEN }}
path: ${{ env.KHIOPS_SAMPLES_DIR }}
- name: Download Conda Package Artifact
uses: actions/download-artifact@v3
with:
name: khiops-conda-package-no-osx-arm64
path: khiops-conda-package
- name: Install Miniconda
if: success() || failure()
uses: conda-incubator/setup-miniconda@v3
with:
miniconda-version: latest # needed for Mac OS 13
python-version: ${{ matrix.python-version }}
activate-environment: khiops-python-env
- name: Install conda-build and setup a local conda channel
run: |
conda install conda-build
conda index ./khiops-conda-package/.
- name: Download Conda Package Artifact
uses: actions/download-artifact@v4
with:
name: khiops-${{ needs.build-packages.pkg-version }}-conda
path: khiops-conda
- name: Install the Khiops Conda pagkage (Windows)
if: runner.os == 'Windows'
run: conda install -c ./khiops-conda-package/ khiops
run: conda install -c ./khiops-conda/ khiops
# In Linux/macOS we need the conda-forge channel to install their pinned versions
- name: Install the Khiops Conda package (Linux/macOS)
if: runner.os != 'Windows'
run: conda install -c conda-forge -c ./khiops-conda-package/ khiops
run: conda install -c conda-forge -c ./khiops-conda/ khiops
- name: Test Khiops Installation Status
run: kh-status
- name: Test Conda Package Installation on Samples
Expand All @@ -140,83 +154,15 @@ jobs:
kh-samples sklearn -i khiops_classifier -e
kh-samples sklearn -i khiops_coclustering -e
# Build and push Conda package release archive
release-conda-package:
needs: test-conda-package
release-packages:
needs: [build-packages, test-packages]
runs-on: ubuntu-latest
permissions:
contents: write
packages: read
steps:
- name: Download Conda Package Artifact
uses: actions/download-artifact@v3
with:
name: khiops-conda-package-no-osx-arm64
path: khiops-conda-package
- name: Login to Github Container Registry
uses: docker/login-action@v2
with:
registry: ghcr.io
username: ${{ github.actor }}
password: ${{ secrets.GITHUB_TOKEN }}
- name: Get Mac OS ARM64 Package from Dedicated Container
run: |
OSX_ARM_64_IMAGE=ghcr.io/khiopsml/khiops-python/conda-osx-arm64:latest
docker pull $OSX_ARM_64_IMAGE
CONTAINER_ID=$(docker run -it -d $OSX_ARM_64_IMAGE)
docker cp $CONTAINER_ID:/osx-arm64/. ./khiops-conda-package/osx-arm64/
docker container rm -f $CONTAINER_ID
docker image rm $OSX_ARM_64_IMAGE
- name: Install Miniconda
if: success() || failure()
uses: conda-incubator/setup-miniconda@v3
with:
activate-environment: khiops-python-env
- name: Create Conda Channel
run: |
conda install conda-build
conda index ./khiops-conda-package/
- name: Extract Conda Package Version
run: |
PKG_VERSION=$(\
conda search --override-channels --channel ./khiops-conda-package/ khiops | \
awk '!/#|channels/ {print $2}' | \
sort -u \
)
echo "PKG_VERSION=$PKG_VERSION" >> "$GITHUB_ENV"
# Flag if single version is detected (all *.bz2 packages have the same version)
# To be removed once GitHub supports Mac OS arm64
SINGLE_VERSION=$( \
if [ $(echo $PKG_VERSION | \
awk '{ for(i = 1 ; i <= NF; i++) {print $i}}' | \
sort -u | \
wc -l \
) -eq 1 ]; \
then
echo true; \
else
echo false; \
fi \
)
echo "SINGLE_VERSION=$SINGLE_VERSION" >> "$GITHUB_ENV"
- name: Upload Khiops Conda Package
uses: actions/upload-artifact@v3
with:
name: khiops-conda-package
path: khiops-conda-package/*/khiops*.tar.bz2
retention-days: 7
# Create release only if single package version was detected and on a tag push
- name: Create the release zip archive
if: env.SINGLE_VERSION == 'true' && github.event_name == 'push'
uses: thedoctor0/[email protected]
with:
type: zip
directory: ./khiops-conda-package/
filename: khiops-conda-${{ env.PKG_VERSION }}.zip
- name: Move Release Archive to the Current Directory
if: env.SINGLE_VERSION == 'true' && github.event_name == 'push'
run: mv ./khiops-conda-package/khiops-conda-${{ env.PKG_VERSION }}.zip .
- name: Release the zip archive
if: env.SINGLE_VERSION == 'true' && github.event_name == 'push'
if: github.event_name == 'push'
uses: ncipollo/release-action@v1
with:
artifacts: khiops-conda-${{ env.PKG_VERSION }}.zip
artifacts: khiops-${{ needs.build-packages.outputs.pkg-version }}-conda
6 changes: 3 additions & 3 deletions packaging/conda/conda_build_config.yaml
View file
Original file line number Diff line number Diff line change
@@ -1,8 +1,8 @@
---
python:
- 3.8
- 3.9
- 3.10
#- 3.8
#- 3.9
#- 3.10
- 3.11

# We need MacOS SDK 10.10 to be able to build on Big Sur for x64
Expand Down
20 changes: 10 additions & 10 deletions packaging/conda/meta.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -39,26 +39,26 @@ build:
# - requires conda-forge
requirements:
build:
- mpich=4.0.3 # [linux]
- mpich-mpicxx=4.0.3 # [linux]
- mpich=3.4.3 # [osx]
- mpich-mpicxx=3.4.3 # [osx]
- mpich 4.0.3 # [linux]
- mpich-mpicxx 4.0.3 # [linux]
- mpich 3.4.3 # [osx]
- mpich-mpicxx 3.4.3 # [osx]
- msmpi # [win]
- cmake
- ninja
- python
- setuptools
- {{ compiler('cxx') }}
host:
- mpich=4.0.3 # [linux]
- mpich-mpicxx=4.0.3 # [linux]
- mpich=3.4.3 # [osx]
- mpich-mpicxx=3.4.3 # [osx]
- mpich 4.0.3 # [linux]
- mpich-mpicxx 4.0.3 # [linux]
- mpich 3.4.3 # [osx]
- mpich-mpicxx 3.4.3 # [osx]
- msmpi # [win]
- python
run:
- mpich=4.0.3 # [linux]
- mpich=3.4.3 # [osx]
- mpich 4.0.3 # [linux]
- mpich 3.4.3 # [osx]
- msmpi # [win]
- pandas >=0.25.3
- python
Expand Down

0 comments on commit 753be4a

Please sign in to comment.