From c96f8c9cd12d3a594663f20772977fe9858108c3 Mon Sep 17 00:00:00 2001 From: Felipe Olmos <92923444+folmos-at-orange@users.noreply.github.com> Date: Wed, 7 Feb 2024 12:44:20 +0100 Subject: [PATCH 1/5] Pin conda-build version We pin the version of conda-build to not have problems with bugs in recent versions. --- .github/workflows/build-conda-package.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build-conda-package.yml b/.github/workflows/build-conda-package.yml index 289f246c..eb12648a 100644 --- a/.github/workflows/build-conda-package.yml +++ b/.github/workflows/build-conda-package.yml @@ -50,7 +50,7 @@ jobs: with: activate-environment: khiops-python-env - name: Install Dependency Requirements for Building Conda Packages - run: conda install conda-build conda-verify ninja + run: conda install conda-build=3.27.0 conda-verify # We need MacOS SDK 10.10 to build on Big Sur - name: Install Mac OS SDK 10.10 if: runner.os == 'macOS' From 720f900a592e9dc221e50b54a6bed7fded8795ff Mon Sep 17 00:00:00 2001 From: Felipe Olmos <92923444+folmos-at-orange@users.noreply.github.com> Date: Fri, 8 Dec 2023 18:22:35 +0100 Subject: [PATCH 2/5] Pin mpich version for Linux and macOS --- .github/workflows/build-conda-package.yml | 34 ++++++++++-------- packaging/conda/build.sh | 2 +- packaging/conda/meta.yaml | 42 +++++++++++------------ 3 files changed, 42 insertions(+), 36 deletions(-) diff --git a/.github/workflows/build-conda-package.yml b/.github/workflows/build-conda-package.yml index eb12648a..0025b753 100644 --- a/.github/workflows/build-conda-package.yml +++ b/.github/workflows/build-conda-package.yml @@ -25,9 +25,8 @@ jobs: strategy: fail-fast: false matrix: - # Use oldest supported Mac OS and Ubuntu versions for GLIBC compatibility - # Use oldest supported Windows version - os: [ubuntu-20.04, windows-2019, macos-11] + # Use the oldest supported Mac OS and Ubuntu versions for GLIBC compatibility + os: [ubuntu-20.04, windows-latest, macos-11] runs-on: ${{ matrix.os }} steps: # We move KHIOPS_REVISION to the environment so that we can use @@ -46,7 +45,7 @@ jobs: fetch-depth: 0 - name: Install Miniconda if: success() || failure() - uses: conda-incubator/setup-miniconda@v2 + uses: conda-incubator/setup-miniconda@v3 with: activate-environment: khiops-python-env - name: Install Dependency Requirements for Building Conda Packages @@ -57,10 +56,17 @@ jobs: run: | wget https://github.com/phracker/MacOSX-SDKs/releases/download/11.3/MacOSX10.10.sdk.tar.xz sudo tar -zxvf MacOSX10.10.sdk.tar.xz -C /opt - - name: Build Khiops Conda Package + - name: Build Khiops Conda Package (Windows) + if: runner.os == 'Windows' run: | mkdir build conda build --output-folder build packaging/conda + # In Linux/macOS we need the conda-forge channel to install mpi 3.4.3 + - name: Build Khiops Conda Package (Linux/macOS) + if: runner.os != 'Windows' + run: | + mkdir build + conda build --channel conda-forge --output-folder build packaging/conda - name: Upload Khiops Conda Package uses: actions/upload-artifact@v3 with: @@ -108,23 +114,23 @@ jobs: path: khiops-conda-package - name: Install Miniconda if: success() || failure() - uses: conda-incubator/setup-miniconda@v2 + uses: conda-incubator/setup-miniconda@v3 with: miniconda-version: latest # needed for Mac OS 13 python-version: ${{ matrix.python-version }} activate-environment: khiops-python-env - - name: Install conda-build and Setup Khiops Conda Channel + - name: Install conda-build and setup a local conda channel run: | conda install conda-build conda index ./khiops-conda-package/. - - name: Install the Khiops Conda package - run: conda install -c ./khiops-conda-package/ khiops - - name: Test Khiops Installation Status (Windows) + - name: Install the Khiops Conda pagkage (Windows) if: runner.os == 'Windows' - shell: cmd /C CALL {0} - run: C:\Miniconda3\envs\khiops-python-env\Scripts\kh-status.exe - - name: Test Khiops Installation Status (Linux/macOS) + run: conda install -c ./khiops-conda-package/ khiops + # In Linux/macOS we need the conda-forge channel to install their pinned versions + - name: Install the Khiops Conda package (Linux/macOS) if: runner.os != 'Windows' + run: conda install -c conda-forge -c ./khiops-conda-package/ khiops + - name: Test Khiops Installation Status run: kh-status - name: Test Conda Package Installation on Samples run: | @@ -162,7 +168,7 @@ jobs: docker image rm $OSX_ARM_64_IMAGE - name: Install Miniconda if: success() || failure() - uses: conda-incubator/setup-miniconda@v2 + uses: conda-incubator/setup-miniconda@v3 with: activate-environment: khiops-python-env - name: Create Conda Channel diff --git a/packaging/conda/build.sh b/packaging/conda/build.sh index 51e7bf04..fa9bf816 100644 --- a/packaging/conda/build.sh +++ b/packaging/conda/build.sh @@ -4,7 +4,7 @@ set -euo pipefail # Clone Khiops sources (we change working directory there) -git clone --depth 1 https://github.com/khiopsml/khiops.git khiops-core +git clone https://github.com/khiopsml/khiops.git khiops-core cd khiops-core git checkout "$KHIOPS_REVISION" diff --git a/packaging/conda/meta.yaml b/packaging/conda/meta.yaml index 2362eafd..1e91a70a 100644 --- a/packaging/conda/meta.yaml +++ b/packaging/conda/meta.yaml @@ -8,7 +8,7 @@ source: build: script_env: - - KHIOPS_REVISION=dev + - KHIOPS_REVISION # Variables for signing the MODL executables in osx-arm64 - KHIOPS_APPLE_CERTIFICATE_BASE64 # [osx and arm64] - KHIOPS_APPLE_CERTIFICATE_PASSWORD # [osx and arm64] @@ -30,36 +30,36 @@ build: - bin/pk-status # [osx and arm64] detect_binary_files_with_prefix: false # [osx and arm64] -# Note for OSX ARM 64: -# - We impose mpich <=3.4.3 until a release with the fix for -# the bug https://github.com/pmodels/mpich/issues/6619 +# Note on version pinning: +# OSX: +# - mpich=3.4.3 because 4.* is still unstable +# - requires conda-forge +# Linux: +# - mpich=4.0.3 because of bugs of the 3.* series +# - requires conda-forge requirements: build: + - mpich=4.0.3 # [linux] + - mpich-mpicxx=4.0.3 # [linux] + - mpich=3.4.3 # [osx] + - mpich-mpicxx=3.4.3 # [osx] + - msmpi # [win] - cmake - - mpich # [linux or osx and x86] - - mpich-mpicc # [linux or osx and x86] - - mpich-mpicxx # [linux or osx and x86] - - mpich <=3.4.3 # [osx and arm64] - - mpich-mpicc <=3.4.3 # [osx and arm64] - - mpich-mpicxx <=3.4.3 # [osx and arm64] - - msmpi # [win] - - ninja # [win] + - ninja - python - setuptools - {{ compiler('cxx') }} host: - - mpich # [linux or osx and x86] - - mpich-mpicc # [linux or osx and x86] - - mpich-mpicxx # [linux or osx and x86] - - mpich <=3.4.3 # [osx and arm64] - - mpich-mpicc <=3.4.3 # [osx and arm64] - - mpich-mpicxx <=3.4.3 # [osx and arm64] + - mpich=4.0.3 # [linux] + - mpich-mpicxx=4.0.3 # [linux] + - mpich=3.4.3 # [osx] + - mpich-mpicxx=3.4.3 # [osx] - msmpi # [win] - python run: - - mpich # [linux or osx and x64] - - msmpi # [win] - - mpich <=3.4.3 # [osx and arm64] + - mpich=4.0.3 # [linux] + - mpich=3.4.3 # [osx] + - msmpi # [win] - pandas >=0.25.3 - python - scikit-learn >=0.22.2 From 721406516ebc2ed5e3cc5a4e458eba0d4b27eb41 Mon Sep 17 00:00:00 2001 From: Felipe Olmos <92923444+folmos-at-orange@users.noreply.github.com> Date: Fri, 2 Feb 2024 17:06:08 +0100 Subject: [PATCH 3/5] Sign conda only if cert variable is set --- packaging/conda/build.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packaging/conda/build.sh b/packaging/conda/build.sh index fa9bf816..01ad0af9 100644 --- a/packaging/conda/build.sh +++ b/packaging/conda/build.sh @@ -55,7 +55,7 @@ $PYTHON -m pip install . --no-deps --ignore-installed --no-cache-dir --no-build- # - KHIOPS_APPLE_CERTIFICATE_BASE64: The identity file .p12 (certificate + private key) in base64 # - KHIOPS_APPLE_TMP_KEYCHAIN_PASSWORD: Password to decrypt the certificate # -if [[ "$(uname)" == "Darwin" && "$(uname -m)" == "arm64" ]] +if [[ "$(uname)" == "Darwin" && "$(uname -m)" == "arm64" && -n "${KHIOPS_APPLE_CERTIFICATE_BASE64-}" ]] then # Delete the rpath of each executable # Delete two times for MODL because for some reason it is there 2 times @@ -98,7 +98,7 @@ then # We make sure to use the default macOS/Xcode codesign tool. This is because the sigtool python # package (installed by conda build as a dependency) makes an alias "codesign" which is prioritary - # in the build environment. The alias, however, alias doen't support signing with a proper + # in the build environment. The alias, however, alias doesn't support signing with a proper # identity and makes the build fail! CODESIGN="/usr/bin/codesign" From ac1f210949c445fbfa0bac5c84166be8d72b06c6 Mon Sep 17 00:00:00 2001 From: Felipe Olmos <92923444+folmos-at-orange@users.noreply.github.com> Date: Wed, 7 Feb 2024 12:46:59 +0100 Subject: [PATCH 4/5] Eliminate docker for osx-arm64 conda packages We eliminate code obtaining the osx-arm64 conda packages to simplify the package workflow. In the meantime we will build and upload the packages for osx-arm64 manually. When we have access to osx-arm64 runners we'll stop to package it manually. --- .github/workflows/build-conda-package.yml | 155 +++++++--------------- packaging/conda/meta.yaml | 20 +-- 2 files changed, 58 insertions(+), 117 deletions(-) diff --git a/.github/workflows/build-conda-package.yml b/.github/workflows/build-conda-package.yml index 0025b753..38fde111 100644 --- a/.github/workflows/build-conda-package.yml +++ b/.github/workflows/build-conda-package.yml @@ -1,5 +1,5 @@ --- -name: Build Conda Packages +name: Conda env: DEFAULT_KHIOPS_REVISION: dev DEFAULT_SAMPLES_REVISION: main @@ -8,12 +8,14 @@ on: inputs: khiops-revision: default: dev - description: khiops repo tag/branch/hash + description: khiops repo revision samples-revision: default: main - description: khiops-samples repo tag/branch/hash + description: khiops-samples repo revision push: - tags: ['*'] + tags: [v*] + pull_request: + paths: [.github/workflows/build-conda-package.yml] defaults: run: shell: bash -el {0} @@ -21,30 +23,32 @@ concurrency: group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} cancel-in-progress: true jobs: - build-conda-package: + build-packages: + name: Build Packages strategy: fail-fast: false matrix: # Use the oldest supported Mac OS and Ubuntu versions for GLIBC compatibility os: [ubuntu-20.04, windows-latest, macos-11] runs-on: ${{ matrix.os }} + outputs: + pkg-version: ${{ steps.get-pkg-version.outputs.pkg-version }} steps: - # We move KHIOPS_REVISION to the environment so that we can use - # them in both push and workflow_dispatch events - - name: Move KHIOPS_REVISION to the environment (push event) - if: github.event_name == 'push' - run: echo "KHIOPS_REVISION=${DEFAULT_KHIOPS_REVISION}" >> "$GITHUB_ENV" - - name: Move KHIOPS_REVISION to the environment (workflow_dispatch event) - if: github.event_name == 'workflow_dispatch' - run: echo "KHIOPS_REVISION=${{ inputs.khiops-revision }}" >> "$GITHUB_ENV" - name: Checkout Sources uses: actions/checkout@v3 with: # Checkout the full repository to have the tags so versioneer works properly # See issue https://github.com/actions/checkout/issues/701 fetch-depth: 0 + # We move KHIOPS_REVISION to the environment so that we can use + # them in both push and workflow_dispatch events + - name: Move KHIOPS_REVISION to the environment (push/PR) + if: github.event_name == 'push' || github.event_name == 'pull_request' + run: echo "KHIOPS_REVISION=${DEFAULT_KHIOPS_REVISION}" >> "$GITHUB_ENV" + - name: Move KHIOPS_REVISION to the environment (workflow_dispatch) + if: github.event_name == 'workflow_dispatch' + run: echo "KHIOPS_REVISION=${{ inputs.khiops-revision }}" >> "$GITHUB_ENV" - name: Install Miniconda - if: success() || failure() uses: conda-incubator/setup-miniconda@v3 with: activate-environment: khiops-python-env @@ -59,23 +63,32 @@ jobs: - name: Build Khiops Conda Package (Windows) if: runner.os == 'Windows' run: | - mkdir build - conda build --output-folder build packaging/conda - # In Linux/macOS we need the conda-forge channel to install mpi 3.4.3 + mkdir khiops-conda + conda build --output-folder khiops-conda ./packaging/conda + # In Linux/macOS we need the conda-forge channel to install their pinned versions - name: Build Khiops Conda Package (Linux/macOS) if: runner.os != 'Windows' run: | - mkdir build - conda build --channel conda-forge --output-folder build packaging/conda + mkdir khiops-conda + conda build --channel conda-forge --output-folder khiops-conda ./packaging/conda + - name: Extract Khiops Package Version + id: get-pkg-version + run: | + PKG_VERSION=$(\ + conda search --override-channels --channel ./khiops-conda/ khiops \ + | awk '!/#|channels/ {print $2}' \ + | sort -u \ + ) + echo "pkg-version=$PKG_VERSION" >> "$GITHUB_OUTPUT" - name: Upload Khiops Conda Package uses: actions/upload-artifact@v3 with: - name: khiops-conda-package-no-osx-arm64 - path: build/*/khiops*.tar.bz2 + name: khiops-${{ steps.get-pkg-version.outputs.pkg-version }}-conda + path: ./khiops-conda retention-days: 7 # Test Conda package on brand new environments - test-conda-package: - needs: build-conda-package + test-packages: + needs: build-packages strategy: fail-fast: false matrix: @@ -107,29 +120,24 @@ jobs: ref: ${{ env.SAMPLES_REVISION }} token: ${{ secrets.GITHUB_TOKEN }} path: ${{ env.KHIOPS_SAMPLES_DIR }} - - name: Download Conda Package Artifact - uses: actions/download-artifact@v3 - with: - name: khiops-conda-package-no-osx-arm64 - path: khiops-conda-package - name: Install Miniconda - if: success() || failure() uses: conda-incubator/setup-miniconda@v3 with: miniconda-version: latest # needed for Mac OS 13 python-version: ${{ matrix.python-version }} activate-environment: khiops-python-env - - name: Install conda-build and setup a local conda channel - run: | - conda install conda-build - conda index ./khiops-conda-package/. + - name: Download Conda Package Artifact + uses: actions/download-artifact@v3 + with: + name: khiops-${{ needs.build-packages.outputs.pkg-version }}-conda + path: khiops-conda - name: Install the Khiops Conda pagkage (Windows) if: runner.os == 'Windows' - run: conda install -c ./khiops-conda-package/ khiops + run: conda install -c ./khiops-conda/ khiops # In Linux/macOS we need the conda-forge channel to install their pinned versions - name: Install the Khiops Conda package (Linux/macOS) if: runner.os != 'Windows' - run: conda install -c conda-forge -c ./khiops-conda-package/ khiops + run: conda install -c conda-forge -c ./khiops-conda/ khiops - name: Test Khiops Installation Status run: kh-status - name: Test Conda Package Installation on Samples @@ -140,83 +148,16 @@ jobs: kh-samples sklearn -i khiops_classifier -e kh-samples sklearn -i khiops_coclustering -e # Build and push Conda package release archive - release-conda-package: - needs: test-conda-package + release-packages: + # We need `build-packages` because we use its output + needs: [build-packages, test-packages] runs-on: ubuntu-latest permissions: contents: write packages: read steps: - - name: Download Conda Package Artifact - uses: actions/download-artifact@v3 - with: - name: khiops-conda-package-no-osx-arm64 - path: khiops-conda-package - - name: Login to Github Container Registry - uses: docker/login-action@v2 - with: - registry: ghcr.io - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - - name: Get Mac OS ARM64 Package from Dedicated Container - run: | - OSX_ARM_64_IMAGE=ghcr.io/khiopsml/khiops-python/conda-osx-arm64:latest - docker pull $OSX_ARM_64_IMAGE - CONTAINER_ID=$(docker run -it -d $OSX_ARM_64_IMAGE) - docker cp $CONTAINER_ID:/osx-arm64/. ./khiops-conda-package/osx-arm64/ - docker container rm -f $CONTAINER_ID - docker image rm $OSX_ARM_64_IMAGE - - name: Install Miniconda - if: success() || failure() - uses: conda-incubator/setup-miniconda@v3 - with: - activate-environment: khiops-python-env - - name: Create Conda Channel - run: | - conda install conda-build - conda index ./khiops-conda-package/ - - name: Extract Conda Package Version - run: | - PKG_VERSION=$(\ - conda search --override-channels --channel ./khiops-conda-package/ khiops | \ - awk '!/#|channels/ {print $2}' | \ - sort -u \ - ) - echo "PKG_VERSION=$PKG_VERSION" >> "$GITHUB_ENV" - # Flag if single version is detected (all *.bz2 packages have the same version) - # To be removed once GitHub supports Mac OS arm64 - SINGLE_VERSION=$( \ - if [ $(echo $PKG_VERSION | \ - awk '{ for(i = 1 ; i <= NF; i++) {print $i}}' | \ - sort -u | \ - wc -l \ - ) -eq 1 ]; \ - then - echo true; \ - else - echo false; \ - fi \ - ) - echo "SINGLE_VERSION=$SINGLE_VERSION" >> "$GITHUB_ENV" - - name: Upload Khiops Conda Package - uses: actions/upload-artifact@v3 - with: - name: khiops-conda-package - path: khiops-conda-package/*/khiops*.tar.bz2 - retention-days: 7 - # Create release only if single package version was detected and on a tag push - - name: Create the release zip archive - if: env.SINGLE_VERSION == 'true' && github.event_name == 'push' - uses: thedoctor0/zip-release@0.7.1 - with: - type: zip - directory: ./khiops-conda-package/ - filename: khiops-conda-${{ env.PKG_VERSION }}.zip - - name: Move Release Archive to the Current Directory - if: env.SINGLE_VERSION == 'true' && github.event_name == 'push' - run: mv ./khiops-conda-package/khiops-conda-${{ env.PKG_VERSION }}.zip . - name: Release the zip archive - if: env.SINGLE_VERSION == 'true' && github.event_name == 'push' + if: github.event_name == 'push' uses: ncipollo/release-action@v1 with: - artifacts: khiops-conda-${{ env.PKG_VERSION }}.zip + artifacts: khiops-${{ needs.build-packages.outputs.pkg-version }}-conda diff --git a/packaging/conda/meta.yaml b/packaging/conda/meta.yaml index 1e91a70a..e758e3a8 100644 --- a/packaging/conda/meta.yaml +++ b/packaging/conda/meta.yaml @@ -39,10 +39,10 @@ build: # - requires conda-forge requirements: build: - - mpich=4.0.3 # [linux] - - mpich-mpicxx=4.0.3 # [linux] - - mpich=3.4.3 # [osx] - - mpich-mpicxx=3.4.3 # [osx] + - mpich 4.0.3 # [linux] + - mpich-mpicxx 4.0.3 # [linux] + - mpich 3.4.3 # [osx] + - mpich-mpicxx 3.4.3 # [osx] - msmpi # [win] - cmake - ninja @@ -50,15 +50,15 @@ requirements: - setuptools - {{ compiler('cxx') }} host: - - mpich=4.0.3 # [linux] - - mpich-mpicxx=4.0.3 # [linux] - - mpich=3.4.3 # [osx] - - mpich-mpicxx=3.4.3 # [osx] + - mpich 4.0.3 # [linux] + - mpich-mpicxx 4.0.3 # [linux] + - mpich 3.4.3 # [osx] + - mpich-mpicxx 3.4.3 # [osx] - msmpi # [win] - python run: - - mpich=4.0.3 # [linux] - - mpich=3.4.3 # [osx] + - mpich 4.0.3 # [linux] + - mpich 3.4.3 # [osx] - msmpi # [win] - pandas >=0.25.3 - python From 2022eec55087458f30ea3fe48da101dfebabc052 Mon Sep 17 00:00:00 2001 From: Felipe Olmos <92923444+folmos-at-orange@users.noreply.github.com> Date: Wed, 7 Feb 2024 14:17:19 +0100 Subject: [PATCH 5/5] Add conda build README --- packaging/conda/README.md | 34 ++++++++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) create mode 100644 packaging/conda/README.md diff --git a/packaging/conda/README.md b/packaging/conda/README.md new file mode 100644 index 00000000..a8dd9bd7 --- /dev/null +++ b/packaging/conda/README.md @@ -0,0 +1,34 @@ +# Khiops Conda Packaging Scripts +In conda, `khiops-python` is packaged with the Khiops executables. These executables are built from +source during the packaging process. + +## How to Build +You'll need `conda-build` installed in your system. The environment variable `KHIOPS_REVISION` must +be set to a Git hash/branch/tag of the `khiops` repository. + +```bash +# At the root of the repo +# These commands will leave a ready to use conda channel in `./khiops-conda-build` +KHIOPS_REVISION=10.2.0 + +# Windows +conda build --output-folder ./khiops-conda-build packaging/conda + +# Linux/macOS +# Note: We need the conda-forge channel to obtain the pinned versions of MPICH +conda build --channel conda-forge --output-folder ./khiops-conda-build packaging/conda +``` + +### Executable Signatures in macOS +The script can sign the Khiops binaries. This is to avoid annoying firewall pop-ups. To enable this +set the following environment variables: +- `KHIOPS_APPLE_CERTIFICATE_ID`: The id of the signing certificate. +- `KHIOPS_APPLE_CERTIFICATE_COMMON_NAME`: The common name of the signing certificate. +-`KHIOPS_APPLE_CERTIFICATE_PASSWORD`: The password of the signing certificate. +- `KHIOPS_APPLE_CERTIFICATE_BASE64`: The base64 encoding of the signing certificate. +- `KHIOPS_APPLE_TMP_KEYCHAIN_PASSWORD` : A password for the temporary keychain created in the process. + +For more details see the comments in the signing section of `build.sh`. + + +