Update conda build workflow

.github/workflows/build-conda-package.yml

@@ -1,5 +1,5 @@
 ---
-name: Build Conda Packages
+name: Conda
 env:
   DEFAULT_KHIOPS_REVISION: dev
   DEFAULT_SAMPLES_REVISION: main
@@ -8,68 +8,87 @@ on:
     inputs:
       khiops-revision:
         default: dev
-        description: khiops repo tag/branch/hash
+        description: khiops repo revision
       samples-revision:
         default: main
-        description: khiops-samples repo tag/branch/hash
+        description: khiops-samples repo revision
   push:
-    tags: ['*']
+    tags: [v*]
+  pull_request:
+    paths: [.github/workflows/build-conda-package.yml]
 defaults:
   run:
     shell: bash -el {0}
 concurrency:
   group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
   cancel-in-progress: true
 jobs:
-  build-conda-package:
+  build-packages:
+    name: Build Packages
     strategy:
       fail-fast: false
       matrix:
-        # Use oldest supported Mac OS and Ubuntu versions for GLIBC compatibility
-        # Use oldest supported Windows version
-        os: [ubuntu-20.04, windows-2019, macos-11]
+        #  Use the oldest supported Mac OS and Ubuntu versions for GLIBC compatibility
+        os: [ubuntu-20.04, windows-latest, macos-11]
     runs-on: ${{ matrix.os }}
+    outputs:
+      pkg-version: ${{ steps.get-pkg-version.outputs.pkg-version }}
     steps:
-      # We move KHIOPS_REVISION to the environment so that we can use
-      # them in both push and workflow_dispatch events
-      - name: Move KHIOPS_REVISION to the environment (push event)
-        if: github.event_name == 'push'
-        run: echo "KHIOPS_REVISION=${DEFAULT_KHIOPS_REVISION}" >> "$GITHUB_ENV"
-      - name: Move KHIOPS_REVISION to the environment (workflow_dispatch event)
-        if: github.event_name == 'workflow_dispatch'
-        run: echo "KHIOPS_REVISION=${{ inputs.khiops-revision }}" >> "$GITHUB_ENV"
       - name: Checkout Sources
         uses: actions/checkout@v3
         with:
           # Checkout the full repository to have the tags so versioneer works properly
           # See issue https://github.com/actions/checkout/issues/701
           fetch-depth: 0
+      # We move KHIOPS_REVISION to the environment so that we can use
+      # them in both push and workflow_dispatch events
+      - name: Move KHIOPS_REVISION to the environment (push/PR)
+        if: github.event_name == 'push' || github.event_name == 'pull_request'
+        run: echo "KHIOPS_REVISION=${DEFAULT_KHIOPS_REVISION}" >> "$GITHUB_ENV"
+      - name: Move KHIOPS_REVISION to the environment (workflow_dispatch)
+        if: github.event_name == 'workflow_dispatch'
+        run: echo "KHIOPS_REVISION=${{ inputs.khiops-revision }}" >> "$GITHUB_ENV"
       - name: Install Miniconda
-        if: success() || failure()
-        uses: conda-incubator/setup-miniconda@v2
+        uses: conda-incubator/setup-miniconda@v3
         with:
           activate-environment: khiops-python-env
       - name: Install Dependency Requirements for Building Conda Packages
-        run: conda install conda-build conda-verify ninja
+        run: conda install conda-build=3.27.0 conda-verify
       # We need MacOS SDK 10.10 to build on Big Sur
       - name: Install Mac OS SDK 10.10
         if: runner.os == 'macOS'
         run: |
           wget https://github.com/phracker/MacOSX-SDKs/releases/download/11.3/MacOSX10.10.sdk.tar.xz
           sudo tar -zxvf MacOSX10.10.sdk.tar.xz -C /opt
-      - name: Build Khiops Conda Package
+      - name: Build Khiops Conda Package (Windows)
+        if: runner.os == 'Windows'
+        run: |
+          mkdir khiops-conda
+          conda build --output-folder khiops-conda ./packaging/conda
+      # In Linux/macOS we need the conda-forge channel to install their pinned versions
+      - name: Build Khiops Conda Package (Linux/macOS)
+        if: runner.os != 'Windows'
         run: |
-          mkdir build
-          conda build --output-folder build packaging/conda
+          mkdir khiops-conda
+          conda build --channel conda-forge --output-folder khiops-conda ./packaging/conda
+      - name: Extract Khiops Package Version
+        id: get-pkg-version
+        run: |
+          PKG_VERSION=$(\
+              conda search --override-channels --channel ./khiops-conda/ khiops \
+                | awk '!/#|channels/ {print $2}' \
+                | sort -u \
+          )
+          echo "pkg-version=$PKG_VERSION" >> "$GITHUB_OUTPUT"
       - name: Upload Khiops Conda Package
         uses: actions/upload-artifact@v3
         with:
-          name: khiops-conda-package-no-osx-arm64
-          path: build/*/khiops*.tar.bz2
+          name: khiops-${{ steps.get-pkg-version.outputs.pkg-version }}-conda
+          path: ./khiops-conda
           retention-days: 7
   # Test Conda package on brand new environments
-  test-conda-package:
-    needs: build-conda-package
+  test-packages:
+    needs: build-packages
     strategy:
       fail-fast: false
       matrix:
@@ -101,30 +120,25 @@ jobs:
           ref: ${{ env.SAMPLES_REVISION }}
           token: ${{ secrets.GITHUB_TOKEN }}
           path: ${{ env.KHIOPS_SAMPLES_DIR }}
-      - name: Download Conda Package Artifact
-        uses: actions/download-artifact@v3
-        with:
-          name: khiops-conda-package-no-osx-arm64
-          path: khiops-conda-package
       - name: Install Miniconda
-        if: success() || failure()
-        uses: conda-incubator/setup-miniconda@v2
+        uses: conda-incubator/setup-miniconda@v3
         with:
           miniconda-version: latest  # needed for Mac OS 13
           python-version: ${{ matrix.python-version }}
           activate-environment: khiops-python-env
-      - name: Install conda-build and Setup Khiops Conda Channel
-        run: |
-          conda install conda-build
-          conda index ./khiops-conda-package/.
-      - name: Install the Khiops Conda package
-        run: conda install -c ./khiops-conda-package/ khiops
-      - name: Test Khiops Installation Status (Windows)
+      - name: Download Conda Package Artifact
+        uses: actions/download-artifact@v3
+        with:
+          name: khiops-${{ needs.build-packages.outputs.pkg-version }}-conda
+          path: khiops-conda
+      - name: Install the Khiops Conda pagkage (Windows)
         if: runner.os == 'Windows'
-        shell: cmd /C CALL {0}
-        run: C:\Miniconda3\envs\khiops-python-env\Scripts\kh-status.exe
-      - name: Test Khiops Installation Status (Linux/macOS)
+        run: conda install -c ./khiops-conda/ khiops
+      # In Linux/macOS we need the conda-forge channel to install their pinned versions
+      - name: Install the Khiops Conda package (Linux/macOS)
         if: runner.os != 'Windows'
+        run: conda install -c conda-forge -c ./khiops-conda/ khiops
+      - name: Test Khiops Installation Status
         run: kh-status
       - name: Test Conda Package Installation on Samples
         run: |
@@ -134,83 +148,16 @@ jobs:
           kh-samples sklearn -i khiops_classifier -e
           kh-samples sklearn -i khiops_coclustering -e
   # Build and push Conda package release archive
-  release-conda-package:
-    needs: test-conda-package
+  release-packages:
+    # We need `build-packages` because we use its output
+    needs: [build-packages, test-packages]
     runs-on: ubuntu-latest
     permissions:
       contents: write
       packages: read
     steps:
-      - name: Download Conda Package Artifact
-        uses: actions/download-artifact@v3
-        with:
-          name: khiops-conda-package-no-osx-arm64
-          path: khiops-conda-package
-      - name: Login to Github Container Registry
-        uses: docker/login-action@v2
-        with:
-          registry: ghcr.io
-          username: ${{ github.actor }}
-          password: ${{ secrets.GITHUB_TOKEN }}
-      - name: Get Mac OS ARM64 Package from Dedicated Container
-        run: |
-          OSX_ARM_64_IMAGE=ghcr.io/khiopsml/khiops-python/conda-osx-arm64:latest
-          docker pull $OSX_ARM_64_IMAGE
-          CONTAINER_ID=$(docker run -it -d $OSX_ARM_64_IMAGE)
-          docker cp $CONTAINER_ID:/osx-arm64/. ./khiops-conda-package/osx-arm64/
-          docker container rm -f $CONTAINER_ID
-          docker image rm $OSX_ARM_64_IMAGE
-      - name: Install Miniconda
-        if: success() || failure()
-        uses: conda-incubator/setup-miniconda@v2
-        with:
-          activate-environment: khiops-python-env
-      - name: Create Conda Channel
-        run: |
-          conda install conda-build
-          conda index ./khiops-conda-package/
-      - name: Extract Conda Package Version
-        run: |
-          PKG_VERSION=$(\
-              conda search --override-channels --channel ./khiops-conda-package/ khiops | \
-              awk '!/#|channels/ {print $2}' | \
-              sort -u \
-          )
-          echo "PKG_VERSION=$PKG_VERSION" >> "$GITHUB_ENV"
-          # Flag if single version is detected (all *.bz2 packages have the same version)
-          # To be removed once GitHub supports Mac OS arm64
-          SINGLE_VERSION=$( \
-              if [ $(echo $PKG_VERSION | \
-                     awk '{ for(i = 1 ; i <= NF; i++) {print $i}}' | \
-                     sort -u | \
-                     wc -l \
-                   ) -eq 1 ]; \
-              then
-                echo true; \
-              else
-                echo false; \
-              fi \
-          )
-          echo "SINGLE_VERSION=$SINGLE_VERSION" >> "$GITHUB_ENV"
-      - name: Upload Khiops Conda Package
-        uses: actions/upload-artifact@v3
-        with:
-          name: khiops-conda-package
-          path: khiops-conda-package/*/khiops*.tar.bz2
-          retention-days: 7
-      # Create release only if single package version was detected and on a tag push
-      - name: Create the release zip archive
-        if: env.SINGLE_VERSION == 'true' && github.event_name == 'push'
-        uses: thedoctor0/[email protected]
-        with:
-          type: zip
-          directory: ./khiops-conda-package/
-          filename: khiops-conda-${{ env.PKG_VERSION }}.zip
-      - name: Move Release Archive to the Current Directory
-        if: env.SINGLE_VERSION == 'true' && github.event_name == 'push'
-        run: mv ./khiops-conda-package/khiops-conda-${{ env.PKG_VERSION }}.zip .
       - name: Release the zip archive
-        if: env.SINGLE_VERSION == 'true' && github.event_name == 'push'
+        if: github.event_name == 'push'
         uses: ncipollo/release-action@v1
         with:
-          artifacts: khiops-conda-${{ env.PKG_VERSION }}.zip
+          artifacts: khiops-${{ needs.build-packages.outputs.pkg-version }}-conda

packaging/conda/README.md

@@ -0,0 +1,34 @@
+# Khiops Conda Packaging Scripts
+In conda, `khiops-python` is packaged with the Khiops executables. These executables are built from
+source during the packaging process.
+
+## How to Build
+You'll need `conda-build` installed in your system. The environment variable `KHIOPS_REVISION` must
+be set to a Git hash/branch/tag of the `khiops` repository.
+
+```bash
+# At the root of the repo
+# These commands will leave a ready to use conda channel in `./khiops-conda-build`
+KHIOPS_REVISION=10.2.0
+
+# Windows
+conda build --output-folder ./khiops-conda-build packaging/conda
+
+# Linux/macOS
+# Note: We need the conda-forge channel to obtain the pinned versions of MPICH
+conda build --channel conda-forge --output-folder ./khiops-conda-build packaging/conda
+```
+
+### Executable Signatures in macOS
+The script can sign the Khiops binaries. This is to avoid annoying firewall pop-ups. To enable this
+set the following environment variables:
+- `KHIOPS_APPLE_CERTIFICATE_ID`: The id of the signing certificate.
+- `KHIOPS_APPLE_CERTIFICATE_COMMON_NAME`: The common name of the signing certificate.
+-`KHIOPS_APPLE_CERTIFICATE_PASSWORD`: The password of the signing certificate.
+- `KHIOPS_APPLE_CERTIFICATE_BASE64`: The base64 encoding of the signing certificate.
+- `KHIOPS_APPLE_TMP_KEYCHAIN_PASSWORD` : A password for the temporary keychain created in the process.
+
+For more details see the comments in the signing section of `build.sh`.
+
+
+

packaging/conda/build.sh

@@ -4,7 +4,7 @@
 set -euo pipefail
 
 # Clone Khiops sources (we change working directory there)
-git clone --depth 1 https://github.com/khiopsml/khiops.git khiops-core
+git clone https://github.com/khiopsml/khiops.git khiops-core
 cd khiops-core
 git checkout "$KHIOPS_REVISION"
 
@@ -55,7 +55,7 @@ $PYTHON -m pip install . --no-deps --ignore-installed --no-cache-dir --no-build-
 # - KHIOPS_APPLE_CERTIFICATE_BASE64: The identity file .p12 (certificate + private key) in base64
 # - KHIOPS_APPLE_TMP_KEYCHAIN_PASSWORD: Password to decrypt the certificate
 #
-if [[ "$(uname)" == "Darwin" && "$(uname -m)" == "arm64" ]]
+if [[ "$(uname)" == "Darwin" && "$(uname -m)" == "arm64" && -n "${KHIOPS_APPLE_CERTIFICATE_BASE64-}" ]]
 then
   # Delete the rpath of each executable
   # Delete two times for MODL because for some reason it is there 2 times
@@ -98,7 +98,7 @@ then
 
   # We make sure to use the default macOS/Xcode codesign tool. This is because the sigtool python
   # package (installed by conda build as a dependency) makes an alias "codesign" which is prioritary
-  # in the build environment. The alias, however, alias doen't support signing with a proper
+  # in the build environment. The alias, however, alias doesn't support signing with a proper
   # identity and makes the build fail!
   CODESIGN="/usr/bin/codesign"
 

packaging/conda/meta.yaml

@@ -8,7 +8,7 @@ source:
 
 build:
   script_env:
-    - KHIOPS_REVISION=dev
+    - KHIOPS_REVISION
     # Variables for signing the MODL executables in osx-arm64
     - KHIOPS_APPLE_CERTIFICATE_BASE64      # [osx and arm64]
     - KHIOPS_APPLE_CERTIFICATE_PASSWORD    # [osx and arm64]
@@ -30,36 +30,36 @@ build:
     - bin/pk-status                      # [osx and arm64]
   detect_binary_files_with_prefix: false # [osx and arm64]
 
-# Note for OSX ARM 64:
-# - We impose mpich <=3.4.3 until a release with the fix for
-#   the bug https://github.com/pmodels/mpich/issues/6619
+# Note on version pinning:
+# OSX:
+# - mpich=3.4.3 because 4.* is still unstable
+# - requires conda-forge
+# Linux:
+# - mpich=4.0.3 because of bugs of the 3.* series
+# - requires conda-forge
 requirements:
   build:
+    - mpich 4.0.3        # [linux]
+    - mpich-mpicxx 4.0.3 # [linux]
+    - mpich 3.4.3        # [osx]
+    - mpich-mpicxx 3.4.3 # [osx]
+    - msmpi              # [win]
     - cmake
-    - mpich                # [linux or osx and x86]
-    - mpich-mpicc          # [linux or osx and x86]
-    - mpich-mpicxx         # [linux or osx and x86]
-    - mpich <=3.4.3        # [osx and arm64]
-    - mpich-mpicc <=3.4.3  # [osx and arm64]
-    - mpich-mpicxx <=3.4.3 # [osx and arm64]
-    - msmpi                # [win]
-    - ninja                # [win]
+    - ninja
     - python
     - setuptools
     - {{ compiler('cxx') }}
   host:
-    - mpich                # [linux or osx and x86]
-    - mpich-mpicc          # [linux or osx and x86]
-    - mpich-mpicxx         # [linux or osx and x86]
-    - mpich <=3.4.3        # [osx and arm64]
-    - mpich-mpicc <=3.4.3  # [osx and arm64]
-    - mpich-mpicxx <=3.4.3 # [osx and arm64]
+    - mpich 4.0.3        # [linux]
+    - mpich-mpicxx 4.0.3 # [linux]
+    - mpich 3.4.3        # [osx]
+    - mpich-mpicxx 3.4.3 # [osx]
     - msmpi                # [win]
     - python
   run:
-    - mpich           # [linux or osx and x64]
-    - msmpi           # [win]
-    - mpich <=3.4.3   # [osx and arm64]
+    - mpich 4.0.3   # [linux]
+    - mpich 3.4.3   # [osx]
+    - msmpi         # [win]
     - pandas >=0.25.3
     - python
     - scikit-learn >=0.22.2