-
Notifications
You must be signed in to change notification settings - Fork 1
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Merge pull request #126 from KhiopsML/full-conda-pkg-tests
Update conda build workflow
- Loading branch information
Showing
4 changed files
with
120 additions
and
139 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,5 +1,5 @@ | ||
| --- | ||
| name: Build Conda Packages | ||
| name: Conda | ||
| env: | ||
| DEFAULT_KHIOPS_REVISION: dev | ||
| DEFAULT_SAMPLES_REVISION: main | ||
|
|
@@ -8,68 +8,87 @@ on: | |
| inputs: | ||
| khiops-revision: | ||
| default: dev | ||
| description: khiops repo tag/branch/hash | ||
| description: khiops repo revision | ||
| samples-revision: | ||
| default: main | ||
| description: khiops-samples repo tag/branch/hash | ||
| description: khiops-samples repo revision | ||
| push: | ||
| tags: ['*'] | ||
| tags: [v*] | ||
| pull_request: | ||
| paths: [.github/workflows/build-conda-package.yml] | ||
| defaults: | ||
| run: | ||
| shell: bash -el {0} | ||
| concurrency: | ||
| group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} | ||
| cancel-in-progress: true | ||
| jobs: | ||
| build-conda-package: | ||
| build-packages: | ||
| name: Build Packages | ||
| strategy: | ||
| fail-fast: false | ||
| matrix: | ||
| # Use oldest supported Mac OS and Ubuntu versions for GLIBC compatibility | ||
| # Use oldest supported Windows version | ||
| os: [ubuntu-20.04, windows-2019, macos-11] | ||
| # Use the oldest supported Mac OS and Ubuntu versions for GLIBC compatibility | ||
| os: [ubuntu-20.04, windows-latest, macos-11] | ||
| runs-on: ${{ matrix.os }} | ||
| outputs: | ||
| pkg-version: ${{ steps.get-pkg-version.outputs.pkg-version }} | ||
| steps: | ||
| # We move KHIOPS_REVISION to the environment so that we can use | ||
| # them in both push and workflow_dispatch events | ||
| - name: Move KHIOPS_REVISION to the environment (push event) | ||
| if: github.event_name == 'push' | ||
| run: echo "KHIOPS_REVISION=${DEFAULT_KHIOPS_REVISION}" >> "$GITHUB_ENV" | ||
| - name: Move KHIOPS_REVISION to the environment (workflow_dispatch event) | ||
| if: github.event_name == 'workflow_dispatch' | ||
| run: echo "KHIOPS_REVISION=${{ inputs.khiops-revision }}" >> "$GITHUB_ENV" | ||
| - name: Checkout Sources | ||
| uses: actions/checkout@v3 | ||
| with: | ||
| # Checkout the full repository to have the tags so versioneer works properly | ||
| # See issue https://github.com/actions/checkout/issues/701 | ||
| fetch-depth: 0 | ||
| # We move KHIOPS_REVISION to the environment so that we can use | ||
| # them in both push and workflow_dispatch events | ||
| - name: Move KHIOPS_REVISION to the environment (push/PR) | ||
| if: github.event_name == 'push' || github.event_name == 'pull_request' | ||
| run: echo "KHIOPS_REVISION=${DEFAULT_KHIOPS_REVISION}" >> "$GITHUB_ENV" | ||
| - name: Move KHIOPS_REVISION to the environment (workflow_dispatch) | ||
| if: github.event_name == 'workflow_dispatch' | ||
| run: echo "KHIOPS_REVISION=${{ inputs.khiops-revision }}" >> "$GITHUB_ENV" | ||
| - name: Install Miniconda | ||
| if: success() || failure() | ||
| uses: conda-incubator/setup-miniconda@v2 | ||
| uses: conda-incubator/setup-miniconda@v3 | ||
| with: | ||
| activate-environment: khiops-python-env | ||
| - name: Install Dependency Requirements for Building Conda Packages | ||
| run: conda install conda-build conda-verify ninja | ||
| run: conda install conda-build=3.27.0 conda-verify | ||
| # We need MacOS SDK 10.10 to build on Big Sur | ||
| - name: Install Mac OS SDK 10.10 | ||
| if: runner.os == 'macOS' | ||
| run: | | ||
| wget https://github.com/phracker/MacOSX-SDKs/releases/download/11.3/MacOSX10.10.sdk.tar.xz | ||
| sudo tar -zxvf MacOSX10.10.sdk.tar.xz -C /opt | ||
| - name: Build Khiops Conda Package | ||
| - name: Build Khiops Conda Package (Windows) | ||
| if: runner.os == 'Windows' | ||
| run: | | ||
| mkdir khiops-conda | ||
| conda build --output-folder khiops-conda ./packaging/conda | ||
| # In Linux/macOS we need the conda-forge channel to install their pinned versions | ||
| - name: Build Khiops Conda Package (Linux/macOS) | ||
| if: runner.os != 'Windows' | ||
| run: | | ||
| mkdir build | ||
| conda build --output-folder build packaging/conda | ||
| mkdir khiops-conda | ||
| conda build --channel conda-forge --output-folder khiops-conda ./packaging/conda | ||
| - name: Extract Khiops Package Version | ||
| id: get-pkg-version | ||
| run: | | ||
| PKG_VERSION=$(\ | ||
| conda search --override-channels --channel ./khiops-conda/ khiops \ | ||
| | awk '!/#|channels/ {print $2}' \ | ||
| | sort -u \ | ||
| ) | ||
| echo "pkg-version=$PKG_VERSION" >> "$GITHUB_OUTPUT" | ||
| - name: Upload Khiops Conda Package | ||
| uses: actions/upload-artifact@v3 | ||
| with: | ||
| name: khiops-conda-package-no-osx-arm64 | ||
| path: build/*/khiops*.tar.bz2 | ||
| name: khiops-${{ steps.get-pkg-version.outputs.pkg-version }}-conda | ||
| path: ./khiops-conda | ||
| retention-days: 7 | ||
| # Test Conda package on brand new environments | ||
| test-conda-package: | ||
| needs: build-conda-package | ||
| test-packages: | ||
| needs: build-packages | ||
| strategy: | ||
| fail-fast: false | ||
| matrix: | ||
|
|
@@ -101,30 +120,25 @@ jobs: | |
| ref: ${{ env.SAMPLES_REVISION }} | ||
| token: ${{ secrets.GITHUB_TOKEN }} | ||
| path: ${{ env.KHIOPS_SAMPLES_DIR }} | ||
| - name: Download Conda Package Artifact | ||
| uses: actions/download-artifact@v3 | ||
| with: | ||
| name: khiops-conda-package-no-osx-arm64 | ||
| path: khiops-conda-package | ||
| - name: Install Miniconda | ||
| if: success() || failure() | ||
| uses: conda-incubator/setup-miniconda@v2 | ||
| uses: conda-incubator/setup-miniconda@v3 | ||
| with: | ||
| miniconda-version: latest # needed for Mac OS 13 | ||
| python-version: ${{ matrix.python-version }} | ||
| activate-environment: khiops-python-env | ||
| - name: Install conda-build and Setup Khiops Conda Channel | ||
| run: | | ||
| conda install conda-build | ||
| conda index ./khiops-conda-package/. | ||
| - name: Install the Khiops Conda package | ||
| run: conda install -c ./khiops-conda-package/ khiops | ||
| - name: Test Khiops Installation Status (Windows) | ||
| - name: Download Conda Package Artifact | ||
| uses: actions/download-artifact@v3 | ||
| with: | ||
| name: khiops-${{ needs.build-packages.outputs.pkg-version }}-conda | ||
| path: khiops-conda | ||
| - name: Install the Khiops Conda pagkage (Windows) | ||
| if: runner.os == 'Windows' | ||
| shell: cmd /C CALL {0} | ||
| run: C:\Miniconda3\envs\khiops-python-env\Scripts\kh-status.exe | ||
| - name: Test Khiops Installation Status (Linux/macOS) | ||
| run: conda install -c ./khiops-conda/ khiops | ||
| # In Linux/macOS we need the conda-forge channel to install their pinned versions | ||
| - name: Install the Khiops Conda package (Linux/macOS) | ||
| if: runner.os != 'Windows' | ||
| run: conda install -c conda-forge -c ./khiops-conda/ khiops | ||
| - name: Test Khiops Installation Status | ||
| run: kh-status | ||
| - name: Test Conda Package Installation on Samples | ||
| run: | | ||
|
|
@@ -134,83 +148,16 @@ jobs: | |
| kh-samples sklearn -i khiops_classifier -e | ||
| kh-samples sklearn -i khiops_coclustering -e | ||
| # Build and push Conda package release archive | ||
| release-conda-package: | ||
| needs: test-conda-package | ||
| release-packages: | ||
| # We need `build-packages` because we use its output | ||
| needs: [build-packages, test-packages] | ||
| runs-on: ubuntu-latest | ||
| permissions: | ||
| contents: write | ||
| packages: read | ||
| steps: | ||
| - name: Download Conda Package Artifact | ||
| uses: actions/download-artifact@v3 | ||
| with: | ||
| name: khiops-conda-package-no-osx-arm64 | ||
| path: khiops-conda-package | ||
| - name: Login to Github Container Registry | ||
| uses: docker/login-action@v2 | ||
| with: | ||
| registry: ghcr.io | ||
| username: ${{ github.actor }} | ||
| password: ${{ secrets.GITHUB_TOKEN }} | ||
| - name: Get Mac OS ARM64 Package from Dedicated Container | ||
| run: | | ||
| OSX_ARM_64_IMAGE=ghcr.io/khiopsml/khiops-python/conda-osx-arm64:latest | ||
| docker pull $OSX_ARM_64_IMAGE | ||
| CONTAINER_ID=$(docker run -it -d $OSX_ARM_64_IMAGE) | ||
| docker cp $CONTAINER_ID:/osx-arm64/. ./khiops-conda-package/osx-arm64/ | ||
| docker container rm -f $CONTAINER_ID | ||
| docker image rm $OSX_ARM_64_IMAGE | ||
| - name: Install Miniconda | ||
| if: success() || failure() | ||
| uses: conda-incubator/setup-miniconda@v2 | ||
| with: | ||
| activate-environment: khiops-python-env | ||
| - name: Create Conda Channel | ||
| run: | | ||
| conda install conda-build | ||
| conda index ./khiops-conda-package/ | ||
| - name: Extract Conda Package Version | ||
| run: | | ||
| PKG_VERSION=$(\ | ||
| conda search --override-channels --channel ./khiops-conda-package/ khiops | \ | ||
| awk '!/#|channels/ {print $2}' | \ | ||
| sort -u \ | ||
| ) | ||
| echo "PKG_VERSION=$PKG_VERSION" >> "$GITHUB_ENV" | ||
| # Flag if single version is detected (all *.bz2 packages have the same version) | ||
| # To be removed once GitHub supports Mac OS arm64 | ||
| SINGLE_VERSION=$( \ | ||
| if [ $(echo $PKG_VERSION | \ | ||
| awk '{ for(i = 1 ; i <= NF; i++) {print $i}}' | \ | ||
| sort -u | \ | ||
| wc -l \ | ||
| ) -eq 1 ]; \ | ||
| then | ||
| echo true; \ | ||
| else | ||
| echo false; \ | ||
| fi \ | ||
| ) | ||
| echo "SINGLE_VERSION=$SINGLE_VERSION" >> "$GITHUB_ENV" | ||
| - name: Upload Khiops Conda Package | ||
| uses: actions/upload-artifact@v3 | ||
| with: | ||
| name: khiops-conda-package | ||
| path: khiops-conda-package/*/khiops*.tar.bz2 | ||
| retention-days: 7 | ||
| # Create release only if single package version was detected and on a tag push | ||
| - name: Create the release zip archive | ||
| if: env.SINGLE_VERSION == 'true' && github.event_name == 'push' | ||
| uses: thedoctor0/[email protected] | ||
| with: | ||
| type: zip | ||
| directory: ./khiops-conda-package/ | ||
| filename: khiops-conda-${{ env.PKG_VERSION }}.zip | ||
| - name: Move Release Archive to the Current Directory | ||
| if: env.SINGLE_VERSION == 'true' && github.event_name == 'push' | ||
| run: mv ./khiops-conda-package/khiops-conda-${{ env.PKG_VERSION }}.zip . | ||
| - name: Release the zip archive | ||
| if: env.SINGLE_VERSION == 'true' && github.event_name == 'push' | ||
| if: github.event_name == 'push' | ||
| uses: ncipollo/release-action@v1 | ||
| with: | ||
| artifacts: khiops-conda-${{ env.PKG_VERSION }}.zip | ||
| artifacts: khiops-${{ needs.build-packages.outputs.pkg-version }}-conda | ||
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,34 @@ | ||
| # Khiops Conda Packaging Scripts | ||
| In conda, `khiops-python` is packaged with the Khiops executables. These executables are built from | ||
| source during the packaging process. | ||
|
|
||
| ## How to Build | ||
| You'll need `conda-build` installed in your system. The environment variable `KHIOPS_REVISION` must | ||
| be set to a Git hash/branch/tag of the `khiops` repository. | ||
|
|
||
| ```bash | ||
| # At the root of the repo | ||
| # These commands will leave a ready to use conda channel in `./khiops-conda-build` | ||
| KHIOPS_REVISION=10.2.0 | ||
|
|
||
| # Windows | ||
| conda build --output-folder ./khiops-conda-build packaging/conda | ||
|
|
||
| # Linux/macOS | ||
| # Note: We need the conda-forge channel to obtain the pinned versions of MPICH | ||
| conda build --channel conda-forge --output-folder ./khiops-conda-build packaging/conda | ||
| ``` | ||
|
|
||
| ### Executable Signatures in macOS | ||
| The script can sign the Khiops binaries. This is to avoid annoying firewall pop-ups. To enable this | ||
| set the following environment variables: | ||
| - `KHIOPS_APPLE_CERTIFICATE_ID`: The id of the signing certificate. | ||
| - `KHIOPS_APPLE_CERTIFICATE_COMMON_NAME`: The common name of the signing certificate. | ||
| -`KHIOPS_APPLE_CERTIFICATE_PASSWORD`: The password of the signing certificate. | ||
| - `KHIOPS_APPLE_CERTIFICATE_BASE64`: The base64 encoding of the signing certificate. | ||
| - `KHIOPS_APPLE_TMP_KEYCHAIN_PASSWORD` : A password for the temporary keychain created in the process. | ||
|
|
||
| For more details see the comments in the signing section of `build.sh`. | ||
|
|
||
|
|
||
|
|
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters