Skip to content

Commit

Permalink
Updates
Browse files Browse the repository at this point in the history 
- Add badge to readme
- Make gitlab ci based on trigger word
- Make conda build, container build as re-usable workflows
  • Loading branch information
@ggeorgakoudis
ggeorgakoudis committed Sep 16, 2024
1 parent 91f5371 commit ebfeb89
Show file tree
Hide file tree
Showing 7 changed files with 286 additions and 383 deletions.
120 changes: 1 addition & 119 deletions .github/workflows/build-containers.yml
View file
Original file line number Diff line number Diff line change
@@ -1,127 +1,9 @@
name: Deploy containers

on:
#release:
# types: [published]
workflow_dispatch:
push:
branches:
- "build-containers"

# Rembember: you have to escape characters like $ with \$.
env:
SETUP_BASE: |
apt-get -qq update > /dev/null;
apt-get -qq upgrade > /dev/null;
apt-get -qq install wget git > /dev/null;
useradd -ms /bin/bash pyompuser;
su pyompuser;
cd /home/pyompuser;
SETUP_MINICONDA: |
wget -q https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-\$(uname -m).sh -O miniconda.sh;
bash miniconda.sh -b -u -p ./miniconda3;
rm -rf miniconda.sh;
export PATH=/home/pyompuser/miniconda3/bin:\${PATH};
conda install -y -c conda-forge conda-build conda-verify anaconda-client;
conda config --set anaconda_upload yes;
SETUP_REPO: |
git clone https://github.com/Python-for-HPC/PyOMP;
cd PyOMP;
# TODO: remove this line when the project is ready for deployment
git switch build-containers;
#git checkout \$(git describe --abbrev=0 --tags);
export GITHUB_HEAD_SHA=\$(git rev-parse --short HEAD);
workflow_call:

jobs:
#conda-deploy-llvm-openmp-dev-linux-arm64:
# name: llvm-openmp-dev linux-arm64
# runs-on: ubuntu-latest
# steps:
# - name: Set up QEMU
# uses: docker/setup-qemu-action@v3
# with:
# platforms: linux/arm64
# - name: Deploy llvm-openmp-dev
# run: |
# docker run --platform linux/arm64 ubuntu:22.04 /bin/bash -xel -c "
# ${{ env.SETUP_BASE }}
# ${{ env.SETUP_MINICONDA }}
# ${{ env.SETUP_REPO }}
# conda build --user python-for-hpc --label main \
# -c python-for-hpc -c conda-forge \
# --token ${{ secrets.ANACONDA_TOKEN }} \
# buildscripts/conda-recipes/llvm-openmp-dev;
# "

#conda-deploy-llvmlite-linux-arm64:
# needs: conda-deploy-llvm-openmp-dev-linux-arm64
# name: llvmlite linux-arm64
# runs-on: ubuntu-latest
# strategy:
# matrix:
# python-version: ["3.8", "3.9", "3.10"]
# steps:
# - name: Set up QEMU
# uses: docker/setup-qemu-action@v3
# with:
# platforms: linux/arm64
# - name: Deploy llvmlite
# run: |
# docker run --platform linux/arm64 ubuntu:22.04 /bin/bash -el -c "
# ${{ env.SETUP_BASE }}
# ${{ env.SETUP_MINICONDA }}
# ${{ env.SETUP_REPO }}
# conda build --user python-for-hpc --label main \
# -c python-for-hpc -c conda-forge \
# --python ${{ matrix.python-version}} --token ${{ secrets.ANACONDA_TOKEN }} \
# buildscripts/conda-recipes/llvmlite
# "

#conda-deploy-numba-linux-arm64:
# #needs: conda-deploy-llvmlite-linux-arm64
# name: numba linux-arm64
# runs-on: ubuntu-latest
# strategy:
# matrix:
# python-version: ["3.8", "3.9", "3.10"]
# steps:
# - name: Set up QEMU
# uses: docker/setup-qemu-action@v3
# with:
# platforms: linux/arm64
# - name: Deploy numba
# run: |
# docker run --platform linux/arm64 ubuntu:22.04 /bin/bash -el -c "
# ${{ env.SETUP_BASE }}
# ${{ env.SETUP_MINICONDA }}
# ${{ env.SETUP_REPO }}
# conda build --user python-for-hpc --label main \
# -c python-for-hpc -c conda-forge \
# --python ${{ matrix.python-version}} --token ${{ secrets.ANACONDA_TOKEN }} \
# buildscripts/conda-recipes/numba;
# "

#conda-deploy-pyomp-linux-arm64:
# needs: conda-deploy-numba-linux-arm64
# name: pyomp linux-arm64
# runs-on: ubuntu-latest
# steps:
# - name: Set up QEMU
# uses: docker/setup-qemu-action@v3
# with:
# platforms: linux/arm64
# - name: Deploy pyomp
# run: |
# docker run --platform linux/arm64 ubuntu:22.04 /bin/bash -el -c "
# ${{ env.SETUP_BASE }}
# ${{ env.SETUP_MINICONDA }}
# ${{ env.SETUP_REPO }}
# conda build --user python-for-hpc --label main \
# -c python-for-hpc -c conda-forge \
# --token ${{ secrets.ANACONDA_TOKEN }} \
# buildscripts/conda-recipes/pyomp;
# "

deploy-containers:
runs-on: ubuntu-latest
steps:
Expand Down
261 changes: 261 additions & 0 deletions .github/workflows/build-upload-conda-base.yml
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,261 @@
name: Deploy conda pkgs base

on:
workflow_call:
inputs:
label:
required: true
type: string
commit:
required: true
type: string
env:
required: true
type: string

# Rembember: you have to escape characters like $ with \$.
env:
SETUP_BASE: |
apt-get -qq update > /dev/null;
apt-get -qq upgrade > /dev/null;
apt-get -qq install wget git > /dev/null;
useradd -ms /bin/bash pyompuser;
su pyompuser;
cd /home/pyompuser;
SETUP_MINICONDA: |
wget -q https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-\$(uname -m).sh -O miniconda.sh;
bash miniconda.sh -b -u -p ./miniconda3;
rm -rf miniconda.sh;
export PATH=/home/pyompuser/miniconda3/bin:\${PATH};
conda install -y -c conda-forge conda-build conda-verify anaconda-client;
conda config --set anaconda_upload yes;
SETUP_REPO: |
git clone https://github.com/Python-for-HPC/PyOMP;
cd PyOMP;
# TODO: remove this line when the project is ready for deployment
git checkout ${{ inputs.commit }};
export GITHUB_HEAD_SHA=${{ inputs.commit }};
jobs:
# Job to deploy llvm-openmp-dev, runs once as it is independent of the python
# version.
conda-deploy-llvm-openmp-dev:
name: llvm-openmp-dev ${{ matrix.os }}
runs-on: ${{ matrix.os }}
strategy:
matrix:
#os: [ubuntu-latest, macOS-latest, windows-latest]
os: [ubuntu-latest, macOS-latest]
steps:
- uses: actions/checkout@v4
- name: Set env for HEAD SHA
run:
echo "GITHUB_HEAD_SHA=${{ inputs.commit }}" >> $GITHUB_ENV
- name: Create and activate conda env
uses: conda-incubator/setup-miniconda@v3
with:
python-version: "3.10"
environment-file: ${{ inputs.env }}
auto-update-conda: false
auto-activate-base: false
show-channel-urls: true
- name: Build and upload llvm-openmp-dev
uses: uibcdf/[email protected]
with:
meta_yaml_dir: buildscripts/conda-recipes/llvm-openmp-dev
python-version: "3.10"
user: python-for-hpc
label: ${{ inputs.label }}
overwrite: true
token: ${{ secrets.ANACONDA_TOKEN }}

# Job to deploy llvmlite and numba, matrixed on os and python version.
conda-deploy-llvmlite:
needs: conda-deploy-llvm-openmp-dev
name: llvmlite ${{ matrix.os }} ${{ matrix.python-version }}
runs-on: ${{ matrix.os }}
strategy:
matrix:
#os: [ubuntu-latest, macOS-latest, windows-latest]
os: [ubuntu-latest, macOS-latest]
python-version: ["3.8", "3.9", "3.10"]
steps:
- uses: actions/checkout@v4
- name: Set env for HEAD SHA
run:
echo "GITHUB_HEAD_SHA=${{ inputs.commit }}" >> $GITHUB_ENV
- name: Create and activate conda env
uses: conda-incubator/setup-miniconda@v3
with:
python-version: ${{ matrix.python-version }}
environment-file: ${{ inputs.env }}
auto-update-conda: false
auto-activate-base: false
show-channel-urls: true
- name: Build and upload llvmlite
uses: uibcdf/[email protected]
with:
meta_yaml_dir: buildscripts/conda-recipes/llvmlite
python-version: ${{ matrix.python-version }}
user: python-for-hpc
label: ${{ inputs.label }}
overwrite: true
token: ${{ secrets.ANACONDA_TOKEN }}

# Job to deploy numba, matrixed on os and python version.
conda-deploy-numba:
needs: conda-deploy-llvmlite
name: numba ${{ matrix.os }} ${{ matrix.python-version }}
runs-on: ${{ matrix.os }}
strategy:
matrix:
#os: [ubuntu-latest, macOS-latest, windows-latest]
os: [ubuntu-latest, macOS-latest]
python-version: ["3.8", "3.9", "3.10"]
steps:
- uses: actions/checkout@v4
- name: Set env for HEAD SHA
run:
echo "GITHUB_HEAD_SHA=${{ inputs.commit }}" >> $GITHUB_ENV
- name: Create and activate conda env
uses: conda-incubator/setup-miniconda@v3
with:
python-version: ${{ matrix.python-version }}
environment-file: ${{ inputs.env }}
auto-update-conda: false
auto-activate-base: false
show-channel-urls: true
- name: Build and upload numba
uses: uibcdf/[email protected]
with:
meta_yaml_dir: buildscripts/conda-recipes/numba
python-version: ${{ matrix.python-version }}
user: python-for-hpc
label: ${{ inputs.label }}
overwrite: true
token: ${{ secrets.ANACONDA_TOKEN }}

# Job to deploy the pyomp metapackage, runs once as it is independent of the
# python version.
conda-deploy-pyomp:
needs: conda-deploy-numba
name: pyomp ${{ matrix.os }}
runs-on: ${{ matrix.os }}
strategy:
matrix:
#os: [ubuntu-latest, macOS-latest, windows-latest]
os: [ubuntu-latest, macOS-latest]
steps:
- uses: actions/checkout@v4
- name: Set env for HEAD SHA
run:
echo "GITHUB_HEAD_SHA=${{ inputs.commit }}" >> $GITHUB_ENV
- name: Create and activate conda env
uses: conda-incubator/setup-miniconda@v3
with:
python-version: ${{ matrix.python-version }}
environment-file: ${{ inputs.env }}
auto-update-conda: false
auto-activate-base: false
show-channel-urls: true
- name: Build and upload pyomp
uses: uibcdf/[email protected]
with:
meta_yaml_dir: buildscripts/conda-recipes/pyomp
python-version: "3.10"
user: python-for-hpc
label: ${{ inputs.label }}
overwrite: true
token: ${{ secrets.ANACONDA_TOKEN }}

################################################################
# Deploy on ARM64 using QEMU and a docker container for building.
################################################################
conda-deploy-llvm-openmp-dev-linux-arm64:
name: llvm-openmp-dev linux-arm64
runs-on: ubuntu-latest
steps:
- name: Set up QEMU
uses: docker/setup-qemu-action@v3
with:
platforms: linux/arm64
- name: Deploy llvm-openmp-dev
run: |
docker run --platform linux/arm64 ubuntu:22.04 /bin/bash -xel -c "
${{ env.SETUP_BASE }}
${{ env.SETUP_MINICONDA }}
${{ env.SETUP_REPO }}
conda build --user python-for-hpc --label main \
-c python-for-hpc -c conda-forge \
--token ${{ secrets.ANACONDA_TOKEN }} \
buildscripts/conda-recipes/llvm-openmp-dev;
"
conda-deploy-llvmlite-linux-arm64:
needs: conda-deploy-llvm-openmp-dev-linux-arm64
name: llvmlite linux-arm64
runs-on: ubuntu-latest
strategy:
matrix:
python-version: ["3.8", "3.9", "3.10"]
steps:
- name: Set up QEMU
uses: docker/setup-qemu-action@v3
with:
platforms: linux/arm64
- name: Deploy llvmlite
run: |
docker run --platform linux/arm64 ubuntu:22.04 /bin/bash -el -c "
${{ env.SETUP_BASE }}
${{ env.SETUP_MINICONDA }}
${{ env.SETUP_REPO }}
conda build --user python-for-hpc --label main \
-c python-for-hpc -c conda-forge \
--python ${{ matrix.python-version}} --token ${{ secrets.ANACONDA_TOKEN }} \
buildscripts/conda-recipes/llvmlite
"
conda-deploy-numba-linux-arm64:
#needs: conda-deploy-llvmlite-linux-arm64
name: numba linux-arm64
runs-on: ubuntu-latest
strategy:
matrix:
python-version: ["3.8", "3.9", "3.10"]
steps:
- name: Set up QEMU
uses: docker/setup-qemu-action@v3
with:
platforms: linux/arm64
- name: Deploy numba
run: |
docker run --platform linux/arm64 ubuntu:22.04 /bin/bash -el -c "
${{ env.SETUP_BASE }}
${{ env.SETUP_MINICONDA }}
${{ env.SETUP_REPO }}
conda build --user python-for-hpc --label main \
-c python-for-hpc -c conda-forge \
--python ${{ matrix.python-version}} --token ${{ secrets.ANACONDA_TOKEN }} \
buildscripts/conda-recipes/numba;
"
conda-deploy-pyomp-linux-arm64:
needs: conda-deploy-numba-linux-arm64
name: pyomp linux-arm64
runs-on: ubuntu-latest
steps:
- name: Set up QEMU
uses: docker/setup-qemu-action@v3
with:
platforms: linux/arm64
- name: Deploy pyomp
run: |
docker run --platform linux/arm64 ubuntu:22.04 /bin/bash -el -c "
${{ env.SETUP_BASE }}
${{ env.SETUP_MINICONDA }}
${{ env.SETUP_REPO }}
conda build --user python-for-hpc --label main \
-c python-for-hpc -c conda-forge \
--token ${{ secrets.ANACONDA_TOKEN }} \
buildscripts/conda-recipes/pyomp;
"
Loading

0 comments on commit ebfeb89

Please sign in to comment.