add ci/cd for github actions (#1070)

* add ci/cd for github actions

* fix workflow and add workflow_dispatch

* remove dist

* fix

* add gcc

* sudo apt install build essential

* add curl

* install curl

* build essential

* update

* test code on mac os

* release on tagged version

* better names

* fix numbers for 3.10

* add cancel workflow for tests

* fix test macOs

* Adding ccache to github action

* creating dependency between jobs

* using cibuildwheel

* ccache path bug

* Testing after building wheels

* misunderstood cibuildwheel

* fixing error in manylinux build

* fixing error in manylinux build (2)

* disabling fail fast (temporary)

* Don't repair macOS wheels

* Not building pypy wheels

* disabling 32-bit linux wheels

* macos-latest only (prevent duplicate)

* fixing musllinux

* fixing ccache

* fixing cache persistence

* adding deploy job

* last debug

Co-authored-by: Thomas Ferreira de Lima <[email protected]>

.github/workflows/build.yml

@@ -0,0 +1,96 @@
+---
+name: Build Python Wheels
+# https://docs.github.com/en/free-pro-team@latest/actions/guides/building-and-testing-python
+
+on:
+  pull_request:
+  push:
+
+jobs:
+  build:
+    runs-on: ${{ matrix.os }}
+    strategy:
+      fail-fast: false
+      max-parallel: 12
+      matrix:
+        include:
+          - os: "macos-latest"
+            cibuild: "*macosx*"
+          - os: "ubuntu-latest"
+            cibuild: "*manylinux*"
+          - os: "ubuntu-latest"
+            cibuild: "*musllinux*"
+    steps:
+      - name: Cancel Workflow Action
+        uses: styfle/[email protected]
+      - uses: actions/checkout@v3
+      - name: ccache
+        uses: hendrikmuhs/[email protected]
+        with:
+          key: ${{ github.job }}-${{ matrix.os }}-${{ matrix.cibuild }}  # Make cache specific to OS
+          max-size: "5G"
+      - name: Install dependencies
+        run: |
+          env
+          export PATH="/usr/lib/ccache:/usr/local/opt/ccache/libexec:$PATH"
+          echo "/usr/lib/ccache:/usr/local/opt/ccache/libexec" >> $GITHUB_PATH
+          HOST_CCACHE_DIR="$(ccache -k cache_dir)"
+          mkdir -p $HOST_CCACHE_DIR
+      - name: Build wheels  # check https://cibuildwheel.readthedocs.io/en/stable/setup/#github-actions
+        uses: pypa/[email protected]
+        # to supply options, put them in 'env', like:
+        # env:
+        #   CIBW_SOME_OPTION: value
+        env:
+          CIBW_BUILD: ${{ matrix.cibuild }}
+      - name: Download Cache from Docker (linux only)
+        if: ${{ runner.os == 'Linux' }}
+        # hack until https://github.com/pypa/cibuildwheel/issues/1030 is fixed
+        run: |
+          env
+          ccache -s
+          HOST_CCACHE_DIR="$(ccache -k cache_dir)"
+          rm -rf $HOST_CCACHE_DIR
+          mv ./wheelhouse/.ccache $HOST_CCACHE_DIR
+          ls -la $HOST_CCACHE_DIR
+          ccache -s
+      - uses: actions/upload-artifact@v2
+        with:
+          path: ./wheelhouse/*.whl
+
+# The following was taken from https://cibuildwheel.readthedocs.io/en/stable/deliver-to-pypi/
+  make_sdist:
+    name: Make SDist
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v2
+      with:
+        fetch-depth: 0  # Optional, use if you use setuptools_scm
+        submodules: true  # Optional, use if you have submodules
+
+    - name: Build SDist
+      run: pipx run build --sdist
+
+    - uses: actions/upload-artifact@v2
+      with:
+        path: dist/*.tar.gz
+
+  upload_all:
+    needs: [build, make_sdist]
+    runs-on: ubuntu-latest
+    # Uncomment for real PyPi
+    # if: github.event_name == 'release' && github.event.action == 'published'
+    steps:
+    - uses: actions/download-artifact@v2
+      with:
+        name: artifact
+        path: dist
+
+    - uses: pypa/[email protected]
+      with:
+        user: __token__
+        # Test PyPI
+        password: ${{ secrets.test_pypi_password }}
+        repository_url: https://test.pypi.org/legacy/
+        # Uncomment for Real Pypi
+        # password: ${{ secrets.pypi_password }}

ci-scripts/docker/docker_prepare.sh

@@ -0,0 +1,50 @@
+#!/usr/bin/env bash
+set -xe
+
+# manylinux prep
+if [[ -f "/etc/centos-release" ]]; then
+    # sometimes the epel server is down. retry 5 times
+    for i in $(seq 1 5); do 
+        yum install -y zlib-devel curl-devel expat-devel ccache && s=0 && break || s=$? && sleep 15; 
+    done
+
+    [ $s -eq 0 ] || exit $s
+
+    if [[ -d "/usr/lib64/ccache" ]]; then
+        ln -s /usr/bin/ccache /usr/lib64/ccache/c++
+        ln -s /usr/bin/ccache /usr/lib64/ccache/cc
+        ln -s /usr/bin/ccache /usr/lib64/ccache/gcc
+        ln -s /usr/bin/ccache /usr/lib64/ccache/g++
+        export PATH="/usr/lib64/ccache:$PATH"
+    elif [[ -d "/usr/lib/ccache" ]]; then
+        ln -s /usr/bin/ccache /usr/lib/ccache/c++
+        ln -s /usr/bin/ccache /usr/lib/ccache/cc
+        ln -s /usr/bin/ccache /usr/lib/ccache/gcc
+        ln -s /usr/bin/ccache /usr/lib/ccache/g++
+        export PATH="/usr/lib/ccache:$PATH"
+    fi
+
+elif [[ -f "/etc/alpine-release" ]]; then
+    # musllinux prep
+    # ccache already present
+    apk add curl-dev expat-dev zlib-dev ccache
+    export PATH="/usr/lib/ccache/bin:$PATH"
+fi
+
+# hack until https://github.com/pypa/cibuildwheel/issues/1030 is fixed
+# Place ccache folder in /outputs
+HOST_CCACHE_DIR="/host${HOST_CCACHE_DIR:-/home/runner/work/klayout/klayout/.ccache}"
+if [ -d $HOST_CCACHE_DIR ]; then
+    mkdir -p /output
+    cp -R $HOST_CCACHE_DIR /output/.ccache
+fi
+
+ls -la /output/
+
+ccache -o cache_dir="/output/.ccache"
+# export CCACHE_DIR="/host/home/runner/work/klayout/klayout/.ccache"
+ccache -M 5 G  # set cache size to 5 G
+
+# Show ccache stats
+echo "Cache stats:"
+ccache -s

pyproject.toml

@@ -0,0 +1,27 @@
+[tool.cibuildwheel]
+build-verbosity = "3"
+test-command = [
+  "python {package}/testdata/pymod/import_db.py",
+  "python {package}/testdata/pymod/import_rdb.py",
+  "python {package}/testdata/pymod/import_tl.py",
+  "python {package}/testdata/pymod/import_lib.py",
+  "python {package}/testdata/pymod/pya_tests.py"
+]
+# Disable building PyPy wheels on all platforms
+skip = "pp*"
+
+[tool.cibuildwheel.linux]
+# beware: the before-all script does not persist environment variables!
+# No point in defining $PATH or $CCACHE_DIR
+before-all = "source {project}/ci-scripts/docker/docker_prepare.sh"
+archs = ["auto64"]
+# Append a directory to the PATH variable (this is expanded in the build environment)
+environment = { PATH="/usr/lib/ccache:/usr/lib64/ccache:/usr/lib/ccache/bin:$PATH" }
+before-build = "ccache -s"
+# Export a variable to docker
+# CCACHE_DIR="/home/runner/work/klayout/klayout/.ccache"
+environment-pass = ["HOST_CCACHE_DIR"]
+
+[tool.cibuildwheel.macos]
+# Don't repair macOS wheels
+repair-wheel-command = ""