From 1f4422f857721d0c31f1645f379093cf4892fec9 Mon Sep 17 00:00:00 2001 From: Joaquin Matres <4514346+joamatab@users.noreply.github.com> Date: Thu, 28 Apr 2022 09:40:09 -0700 Subject: [PATCH 01/34] add ci/cd for github actions --- .github/workflows/pre-commit.yml | 14 ++++++++++ .github/workflows/pythonpublish.yml | 38 +++++++++++++++++++++++++++ .github/workflows/test_code.yml | 40 +++++++++++++++++++++++++++++ .github/workflows/test_codecov.yml | 36 ++++++++++++++++++++++++++ 4 files changed, 128 insertions(+) create mode 100644 .github/workflows/pre-commit.yml create mode 100644 .github/workflows/pythonpublish.yml create mode 100644 .github/workflows/test_code.yml create mode 100644 .github/workflows/test_codecov.yml diff --git a/.github/workflows/pre-commit.yml b/.github/workflows/pre-commit.yml new file mode 100644 index 0000000000..6022454002 --- /dev/null +++ b/.github/workflows/pre-commit.yml @@ -0,0 +1,14 @@ +--- +name: pre-commit + +on: + pull_request: + push: + +jobs: + pre-commit: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - uses: actions/setup-python@v2 + - uses: pre-commit/action@v2.0.0 diff --git a/.github/workflows/pythonpublish.yml b/.github/workflows/pythonpublish.yml new file mode 100644 index 0000000000..1a31ed0e5a --- /dev/null +++ b/.github/workflows/pythonpublish.yml @@ -0,0 +1,38 @@ +--- +name: Upload Python Package + +on: + release: + types: [created, published] + push: + branches: [master] + tags: [v*] + +jobs: + deploy: + runs-on: ${{ matrix.os }} + strategy: + max-parallel: 12 + matrix: + python-version: [3.7, 3.9, 3.10] + os: [ubuntu-latest, macos-latest, windows-latest] + steps: + - uses: actions/checkout@v2 + - name: Set up Python + uses: actions/setup-python@v2 + with: + python-version: 3.x + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install setuptools wheel twine + set "KLAYOUT_BITS=%cd%\klayout-microbits\klayout-microbits-1.0\msvc2017\%PYTHON_ARCHITECTURE%" + echo KLAYOUT_BITS=%KLAYOUT_BITS% + pip install klayout --no-index -f dist + - name: Build and publish + env: + TWINE_USERNAME: ${{ secrets.PYPI_USERNAME }} + TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }} + run: | + python setup.py sdist bdist_wheel + twine upload dist/* diff --git a/.github/workflows/test_code.yml b/.github/workflows/test_code.yml new file mode 100644 index 0000000000..0f400c35ed --- /dev/null +++ b/.github/workflows/test_code.yml @@ -0,0 +1,40 @@ +--- +name: Lint and Test +# https://docs.github.com/en/free-pro-team@latest/actions/guides/building-and-testing-python + +on: + pull_request: + push: + schedule: + - cron: 0 2 * * * # run at 2 AM UTC + +jobs: + build: + runs-on: ${{ matrix.os }} + strategy: + max-parallel: 12 + matrix: + python-version: [3.7, 3.8, 3.9] + os: [ubuntu-latest] + + steps: + - name: Cancel Workflow Action + uses: styfle/cancel-workflow-action@0.9.1 + - uses: actions/checkout@v2 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.python-version }} + - name: Install dependencies + run: | + pip install setuptools wheel twine + set "KLAYOUT_BITS=%cd%\klayout-microbits\klayout-microbits-1.0\msvc2017\%PYTHON_ARCHITECTURE%" + echo KLAYOUT_BITS=%KLAYOUT_BITS% + pip install klayout --no-index -f dist + - name: Test with pytest + run: | + python testdata/pymod/import_db.py + python testdata/pymod/import_rdb.py + python testdata/pymod/import_tl.py + python testdata/pymod/import_lib.py + python testdata/pymod/pya_tests.py diff --git a/.github/workflows/test_codecov.yml b/.github/workflows/test_codecov.yml new file mode 100644 index 0000000000..d27a33bcf9 --- /dev/null +++ b/.github/workflows/test_codecov.yml @@ -0,0 +1,36 @@ +--- +name: Code test coverage +on: + pull_request: + push: + +jobs: + build-linux: + runs-on: ubuntu-latest + + steps: + - name: Cancel Workflow Action + uses: styfle/cancel-workflow-action@0.9.1 + - uses: actions/checkout@v2 + - name: Set up Python 3.9 + uses: actions/setup-python@v2 + with: + python-version: 3.9 + - name: Install dependencies + run: | + make install + - name: Test with pytest + run: | + python -m pip install --upgrade pip + pip install pytest + pip install pytest-cov + pip install setuptools wheel twine + set "KLAYOUT_BITS=%cd%\klayout-microbits\klayout-microbits-1.0\msvc2017\%PYTHON_ARCHITECTURE%" + echo KLAYOUT_BITS=%KLAYOUT_BITS% + pip install klayout --no-index -f dist + pytest --cov=. --cov-report=xml --cov-config=.coveragerc + - name: Upload coverage to Codecov + uses: codecov/codecov-action@v2 + with: + token: ${{ secrets.CODECOV_TOKEN }} + fail_ci_if_error: true From f73e8fdbf87a25ca7a750284678e18af08b1f1b4 Mon Sep 17 00:00:00 2001 From: Joaquin Matres <4514346+joamatab@users.noreply.github.com> Date: Thu, 28 Apr 2022 16:19:26 -0700 Subject: [PATCH 02/34] fix workflow and add workflow_dispatch --- .github/workflows/pre-commit.yml | 14 -------------- .github/workflows/pythonpublish.yml | 3 ++- .github/workflows/test_codecov.yml | 5 ++--- 3 files changed, 4 insertions(+), 18 deletions(-) delete mode 100644 .github/workflows/pre-commit.yml diff --git a/.github/workflows/pre-commit.yml b/.github/workflows/pre-commit.yml deleted file mode 100644 index 6022454002..0000000000 --- a/.github/workflows/pre-commit.yml +++ /dev/null @@ -1,14 +0,0 @@ ---- -name: pre-commit - -on: - pull_request: - push: - -jobs: - pre-commit: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v2 - - uses: actions/setup-python@v2 - - uses: pre-commit/action@v2.0.0 diff --git a/.github/workflows/pythonpublish.yml b/.github/workflows/pythonpublish.yml index 1a31ed0e5a..6d8ad18e09 100644 --- a/.github/workflows/pythonpublish.yml +++ b/.github/workflows/pythonpublish.yml @@ -7,6 +7,7 @@ on: push: branches: [master] tags: [v*] + workflow_dispatch: jobs: deploy: @@ -14,7 +15,7 @@ jobs: strategy: max-parallel: 12 matrix: - python-version: [3.7, 3.9, 3.10] + python-version: [3.8, 3.9, 3.10] os: [ubuntu-latest, macos-latest, windows-latest] steps: - uses: actions/checkout@v2 diff --git a/.github/workflows/test_codecov.yml b/.github/workflows/test_codecov.yml index d27a33bcf9..6ba0d427ff 100644 --- a/.github/workflows/test_codecov.yml +++ b/.github/workflows/test_codecov.yml @@ -17,9 +17,6 @@ jobs: with: python-version: 3.9 - name: Install dependencies - run: | - make install - - name: Test with pytest run: | python -m pip install --upgrade pip pip install pytest @@ -28,6 +25,8 @@ jobs: set "KLAYOUT_BITS=%cd%\klayout-microbits\klayout-microbits-1.0\msvc2017\%PYTHON_ARCHITECTURE%" echo KLAYOUT_BITS=%KLAYOUT_BITS% pip install klayout --no-index -f dist + - name: Test with pytest + run: | pytest --cov=. --cov-report=xml --cov-config=.coveragerc - name: Upload coverage to Codecov uses: codecov/codecov-action@v2 From 6b2b4ca1966d2bafb10f3102accaab0dc3269880 Mon Sep 17 00:00:00 2001 From: Joaquin Matres <4514346+joamatab@users.noreply.github.com> Date: Thu, 28 Apr 2022 16:22:34 -0700 Subject: [PATCH 03/34] remove dist --- .github/workflows/test_code.yml | 2 +- .github/workflows/test_codecov.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test_code.yml b/.github/workflows/test_code.yml index 0f400c35ed..d655e860bd 100644 --- a/.github/workflows/test_code.yml +++ b/.github/workflows/test_code.yml @@ -30,7 +30,7 @@ jobs: pip install setuptools wheel twine set "KLAYOUT_BITS=%cd%\klayout-microbits\klayout-microbits-1.0\msvc2017\%PYTHON_ARCHITECTURE%" echo KLAYOUT_BITS=%KLAYOUT_BITS% - pip install klayout --no-index -f dist + pip install klayout --no-index - name: Test with pytest run: | python testdata/pymod/import_db.py diff --git a/.github/workflows/test_codecov.yml b/.github/workflows/test_codecov.yml index 6ba0d427ff..49c2c13967 100644 --- a/.github/workflows/test_codecov.yml +++ b/.github/workflows/test_codecov.yml @@ -24,7 +24,7 @@ jobs: pip install setuptools wheel twine set "KLAYOUT_BITS=%cd%\klayout-microbits\klayout-microbits-1.0\msvc2017\%PYTHON_ARCHITECTURE%" echo KLAYOUT_BITS=%KLAYOUT_BITS% - pip install klayout --no-index -f dist + pip install klayout --no-index - name: Test with pytest run: | pytest --cov=. --cov-report=xml --cov-config=.coveragerc From 575b769fe24fdb98758a2e6eb7e5ccc44e8ecad9 Mon Sep 17 00:00:00 2001 From: Joaquin Matres <4514346+joamatab@users.noreply.github.com> Date: Thu, 28 Apr 2022 16:24:12 -0700 Subject: [PATCH 04/34] fix --- .github/workflows/test_code.yml | 2 +- .github/workflows/test_codecov.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test_code.yml b/.github/workflows/test_code.yml index d655e860bd..4061a3e247 100644 --- a/.github/workflows/test_code.yml +++ b/.github/workflows/test_code.yml @@ -30,7 +30,7 @@ jobs: pip install setuptools wheel twine set "KLAYOUT_BITS=%cd%\klayout-microbits\klayout-microbits-1.0\msvc2017\%PYTHON_ARCHITECTURE%" echo KLAYOUT_BITS=%KLAYOUT_BITS% - pip install klayout --no-index + pip install -e . --no-index - name: Test with pytest run: | python testdata/pymod/import_db.py diff --git a/.github/workflows/test_codecov.yml b/.github/workflows/test_codecov.yml index 49c2c13967..80b7742612 100644 --- a/.github/workflows/test_codecov.yml +++ b/.github/workflows/test_codecov.yml @@ -24,7 +24,7 @@ jobs: pip install setuptools wheel twine set "KLAYOUT_BITS=%cd%\klayout-microbits\klayout-microbits-1.0\msvc2017\%PYTHON_ARCHITECTURE%" echo KLAYOUT_BITS=%KLAYOUT_BITS% - pip install klayout --no-index + pip install -e . --no-index - name: Test with pytest run: | pytest --cov=. --cov-report=xml --cov-config=.coveragerc From d215c8c365d52ec36bcd76da16ae23acb5cf5f57 Mon Sep 17 00:00:00 2001 From: Joaquin Matres <4514346+joamatab@users.noreply.github.com> Date: Thu, 28 Apr 2022 16:28:26 -0700 Subject: [PATCH 05/34] add gcc --- .github/workflows/pythonpublish.yml | 5 +++++ .github/workflows/test_code.yml | 5 +++++ .github/workflows/test_codecov.yml | 35 ----------------------------- 3 files changed, 10 insertions(+), 35 deletions(-) delete mode 100644 .github/workflows/test_codecov.yml diff --git a/.github/workflows/pythonpublish.yml b/.github/workflows/pythonpublish.yml index 6d8ad18e09..a97296ab18 100644 --- a/.github/workflows/pythonpublish.yml +++ b/.github/workflows/pythonpublish.yml @@ -23,6 +23,11 @@ jobs: uses: actions/setup-python@v2 with: python-version: 3.x + - name: Set up GCC + uses: egor-tensin/setup-gcc@v1 + with: + version: latest + platform: x64 - name: Install dependencies run: | python -m pip install --upgrade pip diff --git a/.github/workflows/test_code.yml b/.github/workflows/test_code.yml index 4061a3e247..0b92b776d8 100644 --- a/.github/workflows/test_code.yml +++ b/.github/workflows/test_code.yml @@ -18,6 +18,11 @@ jobs: os: [ubuntu-latest] steps: + - name: Set up GCC + uses: egor-tensin/setup-gcc@v1 + with: + version: latest + platform: x64 - name: Cancel Workflow Action uses: styfle/cancel-workflow-action@0.9.1 - uses: actions/checkout@v2 diff --git a/.github/workflows/test_codecov.yml b/.github/workflows/test_codecov.yml deleted file mode 100644 index 80b7742612..0000000000 --- a/.github/workflows/test_codecov.yml +++ /dev/null @@ -1,35 +0,0 @@ ---- -name: Code test coverage -on: - pull_request: - push: - -jobs: - build-linux: - runs-on: ubuntu-latest - - steps: - - name: Cancel Workflow Action - uses: styfle/cancel-workflow-action@0.9.1 - - uses: actions/checkout@v2 - - name: Set up Python 3.9 - uses: actions/setup-python@v2 - with: - python-version: 3.9 - - name: Install dependencies - run: | - python -m pip install --upgrade pip - pip install pytest - pip install pytest-cov - pip install setuptools wheel twine - set "KLAYOUT_BITS=%cd%\klayout-microbits\klayout-microbits-1.0\msvc2017\%PYTHON_ARCHITECTURE%" - echo KLAYOUT_BITS=%KLAYOUT_BITS% - pip install -e . --no-index - - name: Test with pytest - run: | - pytest --cov=. --cov-report=xml --cov-config=.coveragerc - - name: Upload coverage to Codecov - uses: codecov/codecov-action@v2 - with: - token: ${{ secrets.CODECOV_TOKEN }} - fail_ci_if_error: true From 455050403fe14dc695321c9a9fc85e457eef3876 Mon Sep 17 00:00:00 2001 From: Joaquin Matres <4514346+joamatab@users.noreply.github.com> Date: Thu, 28 Apr 2022 16:32:46 -0700 Subject: [PATCH 06/34] sudo apt install build essential --- .github/workflows/test_code.yml | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/.github/workflows/test_code.yml b/.github/workflows/test_code.yml index 0b92b776d8..dd19995399 100644 --- a/.github/workflows/test_code.yml +++ b/.github/workflows/test_code.yml @@ -18,11 +18,6 @@ jobs: os: [ubuntu-latest] steps: - - name: Set up GCC - uses: egor-tensin/setup-gcc@v1 - with: - version: latest - platform: x64 - name: Cancel Workflow Action uses: styfle/cancel-workflow-action@0.9.1 - uses: actions/checkout@v2 @@ -33,9 +28,8 @@ jobs: - name: Install dependencies run: | pip install setuptools wheel twine - set "KLAYOUT_BITS=%cd%\klayout-microbits\klayout-microbits-1.0\msvc2017\%PYTHON_ARCHITECTURE%" - echo KLAYOUT_BITS=%KLAYOUT_BITS% pip install -e . --no-index + sudo apt install build-essential - name: Test with pytest run: | python testdata/pymod/import_db.py From dd0573bd68bb672bd55f6a09b1911ae0feb94438 Mon Sep 17 00:00:00 2001 From: Joaquin Matres <4514346+joamatab@users.noreply.github.com> Date: Sun, 1 May 2022 13:00:16 -0700 Subject: [PATCH 07/34] add curl --- .github/workflows/test_code.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test_code.yml b/.github/workflows/test_code.yml index dd19995399..b7a2dffce5 100644 --- a/.github/workflows/test_code.yml +++ b/.github/workflows/test_code.yml @@ -29,7 +29,7 @@ jobs: run: | pip install setuptools wheel twine pip install -e . --no-index - sudo apt install build-essential + sudo apt install build-essential curl - name: Test with pytest run: | python testdata/pymod/import_db.py From 9d70be0998a1393b658670952c7e5bc26ce29544 Mon Sep 17 00:00:00 2001 From: Joaquin Matres <4514346+joamatab@users.noreply.github.com> Date: Sun, 1 May 2022 13:59:24 -0700 Subject: [PATCH 08/34] install curl --- .github/workflows/pythonpublish.yml | 1 + .github/workflows/test_code.yml | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/pythonpublish.yml b/.github/workflows/pythonpublish.yml index a97296ab18..3ce2651a4b 100644 --- a/.github/workflows/pythonpublish.yml +++ b/.github/workflows/pythonpublish.yml @@ -30,6 +30,7 @@ jobs: platform: x64 - name: Install dependencies run: | + sudo apt install build-essential curl python -m pip install --upgrade pip pip install setuptools wheel twine set "KLAYOUT_BITS=%cd%\klayout-microbits\klayout-microbits-1.0\msvc2017\%PYTHON_ARCHITECTURE%" diff --git a/.github/workflows/test_code.yml b/.github/workflows/test_code.yml index b7a2dffce5..ca709e5582 100644 --- a/.github/workflows/test_code.yml +++ b/.github/workflows/test_code.yml @@ -27,9 +27,9 @@ jobs: python-version: ${{ matrix.python-version }} - name: Install dependencies run: | + sudo apt install build-essential curl pip install setuptools wheel twine pip install -e . --no-index - sudo apt install build-essential curl - name: Test with pytest run: | python testdata/pymod/import_db.py From e6d9a9540e4183cf8a7293dd55f586feea0a30ed Mon Sep 17 00:00:00 2001 From: Joaquin Matres <4514346+joamatab@users.noreply.github.com> Date: Sun, 1 May 2022 15:11:36 -0700 Subject: [PATCH 09/34] build essential --- .github/workflows/pythonpublish.yml | 2 +- .github/workflows/test_code.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/pythonpublish.yml b/.github/workflows/pythonpublish.yml index 3ce2651a4b..bbfdd31ad9 100644 --- a/.github/workflows/pythonpublish.yml +++ b/.github/workflows/pythonpublish.yml @@ -30,7 +30,7 @@ jobs: platform: x64 - name: Install dependencies run: | - sudo apt install build-essential curl + sudo apt install -y build-essential curl zlib1g-dev libcurl4-openssl-dev libexpat1-dev python -m pip install --upgrade pip pip install setuptools wheel twine set "KLAYOUT_BITS=%cd%\klayout-microbits\klayout-microbits-1.0\msvc2017\%PYTHON_ARCHITECTURE%" diff --git a/.github/workflows/test_code.yml b/.github/workflows/test_code.yml index ca709e5582..1735c2ecba 100644 --- a/.github/workflows/test_code.yml +++ b/.github/workflows/test_code.yml @@ -27,7 +27,7 @@ jobs: python-version: ${{ matrix.python-version }} - name: Install dependencies run: | - sudo apt install build-essential curl + sudo apt install -y build-essential curl zlib1g-dev libcurl4-openssl-dev libexpat1-dev pip install setuptools wheel twine pip install -e . --no-index - name: Test with pytest From 84f7789a91a84b8776a70491fe11526e68218940 Mon Sep 17 00:00:00 2001 From: Joaquin Matres <4514346+joamatab@users.noreply.github.com> Date: Sun, 1 May 2022 15:14:02 -0700 Subject: [PATCH 10/34] update --- .github/workflows/pythonpublish.yml | 1 + .github/workflows/test_code.yml | 1 + 2 files changed, 2 insertions(+) diff --git a/.github/workflows/pythonpublish.yml b/.github/workflows/pythonpublish.yml index bbfdd31ad9..68afcb2d5a 100644 --- a/.github/workflows/pythonpublish.yml +++ b/.github/workflows/pythonpublish.yml @@ -30,6 +30,7 @@ jobs: platform: x64 - name: Install dependencies run: | + sudo apt update sudo apt install -y build-essential curl zlib1g-dev libcurl4-openssl-dev libexpat1-dev python -m pip install --upgrade pip pip install setuptools wheel twine diff --git a/.github/workflows/test_code.yml b/.github/workflows/test_code.yml index 1735c2ecba..b54136d73f 100644 --- a/.github/workflows/test_code.yml +++ b/.github/workflows/test_code.yml @@ -27,6 +27,7 @@ jobs: python-version: ${{ matrix.python-version }} - name: Install dependencies run: | + sudo apt update sudo apt install -y build-essential curl zlib1g-dev libcurl4-openssl-dev libexpat1-dev pip install setuptools wheel twine pip install -e . --no-index From 69db1938d1f1c25e3debb42aa444e119b1bafab2 Mon Sep 17 00:00:00 2001 From: Joaquin Matres <4514346+joamatab@users.noreply.github.com> Date: Sun, 1 May 2022 16:11:40 -0700 Subject: [PATCH 11/34] test code on mac os --- .github/workflows/pythonpublish-mac.yml | 29 +++++++++++++++++++++++++ .github/workflows/test_code-mac.yml | 29 +++++++++++++++++++++++++ .github/workflows/test_code.yml | 4 ++-- 3 files changed, 60 insertions(+), 2 deletions(-) create mode 100644 .github/workflows/pythonpublish-mac.yml create mode 100644 .github/workflows/test_code-mac.yml diff --git a/.github/workflows/pythonpublish-mac.yml b/.github/workflows/pythonpublish-mac.yml new file mode 100644 index 0000000000..ffdb46db19 --- /dev/null +++ b/.github/workflows/pythonpublish-mac.yml @@ -0,0 +1,29 @@ +--- +name: Mac CI + +on: + pull_request: + push: + +jobs: + mac-build: + name: Wrapper macOS Build + runs-on: ${{ matrix.os }} + strategy: + matrix: + os: ["macos-11", "macos-10.15"] + pyversion: ["python@3.8", "python@3.9", "python@3.10"] + steps: + - name: Checkout + uses: actions/checkout@master + - name: Build the macOS wheels + run: | + pip3 install twine + pip3 install -e . --no-index + - name: Build and publish + env: + TWINE_USERNAME: ${{ secrets.PYPI_USERNAME }} + TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }} + run: | + python3 setup.py sdist bdist_wheel + twine upload dist/* diff --git a/.github/workflows/test_code-mac.yml b/.github/workflows/test_code-mac.yml new file mode 100644 index 0000000000..55ac1305a9 --- /dev/null +++ b/.github/workflows/test_code-mac.yml @@ -0,0 +1,29 @@ +--- +name: Mac test code + +on: + pull_request: + push: + +jobs: + mac-build: + name: Wrapper macOS Build + runs-on: ${{ matrix.os }} + strategy: + matrix: + os: ["macos-11", "macos-10.15"] + pyversion: ["python@3.8", "python@3.9", "python@3.10"] + steps: + - name: Checkout + uses: actions/checkout@master + - name: Build the macOS wheels + run: | + pip3 install twine + pip3 install -e . --no-index + - name: Test code + run: | + python testdata/pymod/import_db.py + python testdata/pymod/import_rdb.py + python testdata/pymod/import_tl.py + python testdata/pymod/import_lib.py + python testdata/pymod/pya_tests.py diff --git a/.github/workflows/test_code.yml b/.github/workflows/test_code.yml index b54136d73f..7c39e68022 100644 --- a/.github/workflows/test_code.yml +++ b/.github/workflows/test_code.yml @@ -14,7 +14,7 @@ jobs: strategy: max-parallel: 12 matrix: - python-version: [3.7, 3.8, 3.9] + python-version: [3.7, 3.8, 3.9, 3.10] os: [ubuntu-latest] steps: @@ -31,7 +31,7 @@ jobs: sudo apt install -y build-essential curl zlib1g-dev libcurl4-openssl-dev libexpat1-dev pip install setuptools wheel twine pip install -e . --no-index - - name: Test with pytest + - name: Test code run: | python testdata/pymod/import_db.py python testdata/pymod/import_rdb.py From a39955db79dd0f5c85cb76398a37b82fe4144735 Mon Sep 17 00:00:00 2001 From: Joaquin Matres <4514346+joamatab@users.noreply.github.com> Date: Sun, 1 May 2022 16:12:10 -0700 Subject: [PATCH 12/34] release on tagged version --- .github/workflows/pythonpublish-mac.yml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.github/workflows/pythonpublish-mac.yml b/.github/workflows/pythonpublish-mac.yml index ffdb46db19..701cc979a5 100644 --- a/.github/workflows/pythonpublish-mac.yml +++ b/.github/workflows/pythonpublish-mac.yml @@ -2,8 +2,12 @@ name: Mac CI on: - pull_request: + release: + types: [created, published] push: + branches: [master] + tags: [v*] + workflow_dispatch: jobs: mac-build: From 4171b6e48b29ac5d7554bd0cb80012e9c94deba1 Mon Sep 17 00:00:00 2001 From: Joaquin Matres <4514346+joamatab@users.noreply.github.com> Date: Sun, 1 May 2022 16:15:51 -0700 Subject: [PATCH 13/34] better names --- .github/workflows/pythonpublish-mac.yml | 2 +- .github/workflows/pythonpublish.yml | 2 +- .github/workflows/test_code.yml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/pythonpublish-mac.yml b/.github/workflows/pythonpublish-mac.yml index 701cc979a5..2f62b21bd8 100644 --- a/.github/workflows/pythonpublish-mac.yml +++ b/.github/workflows/pythonpublish-mac.yml @@ -1,5 +1,5 @@ --- -name: Mac CI +name: Mac publish package on: release: diff --git a/.github/workflows/pythonpublish.yml b/.github/workflows/pythonpublish.yml index 68afcb2d5a..fc68129d26 100644 --- a/.github/workflows/pythonpublish.yml +++ b/.github/workflows/pythonpublish.yml @@ -1,5 +1,5 @@ --- -name: Upload Python Package +name: Upload Python Package for linux on: release: diff --git a/.github/workflows/test_code.yml b/.github/workflows/test_code.yml index 7c39e68022..452d2d5456 100644 --- a/.github/workflows/test_code.yml +++ b/.github/workflows/test_code.yml @@ -1,5 +1,5 @@ --- -name: Lint and Test +name: linux Test code # https://docs.github.com/en/free-pro-team@latest/actions/guides/building-and-testing-python on: From eb5b252f4d1a18ae6a30d7d3a1f8b7340ec2ccf9 Mon Sep 17 00:00:00 2001 From: Joaquin Matres <4514346+joamatab@users.noreply.github.com> Date: Sun, 1 May 2022 16:17:46 -0700 Subject: [PATCH 14/34] fix numbers for 3.10 --- .github/workflows/pythonpublish.yml | 2 +- .github/workflows/test_code.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/pythonpublish.yml b/.github/workflows/pythonpublish.yml index fc68129d26..e153beb978 100644 --- a/.github/workflows/pythonpublish.yml +++ b/.github/workflows/pythonpublish.yml @@ -15,7 +15,7 @@ jobs: strategy: max-parallel: 12 matrix: - python-version: [3.8, 3.9, 3.10] + python-version: [3.8, 3.9, '3.10'] os: [ubuntu-latest, macos-latest, windows-latest] steps: - uses: actions/checkout@v2 diff --git a/.github/workflows/test_code.yml b/.github/workflows/test_code.yml index 452d2d5456..6a9c6f5043 100644 --- a/.github/workflows/test_code.yml +++ b/.github/workflows/test_code.yml @@ -14,7 +14,7 @@ jobs: strategy: max-parallel: 12 matrix: - python-version: [3.7, 3.8, 3.9, 3.10] + python-version: [3.7, 3.8, 3.9, '3.10'] os: [ubuntu-latest] steps: From 2412920b0362d01c06859e5ac1935249bb862637 Mon Sep 17 00:00:00 2001 From: Joaquin Matres <4514346+joamatab@users.noreply.github.com> Date: Sun, 1 May 2022 16:18:39 -0700 Subject: [PATCH 15/34] add cancel workflow for tests --- .github/workflows/test_code-mac.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/test_code-mac.yml b/.github/workflows/test_code-mac.yml index 55ac1305a9..6546e4ebd7 100644 --- a/.github/workflows/test_code-mac.yml +++ b/.github/workflows/test_code-mac.yml @@ -14,6 +14,8 @@ jobs: os: ["macos-11", "macos-10.15"] pyversion: ["python@3.8", "python@3.9", "python@3.10"] steps: + - name: Cancel Workflow Action + uses: styfle/cancel-workflow-action@0.9.1 - name: Checkout uses: actions/checkout@master - name: Build the macOS wheels From 36c36c02595f714d8f5b8e9da46b3c3720098d09 Mon Sep 17 00:00:00 2001 From: Joaquin Matres <4514346+joamatab@users.noreply.github.com> Date: Mon, 2 May 2022 04:24:25 -0700 Subject: [PATCH 16/34] fix test macOs --- .github/workflows/test_code-mac.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/test_code-mac.yml b/.github/workflows/test_code-mac.yml index 6546e4ebd7..d46d191a08 100644 --- a/.github/workflows/test_code-mac.yml +++ b/.github/workflows/test_code-mac.yml @@ -24,8 +24,8 @@ jobs: pip3 install -e . --no-index - name: Test code run: | - python testdata/pymod/import_db.py - python testdata/pymod/import_rdb.py - python testdata/pymod/import_tl.py - python testdata/pymod/import_lib.py - python testdata/pymod/pya_tests.py + python3 testdata/pymod/import_db.py + python3 testdata/pymod/import_rdb.py + python3 testdata/pymod/import_tl.py + python3 testdata/pymod/import_lib.py + python3 testdata/pymod/pya_tests.py From 1708d88a127ef321a8435ff728f1679525ef8162 Mon Sep 17 00:00:00 2001 From: Thomas Ferreira de Lima Date: Mon, 2 May 2022 15:33:44 -0400 Subject: [PATCH 17/34] Adding ccache to github action --- .github/workflows/pythonpublish-mac.yml | 9 ++++++++- .github/workflows/pythonpublish.yml | 11 +++++++++-- .github/workflows/test_code-mac.yml | 9 ++++++++- .github/workflows/test_code.yml | 11 ++++++++++- 4 files changed, 35 insertions(+), 5 deletions(-) diff --git a/.github/workflows/pythonpublish-mac.yml b/.github/workflows/pythonpublish-mac.yml index 2f62b21bd8..0a44c20a29 100644 --- a/.github/workflows/pythonpublish-mac.yml +++ b/.github/workflows/pythonpublish-mac.yml @@ -1,5 +1,5 @@ --- -name: Mac publish package +name: Upload Mac Python Wheel to PyPI on: release: @@ -20,9 +20,15 @@ jobs: steps: - name: Checkout uses: actions/checkout@master + - name: ccache + uses: hendrikmuhs/ccache-action@v1.2 + with: + key: ${{ github.job }}-${{ matrix.os }} # Make cache specific to OS - name: Build the macOS wheels run: | pip3 install twine + export PATH="/usr/lib/ccache:/usr/local/opt/ccache/libexec:$PATH" + ccache -s pip3 install -e . --no-index - name: Build and publish env: @@ -30,4 +36,5 @@ jobs: TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }} run: | python3 setup.py sdist bdist_wheel + ccache -s twine upload dist/* diff --git a/.github/workflows/pythonpublish.yml b/.github/workflows/pythonpublish.yml index e153beb978..63d78693af 100644 --- a/.github/workflows/pythonpublish.yml +++ b/.github/workflows/pythonpublish.yml @@ -1,5 +1,5 @@ --- -name: Upload Python Package for linux +name: Upload Linux Python Wheel to PyPI on: release: @@ -23,6 +23,10 @@ jobs: uses: actions/setup-python@v2 with: python-version: 3.x + - name: ccache + uses: hendrikmuhs/ccache-action@v1.2 + with: + key: ${{ github.job }}-${{ matrix.os }} # Make cache specific to OS - name: Set up GCC uses: egor-tensin/setup-gcc@v1 with: @@ -31,16 +35,19 @@ jobs: - name: Install dependencies run: | sudo apt update + export PATH="/usr/lib/ccache:/usr/local/opt/ccache/libexec:$PATH" sudo apt install -y build-essential curl zlib1g-dev libcurl4-openssl-dev libexpat1-dev python -m pip install --upgrade pip pip install setuptools wheel twine set "KLAYOUT_BITS=%cd%\klayout-microbits\klayout-microbits-1.0\msvc2017\%PYTHON_ARCHITECTURE%" echo KLAYOUT_BITS=%KLAYOUT_BITS% - pip install klayout --no-index -f dist + # pip install klayout --no-index -f dist - name: Build and publish env: TWINE_USERNAME: ${{ secrets.PYPI_USERNAME }} TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }} run: | + ccache -s python setup.py sdist bdist_wheel + ccache -s twine upload dist/* diff --git a/.github/workflows/test_code-mac.yml b/.github/workflows/test_code-mac.yml index d46d191a08..dbcbcb9481 100644 --- a/.github/workflows/test_code-mac.yml +++ b/.github/workflows/test_code-mac.yml @@ -1,5 +1,5 @@ --- -name: Mac test code +name: Mac test python package on: pull_request: @@ -18,10 +18,17 @@ jobs: uses: styfle/cancel-workflow-action@0.9.1 - name: Checkout uses: actions/checkout@master + - name: ccache + uses: hendrikmuhs/ccache-action@v1.2 + with: + key: ${{ github.job }}-${{ matrix.os }} # Make cache specific to OS - name: Build the macOS wheels run: | pip3 install twine + export PATH="/usr/lib/ccache:/usr/local/opt/ccache/libexec:$PATH" + ccache -s pip3 install -e . --no-index + ccache -s - name: Test code run: | python3 testdata/pymod/import_db.py diff --git a/.github/workflows/test_code.yml b/.github/workflows/test_code.yml index 6a9c6f5043..d84f9bcd5d 100644 --- a/.github/workflows/test_code.yml +++ b/.github/workflows/test_code.yml @@ -1,5 +1,5 @@ --- -name: linux Test code +name: linux test python package # https://docs.github.com/en/free-pro-team@latest/actions/guides/building-and-testing-python on: @@ -21,6 +21,10 @@ jobs: - name: Cancel Workflow Action uses: styfle/cancel-workflow-action@0.9.1 - uses: actions/checkout@v2 + - name: ccache + uses: hendrikmuhs/ccache-action@v1.2 + with: + key: ${{ github.job }}-${{ matrix.os }} # Make cache specific to OS - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@v2 with: @@ -28,9 +32,14 @@ jobs: - name: Install dependencies run: | sudo apt update + export PATH="/usr/lib/ccache:/usr/local/opt/ccache/libexec:$PATH" sudo apt install -y build-essential curl zlib1g-dev libcurl4-openssl-dev libexpat1-dev pip install setuptools wheel twine + - name: Build and Install package + run: | + ccache -s pip install -e . --no-index + ccache -s - name: Test code run: | python testdata/pymod/import_db.py From a3f37b4f4f820c102d034f32254dab3821891320 Mon Sep 17 00:00:00 2001 From: Thomas Ferreira de Lima Date: Mon, 2 May 2022 16:13:19 -0400 Subject: [PATCH 18/34] creating dependency between jobs --- .github/workflows/pythonpublish-mac.yml | 16 +++++++++------- .github/workflows/pythonpublish.yml | 17 ++++++++--------- .github/workflows/test_code-mac.yml | 13 ++++++++----- .github/workflows/test_code.yml | 16 ++++++---------- 4 files changed, 31 insertions(+), 31 deletions(-) diff --git a/.github/workflows/pythonpublish-mac.yml b/.github/workflows/pythonpublish-mac.yml index 0a44c20a29..725e038311 100644 --- a/.github/workflows/pythonpublish-mac.yml +++ b/.github/workflows/pythonpublish-mac.yml @@ -10,25 +10,28 @@ on: workflow_dispatch: jobs: - mac-build: - name: Wrapper macOS Build + mac-deploy: + needs: [mac-build, build] runs-on: ${{ matrix.os }} strategy: matrix: os: ["macos-11", "macos-10.15"] - pyversion: ["python@3.8", "python@3.9", "python@3.10"] + python-version: [3.7, 3.8, 3.9, '3.10'] steps: - name: Checkout - uses: actions/checkout@master + uses: actions/checkout@v3 - name: ccache uses: hendrikmuhs/ccache-action@v1.2 with: - key: ${{ github.job }}-${{ matrix.os }} # Make cache specific to OS + key: ${{ matrix.python-version }}-${{ matrix.os }} # Make cache specific to OS + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v3 + with: + python-version: ${{ matrix.python-version }} - name: Build the macOS wheels run: | pip3 install twine export PATH="/usr/lib/ccache:/usr/local/opt/ccache/libexec:$PATH" - ccache -s pip3 install -e . --no-index - name: Build and publish env: @@ -36,5 +39,4 @@ jobs: TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }} run: | python3 setup.py sdist bdist_wheel - ccache -s twine upload dist/* diff --git a/.github/workflows/pythonpublish.yml b/.github/workflows/pythonpublish.yml index 63d78693af..fe7f25726a 100644 --- a/.github/workflows/pythonpublish.yml +++ b/.github/workflows/pythonpublish.yml @@ -10,7 +10,8 @@ on: workflow_dispatch: jobs: - deploy: + linux-deploy: + needs: [mac-build, build] runs-on: ${{ matrix.os }} strategy: max-parallel: 12 @@ -18,15 +19,15 @@ jobs: python-version: [3.8, 3.9, '3.10'] os: [ubuntu-latest, macos-latest, windows-latest] steps: - - uses: actions/checkout@v2 - - name: Set up Python - uses: actions/setup-python@v2 - with: - python-version: 3.x + - uses: actions/checkout@v3 - name: ccache uses: hendrikmuhs/ccache-action@v1.2 with: - key: ${{ github.job }}-${{ matrix.os }} # Make cache specific to OS + key: ${{ matrix.python-version }}-${{ matrix.os }} # Make cache specific to OS + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v3 + with: + python-version: ${{ matrix.python-version }} - name: Set up GCC uses: egor-tensin/setup-gcc@v1 with: @@ -47,7 +48,5 @@ jobs: TWINE_USERNAME: ${{ secrets.PYPI_USERNAME }} TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }} run: | - ccache -s python setup.py sdist bdist_wheel - ccache -s twine upload dist/* diff --git a/.github/workflows/test_code-mac.yml b/.github/workflows/test_code-mac.yml index dbcbcb9481..925497e60b 100644 --- a/.github/workflows/test_code-mac.yml +++ b/.github/workflows/test_code-mac.yml @@ -1,5 +1,5 @@ --- -name: Mac test python package +name: build Mac python package on: pull_request: @@ -7,21 +7,24 @@ on: jobs: mac-build: - name: Wrapper macOS Build runs-on: ${{ matrix.os }} strategy: matrix: os: ["macos-11", "macos-10.15"] - pyversion: ["python@3.8", "python@3.9", "python@3.10"] + python-version: [3.7, 3.8, 3.9, '3.10'] steps: - name: Cancel Workflow Action uses: styfle/cancel-workflow-action@0.9.1 - name: Checkout - uses: actions/checkout@master + uses: actions/checkout@v3 - name: ccache uses: hendrikmuhs/ccache-action@v1.2 with: - key: ${{ github.job }}-${{ matrix.os }} # Make cache specific to OS + key: ${{ matrix.python-version }}-${{ matrix.os }} # Make cache specific to OS + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v3 + with: + python-version: ${{ matrix.python-version }} - name: Build the macOS wheels run: | pip3 install twine diff --git a/.github/workflows/test_code.yml b/.github/workflows/test_code.yml index d84f9bcd5d..f52ab9935f 100644 --- a/.github/workflows/test_code.yml +++ b/.github/workflows/test_code.yml @@ -1,15 +1,13 @@ --- -name: linux test python package +name: build linux python package # https://docs.github.com/en/free-pro-team@latest/actions/guides/building-and-testing-python on: pull_request: push: - schedule: - - cron: 0 2 * * * # run at 2 AM UTC jobs: - build: + linux-build: runs-on: ${{ matrix.os }} strategy: max-parallel: 12 @@ -20,13 +18,13 @@ jobs: steps: - name: Cancel Workflow Action uses: styfle/cancel-workflow-action@0.9.1 - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 - name: ccache uses: hendrikmuhs/ccache-action@v1.2 with: - key: ${{ github.job }}-${{ matrix.os }} # Make cache specific to OS + key: ${{ matrix.python-version }}-${{ matrix.os }} # Make cache specific to OS - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v2 + uses: actions/setup-python@v3 with: python-version: ${{ matrix.python-version }} - name: Install dependencies @@ -37,9 +35,7 @@ jobs: pip install setuptools wheel twine - name: Build and Install package run: | - ccache -s - pip install -e . --no-index - ccache -s + pip install -e . --no-index - name: Test code run: | python testdata/pymod/import_db.py From da67d8fb1bb925e248b696bc6e381776db130b36 Mon Sep 17 00:00:00 2001 From: Thomas Ferreira de Lima Date: Mon, 2 May 2022 16:54:19 -0400 Subject: [PATCH 19/34] using cibuildwheel --- .github/workflows/build.yml | 54 +++++++++++++++++++++++++++++++++ .github/workflows/test_code.yml | 2 +- 2 files changed, 55 insertions(+), 1 deletion(-) create mode 100644 .github/workflows/build.yml diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml new file mode 100644 index 0000000000..fbcca002f1 --- /dev/null +++ b/.github/workflows/build.yml @@ -0,0 +1,54 @@ +--- +name: Build Python Package +# https://docs.github.com/en/free-pro-team@latest/actions/guides/building-and-testing-python + +on: + pull_request: + push: + +jobs: + build: + runs-on: ${{ matrix.os }} + strategy: + max-parallel: 12 + matrix: + os: [ubuntu-latest, "macos-11", "macos-10.15"] + python-version: [3.7, 3.8, 3.9, '3.10'] + + steps: + - name: Cancel Workflow Action + uses: styfle/cancel-workflow-action@0.9.1 + - uses: actions/checkout@v3 + - name: ccache + uses: hendrikmuhs/ccache-action@v1.2 + with: + key: ${{ matrix.python-version }}-${{ matrix.os }} # Make cache specific to OS + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v3 + with: + python-version: ${{ matrix.python-version }} + - name: Install dependencies + run: | + export PATH="/usr/lib/ccache:/usr/local/opt/ccache/libexec:$PATH" + if [ "$RUNNER_OS" == "Linux" ]; then + sudo apt update + sudo apt install -y build-essential curl zlib1g-dev libcurl4-openssl-dev libexpat1-dev + fi + - name: Build and Install package + run: | + pip install -e . --no-index + - name: Test code + run: | + python testdata/pymod/import_db.py + python testdata/pymod/import_rdb.py + python testdata/pymod/import_tl.py + python testdata/pymod/import_lib.py + python testdata/pymod/pya_tests.py + - name: Build wheels # check https://cibuildwheel.readthedocs.io/en/stable/setup/#github-actions + uses: pypa/cibuildwheel@v2.5.0 + # to supply options, put them in 'env', like: + # env: + # CIBW_SOME_OPTION: value + - uses: actions/upload-artifact@v2 + with: + path: ./wheelhouse/*.whl \ No newline at end of file diff --git a/.github/workflows/test_code.yml b/.github/workflows/test_code.yml index f52ab9935f..dfc3890996 100644 --- a/.github/workflows/test_code.yml +++ b/.github/workflows/test_code.yml @@ -29,8 +29,8 @@ jobs: python-version: ${{ matrix.python-version }} - name: Install dependencies run: | - sudo apt update export PATH="/usr/lib/ccache:/usr/local/opt/ccache/libexec:$PATH" + sudo apt update sudo apt install -y build-essential curl zlib1g-dev libcurl4-openssl-dev libexpat1-dev pip install setuptools wheel twine - name: Build and Install package From 809e2a6d66d5dc3833ab4dbd44d614f63d87959a Mon Sep 17 00:00:00 2001 From: Thomas Ferreira de Lima Date: Mon, 2 May 2022 17:00:25 -0400 Subject: [PATCH 20/34] ccache path bug --- .github/workflows/build.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index fbcca002f1..af041a9bdf 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -29,13 +29,14 @@ jobs: python-version: ${{ matrix.python-version }} - name: Install dependencies run: | - export PATH="/usr/lib/ccache:/usr/local/opt/ccache/libexec:$PATH" if [ "$RUNNER_OS" == "Linux" ]; then sudo apt update sudo apt install -y build-essential curl zlib1g-dev libcurl4-openssl-dev libexpat1-dev fi - name: Build and Install package run: | + export PATH="/usr/lib/ccache:/usr/local/opt/ccache/libexec:$PATH" + echo "/usr/lib/ccache:/usr/local/opt/ccache/libexec" >> $GITHUB_PATH pip install -e . --no-index - name: Test code run: | From 05d402bf69d25d9a4be385f053370237e050a13f Mon Sep 17 00:00:00 2001 From: Thomas Ferreira de Lima Date: Mon, 2 May 2022 17:21:41 -0400 Subject: [PATCH 21/34] Testing after building wheels --- .github/workflows/build.yml | 27 ++++++++++----------------- 1 file changed, 10 insertions(+), 17 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index af041a9bdf..15c26f7124 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -1,5 +1,5 @@ --- -name: Build Python Package +name: Build Python Wheels # https://docs.github.com/en/free-pro-team@latest/actions/guides/building-and-testing-python on: @@ -13,7 +13,6 @@ jobs: max-parallel: 12 matrix: os: [ubuntu-latest, "macos-11", "macos-10.15"] - python-version: [3.7, 3.8, 3.9, '3.10'] steps: - name: Cancel Workflow Action @@ -22,34 +21,28 @@ jobs: - name: ccache uses: hendrikmuhs/ccache-action@v1.2 with: - key: ${{ matrix.python-version }}-${{ matrix.os }} # Make cache specific to OS - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v3 - with: - python-version: ${{ matrix.python-version }} + key: ${{ github.job }}-${{ matrix.os }} # Make cache specific to OS + max-size: "5G" - name: Install dependencies run: | if [ "$RUNNER_OS" == "Linux" ]; then sudo apt update sudo apt install -y build-essential curl zlib1g-dev libcurl4-openssl-dev libexpat1-dev fi - - name: Build and Install package - run: | export PATH="/usr/lib/ccache:/usr/local/opt/ccache/libexec:$PATH" echo "/usr/lib/ccache:/usr/local/opt/ccache/libexec" >> $GITHUB_PATH - pip install -e . --no-index - - name: Test code - run: | - python testdata/pymod/import_db.py - python testdata/pymod/import_rdb.py - python testdata/pymod/import_tl.py - python testdata/pymod/import_lib.py - python testdata/pymod/pya_tests.py - name: Build wheels # check https://cibuildwheel.readthedocs.io/en/stable/setup/#github-actions uses: pypa/cibuildwheel@v2.5.0 # to supply options, put them in 'env', like: # env: # CIBW_SOME_OPTION: value + env: + CIBW_TEST_COMMAND: > + python {package}/testdata/pymod/import_db.py && + python {package}/testdata/pymod/import_rdb.py && + python {package}/testdata/pymod/import_tl.py && + python {package}/testdata/pymod/import_lib.py && + python {package}/testdata/pymod/pya_tests.py - uses: actions/upload-artifact@v2 with: path: ./wheelhouse/*.whl \ No newline at end of file From 7f8c27a4ebf9feb8f279c5399e75bd2aa4e31b24 Mon Sep 17 00:00:00 2001 From: Thomas Ferreira de Lima Date: Mon, 2 May 2022 18:09:42 -0400 Subject: [PATCH 22/34] misunderstood cibuildwheel --- .github/workflows/build.yml | 11 ---------- ci-scripts/docker/docker_prepare.sh | 31 +++++++++++++++++++++++++++++ pyproject.toml | 13 ++++++++++++ 3 files changed, 44 insertions(+), 11 deletions(-) create mode 100644 ci-scripts/docker/docker_prepare.sh create mode 100644 pyproject.toml diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 15c26f7124..fd1b1e477d 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -25,10 +25,6 @@ jobs: max-size: "5G" - name: Install dependencies run: | - if [ "$RUNNER_OS" == "Linux" ]; then - sudo apt update - sudo apt install -y build-essential curl zlib1g-dev libcurl4-openssl-dev libexpat1-dev - fi export PATH="/usr/lib/ccache:/usr/local/opt/ccache/libexec:$PATH" echo "/usr/lib/ccache:/usr/local/opt/ccache/libexec" >> $GITHUB_PATH - name: Build wheels # check https://cibuildwheel.readthedocs.io/en/stable/setup/#github-actions @@ -36,13 +32,6 @@ jobs: # to supply options, put them in 'env', like: # env: # CIBW_SOME_OPTION: value - env: - CIBW_TEST_COMMAND: > - python {package}/testdata/pymod/import_db.py && - python {package}/testdata/pymod/import_rdb.py && - python {package}/testdata/pymod/import_tl.py && - python {package}/testdata/pymod/import_lib.py && - python {package}/testdata/pymod/pya_tests.py - uses: actions/upload-artifact@v2 with: path: ./wheelhouse/*.whl \ No newline at end of file diff --git a/ci-scripts/docker/docker_prepare.sh b/ci-scripts/docker/docker_prepare.sh new file mode 100644 index 0000000000..6efe2a8c46 --- /dev/null +++ b/ci-scripts/docker/docker_prepare.sh @@ -0,0 +1,31 @@ +#!/usr/bin/env bash +set -xe + +# sometimes the epel server is down. retry 5 times +for i in $(seq 1 5); do + yum install -y zlib-devel curl-devel expat-devel ccache zip git && s=0 && break || s=$? && sleep 15; +done + +[ $s -eq 0 ] || exit $s + +# if [[ $DOCKER_IMAGE == "quay.io/pypa/manylinux2014_x86_64" ]]; then +ln -s /usr/bin/ccache /usr/lib64/ccache/c++ +ln -s /usr/bin/ccache /usr/lib64/ccache/cc +ln -s /usr/bin/ccache /usr/lib64/ccache/gcc +ln -s /usr/bin/ccache /usr/lib64/ccache/g++ +export PATH="/usr/lib64/ccache/:$PATH" +# elif [[ $DOCKER_IMAGE == "quay.io/pypa/manylinux2014_i686" ]]; then +ln -s /usr/bin/ccache /usr/lib/ccache/c++ +ln -s /usr/bin/ccache /usr/lib/ccache/cc +ln -s /usr/bin/ccache /usr/lib/ccache/gcc +ln -s /usr/bin/ccache /usr/lib/ccache/g++ +export PATH="/usr/lib/ccache/:$PATH" +# fi + +echo $PATH +export CCACHE_DIR="/host/home/runner/work/klayout/klayout/.ccache" +ccache -M 5 G # set cache size to 5 G + +# Show ccache stats +echo "Cache stats:" +ccache -s diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000000..b111e6ac4a --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,13 @@ +[tool.cibuildwheel] +build-verbosity = "3" +test-command = [ + "python {package}/testdata/pymod/import_db.py", + "python {package}/testdata/pymod/import_rdb.py", + "python {package}/testdata/pymod/import_tl.py", + "python {package}/testdata/pymod/import_lib.py", + "python {package}/testdata/pymod/pya_tests.py" +] + +[tool.cibuildwheel.linux] +before-build = "bash {project}/ci-scripts/docker/docker_prepare.sh" +environment = { CFLAGS="-std=c99 -fno-strict-aliasing", LDFLAGS="-Wl,--strip-debug", OPENBLAS64_="/usr/local", NPY_USE_BLAS_ILP64="1", RUNNER_OS="Linux" } From 2aa74fa3bed832c6f87a5b7b74880d59c9f051a3 Mon Sep 17 00:00:00 2001 From: Thomas Ferreira de Lima Date: Mon, 2 May 2022 18:22:23 -0400 Subject: [PATCH 23/34] fixing error in manylinux build --- ci-scripts/docker/docker_prepare.sh | 30 ++++++++++++++--------------- pyproject.toml | 2 +- 2 files changed, 16 insertions(+), 16 deletions(-) diff --git a/ci-scripts/docker/docker_prepare.sh b/ci-scripts/docker/docker_prepare.sh index 6efe2a8c46..dd01f51417 100644 --- a/ci-scripts/docker/docker_prepare.sh +++ b/ci-scripts/docker/docker_prepare.sh @@ -3,27 +3,27 @@ set -xe # sometimes the epel server is down. retry 5 times for i in $(seq 1 5); do - yum install -y zlib-devel curl-devel expat-devel ccache zip git && s=0 && break || s=$? && sleep 15; + yum install -y zlib-devel curl-devel expat-devel ccache && s=0 && break || s=$? && sleep 15; done [ $s -eq 0 ] || exit $s -# if [[ $DOCKER_IMAGE == "quay.io/pypa/manylinux2014_x86_64" ]]; then -ln -s /usr/bin/ccache /usr/lib64/ccache/c++ -ln -s /usr/bin/ccache /usr/lib64/ccache/cc -ln -s /usr/bin/ccache /usr/lib64/ccache/gcc -ln -s /usr/bin/ccache /usr/lib64/ccache/g++ -export PATH="/usr/lib64/ccache/:$PATH" -# elif [[ $DOCKER_IMAGE == "quay.io/pypa/manylinux2014_i686" ]]; then -ln -s /usr/bin/ccache /usr/lib/ccache/c++ -ln -s /usr/bin/ccache /usr/lib/ccache/cc -ln -s /usr/bin/ccache /usr/lib/ccache/gcc -ln -s /usr/bin/ccache /usr/lib/ccache/g++ -export PATH="/usr/lib/ccache/:$PATH" -# fi +if [[ -d "/usr/lib64/ccache" ]]; then + ln -s /usr/bin/ccache /usr/lib64/ccache/c++ + ln -s /usr/bin/ccache /usr/lib64/ccache/cc + ln -s /usr/bin/ccache /usr/lib64/ccache/gcc + ln -s /usr/bin/ccache /usr/lib64/ccache/g++ + export PATH="/usr/lib64/ccache/:$PATH" +elif [[ -d "/usr/lib/ccache" ]]; then + ln -s /usr/bin/ccache /usr/lib/ccache/c++ + ln -s /usr/bin/ccache /usr/lib/ccache/cc + ln -s /usr/bin/ccache /usr/lib/ccache/gcc + ln -s /usr/bin/ccache /usr/lib/ccache/g++ + export PATH="/usr/lib/ccache/:$PATH" +fi echo $PATH -export CCACHE_DIR="/host/home/runner/work/klayout/klayout/.ccache" +# export CCACHE_DIR="/host/home/runner/work/klayout/klayout/.ccache" ccache -M 5 G # set cache size to 5 G # Show ccache stats diff --git a/pyproject.toml b/pyproject.toml index b111e6ac4a..28ad56abd2 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -10,4 +10,4 @@ test-command = [ [tool.cibuildwheel.linux] before-build = "bash {project}/ci-scripts/docker/docker_prepare.sh" -environment = { CFLAGS="-std=c99 -fno-strict-aliasing", LDFLAGS="-Wl,--strip-debug", OPENBLAS64_="/usr/local", NPY_USE_BLAS_ILP64="1", RUNNER_OS="Linux" } +environment = { CCACHE_DIR="/host$(ccache -k cache_dir)" } \ No newline at end of file From 30a39f4d1dc7b4f2f001893439cb95c10489a1f2 Mon Sep 17 00:00:00 2001 From: Thomas Ferreira de Lima Date: Mon, 2 May 2022 18:30:21 -0400 Subject: [PATCH 24/34] fixing error in manylinux build (2) --- .github/workflows/build.yml | 1 + pyproject.toml | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index fd1b1e477d..e3edf3158b 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -27,6 +27,7 @@ jobs: run: | export PATH="/usr/lib/ccache:/usr/local/opt/ccache/libexec:$PATH" echo "/usr/lib/ccache:/usr/local/opt/ccache/libexec" >> $GITHUB_PATH + export CCACHE_DIR="$(ccache -k cache_dir)" - name: Build wheels # check https://cibuildwheel.readthedocs.io/en/stable/setup/#github-actions uses: pypa/cibuildwheel@v2.5.0 # to supply options, put them in 'env', like: diff --git a/pyproject.toml b/pyproject.toml index 28ad56abd2..1b81b0cbf9 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -10,4 +10,4 @@ test-command = [ [tool.cibuildwheel.linux] before-build = "bash {project}/ci-scripts/docker/docker_prepare.sh" -environment = { CCACHE_DIR="/host$(ccache -k cache_dir)" } \ No newline at end of file +environment = { CCACHE_DIR="/host/home/runner/work/klayout/klayout/.ccache" } \ No newline at end of file From fd46957047c71b007f094d7e8625e64faa175c58 Mon Sep 17 00:00:00 2001 From: Thomas Ferreira de Lima Date: Mon, 2 May 2022 19:00:05 -0400 Subject: [PATCH 25/34] disabling fail fast (temporary) --- .github/workflows/build.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index e3edf3158b..ec70d38b73 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -10,6 +10,7 @@ jobs: build: runs-on: ${{ matrix.os }} strategy: + fail-fast: false max-parallel: 12 matrix: os: [ubuntu-latest, "macos-11", "macos-10.15"] From de7d9f27a36f2a1253197b41f9b272ea9fd5fba2 Mon Sep 17 00:00:00 2001 From: Thomas Ferreira de Lima Date: Mon, 2 May 2022 19:10:16 -0400 Subject: [PATCH 26/34] Don't repair macOS wheels --- pyproject.toml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 1b81b0cbf9..a0bc32c3ac 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -10,4 +10,8 @@ test-command = [ [tool.cibuildwheel.linux] before-build = "bash {project}/ci-scripts/docker/docker_prepare.sh" -environment = { CCACHE_DIR="/host/home/runner/work/klayout/klayout/.ccache" } \ No newline at end of file +environment = { CCACHE_DIR="/host/home/runner/work/klayout/klayout/.ccache" } + +[tool.cibuildwheel.macos] +# Don't repair macOS wheels +repair-wheel-command = "" \ No newline at end of file From 67b080904549e9ad483d7b2e4052605990f3c5a0 Mon Sep 17 00:00:00 2001 From: Thomas Ferreira de Lima Date: Mon, 2 May 2022 22:24:26 -0400 Subject: [PATCH 27/34] Not building pypy wheels --- ci-scripts/docker/docker_prepare.sh | 4 ++-- pyproject.toml | 4 +++- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/ci-scripts/docker/docker_prepare.sh b/ci-scripts/docker/docker_prepare.sh index dd01f51417..330a1d2225 100644 --- a/ci-scripts/docker/docker_prepare.sh +++ b/ci-scripts/docker/docker_prepare.sh @@ -13,13 +13,13 @@ if [[ -d "/usr/lib64/ccache" ]]; then ln -s /usr/bin/ccache /usr/lib64/ccache/cc ln -s /usr/bin/ccache /usr/lib64/ccache/gcc ln -s /usr/bin/ccache /usr/lib64/ccache/g++ - export PATH="/usr/lib64/ccache/:$PATH" + export PATH="/usr/lib64/ccache:$PATH" elif [[ -d "/usr/lib/ccache" ]]; then ln -s /usr/bin/ccache /usr/lib/ccache/c++ ln -s /usr/bin/ccache /usr/lib/ccache/cc ln -s /usr/bin/ccache /usr/lib/ccache/gcc ln -s /usr/bin/ccache /usr/lib/ccache/g++ - export PATH="/usr/lib/ccache/:$PATH" + export PATH="/usr/lib/ccache:$PATH" fi echo $PATH diff --git a/pyproject.toml b/pyproject.toml index a0bc32c3ac..c68542e6f0 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -7,9 +7,11 @@ test-command = [ "python {package}/testdata/pymod/import_lib.py", "python {package}/testdata/pymod/pya_tests.py" ] +# Disable building PyPy wheels on all platforms +skip = "pp*" [tool.cibuildwheel.linux] -before-build = "bash {project}/ci-scripts/docker/docker_prepare.sh" +before-all = "source {project}/ci-scripts/docker/docker_prepare.sh" environment = { CCACHE_DIR="/host/home/runner/work/klayout/klayout/.ccache" } [tool.cibuildwheel.macos] From 6623c8958ce6b69e836125d64b0c3d31b689ee1f Mon Sep 17 00:00:00 2001 From: Thomas Ferreira de Lima Date: Tue, 3 May 2022 00:54:33 -0400 Subject: [PATCH 28/34] disabling 32-bit linux wheels --- pyproject.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/pyproject.toml b/pyproject.toml index c68542e6f0..bb26eb9313 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -13,6 +13,7 @@ skip = "pp*" [tool.cibuildwheel.linux] before-all = "source {project}/ci-scripts/docker/docker_prepare.sh" environment = { CCACHE_DIR="/host/home/runner/work/klayout/klayout/.ccache" } +archs = ["auto64"] [tool.cibuildwheel.macos] # Don't repair macOS wheels From 37bde8eb41f8b7710ee4d99dd5f70735e90260f4 Mon Sep 17 00:00:00 2001 From: Thomas Ferreira de Lima Date: Tue, 3 May 2022 01:07:19 -0400 Subject: [PATCH 29/34] macos-latest only (prevent duplicate) --- .github/workflows/build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index ec70d38b73..47076e5f42 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -13,7 +13,7 @@ jobs: fail-fast: false max-parallel: 12 matrix: - os: [ubuntu-latest, "macos-11", "macos-10.15"] + os: [ubuntu-latest, "macos-latest"] steps: - name: Cancel Workflow Action From 3223888bc3f4a04f9f9c0937054514cb0b49d20f Mon Sep 17 00:00:00 2001 From: Thomas Ferreira de Lima Date: Tue, 3 May 2022 12:25:27 -0400 Subject: [PATCH 30/34] fixing musllinux --- ci-scripts/docker/docker_prepare.sh | 43 +++++++++++++++++------------ pyproject.toml | 4 +-- 2 files changed, 28 insertions(+), 19 deletions(-) diff --git a/ci-scripts/docker/docker_prepare.sh b/ci-scripts/docker/docker_prepare.sh index 330a1d2225..99487dd5a9 100644 --- a/ci-scripts/docker/docker_prepare.sh +++ b/ci-scripts/docker/docker_prepare.sh @@ -1,25 +1,34 @@ #!/usr/bin/env bash set -xe -# sometimes the epel server is down. retry 5 times -for i in $(seq 1 5); do - yum install -y zlib-devel curl-devel expat-devel ccache && s=0 && break || s=$? && sleep 15; -done +# manylinux prep +if [[ -f "/etc/centos-release" ]]; then + # sometimes the epel server is down. retry 5 times + for i in $(seq 1 5); do + yum install -y zlib-devel curl-devel expat-devel ccache && s=0 && break || s=$? && sleep 15; + done -[ $s -eq 0 ] || exit $s + [ $s -eq 0 ] || exit $s -if [[ -d "/usr/lib64/ccache" ]]; then - ln -s /usr/bin/ccache /usr/lib64/ccache/c++ - ln -s /usr/bin/ccache /usr/lib64/ccache/cc - ln -s /usr/bin/ccache /usr/lib64/ccache/gcc - ln -s /usr/bin/ccache /usr/lib64/ccache/g++ - export PATH="/usr/lib64/ccache:$PATH" -elif [[ -d "/usr/lib/ccache" ]]; then - ln -s /usr/bin/ccache /usr/lib/ccache/c++ - ln -s /usr/bin/ccache /usr/lib/ccache/cc - ln -s /usr/bin/ccache /usr/lib/ccache/gcc - ln -s /usr/bin/ccache /usr/lib/ccache/g++ - export PATH="/usr/lib/ccache:$PATH" + if [[ -d "/usr/lib64/ccache" ]]; then + ln -s /usr/bin/ccache /usr/lib64/ccache/c++ + ln -s /usr/bin/ccache /usr/lib64/ccache/cc + ln -s /usr/bin/ccache /usr/lib64/ccache/gcc + ln -s /usr/bin/ccache /usr/lib64/ccache/g++ + export PATH="/usr/lib64/ccache:$PATH" + elif [[ -d "/usr/lib/ccache" ]]; then + ln -s /usr/bin/ccache /usr/lib/ccache/c++ + ln -s /usr/bin/ccache /usr/lib/ccache/cc + ln -s /usr/bin/ccache /usr/lib/ccache/gcc + ln -s /usr/bin/ccache /usr/lib/ccache/g++ + export PATH="/usr/lib/ccache:$PATH" + fi + +elif [[ -f "/etc/alpine-release" ]]; then + # musllinux prep + # ccache already present + apk add curl-dev expat-dev zlib-dev ccache + export PATH="/usr/lib/ccache/bin:$PATH" fi echo $PATH diff --git a/pyproject.toml b/pyproject.toml index bb26eb9313..f1e679b4d8 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -8,11 +8,11 @@ test-command = [ "python {package}/testdata/pymod/pya_tests.py" ] # Disable building PyPy wheels on all platforms -skip = "pp*" +skip = "pp* *manylinux*" [tool.cibuildwheel.linux] before-all = "source {project}/ci-scripts/docker/docker_prepare.sh" -environment = { CCACHE_DIR="/host/home/runner/work/klayout/klayout/.ccache" } +# environment = { CCACHE_DIR="/host/home/runner/work/klayout/klayout/.ccache" } archs = ["auto64"] [tool.cibuildwheel.macos] From 54e89e98f79d3881a9f75936a7bcc4310daf5f5f Mon Sep 17 00:00:00 2001 From: Thomas Ferreira de Lima Date: Tue, 3 May 2022 18:14:26 -0400 Subject: [PATCH 31/34] fixing ccache --- .github/workflows/build.yml | 29 ++++++++++++++++++++++++----- ci-scripts/docker/docker_prepare.sh | 9 ++++++++- pyproject.toml | 11 +++++++++-- 3 files changed, 41 insertions(+), 8 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 47076e5f42..74d22e2613 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -13,8 +13,13 @@ jobs: fail-fast: false max-parallel: 12 matrix: - os: [ubuntu-latest, "macos-latest"] - + include: + - os: "macos-latest" + cibuild: "*macosx*" + - os: "ubuntu-latest" + cibuild: "*manylinux*" + - os: "ubuntu-latest" + cibuild: "*musllinux*" steps: - name: Cancel Workflow Action uses: styfle/cancel-workflow-action@0.9.1 @@ -22,18 +27,32 @@ jobs: - name: ccache uses: hendrikmuhs/ccache-action@v1.2 with: - key: ${{ github.job }}-${{ matrix.os }} # Make cache specific to OS + key: ${{ github.job }}-${{ matrix.os }}-${{ matrix.cibuild }} # Make cache specific to OS max-size: "5G" - name: Install dependencies run: | + env export PATH="/usr/lib/ccache:/usr/local/opt/ccache/libexec:$PATH" echo "/usr/lib/ccache:/usr/local/opt/ccache/libexec" >> $GITHUB_PATH - export CCACHE_DIR="$(ccache -k cache_dir)" + HOST_CCACHE_DIR="$(ccache -k cache_dir)" + mkdir -p $HOST_CCACHE_DIR - name: Build wheels # check https://cibuildwheel.readthedocs.io/en/stable/setup/#github-actions uses: pypa/cibuildwheel@v2.5.0 # to supply options, put them in 'env', like: # env: # CIBW_SOME_OPTION: value + env: + CIBW_BUILD: ${{ matrix.cibuild }} + - name: Download Cache from Docker (linux only) + if: ${{ env.RUNNER_OS == 'Linux' }} + # hack until https://github.com/pypa/cibuildwheel/issues/1030 is fixed + run: | + env + ccache -s + HOST_CCACHE_DIR="$(ccache -k cache_dir)" + rm -rf $HOST_CCACHE_DIR + mv ./wheelhouse/.ccache $HOST_CCACHE_DIR + ccache -s - uses: actions/upload-artifact@v2 with: - path: ./wheelhouse/*.whl \ No newline at end of file + path: ./wheelhouse/*.whl diff --git a/ci-scripts/docker/docker_prepare.sh b/ci-scripts/docker/docker_prepare.sh index 99487dd5a9..4e1574a25c 100644 --- a/ci-scripts/docker/docker_prepare.sh +++ b/ci-scripts/docker/docker_prepare.sh @@ -31,7 +31,14 @@ elif [[ -f "/etc/alpine-release" ]]; then export PATH="/usr/lib/ccache/bin:$PATH" fi -echo $PATH +# hack until https://github.com/pypa/cibuildwheel/issues/1030 is fixed +# Place ccache folder in /outputs +HOST_CCACHE_DIR="/host${HOST_CCACHE_DIR:-/home/runner/work/klayout/klayout/.ccache}" +if [ -d $HOST_CCACHE_DIR ]; then + cp -R $HOST_CCACHE_DIR /output/ +fi + +ccache -o cache_dir="/output/.ccache" # export CCACHE_DIR="/host/home/runner/work/klayout/klayout/.ccache" ccache -M 5 G # set cache size to 5 G diff --git a/pyproject.toml b/pyproject.toml index f1e679b4d8..5e3833538c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -8,12 +8,19 @@ test-command = [ "python {package}/testdata/pymod/pya_tests.py" ] # Disable building PyPy wheels on all platforms -skip = "pp* *manylinux*" +skip = "pp*" [tool.cibuildwheel.linux] +# beware: the before-all script does not persist environment variables! +# No point in defining $PATH or $CCACHE_DIR before-all = "source {project}/ci-scripts/docker/docker_prepare.sh" -# environment = { CCACHE_DIR="/host/home/runner/work/klayout/klayout/.ccache" } archs = ["auto64"] +# Append a directory to the PATH variable (this is expanded in the build environment) +environment = { PATH="/usr/lib/ccache:/usr/lib64/ccache:/usr/lib/ccache/bin:$PATH" } +before-build = "ccache -s" +# Export a variable to docker +# CCACHE_DIR="/home/runner/work/klayout/klayout/.ccache" +environment-pass = ["HOST_CCACHE_DIR"] [tool.cibuildwheel.macos] # Don't repair macOS wheels From b259465540ff2bd28954d759d3b8507816d3a911 Mon Sep 17 00:00:00 2001 From: Thomas Ferreira de Lima Date: Wed, 4 May 2022 19:12:48 -0400 Subject: [PATCH 32/34] fixing cache persistence --- .github/workflows/build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 74d22e2613..674b248b98 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -44,7 +44,7 @@ jobs: env: CIBW_BUILD: ${{ matrix.cibuild }} - name: Download Cache from Docker (linux only) - if: ${{ env.RUNNER_OS == 'Linux' }} + if: ${{ runner.os == 'Linux' }} # hack until https://github.com/pypa/cibuildwheel/issues/1030 is fixed run: | env From 597e7ee4b89ea3e3818af60954baeb1b3f0a6146 Mon Sep 17 00:00:00 2001 From: Thomas Ferreira de Lima Date: Wed, 4 May 2022 22:08:10 -0400 Subject: [PATCH 33/34] adding deploy job --- .github/workflows/build.yml | 38 ++++++++++++++++++ .github/workflows/pythonpublish-mac.yml | 42 -------------------- .github/workflows/pythonpublish.yml | 52 ------------------------- .github/workflows/test_code-mac.yml | 41 ------------------- .github/workflows/test_code.yml | 45 --------------------- ci-scripts/docker/docker_prepare.sh | 2 + 6 files changed, 40 insertions(+), 180 deletions(-) delete mode 100644 .github/workflows/pythonpublish-mac.yml delete mode 100644 .github/workflows/pythonpublish.yml delete mode 100644 .github/workflows/test_code-mac.yml delete mode 100644 .github/workflows/test_code.yml diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 674b248b98..3d7a21dd34 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -52,7 +52,45 @@ jobs: HOST_CCACHE_DIR="$(ccache -k cache_dir)" rm -rf $HOST_CCACHE_DIR mv ./wheelhouse/.ccache $HOST_CCACHE_DIR + ls -la $HOST_CCACHE_DIR ccache -s - uses: actions/upload-artifact@v2 with: path: ./wheelhouse/*.whl + +# The following was taken from https://cibuildwheel.readthedocs.io/en/stable/deliver-to-pypi/ + make_sdist: + name: Make SDist + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + with: + fetch-depth: 0 # Optional, use if you use setuptools_scm + submodules: true # Optional, use if you have submodules + + - name: Build SDist + run: pipx run build --sdist + + - uses: actions/upload-artifact@v2 + with: + path: dist/*.tar.gz + + upload_all: + needs: [build, make_sdist] + runs-on: ubuntu-latest + # Uncomment for real PyPi + # if: github.event_name == 'release' && github.event.action == 'published' + steps: + - uses: actions/download-artifact@v2 + with: + name: artifact + path: dist + + - uses: pypa/gh-action-pypi-publish@v1.4.2 + with: + user: __token__ + # Test PyPI + password: ${{ secrets.test_pypi_password }} + repository_url: https://test.pypi.org/legacy/ + # Uncomment for Real Pypi + # password: ${{ secrets.pypi_password }} diff --git a/.github/workflows/pythonpublish-mac.yml b/.github/workflows/pythonpublish-mac.yml deleted file mode 100644 index 725e038311..0000000000 --- a/.github/workflows/pythonpublish-mac.yml +++ /dev/null @@ -1,42 +0,0 @@ ---- -name: Upload Mac Python Wheel to PyPI - -on: - release: - types: [created, published] - push: - branches: [master] - tags: [v*] - workflow_dispatch: - -jobs: - mac-deploy: - needs: [mac-build, build] - runs-on: ${{ matrix.os }} - strategy: - matrix: - os: ["macos-11", "macos-10.15"] - python-version: [3.7, 3.8, 3.9, '3.10'] - steps: - - name: Checkout - uses: actions/checkout@v3 - - name: ccache - uses: hendrikmuhs/ccache-action@v1.2 - with: - key: ${{ matrix.python-version }}-${{ matrix.os }} # Make cache specific to OS - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v3 - with: - python-version: ${{ matrix.python-version }} - - name: Build the macOS wheels - run: | - pip3 install twine - export PATH="/usr/lib/ccache:/usr/local/opt/ccache/libexec:$PATH" - pip3 install -e . --no-index - - name: Build and publish - env: - TWINE_USERNAME: ${{ secrets.PYPI_USERNAME }} - TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }} - run: | - python3 setup.py sdist bdist_wheel - twine upload dist/* diff --git a/.github/workflows/pythonpublish.yml b/.github/workflows/pythonpublish.yml deleted file mode 100644 index fe7f25726a..0000000000 --- a/.github/workflows/pythonpublish.yml +++ /dev/null @@ -1,52 +0,0 @@ ---- -name: Upload Linux Python Wheel to PyPI - -on: - release: - types: [created, published] - push: - branches: [master] - tags: [v*] - workflow_dispatch: - -jobs: - linux-deploy: - needs: [mac-build, build] - runs-on: ${{ matrix.os }} - strategy: - max-parallel: 12 - matrix: - python-version: [3.8, 3.9, '3.10'] - os: [ubuntu-latest, macos-latest, windows-latest] - steps: - - uses: actions/checkout@v3 - - name: ccache - uses: hendrikmuhs/ccache-action@v1.2 - with: - key: ${{ matrix.python-version }}-${{ matrix.os }} # Make cache specific to OS - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v3 - with: - python-version: ${{ matrix.python-version }} - - name: Set up GCC - uses: egor-tensin/setup-gcc@v1 - with: - version: latest - platform: x64 - - name: Install dependencies - run: | - sudo apt update - export PATH="/usr/lib/ccache:/usr/local/opt/ccache/libexec:$PATH" - sudo apt install -y build-essential curl zlib1g-dev libcurl4-openssl-dev libexpat1-dev - python -m pip install --upgrade pip - pip install setuptools wheel twine - set "KLAYOUT_BITS=%cd%\klayout-microbits\klayout-microbits-1.0\msvc2017\%PYTHON_ARCHITECTURE%" - echo KLAYOUT_BITS=%KLAYOUT_BITS% - # pip install klayout --no-index -f dist - - name: Build and publish - env: - TWINE_USERNAME: ${{ secrets.PYPI_USERNAME }} - TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }} - run: | - python setup.py sdist bdist_wheel - twine upload dist/* diff --git a/.github/workflows/test_code-mac.yml b/.github/workflows/test_code-mac.yml deleted file mode 100644 index 925497e60b..0000000000 --- a/.github/workflows/test_code-mac.yml +++ /dev/null @@ -1,41 +0,0 @@ ---- -name: build Mac python package - -on: - pull_request: - push: - -jobs: - mac-build: - runs-on: ${{ matrix.os }} - strategy: - matrix: - os: ["macos-11", "macos-10.15"] - python-version: [3.7, 3.8, 3.9, '3.10'] - steps: - - name: Cancel Workflow Action - uses: styfle/cancel-workflow-action@0.9.1 - - name: Checkout - uses: actions/checkout@v3 - - name: ccache - uses: hendrikmuhs/ccache-action@v1.2 - with: - key: ${{ matrix.python-version }}-${{ matrix.os }} # Make cache specific to OS - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v3 - with: - python-version: ${{ matrix.python-version }} - - name: Build the macOS wheels - run: | - pip3 install twine - export PATH="/usr/lib/ccache:/usr/local/opt/ccache/libexec:$PATH" - ccache -s - pip3 install -e . --no-index - ccache -s - - name: Test code - run: | - python3 testdata/pymod/import_db.py - python3 testdata/pymod/import_rdb.py - python3 testdata/pymod/import_tl.py - python3 testdata/pymod/import_lib.py - python3 testdata/pymod/pya_tests.py diff --git a/.github/workflows/test_code.yml b/.github/workflows/test_code.yml deleted file mode 100644 index dfc3890996..0000000000 --- a/.github/workflows/test_code.yml +++ /dev/null @@ -1,45 +0,0 @@ ---- -name: build linux python package -# https://docs.github.com/en/free-pro-team@latest/actions/guides/building-and-testing-python - -on: - pull_request: - push: - -jobs: - linux-build: - runs-on: ${{ matrix.os }} - strategy: - max-parallel: 12 - matrix: - python-version: [3.7, 3.8, 3.9, '3.10'] - os: [ubuntu-latest] - - steps: - - name: Cancel Workflow Action - uses: styfle/cancel-workflow-action@0.9.1 - - uses: actions/checkout@v3 - - name: ccache - uses: hendrikmuhs/ccache-action@v1.2 - with: - key: ${{ matrix.python-version }}-${{ matrix.os }} # Make cache specific to OS - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v3 - with: - python-version: ${{ matrix.python-version }} - - name: Install dependencies - run: | - export PATH="/usr/lib/ccache:/usr/local/opt/ccache/libexec:$PATH" - sudo apt update - sudo apt install -y build-essential curl zlib1g-dev libcurl4-openssl-dev libexpat1-dev - pip install setuptools wheel twine - - name: Build and Install package - run: | - pip install -e . --no-index - - name: Test code - run: | - python testdata/pymod/import_db.py - python testdata/pymod/import_rdb.py - python testdata/pymod/import_tl.py - python testdata/pymod/import_lib.py - python testdata/pymod/pya_tests.py diff --git a/ci-scripts/docker/docker_prepare.sh b/ci-scripts/docker/docker_prepare.sh index 4e1574a25c..aa81dbede6 100644 --- a/ci-scripts/docker/docker_prepare.sh +++ b/ci-scripts/docker/docker_prepare.sh @@ -38,6 +38,8 @@ if [ -d $HOST_CCACHE_DIR ]; then cp -R $HOST_CCACHE_DIR /output/ fi +ls -la /output/ + ccache -o cache_dir="/output/.ccache" # export CCACHE_DIR="/host/home/runner/work/klayout/klayout/.ccache" ccache -M 5 G # set cache size to 5 G From 0c324f778fef861066e1f60c3845bb15c83ca5da Mon Sep 17 00:00:00 2001 From: Thomas Ferreira de Lima Date: Thu, 5 May 2022 11:01:34 -0400 Subject: [PATCH 34/34] last debug --- ci-scripts/docker/docker_prepare.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/ci-scripts/docker/docker_prepare.sh b/ci-scripts/docker/docker_prepare.sh index aa81dbede6..e380fdc452 100644 --- a/ci-scripts/docker/docker_prepare.sh +++ b/ci-scripts/docker/docker_prepare.sh @@ -35,7 +35,8 @@ fi # Place ccache folder in /outputs HOST_CCACHE_DIR="/host${HOST_CCACHE_DIR:-/home/runner/work/klayout/klayout/.ccache}" if [ -d $HOST_CCACHE_DIR ]; then - cp -R $HOST_CCACHE_DIR /output/ + mkdir -p /output + cp -R $HOST_CCACHE_DIR /output/.ccache fi ls -la /output/