From 83afd0a5b6f7c7d32319e0ba53b9ecd947976da0 Mon Sep 17 00:00:00 2001
From: Shantanu <12621235+hauntsaninja@users.noreply.github.com>
Date: Wed, 6 Jan 2021 19:17:33 -0800
Subject: [PATCH] mypy_mypyc-wheels: rewrite using Github Actions and
 cibuildwheel (#11)

* initial commit

* add mypy_commit

* add github action

* add pure python wheel and sdist

* update mypy_commit

* various improvements

* create release

* skip tests if lxml failed to install

* delete unneeded things

* update README badge

* don't make a release if a build failed

Co-authored-by: hauntsaninja <>
---
 .github/workflows/build.yml | 153 ++++++++++++++++++++++++++++++++++++
 .gitmodules                 |   6 --
 .travis.yml                 |  87 --------------------
 README.md                   |   5 +-
 appveyor.yml                |  61 --------------
 build-linux-wheels.sh       |  33 --------
 config.sh                   |   8 --
 install_python.ps1          |  37 ---------
 multibuild                  |   1 -
 mypy                        |   1 -
 mypy_commit                 |   1 +
 11 files changed, 155 insertions(+), 238 deletions(-)
 create mode 100644 .github/workflows/build.yml
 delete mode 100644 .gitmodules
 delete mode 100644 .travis.yml
 delete mode 100644 appveyor.yml
 delete mode 100755 build-linux-wheels.sh
 delete mode 100644 config.sh
 delete mode 100644 install_python.ps1
 delete mode 160000 multibuild
 delete mode 160000 mypy
 create mode 100644 mypy_commit

diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
new file mode 100644
index 00000000..833034e5
--- /dev/null
+++ b/.github/workflows/build.yml
@@ -0,0 +1,153 @@
+name: Build wheels
+
+on: [push, pull_request]
+
+jobs:
+  build_wheels:
+    name: py${{ matrix.python-version }} on ${{ matrix.os }}
+    runs-on: ${{ matrix.os }}
+    strategy:
+      fail-fast: false
+      matrix:
+        # cibuildwheel builds linux wheels inside a manylinux container
+        # it also takes care of procuring the correct python version for us
+        os: [ubuntu-latest, windows-latest, macos-latest]
+        python-version: [35, 36, 37, 38, 39]
+
+    steps:
+      - uses: actions/checkout@v2
+      - uses: actions/setup-python@v2
+        name: Install Python
+        with:
+          python-version: "3.7"
+      - name: Install cibuildwheel
+        run: |
+          python -m pip install cibuildwheel==1.6
+      - name: Checkout mypy
+        shell: bash
+        # use a commit hash checked into a file to get the mypy revision to build.
+        # submodules prove problematic since .git is outside cibuildwheel's manylinux container
+        run: |
+          COMMIT=$(cat mypy_commit)
+          git clone --recurse-submodules https://github.com/python/mypy.git
+          cd mypy
+          git checkout --force --recurse-submodules $COMMIT
+      - name: Build wheels
+        env:
+          CIBW_BUILD: "cp${{ matrix.python-version }}-*"
+          CIBW_SKIP: "*-manylinux_i686 *-win32"
+          CIBW_BUILD_VERBOSITY: 1
+
+          # mypy's isolated builds don't specify the requirements mypyc needs, so install
+          # requirements and don't use isolated builds
+          CIBW_BEFORE_BUILD: >
+            pip install -r {package}/mypy-requirements.txt
+          # download a copy of clang to use to compile on linux. this was probably built in 2018,
+          # speeds up compilation 2x
+          CIBW_BEFORE_BUILD_LINUX: >
+            (cd / && curl -L https://github.com/mypyc/mypy_mypyc-wheels/releases/download/llvm/llvm-centos-5.tar.gz | tar xzf -) &&
+            pip install -r {package}/mypy-requirements.txt
+
+          # the double negative is counterintuitive, https://github.com/pypa/pip/issues/5735
+          CIBW_ENVIRONMENT: >
+            MYPY_USE_MYPYC=1 MYPYC_OPT_LEVEL=3 PIP_NO_BUILD_ISOLATION=no
+          CIBW_ENVIRONMENT_LINUX: >
+            MYPY_USE_MYPYC=1 MYPYC_OPT_LEVEL=3 PIP_NO_BUILD_ISOLATION=no
+            CC=/opt/llvm/bin/clang
+
+          # lxml is slow to build wheels for new releases, so allow installing reqs to fail
+          # if we failed to install lxml, we'll skip tests, but allow the build to succeed
+          CIBW_BEFORE_TEST: pip install -r {project}/mypy/test-requirements.txt || true
+          # pytest looks for configuration files in the parent directories of where the tests live.
+          # since we are trying to run the tests from their installed location, we copy those into
+          # the venv. Ew ew ew.
+          CIBW_TEST_COMMAND: >
+            ( ! pip list | grep lxml ) || (
+            DIR=$(python -c 'import mypy; from os.path import dirname; print(dirname(dirname(mypy.__path__[0])))')
+            && cp '{project}/mypy/pytest.ini' '{project}/mypy/conftest.py' $DIR
+            && MYPY_TEST_PREFIX='{project}/mypy' pytest $(python -c 'import mypy.test; print(mypy.test.__path__[0])')
+            )
+          # i ran into some flaky tests on windows, so only run testcheck. it looks like we
+          # previously didn't run any tests on windows wheels, so this is a net win.
+          CIBW_TEST_COMMAND_WINDOWS: >
+            bash -c "
+            ( ! pip list | grep lxml ) || (
+            DIR=$(python -c 'import mypy; from os.path import dirname; print(dirname(dirname(mypy.__path__[0])))')
+            && cp '{project}/mypy/pytest.ini' '{project}/mypy/conftest.py' $DIR
+            && MYPY_TEST_PREFIX='{project}/mypy' pytest $(python -c 'import mypy.test; print(mypy.test.__path__[0])')/testcheck.py
+            )
+            "
+        run: |
+          python -m cibuildwheel --output-dir wheelhouse mypy
+      - uses: actions/upload-artifact@v2
+        with:
+          name: dist
+          path: ./wheelhouse/*.whl
+  build_sdist_python_wheel:
+    name: sdist and python wheel
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - uses: actions/setup-python@v2
+        name: Install Python
+        with:
+          python-version: "3.7"
+      - name: Checkout mypy
+        shell: bash
+        run: |
+          COMMIT=$(cat mypy_commit)
+          git clone --recurse-submodules https://github.com/python/mypy.git
+          cd mypy
+          git checkout --force --recurse-submodules $COMMIT
+      - name: Build sdist and wheel
+        run: |
+          cd mypy
+          pip install --upgrade setuptools pip wheel
+          python setup.py sdist bdist_wheel
+      - uses: actions/upload-artifact@v2
+        with:
+          name: dist
+          path: |
+            mypy/dist/*.whl
+            mypy/dist/*.tar.gz
+  release:
+    name: create release
+    needs: [build_wheels, build_sdist_python_wheel]
+    runs-on: ubuntu-latest
+    steps:
+      - name: Download artifact
+        uses: actions/download-artifact@v2
+        with:
+          name: dist
+          path: dist
+      - name: Release
+        # https://github.com/actions/upload-release-asset/issues/47
+        uses: actions/github-script@v2
+        with:
+          github-token: ${{secrets.GITHUB_TOKEN}}
+          script: |
+            const fs = require('fs').promises;
+            const { repo: { owner, repo }, sha } = context;
+
+            console.log('environment', process.versions);
+            console.log({ owner, repo, sha });
+
+            const release = await github.repos.createRelease({
+              owner, repo,
+              // if GITHUB_REF just appears to be a branch, use tag-{commit} as the tag
+              tag_name: process.env.GITHUB_REF.includes("refs/heads/") ? "tag-" + sha : process.env.GITHUB_REF,
+              target_commitish: sha
+            });
+
+            console.log('created release', { release });
+
+            for (let file of await fs.readdir('dist')) {
+              console.log('uploading', file);
+
+              await github.repos.uploadReleaseAsset({
+                owner, repo,
+                release_id: release.data.id,
+                name: file,
+                data: await fs.readFile(`./dist/${file}`)
+              });
+            }
diff --git a/.gitmodules b/.gitmodules
deleted file mode 100644
index 7a31be30..00000000
--- a/.gitmodules
+++ /dev/null
@@ -1,6 +0,0 @@
-[submodule "multibuild"]
-	path = multibuild
-	url = https://github.com/matthew-brett/multibuild
-[submodule "mypy"]
-	path = mypy
-	url = https://github.com/python/mypy
diff --git a/.travis.yml b/.travis.yml
deleted file mode 100644
index a5a2ff1f..00000000
--- a/.travis.yml
+++ /dev/null
@@ -1,87 +0,0 @@
-if: tag IS present
-
-language: generic
-sudo: required
-
-env:
-  global:
-    - DOCKER_IMAGE=quay.io/pypa/manylinux1_x86_64
-
-matrix:
-  include:
-    - os: linux
-      services:
-        - docker
-      env: MB_PYTHON_VERSION=3.5
-    - os: linux
-      services:
-        - docker
-      env: MB_PYTHON_VERSION=3.6
-    - os: linux
-      services:
-        - docker
-      env: MB_PYTHON_VERSION=3.7
-    - os: linux
-      services:
-        - docker
-      env: MB_PYTHON_VERSION=3.8
-    - os: osx
-      env:
-        - MB_PYTHON_VERSION=3.5
-    - os: osx
-      env:
-        - MB_PYTHON_VERSION=3.6
-        - MB_PYTHON_OSX_VER=10.9
-    - os: osx
-      env:
-        - MB_PYTHON_VERSION=3.7
-        - MB_PYTHON_OSX_VER=10.9
-    - os: osx
-      env:
-        - MB_PYTHON_VERSION=3.8
-        - MB_PYTHON_OSX_VER=10.9
-        - MB_ML_VER=2010
-
-before_install: |
-  set -e
-  if [[ "$TRAVIS_OS_NAME" == "linux" ]]; then
-    docker pull $DOCKER_IMAGE
-  elif [[ "$TRAVIS_OS_NAME" == "osx" ]]; then
-    source multibuild/common_utils.sh
-    source multibuild/travis_steps.sh
-    before_install
-  fi
-
-
-install: |
-  # Output something every 10 minutes or Travis kills the job
-  while sleep 9m; do echo "=====[ $SECONDS seconds still running ]====="; done &
-  bg_pid=$!
-  if [[ "$TRAVIS_OS_NAME" == "linux" ]]; then
-    docker run --rm -v `pwd`:/io $DOCKER_IMAGE $PRE_CMD /io/build-linux-wheels.sh $MB_PYTHON_VERSION
-  elif [[ "$TRAVIS_OS_NAME" == "osx" ]]; then
-    export MYPY_USE_MYPYC=1
-    export MYPYC_OPT_LEVEL=3
-    # Note: The following functions are defined by multibuild
-    clean_code mypy HEAD
-    build_bdist_wheel mypy x86_64
-    (cd mypy && ./misc/test_installed_version.sh ../wheelhouse/*.whl)
-  fi
-  kill $bg_pid
-
-notifications:
-  email:
-    recipients:
-      - sully@msully.net
-    on_success: never # default: change
-    on_failure: always # default: always
-
-deploy:
-  provider: releases
-  api_key:
-    secure: KM6lZpvVJ36DNXgCOrnN9CrD+e2Ed9PkR8Ij8aLOzMKiq7+blhWRyA4ZEuF0N0zTiN29nmpKBsC1Qz4II/kZSAFrJQKf32ef67ypQ0XFCJYw4rqgsOcuZqfBf+ZqHVv3IWU3Kb33GoOysG945TyFQ3JFEeN0hWRUtVm8Tav0ths9Ut2+GyOt/jGn+qRcG6xqqt069f65eXC+sGSwcuVlqb1x2kLT31AKOQzMMTS8rN3zFhfGn/Pl/MLobS0DTfm0omWac5WnFW1Bib31zFzbOm19j6PmFc1kNwhpZWCY1yGn8B7QEx/Eq1VKPTu2FC5nQOkBpQILUotDbNfweHcX2w81z2bEkhCOIvu9lVVxOHFaEyDG2a6l9BlgtZuzeLM2loZogjHPHeO7mG3rzj9mq8kRWEIm+xq1xRaSCFDAS9He9iD+Pq5xJ+7yAKwXNcoQoeaLqRtwCgQABpb5RxTwiMUKy9+L8zXNuDj8sMDDmJbfpbegtsb/28oDi1ToZjaCBdKhQ0BjvrUeCmM1bI/4Rb1OkolV/9rx/YaRYACdUw6+WvMQRr9imJXPh7iIfJB0Mz4O/Ptt80jbd6FCXhz1eqR3svhQlFUi5WMytyXz0zCEPKV0vh/F+w6JcxGezeaC9QAS1ei3bG7xU2XztcWSdPrbYlLFe2WBlXRgHtKPi60=
-  file_glob: true
-  file: wheelhouse/*.whl
-  skip_cleanup: true
-  on:
-    tags: true
diff --git a/README.md b/README.md
index 6d1aa9bc..69b7a176 100644
--- a/README.md
+++ b/README.md
@@ -1,7 +1,4 @@
 # mypy_mypyc-wheels
 Automated building and storage of mypyc-compiled mypy binaries
 
-- AppVeyor:
-  [![AppVeyor build status](https://ci.appveyor.com/api/projects/status/github/mypyc/mypy_mypyc-wheels?svg=true)](https://ci.appveyor.com/project/mypyc/mypy-mypyc-wheels)
-- Travis-CI:
-  [![Travis-CI build status](https://api.travis-ci.org/mypyc/mypy_mypyc-wheels.svg)](https://travis-ci.org/mypyc/mypy_mypyc-wheels)
+![Build wheels status](https://github.com/mypyc/mypy_mypyc-wheels/workflows/Build%20wheels/badge.svg)
diff --git a/appveyor.yml b/appveyor.yml
deleted file mode 100644
index 00e767f1..00000000
--- a/appveyor.yml
+++ /dev/null
@@ -1,61 +0,0 @@
-skip_non_tags: true
-
-cache:
-  - '%LOCALAPPDATA%\pip\Cache'
-
-image: Visual Studio 2017
-
-environment:
-  matrix:
-    - PYTHON: "C:\\Python38-x64"
-      PYTHON_VERSION: "3.8.x"
-      PYTHON_ARCH: "64"
-    - PYTHON: "C:\\Python37-x64"
-      PYTHON_VERSION: "3.7.x"
-      PYTHON_ARCH: "64"
-    - PYTHON: "C:\\Python36-x64"
-      PYTHON_VERSION: "3.6.x"
-      PYTHON_ARCH: "64"
-    - PYTHON: "C:\\Python35-x64"
-      PYTHON_VERSION: "3.5.x"
-      PYTHON_ARCH: "64"
-
-install:
-    - "git submodule update --init --recursive"
-    - ps: .\install_python.ps1
-    - "%PYTHON%\\python.exe -m pip install -U pip setuptools wheel virtualenv"
-    - "%PYTHON%\\python.exe -m pip install -r mypy/mypy-requirements.txt"
-
-build: off
-
-test_script:
-    - systeminfo
-    - wmic cpu get SocketDesignation, NumberOfCores, NumberOfLogicalProcessors /Format:List
-    # distutils wants to use the 32-bit cross-compiler since 64-bit might not be
-    # present in "lighter-weight installs". We arrange to always use the 64-bit
-    # version because the 32-bit linker runs out of memory linking mypy.
-    - call "C:\Program Files (x86)\Microsoft Visual Studio\2017\Community\VC\Auxiliary\Build\vcvars64.bat"
-    - "SET DISTUTILS_USE_SDK=1"
-    - "SET MYPYC_OPT_LEVEL=2"
-    - "cd mypy"
-    - "%PYTHON%\\python.exe setup.py --use-mypyc bdist_wheel"
-    - "cd .."
-
-artifacts:
-  # bdist_wheel puts your built wheel in the dist directory
-- path: mypy\dist\*
-  name: wheel
-
-deploy:
-  - provider: GitHub
-    release: $(appveyor_repo_tag_name)
-    artifact:  wheel
-    auth_token:
-        secure: MUA4J5iguzBLtqnbxWLb1kDb2WHZFm7r9tNw1AfwqUB3BhFqp8X89g5MALUY1rvw
-    on:
-     appveyor_repo_tag: true       # deploy on tag push only
-
-#on_success:
-#  You can use this step to upload your artifacts to a public website.
-#  See Appveyor's documentation for more details. Or you can simply
-#  access your wheels from the Appveyor "artifacts" tab for your build.
diff --git a/build-linux-wheels.sh b/build-linux-wheels.sh
deleted file mode 100755
index de910584..00000000
--- a/build-linux-wheels.sh
+++ /dev/null
@@ -1,33 +0,0 @@
-#!/bin/bash -eux
-
-# Unpack a modern clang version
-(cd / && curl -L https://github.com/mypyc/mypy_mypyc-wheels/releases/download/llvm/llvm-centos-5.tar.gz | tar xzf -)
-
-cd /io/mypy
-
-VER="${1//.}"
-TAG="m"
-if [[ $VER -ge 38 ]]; then
-    TAG=""
-fi
-PYBIN="/opt/python/cp${VER}-cp${VER}${TAG}/bin"
-
-# Install mypyc
-"${PYBIN}/pip3" install -r mypy-requirements.txt
-
-# Compile wheels
-CC=/opt/llvm/bin/clang MYPYC_OPT_LEVEL=3 "${PYBIN}/python3" setup.py --use-mypyc bdist_wheel
-
-# Bundle external shared libraries into the wheels
-for whl in dist/*.whl; do
-    auditwheel repair "$whl" -w /io/wheelhouse/
-done
-
-"${PYBIN}/pip3" install virtualenv
-
-# FIXME: For now skip running tests on 3.8 because there are issues
-# installing lxml. Really we shouldn't run the tests in manylinux
-# anyway but...
-if [[ $VER != 38 ]]; then
-    ./misc/test_installed_version.sh /io/wheelhouse/*.whl "${PYBIN}/python"
-fi
diff --git a/config.sh b/config.sh
deleted file mode 100644
index 43ade91d..00000000
--- a/config.sh
+++ /dev/null
@@ -1,8 +0,0 @@
-function pre_build {
-    pip install virtualenv
-    pip install -r mypy/mypy-requirements.txt
-}
-
-function run_tests {
-    true
-}
diff --git a/install_python.ps1 b/install_python.ps1
deleted file mode 100644
index 5307eb63..00000000
--- a/install_python.ps1
+++ /dev/null
@@ -1,37 +0,0 @@
-# This comes from the multibuild project (https://github.com/matthew-brett/multibuild)
-# which is Copyright (c) 2013-2019, Matt Terry and Matthew Brett; all rights reserved.
-# and distributed under a 2-clause BSD license.
-
-# Install specified Python version.
-# Install only if:
-#    Our current matrix entry uses this Python version AND
-#    Python version is not already available.
-
-$py_exe = "${env:PYTHON}\Python.exe"
-if ( [System.IO.File]::Exists($py_exe) ) {
-    exit 0
-}
-$req_nodot = $env:PYTHON -replace '\D+Python(\d+(?:rc\d+)?)(?:-x64)?','$1'
-$req_ver = $req_nodot -replace '(\d)(\d+)','$1.$2.0'
-$req_dir = $req_nodot -replace '(\d)(\d+)(.*)','$1.$2.0'
-
-if ($env:PYTHON -eq "C:\Python${req_nodot}-x64") {
-    $exe_suffix="-amd64"
-} elseif ($env:PYTHON -eq "C:\Python${req_nodot}") {
-    $exe_suffix=""
-} else {
-    exit 0
-}
-
-$py_url = "https://www.python.org/ftp/python"
-Write-Host "Installing Python ${req_ver}$exe_suffix..." -ForegroundColor Cyan
-$exePath = "$env:TEMP\python-${req_ver}${exe_suffix}.exe"
-$downloadFile = "$py_url/${req_dir}/python-${req_ver}${exe_suffix}.exe"
-
-Write-Host "Downloading $downloadFile..."
-(New-Object Net.WebClient).DownloadFile($downloadFile, $exePath)
-Write-Host "Installing..."
-cmd /c start /wait $exePath /quiet TargetDir="$env:PYTHON" Shortcuts=0 Include_launcher=0 InstallLauncherAllUsers=0
-Write-Host "Python ${req_ver} installed to $env:PYTHON"
-
-echo "$(& $py_exe --version 2> $null)"
diff --git a/multibuild b/multibuild
deleted file mode 160000
index dc357469..00000000
--- a/multibuild
+++ /dev/null
@@ -1 +0,0 @@
-Subproject commit dc357469cc8477b8cf4ef407b06eccb58fa8dde0
diff --git a/mypy b/mypy
deleted file mode 160000
index 97d25cfb..00000000
--- a/mypy
+++ /dev/null
@@ -1 +0,0 @@
-Subproject commit 97d25cfb31132e9f9ce84056ecd51084ba7a7d97
diff --git a/mypy_commit b/mypy_commit
new file mode 100644
index 00000000..0686cb52
--- /dev/null
+++ b/mypy_commit
@@ -0,0 +1 @@
+ffed88fb95fcbfdd1363f0f719bd3e13f8fe20e9