diff --git a/misc/build_wheel.py b/misc/build_wheel.py new file mode 100644 index 000000000000..13633405bf77 --- /dev/null +++ b/misc/build_wheel.py @@ -0,0 +1,131 @@ +"""Script to build compiled binary wheels that can be uploaded to PyPI. + +The main GitHub workflow where this script is used: +https://github.com/mypyc/mypy_mypyc-wheels/blob/master/.github/workflows/build.yml + +This uses cibuildwheel (https://github.com/joerick/cibuildwheel) to +build the wheels. + +Usage: + + build_wheel_ci.py --python-version \ + --output-dir + +Wheels for the given Python version will be created in the given directory. +Python version is in form "39". + +This works on macOS, Windows and Linux. + +You can test locally by using --extra-opts. macOS example: + + mypy/misc/build_wheel_ci.py --python-version 39 --output-dir out --extra-opts="--platform macos" + +Other supported values for platform: linux, windows +""" + +import argparse +import os +import subprocess +import sys +from typing import Dict + +# Clang package we use on Linux +LLVM_URL = 'https://github.com/mypyc/mypy_mypyc-wheels/releases/download/llvm/llvm-centos-5.tar.gz' + +# Mypy repository root +ROOT_DIR = os.path.dirname(os.path.dirname(__file__)) + + +def create_environ(python_version: str) -> Dict[str, str]: + """Set up environment variables for cibuildwheel.""" + env = os.environ.copy() + + env['CIBW_BUILD'] = "cp{}-*".format(python_version) + + # Don't build 32-bit wheels + env['CIBW_SKIP'] = "*-manylinux_i686 *-win32" + + env['CIBW_BUILD_VERBOSITY'] = '1' + + # mypy's isolated builds don't specify the requirements mypyc needs, so install + # requirements and don't use isolated builds. we need to use build-requirements.txt + # with recent mypy commits to get stub packages needed for compilation. + # + # TODO: remove use of mypy-requirements.txt once we no longer need to support + # building pre modular typeshed releases + env['CIBW_BEFORE_BUILD'] = """ + pip install -r {package}/mypy-requirements.txt && + (pip install -r {package}/build-requirements.txt || true) + """.replace('\n', ' ') + + # download a copy of clang to use to compile on linux. this was probably built in 2018, + # speeds up compilation 2x + env['CIBW_BEFORE_BUILD_LINUX'] = """ + (cd / && curl -L %s | tar xzf -) && + pip install -r {package}/mypy-requirements.txt && + (pip install -r {package}/build-requirements.txt || true) + """.replace('\n', ' ') % LLVM_URL + + # the double negative is counterintuitive, https://github.com/pypa/pip/issues/5735 + env['CIBW_ENVIRONMENT'] = 'MYPY_USE_MYPYC=1 MYPYC_OPT_LEVEL=3 PIP_NO_BUILD_ISOLATION=no' + env['CIBW_ENVIRONMENT_LINUX'] = ( + 'MYPY_USE_MYPYC=1 MYPYC_OPT_LEVEL=3 PIP_NO_BUILD_ISOLATION=no ' + + 'CC=/opt/llvm/bin/clang' + ) + env['CIBW_ENVIRONMENT_WINDOWS'] = ( + 'MYPY_USE_MYPYC=1 MYPYC_OPT_LEVEL=2 PIP_NO_BUILD_ISOLATION=no' + ) + + # lxml is slow to build wheels for new releases, so allow installing reqs to fail + # if we failed to install lxml, we'll skip tests, but allow the build to succeed + env['CIBW_BEFORE_TEST'] = ( + 'pip install -r {project}/mypy/test-requirements.txt || true' + ) + + # pytest looks for configuration files in the parent directories of where the tests live. + # since we are trying to run the tests from their installed location, we copy those into + # the venv. Ew ew ew. + env['CIBW_TEST_COMMAND'] = """ + ( ! pip list | grep lxml ) || ( + DIR=$(python -c 'import mypy, os; dn = os.path.dirname; print(dn(dn(mypy.__path__[0])))') + && TEST_DIR=$(python -c 'import mypy.test; print(mypy.test.__path__[0])') + && cp '{project}/mypy/pytest.ini' '{project}/mypy/conftest.py' $DIR + && MYPY_TEST_PREFIX='{project}/mypy' pytest $TEST_DIR + ) + """.replace('\n', ' ') + + # i ran into some flaky tests on windows, so only run testcheck. it looks like we + # previously didn't run any tests on windows wheels, so this is a net win. + env['CIBW_TEST_COMMAND_WINDOWS'] = """ + bash -c " + ( ! pip list | grep lxml ) || ( + DIR=$(python -c 'import mypy, os; dn = os.path.dirname; print(dn(dn(mypy.__path__[0])))') + && TEST_DIR=$(python -c 'import mypy.test; print(mypy.test.__path__[0])') + && cp '{project}/mypy/pytest.ini' '{project}/mypy/conftest.py' $DIR + && MYPY_TEST_PREFIX='{project}/mypy' pytest $TEST_DIR/testcheck.py + ) + " + """.replace('\n', ' ') + return env + + +def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument('--python-version', required=True, metavar='XY', + help='Python version (e.g. 38 or 39)') + parser.add_argument('--output-dir', required=True, metavar='DIR', + help='Output directory for created wheels') + parser.add_argument('--extra-opts', default='', metavar='OPTIONS', + help='Extra options passed to cibuildwheel verbatim') + args = parser.parse_args() + python_version = args.python_version + output_dir = args.output_dir + extra_opts = args.extra_opts + environ = create_environ(python_version) + script = 'python -m cibuildwheel {} --output-dir {} {}'.format(extra_opts, output_dir, + ROOT_DIR) + subprocess.check_call(script, shell=True, env=environ) + + +if __name__ == '__main__': + main()