From c92150d1771ba0f9be4299db251e695b437ff0fd Mon Sep 17 00:00:00 2001 From: Shane Harvey Date: Mon, 27 Jul 2020 17:20:43 -0700 Subject: [PATCH] PYTHON-1631 Automate release wheels for Windows and manylinux (#473) --- .evergreen/build-mac.sh | 12 +++++++ .evergreen/build-manylinux-internal.sh | 25 ++++++++++++++ .evergreen/build-manylinux.sh | 40 ++++++++++++++++++++++ .evergreen/build-windows.sh | 18 ++++++++++ .evergreen/config.yml | 47 ++++++++++++++++++++++++-- .evergreen/release.sh | 9 +++++ 6 files changed, 149 insertions(+), 2 deletions(-) create mode 100755 .evergreen/build-mac.sh create mode 100755 .evergreen/build-manylinux-internal.sh create mode 100755 .evergreen/build-manylinux.sh create mode 100755 .evergreen/build-windows.sh create mode 100755 .evergreen/release.sh diff --git a/.evergreen/build-mac.sh b/.evergreen/build-mac.sh new file mode 100755 index 0000000000..95be59c340 --- /dev/null +++ b/.evergreen/build-mac.sh @@ -0,0 +1,12 @@ +#!/bin/bash -ex + +for VERSION in 2.7 3.4 3.5 3.6 3.7 3.8; do + if [[ $VERSION == "2.7" ]]; then + rm -rf build + python$VERSION setup.py bdist_egg + fi + rm -rf build + python$VERSION setup.py bdist_wheel +done + +ls dist diff --git a/.evergreen/build-manylinux-internal.sh b/.evergreen/build-manylinux-internal.sh new file mode 100755 index 0000000000..c810700a29 --- /dev/null +++ b/.evergreen/build-manylinux-internal.sh @@ -0,0 +1,25 @@ +#!/bin/bash -ex +cd /pymongo + +# Compile wheels +for PYBIN in /opt/python/*/bin; do + # Skip Python 3.3 and 3.9. + if [[ "$PYBIN" == *"cp33"* || "$PYBIN" == *"cp39"* ]]; then + continue + fi + # https://github.com/pypa/manylinux/issues/49 + rm -rf build + ${PYBIN}/python setup.py bdist_wheel +done + +# https://github.com/pypa/manylinux/issues/49 +rm -rf build + +# Audit wheels and write multilinux1 tag +for whl in dist/*.whl; do + # Skip already built manylinux1 wheels. + if [[ "$whl" != *"manylinux"* ]]; then + auditwheel repair $whl -w dist + rm $whl + fi +done diff --git a/.evergreen/build-manylinux.sh b/.evergreen/build-manylinux.sh new file mode 100755 index 0000000000..16b05dbea4 --- /dev/null +++ b/.evergreen/build-manylinux.sh @@ -0,0 +1,40 @@ +#!/bin/bash -ex + +docker version + +# 2020-03-20-2fda31c Was the last release to include Python 3.4. +images=(quay.io/pypa/manylinux1_x86_64:2020-03-20-2fda31c \ + quay.io/pypa/manylinux1_i686:2020-03-20-2fda31c \ + quay.io/pypa/manylinux1_x86_64 \ + quay.io/pypa/manylinux1_i686 \ + quay.io/pypa/manylinux2014_x86_64 \ + quay.io/pypa/manylinux2014_i686) +# aarch64/ppc64le/s390x work on macOS locally but not on linux in evergreen: +# [2020/07/23 00:24:00.482] + docker run --rm -v /data/mci/cd100cec6341abda533450fb3f2fab99/src:/pymongo quay.io/pypa/manylinux2014_aarch64 /pymongo/.evergreen/build-manylinux-internal.sh +# [2020/07/23 00:24:01.186] standard_init_linux.go:211: exec user process caused "exec format error" +# +# Could be related to: +# https://github.com/pypa/manylinux/issues/410 +# quay.io/pypa/manylinux2014_aarch64 \ +# quay.io/pypa/manylinux2014_ppc64le \ +# quay.io/pypa/manylinux2014_s390x) + +for image in "${images[@]}"; do + docker pull $image + docker run --rm -v `pwd`:/pymongo $image /pymongo/.evergreen/build-manylinux-internal.sh +done + +ls dist + +# Check for any unexpected files. +unexpected=$(find dist \! \( -iname dist -or \ + -iname '*cp27*' -or \ + -iname '*cp34*' -or \ + -iname '*cp35*' -or \ + -iname '*cp36*' -or \ + -iname '*cp37*' -or \ + -iname '*cp38*' \)) +if [ -n "$unexpected" ]; then + echo "Unexpected files:" $unexpected + exit 1 +fi diff --git a/.evergreen/build-windows.sh b/.evergreen/build-windows.sh new file mode 100755 index 0000000000..a34506f551 --- /dev/null +++ b/.evergreen/build-windows.sh @@ -0,0 +1,18 @@ +#!/bin/bash -ex + +for VERSION in 27 34 35 36 37 38; do + PYTHON=C:/Python/Python${VERSION}/python.exe + PYTHON32=C:/Python/32/Python${VERSION}/python.exe + if [[ $VERSION == "2.7" ]]; then + rm -rf build + $PYTHON setup.py bdist_egg + rm -rf build + $PYTHON32 setup.py bdist_egg + fi + rm -rf build + $PYTHON setup.py bdist_wheel + rm -rf build + $PYTHON32 setup.py bdist_wheel +done + +ls dist diff --git a/.evergreen/config.yml b/.evergreen/config.yml index ac5e4147f9..24cdd26238 100644 --- a/.evergreen/config.yml +++ b/.evergreen/config.yml @@ -756,6 +756,13 @@ functions: -v \ --fault revoked + "teardown_docker": + - command: shell.exec + params: + script: | + # Remove all Docker images + docker rmi -f $(docker images -a -q) &> /dev/null || true + pre: - func: "fetch source" - func: "prepare resources" @@ -773,6 +780,7 @@ post: - func: "upload test results" - func: "stop mongo-orchestration" - func: "cleanup" + - func: "teardown_docker" tasks: @@ -806,6 +814,36 @@ tasks: genhtml --version || true valgrind --version || true + + - name: "release" + tags: ["release"] + git_tag_only: true + commands: + - command: shell.exec + type: test + params: + working_dir: "src" + script: | + set -o xtrace + ${PREPARE_SHELL} + .evergreen/release.sh + - command: archive.targz_pack + params: + target: "release-files.tgz" + source_dir: "src/dist" + include: + - "*" + - command: s3.put + params: + aws_key: ${aws_key} + aws_secret: ${aws_secret} + local_file: release-files.tgz + remote_file: ${UPLOAD_BUCKET}/${build_variant}/${revision}/${version_id}/${build_id}/release/${task_id}-${execution}-release-files.tar.gz + bucket: mciuploads + permissions: public-read + content_type: ${content_type|application/gzip} + display_name: Release files + # Standard test tasks {{{ - name: "mockupdb" @@ -2065,8 +2103,6 @@ buildvariants: - ".latest" - ".4.4" - ".4.2" - variables: - set_xtrace_on: on - matrix_name: "tests-python-version-rhel62-test-ssl" matrix_spec: @@ -2519,6 +2555,13 @@ buildvariants: - name: "aws-auth-test-4.4" - name: "aws-auth-test-latest" +- matrix_name: "Release" + matrix_spec: + platform: [ubuntu-18.04, windows-64-vsMulti-small] + display_name: "Release ${platform}" + tasks: + - name: "release" + # Platform notes # i386 builds of OpenSSL or Cyrus SASL are not available # Ubuntu16.04 ppc64le is only supported by MongoDB 3.4+ diff --git a/.evergreen/release.sh b/.evergreen/release.sh new file mode 100755 index 0000000000..759786b934 --- /dev/null +++ b/.evergreen/release.sh @@ -0,0 +1,9 @@ +#!/bin/bash -ex + +if [ $(uname -s) = "Darwin" ]; then + .evergreen/build-mac.sh +elif [ "Windows_NT" = "$OS" ]; then # Magic variable in cygwin + .evergreen/build-windows.sh +else + .evergreen/build-manylinux.sh +fi