From 6d1d4a10e540317d3f60c40d7aecef0df7cc9ea2 Mon Sep 17 00:00:00 2001 From: Tim Paine Date: Thu, 25 Feb 2021 18:16:40 -0500 Subject: [PATCH 1/3] fixes #1334, adds py39 now that brew defaults this on osx --- azure-pipelines.yml | 22 ++++++++++++++++++++++ python/perspective/setup.py | 15 ++++++++------- scripts/_wheel_python.js | 4 ++-- 3 files changed, 32 insertions(+), 9 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 1a88c743bc..6d09e29d5c 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -111,6 +111,16 @@ jobs: python_flag: '--python38' manylinux_flag: '--manylinux2014' artifact_name: 'cp38-cp38-manylinux2014_x86_64' + Python39ManyLinux2010: + python.version: '3.9' + python_flag: '--python39' + manylinux_flag: '--manylinux2010' + artifact_name: 'cp39-cp39-manylinux2010_x86_64' + Python39ManyLinux2014: + python.version: '3.9' + python_flag: '--python39' + manylinux_flag: '--manylinux2014' + artifact_name: 'cp39-cp39-manylinux2014_x86_64' steps: - task: UsePythonVersion@0 @@ -183,6 +193,9 @@ jobs: Python38: python.version: '3.8' python_flag: '--python38' + Python39: + python.version: '3.9' + python_flag: '--python39' steps: - task: UsePythonVersion@0 @@ -229,6 +242,10 @@ jobs: python.version: '3.7' python_flag: '' artifact_name: 'cp37-cp37m-macosx_10_15_x86_64' + Python39: + python.version: '3.9' + python_flag: '' + artifact_name: 'cp39-cp39-macosx_10_15_x86_64' ${{ if or(startsWith(variables['build.sourceBranch'], 'refs/tags/v'), eq(variables['Build.Reason'], 'Schedule')) }}: Python36: @@ -241,6 +258,11 @@ jobs: python_flag: '--python38' artifact_name: 'cp38-cp38-macosx_10_15_x86_64' + Python39: + python.version: '3.9' + python_flag: '--python39' + artifact_name: 'cp39-cp39-macosx_10_15_x86_64' + steps: - task: UsePythonVersion@0 inputs: diff --git a/python/perspective/setup.py b/python/perspective/setup.py index 5a690e2a41..961aeaeccd 100644 --- a/python/perspective/setup.py +++ b/python/perspective/setup.py @@ -6,20 +6,20 @@ # the Apache License 2.0. The full license can be found in the LICENSE file. # from __future__ import print_function -from setuptools import setup, find_packages, Extension -from setuptools.command.build_ext import build_ext -from setuptools.command.sdist import sdist -from distutils.version import LooseVersion -from codecs import open import io import os import os.path -import re import platform -import sys +import re import subprocess +import sys +from codecs import open +from distutils.version import LooseVersion +from setuptools import Extension, find_packages, setup +from setuptools.command.build_ext import build_ext +from setuptools.command.sdist import sdist try: from shutil import which @@ -75,6 +75,7 @@ def which(x): "pytz>=2018.9", "Sphinx>=1.8.4", "sphinx-markdown-builder>=0.5.2", + "wheel", ] + requires requires_dev = [ diff --git a/scripts/_wheel_python.js b/scripts/_wheel_python.js index b6d9616a09..5cac05c2f9 100644 --- a/scripts/_wheel_python.js +++ b/scripts/_wheel_python.js @@ -12,7 +12,7 @@ const fs = require("fs-extra"); const IS_DOCKER = process.env.PSP_DOCKER; const IS_MACOS = getarg("--macos"); const IS_PY2 = getarg("--python2"); -const PYTHON = IS_PY2 ? "python2" : getarg("--python38") ? "python3.8" : getarg("--python36") ? "python3.6" : "python3.7"; +const PYTHON = IS_PY2 ? "python2" : getarg("--python39") ? "python3.9" : getarg("--python38") ? "python3.8" : getarg("--python36") ? "python3.6" : "python3.7"; let IMAGE = "manylinux2014"; let MANYLINUX_VERSION; @@ -64,7 +64,7 @@ try { // These are system deps that may only be in place from pep-517/518 so // lets reinstall them to be sure - cmd += `${PYTHON} -m pip install 'numpy>=1.13.1' && `; + cmd += `${PYTHON} -m pip install -U 'numpy>=1.13.1' wheel twine && `; // remove the build folder so we completely rebuild (and pick up the // libs we just installed above, since this build method won't use From 99722129fb13ec5bbcf2036930cac23c3eeabf84 Mon Sep 17 00:00:00 2001 From: Tim Paine Date: Thu, 25 Feb 2021 18:20:16 -0500 Subject: [PATCH 2/3] update docker images missing flag fix copypasta --- azure-pipelines.yml | 7 +------ docker/python/manylinux2010/Dockerfile | 5 +++++ docker/python/manylinux2014/Dockerfile | 5 +++++ 3 files changed, 11 insertions(+), 6 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 6d09e29d5c..4cb199f5af 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -244,7 +244,7 @@ jobs: artifact_name: 'cp37-cp37m-macosx_10_15_x86_64' Python39: python.version: '3.9' - python_flag: '' + python_flag: '--python39' artifact_name: 'cp39-cp39-macosx_10_15_x86_64' ${{ if or(startsWith(variables['build.sourceBranch'], 'refs/tags/v'), eq(variables['Build.Reason'], 'Schedule')) }}: @@ -258,11 +258,6 @@ jobs: python_flag: '--python38' artifact_name: 'cp38-cp38-macosx_10_15_x86_64' - Python39: - python.version: '3.9' - python_flag: '--python39' - artifact_name: 'cp39-cp39-macosx_10_15_x86_64' - steps: - task: UsePythonVersion@0 inputs: diff --git a/docker/python/manylinux2010/Dockerfile b/docker/python/manylinux2010/Dockerfile index 7c956aabde..2629cf4164 100644 --- a/docker/python/manylinux2010/Dockerfile +++ b/docker/python/manylinux2010/Dockerfile @@ -25,6 +25,7 @@ RUN rm -rf /opt/python/cp27-cp27m/bin/auditwheel RUN rm -rf /opt/python/cp36-cp36m/bin/auditwheel RUN rm -rf /opt/python/cp37-cp37m/bin/auditwheel RUN rm -rf /opt/python/cp38-cp38/bin/auditwheel +RUN rm -rf /opt/python/cp39-cp39/bin/auditwheel # Copy assets @@ -32,6 +33,7 @@ RUN cp -arf /opt/python/cp27-cp27m/* /usr/local/ RUN cp -arf /opt/python/cp36-cp36m/* /usr/local/ RUN cp -arf /opt/python/cp37-cp37m/* /usr/local/ RUN cp -arf /opt/python/cp38-cp38/* /usr/local/ +RUN cp -arf /opt/python/cp39-cp39/* /usr/local/ ENV PATH=/usr/local/bin:$PATH @@ -40,11 +42,13 @@ RUN python2.7 -m pip install numpy scipy pybind11 cython codecov mock pytest pyt RUN python3.6 -m pip install numpy scipy pybind11 cython codecov mock black>=20 flake8-black pytest pytest-cov traitlets ipywidgets faker psutil RUN python3.7 -m pip install numpy scipy pybind11 cython codecov mock black>=20 flake8-black pytest pytest-cov traitlets ipywidgets faker psutil RUN python3.8 -m pip install numpy scipy pybind11 cython codecov mock black>=20 flake8-black pytest pytest-cov traitlets ipywidgets faker psutil +RUN python3.9 -m pip install numpy scipy pybind11 cython codecov mock black>=20 flake8-black pytest pytest-cov traitlets ipywidgets faker psutil # Install Auditwheel - not available on Python 2 RUN python3.6 -m pip install --ignore-installed auditwheel RUN python3.7 -m pip install --ignore-installed auditwheel RUN python3.8 -m pip install --ignore-installed auditwheel +RUN python3.9 -m pip install --ignore-installed auditwheel # install boost RUN wget https://dl.bintray.com/boostorg/release/1.71.0/source/boost_1_71_0.tar.gz >/dev/null 2>&1 @@ -57,6 +61,7 @@ RUN python2.7 -m pip install 'numpy>=1.13.1' 'pandas>=0.22.0' 'pyarrow>=0.16.0' RUN python3.6 -m pip install 'numpy>=1.13.1' 'pandas>=0.22.0' 'pyarrow>=2.0.0' RUN python3.7 -m pip install 'numpy>=1.13.1' 'pandas>=0.22.0' 'pyarrow>=2.0.0' RUN python3.8 -m pip install 'numpy>=1.13.1' 'pandas>=0.22.0' 'pyarrow>=2.0.0' +RUN python3.9 -m pip install 'numpy>=1.13.1' 'pandas>=0.22.0' 'pyarrow>=2.0.0' # install node RUN curl -sL https://rpm.nodesource.com/setup_10.x | sudo bash - diff --git a/docker/python/manylinux2014/Dockerfile b/docker/python/manylinux2014/Dockerfile index 7ec5b1f843..9745441e0b 100644 --- a/docker/python/manylinux2014/Dockerfile +++ b/docker/python/manylinux2014/Dockerfile @@ -24,12 +24,14 @@ RUN ln -s /opt/cmake/bin/ctest /usr/local/bin/ctest RUN rm -rf /opt/python/cp36-cp36m/bin/auditwheel RUN rm -rf /opt/python/cp37-cp37m/bin/auditwheel RUN rm -rf /opt/python/cp38-cp38/bin/auditwheel +RUN rm -rf /opt/python/cp39-cp39/bin/auditwheel # Copy assets RUN cp -arf /opt/python/cp36-cp36m/* /usr/local/ RUN cp -arf /opt/python/cp37-cp37m/* /usr/local/ RUN cp -arf /opt/python/cp38-cp38/* /usr/local/ +RUN cp -arf /opt/python/cp39-cp39/* /usr/local/ ENV PATH=/usr/local/bin:$PATH @@ -37,11 +39,13 @@ ENV PATH=/usr/local/bin:$PATH RUN python3.6 -m pip install numpy scipy pybind11 cython codecov mock "black==20.8b1" flake8-black pytest pytest-cov traitlets ipywidgets faker psutil RUN python3.7 -m pip install numpy scipy pybind11 cython codecov mock "black==20.8b1" flake8-black pytest pytest-cov traitlets ipywidgets faker psutil RUN python3.8 -m pip install numpy scipy pybind11 cython codecov mock "black==20.8b1" flake8-black pytest pytest-cov traitlets ipywidgets faker psutil +RUN python3.9 -m pip install numpy scipy pybind11 cython codecov mock "black==20.8b1" flake8-black pytest pytest-cov traitlets ipywidgets faker psutil # Install Auditwheel - not available on Python 2 RUN python3.6 -m pip install --ignore-installed auditwheel RUN python3.7 -m pip install --ignore-installed auditwheel RUN python3.8 -m pip install --ignore-installed auditwheel +RUN python3.9 -m pip install --ignore-installed auditwheel # install boost RUN wget https://dl.bintray.com/boostorg/release/1.71.0/source/boost_1_71_0.tar.gz >/dev/null 2>&1 @@ -53,6 +57,7 @@ RUN cd boost_1_71_0 && ./b2 -j8 --with-program_options --with-filesystem --with- RUN python3.6 -m pip install 'numpy>=1.13.1' 'pandas>=0.22.0' 'pyarrow>=2.0.0' RUN python3.7 -m pip install 'numpy>=1.13.1' 'pandas>=0.22.0' 'pyarrow>=2.0.0' RUN python3.8 -m pip install 'numpy>=1.13.1' 'pandas>=0.22.0' 'pyarrow>=2.0.0' +RUN python3.9 -m pip install 'numpy>=1.13.1' 'pandas>=0.22.0' 'pyarrow>=2.0.0' # install node RUN curl -sL https://rpm.nodesource.com/setup_10.x | sudo bash - From fbe4362324da54ecb7ad9cc590f23f254e874fae Mon Sep 17 00:00:00 2001 From: Andrew Stein Date: Fri, 2 Apr 2021 00:04:31 -0400 Subject: [PATCH 3/3] Fix docker images --- azure-pipelines.yml | 10 +++++----- docker/python/manylinux2010/Dockerfile | 7 +++++-- docker/python/manylinux2014/Dockerfile | 6 ++++-- 3 files changed, 14 insertions(+), 9 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 4cb199f5af..11483ee59e 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -73,11 +73,11 @@ jobs: strategy: matrix: - Python27ManyLinux2010: - python.version: '2.7' - python_flag: '--python2' - manylinux_flag: '--manylinux2010' - artifact_name: 'cp27-cp27m-manylinux2010_x86_64' + # Python27ManyLinux2010: + # python.version: '2.7' + # python_flag: '--python2' + # manylinux_flag: '--manylinux2010' + # artifact_name: 'cp27-cp27m-manylinux2010_x86_64' Python37ManyLinux2010: python.version: '3.7' python_flag: '' diff --git a/docker/python/manylinux2010/Dockerfile b/docker/python/manylinux2010/Dockerfile index 2629cf4164..d972fc0294 100644 --- a/docker/python/manylinux2010/Dockerfile +++ b/docker/python/manylinux2010/Dockerfile @@ -6,7 +6,8 @@ # the Apache License 2.0. The full license can be found in the LICENSE file. # -FROM quay.io/pypa/manylinux2010_x86_64 +# https://github.com/pypa/manylinux/issues/428#issuecomment-774729597 +FROM quay.io/pypa/manylinux2010_x86_64:2021-02-06-3d322a5 RUN yum -y install rapidjson-devel sudo wget # Build new cmake @@ -14,6 +15,8 @@ RUN wget https://cmake.org/files/v3.15/cmake-3.15.4-Linux-x86_64.sh -q RUN mkdir /opt/cmake RUN printf "y\nn\n" | sh cmake-3.15.4-Linux-x86_64.sh --prefix=/opt/cmake > /dev/null RUN rm -fr cmake*.sh /opt/cmake/doc +RUN rm -fr /usr/local/bin/cmake +RUN rm -fr /usr/local/bin/ctest RUN rm -fr /opt/cmake/bin/cmake-gui RUN rm -fr /opt/cmake/bin/ccmake RUN rm -fr /opt/cmake/bin/cpack @@ -51,7 +54,7 @@ RUN python3.8 -m pip install --ignore-installed auditwheel RUN python3.9 -m pip install --ignore-installed auditwheel # install boost -RUN wget https://dl.bintray.com/boostorg/release/1.71.0/source/boost_1_71_0.tar.gz >/dev/null 2>&1 +RUN wget https://dl.bintray.com/boostorg/release/1.71.0/source/boost_1_71_0.tar.gz --no-check-certificate >/dev/null 2>&1 RUN tar xfz boost_1_71_0.tar.gz # https://github.com/boostorg/build/issues/468 RUN cd boost_1_71_0 && ./bootstrap.sh diff --git a/docker/python/manylinux2014/Dockerfile b/docker/python/manylinux2014/Dockerfile index 9745441e0b..0ed87fd11a 100644 --- a/docker/python/manylinux2014/Dockerfile +++ b/docker/python/manylinux2014/Dockerfile @@ -14,6 +14,8 @@ RUN wget https://cmake.org/files/v3.15/cmake-3.15.4-Linux-x86_64.sh -q RUN mkdir /opt/cmake RUN printf "y\nn\n" | sh cmake-3.15.4-Linux-x86_64.sh --prefix=/opt/cmake > /dev/null RUN rm -fr cmake*.sh /opt/cmake/doc +RUN rm -fr /usr/local/bin/cmake +RUN rm -fr /usr/local/bin/ctest RUN rm -fr /opt/cmake/bin/cmake-gui RUN rm -fr /opt/cmake/bin/ccmake RUN rm -fr /opt/cmake/bin/cpack @@ -48,7 +50,7 @@ RUN python3.8 -m pip install --ignore-installed auditwheel RUN python3.9 -m pip install --ignore-installed auditwheel # install boost -RUN wget https://dl.bintray.com/boostorg/release/1.71.0/source/boost_1_71_0.tar.gz >/dev/null 2>&1 +RUN wget https://dl.bintray.com/boostorg/release/1.71.0/source/boost_1_71_0.tar.gz --no-check-certificate >/dev/null 2>&1 RUN tar xfz boost_1_71_0.tar.gz # https://github.com/boostorg/build/issues/468 RUN cd boost_1_71_0 && ./bootstrap.sh @@ -60,7 +62,7 @@ RUN python3.8 -m pip install 'numpy>=1.13.1' 'pandas>=0.22.0' 'pyarrow>=2.0.0' RUN python3.9 -m pip install 'numpy>=1.13.1' 'pandas>=0.22.0' 'pyarrow>=2.0.0' # install node -RUN curl -sL https://rpm.nodesource.com/setup_10.x | sudo bash - +RUN curl -sL https://rpm.nodesource.com/setup_14.x | sudo bash - RUN yum install -y nodejs RUN npm install --global yarn