finos · texodus · Oct 4, 2020
diff --git a/azure-pipelines.yml b/azure-pipelines.yml
@@ -106,11 +106,12 @@ jobs:
     - bash: yarn
       displayName: 'Install Deps'
 
-    - bash: python -m pip install -U "black==20.8b1" flake8-black pip
+    - script: |
+        python -m pip install -U "black==20.8b1" flake8-black pip
+      displayName: "Python3 deps"
       condition: and(succeeded(), ne(variables['python.version'], '2.7'))
-      displayName: 'Install Python deps'
 
-    - bash: yarn build_python  --ci $(python_flag) $(manylinux_flag)
+    - bash: yarn build_python --ci $(python_flag) $(manylinux_flag)
       displayName: 'build'
       env:
         PSP_DOCKER: 1
@@ -254,8 +255,14 @@ jobs:
       displayName: "Which python"
 
     - script: |
-        python -m pip install -U delocate wheel numpy "pyarrow>=0.16.0,<1" pip
-      displayName: "Python deps"
+        python -m pip install -U delocate wheel numpy "pyarrow>=1.0.1,<2"
+      displayName: "Python3 deps"
+      condition: not(eq(variables['python.version'], '2.7'))
+
+    - script: |
+        python -m pip install -U delocate wheel numpy "pyarrow<2"
+      displayName: "Python2 deps"
+      condition: eq(variables['python.version'], '2.7')
 
     - script: npm install -g yarn
       displayName: "Install Yarn"

diff --git a/cpp/perspective/CMakeLists.txt b/cpp/perspective/CMakeLists.txt
@@ -645,10 +645,15 @@ elseif(PSP_CPP_BUILD OR PSP_PYTHON_BUILD)
 			# `site-packages`, the relative search path should be able to pick
             # up pyarrow. This is only enabled for *nix (non-manylinux), as `auditwheel`
 			# will not delocate libarrow properly if it is in the rpath.
-            set_property(TARGET psp PROPERTY INSTALL_RPATH ${CMAKE_INSTALL_RPATH} ${module_origin_path} ${module_origin_path}../../pyarrow/ ${PYTHON_PYARROW_LIBRARY_DIR} ${PSP_PYTHON_ARROWINSTALLDIR})
-            set_property(TARGET binding PROPERTY INSTALL_RPATH ${CMAKE_INSTALL_RPATH} ${module_origin_path} ${module_origin_path}../../pyarrow/ ${PYTHON_PYARROW_LIBRARY_DIR} ${PSP_PYTHON_ARROWINSTALLDIR})
+			#
+			# We also include the install-time path to pyarrow so that pep-518
+			# can build properly
+            set_property(TARGET psp PROPERTY INSTALL_RPATH ${CMAKE_INSTALL_RPATH} ${module_origin_path} ${module_origin_path}../../pyarrow ${PYTHON_PYARROW_LIBRARY_DIR} ${PSP_PYTHON_ARROWINSTALLDIR})
+            set_property(TARGET binding PROPERTY INSTALL_RPATH ${CMAKE_INSTALL_RPATH} ${module_origin_path} ${module_origin_path}../../pyarrow ${PYTHON_PYARROW_LIBRARY_DIR} ${PSP_PYTHON_ARROWINSTALLDIR})
 		else()
 			target_compile_options(binding PRIVATE -Wdeprecated-declarations)
+			target_compile_options(binding PRIVATE -Wno-deprecated-register)
+			target_compile_options(psp PRIVATE -Wno-deprecated-register)
 		endif()
 
 		target_link_libraries(psp ${PYTHON_PYARROW_LIBRARIES})
@@ -724,4 +729,4 @@ if(PSP_BUILD_DOCS)
 	# add_dependencies(docs-html doxygen)
 
 endif()
-##########
+##########
diff --git a/docker/python/manylinux2010/Dockerfile b/docker/python/manylinux2010/Dockerfile
@@ -1,4 +1,4 @@
-# *****************************************************************************
+###############################################################################
 #
 # Copyright (c) 2020, the Perspective Authors.
 #
@@ -26,7 +26,6 @@ RUN rm -rf /opt/python/cp36-cp36m/bin/auditwheel
 RUN rm -rf /opt/python/cp37-cp37m/bin/auditwheel
 RUN rm -rf /opt/python/cp38-cp38/bin/auditwheel
 
-
 # Copy assets
 RUN cp -arf /opt/python/cp27-cp27m/* /usr/local/
 RUN cp -arf /opt/python/cp36-cp36m/* /usr/local/
@@ -53,18 +52,10 @@ RUN tar xfz boost_1_71_0.tar.gz
 RUN cd boost_1_71_0 && ./bootstrap.sh
 RUN cd boost_1_71_0 && ./b2 -j8 --with-program_options --with-filesystem --with-system install 
 
-RUN python2.7 -m pip install 'numpy>=1.13.1' 'pandas>=0.22.0' git+https://chromium.googlesource.com/external/gyp
-RUN python3.6 -m pip install 'numpy>=1.13.1' 'pandas>=0.22.0'
-RUN python3.7 -m pip install 'numpy>=1.13.1' 'pandas>=0.22.0'
-RUN python3.8 -m pip install 'numpy>=1.13.1' 'pandas>=0.22.0'
-
-# install pyarrow
-RUN wget https://github.com/apache/arrow/archive/apache-arrow-0.16.0.tar.gz >/dev/null 2>&1  || echo "wget arrow failed"
-RUN tar xfz apache-arrow-0.16.0.tar.gz
-RUN cd arrow-apache-arrow-0.16.0 && mkdir build && cd build && cmake ../cpp/ -DPython_ADDITIONAL_VERSIONS=2.7 -DARROW_RPATH_ORIGIN=ON -DARROW_PYTHON=ON -DARROW_FLIGHT=OFF -DARROW_IPC=ON -DARROW_COMPUTE=ON -DCMAKE_INSTALL_PREFIX=/usr -DARROW_DATASET=OFF -DARROW_BUILD_UTILITIES=OFF -DARROW_JEMALLOC=OFF -DARROW_DEPENDENCY_SOURCE=BUNDLED -DARROW_HDFS=OFF -DARROW_WITH_BACKTRACE=OFF -DARROW_WITH_BROTLI=OFF -DARROW_WITH_BZ2=OFF -DARROW_WITH_LZ4=OFF -DARROW_WITH_SNAPPY=OFF -DARROW_WITH_ZLIB=OFF -DARROW_WITH_ZSTD=OFF && make -j2 && sudo make install && cd ../python && PYARROW_BUNDLE_ARROW_CPP=1 sudo python2.7 setup.py install --single-version-externally-managed --record RECORD 
-RUN cd arrow-apache-arrow-0.16.0 && rm -rf build && mkdir build && cd build && cmake ../cpp/ -DPython_ADDITIONAL_VERSIONS=3.6 -DARROW_RPATH_ORIGIN=ON -DARROW_PYTHON=ON -DARROW_FLIGHT=OFF -DARROW_IPC=ON -DARROW_COMPUTE=ON -DCMAKE_INSTALL_PREFIX=/usr -DARROW_DATASET=OFF -DARROW_BUILD_UTILITIES=OFF -DARROW_JEMALLOC=OFF -DARROW_DEPENDENCY_SOURCE=BUNDLED -DARROW_HDFS=OFF -DARROW_WITH_BACKTRACE=OFF -DARROW_WITH_BROTLI=OFF -DARROW_WITH_BZ2=OFF -DARROW_WITH_LZ4=OFF -DARROW_WITH_SNAPPY=OFF -DARROW_WITH_ZLIB=OFF -DARROW_WITH_ZSTD=OFF && make -j2 && sudo make install && cd ../python && PYARROW_BUNDLE_ARROW_CPP=1 sudo python3.6 setup.py install --single-version-externally-managed --record RECORD 
-RUN cd arrow-apache-arrow-0.16.0 && rm -rf build && mkdir build && cd build && cmake ../cpp/ -DPython_ADDITIONAL_VERSIONS=3.7 -DARROW_RPATH_ORIGIN=ON -DARROW_PYTHON=ON -DARROW_FLIGHT=OFF -DARROW_IPC=ON -DARROW_COMPUTE=ON -DCMAKE_INSTALL_PREFIX=/usr -DARROW_DATASET=OFF -DARROW_BUILD_UTILITIES=OFF -DARROW_JEMALLOC=OFF -DARROW_DEPENDENCY_SOURCE=BUNDLED -DARROW_HDFS=OFF -DARROW_WITH_BACKTRACE=OFF -DARROW_WITH_BROTLI=OFF -DARROW_WITH_BZ2=OFF -DARROW_WITH_LZ4=OFF -DARROW_WITH_SNAPPY=OFF -DARROW_WITH_ZLIB=OFF -DARROW_WITH_ZSTD=OFF && make -j2 && sudo make install && cd ../python && PYARROW_BUNDLE_ARROW_CPP=1 sudo python3.7 setup.py install --single-version-externally-managed --record RECORD 
-RUN cd arrow-apache-arrow-0.16.0 && rm -rf build && mkdir build && cd build && cmake ../cpp/ -DPython_ADDITIONAL_VERSIONS=3.8 -DARROW_RPATH_ORIGIN=ON -DARROW_PYTHON=ON -DARROW_FLIGHT=OFF -DARROW_IPC=ON -DARROW_COMPUTE=ON -DCMAKE_INSTALL_PREFIX=/usr -DARROW_DATASET=OFF -DARROW_BUILD_UTILITIES=OFF -DARROW_JEMALLOC=OFF -DARROW_DEPENDENCY_SOURCE=BUNDLED -DARROW_HDFS=OFF -DARROW_WITH_BACKTRACE=OFF -DARROW_WITH_BROTLI=OFF -DARROW_WITH_BZ2=OFF -DARROW_WITH_LZ4=OFF -DARROW_WITH_SNAPPY=OFF -DARROW_WITH_ZLIB=OFF -DARROW_WITH_ZSTD=OFF && make -j2 && sudo make install && cd ../python && PYARROW_BUNDLE_ARROW_CPP=1 sudo python3.8 setup.py install --single-version-externally-managed --record RECORD 
+RUN python2.7 -m pip install 'numpy>=1.13.1' 'pandas>=0.22.0' 'pyarrow<2' git+https://chromium.googlesource.com/external/gyp
+RUN python3.6 -m pip install 'numpy>=1.13.1' 'pandas>=0.22.0' 'pyarrow>=1.0.1,<2'
+RUN python3.7 -m pip install 'numpy>=1.13.1' 'pandas>=0.22.0' 'pyarrow>=1.0.1,<2'
+RUN python3.8 -m pip install 'numpy>=1.13.1' 'pandas>=0.22.0' 'pyarrow>=1.0.1,<2'
 
 # install node
 RUN curl -sL https://rpm.nodesource.com/setup_10.x | sudo bash -

diff --git a/docker/python/manylinux2014/Dockerfile b/docker/python/manylinux2014/Dockerfile
@@ -1,4 +1,4 @@
-# *****************************************************************************
+###############################################################################
 #
 # Copyright (c) 2020, the Perspective Authors.
 #
@@ -50,16 +50,9 @@ RUN tar xfz boost_1_71_0.tar.gz
 RUN cd boost_1_71_0 && ./bootstrap.sh
 RUN cd boost_1_71_0 && ./b2 -j8 --with-program_options --with-filesystem --with-system install 
 
-RUN python3.6 -m pip install 'numpy>=1.13.1' 'pandas>=0.22.0'
-RUN python3.7 -m pip install 'numpy>=1.13.1' 'pandas>=0.22.0'
-RUN python3.8 -m pip install 'numpy>=1.13.1' 'pandas>=0.22.0'
-
-# install pyarrow
-RUN wget https://github.com/apache/arrow/archive/apache-arrow-0.16.0.tar.gz >/dev/null 2>&1  || echo "wget arrow failed"
-RUN tar xfz apache-arrow-0.16.0.tar.gz
-RUN cd arrow-apache-arrow-0.16.0 && mkdir build && cd build && cmake ../cpp/ -DPython_ADDITIONAL_VERSIONS=3.7 -DARROW_RPATH_ORIGIN=ON -DARROW_PYTHON=ON -DARROW_FLIGHT=OFF -DARROW_IPC=ON -DARROW_COMPUTE=ON -DCMAKE_INSTALL_PREFIX=/usr -DARROW_DATASET=OFF -DARROW_BUILD_UTILITIES=OFF -DARROW_JEMALLOC=OFF -DARROW_DEPENDENCY_SOURCE=BUNDLED -DARROW_HDFS=OFF -DARROW_WITH_BACKTRACE=OFF -DARROW_WITH_BROTLI=OFF -DARROW_WITH_BZ2=OFF -DARROW_WITH_LZ4=OFF -DARROW_WITH_SNAPPY=OFF -DARROW_WITH_ZLIB=OFF -DARROW_WITH_ZSTD=OFF && make -j2 && sudo make install && cd ../python && PYARROW_BUNDLE_ARROW_CPP=1 sudo python3.7 setup.py install --single-version-externally-managed --record RECORD 
-RUN cd arrow-apache-arrow-0.16.0 && rm -rf build && mkdir build && cd build && cmake ../cpp/ -DPython_ADDITIONAL_VERSIONS=3.6 -DARROW_RPATH_ORIGIN=ON -DARROW_PYTHON=ON -DARROW_FLIGHT=OFF -DARROW_IPC=ON -DARROW_COMPUTE=ON -DCMAKE_INSTALL_PREFIX=/usr -DARROW_DATASET=OFF -DARROW_BUILD_UTILITIES=OFF -DARROW_JEMALLOC=OFF -DARROW_DEPENDENCY_SOURCE=BUNDLED -DARROW_HDFS=OFF -DARROW_WITH_BACKTRACE=OFF -DARROW_WITH_BROTLI=OFF -DARROW_WITH_BZ2=OFF -DARROW_WITH_LZ4=OFF -DARROW_WITH_SNAPPY=OFF -DARROW_WITH_ZLIB=OFF -DARROW_WITH_ZSTD=OFF && make -j2 && sudo make install && cd ../python && PYARROW_BUNDLE_ARROW_CPP=1 sudo python3.6 setup.py install --single-version-externally-managed --record RECORD 
-RUN cd arrow-apache-arrow-0.16.0 && rm -rf build && mkdir build && cd build && cmake ../cpp/ -DPython_ADDITIONAL_VERSIONS=3.8 -DARROW_RPATH_ORIGIN=ON -DARROW_PYTHON=ON -DARROW_FLIGHT=OFF -DARROW_IPC=ON -DARROW_COMPUTE=ON -DCMAKE_INSTALL_PREFIX=/usr -DARROW_DATASET=OFF -DARROW_BUILD_UTILITIES=OFF -DARROW_JEMALLOC=OFF -DARROW_DEPENDENCY_SOURCE=BUNDLED -DARROW_HDFS=OFF -DARROW_WITH_BACKTRACE=OFF -DARROW_WITH_BROTLI=OFF -DARROW_WITH_BZ2=OFF -DARROW_WITH_LZ4=OFF -DARROW_WITH_SNAPPY=OFF -DARROW_WITH_ZLIB=OFF -DARROW_WITH_ZSTD=OFF && make -j2 && sudo make install && cd ../python && PYARROW_BUNDLE_ARROW_CPP=1 sudo python3.8 setup.py install --single-version-externally-managed --record RECORD 
+RUN python3.6 -m pip install 'numpy>=1.13.1' 'pandas>=0.22.0' 'pyarrow>=1.0.1,<2'
+RUN python3.7 -m pip install 'numpy>=1.13.1' 'pandas>=0.22.0' 'pyarrow>=1.0.1,<2'
+RUN python3.8 -m pip install 'numpy>=1.13.1' 'pandas>=0.22.0' 'pyarrow>=1.0.1,<2'
 
 # install node
 RUN curl -sL https://rpm.nodesource.com/setup_10.x | sudo bash -

diff --git a/python/perspective/pyproject.toml b/python/perspective/pyproject.toml
@@ -1,3 +1,3 @@
 [build-system]
 # Minimum requirements for the build system to execute.
-requires = ["setuptools", "wheel", "backports.shutil_which ; python_version <'3'", "numpy>=1.13.1", "pyarrow>=0.16.0,<1"]
+requires = ["setuptools", "wheel", "backports.shutil_which ; python_version <'3'", "numpy>=1.13.1", "pyarrow<2"]
diff --git a/python/perspective/setup.py b/python/perspective/setup.py
@@ -5,6 +5,7 @@
 # This file is part of the Perspective library, distributed under the terms of
 # the Apache License 2.0.  The full license can be found in the LICENSE file.
 #
+
 from __future__ import print_function
 from setuptools import setup, find_packages, Extension
 from setuptools.command.build_ext import build_ext
@@ -49,15 +50,17 @@ def which(x):
     "future>=0.16.0",
     "numpy>=1.13.1",
     "pandas>=0.22.0",
-    "pyarrow>=0.16.0,<1",
     "python-dateutil>=2.8.0",
     "six>=1.11.0",
     "tornado>=4.5.3",
     "traitlets>=4.3.2",
 ]
 
 if sys.version_info.major < 3:
-    requires += ["backports.shutil-which"]
+    requires.append("backports.shutil-which")
+    requires.append("pyarrow<2")
+else:
+    requires.append("pyarrow>=1.0.1,<2")
 
 if (sys.version_info.major == 2 and sys.version_info.minor < 7) or (
     sys.version_info.major == 3 and sys.version_info.minor < 6