Upgrade pyarrow requirements, remove FindPyArrow, update dockerfiles …

…to install PyArrow 2
finos · Jan 15, 2021 · 6d66634 · 6d66634
1 parent 871a289
commit 6d66634
Show file tree

Hide file tree

Showing 10 changed files with 23 additions and 142 deletions.
diff --git a/azure-pipelines.yml b/azure-pipelines.yml
@@ -193,7 +193,7 @@ jobs:
       displayName: "Which python"
 
     - script: |
-        python -m pip install -U numpy "pyarrow>=0.16.0,<1" pip
+        python -m pip install -U numpy "pyarrow>=1.0.1" pip
       displayName: "Python deps"
 
     - script: npm install -g yarn
@@ -269,8 +269,15 @@ jobs:
       displayName: "Which python"
 
     - script: |
-        python -m pip install -U delocate wheel numpy "pyarrow>=0.16.0,<1" pip
-      displayName: "Python deps"
+        python -m pip install -U delocate wheel numpy "pyarrow>=1.0.1" pip
+      displayName: "Python 3 deps"
+      condition: eq(variables['python_flag'], '')
+    
+    # Last released PyArrow version on Python 2 is 0.16.0
+    - script: |
+        python -m pip install -U delocate wheel numpy pyarrow pip
+      displayName: "Python 2 deps"
+      condition: eq(variables['python_flag'], '--python2')
 
     - script: npm install -g yarn
       displayName: "Install Yarn"

diff --git a/binder/requirements.txt b/binder/requirements.txt
@@ -1,5 +1,5 @@
 ipywidgets==7.5.1
 jupyterlab==2.2.8
 pandas==0.25.3
-pyarrow==0.17.1
+pyarrow==2.0.0
 voila==0.2.3
diff --git a/cmake/modules/FindPyArrow.cmake b/cmake/modules/FindPyArrow.cmake
diff --git a/cpp/perspective/CMakeLists.txt b/cpp/perspective/CMakeLists.txt
@@ -643,7 +643,6 @@ elseif(PSP_CPP_BUILD OR PSP_PYTHON_BUILD)
 		# building minimal arrow.
 		if (PSP_PYTHON_BUILD AND MANYLINUX AND PSP_PYTHON_VERSION STREQUAL "2.7")
 			target_link_libraries(psp rt)
-			target_link_libraries(binding rt)
 		endif()
 
 		# Link against minimal arrow static library

diff --git a/docker/python/manylinux2010/Dockerfile b/docker/python/manylinux2010/Dockerfile
@@ -53,18 +53,10 @@ RUN tar xfz boost_1_71_0.tar.gz
 RUN cd boost_1_71_0 && ./bootstrap.sh
 RUN cd boost_1_71_0 && ./b2 -j8 --with-program_options --with-filesystem --with-system install 
 
-RUN python2.7 -m pip install 'numpy>=1.13.1' 'pandas>=0.22.0' git+https://chromium.googlesource.com/external/gyp
-RUN python3.6 -m pip install 'numpy>=1.13.1' 'pandas>=0.22.0'
-RUN python3.7 -m pip install 'numpy>=1.13.1' 'pandas>=0.22.0'
-RUN python3.8 -m pip install 'numpy>=1.13.1' 'pandas>=0.22.0'
-
-# install pyarrow
-RUN wget https://github.com/apache/arrow/archive/apache-arrow-0.16.0.tar.gz >/dev/null 2>&1  || echo "wget arrow failed"
-RUN tar xfz apache-arrow-0.16.0.tar.gz
-RUN cd arrow-apache-arrow-0.16.0 && mkdir build && cd build && cmake ../cpp/ -DPython_ADDITIONAL_VERSIONS=2.7 -DARROW_RPATH_ORIGIN=ON -DARROW_PYTHON=ON -DARROW_FLIGHT=OFF -DARROW_IPC=ON -DARROW_COMPUTE=ON -DCMAKE_INSTALL_PREFIX=/usr -DARROW_DATASET=OFF -DARROW_BUILD_UTILITIES=OFF -DARROW_JEMALLOC=OFF -DARROW_DEPENDENCY_SOURCE=BUNDLED -DARROW_HDFS=OFF -DARROW_WITH_BACKTRACE=OFF -DARROW_WITH_BROTLI=OFF -DARROW_WITH_BZ2=OFF -DARROW_WITH_LZ4=OFF -DARROW_WITH_SNAPPY=OFF -DARROW_WITH_ZLIB=OFF -DARROW_WITH_ZSTD=OFF && make -j2 && sudo make install && cd ../python && PYARROW_BUNDLE_ARROW_CPP=1 sudo python2.7 setup.py install --single-version-externally-managed --record RECORD 
-RUN cd arrow-apache-arrow-0.16.0 && rm -rf build && mkdir build && cd build && cmake ../cpp/ -DPython_ADDITIONAL_VERSIONS=3.6 -DARROW_RPATH_ORIGIN=ON -DARROW_PYTHON=ON -DARROW_FLIGHT=OFF -DARROW_IPC=ON -DARROW_COMPUTE=ON -DCMAKE_INSTALL_PREFIX=/usr -DARROW_DATASET=OFF -DARROW_BUILD_UTILITIES=OFF -DARROW_JEMALLOC=OFF -DARROW_DEPENDENCY_SOURCE=BUNDLED -DARROW_HDFS=OFF -DARROW_WITH_BACKTRACE=OFF -DARROW_WITH_BROTLI=OFF -DARROW_WITH_BZ2=OFF -DARROW_WITH_LZ4=OFF -DARROW_WITH_SNAPPY=OFF -DARROW_WITH_ZLIB=OFF -DARROW_WITH_ZSTD=OFF && make -j2 && sudo make install && cd ../python && PYARROW_BUNDLE_ARROW_CPP=1 sudo python3.6 setup.py install --single-version-externally-managed --record RECORD 
-RUN cd arrow-apache-arrow-0.16.0 && rm -rf build && mkdir build && cd build && cmake ../cpp/ -DPython_ADDITIONAL_VERSIONS=3.7 -DARROW_RPATH_ORIGIN=ON -DARROW_PYTHON=ON -DARROW_FLIGHT=OFF -DARROW_IPC=ON -DARROW_COMPUTE=ON -DCMAKE_INSTALL_PREFIX=/usr -DARROW_DATASET=OFF -DARROW_BUILD_UTILITIES=OFF -DARROW_JEMALLOC=OFF -DARROW_DEPENDENCY_SOURCE=BUNDLED -DARROW_HDFS=OFF -DARROW_WITH_BACKTRACE=OFF -DARROW_WITH_BROTLI=OFF -DARROW_WITH_BZ2=OFF -DARROW_WITH_LZ4=OFF -DARROW_WITH_SNAPPY=OFF -DARROW_WITH_ZLIB=OFF -DARROW_WITH_ZSTD=OFF && make -j2 && sudo make install && cd ../python && PYARROW_BUNDLE_ARROW_CPP=1 sudo python3.7 setup.py install --single-version-externally-managed --record RECORD 
-RUN cd arrow-apache-arrow-0.16.0 && rm -rf build && mkdir build && cd build && cmake ../cpp/ -DPython_ADDITIONAL_VERSIONS=3.8 -DARROW_RPATH_ORIGIN=ON -DARROW_PYTHON=ON -DARROW_FLIGHT=OFF -DARROW_IPC=ON -DARROW_COMPUTE=ON -DCMAKE_INSTALL_PREFIX=/usr -DARROW_DATASET=OFF -DARROW_BUILD_UTILITIES=OFF -DARROW_JEMALLOC=OFF -DARROW_DEPENDENCY_SOURCE=BUNDLED -DARROW_HDFS=OFF -DARROW_WITH_BACKTRACE=OFF -DARROW_WITH_BROTLI=OFF -DARROW_WITH_BZ2=OFF -DARROW_WITH_LZ4=OFF -DARROW_WITH_SNAPPY=OFF -DARROW_WITH_ZLIB=OFF -DARROW_WITH_ZSTD=OFF && make -j2 && sudo make install && cd ../python && PYARROW_BUNDLE_ARROW_CPP=1 sudo python3.8 setup.py install --single-version-externally-managed --record RECORD 
+RUN python2.7 -m pip install 'numpy>=1.13.1' 'pandas>=0.22.0' 'pyarrow>=0.16.0' git+https://chromium.googlesource.com/external/gyp
+RUN python3.6 -m pip install 'numpy>=1.13.1' 'pandas>=0.22.0' 'pyarrow>=2.0.0'
+RUN python3.7 -m pip install 'numpy>=1.13.1' 'pandas>=0.22.0' 'pyarrow>=2.0.0'
+RUN python3.8 -m pip install 'numpy>=1.13.1' 'pandas>=0.22.0' 'pyarrow>=2.0.0'
 
 # install node
 RUN curl -sL https://rpm.nodesource.com/setup_10.x | sudo bash -

diff --git a/docker/python/manylinux2014/Dockerfile b/docker/python/manylinux2014/Dockerfile
@@ -50,16 +50,9 @@ RUN tar xfz boost_1_71_0.tar.gz
 RUN cd boost_1_71_0 && ./bootstrap.sh
 RUN cd boost_1_71_0 && ./b2 -j8 --with-program_options --with-filesystem --with-system install 
 
-RUN python3.6 -m pip install 'numpy>=1.13.1' 'pandas>=0.22.0'
-RUN python3.7 -m pip install 'numpy>=1.13.1' 'pandas>=0.22.0'
-RUN python3.8 -m pip install 'numpy>=1.13.1' 'pandas>=0.22.0'
-
-# install pyarrow
-RUN wget https://github.com/apache/arrow/archive/apache-arrow-0.16.0.tar.gz >/dev/null 2>&1  || echo "wget arrow failed"
-RUN tar xfz apache-arrow-0.16.0.tar.gz
-RUN cd arrow-apache-arrow-0.16.0 && mkdir build && cd build && cmake ../cpp/ -DPython_ADDITIONAL_VERSIONS=3.7 -DARROW_RPATH_ORIGIN=ON -DARROW_PYTHON=ON -DARROW_FLIGHT=OFF -DARROW_IPC=ON -DARROW_COMPUTE=ON -DCMAKE_INSTALL_PREFIX=/usr -DARROW_DATASET=OFF -DARROW_BUILD_UTILITIES=OFF -DARROW_JEMALLOC=OFF -DARROW_DEPENDENCY_SOURCE=BUNDLED -DARROW_HDFS=OFF -DARROW_WITH_BACKTRACE=OFF -DARROW_WITH_BROTLI=OFF -DARROW_WITH_BZ2=OFF -DARROW_WITH_LZ4=OFF -DARROW_WITH_SNAPPY=OFF -DARROW_WITH_ZLIB=OFF -DARROW_WITH_ZSTD=OFF && make -j2 && sudo make install && cd ../python && PYARROW_BUNDLE_ARROW_CPP=1 sudo python3.7 setup.py install --single-version-externally-managed --record RECORD 
-RUN cd arrow-apache-arrow-0.16.0 && rm -rf build && mkdir build && cd build && cmake ../cpp/ -DPython_ADDITIONAL_VERSIONS=3.6 -DARROW_RPATH_ORIGIN=ON -DARROW_PYTHON=ON -DARROW_FLIGHT=OFF -DARROW_IPC=ON -DARROW_COMPUTE=ON -DCMAKE_INSTALL_PREFIX=/usr -DARROW_DATASET=OFF -DARROW_BUILD_UTILITIES=OFF -DARROW_JEMALLOC=OFF -DARROW_DEPENDENCY_SOURCE=BUNDLED -DARROW_HDFS=OFF -DARROW_WITH_BACKTRACE=OFF -DARROW_WITH_BROTLI=OFF -DARROW_WITH_BZ2=OFF -DARROW_WITH_LZ4=OFF -DARROW_WITH_SNAPPY=OFF -DARROW_WITH_ZLIB=OFF -DARROW_WITH_ZSTD=OFF && make -j2 && sudo make install && cd ../python && PYARROW_BUNDLE_ARROW_CPP=1 sudo python3.6 setup.py install --single-version-externally-managed --record RECORD 
-RUN cd arrow-apache-arrow-0.16.0 && rm -rf build && mkdir build && cd build && cmake ../cpp/ -DPython_ADDITIONAL_VERSIONS=3.8 -DARROW_RPATH_ORIGIN=ON -DARROW_PYTHON=ON -DARROW_FLIGHT=OFF -DARROW_IPC=ON -DARROW_COMPUTE=ON -DCMAKE_INSTALL_PREFIX=/usr -DARROW_DATASET=OFF -DARROW_BUILD_UTILITIES=OFF -DARROW_JEMALLOC=OFF -DARROW_DEPENDENCY_SOURCE=BUNDLED -DARROW_HDFS=OFF -DARROW_WITH_BACKTRACE=OFF -DARROW_WITH_BROTLI=OFF -DARROW_WITH_BZ2=OFF -DARROW_WITH_LZ4=OFF -DARROW_WITH_SNAPPY=OFF -DARROW_WITH_ZLIB=OFF -DARROW_WITH_ZSTD=OFF && make -j2 && sudo make install && cd ../python && PYARROW_BUNDLE_ARROW_CPP=1 sudo python3.8 setup.py install --single-version-externally-managed --record RECORD 
+RUN python3.6 -m pip install 'numpy>=1.13.1' 'pandas>=0.22.0' 'pyarrow>=2.0.0'
+RUN python3.7 -m pip install 'numpy>=1.13.1' 'pandas>=0.22.0' 'pyarrow>=2.0.0'
+RUN python3.8 -m pip install 'numpy>=1.13.1' 'pandas>=0.22.0' 'pyarrow>=2.0.0'
 
 # install node
 RUN curl -sL https://rpm.nodesource.com/setup_10.x | sudo bash -

diff --git a/docs/md/development.md b/docs/md/development.md
@@ -170,7 +170,7 @@ You can run the test suite simply with the standard NPM command, which will both
 build the test suite for every package and run them.
 
 ```bash
-yarn test
+yarn test [--debug]
 ```
 
 A test name regex can be passed to `jest` via the same `-t` flag:
@@ -238,10 +238,7 @@ Verbosity in the tests can be enabled with the `--verbose` flag.
 ### Troubleshooting installation from source
 
 If you are installing from a source distribution (sdist), make sure you have
-CMake and Boost headers present on your machine:
-
-- CMake (version 3.15.4 or higher)
-- Boost Headers (version 1.67)
+the [System Dependencies](#system-dependencies) installed.
 
 Try installing in verbose mode:
 
@@ -253,13 +250,7 @@ The most common culprits are:
 
 - CMake version too old
 - Boost headers are missing or too old
-- PyArrow not installed prior to installing perspective
-
-Additionally, due to PEP-518 and build isolation, its possible that the version
-of PyArrow that pip uses to build perspective-python is different from the one
-you have installed. To disable this, pass the `--no-build-isolation` flag to
-pip.
-
+- Flatbuffers not installed prior to installing Perspective
 #### Timezones in Python Tests
 
 Python tests are configured to use the `UTC` time zone. If running tests locally,

diff --git a/docs/md/python.md b/docs/md/python.md
@@ -63,16 +63,6 @@ NumPy record arrays are all supported in `perspective-python`.
 pip install perspective-python
 ```
 
-#### Wheels PyArrow linkage
-
-Because we compile Apache Arrow from source to webassembly via Emscripten, we have a tight coupling on the specific version of Apache Arrow that must be used. As such, we link against a specific Apache Arrow version which must be present. Currently, our wheels build against PyArrow==0.17.1 for Python 3.* and PyArrow==0.16.0 for Python 2.7.
-
-To ignore compiled wheels and install from source with pip, install via
-
-```bash
-pip install --no-binary perspective-python
-```
-
 ### Jupyterlab
 
 `PerspectiveWidget` is a JupyterLab widget that implements the same API as

diff --git a/python/perspective/pyproject.toml b/python/perspective/pyproject.toml
@@ -1,3 +1,3 @@
 [build-system]
 # Minimum requirements for the build system to execute.
-requires = ["setuptools", "wheel", "backports.shutil_which ; python_version <'3'", "numpy>=1.13.1", "pyarrow>=0.16.0,<1"]
+requires = ["setuptools", "wheel", "backports.shutil_which ; python_version <'3'", "numpy>=1.13.1"]
diff --git a/python/perspective/setup.py b/python/perspective/setup.py
@@ -10,7 +10,6 @@
 from setuptools.command.build_ext import build_ext
 from setuptools.command.sdist import sdist
 from distutils.version import LooseVersion
-from distutils import sysconfig
 from codecs import open
 
 import io
@@ -49,7 +48,6 @@ def which(x):
     "future>=0.16.0",
     "numpy>=1.13.1",
     "pandas>=0.22.0",
-    "pyarrow>=0.16.0,<1",
     "python-dateutil>=2.8.0",
     "six>=1.11.0",
     "tornado>=4.5.3",
@@ -69,6 +67,7 @@ def which(x):
     "flake8>=3.7.8",
     "mock",
     "pybind11>=2.4.0",
+    "pyarrow>=0.16.0",
     "pytest>=4.3.0",
     "pytest-cov>=2.6.1",
     "pytest-check-links",
@@ -146,14 +145,6 @@ def build_extension_cmake(self, ext):
             "-DPSP_PYTHON_VERSION={}".format(PYTHON_VERSION),
             "-DPython_ADDITIONAL_VERSIONS={}".format(PYTHON_VERSION),
             "-DPython_FIND_VERSION={}".format(PYTHON_VERSION),
-            "-DPSP_PYTHON_ARROWINSTALLDIR={}".format(
-                os.environ.get(
-                    "PSP_PYTHON_ARROWINSTALLDIR",
-                    os.path.join(sysconfig.get_python_lib(), "pyarrow").replace(
-                        "\\", "/"
-                    ),
-                )
-            ),
             "-DPython_EXECUTABLE={}".format(sys.executable).replace("\\", "/"),
             "-DPython_ROOT_DIR={}".format(sys.prefix).replace("\\", "/"),
             "-DPython_ROOT={}".format(sys.prefix).replace("\\", "/"),