Merge pull request #1290 from finos/unlink-pyarrow

Statically link `perspective-python` and `libarrow`
finos · Jan 17, 2021 · 1725f48 · 1725f48
2 parents 0cba720 + 2ee1646
commit 1725f48
Show file tree

Hide file tree

Showing 17 changed files with 175 additions and 244 deletions.
diff --git a/azure-pipelines.yml b/azure-pipelines.yml
@@ -193,7 +193,7 @@ jobs:
       displayName: "Which python"
 
     - script: |
-        python -m pip install -U numpy "pyarrow>=0.16.0,<1" pip
+        python -m pip install -U numpy "pyarrow>=2" pip
       displayName: "Python deps"
 
     - script: npm install -g yarn
@@ -202,7 +202,10 @@ jobs:
     - script: yarn
       displayName: 'Install Deps'
 
-    - script: yarn build_python  --ci $(python_flag)
+    - script: choco install flatc
+      displayName: "Install Flatbuffers"
+
+    - script: yarn build_python --ci $(python_flag)
       displayName: 'build'
       env:
         # Set `BOOST_ROOT` manually, as `BOOST_ROOT` is removed in the VM:
@@ -256,7 +259,7 @@ jobs:
         versionSpec: '12.x'
 
     - script: |
-        brew install boost
+        brew install boost flatbuffers
       displayName: "System deps"
 
     - script: |
@@ -266,8 +269,15 @@ jobs:
       displayName: "Which python"
 
     - script: |
-        python -m pip install -U delocate wheel numpy "pyarrow>=0.16.0,<1" pip
-      displayName: "Python deps"
+        python -m pip install -U delocate wheel numpy "pyarrow>=2" pip
+      displayName: "Python 3 deps"
+      condition: eq(variables['python_flag'], '')
+    
+    # Last released PyArrow version on Python 2 is 0.16.0
+    - script: |
+        python -m pip install -U delocate wheel numpy "pyarrow==0.16.0" pip
+      displayName: "Python 2 deps"
+      condition: eq(variables['python_flag'], '--python2')
 
     - script: npm install -g yarn
       displayName: "Install Yarn"

diff --git a/binder/requirements.txt b/binder/requirements.txt
@@ -1,5 +1,5 @@
 ipywidgets==7.5.1
 jupyterlab==2.2.8
 pandas==0.25.3
-pyarrow==0.17.1
+pyarrow==2.0.0
 voila==0.2.3
diff --git a/cmake/arrow.txt.in b/cmake/arrow.txt.in
@@ -12,5 +12,5 @@ ExternalProject_Add(apachearrow
   BUILD_COMMAND     ""
   INSTALL_COMMAND   ""
   TEST_COMMAND      ""
-  CMAKE_ARGS        "-DCMAKE_TOOLCHAIN_FILE=${CMAKE_TOOLCHAIN_FILE}"
+  CMAKE_ARGS        "-DCMAKE_TOOLCHAIN_FILE=${CMAKE_TOOLCHAIN_FILE} -DFLATBUFFERS_BUILD_TESTS=OFF"
 )
diff --git a/cmake/arrow/CMakeLists.txt b/cmake/arrow/CMakeLists.txt
@@ -101,6 +101,28 @@ set(ARROW_SRCS
     ${CMAKE_BINARY_DIR}/arrow-src/cpp/src/arrow/ipc/options.cc
     ${CMAKE_BINARY_DIR}/arrow-src/cpp/src/arrow/ipc/writer.cc)
 
+if (PSP_PYTHON_BUILD)
+    set(ARROW_SRCS
+        ${ARROW_SRCS}
+        ${CMAKE_BINARY_DIR}/arrow-src/cpp/src/arrow/datum.cc
+        ${CMAKE_BINARY_DIR}/arrow-src/cpp/src/arrow/io/file.cc
+        ${CMAKE_BINARY_DIR}/arrow-src/cpp/src/arrow/tensor/coo_converter.cc
+        ${CMAKE_BINARY_DIR}/arrow-src/cpp/src/arrow/tensor/csf_converter.cc
+        ${CMAKE_BINARY_DIR}/arrow-src/cpp/src/arrow/tensor/csx_converter.cc
+        ${CMAKE_BINARY_DIR}/arrow-src/cpp/src/arrow/util/formatting.cc
+        ${CMAKE_BINARY_DIR}/arrow-src/cpp/src/arrow/util/time.cc
+        ${CMAKE_BINARY_DIR}/arrow-src/cpp/src/arrow/vendored/double-conversion/bignum-dtoa.cc
+        ${CMAKE_BINARY_DIR}/arrow-src/cpp/src/arrow/vendored/double-conversion/fast-dtoa.cc
+        ${CMAKE_BINARY_DIR}/arrow-src/cpp/src/arrow/vendored/double-conversion/fixed-dtoa.cc
+    )
+
+    if(WIN32)
+        set(ARROW_SRCS
+            ${ARROW_SRCS}
+            ${CMAKE_BINARY_DIR}/arrow-src/cpp/src/arrow/vendored/musl/strptime.c)
+    endif()
+endif()
+
 set_property(SOURCE util/io_util.cc
            APPEND_STRING
            PROPERTY COMPILE_FLAGS " -Wno-unused-macros -stdlib=libc++")
@@ -130,13 +152,17 @@ set(FBS_SRC
     ${CMAKE_BINARY_DIR}/arrow-src/cpp/src/arrow/ipc/feather.fbs)
 
 include_directories(src)
+
+# Build Arrow as a static library
 set(ARROW_BUILD_STATIC ON)
 if (PSP_WASM_BUILD)
     set(CMAKE_POSITION_INDEPENDENT_CODE OFF)
 elseif(PSP_CPP_BUILD OR PSP_PYTHON_BUILD)
     set(CMAKE_POSITION_INDEPENDENT_CODE ON)
 endif()
+
 add_library(arrow STATIC ${ARROW_SRCS})
+
 target_compile_definitions(arrow PUBLIC ARROW_NO_DEPRECATED_API)
 target_compile_definitions(arrow PUBLIC ARROW_STATIC)
 

diff --git a/cmake/date.txt.in b/cmake/date.txt.in
@@ -5,7 +5,7 @@ project(date-download NONE)
 include(ExternalProject)
 ExternalProject_Add(date
   GIT_REPOSITORY    https://github.com/HowardHinnant/date.git
-  GIT_TAG           master
+  GIT_TAG           v3.0.0
   SOURCE_DIR        "${CMAKE_BINARY_DIR}/date-src"
   BINARY_DIR        "${CMAKE_BINARY_DIR}/date-build"
   CONFIGURE_COMMAND ""

diff --git a/cmake/flatbuffers.txt.in b/cmake/flatbuffers.txt.in
@@ -0,0 +1,16 @@
+cmake_minimum_required(VERSION 3.7.2)
+
+project(flatbuffers-download NONE)
+
+include(ExternalProject)
+ExternalProject_Add(flatbuffers
+  GIT_REPOSITORY    https://github.com/google/flatbuffers.git
+  GIT_TAG           v1.12.0
+  SOURCE_DIR        "${CMAKE_BINARY_DIR}/flatbuffers-src"
+  BINARY_DIR        "${CMAKE_BINARY_DIR}/flatbuffers-build"
+  CONFIGURE_COMMAND ""
+  BUILD_COMMAND     ""
+  INSTALL_COMMAND   ""
+  TEST_COMMAND      ""
+  CMAKE_ARGS        "-DCMAKE_TOOLCHAIN_FILE=${CMAKE_TOOLCHAIN_FILE}"
+)
diff --git a/cmake/modules/FindFlatbuffers.cmake b/cmake/modules/FindFlatbuffers.cmake
@@ -22,26 +22,48 @@
 # FLATBUFFERS_INCLUDE_DIR, directory containing headers
 # FLATBUFFERS_STATIC_LIB, path to flatbuffers's static library
 # FLATBUFFERS_COMPILER, path to flatc compiler
+#
+# TODO: [01-15-2021] now that we use Flatbuffers on all platforms, it might be
+# a good time to figure out how we can install Flatbuffers as a dependency
+# inside our CMakeLists (we would just need to build the flatc executable
+# before our Arrow build starts). Right now, I've put in some hacks to make
+# sure our Windows build works on Azure by pre-installing flatc (like we do on
+# all other platforms), and then pulling down the headers for Windows so they
+# can be included.
 
 # this might fail
 # https://gitlab.kitware.com/cmake/cmake/issues/19120
-find_path(FLATBUFFERS_INCLUDE_DIR flatbuffers/flatbuffers.h
-  PATHS ${FLATBUFFERS_ROOT}/include
-  HINTS /usr/local /usr/local/flatbuffers /usr/local/Homebrew /usr ~/homebrew/ /usr/local/include /usr/local/flatbuffers/include /usr/include ~/homebrew/include
-  NO_CMAKE_SYSTEM_PATH
-  NO_SYSTEM_ENVIRONMENT_PATH)
+if (WIN32)
+  find_path(FLATBUFFERS_INCLUDE_DIR flatbuffers/flatbuffers.h
+    PATHS ${FLATBUFFERS_ROOT}/include)
+
+  find_program(FLATBUFFERS_COMPILER flatc
+    PATHS ${FLATBUFFERS_ROOT}/bin)
+else()
+  find_path(FLATBUFFERS_INCLUDE_DIR flatbuffers/flatbuffers.h
+    PATHS ${FLATBUFFERS_ROOT}/include
+    HINTS /usr/local /usr/local/flatbuffers /usr/local/Homebrew /usr ~/homebrew/ /usr/local/include /usr/local/flatbuffers/include /usr/include ~/homebrew/include
+    NO_CMAKE_SYSTEM_PATH
+    NO_SYSTEM_ENVIRONMENT_PATH)
 
-find_program(FLATBUFFERS_COMPILER flatc
-  PATHS ${FLATBUFFERS_ROOT}/bin
-  HINTS /usr/local/bin /usr/bin /usr/local/Homebrew/bin ~/homebrew/bin
-  NO_CMAKE_SYSTEM_PATH
-  NO_SYSTEM_ENVIRONMENT_PATH)
+  find_program(FLATBUFFERS_COMPILER flatc
+    PATHS ${FLATBUFFERS_ROOT}/bin
+    HINTS /usr/local/bin /usr/bin /usr/local/Homebrew/bin ~/homebrew/bin
+    NO_CMAKE_SYSTEM_PATH
+    NO_SYSTEM_ENVIRONMENT_PATH)
 
-if(NOT ${FLATBUFFERS_INCLUDE_DIR})
-  # HACK
-  set(FLATBUFFERS_INCLUDE_DIR /usr/local/include)
+  if(NOT ${FLATBUFFERS_INCLUDE_DIR})
+    # HACK
+    set(FLATBUFFERS_INCLUDE_DIR /usr/local/include)
+  endif()
 endif()
 
 include(FindPackageHandleStandardArgs)
-find_package_handle_standard_args(FLATBUFFERS REQUIRED_VARS
-  FLATBUFFERS_INCLUDE_DIR FLATBUFFERS_COMPILER)
+
+if (WIN32)
+  find_package_handle_standard_args(Flatbuffers REQUIRED_VARS
+    FLATBUFFERS_INCLUDE_DIR FLATBUFFERS_COMPILER)
+else()
+  find_package_handle_standard_args(FLATBUFFERS REQUIRED_VARS
+    FLATBUFFERS_INCLUDE_DIR FLATBUFFERS_COMPILER)
+endif()
diff --git a/cmake/modules/FindPyArrow.cmake b/cmake/modules/FindPyArrow.cmake
diff --git a/cmake/modules/FindPybind.cmake b/cmake/modules/FindPybind.cmake
@@ -1,4 +1,4 @@
-# Find the Python PyArrow package
+# Find the Python PyBind package
 # PYTHON_PYBIND_INCLUDE_DIR
 # PYTHON_PYBIND_FOUND
 # will be set by this script