Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

updates and fixes for windows build #884

Merged
merged 11 commits into from
Feb 8, 2020
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
34 changes: 17 additions & 17 deletions .appveyor.yml
Original file line number Diff line number Diff line change
@@ -1,8 +1,8 @@
max_jobs: 8

image:
- Visual Studio 2015
# - Visual Studio 2017
# - Visual Studio 2015
- Visual Studio 2017

configuration:
- Release
Expand All @@ -11,6 +11,8 @@ configuration:
platform:
- x64

stack: python 3.7

environment:
matrix:
- arch: Win64
Expand All @@ -25,25 +27,23 @@ init:
- if "%arch%"=="Win64" ( set arch= Win64)
- echo %arch%
- echo %APPVEYOR_BUILD_WORKER_IMAGE%
- if "%APPVEYOR_BUILD_WORKER_IMAGE%"=="Visual Studio 2017" ( set generator="Visual Studio 15 2017%arch%" )
- if "%APPVEYOR_BUILD_WORKER_IMAGE%"=="Visual Studio 2015" ( set generator="Visual Studio 14 2015%arch%" )

before_build:
- cmd: |-
cd cpp\perspective
mkdir build
cd build
cmake --version
cmake .. -G %generator% -DPSP_WASM_BUILD=0 -DPSP_CPP_BUILD=1 -DPSP_CPP_BUILD_TESTS=1 -DPSP_CPP_BUILD_STRICT=1 -DBOOST_ROOT=C:\Libraries\boost_1_67_0 -DBoost_INCLUDE_DIRS=C:\Libraries\boost_1_67_0
install:
- set PATH=C:\Python37-x64;%PATH%
- which python
- python --version
- ln -s C:\Python37-x64\python C:\Python37-x64\python3.7
- python -m pip install numpy pyarrow==0.15.1
- cmake --version
- set BOOST_ROOT=C:\Libraries\boost_1_67_0
- set Boost_INCLUDE_DIRS=C:\Libraries\boost_1_67_0
- yarn

build_script:
- msbuild psp.sln /p:Platform=x64 /m /p:Configuration=%CONFIGURATION%
- cp %CONFIGURATION%\psp.dll test\%CONFIGURATION%
- cp tbb-build\%CONFIGURATION%\tbb.dll test\%CONFIGURATION%
- .\test\%CONFIGURATION%\psp_test.exe
- yarn build_python --ci

# to disable automatic tests
test: off
test_script:
- yarn test_python

# to disable deployment
deploy: off
7 changes: 6 additions & 1 deletion cmake/modules/FindPyArrow.cmake
Original file line number Diff line number Diff line change
Expand Up @@ -48,7 +48,12 @@ find_path(PYTHON_PYARROW_INCLUDE_DIR arrow/python/api.h

set(PYTHON_PYARROW_LIBRARY_DIR ${__pyarrow_library_dirs})

if (CMAKE_SYSTEM_NAME MATCHES "Darwin")
if(${CMAKE_SYSTEM_NAME} MATCHES "Windows")
# windows its just "arrow.dll"
set(PYTHON_PYARROW_PYTHON_SHARED_LIBRARY "arrow_python")
set(PYTHON_PYARROW_ARROW_SHARED_LIBRARY "arrow")
set(PYTHON_PYARROW_LIBRARIES ${PYTHON_PYARROW_PYTHON_SHARED_LIBRARY} ${PYTHON_PYARROW_ARROW_SHARED_LIBRARY})
elseif (CMAKE_SYSTEM_NAME MATCHES "Darwin")
# Link against pre-built libarrow on MacOS
set(PYTHON_PYARROW_PYTHON_SHARED_LIBRARY ${PYTHON_PYARROW_LIBRARY_DIR}/${CMAKE_SHARED_LIBRARY_PREFIX}arrow_python.15.dylib)
set(PYTHON_PYARROW_ARROW_SHARED_LIBRARY ${PYTHON_PYARROW_LIBRARY_DIR}/${CMAKE_SHARED_LIBRARY_PREFIX}arrow.15.dylib)
Expand Down
89 changes: 62 additions & 27 deletions cpp/perspective/CMakeLists.txt
Original file line number Diff line number Diff line change
@@ -1,9 +1,8 @@
cmake_minimum_required(VERSION 3.7.2)
project (psp)
project(psp)

set(CMAKE_BUILD_TYPE "Release")
set(CMAKE_CXX_FLAGS_RELEASE "-O3 -DNDEBUG")
set(CMAKE_C_FLAGS_RELEASE "-O3 -DNDEBUG")


set(CMAKE_CXX_STANDARD 14)
set(CMAKE_EXPORT_COMPILE_COMMANDS ON)
Expand Down Expand Up @@ -197,6 +196,10 @@ message(WARNING "${BUILD_MESSAGE}")
#######################
include_directories("${CMAKE_SOURCE_DIR}/src/include")

if(NOT WIN32)
set(CMAKE_CXX_FLAGS_RELEASE "-O3 -DNDEBUG")
Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@sc1f note this

set(CMAKE_C_FLAGS_RELEASE "-O3 -DNDEBUG")
endif()

if (PSP_WASM_BUILD)
####################
Expand Down Expand Up @@ -264,16 +267,31 @@ elseif(PSP_CPP_BUILD OR PSP_PYTHON_BUILD)
#####################
# VANILLA CPP BUILD #
#####################
if(CMAKE_BUILD_TYPE_LOWER STREQUAL debug)
set(OPT_FLAGS " \
-O0 \
-g3 \
")
if(WIN32)
if(CMAKE_BUILD_TYPE_LOWER STREQUAL debug)
set(OPT_FLAGS " \
Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@sc1f we should set debug flags (see prev comment) instead of having this opt flags thing

/DEBUG \
/Z7 \
/Zi
")
else()
set(OPT_FLAGS " \
/NDEBUG \
/O2 \
")
endif()
else()
set(OPT_FLAGS " \
-O3 \
-g0 \
")
if(CMAKE_BUILD_TYPE_LOWER STREQUAL debug)
set(OPT_FLAGS " \
-O0 \
-g3 \
")
else()
set(OPT_FLAGS " \
-O3 \
-g0 \
")
endif()
endif()
set(SYNC_MODE_FLAGS "")
set(ASYNC_MODE_FLAGS "")
Expand Down Expand Up @@ -377,9 +395,7 @@ elseif(PSP_CPP_BUILD OR PSP_PYTHON_BUILD)
message(FATAL_ERROR "${Red}PyArrow could not be located${ColorReset}")
else()
message(WARNING "${Cyan}PyArrow found: ${PYTHON_PYARROW_INCLUDE_DIR}${ColorReset}")
if (NOT WIN32)
message(WARNING "${Cyan}Using pre-built ${PYTHON_PYARROW_PYTHON_SHARED_LIBRARY} ${PYTHON_PYARROW_ARROW_SHARED_LIBRARY} from: ${PYTHON_PYARROW_LIBRARY_DIR}${ColorReset}")
endif()
message(WARNING "${Cyan}Using pre-built ${PYTHON_PYARROW_PYTHON_SHARED_LIBRARY} ${PYTHON_PYARROW_ARROW_SHARED_LIBRARY} from: ${PYTHON_PYARROW_LIBRARY_DIR}${ColorReset}")
endif()
include_directories(${PYTHON_PYARROW_INCLUDE_DIR})
link_directories(${PYTHON_PYARROW_LIBRARY_DIR})
Expand All @@ -392,7 +408,7 @@ psp_build_dep("hopscotch" "${PSP_CMAKE_MODULE_PATH}/hopscotch.txt.in")
psp_build_dep("ordered-map" "${PSP_CMAKE_MODULE_PATH}/ordered-map.txt.in")

# For all non-MacOS and non-linux builds, or if building WASM/CPP, build minimal arrow from source
if (WIN32 OR NOT PSP_PYTHON_BUILD)
if (NOT PSP_PYTHON_BUILD)
# build arrow + dependencies from source for Emscripten and C++
message("${Cyan}Building minimal Apache Arrow${ColorReset}")

Expand Down Expand Up @@ -516,7 +532,7 @@ set (PYTHON_SOURCE_FILES
)

if (WIN32)
set(CMAKE_CXX_FLAGS " /EHsc")
set(CMAKE_CXX_FLAGS " /EHsc /MP")
else()
set(CMAKE_CXX_FLAGS " ${CMAKE_CXX_FLAGS} ${CMAKE_C_FLAGS}")
endif()
Expand Down Expand Up @@ -548,7 +564,9 @@ elseif(PSP_CPP_BUILD OR PSP_PYTHON_BUILD)
set(module_install_rpath "@loader_path/")
else()
set(module_install_rpath "\$ORIGIN")
endif()
endif()
else()
set(CMAKE_SHARED_LIBRARY_PREFIX lib)
Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@sc1f we should consider not prefixing with lib, and setting the prefixes to "" (this will mean we need to change python imports from libbinding to just binding, or preferable _binding)

endif()

if(PSP_PYTHON_BUILD)
Expand All @@ -562,28 +580,45 @@ elseif(PSP_CPP_BUILD OR PSP_PYTHON_BUILD)

target_compile_definitions(psp PRIVATE PSP_ENABLE_PYTHON=1)
target_compile_definitions(binding PRIVATE PSP_ENABLE_PYTHON=1)
target_compile_options(binding PRIVATE -Wdeprecated-declarations)

# Link against libarrow 0.15.1 in pyarrow directory for windows
if (WIN32)
target_link_libraries(psp arrow)
target_link_libraries(binding arrow)
target_compile_definitions(binding PRIVATE WIN32=1)
target_compile_definitions(binding PRIVATE _WIN32=1)

target_link_libraries(psp ${Boost_FILESYSTEM_LIBRARY})
target_link_libraries(binding ${Boost_FILESYSTEM_LIBRARY})

# .dll not importable
set_property(TARGET binding PROPERTY SUFFIX .pyd)
else()
target_link_libraries(psp ${PYTHON_PYARROW_LIBRARIES})
target_link_libraries(binding ${PYTHON_PYARROW_LIBRARIES})
target_compile_options(binding PRIVATE -Wdeprecated-declarations)
set_property(TARGET psp PROPERTY INSTALL_RPATH ${CMAKE_INSTALL_RPATH} ${module_install_rpath} ${PYTHON_PYARROW_LIBRARY_DIR})
set_property(TARGET binding PROPERTY INSTALL_RPATH ${CMAKE_INSTALL_RPATH} ${module_install_rpath} ${PYTHON_PYARROW_LIBRARY_DIR})
endif()
endif()

target_link_libraries(psp ${PYTHON_PYARROW_LIBRARIES})
target_link_libraries(binding ${PYTHON_PYARROW_LIBRARIES})

target_link_libraries(psp ${PYTHON_LIBRARIES})
target_link_libraries(binding ${PYTHON_LIBRARIES})
target_link_libraries(binding psp)

target_link_libraries(psp tbb)
target_link_libraries(binding tbb)

target_link_libraries(binding psp)

add_custom_command(TARGET psp POST_BUILD COMMAND ${CMAKE_COMMAND} -E copy $<TARGET_FILE:psp> ${PSP_PYTHON_SRC}/table/)
add_custom_command(TARGET binding POST_BUILD COMMAND ${CMAKE_COMMAND} -E copy $<TARGET_FILE:binding> ${PSP_PYTHON_SRC}/table/)

if(WIN32)
# inline arrow dlls
file(GLOB ARROW_DLLS "${PYTHON_PYARROW_LIBRARY_DIR}/*.dll")
Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@sc1f @texodus open to suggestions to get around copying all arrow dlls into our directory. In a "production" environment with a custom arrow build you won't have to worry about this, but with the packaged arrow wheel bringing a ton of stuff with it, we need to ensure all the links resolve.

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I don't understand what the requirement being solved by copying is, nor why a ton of stuff being brought is the instigator, so I'm not really in a position to offer constructive advice here. Copying is bad of course.


add_custom_command(TARGET binding POST_BUILD COMMAND ${CMAKE_COMMAND} -E copy ${ARROW_DLLS} ${PSP_PYTHON_SRC}/table/)
if(NOT TBB_FOUND)
add_custom_command(TARGET binding POST_BUILD COMMAND ${CMAKE_COMMAND} -E copy $<TARGET_FILE:tbb> ${PSP_PYTHON_SRC}/table/)
endif()
endif()
########################
else()
add_library(psp SHARED ${SOURCE_FILES} ${HEADER_FILES})
Expand All @@ -600,10 +635,10 @@ elseif(PSP_CPP_BUILD OR PSP_PYTHON_BUILD)
elseif(WIN32)
target_compile_definitions(psp PRIVATE PERSPECTIVE_EXPORTS=1)
target_compile_definitions(psp PRIVATE WIN32=1)
target_compile_definitions(psp PRIVATE _WIN32=1)
endif()

target_link_libraries(psp tbb)
set(CMAKE_LIBRARY_PATH ${CMAKE_LIBRARY_PATH} ${TBB_LIBRARY})
endif()


Expand Down
30 changes: 15 additions & 15 deletions cpp/perspective/src/include/perspective/scalar.h
Original file line number Diff line number Diff line change
Expand Up @@ -207,49 +207,49 @@ template <>
t_tscalar t_tscalar::coerce_numeric<bool>() const;

template <>
std::int64_t t_tscalar::get() const;
PERSPECTIVE_EXPORT std::int64_t t_tscalar::get() const;
Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@sc1f i assume all these specializations are necessary?

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yeah, they all need to be explicitly specialized.


template <>
std::int32_t t_tscalar::get() const;
PERSPECTIVE_EXPORT std::int32_t t_tscalar::get() const;

template <>
std::int16_t t_tscalar::get() const;
PERSPECTIVE_EXPORT std::int16_t t_tscalar::get() const;

template <>
std::int8_t t_tscalar::get() const;
PERSPECTIVE_EXPORT std::int8_t t_tscalar::get() const;

template <>
std::uint64_t t_tscalar::get() const;
PERSPECTIVE_EXPORT std::uint64_t t_tscalar::get() const;

template <>
std::uint32_t t_tscalar::get() const;
PERSPECTIVE_EXPORT std::uint32_t t_tscalar::get() const;

template <>
std::uint16_t t_tscalar::get() const;
PERSPECTIVE_EXPORT std::uint16_t t_tscalar::get() const;

template <>
std::uint8_t t_tscalar::get() const;
PERSPECTIVE_EXPORT std::uint8_t t_tscalar::get() const;

template <>
t_date t_tscalar::get() const;
PERSPECTIVE_EXPORT t_date t_tscalar::get() const;

template <>
t_time t_tscalar::get() const;
PERSPECTIVE_EXPORT t_time t_tscalar::get() const;

template <>
const char* t_tscalar::get() const;
PERSPECTIVE_EXPORT const char* t_tscalar::get() const;

template <>
t_none t_tscalar::get() const;
PERSPECTIVE_EXPORT t_none t_tscalar::get() const;

template <>
double t_tscalar::get() const;
PERSPECTIVE_EXPORT double t_tscalar::get() const;

template <>
float t_tscalar::get() const;
PERSPECTIVE_EXPORT float t_tscalar::get() const;

template <>
bool t_tscalar::get() const;
PERSPECTIVE_EXPORT bool t_tscalar::get() const;

template <template <typename COMPARED_T> class COMPARER_T>
bool
Expand Down
8 changes: 4 additions & 4 deletions cpp/perspective/src/include/perspective/sym_table.h
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@

namespace perspective {

class t_symtable {
class PERSPECTIVE_EXPORT t_symtable {
typedef tsl::hopscotch_map<const char*, const char*, t_cchar_umap_hash, t_cchar_umap_cmp>
t_mapping;

Expand All @@ -31,8 +31,8 @@ class t_symtable {
t_mapping m_mapping;
};

const char* get_interned_cstr(const char* s);
t_tscalar get_interned_tscalar(const char* s);
t_tscalar get_interned_tscalar(const t_tscalar& s);
PERSPECTIVE_EXPORT const char* get_interned_cstr(const char* s);
Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@sc1f this is currently being exposed as part of libpsp's public interface (it is used in the python cpp code). should this be public?

PERSPECTIVE_EXPORT t_tscalar get_interned_tscalar(const char* s);
PERSPECTIVE_EXPORT t_tscalar get_interned_tscalar(const t_tscalar& s);

} // end namespace perspective
2 changes: 1 addition & 1 deletion cpp/perspective/src/include/perspective/view_config.h
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,7 @@ namespace perspective {
* C++ types, we allow easy integration with binding languages.
*
*/
class t_view_config {
class PERSPECTIVE_EXPORT t_view_config {
public:
/**
* @brief Construct a `t_view_config` object.
Expand Down
27 changes: 26 additions & 1 deletion python/perspective/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -12,10 +12,32 @@ Python APIs for [perspective](https://github.com/finos/perspective) front end

### Dependencies

*PyArrow 0.15.1* is a required dependency for Perspective. Install it first:
`PyArrow==0.15.1` and `Numpy` are required build-time dependencies, and can both be installed via `pip`.

`pip install pyarrow==0.15.1`

It is recommended to have [https://github.com/intel/tbb](TBB) installed as a system dependency:

On MacOs:

`brew install tbb`

On CentOS/RHEL:

`yum install tbb-devel`

On Ubuntu:

`apt-get install libtbb-dev`

On Windows:

`choco install tbb`

Or install from source.

All other dependencies will be vendored if missing as part of Perspective's CMake build, but you can install them yourself and CMake will detect them via the usual mechanisms.

### Installation

To install the base package from pip:
Expand Down Expand Up @@ -59,3 +81,6 @@ Then run `yarn build`, and if a `.perspectiverc` config file has not been create
If you already have a `.perspectiverc` and want to reset your build configuration, simply run `yarn setup`.

Once the build successfully completes, run `yarn test_python` from the project root in order to verify operation.

### Windows
By default, perspective attempts to build utilizing Microsoft Visual Studio 2017 (MSVC 14.1). You may change this to older versions by editing `setup.py`.
3 changes: 2 additions & 1 deletion python/perspective/package.json
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,8 @@
"name": "perspective-python.node",
"version": "0.4.1",
"dependencies": {
"@finos/perspective": "^0.4.1"
"@finos/perspective": "^0.4.1",
"zerorpc": "^0.9.8"
Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@texodus please confirm

Copy link
Member

@texodus texodus Feb 8, 2020

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

At this point I see honestly zero reason to maintain this feature - perspective-python is better in every way. I propose we delete the perspective-node binding entirely.

},
"devDependencies": {
"@finos/perspective-webpack-plugin": "^0.4.1",
Expand Down
Loading