From 6b0a6e43d64760e35bbf3fcf6e2dc420a14450b2 Mon Sep 17 00:00:00 2001 From: Tim Paine Date: Tue, 28 Jan 2020 04:44:10 +0000 Subject: [PATCH 01/10] updates and fixes for windows build --- .appveyor.yml | 18 +- cmake/modules/FindPyArrow.cmake | 7 +- cpp/perspective/CMakeLists.txt | 89 +- .../src/include/perspective/scalar.h | 30 +- .../src/include/perspective/sym_table.h | 8 +- .../src/include/perspective/view_config.h | 2 +- python/perspective/README.md | 33 +- python/perspective/package.json | 3 +- .../include/perspective/python/numpy.h | 12 +- python/perspective/perspective/src/numpy.cpp | 5 + .../perspective/table/_date_validator.py | 4 +- .../perspective/tests/client/test_client.py | 11 +- .../tests/table/test_table_datetime.py | 1582 +++++++++-------- python/perspective/setup.py | 28 +- scripts/test_python.js | 5 +- yarn.lock | 144 +- 16 files changed, 1093 insertions(+), 888 deletions(-) diff --git a/.appveyor.yml b/.appveyor.yml index 9a40b6baeb..80fd843b2e 100644 --- a/.appveyor.yml +++ b/.appveyor.yml @@ -1,8 +1,8 @@ max_jobs: 8 image: - - Visual Studio 2015 - # - Visual Studio 2017 + # - Visual Studio 2015 + - Visual Studio 2017 configuration: - Release @@ -25,22 +25,16 @@ init: - if "%arch%"=="Win64" ( set arch= Win64) - echo %arch% - echo %APPVEYOR_BUILD_WORKER_IMAGE% - - if "%APPVEYOR_BUILD_WORKER_IMAGE%"=="Visual Studio 2017" ( set generator="Visual Studio 15 2017%arch%" ) - - if "%APPVEYOR_BUILD_WORKER_IMAGE%"=="Visual Studio 2015" ( set generator="Visual Studio 14 2015%arch%" ) before_build: - cmd: |- - cd cpp\perspective - mkdir build - cd build cmake --version - cmake .. -G %generator% -DPSP_WASM_BUILD=0 -DPSP_CPP_BUILD=1 -DPSP_CPP_BUILD_TESTS=1 -DPSP_CPP_BUILD_STRICT=1 -DBOOST_ROOT=C:\Libraries\boost_1_67_0 -DBoost_INCLUDE_DIRS=C:\Libraries\boost_1_67_0 + set BOOST_ROOT=C:\Libraries\boost_1_67_0 + set Boost_INCLUDE_DIRS=C:\Libraries\boost_1_67_0 + yarn build_script: - - msbuild psp.sln /p:Platform=x64 /m /p:Configuration=%CONFIGURATION% - - cp %CONFIGURATION%\psp.dll test\%CONFIGURATION% - - cp tbb-build\%CONFIGURATION%\tbb.dll test\%CONFIGURATION% - - .\test\%CONFIGURATION%\psp_test.exe + - yarn build_python --ci # to disable automatic tests test: off diff --git a/cmake/modules/FindPyArrow.cmake b/cmake/modules/FindPyArrow.cmake index a5e5d0d0b8..1ddd1dbca4 100644 --- a/cmake/modules/FindPyArrow.cmake +++ b/cmake/modules/FindPyArrow.cmake @@ -48,7 +48,12 @@ find_path(PYTHON_PYARROW_INCLUDE_DIR arrow/python/api.h set(PYTHON_PYARROW_LIBRARY_DIR ${__pyarrow_library_dirs}) -if (CMAKE_SYSTEM_NAME MATCHES "Darwin") +if(${CMAKE_SYSTEM_NAME} MATCHES "Windows") + # windows its just "arrow.dll" + set(PYTHON_PYARROW_PYTHON_SHARED_LIBRARY "arrow_python") + set(PYTHON_PYARROW_ARROW_SHARED_LIBRARY "arrow") + set(PYTHON_PYARROW_LIBRARIES ${PYTHON_PYARROW_PYTHON_SHARED_LIBRARY} ${PYTHON_PYARROW_ARROW_SHARED_LIBRARY}) +elseif (CMAKE_SYSTEM_NAME MATCHES "Darwin") # Link against pre-built libarrow on MacOS set(PYTHON_PYARROW_PYTHON_SHARED_LIBRARY ${PYTHON_PYARROW_LIBRARY_DIR}/${CMAKE_SHARED_LIBRARY_PREFIX}arrow_python.15.dylib) set(PYTHON_PYARROW_ARROW_SHARED_LIBRARY ${PYTHON_PYARROW_LIBRARY_DIR}/${CMAKE_SHARED_LIBRARY_PREFIX}arrow.15.dylib) diff --git a/cpp/perspective/CMakeLists.txt b/cpp/perspective/CMakeLists.txt index 5339b6e9c6..2dcf0285e3 100644 --- a/cpp/perspective/CMakeLists.txt +++ b/cpp/perspective/CMakeLists.txt @@ -1,9 +1,8 @@ cmake_minimum_required(VERSION 3.7.2) -project (psp) +project(psp) set(CMAKE_BUILD_TYPE "Release") -set(CMAKE_CXX_FLAGS_RELEASE "-O3 -DNDEBUG") -set(CMAKE_C_FLAGS_RELEASE "-O3 -DNDEBUG") + set(CMAKE_CXX_STANDARD 14) set(CMAKE_EXPORT_COMPILE_COMMANDS ON) @@ -197,6 +196,10 @@ message(WARNING "${BUILD_MESSAGE}") ####################### include_directories("${CMAKE_SOURCE_DIR}/src/include") +if(NOT WIN32) + set(CMAKE_CXX_FLAGS_RELEASE "-O3 -DNDEBUG") + set(CMAKE_C_FLAGS_RELEASE "-O3 -DNDEBUG") +endif() if (PSP_WASM_BUILD) #################### @@ -264,16 +267,31 @@ elseif(PSP_CPP_BUILD OR PSP_PYTHON_BUILD) ##################### # VANILLA CPP BUILD # ##################### - if(CMAKE_BUILD_TYPE_LOWER STREQUAL debug) - set(OPT_FLAGS " \ - -O0 \ - -g3 \ - ") + if(WIN32) + if(CMAKE_BUILD_TYPE_LOWER STREQUAL debug) + set(OPT_FLAGS " \ + /DEBUG \ + /Z7 \ + /Zi + ") + else() + set(OPT_FLAGS " \ + /NDEBUG \ + /O2 \ + ") + endif() else() - set(OPT_FLAGS " \ - -O3 \ - -g0 \ - ") + if(CMAKE_BUILD_TYPE_LOWER STREQUAL debug) + set(OPT_FLAGS " \ + -O0 \ + -g3 \ + ") + else() + set(OPT_FLAGS " \ + -O3 \ + -g0 \ + ") + endif() endif() set(SYNC_MODE_FLAGS "") set(ASYNC_MODE_FLAGS "") @@ -377,9 +395,7 @@ elseif(PSP_CPP_BUILD OR PSP_PYTHON_BUILD) message(FATAL_ERROR "${Red}PyArrow could not be located${ColorReset}") else() message(WARNING "${Cyan}PyArrow found: ${PYTHON_PYARROW_INCLUDE_DIR}${ColorReset}") - if (NOT WIN32) - message(WARNING "${Cyan}Using pre-built ${PYTHON_PYARROW_PYTHON_SHARED_LIBRARY} ${PYTHON_PYARROW_ARROW_SHARED_LIBRARY} from: ${PYTHON_PYARROW_LIBRARY_DIR}${ColorReset}") - endif() + message(WARNING "${Cyan}Using pre-built ${PYTHON_PYARROW_PYTHON_SHARED_LIBRARY} ${PYTHON_PYARROW_ARROW_SHARED_LIBRARY} from: ${PYTHON_PYARROW_LIBRARY_DIR}${ColorReset}") endif() include_directories(${PYTHON_PYARROW_INCLUDE_DIR}) link_directories(${PYTHON_PYARROW_LIBRARY_DIR}) @@ -392,7 +408,7 @@ psp_build_dep("hopscotch" "${PSP_CMAKE_MODULE_PATH}/hopscotch.txt.in") psp_build_dep("ordered-map" "${PSP_CMAKE_MODULE_PATH}/ordered-map.txt.in") # For all non-MacOS and non-linux builds, or if building WASM/CPP, build minimal arrow from source -if (WIN32 OR NOT PSP_PYTHON_BUILD) +if (NOT PSP_PYTHON_BUILD) # build arrow + dependencies from source for Emscripten and C++ message("${Cyan}Building minimal Apache Arrow${ColorReset}") @@ -516,7 +532,7 @@ set (PYTHON_SOURCE_FILES ) if (WIN32) - set(CMAKE_CXX_FLAGS " /EHsc") + set(CMAKE_CXX_FLAGS " /EHsc /MP") else() set(CMAKE_CXX_FLAGS " ${CMAKE_CXX_FLAGS} ${CMAKE_C_FLAGS}") endif() @@ -548,7 +564,9 @@ elseif(PSP_CPP_BUILD OR PSP_PYTHON_BUILD) set(module_install_rpath "@loader_path/") else() set(module_install_rpath "\$ORIGIN") - endif() + endif() + else() + set(CMAKE_SHARED_LIBRARY_PREFIX lib) endif() if(PSP_PYTHON_BUILD) @@ -562,28 +580,45 @@ elseif(PSP_CPP_BUILD OR PSP_PYTHON_BUILD) target_compile_definitions(psp PRIVATE PSP_ENABLE_PYTHON=1) target_compile_definitions(binding PRIVATE PSP_ENABLE_PYTHON=1) - target_compile_options(binding PRIVATE -Wdeprecated-declarations) - # Link against libarrow 0.15.1 in pyarrow directory for windows if (WIN32) - target_link_libraries(psp arrow) - target_link_libraries(binding arrow) + target_compile_definitions(binding PRIVATE WIN32=1) + target_compile_definitions(binding PRIVATE _WIN32=1) + target_link_libraries(psp ${Boost_FILESYSTEM_LIBRARY}) target_link_libraries(binding ${Boost_FILESYSTEM_LIBRARY}) + + # .dll not importable + set_property(TARGET binding PROPERTY SUFFIX .pyd) else() - target_link_libraries(psp ${PYTHON_PYARROW_LIBRARIES}) - target_link_libraries(binding ${PYTHON_PYARROW_LIBRARIES}) + target_compile_options(binding PRIVATE -Wdeprecated-declarations) set_property(TARGET psp PROPERTY INSTALL_RPATH ${CMAKE_INSTALL_RPATH} ${module_install_rpath} ${PYTHON_PYARROW_LIBRARY_DIR}) set_property(TARGET binding PROPERTY INSTALL_RPATH ${CMAKE_INSTALL_RPATH} ${module_install_rpath} ${PYTHON_PYARROW_LIBRARY_DIR}) - endif() + endif() + + target_link_libraries(psp ${PYTHON_PYARROW_LIBRARIES}) + target_link_libraries(binding ${PYTHON_PYARROW_LIBRARIES}) target_link_libraries(psp ${PYTHON_LIBRARIES}) target_link_libraries(binding ${PYTHON_LIBRARIES}) - target_link_libraries(binding psp) + + target_link_libraries(psp tbb) target_link_libraries(binding tbb) + target_link_libraries(binding psp) + add_custom_command(TARGET psp POST_BUILD COMMAND ${CMAKE_COMMAND} -E copy $ ${PSP_PYTHON_SRC}/table/) add_custom_command(TARGET binding POST_BUILD COMMAND ${CMAKE_COMMAND} -E copy $ ${PSP_PYTHON_SRC}/table/) + + if(WIN32) + # inline arrow dlls + file(GLOB ARROW_DLLS "${PYTHON_PYARROW_LIBRARY_DIR}/*.dll") + + add_custom_command(TARGET binding POST_BUILD COMMAND ${CMAKE_COMMAND} -E copy ${ARROW_DLLS} ${PSP_PYTHON_SRC}/table/) + if(NOT TBB_FOUND) + add_custom_command(TARGET binding POST_BUILD COMMAND ${CMAKE_COMMAND} -E copy $ ${PSP_PYTHON_SRC}/table/) + endif() + endif() ######################## else() add_library(psp SHARED ${SOURCE_FILES} ${HEADER_FILES}) @@ -600,10 +635,10 @@ elseif(PSP_CPP_BUILD OR PSP_PYTHON_BUILD) elseif(WIN32) target_compile_definitions(psp PRIVATE PERSPECTIVE_EXPORTS=1) target_compile_definitions(psp PRIVATE WIN32=1) + target_compile_definitions(psp PRIVATE _WIN32=1) endif() target_link_libraries(psp tbb) - set(CMAKE_LIBRARY_PATH ${CMAKE_LIBRARY_PATH} ${TBB_LIBRARY}) endif() diff --git a/cpp/perspective/src/include/perspective/scalar.h b/cpp/perspective/src/include/perspective/scalar.h index 4b77aaa0e8..d55a584aec 100644 --- a/cpp/perspective/src/include/perspective/scalar.h +++ b/cpp/perspective/src/include/perspective/scalar.h @@ -207,49 +207,49 @@ template <> t_tscalar t_tscalar::coerce_numeric() const; template <> -std::int64_t t_tscalar::get() const; +PERSPECTIVE_EXPORT std::int64_t t_tscalar::get() const; template <> -std::int32_t t_tscalar::get() const; +PERSPECTIVE_EXPORT std::int32_t t_tscalar::get() const; template <> -std::int16_t t_tscalar::get() const; +PERSPECTIVE_EXPORT std::int16_t t_tscalar::get() const; template <> -std::int8_t t_tscalar::get() const; +PERSPECTIVE_EXPORT std::int8_t t_tscalar::get() const; template <> -std::uint64_t t_tscalar::get() const; +PERSPECTIVE_EXPORT std::uint64_t t_tscalar::get() const; template <> -std::uint32_t t_tscalar::get() const; +PERSPECTIVE_EXPORT std::uint32_t t_tscalar::get() const; template <> -std::uint16_t t_tscalar::get() const; +PERSPECTIVE_EXPORT std::uint16_t t_tscalar::get() const; template <> -std::uint8_t t_tscalar::get() const; +PERSPECTIVE_EXPORT std::uint8_t t_tscalar::get() const; template <> -t_date t_tscalar::get() const; +PERSPECTIVE_EXPORT t_date t_tscalar::get() const; template <> -t_time t_tscalar::get() const; +PERSPECTIVE_EXPORT t_time t_tscalar::get() const; template <> -const char* t_tscalar::get() const; +PERSPECTIVE_EXPORT const char* t_tscalar::get() const; template <> -t_none t_tscalar::get() const; +PERSPECTIVE_EXPORT t_none t_tscalar::get() const; template <> -double t_tscalar::get() const; +PERSPECTIVE_EXPORT double t_tscalar::get() const; template <> -float t_tscalar::get() const; +PERSPECTIVE_EXPORT float t_tscalar::get() const; template <> -bool t_tscalar::get() const; +PERSPECTIVE_EXPORT bool t_tscalar::get() const; template