Skip to content

Commit

Permalink
ARROW-17016: [C++][Python] Move Arrow Python C++ tests into Cython (a…
Browse files Browse the repository at this point in the history
…pache#14117)

This PR tries to connect the PyArrow C++ tests with PyArrow tests so they can all be run from `pytest`. This will remove GoogleTest as a dependency for PyArrow and therefore a change is needed in the C++ tests so they return a Status which can then be checked through Cython/Python.

Example of pytest run:

```
pyarrow/tests/test_cpp_internals.py::test_owned_ref_moves PASSED
pyarrow/tests/test_cpp_internals.py::test_owned_ref_nogil_moves PASSED
pyarrow/tests/test_cpp_internals.py::test_check_pyerror_status PASSED
pyarrow/tests/test_cpp_internals.py::test_check_pyerror_status_nogil PASSED
pyarrow/tests/test_cpp_internals.py::test_restore_pyerror_basics PASSED
pyarrow/tests/test_cpp_internals.py::test_pybuffer_invalid_input_object PASSED
pyarrow/tests/test_cpp_internals.py::test_pybuffer_numpy_array PASSED
pyarrow/tests/test_cpp_internals.py::test_numpybuffer_numpy_array PASSED
pyarrow/tests/test_cpp_internals.py::test_python_decimal_to_string PASSED
pyarrow/tests/test_cpp_internals.py::test_infer_precision_and_scale PASSED
pyarrow/tests/test_cpp_internals.py::test_infer_precision_and_negative_scale PASSED
pyarrow/tests/test_cpp_internals.py::test_infer_all_leading_zeros PASSED
pyarrow/tests/test_cpp_internals.py::test_infer_all_leading_zeros_exponential_notation_positive PASSED
pyarrow/tests/test_cpp_internals.py::test_infer_all_leading_zeros_exponential_notation_negative PASSED
pyarrow/tests/test_cpp_internals.py::test_object_block_write_fails PASSED
pyarrow/tests/test_cpp_internals.py::test_mixed_type_fails PASSED
pyarrow/tests/test_cpp_internals.py::test_from_python_decimal_rescale_not_truncateable PASSED
pyarrow/tests/test_cpp_internals.py::test_from_python_decimal_rescale_truncateable PASSED
pyarrow/tests/test_cpp_internals.py::test_from_python_negative_decimal_rescale PASSED
pyarrow/tests/test_cpp_internals.py::test_decimal128_from_python_integer PASSED
pyarrow/tests/test_cpp_internals.py::test_decimal256_from_python_integer PASSED
pyarrow/tests/test_cpp_internals.py::test_decimal128_overflow_fails PASSED
pyarrow/tests/test_cpp_internals.py::test_decimal256_overflow_fails PASSED
pyarrow/tests/test_cpp_internals.py::test_none_and_nan PASSED
pyarrow/tests/test_cpp_internals.py::test_mixed_precision_and_scale PASSED
pyarrow/tests/test_cpp_internals.py::test_mixed_precision_and_scale_sequence_convert PASSED
pyarrow/tests/test_cpp_internals.py::test_simple_inference PASSED
pyarrow/tests/test_cpp_internals.py::test_update_with_nan PASSED
```

Lead-authored-by: Alenka Frim <[email protected]>
Co-authored-by: Antoine Pitrou <[email protected]>
Signed-off-by: Antoine Pitrou <[email protected]>
  • Loading branch information
2 people authored and fatemehp committed Oct 17, 2022
1 parent 40bda09 commit 45ff5ff
Show file tree
Hide file tree
Showing 8 changed files with 519 additions and 327 deletions.
3 changes: 2 additions & 1 deletion python/CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -410,7 +410,8 @@ set(CYTHON_EXTENSIONS
_feather
_fs
_hdfsio
_json)
_json
_pyarrow_cpp_tests)

set(LINK_LIBS ArrowPython::arrow_python_shared)

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -15,18 +15,19 @@
# specific language governing permissions and limitations
# under the License.

#
# arrow/python_test_main
#
# distutils: language = c++
# cython: language_level = 3

from pyarrow.includes.common cimport *
from pyarrow.includes.libarrow cimport CStatus


ctypedef CStatus cb_test_func()

cdef extern from "arrow/python/python_test.h" namespace "arrow::py::testing" nogil:

if(PYARROW_BUILD_TESTS)
add_library(arrow/python_test_main STATIC test_main.cc)
cdef cppclass CTestCase "arrow::py::testing::TestCase":
c_string name
cb_test_func func

if(APPLE)
target_link_libraries(arrow/python_test_main GTest::gtest dl)
set_target_properties(arrow/python_test_main PROPERTIES LINK_FLAGS
"-undefined dynamic_lookup")
else()
target_link_libraries(arrow/python_test_main GTest::gtest pthread dl)
endif()
endif()
vector[CTestCase] GetCppTestCases()
62 changes: 62 additions & 0 deletions python/pyarrow/_pyarrow_cpp_tests.pyx
Original file line number Diff line number Diff line change
@@ -0,0 +1,62 @@
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.

# cython: profile=False, binding=True
# distutils: language = c++

from pyarrow.includes.common cimport *
from pyarrow.includes.libarrow cimport *
from pyarrow.lib cimport check_status

from pyarrow.lib import frombytes


cdef class CppTestCase:
"""
A simple wrapper for a C++ test case.
"""
cdef:
CTestCase c_case

@staticmethod
cdef wrap(CTestCase c_case):
cdef:
CppTestCase obj
obj = CppTestCase.__new__(CppTestCase)
obj.c_case = c_case
return obj

@property
def name(self):
return frombytes(self.c_case.name)

def __repr__(self):
return f"<{self.__class__.__name__} {self.name!r}>"

def __call__(self):
check_status(self.c_case.func())


def get_cpp_tests():
"""
Get a list of C++ test cases.
"""
cases = []
c_cases = GetCppTestCases()
for c_case in c_cases:
cases.append(CppTestCase.wrap(c_case))
return cases
175 changes: 1 addition & 174 deletions python/pyarrow/src/CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -75,10 +75,7 @@ set(Python3_FIND_FRAMEWORK "LAST")
find_package(Python3Alt 3.7 REQUIRED)
include_directories(SYSTEM ${NUMPY_INCLUDE_DIRS} ${PYTHON_INCLUDE_DIRS} ${ARROW_INCLUDE_DIR} src)

add_custom_target(arrow_python-all)
add_custom_target(arrow_python)
add_custom_target(arrow_python-tests)
add_dependencies(arrow_python-all arrow_python arrow_python-tests)

set(ARROW_PYTHON_SRCS
arrow_to_pandas.cc
Expand All @@ -96,6 +93,7 @@ set(ARROW_PYTHON_SRCS
ipc.cc
numpy_convert.cc
numpy_to_arrow.cc
python_test.cc
python_to_arrow.cc
pyarrow.cc
serialize.cc
Expand Down Expand Up @@ -312,174 +310,3 @@ if(CMAKE_CXX_COMPILER_ID STREQUAL "AppleClang" OR CMAKE_CXX_COMPILER_ID STREQUAL
endif()

arrow_install_all_headers("arrow/python")

# ----------------------------------------------------------------------

#
# Tests
# The tests will be moved to Cython and are currently supported for bundled GTest
# Follow-up: https://issues.apache.org/jira/browse/ARROW-17016
#

if(ARROW_BUILD_TESTS)

enable_testing()
set(GTEST_ROOT ${ARROW_CPP_SOURCE_DIR}/${ARROW_BUILD_DIR}/googletest_ep-prefix)

# GTest must be built from source
if(EXISTS ${ARROW_CPP_SOURCE_DIR}/${ARROW_BUILD_DIR}/googletest_ep-prefix)

# Set necessary paths for cmake to find GTest
set(GTEST_INCLUDE_DIR "${GTEST_ROOT}/include")
set(GTEST_LIBRARY ${GTEST_ROOT}/lib)
set(GTEST_MAIN_LIBRARY ${GTEST_ROOT}/lib)

#
# Taken from Matlab CMakeLists.txt (enable_gtest and build_gtest)
#

set(ARROW_GTEST_PREFIX "${GTEST_ROOT}")
set(ARROW_GTEST_MAIN_PREFIX "${GTEST_ROOT}")

if(WIN32)
set(ARROW_GTEST_SHARED_LIB_DIR "${ARROW_GTEST_PREFIX}/bin")
set(ARROW_GTEST_MAIN_SHARED_LIB_DIR "${ARROW_GTEST_MAIN_PREFIX}/bin")

set(ARROW_GTEST_LINK_LIB_DIR "${ARROW_GTEST_PREFIX}/lib")
set(ARROW_GTEST_LINK_LIB
"${ARROW_GTEST_LINK_LIB_DIR}/${CMAKE_IMPORT_LIBRARY_PREFIX}gtestd${CMAKE_IMPORT_LIBRARY_SUFFIX}"
)

set(ARROW_GTEST_MAIN_LINK_LIB_DIR "${ARROW_GTEST_MAIN_PREFIX}/lib")
set(ARROW_GTEST_MAIN_LINK_LIB
"${ARROW_GTEST_MAIN_LINK_LIB_DIR}/${CMAKE_IMPORT_LIBRARY_PREFIX}gtest_maind${CMAKE_IMPORT_LIBRARY_SUFFIX}"
)
else()
set(ARROW_GTEST_SHARED_LIB_DIR "${ARROW_GTEST_PREFIX}/lib")
set(ARROW_GTEST_MAIN_SHARED_LIB_DIR "${ARROW_GTEST_MAIN_PREFIX}/lib")
endif()

set(ARROW_GTEST_INCLUDE_DIR "${ARROW_GTEST_PREFIX}/include")
set(ARROW_GTEST_SHARED_LIB
"${ARROW_GTEST_SHARED_LIB_DIR}/${CMAKE_SHARED_LIBRARY_PREFIX}gtestd${CMAKE_SHARED_LIBRARY_SUFFIX}"
)

set(ARROW_GTEST_MAIN_INCLUDE_DIR "${ARROW_GTEST_MAIN_PREFIX}/include")
set(ARROW_GTEST_MAIN_SHARED_LIB
"${ARROW_GTEST_MAIN_SHARED_LIB_DIR}/${CMAKE_SHARED_LIBRARY_PREFIX}gtest_maind${CMAKE_SHARED_LIBRARY_SUFFIX}"
)

file(MAKE_DIRECTORY "${ARROW_GTEST_INCLUDE_DIR}")

# Create target GTest::gtest
add_library(GTest::gtest SHARED IMPORTED)
set_target_properties(GTest::gtest
PROPERTIES IMPORTED_LOCATION ${ARROW_GTEST_SHARED_LIB}
INTERFACE_INCLUDE_DIRECTORIES
${ARROW_GTEST_INCLUDE_DIR})
if(WIN32)
set_target_properties(GTest::gtest PROPERTIES IMPORTED_IMPLIB ${ARROW_GTEST_LINK_LIB})
endif()

# ArrowTesting
# needed to be able to use arrow_testing_shared target
find_package(ArrowTesting REQUIRED)

add_custom_target(all-tests)

add_library(arrow_python_test_main STATIC util/test_main.cc)

target_link_libraries(arrow_python_test_main GTest::gtest)
target_include_directories(arrow_python_test_main SYSTEM
PUBLIC ${ARROW_PYTHON_INCLUDES})

# Link libraries to avoid include error on Linux
if(ARROW_TEST_LINKAGE STREQUAL shared)
target_link_libraries(arrow_python_test_main Arrow::arrow_shared)
else()
target_link_libraries(arrow_python_test_main Arrow::arrow_static)
endif()

if(APPLE)
target_link_libraries(arrow_python_test_main ${CMAKE_DL_LIBS})
set_target_properties(arrow_python_test_main PROPERTIES LINK_FLAGS
"-undefined dynamic_lookup")
elseif(NOT MSVC)
target_link_libraries(arrow_python_test_main pthread ${CMAKE_DL_LIBS})
endif()

if(ARROW_TEST_LINKAGE STREQUAL shared)
set(ARROW_PYTHON_TEST_LINK_LIBS arrow_python_test_main arrow_python_shared
ArrowTesting::arrow_testing_shared)
else()
set(ARROW_PYTHON_TEST_LINK_LIBS arrow_python_test_main arrow_python_static
ArrowTesting::arrow_testing_static)
endif()

#
# Add a test case
#

set(REL_TEST_NAME "python_test")
get_filename_component(TEST_NAME ${REL_TEST_NAME} NAME_WE)
set(TEST_NAME "arrow-${TEST_NAME}")
set(SOURCES "${REL_TEST_NAME}.cc")

# # Make sure the executable name contains only hyphens, not underscores
string(REPLACE "_" "-" TEST_NAME ${TEST_NAME})

set(TEST_PATH "${CMAKE_BINARY_DIR}/${TEST_NAME}")
add_executable(${TEST_NAME} ${SOURCES})

# We need to set the correct RPATH so that dependencies
set_target_properties(${TEST_NAME}
PROPERTIES BUILD_WITH_INSTALL_RPATH TRUE
INSTALL_RPATH_USE_LINK_PATH TRUE
INSTALL_RPATH
"${PYTHON_SOURCE_DIR}/pyarrow;$ENV{CONDA_PREFIX}/lib")

# Customize link libraries
target_link_libraries(${TEST_NAME} PRIVATE "${ARROW_PYTHON_TEST_LINK_LIBS}")
# Extra link libs
target_link_libraries(${TEST_NAME} PRIVATE ${PYTHON_LIBRARIES})
# Extra includes
target_include_directories(${TEST_NAME} SYSTEM PUBLIC "${ARROW_PYTHON_INCLUDES}")

# Add the test
if(WIN32)
add_test(${TEST_NAME} ${TEST_PATH})
else()
add_test(${TEST_NAME}
${ARROW_CPP_SOURCE_DIR}/build-support/run-test.sh
${CMAKE_BINARY_DIR}
test
${TEST_PATH})
endif()

# Add test as dependency of relevant targets
add_dependencies(all-tests ${TEST_NAME})
add_dependencies(arrow_python-tests ${TEST_NAME})

set(LABELS)
list(APPEND LABELS "unittest" arrow_python-tests)

# ensure there is a cmake target which exercises tests with this LABEL
set(LABEL_TEST_NAME "test-arrow_python-tests")
if(NOT TARGET ${LABEL_TEST_NAME})
add_custom_target(${LABEL_TEST_NAME}
ctest -L "${LABEL}" --output-on-failure
USES_TERMINAL)
endif()
# ensure the test is (re)built before the LABEL test runs
add_dependencies(${LABEL_TEST_NAME} ${TEST_NAME})

set_property(TEST ${TEST_NAME}
APPEND
PROPERTY LABELS ${LABELS})

else()
message(STATUS "Tests for PyArrow C++ not build")
message(STATUS "Set -DGTest_SOURCE=BUNDLED when building Arrow C++
to enable building tests for PyArrow C++")
endif()
endif()
Loading

0 comments on commit 45ff5ff

Please sign in to comment.