From 45ff5fffb4cb2c2f010ce58350b7365afc0ca6bb Mon Sep 17 00:00:00 2001 From: Alenka Frim Date: Mon, 3 Oct 2022 18:45:00 +0200 Subject: [PATCH] ARROW-17016: [C++][Python] Move Arrow Python C++ tests into Cython (#14117) This PR tries to connect the PyArrow C++ tests with PyArrow tests so they can all be run from `pytest`. This will remove GoogleTest as a dependency for PyArrow and therefore a change is needed in the C++ tests so they return a Status which can then be checked through Cython/Python. Example of pytest run: ``` pyarrow/tests/test_cpp_internals.py::test_owned_ref_moves PASSED pyarrow/tests/test_cpp_internals.py::test_owned_ref_nogil_moves PASSED pyarrow/tests/test_cpp_internals.py::test_check_pyerror_status PASSED pyarrow/tests/test_cpp_internals.py::test_check_pyerror_status_nogil PASSED pyarrow/tests/test_cpp_internals.py::test_restore_pyerror_basics PASSED pyarrow/tests/test_cpp_internals.py::test_pybuffer_invalid_input_object PASSED pyarrow/tests/test_cpp_internals.py::test_pybuffer_numpy_array PASSED pyarrow/tests/test_cpp_internals.py::test_numpybuffer_numpy_array PASSED pyarrow/tests/test_cpp_internals.py::test_python_decimal_to_string PASSED pyarrow/tests/test_cpp_internals.py::test_infer_precision_and_scale PASSED pyarrow/tests/test_cpp_internals.py::test_infer_precision_and_negative_scale PASSED pyarrow/tests/test_cpp_internals.py::test_infer_all_leading_zeros PASSED pyarrow/tests/test_cpp_internals.py::test_infer_all_leading_zeros_exponential_notation_positive PASSED pyarrow/tests/test_cpp_internals.py::test_infer_all_leading_zeros_exponential_notation_negative PASSED pyarrow/tests/test_cpp_internals.py::test_object_block_write_fails PASSED pyarrow/tests/test_cpp_internals.py::test_mixed_type_fails PASSED pyarrow/tests/test_cpp_internals.py::test_from_python_decimal_rescale_not_truncateable PASSED pyarrow/tests/test_cpp_internals.py::test_from_python_decimal_rescale_truncateable PASSED pyarrow/tests/test_cpp_internals.py::test_from_python_negative_decimal_rescale PASSED pyarrow/tests/test_cpp_internals.py::test_decimal128_from_python_integer PASSED pyarrow/tests/test_cpp_internals.py::test_decimal256_from_python_integer PASSED pyarrow/tests/test_cpp_internals.py::test_decimal128_overflow_fails PASSED pyarrow/tests/test_cpp_internals.py::test_decimal256_overflow_fails PASSED pyarrow/tests/test_cpp_internals.py::test_none_and_nan PASSED pyarrow/tests/test_cpp_internals.py::test_mixed_precision_and_scale PASSED pyarrow/tests/test_cpp_internals.py::test_mixed_precision_and_scale_sequence_convert PASSED pyarrow/tests/test_cpp_internals.py::test_simple_inference PASSED pyarrow/tests/test_cpp_internals.py::test_update_with_nan PASSED ``` Lead-authored-by: Alenka Frim Co-authored-by: Antoine Pitrou Signed-off-by: Antoine Pitrou --- python/CMakeLists.txt | 3 +- .../CMakeLists.txt => _pyarrow_cpp_tests.pxd} | 27 +- python/pyarrow/_pyarrow_cpp_tests.pyx | 62 +++ python/pyarrow/src/CMakeLists.txt | 175 +----- python/pyarrow/src/python_test.cc | 510 +++++++++++++----- .../src/{util/test_main.cc => python_test.h} | 35 +- python/pyarrow/tests/test_cpp_internals.py | 33 ++ python/setup.py | 1 + 8 files changed, 519 insertions(+), 327 deletions(-) rename python/pyarrow/{src/util/CMakeLists.txt => _pyarrow_cpp_tests.pxd} (63%) create mode 100644 python/pyarrow/_pyarrow_cpp_tests.pyx rename python/pyarrow/src/{util/test_main.cc => python_test.h} (67%) create mode 100644 python/pyarrow/tests/test_cpp_internals.py diff --git a/python/CMakeLists.txt b/python/CMakeLists.txt index 846f30f4b546d..1013af4fe856e 100644 --- a/python/CMakeLists.txt +++ b/python/CMakeLists.txt @@ -410,7 +410,8 @@ set(CYTHON_EXTENSIONS _feather _fs _hdfsio - _json) + _json + _pyarrow_cpp_tests) set(LINK_LIBS ArrowPython::arrow_python_shared) diff --git a/python/pyarrow/src/util/CMakeLists.txt b/python/pyarrow/_pyarrow_cpp_tests.pxd similarity index 63% rename from python/pyarrow/src/util/CMakeLists.txt rename to python/pyarrow/_pyarrow_cpp_tests.pxd index 74141bebc8bf2..91c0220d73108 100644 --- a/python/pyarrow/src/util/CMakeLists.txt +++ b/python/pyarrow/_pyarrow_cpp_tests.pxd @@ -15,18 +15,19 @@ # specific language governing permissions and limitations # under the License. -# -# arrow/python_test_main -# +# distutils: language = c++ +# cython: language_level = 3 + +from pyarrow.includes.common cimport * +from pyarrow.includes.libarrow cimport CStatus + + +ctypedef CStatus cb_test_func() + +cdef extern from "arrow/python/python_test.h" namespace "arrow::py::testing" nogil: -if(PYARROW_BUILD_TESTS) - add_library(arrow/python_test_main STATIC test_main.cc) + cdef cppclass CTestCase "arrow::py::testing::TestCase": + c_string name + cb_test_func func - if(APPLE) - target_link_libraries(arrow/python_test_main GTest::gtest dl) - set_target_properties(arrow/python_test_main PROPERTIES LINK_FLAGS - "-undefined dynamic_lookup") - else() - target_link_libraries(arrow/python_test_main GTest::gtest pthread dl) - endif() -endif() + vector[CTestCase] GetCppTestCases() diff --git a/python/pyarrow/_pyarrow_cpp_tests.pyx b/python/pyarrow/_pyarrow_cpp_tests.pyx new file mode 100644 index 0000000000000..adb148351306c --- /dev/null +++ b/python/pyarrow/_pyarrow_cpp_tests.pyx @@ -0,0 +1,62 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# cython: profile=False, binding=True +# distutils: language = c++ + +from pyarrow.includes.common cimport * +from pyarrow.includes.libarrow cimport * +from pyarrow.lib cimport check_status + +from pyarrow.lib import frombytes + + +cdef class CppTestCase: + """ + A simple wrapper for a C++ test case. + """ + cdef: + CTestCase c_case + + @staticmethod + cdef wrap(CTestCase c_case): + cdef: + CppTestCase obj + obj = CppTestCase.__new__(CppTestCase) + obj.c_case = c_case + return obj + + @property + def name(self): + return frombytes(self.c_case.name) + + def __repr__(self): + return f"<{self.__class__.__name__} {self.name!r}>" + + def __call__(self): + check_status(self.c_case.func()) + + +def get_cpp_tests(): + """ + Get a list of C++ test cases. + """ + cases = [] + c_cases = GetCppTestCases() + for c_case in c_cases: + cases.append(CppTestCase.wrap(c_case)) + return cases diff --git a/python/pyarrow/src/CMakeLists.txt b/python/pyarrow/src/CMakeLists.txt index 03c06d32b7a09..f60b3e67ece32 100644 --- a/python/pyarrow/src/CMakeLists.txt +++ b/python/pyarrow/src/CMakeLists.txt @@ -75,10 +75,7 @@ set(Python3_FIND_FRAMEWORK "LAST") find_package(Python3Alt 3.7 REQUIRED) include_directories(SYSTEM ${NUMPY_INCLUDE_DIRS} ${PYTHON_INCLUDE_DIRS} ${ARROW_INCLUDE_DIR} src) -add_custom_target(arrow_python-all) add_custom_target(arrow_python) -add_custom_target(arrow_python-tests) -add_dependencies(arrow_python-all arrow_python arrow_python-tests) set(ARROW_PYTHON_SRCS arrow_to_pandas.cc @@ -96,6 +93,7 @@ set(ARROW_PYTHON_SRCS ipc.cc numpy_convert.cc numpy_to_arrow.cc + python_test.cc python_to_arrow.cc pyarrow.cc serialize.cc @@ -312,174 +310,3 @@ if(CMAKE_CXX_COMPILER_ID STREQUAL "AppleClang" OR CMAKE_CXX_COMPILER_ID STREQUAL endif() arrow_install_all_headers("arrow/python") - -# ---------------------------------------------------------------------- - -# -# Tests -# The tests will be moved to Cython and are currently supported for bundled GTest -# Follow-up: https://issues.apache.org/jira/browse/ARROW-17016 -# - -if(ARROW_BUILD_TESTS) - - enable_testing() - set(GTEST_ROOT ${ARROW_CPP_SOURCE_DIR}/${ARROW_BUILD_DIR}/googletest_ep-prefix) - - # GTest must be built from source - if(EXISTS ${ARROW_CPP_SOURCE_DIR}/${ARROW_BUILD_DIR}/googletest_ep-prefix) - - # Set necessary paths for cmake to find GTest - set(GTEST_INCLUDE_DIR "${GTEST_ROOT}/include") - set(GTEST_LIBRARY ${GTEST_ROOT}/lib) - set(GTEST_MAIN_LIBRARY ${GTEST_ROOT}/lib) - - # - # Taken from Matlab CMakeLists.txt (enable_gtest and build_gtest) - # - - set(ARROW_GTEST_PREFIX "${GTEST_ROOT}") - set(ARROW_GTEST_MAIN_PREFIX "${GTEST_ROOT}") - - if(WIN32) - set(ARROW_GTEST_SHARED_LIB_DIR "${ARROW_GTEST_PREFIX}/bin") - set(ARROW_GTEST_MAIN_SHARED_LIB_DIR "${ARROW_GTEST_MAIN_PREFIX}/bin") - - set(ARROW_GTEST_LINK_LIB_DIR "${ARROW_GTEST_PREFIX}/lib") - set(ARROW_GTEST_LINK_LIB - "${ARROW_GTEST_LINK_LIB_DIR}/${CMAKE_IMPORT_LIBRARY_PREFIX}gtestd${CMAKE_IMPORT_LIBRARY_SUFFIX}" - ) - - set(ARROW_GTEST_MAIN_LINK_LIB_DIR "${ARROW_GTEST_MAIN_PREFIX}/lib") - set(ARROW_GTEST_MAIN_LINK_LIB - "${ARROW_GTEST_MAIN_LINK_LIB_DIR}/${CMAKE_IMPORT_LIBRARY_PREFIX}gtest_maind${CMAKE_IMPORT_LIBRARY_SUFFIX}" - ) - else() - set(ARROW_GTEST_SHARED_LIB_DIR "${ARROW_GTEST_PREFIX}/lib") - set(ARROW_GTEST_MAIN_SHARED_LIB_DIR "${ARROW_GTEST_MAIN_PREFIX}/lib") - endif() - - set(ARROW_GTEST_INCLUDE_DIR "${ARROW_GTEST_PREFIX}/include") - set(ARROW_GTEST_SHARED_LIB - "${ARROW_GTEST_SHARED_LIB_DIR}/${CMAKE_SHARED_LIBRARY_PREFIX}gtestd${CMAKE_SHARED_LIBRARY_SUFFIX}" - ) - - set(ARROW_GTEST_MAIN_INCLUDE_DIR "${ARROW_GTEST_MAIN_PREFIX}/include") - set(ARROW_GTEST_MAIN_SHARED_LIB - "${ARROW_GTEST_MAIN_SHARED_LIB_DIR}/${CMAKE_SHARED_LIBRARY_PREFIX}gtest_maind${CMAKE_SHARED_LIBRARY_SUFFIX}" - ) - - file(MAKE_DIRECTORY "${ARROW_GTEST_INCLUDE_DIR}") - - # Create target GTest::gtest - add_library(GTest::gtest SHARED IMPORTED) - set_target_properties(GTest::gtest - PROPERTIES IMPORTED_LOCATION ${ARROW_GTEST_SHARED_LIB} - INTERFACE_INCLUDE_DIRECTORIES - ${ARROW_GTEST_INCLUDE_DIR}) - if(WIN32) - set_target_properties(GTest::gtest PROPERTIES IMPORTED_IMPLIB ${ARROW_GTEST_LINK_LIB}) - endif() - - # ArrowTesting - # needed to be able to use arrow_testing_shared target - find_package(ArrowTesting REQUIRED) - - add_custom_target(all-tests) - - add_library(arrow_python_test_main STATIC util/test_main.cc) - - target_link_libraries(arrow_python_test_main GTest::gtest) - target_include_directories(arrow_python_test_main SYSTEM - PUBLIC ${ARROW_PYTHON_INCLUDES}) - - # Link libraries to avoid include error on Linux - if(ARROW_TEST_LINKAGE STREQUAL shared) - target_link_libraries(arrow_python_test_main Arrow::arrow_shared) - else() - target_link_libraries(arrow_python_test_main Arrow::arrow_static) - endif() - - if(APPLE) - target_link_libraries(arrow_python_test_main ${CMAKE_DL_LIBS}) - set_target_properties(arrow_python_test_main PROPERTIES LINK_FLAGS - "-undefined dynamic_lookup") - elseif(NOT MSVC) - target_link_libraries(arrow_python_test_main pthread ${CMAKE_DL_LIBS}) - endif() - - if(ARROW_TEST_LINKAGE STREQUAL shared) - set(ARROW_PYTHON_TEST_LINK_LIBS arrow_python_test_main arrow_python_shared - ArrowTesting::arrow_testing_shared) - else() - set(ARROW_PYTHON_TEST_LINK_LIBS arrow_python_test_main arrow_python_static - ArrowTesting::arrow_testing_static) - endif() - - # - # Add a test case - # - - set(REL_TEST_NAME "python_test") - get_filename_component(TEST_NAME ${REL_TEST_NAME} NAME_WE) - set(TEST_NAME "arrow-${TEST_NAME}") - set(SOURCES "${REL_TEST_NAME}.cc") - - # # Make sure the executable name contains only hyphens, not underscores - string(REPLACE "_" "-" TEST_NAME ${TEST_NAME}) - - set(TEST_PATH "${CMAKE_BINARY_DIR}/${TEST_NAME}") - add_executable(${TEST_NAME} ${SOURCES}) - - # We need to set the correct RPATH so that dependencies - set_target_properties(${TEST_NAME} - PROPERTIES BUILD_WITH_INSTALL_RPATH TRUE - INSTALL_RPATH_USE_LINK_PATH TRUE - INSTALL_RPATH - "${PYTHON_SOURCE_DIR}/pyarrow;$ENV{CONDA_PREFIX}/lib") - - # Customize link libraries - target_link_libraries(${TEST_NAME} PRIVATE "${ARROW_PYTHON_TEST_LINK_LIBS}") - # Extra link libs - target_link_libraries(${TEST_NAME} PRIVATE ${PYTHON_LIBRARIES}) - # Extra includes - target_include_directories(${TEST_NAME} SYSTEM PUBLIC "${ARROW_PYTHON_INCLUDES}") - - # Add the test - if(WIN32) - add_test(${TEST_NAME} ${TEST_PATH}) - else() - add_test(${TEST_NAME} - ${ARROW_CPP_SOURCE_DIR}/build-support/run-test.sh - ${CMAKE_BINARY_DIR} - test - ${TEST_PATH}) - endif() - - # Add test as dependency of relevant targets - add_dependencies(all-tests ${TEST_NAME}) - add_dependencies(arrow_python-tests ${TEST_NAME}) - - set(LABELS) - list(APPEND LABELS "unittest" arrow_python-tests) - - # ensure there is a cmake target which exercises tests with this LABEL - set(LABEL_TEST_NAME "test-arrow_python-tests") - if(NOT TARGET ${LABEL_TEST_NAME}) - add_custom_target(${LABEL_TEST_NAME} - ctest -L "${LABEL}" --output-on-failure - USES_TERMINAL) - endif() - # ensure the test is (re)built before the LABEL test runs - add_dependencies(${LABEL_TEST_NAME} ${TEST_NAME}) - - set_property(TEST ${TEST_NAME} - APPEND - PROPERTY LABELS ${LABELS}) - - else() - message(STATUS "Tests for PyArrow C++ not build") - message(STATUS "Set -DGTest_SOURCE=BUNDLED when building Arrow C++ - to enable building tests for PyArrow C++") - endif() -endif() diff --git a/python/pyarrow/src/python_test.cc b/python/pyarrow/src/python_test.cc index 865d4cf5c0f83..844d1b8cccb79 100644 --- a/python/pyarrow/src/python_test.cc +++ b/python/pyarrow/src/python_test.cc @@ -15,8 +15,6 @@ // specific language governing permissions and limitations // under the License. -#include "gtest/gtest.h" - #include #include #include @@ -27,25 +25,86 @@ #include "arrow/array.h" #include "arrow/array/builder_binary.h" #include "arrow/table.h" -#include "arrow/testing/gtest_util.h" #include "arrow/util/decimal.h" +#include "arrow/util/logging.h" #include "arrow_to_pandas.h" #include "decimal.h" #include "helpers.h" #include "numpy_convert.h" #include "numpy_interop.h" +#include "python_test.h" #include "python_to_arrow.h" -#include "arrow/util/checked_cast.h" -#include "arrow/util/logging.h" + +#define ASSERT_EQ(x, y) { \ + auto&& _left = (x); \ + auto&& _right = (y); \ + if (_left != _right) { \ + return Status::Invalid("Expected equality between `", #x, "` and `", #y, \ + "`, but ", _left, " != ", _right); \ + } \ +} + +#define ASSERT_NE(x, y) { \ + auto&& _left = (x); \ + auto&& _right = (y); \ + if (_left == _right) { \ + return Status::Invalid("Expected inequality between `", #x, "` and `", #y, \ + "`, but ", _left, " == ", _right); \ + } \ +} + +#define ASSERT_FALSE(v) { \ + auto&& _v = (v); \ + if (!!_v) { \ + return Status::Invalid("Expected `", #v, "` to evaluate to false, but got ", _v); \ + } \ +} + +#define ASSERT_TRUE(v){ \ + auto&& _v = (v); \ + if (!_v) { \ + return Status::Invalid("Expected `", #v, "` to evaluate to true, but got ", _v); \ + } \ +} + +#define ASSERT_FALSE_MSG(v, msg) { \ + auto&& _v = (v); \ + if (!!_v) { \ + return Status::Invalid("Expected `", #v, "` to evaluate to false, but got ", \ + _v, ": ", msg); \ + } \ +} + +#define ASSERT_TRUE_MSG(v, msg) { \ + auto&& _v = (v); \ + if (!_v) { \ + return Status::Invalid("Expected `", #v, "` to evaluate to true, but got ", \ + _v, ": ", msg); \ + } \ +} + +#define ASSERT_OK(expr) { \ + for (::arrow::Status _st = ::arrow::internal::GenericToStatus((expr)); !_st.ok();) \ + return Status::Invalid("`", #expr, "` failed with ", _st.ToString()); \ +} + +#define ASSERT_RAISES(code, expr) { \ + for (::arrow::Status _st_expr = ::arrow::internal::GenericToStatus((expr)); \ + !_st_expr.Is##code();) \ + return Status::Invalid("Expected `", #expr, "` to fail with ", \ + #code, ", but got ", _st_expr.ToString()); \ +} namespace arrow { using internal::checked_cast; namespace py { +namespace testing { +namespace { -TEST(OwnedRef, TestMoves) { +Status TestOwnedRefMoves() { std::vector vec; PyObject *u, *v; u = PyList_New(0); @@ -59,9 +118,10 @@ TEST(OwnedRef, TestMoves) { vec.emplace_back(v); ASSERT_EQ(Py_REFCNT(u), 1); ASSERT_EQ(Py_REFCNT(v), 1); + return Status::OK(); } -TEST(OwnedRefNoGIL, TestMoves) { +Status TestOwnedRefNoGILMoves() { PyAcquireGIL lock; lock.release(); @@ -82,6 +142,7 @@ TEST(OwnedRefNoGIL, TestMoves) { vec.emplace_back(v); ASSERT_EQ(Py_REFCNT(u), 1); ASSERT_EQ(Py_REFCNT(v), 1); + return Status::OK(); } } @@ -92,8 +153,9 @@ std::string FormatPythonException(const std::string& exc_class_name) { return ss.str(); } -TEST(CheckPyError, TestStatus) { +Status TestCheckPyErrorStatus() { Status st; + std::string expected_detail = ""; auto check_error = [](Status& st, const char* expected_message = "some error", std::string expected_detail = "") { @@ -105,34 +167,35 @@ TEST(CheckPyError, TestStatus) { ASSERT_NE(detail, nullptr); ASSERT_EQ(detail->ToString(), expected_detail); } + return Status::OK(); }; for (PyObject* exc_type : {PyExc_Exception, PyExc_SyntaxError}) { PyErr_SetString(exc_type, "some error"); - check_error(st); + ASSERT_OK(check_error(st)); ASSERT_TRUE(st.IsUnknownError()); } PyErr_SetString(PyExc_TypeError, "some error"); - check_error(st, "some error", FormatPythonException("TypeError")); + ASSERT_OK(check_error(st, "some error", FormatPythonException("TypeError"))); ASSERT_TRUE(st.IsTypeError()); PyErr_SetString(PyExc_ValueError, "some error"); - check_error(st); + ASSERT_OK(check_error(st)); ASSERT_TRUE(st.IsInvalid()); PyErr_SetString(PyExc_KeyError, "some error"); - check_error(st, "'some error'"); + ASSERT_OK(check_error(st, "'some error'")); ASSERT_TRUE(st.IsKeyError()); for (PyObject* exc_type : {PyExc_OSError, PyExc_IOError}) { PyErr_SetString(exc_type, "some error"); - check_error(st); + ASSERT_OK(check_error(st)); ASSERT_TRUE(st.IsIOError()); } PyErr_SetString(PyExc_NotImplementedError, "some error"); - check_error(st, "some error", FormatPythonException("NotImplementedError")); + ASSERT_OK(check_error(st, "some error", FormatPythonException("NotImplementedError"))); ASSERT_TRUE(st.IsNotImplemented()); // No override if a specific status code is given @@ -141,9 +204,11 @@ TEST(CheckPyError, TestStatus) { ASSERT_TRUE(st.IsSerializationError()); ASSERT_EQ(st.message(), "some error"); ASSERT_FALSE(PyErr_Occurred()); + + return Status::OK(); } -TEST(CheckPyError, TestStatusNoGIL) { +Status TestCheckPyErrorStatusNoGIL() { PyAcquireGIL lock; { Status st; @@ -154,10 +219,11 @@ TEST(CheckPyError, TestStatusNoGIL) { ASSERT_TRUE(st.IsUnknownError()); ASSERT_EQ(st.message(), "zzzt"); ASSERT_EQ(st.detail()->ToString(), FormatPythonException("ZeroDivisionError")); + return Status::OK(); } } -TEST(RestorePyError, Basics) { +Status TestRestorePyErrorBasics() { PyErr_SetString(PyExc_ZeroDivisionError, "zzzt"); auto st = ConvertPyError(); ASSERT_FALSE(PyErr_Occurred()); @@ -175,33 +241,36 @@ TEST(RestorePyError, Basics) { std::string py_message; ASSERT_OK(internal::PyObject_StdStringStr(exc_value, &py_message)); ASSERT_EQ(py_message, "zzzt"); + + return Status::OK(); } -TEST(PyBuffer, InvalidInputObject) { +Status TestPyBufferInvalidInputObject() { std::shared_ptr res; PyObject* input = Py_None; auto old_refcnt = Py_REFCNT(input); { Status st = PyBuffer::FromPyObject(input).status(); - ASSERT_TRUE(IsPyError(st)) << st.ToString(); + ASSERT_TRUE_MSG(IsPyError(st), st.ToString()); ASSERT_FALSE(PyErr_Occurred()); } ASSERT_EQ(old_refcnt, Py_REFCNT(input)); + return Status::OK(); } // Because of how it is declared, the Numpy C API instance initialized // within libarrow_python.dll may not be visible in this test under Windows // ("unresolved external symbol arrow_ARRAY_API referenced"). #ifndef _WIN32 -TEST(PyBuffer, NumpyArray) { - const npy_intp dims[1] = {10}; +Status TestPyBufferNumpyArray() { + npy_intp dims[1] = {10}; OwnedRef arr_ref(PyArray_SimpleNew(1, dims, NPY_FLOAT)); PyObject* arr = arr_ref.obj(); ASSERT_NE(arr, nullptr); auto old_refcnt = Py_REFCNT(arr); + auto buf = std::move(PyBuffer::FromPyObject(arr)).ValueOrDie(); - ASSERT_OK_AND_ASSIGN(auto buf, PyBuffer::FromPyObject(arr)); ASSERT_TRUE(buf->is_cpu()); ASSERT_EQ(buf->data(), PyArray_DATA(reinterpret_cast(arr))); ASSERT_TRUE(buf->is_mutable()); @@ -212,16 +281,18 @@ TEST(PyBuffer, NumpyArray) { // Read-only PyArray_CLEARFLAGS(reinterpret_cast(arr), NPY_ARRAY_WRITEABLE); - ASSERT_OK_AND_ASSIGN(buf, PyBuffer::FromPyObject(arr)); + buf = std::move(PyBuffer::FromPyObject(arr)).ValueOrDie(); ASSERT_TRUE(buf->is_cpu()); ASSERT_EQ(buf->data(), PyArray_DATA(reinterpret_cast(arr))); ASSERT_FALSE(buf->is_mutable()); ASSERT_EQ(old_refcnt + 1, Py_REFCNT(arr)); buf.reset(); ASSERT_EQ(old_refcnt, Py_REFCNT(arr)); + + return Status::OK(); } -TEST(NumPyBuffer, NumpyArray) { +Status TestNumPyBufferNumpyArray() { npy_intp dims[1] = {10}; OwnedRef arr_ref(PyArray_SimpleNew(1, dims, NPY_FLOAT)); @@ -247,50 +318,44 @@ TEST(NumPyBuffer, NumpyArray) { ASSERT_EQ(old_refcnt + 1, Py_REFCNT(arr)); buf.reset(); ASSERT_EQ(old_refcnt, Py_REFCNT(arr)); + + return Status::OK(); } #endif -class DecimalTest : public ::testing::Test { - public: - DecimalTest() : lock_(), decimal_constructor_() { - OwnedRef decimal_module; +Status TestPythonDecimalToString(){ + OwnedRef decimal_constructor_; + OwnedRef decimal_module; - Status status = internal::ImportModule("decimal", &decimal_module); - ARROW_CHECK_OK(status); + RETURN_NOT_OK(internal::ImportModule("decimal", &decimal_module)); + RETURN_NOT_OK(internal::ImportFromModule(decimal_module.obj(), "Decimal", + &decimal_constructor_)); - status = internal::ImportFromModule(decimal_module.obj(), "Decimal", - &decimal_constructor_); - ARROW_CHECK_OK(status); - } + std::string decimal_string("-39402950693754869342983"); + PyObject* python_object = internal::DecimalFromString(decimal_constructor_.obj(), + decimal_string); + ASSERT_NE(python_object, nullptr); - OwnedRef CreatePythonDecimal(const std::string& string_value) { - OwnedRef ref(internal::DecimalFromString(decimal_constructor_.obj(), string_value)); - return ref; - } + std::string string_result; + ASSERT_OK(internal::PythonDecimalToString(python_object, &string_result)); - PyObject* decimal_constructor() const { return decimal_constructor_.obj(); } + return Status::OK(); +} - private: - PyAcquireGIL lock_; +Status TestInferPrecisionAndScale(){ OwnedRef decimal_constructor_; -}; + OwnedRef decimal_module; -TEST_F(DecimalTest, TestPythonDecimalToString) { - std::string decimal_string("-39402950693754869342983"); - - OwnedRef python_object(this->CreatePythonDecimal(decimal_string)); - ASSERT_NE(python_object.obj(), nullptr); - - std::string string_result; - ASSERT_OK(internal::PythonDecimalToString(python_object.obj(), &string_result)); -} + RETURN_NOT_OK(internal::ImportModule("decimal", &decimal_module)); + RETURN_NOT_OK(internal::ImportFromModule(decimal_module.obj(), "Decimal", + &decimal_constructor_)); -TEST_F(DecimalTest, TestInferPrecisionAndScale) { std::string decimal_string("-394029506937548693.42983"); - OwnedRef python_decimal(this->CreatePythonDecimal(decimal_string)); + PyObject* python_decimal = internal::DecimalFromString(decimal_constructor_.obj(), + decimal_string); internal::DecimalMetadata metadata; - ASSERT_OK(metadata.Update(python_decimal.obj())); + ASSERT_OK(metadata.Update(python_decimal)); const auto expected_precision = static_cast(decimal_string.size() - 2); // 1 for -, 1 for . @@ -298,51 +363,94 @@ TEST_F(DecimalTest, TestInferPrecisionAndScale) { ASSERT_EQ(expected_precision, metadata.precision()); ASSERT_EQ(expected_scale, metadata.scale()); + + return Status::OK(); } -TEST_F(DecimalTest, TestInferPrecisionAndNegativeScale) { +Status TestInferPrecisionAndNegativeScale(){ + OwnedRef decimal_constructor_; + OwnedRef decimal_module; + + RETURN_NOT_OK(internal::ImportModule("decimal", &decimal_module)); + RETURN_NOT_OK(internal::ImportFromModule(decimal_module.obj(), "Decimal", + &decimal_constructor_)); + std::string decimal_string("-3.94042983E+10"); - OwnedRef python_decimal(this->CreatePythonDecimal(decimal_string)); + PyObject* python_decimal = internal::DecimalFromString(decimal_constructor_.obj(), + decimal_string); internal::DecimalMetadata metadata; - ASSERT_OK(metadata.Update(python_decimal.obj())); + ASSERT_OK(metadata.Update(python_decimal)); const auto expected_precision = 11; const int32_t expected_scale = 0; ASSERT_EQ(expected_precision, metadata.precision()); ASSERT_EQ(expected_scale, metadata.scale()); + + return Status::OK(); } -TEST_F(DecimalTest, TestInferAllLeadingZeros) { +Status TestInferAllLeadingZeros(){ + OwnedRef decimal_constructor_; + OwnedRef decimal_module; + + RETURN_NOT_OK(internal::ImportModule("decimal", &decimal_module)); + RETURN_NOT_OK(internal::ImportFromModule(decimal_module.obj(), "Decimal", + &decimal_constructor_)); + std::string decimal_string("0.001"); - OwnedRef python_decimal(this->CreatePythonDecimal(decimal_string)); + PyObject* python_decimal = internal::DecimalFromString(decimal_constructor_.obj(), + decimal_string); internal::DecimalMetadata metadata; - ASSERT_OK(metadata.Update(python_decimal.obj())); + ASSERT_OK(metadata.Update(python_decimal)); ASSERT_EQ(3, metadata.precision()); ASSERT_EQ(3, metadata.scale()); + + return Status::OK(); } -TEST_F(DecimalTest, TestInferAllLeadingZerosExponentialNotationPositive) { +Status TestInferAllLeadingZerosExponentialNotationPositive(){ + OwnedRef decimal_constructor_; + OwnedRef decimal_module; + + RETURN_NOT_OK(internal::ImportModule("decimal", &decimal_module)); + RETURN_NOT_OK(internal::ImportFromModule(decimal_module.obj(), "Decimal", + &decimal_constructor_)); + std::string decimal_string("0.01E5"); - OwnedRef python_decimal(this->CreatePythonDecimal(decimal_string)); + PyObject* python_decimal = internal::DecimalFromString(decimal_constructor_.obj(), + decimal_string); + internal::DecimalMetadata metadata; - ASSERT_OK(metadata.Update(python_decimal.obj())); + ASSERT_OK(metadata.Update(python_decimal)); ASSERT_EQ(4, metadata.precision()); ASSERT_EQ(0, metadata.scale()); + + return Status::OK(); } -TEST_F(DecimalTest, TestInferAllLeadingZerosExponentialNotationNegative) { +Status TestInferAllLeadingZerosExponentialNotationNegative(){ + OwnedRef decimal_constructor_; + OwnedRef decimal_module; + + RETURN_NOT_OK(internal::ImportModule("decimal", &decimal_module)); + RETURN_NOT_OK(internal::ImportFromModule(decimal_module.obj(), "Decimal", + &decimal_constructor_)); + std::string decimal_string("0.01E3"); - OwnedRef python_decimal(this->CreatePythonDecimal(decimal_string)); + PyObject* python_decimal = internal::DecimalFromString(decimal_constructor_.obj(), + decimal_string); internal::DecimalMetadata metadata; - ASSERT_OK(metadata.Update(python_decimal.obj())); + ASSERT_OK(metadata.Update(python_decimal)); ASSERT_EQ(2, metadata.precision()); ASSERT_EQ(0, metadata.scale()); + + return Status::OK(); } -TEST(PandasConversionTest, TestObjectBlockWriteFails) { +Status TestObjectBlockWriteFails(){ StringBuilder builder; const char value[] = {'\xf1', '\0'}; @@ -370,9 +478,11 @@ TEST(PandasConversionTest, TestObjectBlockWriteFails) { st = ConvertTableToPandas(options, table, &out); Py_END_ALLOW_THREADS; ASSERT_RAISES(UnknownError, st); + + return Status::OK(); } -TEST(BuiltinConversionTest, TestMixedTypeFails) { +Status TestMixedTypeFails(){ OwnedRef list_ref(PyList_New(3)); PyObject* list = list_ref.obj(); @@ -394,110 +504,179 @@ TEST(BuiltinConversionTest, TestMixedTypeFails) { ASSERT_EQ(PyList_SetItem(list, 2, doub), 0); ASSERT_RAISES(TypeError, ConvertPySequence(list, nullptr, {})); + + return Status::OK(); } template -void DecimalTestFromPythonDecimalRescale(std::shared_ptr type, - OwnedRef python_decimal, +Status DecimalTestFromPythonDecimalRescale(std::shared_ptr type, + PyObject* python_decimal, std::optional expected) { DecimalValue value; const auto& decimal_type = checked_cast(*type); if (expected.has_value()) { ASSERT_OK( - internal::DecimalFromPythonDecimal(python_decimal.obj(), decimal_type, &value)); + internal::DecimalFromPythonDecimal(python_decimal, decimal_type, &value)); ASSERT_EQ(expected.value(), value); - ASSERT_OK(internal::DecimalFromPyObject(python_decimal.obj(), decimal_type, &value)); + ASSERT_OK(internal::DecimalFromPyObject(python_decimal, decimal_type, &value)); ASSERT_EQ(expected.value(), value); } else { - ASSERT_RAISES(Invalid, internal::DecimalFromPythonDecimal(python_decimal.obj(), - decimal_type, &value)); - ASSERT_RAISES(Invalid, internal::DecimalFromPyObject(python_decimal.obj(), - decimal_type, &value)); + ASSERT_RAISES(Invalid, + internal::DecimalFromPythonDecimal(python_decimal, + decimal_type, &value)); + ASSERT_RAISES(Invalid, + internal::DecimalFromPyObject(python_decimal, + decimal_type, &value)); } + return Status::OK(); } -TEST_F(DecimalTest, FromPythonDecimalRescaleNotTruncateable) { +Status TestFromPythonDecimalRescaleNotTruncateable(){ + OwnedRef decimal_constructor_; + OwnedRef decimal_module; + + RETURN_NOT_OK(internal::ImportModule("decimal", &decimal_module)); + RETURN_NOT_OK(internal::ImportFromModule(decimal_module.obj(), "Decimal", + &decimal_constructor_)); + + std::string decimal_string("1.001"); + PyObject* python_decimal = internal::DecimalFromString(decimal_constructor_.obj(), + decimal_string); // We fail when truncating values that would lose data if cast to a decimal type with // lower scale - DecimalTestFromPythonDecimalRescale(::arrow::decimal128(10, 2), - this->CreatePythonDecimal("1.001"), {}); - DecimalTestFromPythonDecimalRescale(::arrow::decimal256(10, 2), - this->CreatePythonDecimal("1.001"), {}); + ASSERT_OK(DecimalTestFromPythonDecimalRescale(::arrow::decimal128(10, 2), + python_decimal, {})); + ASSERT_OK(DecimalTestFromPythonDecimalRescale(::arrow::decimal256(10, 2), + python_decimal, {})); + + return Status::OK(); } -TEST_F(DecimalTest, FromPythonDecimalRescaleTruncateable) { +Status TestFromPythonDecimalRescaleTruncateable(){ + OwnedRef decimal_constructor_; + OwnedRef decimal_module; + + RETURN_NOT_OK(internal::ImportModule("decimal", &decimal_module)); + RETURN_NOT_OK(internal::ImportFromModule(decimal_module.obj(), "Decimal", + &decimal_constructor_)); + + std::string decimal_string("1.000"); + PyObject* python_decimal = internal::DecimalFromString(decimal_constructor_.obj(), + decimal_string); // We allow truncation of values that do not lose precision when dividing by 10 * the // difference between the scales, e.g., 1.000 -> 1.00 - DecimalTestFromPythonDecimalRescale( - ::arrow::decimal128(10, 2), this->CreatePythonDecimal("1.000"), 100); - DecimalTestFromPythonDecimalRescale( - ::arrow::decimal256(10, 2), this->CreatePythonDecimal("1.000"), 100); + ASSERT_OK(DecimalTestFromPythonDecimalRescale( + ::arrow::decimal128(10, 2), python_decimal, 100)); + ASSERT_OK(DecimalTestFromPythonDecimalRescale( + ::arrow::decimal256(10, 2), python_decimal, 100)); + + return Status::OK(); } -TEST_F(DecimalTest, FromPythonNegativeDecimalRescale) { - DecimalTestFromPythonDecimalRescale( - ::arrow::decimal128(10, 9), this->CreatePythonDecimal("-1.000"), -1000000000); - DecimalTestFromPythonDecimalRescale( - ::arrow::decimal256(10, 9), this->CreatePythonDecimal("-1.000"), -1000000000); +Status TestFromPythonNegativeDecimalRescale(){ + OwnedRef decimal_constructor_; + OwnedRef decimal_module; + + RETURN_NOT_OK(internal::ImportModule("decimal", &decimal_module)); + RETURN_NOT_OK(internal::ImportFromModule(decimal_module.obj(), "Decimal", + &decimal_constructor_)); + + std::string decimal_string("-1.000"); + PyObject* python_decimal = internal::DecimalFromString(decimal_constructor_.obj(), + decimal_string); + ASSERT_OK(DecimalTestFromPythonDecimalRescale( + ::arrow::decimal128(10, 9), python_decimal, -1000000000)); + ASSERT_OK(DecimalTestFromPythonDecimalRescale( + ::arrow::decimal256(10, 9), python_decimal, -1000000000)); + + return Status::OK(); } -TEST_F(DecimalTest, Decimal128FromPythonInteger) { +Status TestDecimal128FromPythonInteger(){ Decimal128 value; OwnedRef python_long(PyLong_FromLong(42)); auto type = ::arrow::decimal128(10, 2); const auto& decimal_type = checked_cast(*type); ASSERT_OK(internal::DecimalFromPyObject(python_long.obj(), decimal_type, &value)); ASSERT_EQ(4200, value); + return Status::OK(); } -TEST_F(DecimalTest, Decimal256FromPythonInteger) { +Status TestDecimal256FromPythonInteger(){ Decimal256 value; OwnedRef python_long(PyLong_FromLong(42)); auto type = ::arrow::decimal256(10, 2); const auto& decimal_type = checked_cast(*type); ASSERT_OK(internal::DecimalFromPyObject(python_long.obj(), decimal_type, &value)); ASSERT_EQ(4200, value); + return Status::OK(); } -TEST_F(DecimalTest, TestDecimal128OverflowFails) { +Status TestDecimal128OverflowFails(){ Decimal128 value; - OwnedRef python_decimal( - this->CreatePythonDecimal("9999999999999999999999999999999999999.9")); + OwnedRef decimal_constructor_; + OwnedRef decimal_module; + + RETURN_NOT_OK(internal::ImportModule("decimal", &decimal_module)); + RETURN_NOT_OK(internal::ImportFromModule(decimal_module.obj(), "Decimal", + &decimal_constructor_)); + + std::string decimal_string("9999999999999999999999999999999999999.9"); + PyObject* python_decimal = internal::DecimalFromString(decimal_constructor_.obj(), + decimal_string); internal::DecimalMetadata metadata; - ASSERT_OK(metadata.Update(python_decimal.obj())); + ASSERT_OK(metadata.Update(python_decimal)); ASSERT_EQ(38, metadata.precision()); ASSERT_EQ(1, metadata.scale()); auto type = ::arrow::decimal(38, 38); const auto& decimal_type = checked_cast(*type); - ASSERT_RAISES(Invalid, internal::DecimalFromPythonDecimal(python_decimal.obj(), - decimal_type, &value)); + ASSERT_RAISES(Invalid, + internal::DecimalFromPythonDecimal(python_decimal, + decimal_type, &value)); + return Status::OK(); } -TEST_F(DecimalTest, TestDecimal256OverflowFails) { +Status TestDecimal256OverflowFails(){ Decimal256 value; - OwnedRef python_decimal(this->CreatePythonDecimal( - "999999999999999999999999999999999999999999999999999999999999999999999999999.9")); + OwnedRef decimal_constructor_; + OwnedRef decimal_module; + + RETURN_NOT_OK(internal::ImportModule("decimal", &decimal_module)); + RETURN_NOT_OK(internal::ImportFromModule(decimal_module.obj(), "Decimal", + &decimal_constructor_)); + + std::string decimal_string("999999999999999999999999999999999999999999999999999999999999999999999999999.9"); + PyObject* python_decimal = internal::DecimalFromString(decimal_constructor_.obj(), + decimal_string); + internal::DecimalMetadata metadata; - ASSERT_OK(metadata.Update(python_decimal.obj())); + ASSERT_OK(metadata.Update(python_decimal)); ASSERT_EQ(76, metadata.precision()); ASSERT_EQ(1, metadata.scale()); auto type = ::arrow::decimal(76, 76); const auto& decimal_type = checked_cast(*type); - ASSERT_RAISES(Invalid, internal::DecimalFromPythonDecimal(python_decimal.obj(), - decimal_type, &value)); + ASSERT_RAISES(Invalid, + internal::DecimalFromPythonDecimal(python_decimal, + decimal_type, &value)); + return Status::OK(); } -TEST_F(DecimalTest, TestNoneAndNaN) { +Status TestNoneAndNaN(){ OwnedRef list_ref(PyList_New(4)); PyObject* list = list_ref.obj(); ASSERT_NE(list, nullptr); - PyObject* constructor = this->decimal_constructor(); + OwnedRef decimal_constructor_; + OwnedRef decimal_module; + RETURN_NOT_OK(internal::ImportModule("decimal", &decimal_module)); + RETURN_NOT_OK(internal::ImportFromModule(decimal_module.obj(), "Decimal", + &decimal_constructor_)); + PyObject* constructor = decimal_constructor_.obj(); PyObject* decimal_value = internal::DecimalFromString(constructor, "1.234"); ASSERT_NE(decimal_value, nullptr); @@ -519,10 +698,11 @@ TEST_F(DecimalTest, TestNoneAndNaN) { ASSERT_EQ(0, PyList_SetItem(list, 3, missing_value3)); PyConversionOptions options; - ASSERT_RAISES(TypeError, ConvertPySequence(list, nullptr, options)); + ASSERT_RAISES(TypeError, + ConvertPySequence(list, nullptr, options)); options.from_pandas = true; - ASSERT_OK_AND_ASSIGN(auto chunked, ConvertPySequence(list, nullptr, options)); + auto chunked = std::move(ConvertPySequence(list, nullptr, options)).ValueOrDie(); ASSERT_EQ(chunked->num_chunks(), 1); auto arr = chunked->chunk(0); @@ -530,9 +710,11 @@ TEST_F(DecimalTest, TestNoneAndNaN) { ASSERT_TRUE(arr->IsNull(1)); ASSERT_TRUE(arr->IsNull(2)); ASSERT_TRUE(arr->IsNull(3)); + + return Status::OK(); } -TEST_F(DecimalTest, TestMixedPrecisionAndScale) { +Status TestMixedPrecisionAndScale(){ std::vector strings{{"0.001", "1.01E5", "1.01E5"}}; OwnedRef list_ref(PyList_New(static_cast(strings.size()))); @@ -540,28 +722,46 @@ TEST_F(DecimalTest, TestMixedPrecisionAndScale) { ASSERT_NE(list, nullptr); + OwnedRef decimal_constructor_; + OwnedRef decimal_module; + RETURN_NOT_OK(internal::ImportModule("decimal", &decimal_module)); + RETURN_NOT_OK(internal::ImportFromModule(decimal_module.obj(), "Decimal", + &decimal_constructor_)); // PyList_SetItem steals a reference to the item so we don't decref it later - PyObject* decimal_constructor = this->decimal_constructor(); + PyObject* decimal_constructor = decimal_constructor_.obj(); for (Py_ssize_t i = 0; i < static_cast(strings.size()); ++i) { const int result = PyList_SetItem( list, i, internal::DecimalFromString(decimal_constructor, strings.at(i))); ASSERT_EQ(0, result); } - ASSERT_OK_AND_ASSIGN(auto arr, ConvertPySequence(list, nullptr, {})) + auto arr = std::move(ConvertPySequence(list, nullptr, {})).ValueOrDie(); const auto& type = checked_cast(*arr->type()); int32_t expected_precision = 9; int32_t expected_scale = 3; ASSERT_EQ(expected_precision, type.precision()); ASSERT_EQ(expected_scale, type.scale()); + + return Status::OK(); } -TEST_F(DecimalTest, TestMixedPrecisionAndScaleSequenceConvert) { - PyObject* value1 = this->CreatePythonDecimal("0.01").detach(); +Status TestMixedPrecisionAndScaleSequenceConvert(){ + OwnedRef decimal_constructor_; + OwnedRef decimal_module; + + RETURN_NOT_OK(internal::ImportModule("decimal", &decimal_module)); + RETURN_NOT_OK(internal::ImportFromModule(decimal_module.obj(), "Decimal", + &decimal_constructor_)); + + std::string decimal_string_1("0.01"); + PyObject* value1 = internal::DecimalFromString(decimal_constructor_.obj(), + decimal_string_1); ASSERT_NE(value1, nullptr); - PyObject* value2 = this->CreatePythonDecimal("0.001").detach(); + std::string decimal_string_2("0.001"); + PyObject* value2 = internal::DecimalFromString(decimal_constructor_.obj(), + decimal_string_2); ASSERT_NE(value2, nullptr); OwnedRef list_ref(PyList_New(2)); @@ -572,28 +772,94 @@ TEST_F(DecimalTest, TestMixedPrecisionAndScaleSequenceConvert) { ASSERT_EQ(PyList_SetItem(list, 0, value1), 0); ASSERT_EQ(PyList_SetItem(list, 1, value2), 0); - ASSERT_OK_AND_ASSIGN(auto arr, ConvertPySequence(list, nullptr, {})); + auto arr = std::move(ConvertPySequence(list, nullptr, {})).ValueOrDie(); const auto& type = checked_cast(*arr->type()); ASSERT_EQ(3, type.precision()); ASSERT_EQ(3, type.scale()); + + return Status::OK(); } -TEST_F(DecimalTest, SimpleInference) { - OwnedRef value(this->CreatePythonDecimal("0.01")); - ASSERT_NE(value.obj(), nullptr); +Status TestSimpleInference(){ + OwnedRef decimal_constructor_; + OwnedRef decimal_module; + + RETURN_NOT_OK(internal::ImportModule("decimal", &decimal_module)); + RETURN_NOT_OK(internal::ImportFromModule(decimal_module.obj(), "Decimal", + &decimal_constructor_)); + + std::string decimal_string("0.01"); + PyObject* value = internal::DecimalFromString(decimal_constructor_.obj(), + decimal_string); + ASSERT_NE(value, nullptr); internal::DecimalMetadata metadata; - ASSERT_OK(metadata.Update(value.obj())); + ASSERT_OK(metadata.Update(value)); ASSERT_EQ(2, metadata.precision()); ASSERT_EQ(2, metadata.scale()); + + return Status::OK(); } -TEST_F(DecimalTest, UpdateWithNaN) { +Status TestUpdateWithNaN(){ internal::DecimalMetadata metadata; - OwnedRef nan_value(this->CreatePythonDecimal("nan")); - ASSERT_OK(metadata.Update(nan_value.obj())); + OwnedRef decimal_constructor_; + OwnedRef decimal_module; + RETURN_NOT_OK(internal::ImportModule("decimal", &decimal_module)); + RETURN_NOT_OK(internal::ImportFromModule(decimal_module.obj(), "Decimal", + &decimal_constructor_)); + std::string decimal_string("nan"); + PyObject* nan_value = internal::DecimalFromString(decimal_constructor_.obj(), + decimal_string); + + ASSERT_OK(metadata.Update(nan_value)); ASSERT_EQ(std::numeric_limits::min(), metadata.precision()); ASSERT_EQ(std::numeric_limits::min(), metadata.scale()); + + return Status::OK(); +} + +} // namespace + +std::vector GetCppTestCases() { + return { + {"test_owned_ref_moves", TestOwnedRefMoves}, + {"test_owned_ref_nogil_moves", TestOwnedRefNoGILMoves}, + {"test_check_pyerror_status", TestCheckPyErrorStatus}, + {"test_check_pyerror_status_nogil", TestCheckPyErrorStatusNoGIL}, + {"test_restore_pyerror_basics", TestRestorePyErrorBasics}, + {"test_pybuffer_invalid_input_object", TestPyBufferInvalidInputObject}, +#ifndef _WIN32 + {"test_pybuffer_numpy_array", TestPyBufferNumpyArray}, + {"test_numpybuffer_numpy_array", TestNumPyBufferNumpyArray}, +#endif + {"test_python_decimal_to_string", TestPythonDecimalToString}, + {"test_infer_precision_and_scale", TestInferPrecisionAndScale}, + {"test_infer_precision_and_negative_scale", TestInferPrecisionAndNegativeScale}, + {"test_infer_all_leading_zeros", TestInferAllLeadingZeros}, + {"test_infer_all_leading_zeros_exponential_notation_positive", + TestInferAllLeadingZerosExponentialNotationPositive}, + {"test_infer_all_leading_zeros_exponential_notation_negative", + TestInferAllLeadingZerosExponentialNotationNegative}, + {"test_object_block_write_fails", TestObjectBlockWriteFails}, + {"test_mixed_type_fails", TestMixedTypeFails}, + {"test_from_python_decimal_rescale_not_truncateable", + TestFromPythonDecimalRescaleNotTruncateable}, + {"test_from_python_decimal_rescale_truncateable", + TestFromPythonDecimalRescaleTruncateable}, + {"test_from_python_negative_decimal_rescale", TestFromPythonNegativeDecimalRescale}, + {"test_decimal128_from_python_integer", TestDecimal128FromPythonInteger}, + {"test_decimal256_from_python_integer", TestDecimal256FromPythonInteger}, + {"test_decimal128_overflow_fails", TestDecimal128OverflowFails}, + {"test_decimal256_overflow_fails", TestDecimal256OverflowFails}, + {"test_none_and_nan", TestNoneAndNaN}, + {"test_mixed_precision_and_scale", TestMixedPrecisionAndScale}, + {"test_mixed_precision_and_scale_sequence_convert", + TestMixedPrecisionAndScaleSequenceConvert}, + {"test_simple_inference", TestSimpleInference}, + {"test_update_with_nan", TestUpdateWithNaN}, + }; } +} // namespace testing } // namespace py } // namespace arrow diff --git a/python/pyarrow/src/util/test_main.cc b/python/pyarrow/src/python_test.h similarity index 67% rename from python/pyarrow/src/util/test_main.cc rename to python/pyarrow/src/python_test.h index 3ee1657e6440b..db0f7ed313467 100644 --- a/python/pyarrow/src/util/test_main.cc +++ b/python/pyarrow/src/python_test.h @@ -15,27 +15,28 @@ // specific language governing permissions and limitations // under the License. -#include "../platform.h" +#pragma once -#include +#include +#include +#include -#include "../datetime.h" -#include "../init.h" -#include "../pyarrow.h" +#include "arrow/status.h" -int main(int argc, char** argv) { - ::testing::InitGoogleTest(&argc, argv); +#include "visibility.h" - Py_Initialize(); - int ret = arrow_init_numpy(); - if (ret != 0) { - return ret; - } - ::arrow::py::internal::InitDatetime(); +namespace arrow { +namespace py { +namespace testing { - ret = RUN_ALL_TESTS(); +struct TestCase { + std::string name; + std::function func; +}; - Py_Finalize(); +ARROW_PYTHON_EXPORT +std::vector GetCppTestCases(); - return ret; -} +} // namespace testing +} // namespace py +} // namespace arrow diff --git a/python/pyarrow/tests/test_cpp_internals.py b/python/pyarrow/tests/test_cpp_internals.py new file mode 100644 index 0000000000000..1c44bff9b5bdb --- /dev/null +++ b/python/pyarrow/tests/test_cpp_internals.py @@ -0,0 +1,33 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +from pyarrow._pyarrow_cpp_tests import get_cpp_tests + + +def inject_cpp_tests(ns): + """ + Inject C++ tests as Python functions into namespace `ns` (a dict). + """ + for case in get_cpp_tests(): + def wrapper(case=case): + case() + wrapper.__name__ = wrapper.__qualname__ = case.name + wrapper.__module__ = ns['__name__'] + ns[case.name] = wrapper + + +inject_cpp_tests(globals()) diff --git a/python/setup.py b/python/setup.py index 6852a3affb859..cd855e869a856 100755 --- a/python/setup.py +++ b/python/setup.py @@ -220,6 +220,7 @@ def initialize_options(self): '_feather', '_parquet', '_parquet_encryption', + '_pyarrow_cpp_tests', '_orc', '_plasma', '_gcsfs',