diff --git a/DESCRIPTION.md b/DESCRIPTION.md index 47d1f1919..afff30260 100644 --- a/DESCRIPTION.md +++ b/DESCRIPTION.md @@ -8,6 +8,10 @@ Source code is also available at: https://github.com/snowflakedb/snowflake-conne # Release Notes +- v2.9.1(unreleased) + + - Bumped pyarrow dependency from >=8.0.0,<8.1.0 to >=10.0.1,<10.1.0 + - v2.9.0(December 9, 2022) - Fixed a bug where the permission of the file downloaded via GET command is changed diff --git a/pyproject.toml b/pyproject.toml index 87ddbe1a1..2bdafcb5a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -6,7 +6,7 @@ requires = [ "wheel", "cython", # Must be kept in sync with the `setup_requirements` in `setup.cfg` - "pyarrow>=8.0.0,<8.1.0", + "pyarrow>=10.0.1,<10.1.0", ] [tool.cibuildwheel] diff --git a/setup.cfg b/setup.cfg index 114d26862..60c267bec 100644 --- a/setup.cfg +++ b/setup.cfg @@ -91,6 +91,6 @@ development = pytzdata pandas = pandas>=1.0.0,<1.6.0 - pyarrow>=8.0.0,<8.1.0 + pyarrow>=10.0.1,<10.1.0 secure-local-storage = keyring!=16.1.0,<24.0.0 diff --git a/setup.py b/setup.py index d4ccdb7f3..e02374016 100644 --- a/setup.py +++ b/setup.py @@ -71,23 +71,45 @@ class MyBuildExt(build_ext): # this list should be carefully examined when pyarrow lib is # upgraded arrow_libs_to_copy = { - "linux": ["libarrow.so.800", "libarrow_python.so.800", "libparquet.so.800"], + "linux": [ + "libarrow.so.1000", + "libarrow_dataset.so.1000", + "libarrow_python.so.1000", + "libparquet.so.1000", + ], "darwin": [ - "libarrow.800.dylib", - "libarrow_python.800.dylib", - "libparquet.800.dylib", + "libarrow.1000.dylib", + "libarrow_dataset.1000.dylib", + "libarrow_python.1000.dylib", + "libparquet.1000.dylib", + ], + "win32": [ + "arrow.dll", + "arrow_dataset.dll", + "arrow_python.dll", + "parquet.dll", ], - "win32": ["arrow.dll", "arrow_python.dll", "parquet.dll"], } arrow_libs_to_link = { - "linux": ["libarrow.so.800", "libarrow_python.so.800", "libparquet.so.800"], + "linux": [ + "libarrow.so.1000", + "libarrow_dataset.so.1000", + "libarrow_python.so.1000", + "libparquet.so.1000", + ], "darwin": [ - "libarrow.800.dylib", - "libarrow_python.800.dylib", - "libparquet.800.dylib", + "libarrow.1000.dylib", + "libarrow_dataset.1000.dylib", + "libarrow_python.1000.dylib", + "libparquet.1000.dylib", + ], + "win32": [ + "arrow.lib", + "arrow_dataset.lib", + "arrow_python.lib", + "parquet.lib", ], - "win32": ["arrow.lib", "arrow_python.lib", "parquet.lib"], } def build_extension(self, ext): @@ -126,13 +148,15 @@ def build_extension(self, ext): ext.include_dirs.append(LOGGING_SRC_DIR) if sys.platform == "win32": + if not any("/std" not in s for s in ext.extra_compile_args): + ext.extra_compile_args.append("/std:c++17") ext.include_dirs.append(pyarrow.get_include()) ext.include_dirs.append(numpy.get_include()) elif sys.platform == "linux" or sys.platform == "darwin": ext.extra_compile_args.append("-isystem" + pyarrow.get_include()) ext.extra_compile_args.append("-isystem" + numpy.get_include()) if "std=" not in os.environ.get("CXXFLAGS", ""): - ext.extra_compile_args.append("-std=c++11") + ext.extra_compile_args.append("-std=c++17") ext.extra_compile_args.append("-D_GLIBCXX_USE_CXX11_ABI=0") ext.library_dirs.append( @@ -160,9 +184,11 @@ def _get_arrow_lib_dir(self): def _copy_arrow_lib(self): libs_to_bundle = self.arrow_libs_to_copy[sys.platform] + build_dir = os.path.join(self.build_lib, "snowflake", "connector") + os.makedirs(build_dir, exist_ok=True) + for lib in libs_to_bundle: source = f"{self._get_arrow_lib_dir()}/{lib}" - build_dir = os.path.join(self.build_lib, "snowflake", "connector") copy(source, build_dir) def _get_arrow_lib_as_linker_input(self): diff --git a/src/snowflake/connector/cpp/ArrowIterator/BinaryConverter.cpp b/src/snowflake/connector/cpp/ArrowIterator/BinaryConverter.cpp index 76e3edcec..eca3622bb 100644 --- a/src/snowflake/connector/cpp/ArrowIterator/BinaryConverter.cpp +++ b/src/snowflake/connector/cpp/ArrowIterator/BinaryConverter.cpp @@ -18,7 +18,7 @@ PyObject* BinaryConverter::toPyObject(int64_t rowIndex) const { if (m_array->IsValid(rowIndex)) { - arrow::util::string_view sv = m_array->GetView(rowIndex); + std::string_view sv = m_array->GetView(rowIndex); return PyByteArray_FromStringAndSize(sv.data(), sv.size()); } else diff --git a/src/snowflake/connector/cpp/ArrowIterator/StringConverter.cpp b/src/snowflake/connector/cpp/ArrowIterator/StringConverter.cpp index fef6453a5..de650683d 100644 --- a/src/snowflake/connector/cpp/ArrowIterator/StringConverter.cpp +++ b/src/snowflake/connector/cpp/ArrowIterator/StringConverter.cpp @@ -18,7 +18,7 @@ PyObject* StringConverter::toPyObject(int64_t rowIndex) const { if (m_array->IsValid(rowIndex)) { - arrow::util::string_view sv = m_array->GetView(rowIndex); + std::string_view sv = m_array->GetView(rowIndex); return PyUnicode_FromStringAndSize(sv.data(), sv.size()); } else