Merge branch 'master' into fly_0704_cmp

apache · Jul 8, 2019 · 30f946b · 30f946b
2 parents bc880f1 + c2c9e99
commit 30f946b
Show file tree

Hide file tree

Showing 41 changed files with 807 additions and 183 deletions.
diff --git a/cpp/Brewfile b/cpp/Brewfile
@@ -35,7 +35,6 @@ brew "python"
 brew "rapidjson"
 brew "re2"
 brew "snappy"
-brew "openssl"
 brew "thrift"
 brew "wget"
 brew "zstd"
diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt
@@ -52,6 +52,7 @@ endif()
 message(STATUS "Arrow version: "
                "${ARROW_VERSION_MAJOR}.${ARROW_VERSION_MINOR}.${ARROW_VERSION_PATCH} "
                "(full: '${ARROW_VERSION}')")
+message(STATUS "Arrow SO version: ${ARROW_SO_VERSION} (full: ${ARROW_FULL_SO_VERSION})")
 
 set(ARROW_SOURCE_DIR ${PROJECT_SOURCE_DIR})
 set(ARROW_BINARY_DIR ${PROJECT_BINARY_DIR})

diff --git a/cpp/cmake_modules/ThirdpartyToolchain.cmake b/cpp/cmake_modules/ThirdpartyToolchain.cmake
@@ -180,6 +180,19 @@ macro(resolve_dependency DEPENDENCY_NAME)
   endif()
 endmacro()
 
+macro(resolve_dependency_with_version DEPENDENCY_NAME REQUIRED_VERSION)
+  if(${DEPENDENCY_NAME}_SOURCE STREQUAL "AUTO")
+    find_package(${DEPENDENCY_NAME} ${REQUIRED_VERSION} MODULE)
+    if(NOT ${${DEPENDENCY_NAME}_FOUND})
+      build_dependency(${DEPENDENCY_NAME})
+    endif()
+  elseif(${DEPENDENCY_NAME}_SOURCE STREQUAL "BUNDLED")
+    build_dependency(${DEPENDENCY_NAME})
+  elseif(${DEPENDENCY_NAME}_SOURCE STREQUAL "SYSTEM")
+    find_package(${DEPENDENCY_NAME} ${REQUIRED_VERSION} REQUIRED)
+  endif()
+endmacro()
+
 # ----------------------------------------------------------------------
 # Thirdparty versions, environment variables, source URLs
 
@@ -1282,7 +1295,12 @@ macro(build_protobuf)
 endmacro()
 
 if(ARROW_WITH_PROTOBUF)
-  resolve_dependency(Protobuf)
+  if(ARROW_WITH_GRPC)
+    set(ARROW_PROTOBUF_REQUIRED_VERSION "3.6.0")
+  else()
+    set(ARROW_PROTOBUF_REQUIRED_VERSION "2.6.1")
+  endif()
+  resolve_dependency_with_version(Protobuf ${ARROW_PROTOBUF_REQUIRED_VERSION})
 
   if(ARROW_PROTOBUF_USE_SHARED AND MSVC)
     add_definitions(-DPROTOBUF_USE_DLLS)
@@ -2140,6 +2158,9 @@ macro(build_grpc)
       -DCMAKE_INSTALL_LIBDIR=lib
       "-DProtobuf_PROTOC_LIBRARY=${GRPC_Protobuf_PROTOC_LIBRARY}"
       -DBUILD_SHARED_LIBS=OFF)
+  if(OPENSSL_ROOT_DIR)
+    list(APPEND GRPC_CMAKE_ARGS -DOPENSSL_ROOT_DIR=${OPENSSL_ROOT_DIR})
+  endif()
 
   # XXX the gRPC git checkout is huge and takes a long time
   # Ideally, we should be able to use the tarballs, but they don't contain

diff --git a/cpp/src/arrow/array-test.cc b/cpp/src/arrow/array-test.cc
@@ -1300,7 +1300,6 @@ TEST_F(TestFWBinaryArray, ZeroSize) {
 
   ASSERT_OK(builder.Append(""));
   ASSERT_OK(builder.Append(std::string()));
-  ASSERT_OK(builder.Append(static_cast<const uint8_t*>(nullptr)));
   ASSERT_OK(builder.AppendNull());
   ASSERT_OK(builder.AppendNull());
   ASSERT_OK(builder.AppendNull());
@@ -1314,7 +1313,7 @@ TEST_F(TestFWBinaryArray, ZeroSize) {
   ASSERT_EQ(fw_array.values()->size(), 0);
   ASSERT_EQ(0, fw_array.byte_width());
 
-  ASSERT_EQ(6, array->length());
+  ASSERT_EQ(5, array->length());
   ASSERT_EQ(3, array->null_count());
 }
 

diff --git a/cpp/src/arrow/array.cc b/cpp/src/arrow/array.cc
@@ -31,6 +31,7 @@
 #include "arrow/status.h"
 #include "arrow/type.h"
 #include "arrow/type_traits.h"
+#include "arrow/util/atomic_shared_ptr.h"
 #include "arrow/util/bit-util.h"
 #include "arrow/util/checked_cast.h"
 #include "arrow/util/decimal.h"
@@ -530,17 +531,19 @@ const StructType* StructArray::struct_type() const {
 }
 
 std::shared_ptr<Array> StructArray::field(int i) const {
-  if (!boxed_fields_[i]) {
+  std::shared_ptr<Array> result = internal::atomic_load(&boxed_fields_[i]);
+  if (!result) {
     std::shared_ptr<ArrayData> field_data;
     if (data_->offset != 0 || data_->child_data[i]->length != data_->length) {
       field_data = std::make_shared<ArrayData>(
           data_->child_data[i]->Slice(data_->offset, data_->length));
     } else {
       field_data = data_->child_data[i];
     }
-    boxed_fields_[i] = MakeArray(field_data);
+    result = MakeArray(field_data);
+    internal::atomic_store(&boxed_fields_[i], result);
   }
-  return boxed_fields_[i];
+  return result;
 }
 
 std::shared_ptr<Array> StructArray::GetFieldByName(const std::string& name) const {
@@ -709,7 +712,8 @@ Status UnionArray::MakeSparse(const Array& type_ids,
 }
 
 std::shared_ptr<Array> UnionArray::child(int i) const {
-  if (!boxed_fields_[i]) {
+  std::shared_ptr<Array> result = internal::atomic_load(&boxed_fields_[i]);
+  if (!result) {
     std::shared_ptr<ArrayData> child_data = data_->child_data[i]->Copy();
     if (mode() == UnionMode::SPARSE) {
       // Sparse union: need to adjust child if union is sliced
@@ -719,16 +723,10 @@ std::shared_ptr<Array> UnionArray::child(int i) const {
         *child_data = child_data->Slice(data_->offset, data_->length);
       }
     }
-    boxed_fields_[i] = MakeArray(child_data);
+    result = MakeArray(child_data);
+    internal::atomic_store(&boxed_fields_[i], result);
   }
-  return boxed_fields_[i];
-}
-
-const Array* UnionArray::UnsafeChild(int i) const {
-  if (!boxed_fields_[i]) {
-    boxed_fields_[i] = MakeArray(data_->child_data[i]);
-  }
-  return boxed_fields_[i].get();
+  return result;
 }
 
 // ----------------------------------------------------------------------

diff --git a/cpp/src/arrow/array.h b/cpp/src/arrow/array.h
@@ -1042,9 +1042,6 @@ class ARROW_EXPORT UnionArray : public Array {
   // For dense unions, the returned array is unchanged.
   std::shared_ptr<Array> child(int pos) const;
 
-  /// Only use this while the UnionArray is in scope
-  const Array* UnsafeChild(int pos) const;
-
  protected:
   void SetData(const std::shared_ptr<ArrayData>& data);
 

diff --git a/cpp/src/arrow/compute/kernels/cast-test.cc b/cpp/src/arrow/compute/kernels/cast-test.cc
@@ -263,7 +263,7 @@ std::vector<O> UnsafeVectorCast(const std::vector<I>& v) {
 
   for (size_t i = 0; i < v.size(); i++) result[i] = static_cast<O>(v[i]);
 
-  return std::move(result);
+  return result;
 }
 
 TEST_F(TestCast, IntegerSignedToUnsigned) {

diff --git a/cpp/src/arrow/compute/kernels/cast.cc b/cpp/src/arrow/compute/kernels/cast.cc
@@ -22,7 +22,6 @@
 #include <functional>
 #include <limits>
 #include <memory>
-#include <sstream>
 #include <type_traits>
 #include <utility>
 #include <vector>
@@ -46,26 +45,25 @@
 
 #ifdef ARROW_EXTRA_ERROR_CONTEXT
 
-#define FUNC_RETURN_NOT_OK(s)                                                       \
-  do {                                                                              \
-    Status _s = (s);                                                                \
-    if (ARROW_PREDICT_FALSE(!_s.ok())) {                                            \
-      std::stringstream ss;                                                         \
-      ss << __FILE__ << ":" << __LINE__ << " code: " << #s << "\n" << _s.message(); \
-      ctx->SetStatus(Status(_s.code(), ss.str(), s.detail()));                      \
-      return;                                                                       \
-    }                                                                               \
+#define FUNC_RETURN_NOT_OK(expr)                     \
+  do {                                               \
+    Status _st = (expr);                             \
+    if (ARROW_PREDICT_FALSE(!_st.ok())) {            \
+      _st.AddContextLine(__FILE__, __LINE__, #expr); \
+      ctx->SetStatus(_st);                           \
+      return;                                        \
+    }                                                \
   } while (0)
 
 #else
 
-#define FUNC_RETURN_NOT_OK(s)            \
-  do {                                   \
-    Status _s = (s);                     \
-    if (ARROW_PREDICT_FALSE(!_s.ok())) { \
-      ctx->SetStatus(_s);                \
-      return;                            \
-    }                                    \
+#define FUNC_RETURN_NOT_OK(expr)          \
+  do {                                    \
+    Status _st = (expr);                  \
+    if (ARROW_PREDICT_FALSE(!_st.ok())) { \
+      ctx->SetStatus(_st);                \
+      return;                             \
+    }                                     \
   } while (0)
 
 #endif  // ARROW_EXTRA_ERROR_CONTEXT

diff --git a/cpp/src/arrow/compute/kernels/take-benchmark.cc b/cpp/src/arrow/compute/kernels/take-benchmark.cc
@@ -48,7 +48,8 @@ static void TakeInt64(benchmark::State& state) {
 
   auto values = rand.Int64(array_size, -100, 100, args.null_proportion);
 
-  auto indices = rand.Int32(array_size, 0, array_size - 1, args.null_proportion);
+  auto indices = rand.Int32(static_cast<int32_t>(array_size), 0,
+                            static_cast<int32_t>(array_size - 1), args.null_proportion);
 
   TakeBenchmark(state, values, indices);
 }
@@ -64,7 +65,8 @@ static void TakeFixedSizeList1Int64(benchmark::State& state) {
       fixed_size_list(int64(), 1), array_size, int_array, int_array->null_bitmap(),
       int_array->null_count());
 
-  auto indices = rand.Int32(array_size, 0, array_size - 1, args.null_proportion);
+  auto indices = rand.Int32(static_cast<int32_t>(array_size), 0,
+                            static_cast<int32_t>(array_size - 1), args.null_proportion);
 
   TakeBenchmark(state, values, indices);
 }
@@ -110,7 +112,8 @@ static void TakeString(benchmark::State& state) {
   auto values = std::static_pointer_cast<StringArray>(rand.String(
       array_size, string_min_length, string_max_length, args.null_proportion));
 
-  auto indices = rand.Int32(array_size, 0, array_size - 1, args.null_proportion);
+  auto indices = rand.Int32(static_cast<int32_t>(array_size), 0,
+                            static_cast<int32_t>(array_size - 1), args.null_proportion);
 
   TakeBenchmark(state, values, indices);
 }

diff --git a/cpp/src/arrow/csv/column-builder.cc b/cpp/src/arrow/csv/column-builder.cc
@@ -75,8 +75,8 @@ class TypedColumnBuilder : public ColumnBuilder {
       return st;
     } else {
       std::stringstream ss;
-      ss << "In column #" << col_index_ << ": " << st.message();
-      return Status(st.code(), ss.str(), st.detail());
+      ss << "In CSV column #" << col_index_ << ": " << st.message();
+      return st.WithMessage(ss.str());
     }
   }
 

diff --git a/cpp/src/arrow/flight/internal.cc b/cpp/src/arrow/flight/internal.cc
@@ -42,6 +42,8 @@ namespace arrow {
 namespace flight {
 namespace internal {
 
+const char* kGrpcAuthHeader = "auth-token-bin";
+
 Status FromGrpcStatus(const grpc::Status& grpc_status) {
   if (grpc_status.ok()) {
     return Status::OK();

diff --git a/cpp/src/arrow/flight/internal.h b/cpp/src/arrow/flight/internal.h
@@ -64,7 +64,8 @@ namespace flight {
 namespace internal {
 
 /// The name of the header used to pass authentication tokens.
-static const char* kGrpcAuthHeader = "auth-token-bin";
+ARROW_FLIGHT_EXPORT
+extern const char* kGrpcAuthHeader;
 
 ARROW_FLIGHT_EXPORT
 Status SchemaToString(const Schema& schema, std::string* out);

diff --git a/cpp/src/arrow/flight/types.cc b/cpp/src/arrow/flight/types.cc
@@ -31,6 +31,11 @@
 namespace arrow {
 namespace flight {
 
+const char* kSchemeGrpc = "grpc";
+const char* kSchemeGrpcTcp = "grpc+tcp";
+const char* kSchemeGrpcUnix = "grpc+unix";
+const char* kSchemeGrpcTls = "grpc+tls";
+
 bool FlightDescriptor::Equals(const FlightDescriptor& other) const {
   if (type != other.type) {
     return false;

diff --git a/cpp/src/arrow/flight/types.h b/cpp/src/arrow/flight/types.h
@@ -135,10 +135,14 @@ struct ARROW_FLIGHT_EXPORT Ticket {
 class FlightClient;
 class FlightServerBase;
 
-static const char* kSchemeGrpc = "grpc";
-static const char* kSchemeGrpcTcp = "grpc+tcp";
-static const char* kSchemeGrpcUnix = "grpc+unix";
-static const char* kSchemeGrpcTls = "grpc+tls";
+ARROW_FLIGHT_EXPORT
+extern const char* kSchemeGrpc;
+ARROW_FLIGHT_EXPORT
+extern const char* kSchemeGrpcTcp;
+ARROW_FLIGHT_EXPORT
+extern const char* kSchemeGrpcUnix;
+ARROW_FLIGHT_EXPORT
+extern const char* kSchemeGrpcTls;
 
 /// \brief A host location (a URI)
 struct ARROW_FLIGHT_EXPORT Location {