Merge pull request #5 from apache/master

Arrow Master on 2/3

.travis.yml

@@ -188,6 +188,7 @@ matrix:
     - ARROW_TRAVIS_GANDIVA=1
     - ARROW_TRAVIS_GANDIVA_JAVA=1
     - ARROW_TRAVIS_OPTIONAL_INSTALL=1
+    - ARROW_TRAVIS_VERBOSE=0
     - ARROW_BUILD_WARNING_LEVEL=CHECKIN
     # ARROW-3803: The Xcode 8.3 image has Boost libraries in /usr/local/lib
     # which can get loaded before the toolchain Boost libraries. These seem to

LICENSE.txt

@@ -681,7 +681,11 @@ See the License for the specific language governing permissions and
 limitations under the License.
 
 --------------------------------------------------------------------------------
-The file cpp/src/arrow/vendored/date.h has the following license (MIT)
+The files cpp/src/arrow/vendored/datetime/date.h, cpp/src/arrow/vendored/datetime/tz.h,
+cpp/src/arrow/vendored/datetime/tz_private.h, cpp/src/arrow/vendored/datetime/ios.h,
+cpp/src/arrow/vendored/datetime/tz.cpp are adapted from
+Howard Hinnant's date library (https://github.com/HowardHinnant/date)
+It is licensed under MIT license.
 
 The MIT License (MIT)
 Copyright (c) 2015, 2016, 2017 Howard Hinnant

appveyor.yml

@@ -47,6 +47,7 @@ matrix:
 environment:
   global:
     USE_CLCACHE: true
+    ARROW_BUILD_GANDIVA: "OFF"
     PYTHON: "3.6"
     ARCH: "64"
 

c_glib/arrow-glib/table-builder.cpp

@@ -41,7 +41,7 @@ G_BEGIN_DECLS
 
 typedef struct GArrowRecordBatchBuilderPrivate_ {
   arrow::RecordBatchBuilder *record_batch_builder;
-  GPtrArray *fields;
+  GPtrArray *column_builders;
 } GArrowRecordBatchBuilderPrivate;
 
 enum {
@@ -63,13 +63,13 @@ garrow_record_batch_builder_constructed(GObject *object)
 {
   auto priv = GARROW_RECORD_BATCH_BUILDER_GET_PRIVATE(object);
   auto arrow_builder = priv->record_batch_builder;
-  auto n_fields = arrow_builder->num_fields();
-  priv->fields = g_ptr_array_new_full(n_fields, g_object_unref);
-  for (int i = 0; i < n_fields; ++i) {
+  auto n_columns = arrow_builder->num_fields();
+  priv->column_builders = g_ptr_array_new_full(n_columns, g_object_unref);
+  for (int i = 0; i < n_columns; ++i) {
     auto arrow_array_builder = arrow_builder->GetField(i);
     auto array_builder = garrow_array_builder_new_raw(arrow_array_builder);
     garrow_array_builder_release_ownership(array_builder);
-    g_ptr_array_add(priv->fields, array_builder);
+    g_ptr_array_add(priv->column_builders, array_builder);
   }
 
   G_OBJECT_CLASS(garrow_record_batch_builder_parent_class)->constructed(object);
@@ -80,7 +80,7 @@ garrow_record_batch_builder_finalize(GObject *object)
 {
   auto priv = GARROW_RECORD_BATCH_BUILDER_GET_PRIVATE(object);
 
-  g_ptr_array_free(priv->fields, TRUE);
+  g_ptr_array_free(priv->column_builders, TRUE);
   delete priv->record_batch_builder;
 
   G_OBJECT_CLASS(garrow_record_batch_builder_parent_class)->finalize(object);
@@ -223,9 +223,26 @@ garrow_record_batch_builder_get_schema(GArrowRecordBatchBuilder *builder)
  * Returns: The number of fields.
  *
  * Since: 0.8.0
+ *
+ * Deprecated: 0.13.0:
+ *   Use garrow_record_batch_builder_get_n_columns() instead.
  */
 gint
 garrow_record_batch_builder_get_n_fields(GArrowRecordBatchBuilder *builder)
+{
+  return garrow_record_batch_builder_get_n_columns(builder);
+}
+
+/**
+ * garrow_record_batch_builder_get_n_columns:
+ * @builder: A #GArrowRecordBatchBuilder.
+ *
+ * Returns: The number of columns.
+ *
+ * Since: 0.13.0
+ */
+gint
+garrow_record_batch_builder_get_n_columns(GArrowRecordBatchBuilder *builder)
 {
   auto arrow_builder = garrow_record_batch_builder_get_raw(builder);
   return arrow_builder->num_fields();
@@ -241,23 +258,44 @@ garrow_record_batch_builder_get_n_fields(GArrowRecordBatchBuilder *builder)
  *   the `i`-th field on success, %NULL on out of index.
  *
  * Since: 0.8.0
+ *
+ * Deprecated: 0.13.0:
+ *   Use garrow_record_batch_builder_get_column_builder() instead.
  */
 GArrowArrayBuilder *
 garrow_record_batch_builder_get_field(GArrowRecordBatchBuilder *builder,
                                       gint i)
+{
+  return garrow_record_batch_builder_get_column_builder(builder, i);
+}
+
+/**
+ * garrow_record_batch_builder_get_column_builder:
+ * @builder: A #GArrowRecordBatchBuilder.
+ * @i: The column index. If it's negative, index is counted backward
+ *   from the end of the columns. `-1` means the last column.
+ *
+ * Returns: (transfer none) (nullable): The #GArrowArrayBuilder for
+ *   the `i`-th column on success, %NULL on out of index.
+ *
+ * Since: 0.13.0
+ */
+GArrowArrayBuilder *
+garrow_record_batch_builder_get_column_builder(GArrowRecordBatchBuilder *builder,
+                                               gint i)
 {
   auto priv = GARROW_RECORD_BATCH_BUILDER_GET_PRIVATE(builder);
   if (i < 0) {
-    i += priv->fields->len;
+    i += priv->column_builders->len;
   }
   if (i < 0) {
     return NULL;
   }
-  if (static_cast<guint>(i) >= priv->fields->len) {
+  if (static_cast<guint>(i) >= priv->column_builders->len) {
     return NULL;
   }
 
-  return GARROW_ARRAY_BUILDER(g_ptr_array_index(priv->fields, i));
+  return GARROW_ARRAY_BUILDER(g_ptr_array_index(priv->column_builders, i));
 }
 
 /**

c_glib/arrow-glib/table-builder.h

@@ -45,9 +45,22 @@ void garrow_record_batch_builder_set_initial_capacity(GArrowRecordBatchBuilder *
                                                       gint64 capacity);
 GArrowSchema *garrow_record_batch_builder_get_schema(GArrowRecordBatchBuilder *builder);
 
+#ifndef GARROW_DISABLE_DEPRECATED
+GARROW_DEPRECATED_IN_0_13_FOR(garrow_record_batch_builder_get_n_columns)
 gint garrow_record_batch_builder_get_n_fields(GArrowRecordBatchBuilder *builder);
+#endif
+GARROW_AVAILABLE_IN_0_13
+gint
+garrow_record_batch_builder_get_n_columns(GArrowRecordBatchBuilder *builder);
+#ifndef GARROW_DISABLE_DEPRECATED
+GARROW_DEPRECATED_IN_0_13_FOR(garrow_record_batch_builder_get_column_builder)
 GArrowArrayBuilder *garrow_record_batch_builder_get_field(GArrowRecordBatchBuilder *builder,
                                                           gint i);
+#endif
+GARROW_AVAILABLE_IN_0_13
+GArrowArrayBuilder *
+garrow_record_batch_builder_get_column_builder(GArrowRecordBatchBuilder *builder,
+                                               gint i);
 
 GArrowRecordBatch *garrow_record_batch_builder_flush(GArrowRecordBatchBuilder *builder,
                                                      GError **error);

c_glib/test/test-record-batch-builder.rb

@@ -37,27 +37,27 @@ def test_schema
     assert_equal(@schema, @builder.schema)
   end
 
-  def test_n_fields
-    assert_equal(@fields.size, @builder.n_fields)
+  def test_n_columns
+    assert_equal(@fields.size, @builder.n_columns)
   end
 
-  sub_test_case("#get_field") do
+  sub_test_case("#get_column_builder") do
     def test_valid
       assert_equal(Arrow::BooleanArrayBuilder,
-                   @builder.get_field(0).class)
+                   @builder.get_column_builder(0).class)
     end
 
     def test_negative
       assert_equal(Arrow::Int32ArrayBuilder,
-                   @builder.get_field(-1).class)
+                   @builder.get_column_builder(-1).class)
     end
 
     def test_too_negative
-      assert_nil(@builder.get_field(-@fields.size - 1))
+      assert_nil(@builder.get_column_builder(-@fields.size - 1))
     end
 
     def test_too_large
-      assert_nil(@builder.get_field(@fields.size))
+      assert_nil(@builder.get_column_builder(@fields.size))
     end
   end
 
@@ -68,7 +68,7 @@ def test_flush
       "point"   => build_int32_array([1, -1, 0]),
     }
     arrays.each_with_index do |(_, array), i|
-      @builder.get_field(i).append_values(array.values, [])
+      @builder.get_column_builder(i).append_values(array.values, [])
     end
     assert_equal(build_record_batch(arrays),
                  @builder.flush)
@@ -78,7 +78,7 @@ def test_flush
       "point"   => build_int32_array([10, -10]),
     }
     arrays.each_with_index do |(_, array), i|
-      @builder.get_field(i).append_values(array.values, [])
+      @builder.get_column_builder(i).append_values(array.values, [])
     end
     assert_equal(build_record_batch(arrays),
                  @builder.flush)

ci/appveyor-cpp-build.bat

@@ -104,6 +104,13 @@ conda create -n arrow -q -y -c conda-forge ^
 
 call activate arrow
 
+set ARROW_LLVM_VERSION=6.0.1
+
+if "%ARROW_BUILD_GANDIVA%" == "ON" (
+  @rem Install llvmdev in the toolchain if building gandiva.dll
+  conda install -q -y llvmdev=%ARROW_LLVM_VERSION% || exit /B
+)
+
 @rem Use Boost from Anaconda
 set BOOST_ROOT=%CONDA_PREFIX%\Library
 set BOOST_LIBRARYDIR=%CONDA_PREFIX%\Library\lib

ci/conda_env_cpp.yml

@@ -25,6 +25,7 @@ double-conversion
 flatbuffers
 gflags
 glog
+gmock
 gtest
 libprotobuf
 lz4-c

ci/cpp-msvc-build-main.bat

@@ -51,8 +51,11 @@ cmake -G "%GENERATOR%" %CMAKE_ARGS% ^
       -DARROW_BUILD_STATIC=OFF ^
       -DARROW_BUILD_TESTS=ON ^
       -DARROW_BUILD_EXAMPLES=ON ^
+      -DARROW_BUILD_EXAMPLES=ON ^
+      -DARROW_VERBOSE_THIRDPARTY_BUILD=ON ^
       -DARROW_CXXFLAGS="%ARROW_CXXFLAGS%" ^
       -DCMAKE_CXX_FLAGS_RELEASE="/MD %CMAKE_CXX_FLAGS_RELEASE%" ^
+      -DARROW_GANDIVA=%ARROW_BUILD_GANDIVA% ^
       -DARROW_PARQUET=ON ^
       -DARROW_PYTHON=ON ^
       ..  || exit /B

ci/travis_script_python.sh

@@ -48,7 +48,6 @@ fi
 conda create -y -q -p $CONDA_ENV_DIR \
       --file $TRAVIS_BUILD_DIR/ci/conda_env_python.yml \
       nomkl \
-      cmake \
       pip \
       numpy=1.14 \
       python=${PYTHON_VERSION} \

cpp/CMakeLists.txt

@@ -309,8 +309,12 @@ Note that this requires linking Boost statically"
 
     set(BROTLI_MSVC_STATIC_LIB_SUFFIX "-static" CACHE STRING
       "Brotli static lib suffix used on Windows with MSVC (default -static)")
+    set(PROTOBUF_MSVC_STATIC_LIB_SUFFIX "" CACHE STRING
+      "Protobuf static lib suffix used on Windows with MSVC (default is empty string)")
+    set(RE2_MSVC_STATIC_LIB_SUFFIX "_static" CACHE STRING
+      "re2 static lib suffix used on Windows with MSVC (default is _static)")
     set(SNAPPY_MSVC_STATIC_LIB_SUFFIX "_static" CACHE STRING
-      "Snappy static lib suffix used on Windows with MSVC (default is empty string)")
+      "Snappy static lib suffix used on Windows with MSVC (default is _static)")
     set(LZ4_MSVC_STATIC_LIB_SUFFIX "_static" CACHE STRING
       "Lz4 static lib suffix used on Windows with MSVC (default _static)")
     set(ZSTD_MSVC_STATIC_LIB_SUFFIX "_static" CACHE STRING
@@ -796,8 +800,9 @@ set(ARROW_TEST_STATIC_LINK_LIBS
   arrow_testing_static
   arrow_static
   ${ARROW_LINK_LIBS}
-  ${GTEST_MAIN_LIBRARY}
-  ${GTEST_LIBRARY})
+  ${GTEST_LIBRARY}
+  ${GMOCK_MAIN_LIBRARY}
+  ${GMOCK_LIBRARY})
 
 set(ARROW_TEST_SHARED_LINK_LIBS
   arrow_testing_shared
@@ -807,8 +812,9 @@ set(ARROW_TEST_SHARED_LINK_LIBS
   ${BOOST_SYSTEM_LIBRARY}
   ${BOOST_FILESYSTEM_LIBRARY}
   ${BOOST_REGEX_LIBRARY}
-  ${GTEST_MAIN_LIBRARY}
-  ${GTEST_LIBRARY})
+  ${GTEST_LIBRARY}
+  ${GMOCK_MAIN_LIBRARY}
+  ${GMOCK_LIBRARY})
 
 if(NOT MSVC)
   set(ARROW_TEST_SHARED_LINK_LIBS

cpp/build-support/lintutils.py

@@ -15,6 +15,7 @@
 # specific language governing permissions and limitations
 # under the License.
 
+import multiprocessing as mp
 import os
 from fnmatch import fnmatch
 from subprocess import Popen
@@ -49,12 +50,12 @@ def run_parallel(cmds, **kwargs):
     """
     Run each of cmds (with shared **kwargs) using subprocess.Popen
     then wait for all of them to complete.
-    Runs batches of os.cpu_count() * 2 from cmds
+    Runs batches of multiprocessing.cpu_count() * 2 from cmds
     returns a list of tuples containing each process'
     returncode, stdout, stderr
     """
     complete = []
-    for cmds_batch in chunk(cmds, os.cpu_count() * 2):
+    for cmds_batch in chunk(cmds, mp.cpu_count() * 2):
         procs_batch = [Popen(cmd, **kwargs) for cmd in cmds_batch]
         for proc in procs_batch:
             stdout, stderr = proc.communicate()