Fix rebase automerge

.github/workflows/matlab.yml

@@ -37,7 +37,7 @@ concurrency:
 
 jobs:
 
-  matlab:
+  ubuntu:
     name: AMD64 Ubuntu 20.04 MATLAB
     runs-on: ubuntu-latest
     if: ${{ !contains(github.event.pull_request.title, 'WIP') }}
@@ -49,7 +49,7 @@ jobs:
       - name: Install ninja-build
         run: sudo apt-get install ninja-build
       - name: Install MATLAB
-        uses: matlab-actions/setup-matlab@v0
+        uses: matlab-actions/setup-matlab@v1
       - name: Build MATLAB Interface
         run: ci/scripts/matlab_build.sh $(pwd)
       - name: Run MATLAB Tests
@@ -68,3 +68,26 @@ jobs:
         uses: matlab-actions/run-tests@v1
         with:
           select-by-folder: matlab/test
+  macos:
+    name: AMD64 MacOS 10.15 MATLAB
+    runs-on: macos-latest
+    if: ${{ !contains(github.event.pull_request.title, 'WIP') }}
+    steps:
+      - name: Check out repository        
+        uses: actions/checkout@v2
+        with:
+          fetch-depth: 0
+      - name: Install ninja-build
+        run: brew install ninja
+      - name: Install MATLAB
+        uses: matlab-actions/setup-matlab@v1
+      - name: Build MATLAB Interface
+        run: ci/scripts/matlab_build.sh $(pwd)
+      - name: Run MATLAB Tests
+        env:
+          # Add the installation directory to the MATLAB Search Path by
+          # setting the MATLABPATH environment variable.
+          MATLABPATH: matlab/install/arrow_matlab
+        uses: matlab-actions/run-tests@v1
+        with:
+          select-by-folder: matlab/test 

c_glib/arrow-flight-glib/client.cpp

@@ -264,6 +264,26 @@ gaflight_client_new(GAFlightLocation *location,
   }
 }
 
+/**
+ * gaflight_client_close:
+ * @client: A #GAFlightClient.
+ * @error: (nullable): Return location for a #GError or %NULL.
+ *
+ * Returns: %TRUE on success, %FALSE if there was an error.
+ *
+ * Since: 8.0.0
+ */
+gboolean
+gaflight_client_close(GAFlightClient *client,
+                      GError **error)
+{
+  auto flight_client = gaflight_client_get_raw(client);
+  auto status = flight_client->Close();
+  return garrow::check(error,
+                       status,
+                       "[flight-client][close]");
+}
+
 /**
  * gaflight_client_list_flights:
  * @client: A #GAFlightClient.

c_glib/arrow-flight-glib/client.h

@@ -86,6 +86,11 @@ gaflight_client_new(GAFlightLocation *location,
                     GAFlightClientOptions *options,
                     GError **error);
 
+GARROW_AVAILABLE_IN_8_0
+gboolean
+gaflight_client_close(GAFlightClient *client,
+                      GError **error);
+
 GARROW_AVAILABLE_IN_5_0
 GList *
 gaflight_client_list_flights(GAFlightClient *client,

c_glib/test/flight/test-client.rb

@@ -36,6 +36,13 @@ def teardown
     @server.shutdown
   end
 
+  def test_close
+    client = ArrowFlight::Client.new(@location)
+    client.close
+    # Idempotent
+    client.close
+  end
+
   def test_list_flights
     client = ArrowFlight::Client.new(@location)
     generator = Helper::FlightInfoGenerator.new

ci/appveyor-cpp-build.bat

@@ -26,6 +26,8 @@ git submodule update --init || exit /B
 set ARROW_TEST_DATA=%CD%\testing\data
 set PARQUET_TEST_DATA=%CD%\cpp\submodules\parquet-testing\data
 
+set ARROW_DEBUG_MEMORY_POOL=trap
+
 @rem
 @rem In the configurations below we disable building the Arrow static library
 @rem to save some time.  Unfortunately this will still build the Parquet static

ci/conda_env_cpp.txt

@@ -15,7 +15,7 @@
 # specific language governing permissions and limitations
 # under the License.
 
-aws-sdk-cpp
+aws-sdk-cpp=1.8.186
 benchmark>=1.6.0
 boost-cpp>=1.68.0
 brotli
@@ -25,13 +25,18 @@ cmake
 gflags
 glog
 gmock>=1.10.0
-grpc-cpp>=1.27.3
+google-cloud-cpp>=1.34.0
+# 1.45.0 appears to segfault on Windows/AppVeyor
+grpc-cpp>=1.27.3,<1.45.0
 gtest>=1.10.0
 libprotobuf
 libutf8proc
 lz4-c
 make
 ninja
+# Required by google-cloud-cpp, the Conda package is missing the dependency:
+#    https://github.com/conda-forge/google-cloud-cpp-feedstock/issues/28
+nlohmann_json
 pkg-config
 python
 rapidjson

ci/docker/conda-cpp.dockerfile

@@ -22,9 +22,6 @@ FROM ${repo}:${arch}-conda
 COPY ci/scripts/install_minio.sh /arrow/ci/scripts
 RUN /arrow/ci/scripts/install_minio.sh latest /opt/conda
 
-COPY ci/scripts/install_gcs_testbench.sh /arrow/ci/scripts
-RUN /arrow/ci/scripts/install_gcs_testbench.sh default
-
 # install the required conda packages into the test environment
 COPY ci/conda_env_cpp.txt \
      ci/conda_env_gandiva.txt \
@@ -37,12 +34,17 @@ RUN mamba install \
         valgrind && \
     mamba clean --all
 
+# We want to install the GCS testbench using the same Python binary that the Conda code will use.
+COPY ci/scripts/install_gcs_testbench.sh /arrow/ci/scripts
+RUN /arrow/ci/scripts/install_gcs_testbench.sh default
+
 ENV ARROW_BUILD_TESTS=ON \
     ARROW_DATASET=ON \
     ARROW_DEPENDENCY_SOURCE=CONDA \
     ARROW_FLIGHT=ON \
     ARROW_FLIGHT_SQL=ON \
     ARROW_GANDIVA=ON \
+    ARROW_GCS=ON \
     ARROW_HOME=$CONDA_PREFIX \
     ARROW_ORC=ON \
     ARROW_PARQUET=ON \
@@ -57,6 +59,7 @@ ENV ARROW_BUILD_TESTS=ON \
     ARROW_WITH_SNAPPY=ON \
     ARROW_WITH_ZLIB=ON \
     ARROW_WITH_ZSTD=ON \
+    CMAKE_CXX_STANDARD=17 \
     GTest_SOURCE=BUNDLED \
     PARQUET_BUILD_EXAMPLES=ON \
     PARQUET_BUILD_EXECUTABLES=ON \

ci/scripts/c_glib_test.sh

@@ -26,6 +26,9 @@ export LD_LIBRARY_PATH=${ARROW_HOME}/lib:${LD_LIBRARY_PATH}
 export PKG_CONFIG_PATH=${ARROW_HOME}/lib/pkgconfig
 export GI_TYPELIB_PATH=${ARROW_HOME}/lib/girepository-1.0
 
+# Enable memory debug checks.
+export ARROW_DEBUG_MEMORY_POOL=trap
+
 pushd ${source_dir}
 
 ruby test/run-test.rb

ci/scripts/cpp_build.sh

@@ -137,6 +137,7 @@ cmake \
   -DCMAKE_VERBOSE_MAKEFILE=${CMAKE_VERBOSE_MAKEFILE:-OFF} \
   -DCMAKE_C_FLAGS="${CFLAGS:-}" \
   -DCMAKE_CXX_FLAGS="${CXXFLAGS:-}" \
+  -DCMAKE_CXX_STANDARD="${CMAKE_CXX_STANDARD:-11}" \
   -DCMAKE_INSTALL_LIBDIR=${CMAKE_INSTALL_LIBDIR:-lib} \
   -DCMAKE_INSTALL_PREFIX=${CMAKE_INSTALL_PREFIX:-${ARROW_HOME}} \
   -DCMAKE_UNITY_BUILD=${CMAKE_UNITY_BUILD:-OFF} \

ci/scripts/cpp_test.sh

@@ -37,6 +37,9 @@ export LD_LIBRARY_PATH=${ARROW_HOME}/${CMAKE_INSTALL_LIBDIR:-lib}:${LD_LIBRARY_P
 # to retrieve metadata. Disable this so that S3FileSystem tests run faster.
 export AWS_EC2_METADATA_DISABLED=TRUE
 
+# Enable memory debug checks.
+export ARROW_DEBUG_MEMORY_POOL=trap
+
 ctest_options=()
 case "$(uname)" in
   Linux)

ci/scripts/java_jni_macos_build.sh

@@ -50,6 +50,7 @@ mkdir -p "${build_dir}"
 pushd "${build_dir}"
 
 cmake \
+  -GNinja \
   -DARROW_BOOST_USE_SHARED=OFF \
   -DARROW_BROTLI_USE_SHARED=OFF \
   -DARROW_BUILD_TESTS=${ARROW_BUILD_TESTS} \

ci/scripts/python_test.sh

@@ -30,6 +30,9 @@ export ARROW_GDB_SCRIPT=${arrow_dir}/cpp/gdb_arrow.py
 # Enable some checks inside Python itself
 export PYTHONDEVMODE=1
 
+# Enable memory debug checks.
+export ARROW_DEBUG_MEMORY_POOL=trap
+
 # By default, force-test all optional components
 : ${PYARROW_TEST_CUDA:=${ARROW_CUDA:-ON}}
 : ${PYARROW_TEST_DATASET:=${ARROW_DATASET:-ON}}

ci/scripts/r_docker_configure.sh

@@ -30,6 +30,18 @@ fi
 # Ensure parallel compilation of C/C++ code
 echo "MAKEFLAGS=-j$(${R_BIN} -s -e 'cat(parallel::detectCores())')" >> $(R RHOME)/etc/Renviron.site
 
+# Figure out what package manager we have
+if [ "`which dnf`" ]; then
+  PACKAGE_MANAGER=dnf
+elif [ "`which yum`" ]; then
+  PACKAGE_MANAGER=yum
+elif [ "`which zypper`" ]; then
+  PACKAGE_MANAGER=zypper
+else
+  PACKAGE_MANAGER=apt-get
+  apt-get update
+fi
+
 # Special hacking to try to reproduce quirks on fedora-clang-devel on CRAN
 # which uses a bespoke clang compiled to use libc++
 # https://www.stats.ox.ac.uk/pub/bdr/Rconfig/r-devel-linux-x86_64-fedora-clang
@@ -48,26 +60,16 @@ fi
 # Special hacking to try to reproduce quirks on centos using non-default build
 # tooling.
 if [[ "$DEVTOOLSET_VERSION" -gt 0 ]]; then
-  if [ "`which dnf`" ]; then
-    dnf install -y centos-release-scl
-    dnf install -y "devtoolset-$DEVTOOLSET_VERSION"
-  else
-    yum install -y centos-release-scl
-    yum install -y "devtoolset-$DEVTOOLSET_VERSION"
-  fi
+  $PACKAGE_MANAGER install -y centos-release-scl
+  $PACKAGE_MANAGER install -y "devtoolset-$DEVTOOLSET_VERSION"
 fi
 
-# Install openssl for S3 support
 if [ "$ARROW_S3" == "ON" ] || [ "$ARROW_R_DEV" == "TRUE" ]; then
-  if [ "`which dnf`" ]; then
-    dnf install -y libcurl-devel openssl-devel
-  elif [ "`which yum`" ]; then
-    yum install -y libcurl-devel openssl-devel
-  elif [ "`which zypper`" ]; then
-    zypper install -y libcurl-devel libopenssl-devel
-  else
-    apt-get update
+  # Install curl and openssl for S3 support
+  if [ "$PACKAGE_MANAGER" = "apt-get" ]; then
     apt-get install -y libcurl4-openssl-dev libssl-dev
+  else
+    $PACKAGE_MANAGER install -y libcurl-devel openssl-devel
   fi
 
   # The Dockerfile should have put this file here
@@ -80,5 +82,8 @@ if [ "$ARROW_S3" == "ON" ] || [ "$ARROW_R_DEV" == "TRUE" ]; then
   fi
 fi
 
+# Install rsync for bundling cpp source
+$PACKAGE_MANAGER install -y rsync
+
 # Workaround for html help install failure; see https://github.com/r-lib/devtools/issues/2084#issuecomment-530912786
 Rscript -e 'x <- file.path(R.home("doc"), "html"); if (!file.exists(x)) {dir.create(x, recursive=TRUE); file.copy(system.file("html/R.css", package="stats"), x)}'

ci/scripts/r_test.sh

@@ -26,6 +26,13 @@ pushd ${source_dir}
 
 printenv
 
+# Before release, we always copy the relevant parts of the cpp source into the
+# package. In some CI checks, we will use this version of the source:
+# this is done by setting ARROW_SOURCE_HOME to something other than "/arrow"
+# (which is where the arrow git checkout is found in docker and other CI jobs)
+# In the other CI checks the files are synced but ignored.
+make sync-cpp
+
 if [ "$ARROW_USE_PKG_CONFIG" != "false" ]; then
   export LD_LIBRARY_PATH=${ARROW_HOME}/lib:${LD_LIBRARY_PATH}
   export R_LD_LIBRARY_PATH=${LD_LIBRARY_PATH}
@@ -56,6 +63,9 @@ export _R_CHECK_TESTS_NLINES_=0
 # to retrieve metadata. Disable this so that S3FileSystem tests run faster.
 export AWS_EC2_METADATA_DISABLED=TRUE
 
+# Enable memory debug checks.
+export ARROW_DEBUG_MEMORY_POOL=trap
+
 # Hack so that texlive2020 doesn't pollute the home dir
 export TEXMFCONFIG=/tmp/texmf-config
 export TEXMFVAR=/tmp/texmf-var

ci/scripts/ruby_test.sh

@@ -26,4 +26,7 @@ export LD_LIBRARY_PATH=${ARROW_HOME}/lib:${LD_LIBRARY_PATH}
 export PKG_CONFIG_PATH=${ARROW_HOME}/lib/pkgconfig
 export GI_TYPELIB_PATH=${ARROW_HOME}/lib/girepository-1.0
 
+# Enable memory debug checks.
+export ARROW_DEBUG_MEMORY_POOL=trap
+
 rake -f ${source_dir}/Rakefile BUILD_DIR=${build_dir} USE_BUNDLER=yes

cpp/cmake_modules/ThirdpartyToolchain.cmake

@@ -1751,6 +1751,10 @@ if(ARROW_JEMALLOC)
        # See https://github.com/jemalloc/jemalloc/issues/1237
        "--disable-initial-exec-tls"
        ${EP_LOG_OPTIONS})
+  if(${UPPERCASE_BUILD_TYPE} STREQUAL "DEBUG")
+    # Enable jemalloc debug checks when Arrow itself has debugging enabled
+    list(APPEND JEMALLOC_CONFIGURE_COMMAND "--enable-debug")
+  endif()
   set(JEMALLOC_BUILD_COMMAND ${MAKE} ${MAKE_BUILD_ARGS})
   if(CMAKE_OSX_SYSROOT)
     list(APPEND JEMALLOC_BUILD_COMMAND "SDKROOT=${CMAKE_OSX_SYSROOT}")

cpp/examples/arrow/CMakeLists.txt

@@ -92,6 +92,24 @@ if(ARROW_FLIGHT)
                     EXTRA_SOURCES
                     "${CMAKE_CURRENT_BINARY_DIR}/helloworld.pb.cc"
                     "${CMAKE_CURRENT_BINARY_DIR}/helloworld.grpc.pb.cc")
+
+  if(ARROW_FLIGHT_SQL)
+    if(ARROW_GRPC_USE_SHARED)
+      set(FLIGHT_SQL_EXAMPLES_LINK_LIBS arrow_flight_sql_shared)
+    else()
+      set(FLIGHT_SQL_EXAMPLES_LINK_LIBS arrow_flight_sql_static)
+    endif()
+
+    add_arrow_example(flight_sql_example
+                      DEPENDENCIES
+                      flight-sql-test-server
+                      EXTRA_LINK_LIBS
+                      ${FLIGHT_EXAMPLES_LINK_LIBS}
+                      ${FLIGHT_SQL_EXAMPLES_LINK_LIBS}
+                      gRPC::grpc++
+                      ${ARROW_PROTOBUF_LIBPROTOBUF}
+                      ${GFLAGS_LIBRARIES})
+  endif()
 endif()
 
 if(ARROW_PARQUET AND ARROW_DATASET)
@@ -111,5 +129,8 @@ if(ARROW_PARQUET AND ARROW_DATASET)
 
   add_arrow_example(execution_plan_documentation_examples EXTRA_LINK_LIBS
                     ${DATASET_EXAMPLES_LINK_LIBS})
-  add_dependencies(dataset_documentation_example parquet)
+  add_dependencies(execution-plan-documentation-examples parquet)
+
+  add_arrow_example(join_example EXTRA_LINK_LIBS ${DATASET_EXAMPLES_LINK_LIBS})
+  add_dependencies(join-example parquet)
 endif()