Merge remote-tracking branch 'upstream/master' into cmakelint

apache · Feb 21, 2021 · 6b28115 · 6b28115
2 parents f79b8c5 + 5fc0e5e
commit 6b28115
Show file tree

Hide file tree

Showing 220 changed files with 14,212 additions and 8,439 deletions.
diff --git a/.env b/.env
@@ -27,6 +27,12 @@
 # the cache plugin functional
 DOCKER_VOLUME_PREFIX=
 
+# turn on inline build cache, this is a docker buildx feature documented
+# at https://github.com/docker/buildx#--cache-tonametypetypekeyvalue
+COMPOSE_DOCKER_CLI_BUILD=1
+DOCKER_BUILDKIT=1
+BUILDKIT_INLINE_CACHE=1
+
 ULIMIT_CORE=-1
 REPO=apache/arrow-dev
 ARCH=amd64

diff --git a/.github/workflows/cpp.yml b/.github/workflows/cpp.yml
@@ -38,10 +38,8 @@ on:
       - 'format/Flight.proto'
 
 env:
-  DOCKER_BUILDKIT: 0
-  DOCKER_VOLUME_PREFIX: ".docker/"
-  COMPOSE_DOCKER_CLI_BUILD: 1
   ARROW_ENABLE_TIMING_TESTS: OFF
+  DOCKER_VOLUME_PREFIX: ".docker/"
   ARCHERY_DOCKER_USER: ${{ secrets.DOCKERHUB_USER }}
   ARCHERY_DOCKER_PASSWORD: ${{ secrets.DOCKERHUB_TOKEN }}
 
@@ -72,7 +70,7 @@ jobs:
       - name: Free Up Disk Space
         run: ci/scripts/util_cleanup.sh
       - name: Cache Docker Volumes
-        uses: actions/cache@v1
+        uses: actions/cache@v2
         with:
           path: .docker
           key: ${{ matrix.image }}-${{ hashFiles('cpp/**') }}
@@ -367,7 +365,7 @@ jobs:
         run: |
           ci/scripts/msys2_setup.sh cpp
       - name: Cache ccache
-        uses: actions/cache@v1
+        uses: actions/cache@v2
         with:
           path: ccache
           key: cpp-ccache-mingw${{ matrix.mingw-n-bits }}-${{ hashFiles('cpp/**') }}

diff --git a/.github/workflows/cpp_cron.yml b/.github/workflows/cpp_cron.yml
@@ -30,10 +30,8 @@ on:
         0 */12 * * *
 
 env:
-  DOCKER_BUILDKIT: 0
-  DOCKER_VOLUME_PREFIX: ".docker/"
-  COMPOSE_DOCKER_CLI_BUILD: 1
   ARROW_ENABLE_TIMING_TESTS: OFF
+  DOCKER_VOLUME_PREFIX: ".docker/"
   ARCHERY_DOCKER_USER: ${{ secrets.DOCKERHUB_USER }}
   ARCHERY_DOCKER_PASSWORD: ${{ secrets.DOCKERHUB_TOKEN }}
 
@@ -83,7 +81,7 @@ jobs:
       - name: Free Up Disk Space
         run: ci/scripts/util_cleanup.sh
       - name: Cache Docker Volumes
-        uses: actions/cache@v1
+        uses: actions/cache@v2
         with:
           path: .docker
           key: ${{ matrix.name }}-${{ hashFiles('cpp/**') }}

diff --git a/.github/workflows/dev.yml b/.github/workflows/dev.yml
@@ -23,8 +23,6 @@ on:
   pull_request:
 
 env:
-  DOCKER_BUILDKIT: 0
-  COMPOSE_DOCKER_CLI_BUILD: 1
   ARCHERY_DOCKER_USER: ${{ secrets.DOCKERHUB_USER }}
   ARCHERY_DOCKER_PASSWORD: ${{ secrets.DOCKERHUB_TOKEN }}
 

diff --git a/.github/workflows/go.yml b/.github/workflows/go.yml
@@ -33,8 +33,6 @@ on:
       - 'go/**'
 
 env:
-  DOCKER_BUILDKIT: 0
-  COMPOSE_DOCKER_CLI_BUILD: 1
   ARCHERY_DOCKER_USER: ${{ secrets.DOCKERHUB_USER }}
   ARCHERY_DOCKER_PASSWORD: ${{ secrets.DOCKERHUB_TOKEN }}
 

diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml
@@ -44,9 +44,7 @@ on:
       - 'rust/**'
 
 env:
-  DOCKER_BUILDKIT: 0
   DOCKER_VOLUME_PREFIX: ".docker/"
-  COMPOSE_DOCKER_CLI_BUILD: 1
   ARCHERY_DOCKER_USER: ${{ secrets.DOCKERHUB_USER }}
   ARCHERY_DOCKER_PASSWORD: ${{ secrets.DOCKERHUB_TOKEN }}
 
@@ -66,7 +64,7 @@ jobs:
       - name: Free Up Disk Space
         run: ci/scripts/util_cleanup.sh
       - name: Cache Docker Volumes
-        uses: actions/cache@v1
+        uses: actions/cache@v2
         with:
           path: .docker
           key: conda-${{ hashFiles('cpp/**') }}

diff --git a/.github/workflows/java.yml b/.github/workflows/java.yml
@@ -36,9 +36,7 @@ on:
       - 'java/**'
 
 env:
-  DOCKER_BUILDKIT: 0
   DOCKER_VOLUME_PREFIX: ".docker/"
-  COMPOSE_DOCKER_CLI_BUILD: 1
   ARCHERY_DOCKER_USER: ${{ secrets.DOCKERHUB_USER }}
   ARCHERY_DOCKER_PASSWORD: ${{ secrets.DOCKERHUB_TOKEN }}
 
@@ -68,7 +66,7 @@ jobs:
         shell: bash
         run: ci/scripts/util_cleanup.sh
       - name: Cache Docker Volumes
-        uses: actions/cache@v1
+        uses: actions/cache@v2
         with:
           path: .docker
           key: maven-${{ hashFiles('java/**') }}

diff --git a/.github/workflows/java_jni.yml b/.github/workflows/java_jni.yml
@@ -36,9 +36,7 @@ on:
       - 'java/**'
 
 env:
-  DOCKER_BUILDKIT: 0
   DOCKER_VOLUME_PREFIX: ".docker/"
-  COMPOSE_DOCKER_CLI_BUILD: 1
   ARCHERY_DOCKER_USER: ${{ secrets.DOCKERHUB_USER }}
   ARCHERY_DOCKER_PASSWORD: ${{ secrets.DOCKERHUB_TOKEN }}
 
@@ -66,7 +64,7 @@ jobs:
       - name: Free Up Disk Space
         run: ci/scripts/util_cleanup.sh
       - name: Cache Docker Volumes
-        uses: actions/cache@v1
+        uses: actions/cache@v2
         with:
           path: .docker
           key: maven-${{ hashFiles('java/**') }}

diff --git a/.github/workflows/js.yml b/.github/workflows/js.yml
@@ -32,8 +32,6 @@ on:
       - 'js/**'
 
 env:
-  DOCKER_BUILDKIT: 0
-  COMPOSE_DOCKER_CLI_BUILD: 1
   ARCHERY_DOCKER_USER: ${{ secrets.DOCKERHUB_USER }}
   ARCHERY_DOCKER_PASSWORD: ${{ secrets.DOCKERHUB_TOKEN }}
 

diff --git a/.github/workflows/python.yml b/.github/workflows/python.yml
@@ -32,9 +32,7 @@ on:
       - 'python/**'
 
 env:
-  DOCKER_BUILDKIT: 0
   DOCKER_VOLUME_PREFIX: ".docker/"
-  COMPOSE_DOCKER_CLI_BUILD: 1
   ARCHERY_DOCKER_USER: ${{ secrets.DOCKERHUB_USER }}
   ARCHERY_DOCKER_PASSWORD: ${{ secrets.DOCKERHUB_TOKEN }}
 
@@ -84,7 +82,7 @@ jobs:
       - name: Free Up Disk Space
         run: ci/scripts/util_cleanup.sh
       - name: Cache Docker Volumes
-        uses: actions/cache@v1
+        uses: actions/cache@v2
         with:
           path: .docker
           key: ${{ matrix.cache }}-${{ hashFiles('cpp/**') }}

diff --git a/.github/workflows/python_cron.yml b/.github/workflows/python_cron.yml
@@ -29,9 +29,7 @@ on:
         0 */12 * * *
 
 env:
-  DOCKER_BUILDKIT: 0
   DOCKER_VOLUME_PREFIX: ".docker/"
-  COMPOSE_DOCKER_CLI_BUILD: 1
   ARCHERY_DOCKER_USER: ${{ secrets.DOCKERHUB_USER }}
   ARCHERY_DOCKER_PASSWORD: ${{ secrets.DOCKERHUB_TOKEN }}
 
@@ -121,7 +119,7 @@ jobs:
       - name: Free Up Disk Space
         run: ci/scripts/util_cleanup.sh
       - name: Cache Docker Volumes
-        uses: actions/cache@v1
+        uses: actions/cache@v2
         with:
           path: .docker
           key: ${{ matrix.cache }}-${{ hashFiles('cpp/**') }}

diff --git a/.github/workflows/r.yml b/.github/workflows/r.yml
@@ -40,9 +40,7 @@ on:
       - 'r/**'
 
 env:
-  DOCKER_BUILDKIT: 0
   DOCKER_VOLUME_PREFIX: ".docker/"
-  COMPOSE_DOCKER_CLI_BUILD: 1
   ARCHERY_DOCKER_USER: ${{ secrets.DOCKERHUB_USER }}
   ARCHERY_DOCKER_PASSWORD: ${{ secrets.DOCKERHUB_TOKEN }}
 
@@ -69,7 +67,7 @@ jobs:
       - name: Free Up Disk Space
         run: ci/scripts/util_cleanup.sh
       - name: Cache Docker Volumes
-        uses: actions/cache@v1
+        uses: actions/cache@v2
         with:
           path: .docker
           key: ubuntu-${{ matrix.ubuntu }}-r-${{ matrix.r }}-${{ hashFiles('cpp/**') }}
@@ -117,7 +115,7 @@ jobs:
       - name: Free Up Disk Space
         run: ci/scripts/util_cleanup.sh
       - name: Cache Docker Volumes
-        uses: actions/cache@v1
+        uses: actions/cache@v2
         with:
           path: .docker
           key: ${{ matrix.config.image }}-r-${{ hashFiles('cpp/**') }}

diff --git a/.github/workflows/ruby.yml b/.github/workflows/ruby.yml
@@ -44,9 +44,7 @@ on:
       - 'ruby/**'
 
 env:
-  DOCKER_BUILDKIT: 0
   DOCKER_VOLUME_PREFIX: ".docker/"
-  COMPOSE_DOCKER_CLI_BUILD: 1
   ARCHERY_DOCKER_USER: ${{ secrets.DOCKERHUB_USER }}
   ARCHERY_DOCKER_PASSWORD: ${{ secrets.DOCKERHUB_TOKEN }}
 
@@ -76,7 +74,7 @@ jobs:
         shell: bash
         run: ci/scripts/util_cleanup.sh
       - name: Cache Docker Volumes
-        uses: actions/cache@v1
+        uses: actions/cache@v2
         with:
           path: .docker
           key: ubuntu-${{ matrix.ubuntu }}-ruby-${{ hashFiles('cpp/**') }}

diff --git a/c_glib/arrow-glib/reader.cpp b/c_glib/arrow-glib/reader.cpp
@@ -1592,6 +1592,7 @@ garrow_csv_reader_new(GArrowInputStream *input,
 
   auto arrow_reader =
     arrow::csv::TableReader::Make(arrow::default_memory_pool(),
+                                  arrow::io::AsyncContext(),
                                   arrow_input,
                                   read_options,
                                   parse_options,

diff --git a/ci/docker/python-wheel-manylinux-201x.dockerfile b/ci/docker/python-wheel-manylinux-201x.dockerfile
@@ -62,7 +62,8 @@ ARG build_type=release
 ENV CMAKE_BUILD_TYPE=${build_type} \
     VCPKG_FORCE_SYSTEM_BINARIES=1 \
     VCPKG_OVERLAY_TRIPLETS=/arrow/ci/vcpkg \
-    VCPKG_DEFAULT_TRIPLET=x64-linux-static-${build_type}
+    VCPKG_DEFAULT_TRIPLET=x64-linux-static-${build_type} \
+    VCPKG_FEATURE_FLAGS=-manifests
 
 # TODO(kszucs): factor out the package enumeration to a text file and reuse it
 # from the windows image and potentially in a future macos wheel build

diff --git a/ci/docker/python-wheel-windows-vs2017.dockerfile b/ci/docker/python-wheel-windows-vs2017.dockerfile
@@ -46,7 +46,8 @@ COPY ci/vcpkg arrow/ci/vcpkg
 ARG build_type=release
 ENV CMAKE_BUILD_TYPE=${build_type} \
     VCPKG_OVERLAY_TRIPLETS=C:\\arrow\\ci\\vcpkg \
-    VCPKG_DEFAULT_TRIPLET=x64-windows-static-md-${build_type}
+    VCPKG_DEFAULT_TRIPLET=x64-windows-static-md-${build_type} \
+    VCPKG_FEATURE_FLAGS=-manifests
 RUN vcpkg install --clean-after-build \
         abseil \
         aws-sdk-cpp[config,cognito-identity,core,identity-management,s3,sts,transfer] \

diff --git a/ci/scripts/PKGBUILD b/ci/scripts/PKGBUILD
@@ -25,6 +25,8 @@ arch=("any")
 url="https://arrow.apache.org/"
 license=("Apache-2.0")
 depends=("${MINGW_PACKAGE_PREFIX}-aws-sdk-cpp"
+         "${MINGW_PACKAGE_PREFIX}-libutf8proc"
+         "${MINGW_PACKAGE_PREFIX}-re2"
          "${MINGW_PACKAGE_PREFIX}-thrift"
          "${MINGW_PACKAGE_PREFIX}-snappy"
          "${MINGW_PACKAGE_PREFIX}-zlib"
@@ -103,9 +105,7 @@ build() {
     -DARROW_SNAPPY_USE_SHARED=OFF \
     -DARROW_USE_GLOG=OFF \
     -DARROW_WITH_LZ4=ON \
-    -DARROW_WITH_RE2=OFF \
     -DARROW_WITH_SNAPPY=ON \
-    -DARROW_WITH_UTF8PROC=OFF \
     -DARROW_WITH_ZLIB=ON \
     -DARROW_WITH_ZSTD=ON \
     -DARROW_ZSTD_USE_SHARED=OFF \

diff --git a/ci/scripts/cpp_test.sh b/ci/scripts/cpp_test.sh
@@ -86,6 +86,7 @@ if [ "${ARROW_FUZZING}" == "ON" ]; then
     ${binary_output_dir}/arrow-ipc-stream-fuzz ${ARROW_TEST_DATA}/arrow-ipc-stream/crash-*
     ${binary_output_dir}/arrow-ipc-stream-fuzz ${ARROW_TEST_DATA}/arrow-ipc-stream/*-testcase-*
     ${binary_output_dir}/arrow-ipc-file-fuzz ${ARROW_TEST_DATA}/arrow-ipc-file/*-testcase-*
+    ${binary_output_dir}/arrow-ipc-tensor-stream-fuzz ${ARROW_TEST_DATA}/arrow-ipc-tensor-stream/*-testcase-*
     if [ "${ARROW_PARQUET}" == "ON" ]; then
       ${binary_output_dir}/parquet-arrow-fuzz ${ARROW_TEST_DATA}/parquet/fuzzing/*-testcase-*
     fi

diff --git a/ci/scripts/integration_arrow.sh b/ci/scripts/integration_arrow.sh
@@ -30,4 +30,6 @@ pip install -e $arrow_dir/dev/archery
 archery integration --with-all --run-flight \
     --gold-dirs=$gold_dir/0.14.1 \
     --gold-dirs=$gold_dir/0.17.1 \
+    --gold-dirs=$gold_dir/1.0.0-bigendian \
+    --gold-dirs=$gold_dir/1.0.0-littleendian \
     --gold-dirs=$gold_dir/2.0.0-compression \
diff --git a/ci/scripts/python_wheel_manylinux_build.sh b/ci/scripts/python_wheel_manylinux_build.sh
@@ -67,6 +67,7 @@ echo "=== (${PYTHON_VERSION}) Building Arrow C++ libraries ==="
 : ${ARROW_WITH_ZSTD:=ON}
 : ${CMAKE_BUILD_TYPE:=release}
 : ${CMAKE_GENERATOR:=Ninja}
+: ${VCPKG_FEATURE_FLAGS:=-manifests}
 
 mkdir /tmp/arrow-build
 pushd /tmp/arrow-build
@@ -106,6 +107,8 @@ cmake \
     -DCMAKE_UNITY_BUILD=ON \
     -DOPENSSL_USE_STATIC_LIBS=ON \
     -DThrift_ROOT=/opt/vcpkg/installed/x64-linux/lib \
+    -D_VCPKG_INSTALLED_DIR=/opt/vcpkg/installed \
+    -DVCPKG_MANIFEST_MODE=OFF \
     -DVCPKG_TARGET_TRIPLET=x64-linux-static-${CMAKE_BUILD_TYPE} \
     -G ${CMAKE_GENERATOR} \
     /arrow/cpp

diff --git a/ci/scripts/python_wheel_windows_build.bat b/ci/scripts/python_wheel_windows_build.bat
@@ -47,6 +47,7 @@ set ARROW_WITH_ZLIB=ON
 set ARROW_WITH_ZSTD=ON
 set CMAKE_UNITY_BUILD=ON
 set CMAKE_GENERATOR=Visual Studio 15 2017 Win64
+set VCPKG_FEATURE_FLAGS=-manifests
 
 mkdir C:\arrow-build
 pushd C:\arrow-build
@@ -83,6 +84,8 @@ cmake ^
     -DCMAKE_TOOLCHAIN_FILE=C:\vcpkg\scripts\buildsystems\vcpkg.cmake ^
     -DCMAKE_UNITY_BUILD=%CMAKE_UNITY_BUILD% ^
     -DMSVC_LINK_VERBOSE=ON ^
+    -D_VCPKG_INSTALLED_DIR=C:\vcpkg\installed ^
+    -DVCPKG_MANIFEST_MODE=OFF ^
     -DVCPKG_TARGET_TRIPLET=x64-windows-static-md-%CMAKE_BUILD_TYPE% ^
     -G "%CMAKE_GENERATOR%" ^
     C:\arrow\cpp || exit /B

diff --git a/ci/scripts/r_windows_build.sh b/ci/scripts/r_windows_build.sh
@@ -96,8 +96,8 @@ cp $MSYS_LIB_DIR/mingw64/lib/lib{thrift,snappy}.a $DST_DIR/${RWINLIB_LIB_DIR}/x6
 cp $MSYS_LIB_DIR/mingw32/lib/lib{thrift,snappy}.a $DST_DIR/${RWINLIB_LIB_DIR}/i386
 
 # These are from https://dl.bintray.com/rtools/mingw{32,64}/
-cp $MSYS_LIB_DIR/mingw64/lib/lib{zstd,lz4,crypto,aws*}.a $DST_DIR/lib/x64
-cp $MSYS_LIB_DIR/mingw32/lib/lib{zstd,lz4,crypto,aws*}.a $DST_DIR/lib/i386
+cp $MSYS_LIB_DIR/mingw64/lib/lib{zstd,lz4,crypto,utf8proc,re2,aws*}.a $DST_DIR/lib/x64
+cp $MSYS_LIB_DIR/mingw32/lib/lib{zstd,lz4,crypto,utf8proc,re2,aws*}.a $DST_DIR/lib/i386
 
 # Create build artifact
 zip -r ${DST_DIR}.zip $DST_DIR

diff --git a/cpp/CMakeSettings.json b/cpp/CMakeSettings.json
@@ -0,0 +1,21 @@
+{
+  "configurations": [
+  {
+    "name": "x64-Debug (default)",
+    "generator": "Ninja",
+    "configurationType": "Debug",
+    "inheritEnvironments": [ "msvc_x64_x64" ],
+    "buildRoot": "${projectDir}\\out\\build\\${name}",
+    "installRoot": "${projectDir}\\out\\install\\${name}",
+    "cmakeCommandArgs": "",
+    "buildCommandArgs": "",
+    "ctestCommandArgs": "",
+    "variables": [
+        {
+          "name":"VCPKG_MANIFEST_MODE",
+          "value":"OFF"
+        }
+      ]
+    }
+  ]
+}
diff --git a/cpp/examples/minimal_build/example.cc b/cpp/examples/minimal_build/example.cc
@@ -39,6 +39,7 @@ Status RunMain(int argc, char** argv) {
   ARROW_ASSIGN_OR_RAISE(
       auto csv_reader,
       arrow::csv::TableReader::Make(arrow::default_memory_pool(),
+                                    arrow::io::AsyncContext(),
                                     input_file,
                                     arrow::csv::ReadOptions::Defaults(),
                                     arrow::csv::ParseOptions::Defaults(),

diff --git a/cpp/src/arrow/CMakeLists.txt b/cpp/src/arrow/CMakeLists.txt
@@ -189,7 +189,6 @@ set(ARROW_SRCS
     util/future.cc
     util/int_util.cc
     util/io_util.cc
-    util/iterator.cc
     util/logging.cc
     util/key_value_metadata.cc
     util/memory.cc

diff --git a/cpp/src/arrow/array/array_base.h b/cpp/src/arrow/array/array_base.h
@@ -91,7 +91,7 @@ class ARROW_EXPORT Array {
   ///
   /// Note that for `null_count == 0` or for null type, this will be null.
   /// This buffer does not account for any slice offset
-  std::shared_ptr<Buffer> null_bitmap() const { return data_->buffers[0]; }
+  const std::shared_ptr<Buffer>& null_bitmap() const { return data_->buffers[0]; }
 
   /// Raw pointer to the null bitmap.
   ///
@@ -160,7 +160,7 @@ class ARROW_EXPORT Array {
   /// Input-checking variant of Array::Slice
   Result<std::shared_ptr<Array>> SliceSafe(int64_t offset) const;
 
-  std::shared_ptr<ArrayData> data() const { return data_; }
+  const std::shared_ptr<ArrayData>& data() const { return data_; }
 
   int num_fields() const { return static_cast<int>(data_->child_data.size()); }