ARROW-2034: [C++] Filesystem implementation for Azure Blob Storage

.github/workflows/cpp.yml

@@ -129,6 +129,7 @@ jobs:
     if: ${{ !contains(github.event.pull_request.title, 'WIP') }}
     timeout-minutes: 75
     env:
+      ARROW_AZURE: ON
       ARROW_BUILD_TESTS: ON
       ARROW_DATASET: ON
       ARROW_FLIGHT: ON
@@ -168,6 +169,9 @@ jobs:
             ci/scripts/install_minio.sh latest /usr/local
       - name: Install Google Cloud Storage Testbench
         run: ci/scripts/install_gcs_testbench.sh default
+      - name: Install Azurite Storage Emulator
+        shell: bash
+        run: ci/scripts/install_azurite.sh
       - name: Setup ccache
         run: |
           ci/scripts/ccache_setup.sh
@@ -305,6 +309,7 @@ jobs:
           - 32
           - 64
     env:
+      ARROW_AZURE: OFF
       ARROW_BUILD_SHARED: ON
       ARROW_BUILD_STATIC: OFF
       ARROW_BUILD_TESTS: ON

ci/appveyor-cpp-build.bat

@@ -56,6 +56,7 @@ pushd cpp\build
 @rem and enable runtime assertions.
 
 cmake -G "%GENERATOR%" %CMAKE_ARGS% ^
+      -DARROW_AZURE=OFF ^
       -DARROW_BOOST_USE_SHARED=ON ^
       -DARROW_BUILD_EXAMPLES=ON ^
       -DARROW_BUILD_STATIC=OFF ^

ci/docker/ubuntu-18.04-cpp.dockerfile

@@ -80,6 +80,7 @@ RUN apt-get update -y -q && \
         libre2-dev \
         libsnappy-dev \
         libssl-dev \
+        libxml2-dev \
         ninja-build \
         pkg-config \
         protobuf-compiler \
@@ -105,7 +106,8 @@ RUN apt-get update -y -q && \
 COPY ci/scripts/install_sccache.sh /arrow/ci/scripts/
 RUN /arrow/ci/scripts/install_sccache.sh unknown-linux-musl /usr/local/bin
 
-ENV ARROW_BUILD_STATIC=ON \
+ENV ARROW_AZURE=ON \
+    ARROW_BUILD_STATIC=ON \
     ARROW_BUILD_TESTS=ON \
     ARROW_DATASET=ON \
     ARROW_DEPENDENCY_SOURCE=SYSTEM \

ci/docker/ubuntu-20.04-cpp.dockerfile

@@ -91,10 +91,12 @@ RUN apt-get update -y -q && \
         libssl-dev \
         libthrift-dev \
         libutf8proc-dev \
+        libxml2-dev \
         libzstd-dev \
         make \
         ninja-build \
         nlohmann-json3-dev \
+        npm \
         pkg-config \
         protobuf-compiler \
         python3-dev \
@@ -114,6 +116,9 @@ RUN /arrow/ci/scripts/install_minio.sh latest /usr/local
 COPY ci/scripts/install_gcs_testbench.sh /arrow/ci/scripts/
 RUN /arrow/ci/scripts/install_gcs_testbench.sh default
 
+COPY ci/scripts/install_azurite.sh /arrow/ci/scripts/
+RUN /arrow/ci/scripts/install_azurite.sh
+
 COPY ci/scripts/install_ceph.sh /arrow/ci/scripts/
 RUN /arrow/ci/scripts/install_ceph.sh
 
@@ -131,6 +136,7 @@ RUN /arrow/ci/scripts/install_sccache.sh unknown-linux-musl /usr/local/bin
 # ARROW-17051: this build uses static Protobuf, so we must also use
 # static Arrow to run Flight/Flight SQL tests
 ENV absl_SOURCE=BUNDLED \
+    ARROW_AZURE=ON \
     ARROW_BUILD_STATIC=ON \
     ARROW_BUILD_TESTS=ON \
     ARROW_DEPENDENCY_SOURCE=SYSTEM \

ci/docker/ubuntu-22.04-cpp.dockerfile

@@ -90,10 +90,12 @@ RUN apt-get update -y -q && \
         libsqlite3-dev \
         libthrift-dev \
         libutf8proc-dev \
+        libxml2-dev \
         libzstd-dev \
         make \
         ninja-build \
         nlohmann-json3-dev \
+        npm \
         pkg-config \
         protobuf-compiler \
         protobuf-compiler-grpc \
@@ -147,6 +149,9 @@ RUN /arrow/ci/scripts/install_gcs_testbench.sh default
 COPY ci/scripts/install_sccache.sh /arrow/ci/scripts/
 RUN /arrow/ci/scripts/install_sccache.sh unknown-linux-musl /usr/local/bin
 
+COPY ci/scripts/install_azurite.sh /arrow/ci/scripts/
+RUN /arrow/ci/scripts/install_azurite.sh
+
 # Prioritize system packages and local installation
 # The following dependencies will be downloaded due to missing/invalid packages
 # provided by the distribution:
@@ -158,6 +163,7 @@ RUN /arrow/ci/scripts/install_sccache.sh unknown-linux-musl /usr/local/bin
 # ARROW-17051: this build uses static Protobuf, so we must also use
 # static Arrow to run Flight/Flight SQL tests
 ENV absl_SOURCE=BUNDLED \
+    ARROW_AZURE=ON \
     ARROW_BUILD_STATIC=ON \
     ARROW_BUILD_TESTS=ON \
     ARROW_DEPENDENCY_SOURCE=SYSTEM \

ci/scripts/cpp_build.sh

@@ -70,6 +70,7 @@ pushd ${build_dir}
 
 cmake \
   -Dabsl_SOURCE=${absl_SOURCE:-} \
+  -DARROW_AZURE=${ARROW_AZURE:-OFF} \
   -DARROW_BOOST_USE_SHARED=${ARROW_BOOST_USE_SHARED:-ON} \
   -DARROW_BUILD_BENCHMARKS_REFERENCE=${ARROW_BUILD_BENCHMARKS:-OFF} \
   -DARROW_BUILD_BENCHMARKS=${ARROW_BUILD_BENCHMARKS:-OFF} \

ci/scripts/install_azurite.sh

@@ -0,0 +1,37 @@
+#!/usr/bin/env bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+set -e
+
+case "$(uname)" in
+  Darwin)
+    npm install -g azurite
+    which azurite
+    ;;
+  MINGW*)
+    choco install nodejs.install
+    npm install -g azurite
+    ;;
+  Linux)
+    npm install -g azurite
+    which azurite
+    ;;
+esac
+echo "node version = $(node --version)"
+echo "azurite version = $(azurite --version)"

cpp/Brewfile

@@ -32,6 +32,7 @@ brew "llvm"
 brew "llvm@12"
 brew "lz4"
 brew "ninja"
+brew "node"
 brew "numpy"
 brew "[email protected]"
 brew "protobuf"

cpp/CMakeLists.txt

@@ -773,6 +773,11 @@ if(ARROW_WITH_OPENTELEMETRY)
   list(APPEND ARROW_STATIC_INSTALL_INTERFACE_LIBS CURL::libcurl)
 endif()
 
+if(ARROW_AZURE)
+  list(APPEND ARROW_SHARED_LINK_LIBS ${AZURESDK_LINK_LIBRARIES})
+  list(APPEND ARROW_STATIC_LINK_LIBS ${AZURESDK_LINK_LIBRARIES})
+endif()
+
 if(ARROW_WITH_UTF8PROC)
   list(APPEND ARROW_SHARED_LINK_LIBS utf8proc::utf8proc)
   list(APPEND ARROW_STATIC_LINK_LIBS utf8proc::utf8proc)

cpp/cmake_modules/DefineOptions.cmake

@@ -303,6 +303,9 @@ takes precedence over ccache if a storage backend is configured" ON)
   #----------------------------------------------------------------------
   set_option_category("Project component")
 
+  define_option(ARROW_AZURE
+                "Build Arrow with Azure support (requires the Azure SDK for C++)" OFF)
+
   define_option(ARROW_BUILD_UTILITIES "Build Arrow commandline utilities" OFF)
 
   define_option(ARROW_COMPUTE "Build the Arrow Compute Modules" OFF)