GH-36886: [C++] Configure azurite in preparation for testing Azure …

…C++ filesystem (#36988)

### Rationale for this change
We need to write tests for #18014. azurite is like a fake Azure blob storage so it can be used to write integration tests

### What changes are included in this PR?
Extract the `azurite` related changes from #12914 to create a smaller PR that's easier to review. I have made very minimal changes compared to that PR. 

Currently `azurite` is configured for all the environments where `ARROW_AZURE` was enabled by #35701. I assume its deliberate that its not enabled yet for windows, alpine, conda, debian or fedora builds. 

### Are these changes tested?
Its tested by there aren't really any good tests in this PR. I used this `azurite` config in #36835 to make an integration test that uses the Azure C++ SDK. On its own we can't really write tests for this `azurite` setup PR. 

### Are there any user-facing changes?
No

* Closes: #36886

Lead-authored-by: Thomas Newton <[email protected]>
Co-authored-by: Sutou Kouhei <[email protected]>
Signed-off-by: Sutou Kouhei <[email protected]>

.github/workflows/cpp.yml

@@ -23,6 +23,7 @@ on:
       - '.github/workflows/cpp.yml'
       - 'ci/docker/**'
       - 'ci/scripts/cpp_*'
+      - 'ci/scripts/install_azurite.sh'
       - 'ci/scripts/install_gcs_testbench.sh'
       - 'ci/scripts/install_minio.sh'
       - 'ci/scripts/msys2_*'
@@ -34,6 +35,7 @@ on:
       - '.github/workflows/cpp.yml'
       - 'ci/docker/**'
       - 'ci/scripts/cpp_*'
+      - 'ci/scripts/install_azurite.sh'
       - 'ci/scripts/install_gcs_testbench.sh'
       - 'ci/scripts/install_minio.sh'
       - 'ci/scripts/msys2_*'
@@ -201,6 +203,8 @@ jobs:
             ci/scripts/install_minio.sh latest /usr/local
       - name: Install Google Cloud Storage Testbench
         run: ci/scripts/install_gcs_testbench.sh default
+      - name: Install Azurite Storage Emulator
+        run: ci/scripts/install_azurite.sh
       - name: Setup ccache
         run: |
           ci/scripts/ccache_setup.sh

ci/docker/ubuntu-20.04-cpp.dockerfile

@@ -103,6 +103,7 @@ RUN apt-get update -y -q && \
         make \
         ninja-build \
         nlohmann-json3-dev \
+        npm \
         pkg-config \
         protobuf-compiler \
         python3-dev \
@@ -123,6 +124,9 @@ RUN /arrow/ci/scripts/install_minio.sh latest /usr/local
 COPY ci/scripts/install_gcs_testbench.sh /arrow/ci/scripts/
 RUN /arrow/ci/scripts/install_gcs_testbench.sh default
 
+COPY ci/scripts/install_azurite.sh /arrow/ci/scripts/
+RUN /arrow/ci/scripts/install_azurite.sh
+
 COPY ci/scripts/install_ceph.sh /arrow/ci/scripts/
 RUN /arrow/ci/scripts/install_ceph.sh
 

ci/docker/ubuntu-22.04-cpp.dockerfile

@@ -102,6 +102,7 @@ RUN apt-get update -y -q && \
         make \
         ninja-build \
         nlohmann-json3-dev \
+        npm \
         pkg-config \
         protobuf-compiler \
         protobuf-compiler-grpc \
@@ -153,6 +154,9 @@ RUN /arrow/ci/scripts/install_minio.sh latest /usr/local
 COPY ci/scripts/install_gcs_testbench.sh /arrow/ci/scripts/
 RUN /arrow/ci/scripts/install_gcs_testbench.sh default
 
+COPY ci/scripts/install_azurite.sh /arrow/ci/scripts/
+RUN /arrow/ci/scripts/install_azurite.sh
+
 COPY ci/scripts/install_sccache.sh /arrow/ci/scripts/
 RUN /arrow/ci/scripts/install_sccache.sh unknown-linux-musl /usr/local/bin
 

ci/scripts/install_azurite.sh

@@ -0,0 +1,37 @@
+#!/usr/bin/env bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+set -e
+
+case "$(uname)" in
+  Darwin)
+    npm install -g azurite
+    which azurite
+    ;;
+  MINGW*)
+    choco install nodejs.install
+    npm install -g azurite
+    ;;
+  Linux)
+    npm install -g azurite
+    which azurite
+    ;;
+esac
+echo "node version = $(node --version)"
+echo "azurite version = $(azurite --version)"

cpp/Brewfile

@@ -30,6 +30,7 @@ brew "grpc"
 brew "llvm@14"
 brew "lz4"
 brew "ninja"
+brew "node"
 brew "openssl@3"
 brew "protobuf"
 brew "python"

cpp/src/arrow/filesystem/azurefs_test.cc

@@ -15,7 +15,26 @@
 // specific language governing permissions and limitations
 // under the License.
 
+#include <algorithm>  // Missing include in boost/process
+
+// This boost/asio/io_context.hpp include is needless for no MinGW
+// build.
+//
+// This is for including boost/asio/detail/socket_types.hpp before any
+// "#include <windows.h>". boost/asio/detail/socket_types.hpp doesn't
+// work if windows.h is already included. boost/process.h ->
+// boost/process/args.hpp -> boost/process/detail/basic_cmd.hpp
+// includes windows.h. boost/process/args.hpp is included before
+// boost/process/async.h that includes
+// boost/asio/detail/socket_types.hpp implicitly is included.
+#include <boost/asio/io_context.hpp>
+// We need BOOST_USE_WINDOWS_H definition with MinGW when we use
+// boost/process.hpp. See BOOST_USE_WINDOWS_H=1 in
+// cpp/cmake_modules/ThirdpartyToolchain.cmake for details.
+#include <boost/process.hpp>
+
 #include "arrow/filesystem/azurefs.h"
+#include "arrow/util/io_util.h"
 
 #include <gmock/gmock-matchers.h>
 #include <gmock/gmock-more-matchers.h>
@@ -27,15 +46,76 @@
 #include "arrow/testing/util.h"
 
 namespace arrow {
+using internal::TemporaryDir;
 namespace fs {
 namespace {
+namespace bp = boost::process;
 
 using ::testing::IsEmpty;
 using ::testing::Not;
 using ::testing::NotNull;
 
-// Placeholder test for file structure
+class AzuriteEnv : public ::testing::Environment {
+ public:
+  AzuriteEnv() {
+    account_name_ = "devstoreaccount1";
+    account_key_ =
+        "Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/"
+        "KBHBeksoGMGw==";
+    auto exe_path = bp::search_path("azurite");
+    if (exe_path.empty()) {
+      auto error = std::string("Could not find Azurite emulator.");
+      status_ = Status::Invalid(error);
+      return;
+    }
+    auto temp_dir_ = *TemporaryDir::Make("azurefs-test-");
+    server_process_ = bp::child(boost::this_process::environment(), exe_path, "--silent",
+                                "--location", temp_dir_->path().ToString(), "--debug",
+                                temp_dir_->path().ToString() + "/debug.log");
+    if (!(server_process_.valid() && server_process_.running())) {
+      auto error = "Could not start Azurite emulator.";
+      server_process_.terminate();
+      server_process_.wait();
+      status_ = Status::Invalid(error);
+      return;
+    }
+    status_ = Status::OK();
+  }
+
+  ~AzuriteEnv() override {
+    server_process_.terminate();
+    server_process_.wait();
+  }
+
+  const std::string& account_name() const { return account_name_; }
+  const std::string& account_key() const { return account_key_; }
+  const Status status() const { return status_; }
+
+ private:
+  std::string account_name_;
+  std::string account_key_;
+  bp::child server_process_;
+  Status status_;
+  std::unique_ptr<TemporaryDir> temp_dir_;
+};
+
+auto* azurite_env = ::testing::AddGlobalTestEnvironment(new AzuriteEnv);
+
+AzuriteEnv* GetAzuriteEnv() {
+  return ::arrow::internal::checked_cast<AzuriteEnv*>(azurite_env);
+}
+
+// Placeholder tests for file structure
 // TODO: GH-18014 Remove once a proper test is added
+TEST(AzureFileSystem, InitialiseAzurite) {
+  const std::string& account_name = GetAzuriteEnv()->account_name();
+  const std::string& account_key = GetAzuriteEnv()->account_key();
+  EXPECT_EQ(account_name, "devstoreaccount1");
+  EXPECT_EQ(account_key,
+            "Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/"
+            "K1SZFPTOtr/KBHBeksoGMGw==");
+}
+
 TEST(AzureFileSystem, OptionsCompare) {
   AzureOptions options;
   EXPECT_TRUE(options.Equals(options));