ARROW-2034: [C++] Filesystem implementation for Azure Blob Storage

cpp/CMakeLists.txt

@@ -693,7 +693,9 @@ endif()
 
 # Libraries to link statically with libarrow.so
 set(ARROW_LINK_LIBS)
+set(ARROW_AZURE_LINK_LIBS)
 set(ARROW_STATIC_LINK_LIBS)
+set(ARROW_AZURE_STATIC_LINK_LIBS)
 set(ARROW_STATIC_INSTALL_INTERFACE_LIBS)
 
 if(ARROW_USE_OPENSSL)
@@ -792,6 +794,11 @@ if(ARROW_WITH_OPENTELEMETRY)
        opentelemetry-cpp::otlp_http_exporter)
 endif()
 
+if (ARROW_AZURE)
+  list(APPEND ARROW_AZURE_LINK_LIBS ${AZURESDK_LINK_LIBRARIES})
+  list(APPEND ARROW_AZURE_STATIC_LINK_LIBS ${AZURESDK_LINK_LIBRARIES})
+endif()
+
 if(ARROW_WITH_UTF8PROC)
   list(APPEND ARROW_LINK_LIBS utf8proc::utf8proc)
   list(APPEND ARROW_STATIC_LINK_LIBS utf8proc::utf8proc)
@@ -829,6 +836,7 @@ if(ARROW_STATIC_LINK_LIBS)
 endif()
 
 set(ARROW_SHARED_PRIVATE_LINK_LIBS ${ARROW_STATIC_LINK_LIBS})
+set(ARROW_AZURE_SHARED_PRIVATE_LINK_LIBS ${ARROW_AZURE_STATIC_LINK_LIBS})
 
 # boost::filesystem is needed for S3 and Flight tests as a boost::process dependency.
 if(((ARROW_FLIGHT

cpp/cmake_modules/BuildUtils.cmake

@@ -46,6 +46,260 @@ if(WIN32 AND CMAKE_CXX_COMPILER_ID STREQUAL "GNU")
   list(APPEND ARROW_BOOST_PROCESS_COMPILE_DEFINITIONS "BOOST_USE_WINDOWS_H=1")
 endif()
 
+function(ADD_ARROW_LIB_AZURE LIB_NAME)
+  set(options)
+  set(one_value_args 
+      BUILD_SHARED
+      BUILD_STATIC)
+  set(multi_value_args
+      SOURCES
+      STATIC_LINK_LIBS
+      SHARED_LINK_LIBS
+      DEPENDENCIES
+      SHARED_PRIVATE_LINK_LIBS
+      OUTPUT_PATH)
+  cmake_parse_arguments(ARG
+                        "${options}"
+                        "${one_value_args}"
+                        "${multi_value_args}"
+                        ${ARGN})
+  if(ARG_UNPARSED_ARGUMENTS)
+    message(SEND_ERROR "Error: unrecognized arguments: ${ARG_UNPARSED_ARGUMENTS}")
+  endif()
+
+  if(ARG_SOURCES)
+    set(SOURCES ${ARG_SOURCES})
+  else()
+    set(SOURCES "${LIB_NAME}.cc")
+  endif()
+
+  # Allow overriding ARROW_BUILD_SHARED and ARROW_BUILD_STATIC
+  if(DEFINED ARG_BUILD_SHARED)
+    set(BUILD_SHARED ${ARG_BUILD_SHARED})
+  else()
+    set(BUILD_SHARED ${ARROW_BUILD_SHARED})
+  endif()
+  if(DEFINED ARG_BUILD_STATIC)
+    set(BUILD_STATIC ${ARG_BUILD_STATIC})
+  else()
+    set(BUILD_STATIC ${ARROW_BUILD_STATIC})
+  endif()
+  if(ARG_OUTPUT_PATH)
+    set(OUTPUT_PATH ${ARG_OUTPUT_PATH})
+  else()
+    set(OUTPUT_PATH ${BUILD_OUTPUT_ROOT_DIRECTORY})
+  endif()
+
+  if(WIN32 OR (CMAKE_GENERATOR STREQUAL Xcode))
+    # We need to compile C++ separately for each library kind (shared and static)
+    # because of dllexport declarations on Windows.
+    # The Xcode generator doesn't reliably work with Xcode as target names are not
+    # guessed correctly.
+    set(USE_OBJLIB OFF)
+  else()
+    set(USE_OBJLIB ON)
+  endif()
+
+  if(USE_OBJLIB)
+    # Generate a single "objlib" from all C++ modules and link
+    # that "objlib" into each library kind, to avoid compiling twice
+    add_library(${LIB_NAME}_objlib OBJECT ${SOURCES})
+    # Necessary to make static linking into other shared libraries work properly
+    set_property(TARGET ${LIB_NAME}_objlib PROPERTY POSITION_INDEPENDENT_CODE 1)
+    set(LIB_DEPS $<TARGET_OBJECTS:${LIB_NAME}_objlib>)
+  else()
+    set(LIB_DEPS ${ARG_SOURCES})
+  endif()
+
+  set(RUNTIME_INSTALL_DIR bin)
+
+  if(BUILD_SHARED)
+    add_library(${LIB_NAME}_shared SHARED ${LIB_DEPS})
+
+    set_target_properties(${LIB_NAME}_shared
+                          PROPERTIES LIBRARY_OUTPUT_DIRECTORY "${OUTPUT_PATH}"
+                                      RUNTIME_OUTPUT_DIRECTORY "${OUTPUT_PATH}"
+                                      PDB_OUTPUT_DIRECTORY "${OUTPUT_PATH}"
+                                      OUTPUT_NAME ${LIB_NAME}
+                                      VERSION "${ARROW_FULL_SO_VERSION}"
+                                      SOVERSION "${ARROW_SO_VERSION}")
+
+    target_link_libraries(${LIB_NAME}_shared LINK_PRIVATE ${ARG_SHARED_PRIVATE_LINK_LIBS})
+
+    install(TARGETS ${LIB_NAME}_shared
+            EXPORT ${LIB_NAME}_targets
+            RUNTIME DESTINATION ${RUNTIME_INSTALL_DIR}
+            LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR}
+            ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR}
+            INCLUDES
+            DESTINATION ${CMAKE_INSTALL_INCLUDEDIR})
+  endif()
+
+  if(BUILD_STATIC)
+    add_library(${LIB_NAME}_static SHARED ${LIB_DEPS})
+
+    if(MSVC_TOOLCHAIN)
+      set(LIB_NAME_STATIC ${LIB_NAME}_static)
+    else()
+      set(LIB_NAME_STATIC ${LIB_NAME})
+    endif()
+
+    if(ARROW_BUILD_STATIC AND WIN32)
+      target_compile_definitions(${LIB_NAME}_static PUBLIC ARROW_STATIC)
+    endif()
+
+    set_target_properties(${LIB_NAME}_static
+                          PROPERTIES LIBRARY_OUTPUT_DIRECTORY "${OUTPUT_PATH}"
+                                    OUTPUT_NAME ${LIB_NAME_STATIC})
+
+    install(TARGETS ${LIB_NAME}_static
+            EXPORT ${LIB_NAME}_targets
+            RUNTIME DESTINATION ${RUNTIME_INSTALL_DIR}
+            LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR}
+            ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR}
+            INCLUDES
+            DESTINATION ${CMAKE_INSTALL_INCLUDEDIR})
+    endif()
+endfunction()
+
+function(ADD_TEST_CASE_AZURE REL_TEST_NAME)
+  set(options NO_VALGRIND ENABLED)
+  set(one_value_args PRECOMPILED_HEADER_LIB)
+  set(multi_value_args
+      SOURCES
+      PRECOMPILED_HEADERS
+      STATIC_LINK_LIBS
+      EXTRA_LINK_LIBS
+      EXTRA_INCLUDES
+      EXTRA_DEPENDENCIES
+      LABELS
+      EXTRA_LABELS
+      TEST_ARGUMENTS
+      PREFIX)
+  cmake_parse_arguments(ARG
+                        "${options}"
+                        "${one_value_args}"
+                        "${multi_value_args}"
+                        ${ARGN})
+  if(ARG_UNPARSED_ARGUMENTS)
+    message(SEND_ERROR "Error: unrecognized arguments: ${ARG_UNPARSED_ARGUMENTS}")
+  endif()
+
+  if(NO_TESTS AND NOT ARG_ENABLED)
+    return()
+  endif()
+  get_filename_component(TEST_NAME ${REL_TEST_NAME} NAME_WE)
+
+  if(ARG_PREFIX)
+    set(TEST_NAME "${ARG_PREFIX}-${TEST_NAME}")
+  endif()
+
+  if(ARG_SOURCES)
+    set(SOURCES ${ARG_SOURCES})
+  else()
+    set(SOURCES "${REL_TEST_NAME}.cc")
+  endif()
+
+  # Make sure the executable name contains only hyphens, not underscores
+  string(REPLACE "_" "-" TEST_NAME ${TEST_NAME})
+
+  set(TEST_PATH "${EXECUTABLE_OUTPUT_PATH}/${TEST_NAME}")
+  add_executable(${TEST_NAME} ${SOURCES})
+
+  # target_link_libraries(${TEST_NAME} PRIVATE azurefs_shared)
+  # With OSX and conda, we need to set the correct RPATH so that dependencies
+  # are found. The installed libraries with conda have an RPATH that matches
+  # for executables and libraries lying in $ENV{CONDA_PREFIX}/bin or
+  # $ENV{CONDA_PREFIX}/lib but our test libraries and executables are not
+  # installed there.
+  if(NOT "$ENV{CONDA_PREFIX}" STREQUAL "" AND APPLE)
+    set_target_properties(${TEST_NAME}
+                          PROPERTIES BUILD_WITH_INSTALL_RPATH TRUE
+                                     INSTALL_RPATH_USE_LINK_PATH TRUE
+                                     INSTALL_RPATH
+                                     "${EXECUTABLE_OUTPUT_PATH};$ENV{CONDA_PREFIX}/lib")
+  endif()
+
+  if(ARG_STATIC_LINK_LIBS)
+    # Customize link libraries
+    target_link_libraries(${TEST_NAME} PRIVATE ${ARG_STATIC_LINK_LIBS})
+  else()
+    target_link_libraries(${TEST_NAME} PRIVATE ${ARROW_TEST_LINK_LIBS})
+  endif()
+
+  if(ARG_PRECOMPILED_HEADER_LIB)
+    reuse_precompiled_header_lib(${TEST_NAME} ${ARG_PRECOMPILED_HEADER_LIB})
+  endif()
+
+  if(ARG_PRECOMPILED_HEADERS AND ARROW_USE_PRECOMPILED_HEADERS)
+    target_precompile_headers(${TEST_NAME} PRIVATE ${ARG_PRECOMPILED_HEADERS})
+  endif()
+
+  if(ARG_EXTRA_LINK_LIBS)
+    target_link_libraries(${TEST_NAME} PRIVATE ${ARG_EXTRA_LINK_LIBS})
+  endif()
+
+  if(ARG_EXTRA_INCLUDES)
+    target_include_directories(${TEST_NAME} SYSTEM PUBLIC ${ARG_EXTRA_INCLUDES})
+  endif()
+
+  if(ARG_EXTRA_DEPENDENCIES)
+    add_dependencies(${TEST_NAME} ${ARG_EXTRA_DEPENDENCIES})
+  endif()
+
+  if(ARROW_TEST_MEMCHECK AND NOT ARG_NO_VALGRIND)
+    add_test(${TEST_NAME}
+             bash
+             -c
+             "cd '${CMAKE_SOURCE_DIR}'; \
+               valgrind --suppressions=valgrind.supp --tool=memcheck --gen-suppressions=all \
+                 --num-callers=500 --leak-check=full --leak-check-heuristics=stdstring \
+                 --error-exitcode=1 ${TEST_PATH} ${ARG_TEST_ARGUMENTS}")
+  elseif(WIN32)
+    add_test(${TEST_NAME} ${TEST_PATH} ${ARG_TEST_ARGUMENTS})
+  else()
+    add_test(${TEST_NAME}
+             ${BUILD_SUPPORT_DIR}/run-test.sh
+             ${CMAKE_BINARY_DIR}
+             test
+             ${TEST_PATH}
+             ${ARG_TEST_ARGUMENTS})
+  endif()
+
+  # Add test as dependency of relevant targets
+  add_dependencies(all-tests ${TEST_NAME})
+  foreach(TARGET ${ARG_LABELS})
+    add_dependencies(${TARGET} ${TEST_NAME})
+  endforeach()
+
+  set(LABELS)
+  list(APPEND LABELS "unittest")
+  if(ARG_LABELS)
+    list(APPEND LABELS ${ARG_LABELS})
+  endif()
+  # EXTRA_LABELS don't create their own dependencies, they are only used
+  # to ease running certain test categories.
+  if(ARG_EXTRA_LABELS)
+    list(APPEND LABELS ${ARG_EXTRA_LABELS})
+  endif()
+
+  foreach(LABEL ${ARG_LABELS})
+    # ensure there is a cmake target which exercises tests with this LABEL
+    set(LABEL_TEST_NAME "test-${LABEL}")
+    if(NOT TARGET ${LABEL_TEST_NAME})
+      add_custom_target(${LABEL_TEST_NAME}
+                        ctest -L "${LABEL}" --output-on-failure
+                        USES_TERMINAL)
+    endif()
+    # ensure the test is (re)built before the LABEL test runs
+    add_dependencies(${LABEL_TEST_NAME} ${TEST_NAME})
+  endforeach()
+
+  set_property(TEST ${TEST_NAME}
+               APPEND
+               PROPERTY LABELS ${LABELS})
+endfunction()
+
 function(ADD_THIRDPARTY_LIB LIB_NAME)
   set(options)
   set(one_value_args SHARED_LIB STATIC_LIB)

cpp/cmake_modules/DefineOptions.cmake

@@ -215,6 +215,8 @@ if("${CMAKE_SOURCE_DIR}" STREQUAL "${CMAKE_CURRENT_SOURCE_DIR}")
   #----------------------------------------------------------------------
   set_option_category("Project component")
 
+  define_option(ARROW_AZURE "Build Arrow with Azure support (requires the Azure SDK for C++)" OFF)
+
   define_option(ARROW_BUILD_UTILITIES "Build Arrow commandline utilities" OFF)
 
   define_option(ARROW_COMPUTE "Build the Arrow Compute Modules" OFF)

cpp/cmake_modules/ThirdpartyToolchain.cmake

@@ -4513,6 +4513,53 @@ if(ARROW_S3)
   endif()
 endif()
 
+macro(build_azuresdk)
+  message(STATUS "Building Azure C++ SDK from source")
+
+  set(AZURESDK_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/azuresdk_ep-install")
+  set(AZURESDK_INCLUDE_DIR "${AZURESDK_PREFIX}/include")
+
+  set(AZURESDK_CMAKE_ARGS
+      ${EP_COMMON_CMAKE_ARGS}
+      -DBUILD_TESTING=OFF
+      -DCMAKE_INSTALL_LIBDIR=lib
+      "-DCMAKE_INSTALL_PREFIX=${AZURESDK_PREFIX}"
+      -DCMAKE_PREFIX_PATH=${AZURESDK_PREFIX})
+
+  file(MAKE_DIRECTORY ${AZURESDK_INCLUDE_DIR})
+
+  # Azure C++ SDK related libraries to link statically
+  set(_AZURESDK_LIBS
+      azure-core
+      azure-identity
+      azure-storage-blobs
+      azure-storage-common
+      azure-storage-files-datalake)
+  set(AZURESDK_LIBRARIES)
+  set(AZURESDK_LIBRARIES_CPP)
+  foreach(_AZURESDK_LIB ${_AZURESDK_LIBS})
+    string(TOUPPER ${_AZURESDK_LIB} _AZURESDK_LIB_UPPER)
+    string(REPLACE "-" "_" _AZURESDK_LIB_NAME_PREFIX ${_AZURESDK_LIB_UPPER})
+    list(APPEND AZURESDK_LIBRARIES_CPP "${_AZURESDK_LIB}-cpp")
+    set(_AZURESDK_TARGET_NAME Azure::${_AZURESDK_LIB})
+    list(APPEND AZURESDK_LIBRARIES ${_AZURESDK_TARGET_NAME})
+  endforeach()
+
+  set(AZURESDK_LINK_LIBRARIES ${AZURESDK_LIBRARIES})
+endmacro()
+
+if(ARROW_AZURE)
+  build_azuresdk()
+
+  foreach(AZURESDK_LIBRARY_CPP ${AZURESDK_LIBRARIES_CPP})
+    find_package(${AZURESDK_LIBRARY_CPP} CONFIG REQUIRED)
+  endforeach()
+
+  include_directories(SYSTEM ${AZURESDK_INCLUDE_DIR})
+  message(STATUS "Found AZURE SDK headers: ${AZURESDK_INCLUDE_DIR}")
+  message(STATUS "Found AZURE SDK libraries: ${AZURESDK_LINK_LIBRARIES}")
+endif()
+
 message(STATUS "All bundled static libraries: ${ARROW_BUNDLED_STATIC_LIBS}")
 
 # Write out the package configurations.