Add authorization options for azure storage backend

build/setup_azurite.sh

@@ -0,0 +1,32 @@
+#!/bin/bash
+
+export AZURE_STORAGE_CONNECTION_STRING='DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://azurite:10000/devstoreaccount1;'
+
+function wait_for() {
+  retries=10
+  until eval $2 > /dev/null 2>&1
+  do
+    if [ "$retries" -lt "0" ]; then
+      echo "$1 is still offline after 10 retries";
+      exit 1;
+    fi
+    echo "Waiting on $1 to start..."
+    sleep 5
+    retries=$((retries - 1))
+  done
+}
+
+# We need to use azcopy natively and not via az, since the local url is not recognized autmoatically by
+# azcopy as a blob localtion and we cannot pass the '--from-to' flag down into the az commands. 
+wget -O azcopy_v10.tar.gz https://aka.ms/downloadazcopy-v10-linux && tar -xf azcopy_v10.tar.gz --strip-components=1
+cp ./azcopy /usr/bin/
+
+az config set extension.use_dynamic_install=yes_without_prompt
+
+# azcopy does not support authentication via account key, thus we generate an saas key
+SAS=$(az storage account generate-sas --permissions cdlruwap --services b --resource-types sco --expiry 2030-01-01 -o tsv)
+
+# create account and copy data
+az storage fs create -n deltars
+azcopy copy "data/golden" "http://azurite:10000/devstoreaccount1/deltars/?$SAS" --recursive=true --from-to=LocalBlob
+azcopy copy "data/simple_table" "http://azurite:10000/devstoreaccount1/deltars/?$SAS" --recursive=true --from-to=LocalBlob

docker-compose.yml

@@ -28,3 +28,28 @@ services:
       - "./rust/tests/data/simple_table:/data/simple_table"
       - "./rust/tests/data/simple_commit:/data/simple_commit"
       - "./rust/tests/data/concurrent_workers:/data/concurrent_workers"
+
+  azurite:
+    image: mcr.microsoft.com/azure-storage/azurite
+    ports:
+      - "10000:10000"
+      - "10001:10001"
+    command:
+      - "azurite"
+      - "--blobHost"
+      - "0.0.0.0"
+      - "--loose"
+
+  setup-azurite:
+    image: mcr.microsoft.com/azure-cli
+    depends_on:
+      - azurite
+    entrypoint: "/bin/bash"
+    command:
+      - /setup_azurite.sh
+    volumes:
+      - "./build/setup_azurite.sh:/setup_azurite.sh"
+      - "./rust/tests/data/golden:/data/golden"
+      - "./rust/tests/data/simple_table:/data/simple_table"
+      - "./rust/tests/data/simple_commit:/data/simple_commit"
+      - "./rust/tests/data/concurrent_workers:/data/concurrent_workers"

rust/Cargo.toml

@@ -32,10 +32,9 @@ percent-encoding = "2"
 reqwest = { version = "0.11", default-features = false, features = ["rustls-tls", "stream"], optional = true}
 
 # Azure
-azure_core = { git = "https://github.com/Azure/azure-sdk-for-rust", optional = true, rev = "2f1ca2e57801c81143b871fc5bf0df8e7250a56c" }
-azure_storage = { git = "https://github.com/Azure/azure-sdk-for-rust", optional = true, rev = "2f1ca2e57801c81143b871fc5bf0df8e7250a56c", features = ["blob", "account", "data_lake"] }
-# added to avoid cyclic dependecy caused by transient dependencies of azure crates
-indexmap = { version = "~1.6.2", optional = true}
+azure_core = { git = "https://github.com/Azure/azure-sdk-for-rust", optional = true, rev = "c6f94bcd494e74dc71977136fa62362d0b8f6fbf" }
+azure_storage = { git = "https://github.com/Azure/azure-sdk-for-rust", optional = true, rev = "c6f94bcd494e74dc71977136fa62362d0b8f6fbf", features = ["blob", "account"] }
+azure_identity = { git = "https://github.com/Azure/azure-sdk-for-rust", optional = true, rev = "c6f94bcd494e74dc71977136fa62362d0b8f6fbf" }
 
 # S3
 rusoto_core = { version = "0.46", default-features = false, optional = true }
@@ -72,7 +71,7 @@ async-trait = "0.1"
 [features]
 rust-dataframe-ext = []
 datafusion-ext = ["datafusion"]
-azure = ["azure_core", "azure_storage", "reqwest", "indexmap"]
+azure = ["azure_core", "azure_storage", "azure_identity", "reqwest"]
 s3 = ["rusoto_core/native-tls", "rusoto_credential", "rusoto_s3/native-tls", "rusoto_sts/native-tls", "rusoto_dynamodb/native-tls", "maplit"]
 s3-rustls = ["rusoto_core/rustls", "rusoto_credential", "rusoto_s3/rustls", "rusoto_sts/rustls", "rusoto_dynamodb/rustls", "maplit"]
 gcs = ["async-stream", "tame-gcs", "tame-oauth", "reqwest"]