fix: backport AZ Blob Provisioner

.github/workflows/verify.yaml

@@ -167,3 +167,8 @@ jobs:
         run: |
           ./gradlew compileJava compileTestJava
           ./gradlew -p edc-tests/edc-dataplane test -DincludeTags="CloudTransferTest"
+
+      - name: Run Azure/S3 End-To-End tests
+        run: |
+          ./gradlew compileJava compileTestJava
+          ./gradlew -p edc-tests/edc-end2end test -DincludeTags="EndToEndTest"

DEPENDENCIES

@@ -346,6 +346,7 @@ maven/mavencentral/org.eclipse.edc.aws/aws-s3-core/0.7.2, Apache-2.0, approved,
 maven/mavencentral/org.eclipse.edc.aws/aws-spi/0.7.2, Apache-2.0, approved, technology.edc
 maven/mavencentral/org.eclipse.edc.aws/data-plane-aws-s3/0.7.2, Apache-2.0, approved, technology.edc
 maven/mavencentral/org.eclipse.edc.azure/azure-blob-core/0.7.2, Apache-2.0, approved, technology.edc
+maven/mavencentral/org.eclipse.edc.azure/azure-test/0.7.2, Apache-2.0, approved, technology.edc
 maven/mavencentral/org.eclipse.edc.azure/data-plane-azure-storage/0.7.2, Apache-2.0, approved, technology.edc
 maven/mavencentral/org.eclipse.edc.azure/vault-azure/0.7.2, Apache-2.0, approved, technology.edc
 maven/mavencentral/org.eclipse.edc/accesstoken-lib/0.7.2, Apache-2.0, approved, technology.edc
@@ -615,6 +616,7 @@ maven/mavencentral/org.ow2.asm/asm/9.1, BSD-3-Clause, approved, CQ23029
 maven/mavencentral/org.ow2.asm/asm/9.5, BSD-3-Clause, approved, #7554
 maven/mavencentral/org.ow2.asm/asm/9.6, BSD-3-Clause, approved, #10776
 maven/mavencentral/org.ow2.asm/asm/9.7, BSD-3-Clause, approved, #14076
+maven/mavencentral/org.postgresql/postgresql/42.7.2, BSD-2-Clause AND Apache-2.0, approved, #11681
 maven/mavencentral/org.postgresql/postgresql/42.7.3, BSD-2-Clause AND Apache-2.0, approved, #11681
 maven/mavencentral/org.reactivestreams/reactive-streams/1.0.4, CC0-1.0, approved, CQ16332
 maven/mavencentral/org.reflections/reflections/0.10.2, Apache-2.0 AND WTFPL, approved, clearlydefined

edc-controlplane/edc-controlplane-base/build.gradle.kts

@@ -39,7 +39,7 @@ dependencies {
     // Credentials CX policies
     runtimeOnly(project(":edc-extensions:cx-policy"))
 
-    // needed for IATP integration
+    // needed for DCP integration
     runtimeOnly(project(":core:json-ld-core"))
     runtimeOnly(libs.edc.core.did)
     runtimeOnly(libs.edc.identity.did.web)
@@ -75,4 +75,7 @@ dependencies {
     runtimeOnly(libs.edc.controlplane.callback.dispatcher.event)
     runtimeOnly(libs.edc.controlplane.callback.dispatcher.http)
 
+    // cloud provisioner extensions
+    runtimeOnly(project(":edc-extensions:backport:azblob-provisioner"))
+
 }

edc-extensions/backport/README.md

@@ -0,0 +1,2 @@
+All modules in this directory are only temporary and they should be replaced with their upstream counterparts at the
+earliest opportunity. changes made here should be upstreamed ASAP, lest we run the risk of feature divergence!

edc-extensions/backport/azblob-provisioner/README.md

@@ -0,0 +1,77 @@
+# Backport of the Azure Blob Storage Provisioner
+
+This module is a backport, that means the contents of
+the [upstream module](https://github.com/eclipse-edc/Technology-Azure/tree/main/extensions/control-plane/provision/provision-blob)
+are copied here.
+
+## Defining an `Asset` located in AzBlob
+
+Create an asset similar to this on the provider connector's Management API:
+
+```json
+{
+  "@context": {
+    "@vocab": "https://w3id.org/edc/v0.0.1/ns/"
+  },
+  "@id": "blob-test-asset",
+  "properties": {},
+  "dataAddress": {
+    "keyName": "provider-key-alias",
+    "type": "AzureStorage",
+    "@type": "DataAddress",
+    "name": "transfer-test",
+    "container": "provider-container",
+    "account": "provider",
+    "blobPrefix": "folder/"
+  }
+}
+```
+
+Explanation:
+
+- `keyName`: alias under which the Storage Account Key is located in the provider's vault, stored as plain `String`
+- `type`: **must** be `AzureStorage`
+- `container`: the name of the source AzBlob container on the provider side
+- `account`: the name of the AzBlob account on the provider side
+- `blobPrefix`: in case all contents of a "folder" are to be copied, this is the "folder name"
+
+## Creating an AzBlob-to-AzBlob transfer request
+
+Assuming the contract negotiation has succeeded, execute a transfer process request on the consumer's Management API
+endpoint with the following content:
+
+```json
+{
+  "@context": {
+    "@vocab": "https://w3id.org/edc/v0.0.1/ns/"
+  },
+  "@type": "TransferRequest",
+  "protocol": "dataspace-protocol-http",
+  "contractId": "416aed9c-7258-45c8-bdee-c09d5da7c255",
+  "connectorId": "PROVIDER-BPN",
+  "counterPartyAddress": "http://localhost:40950/protocol",
+  "dataDestination": {
+    "@type": "https://w3id.org/edc/v0.0.1/ns/DataAddress",
+    "https://w3id.org/edc/v0.0.1/ns/type": "AzureStorage",
+    "https://w3id.org/edc/v0.0.1/ns/properties": {
+      "https://w3id.org/edc/v0.0.1/ns/type": "AzureStorage",
+      "https://w3id.org/edc/v0.0.1/ns/account": "consumer",
+      "https://w3id.org/edc/v0.0.1/ns/container": "consumer-container"
+    }
+  },
+  "transferType": "AzureStorage-PUSH"
+}
+```
+
+Explanation:
+- `type`: **must** be `AzureStorage`
+- `account`: the consumer account name
+- `container`: the destination container in the consumer's Azure Blob account
+- `transferType`: **must** be `AzureStorage-PUSH` for the provider to push the data into the consumer's AzBlob container
+
+> Note that the Storage Account Key on the consumer side is expected in the Vault under the alias `<ACCOUNTNAME>-key1`,
+here that would be `consumer-key1`. The key **must** be the raw Account Key (no SAS token), stored as plain String.
+
+The AzBlob provisioner on the consumer side will generate a temporary SAS token for the consumer container (
+`"consumer-container"`) and send it to the provider in a DSP `TransferRequestMessage`. The provider will then store it
+in its own Vault, where it gets resolved from the provider Data Plane.

edc-extensions/backport/azblob-provisioner/build.gradle.kts

@@ -0,0 +1,27 @@
+/*
+ *  Copyright (c) 2020, 2021 Microsoft Corporation
+ *
+ *  This program and the accompanying materials are made available under the
+ *  terms of the Apache License, Version 2.0 which is available at
+ *  https://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  SPDX-License-Identifier: Apache-2.0
+ *
+ *  Contributors:
+ *       Microsoft Corporation - initial API and implementation
+ *
+ */
+
+plugins {
+    `java-library`
+}
+
+dependencies {
+    api(libs.edc.spi.core)
+    api(libs.edc.azure.blob.core)
+
+    implementation(libs.azure.storage.blob)
+    testImplementation(testFixtures(libs.edc.azure.test))
+}
+
+

edc-extensions/backport/azblob-provisioner/src/main/java/org/eclipse/edc/connector/provision/azure/AzureProvisionExtension.java

@@ -0,0 +1,77 @@
+/********************************************************************************
+ * Copyright (c) 2020,2021 Microsoft Corporation
+ *
+ * See the NOTICE file(s) distributed with this work for additional
+ * information regarding copyright ownership.
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0.
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ ********************************************************************************/
+
+package org.eclipse.edc.connector.provision.azure;
+
+import dev.failsafe.RetryPolicy;
+import org.eclipse.edc.azure.blob.AzureSasToken;
+import org.eclipse.edc.azure.blob.api.BlobStoreApi;
+import org.eclipse.edc.connector.controlplane.transfer.spi.provision.ProvisionManager;
+import org.eclipse.edc.connector.controlplane.transfer.spi.provision.Provisioner;
+import org.eclipse.edc.connector.controlplane.transfer.spi.provision.ResourceManifestGenerator;
+import org.eclipse.edc.connector.provision.azure.blob.ObjectContainerProvisionedResource;
+import org.eclipse.edc.connector.provision.azure.blob.ObjectStorageConsumerResourceDefinitionGenerator;
+import org.eclipse.edc.connector.provision.azure.blob.ObjectStorageProvisioner;
+import org.eclipse.edc.connector.provision.azure.blob.ObjectStorageResourceDefinition;
+import org.eclipse.edc.runtime.metamodel.annotation.Inject;
+import org.eclipse.edc.spi.system.ServiceExtension;
+import org.eclipse.edc.spi.system.ServiceExtensionContext;
+import org.eclipse.edc.spi.types.TypeManager;
+
+/**
+ * Provides data transfer {@link Provisioner}s backed by Azure services.
+ */
+public class AzureProvisionExtension implements ServiceExtension {
+
+    @Inject
+    private BlobStoreApi blobStoreApi;
+
+    @Inject
+    private RetryPolicy<Object> retryPolicy;
+
+    @Inject
+    private ResourceManifestGenerator manifestGenerator;
+
+    @Inject
+    private TypeManager typeManager;
+
+    @Override
+    public String name() {
+        return "Azure Provision";
+    }
+
+    @Override
+    public void initialize(ServiceExtensionContext context) {
+
+        var monitor = context.getMonitor();
+        var provisionManager = context.getService(ProvisionManager.class);
+
+        provisionManager.register(new ObjectStorageProvisioner(retryPolicy, monitor, blobStoreApi));
+
+        // register the generator
+        manifestGenerator.registerGenerator(new ObjectStorageConsumerResourceDefinitionGenerator());
+
+        registerTypes(typeManager);
+    }
+
+    private void registerTypes(TypeManager typeManager) {
+        typeManager.registerTypes(ObjectContainerProvisionedResource.class, ObjectStorageResourceDefinition.class, AzureSasToken.class);
+    }
+
+}

edc-extensions/backport/azblob-provisioner/src/main/java/org/eclipse/edc/connector/provision/azure/blob/ObjectContainerProvisionedResource.java

@@ -0,0 +1,86 @@
+/********************************************************************************
+ * Copyright (c) 2020,2021 Microsoft Corporation
+ *
+ * See the NOTICE file(s) distributed with this work for additional
+ * information regarding copyright ownership.
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0.
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ ********************************************************************************/
+
+package org.eclipse.edc.connector.provision.azure.blob;
+
+import com.fasterxml.jackson.annotation.JsonCreator;
+import com.fasterxml.jackson.annotation.JsonTypeName;
+import com.fasterxml.jackson.databind.annotation.JsonDeserialize;
+import com.fasterxml.jackson.databind.annotation.JsonPOJOBuilder;
+import org.eclipse.edc.azure.blob.AzureBlobStoreSchema;
+import org.eclipse.edc.connector.controlplane.transfer.spi.types.ProvisionedDataDestinationResource;
+
+import static org.eclipse.edc.azure.blob.AzureBlobStoreSchema.ACCOUNT_NAME;
+import static org.eclipse.edc.azure.blob.AzureBlobStoreSchema.CONTAINER_NAME;
+import static org.eclipse.edc.azure.blob.AzureBlobStoreSchema.FOLDER_NAME;
+import static org.eclipse.edc.spi.constants.CoreConstants.EDC_NAMESPACE;
+
+@JsonDeserialize(builder = ObjectContainerProvisionedResource.Builder.class)
+@JsonTypeName("dataspaceconnector:objectcontainerprovisionedresource")
+public class ObjectContainerProvisionedResource extends ProvisionedDataDestinationResource {
+
+    private ObjectContainerProvisionedResource() {
+    }
+
+    public String getAccountName() {
+        return getDataAddress().getStringProperty(ACCOUNT_NAME);
+    }
+
+    public String getContainerName() {
+        return getDataAddress().getStringProperty(CONTAINER_NAME);
+    }
+
+    @JsonPOJOBuilder(withPrefix = "")
+    public static class Builder extends ProvisionedDataDestinationResource.Builder<ObjectContainerProvisionedResource, Builder> {
+
+        private Builder() {
+            super(new ObjectContainerProvisionedResource());
+            dataAddressBuilder.type(AzureBlobStoreSchema.TYPE);
+        }
+
+        @JsonCreator
+        public static Builder newInstance() {
+            return new Builder();
+        }
+
+        public Builder accountName(String accountName) {
+            dataAddressBuilder.property(EDC_NAMESPACE + ACCOUNT_NAME, accountName);
+            return this;
+        }
+
+        public Builder containerName(String containerName) {
+            dataAddressBuilder.property(EDC_NAMESPACE + CONTAINER_NAME, containerName);
+            return this;
+        }
+
+        @Override
+        public Builder resourceName(String name) {
+            dataAddressBuilder.keyName(name);
+            super.resourceName(name);
+            return this;
+        }
+
+        public Builder folderName(String folderName) {
+            if (folderName != null) {
+                dataAddressBuilder.property(EDC_NAMESPACE + FOLDER_NAME, folderName);
+            }
+            return this;
+        }
+    }
+}

edc-extensions/backport/azblob-provisioner/src/main/java/org/eclipse/edc/connector/provision/azure/blob/ObjectStorageConsumerResourceDefinitionGenerator.java

@@ -0,0 +1,59 @@
+/********************************************************************************
+ * Copyright (c) 2020,2021 Microsoft Corporation
+ *
+ * See the NOTICE file(s) distributed with this work for additional
+ * information regarding copyright ownership.
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0.
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ ********************************************************************************/
+
+package org.eclipse.edc.connector.provision.azure.blob;
+
+import org.eclipse.edc.azure.blob.AzureBlobStoreSchema;
+import org.eclipse.edc.connector.controlplane.transfer.spi.provision.ConsumerResourceDefinitionGenerator;
+import org.eclipse.edc.connector.controlplane.transfer.spi.types.ResourceDefinition;
+import org.eclipse.edc.connector.controlplane.transfer.spi.types.TransferProcess;
+import org.eclipse.edc.policy.model.Policy;
+import org.eclipse.edc.spi.types.domain.DataAddress;
+import org.jetbrains.annotations.Nullable;
+
+import static java.util.Optional.ofNullable;
+import static java.util.UUID.randomUUID;
+
+public class ObjectStorageConsumerResourceDefinitionGenerator implements ConsumerResourceDefinitionGenerator {
+
+    @Override
+    public @Nullable ResourceDefinition generate(TransferProcess transferProcess, Policy policy) {
+        var destination = transferProcess.getDataDestination();
+        var id = randomUUID().toString();
+        var account = destination.getStringProperty(AzureBlobStoreSchema.ACCOUNT_NAME);
+        var container = destination.getStringProperty(AzureBlobStoreSchema.CONTAINER_NAME);
+        var folderName = destination.getStringProperty(AzureBlobStoreSchema.FOLDER_NAME);
+
+        if (container == null) {
+            container = randomUUID().toString();
+        }
+        return ObjectStorageResourceDefinition.Builder.newInstance()
+                .id(id)
+                .accountName(account)
+                .containerName(container)
+                .folderName(folderName)
+                .build();
+    }
+
+    @Override
+    public boolean canGenerate(TransferProcess dataRequest, Policy policy) {
+        var type = ofNullable(dataRequest.getDataDestination()).map(DataAddress::getType).orElse(null);
+        return AzureBlobStoreSchema.TYPE.equals(type);
+    }
+}