Skip to content

Commit

Permalink
[Storage] [DataMovement] Blob Preserve - Http headers, Access Tier an…
Browse files Browse the repository at this point in the history 
…d Metadata (#41551)

* Blob Preserve First Draft

* Draft for DataTransferPropertyOfT

* WIP

* Cleanup; Export API

* WIP - Unit tests; some integration tests

* WIP - service to service integration tests

* WIP - added end to end tests; updated checkpointer indexing and unit tests

* Added testing for end to end pause and resume for blobs

* Minor cleanup

* PR comments + using copy source properties on single put blob from url

* PR Comments + Recording new tests

* Removed unnecessary method in extensions

* NIT - corrected indents

* Actual update to assets.json for recordings

* Update samples

* Rerecord files tests

* Rerecorded tests

* Fix DateTimeOffset checkpointer in Shares

* Fix recordings for content type mismatch

* Cleanup
  • Loading branch information
@amnguye
amnguye authored Mar 22, 2024
1 parent f403667 commit 0ef8e8d
Show file tree
Hide file tree
Showing 53 changed files with 6,300 additions and 453 deletions.
2 changes: 1 addition & 1 deletion sdk/storage/Azure.Storage.Blobs/src/Models/PageBlobUploadPagesOptions.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@ public class PageBlobUploadPagesOptions
{
/// <summary>
/// Optional <see cref="PageBlobRequestConditions"/> to add
/// conditions on the upload of this Append Blob.
/// conditions on the upload of this Page Blob.
/// </summary>
public PageBlobRequestConditions Conditions { get; set; }

Expand Down
9 changes: 5 additions & 4 deletions sdk/storage/Azure.Storage.DataMovement.Blobs/README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -201,10 +201,11 @@ StorageResource virtualDirectoryResource = blobs.FromClient(
```C# Snippet:ResourceConstruction_Blobs_WithOptions_BlockBlob
BlockBlobStorageResourceOptions resourceOptions = new()
{
Metadata = new Dictionary<string, string>
{
{ "key", "value" }
}
Metadata = new DataTransferProperty<IDictionary<string, string>> (
new Dictionary<string, string>
{
{ "key", "value" }
})
};
StorageResource leasedBlockBlobResource = blobs.FromClient(
blockBlobClient,
Expand Down
11 changes: 7 additions & 4 deletions sdk/storage/Azure.Storage.DataMovement.Blobs/api/Azure.Storage.DataMovement.Blobs.net6.0.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -54,10 +54,13 @@ public BlobStorageResourceContainerOptions() { }
public partial class BlobStorageResourceOptions
{
public BlobStorageResourceOptions() { }
public Azure.Storage.Blobs.Models.AccessTier? AccessTier { get { throw null; } set { } }
public Azure.Storage.Blobs.Models.BlobHttpHeaders HttpHeaders { get { throw null; } set { } }
public System.Collections.Generic.IDictionary<string, string> Metadata { get { throw null; } set { } }
public System.Collections.Generic.IDictionary<string, string> Tags { get { throw null; } set { } }
public Azure.Storage.DataMovement.DataTransferProperty<Azure.Storage.Blobs.Models.AccessTier?> AccessTier { get { throw null; } set { } }
public Azure.Storage.DataMovement.DataTransferProperty<string> CacheControl { get { throw null; } set { } }
public Azure.Storage.DataMovement.DataTransferProperty<string> ContentDisposition { get { throw null; } set { } }
public Azure.Storage.DataMovement.DataTransferProperty<string> ContentEncoding { get { throw null; } set { } }
public Azure.Storage.DataMovement.DataTransferProperty<string> ContentLanguage { get { throw null; } set { } }
public Azure.Storage.DataMovement.DataTransferProperty<string> ContentType { get { throw null; } set { } }
public Azure.Storage.DataMovement.DataTransferProperty<System.Collections.Generic.IDictionary<string, string>> Metadata { get { throw null; } set { } }
}
public partial class BlockBlobStorageResourceOptions : Azure.Storage.DataMovement.Blobs.BlobStorageResourceOptions
{
Expand Down
11 changes: 7 additions & 4 deletions ...e/Azure.Storage.DataMovement.Blobs/api/Azure.Storage.DataMovement.Blobs.netstandard2.0.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -54,10 +54,13 @@ public BlobStorageResourceContainerOptions() { }
public partial class BlobStorageResourceOptions
{
public BlobStorageResourceOptions() { }
public Azure.Storage.Blobs.Models.AccessTier? AccessTier { get { throw null; } set { } }
public Azure.Storage.Blobs.Models.BlobHttpHeaders HttpHeaders { get { throw null; } set { } }
public System.Collections.Generic.IDictionary<string, string> Metadata { get { throw null; } set { } }
public System.Collections.Generic.IDictionary<string, string> Tags { get { throw null; } set { } }
public Azure.Storage.DataMovement.DataTransferProperty<Azure.Storage.Blobs.Models.AccessTier?> AccessTier { get { throw null; } set { } }
public Azure.Storage.DataMovement.DataTransferProperty<string> CacheControl { get { throw null; } set { } }
public Azure.Storage.DataMovement.DataTransferProperty<string> ContentDisposition { get { throw null; } set { } }
public Azure.Storage.DataMovement.DataTransferProperty<string> ContentEncoding { get { throw null; } set { } }
public Azure.Storage.DataMovement.DataTransferProperty<string> ContentLanguage { get { throw null; } set { } }
public Azure.Storage.DataMovement.DataTransferProperty<string> ContentType { get { throw null; } set { } }
public Azure.Storage.DataMovement.DataTransferProperty<System.Collections.Generic.IDictionary<string, string>> Metadata { get { throw null; } set { } }
}
public partial class BlockBlobStorageResourceOptions : Azure.Storage.DataMovement.Blobs.BlobStorageResourceOptions
{
Expand Down
9 changes: 5 additions & 4 deletions sdk/storage/Azure.Storage.DataMovement.Blobs/samples/Sample01b_HelloWorldAsync.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -155,10 +155,11 @@ public async Task ConstructFromClientsDemonstration()
#region Snippet:ResourceConstruction_Blobs_WithOptions_BlockBlob
BlockBlobStorageResourceOptions resourceOptions = new()
{
Metadata = new Dictionary<string, string>
{
{ "key", "value" }
}
Metadata = new DataTransferProperty<IDictionary<string, string>> (
new Dictionary<string, string>
{
{ "key", "value" }
})
};
StorageResource leasedBlockBlobResource = blobs.FromClient(
blockBlobClient,
Expand Down
34 changes: 25 additions & 9 deletions sdk/storage/Azure.Storage.DataMovement.Blobs/src/AppendBlobStorageResource.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -136,7 +136,10 @@ protected override async Task CopyFromStreamAsync(
if (position == 0)
{
await BlobClient.CreateAsync(
_options.ToCreateOptions(overwrite),
DataMovementBlobsExtensions.GetCreateOptions(
_options,
overwrite,
options?.SourceProperties),
cancellationToken).ConfigureAwait(false);
}
if (streamLength > 0)
Expand Down Expand Up @@ -174,7 +177,10 @@ protected override async Task CopyFromUriAsync(
{
// Create Append blob beforehand
await BlobClient.CreateAsync(
options: _options.ToCreateOptions(overwrite),
options: DataMovementBlobsExtensions.GetCreateOptions(
_options,
overwrite,
options?.SourceProperties),
cancellationToken: cancellationToken).ConfigureAwait(false);

// There is no synchronous single-call copy API for Append/Page -> Append Blob
Expand Down Expand Up @@ -218,7 +224,10 @@ protected override async Task CopyBlockFromUriAsync(
if (range.Offset == 0)
{
await BlobClient.CreateAsync(
_options.ToCreateOptions(overwrite),
DataMovementBlobsExtensions.GetCreateOptions(
_options,
overwrite,
options?.SourceProperties),
cancellationToken).ConfigureAwait(false);
}

Expand Down Expand Up @@ -269,7 +278,10 @@ protected override async Task<HttpAuthorization> GetCopyAuthorizationHeaderAsync
/// <summary>
/// Commits the block list given.
/// </summary>
protected override Task CompleteTransferAsync(bool overwrite, CancellationToken cancellationToken = default)
protected override Task CompleteTransferAsync(
bool overwrite,
StorageResourceCompleteTransferOptions completeTransferOptions = default,
CancellationToken cancellationToken = default)
{
// no-op for now
return Task.CompletedTask;
Expand Down Expand Up @@ -299,11 +311,15 @@ protected override StorageResourceCheckpointData GetSourceCheckpointData()
protected override StorageResourceCheckpointData GetDestinationCheckpointData()
{
return new BlobDestinationCheckpointData(
BlobType.Append,
_options?.HttpHeaders,
_options?.AccessTier,
_options?.Metadata,
_options?.Tags);
blobType: BlobType.Append,
contentType: _options?.ContentType,
contentEncoding: _options?.ContentEncoding,
contentLanguage: _options?.ContentLanguage,
contentDisposition: _options?.ContentDisposition,
cacheControl: _options?.CacheControl,
accessTier: _options?.AccessTier,
metadata:_options?.Metadata,
tags: default);
}
}
}
1 change: 0 additions & 1 deletion sdk/storage/Azure.Storage.DataMovement.Blobs/src/Azure.Storage.DataMovement.Blobs.csproj
View file
Original file line number Diff line number Diff line change
Expand Up @@ -52,7 +52,6 @@
<Compile Include="$(AzureStorageDataMovementSharedSources)Errors.DataMovement.cs" LinkBase="Shared\DataMovement" />
<Compile Include="$(AzureStorageDataMovementSharedSources)DataMovementConstants.cs" LinkBase="Shared\DataMovement" />
<Compile Include="$(AzureStorageDataMovementSharedSources)DataTransferStatusInternal.cs" LinkBase="Shared\DataMovement" />
<Compile Include="$(AzureStorageDataMovementSharedSources)ResponseExtensions.cs" LinkBase="Shared\DataMovement" />
<Compile Include="$(AzureStorageDataMovementSharedSources)StorageResourceItemInternal.cs" LinkBase="Shared\DataMovement" />
<Compile Include="$(AzureStorageDataMovementSharedSources)StorageResourceContainerInternal.cs" LinkBase="Shared\DataMovement" />
</ItemGroup>
Expand Down
Loading

0 comments on commit 0ef8e8d

Please sign in to comment.