From 0ef8e8d1a13c7712528eda9ccfdd9b178c7832dd Mon Sep 17 00:00:00 2001 From: Amanda Nguyen <48961492+amnguye@users.noreply.github.com> Date: Fri, 22 Mar 2024 14:25:25 -0700 Subject: [PATCH] [Storage] [DataMovement] Blob Preserve - Http headers, Access Tier and Metadata (#41551) * Blob Preserve First Draft * Draft for DataTransferPropertyOfT * WIP * Cleanup; Export API * WIP - Unit tests; some integration tests * WIP - service to service integration tests * WIP - added end to end tests; updated checkpointer indexing and unit tests * Added testing for end to end pause and resume for blobs * Minor cleanup * PR comments + using copy source properties on single put blob from url * PR Comments + Recording new tests * Removed unnecessary method in extensions * NIT - corrected indents * Actual update to assets.json for recordings * Update samples * Rerecord files tests * Rerecorded tests * Fix DateTimeOffset checkpointer in Shares * Fix recordings for content type mismatch * Cleanup --- .../src/Models/PageBlobUploadPagesOptions.cs | 2 +- .../README.md | 9 +- ...Azure.Storage.DataMovement.Blobs.net6.0.cs | 11 +- ...orage.DataMovement.Blobs.netstandard2.0.cs | 11 +- .../samples/Sample01b_HelloWorldAsync.cs | 9 +- .../src/AppendBlobStorageResource.cs | 34 +- .../Azure.Storage.DataMovement.Blobs.csproj | 1 - .../src/BlobDestinationCheckpointData.cs | 325 +++-- .../src/BlobStorageResourceContainer.cs | 15 +- .../src/BlobStorageResourceOptions.cs | 73 +- .../src/BlockBlobStorageResource.cs | 38 +- .../src/DataMovementBlobConstants.cs | 37 +- .../src/DataMovementBlobsExtensions.cs | 173 ++- .../src/PageBlobStorageResource.cs | 35 +- ...re.Storage.DataMovement.Blobs.Tests.csproj | 1 + .../BlobDestinationCheckpointDataTests.cs | 218 +++- .../tests/RehydrateBlobResourceTests.cs | 106 +- .../BlobDestinationCheckpointData.1.bin | Bin 175 -> 0 bytes .../BlobDestinationCheckpointData.2.bin | Bin 0 -> 183 bytes .../assets.json | 2 +- .../src/DataMovementShareConstants.cs | 6 +- .../src/ShareFileStorageResource.cs | 2 +- .../tests/ShareFileResourceTests.cs | 2 - .../api/Azure.Storage.DataMovement.net6.0.cs | 27 +- ...ure.Storage.DataMovement.netstandard2.0.cs | 27 +- .../Azure.Storage.DataMovement/assets.json | 2 +- .../src/CommitChunkHandler.cs | 11 +- .../src/DataTransferProperty.cs | 53 + .../src/DataTransferPropertyOfT.cs | 53 + .../src/LocalFileStorageResource.cs | 5 +- .../src/ServiceToServiceJobPart.cs | 48 +- .../src/Shared/CheckpointerExtensions.cs | 20 +- .../src/Shared/DataMovementConstants.cs | 2 +- .../src/Shared/StorageResourceItemInternal.cs | 10 +- .../StorageResourceCompleteTransferOptions.cs | 20 + .../src/StorageResourceCopyFromUriOptions.cs | 5 + .../src/StorageResourceItem.cs | 8 +- .../StorageResourceWriteToOffsetOptions.cs | 5 + .../src/StreamToUriJobPart.cs | 77 +- .../tests/AppendBlobStorageResourceTests.cs | 991 ++++++++++++++ .../tests/BlockBlobStorageResourceTests.cs | 1138 ++++++++++++++++- .../tests/CleanUpTransferTests.cs | 7 +- .../tests/CommitChunkHandlerTests.cs | 88 +- .../tests/MockStorageResource.cs | 5 +- .../tests/PageBlobStorageResourceTests.cs | 1022 +++++++++++++++ .../tests/PauseResumeTransferTests.cs | 51 +- .../tests/ServiceToServiceJobPartTests.cs | 255 ++++ .../tests/Shared/DataMovementBlobTestBase.cs | 15 +- .../tests/Shared/MemoryStorageResourceItem.cs | 6 +- .../Shared/StartTransferUploadTestBase.cs | 4 +- .../StartTransferSyncCopyDirectoryTests.cs | 481 ++++++- .../tests/StartTransferSyncCopyTests.cs | 879 +++++++++++++ .../tests/StreamToUriJobPartTests.cs | 328 +++++ 53 files changed, 6300 insertions(+), 453 deletions(-) delete mode 100644 sdk/storage/Azure.Storage.DataMovement.Blobs/tests/Resources/BlobDestinationCheckpointData.1.bin create mode 100644 sdk/storage/Azure.Storage.DataMovement.Blobs/tests/Resources/BlobDestinationCheckpointData.2.bin create mode 100644 sdk/storage/Azure.Storage.DataMovement/src/DataTransferProperty.cs create mode 100644 sdk/storage/Azure.Storage.DataMovement/src/DataTransferPropertyOfT.cs create mode 100644 sdk/storage/Azure.Storage.DataMovement/src/StorageResourceCompleteTransferOptions.cs create mode 100644 sdk/storage/Azure.Storage.DataMovement/tests/ServiceToServiceJobPartTests.cs create mode 100644 sdk/storage/Azure.Storage.DataMovement/tests/StreamToUriJobPartTests.cs diff --git a/sdk/storage/Azure.Storage.Blobs/src/Models/PageBlobUploadPagesOptions.cs b/sdk/storage/Azure.Storage.Blobs/src/Models/PageBlobUploadPagesOptions.cs index 95fb9fe6256c5..1d4a417404cdb 100644 --- a/sdk/storage/Azure.Storage.Blobs/src/Models/PageBlobUploadPagesOptions.cs +++ b/sdk/storage/Azure.Storage.Blobs/src/Models/PageBlobUploadPagesOptions.cs @@ -12,7 +12,7 @@ public class PageBlobUploadPagesOptions { /// /// Optional to add - /// conditions on the upload of this Append Blob. + /// conditions on the upload of this Page Blob. /// public PageBlobRequestConditions Conditions { get; set; } diff --git a/sdk/storage/Azure.Storage.DataMovement.Blobs/README.md b/sdk/storage/Azure.Storage.DataMovement.Blobs/README.md index 8614c2f3c8d9e..90cf6b8180055 100644 --- a/sdk/storage/Azure.Storage.DataMovement.Blobs/README.md +++ b/sdk/storage/Azure.Storage.DataMovement.Blobs/README.md @@ -201,10 +201,11 @@ StorageResource virtualDirectoryResource = blobs.FromClient( ```C# Snippet:ResourceConstruction_Blobs_WithOptions_BlockBlob BlockBlobStorageResourceOptions resourceOptions = new() { - Metadata = new Dictionary - { - { "key", "value" } - } + Metadata = new DataTransferProperty> ( + new Dictionary + { + { "key", "value" } + }) }; StorageResource leasedBlockBlobResource = blobs.FromClient( blockBlobClient, diff --git a/sdk/storage/Azure.Storage.DataMovement.Blobs/api/Azure.Storage.DataMovement.Blobs.net6.0.cs b/sdk/storage/Azure.Storage.DataMovement.Blobs/api/Azure.Storage.DataMovement.Blobs.net6.0.cs index a7354a28809d7..7b7ed9ea5e1f9 100644 --- a/sdk/storage/Azure.Storage.DataMovement.Blobs/api/Azure.Storage.DataMovement.Blobs.net6.0.cs +++ b/sdk/storage/Azure.Storage.DataMovement.Blobs/api/Azure.Storage.DataMovement.Blobs.net6.0.cs @@ -54,10 +54,13 @@ public BlobStorageResourceContainerOptions() { } public partial class BlobStorageResourceOptions { public BlobStorageResourceOptions() { } - public Azure.Storage.Blobs.Models.AccessTier? AccessTier { get { throw null; } set { } } - public Azure.Storage.Blobs.Models.BlobHttpHeaders HttpHeaders { get { throw null; } set { } } - public System.Collections.Generic.IDictionary Metadata { get { throw null; } set { } } - public System.Collections.Generic.IDictionary Tags { get { throw null; } set { } } + public Azure.Storage.DataMovement.DataTransferProperty AccessTier { get { throw null; } set { } } + public Azure.Storage.DataMovement.DataTransferProperty CacheControl { get { throw null; } set { } } + public Azure.Storage.DataMovement.DataTransferProperty ContentDisposition { get { throw null; } set { } } + public Azure.Storage.DataMovement.DataTransferProperty ContentEncoding { get { throw null; } set { } } + public Azure.Storage.DataMovement.DataTransferProperty ContentLanguage { get { throw null; } set { } } + public Azure.Storage.DataMovement.DataTransferProperty ContentType { get { throw null; } set { } } + public Azure.Storage.DataMovement.DataTransferProperty> Metadata { get { throw null; } set { } } } public partial class BlockBlobStorageResourceOptions : Azure.Storage.DataMovement.Blobs.BlobStorageResourceOptions { diff --git a/sdk/storage/Azure.Storage.DataMovement.Blobs/api/Azure.Storage.DataMovement.Blobs.netstandard2.0.cs b/sdk/storage/Azure.Storage.DataMovement.Blobs/api/Azure.Storage.DataMovement.Blobs.netstandard2.0.cs index a7354a28809d7..7b7ed9ea5e1f9 100644 --- a/sdk/storage/Azure.Storage.DataMovement.Blobs/api/Azure.Storage.DataMovement.Blobs.netstandard2.0.cs +++ b/sdk/storage/Azure.Storage.DataMovement.Blobs/api/Azure.Storage.DataMovement.Blobs.netstandard2.0.cs @@ -54,10 +54,13 @@ public BlobStorageResourceContainerOptions() { } public partial class BlobStorageResourceOptions { public BlobStorageResourceOptions() { } - public Azure.Storage.Blobs.Models.AccessTier? AccessTier { get { throw null; } set { } } - public Azure.Storage.Blobs.Models.BlobHttpHeaders HttpHeaders { get { throw null; } set { } } - public System.Collections.Generic.IDictionary Metadata { get { throw null; } set { } } - public System.Collections.Generic.IDictionary Tags { get { throw null; } set { } } + public Azure.Storage.DataMovement.DataTransferProperty AccessTier { get { throw null; } set { } } + public Azure.Storage.DataMovement.DataTransferProperty CacheControl { get { throw null; } set { } } + public Azure.Storage.DataMovement.DataTransferProperty ContentDisposition { get { throw null; } set { } } + public Azure.Storage.DataMovement.DataTransferProperty ContentEncoding { get { throw null; } set { } } + public Azure.Storage.DataMovement.DataTransferProperty ContentLanguage { get { throw null; } set { } } + public Azure.Storage.DataMovement.DataTransferProperty ContentType { get { throw null; } set { } } + public Azure.Storage.DataMovement.DataTransferProperty> Metadata { get { throw null; } set { } } } public partial class BlockBlobStorageResourceOptions : Azure.Storage.DataMovement.Blobs.BlobStorageResourceOptions { diff --git a/sdk/storage/Azure.Storage.DataMovement.Blobs/samples/Sample01b_HelloWorldAsync.cs b/sdk/storage/Azure.Storage.DataMovement.Blobs/samples/Sample01b_HelloWorldAsync.cs index 09116515c2671..1a3b79da0f983 100644 --- a/sdk/storage/Azure.Storage.DataMovement.Blobs/samples/Sample01b_HelloWorldAsync.cs +++ b/sdk/storage/Azure.Storage.DataMovement.Blobs/samples/Sample01b_HelloWorldAsync.cs @@ -155,10 +155,11 @@ public async Task ConstructFromClientsDemonstration() #region Snippet:ResourceConstruction_Blobs_WithOptions_BlockBlob BlockBlobStorageResourceOptions resourceOptions = new() { - Metadata = new Dictionary - { - { "key", "value" } - } + Metadata = new DataTransferProperty> ( + new Dictionary + { + { "key", "value" } + }) }; StorageResource leasedBlockBlobResource = blobs.FromClient( blockBlobClient, diff --git a/sdk/storage/Azure.Storage.DataMovement.Blobs/src/AppendBlobStorageResource.cs b/sdk/storage/Azure.Storage.DataMovement.Blobs/src/AppendBlobStorageResource.cs index 12b7db2d7514a..bfcfb71cbcb10 100644 --- a/sdk/storage/Azure.Storage.DataMovement.Blobs/src/AppendBlobStorageResource.cs +++ b/sdk/storage/Azure.Storage.DataMovement.Blobs/src/AppendBlobStorageResource.cs @@ -136,7 +136,10 @@ protected override async Task CopyFromStreamAsync( if (position == 0) { await BlobClient.CreateAsync( - _options.ToCreateOptions(overwrite), + DataMovementBlobsExtensions.GetCreateOptions( + _options, + overwrite, + options?.SourceProperties), cancellationToken).ConfigureAwait(false); } if (streamLength > 0) @@ -174,7 +177,10 @@ protected override async Task CopyFromUriAsync( { // Create Append blob beforehand await BlobClient.CreateAsync( - options: _options.ToCreateOptions(overwrite), + options: DataMovementBlobsExtensions.GetCreateOptions( + _options, + overwrite, + options?.SourceProperties), cancellationToken: cancellationToken).ConfigureAwait(false); // There is no synchronous single-call copy API for Append/Page -> Append Blob @@ -218,7 +224,10 @@ protected override async Task CopyBlockFromUriAsync( if (range.Offset == 0) { await BlobClient.CreateAsync( - _options.ToCreateOptions(overwrite), + DataMovementBlobsExtensions.GetCreateOptions( + _options, + overwrite, + options?.SourceProperties), cancellationToken).ConfigureAwait(false); } @@ -269,7 +278,10 @@ protected override async Task GetCopyAuthorizationHeaderAsync /// /// Commits the block list given. /// - protected override Task CompleteTransferAsync(bool overwrite, CancellationToken cancellationToken = default) + protected override Task CompleteTransferAsync( + bool overwrite, + StorageResourceCompleteTransferOptions completeTransferOptions = default, + CancellationToken cancellationToken = default) { // no-op for now return Task.CompletedTask; @@ -299,11 +311,15 @@ protected override StorageResourceCheckpointData GetSourceCheckpointData() protected override StorageResourceCheckpointData GetDestinationCheckpointData() { return new BlobDestinationCheckpointData( - BlobType.Append, - _options?.HttpHeaders, - _options?.AccessTier, - _options?.Metadata, - _options?.Tags); + blobType: BlobType.Append, + contentType: _options?.ContentType, + contentEncoding: _options?.ContentEncoding, + contentLanguage: _options?.ContentLanguage, + contentDisposition: _options?.ContentDisposition, + cacheControl: _options?.CacheControl, + accessTier: _options?.AccessTier, + metadata:_options?.Metadata, + tags: default); } } } diff --git a/sdk/storage/Azure.Storage.DataMovement.Blobs/src/Azure.Storage.DataMovement.Blobs.csproj b/sdk/storage/Azure.Storage.DataMovement.Blobs/src/Azure.Storage.DataMovement.Blobs.csproj index 09c16659f969b..d0ec61ade002b 100644 --- a/sdk/storage/Azure.Storage.DataMovement.Blobs/src/Azure.Storage.DataMovement.Blobs.csproj +++ b/sdk/storage/Azure.Storage.DataMovement.Blobs/src/Azure.Storage.DataMovement.Blobs.csproj @@ -52,7 +52,6 @@ - diff --git a/sdk/storage/Azure.Storage.DataMovement.Blobs/src/BlobDestinationCheckpointData.cs b/sdk/storage/Azure.Storage.DataMovement.Blobs/src/BlobDestinationCheckpointData.cs index 4aa0586ddd749..0269b13ffdf0e 100644 --- a/sdk/storage/Azure.Storage.DataMovement.Blobs/src/BlobDestinationCheckpointData.cs +++ b/sdk/storage/Azure.Storage.DataMovement.Blobs/src/BlobDestinationCheckpointData.cs @@ -17,51 +17,90 @@ internal class BlobDestinationCheckpointData : BlobCheckpointData /// /// The content headers for the destination blob. /// - public BlobHttpHeaders ContentHeaders; - private byte[] _contentTypeBytes; - private byte[] _contentEncodingBytes; - private byte[] _contentLanguageBytes; - private byte[] _contentDispositionBytes; - private byte[] _cacheControlBytes; + public DataTransferProperty CacheControl; + public bool PreserveCacheControl; + public byte[] CacheControlBytes; + + public DataTransferProperty ContentDisposition; + public bool PreserveContentDisposition; + public byte[] ContentDispositionBytes; + + public DataTransferProperty ContentEncoding; + public bool PreserveContentEncoding; + public byte[] ContentEncodingBytes; + + public DataTransferProperty ContentLanguage; + public bool PreserveContentLanguage; + public byte[] ContentLanguageBytes; + + public DataTransferProperty ContentType; + public bool PreserveContentType; + public byte[] ContentTypeBytes; /// /// The access tier of the destination blob. /// - public AccessTier? AccessTier; + public DataTransferProperty AccessTier; + public bool PreserveAccessTier; /// - /// The metadate for the destination blob. + /// The metadata for the destination blob. /// - public Metadata Metadata; - private byte[] _metadataBytes; + public DataTransferProperty Metadata; + public bool PreserveMetadata; + public byte[] MetadataBytes; /// /// The Blob tags for the destination blob. /// - public Tags Tags; - private byte[] _tagsBytes; + public DataTransferProperty Tags; + public bool PreserveTags; + public byte[] TagsBytes; public override int Length => CalculateLength(); public BlobDestinationCheckpointData( BlobType blobType, - BlobHttpHeaders contentHeaders, - AccessTier? accessTier, - Metadata metadata, - Tags blobTags) + DataTransferProperty contentType, + DataTransferProperty contentEncoding, + DataTransferProperty contentLanguage, + DataTransferProperty contentDisposition, + DataTransferProperty cacheControl, + DataTransferProperty accessTier, + DataTransferProperty metadata, + DataTransferProperty tags) : base(DataMovementBlobConstants.DestinationCheckpointData.SchemaVersion, blobType) { - ContentHeaders = contentHeaders; - _contentTypeBytes = ContentHeaders?.ContentType != default ? Encoding.UTF8.GetBytes(ContentHeaders.ContentType) : Array.Empty(); - _contentEncodingBytes = ContentHeaders?.ContentEncoding != default ? Encoding.UTF8.GetBytes(ContentHeaders.ContentEncoding) : Array.Empty(); - _contentLanguageBytes = ContentHeaders?.ContentLanguage != default ? Encoding.UTF8.GetBytes(ContentHeaders.ContentLanguage) : Array.Empty(); - _contentDispositionBytes = ContentHeaders?.ContentDisposition != default ? Encoding.UTF8.GetBytes(ContentHeaders.ContentDisposition) : Array.Empty(); - _cacheControlBytes = ContentHeaders?.CacheControl != default ? Encoding.UTF8.GetBytes(ContentHeaders.CacheControl) : Array.Empty(); + PreserveAccessTier = accessTier?.Preserve ?? true; AccessTier = accessTier; + + CacheControl = cacheControl; + PreserveCacheControl = cacheControl?.Preserve ?? true; + CacheControlBytes = cacheControl?.Value != default ? Encoding.UTF8.GetBytes(cacheControl.Value) : Array.Empty(); + + ContentDisposition = contentDisposition; + PreserveContentDisposition = contentDisposition?.Preserve ?? true; + ContentDispositionBytes = contentDisposition?.Value != default ? Encoding.UTF8.GetBytes(contentDisposition.Value) : Array.Empty(); + + ContentEncoding = contentEncoding; + PreserveContentEncoding = contentEncoding?.Preserve ?? true; + ContentEncodingBytes = contentEncoding?.Value != default ? Encoding.UTF8.GetBytes(contentEncoding.Value) : Array.Empty(); + + ContentLanguage = contentLanguage; + PreserveContentLanguage = contentLanguage?.Preserve ?? true; + ContentLanguageBytes = contentLanguage?.Value != default ? Encoding.UTF8.GetBytes(contentLanguage.Value) : Array.Empty(); + + ContentType = contentType; + PreserveContentType = contentType?.Preserve ?? true; + ContentTypeBytes = contentType?.Value != default ? Encoding.UTF8.GetBytes(contentType.Value) : Array.Empty(); + Metadata = metadata; - _metadataBytes = Metadata != default ? Encoding.UTF8.GetBytes(Metadata.DictionaryToString()) : Array.Empty(); - Tags = blobTags; - _tagsBytes = Tags != default ? Encoding.UTF8.GetBytes(Tags.DictionaryToString()) : Array.Empty(); + PreserveMetadata = metadata?.Preserve ?? true; + MetadataBytes = metadata?.Value != default ? Encoding.UTF8.GetBytes(metadata.Value.DictionaryToString()) : Array.Empty(); + + Tags = tags; + PreserveTags = tags?.Preserve ?? false; + TagsBytes = tags?.Value != default ? Encoding.UTF8.GetBytes(tags.Value.DictionaryToString()) : Array.Empty(); } protected override void Serialize(Stream stream) @@ -77,37 +116,138 @@ protected override void Serialize(Stream stream) // BlobType writer.Write((byte)BlobType); - // ContentType offset/length - writer.WriteVariableLengthFieldInfo(_contentTypeBytes.Length, ref currentVariableLengthIndex); + // Preserve Content Type + writer.Write(PreserveContentType); + if (!PreserveContentType) + { + // Content Type offset/length + writer.WriteVariableLengthFieldInfo(ContentTypeBytes.Length, ref currentVariableLengthIndex); + } + else + { + // Padding + writer.WriteEmptyLengthOffset(); + } - // ContentEncoding offset/length - writer.WriteVariableLengthFieldInfo(_contentEncodingBytes.Length, ref currentVariableLengthIndex); + // Preserve Content Encoding + writer.Write(PreserveContentEncoding); + if (!PreserveContentEncoding) + { + // ContentEncoding offset/length + writer.WriteVariableLengthFieldInfo(ContentEncodingBytes.Length, ref currentVariableLengthIndex); + } + else + { + // Padding + writer.WriteEmptyLengthOffset(); + } - // ContentLanguage offset/length - writer.WriteVariableLengthFieldInfo(_contentLanguageBytes.Length, ref currentVariableLengthIndex); + // Preserve Content Language + writer.Write(PreserveContentLanguage); + if (!PreserveContentLanguage) + { + // ContentLanguage offset/length + writer.WriteVariableLengthFieldInfo(ContentLanguageBytes.Length, ref currentVariableLengthIndex); + } + else + { + // Padding + writer.WriteEmptyLengthOffset(); + } - // ContentDisposition offset/length - writer.WriteVariableLengthFieldInfo(_contentDispositionBytes.Length, ref currentVariableLengthIndex); + // Preserve Content Disposition + writer.Write(PreserveContentDisposition); + if (!PreserveContentDisposition) + { + // ContentDisposition offset/length + writer.WriteVariableLengthFieldInfo(ContentDispositionBytes.Length, ref currentVariableLengthIndex); + } + else + { + // Padding + writer.WriteEmptyLengthOffset(); + } - // CacheControl offset/length - writer.WriteVariableLengthFieldInfo(_cacheControlBytes.Length, ref currentVariableLengthIndex); + // Preserve Cache Control + writer.Write(PreserveCacheControl); + if (!PreserveCacheControl) + { + // CacheControl offset/length + writer.WriteVariableLengthFieldInfo(CacheControlBytes.Length, ref currentVariableLengthIndex); + } + else + { + // Padding + writer.WriteEmptyLengthOffset(); + } - // AccessTier - writer.Write((byte)AccessTier.ToJobPlanAccessTier()); + // Preserve Access Tier + writer.Write(PreserveAccessTier); + if (!PreserveAccessTier) + { + // AccessTier + writer.Write((byte)AccessTier.Value.ToJobPlanAccessTier()); + } + else + { + // Write null byte value + writer.Write((byte)0); + } - // Metadata offset/length - writer.WriteVariableLengthFieldInfo(_metadataBytes.Length, ref currentVariableLengthIndex); + // Preserve Metadata + writer.Write(PreserveMetadata); + if (!PreserveMetadata) + { + // Metadata offset/length + writer.WriteVariableLengthFieldInfo(MetadataBytes.Length, ref currentVariableLengthIndex); + } + else + { + // Padding + writer.WriteEmptyLengthOffset(); + } - // Tags offset/length - writer.WriteVariableLengthFieldInfo(_tagsBytes.Length, ref currentVariableLengthIndex); + // Preserve Blob Tags + writer.Write(PreserveTags); + if (!PreserveTags) + { + // Tags offset/length + writer.WriteVariableLengthFieldInfo(TagsBytes.Length, ref currentVariableLengthIndex); + } + else + { + // Padding + writer.WriteEmptyLengthOffset(); + } - writer.Write(_contentTypeBytes); - writer.Write(_contentEncodingBytes); - writer.Write(_contentLanguageBytes); - writer.Write(_contentDispositionBytes); - writer.Write(_cacheControlBytes); - writer.Write(_metadataBytes); - writer.Write(_tagsBytes); + if (!PreserveContentType) + { + writer.Write(ContentTypeBytes); + } + if (!PreserveContentEncoding) + { + writer.Write(ContentEncodingBytes); + } + if (!PreserveContentLanguage) + { + writer.Write(ContentLanguageBytes); + } + if (!PreserveContentDisposition) + { + writer.Write(ContentDispositionBytes); + } + if (!PreserveCacheControl) + { + writer.Write(CacheControlBytes); + } + if (!PreserveMetadata) + { + writer.Write(MetadataBytes); + } + if (!PreserveTags) + { + writer.Write(TagsBytes); + } } internal static BlobDestinationCheckpointData Deserialize(Stream stream) @@ -123,45 +263,55 @@ internal static BlobDestinationCheckpointData Deserialize(Stream stream) throw Errors.UnsupportedJobSchemaVersionHeader(version.ToString()); } + // Index Values // BlobType BlobType blobType = (BlobType)reader.ReadByte(); - // ContentType offset/length + // Preserve Content Type and offset/length + bool preserveContentType = reader.ReadBoolean(); int contentTypeOffset = reader.ReadInt32(); int contentTypeLength = reader.ReadInt32(); - // ContentEncoding offset/length + // Preserve Content Encoding and offset/length + bool preserveContentEncoding = reader.ReadBoolean(); int contentEncodingOffset = reader.ReadInt32(); int contentEncodingLength = reader.ReadInt32(); - // ContentLanguage offset/length + // Preserve Content Language and offset/length + bool preserveContentLanguage = reader.ReadBoolean(); int contentLanguageOffset = reader.ReadInt32(); int contentLanguageLength = reader.ReadInt32(); - // ContentDisposition offset/length + // Preserve ContentDisposition and offset/length + bool preserveContentDisposition = reader.ReadBoolean(); int contentDispositionOffset = reader.ReadInt32(); int contentDispositionLength = reader.ReadInt32(); - // CacheControl offset/length + // Preserve CacheControl and offset/length + bool preserveCacheControl = reader.ReadBoolean(); int cacheControlOffset = reader.ReadInt32(); int cacheControlLength = reader.ReadInt32(); - // AccessTier - JobPlanAccessTier jobPlanAccessTier = (JobPlanAccessTier)reader.ReadByte(); + // Preserve AccessTier and offset/length + bool preserveAccessTier = reader.ReadBoolean(); AccessTier? accessTier = default; + JobPlanAccessTier jobPlanAccessTier = (JobPlanAccessTier)reader.ReadByte(); if (!jobPlanAccessTier.Equals(JobPlanAccessTier.None)) { accessTier = new AccessTier(jobPlanAccessTier.ToString()); } - // Metadata offset/length + // Preserve Metadata and offset/length + bool preserveMetadata = reader.ReadBoolean(); int metadataOffset = reader.ReadInt32(); int metadataLength = reader.ReadInt32(); - // Tags offset/length + // Preserve Tags and offset/length + bool preserveTags = reader.ReadBoolean(); int tagsOffset = reader.ReadInt32(); int tagsLength = reader.ReadInt32(); + // Values // ContentType string contentType = null; if (contentTypeOffset > 0) @@ -218,34 +368,51 @@ internal static BlobDestinationCheckpointData Deserialize(Stream stream) tagsString = reader.ReadBytes(tagsLength).AsString(); } - BlobHttpHeaders contentHeaders = new BlobHttpHeaders() - { - ContentType = contentType, - ContentEncoding = contentEncoding, - ContentLanguage = contentLanguage, - ContentDisposition = contentDisposition, - CacheControl = cacheControl, - }; - return new BlobDestinationCheckpointData( - blobType, - contentHeaders, - accessTier, - metadataString.ToDictionary(nameof(metadataString)), - tagsString.ToDictionary(nameof(tagsString))); + blobType: blobType, + contentType: preserveContentType ? new(preserveContentType) : new(contentType), + contentEncoding: preserveContentEncoding ? new(preserveContentEncoding): new(contentEncoding), + contentLanguage: preserveContentLanguage ? new(preserveContentLanguage) : new(contentLanguage), + contentDisposition: preserveContentDisposition ? new(preserveContentDisposition) : new(contentDisposition), + cacheControl: preserveCacheControl ? new(preserveCacheControl): new(cacheControl), + accessTier: preserveAccessTier ? new(preserveAccessTier) : new(accessTier), + metadata: preserveMetadata ? new(preserveMetadata) : new(metadataString.ToDictionary(nameof(metadataString))), + tags: preserveTags ? new(preserveTags) : new(tagsString.ToDictionary(nameof(tagsString)))); } private int CalculateLength() { - // Length is fixed size fields plus length of each variable length field + // Length is calculated based on whether the property is preserved. + // If the property is preserved, the property's length is added to the total length. int length = DataMovementBlobConstants.DestinationCheckpointData.VariableLengthStartIndex; - length += _contentTypeBytes.Length; - length += _contentEncodingBytes.Length; - length += _contentLanguageBytes.Length; - length += _contentDispositionBytes.Length; - length += _cacheControlBytes.Length; - length += _metadataBytes.Length; - length += _tagsBytes.Length; + if (!PreserveContentType) + { + length += ContentTypeBytes.Length; + } + if (!PreserveContentEncoding) + { + length += ContentEncodingBytes.Length; + } + if (!PreserveContentLanguage) + { + length += ContentLanguageBytes.Length; + } + if (!PreserveContentDisposition) + { + length += ContentDispositionBytes.Length; + } + if (!PreserveCacheControl) + { + length += CacheControlBytes.Length; + } + if (!PreserveMetadata) + { + length += MetadataBytes.Length; + } + if (!PreserveTags) + { + length += TagsBytes.Length; + } return length; } } diff --git a/sdk/storage/Azure.Storage.DataMovement.Blobs/src/BlobStorageResourceContainer.cs b/sdk/storage/Azure.Storage.DataMovement.Blobs/src/BlobStorageResourceContainer.cs index 747fce11911ba..42f6b7065c770 100644 --- a/sdk/storage/Azure.Storage.DataMovement.Blobs/src/BlobStorageResourceContainer.cs +++ b/sdk/storage/Azure.Storage.DataMovement.Blobs/src/BlobStorageResourceContainer.cs @@ -3,7 +3,6 @@ using System; using System.Collections.Generic; -using System.Linq; using System.Runtime.CompilerServices; using System.Threading; using System.Threading.Tasks; @@ -173,11 +172,15 @@ protected override StorageResourceCheckpointData GetSourceCheckpointData() protected override StorageResourceCheckpointData GetDestinationCheckpointData() { return new BlobDestinationCheckpointData( - _options?.BlobType ?? BlobType.Block, - _options?.BlobOptions?.HttpHeaders, - _options?.BlobOptions?.AccessTier, - _options?.BlobOptions?.Metadata, - _options?.BlobOptions?.Tags); + blobType: _options?.BlobType ?? BlobType.Block, + contentType: _options?.BlobOptions?.ContentType, + contentEncoding: _options?.BlobOptions?.ContentEncoding, + contentLanguage: _options?.BlobOptions?.ContentLanguage, + contentDisposition: _options?.BlobOptions?.ContentDisposition, + cacheControl: _options?.BlobOptions?.CacheControl, + accessTier: _options?.BlobOptions?.AccessTier, + metadata: _options?.BlobOptions?.Metadata, + tags: default); } private string ApplyOptionalPrefix(string path) diff --git a/sdk/storage/Azure.Storage.DataMovement.Blobs/src/BlobStorageResourceOptions.cs b/sdk/storage/Azure.Storage.DataMovement.Blobs/src/BlobStorageResourceOptions.cs index e59add145a08b..8b3a1dd763a9c 100644 --- a/sdk/storage/Azure.Storage.DataMovement.Blobs/src/BlobStorageResourceOptions.cs +++ b/sdk/storage/Azure.Storage.DataMovement.Blobs/src/BlobStorageResourceOptions.cs @@ -22,42 +22,89 @@ public BlobStorageResourceOptions() internal BlobStorageResourceOptions(BlobStorageResourceOptions other) { Metadata = other?.Metadata; - Tags = other?.Tags; - HttpHeaders = other?.HttpHeaders; + CacheControl = other?.CacheControl; + ContentDisposition = other?.ContentDisposition; + ContentEncoding = other?.ContentEncoding; + ContentLanguage = other?.ContentLanguage; + ContentType = other?.ContentType; AccessTier = other?.AccessTier; } /// - /// Optional. Defines custom metadata to set on the destination blob. + /// Optional. For transferring metadata from the source to the destination storage resource. + /// + /// By default preserves the metadata from the source. + /// + /// Applies to upload and copy transfers. + /// + public DataTransferProperty Metadata { get; set; } + + /// + /// Optional. Sets the Cache Control header which + /// specifies directives for caching mechanisms. + /// + /// By default preserves the Cache Control from the source. + /// + /// Applies to upload and copy transfers. + /// + public DataTransferProperty CacheControl { get; set; } + + /// + /// Optional. Sets the Content Disposition header which + /// conveys additional information about how to process the response + /// payload, and also can be used to attach additional metadata. For + /// example, if set to attachment, it indicates that the user-agent + /// should not display the response, but instead show a Save As dialog + /// with a filename other than the blob name specified. + /// + /// By default preserves the Content Disposition from the source. /// /// Applies to upload and copy transfers. /// -#pragma warning disable CA2227 // Collection properties should be readonly - public Metadata Metadata { get; set; } -#pragma warning restore CA2227 // Collection properties should be readonly + public DataTransferProperty ContentDisposition { get; set; } /// - /// Optional. Defines tags to set on the destination blob. + /// Optional. Sets the Content Encoding header which + /// specifies which content encodings have been applied to the blob. + /// This value is returned to the client when the Get Blob operation + /// is performed on the blob resource. The client can use this value + /// when returned to decode the blob content. + /// + /// By default preserves the Content Encoding from the source. /// /// Applies to upload and copy transfers. /// -#pragma warning disable CA2227 // Collection properties should be readonly - public Tags Tags { get; set; } -#pragma warning restore CA2227 // Collection properties should be readonly + public DataTransferProperty ContentEncoding { get; set; } /// - /// Optional. Standard HTTP header properties that can be set for the new blob. + /// Optional. Sets the Content Language header which + /// specifies the natural languages used by this resource. + /// + /// By default preserves the Content Language from the source. /// /// Applies to upload and copy transfers. /// - public BlobHttpHeaders HttpHeaders { get; set; } + public DataTransferProperty ContentLanguage { get; set; } + + /// + /// Optional. Sets the Content Type header which + /// specifies the MIME content type of the blob. + /// + /// By default preserves the Content Type from the source. + /// + /// Applies to upload and copy transfers. + /// + public DataTransferProperty ContentType { get; set; } /// /// Optional. See . /// Indicates the access tier to be set on the destination blob. /// + /// By default preserves the Access Tier from the source. + /// /// Applies to upload and copy transfers. + /// Also respective Tier Values applies only to Block or Page Blobs. /// - public AccessTier? AccessTier { get; set; } + public DataTransferProperty AccessTier { get; set; } } } diff --git a/sdk/storage/Azure.Storage.DataMovement.Blobs/src/BlockBlobStorageResource.cs b/sdk/storage/Azure.Storage.DataMovement.Blobs/src/BlockBlobStorageResource.cs index 92d27874dd97e..49e16ec0c4634 100644 --- a/sdk/storage/Azure.Storage.DataMovement.Blobs/src/BlockBlobStorageResource.cs +++ b/sdk/storage/Azure.Storage.DataMovement.Blobs/src/BlockBlobStorageResource.cs @@ -157,7 +157,11 @@ protected override async Task CopyFromStreamAsync( // Default to Upload await BlobClient.UploadAsync( stream, - _options.ToBlobUploadOptions(overwrite, _maxInitialSize), + DataMovementBlobsExtensions.GetBlobUploadOptions( + _options, + overwrite, + _maxInitialSize, + options?.SourceProperties), cancellationToken: cancellationToken).ConfigureAwait(false); return; } @@ -204,7 +208,11 @@ protected override async Task CopyFromUriAsync( // TODO: subject to change as we scale to support resource types outside of blobs. await BlobClient.SyncUploadFromUriAsync( sourceResource.Uri, - _options.ToSyncUploadFromUriOptions(overwrite, options?.SourceAuthentication), + DataMovementBlobsExtensions.GetSyncUploadFromUriOptions( + _options, + overwrite, + options?.SourceAuthentication, + options?.SourceProperties), cancellationToken: cancellationToken).ConfigureAwait(false); } @@ -236,7 +244,7 @@ protected override async Task CopyBlockFromUriAsync( { CancellationHelper.ThrowIfCancellationRequested(cancellationToken); - string id = options?.BlockId ?? Shared.StorageExtensions.GenerateBlockId(range.Offset); + string id = options?.BlockId ?? Storage.Shared.StorageExtensions.GenerateBlockId(range.Offset); if (!_blocks.TryAdd(range.Offset, id)) { throw new ArgumentException($"Cannot Stage Block to the specific offset \"{range.Offset}\", it already exists in the block list"); @@ -299,6 +307,9 @@ protected override async Task GetCopyAuthorizationHeaderAsync /// /// If set to true, will overwrite the blob if exists. /// + /// + /// Optional parameters. + /// /// /// Optional to propagate /// notifications that the operation should be cancelled. @@ -306,15 +317,20 @@ protected override async Task GetCopyAuthorizationHeaderAsync /// The Task which Commits the list of ids protected override async Task CompleteTransferAsync( bool overwrite, + StorageResourceCompleteTransferOptions completeTransferOptions = default, CancellationToken cancellationToken = default) { CancellationHelper.ThrowIfCancellationRequested(cancellationToken); + // Call commit block list if the blob was uploaded in chunks. if (_blocks != null && !_blocks.IsEmpty) { IEnumerable blockIds = _blocks.OrderBy(x => x.Key).Select(x => x.Value); await BlobClient.CommitBlockListAsync( blockIds, - _options.ToCommitBlockOptions(overwrite), + DataMovementBlobsExtensions.GetCommitBlockOptions( + _options, + overwrite, + completeTransferOptions?.SourceProperties), cancellationToken).ConfigureAwait(false); _blocks.Clear(); } @@ -344,11 +360,15 @@ protected override StorageResourceCheckpointData GetSourceCheckpointData() protected override StorageResourceCheckpointData GetDestinationCheckpointData() { return new BlobDestinationCheckpointData( - BlobType.Block, - _options?.HttpHeaders, - _options?.AccessTier, - _options?.Metadata, - _options?.Tags); + blobType: BlobType.Block, + contentType: _options?.ContentType, + contentEncoding: _options?.ContentEncoding, + contentLanguage: _options?.ContentLanguage, + contentDisposition: _options?.ContentDisposition, + cacheControl: _options?.CacheControl, + accessTier: _options?.AccessTier, + metadata: _options?.Metadata, + tags: default); } } } diff --git a/sdk/storage/Azure.Storage.DataMovement.Blobs/src/DataMovementBlobConstants.cs b/sdk/storage/Azure.Storage.DataMovement.Blobs/src/DataMovementBlobConstants.cs index 87e26bef13a39..77d5e37043ae4 100644 --- a/sdk/storage/Azure.Storage.DataMovement.Blobs/src/DataMovementBlobConstants.cs +++ b/sdk/storage/Azure.Storage.DataMovement.Blobs/src/DataMovementBlobConstants.cs @@ -18,26 +18,41 @@ internal class SourceCheckpointData internal class DestinationCheckpointData { - internal const int SchemaVersion = 1; + internal const int SchemaVersion = 2; internal const int VersionIndex = 0; internal const int BlobTypeIndex = VersionIndex + IntSizeInBytes; - internal const int ContentTypeOffsetIndex = BlobTypeIndex + OneByte; + internal const int PreserveContentTypeIndex = BlobTypeIndex + OneByte; + internal const int ContentTypeOffsetIndex = PreserveContentTypeIndex + OneByte; internal const int ContentTypeLengthIndex = ContentTypeOffsetIndex + IntSizeInBytes; - internal const int ContentEncodingOffsetIndex = ContentTypeLengthIndex + IntSizeInBytes; + + internal const int PreserveContentEncodingIndex = ContentTypeLengthIndex + IntSizeInBytes; + internal const int ContentEncodingOffsetIndex = PreserveContentEncodingIndex + OneByte; internal const int ContentEncodingLengthIndex = ContentEncodingOffsetIndex + IntSizeInBytes; - internal const int ContentLanguageOffsetIndex = ContentEncodingLengthIndex + IntSizeInBytes; + + internal const int PreserveContentLanguageIndex = ContentEncodingLengthIndex + IntSizeInBytes; + internal const int ContentLanguageOffsetIndex = PreserveContentLanguageIndex + OneByte; internal const int ContentLanguageLengthIndex = ContentLanguageOffsetIndex + IntSizeInBytes; - internal const int ContentDispositionOffsetIndex = ContentLanguageLengthIndex + IntSizeInBytes; + + internal const int PreserveContentDispositionIndex = ContentLanguageLengthIndex + IntSizeInBytes; + internal const int ContentDispositionOffsetIndex = PreserveContentDispositionIndex + OneByte; internal const int ContentDispositionLengthIndex = ContentDispositionOffsetIndex + IntSizeInBytes; - internal const int CacheControlOffsetIndex = ContentDispositionLengthIndex + IntSizeInBytes; + + internal const int PreserveCacheControlIndex = ContentDispositionLengthIndex + IntSizeInBytes; + internal const int CacheControlOffsetIndex = PreserveCacheControlIndex + OneByte; internal const int CacheControlLengthIndex = CacheControlOffsetIndex + IntSizeInBytes; - internal const int AccessTierIndex = CacheControlLengthIndex + IntSizeInBytes; - internal const int MetadataOffsetIndex = AccessTierIndex + OneByte; + + internal const int PreserveAccessTierIndex = CacheControlLengthIndex + IntSizeInBytes; + internal const int AccessTierValueIndex = PreserveAccessTierIndex + OneByte; + + internal const int PreserveMetadataIndex = AccessTierValueIndex + OneByte; + internal const int MetadataOffsetIndex = PreserveMetadataIndex + OneByte; internal const int MetadataLengthIndex = MetadataOffsetIndex + IntSizeInBytes; - internal const int BlobTagsOffsetIndex = MetadataLengthIndex + IntSizeInBytes; - internal const int BlobTagsLengthIndex = BlobTagsOffsetIndex + IntSizeInBytes; - internal const int VariableLengthStartIndex = BlobTagsLengthIndex + IntSizeInBytes; + + internal const int PreserveTagsIndex = MetadataLengthIndex + IntSizeInBytes; + internal const int TagsOffsetIndex = PreserveTagsIndex + OneByte; + internal const int TagsLengthIndex = TagsOffsetIndex + IntSizeInBytes; + internal const int VariableLengthStartIndex = TagsLengthIndex + IntSizeInBytes; } } } diff --git a/sdk/storage/Azure.Storage.DataMovement.Blobs/src/DataMovementBlobsExtensions.cs b/sdk/storage/Azure.Storage.DataMovement.Blobs/src/DataMovementBlobsExtensions.cs index acfe626f65f4d..846ea8b3f9030 100644 --- a/sdk/storage/Azure.Storage.DataMovement.Blobs/src/DataMovementBlobsExtensions.cs +++ b/sdk/storage/Azure.Storage.DataMovement.Blobs/src/DataMovementBlobsExtensions.cs @@ -4,6 +4,8 @@ using Azure.Storage.Blobs.Models; using System.Collections.Generic; using System.IO; +using Metadata = System.Collections.Generic.IDictionary; +using Tags = System.Collections.Generic.IDictionary; namespace Azure.Storage.DataMovement.Blobs { @@ -28,10 +30,6 @@ internal static StorageResourceItemProperties ToStorageResourceProperties(this B { properties.Add(DataMovementConstants.ResourceProperties.ContentType, blobProperties.ContentType); } - if (blobProperties.ContentHash != default) - { - properties.Add(DataMovementConstants.ResourceProperties.ContentHash, blobProperties.ContentHash); - } if (blobProperties.ContentEncoding != default) { properties.Add(DataMovementConstants.ResourceProperties.ContentEncoding, blobProperties.ContentEncoding); @@ -48,6 +46,10 @@ internal static StorageResourceItemProperties ToStorageResourceProperties(this B { properties.Add(DataMovementConstants.ResourceProperties.CacheControl, blobProperties.CacheControl); } + if (blobProperties.AccessTier != default) + { + properties.Add(DataMovementConstants.ResourceProperties.AccessTier, new AccessTier(blobProperties.AccessTier)); + } return new StorageResourceItemProperties( resourceLength: blobProperties.ContentLength, @@ -75,10 +77,6 @@ internal static StorageResourceItemProperties ToStorageResourceItemProperties(th { properties.Add(DataMovementConstants.ResourceProperties.ContentType, result.Details.ContentType); } - if (result.Details.ContentHash != default) - { - properties.Add(DataMovementConstants.ResourceProperties.ContentHash, result.Details.ContentHash); - } if (result.Details.ContentEncoding != default) { properties.Add(DataMovementConstants.ResourceProperties.ContentEncoding, result.Details.ContentEncoding); @@ -129,10 +127,6 @@ internal static StorageResourceReadStreamResult ToReadStreamStorageResourceInfo( { properties.Add(DataMovementConstants.ResourceProperties.ContentType, result.Details.ContentType); } - if (result.Details.ContentHash != default) - { - properties.Add(DataMovementConstants.ResourceProperties.ContentHash, result.Details.ContentHash); - } if (result.Details.ContentEncoding != default) { properties.Add(DataMovementConstants.ResourceProperties.ContentEncoding, result.Details.ContentEncoding); @@ -251,15 +245,15 @@ internal static BlobDownloadOptions ToBlobDownloadOptions( return result; } - internal static AppendBlobCreateOptions ToCreateOptions( - this AppendBlobStorageResourceOptions options, - bool overwrite) + internal static AppendBlobCreateOptions GetCreateOptions( + AppendBlobStorageResourceOptions options, + bool overwrite, + StorageResourceItemProperties sourceProperties) { return new AppendBlobCreateOptions() { - HttpHeaders = options?.HttpHeaders, - Metadata = options?.Metadata, - Tags = options?.Tags, + HttpHeaders = GetHttpHeaders(options, sourceProperties?.RawProperties), + Metadata = GetMetadata(options, sourceProperties?.RawProperties), Conditions = new AppendBlobRequestConditions() { IfMatch = options?.DestinationConditions?.IfMatch, @@ -326,14 +320,17 @@ internal static BlobDownloadOptions ToBlobDownloadOptions( return result; } - internal static BlobUploadOptions ToBlobUploadOptions(this BlockBlobStorageResourceOptions options, bool overwrite, long initialSize) + internal static BlobUploadOptions GetBlobUploadOptions( + BlockBlobStorageResourceOptions options, + bool overwrite, + long initialSize, + StorageResourceItemProperties sourceProperties) { return new BlobUploadOptions() { - HttpHeaders = options?.HttpHeaders, - Metadata = options?.Metadata, - Tags = options?.Tags, - AccessTier = options?.AccessTier, + HttpHeaders = GetHttpHeaders(options, sourceProperties?.RawProperties), + Metadata = GetMetadata(options, sourceProperties?.RawProperties), + AccessTier = GetAccessTier(options, sourceProperties?.RawProperties), TransferOptions = new StorageTransferOptions() { InitialTransferSize = initialSize, @@ -358,20 +355,19 @@ internal static BlockBlobStageBlockOptions ToBlobStageBlockOptions(this BlockBlo }; } - internal static BlobSyncUploadFromUriOptions ToSyncUploadFromUriOptions( - this BlockBlobStorageResourceOptions options, + internal static BlobSyncUploadFromUriOptions GetSyncUploadFromUriOptions( + BlockBlobStorageResourceOptions options, bool overwrite, - HttpAuthorization sourceAuthorization) + HttpAuthorization sourceAuthorization, + StorageResourceItemProperties sourceProperties) { // There's a lot of conditions that cannot be applied to a Copy Blob (async) Request. // We need to omit them, but still apply them to other requests that do accept them. - // See https://learn.microsoft.com/en-us/rest/api/storageservices/copy-blob-from-url#request-headers + // See https://learn.microsoft.com/en-us/rest/api/storageservices/put-blob-from-url?tabs=microsoft-entra-id#request-headers // to see what headers are accepted. - return new BlobSyncUploadFromUriOptions() + BlobSyncUploadFromUriOptions uploadFromUriOptions = new BlobSyncUploadFromUriOptions() { - HttpHeaders = options?.HttpHeaders, - Metadata = options?.Metadata, - Tags = options?.Tags, + AccessTier = GetAccessTier(options, sourceProperties?.RawProperties), SourceConditions = new BlobRequestConditions() { IfMatch = options?.SourceConditions?.IfMatch, @@ -382,6 +378,23 @@ internal static BlobSyncUploadFromUriOptions ToSyncUploadFromUriOptions( DestinationConditions = CreateRequestConditions(options?.DestinationConditions, overwrite), SourceAuthentication = sourceAuthorization, }; + if ((options?.ContentEncoding?.Preserve ?? true) && + (options?.ContentDisposition?.Preserve ?? true) && + (options?.ContentLanguage?.Preserve ?? true) && + (options?.ContentType?.Preserve ?? true) && + (options?.CacheControl?.Preserve ?? true) && + (options?.AccessTier?.Preserve ?? true) && + (options?.Metadata?.Preserve ?? true)) + { + return uploadFromUriOptions; + } + // If all the properties are not being preserved, we need to clear them and manually + // set them from the source. We can't do it the other way around because the service + // does not clear the properties if you send an empty value. + uploadFromUriOptions.CopySourceBlobProperties = false; + uploadFromUriOptions.HttpHeaders = GetHttpHeaders(options, sourceProperties?.RawProperties); + uploadFromUriOptions.Metadata = GetMetadata(options, sourceProperties?.RawProperties); + return uploadFromUriOptions; } internal static StageBlockFromUriOptions ToBlobStageBlockFromUriOptions( @@ -405,7 +418,10 @@ internal static StageBlockFromUriOptions ToBlobStageBlockFromUriOptions( }; } - internal static CommitBlockListOptions ToCommitBlockOptions(this BlockBlobStorageResourceOptions options, bool overwrite) + internal static CommitBlockListOptions GetCommitBlockOptions( + BlockBlobStorageResourceOptions options, + bool overwrite, + StorageResourceItemProperties sourceProperties) { // There's a lot of conditions that cannot be applied to a StageBlock Request. // We need to omit them, but still apply them to other requests that do accept them. @@ -413,10 +429,9 @@ internal static CommitBlockListOptions ToCommitBlockOptions(this BlockBlobStorag // to see what headers are accepted. return new CommitBlockListOptions() { - HttpHeaders = options?.HttpHeaders, - Metadata = options?.Metadata, - Tags = options?.Tags, - AccessTier = options?.AccessTier, + HttpHeaders = GetHttpHeaders(options, sourceProperties?.RawProperties), + Metadata = GetMetadata(options, sourceProperties?.RawProperties), + AccessTier = GetAccessTier(options, sourceProperties?.RawProperties), Conditions = CreateRequestConditions(options?.DestinationConditions, overwrite) }; } @@ -441,16 +456,16 @@ internal static BlobDownloadOptions ToBlobDownloadOptions( return result; } - internal static PageBlobCreateOptions ToCreateOptions( - this PageBlobStorageResourceOptions options, - bool overwrite) + internal static PageBlobCreateOptions GetCreateOptions( + PageBlobStorageResourceOptions options, + bool overwrite, + StorageResourceItemProperties sourceProperties) { return new PageBlobCreateOptions() { SequenceNumber = options?.SequenceNumber, - HttpHeaders = options?.HttpHeaders, - Metadata = options?.Metadata, - Tags = options?.Tags, + HttpHeaders = GetHttpHeaders(options, sourceProperties?.RawProperties), + Metadata = GetMetadata(options, sourceProperties?.RawProperties), Conditions = new PageBlobRequestConditions() { IfMatch = options?.DestinationConditions?.IfMatch, @@ -516,8 +531,11 @@ internal static BlobStorageResourceOptions GetBlobResourceOptions( return new() { Metadata = checkpointData.Metadata, - Tags = checkpointData.Tags, - HttpHeaders = checkpointData.ContentHeaders, + CacheControl = checkpointData.CacheControl, + ContentDisposition = checkpointData.ContentDisposition, + ContentEncoding = checkpointData.ContentEncoding, + ContentLanguage = checkpointData.ContentLanguage, + ContentType = checkpointData.ContentType, AccessTier = checkpointData.AccessTier, }; } @@ -564,8 +582,11 @@ internal static BlobStorageResourceContainerOptions DeepCopy(this BlobStorageRes BlobOptions = new BlobStorageResourceOptions() { Metadata = options?.BlobOptions?.Metadata, - Tags = options?.BlobOptions?.Tags, - HttpHeaders = options?.BlobOptions?.HttpHeaders, + CacheControl = options?.BlobOptions?.CacheControl, + ContentEncoding = options?.BlobOptions?.ContentEncoding, + ContentDisposition = options?.BlobOptions?.ContentDisposition, + ContentLanguage = options?.BlobOptions?.ContentLanguage, + ContentType = options?.BlobOptions?.ContentType, AccessTier = options?.BlobOptions?.AccessTier, } }; @@ -577,10 +598,6 @@ internal static StorageResourceItemProperties ToResourceProperties(this BlobItem { properties.Add(DataMovementConstants.ResourceProperties.Metadata, blobItem.Metadata); } - if (blobItem.Tags != default) - { - properties.Add(DataMovementConstants.ResourceProperties.Tags, blobItem.Tags); - } if (blobItem.Properties.AccessTier.HasValue) { properties.Add(DataMovementConstants.ResourceProperties.AccessTier, blobItem.Properties.AccessTier.Value); @@ -597,10 +614,6 @@ internal static StorageResourceItemProperties ToResourceProperties(this BlobItem { properties.Add(DataMovementConstants.ResourceProperties.ContentType, blobItem.Properties.ContentType); } - if (blobItem.Properties.ContentHash != default) - { - properties.Add(DataMovementConstants.ResourceProperties.ContentHash, blobItem.Properties.ContentHash); - } if (blobItem.Properties.ContentEncoding != default) { properties.Add(DataMovementConstants.ResourceProperties.ContentEncoding, blobItem.Properties.ContentEncoding); @@ -624,5 +637,57 @@ internal static StorageResourceItemProperties ToResourceProperties(this BlobItem lastModifiedTime: blobItem.Properties.LastModified, properties: properties); } + + private static BlobHttpHeaders GetHttpHeaders( + BlobStorageResourceOptions options, + Dictionary properties) + => new() + { + ContentType = (options?.ContentType?.Preserve ?? true) + ? properties?.TryGetValue(DataMovementConstants.ResourceProperties.ContentType, out object contentType) == true + ? (string) contentType + : default + : options?.ContentType?.Value, + ContentEncoding = (options?.ContentEncoding?.Preserve ?? true) + ? properties?.TryGetValue(DataMovementConstants.ResourceProperties.ContentEncoding, out object contentEncoding) == true + ? (string) contentEncoding + : default + : options?.ContentEncoding?.Value, + ContentLanguage = (options?.ContentLanguage?.Preserve ?? true) + ? properties?.TryGetValue(DataMovementConstants.ResourceProperties.ContentLanguage, out object contentLanguage) == true + ? (string) contentLanguage + : default + : options?.ContentLanguage?.Value, + ContentDisposition = (options?.ContentDisposition?.Preserve ?? true) + ? properties?.TryGetValue(DataMovementConstants.ResourceProperties.ContentDisposition, out object contentDisposition) == true + ? (string) contentDisposition + : default + : options?.ContentDisposition?.Value, + CacheControl = (options?.CacheControl?.Preserve ?? true) + ? properties?.TryGetValue(DataMovementConstants.ResourceProperties.CacheControl, out object cacheControl) == true + ? (string) cacheControl + : default + : options?.CacheControl?.Value, + }; + + // By default we preserve the access tier + private static AccessTier? GetAccessTier( + BlobStorageResourceOptions options, + Dictionary properties) + => (options?.AccessTier?.Preserve ?? true) + ? properties?.TryGetValue(DataMovementConstants.ResourceProperties.AccessTier, out object accessTierObject) == true + ? (AccessTier?)accessTierObject + : default + : options?.AccessTier?.Value; + + // By default we preserve the metadata + private static Metadata GetMetadata( + BlobStorageResourceOptions options, + Dictionary properties) + => (options?.Metadata?.Preserve ?? true) + ? properties?.TryGetValue(DataMovementConstants.ResourceProperties.Metadata, out object metadataObject) == true + ? (Metadata) metadataObject + : default + : options?.Metadata?.Value; } } diff --git a/sdk/storage/Azure.Storage.DataMovement.Blobs/src/PageBlobStorageResource.cs b/sdk/storage/Azure.Storage.DataMovement.Blobs/src/PageBlobStorageResource.cs index 0c73af5b0dca1..e68edad774ac8 100644 --- a/sdk/storage/Azure.Storage.DataMovement.Blobs/src/PageBlobStorageResource.cs +++ b/sdk/storage/Azure.Storage.DataMovement.Blobs/src/PageBlobStorageResource.cs @@ -2,6 +2,7 @@ // Licensed under the MIT License. using System; +using System.Collections.Generic; using System.IO; using System.Threading; using System.Threading.Tasks; @@ -136,7 +137,10 @@ protected override async Task CopyFromStreamAsync( { await BlobClient.CreateAsync( size: completeLength, - options: _options.ToCreateOptions(overwrite), + options: DataMovementBlobsExtensions.GetCreateOptions( + _options, + overwrite, + options?.SourceProperties), cancellationToken: cancellationToken).ConfigureAwait(false); } if (streamLength > 0) @@ -175,7 +179,10 @@ protected override async Task CopyFromUriAsync( { await BlobClient.CreateAsync( size: completeLength, - options: _options.ToCreateOptions(overwrite), + options: DataMovementBlobsExtensions.GetCreateOptions( + _options, + overwrite, + options?.SourceProperties), cancellationToken: cancellationToken).ConfigureAwait(false); // There is no synchronous single-call copy API for Append/Page -> Page Blob @@ -223,7 +230,10 @@ protected override async Task CopyBlockFromUriAsync( { await BlobClient.CreateAsync( size: completeLength, - _options.ToCreateOptions(overwrite), + DataMovementBlobsExtensions.GetCreateOptions( + _options, + overwrite, + options?.SourceProperties), cancellationToken).ConfigureAwait(false); } @@ -279,7 +289,10 @@ protected override async Task GetCopyAuthorizationHeaderAsync /// /// Commits the block list given. /// - protected override Task CompleteTransferAsync(bool overwrite, CancellationToken cancellationToken = default) + protected override Task CompleteTransferAsync( + bool overwrite, + StorageResourceCompleteTransferOptions completeTransferOptions = default, + CancellationToken cancellationToken = default) { // no-op for now return Task.CompletedTask; @@ -309,11 +322,15 @@ protected override StorageResourceCheckpointData GetSourceCheckpointData() protected override StorageResourceCheckpointData GetDestinationCheckpointData() { return new BlobDestinationCheckpointData( - BlobType.Page, - _options?.HttpHeaders, - _options?.AccessTier, - _options?.Metadata, - _options?.Tags); + blobType: BlobType.Page, + contentType: _options?.ContentType, + contentEncoding: _options?.ContentEncoding, + contentLanguage: _options?.ContentLanguage, + contentDisposition: _options?.ContentDisposition, + cacheControl: _options?.CacheControl, + accessTier: _options?.AccessTier, + metadata: _options?.Metadata, + tags: default); } } } diff --git a/sdk/storage/Azure.Storage.DataMovement.Blobs/tests/Azure.Storage.DataMovement.Blobs.Tests.csproj b/sdk/storage/Azure.Storage.DataMovement.Blobs/tests/Azure.Storage.DataMovement.Blobs.Tests.csproj index ef14ea316d2bc..970e39bef60bc 100644 --- a/sdk/storage/Azure.Storage.DataMovement.Blobs/tests/Azure.Storage.DataMovement.Blobs.Tests.csproj +++ b/sdk/storage/Azure.Storage.DataMovement.Blobs/tests/Azure.Storage.DataMovement.Blobs.Tests.csproj @@ -13,6 +13,7 @@ + diff --git a/sdk/storage/Azure.Storage.DataMovement.Blobs/tests/BlobDestinationCheckpointDataTests.cs b/sdk/storage/Azure.Storage.DataMovement.Blobs/tests/BlobDestinationCheckpointDataTests.cs index d4997572d9999..96b03b2b7f4cd 100644 --- a/sdk/storage/Azure.Storage.DataMovement.Blobs/tests/BlobDestinationCheckpointDataTests.cs +++ b/sdk/storage/Azure.Storage.DataMovement.Blobs/tests/BlobDestinationCheckpointDataTests.cs @@ -10,6 +10,10 @@ using System.IO; using Metadata = System.Collections.Generic.IDictionary; using Tags = System.Collections.Generic.IDictionary; +using System.Text; +using System.Linq; +using System.Collections.Generic; +using System; namespace Azure.Storage.DataMovement.Blobs.Tests { @@ -21,58 +25,161 @@ public class BlobDestinationCheckpointDataTests private const string DefaultContentLanguage = "en-US"; private const string DefaultContentDisposition = "inline"; private const string DefaultCacheControl = "no-cache"; - private readonly Metadata DefaultMetadata = DataProvider.BuildMetadata(); - private readonly Tags DefaultTags = DataProvider.BuildTags(); + private AccessTier DefaultAccessTier = AccessTier.Hot; + private readonly DataTransferProperty DefaultMetadata = new(DataProvider.BuildMetadata()); + private readonly DataTransferProperty DefaultTags = new(DataProvider.BuildTags()); - private BlobDestinationCheckpointData CreateDefault() + private static byte[] StringToByteArray(string value) => Encoding.UTF8.GetBytes(value); + + private BlobDestinationCheckpointData CreatePreserveValues() { return new BlobDestinationCheckpointData( DefaultBlobType, - new BlobHttpHeaders() - { - ContentType = DefaultContentType, - ContentEncoding = DefaultContentEncoding, - ContentLanguage = DefaultContentLanguage, - ContentDisposition = DefaultContentDisposition, - CacheControl = DefaultCacheControl, - }, - AccessTier.Hot, - DefaultMetadata, - DefaultTags); + default, + default, + default, + default, + default, + default, + default, + default); + } + + private BlobDestinationCheckpointData CreateSetSampleValues() + { + return new BlobDestinationCheckpointData( + blobType: DefaultBlobType, + contentType: new(DefaultContentType), + contentEncoding: new(DefaultContentEncoding), + contentLanguage: new(DefaultContentLanguage), + contentDisposition: new(DefaultContentDisposition), + cacheControl: new(DefaultCacheControl), + accessTier: new(DefaultAccessTier), + metadata: DefaultMetadata, + tags: DefaultTags); + } + + private void TestAssertSerializedData(BlobDestinationCheckpointData data) + { + string samplePath = Path.Combine("Resources", "BlobDestinationCheckpointData.2.bin"); + using (MemoryStream dataStream = new MemoryStream(DataMovementBlobConstants.DestinationCheckpointData.VariableLengthStartIndex)) + using (FileStream fileStream = File.OpenRead(samplePath)) + { + data.Serialize(dataStream); + + BinaryReader reader = new(fileStream); + byte[] expected = reader.ReadBytes((int)fileStream.Length); + byte[] actual = dataStream.ToArray(); + + CollectionAssert.AreEqual(expected, actual); + } } [Test] public void Ctor() { - BlobDestinationCheckpointData data = CreateDefault(); + BlobDestinationCheckpointData data = CreatePreserveValues(); Assert.AreEqual(DataMovementBlobConstants.DestinationCheckpointData.SchemaVersion, data.Version); Assert.AreEqual(DefaultBlobType, data.BlobType); - Assert.AreEqual(DefaultContentType, data.ContentHeaders.ContentType); - Assert.AreEqual(DefaultContentEncoding, data.ContentHeaders.ContentEncoding); - Assert.AreEqual(DefaultContentLanguage, data.ContentHeaders.ContentLanguage); - Assert.AreEqual(DefaultContentDisposition, data.ContentHeaders.ContentDisposition); - Assert.AreEqual(DefaultCacheControl, data.ContentHeaders.CacheControl); - Assert.AreEqual(AccessTier.Hot, data.AccessTier); - CollectionAssert.AreEquivalent(DefaultMetadata, data.Metadata); - CollectionAssert.AreEquivalent(DefaultTags, data.Tags); + Assert.AreEqual(true, data.PreserveContentType); + Assert.IsEmpty(data.ContentTypeBytes); + Assert.AreEqual(true, data.PreserveContentEncoding); + Assert.IsEmpty(data.ContentEncodingBytes); + Assert.AreEqual(true, data.PreserveContentLanguage); + Assert.IsEmpty(data.ContentLanguageBytes); + Assert.AreEqual(true, data.PreserveContentDisposition); + Assert.IsEmpty(data.ContentDispositionBytes); + Assert.AreEqual(true, data.PreserveCacheControl); + Assert.IsEmpty(data.CacheControlBytes); + Assert.AreEqual(true, data.PreserveAccessTier); + Assert.IsNull(data.AccessTier); + Assert.AreEqual(true, data.PreserveMetadata); + Assert.IsNull(data.Metadata); + Assert.AreEqual(false, data.PreserveTags); + Assert.IsNull(data.Tags); + } + + [Test] + public void Ctor_SetValues() + { + BlobDestinationCheckpointData data = CreateSetSampleValues(); + + VerifySampleValues(data, DataMovementBlobConstants.DestinationCheckpointData.SchemaVersion); } [Test] public void Serialize() { - BlobDestinationCheckpointData data = CreateDefault(); + BlobDestinationCheckpointData data = CreateSetSampleValues(); + TestAssertSerializedData(data); + } + + [Test] + public void Serialize_NoPreserveTags() + { + BlobDestinationCheckpointData data = CreateSetSampleValues(); + data.Tags = default; + data.PreserveTags = true; + data.TagsBytes = default; - string samplePath = Path.Combine("Resources", "BlobDestinationCheckpointData.1.bin"); + string samplePath = Path.Combine("Resources", "BlobDestinationCheckpointData.2.bin"); using (MemoryStream dataStream = new MemoryStream(DataMovementBlobConstants.DestinationCheckpointData.VariableLengthStartIndex)) using (FileStream fileStream = File.OpenRead(samplePath)) { + // Act data.Serialize(dataStream); BinaryReader reader = new(fileStream); - byte[] expected = reader.ReadBytes((int)fileStream.Length); + List expected = reader.ReadBytes((int)fileStream.Length).ToList(); + // Change to expected Preserve Tags value - true + expected[DataMovementBlobConstants.DestinationCheckpointData.PreserveTagsIndex] = 1; + int tagsOffset = expected[DataMovementBlobConstants.DestinationCheckpointData.TagsOffsetIndex]; + int tagsLength = expected[DataMovementBlobConstants.DestinationCheckpointData.TagsLengthIndex]; + expected[DataMovementBlobConstants.DestinationCheckpointData.TagsOffsetIndex] = 255; + expected[DataMovementBlobConstants.DestinationCheckpointData.TagsOffsetIndex+1] = 255; + expected[DataMovementBlobConstants.DestinationCheckpointData.TagsOffsetIndex+2] = 255; + expected[DataMovementBlobConstants.DestinationCheckpointData.TagsOffsetIndex+3] = 255; + expected[DataMovementBlobConstants.DestinationCheckpointData.TagsLengthIndex] = 255; + expected[DataMovementBlobConstants.DestinationCheckpointData.TagsLengthIndex+1] = 255; + expected[DataMovementBlobConstants.DestinationCheckpointData.TagsLengthIndex+2] = 255; + expected[DataMovementBlobConstants.DestinationCheckpointData.TagsLengthIndex+3] = 255; + // Remove Tags + expected.RemoveRange(tagsOffset, tagsLength); + + // Get serialized data + byte[] actual = dataStream.ToArray(); + + // Verify + CollectionAssert.AreEqual(expected, actual); + } + } + + [Test] + public void Serialize_PreserveAccessTier() + { + // Arrange + BlobDestinationCheckpointData data = CreateSetSampleValues(); + data.PreserveAccessTier = true; + data.AccessTier = default; + + string samplePath = Path.Combine("Resources", "BlobDestinationCheckpointData.2.bin"); + using (MemoryStream dataStream = new MemoryStream(DataMovementBlobConstants.DestinationCheckpointData.VariableLengthStartIndex)) + using (FileStream fileStream = File.OpenRead(samplePath)) + { + // Act + data.Serialize(dataStream); + + BinaryReader reader = new(fileStream); + List expected = reader.ReadBytes((int)fileStream.Length).ToList(); + // Change to expected AccessTier value - true + expected[DataMovementBlobConstants.DestinationCheckpointData.PreserveAccessTierIndex] = 1; + expected[DataMovementBlobConstants.DestinationCheckpointData.AccessTierValueIndex] = 0; + + // Get serialized data byte[] actual = dataStream.ToArray(); + // Verify CollectionAssert.AreEqual(expected, actual); } } @@ -80,41 +187,64 @@ public void Serialize() [Test] public void Deserialize() { - BlobDestinationCheckpointData data = CreateDefault(); + BlobDestinationCheckpointData data = CreateSetSampleValues(); using (Stream stream = new MemoryStream(DataMovementBlobConstants.DestinationCheckpointData.VariableLengthStartIndex)) { data.Serialize(stream); stream.Position = 0; - DeserializeAndVerify(stream, DataMovementBlobConstants.DestinationCheckpointData.SchemaVersion); + BlobDestinationCheckpointData deserialized = BlobDestinationCheckpointData.Deserialize(stream); + VerifySampleValues(deserialized, DataMovementBlobConstants.DestinationCheckpointData.SchemaVersion); } } [Test] - public void Deserialize_File_Version_1() + public void Deserialize_File_Version_2() { - string samplePath = Path.Combine("Resources", "BlobDestinationCheckpointData.1.bin"); + string samplePath = Path.Combine("Resources", "BlobDestinationCheckpointData.2.bin"); using (FileStream stream = File.OpenRead(samplePath)) { stream.Position = 0; - DeserializeAndVerify(stream, 1); + BlobDestinationCheckpointData deserialized = BlobDestinationCheckpointData.Deserialize(stream); + VerifySampleValues(deserialized, 2); } } - private void DeserializeAndVerify(Stream stream, int version) + private void VerifySampleValues(BlobDestinationCheckpointData data, int version) { - BlobDestinationCheckpointData deserialized = BlobDestinationCheckpointData.Deserialize(stream); - - Assert.AreEqual(version, deserialized.Version); - Assert.AreEqual(DefaultBlobType, deserialized.BlobType); - Assert.AreEqual(DefaultContentType, deserialized.ContentHeaders.ContentType); - Assert.AreEqual(DefaultContentEncoding, deserialized.ContentHeaders.ContentEncoding); - Assert.AreEqual(DefaultContentLanguage, deserialized.ContentHeaders.ContentLanguage); - Assert.AreEqual(DefaultContentDisposition, deserialized.ContentHeaders.ContentDisposition); - Assert.AreEqual(DefaultCacheControl, deserialized.ContentHeaders.CacheControl); - Assert.AreEqual(AccessTier.Hot, deserialized.AccessTier); - CollectionAssert.AreEquivalent(DefaultMetadata, deserialized.Metadata); - CollectionAssert.AreEquivalent(DefaultTags, deserialized.Tags); + Assert.AreEqual(version, data.Version); + Assert.AreEqual(DefaultBlobType, data.BlobType); + Assert.AreEqual(false, data.PreserveContentType); + Assert.AreEqual(StringToByteArray(DefaultContentType), data.ContentTypeBytes); + Assert.AreEqual(false, data.PreserveContentEncoding); + Assert.AreEqual(StringToByteArray(DefaultContentEncoding), data.ContentEncodingBytes); + Assert.AreEqual(false, data.PreserveContentLanguage); + Assert.AreEqual(StringToByteArray(DefaultContentLanguage), data.ContentLanguageBytes); + Assert.AreEqual(false, data.PreserveContentDisposition); + Assert.AreEqual(StringToByteArray(DefaultContentDisposition), data.ContentDispositionBytes); + Assert.AreEqual(false, data.PreserveCacheControl); + Assert.AreEqual(StringToByteArray(DefaultCacheControl), data.CacheControlBytes); + Assert.AreEqual(false, data.PreserveAccessTier); + Assert.AreEqual(DefaultAccessTier, data.AccessTier.Value); + Assert.AreEqual(false, data.PreserveMetadata); + CollectionAssert.AreEquivalent(DefaultMetadata.Value, data.Metadata.Value); + Assert.AreEqual(false, data.PreserveTags); + CollectionAssert.AreEquivalent(DefaultTags.Value, data.Tags.Value); + } + + [Test] + public void Deserialize_IncorrectSchemaVersion() + { + int incorrectSchemaVersion = 1; + BlobDestinationCheckpointData data = CreatePreserveValues(); + data.Version = incorrectSchemaVersion; + + using MemoryStream dataStream = new MemoryStream(DataMovementBlobConstants.DestinationCheckpointData.VariableLengthStartIndex); + data.Serialize(dataStream); + dataStream.Position = 0; + TestHelper.AssertExpectedException( + () => BlobDestinationCheckpointData.Deserialize(dataStream), + new ArgumentException($"The checkpoint file schema version {incorrectSchemaVersion} is not supported by this version of the SDK.")); } } } diff --git a/sdk/storage/Azure.Storage.DataMovement.Blobs/tests/RehydrateBlobResourceTests.cs b/sdk/storage/Azure.Storage.DataMovement.Blobs/tests/RehydrateBlobResourceTests.cs index 814aae3ef2574..6abf88dc2f731 100644 --- a/sdk/storage/Azure.Storage.DataMovement.Blobs/tests/RehydrateBlobResourceTests.cs +++ b/sdk/storage/Azure.Storage.DataMovement.Blobs/tests/RehydrateBlobResourceTests.cs @@ -17,6 +17,11 @@ namespace Azure.Storage.DataMovement.Tests { public class RehydrateBlobResourceTests { + private const string DefaultContentType = "text/plain"; + private const string DefaultContentEncoding = "gzip"; + private const string DefaultContentLanguage = "en-US"; + private const string DefaultContentDisposition = "inline"; + private const string DefaultCacheControl = "no-cache"; public RehydrateBlobResourceTests() { } @@ -47,27 +52,28 @@ private static BlobSourceCheckpointData GetSourceCheckpointData(BlobType blobTyp private static BlobDestinationCheckpointData GetPopulatedDestinationCheckpointData( BlobType blobType, AccessTier? accessTier = default) - { - BlobHttpHeaders headers = new() - { - ContentType = "text/plain", - ContentEncoding = "gzip", - ContentLanguage = "en-US", - ContentDisposition = "inline", - CacheControl = "no-cache", - }; - return new BlobDestinationCheckpointData( - blobType, - headers, - accessTier, - DataProvider.BuildMetadata(), - DataProvider.BuildTags()); - } + => new BlobDestinationCheckpointData( + blobType: blobType, + contentType: new(DefaultContentType), + contentEncoding: new(DefaultContentEncoding), + contentLanguage: new(DefaultContentLanguage), + contentDisposition: new(DefaultContentDisposition), + cacheControl: new(DefaultCacheControl), + accessTier: new(accessTier), + metadata: new(DataProvider.BuildMetadata()), + tags: new(DataProvider.BuildTags())); private static BlobDestinationCheckpointData GetDefaultDestinationCheckpointData(BlobType blobType) - { - return new BlobDestinationCheckpointData(blobType, default, default, default, default); - } + => new BlobDestinationCheckpointData( + blobType, + default, + default, + default, + default, + default, + default, + default, + default); private static byte[] GetBytes(BlobCheckpointData checkpointData) { @@ -155,14 +161,20 @@ public async Task RehydrateBlockBlob_Options() .FromDestinationInternalHookAsync(transferProperties); Assert.AreEqual(destinationPath, storageResource.Uri.AbsoluteUri); - Assert.AreEqual(checkpointData.AccessTier, storageResource._options.AccessTier); - Assert.AreEqual(checkpointData.Metadata, storageResource._options.Metadata); - Assert.AreEqual(checkpointData.Tags, storageResource._options.Tags); - Assert.AreEqual(checkpointData.ContentHeaders.ContentType, storageResource._options.HttpHeaders.ContentType); - Assert.AreEqual(checkpointData.ContentHeaders.ContentEncoding, storageResource._options.HttpHeaders.ContentEncoding); - Assert.AreEqual(checkpointData.ContentHeaders.ContentLanguage, storageResource._options.HttpHeaders.ContentLanguage); - Assert.AreEqual(checkpointData.ContentHeaders.ContentDisposition, storageResource._options.HttpHeaders.ContentDisposition); - Assert.AreEqual(checkpointData.ContentHeaders.CacheControl, storageResource._options.HttpHeaders.CacheControl); + Assert.AreEqual(checkpointData.AccessTier.Preserve, storageResource._options.AccessTier.Preserve); + Assert.AreEqual(checkpointData.AccessTier.Value, storageResource._options.AccessTier.Value); + Assert.AreEqual(checkpointData.Metadata.Preserve, storageResource._options.Metadata.Preserve); + Assert.AreEqual(checkpointData.Metadata.Value, storageResource._options.Metadata.Value); + Assert.AreEqual(checkpointData.CacheControl.Preserve, storageResource._options.CacheControl.Preserve); + Assert.AreEqual(checkpointData.CacheControl.Value, storageResource._options.CacheControl.Value); + Assert.AreEqual(checkpointData.ContentDisposition.Preserve, storageResource._options.ContentDisposition.Preserve); + Assert.AreEqual(checkpointData.ContentDisposition.Value, storageResource._options.ContentDisposition.Value); + Assert.AreEqual(checkpointData.ContentEncoding.Preserve, storageResource._options.ContentEncoding.Preserve); + Assert.AreEqual(checkpointData.ContentEncoding.Value, storageResource._options.ContentEncoding.Value); + Assert.AreEqual(checkpointData.ContentLanguage.Preserve, storageResource._options.ContentLanguage.Preserve); + Assert.AreEqual(checkpointData.ContentLanguage.Value, storageResource._options.ContentLanguage.Value); + Assert.AreEqual(checkpointData.ContentType.Preserve, storageResource._options.ContentType.Preserve); + Assert.AreEqual(checkpointData.ContentType.Value, storageResource._options.ContentType.Value); } [Test] @@ -220,14 +232,18 @@ public async Task RehydratePageBlob_Options() .FromDestinationInternalHookAsync(transferProperties); Assert.AreEqual(destinationPath, storageResource.Uri.AbsoluteUri); - Assert.AreEqual(checkpointData.AccessTier, storageResource._options.AccessTier); - Assert.AreEqual(checkpointData.Metadata, storageResource._options.Metadata); - Assert.AreEqual(checkpointData.Tags, storageResource._options.Tags); - Assert.AreEqual(checkpointData.ContentHeaders.ContentType, storageResource._options.HttpHeaders.ContentType); - Assert.AreEqual(checkpointData.ContentHeaders.ContentEncoding, storageResource._options.HttpHeaders.ContentEncoding); - Assert.AreEqual(checkpointData.ContentHeaders.ContentLanguage, storageResource._options.HttpHeaders.ContentLanguage); - Assert.AreEqual(checkpointData.ContentHeaders.ContentDisposition, storageResource._options.HttpHeaders.ContentDisposition); - Assert.AreEqual(checkpointData.ContentHeaders.CacheControl, storageResource._options.HttpHeaders.CacheControl); + Assert.AreEqual(checkpointData.Metadata.Preserve, storageResource._options.Metadata.Preserve); + Assert.AreEqual(checkpointData.Metadata.Value, storageResource._options.Metadata.Value); + Assert.AreEqual(checkpointData.CacheControl.Preserve, storageResource._options.CacheControl.Preserve); + Assert.AreEqual(checkpointData.CacheControl.Value, storageResource._options.CacheControl.Value); + Assert.AreEqual(checkpointData.ContentDisposition.Preserve, storageResource._options.ContentDisposition.Preserve); + Assert.AreEqual(checkpointData.ContentDisposition.Value, storageResource._options.ContentDisposition.Value); + Assert.AreEqual(checkpointData.ContentEncoding.Preserve, storageResource._options.ContentEncoding.Preserve); + Assert.AreEqual(checkpointData.ContentEncoding.Value, storageResource._options.ContentEncoding.Value); + Assert.AreEqual(checkpointData.ContentLanguage.Preserve, storageResource._options.ContentLanguage.Preserve); + Assert.AreEqual(checkpointData.ContentLanguage.Value, storageResource._options.ContentLanguage.Value); + Assert.AreEqual(checkpointData.ContentType.Preserve, storageResource._options.ContentType.Preserve); + Assert.AreEqual(checkpointData.ContentType.Value, storageResource._options.ContentType.Value); } [Test] @@ -285,14 +301,18 @@ public async Task RehydrateAppendBlob_Options() .FromDestinationInternalHookAsync(transferProperties); Assert.AreEqual(destinationPath, storageResource.Uri.AbsoluteUri); - Assert.AreEqual(checkpointData.AccessTier, storageResource._options.AccessTier); - Assert.AreEqual(checkpointData.Metadata, storageResource._options.Metadata); - Assert.AreEqual(checkpointData.Tags, storageResource._options.Tags); - Assert.AreEqual(checkpointData.ContentHeaders.ContentType, storageResource._options.HttpHeaders.ContentType); - Assert.AreEqual(checkpointData.ContentHeaders.ContentEncoding, storageResource._options.HttpHeaders.ContentEncoding); - Assert.AreEqual(checkpointData.ContentHeaders.ContentLanguage, storageResource._options.HttpHeaders.ContentLanguage); - Assert.AreEqual(checkpointData.ContentHeaders.ContentDisposition, storageResource._options.HttpHeaders.ContentDisposition); - Assert.AreEqual(checkpointData.ContentHeaders.CacheControl, storageResource._options.HttpHeaders.CacheControl); + Assert.AreEqual(checkpointData.Metadata.Preserve, storageResource._options.Metadata.Preserve); + Assert.AreEqual(checkpointData.Metadata.Value, storageResource._options.Metadata.Value); + Assert.AreEqual(checkpointData.CacheControl.Preserve, storageResource._options.CacheControl.Preserve); + Assert.AreEqual(checkpointData.CacheControl.Value, storageResource._options.CacheControl.Value); + Assert.AreEqual(checkpointData.ContentDisposition.Preserve, storageResource._options.ContentDisposition.Preserve); + Assert.AreEqual(checkpointData.ContentDisposition.Value, storageResource._options.ContentDisposition.Value); + Assert.AreEqual(checkpointData.ContentEncoding.Preserve, storageResource._options.ContentEncoding.Preserve); + Assert.AreEqual(checkpointData.ContentEncoding.Value, storageResource._options.ContentEncoding.Value); + Assert.AreEqual(checkpointData.ContentLanguage.Preserve, storageResource._options.ContentLanguage.Preserve); + Assert.AreEqual(checkpointData.ContentLanguage.Value, storageResource._options.ContentLanguage.Value); + Assert.AreEqual(checkpointData.ContentType.Preserve, storageResource._options.ContentType.Preserve); + Assert.AreEqual(checkpointData.ContentType.Value, storageResource._options.ContentType.Value); } [Test] diff --git a/sdk/storage/Azure.Storage.DataMovement.Blobs/tests/Resources/BlobDestinationCheckpointData.1.bin b/sdk/storage/Azure.Storage.DataMovement.Blobs/tests/Resources/BlobDestinationCheckpointData.1.bin deleted file mode 100644 index 7fd46a7780352739486940bd835518ad7db9ae6a..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 175 zcmX}g!3x4K5Cl*QD&lu|wAh|Qyoey^p(@p*tm#?M!`y&Osb_GcZ{XnZp=X z(1#5SU<*Sy!U(bxY6>5ifbvskDUg@;?GsC0@4W;sEHQWFu6&8no~bi8RMPckoFiaNoQ#2ULoZOe?HzJ@Ps0%G7xeljd%)@&wW z*hpf}#u7(1kvOrb#Gr7z7m^<~XM|5$<^Vprx6c^McIRX85o0PX*bc8L=`-}|h6MTo m1ZoA@69gTQ2z%8U(`>bcJ1PRV!B?gE9w0oh`hDvEr&eD@%q;c* literal 0 HcmV?d00001 diff --git a/sdk/storage/Azure.Storage.DataMovement.Files.Shares/assets.json b/sdk/storage/Azure.Storage.DataMovement.Files.Shares/assets.json index 09b1b803612b3..2e94302fb0d1f 100644 --- a/sdk/storage/Azure.Storage.DataMovement.Files.Shares/assets.json +++ b/sdk/storage/Azure.Storage.DataMovement.Files.Shares/assets.json @@ -2,5 +2,5 @@ "AssetsRepo": "Azure/azure-sdk-assets", "AssetsRepoPrefixPath": "net", "TagPrefix": "net/storage/Azure.Storage.DataMovement.Files.Shares", - "Tag": "net/storage/Azure.Storage.DataMovement.Files.Shares_29fb6ca2eb" + "Tag": "net/storage/Azure.Storage.DataMovement.Files.Shares_eb32f81ce1" } diff --git a/sdk/storage/Azure.Storage.DataMovement.Files.Shares/src/DataMovementShareConstants.cs b/sdk/storage/Azure.Storage.DataMovement.Files.Shares/src/DataMovementShareConstants.cs index 3db3ef64c8640..6474b10923ebd 100644 --- a/sdk/storage/Azure.Storage.DataMovement.Files.Shares/src/DataMovementShareConstants.cs +++ b/sdk/storage/Azure.Storage.DataMovement.Files.Shares/src/DataMovementShareConstants.cs @@ -28,9 +28,9 @@ internal class DestinationCheckpointData private const int FileAttributesEncodedSize = OneByte + IntSizeInBytes; private const int FilePermissionKeyOffsetEncodedSize = IntSizeInBytes; private const int FilePermissionKeyLengthEncodedSize = IntSizeInBytes; - private const int FileCreatedOnEncodedSize = OneByte + LongSizeInBytes + LongSizeInBytes; - private const int FileLastWrittenOnEncodedSize = OneByte + LongSizeInBytes + LongSizeInBytes; - private const int FileChangedOnEncodedSize = OneByte + LongSizeInBytes + LongSizeInBytes; + private const int FileCreatedOnEncodedSize = OneByte + LongSizeInBytes; + private const int FileLastWrittenOnEncodedSize = OneByte + LongSizeInBytes; + private const int FileChangedOnEncodedSize = OneByte + LongSizeInBytes; private const int ContentTypeOffsetEncodedSize = IntSizeInBytes; private const int ContentTypeLengthEncodedSize = IntSizeInBytes; private const int ContentEncodingOffsetEncodedSize = IntSizeInBytes; diff --git a/sdk/storage/Azure.Storage.DataMovement.Files.Shares/src/ShareFileStorageResource.cs b/sdk/storage/Azure.Storage.DataMovement.Files.Shares/src/ShareFileStorageResource.cs index be06538fa0b83..b4b8ad9e2edee 100644 --- a/sdk/storage/Azure.Storage.DataMovement.Files.Shares/src/ShareFileStorageResource.cs +++ b/sdk/storage/Azure.Storage.DataMovement.Files.Shares/src/ShareFileStorageResource.cs @@ -3,7 +3,6 @@ using System; using System.IO; -using System.Text; using System.Threading; using System.Threading.Tasks; using Azure.Core; @@ -82,6 +81,7 @@ await ShareFileClient.CreateAsync( protected override Task CompleteTransferAsync( bool overwrite, + StorageResourceCompleteTransferOptions completeTransferOptions, CancellationToken cancellationToken = default) { CancellationHelper.ThrowIfCancellationRequested(cancellationToken); diff --git a/sdk/storage/Azure.Storage.DataMovement.Files.Shares/tests/ShareFileResourceTests.cs b/sdk/storage/Azure.Storage.DataMovement.Files.Shares/tests/ShareFileResourceTests.cs index 9960bf7930976..1a4cf9345832e 100644 --- a/sdk/storage/Azure.Storage.DataMovement.Files.Shares/tests/ShareFileResourceTests.cs +++ b/sdk/storage/Azure.Storage.DataMovement.Files.Shares/tests/ShareFileResourceTests.cs @@ -2,7 +2,6 @@ // Licensed under the MIT License. using System; -using System.Collections; using System.Collections.Generic; using System.IO; using System.Linq; @@ -14,7 +13,6 @@ using Azure.Storage.Files.Shares.Models; using Azure.Storage.Test; using Moq; -using Moq.Protected; using NUnit.Framework; namespace Azure.Storage.DataMovement.Files.Shares.Tests diff --git a/sdk/storage/Azure.Storage.DataMovement/api/Azure.Storage.DataMovement.net6.0.cs b/sdk/storage/Azure.Storage.DataMovement/api/Azure.Storage.DataMovement.net6.0.cs index 0094a090c9bc6..eaa19828a649c 100644 --- a/sdk/storage/Azure.Storage.DataMovement/api/Azure.Storage.DataMovement.net6.0.cs +++ b/sdk/storage/Azure.Storage.DataMovement/api/Azure.Storage.DataMovement.net6.0.cs @@ -71,6 +71,24 @@ protected internal DataTransferProperties() { } public virtual System.Uri SourceUri { get { throw null; } } public virtual string TransferId { get { throw null; } } } + public abstract partial class DataTransferProperty + { + public DataTransferProperty() { } + public DataTransferProperty(bool preserve) { } + public virtual bool Preserve { get { throw null; } } + [System.ComponentModel.EditorBrowsableAttribute(System.ComponentModel.EditorBrowsableState.Never)] + public override bool Equals(object? obj) { throw null; } + [System.ComponentModel.EditorBrowsableAttribute(System.ComponentModel.EditorBrowsableState.Never)] + public override int GetHashCode() { throw null; } + [System.ComponentModel.EditorBrowsableAttribute(System.ComponentModel.EditorBrowsableState.Never)] + public override string? ToString() { throw null; } + } + public partial class DataTransferProperty : Azure.Storage.DataMovement.DataTransferProperty where T : notnull + { + public DataTransferProperty(bool preserve) { } + public DataTransferProperty(T value) { } + public virtual T? Value { get { throw null; } } + } public enum DataTransferState { None = 0, @@ -125,6 +143,11 @@ protected StorageResourceCheckpointData() { } public abstract int Length { get; } protected internal abstract void Serialize(System.IO.Stream stream); } + public partial class StorageResourceCompleteTransferOptions + { + public StorageResourceCompleteTransferOptions() { } + public Azure.Storage.DataMovement.StorageResourceItemProperties SourceProperties { get { throw null; } set { } } + } public abstract partial class StorageResourceContainer : Azure.Storage.DataMovement.StorageResource { protected StorageResourceContainer() { } @@ -139,6 +162,7 @@ public partial class StorageResourceCopyFromUriOptions public StorageResourceCopyFromUriOptions() { } public string BlockId { get { throw null; } } public Azure.HttpAuthorization SourceAuthentication { get { throw null; } set { } } + public Azure.Storage.DataMovement.StorageResourceItemProperties SourceProperties { get { throw null; } set { } } } public enum StorageResourceCreationPreference { @@ -156,7 +180,7 @@ protected StorageResourceItem() { } protected internal abstract string ResourceId { get; } protected Azure.Storage.DataMovement.StorageResourceItemProperties ResourceProperties { get { throw null; } set { } } protected internal abstract Azure.Storage.DataMovement.DataTransferOrder TransferType { get; } - protected internal abstract System.Threading.Tasks.Task CompleteTransferAsync(bool overwrite, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)); + protected internal abstract System.Threading.Tasks.Task CompleteTransferAsync(bool overwrite, Azure.Storage.DataMovement.StorageResourceCompleteTransferOptions completeTransferOptions = null, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)); protected internal abstract System.Threading.Tasks.Task CopyBlockFromUriAsync(Azure.Storage.DataMovement.StorageResourceItem sourceResource, Azure.HttpRange range, bool overwrite, long completeLength, Azure.Storage.DataMovement.StorageResourceCopyFromUriOptions options = null, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)); protected internal abstract System.Threading.Tasks.Task CopyFromStreamAsync(System.IO.Stream stream, long streamLength, bool overwrite, long completeLength, Azure.Storage.DataMovement.StorageResourceWriteToOffsetOptions options = null, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)); protected internal abstract System.Threading.Tasks.Task CopyFromUriAsync(Azure.Storage.DataMovement.StorageResourceItem sourceResource, bool overwrite, long completeLength, Azure.Storage.DataMovement.StorageResourceCopyFromUriOptions options = null, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)); @@ -194,6 +218,7 @@ public partial class StorageResourceWriteToOffsetOptions public StorageResourceWriteToOffsetOptions() { } public string BlockId { get { throw null; } set { } } public long? Position { get { throw null; } set { } } + public Azure.Storage.DataMovement.StorageResourceItemProperties SourceProperties { get { throw null; } set { } } } public partial class TransferCheckpointStoreOptions { diff --git a/sdk/storage/Azure.Storage.DataMovement/api/Azure.Storage.DataMovement.netstandard2.0.cs b/sdk/storage/Azure.Storage.DataMovement/api/Azure.Storage.DataMovement.netstandard2.0.cs index 0094a090c9bc6..eaa19828a649c 100644 --- a/sdk/storage/Azure.Storage.DataMovement/api/Azure.Storage.DataMovement.netstandard2.0.cs +++ b/sdk/storage/Azure.Storage.DataMovement/api/Azure.Storage.DataMovement.netstandard2.0.cs @@ -71,6 +71,24 @@ protected internal DataTransferProperties() { } public virtual System.Uri SourceUri { get { throw null; } } public virtual string TransferId { get { throw null; } } } + public abstract partial class DataTransferProperty + { + public DataTransferProperty() { } + public DataTransferProperty(bool preserve) { } + public virtual bool Preserve { get { throw null; } } + [System.ComponentModel.EditorBrowsableAttribute(System.ComponentModel.EditorBrowsableState.Never)] + public override bool Equals(object? obj) { throw null; } + [System.ComponentModel.EditorBrowsableAttribute(System.ComponentModel.EditorBrowsableState.Never)] + public override int GetHashCode() { throw null; } + [System.ComponentModel.EditorBrowsableAttribute(System.ComponentModel.EditorBrowsableState.Never)] + public override string? ToString() { throw null; } + } + public partial class DataTransferProperty : Azure.Storage.DataMovement.DataTransferProperty where T : notnull + { + public DataTransferProperty(bool preserve) { } + public DataTransferProperty(T value) { } + public virtual T? Value { get { throw null; } } + } public enum DataTransferState { None = 0, @@ -125,6 +143,11 @@ protected StorageResourceCheckpointData() { } public abstract int Length { get; } protected internal abstract void Serialize(System.IO.Stream stream); } + public partial class StorageResourceCompleteTransferOptions + { + public StorageResourceCompleteTransferOptions() { } + public Azure.Storage.DataMovement.StorageResourceItemProperties SourceProperties { get { throw null; } set { } } + } public abstract partial class StorageResourceContainer : Azure.Storage.DataMovement.StorageResource { protected StorageResourceContainer() { } @@ -139,6 +162,7 @@ public partial class StorageResourceCopyFromUriOptions public StorageResourceCopyFromUriOptions() { } public string BlockId { get { throw null; } } public Azure.HttpAuthorization SourceAuthentication { get { throw null; } set { } } + public Azure.Storage.DataMovement.StorageResourceItemProperties SourceProperties { get { throw null; } set { } } } public enum StorageResourceCreationPreference { @@ -156,7 +180,7 @@ protected StorageResourceItem() { } protected internal abstract string ResourceId { get; } protected Azure.Storage.DataMovement.StorageResourceItemProperties ResourceProperties { get { throw null; } set { } } protected internal abstract Azure.Storage.DataMovement.DataTransferOrder TransferType { get; } - protected internal abstract System.Threading.Tasks.Task CompleteTransferAsync(bool overwrite, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)); + protected internal abstract System.Threading.Tasks.Task CompleteTransferAsync(bool overwrite, Azure.Storage.DataMovement.StorageResourceCompleteTransferOptions completeTransferOptions = null, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)); protected internal abstract System.Threading.Tasks.Task CopyBlockFromUriAsync(Azure.Storage.DataMovement.StorageResourceItem sourceResource, Azure.HttpRange range, bool overwrite, long completeLength, Azure.Storage.DataMovement.StorageResourceCopyFromUriOptions options = null, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)); protected internal abstract System.Threading.Tasks.Task CopyFromStreamAsync(System.IO.Stream stream, long streamLength, bool overwrite, long completeLength, Azure.Storage.DataMovement.StorageResourceWriteToOffsetOptions options = null, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)); protected internal abstract System.Threading.Tasks.Task CopyFromUriAsync(Azure.Storage.DataMovement.StorageResourceItem sourceResource, bool overwrite, long completeLength, Azure.Storage.DataMovement.StorageResourceCopyFromUriOptions options = null, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)); @@ -194,6 +218,7 @@ public partial class StorageResourceWriteToOffsetOptions public StorageResourceWriteToOffsetOptions() { } public string BlockId { get { throw null; } set { } } public long? Position { get { throw null; } set { } } + public Azure.Storage.DataMovement.StorageResourceItemProperties SourceProperties { get { throw null; } set { } } } public partial class TransferCheckpointStoreOptions { diff --git a/sdk/storage/Azure.Storage.DataMovement/assets.json b/sdk/storage/Azure.Storage.DataMovement/assets.json index 72a1f4e108b48..44049c7b4149f 100644 --- a/sdk/storage/Azure.Storage.DataMovement/assets.json +++ b/sdk/storage/Azure.Storage.DataMovement/assets.json @@ -2,5 +2,5 @@ "AssetsRepo": "Azure/azure-sdk-assets", "AssetsRepoPrefixPath": "net", "TagPrefix": "net/storage/Azure.Storage.DataMovement", - "Tag": "net/storage/Azure.Storage.DataMovement_3ae44de880" + "Tag": "net/storage/Azure.Storage.DataMovement_63e8363f90" } diff --git a/sdk/storage/Azure.Storage.DataMovement/src/CommitChunkHandler.cs b/sdk/storage/Azure.Storage.DataMovement/src/CommitChunkHandler.cs index c349bb57451af..6c564f91d05f8 100644 --- a/sdk/storage/Azure.Storage.DataMovement/src/CommitChunkHandler.cs +++ b/sdk/storage/Azure.Storage.DataMovement/src/CommitChunkHandler.cs @@ -17,8 +17,8 @@ internal class CommitChunkHandler : IDisposable private static Task _processStageChunkEvents; #region Delegate Definitions - public delegate Task QueuePutBlockTaskInternal(long offset, long blockSize, long expectedLength); - public delegate Task QueueCommitBlockTaskInternal(); + public delegate Task QueuePutBlockTaskInternal(long offset, long blockSize, long expectedLength, StorageResourceItemProperties properties); + public delegate Task QueueCommitBlockTaskInternal(StorageResourceItemProperties sourceProperties); public delegate void ReportProgressInBytes(long bytesWritten); public delegate Task InvokeFailedEventHandlerInternal(Exception ex); #endregion Delegate Definitions @@ -51,6 +51,7 @@ public struct Behaviors private readonly long _blockSize; private readonly DataTransferOrder _transferOrder; private readonly ClientDiagnostics _clientDiagnostics; + private readonly StorageResourceItemProperties _sourceProperties; public CommitChunkHandler( long expectedLength, @@ -58,6 +59,7 @@ public CommitChunkHandler( Behaviors behaviors, DataTransferOrder transferOrder, ClientDiagnostics clientDiagnostics, + StorageResourceItemProperties sourceProperties, CancellationToken cancellationToken) { if (expectedLength <= 0) @@ -103,6 +105,7 @@ public CommitChunkHandler( } _commitBlockHandler += ConcurrentBlockEvent; _clientDiagnostics = clientDiagnostics; + _sourceProperties = sourceProperties; } public void Dispose() @@ -159,7 +162,7 @@ private async Task NotifyOfPendingStageChunkEvents() if (_bytesTransferred == _expectedLength) { // Add CommitBlockList task to the channel - await _queueCommitBlockTask().ConfigureAwait(false); + await _queueCommitBlockTask(_sourceProperties).ConfigureAwait(false); } else if (_bytesTransferred > _expectedLength) { @@ -188,7 +191,7 @@ private async Task SequentialBlockEvent(StageChunkEventArgs args) long blockLength = (newOffset + _blockSize < _expectedLength) ? _blockSize : _expectedLength - newOffset; - await _queuePutBlockTask(newOffset, blockLength, _expectedLength).ConfigureAwait(false); + await _queuePutBlockTask(newOffset, blockLength, _expectedLength, _sourceProperties).ConfigureAwait(false); } } else diff --git a/sdk/storage/Azure.Storage.DataMovement/src/DataTransferProperty.cs b/sdk/storage/Azure.Storage.DataMovement/src/DataTransferProperty.cs new file mode 100644 index 0000000000000..30d379a4d4fdb --- /dev/null +++ b/sdk/storage/Azure.Storage.DataMovement/src/DataTransferProperty.cs @@ -0,0 +1,53 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +using System.ComponentModel; + +#nullable enable + +namespace Azure.Storage.DataMovement +{ + /// + /// Represents a property on the storage resource. + /// + public abstract class DataTransferProperty + { + internal bool _preserve; + + /// + /// Defines whether the preserve the property on the storage resource. True to preserve, false to not. + /// + public virtual bool Preserve { + get => _preserve; + internal set => _preserve = value; } + + /// + /// Default constructor for . Defaults to preserve the respective property the destination. + /// + public DataTransferProperty() + { + Preserve = true; + } + + /// + /// Constructs to preserves the respective property. + /// + /// + public DataTransferProperty(bool preserve) + { + Preserve = preserve; + } + + /// + [EditorBrowsable(EditorBrowsableState.Never)] + public override bool Equals(object? obj) => base.Equals(obj); + + /// + [EditorBrowsable(EditorBrowsableState.Never)] + public override int GetHashCode() => base.GetHashCode(); + + /// + [EditorBrowsable(EditorBrowsableState.Never)] + public override string? ToString() => base.ToString(); + } +} diff --git a/sdk/storage/Azure.Storage.DataMovement/src/DataTransferPropertyOfT.cs b/sdk/storage/Azure.Storage.DataMovement/src/DataTransferPropertyOfT.cs new file mode 100644 index 0000000000000..eecae62aa5c88 --- /dev/null +++ b/sdk/storage/Azure.Storage.DataMovement/src/DataTransferPropertyOfT.cs @@ -0,0 +1,53 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +using System; +using System.Collections.Generic; +using System.Text; + +#nullable enable + +namespace Azure.Storage.DataMovement +{ + /// + /// Represents a property on the storage resource. + /// + /// The property of the storage resource + #pragma warning disable SA1649 // File name should match first type name + public class DataTransferProperty : DataTransferProperty where T : notnull +#pragma warning restore SA1649 // File name should match first type name + { + internal T? _value; + + /// + /// Represents the value of the DataTransferProperty. + /// + /// + /// This property can be accessed only if the property as been set. (HasValue is true). + /// + public virtual T? Value { + get => _value; + internal set => _value = value; + } + + /// + /// Constructs to preserves the respective property. + /// + /// Specifies whether or to preserve the property value from the source. + public DataTransferProperty(bool preserve) : base(preserve) + { + _value = default; + } + + /// + /// Constructor for to set value on the destination. + /// This will overwrite the property on the destination with the parameter value. + /// + /// The value to set on the property. + public DataTransferProperty(T value) + { + _value = value; + Preserve = false; + } + } +} diff --git a/sdk/storage/Azure.Storage.DataMovement/src/LocalFileStorageResource.cs b/sdk/storage/Azure.Storage.DataMovement/src/LocalFileStorageResource.cs index 07fd5548da878..6949d5f08a117 100644 --- a/sdk/storage/Azure.Storage.DataMovement/src/LocalFileStorageResource.cs +++ b/sdk/storage/Azure.Storage.DataMovement/src/LocalFileStorageResource.cs @@ -245,7 +245,10 @@ protected internal override Task GetCopyAuthorizationHeaderAs /// If the transfer requires client-side encryption, necessary /// operations will occur here. /// - protected internal override Task CompleteTransferAsync(bool overwrite, CancellationToken cancellationToken = default) + protected internal override Task CompleteTransferAsync( + bool overwrite, + StorageResourceCompleteTransferOptions completeTransferOptions = default, + CancellationToken cancellationToken = default) { if (File.Exists(_uri.LocalPath)) { diff --git a/sdk/storage/Azure.Storage.DataMovement/src/ServiceToServiceJobPart.cs b/sdk/storage/Azure.Storage.DataMovement/src/ServiceToServiceJobPart.cs index f68972a43dd52..4dfa03df16367 100644 --- a/sdk/storage/Azure.Storage.DataMovement/src/ServiceToServiceJobPart.cs +++ b/sdk/storage/Azure.Storage.DataMovement/src/ServiceToServiceJobPart.cs @@ -180,10 +180,11 @@ public override async Task ProcessPartToChunkAsync() await OnTransferStateChangedAsync(DataTransferState.InProgress).ConfigureAwait(false); long? fileLength = _sourceResource.Length; + StorageResourceItemProperties sourceProperties = default; try { - StorageResourceItemProperties properties = await _sourceResource.GetPropertiesAsync(_cancellationToken).ConfigureAwait(false); - fileLength = properties.ResourceLength; + sourceProperties = await _sourceResource.GetPropertiesAsync(_cancellationToken).ConfigureAwait(false); + fileLength = sourceProperties.ResourceLength; } catch (Exception ex) { @@ -215,19 +216,24 @@ await StartSingleCallCopy(length).ConfigureAwait(false)) expectedLength: length, blockSize: blockSize, this, - _destinationResource.TransferType); + _destinationResource.TransferType, + sourceProperties); // If we cannot upload in one shot, initiate the parallel block uploader if (await CreateDestinationResource(length, blockSize).ConfigureAwait(false)) { List<(long Offset, long Length)> commitBlockList = GetRangeList(blockSize, length); if (_destinationResource.TransferType == DataTransferOrder.Unordered) { - await QueueStageBlockRequests(commitBlockList, length).ConfigureAwait(false); + await QueueStageBlockRequests(commitBlockList, length, sourceProperties).ConfigureAwait(false); } else // Sequential { // Queue the first partitioned block task - await QueueStageBlockRequest(commitBlockList[0].Offset, commitBlockList[0].Length, length).ConfigureAwait(false); + await QueueStageBlockRequest( + commitBlockList[0].Offset, + commitBlockList[0].Length, + length, + sourceProperties).ConfigureAwait(false); } } else @@ -295,7 +301,7 @@ await _destinationResource.CopyBlockFromUriAsync( if (blockSize == length) { - await CompleteTransferAsync().ConfigureAwait(false); + await CompleteTransferAsync(options.SourceProperties).ConfigureAwait(false); return false; } return true; @@ -318,13 +324,15 @@ internal CommitChunkHandler GetCommitController( long expectedLength, long blockSize, ServiceToServiceJobPart jobPart, - DataTransferOrder transferType) + DataTransferOrder transferType, + StorageResourceItemProperties sourceProperties) => new CommitChunkHandler( expectedLength, blockSize, GetBlockListCommitHandlerBehaviors(jobPart), transferType, ClientDiagnostics, + sourceProperties, _cancellationToken); internal static CommitChunkHandler.Behaviors GetBlockListCommitHandlerBehaviors( @@ -340,13 +348,14 @@ internal static CommitChunkHandler.Behaviors GetBlockListCommitHandlerBehaviors( } #endregion - internal async Task CompleteTransferAsync() + internal async Task CompleteTransferAsync(StorageResourceItemProperties sourceProperties) { try { // Apply necessary transfer completions on the destination. await _destinationResource.CompleteTransferAsync( overwrite: _createMode == StorageResourceCreationPreference.OverwriteIfExists, + completeTransferOptions: new() { SourceProperties = sourceProperties }, cancellationToken: _cancellationToken).ConfigureAwait(false); // Dispose the handlers @@ -361,7 +370,10 @@ await _destinationResource.CompleteTransferAsync( } } - private async Task QueueStageBlockRequests(List<(long Offset, long Size)> commitBlockList, long expectedLength) + private async Task QueueStageBlockRequests( + List<(long Offset, long Size)> commitBlockList, + long expectedLength, + StorageResourceItemProperties sourceProperties) { _queueingTasks = true; // Partition the stream into individual blocks @@ -373,14 +385,22 @@ private async Task QueueStageBlockRequests(List<(long Offset, long Size)> commit } // Queue partitioned block task - await QueueStageBlockRequest(block.Offset, block.Length, expectedLength).ConfigureAwait(false); + await QueueStageBlockRequest( + block.Offset, + block.Length, + expectedLength, + sourceProperties).ConfigureAwait(false); } _queueingTasks = false; await CheckAndUpdateCancellationStateAsync().ConfigureAwait(false); } - private Task QueueStageBlockRequest(long offset, long blockSize, long expectedLength) + private Task QueueStageBlockRequest( + long offset, + long blockSize, + long expectedLength, + StorageResourceItemProperties properties) { return QueueChunkToChannelAsync( async () => @@ -483,7 +503,11 @@ internal void DisposeHandlers() private async Task GetCopyFromUriOptionsAsync(CancellationToken cancellationToken) { - StorageResourceCopyFromUriOptions options = default; + StorageResourceItemProperties properties = await _sourceResource.GetPropertiesAsync(cancellationToken).ConfigureAwait(false); + StorageResourceCopyFromUriOptions options = new() + { + SourceProperties = properties + }; HttpAuthorization authorization = await _sourceResource.GetCopyAuthorizationHeaderAsync(cancellationToken).ConfigureAwait(false); if (authorization != null) { diff --git a/sdk/storage/Azure.Storage.DataMovement/src/Shared/CheckpointerExtensions.cs b/sdk/storage/Azure.Storage.DataMovement/src/Shared/CheckpointerExtensions.cs index f7de11afdec13..cc2ba042cb5b7 100644 --- a/sdk/storage/Azure.Storage.DataMovement/src/Shared/CheckpointerExtensions.cs +++ b/sdk/storage/Azure.Storage.DataMovement/src/Shared/CheckpointerExtensions.cs @@ -139,25 +139,33 @@ internal static void Write(this BinaryWriter writer, long? value) /// /// Writes a boolean plus two int64s to represent a nullable DateTimeOffset. - /// The first long is datetime ticks and the second is offset ticks. + /// The first long is datetime ticks and the second is a short for offset minutes. /// internal static void Write(this BinaryWriter writer, DateTimeOffset? value) { writer.Write(value.HasValue); - writer.Write(value?.Ticks ?? 0L); - writer.Write(value?.Offset.Ticks ?? 0L); + writer.Write(value?.UtcTicks ?? 0L); + } + + /// + /// Writes two int zero values to represent the length and offset of a value + /// that is preserved. + /// + internal static void WriteEmptyLengthOffset(this BinaryWriter writer) + { + writer.Write(-1); + writer.Write(-1); } /// /// Reads a boolean plus two int64s as a nullable DateTimeOffset. - /// The first long is datetime ticks and the second is offset ticks. + /// The first long is datetime ticks and the second is a short for offset minutes. /// internal static DateTimeOffset? ReadNullableDateTimeOffset(this BinaryReader reader) { bool hasValue = reader.ReadBoolean(); long valueTicks = reader.ReadInt64(); - long valueOffsetTicks = reader.ReadInt64(); - return hasValue ? new DateTimeOffset(valueTicks, new TimeSpan(valueOffsetTicks)) : default; + return hasValue ? new DateTimeOffset(valueTicks, TimeSpan.Zero) : default; } internal static string ReadPaddedString(this BinaryReader reader, int numBytes) diff --git a/sdk/storage/Azure.Storage.DataMovement/src/Shared/DataMovementConstants.cs b/sdk/storage/Azure.Storage.DataMovement/src/Shared/DataMovementConstants.cs index 00d72c7f603a0..6e4822fb440a3 100644 --- a/sdk/storage/Azure.Storage.DataMovement/src/Shared/DataMovementConstants.cs +++ b/sdk/storage/Azure.Storage.DataMovement/src/Shared/DataMovementConstants.cs @@ -58,6 +58,7 @@ internal static class Log } internal const int OneByte = 1; + internal const int ShortSizeInBytes = 2; internal const int LongSizeInBytes = 8; internal const int IntSizeInBytes = 4; internal const int GuidSizeInBytes = 16; @@ -156,7 +157,6 @@ internal static class ResourceProperties internal const string ETag = "ETag"; internal const string LastModified = "LastModified"; internal const string Metadata = "Metadata"; - internal const string Tags = "Tags"; } } } diff --git a/sdk/storage/Azure.Storage.DataMovement/src/Shared/StorageResourceItemInternal.cs b/sdk/storage/Azure.Storage.DataMovement/src/Shared/StorageResourceItemInternal.cs index 4f7105b168b6a..1c50f4681745f 100644 --- a/sdk/storage/Azure.Storage.DataMovement/src/Shared/StorageResourceItemInternal.cs +++ b/sdk/storage/Azure.Storage.DataMovement/src/Shared/StorageResourceItemInternal.cs @@ -12,8 +12,14 @@ namespace Azure.Storage.DataMovement /// internal abstract class StorageResourceItemInternal : StorageResourceItem { - internal Task CompleteTransferInternalAsync(bool overwrite, CancellationToken cancellationToken = default) - => CompleteTransferAsync(overwrite, cancellationToken); + internal Task CompleteTransferInternalAsync( + bool overwrite, + StorageResourceCompleteTransferOptions completeTransferOptions, + CancellationToken cancellationToken = default) + => CompleteTransferAsync( + overwrite, + completeTransferOptions, + cancellationToken); internal Task CopyBlockFromUriInternalAsync( StorageResourceItem sourceResource, diff --git a/sdk/storage/Azure.Storage.DataMovement/src/StorageResourceCompleteTransferOptions.cs b/sdk/storage/Azure.Storage.DataMovement/src/StorageResourceCompleteTransferOptions.cs new file mode 100644 index 0000000000000..15e2cf8b0c986 --- /dev/null +++ b/sdk/storage/Azure.Storage.DataMovement/src/StorageResourceCompleteTransferOptions.cs @@ -0,0 +1,20 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +using System; +using System.Collections.Generic; +using System.Text; + +namespace Azure.Storage.DataMovement +{ + /// + /// Options for . + /// + public class StorageResourceCompleteTransferOptions + { + /// + /// Optional. Specifies the source properties to set in the destination. + /// + public StorageResourceItemProperties SourceProperties { get; set; } + } +} diff --git a/sdk/storage/Azure.Storage.DataMovement/src/StorageResourceCopyFromUriOptions.cs b/sdk/storage/Azure.Storage.DataMovement/src/StorageResourceCopyFromUriOptions.cs index 31d5bb8f15cef..4b50e4c718e4a 100644 --- a/sdk/storage/Azure.Storage.DataMovement/src/StorageResourceCopyFromUriOptions.cs +++ b/sdk/storage/Azure.Storage.DataMovement/src/StorageResourceCopyFromUriOptions.cs @@ -25,5 +25,10 @@ public class StorageResourceCopyFromUriOptions /// Only applies to copy operations, not local operations. /// public HttpAuthorization SourceAuthentication { get; set; } + + /// + /// Optional. Specifies the source properties to set in the destination. + /// + public StorageResourceItemProperties SourceProperties { get; set; } } } diff --git a/sdk/storage/Azure.Storage.DataMovement/src/StorageResourceItem.cs b/sdk/storage/Azure.Storage.DataMovement/src/StorageResourceItem.cs index afa545ea04406..b79a190b8f604 100644 --- a/sdk/storage/Azure.Storage.DataMovement/src/StorageResourceItem.cs +++ b/sdk/storage/Azure.Storage.DataMovement/src/StorageResourceItem.cs @@ -170,12 +170,18 @@ protected internal abstract Task CopyBlockFromUriAsync( /// /// If set to true, will overwrite the blob if exists. /// + /// + /// Optional parameters. + /// /// /// Optional to propagate /// notifications that the operation should be cancelled. /// /// The Task which Commits the list of ids - protected internal abstract Task CompleteTransferAsync(bool overwrite, CancellationToken cancellationToken = default); + protected internal abstract Task CompleteTransferAsync( + bool overwrite, + StorageResourceCompleteTransferOptions completeTransferOptions = default, + CancellationToken cancellationToken = default); /// /// Deletes the respective storage resource. diff --git a/sdk/storage/Azure.Storage.DataMovement/src/StorageResourceWriteToOffsetOptions.cs b/sdk/storage/Azure.Storage.DataMovement/src/StorageResourceWriteToOffsetOptions.cs index b4ecdae36a7bd..a041f20a07f6f 100644 --- a/sdk/storage/Azure.Storage.DataMovement/src/StorageResourceWriteToOffsetOptions.cs +++ b/sdk/storage/Azure.Storage.DataMovement/src/StorageResourceWriteToOffsetOptions.cs @@ -23,5 +23,10 @@ public class StorageResourceWriteToOffsetOptions /// Optional. Specifies the position to write to. Will default to 0 if not specified. /// public long? Position { get; set; } + + /// + /// Optional. Specifies the source properties to set in the destination. + /// + public StorageResourceItemProperties SourceProperties { get; set; } } } diff --git a/sdk/storage/Azure.Storage.DataMovement/src/StreamToUriJobPart.cs b/sdk/storage/Azure.Storage.DataMovement/src/StreamToUriJobPart.cs index 0bb2090442386..b88ebb8d1089b 100644 --- a/sdk/storage/Azure.Storage.DataMovement/src/StreamToUriJobPart.cs +++ b/sdk/storage/Azure.Storage.DataMovement/src/StreamToUriJobPart.cs @@ -202,7 +202,8 @@ await QueueChunkToChannelAsync( await CreateDestinationResource( blockSize: length, length: length, - singleCall: true).ConfigureAwait(false)).ConfigureAwait(false); + singleCall: true, + sourceProperties: properties).ConfigureAwait(false)).ConfigureAwait(false); return; } long blockSize = _transferChunkSize; @@ -211,21 +212,30 @@ await CreateDestinationResource( expectedLength: length, blockSize: blockSize, this, - _destinationResource.TransferType); + _destinationResource.TransferType, + properties); - bool destinationCreated = await CreateDestinationResource(blockSize, length, false).ConfigureAwait(false); + bool destinationCreated = await CreateDestinationResource( + blockSize, + length, + false, + properties).ConfigureAwait(false); if (destinationCreated) { // If we cannot upload in one shot, initiate the parallel block uploader List<(long Offset, long Length)> rangeList = GetRangeList(blockSize, length); if (_destinationResource.TransferType == DataTransferOrder.Unordered) { - await QueueStageBlockRequests(rangeList, length).ConfigureAwait(false); + await QueueStageBlockRequests(rangeList, length, properties).ConfigureAwait(false); } else // Sequential { // Queue the first partitioned block task - await QueueStageBlockRequest(rangeList[0].Offset, rangeList[0].Length, length).ConfigureAwait(false); + await QueueStageBlockRequest( + rangeList[0].Offset, + rangeList[0].Length, + length, + properties).ConfigureAwait(false); } } } @@ -244,11 +254,19 @@ await CreateDestinationResource( /// /// Return whether we need to do more after creating the destination resource /// - private async Task CreateDestinationResource(long blockSize, long length, bool singleCall) + private async Task CreateDestinationResource( + long blockSize, + long length, + bool singleCall, + StorageResourceItemProperties sourceProperties) { try { - await InitialUploadCall(blockSize, length, singleCall).ConfigureAwait(false); + await InitialUploadCall( + blockSize, + length, + singleCall, + sourceProperties).ConfigureAwait(false); // Whether or not we continue is up to whether this was single put call or not. return !singleCall; } @@ -275,7 +293,11 @@ private async Task CreateDestinationResource(long blockSize, long length, /// Made to do the initial creation of the blob (if needed). And also /// to make an write if necessary. /// - private async Task InitialUploadCall(long blockSize, long expectedLength, bool singleCall) + private async Task InitialUploadCall( + long blockSize, + long expectedLength, + bool singleCall, + StorageResourceItemProperties sourceProperties) { if (singleCall) { @@ -288,6 +310,10 @@ await _destinationResource.CopyFromStreamAsync( overwrite: _createMode == StorageResourceCreationPreference.OverwriteIfExists, streamLength: blockSize, completeLength: expectedLength, + options: new() + { + SourceProperties = sourceProperties + }, cancellationToken: _cancellationToken).ConfigureAwait(false); // Report bytes written before completion @@ -316,6 +342,10 @@ await _destinationResource.CopyFromStreamAsync( streamLength: blockSize, overwrite: _createMode == StorageResourceCreationPreference.OverwriteIfExists, completeLength: expectedLength, + options: new() + { + SourceProperties = sourceProperties, + }, cancellationToken: _cancellationToken).ConfigureAwait(false); } @@ -328,13 +358,15 @@ internal CommitChunkHandler GetCommitController( long expectedLength, long blockSize, StreamToUriJobPart jobPart, - DataTransferOrder transferType) + DataTransferOrder transferType, + StorageResourceItemProperties sourceProperties) => new CommitChunkHandler( expectedLength, blockSize, GetBlockListCommitHandlerBehaviors(jobPart), transferType, ClientDiagnostics, + sourceProperties, _cancellationToken); internal static CommitChunkHandler.Behaviors GetBlockListCommitHandlerBehaviors( @@ -353,7 +385,8 @@ internal static CommitChunkHandler.Behaviors GetBlockListCommitHandlerBehaviors( internal async Task StageBlockInternal( long offset, long blockLength, - long completeLength) + long completeLength, + StorageResourceItemProperties sourceProperties) { try { @@ -378,6 +411,7 @@ await _destinationResource.CopyFromStreamAsync( options: new StorageResourceWriteToOffsetOptions() { Position = offset, + SourceProperties = sourceProperties }, cancellationToken: _cancellationToken).ConfigureAwait(false); } @@ -420,13 +454,14 @@ await _commitBlockHandler.InvokeEvent( } } - internal async Task CompleteTransferAsync() + internal async Task CompleteTransferAsync(StorageResourceItemProperties sourceProperties) { CancellationHelper.ThrowIfCancellationRequested(_cancellationToken); // Apply necessary transfer completions on the destination. await _destinationResource.CompleteTransferAsync( overwrite: _createMode == StorageResourceCreationPreference.OverwriteIfExists, + completeTransferOptions: new() { SourceProperties = sourceProperties }, cancellationToken: _cancellationToken).ConfigureAwait(false); // Dispose the handlers @@ -436,7 +471,10 @@ await _destinationResource.CompleteTransferAsync( await OnTransferStateChangedAsync(DataTransferState.Completed).ConfigureAwait(false); } - private async Task QueueStageBlockRequests(List<(long Offset, long Size)> rangeList, long completeLength) + private async Task QueueStageBlockRequests( + List<(long Offset, long Size)> rangeList, + long completeLength, + StorageResourceItemProperties sourceProperties) { _queueingTasks = true; // Partition the stream into individual blocks @@ -448,21 +486,30 @@ private async Task QueueStageBlockRequests(List<(long Offset, long Size)> rangeL } // Queue partitioned block task - await QueueStageBlockRequest(block.Offset, block.Length, completeLength).ConfigureAwait(false); + await QueueStageBlockRequest( + block.Offset, + block.Length, + completeLength, + sourceProperties).ConfigureAwait(false); } _queueingTasks = false; await CheckAndUpdateCancellationStateAsync().ConfigureAwait(false); } - private Task QueueStageBlockRequest(long offset, long blockSize, long expectedLength) + private Task QueueStageBlockRequest( + long offset, + long blockSize, + long expectedLength, + StorageResourceItemProperties sourceProperties) { return QueueChunkToChannelAsync( async () => await StageBlockInternal( offset, blockSize, - expectedLength).ConfigureAwait(false)); + expectedLength, + sourceProperties).ConfigureAwait(false)); } /// diff --git a/sdk/storage/Azure.Storage.DataMovement/tests/AppendBlobStorageResourceTests.cs b/sdk/storage/Azure.Storage.DataMovement/tests/AppendBlobStorageResourceTests.cs index 6883b2b4e8410..79c00a92e73b8 100644 --- a/sdk/storage/Azure.Storage.DataMovement/tests/AppendBlobStorageResourceTests.cs +++ b/sdk/storage/Azure.Storage.DataMovement/tests/AppendBlobStorageResourceTests.cs @@ -3,6 +3,7 @@ extern alias DMBlobs; using System; +using System.Collections.Generic; using System.IO; using System.Linq; using System.Net; @@ -17,12 +18,20 @@ using Azure.Storage.DataMovement.Tests; using Azure.Storage.Test; using DMBlobs::Azure.Storage.DataMovement.Blobs; +using Moq; using NUnit.Framework; +using Metadata = System.Collections.Generic.IDictionary; namespace Azure.Storage.DataMovement.Blobs.Tests { public class AppendBlobStorageResourceTests : DataMovementBlobTestBase { + private const string DefaultContentType = "text/plain"; + private const string DefaultContentEncoding = "gzip"; + private const string DefaultContentLanguage = "en-US"; + private const string DefaultContentDisposition = "inline"; + private const string DefaultCacheControl = "no-cache"; + public AppendBlobStorageResourceTests(bool async, BlobClientOptions.ServiceVersion serviceVersion) : base(async, serviceVersion, null /* RecordedTestMode.Record /* to re-record */) { } @@ -231,6 +240,294 @@ await TestHelper.AssertExpectedExceptionAsync( } } + [Test] + public async Task CopyFromStreamAsync_PropertiesDefault() + { + // Arrange + Mock mock = new( + new Uri("https://storageaccount.blob.core.windows.net/container/destination"), + new BlobClientOptions()); + + int length = 1024; + var data = GetRandomBuffer(length); + using var stream = new MemoryStream(data); + using var fileContentStream = new MemoryStream(); + mock.Setup(b => b.CreateAsync(It.IsAny(), It.IsAny())) + .Returns(Task.FromResult(Response.FromValue( + BlobsModelFactory.BlobContentInfo( + eTag: new ETag("eTag"), + lastModified: DateTimeOffset.UtcNow, + contentHash: default, + versionId: "version", + encryptionKeySha256: default, + encryptionScope: default, + blobSequenceNumber: default), + new MockResponse(201)))); + mock.Setup(b => b.AppendBlockAsync(It.IsAny(), It.IsAny(), It.IsAny())) + .Callback( + async (stream, options, token) => + { + await stream.CopyToAsync(fileContentStream).ConfigureAwait(false); + fileContentStream.Position = 0; + }) + .Returns(Task.FromResult(Response.FromValue( + BlobsModelFactory.BlobAppendInfo( + eTag: new ETag("eTag"), + lastModified: DateTimeOffset.UtcNow, + contentHash: default, + contentCrc64: default, + blobAppendOffset: "0", + blobCommittedBlockCount: 1, + isServerEncrypted: false, + encryptionKeySha256: default, + encryptionScope: default), + new MockResponse(201)))); + + AppendBlobStorageResource destinationResource = new AppendBlobStorageResource(mock.Object); + + // Act + IDictionary metadata = DataProvider.BuildMetadata(); + + Dictionary sourceProperties = new() + { + { DataMovementConstants.ResourceProperties.ContentType, DefaultContentType }, + { DataMovementConstants.ResourceProperties.ContentEncoding, DefaultContentEncoding }, + { DataMovementConstants.ResourceProperties.ContentLanguage, DefaultContentLanguage }, + { DataMovementConstants.ResourceProperties.ContentDisposition, DefaultContentDisposition }, + { DataMovementConstants.ResourceProperties.CacheControl, DefaultCacheControl }, + { DataMovementConstants.ResourceProperties.Metadata, metadata } + }; + StorageResourceWriteToOffsetOptions copyFromStreamOptions = new() + { + SourceProperties = new StorageResourceItemProperties( + resourceLength: length, + eTag: new("ETag"), + lastModifiedTime: DateTimeOffset.UtcNow.AddHours(-1), + properties: sourceProperties) + }; + await destinationResource.CopyFromStreamInternalAsync( + stream, + length, + false, + length, + copyFromStreamOptions); + + Assert.That(data, Is.EqualTo(fileContentStream.AsBytes().ToArray())); + mock.Verify(b => b.CreateAsync( + It.Is( + options => + options.HttpHeaders.ContentType == DefaultContentType && + options.HttpHeaders.ContentEncoding == DefaultContentEncoding && + options.HttpHeaders.ContentLanguage == DefaultContentLanguage && + options.HttpHeaders.ContentDisposition == DefaultContentDisposition && + options.HttpHeaders.CacheControl == DefaultCacheControl && + options.Metadata.SequenceEqual(metadata)), + It.IsAny()), + Times.Once()); + mock.Verify(b => b.AppendBlockAsync( + stream, + It.IsAny(), + It.IsAny()), + Times.Once()); + mock.VerifyNoOtherCalls(); + } + + [Test] + public async Task CopyFromStreamAsync_PropertiesPreserve() + { + // Arrange + Mock mock = new( + new Uri("https://storageaccount.blob.core.windows.net/container/destination"), + new BlobClientOptions()); + + int length = 1024; + var data = GetRandomBuffer(length); + using var stream = new MemoryStream(data); + using var fileContentStream = new MemoryStream(); + mock.Setup(b => b.CreateAsync(It.IsAny(), It.IsAny())) + .Returns(Task.FromResult(Response.FromValue( + BlobsModelFactory.BlobContentInfo( + eTag: new ETag("eTag"), + lastModified: DateTimeOffset.UtcNow, + contentHash: default, + versionId: "version", + encryptionKeySha256: default, + encryptionScope: default, + blobSequenceNumber: default), + new MockResponse(201)))); + mock.Setup(b => b.AppendBlockAsync(It.IsAny(), It.IsAny(), It.IsAny())) + .Callback( + async (stream, options, token) => + { + await stream.CopyToAsync(fileContentStream).ConfigureAwait(false); + fileContentStream.Position = 0; + }) + .Returns(Task.FromResult(Response.FromValue( + BlobsModelFactory.BlobAppendInfo( + eTag: new ETag("eTag"), + lastModified: DateTimeOffset.UtcNow, + contentHash: default, + contentCrc64: default, + blobAppendOffset: "0", + blobCommittedBlockCount: 1, + isServerEncrypted: false, + encryptionKeySha256: default, + encryptionScope: default), + new MockResponse(201)))); + + AppendBlobStorageResourceOptions resourceOptions = new() + { + CacheControl = new(true), + ContentDisposition = new(true), + ContentLanguage = new(true), + ContentEncoding = new(true), + ContentType = new(true), + Metadata = new(true) + }; + AppendBlobStorageResource destinationResource = new AppendBlobStorageResource(mock.Object, resourceOptions); + + // Act + IDictionary metadata = DataProvider.BuildMetadata(); + + Dictionary sourceProperties = new() + { + { DataMovementConstants.ResourceProperties.ContentType, DefaultContentType }, + { DataMovementConstants.ResourceProperties.ContentEncoding, DefaultContentEncoding }, + { DataMovementConstants.ResourceProperties.ContentLanguage, DefaultContentLanguage }, + { DataMovementConstants.ResourceProperties.ContentDisposition, DefaultContentDisposition }, + { DataMovementConstants.ResourceProperties.CacheControl, DefaultCacheControl }, + { DataMovementConstants.ResourceProperties.Metadata, metadata } + }; + StorageResourceWriteToOffsetOptions copyFromStreamOptions = new() + { + SourceProperties = new StorageResourceItemProperties( + resourceLength: length, + eTag: new("ETag"), + lastModifiedTime: DateTimeOffset.UtcNow.AddHours(-1), + properties: sourceProperties) + }; + await destinationResource.CopyFromStreamInternalAsync( + stream, + length, + false, + length, + copyFromStreamOptions); + + Assert.That(data, Is.EqualTo(fileContentStream.AsBytes().ToArray())); + mock.Verify(b => b.CreateAsync( + It.Is( + options => + options.HttpHeaders.ContentType == DefaultContentType && + options.HttpHeaders.ContentEncoding == DefaultContentEncoding && + options.HttpHeaders.ContentLanguage == DefaultContentLanguage && + options.HttpHeaders.ContentDisposition == DefaultContentDisposition && + options.HttpHeaders.CacheControl == DefaultCacheControl && + options.Metadata.SequenceEqual(metadata)), + It.IsAny()), + Times.Once()); + mock.Verify(b => b.AppendBlockAsync( + stream, + It.IsAny(), + It.IsAny()), + Times.Once()); + mock.VerifyNoOtherCalls(); + } + + [Test] + public async Task CopyFromStreamAsync_PropertiesNoPreserve() + { + // Arrange + Mock mock = new( + new Uri("https://storageaccount.blob.core.windows.net/container/destination"), + new BlobClientOptions()); + + int length = 1024; + var data = GetRandomBuffer(length); + using var stream = new MemoryStream(data); + using var fileContentStream = new MemoryStream(); + mock.Setup(b => b.CreateAsync(It.IsAny(), It.IsAny())) + .Returns(Task.FromResult(Response.FromValue( + BlobsModelFactory.BlobContentInfo( + eTag: new ETag("eTag"), + lastModified: DateTimeOffset.UtcNow, + contentHash: default, + versionId: "version", + encryptionKeySha256: default, + encryptionScope: default, + blobSequenceNumber: default), + new MockResponse(201)))); + mock.Setup(b => b.AppendBlockAsync(It.IsAny(), It.IsAny(), It.IsAny())) + .Callback( + async (stream, options, token) => + { + await stream.CopyToAsync(fileContentStream).ConfigureAwait(false); + fileContentStream.Position = 0; + }) + .Returns(Task.FromResult(Response.FromValue( + BlobsModelFactory.BlobAppendInfo( + eTag: new ETag("eTag"), + lastModified: DateTimeOffset.UtcNow, + contentHash: default, + contentCrc64: default, + blobAppendOffset: "0", + blobCommittedBlockCount: 1, + isServerEncrypted: false, + encryptionKeySha256: default, + encryptionScope: default), + new MockResponse(201)))); + + AppendBlobStorageResourceOptions resourceOptions = new() + { + CacheControl = new(false), + ContentDisposition = new(false), + ContentLanguage = new(false), + ContentEncoding = new(false), + ContentType = new(false), + Metadata = new(false) + }; + AppendBlobStorageResource destinationResource = new AppendBlobStorageResource(mock.Object, resourceOptions); + + // Act + IDictionary metadata = DataProvider.BuildMetadata(); + + Dictionary sourceProperties = new() + { + { DataMovementConstants.ResourceProperties.ContentType, DefaultContentType }, + { DataMovementConstants.ResourceProperties.ContentEncoding, DefaultContentEncoding }, + { DataMovementConstants.ResourceProperties.ContentLanguage, DefaultContentLanguage }, + { DataMovementConstants.ResourceProperties.ContentDisposition, DefaultContentDisposition }, + { DataMovementConstants.ResourceProperties.CacheControl, DefaultCacheControl }, + { DataMovementConstants.ResourceProperties.Metadata, metadata } + }; + StorageResourceWriteToOffsetOptions copyFromStreamOptions = new() + { + SourceProperties = new StorageResourceItemProperties( + resourceLength: length, + eTag: new("ETag"), + lastModifiedTime: DateTimeOffset.UtcNow.AddHours(-1), + properties: sourceProperties) + }; + await destinationResource.CopyFromStreamInternalAsync( + stream, + length, + false, + length, + copyFromStreamOptions); + + Assert.That(data, Is.EqualTo(fileContentStream.AsBytes().ToArray())); + mock.Verify(b => b.CreateAsync( + It.Is( + options => options.Metadata == default), + It.IsAny()), + Times.Once()); + mock.Verify(b => b.AppendBlockAsync( + stream, + It.IsAny(), + It.IsAny()), + Times.Once()); + mock.VerifyNoOtherCalls(); + } + [RecordedTest] public async Task CopyFromUriAsync() { @@ -346,6 +643,302 @@ await destinationResource.CopyFromUriAsync( TestHelper.AssertSequenceEqual(data, result.Content.AsBytes().ToArray()); } + [Test] + public async Task CopyFromUriAsync_PropertiesDefault() + { + // Arrange + Uri sourceUri = new Uri("https://storageaccount.blob.core.windows.net/container/source"); + Mock sourceResource = new(); + sourceResource.Setup(b => b.Uri) + .Returns(sourceUri); + + Mock mockDestination = new( + new Uri("https://storageaccount.blob.core.windows.net/container/destination"), + new BlobClientOptions()); + + int length = 1024; + var data = GetRandomBuffer(length); + using var stream = new MemoryStream(data); + using var fileContentStream = new MemoryStream(); + mockDestination.Setup(b => b.CreateAsync(It.IsAny(), It.IsAny())) + .Returns(Task.FromResult(Response.FromValue( + BlobsModelFactory.BlobContentInfo( + eTag: new ETag("eTag"), + lastModified: DateTimeOffset.UtcNow, + contentHash: default, + versionId: "version", + encryptionKeySha256: default, + encryptionScope: default, + blobSequenceNumber: default), + new MockResponse(201)))); + mockDestination.Setup(b => b.AppendBlockFromUriAsync(It.IsAny(), It.IsAny(), It.IsAny())) + .Callback( + async (uri, options, token) => + { + await stream.CopyToAsync(fileContentStream).ConfigureAwait(false); + fileContentStream.Position = 0; + }) + .Returns(Task.FromResult(Response.FromValue( + BlobsModelFactory.BlobAppendInfo( + eTag: new ETag("eTag"), + lastModified: DateTimeOffset.UtcNow, + contentHash: default, + contentCrc64: default, + blobAppendOffset: "0", + blobCommittedBlockCount: 1, + isServerEncrypted: false, + encryptionKeySha256: default, + encryptionScope: default), + new MockResponse(201)))); + AppendBlobStorageResource destinationResource = new AppendBlobStorageResource(mockDestination.Object); + + // Act + IDictionary metadata = DataProvider.BuildMetadata(); + + Dictionary sourceProperties = new() + { + { DataMovementConstants.ResourceProperties.ContentType, DefaultContentType }, + { DataMovementConstants.ResourceProperties.ContentEncoding, DefaultContentEncoding }, + { DataMovementConstants.ResourceProperties.ContentLanguage, DefaultContentLanguage }, + { DataMovementConstants.ResourceProperties.ContentDisposition, DefaultContentDisposition }, + { DataMovementConstants.ResourceProperties.CacheControl, DefaultCacheControl }, + { DataMovementConstants.ResourceProperties.Metadata, metadata } + }; + StorageResourceCopyFromUriOptions copyFromUriOptions = new() + { + SourceProperties = new StorageResourceItemProperties( + resourceLength: length, + eTag: new("ETag"), + lastModifiedTime: DateTimeOffset.UtcNow.AddHours(-1), + properties: sourceProperties) + }; + await destinationResource.CopyFromUriInternalAsync( + sourceResource.Object, + false, + length, + copyFromUriOptions); + + Assert.That(data, Is.EqualTo(fileContentStream.AsBytes().ToArray())); + mockDestination.Verify(b => b.CreateAsync( + It.Is( + options => + options.HttpHeaders.ContentType == DefaultContentType && + options.HttpHeaders.ContentEncoding == DefaultContentEncoding && + options.HttpHeaders.ContentLanguage == DefaultContentLanguage && + options.HttpHeaders.ContentDisposition == DefaultContentDisposition && + options.HttpHeaders.CacheControl == DefaultCacheControl && + options.Metadata.SequenceEqual(metadata)), + It.IsAny()), + Times.Once()); + mockDestination.Verify(b => b.AppendBlockFromUriAsync( + sourceUri, + It.IsAny(), + It.IsAny()), + Times.Once()); + } + + [Test] + public async Task CopyFromUriAsync_PropertiesPreserve() + { + // Arrange + Uri sourceUri = new Uri("https://storageaccount.blob.core.windows.net/container/source"); + Mock sourceResource = new(); + sourceResource.Setup(b => b.Uri) + .Returns(sourceUri); + + Mock mockDestination = new( + new Uri("https://storageaccount.blob.core.windows.net/container/destination"), + new BlobClientOptions()); + + int length = 1024; + var data = GetRandomBuffer(length); + using var stream = new MemoryStream(data); + using var fileContentStream = new MemoryStream(); + mockDestination.Setup(b => b.CreateAsync(It.IsAny(), It.IsAny())) + .Returns(Task.FromResult(Response.FromValue( + BlobsModelFactory.BlobContentInfo( + eTag: new ETag("eTag"), + lastModified: DateTimeOffset.UtcNow, + contentHash: default, + versionId: "version", + encryptionKeySha256: default, + encryptionScope: default, + blobSequenceNumber: default), + new MockResponse(201)))); + mockDestination.Setup(b => b.AppendBlockFromUriAsync(It.IsAny(), It.IsAny(), It.IsAny())) + .Callback( + async (uri, options, token) => + { + await stream.CopyToAsync(fileContentStream).ConfigureAwait(false); + fileContentStream.Position = 0; + }) + .Returns(Task.FromResult(Response.FromValue( + BlobsModelFactory.BlobAppendInfo( + eTag: new ETag("eTag"), + lastModified: DateTimeOffset.UtcNow, + contentHash: default, + contentCrc64: default, + blobAppendOffset: "0", + blobCommittedBlockCount: 1, + isServerEncrypted: false, + encryptionKeySha256: default, + encryptionScope: default), + new MockResponse(201)))); + AppendBlobStorageResourceOptions resourceOptions = new() + { + CacheControl = new(true), + ContentDisposition = new(true), + ContentLanguage = new(true), + ContentEncoding = new(true), + ContentType = new(true), + Metadata = new(true) + }; + AppendBlobStorageResource destinationResource = new AppendBlobStorageResource(mockDestination.Object, resourceOptions); + + // Act + IDictionary metadata = DataProvider.BuildMetadata(); + + Dictionary sourceProperties = new() + { + { DataMovementConstants.ResourceProperties.ContentType, DefaultContentType }, + { DataMovementConstants.ResourceProperties.ContentEncoding, DefaultContentEncoding }, + { DataMovementConstants.ResourceProperties.ContentLanguage, DefaultContentLanguage }, + { DataMovementConstants.ResourceProperties.ContentDisposition, DefaultContentDisposition }, + { DataMovementConstants.ResourceProperties.CacheControl, DefaultCacheControl }, + { DataMovementConstants.ResourceProperties.Metadata, metadata } + }; + StorageResourceCopyFromUriOptions copyFromUriOptions = new() + { + SourceProperties = new StorageResourceItemProperties( + resourceLength: length, + eTag: new("ETag"), + lastModifiedTime: DateTimeOffset.UtcNow.AddHours(-1), + properties: sourceProperties) + }; + await destinationResource.CopyFromUriInternalAsync( + sourceResource.Object, + false, + length, + copyFromUriOptions); + + Assert.That(data, Is.EqualTo(fileContentStream.AsBytes().ToArray())); + mockDestination.Verify(b => b.CreateAsync( + It.Is( + options => + options.HttpHeaders.ContentType == DefaultContentType && + options.HttpHeaders.ContentEncoding == DefaultContentEncoding && + options.HttpHeaders.ContentLanguage == DefaultContentLanguage && + options.HttpHeaders.ContentDisposition == DefaultContentDisposition && + options.HttpHeaders.CacheControl == DefaultCacheControl && + options.Metadata.SequenceEqual(metadata)), + It.IsAny()), + Times.Once()); + mockDestination.Verify(b => b.AppendBlockFromUriAsync( + sourceUri, + It.IsAny(), + It.IsAny()), + Times.Once()); + mockDestination.VerifyNoOtherCalls(); + } + + [Test] + public async Task CopyFromUriAsync_PropertiesNoPreserve() + { + // Arrange + Uri sourceUri = new Uri("https://storageaccount.blob.core.windows.net/container/source"); + Mock sourceResource = new(); + sourceResource.Setup(b => b.Uri) + .Returns(sourceUri); + + Mock mockDestination = new( + new Uri("https://storageaccount.blob.core.windows.net/container/destination"), + new BlobClientOptions()); + + int length = 1024; + var data = GetRandomBuffer(length); + using var stream = new MemoryStream(data); + using var fileContentStream = new MemoryStream(); + mockDestination.Setup(b => b.CreateAsync(It.IsAny(), It.IsAny())) + .Returns(Task.FromResult(Response.FromValue( + BlobsModelFactory.BlobContentInfo( + eTag: new ETag("eTag"), + lastModified: DateTimeOffset.UtcNow, + contentHash: default, + versionId: "version", + encryptionKeySha256: default, + encryptionScope: default, + blobSequenceNumber: default), + new MockResponse(201)))); + mockDestination.Setup(b => b.AppendBlockFromUriAsync(It.IsAny(), It.IsAny(), It.IsAny())) + .Callback( + async (uri, options, token) => + { + await stream.CopyToAsync(fileContentStream).ConfigureAwait(false); + fileContentStream.Position = 0; + }) + .Returns(Task.FromResult(Response.FromValue( + BlobsModelFactory.BlobAppendInfo( + eTag: new ETag("eTag"), + lastModified: DateTimeOffset.UtcNow, + contentHash: default, + contentCrc64: default, + blobAppendOffset: "0", + blobCommittedBlockCount: 1, + isServerEncrypted: false, + encryptionKeySha256: default, + encryptionScope: default), + new MockResponse(201)))); + AppendBlobStorageResourceOptions resourceOptions = new() + { + CacheControl = new(false), + ContentDisposition = new(false), + ContentLanguage = new(false), + ContentEncoding = new(false), + ContentType = new(false), + Metadata = new(false) + }; + AppendBlobStorageResource destinationResource = new AppendBlobStorageResource(mockDestination.Object, resourceOptions); + + // Act + IDictionary metadata = DataProvider.BuildMetadata(); + + Dictionary sourceProperties = new() + { + { DataMovementConstants.ResourceProperties.ContentType, DefaultContentType }, + { DataMovementConstants.ResourceProperties.ContentEncoding, DefaultContentEncoding }, + { DataMovementConstants.ResourceProperties.ContentLanguage, DefaultContentLanguage }, + { DataMovementConstants.ResourceProperties.ContentDisposition, DefaultContentDisposition }, + { DataMovementConstants.ResourceProperties.CacheControl, DefaultCacheControl }, + { DataMovementConstants.ResourceProperties.Metadata, metadata } + }; + StorageResourceCopyFromUriOptions copyFromUriOptions = new() + { + SourceProperties = new StorageResourceItemProperties( + resourceLength: length, + eTag: new("ETag"), + lastModifiedTime: DateTimeOffset.UtcNow.AddHours(-1), + properties: sourceProperties) + }; + await destinationResource.CopyFromUriInternalAsync( + sourceResource.Object, + false, + length, + copyFromUriOptions); + + Assert.That(data, Is.EqualTo(fileContentStream.AsBytes().ToArray())); + mockDestination.Verify(b => b.CreateAsync( + It.Is( + options => options.Metadata == default), + It.IsAny()), + Times.Once()); + mockDestination.Verify(b => b.AppendBlockFromUriAsync( + sourceUri, + It.IsAny(), + It.IsAny()), + Times.Once()); + mockDestination.VerifyNoOtherCalls(); + } + [RecordedTest] public async Task CopyFromUriAsync_Error() { @@ -550,6 +1143,306 @@ await destinationResource.CopyBlockFromUriAsync( TestHelper.AssertSequenceEqual(blockData, result.Content.AsBytes().ToArray()); } + [Test] + public async Task CopyBlockFromUriAsync_PropertiesDefault() + { + // Arrange + Uri sourceUri = new Uri("https://storageaccount.blob.core.windows.net/container/source"); + Mock sourceResource = new(); + sourceResource.Setup(b => b.Uri) + .Returns(sourceUri); + + Mock mockDestination = new( + new Uri("https://storageaccount.blob.core.windows.net/container/destination"), + new BlobClientOptions()); + + int length = 1024; + var data = GetRandomBuffer(length); + using var stream = new MemoryStream(data); + using var fileContentStream = new MemoryStream(); + mockDestination.Setup(b => b.CreateAsync(It.IsAny(), It.IsAny())) + .Returns(Task.FromResult(Response.FromValue( + BlobsModelFactory.BlobContentInfo( + eTag: new ETag("eTag"), + lastModified: DateTimeOffset.UtcNow, + contentHash: default, + versionId: "version", + encryptionKeySha256: default, + encryptionScope: default, + blobSequenceNumber: default), + new MockResponse(201)))); + mockDestination.Setup(b => b.AppendBlockFromUriAsync(It.IsAny(), It.IsAny(), It.IsAny())) + .Callback( + async (uri, options, token) => + { + await stream.CopyToAsync(fileContentStream).ConfigureAwait(false); + fileContentStream.Position = 0; + }) + .Returns(Task.FromResult(Response.FromValue( + BlobsModelFactory.BlobAppendInfo( + eTag: new ETag("eTag"), + lastModified: DateTimeOffset.UtcNow, + contentHash: default, + contentCrc64: default, + blobAppendOffset: "0", + blobCommittedBlockCount: 1, + isServerEncrypted: false, + encryptionKeySha256: default, + encryptionScope: default), + new MockResponse(201)))); + AppendBlobStorageResource destinationResource = new AppendBlobStorageResource(mockDestination.Object); + + // Act + IDictionary metadata = DataProvider.BuildMetadata(); + + Dictionary sourceProperties = new() + { + { DataMovementConstants.ResourceProperties.ContentType, DefaultContentType }, + { DataMovementConstants.ResourceProperties.ContentEncoding, DefaultContentEncoding }, + { DataMovementConstants.ResourceProperties.ContentLanguage, DefaultContentLanguage }, + { DataMovementConstants.ResourceProperties.ContentDisposition, DefaultContentDisposition }, + { DataMovementConstants.ResourceProperties.CacheControl, DefaultCacheControl }, + { DataMovementConstants.ResourceProperties.Metadata, metadata } + }; + StorageResourceCopyFromUriOptions copyFromUriOptions = new() + { + SourceProperties = new StorageResourceItemProperties( + resourceLength: length, + eTag: new("ETag"), + lastModifiedTime: DateTimeOffset.UtcNow.AddHours(-1), + properties: sourceProperties) + }; + await destinationResource.CopyBlockFromUriInternalAsync( + sourceResource.Object, + new HttpRange(0, length), + false, + length, + copyFromUriOptions); + + Assert.That(data, Is.EqualTo(fileContentStream.AsBytes().ToArray())); + mockDestination.Verify(b => b.CreateAsync( + It.Is( + options => + options.HttpHeaders.ContentType == DefaultContentType && + options.HttpHeaders.ContentEncoding == DefaultContentEncoding && + options.HttpHeaders.ContentLanguage == DefaultContentLanguage && + options.HttpHeaders.ContentDisposition == DefaultContentDisposition && + options.HttpHeaders.CacheControl == DefaultCacheControl && + options.Metadata.SequenceEqual(metadata)), + It.IsAny()), + Times.Once()); + mockDestination.Verify(b => b.AppendBlockFromUriAsync( + sourceUri, + It.IsAny(), + It.IsAny()), + Times.Once()); + mockDestination.VerifyNoOtherCalls(); + } + + [Test] + public async Task CopyBlockFromUriAsync_PropertiesPreserve() + { + // Arrange + Uri sourceUri = new Uri("https://storageaccount.blob.core.windows.net/container/source"); + Mock sourceResource = new(); + sourceResource.Setup(b => b.Uri) + .Returns(sourceUri); + + Mock mockDestination = new( + new Uri("https://storageaccount.blob.core.windows.net/container/destination"), + new BlobClientOptions()); + + int length = 1024; + var data = GetRandomBuffer(length); + using var stream = new MemoryStream(data); + using var fileContentStream = new MemoryStream(); + mockDestination.Setup(b => b.CreateAsync(It.IsAny(), It.IsAny())) + .Returns(Task.FromResult(Response.FromValue( + BlobsModelFactory.BlobContentInfo( + eTag: new ETag("eTag"), + lastModified: DateTimeOffset.UtcNow, + contentHash: default, + versionId: "version", + encryptionKeySha256: default, + encryptionScope: default, + blobSequenceNumber: default), + new MockResponse(201)))); + mockDestination.Setup(b => b.AppendBlockFromUriAsync(It.IsAny(), It.IsAny(), It.IsAny())) + .Callback( + async (uri, options, token) => + { + await stream.CopyToAsync(fileContentStream).ConfigureAwait(false); + fileContentStream.Position = 0; + }) + .Returns(Task.FromResult(Response.FromValue( + BlobsModelFactory.BlobAppendInfo( + eTag: new ETag("eTag"), + lastModified: DateTimeOffset.UtcNow, + contentHash: default, + contentCrc64: default, + blobAppendOffset: "0", + blobCommittedBlockCount: 1, + isServerEncrypted: false, + encryptionKeySha256: default, + encryptionScope: default), + new MockResponse(201)))); + AppendBlobStorageResourceOptions resourceOptions = new() + { + CacheControl = new(true), + ContentDisposition = new(true), + ContentLanguage = new(true), + ContentEncoding = new(true), + ContentType = new(true), + Metadata = new(true) + }; + AppendBlobStorageResource destinationResource = new AppendBlobStorageResource(mockDestination.Object, resourceOptions); + + // Act + IDictionary metadata = DataProvider.BuildMetadata(); + + Dictionary sourceProperties = new() + { + { DataMovementConstants.ResourceProperties.ContentType, DefaultContentType }, + { DataMovementConstants.ResourceProperties.ContentEncoding, DefaultContentEncoding }, + { DataMovementConstants.ResourceProperties.ContentLanguage, DefaultContentLanguage }, + { DataMovementConstants.ResourceProperties.ContentDisposition, DefaultContentDisposition }, + { DataMovementConstants.ResourceProperties.CacheControl, DefaultCacheControl }, + { DataMovementConstants.ResourceProperties.Metadata, metadata } + }; + StorageResourceCopyFromUriOptions copyFromUriOptions = new() + { + SourceProperties = new StorageResourceItemProperties( + resourceLength: length, + eTag: new("ETag"), + lastModifiedTime: DateTimeOffset.UtcNow.AddHours(-1), + properties: sourceProperties) + }; + await destinationResource.CopyBlockFromUriInternalAsync( + sourceResource.Object, + new HttpRange(0, length), + false, + length, + copyFromUriOptions); + + Assert.That(data, Is.EqualTo(fileContentStream.AsBytes().ToArray())); + mockDestination.Verify(b => b.CreateAsync( + It.Is( + options => + options.HttpHeaders.ContentType == DefaultContentType && + options.HttpHeaders.ContentEncoding == DefaultContentEncoding && + options.HttpHeaders.ContentLanguage == DefaultContentLanguage && + options.HttpHeaders.ContentDisposition == DefaultContentDisposition && + options.HttpHeaders.CacheControl == DefaultCacheControl && + options.Metadata.SequenceEqual(metadata)), + It.IsAny()), + Times.Once()); + mockDestination.Verify(b => b.AppendBlockFromUriAsync( + sourceUri, + It.IsAny(), + It.IsAny()), + Times.Once()); + mockDestination.VerifyNoOtherCalls(); + } + + [Test] + public async Task CopyBlockFromUriAsync_PropertiesNoPreserve() + { + // Arrange + Uri sourceUri = new Uri("https://storageaccount.blob.core.windows.net/container/source"); + Mock sourceResource = new(); + sourceResource.Setup(b => b.Uri) + .Returns(sourceUri); + + Mock mockDestination = new( + new Uri("https://storageaccount.blob.core.windows.net/container/destination"), + new BlobClientOptions()); + + int length = 1024; + var data = GetRandomBuffer(length); + using var stream = new MemoryStream(data); + using var fileContentStream = new MemoryStream(); + mockDestination.Setup(b => b.CreateAsync(It.IsAny(), It.IsAny())) + .Returns(Task.FromResult(Response.FromValue( + BlobsModelFactory.BlobContentInfo( + eTag: new ETag("eTag"), + lastModified: DateTimeOffset.UtcNow, + contentHash: default, + versionId: "version", + encryptionKeySha256: default, + encryptionScope: default, + blobSequenceNumber: default), + new MockResponse(201)))); + mockDestination.Setup(b => b.AppendBlockFromUriAsync(It.IsAny(), It.IsAny(), It.IsAny())) + .Callback( + async (uri, options, token) => + { + await stream.CopyToAsync(fileContentStream).ConfigureAwait(false); + fileContentStream.Position = 0; + }) + .Returns(Task.FromResult(Response.FromValue( + BlobsModelFactory.BlobAppendInfo( + eTag: new ETag("eTag"), + lastModified: DateTimeOffset.UtcNow, + contentHash: default, + contentCrc64: default, + blobAppendOffset: "0", + blobCommittedBlockCount: 1, + isServerEncrypted: false, + encryptionKeySha256: default, + encryptionScope: default), + new MockResponse(201)))); + AppendBlobStorageResourceOptions resourceOptions = new() + { + CacheControl = new(false), + ContentDisposition = new(false), + ContentLanguage = new(false), + ContentEncoding = new(false), + ContentType = new(false), + Metadata = new(false) + }; + AppendBlobStorageResource destinationResource = new AppendBlobStorageResource(mockDestination.Object, resourceOptions); + + // Act + IDictionary metadata = DataProvider.BuildMetadata(); + + Dictionary sourceProperties = new() + { + { DataMovementConstants.ResourceProperties.ContentType, DefaultContentType }, + { DataMovementConstants.ResourceProperties.ContentEncoding, DefaultContentEncoding }, + { DataMovementConstants.ResourceProperties.ContentLanguage, DefaultContentLanguage }, + { DataMovementConstants.ResourceProperties.ContentDisposition, DefaultContentDisposition }, + { DataMovementConstants.ResourceProperties.CacheControl, DefaultCacheControl }, + { DataMovementConstants.ResourceProperties.Metadata, metadata } + }; + StorageResourceCopyFromUriOptions copyFromUriOptions = new() + { + SourceProperties = new StorageResourceItemProperties( + resourceLength: length, + eTag: new("ETag"), + lastModifiedTime: DateTimeOffset.UtcNow.AddHours(-1), + properties: sourceProperties) + }; + await destinationResource.CopyBlockFromUriInternalAsync( + sourceResource.Object, + new HttpRange(0, length), + false, + length, + copyFromUriOptions); + + Assert.That(data, Is.EqualTo(fileContentStream.AsBytes().ToArray())); + mockDestination.Verify(b => b.CreateAsync( + It.Is( + options => options.Metadata == default), + It.IsAny()), + Times.Once()); + mockDestination.Verify(b => b.AppendBlockFromUriAsync( + sourceUri, + It.IsAny(), + It.IsAny()), + Times.Once()); + mockDestination.VerifyNoOtherCalls(); + } + [RecordedTest] public async Task CopyBlockFromUriAsync_Error() { @@ -615,6 +1508,104 @@ await TestHelper.AssertExpectedExceptionAsync( }); } + [Test] + public async Task GetPropertiesAsync_NotCached() + { + // Arrange + Mock mock = new( + new Uri("https://storageaccount.file.core.windows.net/container/file"), + new BlobClientOptions()); + + long length = 1024; + ETag eTag = new ETag("etag"); + string source = "https://storageaccount.file.core.windows.net/container/file2"; + Metadata metadata = DataProvider.BuildMetadata(); + mock.Setup(b => b.GetPropertiesAsync(It.IsAny(), It.IsAny())) + .Returns(Task.FromResult(Response.FromValue( + BlobsModelFactory.BlobProperties( + lastModified: DateTime.MinValue, + leaseStatus: LeaseStatus.Unlocked, + contentLength: length, + eTag: eTag, + contentEncoding: DefaultContentEncoding, + contentDisposition: DefaultContentDisposition, + contentLanguage: DefaultContentLanguage, + contentType: DefaultContentType, + cacheControl: DefaultCacheControl, + copySource: new Uri(source), + accessTier: default, + copyCompletedOn: DateTimeOffset.MinValue, + accessTierChangedOn: DateTimeOffset.MinValue, + blobType: BlobType.Block, + metadata: metadata, + tagCount: 5), + new MockResponse(200)))); + + BlockBlobStorageResource storageResource = new BlockBlobStorageResource(mock.Object); + + // Act + StorageResourceItemProperties result = await storageResource.GetPropertiesInternalAsync(); + string contentEncodingResult = (string)result.RawProperties[DataMovementConstants.ResourceProperties.ContentEncoding]; + string contentDispositionResult = (string)result.RawProperties[DataMovementConstants.ResourceProperties.ContentDisposition]; + string contentLanguageResult = (string)result.RawProperties[DataMovementConstants.ResourceProperties.ContentLanguage]; + string contentTypeResult = (string)result.RawProperties[DataMovementConstants.ResourceProperties.ContentType]; + string cacheControlResult = (string)result.RawProperties[DataMovementConstants.ResourceProperties.CacheControl]; + Metadata metadataResult = (Metadata)result.RawProperties[DataMovementConstants.ResourceProperties.Metadata]; + + // Assert + Assert.AreEqual(eTag, result.ETag); + Assert.AreEqual(length, result.ResourceLength); + Assert.AreEqual(contentEncodingResult, DefaultContentEncoding); + Assert.AreEqual(contentDispositionResult, DefaultContentDisposition); + Assert.AreEqual(contentLanguageResult, DefaultContentLanguage); + Assert.AreEqual(contentTypeResult, DefaultContentType); + Assert.AreEqual(cacheControlResult, DefaultCacheControl); + Assert.That(metadata, Is.EqualTo(metadataResult)); + mock.Verify(b => b.GetPropertiesAsync(It.IsAny(), It.IsAny()), + Times.Once()); + mock.VerifyNoOtherCalls(); + } + + [Test] + public async Task GetPropertiesAsync_Cached() + { + // Arrange + Mock mock = new( + new Uri("https://storageaccount.file.core.windows.net/container/file"), + new BlobClientOptions()); + + long length = 1024; + ETag eTag = new ETag("etag"); + DateTimeOffset lastModified = DateTimeOffset.UtcNow.AddHours(-1); + Metadata metadata = DataProvider.BuildMetadata(); + Dictionary rawProperties = new() + { + { DataMovementConstants.ResourceProperties.ContentEncoding, DefaultContentEncoding }, + { DataMovementConstants.ResourceProperties.ContentDisposition, DefaultContentDisposition}, + { DataMovementConstants.ResourceProperties.ContentLanguage, DefaultContentLanguage}, + { DataMovementConstants.ResourceProperties.ContentType, DefaultContentType}, + { DataMovementConstants.ResourceProperties.CacheControl, DefaultCacheControl}, + { DataMovementConstants.ResourceProperties.Metadata, metadata }, + }; + + BlockBlobStorageResource storageResource = new BlockBlobStorageResource( + mock.Object, + new StorageResourceItemProperties( + length, + eTag, + lastModified, + rawProperties)); + + // Act + StorageResourceItemProperties result = await storageResource.GetPropertiesInternalAsync(); + + // Assert + Assert.That(rawProperties, Is.EqualTo(result.RawProperties)); + mock.Verify(b => b.GetPropertiesAsync(It.IsAny(), It.IsAny()), + Times.Never()); + mock.VerifyNoOtherCalls(); + } + [RecordedTest] public async Task CompleteTransferAsync() { diff --git a/sdk/storage/Azure.Storage.DataMovement/tests/BlockBlobStorageResourceTests.cs b/sdk/storage/Azure.Storage.DataMovement/tests/BlockBlobStorageResourceTests.cs index 0010b544148ac..d894e34d993b9 100644 --- a/sdk/storage/Azure.Storage.DataMovement/tests/BlockBlobStorageResourceTests.cs +++ b/sdk/storage/Azure.Storage.DataMovement/tests/BlockBlobStorageResourceTests.cs @@ -3,6 +3,7 @@ extern alias DMBlobs; using System; +using System.Collections.Generic; using System.IO; using System.Linq; using System.Threading; @@ -16,13 +17,24 @@ using Azure.Storage.DataMovement.Tests; using Azure.Storage.Test; using DMBlobs::Azure.Storage.DataMovement.Blobs; +using Microsoft.CodeAnalysis; +using Microsoft.Extensions.Options; +using Moq; using NUnit.Framework; +using Metadata = System.Collections.Generic.IDictionary; namespace Azure.Storage.DataMovement.Blobs.Tests { public class BlockBlobStorageResourceTests : DataMovementBlobTestBase { - public BlockBlobStorageResourceTests(bool async, BlobClientOptions.ServiceVersion serviceVersion) + private const string DefaultContentType = "text/plain"; + private const string DefaultContentEncoding = "gzip"; + private const string DefaultContentLanguage = "en-US"; + private const string DefaultContentDisposition = "inline"; + private const string DefaultCacheControl = "no-cache"; + private AccessTier DefaultAccessTier = AccessTier.Cold; + + public BlockBlobStorageResourceTests(bool async, BlobClientOptions.ServiceVersion serviceVersion) : base(async, serviceVersion, null /* RecordedTestMode.Record /* to re-record */) { } @@ -223,6 +235,252 @@ await TestHelper.AssertExpectedExceptionAsync( } } + [Test] + public async Task CopyFromStreamAsync_PropertiesDefault() + { + // Arrange + Mock mock = new( + new Uri("https://storageaccount.blob.core.windows.net/container/blob"), + new BlobClientOptions()); + int length = 1024; + var data = GetRandomBuffer(length); + using var stream = new MemoryStream(data); + using var fileContentStream = new MemoryStream(); + mock.Setup(b => b.UploadAsync(It.IsAny(), It.IsAny(), It.IsAny())) + .Callback( + async (stream, options, token) => + { + await stream.CopyToAsync(fileContentStream).ConfigureAwait(false); + fileContentStream.Position = 0; + }) + .Returns(Task.FromResult(Response.FromValue( + BlobsModelFactory.BlobContentInfo( + eTag: new ETag("eTag"), + lastModified: DateTimeOffset.UtcNow, + contentHash: default, + versionId: "version", + encryptionKeySha256: default, + encryptionScope: default, + blobSequenceNumber: default), + new MockResponse(201)))); + + BlockBlobStorageResource storageResource = new BlockBlobStorageResource(mock.Object); + + // Act + IDictionary metadata = DataProvider.BuildMetadata(); + + Dictionary sourceProperties = new() + { + { DataMovementConstants.ResourceProperties.AccessTier, DefaultAccessTier }, + { DataMovementConstants.ResourceProperties.ContentType, DefaultContentType }, + { DataMovementConstants.ResourceProperties.ContentEncoding, DefaultContentEncoding }, + { DataMovementConstants.ResourceProperties.ContentLanguage, DefaultContentLanguage }, + { DataMovementConstants.ResourceProperties.ContentDisposition, DefaultContentDisposition }, + { DataMovementConstants.ResourceProperties.CacheControl, DefaultCacheControl }, + { DataMovementConstants.ResourceProperties.Metadata, metadata } + }; + StorageResourceWriteToOffsetOptions copyFromStreamOptions = new() + { + SourceProperties = new StorageResourceItemProperties( + resourceLength: length, + eTag: new("ETag"), + lastModifiedTime: DateTimeOffset.UtcNow.AddHours(-1), + properties: sourceProperties) + }; + await storageResource.CopyFromStreamInternalAsync( + stream: stream, + streamLength: length, + overwrite: false, + options: copyFromStreamOptions, + completeLength: length); + + Assert.That(data, Is.EqualTo(fileContentStream.AsBytes().ToArray())); + mock.Verify(b => b.UploadAsync( + stream, + It.Is( + options => + options.AccessTier == DefaultAccessTier && + options.HttpHeaders.ContentType == DefaultContentType && + options.HttpHeaders.ContentEncoding == DefaultContentEncoding && + options.HttpHeaders.ContentLanguage == DefaultContentLanguage && + options.HttpHeaders.ContentDisposition == DefaultContentDisposition && + options.HttpHeaders.CacheControl == DefaultCacheControl && + options.Metadata.SequenceEqual(metadata)), + It.IsAny()), + Times.Once()); + mock.VerifyNoOtherCalls(); + } + + [Test] + public async Task CopyFromStreamAsync_PropertiesPreserve() + { + // Arrange + Mock mock = new( + new Uri("https://storageaccount.blob.core.windows.net/container/blob"), + new BlobClientOptions()); + int length = 1024; + var data = GetRandomBuffer(length); + using var stream = new MemoryStream(data); + using var fileContentStream = new MemoryStream(); + mock.Setup(b => b.UploadAsync(It.IsAny(), It.IsAny(), It.IsAny())) + .Callback( + async (stream, options, token) => + { + await stream.CopyToAsync(fileContentStream).ConfigureAwait(false); + fileContentStream.Position = 0; + }) + .Returns(Task.FromResult(Response.FromValue( + BlobsModelFactory.BlobContentInfo( + eTag: new ETag("eTag"), + lastModified: DateTimeOffset.UtcNow, + contentHash: default, + versionId: "version", + encryptionKeySha256: default, + encryptionScope: default, + blobSequenceNumber: default), + new MockResponse(201)))); + BlockBlobStorageResourceOptions resourceOptions = new() + { + AccessTier = new(true), + CacheControl = new(true), + ContentDisposition = new(true), + ContentEncoding = new(true), + ContentLanguage = new(true), + ContentType = new(true), + Metadata = new(true) + }; + BlockBlobStorageResource storageResource = new BlockBlobStorageResource(mock.Object, resourceOptions); + + // Act + IDictionary metadata = DataProvider.BuildMetadata(); + + Dictionary sourceProperties = new() + { + { DataMovementConstants.ResourceProperties.AccessTier, DefaultAccessTier }, + { DataMovementConstants.ResourceProperties.ContentType, DefaultContentType }, + { DataMovementConstants.ResourceProperties.ContentEncoding, DefaultContentEncoding }, + { DataMovementConstants.ResourceProperties.ContentLanguage, DefaultContentLanguage }, + { DataMovementConstants.ResourceProperties.ContentDisposition, DefaultContentDisposition }, + { DataMovementConstants.ResourceProperties.CacheControl, DefaultCacheControl }, + { DataMovementConstants.ResourceProperties.Metadata, metadata } + }; + StorageResourceWriteToOffsetOptions copyFromStreamOptions = new() + { + SourceProperties = new StorageResourceItemProperties( + resourceLength: length, + eTag: new("ETag"), + lastModifiedTime: DateTimeOffset.UtcNow.AddHours(-1), + properties: sourceProperties) + }; + await storageResource.CopyFromStreamInternalAsync( + stream: stream, + streamLength: length, + overwrite: false, + options: copyFromStreamOptions, + completeLength: length); + + Assert.That(data, Is.EqualTo(fileContentStream.AsBytes().ToArray())); + mock.Verify(b => b.UploadAsync( + stream, + It.Is( + options => + options.AccessTier == DefaultAccessTier && + options.HttpHeaders.ContentType == DefaultContentType && + options.HttpHeaders.ContentEncoding == DefaultContentEncoding && + options.HttpHeaders.ContentLanguage == DefaultContentLanguage && + options.HttpHeaders.ContentDisposition == DefaultContentDisposition && + options.HttpHeaders.CacheControl == DefaultCacheControl && + options.Metadata.SequenceEqual(metadata)), + It.IsAny()), + Times.Once()); + mock.VerifyNoOtherCalls(); + } + + [Test] + public async Task CopyFromStreamAsync_PropertiesNoPreserve() + { + // Arrange + Mock mock = new( + new Uri("https://storageaccount.blob.core.windows.net/container/blob"), + new BlobClientOptions()); + int length = 1024; + var data = GetRandomBuffer(length); + using var stream = new MemoryStream(data); + using var fileContentStream = new MemoryStream(); + mock.Setup(b => b.UploadAsync(It.IsAny(), It.IsAny(), It.IsAny())) + .Callback( + async (stream, options, token) => + { + await stream.CopyToAsync(fileContentStream).ConfigureAwait(false); + fileContentStream.Position = 0; + }) + .Returns(Task.FromResult(Response.FromValue( + BlobsModelFactory.BlobContentInfo( + eTag: new ETag("eTag"), + lastModified: DateTimeOffset.UtcNow, + contentHash: default, + versionId: "version", + encryptionKeySha256: default, + encryptionScope: default, + blobSequenceNumber: default), + new MockResponse(201)))); + BlockBlobStorageResourceOptions resourceOptions = new() + { + AccessTier = new(false), + CacheControl = new(false), + ContentDisposition = new(false), + ContentEncoding = new(false), + ContentLanguage = new(false), + ContentType = new(false), + Metadata = new(false) + }; + BlockBlobStorageResource storageResource = new BlockBlobStorageResource(mock.Object, resourceOptions); + + // Act + IDictionary metadata = DataProvider.BuildMetadata(); + + Dictionary sourceProperties = new() + { + { DataMovementConstants.ResourceProperties.AccessTier, DefaultAccessTier }, + { DataMovementConstants.ResourceProperties.ContentType, DefaultContentType }, + { DataMovementConstants.ResourceProperties.ContentEncoding, DefaultContentEncoding }, + { DataMovementConstants.ResourceProperties.ContentLanguage, DefaultContentLanguage }, + { DataMovementConstants.ResourceProperties.ContentDisposition, DefaultContentDisposition }, + { DataMovementConstants.ResourceProperties.CacheControl, DefaultCacheControl }, + { DataMovementConstants.ResourceProperties.Metadata, metadata } + }; + StorageResourceWriteToOffsetOptions copyFromStreamOptions = new() + { + SourceProperties = new StorageResourceItemProperties( + resourceLength: length, + eTag: new("ETag"), + lastModifiedTime: DateTimeOffset.UtcNow.AddHours(-1), + properties: sourceProperties) + }; + await storageResource.CopyFromStreamInternalAsync( + stream: stream, + streamLength: length, + overwrite: false, + options: copyFromStreamOptions, + completeLength: length); + + Assert.That(data, Is.EqualTo(fileContentStream.AsBytes().ToArray())); + mock.Verify(b => b.UploadAsync( + stream, + It.Is( + options => + options.AccessTier == default && + options.HttpHeaders.ContentType == default && + options.HttpHeaders.ContentEncoding == default && + options.HttpHeaders.ContentLanguage == default && + options.HttpHeaders.ContentDisposition == default && + options.HttpHeaders.CacheControl == default && + options.Metadata == default), + It.IsAny()), + Times.Once()); + mock.VerifyNoOtherCalls(); + } + [RecordedTest] public async Task CopyFromUriAsync() { @@ -359,6 +617,416 @@ await TestHelper.AssertExpectedExceptionAsync( }); } + private async Task> CopyFromUriInternalPreserveAsync( + BlockBlobStorageResourceOptions resourceOptions, + Uri sourceUri, + Metadata metadata) + { + // Arrange + Mock mockSource = new( + sourceUri, + new BlobClientOptions()); + mockSource.Setup(b => b.Uri).Returns(sourceUri); + + Mock mockDestination = new( + new Uri("https://storageaccount.blob.core.windows.net/container/destination"), + new BlobClientOptions()); + int length = 1024; + var data = GetRandomBuffer(length); + using var stream = new MemoryStream(data); + using var fileContentStream = new MemoryStream(); + mockDestination.Setup(b => b.SyncUploadFromUriAsync(It.IsAny(), It.IsAny(), It.IsAny())) + .Callback( + async (uri, options, token) => + { + await stream.CopyToAsync(fileContentStream).ConfigureAwait(false); + fileContentStream.Position = 0; + }) + .Returns(Task.FromResult(Response.FromValue( + BlobsModelFactory.BlobContentInfo( + eTag: new ETag("eTag"), + lastModified: DateTimeOffset.UtcNow, + contentHash: default, + versionId: "version", + encryptionKeySha256: default, + encryptionScope: default, + blobSequenceNumber: default), + new MockResponse(201)))); + + BlockBlobStorageResource sourceResource = new BlockBlobStorageResource(mockSource.Object); + BlockBlobStorageResource destinationResource = new BlockBlobStorageResource(mockDestination.Object, resourceOptions); + + // Act + Dictionary sourceProperties = new() + { + { DataMovementConstants.ResourceProperties.AccessTier, DefaultAccessTier }, + { DataMovementConstants.ResourceProperties.ContentType, DefaultContentType }, + { DataMovementConstants.ResourceProperties.ContentEncoding, DefaultContentEncoding }, + { DataMovementConstants.ResourceProperties.ContentLanguage, DefaultContentLanguage }, + { DataMovementConstants.ResourceProperties.ContentDisposition, DefaultContentDisposition }, + { DataMovementConstants.ResourceProperties.CacheControl, DefaultCacheControl }, + { DataMovementConstants.ResourceProperties.Metadata, metadata } + }; + StorageResourceCopyFromUriOptions copyFromUriOptions = new() + { + SourceProperties = new StorageResourceItemProperties( + resourceLength: length, + eTag: new("ETag"), + lastModifiedTime: DateTimeOffset.UtcNow.AddHours(-1), + properties: sourceProperties) + }; + await destinationResource.CopyFromUriInternalAsync( + sourceResource: sourceResource, + overwrite: false, + completeLength: length, + options: copyFromUriOptions); + + Assert.That(data, Is.EqualTo(fileContentStream.AsBytes().ToArray())); + return mockDestination; + } + + private void VerifyHttpHeaders( + Mock mockDestination, + BlobStorageResourceOptions resourceOptions, + Uri expectedUri, + Metadata expectedMetdata) + { + AccessTier? expectedAccessTier = resourceOptions.AccessTier != default ? resourceOptions.AccessTier.Value : DefaultAccessTier; + string expectedContentDisposition = resourceOptions.ContentDisposition != default ? resourceOptions.ContentDisposition.Value : DefaultContentDisposition; + string expectedContentEncoding = resourceOptions.ContentEncoding != default ? resourceOptions.ContentEncoding.Value : DefaultContentEncoding; + string expectedContentLanguage = resourceOptions.ContentLanguage != default ? resourceOptions.ContentLanguage.Value : DefaultContentLanguage; + string expectedContentType = resourceOptions.ContentType != default ? resourceOptions.ContentType.Value : DefaultContentType; + string expectedCacheControl = resourceOptions.CacheControl != default ? resourceOptions.CacheControl.Value : DefaultCacheControl; + Metadata expectedMetadata = resourceOptions.Metadata != default ? resourceOptions.Metadata.Value : expectedMetdata; + + mockDestination.Verify(b => b.SyncUploadFromUriAsync( + expectedUri, + It.Is( + options => + options.CopySourceBlobProperties == false && + options.AccessTier == expectedAccessTier && + options.HttpHeaders.ContentType == expectedContentType && + options.HttpHeaders.ContentEncoding == expectedContentEncoding && + options.HttpHeaders.ContentLanguage == expectedContentLanguage && + options.HttpHeaders.ContentDisposition == expectedContentDisposition && + options.HttpHeaders.CacheControl == expectedCacheControl && + options.Metadata.SequenceEqual(expectedMetadata)), + It.IsAny()), + Times.Once()); + mockDestination.VerifyNoOtherCalls(); + } + + [Test] + public async Task CopyFromUriAsync_PropertiesDefault() + { + Uri sourceUri = new Uri("https://storageaccount.blob.core.windows.net/container/source"); + Metadata metadata = DataProvider.BuildMetadata(); + Mock mockDestination = await CopyFromUriInternalPreserveAsync( + default, + sourceUri, + metadata); + mockDestination.Verify(b => b.SyncUploadFromUriAsync( + sourceUri, + It.Is( + options => + options.CopySourceBlobProperties == default && + options.AccessTier == DefaultAccessTier && + options.HttpHeaders == default && + options.Metadata == default), + It.IsAny()), + Times.Once()); + mockDestination.VerifyNoOtherCalls(); + } + + [Test] + public async Task CopyFromUriAsync_PropertiesPreserve() + { + Uri sourceUri = new Uri("https://storageaccount.blob.core.windows.net/container/source"); + Metadata metadata = DataProvider.BuildMetadata(); + BlockBlobStorageResourceOptions resourceOptions = new() + { + AccessTier = new(true), + ContentDisposition = new(true), + ContentEncoding = new(true), + ContentLanguage = new(true), + ContentType = new(true), + CacheControl = new(true), + Metadata = new(true) + }; + Mock mockDestination = await CopyFromUriInternalPreserveAsync( + default, + sourceUri, + metadata); + mockDestination.Verify(b => b.SyncUploadFromUriAsync( + sourceUri, + It.Is( + options => + options.CopySourceBlobProperties == default && + options.AccessTier == DefaultAccessTier && + options.HttpHeaders == default && + options.Metadata == default), + It.IsAny()), + Times.Once()); + mockDestination.VerifyNoOtherCalls(); + } + + [Test] + public async Task CopyFromUriAsync_NoPreserveContentType() + { + Uri sourceUri = new Uri("https://storageaccount.blob.core.windows.net/container/source"); + Metadata metadata = DataProvider.BuildMetadata(); + BlockBlobStorageResourceOptions resourceOptions = new() + { + ContentType = new(false) + }; + Mock destinationMock = await CopyFromUriInternalPreserveAsync( + resourceOptions, + sourceUri, + metadata); + VerifyHttpHeaders( + destinationMock, + resourceOptions, + sourceUri, + metadata); + } + + [Test] + public async Task CopyFromUriAsync_NoPreserveContentDisposition() + { + Uri sourceUri = new Uri("https://storageaccount.blob.core.windows.net/container/source"); + Metadata metadata = DataProvider.BuildMetadata(); + BlockBlobStorageResourceOptions resourceOptions = new() + { + ContentDisposition = new(false) + }; + Mock destinationMock = await CopyFromUriInternalPreserveAsync( + resourceOptions, + sourceUri, + metadata); + VerifyHttpHeaders( + destinationMock, + resourceOptions, + sourceUri, + metadata); + } + + [Test] + public async Task CopyFromUriAsync_NoPreserveContentLanguage() + { + Uri sourceUri = new Uri("https://storageaccount.blob.core.windows.net/container/source"); + Metadata metadata = DataProvider.BuildMetadata(); + BlockBlobStorageResourceOptions resourceOptions = new() + { + ContentLanguage= new(false) + }; + Mock destinationMock = await CopyFromUriInternalPreserveAsync( + resourceOptions, + sourceUri, + metadata); + VerifyHttpHeaders( + destinationMock, + resourceOptions, + sourceUri, + metadata); + } + + [Test] + public async Task CopyFromUriAsync_NoPreserveCacheControl() + { + Uri sourceUri = new Uri("https://storageaccount.blob.core.windows.net/container/source"); + Metadata metadata = DataProvider.BuildMetadata(); + BlockBlobStorageResourceOptions resourceOptions = new() + { + CacheControl = new(false) + }; + Mock destinationMock = await CopyFromUriInternalPreserveAsync( + resourceOptions, + sourceUri, + metadata); + VerifyHttpHeaders( + destinationMock, + resourceOptions, + sourceUri, + metadata); + } + + [Test] + public async Task CopyFromUriAsync_NoPreserveMetadata() + { + Uri sourceUri = new Uri("https://storageaccount.blob.core.windows.net/container/source"); + Metadata metadata = DataProvider.BuildMetadata(); + BlockBlobStorageResourceOptions resourceOptions = new() + { + Metadata = new(false) + }; + Mock mockDestination = await CopyFromUriInternalPreserveAsync( + resourceOptions, + sourceUri, + metadata); + mockDestination.Verify(b => b.SyncUploadFromUriAsync( + sourceUri, + It.Is( + options => + options.CopySourceBlobProperties == false && + options.AccessTier == DefaultAccessTier && + options.HttpHeaders.ContentType == DefaultContentType && + options.HttpHeaders.ContentEncoding == DefaultContentEncoding && + options.HttpHeaders.ContentLanguage == DefaultContentLanguage && + options.HttpHeaders.ContentDisposition == DefaultContentDisposition && + options.HttpHeaders.CacheControl == DefaultCacheControl && + options.Metadata == default), + It.IsAny()), + Times.Once()); + mockDestination.VerifyNoOtherCalls(); + } + + [Test] + public async Task CopyFromUriAsync_NoPreserveAccessTier() + { + Uri sourceUri = new Uri("https://storageaccount.blob.core.windows.net/container/source"); + Metadata metadata = DataProvider.BuildMetadata(); + BlockBlobStorageResourceOptions resourceOptions = new() + { + AccessTier = new(false) + }; + Mock mockDestination = await CopyFromUriInternalPreserveAsync( + resourceOptions, + sourceUri, + metadata); + mockDestination.Verify(b => b.SyncUploadFromUriAsync( + sourceUri, + It.Is( + options => + options.CopySourceBlobProperties == false && + options.AccessTier == default && + options.HttpHeaders.ContentType == DefaultContentType && + options.HttpHeaders.ContentEncoding == DefaultContentEncoding && + options.HttpHeaders.ContentLanguage == DefaultContentLanguage && + options.HttpHeaders.ContentDisposition == DefaultContentDisposition && + options.HttpHeaders.CacheControl == DefaultCacheControl && + options.Metadata.SequenceEqual(metadata)), + It.IsAny()), + Times.Once()); + mockDestination.VerifyNoOtherCalls(); + } + + [Test] + public async Task CopyFromUriAsync_PropertiesNoPreserve() + { + Uri sourceUri = new Uri("https://storageaccount.blob.core.windows.net/container/source"); + Metadata metadata = DataProvider.BuildMetadata(); + BlockBlobStorageResourceOptions resourceOptions = new() + { + AccessTier = new(false), + CacheControl = new(false), + ContentDisposition = new(false), + ContentEncoding = new(false), + ContentLanguage = new(false), + ContentType = new(false), + Metadata = new(false) + }; + Mock mockDestination = await CopyFromUriInternalPreserveAsync( + resourceOptions, + sourceUri, + metadata); + mockDestination.Verify(b => b.SyncUploadFromUriAsync( + sourceUri, + It.Is( + options => + options.CopySourceBlobProperties == false && + options.AccessTier == default && + options.HttpHeaders.ContentType == default && + options.HttpHeaders.ContentEncoding == default && + options.HttpHeaders.ContentLanguage == default && + options.HttpHeaders.ContentDisposition == default && + options.HttpHeaders.CacheControl == default && + options.Metadata == default), + It.IsAny()), + Times.Once()); + mockDestination.VerifyNoOtherCalls(); + } + + [Test] + public async Task CopyFromUriAsync_SetProperties() + { + // Arrange + Uri sourceUri = new Uri("https://storageaccount.blob.core.windows.net/container/source"); + Mock mockSource = new( + sourceUri, + new BlobClientOptions()); + mockSource.Setup(b => b.Uri).Returns(sourceUri); + + Mock mockDestination = new( + new Uri("https://storageaccount.blob.core.windows.net/container/destination"), + new BlobClientOptions()); + int length = 1024; + var data = GetRandomBuffer(length); + using var stream = new MemoryStream(data); + using var fileContentStream = new MemoryStream(); + Metadata metadata = DataProvider.BuildMetadata(); + mockDestination.Setup(b => b.SyncUploadFromUriAsync(It.IsAny(), It.IsAny(), It.IsAny())) + .Callback( + async (uri, options, token) => + { + await stream.CopyToAsync(fileContentStream).ConfigureAwait(false); + fileContentStream.Position = 0; + }) + .Returns(Task.FromResult(Response.FromValue( + BlobsModelFactory.BlobContentInfo( + eTag: new ETag("eTag"), + lastModified: DateTimeOffset.UtcNow, + contentHash: default, + versionId: "version", + encryptionKeySha256: default, + encryptionScope: default, + blobSequenceNumber: default), + new MockResponse(201)))); + + BlockBlobStorageResource sourceResource = new BlockBlobStorageResource(mockSource.Object); + BlockBlobStorageResourceOptions resourceOptions = new() + { + AccessTier = new(DefaultAccessTier), + CacheControl = new(DefaultCacheControl), + ContentDisposition = new(DefaultContentDisposition), + ContentEncoding = new(DefaultContentEncoding), + ContentLanguage = new(DefaultContentLanguage), + ContentType = new(DefaultContentType), + Metadata = new(metadata) + }; + BlockBlobStorageResource destinationResource = new BlockBlobStorageResource(mockDestination.Object, resourceOptions); + + // Act + StorageResourceCopyFromUriOptions copyFromUriOptions = new() + { + SourceProperties = new StorageResourceItemProperties( + resourceLength: length, + eTag: new("ETag"), + lastModifiedTime: DateTimeOffset.UtcNow.AddHours(-1), + properties: default) + }; + await destinationResource.CopyFromUriInternalAsync( + sourceResource: sourceResource, + overwrite: false, + completeLength: length, + options: copyFromUriOptions); + + Assert.That(data, Is.EqualTo(fileContentStream.AsBytes().ToArray())); + mockDestination.Verify(b => b.SyncUploadFromUriAsync( + sourceUri, + It.Is( + options => + options.AccessTier == DefaultAccessTier && + options.HttpHeaders.ContentType == DefaultContentType && + options.HttpHeaders.ContentEncoding == DefaultContentEncoding && + options.HttpHeaders.ContentLanguage == DefaultContentLanguage && + options.HttpHeaders.ContentDisposition == DefaultContentDisposition && + options.HttpHeaders.CacheControl == DefaultCacheControl && + options.Metadata.SequenceEqual(metadata)), + It.IsAny()), + Times.Once()); + mockDestination.VerifyNoOtherCalls(); + } + [RecordedTest] public async Task CopyBlockFromUriAsync() { @@ -602,6 +1270,107 @@ await TestHelper.AssertExpectedExceptionAsync( }); } + [Test] + public async Task GetPropertiesAsync_NotCached() + { + // Arrange + Mock mock = new( + new Uri("https://storageaccount.file.core.windows.net/container/file"), + new BlobClientOptions()); + + long length = 1024; + ETag eTag = new ETag("etag"); + string source = "https://storageaccount.file.core.windows.net/container/file2"; + Metadata metadata = DataProvider.BuildMetadata(); + mock.Setup(b => b.GetPropertiesAsync(It.IsAny(), It.IsAny())) + .Returns(Task.FromResult(Response.FromValue( + BlobsModelFactory.BlobProperties( + lastModified: DateTime.MinValue, + leaseStatus: LeaseStatus.Unlocked, + contentLength: length, + eTag: eTag, + contentEncoding: DefaultContentEncoding, + contentDisposition: DefaultContentDisposition, + contentLanguage: DefaultContentLanguage, + contentType: DefaultContentType, + cacheControl: DefaultCacheControl, + copySource: new Uri(source), + accessTier: DefaultAccessTier.ToString(), + copyCompletedOn: DateTimeOffset.MinValue, + accessTierChangedOn: DateTimeOffset.MinValue, + blobType: BlobType.Block, + metadata: metadata, + tagCount: 5), + new MockResponse(200)))); + + BlockBlobStorageResource storageResource = new BlockBlobStorageResource(mock.Object); + + // Act + StorageResourceItemProperties result = await storageResource.GetPropertiesInternalAsync(); + AccessTier accessTierResult = (AccessTier) result.RawProperties[DataMovementConstants.ResourceProperties.AccessTier]; + string contentEncodingResult = (string) result.RawProperties[DataMovementConstants.ResourceProperties.ContentEncoding]; + string contentDispositionResult = (string) result.RawProperties[DataMovementConstants.ResourceProperties.ContentDisposition]; + string contentLanguageResult = (string) result.RawProperties[DataMovementConstants.ResourceProperties.ContentLanguage]; + string contentTypeResult = (string) result.RawProperties[DataMovementConstants.ResourceProperties.ContentType]; + string cacheControlResult = (string) result.RawProperties[DataMovementConstants.ResourceProperties.CacheControl]; + Metadata metadataResult = (Metadata) result.RawProperties[DataMovementConstants.ResourceProperties.Metadata]; + + // Assert + Assert.AreEqual(eTag, result.ETag); + Assert.AreEqual(length, result.ResourceLength); + Assert.AreEqual(accessTierResult, DefaultAccessTier); + Assert.AreEqual(contentEncodingResult, DefaultContentEncoding); + Assert.AreEqual(contentDispositionResult, DefaultContentDisposition); + Assert.AreEqual(contentLanguageResult, DefaultContentLanguage); + Assert.AreEqual(contentTypeResult, DefaultContentType); + Assert.AreEqual(cacheControlResult, DefaultCacheControl); + Assert.That(metadata, Is.EqualTo(metadataResult)); + mock.Verify(b => b.GetPropertiesAsync(It.IsAny(), It.IsAny()), + Times.Once()); + mock.VerifyNoOtherCalls(); + } + + [Test] + public async Task GetPropertiesAsync_Cached() + { + // Arrange + Mock mock = new( + new Uri("https://storageaccount.file.core.windows.net/container/file"), + new BlobClientOptions()); + + long length = 1024; + ETag eTag = new ETag("etag"); + DateTimeOffset lastModified = DateTimeOffset.UtcNow.AddHours(-1); + Metadata metadata = DataProvider.BuildMetadata(); + Dictionary rawProperties = new() + { + { DataMovementConstants.ResourceProperties.AccessTier, DefaultAccessTier }, + { DataMovementConstants.ResourceProperties.ContentEncoding, DefaultContentEncoding }, + { DataMovementConstants.ResourceProperties.ContentDisposition, DefaultContentDisposition}, + { DataMovementConstants.ResourceProperties.ContentLanguage, DefaultContentLanguage}, + { DataMovementConstants.ResourceProperties.ContentType, DefaultContentType}, + { DataMovementConstants.ResourceProperties.CacheControl, DefaultCacheControl}, + { DataMovementConstants.ResourceProperties.Metadata, metadata }, + }; + + BlockBlobStorageResource storageResource = new BlockBlobStorageResource( + mock.Object, + new StorageResourceItemProperties( + length, + eTag, + lastModified, + rawProperties)); + + // Act + StorageResourceItemProperties result = await storageResource.GetPropertiesInternalAsync(); + + // Assert + Assert.That(rawProperties, Is.EqualTo(result.RawProperties)); + mock.Verify(b => b.GetPropertiesAsync(It.IsAny(), It.IsAny()), + Times.Never()); + mock.VerifyNoOtherCalls(); + } + [RecordedTest] public async Task CompleteTransferAsync() { @@ -664,6 +1433,373 @@ await storageResource.CopyFromStreamAsync( Assert.IsTrue(await blobClient.ExistsAsync()); } + [Test] + public async Task CopyFromUriAsync_DefaultMetadata() + { + // Arrange + Uri sourceUri = new Uri("https://storageaccount.blob.core.windows.net/container/source"); + Mock mockSource = new( + sourceUri, + new BlobClientOptions()); + mockSource.Setup(b => b.Uri).Returns(sourceUri); + Mock mockDestination = new( + new Uri("https://storageaccount.blob.core.windows.net/container/destination"), + new BlobClientOptions()); + int length = 1024; + var data = GetRandomBuffer(length); + using var stream = new MemoryStream(data); + using var fileContentStream = new MemoryStream(); + mockDestination.Setup(b => b.SyncUploadFromUriAsync(It.IsAny(), It.IsAny(), It.IsAny())) + .Returns(Task.FromResult(Response.FromValue( + BlobsModelFactory.BlobContentInfo( + eTag: new ETag("eTag"), + lastModified: DateTimeOffset.UtcNow, + contentHash: default, + versionId: "version", + encryptionKeySha256: default, + encryptionScope: default, + blobSequenceNumber: default), + new MockResponse(201)))); + mockDestination.Setup(b => b.SetMetadataAsync(It.IsAny(), It.IsAny(), It.IsAny())) + .Returns(Task.FromResult(Response.FromValue( + BlobsModelFactory.BlobInfo( + eTag: new ETag("eTag"), + lastModified: DateTimeOffset.UtcNow), + new MockResponse(200)))); + + BlockBlobStorageResource sourceResource = new BlockBlobStorageResource(mockSource.Object); + BlockBlobStorageResource destinationResource = new BlockBlobStorageResource(mockDestination.Object); + + // Act + IDictionary sourceMetdata = DataProvider.BuildMetadata(); + Dictionary rawProperties = new() + { + { DataMovementConstants.ResourceProperties.Metadata, sourceMetdata }, + }; + StorageResourceItemProperties sourceProperties = new( + length, + new ETag("etag"), + DateTimeOffset.UtcNow.AddHours(-1), + rawProperties); + StorageResourceCopyFromUriOptions copyFromUriOptions = new() + { + SourceProperties = new StorageResourceItemProperties( + resourceLength: length, + eTag: new("ETag"), + lastModifiedTime: DateTimeOffset.UtcNow.AddHours(-1), + properties: rawProperties) + }; + await destinationResource.CopyFromUriInternalAsync( + sourceResource, + overwrite: false, + completeLength: length, + options: copyFromUriOptions, + cancellationToken: default); + + // Assert + mockDestination.Verify(b => b.SyncUploadFromUriAsync( + sourceUri, + It.IsAny(), + It.IsAny()), + Times.Once()); + mockDestination.Verify(b => b.SetMetadataAsync( + default, + It.IsAny(), + It.IsAny()), + Times.Never()); + mockDestination.VerifyNoOtherCalls(); + } + + [Test] + public async Task CompleteTransferAsync_PropertiesDefaultChunks() + { + // Arrange + Mock mockDestination = new( + new Uri("https://storageaccount.blob.core.windows.net/container/destination"), + new BlobClientOptions()); + int blockLength = 512; + int completeLength = 1024; + var data = GetRandomBuffer(completeLength); + using var stream = new MemoryStream(data); + using var fileContentStream = new MemoryStream(); + mockDestination.Setup(b => b.StageBlockAsync(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny())) + .Returns(Task.FromResult(Response.FromValue( + BlobsModelFactory.BlockInfo( + contentHash: default, + contentCrc64: default, + encryptionKeySha256: default, + encryptionScope: default), + new MockResponse(201)))); + mockDestination.Setup(b => b.CommitBlockListAsync(It.IsAny>(), It.IsAny(), It.IsAny())) + .Returns(Task.FromResult(Response.FromValue( + BlobsModelFactory.BlobContentInfo( + eTag: new ETag("eTag"), + lastModified: DateTimeOffset.UtcNow, + contentHash: default, + versionId: "version", + encryptionKeySha256: default, + encryptionScope: default, + blobSequenceNumber: default), + new MockResponse(201)))); + + BlockBlobStorageResource destinationResource = new BlockBlobStorageResource(mockDestination.Object); + await destinationResource.CopyFromStreamInternalAsync( + stream: stream, + streamLength: blockLength, + overwrite: false, + options: new(){ Position = 0 }, + completeLength: completeLength); + await destinationResource.CopyFromStreamInternalAsync( + stream: stream, + streamLength: blockLength, + overwrite: false, + options: new() { Position = blockLength }, + completeLength: completeLength); + + // Act + IDictionary metadata = DataProvider.BuildMetadata(); + Dictionary rawProperties = new() + { + { DataMovementConstants.ResourceProperties.AccessTier, DefaultAccessTier }, + { DataMovementConstants.ResourceProperties.ContentType, DefaultContentType }, + { DataMovementConstants.ResourceProperties.ContentEncoding, DefaultContentEncoding }, + { DataMovementConstants.ResourceProperties.ContentLanguage, DefaultContentLanguage }, + { DataMovementConstants.ResourceProperties.ContentDisposition, DefaultContentDisposition }, + { DataMovementConstants.ResourceProperties.CacheControl, DefaultCacheControl }, + { DataMovementConstants.ResourceProperties.Metadata, metadata } + }; + StorageResourceItemProperties sourceProperties = new( + completeLength, + new ETag("etag"), + DateTimeOffset.UtcNow.AddHours(-1), + rawProperties); + await destinationResource.CompleteTransferAsync( + overwrite: false, + completeTransferOptions: new() { SourceProperties = sourceProperties }); + + // Assert + mockDestination.Verify(b => b.CommitBlockListAsync( + It.IsAny>(), + It.Is( + options => + options.AccessTier == DefaultAccessTier && + options.HttpHeaders.ContentType == DefaultContentType && + options.HttpHeaders.ContentEncoding == DefaultContentEncoding && + options.HttpHeaders.ContentLanguage == DefaultContentLanguage && + options.HttpHeaders.ContentDisposition == DefaultContentDisposition && + options.HttpHeaders.CacheControl == DefaultCacheControl && + options.Metadata.SequenceEqual(metadata)), + It.IsAny()), + Times.Once()); + mockDestination.Verify(b => b.StageBlockAsync( + It.IsAny(), + It.IsAny(), + It.IsAny(), + It.IsAny()), + Times.Exactly(2)); + mockDestination.VerifyNoOtherCalls(); + } + + [Test] + public async Task CompleteTransferAsync_PropertiesPreserveChunks() + { + // Arrange + Mock mockDestination = new( + new Uri("https://storageaccount.blob.core.windows.net/container/destination"), + new BlobClientOptions()); + int blockLength = 512; + int completeLength = 1024; + var data = GetRandomBuffer(completeLength); + using var stream = new MemoryStream(data); + using var fileContentStream = new MemoryStream(); + mockDestination.Setup(b => b.StageBlockAsync(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny())) + .Returns(Task.FromResult(Response.FromValue( + BlobsModelFactory.BlockInfo( + contentHash: default, + contentCrc64: default, + encryptionKeySha256: default, + encryptionScope: default), + new MockResponse(201)))); + mockDestination.Setup(b => b.CommitBlockListAsync(It.IsAny>(), It.IsAny(), It.IsAny())) + .Returns(Task.FromResult(Response.FromValue( + BlobsModelFactory.BlobContentInfo( + eTag: new ETag("eTag"), + lastModified: DateTimeOffset.UtcNow, + contentHash: default, + versionId: "version", + encryptionKeySha256: default, + encryptionScope: default, + blobSequenceNumber: default), + new MockResponse(201)))); + + BlockBlobStorageResourceOptions resourceOptions = new() + { + AccessTier = new(true), + CacheControl = new(true), + ContentDisposition = new(true), + ContentEncoding = new(true), + ContentLanguage = new(true), + ContentType = new(true), + Metadata = new(true) + }; + BlockBlobStorageResource destinationResource = new BlockBlobStorageResource(mockDestination.Object, resourceOptions); + await destinationResource.CopyFromStreamInternalAsync( + stream: stream, + streamLength: blockLength, + overwrite: false, + options: new() { Position = 0 }, + completeLength: completeLength); + await destinationResource.CopyFromStreamInternalAsync( + stream: stream, + streamLength: blockLength, + overwrite: false, + options: new() { Position = blockLength }, + completeLength: completeLength); + + // Act + IDictionary metadata = DataProvider.BuildMetadata(); + Dictionary rawProperties = new() + { + { DataMovementConstants.ResourceProperties.AccessTier, DefaultAccessTier }, + { DataMovementConstants.ResourceProperties.ContentType, DefaultContentType }, + { DataMovementConstants.ResourceProperties.ContentEncoding, DefaultContentEncoding }, + { DataMovementConstants.ResourceProperties.ContentLanguage, DefaultContentLanguage }, + { DataMovementConstants.ResourceProperties.ContentDisposition, DefaultContentDisposition }, + { DataMovementConstants.ResourceProperties.CacheControl, DefaultCacheControl }, + { DataMovementConstants.ResourceProperties.Metadata, metadata } + }; + StorageResourceItemProperties sourceProperties = new( + completeLength, + new ETag("etag"), + DateTimeOffset.UtcNow.AddHours(-1), + rawProperties); + await destinationResource.CompleteTransferAsync( + overwrite: false, + completeTransferOptions: new() { SourceProperties = sourceProperties }); + + // Assert + mockDestination.Verify(b => b.CommitBlockListAsync( + It.IsAny>(), + It.Is( + options => + options.AccessTier == DefaultAccessTier && + options.HttpHeaders.ContentType == DefaultContentType && + options.HttpHeaders.ContentEncoding == DefaultContentEncoding && + options.HttpHeaders.ContentLanguage == DefaultContentLanguage && + options.HttpHeaders.ContentDisposition == DefaultContentDisposition && + options.HttpHeaders.CacheControl == DefaultCacheControl && + options.Metadata.SequenceEqual(metadata)), + It.IsAny()), + Times.Once()); + mockDestination.Verify(b => b.StageBlockAsync( + It.IsAny(), + It.IsAny(), + It.IsAny(), + It.IsAny()), + Times.Exactly(2)); + mockDestination.VerifyNoOtherCalls(); + } + + [Test] + public async Task CompleteTransferAsync_PropertiesNoPreserveChunks() + { + // Arrange + Mock mockDestination = new( + new Uri("https://storageaccount.blob.core.windows.net/container/destination"), + new BlobClientOptions()); + int blockLength = 512; + int completeLength = 1024; + var data = GetRandomBuffer(completeLength); + using var stream = new MemoryStream(data); + using var fileContentStream = new MemoryStream(); + mockDestination.Setup(b => b.StageBlockAsync(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny())) + .Returns(Task.FromResult(Response.FromValue( + BlobsModelFactory.BlockInfo( + contentHash: default, + contentCrc64: default, + encryptionKeySha256: default, + encryptionScope: default), + new MockResponse(201)))); + mockDestination.Setup(b => b.CommitBlockListAsync(It.IsAny>(), It.IsAny(), It.IsAny())) + .Returns(Task.FromResult(Response.FromValue( + BlobsModelFactory.BlobContentInfo( + eTag: new ETag("eTag"), + lastModified: DateTimeOffset.UtcNow, + contentHash: default, + versionId: "version", + encryptionKeySha256: default, + encryptionScope: default, + blobSequenceNumber: default), + new MockResponse(201)))); + + BlockBlobStorageResourceOptions resourceOptions = new() + { + AccessTier = new(false), + CacheControl = new(false), + ContentDisposition = new(false), + ContentEncoding = new(false), + ContentLanguage = new(false), + ContentType = new(false), + Metadata = new(false) + }; + BlockBlobStorageResource destinationResource = new BlockBlobStorageResource(mockDestination.Object, resourceOptions); + await destinationResource.CopyFromStreamInternalAsync( + stream: stream, + streamLength: blockLength, + overwrite: false, + options: new() { Position = 0 }, + completeLength: completeLength); + await destinationResource.CopyFromStreamInternalAsync( + stream: stream, + streamLength: blockLength, + overwrite: false, + options: new() { Position = blockLength }, + completeLength: completeLength); + + // Act + IDictionary metadata = DataProvider.BuildMetadata(); + Dictionary rawProperties = new() + { + { DataMovementConstants.ResourceProperties.AccessTier, DefaultAccessTier }, + { DataMovementConstants.ResourceProperties.ContentType, DefaultContentType }, + { DataMovementConstants.ResourceProperties.ContentEncoding, DefaultContentEncoding }, + { DataMovementConstants.ResourceProperties.ContentLanguage, DefaultContentLanguage }, + { DataMovementConstants.ResourceProperties.ContentDisposition, DefaultContentDisposition }, + { DataMovementConstants.ResourceProperties.CacheControl, DefaultCacheControl }, + { DataMovementConstants.ResourceProperties.Metadata, metadata } + }; + StorageResourceItemProperties sourceProperties = new( + completeLength, + new ETag("etag"), + DateTimeOffset.UtcNow.AddHours(-1), + rawProperties); + await destinationResource.CompleteTransferAsync( + overwrite: false, + completeTransferOptions: new() { SourceProperties = sourceProperties }); + + // Assert + mockDestination.Verify(b => b.CommitBlockListAsync( + It.IsAny>(), + It.Is( + options => + options.AccessTier == default && + options.HttpHeaders.ContentType == default && + options.HttpHeaders.ContentEncoding == default && + options.HttpHeaders.ContentLanguage == default && + options.HttpHeaders.ContentDisposition == default && + options.HttpHeaders.CacheControl == default && + options.Metadata == default), + It.IsAny()), + Times.Once()); + mockDestination.Verify(b => b.StageBlockAsync( + It.IsAny(), + It.IsAny(), + It.IsAny(), + It.IsAny()), + Times.Exactly(2)); + mockDestination.VerifyNoOtherCalls(); + } + [RecordedTest] public async Task GetCopyAuthorizationHeaderAsync() { diff --git a/sdk/storage/Azure.Storage.DataMovement/tests/CleanUpTransferTests.cs b/sdk/storage/Azure.Storage.DataMovement/tests/CleanUpTransferTests.cs index 325ce2b0bd16c..ecf1e0742c7a6 100644 --- a/sdk/storage/Azure.Storage.DataMovement/tests/CleanUpTransferTests.cs +++ b/sdk/storage/Azure.Storage.DataMovement/tests/CleanUpTransferTests.cs @@ -2,6 +2,7 @@ // Licensed under the MIT License. using System; +using System.Collections.Generic; using System.Threading; using System.Threading.Tasks; using Moq; @@ -52,7 +53,7 @@ private Mock GetRemoteDestinationResource(bool throwOnDelet .Returns(new MockResourceCheckpointData()); mock.Setup(b => b.GetDestinationCheckpointData()) .Returns(new MockResourceCheckpointData()); - mock.Setup(b => b.CompleteTransferAsync(It.IsAny(), It.IsAny())) + mock.Setup(b => b.CompleteTransferAsync(It.IsAny(), It.IsAny(), It.IsAny())) .Returns(Task.CompletedTask); // Throw a failure when doing a CopyFromUri call to trigger a failed state mock.Setup(b => b.CopyFromUriAsync(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny())) @@ -119,7 +120,7 @@ public async Task CleanupAfterFailureAsync() sourceMock.Object, false, sourceLength, - default, + It.IsAny(), It.IsAny()), Times.Once()); destMock.Verify(b => b.DeleteIfExistsAsync(It.IsAny()), @@ -158,7 +159,7 @@ public async Task ErrorThrownDuringCleanup() sourceMock.Object, false, sourceLength, - default, + It.IsAny(), It.IsAny()), Times.Once()); destMock.Verify(b => b.DeleteIfExistsAsync(It.IsAny()), diff --git a/sdk/storage/Azure.Storage.DataMovement/tests/CommitChunkHandlerTests.cs b/sdk/storage/Azure.Storage.DataMovement/tests/CommitChunkHandlerTests.cs index e9f160fab129c..116f62214d1aa 100644 --- a/sdk/storage/Azure.Storage.DataMovement/tests/CommitChunkHandlerTests.cs +++ b/sdk/storage/Azure.Storage.DataMovement/tests/CommitChunkHandlerTests.cs @@ -8,7 +8,7 @@ using NUnit.Framework; using System.Threading; using Azure.Core; -using Azure.Storage.Tests.Shared; +using Azure.Storage.Test; using Azure.Core.Pipeline; namespace Azure.Storage.DataMovement.Tests @@ -16,6 +16,12 @@ namespace Azure.Storage.DataMovement.Tests [TestFixture] public class CommitChunkHandlerTests { + private const string DefaultContentType = "text/plain"; + private const string DefaultContentEncoding = "gzip"; + private const string DefaultContentLanguage = "en-US"; + private const string DefaultContentDisposition = "inline"; + private const string DefaultCacheControl = "no-cache"; + private readonly int _maxDelayInSec = 1; private readonly string _failedEventMsg = "Amount of Failed Event Handler calls was incorrect."; private readonly string _putBlockMsg = "Amount of Put Block Task calls were incorrect"; @@ -77,7 +83,7 @@ public CommitChunkHandlerTests() { } private Mock GetPutBlockTask() { var mock = new Mock(MockBehavior.Strict); - mock.Setup(del => del(It.IsNotNull(), It.IsNotNull(), It.IsNotNull())) + mock.Setup(del => del(It.IsNotNull(), It.IsNotNull(), It.IsNotNull(), It.IsAny())) .Returns(Task.CompletedTask); return mock; } @@ -85,7 +91,7 @@ public CommitChunkHandlerTests() { } private Mock GetExceptionPutBlockTask() { var mock = new Mock(MockBehavior.Strict); - mock.Setup(del => del(It.IsNotNull(), It.IsNotNull(), It.IsNotNull())) + mock.Setup(del => del(It.IsNotNull(), It.IsNotNull(), It.IsNotNull(), It.IsNotNull())) .Throws(new RequestFailedException("Mock Request Error")); return mock; } @@ -93,7 +99,7 @@ public CommitChunkHandlerTests() { } private Mock GetCommitBlockTask() { var mock = new Mock(MockBehavior.Strict); - mock.Setup(del => del()) + mock.Setup(del => del(It.IsAny())) .Returns(Task.CompletedTask); return mock; } @@ -101,7 +107,7 @@ public CommitChunkHandlerTests() { } private Mock GetExceptionCommitBlockTask() { var mock = new Mock(MockBehavior.Strict); - mock.Setup(del => del()) + mock.Setup(del => del(It.IsAny())) .Throws(new RequestFailedException("Mock Request Error")); return mock; } @@ -161,6 +167,7 @@ public async Task OneChunkTransfer(long blockSize) }, DataTransferOrder.Unordered, ClientDiagnostics, + default, CancellationToken.None); // Make one chunk that would meet the expected length @@ -203,6 +210,7 @@ public async Task ParallelChunkTransfer(long blockSize) }, DataTransferOrder.Unordered, ClientDiagnostics, + default, CancellationToken.None); // Make one chunk that would update the bytes but not cause a commit block list to occur @@ -265,6 +273,7 @@ public async Task ParallelChunkTransfer_ExceedError(long blockSize) }, DataTransferOrder.Unordered, ClientDiagnostics, + default, CancellationToken.None); // Make one chunk that would update the bytes that would cause the bytes to exceed the expected amount @@ -310,6 +319,7 @@ public async Task ParallelChunkTransfer_MultipleProcesses(long blockSize, int ta }, DataTransferOrder.Unordered, ClientDiagnostics, + default, CancellationToken.None); List runningTasks = new List(); @@ -362,6 +372,7 @@ public async Task SequentialChunkTransfer(long blockSize) }, DataTransferOrder.Sequential, ClientDiagnostics, + default, CancellationToken.None); // Make one chunk that would update the bytes but not cause a commit block list to occur @@ -424,6 +435,7 @@ public async Task SequentialChunkTransfer_ExceedError(long blockSize) }, DataTransferOrder.Sequential, ClientDiagnostics, + default, CancellationToken.None); // Make one chunk that would update the bytes that would cause the bytes to exceed the expected amount @@ -468,6 +480,7 @@ public async Task GetPutBlockTask_ExpectedFailure() }, transferOrder: DataTransferOrder.Sequential, ClientDiagnostics, + default, CancellationToken.None); // Act @@ -510,6 +523,7 @@ public async Task GetCommitBlockTask_ExpectedFailure() }, transferOrder: DataTransferOrder.Unordered, ClientDiagnostics, + default, CancellationToken.None); // Act @@ -534,7 +548,7 @@ await commitBlockHandler.InvokeEvent(new StageChunkEventArgs( [Test] public async Task DisposedEventHandler() { - // Arrange - Create DownloadChunkHandler then Dispose it so the event handler is disposed + // Arrange - Create CommitChunkHandler then Dispose it so the event handler is disposed MockCommitChunkBehaviors mockCommitChunkBehaviors = GetCommitChunkBehaviors(); int blockSize = 512; long expectedLength = blockSize * 2; @@ -551,6 +565,7 @@ public async Task DisposedEventHandler() }, transferOrder: DataTransferOrder.Unordered, ClientDiagnostics, + default, CancellationToken.None); // Act @@ -566,5 +581,66 @@ public async Task DisposedEventHandler() expectedReportProgressCount: 0, expectedCompleteFileCount: 0); } + + [Test] + public async Task CompleteTransferTask_Properties() + { + // Set up tasks + MockCommitChunkBehaviors mockCommitChunkBehaviors = GetCommitChunkBehaviors(); + int blockSize = 512; + long expectedLength = blockSize * 2; + + IDictionary metadata = DataProvider.BuildMetadata(); + IDictionary tags = DataProvider.BuildTags(); + Dictionary sourceProperties = new() + { + { "ContentType", DefaultContentType }, + { "ContentEncoding", DefaultContentEncoding }, + { "ContentLanguage", DefaultContentLanguage }, + { "ContentDisposition", DefaultContentDisposition }, + { "CacheControl", DefaultCacheControl }, + { "Metadata", metadata }, + { "Tags", tags } + }; + StorageResourceItemProperties properties = new( + resourceLength: expectedLength, + eTag: new ETag("etag"), + lastModifiedTime: DateTimeOffset.UtcNow.AddHours(-1), + properties: sourceProperties); + var commitBlockHandler = new CommitChunkHandler( + expectedLength: expectedLength, + blockSize: blockSize, + new CommitChunkHandler.Behaviors + { + QueuePutBlockTask = mockCommitChunkBehaviors.PutBlockTask.Object, + QueueCommitBlockTask = mockCommitChunkBehaviors.QueueCommitBlockTask.Object, + ReportProgressInBytes = mockCommitChunkBehaviors.ReportProgressInBytesTask.Object, + InvokeFailedHandler = mockCommitChunkBehaviors.InvokeFailedEventHandlerTask.Object, + }, + DataTransferOrder.Unordered, + ClientDiagnostics, + properties, + CancellationToken.None); + + // Make one chunk that would meet the expected length + await commitBlockHandler.InvokeEvent(new StageChunkEventArgs( + transferId: "fake-id", + success: true, + // Before commit block is called, one block chunk has already been added when creating the destination + offset: blockSize, + bytesTransferred: blockSize, + exception: default, + isRunningSynchronously: false, + cancellationToken: CancellationToken.None)); + + VerifyDelegateInvocations( + behaviors: mockCommitChunkBehaviors, + expectedFailureCount: 0, + expectedPutBlockCount: 0, + expectedReportProgressCount: 1, + expectedCompleteFileCount: 1); + + mockCommitChunkBehaviors.QueueCommitBlockTask.Verify(b => b(properties)); + } } } diff --git a/sdk/storage/Azure.Storage.DataMovement/tests/MockStorageResource.cs b/sdk/storage/Azure.Storage.DataMovement/tests/MockStorageResource.cs index 7d7a64d12ff22..0180982aff831 100644 --- a/sdk/storage/Azure.Storage.DataMovement/tests/MockStorageResource.cs +++ b/sdk/storage/Azure.Storage.DataMovement/tests/MockStorageResource.cs @@ -44,7 +44,10 @@ public static MockStorageResource MakeDestinationResource(Uri uri = default, Dat return new MockStorageResource(default, uri, failAfter, transferOrder); } - protected internal override Task CompleteTransferAsync(bool overwrite, CancellationToken cancellationToken = default) + protected internal override Task CompleteTransferAsync( + bool overwrite, + StorageResourceCompleteTransferOptions completeTransferOptions = default, + CancellationToken cancellationToken = default) { return Task.CompletedTask; } diff --git a/sdk/storage/Azure.Storage.DataMovement/tests/PageBlobStorageResourceTests.cs b/sdk/storage/Azure.Storage.DataMovement/tests/PageBlobStorageResourceTests.cs index b8554c0eeae20..586da1bd66c72 100644 --- a/sdk/storage/Azure.Storage.DataMovement/tests/PageBlobStorageResourceTests.cs +++ b/sdk/storage/Azure.Storage.DataMovement/tests/PageBlobStorageResourceTests.cs @@ -3,6 +3,7 @@ extern alias DMBlobs; using System; +using System.Collections.Generic; using System.IO; using System.Linq; using System.Net; @@ -17,12 +18,20 @@ using Azure.Storage.DataMovement.Tests; using Azure.Storage.Test; using DMBlobs::Azure.Storage.DataMovement.Blobs; +using Moq; using NUnit.Framework; +using Metadata = System.Collections.Generic.IDictionary; namespace Azure.Storage.DataMovement.Blobs.Tests { public class PageBlobStorageResourceTests : DataMovementBlobTestBase { + private const string DefaultContentType = "text/plain"; + private const string DefaultContentEncoding = "gzip"; + private const string DefaultContentLanguage = "en-US"; + private const string DefaultContentDisposition = "inline"; + private const string DefaultCacheControl = "no-cache"; + public PageBlobStorageResourceTests(bool async, BlobClientOptions.ServiceVersion serviceVersion) : base(async, serviceVersion, null /* RecordedTestMode.Record /* to re-record */) { } @@ -236,6 +245,295 @@ await TestHelper.AssertExpectedExceptionAsync( } } + [Test] + public async Task CopyFromStreamAsync_PropertiesDefault() + { + // Arrange + Mock mock = new( + new Uri("https://storageaccount.blob.core.windows.net/container/destination"), + new BlobClientOptions()); + + int length = 1024; + var data = GetRandomBuffer(length); + using var stream = new MemoryStream(data); + using var fileContentStream = new MemoryStream(); + mock.Setup(b => b.CreateAsync(It.IsAny(), It.IsAny(), It.IsAny())) + .Returns(Task.FromResult(Response.FromValue( + BlobsModelFactory.BlobContentInfo( + eTag: new ETag("eTag"), + lastModified: DateTimeOffset.UtcNow, + contentHash: default, + versionId: "version", + encryptionKeySha256: default, + encryptionScope: default, + blobSequenceNumber: default), + new MockResponse(201)))); + mock.Setup(b => b.UploadPagesAsync(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny())) + .Callback( + async (stream, offset, options, token) => + { + await stream.CopyToAsync(fileContentStream).ConfigureAwait(false); + fileContentStream.Position = 0; + }) + .Returns(Task.FromResult(Response.FromValue( + BlobsModelFactory.PageInfo( + eTag: new ETag("eTag"), + lastModified: DateTimeOffset.UtcNow, + contentHash: default, + contentCrc64: default, + encryptionKeySha256: default, + encryptionScope: default, + blobSequenceNumber: default), + new MockResponse(201)))); + + PageBlobStorageResource destinationResource = new PageBlobStorageResource(mock.Object); + + // Act + IDictionary metadata = DataProvider.BuildMetadata(); + + Dictionary sourceProperties = new() + { + { DataMovementConstants.ResourceProperties.ContentType, DefaultContentType }, + { DataMovementConstants.ResourceProperties.ContentEncoding, DefaultContentEncoding }, + { DataMovementConstants.ResourceProperties.ContentLanguage, DefaultContentLanguage }, + { DataMovementConstants.ResourceProperties.ContentDisposition, DefaultContentDisposition }, + { DataMovementConstants.ResourceProperties.CacheControl, DefaultCacheControl }, + { DataMovementConstants.ResourceProperties.Metadata, metadata } + }; + StorageResourceWriteToOffsetOptions copyFromStreamOptions = new() + { + SourceProperties = new StorageResourceItemProperties( + resourceLength: length, + eTag: new("ETag"), + lastModifiedTime: DateTimeOffset.UtcNow.AddHours(-1), + properties: sourceProperties) + }; + await destinationResource.CopyFromStreamInternalAsync( + stream, + length, + false, + length, + copyFromStreamOptions); + + Assert.That(data, Is.EqualTo(fileContentStream.AsBytes().ToArray())); + mock.Verify(b => b.CreateAsync( + length, + It.Is( + options => + options.HttpHeaders.ContentType == DefaultContentType && + options.HttpHeaders.ContentEncoding == DefaultContentEncoding && + options.HttpHeaders.ContentLanguage == DefaultContentLanguage && + options.HttpHeaders.ContentDisposition == DefaultContentDisposition && + options.HttpHeaders.CacheControl == DefaultCacheControl && + options.Metadata.SequenceEqual(metadata)), + It.IsAny()), + Times.Once()); + mock.Verify(b => b.UploadPagesAsync( + stream, + 0, + It.IsAny(), + It.IsAny()), + Times.Once()); + mock.VerifyNoOtherCalls(); + } + + [Test] + public async Task CopyFromStreamAsync_PropertiesPreserve() + { + // Arrange + Mock mock = new( + new Uri("https://storageaccount.blob.core.windows.net/container/destination"), + new BlobClientOptions()); + + int length = 1024; + var data = GetRandomBuffer(length); + using var stream = new MemoryStream(data); + using var fileContentStream = new MemoryStream(); + mock.Setup(b => b.CreateAsync(It.IsAny(), It.IsAny(), It.IsAny())) + .Returns(Task.FromResult(Response.FromValue( + BlobsModelFactory.BlobContentInfo( + eTag: new ETag("eTag"), + lastModified: DateTimeOffset.UtcNow, + contentHash: default, + versionId: "version", + encryptionKeySha256: default, + encryptionScope: default, + blobSequenceNumber: default), + new MockResponse(201)))); + mock.Setup(b => b.UploadPagesAsync(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny())) + .Callback( + async (stream, offset, options, token) => + { + await stream.CopyToAsync(fileContentStream).ConfigureAwait(false); + fileContentStream.Position = 0; + }) + .Returns(Task.FromResult(Response.FromValue( + BlobsModelFactory.PageInfo( + eTag: new ETag("eTag"), + lastModified: DateTimeOffset.UtcNow, + contentHash: default, + contentCrc64: default, + encryptionKeySha256: default, + encryptionScope: default, + blobSequenceNumber: default), + new MockResponse(201)))); + + PageBlobStorageResourceOptions resourceOptions = new() + { + CacheControl = new(true), + ContentDisposition = new(true), + ContentLanguage = new(true), + ContentEncoding = new(true), + ContentType = new(true), + Metadata = new(true) + }; + PageBlobStorageResource destinationResource = new PageBlobStorageResource(mock.Object, resourceOptions); + + // Act + IDictionary metadata = DataProvider.BuildMetadata(); + + Dictionary sourceProperties = new() + { + { DataMovementConstants.ResourceProperties.ContentType, DefaultContentType }, + { DataMovementConstants.ResourceProperties.ContentEncoding, DefaultContentEncoding }, + { DataMovementConstants.ResourceProperties.ContentLanguage, DefaultContentLanguage }, + { DataMovementConstants.ResourceProperties.ContentDisposition, DefaultContentDisposition }, + { DataMovementConstants.ResourceProperties.CacheControl, DefaultCacheControl }, + { DataMovementConstants.ResourceProperties.Metadata, metadata } + }; + StorageResourceWriteToOffsetOptions copyFromStreamOptions = new() + { + SourceProperties = new StorageResourceItemProperties( + resourceLength: length, + eTag: new("ETag"), + lastModifiedTime: DateTimeOffset.UtcNow.AddHours(-1), + properties: sourceProperties) + }; + await destinationResource.CopyFromStreamInternalAsync( + stream, + length, + false, + length, + copyFromStreamOptions); + + Assert.That(data, Is.EqualTo(fileContentStream.AsBytes().ToArray())); + mock.Verify(b => b.CreateAsync( + length, + It.Is( + options => + options.HttpHeaders.ContentType == DefaultContentType && + options.HttpHeaders.ContentEncoding == DefaultContentEncoding && + options.HttpHeaders.ContentLanguage == DefaultContentLanguage && + options.HttpHeaders.ContentDisposition == DefaultContentDisposition && + options.HttpHeaders.CacheControl == DefaultCacheControl && + options.Metadata.SequenceEqual(metadata)), + It.IsAny()), + Times.Once()); + mock.Verify(b => b.UploadPagesAsync( + stream, + 0, + It.IsAny(), + It.IsAny()), + Times.Once()); + mock.VerifyNoOtherCalls(); + } + + [Test] + public async Task CopyFromStreamAsync_PropertiesNoPreserve() + { + // Arrange + Mock mock = new( + new Uri("https://storageaccount.blob.core.windows.net/container/destination"), + new BlobClientOptions()); + + int length = 1024; + var data = GetRandomBuffer(length); + using var stream = new MemoryStream(data); + using var fileContentStream = new MemoryStream(); + mock.Setup(b => b.CreateAsync(It.IsAny(), It.IsAny(), It.IsAny())) + .Returns(Task.FromResult(Response.FromValue( + BlobsModelFactory.BlobContentInfo( + eTag: new ETag("eTag"), + lastModified: DateTimeOffset.UtcNow, + contentHash: default, + versionId: "version", + encryptionKeySha256: default, + encryptionScope: default, + blobSequenceNumber: default), + new MockResponse(201)))); + mock.Setup(b => b.UploadPagesAsync(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny())) + .Callback( + async (stream, offset, options, token) => + { + await stream.CopyToAsync(fileContentStream).ConfigureAwait(false); + fileContentStream.Position = 0; + }) + .Returns(Task.FromResult(Response.FromValue( + BlobsModelFactory.PageInfo( + eTag: new ETag("eTag"), + lastModified: DateTimeOffset.UtcNow, + contentHash: default, + contentCrc64: default, + encryptionKeySha256: default, + encryptionScope: default, + blobSequenceNumber: default), + new MockResponse(201)))); + + PageBlobStorageResourceOptions resourceOptions = new() + { + CacheControl = new(false), + ContentDisposition = new(false), + ContentLanguage = new(false), + ContentEncoding = new(false), + ContentType = new(false), + Metadata = new(false) + }; + PageBlobStorageResource destinationResource = new PageBlobStorageResource(mock.Object, resourceOptions); + + // Act + IDictionary metadata = DataProvider.BuildMetadata(); + + Dictionary sourceProperties = new() + { + { DataMovementConstants.ResourceProperties.ContentType, DefaultContentType }, + { DataMovementConstants.ResourceProperties.ContentEncoding, DefaultContentEncoding }, + { DataMovementConstants.ResourceProperties.ContentLanguage, DefaultContentLanguage }, + { DataMovementConstants.ResourceProperties.ContentDisposition, DefaultContentDisposition }, + { DataMovementConstants.ResourceProperties.CacheControl, DefaultCacheControl }, + { DataMovementConstants.ResourceProperties.Metadata, metadata } + }; + StorageResourceWriteToOffsetOptions copyFromStreamOptions = new() + { + SourceProperties = new StorageResourceItemProperties( + resourceLength: length, + eTag: new("ETag"), + lastModifiedTime: DateTimeOffset.UtcNow.AddHours(-1), + properties: sourceProperties) + }; + await destinationResource.CopyFromStreamInternalAsync( + stream, + length, + false, + length, + copyFromStreamOptions); + + Assert.That(data, Is.EqualTo(fileContentStream.AsBytes().ToArray())); + mock.Verify(b => b.CreateAsync( + length, + It.Is( + options => + options.Metadata == default), + It.IsAny()), + Times.Once()); + mock.Verify(b => b.UploadPagesAsync( + stream, + 0, + It.IsAny(), + It.IsAny()), + Times.Once()); + mock.VerifyNoOtherCalls(); + } + [RecordedTest] public async Task CopyFromUriAsync() { @@ -358,6 +656,317 @@ await destinationResource.CopyFromUriAsync( TestHelper.AssertSequenceEqual(data, result.Content.AsBytes().ToArray()); } + [Test] + public async Task CopyFromUriAsync_PropertiesDefault() + { + // Arrange + Uri sourceUri = new Uri("https://storageaccount.blob.core.windows.net/container/source"); + Mock sourceResource = new(); + sourceResource.Setup(b => b.Uri) + .Returns(sourceUri); + + Mock mockDestination = new( + new Uri("https://storageaccount.blob.core.windows.net/container/destination"), + new BlobClientOptions()); + + int length = 1024; + var data = GetRandomBuffer(length); + using var stream = new MemoryStream(data); + using var fileContentStream = new MemoryStream(); + mockDestination.Setup(b => b.CreateAsync(It.IsAny(), It.IsAny(), It.IsAny())) + .Returns(Task.FromResult(Response.FromValue( + BlobsModelFactory.BlobContentInfo( + eTag: new ETag("eTag"), + lastModified: DateTimeOffset.UtcNow, + contentHash: default, + versionId: "version", + encryptionKeySha256: default, + encryptionScope: default, + blobSequenceNumber: default), + new MockResponse(201)))); + mockDestination.Setup(b => b.UploadPagesFromUriAsync(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny())) + .Callback( + async (uri, sourceRange, destinationRange, options, token) => + { + await stream.CopyToAsync(fileContentStream).ConfigureAwait(false); + fileContentStream.Position = 0; + }) + .Returns(Task.FromResult(Response.FromValue( + BlobsModelFactory.PageInfo( + eTag: new ETag("eTag"), + lastModified: DateTimeOffset.UtcNow, + contentHash: default, + contentCrc64: default, + encryptionKeySha256: default, + encryptionScope: default, + blobSequenceNumber: default), + new MockResponse(201)))); + PageBlobStorageResource destinationResource = new PageBlobStorageResource(mockDestination.Object); + + // Act + IDictionary metadata = DataProvider.BuildMetadata(); + + Dictionary sourceProperties = new() + { + { DataMovementConstants.ResourceProperties.ContentType, DefaultContentType }, + { DataMovementConstants.ResourceProperties.ContentEncoding, DefaultContentEncoding }, + { DataMovementConstants.ResourceProperties.ContentLanguage, DefaultContentLanguage }, + { DataMovementConstants.ResourceProperties.ContentDisposition, DefaultContentDisposition }, + { DataMovementConstants.ResourceProperties.CacheControl, DefaultCacheControl }, + { DataMovementConstants.ResourceProperties.Metadata, metadata } + }; + StorageResourceCopyFromUriOptions copyFromUriOptions = new() + { + SourceProperties = new StorageResourceItemProperties( + resourceLength: length, + eTag: new("ETag"), + lastModifiedTime: DateTimeOffset.UtcNow.AddHours(-1), + properties: sourceProperties) + }; + await destinationResource.CopyFromUriInternalAsync( + sourceResource.Object, + false, + length, + copyFromUriOptions); + + Assert.That(data, Is.EqualTo(fileContentStream.AsBytes().ToArray())); + mockDestination.Verify(b => b.CreateAsync( + length, + It.Is( + options => + options.HttpHeaders.ContentType == DefaultContentType && + options.HttpHeaders.ContentEncoding == DefaultContentEncoding && + options.HttpHeaders.ContentLanguage == DefaultContentLanguage && + options.HttpHeaders.ContentDisposition == DefaultContentDisposition && + options.HttpHeaders.CacheControl == DefaultCacheControl && + options.Metadata.SequenceEqual(metadata)), + It.IsAny()), + Times.Once()); + mockDestination.Verify(b => b.UploadPagesFromUriAsync( + sourceUri, + It.Is(range => + range.Offset == 0 && + range.Length == length), + It.Is(range => + range.Offset == 0 && + range.Length == length), + It.IsAny(), + It.IsAny()), + Times.Once()); + } + + [Test] + public async Task CopyFromUriAsync_PropertiesPreserve() + { + // Arrange + Uri sourceUri = new Uri("https://storageaccount.blob.core.windows.net/container/source"); + Mock sourceResource = new(); + sourceResource.Setup(b => b.Uri) + .Returns(sourceUri); + + Mock mockDestination = new( + new Uri("https://storageaccount.blob.core.windows.net/container/destination"), + new BlobClientOptions()); + + int length = 1024; + var data = GetRandomBuffer(length); + using var stream = new MemoryStream(data); + using var fileContentStream = new MemoryStream(); + mockDestination.Setup(b => b.CreateAsync(It.IsAny(), It.IsAny(), It.IsAny())) + .Returns(Task.FromResult(Response.FromValue( + BlobsModelFactory.BlobContentInfo( + eTag: new ETag("eTag"), + lastModified: DateTimeOffset.UtcNow, + contentHash: default, + versionId: "version", + encryptionKeySha256: default, + encryptionScope: default, + blobSequenceNumber: default), + new MockResponse(201)))); + mockDestination.Setup(b => b.UploadPagesFromUriAsync(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny())) + .Callback( + async (uri, sourceRange, destinationRange, options, token) => + { + await stream.CopyToAsync(fileContentStream).ConfigureAwait(false); + fileContentStream.Position = 0; + }) + .Returns(Task.FromResult(Response.FromValue( + BlobsModelFactory.PageInfo( + eTag: new ETag("eTag"), + lastModified: DateTimeOffset.UtcNow, + contentHash: default, + contentCrc64: default, + encryptionKeySha256: default, + encryptionScope: default, + blobSequenceNumber: default), + new MockResponse(201)))); + PageBlobStorageResourceOptions resourceOptions = new() + { + CacheControl = new(true), + ContentDisposition = new(true), + ContentLanguage = new(true), + ContentEncoding = new(true), + ContentType = new(true), + Metadata = new(true) + }; + PageBlobStorageResource destinationResource = new PageBlobStorageResource(mockDestination.Object, resourceOptions); + + // Act + IDictionary metadata = DataProvider.BuildMetadata(); + + Dictionary sourceProperties = new() + { + { DataMovementConstants.ResourceProperties.ContentType, DefaultContentType }, + { DataMovementConstants.ResourceProperties.ContentEncoding, DefaultContentEncoding }, + { DataMovementConstants.ResourceProperties.ContentLanguage, DefaultContentLanguage }, + { DataMovementConstants.ResourceProperties.ContentDisposition, DefaultContentDisposition }, + { DataMovementConstants.ResourceProperties.CacheControl, DefaultCacheControl }, + { DataMovementConstants.ResourceProperties.Metadata, metadata } + }; + StorageResourceCopyFromUriOptions copyFromUriOptions = new() + { + SourceProperties = new StorageResourceItemProperties( + resourceLength: length, + eTag: new("ETag"), + lastModifiedTime: DateTimeOffset.UtcNow.AddHours(-1), + properties: sourceProperties) + }; + await destinationResource.CopyFromUriInternalAsync( + sourceResource.Object, + false, + length, + copyFromUriOptions); + + Assert.That(data, Is.EqualTo(fileContentStream.AsBytes().ToArray())); + mockDestination.Verify(b => b.CreateAsync( + length, + It.Is( + options => + options.HttpHeaders.ContentType == DefaultContentType && + options.HttpHeaders.ContentEncoding == DefaultContentEncoding && + options.HttpHeaders.ContentLanguage == DefaultContentLanguage && + options.HttpHeaders.ContentDisposition == DefaultContentDisposition && + options.HttpHeaders.CacheControl == DefaultCacheControl && + options.Metadata.SequenceEqual(metadata)), + It.IsAny()), + Times.Once()); + mockDestination.Verify(b => b.UploadPagesFromUriAsync( + sourceUri, + It.Is(range => + range.Offset == 0 && + range.Length == length), + It.Is(range => + range.Offset == 0 && + range.Length == length), + It.IsAny(), + It.IsAny()), + Times.Once()); + mockDestination.VerifyNoOtherCalls(); + } + + [Test] + public async Task CopyFromUriAsync_PropertiesNoPreserve() + { + // Arrange + Uri sourceUri = new Uri("https://storageaccount.blob.core.windows.net/container/source"); + Mock sourceResource = new(); + sourceResource.Setup(b => b.Uri) + .Returns(sourceUri); + + Mock mockDestination = new( + new Uri("https://storageaccount.blob.core.windows.net/container/destination"), + new BlobClientOptions()); + + int length = 1024; + var data = GetRandomBuffer(length); + using var stream = new MemoryStream(data); + using var fileContentStream = new MemoryStream(); + mockDestination.Setup(b => b.CreateAsync(It.IsAny(), It.IsAny(), It.IsAny())) + .Returns(Task.FromResult(Response.FromValue( + BlobsModelFactory.BlobContentInfo( + eTag: new ETag("eTag"), + lastModified: DateTimeOffset.UtcNow, + contentHash: default, + versionId: "version", + encryptionKeySha256: default, + encryptionScope: default, + blobSequenceNumber: default), + new MockResponse(201)))); + mockDestination.Setup(b => b.UploadPagesFromUriAsync(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny())) + .Callback( + async (uri, sourceRange, destinationRange, options, token) => + { + await stream.CopyToAsync(fileContentStream).ConfigureAwait(false); + fileContentStream.Position = 0; + }) + .Returns(Task.FromResult(Response.FromValue( + BlobsModelFactory.PageInfo( + eTag: new ETag("eTag"), + lastModified: DateTimeOffset.UtcNow, + contentHash: default, + contentCrc64: default, + encryptionKeySha256: default, + encryptionScope: default, + blobSequenceNumber: default), + new MockResponse(201)))); + PageBlobStorageResourceOptions resourceOptions = new() + { + CacheControl = new(false), + ContentDisposition = new(false), + ContentLanguage = new(false), + ContentEncoding = new(false), + ContentType = new(false), + Metadata = new(false) + }; + PageBlobStorageResource destinationResource = new PageBlobStorageResource(mockDestination.Object, resourceOptions); + + // Act + IDictionary metadata = DataProvider.BuildMetadata(); + + Dictionary sourceProperties = new() + { + { DataMovementConstants.ResourceProperties.ContentType, DefaultContentType }, + { DataMovementConstants.ResourceProperties.ContentEncoding, DefaultContentEncoding }, + { DataMovementConstants.ResourceProperties.ContentLanguage, DefaultContentLanguage }, + { DataMovementConstants.ResourceProperties.ContentDisposition, DefaultContentDisposition }, + { DataMovementConstants.ResourceProperties.CacheControl, DefaultCacheControl }, + { DataMovementConstants.ResourceProperties.Metadata, metadata } + }; + StorageResourceCopyFromUriOptions copyFromUriOptions = new() + { + SourceProperties = new StorageResourceItemProperties( + resourceLength: length, + eTag: new("ETag"), + lastModifiedTime: DateTimeOffset.UtcNow.AddHours(-1), + properties: sourceProperties) + }; + await destinationResource.CopyFromUriInternalAsync( + sourceResource.Object, + false, + length, + copyFromUriOptions); + + Assert.That(data, Is.EqualTo(fileContentStream.AsBytes().ToArray())); + mockDestination.Verify(b => b.CreateAsync( + length, + It.Is( + options => options.Metadata == default), + It.IsAny()), + Times.Once()); + mockDestination.Verify(b => b.UploadPagesFromUriAsync( + sourceUri, + It.Is(range => + range.Offset == 0 && + range.Length == length), + It.Is(range => + range.Offset == 0 && + range.Length == length), + It.IsAny(), + It.IsAny()), + Times.Once()); + mockDestination.VerifyNoOtherCalls(); + } + [RecordedTest] public async Task CopyFromUriAsync_Error() { @@ -567,6 +1176,321 @@ await destinationResource.CopyBlockFromUriAsync( TestHelper.AssertSequenceEqual(blockData, result.Content.AsBytes().ToArray()); } + [Test] + public async Task CopyBlockFromUriAsync_PropertiesDefault() + { + // Arrange + Uri sourceUri = new Uri("https://storageaccount.blob.core.windows.net/container/source"); + Mock sourceResource = new(); + sourceResource.Setup(b => b.Uri) + .Returns(sourceUri); + + Mock mockDestination = new( + new Uri("https://storageaccount.blob.core.windows.net/container/destination"), + new BlobClientOptions()); + + int length = 1024; + var data = GetRandomBuffer(length); + using var stream = new MemoryStream(data); + using var fileContentStream = new MemoryStream(); + mockDestination.Setup(b => b.CreateAsync(It.IsAny(), It.IsAny(), It.IsAny())) + .Returns(Task.FromResult(Response.FromValue( + BlobsModelFactory.BlobContentInfo( + eTag: new ETag("eTag"), + lastModified: DateTimeOffset.UtcNow, + contentHash: default, + versionId: "version", + encryptionKeySha256: default, + encryptionScope: default, + blobSequenceNumber: default), + new MockResponse(201)))); + mockDestination.Setup(b => b.UploadPagesFromUriAsync(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny())) + .Callback( + async (uri, sourceRange, destinationRange, options, token) => + { + await stream.CopyToAsync(fileContentStream).ConfigureAwait(false); + fileContentStream.Position = 0; + }) + .Returns(Task.FromResult(Response.FromValue( + BlobsModelFactory.PageInfo( + eTag: new ETag("eTag"), + lastModified: DateTimeOffset.UtcNow, + contentHash: default, + contentCrc64: default, + encryptionKeySha256: default, + encryptionScope: default, + blobSequenceNumber: default), + new MockResponse(201)))); + PageBlobStorageResource destinationResource = new PageBlobStorageResource(mockDestination.Object); + + // Act + IDictionary metadata = DataProvider.BuildMetadata(); + + Dictionary sourceProperties = new() + { + { DataMovementConstants.ResourceProperties.ContentType, DefaultContentType }, + { DataMovementConstants.ResourceProperties.ContentEncoding, DefaultContentEncoding }, + { DataMovementConstants.ResourceProperties.ContentLanguage, DefaultContentLanguage }, + { DataMovementConstants.ResourceProperties.ContentDisposition, DefaultContentDisposition }, + { DataMovementConstants.ResourceProperties.CacheControl, DefaultCacheControl }, + { DataMovementConstants.ResourceProperties.Metadata, metadata } + }; + StorageResourceCopyFromUriOptions copyFromUriOptions = new() + { + SourceProperties = new StorageResourceItemProperties( + resourceLength: length, + eTag: new("ETag"), + lastModifiedTime: DateTimeOffset.UtcNow.AddHours(-1), + properties: sourceProperties) + }; + await destinationResource.CopyBlockFromUriInternalAsync( + sourceResource.Object, + new HttpRange(0, length), + false, + length, + copyFromUriOptions); + + Assert.That(data, Is.EqualTo(fileContentStream.AsBytes().ToArray())); + mockDestination.Verify(b => b.CreateAsync( + length, + It.Is( + options => + options.HttpHeaders.ContentType == DefaultContentType && + options.HttpHeaders.ContentEncoding == DefaultContentEncoding && + options.HttpHeaders.ContentLanguage == DefaultContentLanguage && + options.HttpHeaders.ContentDisposition == DefaultContentDisposition && + options.HttpHeaders.CacheControl == DefaultCacheControl && + options.Metadata.SequenceEqual(metadata)), + It.IsAny()), + Times.Once()); + mockDestination.Verify(b => b.UploadPagesFromUriAsync( + sourceUri, + It.Is(range => + range.Offset == 0 && + range.Length == length), + It.Is(range => + range.Offset == 0 && + range.Length == length), + It.IsAny(), + It.IsAny()), + Times.Once()); + mockDestination.VerifyNoOtherCalls(); + } + + [Test] + public async Task CopyBlockFromUriAsync_PropertiesPreserve() + { + // Arrange + Uri sourceUri = new Uri("https://storageaccount.blob.core.windows.net/container/source"); + Mock sourceResource = new(); + sourceResource.Setup(b => b.Uri) + .Returns(sourceUri); + + Mock mockDestination = new( + new Uri("https://storageaccount.blob.core.windows.net/container/destination"), + new BlobClientOptions()); + + int length = 1024; + var data = GetRandomBuffer(length); + using var stream = new MemoryStream(data); + using var fileContentStream = new MemoryStream(); + mockDestination.Setup(b => b.CreateAsync(It.IsAny(), It.IsAny(), It.IsAny())) + .Returns(Task.FromResult(Response.FromValue( + BlobsModelFactory.BlobContentInfo( + eTag: new ETag("eTag"), + lastModified: DateTimeOffset.UtcNow, + contentHash: default, + versionId: "version", + encryptionKeySha256: default, + encryptionScope: default, + blobSequenceNumber: default), + new MockResponse(201)))); + mockDestination.Setup(b => b.UploadPagesFromUriAsync(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny())) + .Callback( + async (uri, sourceRange, destinationRange, options, token) => + { + await stream.CopyToAsync(fileContentStream).ConfigureAwait(false); + fileContentStream.Position = 0; + }) + .Returns(Task.FromResult(Response.FromValue( + BlobsModelFactory.PageInfo( + eTag: new ETag("eTag"), + lastModified: DateTimeOffset.UtcNow, + contentHash: default, + contentCrc64: default, + encryptionKeySha256: default, + encryptionScope: default, + blobSequenceNumber: default), + new MockResponse(201)))); + PageBlobStorageResourceOptions resourceOptions = new() + { + CacheControl = new(true), + ContentDisposition = new(true), + ContentLanguage = new(true), + ContentEncoding = new(true), + ContentType = new(true), + Metadata = new(true) + }; + PageBlobStorageResource destinationResource = new PageBlobStorageResource(mockDestination.Object, resourceOptions); + + // Act + IDictionary metadata = DataProvider.BuildMetadata(); + + Dictionary sourceProperties = new() + { + { DataMovementConstants.ResourceProperties.ContentType, DefaultContentType }, + { DataMovementConstants.ResourceProperties.ContentEncoding, DefaultContentEncoding }, + { DataMovementConstants.ResourceProperties.ContentLanguage, DefaultContentLanguage }, + { DataMovementConstants.ResourceProperties.ContentDisposition, DefaultContentDisposition }, + { DataMovementConstants.ResourceProperties.CacheControl, DefaultCacheControl }, + { DataMovementConstants.ResourceProperties.Metadata, metadata } + }; + StorageResourceCopyFromUriOptions copyFromUriOptions = new() + { + SourceProperties = new StorageResourceItemProperties( + resourceLength: length, + eTag: new("ETag"), + lastModifiedTime: DateTimeOffset.UtcNow.AddHours(-1), + properties: sourceProperties) + }; + await destinationResource.CopyBlockFromUriInternalAsync( + sourceResource.Object, + new HttpRange(0, length), + false, + length, + copyFromUriOptions); + + Assert.That(data, Is.EqualTo(fileContentStream.AsBytes().ToArray())); + mockDestination.Verify(b => b.CreateAsync( + length, + It.Is( + options => + options.HttpHeaders.ContentType == DefaultContentType && + options.HttpHeaders.ContentEncoding == DefaultContentEncoding && + options.HttpHeaders.ContentLanguage == DefaultContentLanguage && + options.HttpHeaders.ContentDisposition == DefaultContentDisposition && + options.HttpHeaders.CacheControl == DefaultCacheControl && + options.Metadata.SequenceEqual(metadata)), + It.IsAny()), + Times.Once()); + mockDestination.Verify(b => b.UploadPagesFromUriAsync( + sourceUri, + It.Is(range => + range.Offset == 0 && + range.Length == length), + It.Is(range => + range.Offset == 0 && + range.Length == length), + It.IsAny(), + It.IsAny()), + Times.Once()); + mockDestination.VerifyNoOtherCalls(); + } + + [Test] + public async Task CopyBlockFromUriAsync_PropertiesNoPreserve() + { + // Arrange + Uri sourceUri = new Uri("https://storageaccount.blob.core.windows.net/container/source"); + Mock sourceResource = new(); + sourceResource.Setup(b => b.Uri) + .Returns(sourceUri); + + Mock mockDestination = new( + new Uri("https://storageaccount.blob.core.windows.net/container/destination"), + new BlobClientOptions()); + + int length = 1024; + var data = GetRandomBuffer(length); + using var stream = new MemoryStream(data); + using var fileContentStream = new MemoryStream(); + mockDestination.Setup(b => b.CreateAsync(It.IsAny(), It.IsAny(), It.IsAny())) + .Returns(Task.FromResult(Response.FromValue( + BlobsModelFactory.BlobContentInfo( + eTag: new ETag("eTag"), + lastModified: DateTimeOffset.UtcNow, + contentHash: default, + versionId: "version", + encryptionKeySha256: default, + encryptionScope: default, + blobSequenceNumber: default), + new MockResponse(201)))); + mockDestination.Setup(b => b.UploadPagesFromUriAsync(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny())) + .Callback( + async (uri, sourceRange, destinationRange, options, token) => + { + await stream.CopyToAsync(fileContentStream).ConfigureAwait(false); + fileContentStream.Position = 0; + }) + .Returns(Task.FromResult(Response.FromValue( + BlobsModelFactory.PageInfo( + eTag: new ETag("eTag"), + lastModified: DateTimeOffset.UtcNow, + contentHash: default, + contentCrc64: default, + encryptionKeySha256: default, + encryptionScope: default, + blobSequenceNumber: default), + new MockResponse(201)))); + PageBlobStorageResourceOptions resourceOptions = new() + { + CacheControl = new(false), + ContentDisposition = new(false), + ContentLanguage = new(false), + ContentEncoding = new(false), + ContentType = new(false), + Metadata = new(false) + }; + PageBlobStorageResource destinationResource = new PageBlobStorageResource(mockDestination.Object, resourceOptions); + + // Act + IDictionary metadata = DataProvider.BuildMetadata(); + + Dictionary sourceProperties = new() + { + { DataMovementConstants.ResourceProperties.ContentType, DefaultContentType }, + { DataMovementConstants.ResourceProperties.ContentEncoding, DefaultContentEncoding }, + { DataMovementConstants.ResourceProperties.ContentLanguage, DefaultContentLanguage }, + { DataMovementConstants.ResourceProperties.ContentDisposition, DefaultContentDisposition }, + { DataMovementConstants.ResourceProperties.CacheControl, DefaultCacheControl }, + { DataMovementConstants.ResourceProperties.Metadata, metadata } + }; + StorageResourceCopyFromUriOptions copyFromUriOptions = new() + { + SourceProperties = new StorageResourceItemProperties( + resourceLength: length, + eTag: new("ETag"), + lastModifiedTime: DateTimeOffset.UtcNow.AddHours(-1), + properties: sourceProperties) + }; + await destinationResource.CopyBlockFromUriInternalAsync( + sourceResource.Object, + new HttpRange(0, length), + false, + length, + copyFromUriOptions); + + Assert.That(data, Is.EqualTo(fileContentStream.AsBytes().ToArray())); + mockDestination.Verify(b => b.CreateAsync( + length, + It.Is( + options => options.Metadata == default), + It.IsAny()), + Times.Once()); + mockDestination.Verify(b => b.UploadPagesFromUriAsync( + sourceUri, + It.Is(range => + range.Offset == 0 && + range.Length == length), + It.Is(range => + range.Offset == 0 && + range.Length == length), + It.IsAny(), + It.IsAny()), + Times.Once()); + mockDestination.VerifyNoOtherCalls(); + } + [RecordedTest] public async Task CopyBlockFromUriAsync_Error() { @@ -633,6 +1557,104 @@ await TestHelper.AssertExpectedExceptionAsync( }); } + [Test] + public async Task GetPropertiesAsync_NotCached() + { + // Arrange + Mock mock = new( + new Uri("https://storageaccount.file.core.windows.net/container/file"), + new BlobClientOptions()); + + long length = 1024; + ETag eTag = new ETag("etag"); + string source = "https://storageaccount.file.core.windows.net/container/file2"; + Metadata metadata = DataProvider.BuildMetadata(); + mock.Setup(b => b.GetPropertiesAsync(It.IsAny(), It.IsAny())) + .Returns(Task.FromResult(Response.FromValue( + BlobsModelFactory.BlobProperties( + lastModified: DateTime.MinValue, + leaseStatus: LeaseStatus.Unlocked, + contentLength: length, + eTag: eTag, + contentEncoding: DefaultContentEncoding, + contentDisposition: DefaultContentDisposition, + contentLanguage: DefaultContentLanguage, + contentType: DefaultContentType, + cacheControl: DefaultCacheControl, + copySource: new Uri(source), + accessTier: default, + copyCompletedOn: DateTimeOffset.MinValue, + accessTierChangedOn: DateTimeOffset.MinValue, + blobType: BlobType.Block, + metadata: metadata, + tagCount: 5), + new MockResponse(200)))); + + BlockBlobStorageResource storageResource = new BlockBlobStorageResource(mock.Object); + + // Act + StorageResourceItemProperties result = await storageResource.GetPropertiesInternalAsync(); + string contentEncodingResult = (string)result.RawProperties[DataMovementConstants.ResourceProperties.ContentEncoding]; + string contentDispositionResult = (string)result.RawProperties[DataMovementConstants.ResourceProperties.ContentDisposition]; + string contentLanguageResult = (string)result.RawProperties[DataMovementConstants.ResourceProperties.ContentLanguage]; + string contentTypeResult = (string)result.RawProperties[DataMovementConstants.ResourceProperties.ContentType]; + string cacheControlResult = (string)result.RawProperties[DataMovementConstants.ResourceProperties.CacheControl]; + Metadata metadataResult = (Metadata)result.RawProperties[DataMovementConstants.ResourceProperties.Metadata]; + + // Assert + Assert.AreEqual(eTag, result.ETag); + Assert.AreEqual(length, result.ResourceLength); + Assert.AreEqual(contentEncodingResult, DefaultContentEncoding); + Assert.AreEqual(contentDispositionResult, DefaultContentDisposition); + Assert.AreEqual(contentLanguageResult, DefaultContentLanguage); + Assert.AreEqual(contentTypeResult, DefaultContentType); + Assert.AreEqual(cacheControlResult, DefaultCacheControl); + Assert.That(metadata, Is.EqualTo(metadataResult)); + mock.Verify(b => b.GetPropertiesAsync(It.IsAny(), It.IsAny()), + Times.Once()); + mock.VerifyNoOtherCalls(); + } + + [Test] + public async Task GetPropertiesAsync_Cached() + { + // Arrange + Mock mock = new( + new Uri("https://storageaccount.file.core.windows.net/container/file"), + new BlobClientOptions()); + + long length = 1024; + ETag eTag = new ETag("etag"); + DateTimeOffset lastModified = DateTimeOffset.UtcNow.AddHours(-1); + Metadata metadata = DataProvider.BuildMetadata(); + Dictionary rawProperties = new() + { + { DataMovementConstants.ResourceProperties.ContentEncoding, DefaultContentEncoding }, + { DataMovementConstants.ResourceProperties.ContentDisposition, DefaultContentDisposition}, + { DataMovementConstants.ResourceProperties.ContentLanguage, DefaultContentLanguage}, + { DataMovementConstants.ResourceProperties.ContentType, DefaultContentType}, + { DataMovementConstants.ResourceProperties.CacheControl, DefaultCacheControl}, + { DataMovementConstants.ResourceProperties.Metadata, metadata }, + }; + + BlockBlobStorageResource storageResource = new BlockBlobStorageResource( + mock.Object, + new StorageResourceItemProperties( + length, + eTag, + lastModified, + rawProperties)); + + // Act + StorageResourceItemProperties result = await storageResource.GetPropertiesInternalAsync(); + + // Assert + Assert.That(rawProperties, Is.EqualTo(result.RawProperties)); + mock.Verify(b => b.GetPropertiesAsync(It.IsAny(), It.IsAny()), + Times.Never()); + mock.VerifyNoOtherCalls(); + } + [RecordedTest] public async Task CompleteTransferAsync() { diff --git a/sdk/storage/Azure.Storage.DataMovement/tests/PauseResumeTransferTests.cs b/sdk/storage/Azure.Storage.DataMovement/tests/PauseResumeTransferTests.cs index 13d7d39ac883f..4f67fcf6d8bf2 100644 --- a/sdk/storage/Azure.Storage.DataMovement/tests/PauseResumeTransferTests.cs +++ b/sdk/storage/Azure.Storage.DataMovement/tests/PauseResumeTransferTests.cs @@ -20,6 +20,7 @@ using DMBlobs::Azure.Storage.DataMovement.Blobs; using Moq; using NUnit.Framework; +using Metadata = System.Collections.Generic.IDictionary; namespace Azure.Storage.DataMovement.Tests { @@ -464,7 +465,11 @@ public async Task PauseThenResumeTransferAsync(TransferDirection transferType) await testEventsRaised.AssertPausedCheck(); // Act - Resume Job - DataTransferOptions resumeOptions = new DataTransferOptions(); + DataTransferOptions resumeOptions = new DataTransferOptions() + { + // Enable overwrite on resume, to overwrite destination. + CreationPreference = StorageResourceCreationPreference.OverwriteIfExists + }; TestEventsRaised testEventRaised2 = new TestEventsRaised(resumeOptions); DataTransfer resumeTransfer = await transferManager.ResumeTransferAsync( transferId: transfer.Id, @@ -498,7 +503,7 @@ public async Task ResumeTransferAsync(TransferDirection transferType) using DisposingLocalDirectory checkpointerDirectory = DisposingLocalDirectory.GetTestDirectory(); using DisposingLocalDirectory localDirectory = DisposingLocalDirectory.GetTestDirectory(); await using DisposingContainer sourceContainer = await GetTestContainerAsync(publicAccessType: PublicAccessType.BlobContainer); - await using DisposingContainer destinationContainer = await GetTestContainerAsync(publicAccessType: PublicAccessType.BlobContainer); + await using DisposingContainer destinationContainer = await GetTestContainerAsync(); BlobsStorageResourceProvider blobProvider = new(GetSharedKeyCredential()); LocalFilesStorageResourceProvider localProvider = new(); @@ -584,15 +589,12 @@ public async Task ResumeTransferAsync_Options(TransferDirection transferType) }; TransferManager transferManager = new TransferManager(options); + Metadata metadata = DataProvider.BuildMetadata(); BlockBlobStorageResourceOptions testOptions = new() { - Metadata = DataProvider.BuildMetadata(), - Tags = DataProvider.BuildTags(), - AccessTier = AccessTier.Cool, - HttpHeaders = new BlobHttpHeaders() - { - ContentLanguage = "en-US", - }, + Metadata = new(DataProvider.BuildMetadata()), + AccessTier =new(AccessTier.Cool), + ContentLanguage = new("en-US"), }; long size = Constants.KB; @@ -628,10 +630,9 @@ public async Task ResumeTransferAsync_Options(TransferDirection transferType) BlobUriBuilder builder = new BlobUriBuilder(destination.Uri); BlockBlobClient blob = blobContainer.Container.GetBlockBlobClient(builder.BlobName); BlobProperties props = (await blob.GetPropertiesAsync()).Value; - Assert.AreEqual(testOptions.Metadata, props.Metadata); - Assert.AreEqual(testOptions.Tags.Count, props.TagCount); - Assert.AreEqual(testOptions.AccessTier, new AccessTier(props.AccessTier)); - Assert.AreEqual(testOptions.HttpHeaders.ContentLanguage, props.ContentLanguage); + Assert.That(props.Metadata, Is.EqualTo(metadata)); + Assert.AreEqual(testOptions.AccessTier.Value, new AccessTier(props.AccessTier)); + Assert.AreEqual(testOptions.ContentLanguage.Value, props.ContentLanguage); } private async Task CreateBlobDirectorySourceResourceAsync( @@ -843,7 +844,7 @@ public async Task TryPauseTransferAsync_DataTransfer_Directory(TransferDirection using DisposingLocalDirectory checkpointerDirectory = DisposingLocalDirectory.GetTestDirectory(); using DisposingLocalDirectory sourceDirectory = DisposingLocalDirectory.GetTestDirectory(); using DisposingLocalDirectory destinationDirectory = DisposingLocalDirectory.GetTestDirectory(); - await using DisposingContainer sourceContainer = await GetTestContainerAsync(); + await using DisposingContainer sourceContainer = await GetTestContainerAsync(publicAccessType: PublicAccessType.BlobContainer); await using DisposingContainer destinationContainer = await GetTestContainerAsync(); BlobsStorageResourceProvider blobProvider = new(GetSharedKeyCredential()); @@ -894,7 +895,7 @@ public async Task TryPauseTransferAsync_AlreadyPaused_Directory(TransferDirectio using DisposingLocalDirectory checkpointerDirectory = DisposingLocalDirectory.GetTestDirectory(); using DisposingLocalDirectory sourceDirectory = DisposingLocalDirectory.GetTestDirectory(); using DisposingLocalDirectory destinationDirectory = DisposingLocalDirectory.GetTestDirectory(); - await using DisposingContainer sourceContainer = await GetTestContainerAsync(); + await using DisposingContainer sourceContainer = await GetTestContainerAsync(publicAccessType: PublicAccessType.BlobContainer); await using DisposingContainer destinationContainer = await GetTestContainerAsync(); BlobsStorageResourceProvider blobProvider = new(GetSharedKeyCredential()); @@ -963,7 +964,11 @@ public async Task PauseThenResumeTransferAsync_Directory(TransferDirection trans ResumeProviders = new() { blobProvider, localProvider }, }; TransferManager transferManager = new TransferManager(options); - DataTransferOptions transferOptions = new DataTransferOptions(); + DataTransferOptions transferOptions = new DataTransferOptions() + { + InitialTransferSize = Constants.KB, + MaximumTransferChunkSize = Constants.KB + }; TestEventsRaised testEventsRaised = new TestEventsRaised(transferOptions); long size = Constants.KB * 4; int partCount = 4; @@ -1046,7 +1051,11 @@ public async Task ResumeTransferAsync_Directory(TransferDirection transferType) ResumeProviders = new() { blobProvider, localProvider }, }; TransferManager transferManager = new TransferManager(options); - DataTransferOptions transferOptions = new DataTransferOptions(); + DataTransferOptions transferOptions = new DataTransferOptions() + { + InitialTransferSize = Constants.KB, + MaximumTransferChunkSize = Constants.KB + }; TestEventsRaised testEventsRaised = new TestEventsRaised(transferOptions); long size = Constants.KB * 4; int partCount = 4; @@ -1149,8 +1158,14 @@ public async Task ResumeTransferAsync_Directory_Large( blobProvider: blobProvider, localProvider: localProvider); + DataTransferOptions transferOptions = new() + { + InitialTransferSize = size / 4, + MaximumTransferChunkSize = size / 4 + }; + // Start transfer - DataTransfer transfer = await transferManager.StartTransferAsync(sResource, dResource); + DataTransfer transfer = await transferManager.StartTransferAsync(sResource, dResource, transferOptions); // Sleep before pausing await Task.Delay(delayInMs); diff --git a/sdk/storage/Azure.Storage.DataMovement/tests/ServiceToServiceJobPartTests.cs b/sdk/storage/Azure.Storage.DataMovement/tests/ServiceToServiceJobPartTests.cs new file mode 100644 index 0000000000000..586702c6393b6 --- /dev/null +++ b/sdk/storage/Azure.Storage.DataMovement/tests/ServiceToServiceJobPartTests.cs @@ -0,0 +1,255 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +using System; +using System.Buffers; +using System.Collections.Generic; +using System.Linq.Expressions; +using System.Threading; +using System.Threading.Tasks; +using Azure.Core; +using Azure.Core.Pipeline; +using Moq; +using NUnit.Framework; +using Azure.Storage.Test; + +namespace Azure.Storage.DataMovement.Tests +{ + [TestFixture] + public class ServiceToServiceJobPartTests + { + private readonly int _maxDelayInSec = 1; + private const string DefaultContentType = "text/plain"; + private const string DefaultContentEncoding = "gzip"; + private const string DefaultContentLanguage = "en-US"; + private const string DefaultContentDisposition = "inline"; + private const string DefaultCacheControl = "no-cache"; + public ServiceToServiceJobPartTests() { } + + private Mock GetQueueChunkTask() + { + var mock = new Mock(MockBehavior.Strict); + mock.Setup(del => del(It.IsAny>())) + .Returns(Task.CompletedTask); + return mock; + } + + private Mock GetPartQueueChunkTask() + { + var mock = new Mock(MockBehavior.Strict); + mock.Setup(del => del(It.IsAny>())) + .Callback>( + async(funcTask) => + { + await funcTask().ConfigureAwait(false); + }) + .Returns(Task.CompletedTask); + return mock; + } + + private StorageResourceItemProperties GetResourceProperties(int length) + { + IDictionary metadata = DataProvider.BuildMetadata(); + IDictionary tags = DataProvider.BuildTags(); + + Dictionary sourceProperties = new() + { + { "ContentType", DefaultContentType }, + { "ContentEncoding", DefaultContentEncoding }, + { "ContentLanguage", DefaultContentLanguage }, + { "ContentDisposition", DefaultContentDisposition }, + { "CacheControl", DefaultCacheControl }, + { "Metadata", metadata }, + { "Tags", tags } + }; + return new( + resourceLength: length, + eTag: new("ETag"), + lastModifiedTime: DateTimeOffset.UtcNow.AddHours(-1), + properties: sourceProperties); + } + + private Mock GetStorageResourceItem(int length = Constants.KB) + { + Mock mock = new(); + mock.Setup(r => r.Length).Returns(length); + mock.Setup(r => r.Uri).Returns(new Uri("https://storageacount.blob.core.windows.net/container/source")); + mock.Setup(r => r.ResourceId).Returns("mock"); + mock.Setup(r => r.ProviderId).Returns("mock"); + mock.Setup(r => r.GetSourceCheckpointData()) + .Returns(new MockResourceCheckpointData()); + mock.Setup(r => r.GetDestinationCheckpointData()) + .Returns(new MockResourceCheckpointData()); + return mock; + } + + private void VerifyInvocation( + Mock destinationMock, + Expression> expectedInvocation, + int numberOfInvocationCalls = 1, + int maxWaitTimeInSec = 6) + { + CancellationTokenSource cancellationSource = new CancellationTokenSource(TimeSpan.FromSeconds(maxWaitTimeInSec)); + CancellationToken cancellationToken = cancellationSource.Token; + bool verified = false; + + try + { + do + { + CancellationHelper.ThrowIfCancellationRequested(cancellationToken); + // If it exceeds the count we should just fail. But if it's less, + // we can retry and see if the invocation we expected will be called. + Thread.Sleep(TimeSpan.FromSeconds(_maxDelayInSec)); + + try + { + destinationMock.Verify(expectedInvocation, Times.Exactly(numberOfInvocationCalls)); + verified = true; + } catch (MockException) + { + // This exception tells us it hasn't seen the expected invocation + // which might happen due to parallelism. + } + } while (!verified); + } + catch (TaskCanceledException) + { + string message = "Timed out waiting for the correct amount of invocations for the task"; + Assert.Fail(message); + } + } + + [Test] + public async Task ProcessPartToChunkAsync_OneShot() + { + //Arrange + string transferId = Guid.NewGuid().ToString(); + int length = Constants.KB; + + // Set up source with properties + Mock mockSource = GetStorageResourceItem(length); + StorageResourceItemProperties properties = GetResourceProperties(length); + mockSource.Setup(r => r.GetPropertiesAsync(It.IsAny())) + .Returns(Task.FromResult(properties)); + + // Set up Destination to copy in one shot with a large chunk size and smaller total length. + Mock mockDestination = GetStorageResourceItem(); + mockDestination.Setup(resource => resource.CopyFromUriAsync(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny())) + .Returns(Task.CompletedTask); + mockDestination.Setup(r => r.MaxSupportedChunkSize).Returns(Constants.MB); + + // Set up default checkpointer with transfer job + LocalTransferCheckpointer checkpointer = new(default); + await checkpointer.AddNewJobAsync( + transferId: transferId, + source: mockSource.Object, + destination: mockDestination.Object); + + Mock mockQueueChunkTask = GetQueueChunkTask(); + Mock mockPartQueueChunkTask = GetPartQueueChunkTask(); + + ServiceToServiceTransferJob job = new( + new DataTransfer( + id: transferId, + transferManager: new TransferManager()), + mockSource.Object, + mockDestination.Object, + new DataTransferOptions(), + mockQueueChunkTask.Object, + checkpointer, + DataTransferErrorMode.StopOnAnyFailure, + ArrayPool.Shared, + new ClientDiagnostics(ClientOptions.Default)); + ServiceToServiceJobPart jobPart = await ServiceToServiceJobPart.CreateJobPartAsync( + job, + 1); + jobPart.SetQueueChunkDelegate(mockPartQueueChunkTask.Object); + + // Act + await jobPart.ProcessPartToChunkAsync(); + + // Verify + VerifyInvocation( + mockDestination, + resource => resource.CopyFromUriAsync( + mockSource.Object, + It.IsAny(), + length, + It.Is( options => + options.SourceProperties.Equals(properties)), + It.IsAny())); + } + + [Test] + public async Task ProcessPartToChunkAsync_Chunks() + { + // Arrange + string transferId = Guid.NewGuid().ToString(); + int length = Constants.KB * 4; + int chunkSize = Constants.KB; + int chunkAmount = length / chunkSize; + Mock mockSource = GetStorageResourceItem(length); + StorageResourceItemProperties properties = GetResourceProperties(length); + mockSource.Setup(r => r.GetPropertiesAsync(It.IsAny())) + .Returns(Task.FromResult(properties)); + + // Setup destination with small chunk size and a larger source total length + // to cause chunked copy + Mock mockDestination = GetStorageResourceItem(); + mockDestination.Setup(resource => resource.CopyBlockFromUriAsync(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny())) + .Returns(Task.CompletedTask); + mockDestination.Setup(resource => resource.CompleteTransferAsync(It.IsAny(), It.IsAny(), It.IsAny())) + .Returns(Task.CompletedTask); + mockDestination.Setup(r => r.MaxSupportedChunkSize).Returns(chunkSize); + + // Set up default checkpointer with transfer job + LocalTransferCheckpointer checkpointer = new(default); + await checkpointer.AddNewJobAsync( + transferId: transferId, + source: mockSource.Object, + destination: mockDestination.Object); + + Mock mockQueueChunkTask = GetQueueChunkTask(); + Mock mockPartQueueChunkTask = GetPartQueueChunkTask(); + + ServiceToServiceTransferJob job = new( + new DataTransfer( + id: transferId, + transferManager: new TransferManager()), + mockSource.Object, + mockDestination.Object, + new DataTransferOptions(), + mockQueueChunkTask.Object, + checkpointer, + DataTransferErrorMode.StopOnAnyFailure, + ArrayPool.Shared, + new ClientDiagnostics(ClientOptions.Default)); + ServiceToServiceJobPart jobPart = await ServiceToServiceJobPart.CreateJobPartAsync( + job, + 1); + jobPart.SetQueueChunkDelegate(mockPartQueueChunkTask.Object); + + await jobPart.ProcessPartToChunkAsync(); + + VerifyInvocation( + mockDestination, + resource => resource.CopyBlockFromUriAsync( + mockSource.Object, + It.IsAny(), + It.IsAny(), + length, + It.Is(options => + options.SourceProperties.Equals(properties)), + It.IsAny()), + chunkAmount); + VerifyInvocation( + mockDestination, + resource => resource.CompleteTransferAsync( + It.IsAny(), + It.Is(options => + options.SourceProperties.Equals(properties)), + It.IsAny())); + } + } +} diff --git a/sdk/storage/Azure.Storage.DataMovement/tests/Shared/DataMovementBlobTestBase.cs b/sdk/storage/Azure.Storage.DataMovement/tests/Shared/DataMovementBlobTestBase.cs index 4b93c791bb798..8c4c45d1019ff 100644 --- a/sdk/storage/Azure.Storage.DataMovement/tests/Shared/DataMovementBlobTestBase.cs +++ b/sdk/storage/Azure.Storage.DataMovement/tests/Shared/DataMovementBlobTestBase.cs @@ -618,10 +618,11 @@ internal async Task CreateAppendBlob( BlobContainerClient containerClient, string localSourceFile, string blobName, - long size) + long size, + AppendBlobCreateOptions createOptions = default) { AppendBlobClient blobClient = containerClient.GetAppendBlobClient(blobName); - await blobClient.CreateIfNotExistsAsync().ConfigureAwait(false); + await blobClient.CreateIfNotExistsAsync(createOptions).ConfigureAwait(false); if (size > 0) { long offset = 0; @@ -652,7 +653,8 @@ internal async Task CreateBlockBlob( BlobContainerClient containerClient, string localSourceFile, string blobName, - long size) + long size, + BlobUploadOptions options = default) { BlockBlobClient blobClient = containerClient.GetBlockBlobClient(blobName); @@ -665,7 +667,7 @@ internal async Task CreateBlockBlob( await originalStream.CopyToAsync(fileStream); // Upload blob to storage account originalStream.Position = 0; - await blobClient.UploadAsync(originalStream); + await blobClient.UploadAsync(originalStream, options); } return blobClient; } @@ -674,12 +676,13 @@ internal async Task CreatePageBlob( BlobContainerClient containerClient, string localSourceFile, string blobName, - long size) + long size, + PageBlobCreateOptions options = default) { Assert.IsTrue(size % (Constants.KB / 2) == 0, "Cannot create page blob that's not a multiple of 512"); PageBlobClient blobClient = containerClient.GetPageBlobClient(blobName); - await blobClient.CreateIfNotExistsAsync(size).ConfigureAwait(false); + await blobClient.CreateIfNotExistsAsync(size, options).ConfigureAwait(false); if (size > 0) { long offset = 0; diff --git a/sdk/storage/Azure.Storage.DataMovement/tests/Shared/MemoryStorageResourceItem.cs b/sdk/storage/Azure.Storage.DataMovement/tests/Shared/MemoryStorageResourceItem.cs index 9387fd545bff4..33b9dfa424c3a 100644 --- a/sdk/storage/Azure.Storage.DataMovement/tests/Shared/MemoryStorageResourceItem.cs +++ b/sdk/storage/Azure.Storage.DataMovement/tests/Shared/MemoryStorageResourceItem.cs @@ -2,6 +2,7 @@ // Licensed under the MIT License. using System; +using System.Collections.Generic; using System.IO; using System.Threading; using System.Threading.Tasks; @@ -29,7 +30,10 @@ public MemoryStorageResourceItem(Uri uri = default) Uri = uri ?? new Uri($"memory://localhost/mycontainer/mypath-{Guid.NewGuid()}/resource-item-{Guid.NewGuid()}"); } - protected internal override Task CompleteTransferAsync(bool overwrite, CancellationToken cancellationToken = default) + protected internal override Task CompleteTransferAsync( + bool overwrite, + StorageResourceCompleteTransferOptions completeTransferOptions, + CancellationToken cancellationToken = default) { return Task.CompletedTask; } diff --git a/sdk/storage/Azure.Storage.DataMovement/tests/Shared/StartTransferUploadTestBase.cs b/sdk/storage/Azure.Storage.DataMovement/tests/Shared/StartTransferUploadTestBase.cs index 4e7d5eac156b4..c28327365db80 100644 --- a/sdk/storage/Azure.Storage.DataMovement/tests/Shared/StartTransferUploadTestBase.cs +++ b/sdk/storage/Azure.Storage.DataMovement/tests/Shared/StartTransferUploadTestBase.cs @@ -284,7 +284,7 @@ public VerifyUploadObjectContentInfo( /// /// Upload and verify the contents of the object /// - /// By default in this function an event arguement will be added to the options event handler + /// By default in this function an event argument will be added to the options event handler /// to detect when the upload has finished. /// /// @@ -755,6 +755,6 @@ await UploadResourceAndVerify( waitTimeInSec: waitTimeInSec, objectCount: objectCount); } -#endregion + #endregion } } diff --git a/sdk/storage/Azure.Storage.DataMovement/tests/StartTransferSyncCopyDirectoryTests.cs b/sdk/storage/Azure.Storage.DataMovement/tests/StartTransferSyncCopyDirectoryTests.cs index 24777b532de8b..7d6faf158e4a6 100644 --- a/sdk/storage/Azure.Storage.DataMovement/tests/StartTransferSyncCopyDirectoryTests.cs +++ b/sdk/storage/Azure.Storage.DataMovement/tests/StartTransferSyncCopyDirectoryTests.cs @@ -13,17 +13,86 @@ using Azure.Storage.Blobs.Models; using Azure.Storage.Blobs.Specialized; using Azure.Storage.Blobs.Tests; +using Azure.Storage.Test; using DMBlobs::Azure.Storage.DataMovement.Blobs; using NUnit.Framework; +using Metadata = System.Collections.Generic.Dictionary; namespace Azure.Storage.DataMovement.Tests { public class StartTransferSyncCopyDirectoryTests : DataMovementBlobTestBase { + private static AccessTier DefaultAccessTier = AccessTier.Cold; + private const string DefaultContentType = "text/plain"; + private const string DefaultContentEncoding = "gzip"; + private const string DefaultContentLanguage = "en-US"; + private const string DefaultContentDisposition = "inline"; + private const string DefaultCacheControl = "no-cache"; public StartTransferSyncCopyDirectoryTests(bool async, BlobClientOptions.ServiceVersion serviceVersion) : base(async, serviceVersion, null /* RecordedTestMode.Record /* to re-record */) { } + private async Task VerifyBlobDirectoryCopyAsync( + DataTransfer transfer, + BlobContainerClient container, + string sourcePrefix, + string destinationPrefix, + TestEventsRaised testEventsRaised, + bool verifyPropertiesMatch) + { + Assert.IsTrue(transfer.HasCompleted); + Assert.AreEqual(DataTransferState.Completed, transfer.TransferStatus.State); + + // List all files in source blob folder path + List sourceblobNames = new List(); + await foreach (Page page in container.GetBlobsAsync(prefix: sourcePrefix).AsPages()) + { + sourceblobNames.AddRange(page.Values.Select((BlobItem item) => item.Name)); + } + + // List all files in the destination blob folder path + List destblobNames = new List(); + await foreach (Page page in container.GetBlobsAsync(prefix: destinationPrefix).AsPages()) + { + destblobNames.AddRange(page.Values.Select((BlobItem item) => item.Name)); + } + await testEventsRaised.AssertContainerCompletedCheck(sourceblobNames.Count); + Assert.AreEqual(sourceblobNames.Count, destblobNames.Count); + sourceblobNames.Sort(); + destblobNames.Sort(); + for (int i = 0; i < sourceblobNames.Count; i++) + { + // Verify file name to match the + // (prefix folder path) + (the blob name without the blob folder prefix) + string sourceNonPrefixed = sourceblobNames[i].Substring(sourcePrefix.Length + 1); + Assert.AreEqual( + sourceNonPrefixed, + destblobNames[i].Substring(destinationPrefix.Length + 1)); + + // Verify Copy + BlockBlobClient sourceBlob = container.GetBlockBlobClient(sourceblobNames[i]); + BlockBlobClient destinationBlob = container.GetBlockBlobClient(destblobNames[i]); + using (Stream sourceStream = await sourceBlob.OpenReadAsync()) + { + Assert.IsTrue(await destinationBlob.ExistsAsync()); + await DownloadAndAssertAsync(sourceStream, destinationBlob); + } + if (verifyPropertiesMatch) + { + BlobProperties sourceProperties = await sourceBlob.GetPropertiesAsync(); + BlobProperties destinationProperties = await destinationBlob.GetPropertiesAsync(); + + Assert.That(sourceProperties.Metadata, Is.EqualTo(destinationProperties.Metadata)); + Assert.AreEqual(sourceProperties.ContentDisposition, destinationProperties.ContentDisposition); + Assert.AreEqual(sourceProperties.ContentEncoding, destinationProperties.ContentEncoding); + Assert.AreEqual(sourceProperties.ContentLanguage, destinationProperties.ContentLanguage); + Assert.AreEqual(sourceProperties.CacheControl, destinationProperties.CacheControl); + Assert.AreEqual(sourceProperties.AccessTier, destinationProperties.AccessTier); + Assert.Zero(destinationProperties.TagCount); + } + } + } + /// /// Upload and verify the contents of the blob /// @@ -32,8 +101,6 @@ public StartTransferSyncCopyDirectoryTests(bool async, BlobClientOptions.Service /// /// The source container which will contains the source blobs /// The source blob prefix/folder - /// The local source file prefix to join together with the source prefixes below. - /// The source file paths relative to the sourceFilePrefix /// The destination local path to download the blobs to /// /// How long we should wait until we cancel the operation. If this timeout is reached the test will fail. @@ -44,12 +111,11 @@ public StartTransferSyncCopyDirectoryTests(bool async, BlobClientOptions.Service private async Task CopyBlobDirectoryAndVerify( BlobContainerClient container, string sourceBlobPrefix, - string sourceFilePrefix, string destinationBlobPrefix, - List sourceFiles, int waitTimeInSec = 30, TransferManagerOptions transferManagerOptions = default, - DataTransferOptions options = default) + DataTransferOptions options = default, + bool verifyPropertiesMatch = false) { // Set transfer options options ??= new DataTransferOptions(); @@ -81,45 +147,14 @@ await TestTransferWithTimeout.WaitForCompletionAsync( testEventFailed, tokenSource.Token); - await testEventFailed.AssertContainerCompletedCheck(sourceFiles.Count); - Assert.IsTrue(transfer.HasCompleted); - Assert.AreEqual(DataTransferState.Completed, transfer.TransferStatus.State); - - // List all files in source blob folder path - List sourceblobNames = new List(); - await foreach (Page page in container.GetBlobsAsync(prefix: sourceBlobPrefix).AsPages()) - { - sourceblobNames.AddRange(page.Values.Select((BlobItem item) => item.Name)); - } - - // List all files in the destination blob folder path - List destblobNames = new List(); - await foreach (Page page in container.GetBlobsAsync(prefix: destinationBlobPrefix).AsPages()) - { - destblobNames.AddRange(page.Values.Select((BlobItem item) => item.Name)); - } - Assert.AreEqual(sourceblobNames.Count, destblobNames.Count); - sourceFiles.Sort(); - sourceblobNames.Sort(); - destblobNames.Sort(); - for (int i = 0; i < sourceFiles.Count; i++) - { - // Verify file name to match the - // (prefix folder path) + (the blob name without the blob folder prefix) - string sourceNonPrefixed = sourceblobNames[i].Substring(sourceBlobPrefix.Length + 1); - Assert.AreEqual( - sourceNonPrefixed, - destblobNames[i].Substring(destinationBlobPrefix.Length+1)); - - // Verify Download - string sourceFileName = Path.Combine(sourceFilePrefix, sourceNonPrefixed); - using (FileStream fileStream = File.OpenRead(sourceFileName)) - { - BlockBlobClient destinationBlob = container.GetBlockBlobClient(destblobNames[i]); - Assert.IsTrue(await destinationBlob.ExistsAsync()); - await DownloadAndAssertAsync(fileStream, destinationBlob); - } - } + // Verify + await VerifyBlobDirectoryCopyAsync( + transfer, + container, + sourceBlobPrefix, + destinationBlobPrefix, + testEventFailed, + verifyPropertiesMatch).ConfigureAwait(false); } [Test] @@ -161,9 +196,7 @@ public async Task BlockBlobDirectoryToDirectory_SmallSize(long size, int waitTim await CopyBlobDirectoryAndVerify( test.Container, sourceBlobDirectoryName, - sourceFolderPath, destinationFolder, - blobNames, waitTimeInSec).ConfigureAwait(false); } @@ -208,9 +241,7 @@ public async Task BlockBlobDirectoryToDirectory_LargeSize(long size, int waitTim await CopyBlobDirectoryAndVerify( test.Container, sourceBlobDirectoryName, - sourceFolderPath, destinationFolder, - blobNames, waitTimeInSec).ConfigureAwait(false); } @@ -283,9 +314,7 @@ public async Task BlockBlobDirectoryToDirectory_SingleFile() await CopyBlobDirectoryAndVerify( container: test.Container, sourceBlobPrefix: sourceFolderName, - sourceFilePrefix: sourceFolderPath, - destinationBlobPrefix: destinationFolder, - blobNames).ConfigureAwait(false); + destinationBlobPrefix: destinationFolder).ConfigureAwait(false); } [Test] @@ -320,9 +349,7 @@ public async Task BlockBlobDirectoryToDirectory_ManySubDirectories() await CopyBlobDirectoryAndVerify( container: test.Container, sourceBlobPrefix: sourceBlobPrefix, - sourceFilePrefix: fullSourceFolderPath, - destinationBlobPrefix: destinationFolder, - blobNames).ConfigureAwait(false); + destinationBlobPrefix: destinationFolder).ConfigureAwait(false); } [Test] @@ -355,9 +382,7 @@ public async Task BlockBlobDirectoryToDirectory_SubDirectoriesLevels(int level) await CopyBlobDirectoryAndVerify( test.Container, sourceBlobDirectoryName, - fullSourceFolderPath, - destinationBlobPrefix: destinationFolder, - blobNames).ConfigureAwait(false); + destinationBlobPrefix: destinationFolder).ConfigureAwait(false); } [Test] @@ -403,9 +428,7 @@ public async Task BlockBlobDirectoryToDirectory_OverwriteTrue() await CopyBlobDirectoryAndVerify( test.Container, sourceBlobDirectoryName, - sourceFolderPath, destinationBlobPrefix: destinationFolder, - blobNames, options: options).ConfigureAwait(false); } @@ -452,9 +475,7 @@ public async Task BlockBlobDirectoryToDirectory_OverwriteFalse() await CopyBlobDirectoryAndVerify( test.Container, sourceBlobDirectoryName, - sourceFolderPath, destinationBlobPrefix: destinationFolder, - blobNames, options: options).ConfigureAwait(false); } @@ -501,9 +522,7 @@ public async Task BlockBlobDirectoryToDirectory_OAuth() await CopyBlobDirectoryAndVerify( testContainer.Container, sourceBlobDirectoryName, - sourceFolderPath, destinationFolder, - blobNames, waitTimeInSec).ConfigureAwait(false); } @@ -512,22 +531,23 @@ private async Task CreateBlobDirectoryTree( BlobContainerClient client, string sourceFolderPath, string sourceBlobDirectoryName, - int size) + int size, + BlobUploadOptions uploadOptions = default) { string blobName1 = Path.Combine(sourceBlobDirectoryName, "blob1"); string blobName2 = Path.Combine(sourceBlobDirectoryName, "blob2"); - await CreateBlockBlob(client, Path.GetTempFileName(), blobName1, size); - await CreateBlockBlob(client, Path.GetTempFileName(), blobName2, size); + await CreateBlockBlob(client, Path.GetTempFileName(), blobName1, size, uploadOptions); + await CreateBlockBlob(client, Path.GetTempFileName(), blobName2, size, uploadOptions); string subDirName = "bar"; CreateRandomDirectory(sourceFolderPath, subDirName).Substring(sourceFolderPath.Length + 1); string blobName3 = Path.Combine(sourceBlobDirectoryName, subDirName, "blob3"); - await CreateBlockBlob(client, Path.GetTempFileName(), blobName3, size); + await CreateBlockBlob(client, Path.GetTempFileName(), blobName3, size, uploadOptions); string subDirName2 = "pik"; CreateRandomDirectory(sourceFolderPath, subDirName2).Substring(sourceFolderPath.Length + 1); string blobName4 = Path.Combine(sourceBlobDirectoryName, subDirName2, "blob4"); - await CreateBlockBlob(client, Path.GetTempFileName(), blobName4, size); + await CreateBlockBlob(client, Path.GetTempFileName(), blobName4, size, uploadOptions); } private async Task CreateStartTransfer( @@ -812,5 +832,330 @@ public async Task StartTransfer_EnsureCompleted_Failed_SmallChunks() await testEventsRaised.AssertContainerCompletedWithFailedCheck(1); } #endregion + + #region Properties + private async Task CreateStartTransferPropertiesAsync( + BlobContainerClient containerClient, + Metadata metadata, + string sourcePrefix, + string destinationPrefix, + DataTransferOptions transferOptions, + BlobStorageResourceOptions destinationBlobOptions = default) + { + // Arrange + using DisposingLocalDirectory testDirectory = DisposingLocalDirectory.GetTestDirectory(); + BlobUploadOptions uploadOptions = new() + { + AccessTier = DefaultAccessTier, + Metadata = metadata, + HttpHeaders = new() + { + CacheControl = DefaultCacheControl, + ContentDisposition = DefaultContentDisposition, + ContentEncoding = DefaultContentEncoding, + ContentLanguage = DefaultContentLanguage, + ContentType = DefaultContentType + } + }; + string sourceFolderPath = CreateRandomDirectory(testDirectory.DirectoryPath, sourcePrefix); + // Create source blob directory tree with properties. + await CreateBlobDirectoryTree( + containerClient, + sourceFolderPath, + sourcePrefix, + Constants.KB, + uploadOptions); + + StorageResourceContainer sourceResource = new BlobStorageResourceContainer(containerClient, new() { BlobDirectoryPrefix = sourcePrefix }); + StorageResourceContainer destinationResource = new BlobStorageResourceContainer(containerClient, new() + { + BlobDirectoryPrefix = destinationPrefix, + BlobOptions = destinationBlobOptions + }); + + // Create Transfer Manager with single threaded operation + TransferManager transferManager = new TransferManager(); + + // Start transfer and await for completion. + return await transferManager.StartTransferAsync( + sourceResource, + destinationResource, + transferOptions).ConfigureAwait(false); + } + + [LiveOnly] + [Test] + public async Task BlobDirectoryToDirectoryAsync_DefaultProperties() + { + // Arrange + await using DisposingContainer test = await GetTestContainerAsync(publicAccessType: PublicAccessType.BlobContainer); + string sourceBlobPrefix = "sourceFolder"; + string destBlobPrefix = "destFolder"; + + Metadata metadata = DataProvider.BuildMetadata(); + + // Act + DataTransferOptions transferOptions = new DataTransferOptions(); + TestEventsRaised testEventsRaised = new TestEventsRaised(transferOptions); + DataTransfer transfer = await CreateStartTransferPropertiesAsync( + test.Container, + metadata, + sourcePrefix: sourceBlobPrefix, + destinationPrefix: destBlobPrefix, + transferOptions: transferOptions); + CancellationTokenSource cancellationTokenSource = new CancellationTokenSource(TimeSpan.FromSeconds(30)); + await TestTransferWithTimeout.WaitForCompletionAsync( + transfer, + testEventsRaised, + cancellationTokenSource.Token); + + // Verify + await VerifyBlobDirectoryCopyAsync( + transfer, + test.Container, + sourceBlobPrefix, + destBlobPrefix, + testEventsRaised, + verifyPropertiesMatch: true).ConfigureAwait(false); + } + + [LiveOnly] + [Test] + public async Task BlobDirectoryToDirectoryAsync_PropertiesPreserve() + { + // Arrange + await using DisposingContainer test = await GetTestContainerAsync(publicAccessType: PublicAccessType.BlobContainer); + string sourceBlobPrefix = "sourceFolder"; + string destBlobPrefix = "destFolder"; + + Metadata metadata = DataProvider.BuildMetadata(); + + // Act + DataTransferOptions transferOptions = new DataTransferOptions(); + TestEventsRaised testEventsRaised = new TestEventsRaised(transferOptions); + DataTransfer transfer = await CreateStartTransferPropertiesAsync( + test.Container, + metadata, + sourcePrefix: sourceBlobPrefix, + destinationPrefix: destBlobPrefix, + transferOptions: transferOptions, + destinationBlobOptions: new() // Preserve all properties + { + AccessTier = new(true), + CacheControl = new(true), + ContentDisposition = new(true), + ContentEncoding = new(true), + ContentLanguage = new(true), + ContentType = new(true), + Metadata = new(true) + }); + CancellationTokenSource cancellationTokenSource = new CancellationTokenSource(TimeSpan.FromSeconds(30)); + await TestTransferWithTimeout.WaitForCompletionAsync( + transfer, + testEventsRaised, + cancellationTokenSource.Token); + + // Verify + await VerifyBlobDirectoryCopyAsync( + transfer, + test.Container, + sourceBlobPrefix, + destBlobPrefix, + testEventsRaised, + verifyPropertiesMatch: true).ConfigureAwait(false); + } + + [LiveOnly] + [Test] + public async Task BlobDirectoryToDirectoryAsync_PropertiesNoPreserve() + { + // Arrange + await using DisposingContainer test = await GetTestContainerAsync(publicAccessType: PublicAccessType.BlobContainer); + string sourceBlobPrefix = "sourceFolder"; + string destBlobPrefix = "destFolder"; + + Metadata metadata = DataProvider.BuildMetadata(); + + // Act + DataTransferOptions transferOptions = new DataTransferOptions(); + TestEventsRaised testEventsRaised = new TestEventsRaised(transferOptions); + DataTransfer transfer = await CreateStartTransferPropertiesAsync( + test.Container, + metadata, + sourcePrefix: sourceBlobPrefix, + destinationPrefix: destBlobPrefix, + transferOptions: transferOptions, + destinationBlobOptions: new() // Do NOT preserve any property + { + AccessTier = new(false), + CacheControl = new(false), + ContentDisposition = new(false), + ContentEncoding = new(false), + ContentLanguage = new(false), + ContentType = new(false), + Metadata = new(false) + }); + CancellationTokenSource cancellationTokenSource = new CancellationTokenSource(TimeSpan.FromSeconds(30)); + await TestTransferWithTimeout.WaitForCompletionAsync( + transfer, + testEventsRaised, + cancellationTokenSource.Token); + + // Verify + Assert.IsTrue(transfer.HasCompleted); + Assert.AreEqual(DataTransferState.Completed, transfer.TransferStatus.State); + + // List all files in source blob folder path + List sourceblobNames = new List(); + await foreach (Page page in test.Container.GetBlobsAsync(prefix: sourceBlobPrefix).AsPages()) + { + sourceblobNames.AddRange(page.Values.Select((BlobItem item) => item.Name)); + } + + // List all files in the destination blob folder path + List destblobNames = new List(); + await foreach (Page page in test.Container.GetBlobsAsync(prefix: destBlobPrefix).AsPages()) + { + destblobNames.AddRange(page.Values.Select((BlobItem item) => item.Name)); + } + await testEventsRaised.AssertContainerCompletedCheck(sourceblobNames.Count); + Assert.AreEqual(sourceblobNames.Count, destblobNames.Count); + sourceblobNames.Sort(); + destblobNames.Sort(); + for (int i = 0; i < sourceblobNames.Count; i++) + { + // Verify file name to match the + // (prefix folder path) + (the blob name without the blob folder prefix) + string sourceNonPrefixed = sourceblobNames[i].Substring(sourceBlobPrefix.Length + 1); + Assert.AreEqual( + sourceNonPrefixed, + destblobNames[i].Substring(destBlobPrefix.Length + 1)); + + // Verify Copy + BlockBlobClient sourceBlob = test.Container.GetBlockBlobClient(sourceblobNames[i]); + BlockBlobClient destinationBlob = test.Container.GetBlockBlobClient(destblobNames[i]); + using (Stream sourceStream = await sourceBlob.OpenReadAsync()) + { + Assert.IsTrue(await destinationBlob.ExistsAsync()); + await DownloadAndAssertAsync(sourceStream, destinationBlob); + } + BlobProperties destinationProperties = await destinationBlob.GetPropertiesAsync(); + + // Check if the properties are empty on the destination + Assert.IsEmpty(destinationProperties.Metadata); + Assert.IsNull(destinationProperties.ContentDisposition); + Assert.IsNull(destinationProperties.ContentEncoding); + Assert.IsNull(destinationProperties.ContentLanguage); + Assert.IsNull(destinationProperties.CacheControl); + Assert.AreEqual(AccessTier.Hot.ToString(), destinationProperties.AccessTier); + Assert.Zero(destinationProperties.TagCount); + } + } + + [LiveOnly] + [Test] + public async Task BlobDirectoryToDirectoryAsync_SetProperties() + { + // Arrange + await using DisposingContainer test = await GetTestContainerAsync(publicAccessType: PublicAccessType.BlobContainer); + string sourceBlobPrefix = "sourceFolder"; + string destBlobPrefix = "destFolder"; + + Metadata metadata = DataProvider.BuildMetadata(); + + // Act + DataTransferOptions transferOptions = new DataTransferOptions(); + TestEventsRaised testEventsRaised = new TestEventsRaised(transferOptions); + // Arrange + using DisposingLocalDirectory testDirectory = DisposingLocalDirectory.GetTestDirectory(); + string sourceFolderPath = CreateRandomDirectory(testDirectory.DirectoryPath, sourceBlobPrefix); + // Upload without any properties set on the source blobs. + await CreateBlobDirectoryTree( + test.Container, + sourceFolderPath, + sourceBlobPrefix, + Constants.KB); + + StorageResourceContainer sourceResource = new BlobStorageResourceContainer(test.Container, new() { BlobDirectoryPrefix = sourceBlobPrefix }); + StorageResourceContainer destinationResource = new BlobStorageResourceContainer(test.Container, new() + { + BlobDirectoryPrefix = destBlobPrefix, + BlobOptions = new() + { + AccessTier = new(DefaultAccessTier), + CacheControl = new(DefaultCacheControl), + ContentDisposition = new(DefaultContentDisposition), + ContentEncoding = new(DefaultContentEncoding), + ContentLanguage = new(DefaultContentLanguage), + ContentType = new(DefaultContentType), + Metadata = new(metadata) + } + }); + + // Create Transfer Manager with single threaded operation + TransferManager transferManager = new TransferManager(); + + // Start transfer and await for completion. + DataTransfer transfer = await transferManager.StartTransferAsync( + sourceResource, + destinationResource, + transferOptions).ConfigureAwait(false); + CancellationTokenSource cancellationTokenSource = new CancellationTokenSource(TimeSpan.FromSeconds(30)); + await TestTransferWithTimeout.WaitForCompletionAsync( + transfer, + testEventsRaised, + cancellationTokenSource.Token); + + // Verify + Assert.IsTrue(transfer.HasCompleted); + Assert.AreEqual(DataTransferState.Completed, transfer.TransferStatus.State); + + // List all files in source blob folder path + List sourceblobNames = new List(); + await foreach (Page page in test.Container.GetBlobsAsync(prefix: sourceBlobPrefix).AsPages()) + { + sourceblobNames.AddRange(page.Values.Select((BlobItem item) => item.Name)); + } + + // List all files in the destination blob folder path + List destblobNames = new List(); + await foreach (Page page in test.Container.GetBlobsAsync(prefix: destBlobPrefix).AsPages()) + { + destblobNames.AddRange(page.Values.Select((BlobItem item) => item.Name)); + } + await testEventsRaised.AssertContainerCompletedCheck(sourceblobNames.Count); + Assert.AreEqual(sourceblobNames.Count, destblobNames.Count); + sourceblobNames.Sort(); + destblobNames.Sort(); + for (int i = 0; i < sourceblobNames.Count; i++) + { + // Verify file name to match the + // (prefix folder path) + (the blob name without the blob folder prefix) + string sourceNonPrefixed = sourceblobNames[i].Substring(sourceBlobPrefix.Length + 1); + Assert.AreEqual( + sourceNonPrefixed, + destblobNames[i].Substring(destBlobPrefix.Length + 1)); + + // Verify Copy + BlockBlobClient sourceBlob = test.Container.GetBlockBlobClient(sourceblobNames[i]); + BlockBlobClient destinationBlob = test.Container.GetBlockBlobClient(destblobNames[i]); + using (Stream sourceStream = await sourceBlob.OpenReadAsync()) + { + Assert.IsTrue(await destinationBlob.ExistsAsync()); + await DownloadAndAssertAsync(sourceStream, destinationBlob); + } + // Check if the properties are correct on destination + BlobProperties destinationProperties = await destinationBlob.GetPropertiesAsync(); + + Assert.That(metadata, Is.EqualTo(destinationProperties.Metadata));; + Assert.AreEqual(DefaultContentDisposition, destinationProperties.ContentDisposition); + Assert.AreEqual(DefaultContentEncoding, destinationProperties.ContentEncoding); + Assert.AreEqual(DefaultContentLanguage, destinationProperties.ContentLanguage); + Assert.AreEqual(DefaultCacheControl, destinationProperties.CacheControl); + Assert.AreEqual(DefaultAccessTier.ToString(), destinationProperties.AccessTier); + Assert.Zero(destinationProperties.TagCount); + } + } + #endregion Properties } } diff --git a/sdk/storage/Azure.Storage.DataMovement/tests/StartTransferSyncCopyTests.cs b/sdk/storage/Azure.Storage.DataMovement/tests/StartTransferSyncCopyTests.cs index 34bae6bc0bd04..3c99e475eb4d8 100644 --- a/sdk/storage/Azure.Storage.DataMovement/tests/StartTransferSyncCopyTests.cs +++ b/sdk/storage/Azure.Storage.DataMovement/tests/StartTransferSyncCopyTests.cs @@ -13,13 +13,23 @@ using Azure.Storage.Blobs.Models; using Azure.Storage.Blobs.Specialized; using Azure.Storage.Blobs.Tests; +using Azure.Storage.Test; using DMBlobs::Azure.Storage.DataMovement.Blobs; using NUnit.Framework; +using Metadata = System.Collections.Generic.IDictionary; +using Tags = System.Collections.Generic.IDictionary; namespace Azure.Storage.DataMovement.Tests { public class StartTransferSyncCopyTests : DataMovementBlobTestBase { + private static AccessTier DefaultAccessTier = AccessTier.Cold; + private const string DefaultContentType = "text/plain"; + private const string DefaultContentEncoding = "gzip"; + private const string DefaultContentLanguage = "en-US"; + private const string DefaultContentDisposition = "inline"; + private const string DefaultCacheControl = "no-cache"; + public StartTransferSyncCopyTests(bool async, BlobClientOptions.ServiceVersion serviceVersion) : base(async, serviceVersion, null /* RecordedTestMode.Record /* to re-record */) { } @@ -51,6 +61,52 @@ public VerifyBlockBlobCopyFromUriInfo( DataTransfer = default; } }; + + private async Task VerifyBlobPropertiesCopyAsync( + DataTransfer transfer, + TestEventsRaised testEventsRaised, + BlobBaseClient sourceClient, + BlobBaseClient destinationClient) + { + Assert.NotNull(transfer); + Assert.IsTrue(transfer.HasCompleted); + Assert.AreEqual(DataTransferState.Completed, transfer.TransferStatus.State); + // Verify Copy - using original source File and Copying the destination + await testEventsRaised.AssertSingleCompletedCheck(); + using Stream sourceStream = await sourceClient.OpenReadAsync(); + using Stream destinationStream = await destinationClient.OpenReadAsync(); + Assert.AreEqual(sourceStream, destinationStream); + // Verify Properties + BlobProperties sourceProperties = await sourceClient.GetPropertiesAsync(); + BlobProperties destinationProperties = await destinationClient.GetPropertiesAsync(); + + Assert.That(sourceProperties.Metadata, Is.EqualTo(destinationProperties.Metadata)); + Assert.AreEqual(sourceProperties.AccessTier, destinationProperties.AccessTier); + Assert.AreEqual(sourceProperties.ContentDisposition, destinationProperties.ContentDisposition); + Assert.AreEqual(sourceProperties.ContentEncoding, destinationProperties.ContentEncoding); + Assert.AreEqual(sourceProperties.ContentLanguage, destinationProperties.ContentLanguage); + Assert.AreEqual(sourceProperties.CacheControl, destinationProperties.CacheControl); + } + + private async Task VerifyEmptyPropertiesAsync( + BlobBaseClient destinationClient, + bool checkAccessTier = false) + { + BlobProperties destinationProperties = await destinationClient.GetPropertiesAsync(); + + Assert.IsEmpty(destinationProperties.Metadata); + Assert.IsNull(destinationProperties.ContentDisposition); + Assert.IsNull(destinationProperties.ContentEncoding); + Assert.IsNull(destinationProperties.ContentLanguage); + Assert.IsNull(destinationProperties.CacheControl); + if (checkAccessTier) + { + Assert.AreEqual(AccessTier.Hot.ToString(), destinationProperties.AccessTier); + } + + GetBlobTagResult destinationTags = await destinationClient.GetTagsAsync(); + Assert.IsEmpty(destinationTags.Tags); + } #region SyncCopy BlockBlob /// /// Upload the blob, then copy the contents to another blob. @@ -842,5 +898,828 @@ public async Task StartTransfer_EnsureCompleted_Skipped() Assert.AreEqual(true, transfer.TransferStatus.HasSkippedItems); } #endregion + + #region Block Blob Properties + private async Task SetupSourceBlockBlobAsync( + BlobContainerClient container, + Metadata metadata, + Tags tags) + { + using DisposingLocalDirectory testDirectory = DisposingLocalDirectory.GetTestDirectory(); + string blobName = GetNewBlobName(); + int size = Constants.KB; + string newSourceFile = Path.Combine(testDirectory.DirectoryPath, GetNewBlobName()); + + BlobUploadOptions uploadOptions = new() + { + AccessTier = DefaultAccessTier, + // We can't include Content Encoding since we can't record the value. + HttpHeaders = new() + { + CacheControl = DefaultCacheControl, + ContentType = DefaultContentType, + ContentDisposition = DefaultContentDisposition, + ContentLanguage = DefaultContentLanguage, + }, + Metadata = metadata, + Tags = tags + }; + return await CreateBlockBlob( + container, + newSourceFile, + GetNewBlobName(), + size, + uploadOptions); + } + + [RecordedTest] + public async Task BlockBlobToBlockBlob_DefaultProperties() + { + // Arrange + // Create source local file for checking, and source blob + await using DisposingContainer testContainer = await GetTestContainerAsync(publicAccessType: PublicAccessType.BlobContainer); + Metadata metadata = DataProvider.BuildMetadata(); + Tags tags = DataProvider.BuildTags(); + + // Act + // Create blob with properties + BlockBlobClient sourceClient = await SetupSourceBlockBlobAsync( + testContainer.Container, + metadata, + tags); + + // Set preserve properties + StorageResourceItem sourceResource = new BlockBlobStorageResource(sourceClient); + + // Destination client - Set Properties + BlockBlobClient destinationClient = testContainer.Container.GetBlockBlobClient(GetNewBlobName()); + StorageResourceItem destinationResource = new BlockBlobStorageResource(destinationClient); + + DataTransferOptions options = new DataTransferOptions(); + TestEventsRaised testEventsRaised = new TestEventsRaised(options); + TransferManager transferManager = new TransferManager(); + + // Start transfer and await for completion. + DataTransfer transfer = await transferManager.StartTransferAsync( + sourceResource, + destinationResource, + options); + CancellationTokenSource cancellationTokenSource = new CancellationTokenSource(TimeSpan.FromSeconds(30)); + await TestTransferWithTimeout.WaitForCompletionAsync( + transfer, + testEventsRaised, + cancellationTokenSource.Token); + + // Assert + await VerifyBlobPropertiesCopyAsync( + transfer, + testEventsRaised, + sourceClient, + destinationClient); + // Verify Tags are NOT preserved + GetBlobTagResult destinationTags = await destinationClient.GetTagsAsync(); + Assert.IsEmpty(destinationTags.Tags); + } + + [RecordedTest] + public async Task BlockBlobToBlockBlob_PreservePropertiesNoTags() + { + // Arrange + // Create source local file for checking, and source blob + await using DisposingContainer testContainer = await GetTestContainerAsync(publicAccessType: PublicAccessType.BlobContainer); + Metadata metadata = DataProvider.BuildMetadata(); + Tags tags = DataProvider.BuildTags(); + + // Act + // Create blob with properties + BlockBlobClient sourceClient = await SetupSourceBlockBlobAsync( + testContainer.Container, + metadata, + tags); + + // Set preserve properties + StorageResourceItem sourceResource = new BlockBlobStorageResource(sourceClient); + + // Destination client - Set Properties + BlockBlobClient destinationClient = testContainer.Container.GetBlockBlobClient(GetNewBlobName()); + StorageResourceItem destinationResource = new BlockBlobStorageResource( + destinationClient, + new() + { + AccessTier = new(preserve: true), + ContentType = new(preserve: true), + ContentEncoding = new(preserve: true), + ContentDisposition = new(preserve: true), + ContentLanguage = new(preserve: true), + CacheControl = new(preserve: true), + Metadata = new(preserve: true), + }); + + DataTransferOptions options = new DataTransferOptions(); + TestEventsRaised testEventsRaised = new TestEventsRaised(options); + TransferManager transferManager = new TransferManager(); + + // Start transfer and await for completion. + DataTransfer transfer = await transferManager.StartTransferAsync( + sourceResource, + destinationResource, + options); + CancellationTokenSource cancellationTokenSource = new CancellationTokenSource(TimeSpan.FromSeconds(30)); + await TestTransferWithTimeout.WaitForCompletionAsync( + transfer, + testEventsRaised, + cancellationTokenSource.Token); + + // Assert + await VerifyBlobPropertiesCopyAsync( + transfer, + testEventsRaised, + sourceClient, + destinationClient); + // Verify Tags are NOT preserved + GetBlobTagResult destinationTags = await destinationClient.GetTagsAsync(); + Assert.IsEmpty(destinationTags.Tags); + } + + [RecordedTest] + public async Task BlockBlobToBlockBlob_NoPreserveProperties() + { + // Arrange + // Create source local file for checking, and source blob + await using DisposingContainer testContainer = await GetTestContainerAsync(publicAccessType: PublicAccessType.BlobContainer); + Metadata metadata = DataProvider.BuildMetadata(); + Tags tags = DataProvider.BuildTags(); + + // Act + // Create blob with properties + BlockBlobClient sourceClient = await SetupSourceBlockBlobAsync( + testContainer.Container, + metadata, + tags); + + StorageResourceItem sourceResource = new BlockBlobStorageResource(sourceClient); + + // Destination client - Set to not preserve properties + BlockBlobClient destinationClient = testContainer.Container.GetBlockBlobClient(GetNewBlobName()); + StorageResourceItem destinationResource = new BlockBlobStorageResource( + destinationClient, + new() + { + AccessTier = new(preserve: false), + ContentType = new(true), // For test recording content type has to be the same + ContentEncoding = new(false), + ContentDisposition = new(false), + ContentLanguage = new(false), + CacheControl = new(false), + Metadata = new(false), + }); + + DataTransferOptions options = new DataTransferOptions(); + TestEventsRaised testEventsRaised = new TestEventsRaised(options); + TransferManager transferManager = new TransferManager(); + + // Start transfer and await for completion. + DataTransfer transfer = await transferManager.StartTransferAsync( + sourceResource, + destinationResource, + options); + CancellationTokenSource cancellationTokenSource = new CancellationTokenSource(TimeSpan.FromSeconds(30)); + await TestTransferWithTimeout.WaitForCompletionAsync( + transfer, + testEventsRaised, + cancellationTokenSource.Token); + + // Assert + Assert.NotNull(transfer); + Assert.IsTrue(transfer.HasCompleted); + Assert.AreEqual(DataTransferState.Completed, transfer.TransferStatus.State); + // Verify Copy - using original source File and Copying the destination + await testEventsRaised.AssertSingleCompletedCheck(); + using Stream sourceStream = await sourceClient.OpenReadAsync(); + using Stream destinationStream = await destinationClient.OpenReadAsync(); + Assert.AreEqual(sourceStream, destinationStream); + // Verify Properties + await VerifyEmptyPropertiesAsync( + destinationClient, + checkAccessTier: true); + } + + [RecordedTest] + public async Task BlockBlobToBlockBlob_NewProperties() + { + // Arrange + // Create source local file for checking, and source blob + await using DisposingContainer testContainer = await GetTestContainerAsync(publicAccessType: PublicAccessType.BlobContainer); + Metadata sourceMetadata = DataProvider.BuildMetadata(); + Tags tags = DataProvider.BuildTags(); + + // Act + // Create blob with properties + BlockBlobClient sourceClient = await SetupSourceBlockBlobAsync( + testContainer.Container, + sourceMetadata, + tags); + + StorageResourceItem sourceResource = new BlockBlobStorageResource(sourceClient); + + // Destination client - Set to not preserve properties + BlockBlobClient destinationClient = testContainer.Container.GetBlockBlobClient(GetNewBlobName()); + Metadata destinationMetadata = new Dictionary(StringComparer.OrdinalIgnoreCase) + { + { "data", "meta" }, + { "lower", "case" }, + { "uni", "corn" } + }; + StorageResourceItem destinationResource = new BlockBlobStorageResource( + destinationClient, + new() + { + AccessTier = new(DefaultAccessTier), + ContentType = new(DefaultContentType), + ContentEncoding = new(false), + ContentDisposition = new(DefaultContentDisposition), + ContentLanguage = new(DefaultContentLanguage), + CacheControl = new(DefaultCacheControl), + Metadata = new(destinationMetadata), + }); + + DataTransferOptions options = new DataTransferOptions(); + TestEventsRaised testEventsRaised = new TestEventsRaised(options); + TransferManager transferManager = new TransferManager(); + + // Start transfer and await for completion. + DataTransfer transfer = await transferManager.StartTransferAsync( + sourceResource, + destinationResource, + options); + CancellationTokenSource cancellationTokenSource = new CancellationTokenSource(TimeSpan.FromSeconds(30)); + await TestTransferWithTimeout.WaitForCompletionAsync( + transfer, + testEventsRaised, + cancellationTokenSource.Token); + + // Assert + Assert.NotNull(transfer); + Assert.IsTrue(transfer.HasCompleted); + Assert.AreEqual(DataTransferState.Completed, transfer.TransferStatus.State); + // Verify Copy - using original source File and Copying the destination + await testEventsRaised.AssertSingleCompletedCheck(); + using Stream sourceStream = await sourceClient.OpenReadAsync(); + using Stream destinationStream = await destinationClient.OpenReadAsync(); + Assert.AreEqual(sourceStream, destinationStream); + // Verify Properties + BlobProperties destinationProperties = await destinationClient.GetPropertiesAsync(); + Assert.That(destinationMetadata, Is.EqualTo(destinationProperties.Metadata)); + Assert.AreEqual(DefaultAccessTier.ToString(), destinationProperties.AccessTier); + Assert.AreEqual(DefaultContentDisposition, destinationProperties.ContentDisposition); + Assert.AreEqual(DefaultContentLanguage, destinationProperties.ContentLanguage); + Assert.AreEqual(DefaultCacheControl, destinationProperties.CacheControl); + } + #endregion Block Blob Properties + + #region Page Blob Properties + private async Task SetupSourcePageBlobAsync( + BlobContainerClient container, + Metadata metadata, + Tags tags) + { + using DisposingLocalDirectory testDirectory = DisposingLocalDirectory.GetTestDirectory(); + string blobName = GetNewBlobName(); + int size = Constants.KB; + string newSourceFile = Path.Combine(testDirectory.DirectoryPath, GetNewBlobName()); + + PageBlobCreateOptions createOptions = new() + { + // We can't include Content Encoding since we can't record the value. + HttpHeaders = new() + { + ContentType = DefaultContentType, + ContentDisposition = DefaultContentDisposition, + ContentLanguage = DefaultContentLanguage, + CacheControl = DefaultCacheControl, + }, + Metadata = metadata, + Tags = tags + }; + return await CreatePageBlob( + container, + newSourceFile, + GetNewBlobName(), + size, + createOptions); + } + + [RecordedTest] + public async Task PageBlobToPageBlob_DefaultProperties() + { + // Arrange + // Create source local file for checking, and source blob + await using DisposingContainer testContainer = await GetTestContainerAsync(publicAccessType: PublicAccessType.BlobContainer); + Metadata metadata = DataProvider.BuildMetadata(); + Tags tags = DataProvider.BuildTags(); + + // Act + // Create blob with properties + PageBlobClient sourceClient = await SetupSourcePageBlobAsync( + testContainer.Container, + metadata, + tags); + + // Set preserve properties + StorageResourceItem sourceResource = new PageBlobStorageResource(sourceClient); + + // Destination client - Set Properties + PageBlobClient destinationClient = testContainer.Container.GetPageBlobClient(GetNewBlobName()); + StorageResourceItem destinationResource = new PageBlobStorageResource(destinationClient); + + DataTransferOptions options = new DataTransferOptions(); + TestEventsRaised testEventsRaised = new TestEventsRaised(options); + TransferManager transferManager = new TransferManager(); + + // Start transfer and await for completion. + DataTransfer transfer = await transferManager.StartTransferAsync( + sourceResource, + destinationResource, + options); + CancellationTokenSource cancellationTokenSource = new CancellationTokenSource(TimeSpan.FromSeconds(30)); + await TestTransferWithTimeout.WaitForCompletionAsync( + transfer, + testEventsRaised, + cancellationTokenSource.Token); + + // Assert + await VerifyBlobPropertiesCopyAsync( + transfer, + testEventsRaised, + sourceClient, + destinationClient); + // Verify Tags are NOT preserved + GetBlobTagResult destinationTags = await destinationClient.GetTagsAsync(); + Assert.IsEmpty(destinationTags.Tags); + } + + [RecordedTest] + public async Task PageBlobToPageBlob_PreservePropertiesNoTags() + { + // Arrange + // Create source local file for checking, and source blob + await using DisposingContainer testContainer = await GetTestContainerAsync(publicAccessType: PublicAccessType.BlobContainer); + Metadata metadata = DataProvider.BuildMetadata(); + Tags tags = DataProvider.BuildTags(); + + // Act + // Create blob with properties + PageBlobClient sourceClient = await SetupSourcePageBlobAsync( + testContainer.Container, + metadata, + tags); + + // Set preserve properties + StorageResourceItem sourceResource = new PageBlobStorageResource(sourceClient); + + // Destination client - Set Properties + PageBlobClient destinationClient = testContainer.Container.GetPageBlobClient(GetNewBlobName()); + StorageResourceItem destinationResource = new PageBlobStorageResource( + destinationClient, + new() + { + ContentType = new(preserve: true), + ContentEncoding = new(preserve: true), + ContentDisposition = new(preserve: true), + ContentLanguage = new(preserve: true), + CacheControl = new(preserve: true), + Metadata = new(preserve: true), + }); + + DataTransferOptions options = new DataTransferOptions(); + TestEventsRaised testEventsRaised = new TestEventsRaised(options); + TransferManager transferManager = new TransferManager(); + + // Start transfer and await for completion. + DataTransfer transfer = await transferManager.StartTransferAsync( + sourceResource, + destinationResource, + options); + CancellationTokenSource cancellationTokenSource = new CancellationTokenSource(TimeSpan.FromSeconds(30)); + await TestTransferWithTimeout.WaitForCompletionAsync( + transfer, + testEventsRaised, + cancellationTokenSource.Token); + + // Assert + await VerifyBlobPropertiesCopyAsync( + transfer, + testEventsRaised, + sourceClient, + destinationClient); + // Verify Tags are NOT preserved + GetBlobTagResult destinationTags = await destinationClient.GetTagsAsync(); + Assert.IsEmpty(destinationTags.Tags); + } + + [RecordedTest] + public async Task PageBlobToPageBlob_NoPreserveProperties() + { + // Arrange + // Create source local file for checking, and source blob + await using DisposingContainer testContainer = await GetTestContainerAsync(publicAccessType: PublicAccessType.BlobContainer); + Metadata metadata = DataProvider.BuildMetadata(); + Tags tags = DataProvider.BuildTags(); + + // Act + // Create blob with properties + PageBlobClient sourceClient = await SetupSourcePageBlobAsync( + testContainer.Container, + metadata, + tags); + + StorageResourceItem sourceResource = new PageBlobStorageResource(sourceClient); + + // Destination client - Set to not preserve properties + PageBlobClient destinationClient = testContainer.Container.GetPageBlobClient(GetNewBlobName()); + StorageResourceItem destinationResource = new PageBlobStorageResource( + destinationClient, + new() + { + ContentType = new(true), // For test recording content type has to be the same + ContentEncoding = new(false), + ContentDisposition = new(false), + ContentLanguage = new(false), + CacheControl = new(false), + Metadata = new(false), + }); + + DataTransferOptions options = new DataTransferOptions(); + TestEventsRaised testEventsRaised = new TestEventsRaised(options); + TransferManager transferManager = new TransferManager(); + + // Start transfer and await for completion. + DataTransfer transfer = await transferManager.StartTransferAsync( + sourceResource, + destinationResource, + options); + CancellationTokenSource cancellationTokenSource = new CancellationTokenSource(TimeSpan.FromSeconds(30)); + await TestTransferWithTimeout.WaitForCompletionAsync( + transfer, + testEventsRaised, + cancellationTokenSource.Token); + + // Assert + Assert.NotNull(transfer); + Assert.IsTrue(transfer.HasCompleted); + Assert.AreEqual(DataTransferState.Completed, transfer.TransferStatus.State); + // Verify Copy - using original source File and Copying the destination + await testEventsRaised.AssertSingleCompletedCheck(); + using Stream sourceStream = await sourceClient.OpenReadAsync(); + using Stream destinationStream = await destinationClient.OpenReadAsync(); + Assert.AreEqual(sourceStream, destinationStream); + // Verify Properties + await VerifyEmptyPropertiesAsync(destinationClient); + } + + [RecordedTest] + public async Task PageBlobToPageBlob_NewProperties() + { + // Arrange + // Create source local file for checking, and source blob + await using DisposingContainer testContainer = await GetTestContainerAsync(publicAccessType: PublicAccessType.BlobContainer); + Metadata sourceMetadata = DataProvider.BuildMetadata(); + Tags tags = DataProvider.BuildTags(); + + // Act + // Create blob with properties + PageBlobClient sourceClient = await SetupSourcePageBlobAsync( + testContainer.Container, + sourceMetadata, + tags); + + StorageResourceItem sourceResource = new PageBlobStorageResource(sourceClient); + + // Destination client - Set to not preserve properties + PageBlobClient destinationClient = testContainer.Container.GetPageBlobClient(GetNewBlobName()); + Metadata destinationMetadata = new Dictionary(StringComparer.OrdinalIgnoreCase) + { + { "data", "meta" }, + { "lower", "case" }, + { "uni", "corn" } + }; + StorageResourceItem destinationResource = new PageBlobStorageResource( + destinationClient, + new() + { + ContentType = new(DefaultContentType), + ContentEncoding = new(false), + ContentDisposition = new(DefaultContentDisposition), + ContentLanguage = new(DefaultContentLanguage), + CacheControl = new(DefaultCacheControl), + Metadata = new(destinationMetadata), + }); + + DataTransferOptions options = new DataTransferOptions(); + TestEventsRaised testEventsRaised = new TestEventsRaised(options); + TransferManager transferManager = new TransferManager(); + + // Start transfer and await for completion. + DataTransfer transfer = await transferManager.StartTransferAsync( + sourceResource, + destinationResource, + options); + CancellationTokenSource cancellationTokenSource = new CancellationTokenSource(TimeSpan.FromSeconds(30)); + await TestTransferWithTimeout.WaitForCompletionAsync( + transfer, + testEventsRaised, + cancellationTokenSource.Token); + + // Assert + Assert.NotNull(transfer); + Assert.IsTrue(transfer.HasCompleted); + Assert.AreEqual(DataTransferState.Completed, transfer.TransferStatus.State); + // Verify Copy - using original source File and Copying the destination + await testEventsRaised.AssertSingleCompletedCheck(); + using Stream sourceStream = await sourceClient.OpenReadAsync(); + using Stream destinationStream = await destinationClient.OpenReadAsync(); + Assert.AreEqual(sourceStream, destinationStream); + // Verify Properties + BlobProperties destinationProperties = await destinationClient.GetPropertiesAsync(); + Assert.That(destinationMetadata, Is.EqualTo(destinationProperties.Metadata)); + Assert.AreEqual(DefaultContentDisposition, destinationProperties.ContentDisposition); + Assert.AreEqual(DefaultContentLanguage, destinationProperties.ContentLanguage); + Assert.AreEqual(DefaultCacheControl, destinationProperties.CacheControl); + Assert.AreEqual(DefaultContentType, destinationProperties.ContentType); + } + #endregion Page Blob Properties + + #region Append Blob Properties + private async Task SetupSourceAppendBlobAsync( + BlobContainerClient container, + Metadata metadata, + Tags tags) + { + using DisposingLocalDirectory testDirectory = DisposingLocalDirectory.GetTestDirectory(); + string blobName = GetNewBlobName(); + int size = Constants.KB; + string newSourceFile = Path.Combine(testDirectory.DirectoryPath, GetNewBlobName()); + + AppendBlobCreateOptions createOptions = new() + { + // We can't include Content Encoding since we can't record the value. + HttpHeaders = new() + { + ContentType = DefaultContentType, + ContentDisposition = DefaultContentDisposition, + ContentLanguage = DefaultContentLanguage, + CacheControl = DefaultCacheControl, + }, + Metadata = metadata, + Tags = tags + }; + return await CreateAppendBlob( + container, + newSourceFile, + GetNewBlobName(), + size, + createOptions); + } + + [RecordedTest] + public async Task AppendBlobToAppendBlob_DefaultProperties() + { + // Arrange + // Create source local file for checking, and source blob + await using DisposingContainer testContainer = await GetTestContainerAsync(publicAccessType: PublicAccessType.BlobContainer); + Metadata metadata = DataProvider.BuildMetadata(); + + // Act + // Create blob with properties + AppendBlobClient sourceClient = await SetupSourceAppendBlobAsync( + testContainer.Container, + metadata, + default); + + // Set preserve properties + StorageResourceItem sourceResource = new AppendBlobStorageResource(sourceClient); + + // Destination client - Set Properties + AppendBlobClient destinationClient = testContainer.Container.GetAppendBlobClient(GetNewBlobName()); + StorageResourceItem destinationResource = new AppendBlobStorageResource(destinationClient); + + DataTransferOptions options = new DataTransferOptions(); + TestEventsRaised testEventsRaised = new TestEventsRaised(options); + TransferManager transferManager = new TransferManager(); + + // Start transfer and await for completion. + DataTransfer transfer = await transferManager.StartTransferAsync( + sourceResource, + destinationResource, + options); + CancellationTokenSource cancellationTokenSource = new CancellationTokenSource(TimeSpan.FromSeconds(30)); + await TestTransferWithTimeout.WaitForCompletionAsync( + transfer, + testEventsRaised, + cancellationTokenSource.Token); + + // Assert + await VerifyBlobPropertiesCopyAsync( + transfer, + testEventsRaised, + sourceClient, + destinationClient); + // Verify Tags are NOT preserved + GetBlobTagResult destinationTags = await destinationClient.GetTagsAsync(); + Assert.IsEmpty(destinationTags.Tags); + } + + [RecordedTest] + public async Task AppendBlobToAppendBlob_PreservePropertiesNoTags() + { + // Arrange + // Create source local file for checking, and source blob + await using DisposingContainer testContainer = await GetTestContainerAsync(publicAccessType: PublicAccessType.BlobContainer); + Metadata metadata = DataProvider.BuildMetadata(); + Tags tags = DataProvider.BuildTags(); + + // Act + // Create blob with properties + AppendBlobClient sourceClient = await SetupSourceAppendBlobAsync( + testContainer.Container, + metadata, + tags); + + // Set preserve properties + StorageResourceItem sourceResource = new AppendBlobStorageResource(sourceClient); + + // Destination client - Set Properties + AppendBlobClient destinationClient = testContainer.Container.GetAppendBlobClient(GetNewBlobName()); + StorageResourceItem destinationResource = new AppendBlobStorageResource( + destinationClient, + new() + { + ContentType = new(preserve: true), + ContentEncoding = new(preserve: true), + ContentDisposition = new(preserve: true), + ContentLanguage = new(preserve: true), + CacheControl = new(preserve: true), + Metadata = new(preserve: true), + }); + + DataTransferOptions options = new DataTransferOptions(); + TestEventsRaised testEventsRaised = new TestEventsRaised(options); + TransferManager transferManager = new TransferManager(); + + // Start transfer and await for completion. + DataTransfer transfer = await transferManager.StartTransferAsync( + sourceResource, + destinationResource, + options); + CancellationTokenSource cancellationTokenSource = new CancellationTokenSource(TimeSpan.FromSeconds(30)); + await TestTransferWithTimeout.WaitForCompletionAsync( + transfer, + testEventsRaised, + cancellationTokenSource.Token); + + // Assert + await VerifyBlobPropertiesCopyAsync( + transfer, + testEventsRaised, + sourceClient, + destinationClient); + // Verify Tags are NOT preserved + GetBlobTagResult destinationTags = await destinationClient.GetTagsAsync(); + Assert.IsEmpty(destinationTags.Tags); + } + + [RecordedTest] + public async Task AppendBlobToAppendBlob_NoPreserveProperties() + { + // Arrange + // Create source local file for checking, and source blob + await using DisposingContainer testContainer = await GetTestContainerAsync(publicAccessType: PublicAccessType.BlobContainer); + Metadata sourceMetadata = DataProvider.BuildMetadata(); + Tags tags = DataProvider.BuildTags(); + + // Act + // Create blob with properties + AppendBlobClient sourceClient = await SetupSourceAppendBlobAsync( + testContainer.Container, + sourceMetadata, + tags); + + StorageResourceItem sourceResource = new AppendBlobStorageResource(sourceClient); + + // Destination client - Set to not preserve properties + AppendBlobClient destinationClient = testContainer.Container.GetAppendBlobClient(GetNewBlobName()); + StorageResourceItem destinationResource = new AppendBlobStorageResource( + destinationClient, + new() + { + ContentType = new(true), // For test recording content type has to be the same + ContentEncoding = new(false), + ContentDisposition = new(false), + ContentLanguage = new(false), + CacheControl = new(false), + Metadata = new(false), + }); + + DataTransferOptions options = new DataTransferOptions(); + TestEventsRaised testEventsRaised = new TestEventsRaised(options); + TransferManager transferManager = new TransferManager(); + + // Start transfer and await for completion. + DataTransfer transfer = await transferManager.StartTransferAsync( + sourceResource, + destinationResource, + options); + CancellationTokenSource cancellationTokenSource = new CancellationTokenSource(TimeSpan.FromSeconds(30)); + await TestTransferWithTimeout.WaitForCompletionAsync( + transfer, + testEventsRaised, + cancellationTokenSource.Token); + + // Assert + Assert.NotNull(transfer); + Assert.IsTrue(transfer.HasCompleted); + Assert.AreEqual(DataTransferState.Completed, transfer.TransferStatus.State); + // Verify Copy - using original source File and Copying the destination + await testEventsRaised.AssertSingleCompletedCheck(); + using Stream sourceStream = await sourceClient.OpenReadAsync(); + using Stream destinationStream = await destinationClient.OpenReadAsync(); + Assert.AreEqual(sourceStream, destinationStream); + // Verify Properties + BlobProperties destinationProperties = await destinationClient.GetPropertiesAsync(); + Assert.IsNull(destinationProperties.ContentDisposition); + Assert.IsNull(destinationProperties.ContentLanguage); + Assert.IsNull(destinationProperties.CacheControl); + } + + [RecordedTest] + public async Task AppendBlobToAppendBlob_NewProperties() + { + // Arrange + // Create source local file for checking, and source blob + await using DisposingContainer testContainer = await GetTestContainerAsync(publicAccessType: PublicAccessType.BlobContainer); + Metadata sourceMetadata = DataProvider.BuildMetadata(); + Tags tags = DataProvider.BuildTags(); + + // Act + // Create blob with properties + AppendBlobClient sourceClient = await SetupSourceAppendBlobAsync( + testContainer.Container, + sourceMetadata, + tags); + + StorageResourceItem sourceResource = new AppendBlobStorageResource(sourceClient); + + // Destination client - Set to not preserve properties + AppendBlobClient destinationClient = testContainer.Container.GetAppendBlobClient(GetNewBlobName()); + Metadata destinationMetadata = new Dictionary(StringComparer.OrdinalIgnoreCase) + { + { "data", "meta" }, + { "lower", "case" }, + { "uni", "corn" } + }; + StorageResourceItem destinationResource = new AppendBlobStorageResource( + destinationClient, + new() + { + ContentType = new(DefaultContentType), + ContentEncoding = new(false), + ContentDisposition = new(DefaultContentDisposition), + ContentLanguage = new(DefaultContentLanguage), + CacheControl = new(DefaultCacheControl), + Metadata = new(destinationMetadata), + }); + + DataTransferOptions options = new DataTransferOptions(); + TestEventsRaised testEventsRaised = new TestEventsRaised(options); + TransferManager transferManager = new TransferManager(); + + // Start transfer and await for completion. + DataTransfer transfer = await transferManager.StartTransferAsync( + sourceResource, + destinationResource, + options); + CancellationTokenSource cancellationTokenSource = new CancellationTokenSource(TimeSpan.FromSeconds(30)); + await TestTransferWithTimeout.WaitForCompletionAsync( + transfer, + testEventsRaised, + cancellationTokenSource.Token); + + // Assert + Assert.NotNull(transfer); + Assert.IsTrue(transfer.HasCompleted); + Assert.AreEqual(DataTransferState.Completed, transfer.TransferStatus.State); + // Verify Copy - using original source File and Copying the destination + await testEventsRaised.AssertSingleCompletedCheck(); + using Stream sourceStream = await sourceClient.OpenReadAsync(); + using Stream destinationStream = await destinationClient.OpenReadAsync(); + Assert.AreEqual(sourceStream, destinationStream); + // Verify Properties + BlobProperties destinationProperties = await destinationClient.GetPropertiesAsync(); + Assert.That(destinationMetadata, Is.EqualTo(destinationProperties.Metadata)); + Assert.AreEqual(DefaultContentDisposition, destinationProperties.ContentDisposition); + Assert.AreEqual(DefaultContentLanguage, destinationProperties.ContentLanguage); + Assert.AreEqual(DefaultCacheControl, destinationProperties.CacheControl); + } + #endregion Append Blob Properties } } diff --git a/sdk/storage/Azure.Storage.DataMovement/tests/StreamToUriJobPartTests.cs b/sdk/storage/Azure.Storage.DataMovement/tests/StreamToUriJobPartTests.cs new file mode 100644 index 0000000000000..21cb09c8df046 --- /dev/null +++ b/sdk/storage/Azure.Storage.DataMovement/tests/StreamToUriJobPartTests.cs @@ -0,0 +1,328 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +using System; +using System.Buffers; +using System.Collections.Generic; +using System.Linq.Expressions; +using System.Threading; +using System.Threading.Tasks; +using Azure.Core; +using Azure.Core.Pipeline; +using Moq; +using NUnit.Framework; +using Azure.Storage.Test; +using System.IO; + +namespace Azure.Storage.DataMovement.Tests +{ + [TestFixture] + public class StreamToUriJobPartTests + { + private readonly int _maxDelayInSec = 1; + private const string DefaultContentType = "text/plain"; + private const string DefaultContentEncoding = "gzip"; + private const string DefaultContentLanguage = "en-US"; + private const string DefaultContentDisposition = "inline"; + private const string DefaultCacheControl = "no-cache"; + public StreamToUriJobPartTests() { } + + private static byte[] GetRandomBuffer(long size, Random random = null) + { + random ??= new Random(Environment.TickCount); + var buffer = new byte[size]; + random.NextBytes(buffer); + return buffer; + } + + private Mock GetQueueChunkTask() + { + var mock = new Mock(MockBehavior.Strict); + mock.Setup(del => del(It.IsAny>())) + .Returns(Task.CompletedTask); + return mock; + } + + private Mock GetPartQueueChunkTask() + { + var mock = new Mock(MockBehavior.Strict); + mock.Setup(del => del(It.IsAny>())) + .Callback>( + async (funcTask) => + { + await funcTask().ConfigureAwait(false); + }) + .Returns(Task.CompletedTask); + return mock; + } + + private StorageResourceItemProperties GetResourceProperties(long length) + { + IDictionary metadata = DataProvider.BuildMetadata(); + IDictionary tags = DataProvider.BuildTags(); + + Dictionary sourceProperties = new() + { + { "ContentType", DefaultContentType }, + { "ContentEncoding", DefaultContentEncoding }, + { "ContentLanguage", DefaultContentLanguage }, + { "ContentDisposition", DefaultContentDisposition }, + { "CacheControl", DefaultCacheControl }, + { "Metadata", metadata }, + { "Tags", tags } + }; + return new( + resourceLength: length, + eTag: new("ETag"), + lastModifiedTime: DateTimeOffset.UtcNow.AddHours(-1), + properties: sourceProperties); + } + + private Mock GetLocalStorageResourceItem(long length = Constants.KB) + { + Mock mock = new(); + mock.Setup(r => r.Length).Returns(length); + mock.Setup(r => r.Uri).Returns(new Uri("C:\\User\\folder\\file")); + mock.Setup(r => r.ResourceId).Returns("mock"); + mock.Setup(r => r.ProviderId).Returns("mock"); + mock.Setup(r => r.GetSourceCheckpointData()) + .Returns(new MockResourceCheckpointData()); + mock.Setup(r => r.GetDestinationCheckpointData()) + .Returns(new MockResourceCheckpointData()); + return mock; + } + + private Mock GetServiceStorageResourceItem(long length = Constants.KB) + { + Mock mock = new(); + mock.Setup(r => r.Length).Returns(length); + mock.Setup(r => r.Uri).Returns(new Uri("https://storageacount.blob.core.windows.net/container/source")); + mock.Setup(r => r.ResourceId).Returns("mock"); + mock.Setup(r => r.ProviderId).Returns("mock"); + mock.Setup(r => r.GetSourceCheckpointData()) + .Returns(new MockResourceCheckpointData()); + mock.Setup(r => r.GetDestinationCheckpointData()) + .Returns(new MockResourceCheckpointData()); + return mock; + } + + private void VerifyInvocation( + Mock destinationMock, + Expression> expectedInvocation, + int numberOfInvocationCalls = 1, + int maxWaitTimeInSec = 6) + { + CancellationTokenSource cancellationSource = new CancellationTokenSource(TimeSpan.FromSeconds(maxWaitTimeInSec)); + CancellationToken cancellationToken = cancellationSource.Token; + bool verified = false; + + try + { + do + { + CancellationHelper.ThrowIfCancellationRequested(cancellationToken); + // If it exceeds the count we should just fail. But if it's less, + // we can retry and see if the invocation we expected will be called. + Thread.Sleep(TimeSpan.FromSeconds(_maxDelayInSec)); + + try + { + destinationMock.Verify(expectedInvocation, Times.Exactly(numberOfInvocationCalls)); + verified = true; + } + catch (MockException) + { + // This exception tells us it hasn't seen the expected invocation + // which might happen due to parallelism. + } + } while (!verified); + } + catch (TaskCanceledException) + { + string message = "Timed out waiting for the correct amount of invocations for the task"; + Assert.Fail(message); + } + } + + [Test] + public async Task ProcessPartToChunkAsync_OneShot() + { + //Arrange + string transferId = Guid.NewGuid().ToString(); + long length = Constants.KB; + Mock mockQueueChunkTask = GetQueueChunkTask(); + Mock mockPartQueueChunkTask = GetPartQueueChunkTask(); + + // Set up Destination to copy in one shot with a large chunk size and smaller total length. + Mock mockDestination = GetServiceStorageResourceItem(); + mockDestination.Setup(resource => resource.CopyFromStreamAsync(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny())) + .Returns(Task.CompletedTask); + mockDestination.Setup(r => r.MaxSupportedChunkSize).Returns(Constants.MB); + + // Set up source with properties and read stream + Mock mockSource = GetLocalStorageResourceItem(length); + StorageResourceItemProperties properties = GetResourceProperties(length); + mockSource.Setup(r => r.GetPropertiesAsync(It.IsAny())) + .Returns(Task.FromResult(properties)); + var data = GetRandomBuffer(length); + using var stream = new MemoryStream(data); + StorageResourceReadStreamResult readStreamResult = new( + stream, + new HttpRange(0, length), + properties); + mockSource.Setup(r => r.ReadStreamAsync(It.IsAny(), It.IsAny(), It.IsAny())) + .Returns(Task.FromResult(readStreamResult)); + + // Set up default checkpointer with transfer job + LocalTransferCheckpointer checkpointer = new(default); + await checkpointer.AddNewJobAsync( + transferId: transferId, + source: mockSource.Object, + destination: mockDestination.Object); + + StreamToUriTransferJob job = new( + new DataTransfer( + id: transferId, + transferManager: new TransferManager()), + mockSource.Object, + mockDestination.Object, + new DataTransferOptions(), + mockQueueChunkTask.Object, + checkpointer, + DataTransferErrorMode.StopOnAnyFailure, + ArrayPool.Shared, + new ClientDiagnostics(ClientOptions.Default)); + StreamToUriJobPart jobPart = await StreamToUriJobPart.CreateJobPartAsync( + job, + 1); + jobPart.SetQueueChunkDelegate(mockPartQueueChunkTask.Object); + + // Act + await jobPart.ProcessPartToChunkAsync(); + + // Verify + VerifyInvocation( + mockDestination, + resource => resource.CopyFromStreamAsync( + stream, + length, + It.IsAny(), + length, + It.Is(options => + options != default && options.SourceProperties != default && + options.SourceProperties.Equals(properties)), + It.IsAny())); + } + + [Test] + public async Task ProcessPartToChunkAsync_Chunks() + { + // Arrange + string transferId = Guid.NewGuid().ToString(); + int length = Constants.KB * 4; + int chunkSize = Constants.KB; + int chunkAmount = length / chunkSize; + Mock mockSource = GetLocalStorageResourceItem(length); + StorageResourceItemProperties properties = GetResourceProperties(length); + mockSource.Setup(r => r.GetPropertiesAsync(It.IsAny())) + .Returns(Task.FromResult(properties)); + + // Setup destination with small chunk size and a larger source total length + // to cause chunked copy + Mock mockDestination = GetServiceStorageResourceItem(); + mockDestination.Setup(resource => resource.CopyFromStreamAsync(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny())) + .Returns(Task.CompletedTask); + mockDestination.Setup(resource => resource.CompleteTransferAsync(It.IsAny(), It.IsAny(), It.IsAny())) + .Returns(Task.CompletedTask); + mockDestination.Setup(r => r.MaxSupportedChunkSize).Returns(chunkSize); + var data = GetRandomBuffer(length); + using var stream = new MemoryStream(data); + using var stream2 = new MemoryStream(data); + using var stream3 = new MemoryStream(data); + using var stream4 = new MemoryStream(data); + mockSource.Setup(r => r.ReadStreamAsync(0, It.IsAny(), It.IsAny())) + .ReturnsAsync((long position, long? length, CancellationToken token) => + { + // Create a custom StorageResourceReadStreamResult + return new StorageResourceReadStreamResult( + stream, // Your actual stream + new HttpRange(position, chunkSize), // Your actual HttpRange + properties); // Your actual properties + }); + mockSource.Setup(r => r.ReadStreamAsync(chunkSize, It.IsAny(), It.IsAny())) + .ReturnsAsync((long position, long? length, CancellationToken token) => + { + // Create a custom StorageResourceReadStreamResult + return new StorageResourceReadStreamResult( + stream2, // Your actual stream + new HttpRange(position, chunkSize), // Your actual HttpRange + properties); // Your actual properties + }); + mockSource.Setup(r => r.ReadStreamAsync(chunkSize*2, It.IsAny(), It.IsAny())) + .ReturnsAsync((long position, long? length, CancellationToken token) => + { + // Create a custom StorageResourceReadStreamResult + return new StorageResourceReadStreamResult( + stream3, // Your actual stream + new HttpRange(position, chunkSize), // Your actual HttpRange + properties); // Your actual properties + }); + mockSource.Setup(r => r.ReadStreamAsync(chunkSize * 3, It.IsAny(), It.IsAny())) + .ReturnsAsync((long position, long? length, CancellationToken token) => + { + // Create a custom StorageResourceReadStreamResult + return new StorageResourceReadStreamResult( + stream4, // Your actual stream + new HttpRange(position, chunkSize), // Your actual HttpRange + properties); // Your actual properties + }); + + // Set up default checkpointer with transfer job + LocalTransferCheckpointer checkpointer = new(default); + await checkpointer.AddNewJobAsync( + transferId: transferId, + source: mockSource.Object, + destination: mockDestination.Object); + + Mock mockQueueChunkTask = GetQueueChunkTask(); + Mock mockPartQueueChunkTask = GetPartQueueChunkTask(); + + StreamToUriTransferJob job = new( + new DataTransfer( + id: transferId, + transferManager: new TransferManager()), + mockSource.Object, + mockDestination.Object, + new DataTransferOptions(), + mockQueueChunkTask.Object, + checkpointer, + DataTransferErrorMode.StopOnAnyFailure, + ArrayPool.Shared, + new ClientDiagnostics(ClientOptions.Default)); + StreamToUriJobPart jobPart = await StreamToUriJobPart.CreateJobPartAsync( + job, + 1); + jobPart.SetQueueChunkDelegate(mockPartQueueChunkTask.Object); + + await jobPart.ProcessPartToChunkAsync(); + + VerifyInvocation( + mockDestination, + resource => resource.CopyFromStreamAsync( + It.IsAny(), + It.IsAny(), + It.IsAny(), + It.IsAny(), + It.Is(options => options.SourceProperties.Equals(properties)), + It.IsAny()), + chunkAmount); + VerifyInvocation( + mockDestination, + resource => resource.CompleteTransferAsync( + It.IsAny(), + It.Is(options => + options.SourceProperties.Equals(properties)), + It.IsAny())); + } + } +}