diff --git a/sdk/storage/Azure.Storage.DataMovement.Files.Shares/src/Azure.Storage.DataMovement.Files.Shares.csproj b/sdk/storage/Azure.Storage.DataMovement.Files.Shares/src/Azure.Storage.DataMovement.Files.Shares.csproj index 3126c27f07636..6da055363dd06 100644 --- a/sdk/storage/Azure.Storage.DataMovement.Files.Shares/src/Azure.Storage.DataMovement.Files.Shares.csproj +++ b/sdk/storage/Azure.Storage.DataMovement.Files.Shares/src/Azure.Storage.DataMovement.Files.Shares.csproj @@ -40,6 +40,7 @@ + diff --git a/sdk/storage/Azure.Storage.DataMovement.Files.Shares/src/ShareFileDestinationCheckpointData.cs b/sdk/storage/Azure.Storage.DataMovement.Files.Shares/src/ShareFileDestinationCheckpointData.cs index 4001b2d719ab7..574110d3047a1 100644 --- a/sdk/storage/Azure.Storage.DataMovement.Files.Shares/src/ShareFileDestinationCheckpointData.cs +++ b/sdk/storage/Azure.Storage.DataMovement.Files.Shares/src/ShareFileDestinationCheckpointData.cs @@ -10,7 +10,7 @@ namespace Azure.Storage.DataMovement.Files.Shares { - internal class ShareFileDestinationCheckpointData : StorageResourceCheckpointData + internal class ShareFileDestinationCheckpointData : StorageResourceCheckpointDataInternal { private const char HeaderDelimiter = Constants.CommaChar; @@ -67,8 +67,6 @@ public ShareFileDestinationCheckpointData( _filePermissionKeyBytes = SmbProperties?.FilePermissionKey != default ? Encoding.UTF8.GetBytes(SmbProperties.FilePermissionKey) : Array.Empty(); } - internal void SerializeInternal(Stream stream) => Serialize(stream); - protected override void Serialize(Stream stream) { Argument.AssertNotNull(stream, nameof(stream)); @@ -204,8 +202,8 @@ internal static ShareFileDestinationCheckpointData Deserialize(Stream stream) ShareFileHttpHeaders contentHeaders = new() { ContentType = contentType, - ContentEncoding = contentEncoding.Split(HeaderDelimiter), - ContentLanguage = contentLanguage.Split(HeaderDelimiter), + ContentEncoding = contentEncoding?.Split(HeaderDelimiter), + ContentLanguage = contentLanguage?.Split(HeaderDelimiter), ContentDisposition = contentDisposition, CacheControl = cacheControl, }; diff --git a/sdk/storage/Azure.Storage.DataMovement.Files.Shares/src/ShareFileSourceCheckpointData.cs b/sdk/storage/Azure.Storage.DataMovement.Files.Shares/src/ShareFileSourceCheckpointData.cs index 29f104eedc1d2..c00ac03ce9fb8 100644 --- a/sdk/storage/Azure.Storage.DataMovement.Files.Shares/src/ShareFileSourceCheckpointData.cs +++ b/sdk/storage/Azure.Storage.DataMovement.Files.Shares/src/ShareFileSourceCheckpointData.cs @@ -5,12 +5,10 @@ namespace Azure.Storage.DataMovement.Files.Shares { - internal class ShareFileSourceCheckpointData : StorageResourceCheckpointData + internal class ShareFileSourceCheckpointData : StorageResourceCheckpointDataInternal { public override int Length => 0; - internal void SerializeInternal(Stream stream) => Serialize(stream); - protected override void Serialize(Stream stream) { } diff --git a/sdk/storage/Azure.Storage.DataMovement.Files.Shares/src/ShareFilesStorageResourceProvider.cs b/sdk/storage/Azure.Storage.DataMovement.Files.Shares/src/ShareFilesStorageResourceProvider.cs index 9735d98f3d669..ceb7a052577bf 100644 --- a/sdk/storage/Azure.Storage.DataMovement.Files.Shares/src/ShareFilesStorageResourceProvider.cs +++ b/sdk/storage/Azure.Storage.DataMovement.Files.Shares/src/ShareFilesStorageResourceProvider.cs @@ -2,6 +2,7 @@ // Licensed under the MIT License. using System; +using System.IO; using System.Threading; using System.Threading.Tasks; using Azure.Core; @@ -222,19 +223,35 @@ public ShareFilesStorageResourceProvider(GetAzureSasCredential getAzureSasCreden #region Abstract Class Implementation /// - protected override async Task FromSourceAsync(DataTransferProperties properties, CancellationToken cancellationToken) - => await FromTransferPropertiesAsync(properties, getSource: true, cancellationToken).ConfigureAwait(false); + protected override Task FromSourceAsync(DataTransferProperties properties, CancellationToken cancellationToken) + { + // Source share file data currently empty, so no specific properties to grab - /// - protected override async Task FromDestinationAsync(DataTransferProperties properties, CancellationToken cancellationToken) - => await FromTransferPropertiesAsync(properties, getSource: false, cancellationToken).ConfigureAwait(false); + return Task.FromResult(properties.IsContainer + ? FromDirectory(properties.SourceUri.AbsoluteUri) + : FromFile(properties.SourceUri.AbsoluteUri)); + } - private Task FromTransferPropertiesAsync( - DataTransferProperties properties, - bool getSource, - CancellationToken cancellationToken) + /// + protected override Task FromDestinationAsync(DataTransferProperties properties, CancellationToken cancellationToken) { - throw new NotImplementedException(); + ShareFileDestinationCheckpointData checkpointData; + using (MemoryStream stream = new(properties.DestinationCheckpointData)) + { + checkpointData = ShareFileDestinationCheckpointData.Deserialize(stream); + } + + ShareFileStorageResourceOptions options = new() + { + SmbProperties = checkpointData.SmbProperties, + HttpHeaders = checkpointData.ContentHeaders, + DirectoryMetadata = checkpointData.DirectoryMetadata, + FileMetadata = checkpointData.FileMetadata, + }; + + return Task.FromResult(properties.IsContainer + ? FromDirectory(properties.DestinationUri.AbsoluteUri, options) + : FromFile(properties.DestinationUri.AbsoluteUri, options)); } /// diff --git a/sdk/storage/Azure.Storage.DataMovement.Files.Shares/tests/RehydrateShareResourceTests.cs b/sdk/storage/Azure.Storage.DataMovement.Files.Shares/tests/RehydrateShareResourceTests.cs new file mode 100644 index 0000000000000..4c57d09495878 --- /dev/null +++ b/sdk/storage/Azure.Storage.DataMovement.Files.Shares/tests/RehydrateShareResourceTests.cs @@ -0,0 +1,176 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using Azure.Storage.Files.Shares.Models; +using Azure.Storage.Test; +using Azure.Storage.Tests; +using Moq; +using NUnit.Framework; + +namespace Azure.Storage.DataMovement.Files.Shares.Tests +{ + public class RehydrateShareResourceTests + { + public const string ShareProviderId = "share"; + + private static byte[] GetBytes(StorageResourceCheckpointDataInternal checkpointData) + { + using MemoryStream stream = new(); + checkpointData.SerializeInternal(stream); + return stream.ToArray(); + } + + private static Mock GetProperties( + string transferId, + string sourcePath, + string destinationPath, + string sourceProviderId, + string destinationProviderId, + bool isContainer, + ShareFileSourceCheckpointData sourceCheckpointData, + ShareFileDestinationCheckpointData destinationCheckpointData) + { + var mock = new Mock(MockBehavior.Strict); + mock.Setup(p => p.TransferId).Returns(transferId); + mock.Setup(p => p.SourceUri).Returns(new Uri(sourcePath)); + mock.Setup(p => p.DestinationUri).Returns(new Uri(destinationPath)); + mock.Setup(p => p.SourceProviderId).Returns(sourceProviderId); + mock.Setup(p => p.DestinationProviderId).Returns(destinationProviderId); + mock.Setup(p => p.SourceCheckpointData).Returns(GetBytes(sourceCheckpointData)); + mock.Setup(p => p.DestinationCheckpointData).Returns(GetBytes(destinationCheckpointData)); + mock.Setup(p => p.IsContainer).Returns(isContainer); + return mock; + } + + [Test] + public async Task RehydrateFile( + [Values(true, false)] bool isSource) + { + string transferId = Guid.NewGuid().ToString(); + string sourcePath = "https://storageaccount.file.core.windows.net/share/dir1/file1"; + string destinationPath = "https://storageaccount.file.core.windows.net/share/dir2/file2"; + string originalPath = isSource ? sourcePath : destinationPath; + + DataTransferProperties transferProperties = GetProperties( + transferId, + sourcePath, + destinationPath, + ShareProviderId, + ShareProviderId, + isContainer: false, + new ShareFileSourceCheckpointData(), + new ShareFileDestinationCheckpointData(null, null, null, null)).Object; + + StorageResource storageResource = isSource + ? await new ShareFilesStorageResourceProvider().FromSourceInternalHookAsync(transferProperties) + : await new ShareFilesStorageResourceProvider().FromDestinationInternalHookAsync(transferProperties); + + Assert.That(originalPath, Is.EqualTo(storageResource.Uri.AbsoluteUri)); + Assert.That(storageResource, Is.TypeOf()); + } + + [Test] + public async Task RehydrateFile_DestinationOptions() + { + string transferId = Guid.NewGuid().ToString(); + string sourcePath = "https://storageaccount.file.core.windows.net/share/dir1/file1"; + string destinationPath = "https://storageaccount.file.core.windows.net/share/dir2/file2"; + + Random r = new(); + ShareFileDestinationCheckpointData originalDestinationData = new( + new ShareFileHttpHeaders + { + ContentType = "text/plain", + ContentEncoding = new string[] { "gzip" }, + ContentLanguage = new string[] { "en-US" }, + ContentDisposition = "inline", + CacheControl = "no-cache", + }, + new Dictionary + { + { r.NextString(8), r.NextString(8) } + }, + new Dictionary + { + { r.NextString(8), r.NextString(8) } + }, + new FileSmbProperties + { + FileAttributes = NtfsFileAttributes.Archive, + FilePermissionKey = r.NextString(8), + FileLastWrittenOn = DateTimeOffset.Now, + FileChangedOn = DateTimeOffset.Now, + FileCreatedOn = DateTimeOffset.Now, + }); + DataTransferProperties transferProperties = GetProperties( + transferId, + sourcePath, + destinationPath, + ShareProviderId, + ShareProviderId, + isContainer: false, + new ShareFileSourceCheckpointData(), + originalDestinationData).Object; + + ShareFileStorageResource storageResource = (ShareFileStorageResource) + await new ShareFilesStorageResourceProvider().FromDestinationInternalHookAsync(transferProperties); + + Assert.That(destinationPath, Is.EqualTo(storageResource.Uri.AbsoluteUri)); + Assert.That(storageResource, Is.TypeOf()); + Assert.That(storageResource._options.HttpHeaders.ContentType, Is.EqualTo(originalDestinationData.ContentHeaders.ContentType)); + Assert.That(storageResource._options.HttpHeaders.ContentEncoding, Is.EqualTo(originalDestinationData.ContentHeaders.ContentEncoding)); + Assert.That(storageResource._options.HttpHeaders.ContentLanguage, Is.EqualTo(originalDestinationData.ContentHeaders.ContentLanguage)); + Assert.That(storageResource._options.HttpHeaders.ContentDisposition, Is.EqualTo(originalDestinationData.ContentHeaders.ContentDisposition)); + Assert.That(storageResource._options.HttpHeaders.CacheControl, Is.EqualTo(originalDestinationData.ContentHeaders.CacheControl)); + Assert.That(storageResource._options.FileMetadata, Is.EqualTo(originalDestinationData.FileMetadata)); + Assert.That(storageResource._options.DirectoryMetadata, Is.EqualTo(originalDestinationData.DirectoryMetadata)); + Assert.That(storageResource._options.SmbProperties.FileAttributes, Is.EqualTo(originalDestinationData.SmbProperties.FileAttributes)); + Assert.That(storageResource._options.SmbProperties.FilePermissionKey, Is.EqualTo(originalDestinationData.SmbProperties.FilePermissionKey)); + Assert.That(storageResource._options.SmbProperties.FileCreatedOn, Is.EqualTo(originalDestinationData.SmbProperties.FileCreatedOn)); + Assert.That(storageResource._options.SmbProperties.FileLastWrittenOn, Is.EqualTo(originalDestinationData.SmbProperties.FileLastWrittenOn)); + Assert.That(storageResource._options.SmbProperties.FileChangedOn, Is.EqualTo(originalDestinationData.SmbProperties.FileChangedOn)); + } + + [Test] + public async Task RehydrateDirectory( + [Values(true, false)] bool isSource) + { + string transferId = Guid.NewGuid().ToString(); + List sourcePaths = new List(); + string sourcePath = "https://storageaccount.file.core.windows.net/share/dir1"; + List destinationPaths = new List(); + string destinationPath = "https://storageaccount.file.core.windows.net/share/dir2"; + string originalPath = isSource ? sourcePath : destinationPath; + int jobPartCount = 10; + for (int i = 0; i < jobPartCount; i++) + { + string childPath = DataProvider.GetNewString(5); + sourcePaths.Add(string.Join("/", sourcePath, childPath)); + destinationPaths.Add(string.Join("/", destinationPath, childPath)); + } + + DataTransferProperties transferProperties = GetProperties( + transferId, + sourcePath, + destinationPath, + ShareProviderId, + ShareProviderId, + isContainer: true, + new ShareFileSourceCheckpointData(), + new ShareFileDestinationCheckpointData(null, null, null, null)).Object; + + StorageResource storageResource = isSource + ? await new ShareFilesStorageResourceProvider().FromSourceInternalHookAsync(transferProperties) + : await new ShareFilesStorageResourceProvider().FromDestinationInternalHookAsync(transferProperties); + + Assert.That(originalPath, Is.EqualTo(storageResource.Uri.AbsoluteUri)); + Assert.That(storageResource, Is.TypeOf()); + } + } +} diff --git a/sdk/storage/Azure.Storage.DataMovement/src/Shared/StorageResourceCheckpointDataInternal.cs b/sdk/storage/Azure.Storage.DataMovement/src/Shared/StorageResourceCheckpointDataInternal.cs new file mode 100644 index 0000000000000..d7eedfd3576ae --- /dev/null +++ b/sdk/storage/Azure.Storage.DataMovement/src/Shared/StorageResourceCheckpointDataInternal.cs @@ -0,0 +1,21 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +using System.Collections.Generic; +using System.IO; +using System.Threading; +using System.Threading.Tasks; + +namespace Azure.Storage.DataMovement +{ + /// + /// Middle class between the public type and implementation class. + /// Gives internal hook methods to protected methods of + /// , allowing for internal + /// package use as well as testing access. + /// + internal abstract class StorageResourceCheckpointDataInternal : StorageResourceCheckpointData + { + internal void SerializeInternal(Stream stream) => Serialize(stream); + } +}