diff --git a/src/Accounts/Authentication/Utilities/CustomAssemblyResolver.cs b/src/Accounts/Authentication/Utilities/CustomAssemblyResolver.cs index d82119960ca2..e6ea09556d8a 100644 --- a/src/Accounts/Authentication/Utilities/CustomAssemblyResolver.cs +++ b/src/Accounts/Authentication/Utilities/CustomAssemblyResolver.cs @@ -10,7 +10,7 @@ public static class CustomAssemblyResolver private static IDictionary NetFxPreloadAssemblies = new Dictionary(StringComparer.InvariantCultureIgnoreCase) { - {"Azure.Core", new Version("1.5.1.0")}, + {"Azure.Core", new Version("1.6.0.0")}, {"Microsoft.Bcl.AsyncInterfaces", new Version("1.0.0.0")}, {"Microsoft.Identity.Client", new Version("4.21.0.0") }, {"Microsoft.Identity.Client.Extensions.Msal", new Version("2.16.2.0") }, diff --git a/src/DataLakeStore/DataLakeStore/DataLakeStore.csproj b/src/DataLakeStore/DataLakeStore/DataLakeStore.csproj index e690af1599ac..a066619fef86 100644 --- a/src/DataLakeStore/DataLakeStore/DataLakeStore.csproj +++ b/src/DataLakeStore/DataLakeStore/DataLakeStore.csproj @@ -15,7 +15,7 @@ - + diff --git a/src/Storage/Storage.Management.Test/Storage.Management.Test.csproj b/src/Storage/Storage.Management.Test/Storage.Management.Test.csproj index d927c15535a1..de8f7bdda595 100644 --- a/src/Storage/Storage.Management.Test/Storage.Management.Test.csproj +++ b/src/Storage/Storage.Management.Test/Storage.Management.Test.csproj @@ -11,9 +11,9 @@ - - - + + + diff --git a/src/Storage/Storage.Management/ChangeLog.md b/src/Storage/Storage.Management/ChangeLog.md index a02fb5617138..e2eef04bcc4c 100644 --- a/src/Storage/Storage.Management/ChangeLog.md +++ b/src/Storage/Storage.Management/ChangeLog.md @@ -18,6 +18,11 @@ - Additional information about change #1 --> ## Upcoming Release +* Support upload Azure File size up to 4 TiB + - `Set-AzStorageFileContent` +* Upgraded Azure.Storage..Blobs to 12.7.0 +* Upgraded Azure.Storage.Files.Shares to 12.5.0 +* Upgraded Azure.Storage.Files.DataLake to 12.5.0 ## Version 3.0.0 * Removed obsolete property RestorePolicy.LastEnabledTime diff --git a/src/Storage/Storage/Common/AzureStorageFile.cs b/src/Storage/Storage/Common/AzureStorageFile.cs index cfc01d2c7c79..1ddecc6bb8f0 100644 --- a/src/Storage/Storage/Common/AzureStorageFile.cs +++ b/src/Storage/Storage/Common/AzureStorageFile.cs @@ -92,7 +92,7 @@ public AzureStorageFile(CloudFile file, AzureStorageContext storageContext) } // Convert Track1 File object to Track 2 file Client - protected static ShareFileClient GetTrack2FileClient(CloudFile cloudFile, AzureStorageContext context) + public static ShareFileClient GetTrack2FileClient(CloudFile cloudFile, AzureStorageContext context) { ShareFileClient fileClient; if (cloudFile.ServiceClient.Credentials.IsSAS) //SAS diff --git a/src/Storage/Storage/Common/Util.cs b/src/Storage/Storage/Common/Util.cs index 80c3b1b3e978..fa9db1f672ec 100644 --- a/src/Storage/Storage/Common/Util.cs +++ b/src/Storage/Storage/Common/Util.cs @@ -20,6 +20,7 @@ namespace Microsoft.WindowsAzure.Commands.Storage.Common using Microsoft.Azure.Storage.File; using XTable = Microsoft.Azure.Cosmos.Table; using System; + using System.IO; using System.Globalization; using System.Net; using System.Threading.Tasks; @@ -29,6 +30,7 @@ namespace Microsoft.WindowsAzure.Commands.Storage.Common using System.Collections; using global::Azure.Storage.Blobs; using global::Azure.Storage; + using global::Azure.Storage.Files.Shares.Models; internal static class Util { @@ -596,5 +598,86 @@ public static void ValidateUserDelegationKeyStartEndTime(DateTimeOffset userDele return null; } } + + public static FileAttributes AzureFileNtfsAttributesToLocalAttributes(NtfsFileAttributes cloudFileNtfsAttributes) + { + FileAttributes fileAttributes = FileAttributes.Normal; + + if ((cloudFileNtfsAttributes & NtfsFileAttributes.ReadOnly) == NtfsFileAttributes.ReadOnly) + fileAttributes |= FileAttributes.ReadOnly; + + if ((cloudFileNtfsAttributes & NtfsFileAttributes.Hidden) == NtfsFileAttributes.Hidden) + fileAttributes |= FileAttributes.Hidden; + + if ((cloudFileNtfsAttributes & NtfsFileAttributes.System) == NtfsFileAttributes.System) + fileAttributes |= FileAttributes.System; + + if ((cloudFileNtfsAttributes & NtfsFileAttributes.Directory) == NtfsFileAttributes.Directory) + fileAttributes |= FileAttributes.Directory; + + if ((cloudFileNtfsAttributes & NtfsFileAttributes.Archive) == NtfsFileAttributes.Archive) + fileAttributes |= FileAttributes.Archive; + + if ((cloudFileNtfsAttributes & NtfsFileAttributes.Temporary) == NtfsFileAttributes.Temporary) + fileAttributes |= FileAttributes.Temporary; + + if ((cloudFileNtfsAttributes & NtfsFileAttributes.Offline) == NtfsFileAttributes.Offline) + fileAttributes |= FileAttributes.Offline; + + if ((cloudFileNtfsAttributes & NtfsFileAttributes.NotContentIndexed) == NtfsFileAttributes.NotContentIndexed) + fileAttributes |= FileAttributes.NotContentIndexed; + + if ((cloudFileNtfsAttributes & NtfsFileAttributes.NoScrubData) == NtfsFileAttributes.NoScrubData) + fileAttributes |= FileAttributes.NoScrubData; + + if ((cloudFileNtfsAttributes & NtfsFileAttributes.None) == NtfsFileAttributes.None) + { + if (fileAttributes != FileAttributes.Normal) + { + fileAttributes = fileAttributes & (~FileAttributes.Normal); + } + } + + return fileAttributes; + } + + public static NtfsFileAttributes LocalAttributesToAzureFileNtfsAttributes(FileAttributes fileAttributes) + { + NtfsFileAttributes cloudFileNtfsAttributes = NtfsFileAttributes.None; + + if ((fileAttributes & FileAttributes.ReadOnly) == FileAttributes.ReadOnly) + cloudFileNtfsAttributes |= NtfsFileAttributes.ReadOnly; + + if ((fileAttributes & FileAttributes.Hidden) == FileAttributes.Hidden) + cloudFileNtfsAttributes |= NtfsFileAttributes.Hidden; + + if ((fileAttributes & FileAttributes.System) == FileAttributes.System) + cloudFileNtfsAttributes |= NtfsFileAttributes.System; + + if ((fileAttributes & FileAttributes.Directory) == FileAttributes.Directory) + cloudFileNtfsAttributes |= NtfsFileAttributes.Directory; + + if ((fileAttributes & FileAttributes.Archive) == FileAttributes.Archive) + cloudFileNtfsAttributes |= NtfsFileAttributes.Archive; + + if ((fileAttributes & FileAttributes.Normal) == FileAttributes.Normal) + cloudFileNtfsAttributes |= NtfsFileAttributes.None; + + if ((fileAttributes & FileAttributes.Temporary) == FileAttributes.Temporary) + cloudFileNtfsAttributes |= NtfsFileAttributes.Temporary; + + if ((fileAttributes & FileAttributes.Offline) == FileAttributes.Offline) + cloudFileNtfsAttributes |= NtfsFileAttributes.Offline; + + if ((fileAttributes & FileAttributes.NotContentIndexed) == FileAttributes.NotContentIndexed) + cloudFileNtfsAttributes |= NtfsFileAttributes.NotContentIndexed; + + if ((fileAttributes & FileAttributes.NoScrubData) == FileAttributes.NoScrubData) + cloudFileNtfsAttributes |= NtfsFileAttributes.NoScrubData; + + if (cloudFileNtfsAttributes != NtfsFileAttributes.None) cloudFileNtfsAttributes &= (~NtfsFileAttributes.None); + + return cloudFileNtfsAttributes; + } } } diff --git a/src/Storage/Storage/File/Cmdlet/SetAzureStorageFileContent.cs b/src/Storage/Storage/File/Cmdlet/SetAzureStorageFileContent.cs index 67daeb18a0c4..960e1b3db701 100644 --- a/src/Storage/Storage/File/Cmdlet/SetAzureStorageFileContent.cs +++ b/src/Storage/Storage/File/Cmdlet/SetAzureStorageFileContent.cs @@ -14,22 +14,26 @@ namespace Microsoft.WindowsAzure.Commands.Storage.File.Cmdlet { + using global::Azure; + using global::Azure.Storage.Files.Shares; + using global::Azure.Storage.Files.Shares.Models; + using Microsoft.Azure.Storage; + using Microsoft.Azure.Storage.DataMovement; + using Microsoft.Azure.Storage.File; using Microsoft.WindowsAzure.Commands.Common; + using Microsoft.WindowsAzure.Commands.Common.Storage.ResourceModel; using Microsoft.WindowsAzure.Commands.Storage.Common; using Microsoft.WindowsAzure.Commands.Utilities.Common; - using Microsoft.Azure.Storage; - using Microsoft.Azure.Storage.File; using System; + using System.Collections.Generic; using System.Globalization; using System.IO; using System.Management.Automation; using System.Net; + using System.Runtime.InteropServices; + using System.Security.Cryptography; using System.Threading.Tasks; using LocalConstants = Microsoft.WindowsAzure.Commands.Storage.File.Constants; - using System.Runtime.InteropServices; - using Microsoft.Azure.Storage.DataMovement; - using Microsoft.WindowsAzure.Commands.Common.CustomAttributes; - using Microsoft.WindowsAzure.Commands.Common.Storage.ResourceModel; [Cmdlet("Set", Azure.Commands.ResourceManager.Common.AzureRMConstants.AzurePrefix + "StorageFileContent", SupportsShouldProcess = true, DefaultParameterSetName = LocalConstants.ShareNameParameterSetName), OutputType(typeof(AzureStorageFile))] public class SetAzureStorageFileContent : StorageFileDataManagementCmdletBase, IDynamicParameters @@ -110,11 +114,19 @@ public override void ExecuteCmdlet() { throw new FileNotFoundException(string.Format(CultureInfo.CurrentCulture, Resources.SourceFileNotFound, this.Source)); } + long fileSize = localFile.Length; - // if FIPS policy is enabled, must use native MD5 + // if FIPS policy is enabled, must use native MD5 for DMlib. if (fipsEnabled) { - CloudStorageAccount.UseV1MD5 = false; + if (fileSize < sizeTB) + { + CloudStorageAccount.UseV1MD5 = false; + } + else // use Track2 SDK + { + WriteWarning("The uploaded file won't have Content MD5 hash, since caculate MD5 hash fail, most possiblly caused by FIPS is enabled on this machine."); + } } bool isDirectory; @@ -133,19 +145,141 @@ public override void ExecuteCmdlet() cloudFileToBeUploaded.GetFullPath(), cloudFileToBeUploaded.Share.Name), Resources.PrepareUploadingFile); - await DataMovementTransferHelper.DoTransfer(() => - this.TransferManager.UploadAsync( - localFile.FullName, - cloudFileToBeUploaded, - new UploadOptions + if (fileSize <= sizeTB) + { + + + await DataMovementTransferHelper.DoTransfer(() => + this.TransferManager.UploadAsync( + localFile.FullName, + cloudFileToBeUploaded, + new UploadOptions + { + PreserveSMBAttributes = context is null ? false : context.PreserveSMBAttribute.IsPresent + }, + this.GetTransferContext(progressRecord, localFile.Length), + this.CmdletCancellationToken), + progressRecord, + this.OutputStream).ConfigureAwait(false); + } + else // use Track2 SDK + { + //Create File + ShareFileClient fileClient = AzureStorageFile.GetTrack2FileClient(cloudFileToBeUploaded, Channel.StorageContext); + + // confirm overwrite if file exist + if(!this.Force.IsPresent && + fileClient.Exists(this.CmdletCancellationToken) && + !await this.OutputStream.ConfirmAsync(string.Format(CultureInfo.CurrentCulture, Resources.OverwriteConfirmation, Util.ConvertToString(cloudFileToBeUploaded)))) + { + return; + } + + await fileClient.CreateAsync(fileSize, cancellationToken: this.CmdletCancellationToken).ConfigureAwait(false); + + //Prepare progress Handler + IProgress progressHandler = new Progress((finishedBytes) => + { + if (progressRecord != null) + { + // Size of the source file might be 0, when it is, directly treat the progress as 100 percent. + progressRecord.PercentComplete = 0 == fileSize ? 100 : (int)(finishedBytes * 100 / fileSize); + progressRecord.StatusDescription = string.Format(CultureInfo.CurrentCulture, Resources.FileTransmitStatus, progressRecord.PercentComplete); + this.OutputStream.WriteProgress(progressRecord); + } + }); + + long blockSize = 4 * 1024 * 1024; + int maxWorkers = 4; + + List runningTasks = new List(); + + IncrementalHash hash = null; + if (!fipsEnabled) + { + hash = IncrementalHash.CreateHash(HashAlgorithmName.MD5); + } + + using (FileStream stream = File.OpenRead(localFile.FullName)) + { + byte[] buffer = null; + long lastBlockSize = 0; + for (long offset = 0; offset < fileSize; offset += blockSize) + { + long currentBlockSize = offset + blockSize < fileSize ? blockSize : fileSize - offset; + + // Only need to create new buffer when chunk size change + if (currentBlockSize != lastBlockSize) + { + buffer = new byte[currentBlockSize]; + lastBlockSize = currentBlockSize; + } + await stream.ReadAsync(buffer: buffer, offset: 0, count: (int)currentBlockSize); + if (!fipsEnabled && hash != null) + { + hash.AppendData(buffer); + } + + Task task = UploadFileRangAsync(fileClient, + new HttpRange(offset, currentBlockSize), + new MemoryStream(buffer), + progressHandler); + runningTasks.Add(task); + + // Check if any of upload range tasks are still busy + if (runningTasks.Count >= maxWorkers) + { + await Task.WhenAny(runningTasks).ConfigureAwait(false); + + // Clear any completed blocks from the task list + for (int i = 0; i < runningTasks.Count; i++) + { + Task runningTask = runningTasks[i]; + if (!runningTask.IsCompleted) + { + continue; + } + + await runningTask.ConfigureAwait(false); + runningTasks.RemoveAt(i); + i--; + } + } + } + // Wait for all upload range tasks finished + await Task.WhenAll(runningTasks).ConfigureAwait(false); + } + + // Need set file properties + if ((!fipsEnabled && hash != null) || (context != null && context.PreserveSMBAttribute.IsPresent)) + { + ShareFileHttpHeaders header = null; + if (!fipsEnabled && hash != null) + { + header = new ShareFileHttpHeaders(); + header.ContentHash = hash.GetHashAndReset(); + } + + FileSmbProperties smbProperties = null; + if (context != null && context.PreserveSMBAttribute.IsPresent) { - PreserveSMBAttributes = context is null ? false : context.PreserveSMBAttribute.IsPresent - }, - this.GetTransferContext(progressRecord, localFile.Length), - this.CmdletCancellationToken), - progressRecord, - this.OutputStream).ConfigureAwait(false); + FileInfo sourceFileInfo = new FileInfo(localFile.FullName); + smbProperties = new FileSmbProperties(); + smbProperties.FileCreatedOn = sourceFileInfo.CreationTimeUtc; + smbProperties.FileLastWrittenOn = sourceFileInfo.LastWriteTimeUtc; + smbProperties.FileAttributes = Util.LocalAttributesToAzureFileNtfsAttributes(File.GetAttributes(localFile.FullName)); + } + // set file header and attributes to the file + fileClient.SetHttpHeaders(httpHeaders: header, smbProperties: smbProperties); + } + + if (this.PassThru) + { + // fetch latest file properties for output + cloudFileToBeUploaded.FetchAttributes(); + } + } if (this.PassThru) { @@ -160,6 +294,17 @@ await DataMovementTransferHelper.DoTransfer(() => } } + private long Finishedbytes = 0; + private async Task UploadFileRangAsync(ShareFileClient file, HttpRange range, Stream content, IProgress progressHandler = null) + { + await file.UploadRangeAsync( + range, + content, + cancellationToken: this.CmdletCancellationToken).ConfigureAwait(false); + Finishedbytes += range.Length is null? 0 : range.Length.Value; + progressHandler.Report(Finishedbytes); + } + private async Task BuildCloudFileInstanceFromPathAsync(string defaultFileName, string[] path, bool pathIsDirectory) { CloudFileDirectory baseDirectory = null; diff --git a/src/Storage/Storage/File/StorageFileDataManagementCmdletBase.cs b/src/Storage/Storage/File/StorageFileDataManagementCmdletBase.cs index 9ae98b7fdfc0..420640b4a4b4 100644 --- a/src/Storage/Storage/File/StorageFileDataManagementCmdletBase.cs +++ b/src/Storage/Storage/File/StorageFileDataManagementCmdletBase.cs @@ -32,6 +32,8 @@ public abstract class StorageFileDataManagementCmdletBase : AzureStorageFileCmdl /// private const int DefaultConcurrentTaskCount = 10; + public const long sizeTB = (long)1024 * 1024 * 1024 * 1024; + /// /// Stores the transfer manager instance. /// @@ -54,7 +56,7 @@ public SwitchParameter Force [Parameter(Mandatory = false, HelpMessage = "Run cmdlet in the background")] public virtual SwitchParameter AsJob { get; set; } - + /// /// Confirm the overwrite operation /// @@ -101,7 +103,7 @@ protected override void EndProcessing() } } - protected void DoEndProcessing() + protected void DoEndProcessing() { try { diff --git a/src/Storage/Storage/Storage.csproj b/src/Storage/Storage/Storage.csproj index 347f6fb170b5..23a07b54247e 100644 --- a/src/Storage/Storage/Storage.csproj +++ b/src/Storage/Storage/Storage.csproj @@ -12,9 +12,9 @@ - - - + + + diff --git a/src/Storage/Storage/Storage.format.ps1xml b/src/Storage/Storage/Storage.format.ps1xml index 6043deb84d8f..fb6b511df4c5 100644 --- a/src/Storage/Storage/Storage.format.ps1xml +++ b/src/Storage/Storage/Storage.format.ps1xml @@ -84,7 +84,7 @@ $_.DirectoryTag - $_.Length + $_.Properties.Length $_.Name diff --git a/src/Storage/Storage/Storage.generated.format.ps1xml b/src/Storage/Storage/Storage.generated.format.ps1xml index a9070fce1621..0280e4be021c 100644 --- a/src/Storage/Storage/Storage.generated.format.ps1xml +++ b/src/Storage/Storage/Storage.generated.format.ps1xml @@ -348,7 +348,7 @@ if ($_.CloudFile -eq $null) {"Directory"} else {"File"} - $_.Properties.Length + $_.Length $_.Name diff --git a/src/lib/NetCorePreloadAssemblies/Azure.Core.dll b/src/lib/NetCorePreloadAssemblies/Azure.Core.dll index 8b8cba478b0a..d5fc2d1bcf53 100644 Binary files a/src/lib/NetCorePreloadAssemblies/Azure.Core.dll and b/src/lib/NetCorePreloadAssemblies/Azure.Core.dll differ diff --git a/src/lib/NetFxPreloadAssemblies/Azure.Core.dll b/src/lib/NetFxPreloadAssemblies/Azure.Core.dll index d718ed462327..7dab01e5906a 100644 Binary files a/src/lib/NetFxPreloadAssemblies/Azure.Core.dll and b/src/lib/NetFxPreloadAssemblies/Azure.Core.dll differ diff --git a/tools/Common.Netcore.Dependencies.targets b/tools/Common.Netcore.Dependencies.targets index 873343ee127d..474b55a97121 100644 --- a/tools/Common.Netcore.Dependencies.targets +++ b/tools/Common.Netcore.Dependencies.targets @@ -19,6 +19,9 @@ + + + All diff --git a/tools/StaticAnalysis/Exceptions/Az.Batch/MissingAssemblies.csv b/tools/StaticAnalysis/Exceptions/Az.Batch/MissingAssemblies.csv index f2dd9c4977b5..f4c3c324b6e8 100644 --- a/tools/StaticAnalysis/Exceptions/Az.Batch/MissingAssemblies.csv +++ b/tools/StaticAnalysis/Exceptions/Az.Batch/MissingAssemblies.csv @@ -1,3 +1,5 @@ "Directory","Assembly Name","Assembly Version","Referencing Assembly","Severity","ProblemId","Description","Remediation" "Az.Batch","Microsoft.Win32.Registry","4.1.1.0","System.Management.Automation","0","3000","Missing assembly Microsoft.Win32.Registry referenced from System.Management.Automation","Ensure that the assembly is included in the Wix file or directory" "Az.Batch","Microsoft.PowerShell.CoreCLR.Eventing","6.1.0.0","System.Management.Automation","0","3000","Missing assembly Microsoft.PowerShell.CoreCLR.Eventing referenced from System.Management.Automation","Ensure that the assembly is included in the Wix file or directory" +"Az.Batch","System.Runtime.CompilerServices.Unsafe","4.0.4.0","Microsoft.Extensions.Primitives","0","3000","Missing assembly System.Runtime.CompilerServices.Unsafe referenced from Microsoft.Extensions.Primitives","Ensure that the assembly is included in the Wix file or directory" +"Az.Batch","System.Text.Encodings.Web","4.0.3.0","Microsoft.AspNetCore.WebUtilities","0","3000","Missing assembly System.Text.Encodings.Web referenced from Microsoft.AspNetCore.WebUtilities","Ensure that the assembly is included in the Wix file or directory" diff --git a/tools/StaticAnalysis/Exceptions/Az.Resources/MissingAssemblies.csv b/tools/StaticAnalysis/Exceptions/Az.Resources/MissingAssemblies.csv index e8bfcff58a34..36343204d28d 100644 --- a/tools/StaticAnalysis/Exceptions/Az.Resources/MissingAssemblies.csv +++ b/tools/StaticAnalysis/Exceptions/Az.Resources/MissingAssemblies.csv @@ -1,3 +1,4 @@ "Directory","Assembly Name","Assembly Version","Referencing Assembly","Severity","ProblemId","Description","Remediation" "Az.Resources","Microsoft.Win32.Registry","4.1.1.0","System.Management.Automation","0","3000","Missing assembly Microsoft.Win32.Registry referenced from System.Management.Automation","Ensure that the assembly is included in the Wix file or directory" "Az.Resources","Microsoft.PowerShell.CoreCLR.Eventing","6.1.0.0","System.Management.Automation","0","3000","Missing assembly Microsoft.PowerShell.CoreCLR.Eventing referenced from System.Management.Automation","Ensure that the assembly is included in the Wix file or directory" +"Az.Resources","System.Runtime.CompilerServices.Unsafe","4.0.4.0","Microsoft.Extensions.Primitives","0","3000","Missing assembly System.Runtime.CompilerServices.Unsafe referenced from Microsoft.Extensions.Primitives","Ensure that the assembly is included in the Wix file or directory" \ No newline at end of file