From b5b461033fef077107d008da6427e8ecce369779 Mon Sep 17 00:00:00 2001 From: Jocelyn <41338290+jaschrep-msft@users.noreply.github.com> Date: Wed, 21 Feb 2024 15:51:10 -0500 Subject: [PATCH 01/22] Structured Message Decode Stream (#42079) * Initial implementation and basic test * seek/write tests * fix test param * fix exceptions --- .../src/Azure.Storage.Blobs.csproj | 3 + .../Azure.Storage.Common/src/Shared/Errors.cs | 16 + .../src/Shared/StructuredMessage.cs | 191 ++++++++ .../Shared/StructuredMessageDecodingStream.cs | 429 ++++++++++++++++++ .../tests/Azure.Storage.Common.Tests.csproj | 3 + .../StructuredMessageDecodingStreamTests.cs | 137 ++++++ .../tests/StructuredMessageHelper.cs | 56 +++ 7 files changed, 835 insertions(+) create mode 100644 sdk/storage/Azure.Storage.Common/src/Shared/StructuredMessage.cs create mode 100644 sdk/storage/Azure.Storage.Common/src/Shared/StructuredMessageDecodingStream.cs create mode 100644 sdk/storage/Azure.Storage.Common/tests/StructuredMessageDecodingStreamTests.cs create mode 100644 sdk/storage/Azure.Storage.Common/tests/StructuredMessageHelper.cs diff --git a/sdk/storage/Azure.Storage.Blobs/src/Azure.Storage.Blobs.csproj b/sdk/storage/Azure.Storage.Blobs/src/Azure.Storage.Blobs.csproj index 32b8511ab6ab4..11a4fdff8aba4 100644 --- a/sdk/storage/Azure.Storage.Blobs/src/Azure.Storage.Blobs.csproj +++ b/sdk/storage/Azure.Storage.Blobs/src/Azure.Storage.Blobs.csproj @@ -91,6 +91,9 @@ + + + diff --git a/sdk/storage/Azure.Storage.Common/src/Shared/Errors.cs b/sdk/storage/Azure.Storage.Common/src/Shared/Errors.cs index 6b89a59011d51..6e50037782e0d 100644 --- a/sdk/storage/Azure.Storage.Common/src/Shared/Errors.cs +++ b/sdk/storage/Azure.Storage.Common/src/Shared/Errors.cs @@ -80,6 +80,22 @@ internal static void VerifyStreamPosition(Stream stream, string streamName) } } + internal static void AssertBufferMinimumSize(ReadOnlySpan buffer, int minSize, string paramName) + { + if (buffer.Length < minSize) + { + throw new ArgumentException($"Expected buffer Length of at least {minSize} bytes. Got {buffer.Length}.", paramName); + } + } + + internal static void AssertBufferExactSize(ReadOnlySpan buffer, int size, string paramName) + { + if (buffer.Length != size) + { + throw new ArgumentException($"Expected buffer Length of exactly {size} bytes. Got {buffer.Length}.", paramName); + } + } + public static void ThrowIfParamNull(object obj, string paramName) { if (obj == null) diff --git a/sdk/storage/Azure.Storage.Common/src/Shared/StructuredMessage.cs b/sdk/storage/Azure.Storage.Common/src/Shared/StructuredMessage.cs new file mode 100644 index 0000000000000..5c19fe0469074 --- /dev/null +++ b/sdk/storage/Azure.Storage.Common/src/Shared/StructuredMessage.cs @@ -0,0 +1,191 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +using System; +using System.Buffers; +using System.Buffers.Binary; +using System.IO; +using System.Security.Cryptography; +using Azure.Core; + +namespace Azure.Storage.Shared; + +internal static class StructuredMessage +{ + public const int Crc64Length = 8; + + [Flags] + public enum Flags + { + None = 0, + CrcSegment = 1, + } + + public static class V1_0 + { + public const byte MessageVersionByte = 1; + + public const int StreamHeaderLength = 13; + public const int SegmentHeaderLength = 10; + + #region Stream Header + public static void ReadStreamHeader( + ReadOnlySpan buffer, + out long messageLength, + out Flags flags, + out int totalSegments) + { + Errors.AssertBufferExactSize(buffer, 13, nameof(buffer)); + if (buffer[0] != 1) + { + throw new InvalidDataException("Unrecognized version of structured message."); + } + messageLength = (long)BinaryPrimitives.ReadUInt64LittleEndian(buffer.Slice(1, 8)); + flags = (Flags)BinaryPrimitives.ReadUInt16LittleEndian(buffer.Slice(9, 2)); + totalSegments = BinaryPrimitives.ReadUInt16LittleEndian(buffer.Slice(11, 2)); + } + + public static int WriteStreamHeader( + Span buffer, + long messageLength, + Flags flags, + int totalSegments) + { + const int versionOffset = 0; + const int messageLengthOffset = 1; + const int flagsOffset = 9; + const int numSegmentsOffset = 11; + + Errors.AssertBufferMinimumSize(buffer, StreamHeaderLength, nameof(buffer)); + + buffer[versionOffset] = MessageVersionByte; + BinaryPrimitives.WriteUInt64LittleEndian(buffer.Slice(messageLengthOffset, 8), (ulong)messageLength); + BinaryPrimitives.WriteUInt16LittleEndian(buffer.Slice(flagsOffset, 2), (ushort)flags); + BinaryPrimitives.WriteUInt16LittleEndian(buffer.Slice(numSegmentsOffset, 2), (ushort)totalSegments); + + return StreamHeaderLength; + } + + /// + /// Gets stream header in a buffer rented from the provided ArrayPool. + /// + /// + /// Disposable to return the buffer to the pool. + /// + public static IDisposable GetStreamHeaderBytes( + ArrayPool pool, + out Memory bytes, + long messageLength, + Flags flags, + int totalSegments) + { + Argument.AssertNotNull(pool, nameof(pool)); + IDisposable disposable = pool.RentAsMemoryDisposable(StreamHeaderLength, out bytes); + WriteStreamHeader(bytes.Span, messageLength, flags, totalSegments); + return disposable; + } + #endregion + + // no stream footer content in 1.0 + + #region SegmentHeader + public static void ReadSegmentHeader( + ReadOnlySpan buffer, + out int segmentNum, + out long contentLength) + { + Errors.AssertBufferExactSize(buffer, 10, nameof(buffer)); + segmentNum = BinaryPrimitives.ReadUInt16LittleEndian(buffer.Slice(0, 2)); + contentLength = (long)BinaryPrimitives.ReadUInt64LittleEndian(buffer.Slice(2, 8)); + } + + public static int WriteSegmentHeader(Span buffer, int segmentNum, long segmentLength) + { + const int segmentNumOffset = 0; + const int segmentLengthOffset = 2; + + Errors.AssertBufferMinimumSize(buffer, SegmentHeaderLength, nameof(buffer)); + + BinaryPrimitives.WriteUInt16LittleEndian(buffer.Slice(segmentNumOffset, 2), (ushort)segmentNum); + BinaryPrimitives.WriteUInt64LittleEndian(buffer.Slice(segmentLengthOffset, 8), (ulong)segmentLength); + + return SegmentHeaderLength; + } + + /// + /// Gets segment header in a buffer rented from the provided ArrayPool. + /// + /// + /// Disposable to return the buffer to the pool. + /// + public static IDisposable GetSegmentHeaderBytes( + ArrayPool pool, + out Memory bytes, + int segmentNum, + long segmentLength) + { + Argument.AssertNotNull(pool, nameof(pool)); + IDisposable disposable = pool.RentAsMemoryDisposable(SegmentHeaderLength, out bytes); + WriteSegmentHeader(bytes.Span, segmentNum, segmentLength); + return disposable; + } + #endregion + + #region SegmentFooter + public static void ReadSegmentFooter( + ReadOnlySpan buffer, + Span crc64 = default) + { + int expectedBufferSize = 0; + if (!crc64.IsEmpty) + { + Errors.AssertBufferExactSize(crc64, Crc64Length, nameof(crc64)); + expectedBufferSize += Crc64Length; + } + Errors.AssertBufferExactSize(buffer, expectedBufferSize, nameof(buffer)); + + if (!crc64.IsEmpty) + { + buffer.Slice(0, Crc64Length).CopyTo(crc64); + } + } + + public static int WriteSegmentFooter(Span buffer, ReadOnlySpan crc64 = default) + { + int requiredSpace = 0; + if (!crc64.IsEmpty) + { + Errors.AssertBufferExactSize(crc64, Crc64Length, nameof(crc64)); + requiredSpace += Crc64Length; + } + + Errors.AssertBufferMinimumSize(buffer, requiredSpace, nameof(buffer)); + int offset = 0; + if (!crc64.IsEmpty) + { + crc64.CopyTo(buffer.Slice(offset, Crc64Length)); + offset += Crc64Length; + } + + return offset; + } + + /// + /// Gets stream header in a buffer rented from the provided ArrayPool. + /// + /// + /// Disposable to return the buffer to the pool. + /// + public static IDisposable GetSegmentFooterBytes( + ArrayPool pool, + out Memory bytes, + ReadOnlySpan crc64 = default) + { + Argument.AssertNotNull(pool, nameof(pool)); + IDisposable disposable = pool.RentAsMemoryDisposable(StreamHeaderLength, out bytes); + WriteSegmentFooter(bytes.Span, crc64); + return disposable; + } + #endregion + } +} diff --git a/sdk/storage/Azure.Storage.Common/src/Shared/StructuredMessageDecodingStream.cs b/sdk/storage/Azure.Storage.Common/src/Shared/StructuredMessageDecodingStream.cs new file mode 100644 index 0000000000000..b979cfb85a059 --- /dev/null +++ b/sdk/storage/Azure.Storage.Common/src/Shared/StructuredMessageDecodingStream.cs @@ -0,0 +1,429 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +using System; +using System.Buffers; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using Azure.Core; + +namespace Azure.Storage.Shared; + +/// +/// Decodes a structured message stream as the data is read. +/// +/// +/// Wraps the inner stream in a , which avoids using its internal +/// buffer if individual Read() calls are larger than it. This ensures one of the three scenarios +/// +/// +/// Read buffer >= stream buffer: +/// There is enough space in the read buffer for inline metadata to be safely +/// extracted in only one read to the true inner stream. +/// +/// +/// Read buffer < next inline metadata: +/// The stream buffer has been activated, and we can read multiple small times from the inner stream +/// without multi-reading the real stream, even when partway through an existing stream buffer. +/// +/// +/// Else: +/// Same as #1, but also the already-allocated stream buffer has been used to slightly improve +/// resource churn when reading inner stream. +/// +/// +/// +internal class StructuredMessageDecodingStream : Stream +{ + private readonly Stream _innerBufferedStream; + + private byte[] _metadataBuffer = ArrayPool.Shared.Rent(Constants.KB); + private int _metadataBufferOffset = 0; + private int _metadataBufferLength = 0; + + private int _streamHeaderLength; + // private readonly int _streamFooterLength; // no stream footer in 1.0 + private int _segmentHeaderLength; + private int _segmentFooterLength; + private int _totalSegments; + + private StructuredMessage.Flags _flags; + private bool _disposed; + + private StorageCrc64HashAlgorithm _totalContentCrc; + private StorageCrc64HashAlgorithm _segmentCrc; + + public override bool CanRead => true; + + public override bool CanWrite => false; + + public override bool CanSeek => false; + + public override bool CanTimeout => _innerBufferedStream.CanTimeout; + + public override int ReadTimeout => _innerBufferedStream.ReadTimeout; + + public override int WriteTimeout => _innerBufferedStream.WriteTimeout; + + public override long Length => throw new NotSupportedException(); + + public override long Position + { + get => throw new NotSupportedException(); + set => throw new NotSupportedException(); + } + + #region Position + private long _innerStreamLength; + + private enum SMRegion + { + StreamHeader, + StreamFooter, + SegmentHeader, + SegmentFooter, + SegmentContent, + } + #endregion + + public StructuredMessageDecodingStream( + Stream innerStream) + { + Argument.AssertNotNull(innerStream, nameof(innerStream)); + _innerBufferedStream = new BufferedStream(innerStream); + + // Assumes stream will be structured message 1.0. Will validate this when consuming stream. + _streamHeaderLength = StructuredMessage.V1_0.StreamHeaderLength; + _segmentHeaderLength = StructuredMessage.V1_0.SegmentHeaderLength; + } + + #region Write + public override void Flush() => throw new NotSupportedException(); + + public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException(); + + public override void SetLength(long value) => throw new NotSupportedException(); + #endregion + + #region Read + public override int Read(byte[] buf, int offset, int count) + { + int decodedRead; + int read; + do + { + read = _innerBufferedStream.Read(buf, offset, count); + decodedRead = Decode(new Span(buf, offset, read)); + } while (decodedRead <= 0 && read > 0); + + return decodedRead; + } + + public override async Task ReadAsync(byte[] buf, int offset, int count, CancellationToken cancellationToken) + { + int decodedRead; + int read; + do + { + read = await _innerBufferedStream.ReadAsync(buf, offset, count, cancellationToken).ConfigureAwait(false); + decodedRead = Decode(new Span(buf, offset, read)); + } while (decodedRead <= 0 && read > 0); + + return decodedRead; + } + +#if NETSTANDARD2_1_OR_GREATER || NETCOREAPP3_0_OR_GREATER + public override int Read(Span buf) + { + int decodedRead; + int read; + do + { + read = _innerBufferedStream.Read(buf); + decodedRead = Decode(buf.Slice(0, read)); + } while (decodedRead <= 0 && read > 0); + + return decodedRead; + } + + public override async ValueTask ReadAsync(Memory buf, CancellationToken cancellationToken = default) + { + int decodedRead; + int read; + do + { + read = await _innerBufferedStream.ReadAsync(buf).ConfigureAwait(false); + decodedRead = Decode(buf.Slice(0, read).Span); + } while (decodedRead <= 0 && read > 0); + + return decodedRead; + } +#endif + + private SMRegion _currentRegion; + private int _currentSegmentNum; + private long _currentSegmentContentLength; + private long _currentSegmentContentRemaining; + private long CurrentRegionLength => _currentRegion switch + { + SMRegion.StreamHeader => _streamHeaderLength, + SMRegion.StreamFooter => 0, + SMRegion.SegmentHeader => _segmentHeaderLength, + SMRegion.SegmentFooter => _segmentFooterLength, + SMRegion.SegmentContent => _currentSegmentContentLength, + _ => 0, + }; + + /// + /// Decodes given bytes in place. Decoding based on internal stream position info. + /// Decoded data size will be less than or equal to encoded data length. + /// + /// + /// Length of the decoded data in . + /// + private int Decode(Span buffer) + { + if (buffer.IsEmpty) + { + return 0; + } + List<(int Offset, int Count)> gaps = new(); + + int bufferConsumed = ProcessMetadataBuffer(buffer); + + if (bufferConsumed > 0) + { + gaps.Add((0, bufferConsumed)); + } + + while (bufferConsumed < buffer.Length) + { + if (_currentRegion == SMRegion.SegmentContent) + { + int read = (int)Math.Min(buffer.Length - bufferConsumed, _currentSegmentContentRemaining); + _totalContentCrc?.Append(buffer.Slice(bufferConsumed, read)); + _segmentCrc?.Append(buffer.Slice(bufferConsumed, read)); + bufferConsumed += read; + _currentSegmentContentRemaining -= read; + if (_currentSegmentContentRemaining == 0) + { + _currentRegion = SMRegion.SegmentFooter; + } + } + else if (buffer.Length - bufferConsumed < CurrentRegionLength) + { + SavePartialMetadata(buffer.Slice(bufferConsumed)); + gaps.Add((bufferConsumed, buffer.Length - bufferConsumed)); + bufferConsumed = buffer.Length; + } + else + { + int processed = _currentRegion switch + { + SMRegion.StreamHeader => ProcessStreamHeader(buffer.Slice(bufferConsumed)), + SMRegion.StreamFooter => ProcessStreamFooter(buffer.Slice(bufferConsumed)), + SMRegion.SegmentHeader => ProcessSegmentHeader(buffer.Slice(bufferConsumed)), + SMRegion.SegmentFooter => ProcessSegmentFooter(buffer.Slice(bufferConsumed)), + _ => 0, + }; + gaps.Add((bufferConsumed, processed)); + bufferConsumed += processed; + } + } + + if (gaps.Count == 0) + { + return buffer.Length; + } + + // gaps is already sorted by offset due to how it was assembled + int gap = 0; + for (int i = gaps.First().Offset; i < buffer.Length; i++) + { + if (gaps.Count > 0 && gaps.First().Offset == i) + { + int count = gaps.First().Count; + gap += count; + i += count - 1; + gaps.RemoveAt(0); + } + else + { + buffer[i - gap] = buffer[i]; + } + } + return buffer.Length - gap; + } + + /// + /// Processes metadata in the internal buffer, if any. Appends any necessary data + /// from the append buffer to complete metadata. + /// + /// + /// Bytes consumed from . + /// + private int ProcessMetadataBuffer(ReadOnlySpan append) + { + if (_metadataBufferLength == 0) + { + return 0; + } + if (_currentRegion == SMRegion.SegmentContent) + { + return 0; + } + int appended = 0; + if (_metadataBufferLength < CurrentRegionLength && append.Length > 0) + { + appended = Math.Min((int)CurrentRegionLength - _metadataBufferLength, append.Length); + SavePartialMetadata(append.Slice(0, appended)); + } + if (_metadataBufferLength == CurrentRegionLength) + { + Span metadata = new(_metadataBuffer, _metadataBufferOffset, (int)CurrentRegionLength); + switch (_currentRegion) + { + case SMRegion.StreamHeader: + ProcessStreamHeader(metadata); + break; + case SMRegion.StreamFooter: + ProcessStreamFooter(metadata); + break; + case SMRegion.SegmentHeader: + ProcessSegmentHeader(metadata); + break; + case SMRegion.SegmentFooter: + ProcessSegmentFooter(metadata); + break; + } + _metadataBufferOffset = 0; + _metadataBufferLength = 0; + } + return appended; + } + + private void SavePartialMetadata(ReadOnlySpan span) + { + // safety array resize w/ArrayPool + if (_metadataBufferLength + span.Length > _metadataBuffer.Length) + { + ResizeMetadataBuffer(2 * (_metadataBufferLength + span.Length)); + } + + // realign any existing content if necessary + if (_metadataBufferLength != 0 && _metadataBufferOffset != 0) + { + // don't use Array.Copy() to move elements in the same array + for (int i = 0; i < _metadataBufferLength; i++) + { + _metadataBuffer[i] = _metadataBuffer[i + _metadataBufferOffset]; + } + _metadataBufferOffset = 0; + } + + span.CopyTo(new Span(_metadataBuffer, _metadataBufferOffset + _metadataBufferLength, span.Length)); + _metadataBufferLength += span.Length; + } + + private int ProcessStreamHeader(ReadOnlySpan span) + { + StructuredMessage.V1_0.ReadStreamHeader( + span.Slice(0, _streamHeaderLength), + out _innerStreamLength, + out _flags, + out _totalSegments); + if (_flags.HasFlag(StructuredMessage.Flags.CrcSegment)) + { + _segmentFooterLength = _flags.HasFlag(StructuredMessage.Flags.CrcSegment) ? StructuredMessage.Crc64Length : 0; + _segmentCrc = StorageCrc64HashAlgorithm.Create(); + _totalContentCrc = StorageCrc64HashAlgorithm.Create(); + } + _currentRegion = SMRegion.SegmentHeader; + return _streamHeaderLength; + } + + private int ProcessStreamFooter(ReadOnlySpan span) + { + return 0; + } + + private int ProcessSegmentHeader(ReadOnlySpan span) + { + StructuredMessage.V1_0.ReadSegmentHeader( + span.Slice(0, _segmentHeaderLength), + out int newSegNum, + out _currentSegmentContentLength); + _currentSegmentContentRemaining = _currentSegmentContentLength; + if (newSegNum != _currentSegmentNum + 1) + { + throw new InvalidDataException("Unexpected segment number in structured message."); + } + _currentSegmentNum = newSegNum; + _currentRegion = SMRegion.SegmentContent; + return _segmentHeaderLength; + } + + private int ProcessSegmentFooter(ReadOnlySpan span) + { + int totalProcessed = 0; + if (_flags.HasFlag(StructuredMessage.Flags.CrcSegment)) + { + totalProcessed += StructuredMessage.Crc64Length; + using (ArrayPool.Shared.RentAsSpanDisposable(StructuredMessage.Crc64Length, out Span calculated)) + { + _segmentCrc.GetCurrentHash(calculated); + _segmentCrc = StorageCrc64HashAlgorithm.Create(); + ReadOnlySpan expected = span.Slice(0, StructuredMessage.Crc64Length); + if (!calculated.SequenceEqual(expected)) + { + throw Errors.ChecksumMismatch(calculated, expected); + } + } + } + _currentRegion = _currentSegmentNum == _totalSegments ? SMRegion.StreamFooter : SMRegion.SegmentHeader; + return totalProcessed; + } + #endregion + + public override long Seek(long offset, SeekOrigin origin) + => throw new NotSupportedException(); + + protected override void Dispose(bool disposing) + { + base.Dispose(disposing); + + if (_disposed) + { + return; + } + + if (disposing) + { + _innerBufferedStream.Dispose(); + _disposed = true; + } + } + + private void ResizeMetadataBuffer(int newSize) + { + byte[] newBuf = ArrayPool.Shared.Rent(newSize); + Array.Copy(_metadataBuffer, _metadataBufferOffset, newBuf, 0, _metadataBufferLength); + ArrayPool.Shared.Return(_metadataBuffer); + _metadataBuffer = newBuf; + } + + private void AlignMetadataBuffer() + { + if (_metadataBufferOffset != 0 && _metadataBufferLength != 0) + { + for (int i = 0; i < _metadataBufferLength; i++) + { + _metadataBuffer[i] = _metadataBuffer[_metadataBufferOffset + i]; + } + _metadataBufferOffset = 0; + } + } +} diff --git a/sdk/storage/Azure.Storage.Common/tests/Azure.Storage.Common.Tests.csproj b/sdk/storage/Azure.Storage.Common/tests/Azure.Storage.Common.Tests.csproj index 5db86ebee984b..bef5c97ccd50a 100644 --- a/sdk/storage/Azure.Storage.Common/tests/Azure.Storage.Common.Tests.csproj +++ b/sdk/storage/Azure.Storage.Common/tests/Azure.Storage.Common.Tests.csproj @@ -46,6 +46,9 @@ + + + diff --git a/sdk/storage/Azure.Storage.Common/tests/StructuredMessageDecodingStreamTests.cs b/sdk/storage/Azure.Storage.Common/tests/StructuredMessageDecodingStreamTests.cs new file mode 100644 index 0000000000000..5525b52619b9c --- /dev/null +++ b/sdk/storage/Azure.Storage.Common/tests/StructuredMessageDecodingStreamTests.cs @@ -0,0 +1,137 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +using System; +using System.IO; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using Azure.Storage.Blobs.Tests; +using Azure.Storage.Shared; +using NUnit.Framework; +using static Azure.Storage.Shared.StructuredMessage; + +namespace Azure.Storage.Tests +{ + [TestFixture(ReadMethod.SyncArray)] + [TestFixture(ReadMethod.AsyncArray)] +#if NETSTANDARD2_1_OR_GREATER || NETCOREAPP3_0_OR_GREATER + [TestFixture(ReadMethod.SyncSpan)] + [TestFixture(ReadMethod.AsyncMemory)] +#endif + public class StructuredMessageDecodingStreamTests + { + // Cannot just implement as passthru in the stream + // Must test each one + public enum ReadMethod + { + SyncArray, + AsyncArray, +#if NETSTANDARD2_1_OR_GREATER || NETCOREAPP3_0_OR_GREATER + SyncSpan, + AsyncMemory +#endif + } + + public ReadMethod Method { get; } + + public StructuredMessageDecodingStreamTests(ReadMethod method) + { + Method = method; + } + + private async ValueTask CopyStream(Stream source, Stream destination, int bufferSize = 81920) // number default for CopyTo impl + { + byte[] buf = new byte[bufferSize]; + int read; + switch (Method) + { + case ReadMethod.SyncArray: + while ((read = source.Read(buf, 0, bufferSize)) > 0) + { + destination.Write(buf, 0, read); + } + break; + case ReadMethod.AsyncArray: + while ((read = await source.ReadAsync(buf, 0, bufferSize)) > 0) + { + await destination.WriteAsync(buf, 0, read); + } + break; +#if NETSTANDARD2_1_OR_GREATER || NETCOREAPP3_0_OR_GREATER + case ReadMethod.SyncSpan: + while ((read = source.Read(new Span(buf))) > 0) + { + destination.Write(new Span(buf, 0, read)); + } + break; + case ReadMethod.AsyncMemory: + while ((read = await source.ReadAsync(new Memory(buf))) > 0) + { + await destination.WriteAsync(new Memory(buf, 0, read)); + } + break; +#endif + } + destination.Flush(); + } + + [Test] + [Pairwise] + public async Task DecodesData( + [Values(2048, 2005)] int dataLength, + [Values(default, 512)] int? seglen, + [Values(8*Constants.KB, 512, 530, 3)] int readLen, + [Values(true, false)] bool useCrc) + { + int segmentContentLength = seglen ?? int.MaxValue; + Flags flags = useCrc ? Flags.CrcSegment : Flags.None; + + byte[] originalData = new byte[dataLength]; + new Random().NextBytes(originalData); + byte[] encodedData = StructuredMessageHelper.MakeEncodedData(originalData, segmentContentLength, flags); + + Stream encodingStream = new StructuredMessageDecodingStream(new MemoryStream(encodedData)); + byte[] decodedData; + using (MemoryStream dest = new()) + { + await CopyStream(encodingStream, dest, readLen); + decodedData = dest.ToArray(); + } + + Assert.That(new Span(decodedData).SequenceEqual(originalData)); + } + + [Test] + public void NoSeek() + { + StructuredMessageDecodingStream stream = new(new MemoryStream()); + + Assert.That(stream.CanSeek, Is.False); + Assert.That(() => stream.Length, Throws.TypeOf()); + Assert.That(() => stream.Position, Throws.TypeOf()); + Assert.That(() => stream.Position = 0, Throws.TypeOf()); + Assert.That(() => stream.Seek(0, SeekOrigin.Begin), Throws.TypeOf()); + } + + [Test] + public void NoWrite() + { + StructuredMessageDecodingStream stream = new(new MemoryStream()); + byte[] data = new byte[1024]; + new Random().NextBytes(data); + + Assert.That(stream.CanWrite, Is.False); + Assert.That(() => stream.Write(data, 0, data.Length), + Throws.TypeOf()); + Assert.That(async () => await stream.WriteAsync(data, 0, data.Length, CancellationToken.None), + Throws.TypeOf()); +#if NETSTANDARD2_1_OR_GREATER || NETCOREAPP3_0_OR_GREATER + Assert.That(() => stream.Write(new Span(data)), + Throws.TypeOf()); + Assert.That(async () => await stream.WriteAsync(new Memory(data), CancellationToken.None), + Throws.TypeOf()); +#endif + } + } +} diff --git a/sdk/storage/Azure.Storage.Common/tests/StructuredMessageHelper.cs b/sdk/storage/Azure.Storage.Common/tests/StructuredMessageHelper.cs new file mode 100644 index 0000000000000..ffc8c9e935a22 --- /dev/null +++ b/sdk/storage/Azure.Storage.Common/tests/StructuredMessageHelper.cs @@ -0,0 +1,56 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using Azure.Storage.Shared; +using static Azure.Storage.Shared.StructuredMessage; + +namespace Azure.Storage.Blobs.Tests +{ + internal class StructuredMessageHelper + { + public static byte[] MakeEncodedData(byte[] data, long segmentContentLength, Flags flags) + { + int segmentCount = (int) Math.Ceiling(data.Length / (double)segmentContentLength); + int segmentFooterLen = flags.HasFlag(Flags.CrcSegment) ? 8 : 0; + + byte[] encodedData = new byte[V1_0.StreamHeaderLength + segmentCount*(V1_0.SegmentHeaderLength + segmentFooterLen) + data.Length]; + V1_0.WriteStreamHeader( + new Span(encodedData, 0, V1_0.StreamHeaderLength), + encodedData.Length, + flags, + segmentCount); + + int i = V1_0.StreamHeaderLength; + int j = 0; + foreach (int seg in Enumerable.Range(1, segmentCount)) + { + int segContentLen = Math.Min((int)segmentContentLength, data.Length - j); + V1_0.WriteSegmentHeader( + new Span(encodedData, i, V1_0.SegmentHeaderLength), + seg, + segContentLen); + i += V1_0.SegmentHeaderLength; + + new Span(data, j, segContentLen) + .CopyTo(new Span(encodedData).Slice(i)); + i += segContentLen; + + if (flags.HasFlag(Flags.CrcSegment)) + { + var crc = StorageCrc64HashAlgorithm.Create(); + crc.Append(new Span(data).Slice(j, segContentLen)); + crc.GetCurrentHash(new Span(encodedData, i, Crc64Length)); + i += Crc64Length; + } + j += segContentLen; + } + + return encodedData; + } + } +} From fc1396279a389391d9d38451c1a65382e354f166 Mon Sep 17 00:00:00 2001 From: Jocelyn <41338290+jaschrep-msft@users.noreply.github.com> Date: Wed, 28 Feb 2024 10:58:20 -0500 Subject: [PATCH 02/22] Content validation update spec (#42191) * enum rename and footer read/write methods * align encode/decode tests | update encoding stream * decode stream footer * rename --- .../src/Shared/StructuredMessage.cs | 60 +- .../Shared/StructuredMessageDecodingStream.cs | 28 +- .../Shared/StructuredMessageEncodingStream.cs | 545 ++++++++++++++++++ .../StructuredMessageDecodingStreamTests.cs | 6 +- .../StructuredMessageEncodingStreamTests.cs | 246 ++++++++ .../tests/StructuredMessageHelper.cs | 26 +- .../tests/StructuredMessageTests.cs | 114 ++++ 7 files changed, 1006 insertions(+), 19 deletions(-) create mode 100644 sdk/storage/Azure.Storage.Common/src/Shared/StructuredMessageEncodingStream.cs create mode 100644 sdk/storage/Azure.Storage.Common/tests/StructuredMessageEncodingStreamTests.cs create mode 100644 sdk/storage/Azure.Storage.Common/tests/StructuredMessageTests.cs diff --git a/sdk/storage/Azure.Storage.Common/src/Shared/StructuredMessage.cs b/sdk/storage/Azure.Storage.Common/src/Shared/StructuredMessage.cs index 5c19fe0469074..d963224536fb3 100644 --- a/sdk/storage/Azure.Storage.Common/src/Shared/StructuredMessage.cs +++ b/sdk/storage/Azure.Storage.Common/src/Shared/StructuredMessage.cs @@ -5,7 +5,6 @@ using System.Buffers; using System.Buffers.Binary; using System.IO; -using System.Security.Cryptography; using Azure.Core; namespace Azure.Storage.Shared; @@ -18,7 +17,7 @@ internal static class StructuredMessage public enum Flags { None = 0, - CrcSegment = 1, + StorageCrc64 = 1, } public static class V1_0 @@ -86,7 +85,62 @@ public static IDisposable GetStreamHeaderBytes( } #endregion - // no stream footer content in 1.0 + #region StreamFooter + public static void ReadStreamFooter( + ReadOnlySpan buffer, + Span crc64 = default) + { + int expectedBufferSize = 0; + if (!crc64.IsEmpty) + { + Errors.AssertBufferExactSize(crc64, Crc64Length, nameof(crc64)); + expectedBufferSize += Crc64Length; + } + Errors.AssertBufferExactSize(buffer, expectedBufferSize, nameof(buffer)); + + if (!crc64.IsEmpty) + { + buffer.Slice(0, Crc64Length).CopyTo(crc64); + } + } + + public static int WriteStreamFooter(Span buffer, ReadOnlySpan crc64 = default) + { + int requiredSpace = 0; + if (!crc64.IsEmpty) + { + Errors.AssertBufferExactSize(crc64, Crc64Length, nameof(crc64)); + requiredSpace += Crc64Length; + } + + Errors.AssertBufferMinimumSize(buffer, requiredSpace, nameof(buffer)); + int offset = 0; + if (!crc64.IsEmpty) + { + crc64.CopyTo(buffer.Slice(offset, Crc64Length)); + offset += Crc64Length; + } + + return offset; + } + + /// + /// Gets stream header in a buffer rented from the provided ArrayPool. + /// + /// + /// Disposable to return the buffer to the pool. + /// + public static IDisposable GetStreamFooterBytes( + ArrayPool pool, + out Memory bytes, + ReadOnlySpan crc64 = default) + { + Argument.AssertNotNull(pool, nameof(pool)); + IDisposable disposable = pool.RentAsMemoryDisposable(StreamHeaderLength, out bytes); + WriteStreamFooter(bytes.Span, crc64); + return disposable; + } + #endregion #region SegmentHeader public static void ReadSegmentHeader( diff --git a/sdk/storage/Azure.Storage.Common/src/Shared/StructuredMessageDecodingStream.cs b/sdk/storage/Azure.Storage.Common/src/Shared/StructuredMessageDecodingStream.cs index b979cfb85a059..ef6e1af60604d 100644 --- a/sdk/storage/Azure.Storage.Common/src/Shared/StructuredMessageDecodingStream.cs +++ b/sdk/storage/Azure.Storage.Common/src/Shared/StructuredMessageDecodingStream.cs @@ -45,7 +45,7 @@ internal class StructuredMessageDecodingStream : Stream private int _metadataBufferLength = 0; private int _streamHeaderLength; - // private readonly int _streamFooterLength; // no stream footer in 1.0 + private int _streamFooterLength; private int _segmentHeaderLength; private int _segmentFooterLength; private int _totalSegments; @@ -170,7 +170,7 @@ public override async ValueTask ReadAsync(Memory buf, CancellationTok private long CurrentRegionLength => _currentRegion switch { SMRegion.StreamHeader => _streamHeaderLength, - SMRegion.StreamFooter => 0, + SMRegion.StreamFooter => _streamFooterLength, SMRegion.SegmentHeader => _segmentHeaderLength, SMRegion.SegmentFooter => _segmentFooterLength, SMRegion.SegmentContent => _currentSegmentContentLength, @@ -229,6 +229,7 @@ private int Decode(Span buffer) SMRegion.SegmentFooter => ProcessSegmentFooter(buffer.Slice(bufferConsumed)), _ => 0, }; + // TODO surface error if processed is 0 gaps.Add((bufferConsumed, processed)); bufferConsumed += processed; } @@ -335,9 +336,10 @@ private int ProcessStreamHeader(ReadOnlySpan span) out _innerStreamLength, out _flags, out _totalSegments); - if (_flags.HasFlag(StructuredMessage.Flags.CrcSegment)) + if (_flags.HasFlag(StructuredMessage.Flags.StorageCrc64)) { - _segmentFooterLength = _flags.HasFlag(StructuredMessage.Flags.CrcSegment) ? StructuredMessage.Crc64Length : 0; + _segmentFooterLength = _flags.HasFlag(StructuredMessage.Flags.StorageCrc64) ? StructuredMessage.Crc64Length : 0; + _streamFooterLength = _flags.HasFlag(StructuredMessage.Flags.StorageCrc64) ? StructuredMessage.Crc64Length : 0; _segmentCrc = StorageCrc64HashAlgorithm.Create(); _totalContentCrc = StorageCrc64HashAlgorithm.Create(); } @@ -347,7 +349,21 @@ private int ProcessStreamHeader(ReadOnlySpan span) private int ProcessStreamFooter(ReadOnlySpan span) { - return 0; + int totalProcessed = 0; + if (_flags.HasFlag(StructuredMessage.Flags.StorageCrc64)) + { + totalProcessed += StructuredMessage.Crc64Length; + using (ArrayPool.Shared.RentAsSpanDisposable(StructuredMessage.Crc64Length, out Span calculated)) + { + _totalContentCrc.GetCurrentHash(calculated); + ReadOnlySpan expected = span.Slice(0, StructuredMessage.Crc64Length); + if (!calculated.SequenceEqual(expected)) + { + throw Errors.ChecksumMismatch(calculated, expected); + } + } + } + return totalProcessed; } private int ProcessSegmentHeader(ReadOnlySpan span) @@ -369,7 +385,7 @@ private int ProcessSegmentHeader(ReadOnlySpan span) private int ProcessSegmentFooter(ReadOnlySpan span) { int totalProcessed = 0; - if (_flags.HasFlag(StructuredMessage.Flags.CrcSegment)) + if (_flags.HasFlag(StructuredMessage.Flags.StorageCrc64)) { totalProcessed += StructuredMessage.Crc64Length; using (ArrayPool.Shared.RentAsSpanDisposable(StructuredMessage.Crc64Length, out Span calculated)) diff --git a/sdk/storage/Azure.Storage.Common/src/Shared/StructuredMessageEncodingStream.cs b/sdk/storage/Azure.Storage.Common/src/Shared/StructuredMessageEncodingStream.cs new file mode 100644 index 0000000000000..935b830ec5d7f --- /dev/null +++ b/sdk/storage/Azure.Storage.Common/src/Shared/StructuredMessageEncodingStream.cs @@ -0,0 +1,545 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +using System; +using System.Buffers; +using System.IO; +using System.Threading; +using System.Threading.Tasks; +using Azure.Core; +using Azure.Core.Pipeline; + +namespace Azure.Storage.Shared; + +internal class StructuredMessageEncodingStream : Stream +{ + private readonly Stream _innerStream; + + private readonly int _streamHeaderLength; + private readonly int _streamFooterLength; + private readonly int _segmentHeaderLength; + private readonly int _segmentFooterLength; + private readonly int _segmentContentLength; + + private readonly StructuredMessage.Flags _flags; + private bool _disposed; + + private bool UseCrcSegment => _flags.HasFlag(StructuredMessage.Flags.StorageCrc64); + private readonly StorageCrc64HashAlgorithm _totalCrc; + private StorageCrc64HashAlgorithm _segmentCrc; + private readonly byte[] _segmentCrcs; + private int _latestSegmentCrcd = 0; + + #region Segments + /// + /// Gets the 1-indexed segment number the underlying stream is currently positioned in. + /// 1-indexed to match segment labelling as specified by SM spec. + /// + private int CurrentInnerSegment => (int)Math.Floor(_innerStream.Position / (float)_segmentContentLength) + 1; + + /// + /// Gets the 1-indexed segment number the encoded data stream is currently positioned in. + /// 1-indexed to match segment labelling as specified by SM spec. + /// + private int CurrentEncodingSegment + { + get + { + // edge case: always on final segment when at end of inner stream + if (_innerStream.Position == _innerStream.Length) + { + return TotalSegments; + } + // when writing footer, inner stream is positioned at next segment, + // but this stream is still writing the previous one + if (_currentRegion == SMRegion.SegmentFooter) + { + return CurrentInnerSegment - 1; + } + return CurrentInnerSegment; + } + } + + /// + /// Segment length including header and footer. + /// + private int SegmentTotalLength => _segmentHeaderLength + _segmentContentLength + _segmentFooterLength; + + private int TotalSegments => GetTotalSegments(_innerStream, _segmentContentLength); + private static int GetTotalSegments(Stream innerStream, long segmentContentLength) + { + return (int)Math.Ceiling(innerStream.Length / (float)segmentContentLength); + } + #endregion + + public override bool CanRead => true; + + public override bool CanWrite => false; + + public override bool CanSeek => _innerStream.CanSeek; + + public override bool CanTimeout => _innerStream.CanTimeout; + + public override int ReadTimeout => _innerStream.ReadTimeout; + + public override int WriteTimeout => _innerStream.WriteTimeout; + + public override long Length => + _streamHeaderLength + _streamFooterLength + + (_segmentHeaderLength + _segmentFooterLength) * TotalSegments + + _innerStream.Length; + + #region Position + private enum SMRegion + { + StreamHeader, + StreamFooter, + SegmentHeader, + SegmentFooter, + SegmentContent, + } + + private SMRegion _currentRegion = SMRegion.StreamHeader; + private int _currentRegionPosition = 0; + + private long _maxSeekPosition = 0; + + public override long Position + { + get + { + return _currentRegion switch + { + SMRegion.StreamHeader => _currentRegionPosition, + SMRegion.StreamFooter => _streamHeaderLength + + TotalSegments * (_segmentHeaderLength + _segmentFooterLength) + + _innerStream.Length + + _currentRegionPosition, + SMRegion.SegmentHeader => _innerStream.Position + + _streamHeaderLength + + (CurrentEncodingSegment - 1) * (_segmentHeaderLength + _segmentFooterLength) + + _currentRegionPosition, + SMRegion.SegmentFooter => _innerStream.Position + + _streamHeaderLength + + // Inner stream has moved to next segment but we're still writing the previous segment footer + CurrentEncodingSegment * (_segmentHeaderLength + _segmentFooterLength) - + _segmentFooterLength + _currentRegionPosition, + SMRegion.SegmentContent => _innerStream.Position + + _streamHeaderLength + + CurrentEncodingSegment * (_segmentHeaderLength + _segmentFooterLength) - + _segmentFooterLength, + _ => throw new InvalidDataException($"{nameof(StructuredMessageEncodingStream)} invalid state."), + }; + } + set + { + Argument.AssertInRange(value, 0, _maxSeekPosition, nameof(value)); + if (value < _streamHeaderLength) + { + _currentRegion = SMRegion.StreamHeader; + _currentRegionPosition = (int)value; + _innerStream.Position = 0; + return; + } + if (value >= Length - _streamFooterLength) + { + _currentRegion = SMRegion.StreamFooter; + _currentRegionPosition = (int)(value - (Length - _streamFooterLength)); + _innerStream.Position = _innerStream.Length; + return; + } + int newSegmentNum = 1 + (int)Math.Floor((value - _streamHeaderLength) / (double)(_segmentHeaderLength + _segmentFooterLength + _segmentContentLength)); + int segmentPosition = (int)(value - _streamHeaderLength - + ((newSegmentNum - 1) * (_segmentHeaderLength + _segmentFooterLength + _segmentContentLength))); + + if (segmentPosition < _segmentHeaderLength) + { + _currentRegion = SMRegion.SegmentHeader; + _currentRegionPosition = (int)((value - _streamHeaderLength) % SegmentTotalLength); + _innerStream.Position = (newSegmentNum - 1) * _segmentContentLength; + return; + } + if (segmentPosition < _segmentHeaderLength + _segmentContentLength) + { + _currentRegion = SMRegion.SegmentContent; + _currentRegionPosition = (int)((value - _streamHeaderLength) % SegmentTotalLength) - + _segmentHeaderLength; + _innerStream.Position = (newSegmentNum - 1) * _segmentContentLength + _currentRegionPosition; + return; + } + + _currentRegion = SMRegion.SegmentFooter; + _currentRegionPosition = (int)((value - _streamHeaderLength) % SegmentTotalLength) - + _segmentHeaderLength - _segmentContentLength; + _innerStream.Position = newSegmentNum * _segmentContentLength; + } + } + #endregion + + public StructuredMessageEncodingStream( + Stream innerStream, + int segmentContentLength, + StructuredMessage.Flags flags) + { + Argument.AssertNotNull(innerStream, nameof(innerStream)); + if (innerStream.GetLengthOrDefault() == default) + { + throw new ArgumentException("Stream must have known length.", nameof(innerStream)); + } + if (innerStream.Position != 0) + { + throw new ArgumentException("Stream must be at starting position.", nameof(innerStream)); + } + // stream logic likely breaks down with segment length of 1; enforce >=2 rather than just positive number + // real world scenarios will probably use a minimum of tens of KB + Argument.AssertInRange(segmentContentLength, 2, int.MaxValue, nameof(segmentContentLength)); + + _flags = flags; + _segmentContentLength = segmentContentLength; + + _streamHeaderLength = StructuredMessage.V1_0.StreamHeaderLength; + _streamFooterLength = UseCrcSegment ? StructuredMessage.Crc64Length : 0; + _segmentHeaderLength = StructuredMessage.V1_0.SegmentHeaderLength; + _segmentFooterLength = UseCrcSegment ? StructuredMessage.Crc64Length : 0; + + if (UseCrcSegment) + { + _totalCrc = StorageCrc64HashAlgorithm.Create(); + _segmentCrc = StorageCrc64HashAlgorithm.Create(); + _segmentCrcs = ArrayPool.Shared.Rent( + GetTotalSegments(innerStream, segmentContentLength) * StructuredMessage.Crc64Length); + innerStream = ChecksumCalculatingStream.GetReadStream(innerStream, span => + { + _totalCrc.Append(span); + _segmentCrc.Append(span); + }); + } + + _innerStream = innerStream; + } + + #region Write + public override void Flush() => throw new NotSupportedException(); + + public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException(); + + public override void SetLength(long value) => throw new NotSupportedException(); + #endregion + + #region Read + public override int Read(byte[] buffer, int offset, int count) + => ReadInternal(buffer, offset, count, async: false, cancellationToken: default).EnsureCompleted(); + + public override async Task ReadAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) + => await ReadInternal(buffer, offset, count, async: true, cancellationToken).ConfigureAwait(false); + + private async ValueTask ReadInternal(byte[] buffer, int offset, int count, bool async, CancellationToken cancellationToken) + { + int totalRead = 0; + bool readInner = false; + while (totalRead < count && Position < Length) + { + int subreadOffset = offset + totalRead; + int subreadCount = count - totalRead; + switch (_currentRegion) + { + case SMRegion.StreamHeader: + totalRead += ReadFromStreamHeader(new Span(buffer, subreadOffset, subreadCount)); + break; + case SMRegion.StreamFooter: + totalRead += ReadFromStreamFooter(new Span(buffer, subreadOffset, subreadCount)); + break; + case SMRegion.SegmentHeader: + totalRead += ReadFromSegmentHeader(new Span(buffer, subreadOffset, subreadCount)); + break; + case SMRegion.SegmentFooter: + totalRead += ReadFromSegmentFooter(new Span(buffer, subreadOffset, subreadCount)); + break; + case SMRegion.SegmentContent: + // don't double read from stream. Allow caller to multi-read when desired. + if (readInner) + { + UpdateLatestPosition(); + return totalRead; + } + totalRead += await ReadFromInnerStreamInternal( + buffer, subreadOffset, subreadCount, async, cancellationToken).ConfigureAwait(false); + readInner = true; + break; + default: + break; + } + } + UpdateLatestPosition(); + return totalRead; + } + +#if NETSTANDARD2_1_OR_GREATER || NETCOREAPP3_0_OR_GREATER + public override int Read(Span buffer) + { + int totalRead = 0; + bool readInner = false; + while (totalRead < buffer.Length && Position < Length) + { + switch (_currentRegion) + { + case SMRegion.StreamHeader: + totalRead += ReadFromStreamHeader(buffer.Slice(totalRead)); + break; + case SMRegion.StreamFooter: + totalRead += ReadFromStreamFooter(buffer.Slice(totalRead)); + break; + case SMRegion.SegmentHeader: + totalRead += ReadFromSegmentHeader(buffer.Slice(totalRead)); + break; + case SMRegion.SegmentFooter: + totalRead += ReadFromSegmentFooter(buffer.Slice(totalRead)); + break; + case SMRegion.SegmentContent: + // don't double read from stream. Allow caller to multi-read when desired. + if (readInner) + { + UpdateLatestPosition(); + return totalRead; + } + totalRead += ReadFromInnerStream(buffer.Slice(totalRead)); + readInner = true; + break; + default: + break; + } + } + UpdateLatestPosition(); + return totalRead; + } + + public override async ValueTask ReadAsync(Memory buffer, CancellationToken cancellationToken = default) + { + int totalRead = 0; + bool readInner = false; + while (totalRead < buffer.Length && Position < Length) + { + switch (_currentRegion) + { + case SMRegion.StreamHeader: + totalRead += ReadFromStreamHeader(buffer.Slice(totalRead).Span); + break; + case SMRegion.StreamFooter: + totalRead += ReadFromStreamFooter(buffer.Slice(totalRead).Span); + break; + case SMRegion.SegmentHeader: + totalRead += ReadFromSegmentHeader(buffer.Slice(totalRead).Span); + break; + case SMRegion.SegmentFooter: + totalRead += ReadFromSegmentFooter(buffer.Slice(totalRead).Span); + break; + case SMRegion.SegmentContent: + // don't double read from stream. Allow caller to multi-read when desired. + if (readInner) + { + UpdateLatestPosition(); + return totalRead; + } + totalRead += await ReadFromInnerStreamAsync(buffer.Slice(totalRead), cancellationToken).ConfigureAwait(false); + readInner = true; + break; + default: + break; + } + } + UpdateLatestPosition(); + return totalRead; + } +#endif + + #region Read Headers/Footers + private int ReadFromStreamHeader(Span buffer) + { + int read = Math.Min(buffer.Length, _streamHeaderLength - _currentRegionPosition); + using IDisposable _ = StructuredMessage.V1_0.GetStreamHeaderBytes( + ArrayPool.Shared, out Memory headerBytes, Length, _flags, TotalSegments); + headerBytes.Slice(_currentRegionPosition, read).Span.CopyTo(buffer); + _currentRegionPosition += read; + + if (_currentRegionPosition == _streamHeaderLength) + { + _currentRegion = SMRegion.SegmentHeader; + _currentRegionPosition = 0; + } + + return read; + } + + private int ReadFromStreamFooter(Span buffer) + { + int read = Math.Min(buffer.Length, _segmentFooterLength - _currentRegionPosition); + if (read <= 0) + { + return 0; + } + + using IDisposable _ = StructuredMessage.V1_0.GetStreamFooterBytes( + ArrayPool.Shared, + out Memory footerBytes, + crc64: UseCrcSegment + ? _totalCrc.GetCurrentHash() // TODO array pooling + : default); + footerBytes.Slice(_currentRegionPosition, read).Span.CopyTo(buffer); + _currentRegionPosition += read; + + return read; + } + + private int ReadFromSegmentHeader(Span buffer) + { + int read = Math.Min(buffer.Length, _segmentHeaderLength - _currentRegionPosition); + using IDisposable _ = StructuredMessage.V1_0.GetSegmentHeaderBytes( + ArrayPool.Shared, + out Memory headerBytes, + CurrentInnerSegment, + Math.Min(_segmentContentLength, _innerStream.Length - _innerStream.Position)); + headerBytes.Slice(_currentRegionPosition, read).Span.CopyTo(buffer); + _currentRegionPosition += read; + + if (_currentRegionPosition == _segmentHeaderLength) + { + _currentRegion = SMRegion.SegmentContent; + _currentRegionPosition = 0; + } + + return read; + } + + private int ReadFromSegmentFooter(Span buffer) + { + int read = Math.Min(buffer.Length, _segmentFooterLength - _currentRegionPosition); + if (read < 0) + { + return 0; + } + + using IDisposable _ = StructuredMessage.V1_0.GetSegmentFooterBytes( + ArrayPool.Shared, + out Memory headerBytes, + crc64: UseCrcSegment + ? new Span( + _segmentCrcs, + (CurrentEncodingSegment-1) * _totalCrc.HashLengthInBytes, + _totalCrc.HashLengthInBytes) + : default); + headerBytes.Slice(_currentRegionPosition, read).Span.CopyTo(buffer); + _currentRegionPosition += read; + + if (_currentRegionPosition == _segmentFooterLength) + { + _currentRegion = _innerStream.Position == _innerStream.Length + ? SMRegion.StreamFooter : SMRegion.SegmentHeader; + _currentRegionPosition = 0; + } + + return read; + } + #endregion + + #region ReadUnderlyingStream + private int MaxInnerStreamRead => _segmentContentLength - _currentRegionPosition; + + private void CleanupContentSegment() + { + if (_currentRegionPosition == _segmentContentLength || _innerStream.Position >= _innerStream.Length) + { + _currentRegion = SMRegion.SegmentFooter; + _currentRegionPosition = 0; + if (UseCrcSegment && CurrentEncodingSegment - 1 == _latestSegmentCrcd) + { + _segmentCrc.GetCurrentHash(new Span( + _segmentCrcs, + _latestSegmentCrcd * _segmentCrc.HashLengthInBytes, + _segmentCrc.HashLengthInBytes)); + _latestSegmentCrcd++; + _segmentCrc = StorageCrc64HashAlgorithm.Create(); + } + } + } + + private async ValueTask ReadFromInnerStreamInternal( + byte[] buffer, int offset, int count, bool async, CancellationToken cancellationToken) + { + int read = async + ? await _innerStream.ReadAsync(buffer, offset, Math.Min(count, MaxInnerStreamRead)).ConfigureAwait(false) + : _innerStream.Read(buffer, offset, Math.Min(count, MaxInnerStreamRead)); + _currentRegionPosition += read; + CleanupContentSegment(); + return read; + } + +#if NETSTANDARD2_1_OR_GREATER || NETCOREAPP3_0_OR_GREATER + private int ReadFromInnerStream(Span buffer) + { + if (MaxInnerStreamRead < buffer.Length) + { + buffer = buffer.Slice(0, MaxInnerStreamRead); + } + int read = _innerStream.Read(buffer); + _currentRegionPosition += read; + CleanupContentSegment(); + return read; + } + + private async ValueTask ReadFromInnerStreamAsync(Memory buffer, CancellationToken cancellationToken) + { + if (MaxInnerStreamRead < buffer.Length) + { + buffer = buffer.Slice(0, MaxInnerStreamRead); + } + int read = await _innerStream.ReadAsync(buffer, cancellationToken).ConfigureAwait(false); + _currentRegionPosition += read; + CleanupContentSegment(); + return read; + } +#endif + #endregion + + // don't allow stream to seek too far forward. track how far the stream has been naturally read. + private void UpdateLatestPosition() + { + if (_maxSeekPosition < Position) + { + _maxSeekPosition = Position; + } + } + #endregion + + public override long Seek(long offset, SeekOrigin origin) + { + switch (origin) + { + case SeekOrigin.Begin: + Position = offset; + break; + case SeekOrigin.Current: + Position += offset; + break; + case SeekOrigin.End: + Position = Length + offset; + break; + } + return Position; + } + + protected override void Dispose(bool disposing) + { + base.Dispose(disposing); + + if (_disposed) + { + return; + } + + if (disposing) + { + _innerStream.Dispose(); + _disposed = true; + } + } +} diff --git a/sdk/storage/Azure.Storage.Common/tests/StructuredMessageDecodingStreamTests.cs b/sdk/storage/Azure.Storage.Common/tests/StructuredMessageDecodingStreamTests.cs index 5525b52619b9c..04a69d68169e3 100644 --- a/sdk/storage/Azure.Storage.Common/tests/StructuredMessageDecodingStreamTests.cs +++ b/sdk/storage/Azure.Storage.Common/tests/StructuredMessageDecodingStreamTests.cs @@ -85,17 +85,17 @@ public async Task DecodesData( [Values(true, false)] bool useCrc) { int segmentContentLength = seglen ?? int.MaxValue; - Flags flags = useCrc ? Flags.CrcSegment : Flags.None; + Flags flags = useCrc ? Flags.StorageCrc64 : Flags.None; byte[] originalData = new byte[dataLength]; new Random().NextBytes(originalData); byte[] encodedData = StructuredMessageHelper.MakeEncodedData(originalData, segmentContentLength, flags); - Stream encodingStream = new StructuredMessageDecodingStream(new MemoryStream(encodedData)); + Stream decodingStream = new StructuredMessageDecodingStream(new MemoryStream(encodedData)); byte[] decodedData; using (MemoryStream dest = new()) { - await CopyStream(encodingStream, dest, readLen); + await CopyStream(decodingStream, dest, readLen); decodedData = dest.ToArray(); } diff --git a/sdk/storage/Azure.Storage.Common/tests/StructuredMessageEncodingStreamTests.cs b/sdk/storage/Azure.Storage.Common/tests/StructuredMessageEncodingStreamTests.cs new file mode 100644 index 0000000000000..d4c667b937bf5 --- /dev/null +++ b/sdk/storage/Azure.Storage.Common/tests/StructuredMessageEncodingStreamTests.cs @@ -0,0 +1,246 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +using System; +using System.Buffers.Binary; +using System.IO; +using System.Linq; +using System.Threading.Tasks; +using Azure.Storage.Blobs.Tests; +using Azure.Storage.Shared; +using NUnit.Framework; +using static Azure.Storage.Shared.StructuredMessage; + +namespace Azure.Storage.Tests +{ + [TestFixture(ReadMethod.SyncArray)] + [TestFixture(ReadMethod.AsyncArray)] +#if NETSTANDARD2_1_OR_GREATER || NETCOREAPP3_0_OR_GREATER + [TestFixture(ReadMethod.SyncSpan)] + [TestFixture(ReadMethod.AsyncMemory)] +#endif + public class StructuredMessageEncodingStreamTests + { + // Cannot just implement as passthru in the stream + // Must test each one + public enum ReadMethod + { + SyncArray, + AsyncArray, +#if NETSTANDARD2_1_OR_GREATER || NETCOREAPP3_0_OR_GREATER + SyncSpan, + AsyncMemory +#endif + } + + public ReadMethod Method { get; } + + public StructuredMessageEncodingStreamTests(ReadMethod method) + { + Method = method; + } + + private async ValueTask CopyStream(Stream source, Stream destination, int bufferSize = 81920) // number default for CopyTo impl + { + byte[] buf = new byte[bufferSize]; + int read; + switch (Method) + { + case ReadMethod.SyncArray: + while ((read = source.Read(buf, 0, bufferSize)) > 0) + { + destination.Write(buf, 0, read); + } + break; + case ReadMethod.AsyncArray: + while ((read = await source.ReadAsync(buf, 0, bufferSize)) > 0) + { + await destination.WriteAsync(buf, 0, read); + } + break; +#if NETSTANDARD2_1_OR_GREATER || NETCOREAPP3_0_OR_GREATER + case ReadMethod.SyncSpan: + while ((read = source.Read(new Span(buf))) > 0) + { + destination.Write(new Span(buf, 0, read)); + } + break; + case ReadMethod.AsyncMemory: + while ((read = await source.ReadAsync(new Memory(buf))) > 0) + { + await destination.WriteAsync(new Memory(buf, 0, read)); + } + break; +#endif + } + destination.Flush(); + } + + [Test] + [Pairwise] + public async Task EncodesData( + [Values(2048, 2005)] int dataLength, + [Values(default, 512)] int? seglen, + [Values(8 * Constants.KB, 512, 530, 3)] int readLen, + [Values(true, false)] bool useCrc) + { + int segmentContentLength = seglen ?? int.MaxValue; + Flags flags = useCrc ? Flags.StorageCrc64 : Flags.None; + + byte[] originalData = new byte[dataLength]; + new Random().NextBytes(originalData); + byte[] expectedEncodedData = StructuredMessageHelper.MakeEncodedData(originalData, segmentContentLength, flags); + + Stream encodingStream = new StructuredMessageEncodingStream(new MemoryStream(originalData), segmentContentLength, flags); + byte[] encodedData; + using (MemoryStream dest = new()) + { + await CopyStream(encodingStream, dest, readLen); + encodedData = dest.ToArray(); + } + + Assert.That(new Span(encodedData).SequenceEqual(expectedEncodedData)); + } + + [TestCase(0, 0)] // start + [TestCase(5, 0)] // partway through stream header + [TestCase(V1_0.StreamHeaderLength, 0)] // start of segment + [TestCase(V1_0.StreamHeaderLength + 3, 0)] // partway through segment header + [TestCase(V1_0.StreamHeaderLength + V1_0.SegmentHeaderLength, 0)] // start of segment content + [TestCase(V1_0.StreamHeaderLength + V1_0.SegmentHeaderLength + 123, 123)] // partway through segment content + [TestCase(V1_0.StreamHeaderLength + V1_0.SegmentHeaderLength + 512, 512)] // start of segment footer + [TestCase(V1_0.StreamHeaderLength + V1_0.SegmentHeaderLength + 515, 512)] // partway through segment footer + [TestCase(V1_0.StreamHeaderLength + 3*V1_0.SegmentHeaderLength + 2*Crc64Length + 1500, 1500)] // partway through not first segment content + public async Task Seek(int targetRewindOffset, int expectedInnerStreamPosition) + { + const int segmentLength = 512; + const int dataLength = 2055; + byte[] data = new byte[dataLength]; + new Random().NextBytes(data); + + MemoryStream dataStream = new(data); + StructuredMessageEncodingStream encodingStream = new(dataStream, segmentLength, Flags.StorageCrc64); + + // no support for seeking past existing read, need to consume whole stream before seeking + await CopyStream(encodingStream, Stream.Null); + + encodingStream.Position = targetRewindOffset; + Assert.That(encodingStream.Position, Is.EqualTo(targetRewindOffset)); + Assert.That(dataStream.Position, Is.EqualTo(expectedInnerStreamPosition)); + } + + [TestCase(0)] // start + [TestCase(5)] // partway through stream header + [TestCase(V1_0.StreamHeaderLength)] // start of segment + [TestCase(V1_0.StreamHeaderLength + 3)] // partway through segment header + [TestCase(V1_0.StreamHeaderLength + V1_0.SegmentHeaderLength)] // start of segment content + [TestCase(V1_0.StreamHeaderLength + V1_0.SegmentHeaderLength + 123)] // partway through segment content + [TestCase(V1_0.StreamHeaderLength + V1_0.SegmentHeaderLength + 512)] // start of segment footer + [TestCase(V1_0.StreamHeaderLength + V1_0.SegmentHeaderLength + 515)] // partway through segment footer + [TestCase(V1_0.StreamHeaderLength + 2 * V1_0.SegmentHeaderLength + Crc64Length + 1500)] // partway through not first segment content + public async Task SupportsRewind(int targetRewindOffset) + { + const int segmentLength = 512; + const int dataLength = 2055; + byte[] data = new byte[dataLength]; + new Random().NextBytes(data); + + Stream encodingStream = new StructuredMessageEncodingStream(new MemoryStream(data), segmentLength, Flags.StorageCrc64); + byte[] encodedData1; + using (MemoryStream dest = new()) + { + await CopyStream(encodingStream, dest); + encodedData1 = dest.ToArray(); + } + encodingStream.Position = targetRewindOffset; + byte[] encodedData2; + using (MemoryStream dest = new()) + { + await CopyStream(encodingStream, dest); + encodedData2 = dest.ToArray(); + } + + Assert.That(new Span(encodedData1).Slice(targetRewindOffset).SequenceEqual(encodedData2)); + } + + [Test] + public async Task SupportsFastForward() + { + const int segmentLength = 512; + const int dataLength = 2055; + byte[] data = new byte[dataLength]; + new Random().NextBytes(data); + + // must have read stream to fastforward. so read whole stream upfront & save result to check later + Stream encodingStream = new StructuredMessageEncodingStream(new MemoryStream(data), segmentLength, Flags.StorageCrc64); + byte[] encodedData; + using (MemoryStream dest = new()) + { + await CopyStream(encodingStream, dest); + encodedData = dest.ToArray(); + } + + encodingStream.Position = 0; + + bool skip = false; + const int increment = 499; + while (encodingStream.Position < encodingStream.Length) + { + if (skip) + { + encodingStream.Position = Math.Min(dataLength, encodingStream.Position + increment); + skip = !skip; + continue; + } + ReadOnlyMemory expected = new(encodedData, (int)encodingStream.Position, + (int)Math.Min(increment, encodedData.Length - encodingStream.Position)); + ReadOnlyMemory actual; + using (MemoryStream dest = new(increment)) + { + await CopyStream(WindowStream.GetWindow(encodingStream, increment), dest); + actual = dest.ToArray(); + } + Assert.That(expected.Span.SequenceEqual(actual.Span)); + skip = !skip; + } + } + + [Test] + public void NotSupportsFastForwardBeyondLatestRead() + { + const int segmentLength = 512; + const int dataLength = 2055; + byte[] data = new byte[dataLength]; + new Random().NextBytes(data); + + Stream encodingStream = new StructuredMessageEncodingStream(new MemoryStream(data), segmentLength, Flags.StorageCrc64); + + Assert.That(() => encodingStream.Position = 123, Throws.TypeOf()); + } + + private static void AssertExpectedStreamHeader(ReadOnlySpan actual, int originalDataLength, Flags flags, int expectedSegments) + { + int expectedFooterLen = flags.HasFlag(Flags.StorageCrc64) ? Crc64Length : 0; + + Assert.That(actual.Length, Is.EqualTo(V1_0.StreamHeaderLength)); + Assert.That(actual[0], Is.EqualTo(1)); + Assert.That(BinaryPrimitives.ReadInt64LittleEndian(actual.Slice(1, 8)), + Is.EqualTo(V1_0.StreamHeaderLength + expectedSegments * (V1_0.SegmentHeaderLength + expectedFooterLen) + originalDataLength)); + Assert.That(BinaryPrimitives.ReadInt16LittleEndian(actual.Slice(9, 2)), Is.EqualTo((short)flags)); + Assert.That(BinaryPrimitives.ReadInt16LittleEndian(actual.Slice(11, 2)), Is.EqualTo((short)expectedSegments)); + } + + private static void AssertExpectedSegmentHeader(ReadOnlySpan actual, int segmentNum, long contentLength) + { + Assert.That(BinaryPrimitives.ReadInt16LittleEndian(actual.Slice(0, 2)), Is.EqualTo((short) segmentNum)); + Assert.That(BinaryPrimitives.ReadInt64LittleEndian(actual.Slice(2, 8)), Is.EqualTo(contentLength)); + } + + private static byte[] CrcInline(ReadOnlySpan data) + { + var crc = StorageCrc64HashAlgorithm.Create(); + crc.Append(data); + return crc.GetCurrentHash(); + } + } +} diff --git a/sdk/storage/Azure.Storage.Common/tests/StructuredMessageHelper.cs b/sdk/storage/Azure.Storage.Common/tests/StructuredMessageHelper.cs index ffc8c9e935a22..59e80320d96a0 100644 --- a/sdk/storage/Azure.Storage.Common/tests/StructuredMessageHelper.cs +++ b/sdk/storage/Azure.Storage.Common/tests/StructuredMessageHelper.cs @@ -13,12 +13,17 @@ namespace Azure.Storage.Blobs.Tests { internal class StructuredMessageHelper { - public static byte[] MakeEncodedData(byte[] data, long segmentContentLength, Flags flags) + public static byte[] MakeEncodedData(ReadOnlySpan data, long segmentContentLength, Flags flags) { int segmentCount = (int) Math.Ceiling(data.Length / (double)segmentContentLength); - int segmentFooterLen = flags.HasFlag(Flags.CrcSegment) ? 8 : 0; + int segmentFooterLen = flags.HasFlag(Flags.StorageCrc64) ? 8 : 0; + int streamFooterLen = flags.HasFlag(Flags.StorageCrc64) ? 8 : 0; - byte[] encodedData = new byte[V1_0.StreamHeaderLength + segmentCount*(V1_0.SegmentHeaderLength + segmentFooterLen) + data.Length]; + byte[] encodedData = new byte[ + V1_0.StreamHeaderLength + + segmentCount*(V1_0.SegmentHeaderLength + segmentFooterLen) + + streamFooterLen + + data.Length]; V1_0.WriteStreamHeader( new Span(encodedData, 0, V1_0.StreamHeaderLength), encodedData.Length, @@ -33,23 +38,30 @@ public static byte[] MakeEncodedData(byte[] data, long segmentContentLength, Fla V1_0.WriteSegmentHeader( new Span(encodedData, i, V1_0.SegmentHeaderLength), seg, - segContentLen); + segContentLen); i += V1_0.SegmentHeaderLength; - new Span(data, j, segContentLen) + data.Slice(j, segContentLen) .CopyTo(new Span(encodedData).Slice(i)); i += segContentLen; - if (flags.HasFlag(Flags.CrcSegment)) + if (flags.HasFlag(Flags.StorageCrc64)) { var crc = StorageCrc64HashAlgorithm.Create(); - crc.Append(new Span(data).Slice(j, segContentLen)); + crc.Append(data.Slice(j, segContentLen)); crc.GetCurrentHash(new Span(encodedData, i, Crc64Length)); i += Crc64Length; } j += segContentLen; } + if (flags.HasFlag(Flags.StorageCrc64)) + { + var crc = StorageCrc64HashAlgorithm.Create(); + crc.Append(data); + crc.GetCurrentHash(new Span(encodedData, i, Crc64Length)); + } + return encodedData; } } diff --git a/sdk/storage/Azure.Storage.Common/tests/StructuredMessageTests.cs b/sdk/storage/Azure.Storage.Common/tests/StructuredMessageTests.cs new file mode 100644 index 0000000000000..b4f1dfe178246 --- /dev/null +++ b/sdk/storage/Azure.Storage.Common/tests/StructuredMessageTests.cs @@ -0,0 +1,114 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +using System; +using System.Buffers.Binary; +using System.Collections.Generic; +using NUnit.Framework; +using static Azure.Storage.Shared.StructuredMessage; + +namespace Azure.Storage.Tests +{ + public class StructuredMessageTests + { + [TestCase(1024, Flags.None, 2)] + [TestCase(2000, Flags.StorageCrc64, 4)] + public void EncodeStreamHeader(int messageLength, int flags, int numSegments) + { + Span encoding = new(new byte[V1_0.StreamHeaderLength]); + V1_0.WriteStreamHeader(encoding, messageLength, (Flags)flags, numSegments); + + Assert.That(encoding[0], Is.EqualTo((byte)1)); + Assert.That(BinaryPrimitives.ReadUInt64LittleEndian(encoding.Slice(1, 8)), Is.EqualTo(messageLength)); + Assert.That(BinaryPrimitives.ReadUInt16LittleEndian(encoding.Slice(9, 2)), Is.EqualTo(flags)); + Assert.That(BinaryPrimitives.ReadUInt16LittleEndian(encoding.Slice(11, 2)), Is.EqualTo(numSegments)); + } + + [TestCase(V1_0.StreamHeaderLength)] + [TestCase(V1_0.StreamHeaderLength + 1)] + [TestCase(V1_0.StreamHeaderLength - 1)] + public void EncodeStreamHeaderRejectBadBufferSize(int bufferSize) + { + Random r = new(); + byte[] encoding = new byte[bufferSize]; + + void Action() => V1_0.WriteStreamHeader(encoding, r.Next(2, int.MaxValue), Flags.StorageCrc64, r.Next(2, int.MaxValue)); + if (bufferSize < V1_0.StreamHeaderLength) + { + Assert.That(Action, Throws.ArgumentException); + } + else + { + Assert.That(Action, Throws.Nothing); + } + } + + [TestCase(1, 1024)] + [TestCase(5, 39578)] + public void EncodeSegmentHeader(int segmentNum, int contentLength) + { + Span encoding = new(new byte[V1_0.SegmentHeaderLength]); + V1_0.WriteSegmentHeader(encoding, segmentNum, contentLength); + + Assert.That(BinaryPrimitives.ReadUInt16LittleEndian(encoding.Slice(0, 2)), Is.EqualTo(segmentNum)); + Assert.That(BinaryPrimitives.ReadUInt64LittleEndian(encoding.Slice(2, 8)), Is.EqualTo(contentLength)); + } + + [TestCase(V1_0.SegmentHeaderLength)] + [TestCase(V1_0.SegmentHeaderLength + 1)] + [TestCase(V1_0.SegmentHeaderLength - 1)] + public void EncodeSegmentHeaderRejectBadBufferSize(int bufferSize) + { + Random r = new(); + byte[] encoding = new byte[bufferSize]; + + void Action() => V1_0.WriteSegmentHeader(encoding, r.Next(1, int.MaxValue), r.Next(2, int.MaxValue)); + if (bufferSize < V1_0.SegmentHeaderLength) + { + Assert.That(Action, Throws.ArgumentException); + } + else + { + Assert.That(Action, Throws.Nothing); + } + } + + [TestCase(true)] + [TestCase(false)] + public void EncodeSegmentFooter(bool useCrc) + { + Span encoding = new(new byte[Crc64Length]); + Span crc = useCrc ? new Random().NextBytesInline(Crc64Length) : default; + V1_0.WriteSegmentFooter(encoding, crc); + + if (useCrc) + { + Assert.That(encoding.SequenceEqual(crc), Is.True); + } + else + { + Assert.That(encoding.SequenceEqual(new Span(new byte[Crc64Length])), Is.True); + } + } + + [TestCase(Crc64Length)] + [TestCase(Crc64Length + 1)] + [TestCase(Crc64Length - 1)] + public void EncodeSegmentFooterRejectBadBufferSize(int bufferSize) + { + byte[] encoding = new byte[bufferSize]; + byte[] crc = new byte[Crc64Length]; + new Random().NextBytes(crc); + + void Action() => V1_0.WriteSegmentFooter(encoding, crc); + if (bufferSize < Crc64Length) + { + Assert.That(Action, Throws.ArgumentException); + } + else + { + Assert.That(Action, Throws.Nothing); + } + } + } +} From 34df306916889f812d9bbb46ffc7366b5fcbf565 Mon Sep 17 00:00:00 2001 From: Jocelyn <41338290+jaschrep-msft@users.noreply.github.com> Date: Thu, 29 Feb 2024 16:04:33 -0500 Subject: [PATCH 03/22] decode tests & bugfixes (#42256) * decode tests & bugfixes * roundtrip tests * more tests * better errors | remove duplicate test --- .../Azure.Storage.Common/src/Shared/Errors.cs | 3 + .../src/Shared/StructuredMessage.cs | 15 +- .../Shared/StructuredMessageDecodingStream.cs | 76 +++++-- .../StructuredMessageDecodingStreamTests.cs | 193 +++++++++++++++--- .../StructuredMessageStreamRoundtripTests.cs | 127 ++++++++++++ 5 files changed, 367 insertions(+), 47 deletions(-) create mode 100644 sdk/storage/Azure.Storage.Common/tests/StructuredMessageStreamRoundtripTests.cs diff --git a/sdk/storage/Azure.Storage.Common/src/Shared/Errors.cs b/sdk/storage/Azure.Storage.Common/src/Shared/Errors.cs index 6e50037782e0d..e3372665928c1 100644 --- a/sdk/storage/Azure.Storage.Common/src/Shared/Errors.cs +++ b/sdk/storage/Azure.Storage.Common/src/Shared/Errors.cs @@ -72,6 +72,9 @@ public static ArgumentException CannotDeferTransactionalHashVerification() public static ArgumentException CannotInitializeWriteStreamWithData() => new ArgumentException("Initialized buffer for StorageWriteStream must be empty."); + public static InvalidDataException InvalidStructuredMessage(string optionalMessage = default) + => new InvalidDataException(("Invalid structured message data. " + optionalMessage ?? "").Trim()); + internal static void VerifyStreamPosition(Stream stream, string streamName) { if (stream != null && stream.CanSeek && stream.Length > 0 && stream.Position >= stream.Length) diff --git a/sdk/storage/Azure.Storage.Common/src/Shared/StructuredMessage.cs b/sdk/storage/Azure.Storage.Common/src/Shared/StructuredMessage.cs index d963224536fb3..5e31dd4ac0ed8 100644 --- a/sdk/storage/Azure.Storage.Common/src/Shared/StructuredMessage.cs +++ b/sdk/storage/Azure.Storage.Common/src/Shared/StructuredMessage.cs @@ -25,7 +25,14 @@ public static class V1_0 public const byte MessageVersionByte = 1; public const int StreamHeaderLength = 13; + public const int StreamHeaderVersionOffset = 0; + public const int StreamHeaderMessageLengthOffset = 1; + public const int StreamHeaderFlagsOffset = 9; + public const int StreamHeaderSegmentCountOffset = 11; + public const int SegmentHeaderLength = 10; + public const int SegmentHeaderNumOffset = 0; + public const int SegmentHeaderContentLengthOffset = 2; #region Stream Header public static void ReadStreamHeader( @@ -35,13 +42,13 @@ public static void ReadStreamHeader( out int totalSegments) { Errors.AssertBufferExactSize(buffer, 13, nameof(buffer)); - if (buffer[0] != 1) + if (buffer[StreamHeaderVersionOffset] != 1) { throw new InvalidDataException("Unrecognized version of structured message."); } - messageLength = (long)BinaryPrimitives.ReadUInt64LittleEndian(buffer.Slice(1, 8)); - flags = (Flags)BinaryPrimitives.ReadUInt16LittleEndian(buffer.Slice(9, 2)); - totalSegments = BinaryPrimitives.ReadUInt16LittleEndian(buffer.Slice(11, 2)); + messageLength = (long)BinaryPrimitives.ReadUInt64LittleEndian(buffer.Slice(StreamHeaderMessageLengthOffset, 8)); + flags = (Flags)BinaryPrimitives.ReadUInt16LittleEndian(buffer.Slice(StreamHeaderFlagsOffset, 2)); + totalSegments = BinaryPrimitives.ReadUInt16LittleEndian(buffer.Slice(StreamHeaderSegmentCountOffset, 2)); } public static int WriteStreamHeader( diff --git a/sdk/storage/Azure.Storage.Common/src/Shared/StructuredMessageDecodingStream.cs b/sdk/storage/Azure.Storage.Common/src/Shared/StructuredMessageDecodingStream.cs index ef6e1af60604d..806a968cf4ec7 100644 --- a/sdk/storage/Azure.Storage.Common/src/Shared/StructuredMessageDecodingStream.cs +++ b/sdk/storage/Azure.Storage.Common/src/Shared/StructuredMessageDecodingStream.cs @@ -38,6 +38,15 @@ namespace Azure.Storage.Shared; /// internal class StructuredMessageDecodingStream : Stream { + private enum SMRegion + { + StreamHeader, + StreamFooter, + SegmentHeader, + SegmentFooter, + SegmentContent, + } + private readonly Stream _innerBufferedStream; private byte[] _metadataBuffer = ArrayPool.Shared.Rent(Constants.KB); @@ -48,9 +57,12 @@ internal class StructuredMessageDecodingStream : Stream private int _streamFooterLength; private int _segmentHeaderLength; private int _segmentFooterLength; + private int _totalSegments; + private long _innerStreamLength; private StructuredMessage.Flags _flags; + private bool _processedFooter = false; private bool _disposed; private StorageCrc64HashAlgorithm _totalContentCrc; @@ -76,19 +88,6 @@ public override long Position set => throw new NotSupportedException(); } - #region Position - private long _innerStreamLength; - - private enum SMRegion - { - StreamHeader, - StreamFooter, - SegmentHeader, - SegmentFooter, - SegmentContent, - } - #endregion - public StructuredMessageDecodingStream( Stream innerStream) { @@ -116,9 +115,15 @@ public override int Read(byte[] buf, int offset, int count) do { read = _innerBufferedStream.Read(buf, offset, count); + _innerStreamConsumed += read; decodedRead = Decode(new Span(buf, offset, read)); } while (decodedRead <= 0 && read > 0); + if (read <= 0) + { + AssertDecodeFinished(); + } + return decodedRead; } @@ -129,9 +134,15 @@ public override async Task ReadAsync(byte[] buf, int offset, int count, Can do { read = await _innerBufferedStream.ReadAsync(buf, offset, count, cancellationToken).ConfigureAwait(false); + _innerStreamConsumed += read; decodedRead = Decode(new Span(buf, offset, read)); } while (decodedRead <= 0 && read > 0); + if (read <= 0) + { + AssertDecodeFinished(); + } + return decodedRead; } @@ -143,9 +154,15 @@ public override int Read(Span buf) do { read = _innerBufferedStream.Read(buf); + _innerStreamConsumed += read; decodedRead = Decode(buf.Slice(0, read)); } while (decodedRead <= 0 && read > 0); + if (read <= 0) + { + AssertDecodeFinished(); + } + return decodedRead; } @@ -156,15 +173,31 @@ public override async ValueTask ReadAsync(Memory buf, CancellationTok do { read = await _innerBufferedStream.ReadAsync(buf).ConfigureAwait(false); + _innerStreamConsumed += read; decodedRead = Decode(buf.Slice(0, read).Span); } while (decodedRead <= 0 && read > 0); + if (read <= 0) + { + AssertDecodeFinished(); + } + return decodedRead; } #endif - private SMRegion _currentRegion; - private int _currentSegmentNum; + private void AssertDecodeFinished() + { + if (_streamFooterLength > 0 && !_processedFooter) + { + throw Errors.InvalidStructuredMessage("Missing or incomplete trailer."); + } + _processedFooter = true; + } + + private long _innerStreamConsumed = 0; + private SMRegion _currentRegion = SMRegion.StreamHeader; + private int _currentSegmentNum = 0; private long _currentSegmentContentLength; private long _currentSegmentContentRemaining; private long CurrentRegionLength => _currentRegion switch @@ -363,6 +396,17 @@ private int ProcessStreamFooter(ReadOnlySpan span) } } } + + if (_innerStreamConsumed != _innerStreamLength) + { + throw Errors.InvalidStructuredMessage("Unexpected message size."); + } + if (_currentSegmentNum != _totalSegments) + { + throw Errors.InvalidStructuredMessage("Missing expected message segments."); + } + + _processedFooter = true; return totalProcessed; } @@ -375,7 +419,7 @@ private int ProcessSegmentHeader(ReadOnlySpan span) _currentSegmentContentRemaining = _currentSegmentContentLength; if (newSegNum != _currentSegmentNum + 1) { - throw new InvalidDataException("Unexpected segment number in structured message."); + throw Errors.InvalidStructuredMessage("Unexpected segment number in structured message."); } _currentSegmentNum = newSegNum; _currentRegion = SMRegion.SegmentContent; diff --git a/sdk/storage/Azure.Storage.Common/tests/StructuredMessageDecodingStreamTests.cs b/sdk/storage/Azure.Storage.Common/tests/StructuredMessageDecodingStreamTests.cs index 04a69d68169e3..15c36f19d8763 100644 --- a/sdk/storage/Azure.Storage.Common/tests/StructuredMessageDecodingStreamTests.cs +++ b/sdk/storage/Azure.Storage.Common/tests/StructuredMessageDecodingStreamTests.cs @@ -2,6 +2,8 @@ // Licensed under the MIT License. using System; +using System.Buffers.Binary; +using System.Dynamic; using System.IO; using System.Linq; using System.Threading; @@ -40,40 +42,63 @@ public StructuredMessageDecodingStreamTests(ReadMethod method) Method = method; } - private async ValueTask CopyStream(Stream source, Stream destination, int bufferSize = 81920) // number default for CopyTo impl + private class CopyStreamException : Exception + { + public long TotalCopied { get; } + + public CopyStreamException(Exception inner, long totalCopied) + : base($"Failed read after {totalCopied}-many bytes.", inner) + { + TotalCopied = totalCopied; + } + } + private async ValueTask CopyStream(Stream source, Stream destination, int bufferSize = 81920) // number default for CopyTo impl { byte[] buf = new byte[bufferSize]; int read; - switch (Method) + long totalRead = 0; + try { - case ReadMethod.SyncArray: - while ((read = source.Read(buf, 0, bufferSize)) > 0) - { - destination.Write(buf, 0, read); - } - break; - case ReadMethod.AsyncArray: - while ((read = await source.ReadAsync(buf, 0, bufferSize)) > 0) - { - await destination.WriteAsync(buf, 0, read); - } - break; + switch (Method) + { + case ReadMethod.SyncArray: + while ((read = source.Read(buf, 0, bufferSize)) > 0) + { + totalRead += read; + destination.Write(buf, 0, read); + } + break; + case ReadMethod.AsyncArray: + while ((read = await source.ReadAsync(buf, 0, bufferSize)) > 0) + { + totalRead += read; + await destination.WriteAsync(buf, 0, read); + } + break; #if NETSTANDARD2_1_OR_GREATER || NETCOREAPP3_0_OR_GREATER - case ReadMethod.SyncSpan: - while ((read = source.Read(new Span(buf))) > 0) - { - destination.Write(new Span(buf, 0, read)); - } - break; - case ReadMethod.AsyncMemory: - while ((read = await source.ReadAsync(new Memory(buf))) > 0) - { - await destination.WriteAsync(new Memory(buf, 0, read)); - } - break; + case ReadMethod.SyncSpan: + while ((read = source.Read(new Span(buf))) > 0) + { + totalRead += read; + destination.Write(new Span(buf, 0, read)); + } + break; + case ReadMethod.AsyncMemory: + while ((read = await source.ReadAsync(new Memory(buf))) > 0) + { + totalRead += read; + await destination.WriteAsync(new Memory(buf, 0, read)); + } + break; #endif + } + destination.Flush(); } - destination.Flush(); + catch (Exception ex) + { + throw new CopyStreamException(ex, totalRead); + } + return totalRead; } [Test] @@ -102,6 +127,120 @@ public async Task DecodesData( Assert.That(new Span(decodedData).SequenceEqual(originalData)); } + [Test] + public void BadStreamBadVersion() + { + byte[] originalData = new byte[1024]; + new Random().NextBytes(originalData); + byte[] encodedData = StructuredMessageHelper.MakeEncodedData(originalData, 256, Flags.StorageCrc64); + + encodedData[0] = byte.MaxValue; + + Stream decodingStream = new StructuredMessageDecodingStream(new MemoryStream(encodedData)); + Assert.That(async () => await CopyStream(decodingStream, Stream.Null), Throws.InnerException.TypeOf()); + } + + [Test] + public async Task BadSegmentCrcThrows() + { + const int segmentLength = 256; + Random r = new(); + + byte[] originalData = new byte[2048]; + r.NextBytes(originalData); + byte[] encodedData = StructuredMessageHelper.MakeEncodedData(originalData, segmentLength, Flags.StorageCrc64); + + const int badBytePos = 1024; + encodedData[badBytePos] = (byte)~encodedData[badBytePos]; + + MemoryStream encodedDataStream = new(encodedData); + Stream decodingStream = new StructuredMessageDecodingStream(encodedDataStream); + + // manual try/catch to validate the proccess failed mid-stream rather than the end + const int copyBufferSize = 4; + bool caught = false; + try + { + await CopyStream(decodingStream, Stream.Null, copyBufferSize); + } + catch (CopyStreamException ex) + { + caught = true; + Assert.That(ex.TotalCopied, Is.LessThanOrEqualTo(badBytePos)); + } + Assert.That(caught); + } + + [Test] + public void BadStreamCrcThrows() + { + const int segmentLength = 256; + Random r = new(); + + byte[] originalData = new byte[2048]; + r.NextBytes(originalData); + byte[] encodedData = StructuredMessageHelper.MakeEncodedData(originalData, segmentLength, Flags.StorageCrc64); + + encodedData[originalData.Length - 1] = (byte)~encodedData[originalData.Length - 1]; + + Stream decodingStream = new StructuredMessageDecodingStream(new MemoryStream(encodedData)); + Assert.That(async () => await CopyStream(decodingStream, Stream.Null), Throws.InnerException.TypeOf()); + } + + [Test] + public void BadStreamWrongContentLength() + { + byte[] originalData = new byte[1024]; + new Random().NextBytes(originalData); + byte[] encodedData = StructuredMessageHelper.MakeEncodedData(originalData, 256, Flags.StorageCrc64); + + BinaryPrimitives.WriteInt64LittleEndian(new Span(encodedData, V1_0.StreamHeaderMessageLengthOffset, 8), 123456789L); + + Stream decodingStream = new StructuredMessageDecodingStream(new MemoryStream(encodedData)); + Assert.That(async () => await CopyStream(decodingStream, Stream.Null), Throws.InnerException.TypeOf()); + } + + [Test] + public void BadStreamWrongSegmentCount() + { + byte[] originalData = new byte[1024]; + new Random().NextBytes(originalData); + byte[] encodedData = StructuredMessageHelper.MakeEncodedData(originalData, 256, Flags.StorageCrc64); + + BinaryPrimitives.WriteInt16LittleEndian(new Span(encodedData, V1_0.StreamHeaderSegmentCountOffset, 2), 123); + + Stream decodingStream = new StructuredMessageDecodingStream(new MemoryStream(encodedData)); + Assert.That(async () => await CopyStream(decodingStream, Stream.Null), Throws.InnerException.TypeOf()); + } + + [Test] + public void BadStreamWrongSegmentNum() + { + byte[] originalData = new byte[1024]; + new Random().NextBytes(originalData); + byte[] encodedData = StructuredMessageHelper.MakeEncodedData(originalData, 256, Flags.StorageCrc64); + + BinaryPrimitives.WriteInt16LittleEndian( + new Span(encodedData, V1_0.StreamHeaderLength + V1_0.SegmentHeaderNumOffset, 2), 123); + + Stream decodingStream = new StructuredMessageDecodingStream(new MemoryStream(encodedData)); + Assert.That(async () => await CopyStream(decodingStream, Stream.Null), Throws.InnerException.TypeOf()); + } + + [Test] + public void BadStreamMissingExpectedStreamFooter() + { + byte[] originalData = new byte[1024]; + new Random().NextBytes(originalData); + byte[] encodedData = StructuredMessageHelper.MakeEncodedData(originalData, 256, Flags.StorageCrc64); + + byte[] brokenData = new byte[encodedData.Length - Crc64Length]; + new Span(encodedData, 0, encodedData.Length - Crc64Length).CopyTo(brokenData); + + Stream decodingStream = new StructuredMessageDecodingStream(new MemoryStream(brokenData)); + Assert.That(async () => await CopyStream(decodingStream, Stream.Null), Throws.InnerException.TypeOf()); + } + [Test] public void NoSeek() { diff --git a/sdk/storage/Azure.Storage.Common/tests/StructuredMessageStreamRoundtripTests.cs b/sdk/storage/Azure.Storage.Common/tests/StructuredMessageStreamRoundtripTests.cs new file mode 100644 index 0000000000000..633233db2e73c --- /dev/null +++ b/sdk/storage/Azure.Storage.Common/tests/StructuredMessageStreamRoundtripTests.cs @@ -0,0 +1,127 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +using System; +using System.IO; +using System.Linq; +using System.Threading.Tasks; +using Azure.Storage.Shared; +using NUnit.Framework; +using static Azure.Storage.Shared.StructuredMessage; + +namespace Azure.Storage.Tests +{ + [TestFixture(ReadMethod.SyncArray)] + [TestFixture(ReadMethod.AsyncArray)] +#if NETSTANDARD2_1_OR_GREATER || NETCOREAPP3_0_OR_GREATER + [TestFixture(ReadMethod.SyncSpan)] + [TestFixture(ReadMethod.AsyncMemory)] +#endif + public class StructuredMessageStreamRoundtripTests + { + // Cannot just implement as passthru in the stream + // Must test each one + public enum ReadMethod + { + SyncArray, + AsyncArray, +#if NETSTANDARD2_1_OR_GREATER || NETCOREAPP3_0_OR_GREATER + SyncSpan, + AsyncMemory +#endif + } + + public ReadMethod Method { get; } + + public StructuredMessageStreamRoundtripTests(ReadMethod method) + { + Method = method; + } + + private class CopyStreamException : Exception + { + public long TotalCopied { get; } + + public CopyStreamException(Exception inner, long totalCopied) + : base($"Failed read after {totalCopied}-many bytes.", inner) + { + TotalCopied = totalCopied; + } + } + private async ValueTask CopyStream(Stream source, Stream destination, int bufferSize = 81920) // number default for CopyTo impl + { + byte[] buf = new byte[bufferSize]; + int read; + long totalRead = 0; + try + { + switch (Method) + { + case ReadMethod.SyncArray: + while ((read = source.Read(buf, 0, bufferSize)) > 0) + { + totalRead += read; + destination.Write(buf, 0, read); + } + break; + case ReadMethod.AsyncArray: + while ((read = await source.ReadAsync(buf, 0, bufferSize)) > 0) + { + totalRead += read; + await destination.WriteAsync(buf, 0, read); + } + break; +#if NETSTANDARD2_1_OR_GREATER || NETCOREAPP3_0_OR_GREATER + case ReadMethod.SyncSpan: + while ((read = source.Read(new Span(buf))) > 0) + { + totalRead += read; + destination.Write(new Span(buf, 0, read)); + } + break; + case ReadMethod.AsyncMemory: + while ((read = await source.ReadAsync(new Memory(buf))) > 0) + { + totalRead += read; + await destination.WriteAsync(new Memory(buf, 0, read)); + } + break; +#endif + } + destination.Flush(); + } + catch (Exception ex) + { + throw new CopyStreamException(ex, totalRead); + } + return totalRead; + } + + [Test] + [Pairwise] + public async Task RoundTrip( + [Values(2048, 2005)] int dataLength, + [Values(default, 512)] int? seglen, + [Values(8 * Constants.KB, 512, 530, 3)] int readLen, + [Values(true, false)] bool useCrc) + { + int segmentLength = seglen ?? int.MaxValue; + Flags flags = useCrc ? Flags.StorageCrc64 : Flags.None; + + byte[] originalData = new byte[dataLength]; + new Random().NextBytes(originalData); + + byte[] roundtripData; + using (MemoryStream source = new(originalData)) + using (StructuredMessageEncodingStream encode = new(source, segmentLength, flags)) + using (StructuredMessageDecodingStream decode = new(encode)) + using (MemoryStream dest = new()) + { + await CopyStream(source, dest, readLen); + roundtripData = dest.ToArray(); + } + + Assert.That(originalData.SequenceEqual(roundtripData)); + } + } +} From c14d574ae523ac4c17e211baff5afe8ffa615c74 Mon Sep 17 00:00:00 2001 From: Jocelyn <41338290+jaschrep-msft@users.noreply.github.com> Date: Wed, 6 Mar 2024 13:13:03 -0500 Subject: [PATCH 04/22] test coverage | exception message (#42363) --- .../Shared/StructuredMessageDecodingStream.cs | 2 +- .../StructuredMessageDecodingStreamTests.cs | 17 ++++++++++++----- 2 files changed, 13 insertions(+), 6 deletions(-) diff --git a/sdk/storage/Azure.Storage.Common/src/Shared/StructuredMessageDecodingStream.cs b/sdk/storage/Azure.Storage.Common/src/Shared/StructuredMessageDecodingStream.cs index 806a968cf4ec7..1e2af40a594f2 100644 --- a/sdk/storage/Azure.Storage.Common/src/Shared/StructuredMessageDecodingStream.cs +++ b/sdk/storage/Azure.Storage.Common/src/Shared/StructuredMessageDecodingStream.cs @@ -190,7 +190,7 @@ private void AssertDecodeFinished() { if (_streamFooterLength > 0 && !_processedFooter) { - throw Errors.InvalidStructuredMessage("Missing or incomplete trailer."); + throw Errors.InvalidStructuredMessage("Premature end of stream."); } _processedFooter = true; } diff --git a/sdk/storage/Azure.Storage.Common/tests/StructuredMessageDecodingStreamTests.cs b/sdk/storage/Azure.Storage.Common/tests/StructuredMessageDecodingStreamTests.cs index 15c36f19d8763..b115253009705 100644 --- a/sdk/storage/Azure.Storage.Common/tests/StructuredMessageDecodingStreamTests.cs +++ b/sdk/storage/Azure.Storage.Common/tests/StructuredMessageDecodingStreamTests.cs @@ -200,14 +200,21 @@ public void BadStreamWrongContentLength() Assert.That(async () => await CopyStream(decodingStream, Stream.Null), Throws.InnerException.TypeOf()); } - [Test] - public void BadStreamWrongSegmentCount() + [TestCase(-1)] + [TestCase(1)] + public void BadStreamWrongSegmentCount(int difference) { - byte[] originalData = new byte[1024]; + const int dataSize = 1024; + const int segmentSize = 256; + const int numSegments = 4; + + byte[] originalData = new byte[dataSize]; new Random().NextBytes(originalData); - byte[] encodedData = StructuredMessageHelper.MakeEncodedData(originalData, 256, Flags.StorageCrc64); + byte[] encodedData = StructuredMessageHelper.MakeEncodedData(originalData, segmentSize, Flags.StorageCrc64); - BinaryPrimitives.WriteInt16LittleEndian(new Span(encodedData, V1_0.StreamHeaderSegmentCountOffset, 2), 123); + // rewrite the segment count to be different than the actual number of segments + BinaryPrimitives.WriteInt16LittleEndian( + new Span(encodedData, V1_0.StreamHeaderSegmentCountOffset, 2), (short)(numSegments + difference)); Stream decodingStream = new StructuredMessageDecodingStream(new MemoryStream(encodedData)); Assert.That(async () => await CopyStream(decodingStream, Stream.Null), Throws.InnerException.TypeOf()); From 26d8a12f062bb5dbe74b5694e7b9b165097bfb58 Mon Sep 17 00:00:00 2001 From: Jocelyn <41338290+jaschrep-msft@users.noreply.github.com> Date: Wed, 6 Mar 2024 13:13:14 -0500 Subject: [PATCH 05/22] Structured Message Decode: Validate Content Length (#42370) * validate stream length * tests --- .../Shared/StructuredMessageDecodingStream.cs | 17 +++++++- .../StructuredMessageDecodingStreamTests.cs | 40 +++++++++++++++++++ 2 files changed, 55 insertions(+), 2 deletions(-) diff --git a/sdk/storage/Azure.Storage.Common/src/Shared/StructuredMessageDecodingStream.cs b/sdk/storage/Azure.Storage.Common/src/Shared/StructuredMessageDecodingStream.cs index 1e2af40a594f2..aa94b8df350d2 100644 --- a/sdk/storage/Azure.Storage.Common/src/Shared/StructuredMessageDecodingStream.cs +++ b/sdk/storage/Azure.Storage.Common/src/Shared/StructuredMessageDecodingStream.cs @@ -89,9 +89,12 @@ public override long Position } public StructuredMessageDecodingStream( - Stream innerStream) + Stream innerStream, + long? expectedStreamLength = default) { Argument.AssertNotNull(innerStream, nameof(innerStream)); + + _innerStreamLength = expectedStreamLength ?? -1; _innerBufferedStream = new BufferedStream(innerStream); // Assumes stream will be structured message 1.0. Will validate this when consuming stream. @@ -366,9 +369,19 @@ private int ProcessStreamHeader(ReadOnlySpan span) { StructuredMessage.V1_0.ReadStreamHeader( span.Slice(0, _streamHeaderLength), - out _innerStreamLength, + out long streamLength, out _flags, out _totalSegments); + + if (_innerStreamLength > 0 && streamLength != _innerStreamLength) + { + throw Errors.InvalidStructuredMessage("Unexpected message size."); + } + else + { + _innerStreamLength = streamLength; + } + if (_flags.HasFlag(StructuredMessage.Flags.StorageCrc64)) { _segmentFooterLength = _flags.HasFlag(StructuredMessage.Flags.StorageCrc64) ? StructuredMessage.Crc64Length : 0; diff --git a/sdk/storage/Azure.Storage.Common/tests/StructuredMessageDecodingStreamTests.cs b/sdk/storage/Azure.Storage.Common/tests/StructuredMessageDecodingStreamTests.cs index b115253009705..f881a70c8e78f 100644 --- a/sdk/storage/Azure.Storage.Common/tests/StructuredMessageDecodingStreamTests.cs +++ b/sdk/storage/Azure.Storage.Common/tests/StructuredMessageDecodingStreamTests.cs @@ -234,6 +234,46 @@ public void BadStreamWrongSegmentNum() Assert.That(async () => await CopyStream(decodingStream, Stream.Null), Throws.InnerException.TypeOf()); } + [Test] + [Combinatorial] + public async Task BadStreamWrongContentLength( + [Values(-1, 1)] int difference, + [Values(true, false)] bool lengthProvided) + { + byte[] originalData = new byte[1024]; + new Random().NextBytes(originalData); + byte[] encodedData = StructuredMessageHelper.MakeEncodedData(originalData, 256, Flags.StorageCrc64); + + BinaryPrimitives.WriteInt64LittleEndian( + new Span(encodedData, V1_0.StreamHeaderMessageLengthOffset, 8), + encodedData.Length + difference); + + Stream decodingStream = new StructuredMessageDecodingStream( + new MemoryStream(encodedData), + lengthProvided ? (long?)encodedData.Length : default); + + // manual try/catch with tiny buffer to validate the proccess failed mid-stream rather than the end + const int copyBufferSize = 4; + bool caught = false; + try + { + await CopyStream(decodingStream, Stream.Null, copyBufferSize); + } + catch (CopyStreamException ex) + { + caught = true; + if (lengthProvided) + { + Assert.That(ex.TotalCopied, Is.EqualTo(0)); + } + else + { + Assert.That(ex.TotalCopied, Is.EqualTo(originalData.Length)); + } + } + Assert.That(caught); + } + [Test] public void BadStreamMissingExpectedStreamFooter() { From 305ee6f2ce8fbbe8a264dd3efa5174523338a916 Mon Sep 17 00:00:00 2001 From: Jocelyn <41338290+jaschrep-msft@users.noreply.github.com> Date: Thu, 21 Mar 2024 10:18:34 -0400 Subject: [PATCH 06/22] stageblock | appendblock | putpages StructuredMessage (#42699) * regenerate and stage block uses structured message * page and append * testproxy * cleanup * fix datalake/share tests * testproxy * testproxy * re-add null-safe access --- sdk/storage/Azure.Storage.Blobs/assets.json | 2 +- .../src/AppendBlobClient.cs | 42 ++++-- .../src/BlockBlobClient.cs | 46 +++++-- .../Generated/AppendBlobAppendBlockHeaders.cs | 2 + .../src/Generated/AppendBlobRestClient.cs | 24 +++- .../src/Generated/BlobDownloadHeaders.cs | 4 + .../src/Generated/BlobRestClient.cs | 18 ++- .../src/Generated/BlockBlobRestClient.cs | 46 +++++-- .../Generated/BlockBlobStageBlockHeaders.cs | 2 + .../src/Generated/BlockBlobUploadHeaders.cs | 2 + .../src/Generated/ContainerRestClient.cs | 2 +- .../src/Generated/PageBlobRestClient.cs | 24 +++- .../Generated/PageBlobUploadPagesHeaders.cs | 2 + .../src/Generated/ServiceRestClient.cs | 2 +- .../Azure.Storage.Blobs/src/PageBlobClient.cs | 46 +++++-- .../Azure.Storage.Blobs/src/autorest.md | 6 +- .../tests/Azure.Storage.Blobs.Tests.csproj | 3 + .../BlobBaseClientTransferValidationTests.cs | 5 +- .../src/Shared/Constants.cs | 6 + .../src/Shared/StructuredMessage.cs | 2 +- .../Shared/StructuredMessageDecodingStream.cs | 2 +- .../Shared/StructuredMessageEncodingStream.cs | 2 +- .../Shared/TransferValidationTestBase.cs | 124 ++++++++++++++---- .../src/Azure.Storage.DataMovement.csproj | 2 +- .../Azure.Storage.Files.DataLake/assets.json | 2 +- .../src/Azure.Storage.Files.DataLake.csproj | 1 + ...taLakeFileClientTransferValidationTests.cs | 5 +- .../Azure.Storage.Files.Shares/assets.json | 2 +- .../src/Azure.Storage.Files.Shares.csproj | 1 + 29 files changed, 330 insertions(+), 97 deletions(-) diff --git a/sdk/storage/Azure.Storage.Blobs/assets.json b/sdk/storage/Azure.Storage.Blobs/assets.json index e0cc7497a2f22..3de365ac947fb 100644 --- a/sdk/storage/Azure.Storage.Blobs/assets.json +++ b/sdk/storage/Azure.Storage.Blobs/assets.json @@ -2,5 +2,5 @@ "AssetsRepo": "Azure/azure-sdk-assets", "AssetsRepoPrefixPath": "net", "TagPrefix": "net/storage/Azure.Storage.Blobs", - "Tag": "net/storage/Azure.Storage.Blobs_14eb1d6279" + "Tag": "net/storage/Azure.Storage.Blobs_5d0f81abfc" } diff --git a/sdk/storage/Azure.Storage.Blobs/src/AppendBlobClient.cs b/sdk/storage/Azure.Storage.Blobs/src/AppendBlobClient.cs index e70d5e02c82d7..5a396e60a0598 100644 --- a/sdk/storage/Azure.Storage.Blobs/src/AppendBlobClient.cs +++ b/sdk/storage/Azure.Storage.Blobs/src/AppendBlobClient.cs @@ -1242,14 +1242,36 @@ internal async Task> AppendBlockInternal( BlobErrors.VerifyHttpsCustomerProvidedKey(Uri, ClientConfiguration.CustomerProvidedKey); Errors.VerifyStreamPosition(content, nameof(content)); - // compute hash BEFORE attaching progress handler - ContentHasher.GetHashResult hashResult = await ContentHasher.GetHashOrDefaultInternal( - content, - validationOptions, - async, - cancellationToken).ConfigureAwait(false); - - content = content.WithNoDispose().WithProgress(progressHandler); + ContentHasher.GetHashResult hashResult = null; + long contentLength = (content?.Length - content?.Position) ?? 0; + long? structuredContentLength = default; + string structuredBodyType = null; + if (validationOptions != null && + validationOptions.ChecksumAlgorithm.ResolveAuto() == StorageChecksumAlgorithm.StorageCrc64 && + validationOptions.PrecalculatedChecksum.IsEmpty && + ClientSideEncryption == null) // don't allow feature combination + { + // report progress in terms of caller bytes, not encoded bytes + structuredContentLength = contentLength; + contentLength = (content?.Length - content?.Position) ?? 0; + structuredBodyType = Constants.StructuredMessage.CrcStructuredMessage; + content = content.WithNoDispose().WithProgress(progressHandler); + content = new StructuredMessageEncodingStream( + content, + Constants.StructuredMessage.DefaultSegmentContentLength, + StructuredMessage.Flags.StorageCrc64); + contentLength = (content?.Length - content?.Position) ?? 0; + } + else + { + // compute hash BEFORE attaching progress handler + hashResult = await ContentHasher.GetHashOrDefaultInternal( + content, + validationOptions, + async, + cancellationToken).ConfigureAwait(false); + content = content.WithNoDispose().WithProgress(progressHandler); + } ResponseWithHeaders response; @@ -1267,6 +1289,8 @@ internal async Task> AppendBlockInternal( encryptionKeySha256: ClientConfiguration.CustomerProvidedKey?.EncryptionKeyHash, encryptionAlgorithm: ClientConfiguration.CustomerProvidedKey?.EncryptionAlgorithm == null ? null : EncryptionAlgorithmTypeInternal.AES256, encryptionScope: ClientConfiguration.EncryptionScope, + structuredBodyType: structuredBodyType, + structuredContentLength: structuredContentLength, ifModifiedSince: conditions?.IfModifiedSince, ifUnmodifiedSince: conditions?.IfUnmodifiedSince, ifMatch: conditions?.IfMatch?.ToString(), @@ -1289,6 +1313,8 @@ internal async Task> AppendBlockInternal( encryptionKeySha256: ClientConfiguration.CustomerProvidedKey?.EncryptionKeyHash, encryptionAlgorithm: ClientConfiguration.CustomerProvidedKey?.EncryptionAlgorithm == null ? null : EncryptionAlgorithmTypeInternal.AES256, encryptionScope: ClientConfiguration.EncryptionScope, + structuredBodyType: structuredBodyType, + structuredContentLength: structuredContentLength, ifModifiedSince: conditions?.IfModifiedSince, ifUnmodifiedSince: conditions?.IfUnmodifiedSince, ifMatch: conditions?.IfMatch?.ToString(), diff --git a/sdk/storage/Azure.Storage.Blobs/src/BlockBlobClient.cs b/sdk/storage/Azure.Storage.Blobs/src/BlockBlobClient.cs index cd6bc3788fc26..38c3186c13f4c 100644 --- a/sdk/storage/Azure.Storage.Blobs/src/BlockBlobClient.cs +++ b/sdk/storage/Azure.Storage.Blobs/src/BlockBlobClient.cs @@ -1305,14 +1305,36 @@ internal virtual async Task> StageBlockInternal( Errors.VerifyStreamPosition(content, nameof(content)); - // compute hash BEFORE attaching progress handler - ContentHasher.GetHashResult hashResult = await ContentHasher.GetHashOrDefaultInternal( - content, - validationOptions, - async, - cancellationToken).ConfigureAwait(false); - - content = content.WithNoDispose().WithProgress(progressHandler); + ContentHasher.GetHashResult hashResult = null; + long contentLength = (content?.Length - content?.Position) ?? 0; + long? structuredContentLength = default; + string structuredBodyType = null; + if (validationOptions != null && + validationOptions.ChecksumAlgorithm.ResolveAuto() == StorageChecksumAlgorithm.StorageCrc64 && + validationOptions.PrecalculatedChecksum.IsEmpty && + ClientSideEncryption == null) // don't allow feature combination + { + // report progress in terms of caller bytes, not encoded bytes + structuredContentLength = contentLength; + contentLength = (content?.Length - content?.Position) ?? 0; + structuredBodyType = Constants.StructuredMessage.CrcStructuredMessage; + content = content.WithNoDispose().WithProgress(progressHandler); + content = new StructuredMessageEncodingStream( + content, + Constants.StructuredMessage.DefaultSegmentContentLength, + StructuredMessage.Flags.StorageCrc64); + contentLength = (content?.Length - content?.Position) ?? 0; + } + else + { + // compute hash BEFORE attaching progress handler + hashResult = await ContentHasher.GetHashOrDefaultInternal( + content, + validationOptions, + async, + cancellationToken).ConfigureAwait(false); + content = content.WithNoDispose().WithProgress(progressHandler); + } ResponseWithHeaders response; @@ -1320,7 +1342,7 @@ internal virtual async Task> StageBlockInternal( { response = await BlockBlobRestClient.StageBlockAsync( blockId: base64BlockId, - contentLength: (content?.Length - content?.Position) ?? 0, + contentLength: contentLength, body: content, transactionalContentCrc64: hashResult?.StorageCrc64AsArray, transactionalContentMD5: hashResult?.MD5AsArray, @@ -1329,6 +1351,8 @@ internal virtual async Task> StageBlockInternal( encryptionKeySha256: ClientConfiguration.CustomerProvidedKey?.EncryptionKeyHash, encryptionAlgorithm: ClientConfiguration.CustomerProvidedKey?.EncryptionAlgorithm == null ? null : EncryptionAlgorithmTypeInternal.AES256, encryptionScope: ClientConfiguration.EncryptionScope, + structuredBodyType: structuredBodyType, + structuredContentLength: structuredContentLength, cancellationToken: cancellationToken) .ConfigureAwait(false); } @@ -1336,7 +1360,7 @@ internal virtual async Task> StageBlockInternal( { response = BlockBlobRestClient.StageBlock( blockId: base64BlockId, - contentLength: (content?.Length - content?.Position) ?? 0, + contentLength: contentLength, body: content, transactionalContentCrc64: hashResult?.StorageCrc64AsArray, transactionalContentMD5: hashResult?.MD5AsArray, @@ -1345,6 +1369,8 @@ internal virtual async Task> StageBlockInternal( encryptionKeySha256: ClientConfiguration.CustomerProvidedKey?.EncryptionKeyHash, encryptionAlgorithm: ClientConfiguration.CustomerProvidedKey?.EncryptionAlgorithm == null ? null : EncryptionAlgorithmTypeInternal.AES256, encryptionScope: ClientConfiguration.EncryptionScope, + structuredBodyType: structuredBodyType, + structuredContentLength: structuredContentLength, cancellationToken: cancellationToken); } diff --git a/sdk/storage/Azure.Storage.Blobs/src/Generated/AppendBlobAppendBlockHeaders.cs b/sdk/storage/Azure.Storage.Blobs/src/Generated/AppendBlobAppendBlockHeaders.cs index 9303ec3a3d653..48139cc16a682 100644 --- a/sdk/storage/Azure.Storage.Blobs/src/Generated/AppendBlobAppendBlockHeaders.cs +++ b/sdk/storage/Azure.Storage.Blobs/src/Generated/AppendBlobAppendBlockHeaders.cs @@ -35,5 +35,7 @@ public AppendBlobAppendBlockHeaders(Response response) public string EncryptionKeySha256 => _response.Headers.TryGetValue("x-ms-encryption-key-sha256", out string value) ? value : null; /// Returns the name of the encryption scope used to encrypt the blob contents and application metadata. Note that the absence of this header implies use of the default account encryption scope. public string EncryptionScope => _response.Headers.TryGetValue("x-ms-encryption-scope", out string value) ? value : null; + /// Indicates the structured message body was accepted and mirrors back the message schema version and properties. + public string StructuredBodyType => _response.Headers.TryGetValue("x-ms-structured-body", out string value) ? value : null; } } diff --git a/sdk/storage/Azure.Storage.Blobs/src/Generated/AppendBlobRestClient.cs b/sdk/storage/Azure.Storage.Blobs/src/Generated/AppendBlobRestClient.cs index 88104aa95bb00..a3d0eca1ec405 100644 --- a/sdk/storage/Azure.Storage.Blobs/src/Generated/AppendBlobRestClient.cs +++ b/sdk/storage/Azure.Storage.Blobs/src/Generated/AppendBlobRestClient.cs @@ -29,7 +29,7 @@ internal partial class AppendBlobRestClient /// The handler for diagnostic messaging in the client. /// The HTTP pipeline for sending and receiving REST requests and responses. /// The URL of the service account, container, or blob that is the target of the desired operation. - /// Specifies the version of the operation to use for this request. The default value is "2024-08-04". + /// Specifies the version of the operation to use for this request. The default value is "2025-01-05". /// , , or is null. public AppendBlobRestClient(ClientDiagnostics clientDiagnostics, HttpPipeline pipeline, string url, string version) { @@ -219,7 +219,7 @@ public ResponseWithHeaders Create(long contentLength, i } } - internal HttpMessage CreateAppendBlockRequest(long contentLength, Stream body, int? timeout, byte[] transactionalContentMD5, byte[] transactionalContentCrc64, string leaseId, long? maxSize, long? appendPosition, string encryptionKey, string encryptionKeySha256, EncryptionAlgorithmTypeInternal? encryptionAlgorithm, string encryptionScope, DateTimeOffset? ifModifiedSince, DateTimeOffset? ifUnmodifiedSince, string ifMatch, string ifNoneMatch, string ifTags) + internal HttpMessage CreateAppendBlockRequest(long contentLength, Stream body, int? timeout, byte[] transactionalContentMD5, byte[] transactionalContentCrc64, string leaseId, long? maxSize, long? appendPosition, string encryptionKey, string encryptionKeySha256, EncryptionAlgorithmTypeInternal? encryptionAlgorithm, string encryptionScope, DateTimeOffset? ifModifiedSince, DateTimeOffset? ifUnmodifiedSince, string ifMatch, string ifNoneMatch, string ifTags, string structuredBodyType, long? structuredContentLength) { var message = _pipeline.CreateMessage(); var request = message.Request; @@ -285,6 +285,14 @@ internal HttpMessage CreateAppendBlockRequest(long contentLength, Stream body, i request.Headers.Add("x-ms-if-tags", ifTags); } request.Headers.Add("x-ms-version", _version); + if (structuredBodyType != null) + { + request.Headers.Add("x-ms-structured-body", structuredBodyType); + } + if (structuredContentLength != null) + { + request.Headers.Add("x-ms-structured-content-length", structuredContentLength.Value); + } request.Headers.Add("Accept", "application/xml"); request.Headers.Add("Content-Length", contentLength); if (transactionalContentMD5 != null) @@ -314,16 +322,18 @@ internal HttpMessage CreateAppendBlockRequest(long contentLength, Stream body, i /// Specify an ETag value to operate only on blobs with a matching value. /// Specify an ETag value to operate only on blobs without a matching value. /// Specify a SQL where clause on blob tags to operate only on blobs with a matching value. + /// Required if the request body is a structured message. Specifies the message schema version and properties. + /// Required if the request body is a structured message. Specifies the length of the blob/file content inside the message body. Will always be smaller than Content-Length. /// The cancellation token to use. /// is null. - public async Task> AppendBlockAsync(long contentLength, Stream body, int? timeout = null, byte[] transactionalContentMD5 = null, byte[] transactionalContentCrc64 = null, string leaseId = null, long? maxSize = null, long? appendPosition = null, string encryptionKey = null, string encryptionKeySha256 = null, EncryptionAlgorithmTypeInternal? encryptionAlgorithm = null, string encryptionScope = null, DateTimeOffset? ifModifiedSince = null, DateTimeOffset? ifUnmodifiedSince = null, string ifMatch = null, string ifNoneMatch = null, string ifTags = null, CancellationToken cancellationToken = default) + public async Task> AppendBlockAsync(long contentLength, Stream body, int? timeout = null, byte[] transactionalContentMD5 = null, byte[] transactionalContentCrc64 = null, string leaseId = null, long? maxSize = null, long? appendPosition = null, string encryptionKey = null, string encryptionKeySha256 = null, EncryptionAlgorithmTypeInternal? encryptionAlgorithm = null, string encryptionScope = null, DateTimeOffset? ifModifiedSince = null, DateTimeOffset? ifUnmodifiedSince = null, string ifMatch = null, string ifNoneMatch = null, string ifTags = null, string structuredBodyType = null, long? structuredContentLength = null, CancellationToken cancellationToken = default) { if (body == null) { throw new ArgumentNullException(nameof(body)); } - using var message = CreateAppendBlockRequest(contentLength, body, timeout, transactionalContentMD5, transactionalContentCrc64, leaseId, maxSize, appendPosition, encryptionKey, encryptionKeySha256, encryptionAlgorithm, encryptionScope, ifModifiedSince, ifUnmodifiedSince, ifMatch, ifNoneMatch, ifTags); + using var message = CreateAppendBlockRequest(contentLength, body, timeout, transactionalContentMD5, transactionalContentCrc64, leaseId, maxSize, appendPosition, encryptionKey, encryptionKeySha256, encryptionAlgorithm, encryptionScope, ifModifiedSince, ifUnmodifiedSince, ifMatch, ifNoneMatch, ifTags, structuredBodyType, structuredContentLength); await _pipeline.SendAsync(message, cancellationToken).ConfigureAwait(false); var headers = new AppendBlobAppendBlockHeaders(message.Response); switch (message.Response.Status) @@ -353,16 +363,18 @@ public async Task> AppendBlock /// Specify an ETag value to operate only on blobs with a matching value. /// Specify an ETag value to operate only on blobs without a matching value. /// Specify a SQL where clause on blob tags to operate only on blobs with a matching value. + /// Required if the request body is a structured message. Specifies the message schema version and properties. + /// Required if the request body is a structured message. Specifies the length of the blob/file content inside the message body. Will always be smaller than Content-Length. /// The cancellation token to use. /// is null. - public ResponseWithHeaders AppendBlock(long contentLength, Stream body, int? timeout = null, byte[] transactionalContentMD5 = null, byte[] transactionalContentCrc64 = null, string leaseId = null, long? maxSize = null, long? appendPosition = null, string encryptionKey = null, string encryptionKeySha256 = null, EncryptionAlgorithmTypeInternal? encryptionAlgorithm = null, string encryptionScope = null, DateTimeOffset? ifModifiedSince = null, DateTimeOffset? ifUnmodifiedSince = null, string ifMatch = null, string ifNoneMatch = null, string ifTags = null, CancellationToken cancellationToken = default) + public ResponseWithHeaders AppendBlock(long contentLength, Stream body, int? timeout = null, byte[] transactionalContentMD5 = null, byte[] transactionalContentCrc64 = null, string leaseId = null, long? maxSize = null, long? appendPosition = null, string encryptionKey = null, string encryptionKeySha256 = null, EncryptionAlgorithmTypeInternal? encryptionAlgorithm = null, string encryptionScope = null, DateTimeOffset? ifModifiedSince = null, DateTimeOffset? ifUnmodifiedSince = null, string ifMatch = null, string ifNoneMatch = null, string ifTags = null, string structuredBodyType = null, long? structuredContentLength = null, CancellationToken cancellationToken = default) { if (body == null) { throw new ArgumentNullException(nameof(body)); } - using var message = CreateAppendBlockRequest(contentLength, body, timeout, transactionalContentMD5, transactionalContentCrc64, leaseId, maxSize, appendPosition, encryptionKey, encryptionKeySha256, encryptionAlgorithm, encryptionScope, ifModifiedSince, ifUnmodifiedSince, ifMatch, ifNoneMatch, ifTags); + using var message = CreateAppendBlockRequest(contentLength, body, timeout, transactionalContentMD5, transactionalContentCrc64, leaseId, maxSize, appendPosition, encryptionKey, encryptionKeySha256, encryptionAlgorithm, encryptionScope, ifModifiedSince, ifUnmodifiedSince, ifMatch, ifNoneMatch, ifTags, structuredBodyType, structuredContentLength); _pipeline.Send(message, cancellationToken); var headers = new AppendBlobAppendBlockHeaders(message.Response); switch (message.Response.Status) diff --git a/sdk/storage/Azure.Storage.Blobs/src/Generated/BlobDownloadHeaders.cs b/sdk/storage/Azure.Storage.Blobs/src/Generated/BlobDownloadHeaders.cs index ad17079901a72..1897117cb01d8 100644 --- a/sdk/storage/Azure.Storage.Blobs/src/Generated/BlobDownloadHeaders.cs +++ b/sdk/storage/Azure.Storage.Blobs/src/Generated/BlobDownloadHeaders.cs @@ -96,6 +96,10 @@ public BlobDownloadHeaders(Response response) public BlobImmutabilityPolicyMode? ImmutabilityPolicyMode => _response.Headers.TryGetValue("x-ms-immutability-policy-mode", out string value) ? value.ToBlobImmutabilityPolicyMode() : null; /// Indicates if a legal hold is present on the blob. public bool? LegalHold => _response.Headers.TryGetValue("x-ms-legal-hold", out bool? value) ? value : null; + /// Indicates the response body contains a structured message and specifies the message schema version and properties. + public string StructuredBodyType => _response.Headers.TryGetValue("x-ms-structured-body", out string value) ? value : null; + /// The length of the blob/file content inside the message body when the response body is returned as a structured message. Will always be smaller than Content-Length. + public long? StructuredContentLength => _response.Headers.TryGetValue("x-ms-structured-content-length", out long? value) ? value : null; /// If the request is to read a specified range and the x-ms-range-get-content-crc64 is set to true, then the request returns a crc64 for the range, as long as the range size is less than or equal to 4 MB. If both x-ms-range-get-content-crc64 & x-ms-range-get-content-md5 is specified in the same request, it will fail with 400(Bad Request). public byte[] ContentCrc64 => _response.Headers.TryGetValue("x-ms-content-crc64", out byte[] value) ? value : null; public string ErrorCode => _response.Headers.TryGetValue("x-ms-error-code", out string value) ? value : null; diff --git a/sdk/storage/Azure.Storage.Blobs/src/Generated/BlobRestClient.cs b/sdk/storage/Azure.Storage.Blobs/src/Generated/BlobRestClient.cs index 615257741b781..4f891a0a14684 100644 --- a/sdk/storage/Azure.Storage.Blobs/src/Generated/BlobRestClient.cs +++ b/sdk/storage/Azure.Storage.Blobs/src/Generated/BlobRestClient.cs @@ -30,7 +30,7 @@ internal partial class BlobRestClient /// The handler for diagnostic messaging in the client. /// The HTTP pipeline for sending and receiving REST requests and responses. /// The URL of the service account, container, or blob that is the target of the desired operation. - /// Specifies the version of the operation to use for this request. The default value is "2024-08-04". + /// Specifies the version of the operation to use for this request. The default value is "2025-01-05". /// , , or is null. public BlobRestClient(ClientDiagnostics clientDiagnostics, HttpPipeline pipeline, string url, string version) { @@ -40,7 +40,7 @@ public BlobRestClient(ClientDiagnostics clientDiagnostics, HttpPipeline pipeline _version = version ?? throw new ArgumentNullException(nameof(version)); } - internal HttpMessage CreateDownloadRequest(string snapshot, string versionId, int? timeout, string range, string leaseId, bool? rangeGetContentMD5, bool? rangeGetContentCRC64, string encryptionKey, string encryptionKeySha256, EncryptionAlgorithmTypeInternal? encryptionAlgorithm, DateTimeOffset? ifModifiedSince, DateTimeOffset? ifUnmodifiedSince, string ifMatch, string ifNoneMatch, string ifTags) + internal HttpMessage CreateDownloadRequest(string snapshot, string versionId, int? timeout, string range, string leaseId, bool? rangeGetContentMD5, bool? rangeGetContentCRC64, string structuredBodyType, string encryptionKey, string encryptionKeySha256, EncryptionAlgorithmTypeInternal? encryptionAlgorithm, DateTimeOffset? ifModifiedSince, DateTimeOffset? ifUnmodifiedSince, string ifMatch, string ifNoneMatch, string ifTags) { var message = _pipeline.CreateMessage(); var request = message.Request; @@ -77,6 +77,10 @@ internal HttpMessage CreateDownloadRequest(string snapshot, string versionId, in { request.Headers.Add("x-ms-range-get-content-crc64", rangeGetContentCRC64.Value); } + if (structuredBodyType != null) + { + request.Headers.Add("x-ms-structured-body", structuredBodyType); + } if (encryptionKey != null) { request.Headers.Add("x-ms-encryption-key", encryptionKey); @@ -122,6 +126,7 @@ internal HttpMessage CreateDownloadRequest(string snapshot, string versionId, in /// If specified, the operation only succeeds if the resource's lease is active and matches this ID. /// When set to true and specified together with the Range, the service returns the MD5 hash for the range, as long as the range is less than or equal to 4 MB in size. /// When set to true and specified together with the Range, the service returns the CRC64 hash for the range, as long as the range is less than or equal to 4 MB in size. + /// Specifies the response content should be returned as a structured message and specifies the message schema version and properties. /// Optional. Specifies the encryption key to use to encrypt the data provided in the request. If not specified, encryption is performed with the root account encryption key. For more information, see Encryption at Rest for Azure Storage Services. /// The SHA-256 hash of the provided encryption key. Must be provided if the x-ms-encryption-key header is provided. /// The algorithm used to produce the encryption key hash. Currently, the only accepted value is "AES256". Must be provided if the x-ms-encryption-key header is provided. @@ -131,9 +136,9 @@ internal HttpMessage CreateDownloadRequest(string snapshot, string versionId, in /// Specify an ETag value to operate only on blobs without a matching value. /// Specify a SQL where clause on blob tags to operate only on blobs with a matching value. /// The cancellation token to use. - public async Task> DownloadAsync(string snapshot = null, string versionId = null, int? timeout = null, string range = null, string leaseId = null, bool? rangeGetContentMD5 = null, bool? rangeGetContentCRC64 = null, string encryptionKey = null, string encryptionKeySha256 = null, EncryptionAlgorithmTypeInternal? encryptionAlgorithm = null, DateTimeOffset? ifModifiedSince = null, DateTimeOffset? ifUnmodifiedSince = null, string ifMatch = null, string ifNoneMatch = null, string ifTags = null, CancellationToken cancellationToken = default) + public async Task> DownloadAsync(string snapshot = null, string versionId = null, int? timeout = null, string range = null, string leaseId = null, bool? rangeGetContentMD5 = null, bool? rangeGetContentCRC64 = null, string structuredBodyType = null, string encryptionKey = null, string encryptionKeySha256 = null, EncryptionAlgorithmTypeInternal? encryptionAlgorithm = null, DateTimeOffset? ifModifiedSince = null, DateTimeOffset? ifUnmodifiedSince = null, string ifMatch = null, string ifNoneMatch = null, string ifTags = null, CancellationToken cancellationToken = default) { - using var message = CreateDownloadRequest(snapshot, versionId, timeout, range, leaseId, rangeGetContentMD5, rangeGetContentCRC64, encryptionKey, encryptionKeySha256, encryptionAlgorithm, ifModifiedSince, ifUnmodifiedSince, ifMatch, ifNoneMatch, ifTags); + using var message = CreateDownloadRequest(snapshot, versionId, timeout, range, leaseId, rangeGetContentMD5, rangeGetContentCRC64, structuredBodyType, encryptionKey, encryptionKeySha256, encryptionAlgorithm, ifModifiedSince, ifUnmodifiedSince, ifMatch, ifNoneMatch, ifTags); await _pipeline.SendAsync(message, cancellationToken).ConfigureAwait(false); var headers = new BlobDownloadHeaders(message.Response); switch (message.Response.Status) @@ -159,6 +164,7 @@ public async Task> DownloadAsyn /// If specified, the operation only succeeds if the resource's lease is active and matches this ID. /// When set to true and specified together with the Range, the service returns the MD5 hash for the range, as long as the range is less than or equal to 4 MB in size. /// When set to true and specified together with the Range, the service returns the CRC64 hash for the range, as long as the range is less than or equal to 4 MB in size. + /// Specifies the response content should be returned as a structured message and specifies the message schema version and properties. /// Optional. Specifies the encryption key to use to encrypt the data provided in the request. If not specified, encryption is performed with the root account encryption key. For more information, see Encryption at Rest for Azure Storage Services. /// The SHA-256 hash of the provided encryption key. Must be provided if the x-ms-encryption-key header is provided. /// The algorithm used to produce the encryption key hash. Currently, the only accepted value is "AES256". Must be provided if the x-ms-encryption-key header is provided. @@ -168,9 +174,9 @@ public async Task> DownloadAsyn /// Specify an ETag value to operate only on blobs without a matching value. /// Specify a SQL where clause on blob tags to operate only on blobs with a matching value. /// The cancellation token to use. - public ResponseWithHeaders Download(string snapshot = null, string versionId = null, int? timeout = null, string range = null, string leaseId = null, bool? rangeGetContentMD5 = null, bool? rangeGetContentCRC64 = null, string encryptionKey = null, string encryptionKeySha256 = null, EncryptionAlgorithmTypeInternal? encryptionAlgorithm = null, DateTimeOffset? ifModifiedSince = null, DateTimeOffset? ifUnmodifiedSince = null, string ifMatch = null, string ifNoneMatch = null, string ifTags = null, CancellationToken cancellationToken = default) + public ResponseWithHeaders Download(string snapshot = null, string versionId = null, int? timeout = null, string range = null, string leaseId = null, bool? rangeGetContentMD5 = null, bool? rangeGetContentCRC64 = null, string structuredBodyType = null, string encryptionKey = null, string encryptionKeySha256 = null, EncryptionAlgorithmTypeInternal? encryptionAlgorithm = null, DateTimeOffset? ifModifiedSince = null, DateTimeOffset? ifUnmodifiedSince = null, string ifMatch = null, string ifNoneMatch = null, string ifTags = null, CancellationToken cancellationToken = default) { - using var message = CreateDownloadRequest(snapshot, versionId, timeout, range, leaseId, rangeGetContentMD5, rangeGetContentCRC64, encryptionKey, encryptionKeySha256, encryptionAlgorithm, ifModifiedSince, ifUnmodifiedSince, ifMatch, ifNoneMatch, ifTags); + using var message = CreateDownloadRequest(snapshot, versionId, timeout, range, leaseId, rangeGetContentMD5, rangeGetContentCRC64, structuredBodyType, encryptionKey, encryptionKeySha256, encryptionAlgorithm, ifModifiedSince, ifUnmodifiedSince, ifMatch, ifNoneMatch, ifTags); _pipeline.Send(message, cancellationToken); var headers = new BlobDownloadHeaders(message.Response); switch (message.Response.Status) diff --git a/sdk/storage/Azure.Storage.Blobs/src/Generated/BlockBlobRestClient.cs b/sdk/storage/Azure.Storage.Blobs/src/Generated/BlockBlobRestClient.cs index 0723c07204ac2..78ef424f66b13 100644 --- a/sdk/storage/Azure.Storage.Blobs/src/Generated/BlockBlobRestClient.cs +++ b/sdk/storage/Azure.Storage.Blobs/src/Generated/BlockBlobRestClient.cs @@ -30,7 +30,7 @@ internal partial class BlockBlobRestClient /// The handler for diagnostic messaging in the client. /// The HTTP pipeline for sending and receiving REST requests and responses. /// The URL of the service account, container, or blob that is the target of the desired operation. - /// Specifies the version of the operation to use for this request. The default value is "2024-08-04". + /// Specifies the version of the operation to use for this request. The default value is "2025-01-05". /// , , or is null. public BlockBlobRestClient(ClientDiagnostics clientDiagnostics, HttpPipeline pipeline, string url, string version) { @@ -40,7 +40,7 @@ public BlockBlobRestClient(ClientDiagnostics clientDiagnostics, HttpPipeline pip _version = version ?? throw new ArgumentNullException(nameof(version)); } - internal HttpMessage CreateUploadRequest(long contentLength, Stream body, int? timeout, byte[] transactionalContentMD5, string blobContentType, string blobContentEncoding, string blobContentLanguage, byte[] blobContentMD5, string blobCacheControl, IDictionary metadata, string leaseId, string blobContentDisposition, string encryptionKey, string encryptionKeySha256, EncryptionAlgorithmTypeInternal? encryptionAlgorithm, string encryptionScope, AccessTier? tier, DateTimeOffset? ifModifiedSince, DateTimeOffset? ifUnmodifiedSince, string ifMatch, string ifNoneMatch, string ifTags, string blobTagsString, DateTimeOffset? immutabilityPolicyExpiry, BlobImmutabilityPolicyMode? immutabilityPolicyMode, bool? legalHold, byte[] transactionalContentCrc64) + internal HttpMessage CreateUploadRequest(long contentLength, Stream body, int? timeout, byte[] transactionalContentMD5, string blobContentType, string blobContentEncoding, string blobContentLanguage, byte[] blobContentMD5, string blobCacheControl, IDictionary metadata, string leaseId, string blobContentDisposition, string encryptionKey, string encryptionKeySha256, EncryptionAlgorithmTypeInternal? encryptionAlgorithm, string encryptionScope, AccessTier? tier, DateTimeOffset? ifModifiedSince, DateTimeOffset? ifUnmodifiedSince, string ifMatch, string ifNoneMatch, string ifTags, string blobTagsString, DateTimeOffset? immutabilityPolicyExpiry, BlobImmutabilityPolicyMode? immutabilityPolicyMode, bool? legalHold, byte[] transactionalContentCrc64, string structuredBodyType, long? structuredContentLength) { var message = _pipeline.CreateMessage(); var request = message.Request; @@ -146,6 +146,14 @@ internal HttpMessage CreateUploadRequest(long contentLength, Stream body, int? t { request.Headers.Add("x-ms-content-crc64", transactionalContentCrc64, "D"); } + if (structuredBodyType != null) + { + request.Headers.Add("x-ms-structured-body", structuredBodyType); + } + if (structuredContentLength != null) + { + request.Headers.Add("x-ms-structured-content-length", structuredContentLength.Value); + } request.Headers.Add("Accept", "application/xml"); if (transactionalContentMD5 != null) { @@ -185,16 +193,18 @@ internal HttpMessage CreateUploadRequest(long contentLength, Stream body, int? t /// Specifies the immutability policy mode to set on the blob. /// Specified if a legal hold should be set on the blob. /// Specify the transactional crc64 for the body, to be validated by the service. + /// Required if the request body is a structured message. Specifies the message schema version and properties. + /// Required if the request body is a structured message. Specifies the length of the blob/file content inside the message body. Will always be smaller than Content-Length. /// The cancellation token to use. /// is null. - public async Task> UploadAsync(long contentLength, Stream body, int? timeout = null, byte[] transactionalContentMD5 = null, string blobContentType = null, string blobContentEncoding = null, string blobContentLanguage = null, byte[] blobContentMD5 = null, string blobCacheControl = null, IDictionary metadata = null, string leaseId = null, string blobContentDisposition = null, string encryptionKey = null, string encryptionKeySha256 = null, EncryptionAlgorithmTypeInternal? encryptionAlgorithm = null, string encryptionScope = null, AccessTier? tier = null, DateTimeOffset? ifModifiedSince = null, DateTimeOffset? ifUnmodifiedSince = null, string ifMatch = null, string ifNoneMatch = null, string ifTags = null, string blobTagsString = null, DateTimeOffset? immutabilityPolicyExpiry = null, BlobImmutabilityPolicyMode? immutabilityPolicyMode = null, bool? legalHold = null, byte[] transactionalContentCrc64 = null, CancellationToken cancellationToken = default) + public async Task> UploadAsync(long contentLength, Stream body, int? timeout = null, byte[] transactionalContentMD5 = null, string blobContentType = null, string blobContentEncoding = null, string blobContentLanguage = null, byte[] blobContentMD5 = null, string blobCacheControl = null, IDictionary metadata = null, string leaseId = null, string blobContentDisposition = null, string encryptionKey = null, string encryptionKeySha256 = null, EncryptionAlgorithmTypeInternal? encryptionAlgorithm = null, string encryptionScope = null, AccessTier? tier = null, DateTimeOffset? ifModifiedSince = null, DateTimeOffset? ifUnmodifiedSince = null, string ifMatch = null, string ifNoneMatch = null, string ifTags = null, string blobTagsString = null, DateTimeOffset? immutabilityPolicyExpiry = null, BlobImmutabilityPolicyMode? immutabilityPolicyMode = null, bool? legalHold = null, byte[] transactionalContentCrc64 = null, string structuredBodyType = null, long? structuredContentLength = null, CancellationToken cancellationToken = default) { if (body == null) { throw new ArgumentNullException(nameof(body)); } - using var message = CreateUploadRequest(contentLength, body, timeout, transactionalContentMD5, blobContentType, blobContentEncoding, blobContentLanguage, blobContentMD5, blobCacheControl, metadata, leaseId, blobContentDisposition, encryptionKey, encryptionKeySha256, encryptionAlgorithm, encryptionScope, tier, ifModifiedSince, ifUnmodifiedSince, ifMatch, ifNoneMatch, ifTags, blobTagsString, immutabilityPolicyExpiry, immutabilityPolicyMode, legalHold, transactionalContentCrc64); + using var message = CreateUploadRequest(contentLength, body, timeout, transactionalContentMD5, blobContentType, blobContentEncoding, blobContentLanguage, blobContentMD5, blobCacheControl, metadata, leaseId, blobContentDisposition, encryptionKey, encryptionKeySha256, encryptionAlgorithm, encryptionScope, tier, ifModifiedSince, ifUnmodifiedSince, ifMatch, ifNoneMatch, ifTags, blobTagsString, immutabilityPolicyExpiry, immutabilityPolicyMode, legalHold, transactionalContentCrc64, structuredBodyType, structuredContentLength); await _pipeline.SendAsync(message, cancellationToken).ConfigureAwait(false); var headers = new BlockBlobUploadHeaders(message.Response); switch (message.Response.Status) @@ -234,16 +244,18 @@ public async Task> UploadAsync(long /// Specifies the immutability policy mode to set on the blob. /// Specified if a legal hold should be set on the blob. /// Specify the transactional crc64 for the body, to be validated by the service. + /// Required if the request body is a structured message. Specifies the message schema version and properties. + /// Required if the request body is a structured message. Specifies the length of the blob/file content inside the message body. Will always be smaller than Content-Length. /// The cancellation token to use. /// is null. - public ResponseWithHeaders Upload(long contentLength, Stream body, int? timeout = null, byte[] transactionalContentMD5 = null, string blobContentType = null, string blobContentEncoding = null, string blobContentLanguage = null, byte[] blobContentMD5 = null, string blobCacheControl = null, IDictionary metadata = null, string leaseId = null, string blobContentDisposition = null, string encryptionKey = null, string encryptionKeySha256 = null, EncryptionAlgorithmTypeInternal? encryptionAlgorithm = null, string encryptionScope = null, AccessTier? tier = null, DateTimeOffset? ifModifiedSince = null, DateTimeOffset? ifUnmodifiedSince = null, string ifMatch = null, string ifNoneMatch = null, string ifTags = null, string blobTagsString = null, DateTimeOffset? immutabilityPolicyExpiry = null, BlobImmutabilityPolicyMode? immutabilityPolicyMode = null, bool? legalHold = null, byte[] transactionalContentCrc64 = null, CancellationToken cancellationToken = default) + public ResponseWithHeaders Upload(long contentLength, Stream body, int? timeout = null, byte[] transactionalContentMD5 = null, string blobContentType = null, string blobContentEncoding = null, string blobContentLanguage = null, byte[] blobContentMD5 = null, string blobCacheControl = null, IDictionary metadata = null, string leaseId = null, string blobContentDisposition = null, string encryptionKey = null, string encryptionKeySha256 = null, EncryptionAlgorithmTypeInternal? encryptionAlgorithm = null, string encryptionScope = null, AccessTier? tier = null, DateTimeOffset? ifModifiedSince = null, DateTimeOffset? ifUnmodifiedSince = null, string ifMatch = null, string ifNoneMatch = null, string ifTags = null, string blobTagsString = null, DateTimeOffset? immutabilityPolicyExpiry = null, BlobImmutabilityPolicyMode? immutabilityPolicyMode = null, bool? legalHold = null, byte[] transactionalContentCrc64 = null, string structuredBodyType = null, long? structuredContentLength = null, CancellationToken cancellationToken = default) { if (body == null) { throw new ArgumentNullException(nameof(body)); } - using var message = CreateUploadRequest(contentLength, body, timeout, transactionalContentMD5, blobContentType, blobContentEncoding, blobContentLanguage, blobContentMD5, blobCacheControl, metadata, leaseId, blobContentDisposition, encryptionKey, encryptionKeySha256, encryptionAlgorithm, encryptionScope, tier, ifModifiedSince, ifUnmodifiedSince, ifMatch, ifNoneMatch, ifTags, blobTagsString, immutabilityPolicyExpiry, immutabilityPolicyMode, legalHold, transactionalContentCrc64); + using var message = CreateUploadRequest(contentLength, body, timeout, transactionalContentMD5, blobContentType, blobContentEncoding, blobContentLanguage, blobContentMD5, blobCacheControl, metadata, leaseId, blobContentDisposition, encryptionKey, encryptionKeySha256, encryptionAlgorithm, encryptionScope, tier, ifModifiedSince, ifUnmodifiedSince, ifMatch, ifNoneMatch, ifTags, blobTagsString, immutabilityPolicyExpiry, immutabilityPolicyMode, legalHold, transactionalContentCrc64, structuredBodyType, structuredContentLength); _pipeline.Send(message, cancellationToken); var headers = new BlockBlobUploadHeaders(message.Response); switch (message.Response.Status) @@ -494,7 +506,7 @@ public ResponseWithHeaders PutBlobFromUrl(long c } } - internal HttpMessage CreateStageBlockRequest(string blockId, long contentLength, Stream body, byte[] transactionalContentMD5, byte[] transactionalContentCrc64, int? timeout, string leaseId, string encryptionKey, string encryptionKeySha256, EncryptionAlgorithmTypeInternal? encryptionAlgorithm, string encryptionScope) + internal HttpMessage CreateStageBlockRequest(string blockId, long contentLength, Stream body, byte[] transactionalContentMD5, byte[] transactionalContentCrc64, int? timeout, string leaseId, string encryptionKey, string encryptionKeySha256, EncryptionAlgorithmTypeInternal? encryptionAlgorithm, string encryptionScope, string structuredBodyType, long? structuredContentLength) { var message = _pipeline.CreateMessage(); var request = message.Request; @@ -533,6 +545,14 @@ internal HttpMessage CreateStageBlockRequest(string blockId, long contentLength, request.Headers.Add("x-ms-encryption-scope", encryptionScope); } request.Headers.Add("x-ms-version", _version); + if (structuredBodyType != null) + { + request.Headers.Add("x-ms-structured-body", structuredBodyType); + } + if (structuredContentLength != null) + { + request.Headers.Add("x-ms-structured-content-length", structuredContentLength.Value); + } request.Headers.Add("Accept", "application/xml"); request.Headers.Add("Content-Length", contentLength); if (transactionalContentMD5 != null) @@ -556,9 +576,11 @@ internal HttpMessage CreateStageBlockRequest(string blockId, long contentLength, /// The SHA-256 hash of the provided encryption key. Must be provided if the x-ms-encryption-key header is provided. /// The algorithm used to produce the encryption key hash. Currently, the only accepted value is "AES256". Must be provided if the x-ms-encryption-key header is provided. /// Optional. Version 2019-07-07 and later. Specifies the name of the encryption scope to use to encrypt the data provided in the request. If not specified, encryption is performed with the default account encryption scope. For more information, see Encryption at Rest for Azure Storage Services. + /// Required if the request body is a structured message. Specifies the message schema version and properties. + /// Required if the request body is a structured message. Specifies the length of the blob/file content inside the message body. Will always be smaller than Content-Length. /// The cancellation token to use. /// or is null. - public async Task> StageBlockAsync(string blockId, long contentLength, Stream body, byte[] transactionalContentMD5 = null, byte[] transactionalContentCrc64 = null, int? timeout = null, string leaseId = null, string encryptionKey = null, string encryptionKeySha256 = null, EncryptionAlgorithmTypeInternal? encryptionAlgorithm = null, string encryptionScope = null, CancellationToken cancellationToken = default) + public async Task> StageBlockAsync(string blockId, long contentLength, Stream body, byte[] transactionalContentMD5 = null, byte[] transactionalContentCrc64 = null, int? timeout = null, string leaseId = null, string encryptionKey = null, string encryptionKeySha256 = null, EncryptionAlgorithmTypeInternal? encryptionAlgorithm = null, string encryptionScope = null, string structuredBodyType = null, long? structuredContentLength = null, CancellationToken cancellationToken = default) { if (blockId == null) { @@ -569,7 +591,7 @@ public async Task> StageBlockAsy throw new ArgumentNullException(nameof(body)); } - using var message = CreateStageBlockRequest(blockId, contentLength, body, transactionalContentMD5, transactionalContentCrc64, timeout, leaseId, encryptionKey, encryptionKeySha256, encryptionAlgorithm, encryptionScope); + using var message = CreateStageBlockRequest(blockId, contentLength, body, transactionalContentMD5, transactionalContentCrc64, timeout, leaseId, encryptionKey, encryptionKeySha256, encryptionAlgorithm, encryptionScope, structuredBodyType, structuredContentLength); await _pipeline.SendAsync(message, cancellationToken).ConfigureAwait(false); var headers = new BlockBlobStageBlockHeaders(message.Response); switch (message.Response.Status) @@ -593,9 +615,11 @@ public async Task> StageBlockAsy /// The SHA-256 hash of the provided encryption key. Must be provided if the x-ms-encryption-key header is provided. /// The algorithm used to produce the encryption key hash. Currently, the only accepted value is "AES256". Must be provided if the x-ms-encryption-key header is provided. /// Optional. Version 2019-07-07 and later. Specifies the name of the encryption scope to use to encrypt the data provided in the request. If not specified, encryption is performed with the default account encryption scope. For more information, see Encryption at Rest for Azure Storage Services. + /// Required if the request body is a structured message. Specifies the message schema version and properties. + /// Required if the request body is a structured message. Specifies the length of the blob/file content inside the message body. Will always be smaller than Content-Length. /// The cancellation token to use. /// or is null. - public ResponseWithHeaders StageBlock(string blockId, long contentLength, Stream body, byte[] transactionalContentMD5 = null, byte[] transactionalContentCrc64 = null, int? timeout = null, string leaseId = null, string encryptionKey = null, string encryptionKeySha256 = null, EncryptionAlgorithmTypeInternal? encryptionAlgorithm = null, string encryptionScope = null, CancellationToken cancellationToken = default) + public ResponseWithHeaders StageBlock(string blockId, long contentLength, Stream body, byte[] transactionalContentMD5 = null, byte[] transactionalContentCrc64 = null, int? timeout = null, string leaseId = null, string encryptionKey = null, string encryptionKeySha256 = null, EncryptionAlgorithmTypeInternal? encryptionAlgorithm = null, string encryptionScope = null, string structuredBodyType = null, long? structuredContentLength = null, CancellationToken cancellationToken = default) { if (blockId == null) { @@ -606,7 +630,7 @@ public ResponseWithHeaders StageBlock(string blockId throw new ArgumentNullException(nameof(body)); } - using var message = CreateStageBlockRequest(blockId, contentLength, body, transactionalContentMD5, transactionalContentCrc64, timeout, leaseId, encryptionKey, encryptionKeySha256, encryptionAlgorithm, encryptionScope); + using var message = CreateStageBlockRequest(blockId, contentLength, body, transactionalContentMD5, transactionalContentCrc64, timeout, leaseId, encryptionKey, encryptionKeySha256, encryptionAlgorithm, encryptionScope, structuredBodyType, structuredContentLength); _pipeline.Send(message, cancellationToken); var headers = new BlockBlobStageBlockHeaders(message.Response); switch (message.Response.Status) diff --git a/sdk/storage/Azure.Storage.Blobs/src/Generated/BlockBlobStageBlockHeaders.cs b/sdk/storage/Azure.Storage.Blobs/src/Generated/BlockBlobStageBlockHeaders.cs index 7888b27dd7383..b13a3b7d1609a 100644 --- a/sdk/storage/Azure.Storage.Blobs/src/Generated/BlockBlobStageBlockHeaders.cs +++ b/sdk/storage/Azure.Storage.Blobs/src/Generated/BlockBlobStageBlockHeaders.cs @@ -29,5 +29,7 @@ public BlockBlobStageBlockHeaders(Response response) public string EncryptionKeySha256 => _response.Headers.TryGetValue("x-ms-encryption-key-sha256", out string value) ? value : null; /// Returns the name of the encryption scope used to encrypt the blob contents and application metadata. Note that the absence of this header implies use of the default account encryption scope. public string EncryptionScope => _response.Headers.TryGetValue("x-ms-encryption-scope", out string value) ? value : null; + /// Indicates the structured message body was accepted and mirrors back the message schema version and properties. + public string StructuredBodyType => _response.Headers.TryGetValue("x-ms-structured-body", out string value) ? value : null; } } diff --git a/sdk/storage/Azure.Storage.Blobs/src/Generated/BlockBlobUploadHeaders.cs b/sdk/storage/Azure.Storage.Blobs/src/Generated/BlockBlobUploadHeaders.cs index 1cfbd3924fa55..ca024b1fb5d84 100644 --- a/sdk/storage/Azure.Storage.Blobs/src/Generated/BlockBlobUploadHeaders.cs +++ b/sdk/storage/Azure.Storage.Blobs/src/Generated/BlockBlobUploadHeaders.cs @@ -31,5 +31,7 @@ public BlockBlobUploadHeaders(Response response) public string EncryptionKeySha256 => _response.Headers.TryGetValue("x-ms-encryption-key-sha256", out string value) ? value : null; /// Returns the name of the encryption scope used to encrypt the blob contents and application metadata. Note that the absence of this header implies use of the default account encryption scope. public string EncryptionScope => _response.Headers.TryGetValue("x-ms-encryption-scope", out string value) ? value : null; + /// Indicates the structured message body was accepted and mirrors back the message schema version and properties. + public string StructuredBodyType => _response.Headers.TryGetValue("x-ms-structured-body", out string value) ? value : null; } } diff --git a/sdk/storage/Azure.Storage.Blobs/src/Generated/ContainerRestClient.cs b/sdk/storage/Azure.Storage.Blobs/src/Generated/ContainerRestClient.cs index 024bfecd4e90b..9dd20ee7e1811 100644 --- a/sdk/storage/Azure.Storage.Blobs/src/Generated/ContainerRestClient.cs +++ b/sdk/storage/Azure.Storage.Blobs/src/Generated/ContainerRestClient.cs @@ -31,7 +31,7 @@ internal partial class ContainerRestClient /// The handler for diagnostic messaging in the client. /// The HTTP pipeline for sending and receiving REST requests and responses. /// The URL of the service account, container, or blob that is the target of the desired operation. - /// Specifies the version of the operation to use for this request. The default value is "2024-08-04". + /// Specifies the version of the operation to use for this request. The default value is "2025-01-05". /// , , or is null. public ContainerRestClient(ClientDiagnostics clientDiagnostics, HttpPipeline pipeline, string url, string version) { diff --git a/sdk/storage/Azure.Storage.Blobs/src/Generated/PageBlobRestClient.cs b/sdk/storage/Azure.Storage.Blobs/src/Generated/PageBlobRestClient.cs index 260d8021543e2..68a9e85b00d1b 100644 --- a/sdk/storage/Azure.Storage.Blobs/src/Generated/PageBlobRestClient.cs +++ b/sdk/storage/Azure.Storage.Blobs/src/Generated/PageBlobRestClient.cs @@ -30,7 +30,7 @@ internal partial class PageBlobRestClient /// The handler for diagnostic messaging in the client. /// The HTTP pipeline for sending and receiving REST requests and responses. /// The URL of the service account, container, or blob that is the target of the desired operation. - /// Specifies the version of the operation to use for this request. The default value is "2024-08-04". + /// Specifies the version of the operation to use for this request. The default value is "2025-01-05". /// , , or is null. public PageBlobRestClient(ClientDiagnostics clientDiagnostics, HttpPipeline pipeline, string url, string version) { @@ -235,7 +235,7 @@ public ResponseWithHeaders Create(long contentLength, lon } } - internal HttpMessage CreateUploadPagesRequest(long contentLength, Stream body, byte[] transactionalContentMD5, byte[] transactionalContentCrc64, int? timeout, string range, string leaseId, string encryptionKey, string encryptionKeySha256, EncryptionAlgorithmTypeInternal? encryptionAlgorithm, string encryptionScope, long? ifSequenceNumberLessThanOrEqualTo, long? ifSequenceNumberLessThan, long? ifSequenceNumberEqualTo, DateTimeOffset? ifModifiedSince, DateTimeOffset? ifUnmodifiedSince, string ifMatch, string ifNoneMatch, string ifTags) + internal HttpMessage CreateUploadPagesRequest(long contentLength, Stream body, byte[] transactionalContentMD5, byte[] transactionalContentCrc64, int? timeout, string range, string leaseId, string encryptionKey, string encryptionKeySha256, EncryptionAlgorithmTypeInternal? encryptionAlgorithm, string encryptionScope, long? ifSequenceNumberLessThanOrEqualTo, long? ifSequenceNumberLessThan, long? ifSequenceNumberEqualTo, DateTimeOffset? ifModifiedSince, DateTimeOffset? ifUnmodifiedSince, string ifMatch, string ifNoneMatch, string ifTags, string structuredBodyType, long? structuredContentLength) { var message = _pipeline.CreateMessage(); var request = message.Request; @@ -310,6 +310,14 @@ internal HttpMessage CreateUploadPagesRequest(long contentLength, Stream body, b request.Headers.Add("x-ms-if-tags", ifTags); } request.Headers.Add("x-ms-version", _version); + if (structuredBodyType != null) + { + request.Headers.Add("x-ms-structured-body", structuredBodyType); + } + if (structuredContentLength != null) + { + request.Headers.Add("x-ms-structured-content-length", structuredContentLength.Value); + } request.Headers.Add("Accept", "application/xml"); request.Headers.Add("Content-Length", contentLength); if (transactionalContentMD5 != null) @@ -341,16 +349,18 @@ internal HttpMessage CreateUploadPagesRequest(long contentLength, Stream body, b /// Specify an ETag value to operate only on blobs with a matching value. /// Specify an ETag value to operate only on blobs without a matching value. /// Specify a SQL where clause on blob tags to operate only on blobs with a matching value. + /// Required if the request body is a structured message. Specifies the message schema version and properties. + /// Required if the request body is a structured message. Specifies the length of the blob/file content inside the message body. Will always be smaller than Content-Length. /// The cancellation token to use. /// is null. - public async Task> UploadPagesAsync(long contentLength, Stream body, byte[] transactionalContentMD5 = null, byte[] transactionalContentCrc64 = null, int? timeout = null, string range = null, string leaseId = null, string encryptionKey = null, string encryptionKeySha256 = null, EncryptionAlgorithmTypeInternal? encryptionAlgorithm = null, string encryptionScope = null, long? ifSequenceNumberLessThanOrEqualTo = null, long? ifSequenceNumberLessThan = null, long? ifSequenceNumberEqualTo = null, DateTimeOffset? ifModifiedSince = null, DateTimeOffset? ifUnmodifiedSince = null, string ifMatch = null, string ifNoneMatch = null, string ifTags = null, CancellationToken cancellationToken = default) + public async Task> UploadPagesAsync(long contentLength, Stream body, byte[] transactionalContentMD5 = null, byte[] transactionalContentCrc64 = null, int? timeout = null, string range = null, string leaseId = null, string encryptionKey = null, string encryptionKeySha256 = null, EncryptionAlgorithmTypeInternal? encryptionAlgorithm = null, string encryptionScope = null, long? ifSequenceNumberLessThanOrEqualTo = null, long? ifSequenceNumberLessThan = null, long? ifSequenceNumberEqualTo = null, DateTimeOffset? ifModifiedSince = null, DateTimeOffset? ifUnmodifiedSince = null, string ifMatch = null, string ifNoneMatch = null, string ifTags = null, string structuredBodyType = null, long? structuredContentLength = null, CancellationToken cancellationToken = default) { if (body == null) { throw new ArgumentNullException(nameof(body)); } - using var message = CreateUploadPagesRequest(contentLength, body, transactionalContentMD5, transactionalContentCrc64, timeout, range, leaseId, encryptionKey, encryptionKeySha256, encryptionAlgorithm, encryptionScope, ifSequenceNumberLessThanOrEqualTo, ifSequenceNumberLessThan, ifSequenceNumberEqualTo, ifModifiedSince, ifUnmodifiedSince, ifMatch, ifNoneMatch, ifTags); + using var message = CreateUploadPagesRequest(contentLength, body, transactionalContentMD5, transactionalContentCrc64, timeout, range, leaseId, encryptionKey, encryptionKeySha256, encryptionAlgorithm, encryptionScope, ifSequenceNumberLessThanOrEqualTo, ifSequenceNumberLessThan, ifSequenceNumberEqualTo, ifModifiedSince, ifUnmodifiedSince, ifMatch, ifNoneMatch, ifTags, structuredBodyType, structuredContentLength); await _pipeline.SendAsync(message, cancellationToken).ConfigureAwait(false); var headers = new PageBlobUploadPagesHeaders(message.Response); switch (message.Response.Status) @@ -382,16 +392,18 @@ public async Task> UploadPagesAs /// Specify an ETag value to operate only on blobs with a matching value. /// Specify an ETag value to operate only on blobs without a matching value. /// Specify a SQL where clause on blob tags to operate only on blobs with a matching value. + /// Required if the request body is a structured message. Specifies the message schema version and properties. + /// Required if the request body is a structured message. Specifies the length of the blob/file content inside the message body. Will always be smaller than Content-Length. /// The cancellation token to use. /// is null. - public ResponseWithHeaders UploadPages(long contentLength, Stream body, byte[] transactionalContentMD5 = null, byte[] transactionalContentCrc64 = null, int? timeout = null, string range = null, string leaseId = null, string encryptionKey = null, string encryptionKeySha256 = null, EncryptionAlgorithmTypeInternal? encryptionAlgorithm = null, string encryptionScope = null, long? ifSequenceNumberLessThanOrEqualTo = null, long? ifSequenceNumberLessThan = null, long? ifSequenceNumberEqualTo = null, DateTimeOffset? ifModifiedSince = null, DateTimeOffset? ifUnmodifiedSince = null, string ifMatch = null, string ifNoneMatch = null, string ifTags = null, CancellationToken cancellationToken = default) + public ResponseWithHeaders UploadPages(long contentLength, Stream body, byte[] transactionalContentMD5 = null, byte[] transactionalContentCrc64 = null, int? timeout = null, string range = null, string leaseId = null, string encryptionKey = null, string encryptionKeySha256 = null, EncryptionAlgorithmTypeInternal? encryptionAlgorithm = null, string encryptionScope = null, long? ifSequenceNumberLessThanOrEqualTo = null, long? ifSequenceNumberLessThan = null, long? ifSequenceNumberEqualTo = null, DateTimeOffset? ifModifiedSince = null, DateTimeOffset? ifUnmodifiedSince = null, string ifMatch = null, string ifNoneMatch = null, string ifTags = null, string structuredBodyType = null, long? structuredContentLength = null, CancellationToken cancellationToken = default) { if (body == null) { throw new ArgumentNullException(nameof(body)); } - using var message = CreateUploadPagesRequest(contentLength, body, transactionalContentMD5, transactionalContentCrc64, timeout, range, leaseId, encryptionKey, encryptionKeySha256, encryptionAlgorithm, encryptionScope, ifSequenceNumberLessThanOrEqualTo, ifSequenceNumberLessThan, ifSequenceNumberEqualTo, ifModifiedSince, ifUnmodifiedSince, ifMatch, ifNoneMatch, ifTags); + using var message = CreateUploadPagesRequest(contentLength, body, transactionalContentMD5, transactionalContentCrc64, timeout, range, leaseId, encryptionKey, encryptionKeySha256, encryptionAlgorithm, encryptionScope, ifSequenceNumberLessThanOrEqualTo, ifSequenceNumberLessThan, ifSequenceNumberEqualTo, ifModifiedSince, ifUnmodifiedSince, ifMatch, ifNoneMatch, ifTags, structuredBodyType, structuredContentLength); _pipeline.Send(message, cancellationToken); var headers = new PageBlobUploadPagesHeaders(message.Response); switch (message.Response.Status) diff --git a/sdk/storage/Azure.Storage.Blobs/src/Generated/PageBlobUploadPagesHeaders.cs b/sdk/storage/Azure.Storage.Blobs/src/Generated/PageBlobUploadPagesHeaders.cs index 77d37d90027aa..c04659bc43322 100644 --- a/sdk/storage/Azure.Storage.Blobs/src/Generated/PageBlobUploadPagesHeaders.cs +++ b/sdk/storage/Azure.Storage.Blobs/src/Generated/PageBlobUploadPagesHeaders.cs @@ -33,5 +33,7 @@ public PageBlobUploadPagesHeaders(Response response) public string EncryptionKeySha256 => _response.Headers.TryGetValue("x-ms-encryption-key-sha256", out string value) ? value : null; /// Returns the name of the encryption scope used to encrypt the blob contents and application metadata. Note that the absence of this header implies use of the default account encryption scope. public string EncryptionScope => _response.Headers.TryGetValue("x-ms-encryption-scope", out string value) ? value : null; + /// Indicates the structured message body was accepted and mirrors back the message schema version and properties. + public string StructuredBodyType => _response.Headers.TryGetValue("x-ms-structured-body", out string value) ? value : null; } } diff --git a/sdk/storage/Azure.Storage.Blobs/src/Generated/ServiceRestClient.cs b/sdk/storage/Azure.Storage.Blobs/src/Generated/ServiceRestClient.cs index e274940f81e8d..2abac369c0cae 100644 --- a/sdk/storage/Azure.Storage.Blobs/src/Generated/ServiceRestClient.cs +++ b/sdk/storage/Azure.Storage.Blobs/src/Generated/ServiceRestClient.cs @@ -31,7 +31,7 @@ internal partial class ServiceRestClient /// The handler for diagnostic messaging in the client. /// The HTTP pipeline for sending and receiving REST requests and responses. /// The URL of the service account, container, or blob that is the target of the desired operation. - /// Specifies the version of the operation to use for this request. The default value is "2024-08-04". + /// Specifies the version of the operation to use for this request. The default value is "2025-01-05". /// , , or is null. public ServiceRestClient(ClientDiagnostics clientDiagnostics, HttpPipeline pipeline, string url, string version) { diff --git a/sdk/storage/Azure.Storage.Blobs/src/PageBlobClient.cs b/sdk/storage/Azure.Storage.Blobs/src/PageBlobClient.cs index fa575e41b8ebe..34befc6d4efe7 100644 --- a/sdk/storage/Azure.Storage.Blobs/src/PageBlobClient.cs +++ b/sdk/storage/Azure.Storage.Blobs/src/PageBlobClient.cs @@ -1363,15 +1363,39 @@ internal async Task> UploadPagesInternal( scope.Start(); Errors.VerifyStreamPosition(content, nameof(content)); - // compute hash BEFORE attaching progress handler - ContentHasher.GetHashResult hashResult = await ContentHasher.GetHashOrDefaultInternal( - content, - validationOptions, - async, - cancellationToken).ConfigureAwait(false); - - content = content?.WithNoDispose().WithProgress(progressHandler); - HttpRange range = new HttpRange(offset, (content?.Length - content?.Position) ?? null); + ContentHasher.GetHashResult hashResult = null; + long contentLength = (content?.Length - content?.Position) ?? 0; + long? structuredContentLength = default; + string structuredBodyType = null; + HttpRange range; + if (validationOptions != null && + validationOptions.ChecksumAlgorithm.ResolveAuto() == StorageChecksumAlgorithm.StorageCrc64 && + validationOptions.PrecalculatedChecksum.IsEmpty && + ClientSideEncryption == null) // don't allow feature combination + { + // report progress in terms of caller bytes, not encoded bytes + structuredContentLength = contentLength; + contentLength = (content?.Length - content?.Position) ?? 0; + range = new HttpRange(offset, (content?.Length - content?.Position) ?? null); + structuredBodyType = Constants.StructuredMessage.CrcStructuredMessage; + content = content?.WithNoDispose().WithProgress(progressHandler); + content = new StructuredMessageEncodingStream( + content, + Constants.StructuredMessage.DefaultSegmentContentLength, + StructuredMessage.Flags.StorageCrc64); + contentLength = (content?.Length - content?.Position) ?? 0; + } + else + { + // compute hash BEFORE attaching progress handler + hashResult = await ContentHasher.GetHashOrDefaultInternal( + content, + validationOptions, + async, + cancellationToken).ConfigureAwait(false); + content = content?.WithNoDispose().WithProgress(progressHandler); + range = new HttpRange(offset, (content?.Length - content?.Position) ?? null); + } ResponseWithHeaders response; @@ -1388,6 +1412,8 @@ internal async Task> UploadPagesInternal( encryptionKeySha256: ClientConfiguration.CustomerProvidedKey?.EncryptionKeyHash, encryptionAlgorithm: ClientConfiguration.CustomerProvidedKey?.EncryptionAlgorithm == null ? null : EncryptionAlgorithmTypeInternal.AES256, encryptionScope: ClientConfiguration.EncryptionScope, + structuredBodyType: structuredBodyType, + structuredContentLength: structuredContentLength, ifSequenceNumberLessThanOrEqualTo: conditions?.IfSequenceNumberLessThanOrEqual, ifSequenceNumberLessThan: conditions?.IfSequenceNumberLessThan, ifSequenceNumberEqualTo: conditions?.IfSequenceNumberEqual, @@ -1412,6 +1438,8 @@ internal async Task> UploadPagesInternal( encryptionKeySha256: ClientConfiguration.CustomerProvidedKey?.EncryptionKeyHash, encryptionAlgorithm: ClientConfiguration.CustomerProvidedKey?.EncryptionAlgorithm == null ? null : EncryptionAlgorithmTypeInternal.AES256, encryptionScope: ClientConfiguration.EncryptionScope, + structuredBodyType: structuredBodyType, + structuredContentLength: structuredContentLength, ifSequenceNumberLessThanOrEqualTo: conditions?.IfSequenceNumberLessThanOrEqual, ifSequenceNumberLessThan: conditions?.IfSequenceNumberLessThan, ifSequenceNumberEqualTo: conditions?.IfSequenceNumberEqual, diff --git a/sdk/storage/Azure.Storage.Blobs/src/autorest.md b/sdk/storage/Azure.Storage.Blobs/src/autorest.md index 85fb92c2349cd..34efb5857c4a4 100644 --- a/sdk/storage/Azure.Storage.Blobs/src/autorest.md +++ b/sdk/storage/Azure.Storage.Blobs/src/autorest.md @@ -4,7 +4,7 @@ Run `dotnet build /t:GenerateCode` to generate code. ``` yaml input-file: - - https://raw.githubusercontent.com/Azure/azure-rest-api-specs/f6f50c6388fd5836fa142384641b8353a99874ef/specification/storage/data-plane/Microsoft.BlobStorage/stable/2024-08-04/blob.json + - https://github.com/Azure/azure-rest-api-specs/blob/794c6178bc06c6c9dceb139e9f9d1b35b1a99701/specification/storage/data-plane/Microsoft.BlobStorage/preview/2025-01-05/blob.json generation1-convenience-client: true # https://github.com/Azure/autorest/issues/4075 skip-semantics-validation: true @@ -34,7 +34,7 @@ directive: if (property.includes('/{containerName}/{blob}')) { $[property]["parameters"] = $[property]["parameters"].filter(function(param) { return (typeof param['$ref'] === "undefined") || (false == param['$ref'].endsWith("#/parameters/ContainerName") && false == param['$ref'].endsWith("#/parameters/Blob"))}); - } + } else if (property.includes('/{containerName}')) { $[property]["parameters"] = $[property]["parameters"].filter(function(param) { return (typeof param['$ref'] === "undefined") || (false == param['$ref'].endsWith("#/parameters/ContainerName"))}); @@ -158,7 +158,7 @@ directive: var newName = property.replace('/{containerName}/{blob}', ''); $[newName] = $[oldName]; delete $[oldName]; - } + } else if (property.includes('/{containerName}')) { var oldName = property; diff --git a/sdk/storage/Azure.Storage.Blobs/tests/Azure.Storage.Blobs.Tests.csproj b/sdk/storage/Azure.Storage.Blobs/tests/Azure.Storage.Blobs.Tests.csproj index 62c7b6d17e63e..1c3856c83b64e 100644 --- a/sdk/storage/Azure.Storage.Blobs/tests/Azure.Storage.Blobs.Tests.csproj +++ b/sdk/storage/Azure.Storage.Blobs/tests/Azure.Storage.Blobs.Tests.csproj @@ -6,6 +6,9 @@ Microsoft Azure.Storage.Blobs client library tests false + + BlobSDK + diff --git a/sdk/storage/Azure.Storage.Blobs/tests/BlobBaseClientTransferValidationTests.cs b/sdk/storage/Azure.Storage.Blobs/tests/BlobBaseClientTransferValidationTests.cs index 73d11612f1d8c..83e302902f1d5 100644 --- a/sdk/storage/Azure.Storage.Blobs/tests/BlobBaseClientTransferValidationTests.cs +++ b/sdk/storage/Azure.Storage.Blobs/tests/BlobBaseClientTransferValidationTests.cs @@ -37,7 +37,10 @@ protected override async Task> GetDispo StorageChecksumAlgorithm uploadAlgorithm = StorageChecksumAlgorithm.None, StorageChecksumAlgorithm downloadAlgorithm = StorageChecksumAlgorithm.None) { - var disposingContainer = await ClientBuilder.GetTestContainerAsync(service: service, containerName: containerName); + var disposingContainer = await ClientBuilder.GetTestContainerAsync( + service: service, + containerName: containerName, + publicAccessType: PublicAccessType.None); disposingContainer.Container.ClientConfiguration.TransferValidation.Upload.ChecksumAlgorithm = uploadAlgorithm; disposingContainer.Container.ClientConfiguration.TransferValidation.Download.ChecksumAlgorithm = downloadAlgorithm; diff --git a/sdk/storage/Azure.Storage.Common/src/Shared/Constants.cs b/sdk/storage/Azure.Storage.Common/src/Shared/Constants.cs index 17a32b2d46d41..17f02b3596735 100644 --- a/sdk/storage/Azure.Storage.Common/src/Shared/Constants.cs +++ b/sdk/storage/Azure.Storage.Common/src/Shared/Constants.cs @@ -665,6 +665,12 @@ internal static class AccountResources internal static readonly int[] PathStylePorts = { 10000, 10001, 10002, 10003, 10004, 10100, 10101, 10102, 10103, 10104, 11000, 11001, 11002, 11003, 11004, 11100, 11101, 11102, 11103, 11104 }; } + internal static class StructuredMessage + { + public const string CrcStructuredMessage = "XSM/1.0; properties=crc64"; + public const int DefaultSegmentContentLength = 4 * MB; + } + internal static class ClientSideEncryption { public const string HttpMessagePropertyKeyV1 = "Azure.Storage.StorageTelemetryPolicy.ClientSideEncryption.V1"; diff --git a/sdk/storage/Azure.Storage.Common/src/Shared/StructuredMessage.cs b/sdk/storage/Azure.Storage.Common/src/Shared/StructuredMessage.cs index 5e31dd4ac0ed8..1757442037f92 100644 --- a/sdk/storage/Azure.Storage.Common/src/Shared/StructuredMessage.cs +++ b/sdk/storage/Azure.Storage.Common/src/Shared/StructuredMessage.cs @@ -5,7 +5,7 @@ using System.Buffers; using System.Buffers.Binary; using System.IO; -using Azure.Core; +using Azure.Storage.Common; namespace Azure.Storage.Shared; diff --git a/sdk/storage/Azure.Storage.Common/src/Shared/StructuredMessageDecodingStream.cs b/sdk/storage/Azure.Storage.Common/src/Shared/StructuredMessageDecodingStream.cs index aa94b8df350d2..db5b58ac74189 100644 --- a/sdk/storage/Azure.Storage.Common/src/Shared/StructuredMessageDecodingStream.cs +++ b/sdk/storage/Azure.Storage.Common/src/Shared/StructuredMessageDecodingStream.cs @@ -8,7 +8,7 @@ using System.Linq; using System.Threading; using System.Threading.Tasks; -using Azure.Core; +using Azure.Storage.Common; namespace Azure.Storage.Shared; diff --git a/sdk/storage/Azure.Storage.Common/src/Shared/StructuredMessageEncodingStream.cs b/sdk/storage/Azure.Storage.Common/src/Shared/StructuredMessageEncodingStream.cs index 935b830ec5d7f..cb0ef340155ec 100644 --- a/sdk/storage/Azure.Storage.Common/src/Shared/StructuredMessageEncodingStream.cs +++ b/sdk/storage/Azure.Storage.Common/src/Shared/StructuredMessageEncodingStream.cs @@ -6,8 +6,8 @@ using System.IO; using System.Threading; using System.Threading.Tasks; -using Azure.Core; using Azure.Core.Pipeline; +using Azure.Storage.Common; namespace Azure.Storage.Shared; diff --git a/sdk/storage/Azure.Storage.Common/tests/Shared/TransferValidationTestBase.cs b/sdk/storage/Azure.Storage.Common/tests/Shared/TransferValidationTestBase.cs index c18492d2fb4dd..0923fe27d439f 100644 --- a/sdk/storage/Azure.Storage.Common/tests/Shared/TransferValidationTestBase.cs +++ b/sdk/storage/Azure.Storage.Common/tests/Shared/TransferValidationTestBase.cs @@ -8,7 +8,7 @@ using System.Threading.Tasks; using Azure.Core; using Azure.Core.TestFramework; -using FastSerialization; +using Azure.Storage.Shared; using NUnit.Framework; namespace Azure.Storage.Test.Shared @@ -190,7 +190,7 @@ protected string GetNewResourceName() /// The actual checksum value expected to be on the request, if known. Defaults to no specific value expected or checked. /// /// An assertion to put into a pipeline policy. - internal static Action GetRequestChecksumAssertion(StorageChecksumAlgorithm algorithm, Func isChecksumExpected = default, byte[] expectedChecksum = default) + internal static Action GetRequestChecksumHeaderAssertion(StorageChecksumAlgorithm algorithm, Func isChecksumExpected = default, byte[] expectedChecksum = default) { // action to assert a request header is as expected void AssertChecksum(RequestHeaders headers, string headerName) @@ -225,11 +225,38 @@ void AssertChecksum(RequestHeaders headers, string headerName) AssertChecksum(request.Headers, "x-ms-content-crc64"); break; default: - throw new Exception($"Bad {nameof(StorageChecksumAlgorithm)} provided to {nameof(GetRequestChecksumAssertion)}."); + throw new Exception($"Bad {nameof(StorageChecksumAlgorithm)} provided to {nameof(GetRequestChecksumHeaderAssertion)}."); } }; } +#if BlobSDK + internal static Action GetRequestStructuredMessageAssertion( + StructuredMessage.Flags flags, + Func isStructuredMessageExpected = default, + long? structuredContentSegmentLength = default) + { + return request => + { + // filter some requests out with predicate + if (isStructuredMessageExpected != default && !isStructuredMessageExpected(request)) + { + return; + } + + Assert.That(request.Headers.TryGetValue("x-ms-structured-body", out string structuredBody)); + Assert.That(structuredBody, Does.Contain("XSM/1.0")); + if (flags.HasFlag(StructuredMessage.Flags.StorageCrc64)) + { + Assert.That(structuredBody, Does.Contain("crc64")); + } + + Assert.That(request.Headers.TryGetValue("Content-Length", out string contentLength)); + Assert.That(request.Headers.TryGetValue("x-ms-structured-content-length", out string structuredContentLength)); + }; + } +#endif + /// /// Gets an assertion as to whether a transactional checksum appeared on a returned response. /// Meant to be injected into a pipeline. @@ -281,7 +308,7 @@ void AssertChecksum(ResponseHeaders headers, string headerName) AssertChecksum(response.Headers, "x-ms-content-crc64"); break; default: - throw new Exception($"Bad {nameof(StorageChecksumAlgorithm)} provided to {nameof(GetRequestChecksumAssertion)}."); + throw new Exception($"Bad {nameof(StorageChecksumAlgorithm)} provided to {nameof(GetRequestChecksumHeaderAssertion)}."); } }; } @@ -291,19 +318,29 @@ void AssertChecksum(ResponseHeaders headers, string headerName) /// /// Async action to upload data to service. /// Checksum algorithm used. - internal static void AssertWriteChecksumMismatch(AsyncTestDelegate writeAction, StorageChecksumAlgorithm algorithm) + internal static void AssertWriteChecksumMismatch( + AsyncTestDelegate writeAction, + StorageChecksumAlgorithm algorithm, + bool expectStructuredMessage = false) { var exception = ThrowsOrInconclusiveAsync(writeAction); - switch (algorithm.ResolveAuto()) + if (expectStructuredMessage) { - case StorageChecksumAlgorithm.MD5: - Assert.AreEqual("Md5Mismatch", exception.ErrorCode); - break; - case StorageChecksumAlgorithm.StorageCrc64: - Assert.AreEqual("Crc64Mismatch", exception.ErrorCode); - break; - default: - throw new ArgumentException("Test arguments contain bad algorithm specifier."); + Assert.That(exception.ErrorCode, Is.EqualTo("InvalidStructuredMessage")); + } + else + { + switch (algorithm.ResolveAuto()) + { + case StorageChecksumAlgorithm.MD5: + Assert.That(exception.ErrorCode, Is.EqualTo("Md5Mismatch")); + break; + case StorageChecksumAlgorithm.StorageCrc64: + Assert.That(exception.ErrorCode, Is.EqualTo("Crc64Mismatch")); + break; + default: + throw new ArgumentException("Test arguments contain bad algorithm specifier."); + } } } #endregion @@ -348,6 +385,7 @@ public virtual async Task UploadPartitionSuccessfulHashComputation(StorageChecks await using IDisposingContainer disposingContainer = await GetDisposingContainerAsync(); // Arrange + bool expectStructuredMessage = algorithm.ResolveAuto() == StorageChecksumAlgorithm.StorageCrc64; const int dataLength = Constants.KB; var data = GetRandomBuffer(dataLength); var validationOptions = new UploadTransferValidationOptions @@ -356,7 +394,14 @@ public virtual async Task UploadPartitionSuccessfulHashComputation(StorageChecks }; // make pipeline assertion for checking checksum was present on upload - var checksumPipelineAssertion = new AssertMessageContentsPolicy(checkRequest: GetRequestChecksumAssertion(algorithm)); +#if BlobSDK + var assertion = algorithm.ResolveAuto() == StorageChecksumAlgorithm.StorageCrc64 + ? GetRequestStructuredMessageAssertion(StructuredMessage.Flags.StorageCrc64, null, dataLength) + : GetRequestChecksumHeaderAssertion(algorithm); +#else + var assertion = GetRequestChecksumHeaderAssertion(algorithm); +#endif + var checksumPipelineAssertion = new AssertMessageContentsPolicy(checkRequest: assertion); var clientOptions = ClientBuilder.GetOptions(); clientOptions.AddPolicy(checksumPipelineAssertion, HttpPipelinePosition.PerCall); @@ -406,7 +451,9 @@ public virtual async Task UploadPartitionUsePrecalculatedHash(StorageChecksumAlg }; // make pipeline assertion for checking precalculated checksum was present on upload - var checksumPipelineAssertion = new AssertMessageContentsPolicy(checkRequest: GetRequestChecksumAssertion(algorithm, expectedChecksum: precalculatedChecksum)); + // precalculated partition upload will never use structured message. always check header + var assertion = GetRequestChecksumHeaderAssertion(algorithm, expectedChecksum: precalculatedChecksum); + var checksumPipelineAssertion = new AssertMessageContentsPolicy(checkRequest: assertion); var clientOptions = ClientBuilder.GetOptions(); clientOptions.AddPolicy(checksumPipelineAssertion, HttpPipelinePosition.PerCall); @@ -428,7 +475,7 @@ public virtual async Task UploadPartitionUsePrecalculatedHash(StorageChecksumAlg } [TestCaseSource(nameof(GetValidationAlgorithms))] - public virtual async Task UploadPartitionMismatchedHashThrows(StorageChecksumAlgorithm algorithm) + public virtual async Task UploadPartitionTamperedStreamThrows(StorageChecksumAlgorithm algorithm) { await using IDisposingContainer disposingContainer = await GetDisposingContainerAsync(); @@ -458,7 +505,12 @@ public virtual async Task UploadPartitionMismatchedHashThrows(StorageChecksumAlg AsyncTestDelegate operation = async () => await UploadPartitionAsync(client, stream, validationOptions); // Assert +#if BlobSDK + AssertWriteChecksumMismatch(operation, algorithm, + expectStructuredMessage: algorithm.ResolveAuto() == StorageChecksumAlgorithm.StorageCrc64); +#else AssertWriteChecksumMismatch(operation, algorithm); +#endif } } @@ -473,7 +525,14 @@ public virtual async Task UploadPartitionUsesDefaultClientValidationOptions( var data = GetRandomBuffer(dataLength); // make pipeline assertion for checking checksum was present on upload - var checksumPipelineAssertion = new AssertMessageContentsPolicy(checkRequest: GetRequestChecksumAssertion(clientAlgorithm)); +#if BlobSDK + var assertion = clientAlgorithm.ResolveAuto() == StorageChecksumAlgorithm.StorageCrc64 + ? GetRequestStructuredMessageAssertion(StructuredMessage.Flags.StorageCrc64, null, dataLength) + : GetRequestChecksumHeaderAssertion(clientAlgorithm); +#else + var assertion = GetRequestChecksumHeaderAssertion(clientAlgorithm); +#endif + var checksumPipelineAssertion = new AssertMessageContentsPolicy(checkRequest: assertion); var clientOptions = ClientBuilder.GetOptions(); clientOptions.AddPolicy(checksumPipelineAssertion, HttpPipelinePosition.PerCall); @@ -512,7 +571,14 @@ public virtual async Task UploadPartitionOverwritesDefaultClientValidationOption }; // make pipeline assertion for checking checksum was present on upload - var checksumPipelineAssertion = new AssertMessageContentsPolicy(checkRequest: GetRequestChecksumAssertion(overrideAlgorithm)); +#if BlobSDK + var assertion = overrideAlgorithm.ResolveAuto() == StorageChecksumAlgorithm.StorageCrc64 + ? GetRequestStructuredMessageAssertion(StructuredMessage.Flags.StorageCrc64, null, dataLength) + : GetRequestChecksumHeaderAssertion(overrideAlgorithm); +#else + var assertion = GetRequestChecksumHeaderAssertion(overrideAlgorithm); +#endif + var checksumPipelineAssertion = new AssertMessageContentsPolicy(checkRequest: assertion); var clientOptions = ClientBuilder.GetOptions(); clientOptions.AddPolicy(checksumPipelineAssertion, HttpPipelinePosition.PerCall); @@ -559,6 +625,10 @@ public virtual async Task UploadPartitionDisablesDefaultClientValidationOptions( { Assert.Fail($"Hash found when none expected."); } + if (request.Headers.Contains("x-ms-structured-body")) + { + Assert.Fail($"Structured body used when none expected."); + } }); var clientOptions = ClientBuilder.GetOptions(); clientOptions.AddPolicy(checksumPipelineAssertion, HttpPipelinePosition.PerCall); @@ -601,7 +671,7 @@ public virtual async Task OpenWriteSuccessfulHashComputation( }; // make pipeline assertion for checking checksum was present on upload - var checksumPipelineAssertion = new AssertMessageContentsPolicy(checkRequest: GetRequestChecksumAssertion(algorithm)); + var checksumPipelineAssertion = new AssertMessageContentsPolicy(checkRequest: GetRequestChecksumHeaderAssertion(algorithm)); var clientOptions = ClientBuilder.GetOptions(); clientOptions.AddPolicy(checksumPipelineAssertion, HttpPipelinePosition.PerCall); @@ -682,7 +752,7 @@ public virtual async Task OpenWriteUsesDefaultClientValidationOptions( var data = GetRandomBuffer(dataLength); // make pipeline assertion for checking checksum was present on upload - var checksumPipelineAssertion = new AssertMessageContentsPolicy(checkRequest: GetRequestChecksumAssertion(clientAlgorithm)); + var checksumPipelineAssertion = new AssertMessageContentsPolicy(checkRequest: GetRequestChecksumHeaderAssertion(clientAlgorithm)); var clientOptions = ClientBuilder.GetOptions(); clientOptions.AddPolicy(checksumPipelineAssertion, HttpPipelinePosition.PerCall); @@ -726,7 +796,7 @@ public virtual async Task OpenWriteOverwritesDefaultClientValidationOptions( }; // make pipeline assertion for checking checksum was present on upload - var checksumPipelineAssertion = new AssertMessageContentsPolicy(checkRequest: GetRequestChecksumAssertion(overrideAlgorithm)); + var checksumPipelineAssertion = new AssertMessageContentsPolicy(checkRequest: GetRequestChecksumHeaderAssertion(overrideAlgorithm)); var clientOptions = ClientBuilder.GetOptions(); clientOptions.AddPolicy(checksumPipelineAssertion, HttpPipelinePosition.PerCall); @@ -886,7 +956,7 @@ public virtual async Task ParallelUploadSplitSuccessfulHashComputation(StorageCh // make pipeline assertion for checking checksum was present on upload var checksumPipelineAssertion = new AssertMessageContentsPolicy( - checkRequest: GetRequestChecksumAssertion(algorithm, isChecksumExpected: ParallelUploadIsChecksumExpected)); + checkRequest: GetRequestChecksumHeaderAssertion(algorithm, isChecksumExpected: ParallelUploadIsChecksumExpected)); var clientOptions = ClientBuilder.GetOptions(); clientOptions.AddPolicy(checksumPipelineAssertion, HttpPipelinePosition.PerCall); @@ -924,7 +994,7 @@ public virtual async Task ParallelUploadOneShotSuccessfulHashComputation(Storage // make pipeline assertion for checking checksum was present on upload var checksumPipelineAssertion = new AssertMessageContentsPolicy( - checkRequest: GetRequestChecksumAssertion(algorithm, isChecksumExpected: ParallelUploadIsChecksumExpected)); + checkRequest: GetRequestChecksumHeaderAssertion(algorithm, isChecksumExpected: ParallelUploadIsChecksumExpected)); var clientOptions = ClientBuilder.GetOptions(); clientOptions.AddPolicy(checksumPipelineAssertion, HttpPipelinePosition.PerCall); @@ -1011,7 +1081,7 @@ public virtual async Task ParallelUploadUsesDefaultClientValidationOptions( }; // make pipeline assertion for checking checksum was present on upload - var checksumPipelineAssertion = new AssertMessageContentsPolicy(checkRequest: GetRequestChecksumAssertion( + var checksumPipelineAssertion = new AssertMessageContentsPolicy(checkRequest: GetRequestChecksumHeaderAssertion( clientAlgorithm, isChecksumExpected: ParallelUploadIsChecksumExpected)); var clientOptions = ClientBuilder.GetOptions(); clientOptions.AddPolicy(checksumPipelineAssertion, HttpPipelinePosition.PerCall); @@ -1063,7 +1133,7 @@ public virtual async Task ParallelUploadOverwritesDefaultClientValidationOptions }; // make pipeline assertion for checking checksum was present on upload - var checksumPipelineAssertion = new AssertMessageContentsPolicy(checkRequest: GetRequestChecksumAssertion( + var checksumPipelineAssertion = new AssertMessageContentsPolicy(checkRequest: GetRequestChecksumHeaderAssertion( overrideAlgorithm, isChecksumExpected: ParallelUploadIsChecksumExpected)); var clientOptions = ClientBuilder.GetOptions(); clientOptions.AddPolicy(checksumPipelineAssertion, HttpPipelinePosition.PerCall); @@ -1891,7 +1961,7 @@ public async Task RoundtripWIthDefaults() // make pipeline assertion for checking checksum was present on upload AND download var checksumPipelineAssertion = new AssertMessageContentsPolicy( - checkRequest: GetRequestChecksumAssertion(expectedAlgorithm, isChecksumExpected: ParallelUploadIsChecksumExpected), + checkRequest: GetRequestChecksumHeaderAssertion(expectedAlgorithm, isChecksumExpected: ParallelUploadIsChecksumExpected), checkResponse: GetResponseChecksumAssertion(expectedAlgorithm)); clientOptions.AddPolicy(checksumPipelineAssertion, HttpPipelinePosition.PerCall); diff --git a/sdk/storage/Azure.Storage.DataMovement/src/Azure.Storage.DataMovement.csproj b/sdk/storage/Azure.Storage.DataMovement/src/Azure.Storage.DataMovement.csproj index 5aaf548493b15..dd30659cf0a5d 100644 --- a/sdk/storage/Azure.Storage.DataMovement/src/Azure.Storage.DataMovement.csproj +++ b/sdk/storage/Azure.Storage.DataMovement/src/Azure.Storage.DataMovement.csproj @@ -1,4 +1,4 @@ - + $(RequiredTargetFrameworks);net6.0 diff --git a/sdk/storage/Azure.Storage.Files.DataLake/assets.json b/sdk/storage/Azure.Storage.Files.DataLake/assets.json index 442889d04be63..39ee762ad9a8a 100644 --- a/sdk/storage/Azure.Storage.Files.DataLake/assets.json +++ b/sdk/storage/Azure.Storage.Files.DataLake/assets.json @@ -2,5 +2,5 @@ "AssetsRepo": "Azure/azure-sdk-assets", "AssetsRepoPrefixPath": "net", "TagPrefix": "net/storage/Azure.Storage.Files.DataLake", - "Tag": "net/storage/Azure.Storage.Files.DataLake_186c14971d" + "Tag": "net/storage/Azure.Storage.Files.DataLake_6d44446b20" } diff --git a/sdk/storage/Azure.Storage.Files.DataLake/src/Azure.Storage.Files.DataLake.csproj b/sdk/storage/Azure.Storage.Files.DataLake/src/Azure.Storage.Files.DataLake.csproj index 7adb79645b0a9..b5ffe9a99116c 100644 --- a/sdk/storage/Azure.Storage.Files.DataLake/src/Azure.Storage.Files.DataLake.csproj +++ b/sdk/storage/Azure.Storage.Files.DataLake/src/Azure.Storage.Files.DataLake.csproj @@ -81,6 +81,7 @@ + diff --git a/sdk/storage/Azure.Storage.Files.DataLake/tests/DataLakeFileClientTransferValidationTests.cs b/sdk/storage/Azure.Storage.Files.DataLake/tests/DataLakeFileClientTransferValidationTests.cs index 4bdefdbf756cd..5067f98517bd2 100644 --- a/sdk/storage/Azure.Storage.Files.DataLake/tests/DataLakeFileClientTransferValidationTests.cs +++ b/sdk/storage/Azure.Storage.Files.DataLake/tests/DataLakeFileClientTransferValidationTests.cs @@ -34,7 +34,10 @@ protected override async Task> Get StorageChecksumAlgorithm uploadAlgorithm = StorageChecksumAlgorithm.None, StorageChecksumAlgorithm downloadAlgorithm = StorageChecksumAlgorithm.None) { - var disposingFileSystem = await ClientBuilder.GetNewFileSystem(service: service, fileSystemName: containerName); + var disposingFileSystem = await ClientBuilder.GetNewFileSystem( + service: service, + fileSystemName: containerName, + publicAccessType: PublicAccessType.None); disposingFileSystem.FileSystem.ClientConfiguration.TransferValidation.Upload.ChecksumAlgorithm = uploadAlgorithm; disposingFileSystem.FileSystem.ClientConfiguration.TransferValidation.Download.ChecksumAlgorithm = downloadAlgorithm; diff --git a/sdk/storage/Azure.Storage.Files.Shares/assets.json b/sdk/storage/Azure.Storage.Files.Shares/assets.json index 9ca749681b79e..d46664532748f 100644 --- a/sdk/storage/Azure.Storage.Files.Shares/assets.json +++ b/sdk/storage/Azure.Storage.Files.Shares/assets.json @@ -2,5 +2,5 @@ "AssetsRepo": "Azure/azure-sdk-assets", "AssetsRepoPrefixPath": "net", "TagPrefix": "net/storage/Azure.Storage.Files.Shares", - "Tag": "net/storage/Azure.Storage.Files.Shares_14e0fa0c22" + "Tag": "net/storage/Azure.Storage.Files.Shares_997e3d57ce" } diff --git a/sdk/storage/Azure.Storage.Files.Shares/src/Azure.Storage.Files.Shares.csproj b/sdk/storage/Azure.Storage.Files.Shares/src/Azure.Storage.Files.Shares.csproj index 60f6f200fd402..4b568316d32a2 100644 --- a/sdk/storage/Azure.Storage.Files.Shares/src/Azure.Storage.Files.Shares.csproj +++ b/sdk/storage/Azure.Storage.Files.Shares/src/Azure.Storage.Files.Shares.csproj @@ -85,6 +85,7 @@ + From 73d5345e2a4b352fa592af84b6469da91c23271e Mon Sep 17 00:00:00 2001 From: Jocelyn <41338290+jaschrep-msft@users.noreply.github.com> Date: Tue, 26 Mar 2024 16:05:04 -0400 Subject: [PATCH 07/22] GET Blob Structured Message (#42959) * download range structured message * testproxy * throw when service fails to give back structured message * test * testproxy --- sdk/storage/Azure.Storage.Blobs/assets.json | 2 +- .../Azure.Storage.Blobs/src/BlobBaseClient.cs | 56 ++++++++++-- .../BlobBaseClientTransferValidationTests.cs | 31 +++++++ .../src/Shared/Constants.cs | 3 + .../src/Shared/Errors.Clients.cs | 4 + .../Shared/TransferValidationTestBase.cs | 86 +++++++++++++++++++ 6 files changed, 175 insertions(+), 7 deletions(-) diff --git a/sdk/storage/Azure.Storage.Blobs/assets.json b/sdk/storage/Azure.Storage.Blobs/assets.json index 3de365ac947fb..700e2c053c45b 100644 --- a/sdk/storage/Azure.Storage.Blobs/assets.json +++ b/sdk/storage/Azure.Storage.Blobs/assets.json @@ -2,5 +2,5 @@ "AssetsRepo": "Azure/azure-sdk-assets", "AssetsRepoPrefixPath": "net", "TagPrefix": "net/storage/Azure.Storage.Blobs", - "Tag": "net/storage/Azure.Storage.Blobs_5d0f81abfc" + "Tag": "net/storage/Azure.Storage.Blobs_efe2c4ee4f" } diff --git a/sdk/storage/Azure.Storage.Blobs/src/BlobBaseClient.cs b/sdk/storage/Azure.Storage.Blobs/src/BlobBaseClient.cs index c1416524f0221..a0fc9a45dc47c 100644 --- a/sdk/storage/Azure.Storage.Blobs/src/BlobBaseClient.cs +++ b/sdk/storage/Azure.Storage.Blobs/src/BlobBaseClient.cs @@ -1578,7 +1578,11 @@ internal virtual async ValueTask> Download * Buffer response stream and ensure it matches the transactional checksum if any. * Storage will not return a checksum for payload >4MB, so this buffer is capped similarly. * Checksum validation is opt-in, so this buffer is part of that opt-in. */ - if (validationOptions != default && validationOptions.ChecksumAlgorithm != StorageChecksumAlgorithm.None && validationOptions.AutoValidateChecksum) + if (validationOptions != default && + validationOptions.ChecksumAlgorithm != StorageChecksumAlgorithm.None && + validationOptions.AutoValidateChecksum && + // structured message decoding does the validation for us + !response.GetRawResponse().Headers.Contains(Constants.StructuredMessage.CrcStructuredMessageHeader)) { // safe-buffer; transactional hash download limit well below maxInt var readDestStream = new MemoryStream((int)response.Value.Details.ContentLength); @@ -1689,13 +1693,36 @@ private async ValueTask> StartDownloadAsyn operationName: nameof(BlobBaseClient.Download), parameterName: nameof(conditions)); + bool? rangeGetContentMD5 = null; + bool? rangeGetContentCRC64 = null; + string structuredBodyType = null; + switch (validationOptions?.ChecksumAlgorithm.ResolveAuto()) + { + case StorageChecksumAlgorithm.MD5: + rangeGetContentMD5 = true; + break; + case StorageChecksumAlgorithm.StorageCrc64: + if (pageRange?.Length <= Constants.StructuredMessage.MaxDownloadCrcWithHeader) + { + rangeGetContentCRC64 = true; + } + else + { + structuredBodyType = Constants.StructuredMessage.CrcStructuredMessage; + } + break; + default: + break; + } + if (async) { response = await BlobRestClient.DownloadAsync( range: pageRange?.ToString(), leaseId: conditions?.LeaseId, - rangeGetContentMD5: validationOptions?.ChecksumAlgorithm.ResolveAuto() == StorageChecksumAlgorithm.MD5 ? true : null, - rangeGetContentCRC64: validationOptions?.ChecksumAlgorithm.ResolveAuto() == StorageChecksumAlgorithm.StorageCrc64 ? true : null, + rangeGetContentMD5: rangeGetContentMD5, + rangeGetContentCRC64: rangeGetContentCRC64, + structuredBodyType: structuredBodyType, encryptionKey: ClientConfiguration.CustomerProvidedKey?.EncryptionKey, encryptionKeySha256: ClientConfiguration.CustomerProvidedKey?.EncryptionKeyHash, encryptionAlgorithm: ClientConfiguration.CustomerProvidedKey?.EncryptionAlgorithm == null ? null : EncryptionAlgorithmTypeInternal.AES256, @@ -1712,8 +1739,9 @@ private async ValueTask> StartDownloadAsyn response = BlobRestClient.Download( range: pageRange?.ToString(), leaseId: conditions?.LeaseId, - rangeGetContentMD5: validationOptions?.ChecksumAlgorithm.ResolveAuto() == StorageChecksumAlgorithm.MD5 ? true : null, - rangeGetContentCRC64: validationOptions?.ChecksumAlgorithm.ResolveAuto() == StorageChecksumAlgorithm.StorageCrc64 ? true : null, + rangeGetContentMD5: rangeGetContentMD5, + rangeGetContentCRC64: rangeGetContentCRC64, + structuredBodyType: structuredBodyType, encryptionKey: ClientConfiguration.CustomerProvidedKey?.EncryptionKey, encryptionKeySha256: ClientConfiguration.CustomerProvidedKey?.EncryptionKeyHash, encryptionAlgorithm: ClientConfiguration.CustomerProvidedKey?.EncryptionAlgorithm == null ? null : EncryptionAlgorithmTypeInternal.AES256, @@ -1729,8 +1757,24 @@ private async ValueTask> StartDownloadAsyn long length = response.IsUnavailable() ? 0 : response.Headers.ContentLength ?? 0; ClientConfiguration.Pipeline.LogTrace($"Response: {response.GetRawResponse().Status}, ContentLength: {length}"); + BlobDownloadStreamingResult result = response.ToBlobDownloadStreamingResult(); + if (response.GetRawResponse().Headers.TryGetValue(Constants.StructuredMessage.CrcStructuredMessageHeader, out string _) && + response.GetRawResponse().Headers.TryGetValue(Constants.HeaderNames.ContentLength, out string rawContentLength)) + { + result.Content = new StructuredMessageDecodingStream(result.Content, long.Parse(rawContentLength)); + } + // if not null, we expected a structured message response + // but we didn't find one in the above condition + else if (structuredBodyType != null) + { + // okay to throw here. due to 4MB checksum limit on service downloads, and how we don't + // request structured message until we exceed that, we are not throwing on a request + // that would have otherwise succeeded and still gotten the desired checksum + throw Errors.ExpectedStructuredMessage(); + } + return Response.FromValue( - response.ToBlobDownloadStreamingResult(), + result, response.GetRawResponse()); } #endregion diff --git a/sdk/storage/Azure.Storage.Blobs/tests/BlobBaseClientTransferValidationTests.cs b/sdk/storage/Azure.Storage.Blobs/tests/BlobBaseClientTransferValidationTests.cs index 83e302902f1d5..5653e40af16e3 100644 --- a/sdk/storage/Azure.Storage.Blobs/tests/BlobBaseClientTransferValidationTests.cs +++ b/sdk/storage/Azure.Storage.Blobs/tests/BlobBaseClientTransferValidationTests.cs @@ -1,6 +1,7 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License. +using System; using System.IO; using System.Threading.Tasks; using Azure.Core.TestFramework; @@ -146,6 +147,36 @@ public async Task ExpectedDownloadStreamingStreamTypeReturned_None() // unvalidated stream type is private; just check we didn't get back a buffered stream Assert.AreNotEqual(typeof(MemoryStream), response.Value.Content.GetType()); } + + [Test] + public virtual async Task OlderServiceVersionThrowsOnStructuredMessage() + { + // use service version before structured message was introduced + await using DisposingContainer disposingContainer = await ClientBuilder.GetTestContainerAsync( + service: ClientBuilder.GetServiceClient_SharedKey( + InstrumentClientOptions(new BlobClientOptions(BlobClientOptions.ServiceVersion.V2024_11_04))), + publicAccessType: PublicAccessType.None); + + // Arrange + const int dataLength = Constants.KB; + var data = GetRandomBuffer(dataLength); + + var resourceName = GetNewResourceName(); + var blob = InstrumentClient(disposingContainer.Container.GetBlobClient(GetNewResourceName())); + await blob.UploadAsync(BinaryData.FromBytes(data)); + + var validationOptions = new DownloadTransferValidationOptions + { + ChecksumAlgorithm = StorageChecksumAlgorithm.StorageCrc64 + }; + AsyncTestDelegate operation = async () => await (await blob.DownloadStreamingAsync( + new BlobDownloadOptions + { + Range = new HttpRange(length: Constants.StructuredMessage.MaxDownloadCrcWithHeader + 1), + TransferValidation = validationOptions, + })).Value.Content.CopyToAsync(Stream.Null); + Assert.That(operation, Throws.TypeOf()); + } #endregion } } diff --git a/sdk/storage/Azure.Storage.Common/src/Shared/Constants.cs b/sdk/storage/Azure.Storage.Common/src/Shared/Constants.cs index 17f02b3596735..0636041a65134 100644 --- a/sdk/storage/Azure.Storage.Common/src/Shared/Constants.cs +++ b/sdk/storage/Azure.Storage.Common/src/Shared/Constants.cs @@ -667,8 +667,11 @@ internal static class AccountResources internal static class StructuredMessage { + public const string CrcStructuredMessageHeader = "x-ms-structured-body"; + public const string CrcStructuredContentLength = "x-ms-structured-content-length"; public const string CrcStructuredMessage = "XSM/1.0; properties=crc64"; public const int DefaultSegmentContentLength = 4 * MB; + public const int MaxDownloadCrcWithHeader = 4 * MB; } internal static class ClientSideEncryption diff --git a/sdk/storage/Azure.Storage.Common/src/Shared/Errors.Clients.cs b/sdk/storage/Azure.Storage.Common/src/Shared/Errors.Clients.cs index 4e5464fa17e6e..22def52ec719c 100644 --- a/sdk/storage/Azure.Storage.Common/src/Shared/Errors.Clients.cs +++ b/sdk/storage/Azure.Storage.Common/src/Shared/Errors.Clients.cs @@ -3,6 +3,7 @@ using System; using System.Globalization; +using System.IO; using System.Linq; using System.Security.Authentication; using System.Xml.Serialization; @@ -108,6 +109,9 @@ public static RequestFailedException ClientRequestIdMismatch(Response response, public static ArgumentException TransactionalHashingNotSupportedWithClientSideEncryption() => new ArgumentException("Client-side encryption and transactional hashing are not supported at the same time."); + public static InvalidDataException ExpectedStructuredMessage() + => new InvalidDataException($"Expected {Constants.StructuredMessage.CrcStructuredMessageHeader} in response, but found none."); + public static void VerifyHttpsTokenAuth(Uri uri) { if (uri.Scheme != Constants.Https) diff --git a/sdk/storage/Azure.Storage.Common/tests/Shared/TransferValidationTestBase.cs b/sdk/storage/Azure.Storage.Common/tests/Shared/TransferValidationTestBase.cs index 0923fe27d439f..05918941c9309 100644 --- a/sdk/storage/Azure.Storage.Common/tests/Shared/TransferValidationTestBase.cs +++ b/sdk/storage/Azure.Storage.Common/tests/Shared/TransferValidationTestBase.cs @@ -7,6 +7,7 @@ using System.Linq; using System.Threading.Tasks; using Azure.Core; +using Azure.Core.Pipeline; using Azure.Core.TestFramework; using Azure.Storage.Shared; using NUnit.Framework; @@ -313,6 +314,32 @@ void AssertChecksum(ResponseHeaders headers, string headerName) }; } +#if BlobSDK + internal static Action GetResponseStructuredMessageAssertion( + StructuredMessage.Flags flags, + Func isStructuredMessageExpected = default) + { + return response => + { + // filter some requests out with predicate + if (isStructuredMessageExpected != default && !isStructuredMessageExpected(response)) + { + return; + } + + Assert.That(response.Headers.TryGetValue("x-ms-structured-body", out string structuredBody)); + Assert.That(structuredBody, Does.Contain("XSM/1.0")); + if (flags.HasFlag(StructuredMessage.Flags.StorageCrc64)) + { + Assert.That(structuredBody, Does.Contain("crc64")); + } + + Assert.That(response.Headers.TryGetValue("Content-Length", out string contentLength)); + Assert.That(response.Headers.TryGetValue("x-ms-structured-content-length", out string structuredContentLength)); + }; + } +#endif + /// /// Asserts the service returned an error that expected checksum did not match checksum on upload. /// @@ -1705,6 +1732,65 @@ public virtual async Task DownloadSuccessfulHashVerification(StorageChecksumAlgo Assert.IsTrue(dest.ToArray().SequenceEqual(data)); } +#if BlobSDK + [TestCase(StorageChecksumAlgorithm.StorageCrc64, Constants.StructuredMessage.MaxDownloadCrcWithHeader, false, false)] + [TestCase(StorageChecksumAlgorithm.StorageCrc64, Constants.StructuredMessage.MaxDownloadCrcWithHeader-1, false, false)] + [TestCase(StorageChecksumAlgorithm.StorageCrc64, Constants.StructuredMessage.MaxDownloadCrcWithHeader+1, true, false)] + [TestCase(StorageChecksumAlgorithm.MD5, Constants.StructuredMessage.MaxDownloadCrcWithHeader+1, false, true)] + public virtual async Task DownloadApporpriatelyUsesStructuredMessage( + StorageChecksumAlgorithm algorithm, + int? downloadLen, + bool expectStructuredMessage, + bool expectThrow) + { + await using IDisposingContainer disposingContainer = await GetDisposingContainerAsync(); + + // Arrange + const int dataLength = Constants.KB; + var data = GetRandomBuffer(dataLength); + + var resourceName = GetNewResourceName(); + var client = await GetResourceClientAsync( + disposingContainer.Container, + resourceLength: dataLength, + createResource: true, + resourceName: resourceName); + await SetupDataAsync(client, new MemoryStream(data)); + + // make pipeline assertion for checking checksum was present on download + HttpPipelinePolicy checksumPipelineAssertion = new AssertMessageContentsPolicy(checkResponse: expectStructuredMessage + ? GetResponseStructuredMessageAssertion(StructuredMessage.Flags.StorageCrc64) + : GetResponseChecksumAssertion(algorithm)); + TClientOptions clientOptions = ClientBuilder.GetOptions(); + clientOptions.AddPolicy(checksumPipelineAssertion, HttpPipelinePosition.PerCall); + + client = await GetResourceClientAsync( + disposingContainer.Container, + resourceLength: dataLength, + resourceName: resourceName, + createResource: false, + downloadAlgorithm: algorithm, + options: clientOptions); + + var validationOptions = new DownloadTransferValidationOptions { ChecksumAlgorithm = algorithm }; + + // Act + var dest = new MemoryStream(); + AsyncTestDelegate operation = async () => await DownloadPartitionAsync( + client, dest, validationOptions, downloadLen.HasValue ? new HttpRange(length: downloadLen.Value) : default); + // Assert (policies checked use of content validation) + if (expectThrow) + { + Assert.That(operation, Throws.TypeOf()); + } + else + { + Assert.That(operation, Throws.Nothing); + Assert.IsTrue(dest.ToArray().SequenceEqual(data)); + } + } +#endif + [Test, Combinatorial] public virtual async Task DownloadHashMismatchThrows( [ValueSource(nameof(GetValidationAlgorithms))] StorageChecksumAlgorithm algorithm, From 003b439dafd9df263b3303fd12015675411c7078 Mon Sep 17 00:00:00 2001 From: Jocelyn <41338290+jaschrep-msft@users.noreply.github.com> Date: Thu, 4 Apr 2024 14:57:54 -0400 Subject: [PATCH 08/22] PUT Blob Structured Message (#43130) * putblob structured message * testproxy * fixes --- .../src/BlockBlobClient.cs | 44 +++++++++++++++---- .../Shared/TransferValidationTestBase.cs | 31 ++++++++++--- 2 files changed, 60 insertions(+), 15 deletions(-) diff --git a/sdk/storage/Azure.Storage.Blobs/src/BlockBlobClient.cs b/sdk/storage/Azure.Storage.Blobs/src/BlockBlobClient.cs index 38c3186c13f4c..0a90c66c851b8 100644 --- a/sdk/storage/Azure.Storage.Blobs/src/BlockBlobClient.cs +++ b/sdk/storage/Azure.Storage.Blobs/src/BlockBlobClient.cs @@ -875,14 +875,36 @@ internal virtual async Task> UploadInternal( scope.Start(); Errors.VerifyStreamPosition(content, nameof(content)); - // compute hash BEFORE attaching progress handler - ContentHasher.GetHashResult hashResult = await ContentHasher.GetHashOrDefaultInternal( - content, - validationOptions, - async, - cancellationToken).ConfigureAwait(false); - - content = content?.WithNoDispose().WithProgress(progressHandler); + ContentHasher.GetHashResult hashResult = null; + long contentLength = (content?.Length - content?.Position) ?? 0; + long? structuredContentLength = default; + string structuredBodyType = null; + if (content != null && + validationOptions != null && + validationOptions.ChecksumAlgorithm.ResolveAuto() == StorageChecksumAlgorithm.StorageCrc64 && + validationOptions.PrecalculatedChecksum.IsEmpty && + ClientSideEncryption == null) // don't allow feature combination + { + // report progress in terms of caller bytes, not encoded bytes + structuredContentLength = contentLength; + structuredBodyType = Constants.StructuredMessage.CrcStructuredMessage; + content = content.WithNoDispose().WithProgress(progressHandler); + content = new StructuredMessageEncodingStream( + content, + Constants.StructuredMessage.DefaultSegmentContentLength, + StructuredMessage.Flags.StorageCrc64); + contentLength = content.Length - content.Position; + } + else + { + // compute hash BEFORE attaching progress handler + hashResult = await ContentHasher.GetHashOrDefaultInternal( + content, + validationOptions, + async, + cancellationToken).ConfigureAwait(false); + content = content.WithNoDispose().WithProgress(progressHandler); + } ResponseWithHeaders response; @@ -921,6 +943,8 @@ internal virtual async Task> UploadInternal( legalHold: legalHold, transactionalContentMD5: hashResult?.MD5AsArray, transactionalContentCrc64: hashResult?.StorageCrc64AsArray, + structuredBodyType: structuredBodyType, + structuredContentLength: structuredContentLength, cancellationToken: cancellationToken) .ConfigureAwait(false); } @@ -953,6 +977,8 @@ internal virtual async Task> UploadInternal( legalHold: legalHold, transactionalContentMD5: hashResult?.MD5AsArray, transactionalContentCrc64: hashResult?.StorageCrc64AsArray, + structuredBodyType: structuredBodyType, + structuredContentLength: structuredContentLength, cancellationToken: cancellationToken); } @@ -2817,7 +2843,7 @@ internal async Task OpenWriteInternal( immutabilityPolicy: default, legalHold: default, progressHandler: default, - transferValidationOverride: default, + transferValidationOverride: new() { ChecksumAlgorithm = StorageChecksumAlgorithm.None }, operationName: default, async: async, cancellationToken: cancellationToken) diff --git a/sdk/storage/Azure.Storage.Common/tests/Shared/TransferValidationTestBase.cs b/sdk/storage/Azure.Storage.Common/tests/Shared/TransferValidationTestBase.cs index 05918941c9309..eda16484d7b19 100644 --- a/sdk/storage/Azure.Storage.Common/tests/Shared/TransferValidationTestBase.cs +++ b/sdk/storage/Azure.Storage.Common/tests/Shared/TransferValidationTestBase.cs @@ -5,6 +5,7 @@ using System.Collections.Generic; using System.IO; using System.Linq; +using System.Security.Cryptography; using System.Threading.Tasks; using Azure.Core; using Azure.Core.Pipeline; @@ -1020,8 +1021,14 @@ public virtual async Task ParallelUploadOneShotSuccessfulHashComputation(Storage }; // make pipeline assertion for checking checksum was present on upload - var checksumPipelineAssertion = new AssertMessageContentsPolicy( - checkRequest: GetRequestChecksumHeaderAssertion(algorithm, isChecksumExpected: ParallelUploadIsChecksumExpected)); +#if BlobSDK + var assertion = algorithm.ResolveAuto() == StorageChecksumAlgorithm.StorageCrc64 + ? GetRequestStructuredMessageAssertion(StructuredMessage.Flags.StorageCrc64, ParallelUploadIsChecksumExpected, dataLength) + : GetRequestChecksumHeaderAssertion(algorithm, isChecksumExpected: ParallelUploadIsChecksumExpected); +#else + var assertion = GetRequestChecksumHeaderAssertion(algorithm, isChecksumExpected: ParallelUploadIsChecksumExpected); +#endif + var checksumPipelineAssertion = new AssertMessageContentsPolicy(checkRequest: assertion); var clientOptions = ClientBuilder.GetOptions(); clientOptions.AddPolicy(checksumPipelineAssertion, HttpPipelinePosition.PerCall); @@ -1108,8 +1115,14 @@ public virtual async Task ParallelUploadUsesDefaultClientValidationOptions( }; // make pipeline assertion for checking checksum was present on upload - var checksumPipelineAssertion = new AssertMessageContentsPolicy(checkRequest: GetRequestChecksumHeaderAssertion( - clientAlgorithm, isChecksumExpected: ParallelUploadIsChecksumExpected)); +#if BlobSDK + var assertion = clientAlgorithm.ResolveAuto() == StorageChecksumAlgorithm.StorageCrc64 && !split + ? GetRequestStructuredMessageAssertion(StructuredMessage.Flags.StorageCrc64, ParallelUploadIsChecksumExpected, dataLength) + : GetRequestChecksumHeaderAssertion(clientAlgorithm, isChecksumExpected: ParallelUploadIsChecksumExpected); +#else + var assertion = GetRequestChecksumHeaderAssertion(clientAlgorithm, isChecksumExpected: ParallelUploadIsChecksumExpected); +#endif + var checksumPipelineAssertion = new AssertMessageContentsPolicy(checkRequest: assertion); var clientOptions = ClientBuilder.GetOptions(); clientOptions.AddPolicy(checksumPipelineAssertion, HttpPipelinePosition.PerCall); @@ -1160,8 +1173,14 @@ public virtual async Task ParallelUploadOverwritesDefaultClientValidationOptions }; // make pipeline assertion for checking checksum was present on upload - var checksumPipelineAssertion = new AssertMessageContentsPolicy(checkRequest: GetRequestChecksumHeaderAssertion( - overrideAlgorithm, isChecksumExpected: ParallelUploadIsChecksumExpected)); +#if BlobSDK + var assertion = overrideAlgorithm.ResolveAuto() == StorageChecksumAlgorithm.StorageCrc64 && !split + ? GetRequestStructuredMessageAssertion(StructuredMessage.Flags.StorageCrc64, ParallelUploadIsChecksumExpected, dataLength) + : GetRequestChecksumHeaderAssertion(overrideAlgorithm, isChecksumExpected: ParallelUploadIsChecksumExpected); +#else + var assertion = GetRequestChecksumHeaderAssertion(overrideAlgorithm, isChecksumExpected: ParallelUploadIsChecksumExpected); +#endif + var checksumPipelineAssertion = new AssertMessageContentsPolicy(checkRequest: assertion); var clientOptions = ClientBuilder.GetOptions(); clientOptions.AddPolicy(checksumPipelineAssertion, HttpPipelinePosition.PerCall); From d3010dbb47a4494028ef2c361308aa02aea43dd7 Mon Sep 17 00:00:00 2001 From: Jocelyn <41338290+jaschrep-msft@users.noreply.github.com> Date: Wed, 17 Apr 2024 11:05:18 -0400 Subject: [PATCH 09/22] Structured Message: DataLake Append (#43275) * datalake append * null fix * fixes --- .../Shared/TransferValidationTestBase.cs | 23 +++++----- .../src/Azure.Storage.Files.DataLake.csproj | 2 + .../src/DataLakeFileClient.cs | 40 +++++++++++++--- .../src/Generated/FileSystemRestClient.cs | 2 +- .../src/Generated/PathAppendDataHeaders.cs | 2 + .../src/Generated/PathRestClient.cs | 46 ++++++++++++++----- .../src/Generated/PathUpdateHeaders.cs | 2 + .../src/Generated/ServiceRestClient.cs | 2 +- .../src/autorest.md | 6 +-- .../Azure.Storage.Files.DataLake.Tests.csproj | 3 ++ 10 files changed, 95 insertions(+), 33 deletions(-) diff --git a/sdk/storage/Azure.Storage.Common/tests/Shared/TransferValidationTestBase.cs b/sdk/storage/Azure.Storage.Common/tests/Shared/TransferValidationTestBase.cs index eda16484d7b19..764cecb599ac5 100644 --- a/sdk/storage/Azure.Storage.Common/tests/Shared/TransferValidationTestBase.cs +++ b/sdk/storage/Azure.Storage.Common/tests/Shared/TransferValidationTestBase.cs @@ -8,6 +8,7 @@ using System.Security.Cryptography; using System.Threading.Tasks; using Azure.Core; +using Azure.Core.Diagnostics; using Azure.Core.Pipeline; using Azure.Core.TestFramework; using Azure.Storage.Shared; @@ -232,7 +233,7 @@ void AssertChecksum(RequestHeaders headers, string headerName) }; } -#if BlobSDK +#if BlobSDK || DataLakeSDK internal static Action GetRequestStructuredMessageAssertion( StructuredMessage.Flags flags, Func isStructuredMessageExpected = default, @@ -315,7 +316,7 @@ void AssertChecksum(ResponseHeaders headers, string headerName) }; } -#if BlobSDK +#if BlobSDK || DataLakeSDK internal static Action GetResponseStructuredMessageAssertion( StructuredMessage.Flags flags, Func isStructuredMessageExpected = default) @@ -422,7 +423,7 @@ public virtual async Task UploadPartitionSuccessfulHashComputation(StorageChecks }; // make pipeline assertion for checking checksum was present on upload -#if BlobSDK +#if BlobSDK || DataLakeSDK var assertion = algorithm.ResolveAuto() == StorageChecksumAlgorithm.StorageCrc64 ? GetRequestStructuredMessageAssertion(StructuredMessage.Flags.StorageCrc64, null, dataLength) : GetRequestChecksumHeaderAssertion(algorithm); @@ -531,9 +532,9 @@ public virtual async Task UploadPartitionTamperedStreamThrows(StorageChecksumAlg // Act streamTamperPolicy.TransformRequestBody = true; AsyncTestDelegate operation = async () => await UploadPartitionAsync(client, stream, validationOptions); - + using var listener = AzureEventSourceListener.CreateConsoleLogger(); // Assert -#if BlobSDK +#if BlobSDK || DataLakeSDK AssertWriteChecksumMismatch(operation, algorithm, expectStructuredMessage: algorithm.ResolveAuto() == StorageChecksumAlgorithm.StorageCrc64); #else @@ -553,7 +554,7 @@ public virtual async Task UploadPartitionUsesDefaultClientValidationOptions( var data = GetRandomBuffer(dataLength); // make pipeline assertion for checking checksum was present on upload -#if BlobSDK +#if BlobSDK || DataLakeSDK var assertion = clientAlgorithm.ResolveAuto() == StorageChecksumAlgorithm.StorageCrc64 ? GetRequestStructuredMessageAssertion(StructuredMessage.Flags.StorageCrc64, null, dataLength) : GetRequestChecksumHeaderAssertion(clientAlgorithm); @@ -599,7 +600,7 @@ public virtual async Task UploadPartitionOverwritesDefaultClientValidationOption }; // make pipeline assertion for checking checksum was present on upload -#if BlobSDK +#if BlobSDK || DataLakeSDK var assertion = overrideAlgorithm.ResolveAuto() == StorageChecksumAlgorithm.StorageCrc64 ? GetRequestStructuredMessageAssertion(StructuredMessage.Flags.StorageCrc64, null, dataLength) : GetRequestChecksumHeaderAssertion(overrideAlgorithm); @@ -1021,7 +1022,7 @@ public virtual async Task ParallelUploadOneShotSuccessfulHashComputation(Storage }; // make pipeline assertion for checking checksum was present on upload -#if BlobSDK +#if BlobSDK || DataLakeSDK var assertion = algorithm.ResolveAuto() == StorageChecksumAlgorithm.StorageCrc64 ? GetRequestStructuredMessageAssertion(StructuredMessage.Flags.StorageCrc64, ParallelUploadIsChecksumExpected, dataLength) : GetRequestChecksumHeaderAssertion(algorithm, isChecksumExpected: ParallelUploadIsChecksumExpected); @@ -1115,7 +1116,7 @@ public virtual async Task ParallelUploadUsesDefaultClientValidationOptions( }; // make pipeline assertion for checking checksum was present on upload -#if BlobSDK +#if BlobSDK || DataLakeSDK var assertion = clientAlgorithm.ResolveAuto() == StorageChecksumAlgorithm.StorageCrc64 && !split ? GetRequestStructuredMessageAssertion(StructuredMessage.Flags.StorageCrc64, ParallelUploadIsChecksumExpected, dataLength) : GetRequestChecksumHeaderAssertion(clientAlgorithm, isChecksumExpected: ParallelUploadIsChecksumExpected); @@ -1173,7 +1174,7 @@ public virtual async Task ParallelUploadOverwritesDefaultClientValidationOptions }; // make pipeline assertion for checking checksum was present on upload -#if BlobSDK +#if BlobSDK || DataLakeSDK var assertion = overrideAlgorithm.ResolveAuto() == StorageChecksumAlgorithm.StorageCrc64 && !split ? GetRequestStructuredMessageAssertion(StructuredMessage.Flags.StorageCrc64, ParallelUploadIsChecksumExpected, dataLength) : GetRequestChecksumHeaderAssertion(overrideAlgorithm, isChecksumExpected: ParallelUploadIsChecksumExpected); @@ -1751,7 +1752,7 @@ public virtual async Task DownloadSuccessfulHashVerification(StorageChecksumAlgo Assert.IsTrue(dest.ToArray().SequenceEqual(data)); } -#if BlobSDK +#if BlobSDK || DataLakeSDK [TestCase(StorageChecksumAlgorithm.StorageCrc64, Constants.StructuredMessage.MaxDownloadCrcWithHeader, false, false)] [TestCase(StorageChecksumAlgorithm.StorageCrc64, Constants.StructuredMessage.MaxDownloadCrcWithHeader-1, false, false)] [TestCase(StorageChecksumAlgorithm.StorageCrc64, Constants.StructuredMessage.MaxDownloadCrcWithHeader+1, true, false)] diff --git a/sdk/storage/Azure.Storage.Files.DataLake/src/Azure.Storage.Files.DataLake.csproj b/sdk/storage/Azure.Storage.Files.DataLake/src/Azure.Storage.Files.DataLake.csproj index b5ffe9a99116c..7467863a61be0 100644 --- a/sdk/storage/Azure.Storage.Files.DataLake/src/Azure.Storage.Files.DataLake.csproj +++ b/sdk/storage/Azure.Storage.Files.DataLake/src/Azure.Storage.Files.DataLake.csproj @@ -82,6 +82,8 @@ + + diff --git a/sdk/storage/Azure.Storage.Files.DataLake/src/DataLakeFileClient.cs b/sdk/storage/Azure.Storage.Files.DataLake/src/DataLakeFileClient.cs index 3d2bd710e25aa..6088e970ec9e0 100644 --- a/sdk/storage/Azure.Storage.Files.DataLake/src/DataLakeFileClient.cs +++ b/sdk/storage/Azure.Storage.Files.DataLake/src/DataLakeFileClient.cs @@ -16,6 +16,7 @@ using Azure.Storage.Common; using Azure.Storage.Files.DataLake.Models; using Azure.Storage.Sas; +using Azure.Storage.Shared; using Metadata = System.Collections.Generic.IDictionary; namespace Azure.Storage.Files.DataLake @@ -2332,13 +2333,36 @@ internal virtual async Task AppendInternal( using (ClientConfiguration.Pipeline.BeginLoggingScope(nameof(DataLakeFileClient))) { // compute hash BEFORE attaching progress handler - ContentHasher.GetHashResult hashResult = await ContentHasher.GetHashOrDefaultInternal( - content, - validationOptions, - async, - cancellationToken).ConfigureAwait(false); + ContentHasher.GetHashResult hashResult = null; + long contentLength = (content?.Length - content?.Position) ?? 0; + long? structuredContentLength = default; + string structuredBodyType = null; + if (content != null && + validationOptions != null && + validationOptions.ChecksumAlgorithm.ResolveAuto() == StorageChecksumAlgorithm.StorageCrc64 && + validationOptions.PrecalculatedChecksum.IsEmpty) + { + // report progress in terms of caller bytes, not encoded bytes + structuredContentLength = contentLength; + structuredBodyType = Constants.StructuredMessage.CrcStructuredMessage; + content = content.WithNoDispose().WithProgress(progressHandler); + content = new StructuredMessageEncodingStream( + content, + Constants.StructuredMessage.DefaultSegmentContentLength, + StructuredMessage.Flags.StorageCrc64); + contentLength = content.Length - content.Position; + } + else + { + // compute hash BEFORE attaching progress handler + hashResult = await ContentHasher.GetHashOrDefaultInternal( + content, + validationOptions, + async, + cancellationToken).ConfigureAwait(false); + content = content?.WithNoDispose().WithProgress(progressHandler); + } - content = content?.WithNoDispose().WithProgress(progressHandler); ClientConfiguration.Pipeline.LogMethodEnter( nameof(DataLakeFileClient), message: @@ -2373,6 +2397,8 @@ internal virtual async Task AppendInternal( encryptionKey: ClientConfiguration.CustomerProvidedKey?.EncryptionKey, encryptionKeySha256: ClientConfiguration.CustomerProvidedKey?.EncryptionKeyHash, encryptionAlgorithm: ClientConfiguration.CustomerProvidedKey?.EncryptionAlgorithm == null ? null : EncryptionAlgorithmTypeInternal.AES256, + structuredBodyType: structuredBodyType, + structuredContentLength: structuredContentLength, leaseId: leaseId, leaseAction: leaseAction, leaseDuration: leaseDurationLong, @@ -2392,6 +2418,8 @@ internal virtual async Task AppendInternal( encryptionKey: ClientConfiguration.CustomerProvidedKey?.EncryptionKey, encryptionKeySha256: ClientConfiguration.CustomerProvidedKey?.EncryptionKeyHash, encryptionAlgorithm: ClientConfiguration.CustomerProvidedKey?.EncryptionAlgorithm == null ? null : EncryptionAlgorithmTypeInternal.AES256, + structuredBodyType: structuredBodyType, + structuredContentLength: structuredContentLength, leaseId: leaseId, leaseAction: leaseAction, leaseDuration: leaseDurationLong, diff --git a/sdk/storage/Azure.Storage.Files.DataLake/src/Generated/FileSystemRestClient.cs b/sdk/storage/Azure.Storage.Files.DataLake/src/Generated/FileSystemRestClient.cs index 719932d5cd500..4144d908b7549 100644 --- a/sdk/storage/Azure.Storage.Files.DataLake/src/Generated/FileSystemRestClient.cs +++ b/sdk/storage/Azure.Storage.Files.DataLake/src/Generated/FileSystemRestClient.cs @@ -33,7 +33,7 @@ internal partial class FileSystemRestClient /// The HTTP pipeline for sending and receiving REST requests and responses. /// The URL of the service account, container, or blob that is the target of the desired operation. /// The value must be "filesystem" for all filesystem operations. The default value is "filesystem". - /// Specifies the version of the operation to use for this request. The default value is "2023-05-03". + /// Specifies the version of the operation to use for this request. The default value is "2025-01-05". /// , , , or is null. public FileSystemRestClient(ClientDiagnostics clientDiagnostics, HttpPipeline pipeline, string url, string resource, string version) { diff --git a/sdk/storage/Azure.Storage.Files.DataLake/src/Generated/PathAppendDataHeaders.cs b/sdk/storage/Azure.Storage.Files.DataLake/src/Generated/PathAppendDataHeaders.cs index 6ec456a438564..502dd557f4822 100644 --- a/sdk/storage/Azure.Storage.Files.DataLake/src/Generated/PathAppendDataHeaders.cs +++ b/sdk/storage/Azure.Storage.Files.DataLake/src/Generated/PathAppendDataHeaders.cs @@ -29,5 +29,7 @@ public PathAppendDataHeaders(Response response) public string EncryptionKeySha256 => _response.Headers.TryGetValue("x-ms-encryption-key-sha256", out string value) ? value : null; /// If the lease was auto-renewed with this request. public bool? LeaseRenewed => _response.Headers.TryGetValue("x-ms-lease-renewed", out bool? value) ? value : null; + /// Indicates the structured message body was accepted and mirrors back the message schema version and properties. + public string StructuredBodyType => _response.Headers.TryGetValue("x-ms-structured-body", out string value) ? value : null; } } diff --git a/sdk/storage/Azure.Storage.Files.DataLake/src/Generated/PathRestClient.cs b/sdk/storage/Azure.Storage.Files.DataLake/src/Generated/PathRestClient.cs index 6b1e970bd2fc8..d328c3079de6b 100644 --- a/sdk/storage/Azure.Storage.Files.DataLake/src/Generated/PathRestClient.cs +++ b/sdk/storage/Azure.Storage.Files.DataLake/src/Generated/PathRestClient.cs @@ -30,7 +30,7 @@ internal partial class PathRestClient /// The handler for diagnostic messaging in the client. /// The HTTP pipeline for sending and receiving REST requests and responses. /// The URL of the service account, container, or blob that is the target of the desired operation. - /// Specifies the version of the operation to use for this request. The default value is "2023-05-03". + /// Specifies the version of the operation to use for this request. The default value is "2025-01-05". /// The lease duration is required to acquire a lease, and specifies the duration of the lease in seconds. The lease duration must be between 15 and 60 seconds or -1 for infinite lease. /// , , or is null. public PathRestClient(ClientDiagnostics clientDiagnostics, HttpPipeline pipeline, string url, string version, int? xMsLeaseDuration = null) @@ -293,7 +293,7 @@ public ResponseWithHeaders Create(int? timeout = null, PathRe } } - internal HttpMessage CreateUpdateRequest(PathUpdateAction action, PathSetAccessControlRecursiveMode mode, Stream body, int? timeout, int? maxRecords, string continuation, bool? forceFlag, long? position, bool? retainUncommittedData, bool? close, long? contentLength, byte[] contentMD5, string leaseId, string cacheControl, string contentType, string contentDisposition, string contentEncoding, string contentLanguage, string properties, string owner, string group, string permissions, string acl, string ifMatch, string ifNoneMatch, DateTimeOffset? ifModifiedSince, DateTimeOffset? ifUnmodifiedSince) + internal HttpMessage CreateUpdateRequest(PathUpdateAction action, PathSetAccessControlRecursiveMode mode, Stream body, int? timeout, int? maxRecords, string continuation, bool? forceFlag, long? position, bool? retainUncommittedData, bool? close, long? contentLength, byte[] contentMD5, string leaseId, string cacheControl, string contentType, string contentDisposition, string contentEncoding, string contentLanguage, string properties, string owner, string group, string permissions, string acl, string ifMatch, string ifNoneMatch, DateTimeOffset? ifModifiedSince, DateTimeOffset? ifUnmodifiedSince, string structuredBodyType, long? structuredContentLength) { var message = _pipeline.CreateMessage(); var request = message.Request; @@ -396,6 +396,14 @@ internal HttpMessage CreateUpdateRequest(PathUpdateAction action, PathSetAccessC { request.Headers.Add("If-Unmodified-Since", ifUnmodifiedSince.Value, "R"); } + if (structuredBodyType != null) + { + request.Headers.Add("x-ms-structured-body", structuredBodyType); + } + if (structuredContentLength != null) + { + request.Headers.Add("x-ms-structured-content-length", structuredContentLength.Value); + } request.Headers.Add("Accept", "application/json"); if (contentLength != null) { @@ -434,17 +442,19 @@ internal HttpMessage CreateUpdateRequest(PathUpdateAction action, PathSetAccessC /// Specify an ETag value to operate only on blobs without a matching value. /// Specify this header value to operate only on a blob if it has been modified since the specified date/time. /// Specify this header value to operate only on a blob if it has not been modified since the specified date/time. + /// Required if the request body is a structured message. Specifies the message schema version and properties. + /// Required if the request body is a structured message. Specifies the length of the blob/file content inside the message body. Will always be smaller than Content-Length. /// The cancellation token to use. /// is null. /// Uploads data to be appended to a file, flushes (writes) previously uploaded data to a file, sets properties for a file or directory, or sets access control for a file or directory. Data can only be appended to a file. Concurrent writes to the same file using multiple clients are not supported. This operation supports conditional HTTP requests. For more information, see [Specifying Conditional Headers for Blob Service Operations](https://docs.microsoft.com/en-us/rest/api/storageservices/specifying-conditional-headers-for-blob-service-operations). - public async Task> UpdateAsync(PathUpdateAction action, PathSetAccessControlRecursiveMode mode, Stream body, int? timeout = null, int? maxRecords = null, string continuation = null, bool? forceFlag = null, long? position = null, bool? retainUncommittedData = null, bool? close = null, long? contentLength = null, byte[] contentMD5 = null, string leaseId = null, string cacheControl = null, string contentType = null, string contentDisposition = null, string contentEncoding = null, string contentLanguage = null, string properties = null, string owner = null, string group = null, string permissions = null, string acl = null, string ifMatch = null, string ifNoneMatch = null, DateTimeOffset? ifModifiedSince = null, DateTimeOffset? ifUnmodifiedSince = null, CancellationToken cancellationToken = default) + public async Task> UpdateAsync(PathUpdateAction action, PathSetAccessControlRecursiveMode mode, Stream body, int? timeout = null, int? maxRecords = null, string continuation = null, bool? forceFlag = null, long? position = null, bool? retainUncommittedData = null, bool? close = null, long? contentLength = null, byte[] contentMD5 = null, string leaseId = null, string cacheControl = null, string contentType = null, string contentDisposition = null, string contentEncoding = null, string contentLanguage = null, string properties = null, string owner = null, string group = null, string permissions = null, string acl = null, string ifMatch = null, string ifNoneMatch = null, DateTimeOffset? ifModifiedSince = null, DateTimeOffset? ifUnmodifiedSince = null, string structuredBodyType = null, long? structuredContentLength = null, CancellationToken cancellationToken = default) { if (body == null) { throw new ArgumentNullException(nameof(body)); } - using var message = CreateUpdateRequest(action, mode, body, timeout, maxRecords, continuation, forceFlag, position, retainUncommittedData, close, contentLength, contentMD5, leaseId, cacheControl, contentType, contentDisposition, contentEncoding, contentLanguage, properties, owner, group, permissions, acl, ifMatch, ifNoneMatch, ifModifiedSince, ifUnmodifiedSince); + using var message = CreateUpdateRequest(action, mode, body, timeout, maxRecords, continuation, forceFlag, position, retainUncommittedData, close, contentLength, contentMD5, leaseId, cacheControl, contentType, contentDisposition, contentEncoding, contentLanguage, properties, owner, group, permissions, acl, ifMatch, ifNoneMatch, ifModifiedSince, ifUnmodifiedSince, structuredBodyType, structuredContentLength); await _pipeline.SendAsync(message, cancellationToken).ConfigureAwait(false); var headers = new PathUpdateHeaders(message.Response); switch (message.Response.Status) @@ -491,17 +501,19 @@ public async Task Specify an ETag value to operate only on blobs without a matching value. /// Specify this header value to operate only on a blob if it has been modified since the specified date/time. /// Specify this header value to operate only on a blob if it has not been modified since the specified date/time. + /// Required if the request body is a structured message. Specifies the message schema version and properties. + /// Required if the request body is a structured message. Specifies the length of the blob/file content inside the message body. Will always be smaller than Content-Length. /// The cancellation token to use. /// is null. /// Uploads data to be appended to a file, flushes (writes) previously uploaded data to a file, sets properties for a file or directory, or sets access control for a file or directory. Data can only be appended to a file. Concurrent writes to the same file using multiple clients are not supported. This operation supports conditional HTTP requests. For more information, see [Specifying Conditional Headers for Blob Service Operations](https://docs.microsoft.com/en-us/rest/api/storageservices/specifying-conditional-headers-for-blob-service-operations). - public ResponseWithHeaders Update(PathUpdateAction action, PathSetAccessControlRecursiveMode mode, Stream body, int? timeout = null, int? maxRecords = null, string continuation = null, bool? forceFlag = null, long? position = null, bool? retainUncommittedData = null, bool? close = null, long? contentLength = null, byte[] contentMD5 = null, string leaseId = null, string cacheControl = null, string contentType = null, string contentDisposition = null, string contentEncoding = null, string contentLanguage = null, string properties = null, string owner = null, string group = null, string permissions = null, string acl = null, string ifMatch = null, string ifNoneMatch = null, DateTimeOffset? ifModifiedSince = null, DateTimeOffset? ifUnmodifiedSince = null, CancellationToken cancellationToken = default) + public ResponseWithHeaders Update(PathUpdateAction action, PathSetAccessControlRecursiveMode mode, Stream body, int? timeout = null, int? maxRecords = null, string continuation = null, bool? forceFlag = null, long? position = null, bool? retainUncommittedData = null, bool? close = null, long? contentLength = null, byte[] contentMD5 = null, string leaseId = null, string cacheControl = null, string contentType = null, string contentDisposition = null, string contentEncoding = null, string contentLanguage = null, string properties = null, string owner = null, string group = null, string permissions = null, string acl = null, string ifMatch = null, string ifNoneMatch = null, DateTimeOffset? ifModifiedSince = null, DateTimeOffset? ifUnmodifiedSince = null, string structuredBodyType = null, long? structuredContentLength = null, CancellationToken cancellationToken = default) { if (body == null) { throw new ArgumentNullException(nameof(body)); } - using var message = CreateUpdateRequest(action, mode, body, timeout, maxRecords, continuation, forceFlag, position, retainUncommittedData, close, contentLength, contentMD5, leaseId, cacheControl, contentType, contentDisposition, contentEncoding, contentLanguage, properties, owner, group, permissions, acl, ifMatch, ifNoneMatch, ifModifiedSince, ifUnmodifiedSince); + using var message = CreateUpdateRequest(action, mode, body, timeout, maxRecords, continuation, forceFlag, position, retainUncommittedData, close, contentLength, contentMD5, leaseId, cacheControl, contentType, contentDisposition, contentEncoding, contentLanguage, properties, owner, group, permissions, acl, ifMatch, ifNoneMatch, ifModifiedSince, ifUnmodifiedSince, structuredBodyType, structuredContentLength); _pipeline.Send(message, cancellationToken); var headers = new PathUpdateHeaders(message.Response); switch (message.Response.Status) @@ -1315,7 +1327,7 @@ public ResponseWithHeaders FlushData(int? timeout = null, } } - internal HttpMessage CreateAppendDataRequest(Stream body, long? position, int? timeout, long? contentLength, byte[] transactionalContentHash, byte[] transactionalContentCrc64, string leaseId, DataLakeLeaseAction? leaseAction, long? leaseDuration, string proposedLeaseId, string encryptionKey, string encryptionKeySha256, EncryptionAlgorithmTypeInternal? encryptionAlgorithm, bool? flush) + internal HttpMessage CreateAppendDataRequest(Stream body, long? position, int? timeout, long? contentLength, byte[] transactionalContentHash, byte[] transactionalContentCrc64, string leaseId, DataLakeLeaseAction? leaseAction, long? leaseDuration, string proposedLeaseId, string encryptionKey, string encryptionKeySha256, EncryptionAlgorithmTypeInternal? encryptionAlgorithm, bool? flush, string structuredBodyType, long? structuredContentLength) { var message = _pipeline.CreateMessage(); var request = message.Request; @@ -1369,6 +1381,14 @@ internal HttpMessage CreateAppendDataRequest(Stream body, long? position, int? t { request.Headers.Add("x-ms-encryption-algorithm", encryptionAlgorithm.Value.ToSerialString()); } + if (structuredBodyType != null) + { + request.Headers.Add("x-ms-structured-body", structuredBodyType); + } + if (structuredContentLength != null) + { + request.Headers.Add("x-ms-structured-content-length", structuredContentLength.Value); + } request.Headers.Add("Accept", "application/json"); if (contentLength != null) { @@ -1398,16 +1418,18 @@ internal HttpMessage CreateAppendDataRequest(Stream body, long? position, int? t /// The SHA-256 hash of the provided encryption key. Must be provided if the x-ms-encryption-key header is provided. /// The algorithm used to produce the encryption key hash. Currently, the only accepted value is "AES256". Must be provided if the x-ms-encryption-key header is provided. /// If file should be flushed after the append. + /// Required if the request body is a structured message. Specifies the message schema version and properties. + /// Required if the request body is a structured message. Specifies the length of the blob/file content inside the message body. Will always be smaller than Content-Length. /// The cancellation token to use. /// is null. - public async Task> AppendDataAsync(Stream body, long? position = null, int? timeout = null, long? contentLength = null, byte[] transactionalContentHash = null, byte[] transactionalContentCrc64 = null, string leaseId = null, DataLakeLeaseAction? leaseAction = null, long? leaseDuration = null, string proposedLeaseId = null, string encryptionKey = null, string encryptionKeySha256 = null, EncryptionAlgorithmTypeInternal? encryptionAlgorithm = null, bool? flush = null, CancellationToken cancellationToken = default) + public async Task> AppendDataAsync(Stream body, long? position = null, int? timeout = null, long? contentLength = null, byte[] transactionalContentHash = null, byte[] transactionalContentCrc64 = null, string leaseId = null, DataLakeLeaseAction? leaseAction = null, long? leaseDuration = null, string proposedLeaseId = null, string encryptionKey = null, string encryptionKeySha256 = null, EncryptionAlgorithmTypeInternal? encryptionAlgorithm = null, bool? flush = null, string structuredBodyType = null, long? structuredContentLength = null, CancellationToken cancellationToken = default) { if (body == null) { throw new ArgumentNullException(nameof(body)); } - using var message = CreateAppendDataRequest(body, position, timeout, contentLength, transactionalContentHash, transactionalContentCrc64, leaseId, leaseAction, leaseDuration, proposedLeaseId, encryptionKey, encryptionKeySha256, encryptionAlgorithm, flush); + using var message = CreateAppendDataRequest(body, position, timeout, contentLength, transactionalContentHash, transactionalContentCrc64, leaseId, leaseAction, leaseDuration, proposedLeaseId, encryptionKey, encryptionKeySha256, encryptionAlgorithm, flush, structuredBodyType, structuredContentLength); await _pipeline.SendAsync(message, cancellationToken).ConfigureAwait(false); var headers = new PathAppendDataHeaders(message.Response); switch (message.Response.Status) @@ -1434,16 +1456,18 @@ public async Task> AppendDataAsync(St /// The SHA-256 hash of the provided encryption key. Must be provided if the x-ms-encryption-key header is provided. /// The algorithm used to produce the encryption key hash. Currently, the only accepted value is "AES256". Must be provided if the x-ms-encryption-key header is provided. /// If file should be flushed after the append. + /// Required if the request body is a structured message. Specifies the message schema version and properties. + /// Required if the request body is a structured message. Specifies the length of the blob/file content inside the message body. Will always be smaller than Content-Length. /// The cancellation token to use. /// is null. - public ResponseWithHeaders AppendData(Stream body, long? position = null, int? timeout = null, long? contentLength = null, byte[] transactionalContentHash = null, byte[] transactionalContentCrc64 = null, string leaseId = null, DataLakeLeaseAction? leaseAction = null, long? leaseDuration = null, string proposedLeaseId = null, string encryptionKey = null, string encryptionKeySha256 = null, EncryptionAlgorithmTypeInternal? encryptionAlgorithm = null, bool? flush = null, CancellationToken cancellationToken = default) + public ResponseWithHeaders AppendData(Stream body, long? position = null, int? timeout = null, long? contentLength = null, byte[] transactionalContentHash = null, byte[] transactionalContentCrc64 = null, string leaseId = null, DataLakeLeaseAction? leaseAction = null, long? leaseDuration = null, string proposedLeaseId = null, string encryptionKey = null, string encryptionKeySha256 = null, EncryptionAlgorithmTypeInternal? encryptionAlgorithm = null, bool? flush = null, string structuredBodyType = null, long? structuredContentLength = null, CancellationToken cancellationToken = default) { if (body == null) { throw new ArgumentNullException(nameof(body)); } - using var message = CreateAppendDataRequest(body, position, timeout, contentLength, transactionalContentHash, transactionalContentCrc64, leaseId, leaseAction, leaseDuration, proposedLeaseId, encryptionKey, encryptionKeySha256, encryptionAlgorithm, flush); + using var message = CreateAppendDataRequest(body, position, timeout, contentLength, transactionalContentHash, transactionalContentCrc64, leaseId, leaseAction, leaseDuration, proposedLeaseId, encryptionKey, encryptionKeySha256, encryptionAlgorithm, flush, structuredBodyType, structuredContentLength); _pipeline.Send(message, cancellationToken); var headers = new PathAppendDataHeaders(message.Response); switch (message.Response.Status) diff --git a/sdk/storage/Azure.Storage.Files.DataLake/src/Generated/PathUpdateHeaders.cs b/sdk/storage/Azure.Storage.Files.DataLake/src/Generated/PathUpdateHeaders.cs index 35668cb1c3a1d..026c78e72481a 100644 --- a/sdk/storage/Azure.Storage.Files.DataLake/src/Generated/PathUpdateHeaders.cs +++ b/sdk/storage/Azure.Storage.Files.DataLake/src/Generated/PathUpdateHeaders.cs @@ -43,5 +43,7 @@ public PathUpdateHeaders(Response response) public string XMsContinuation => _response.Headers.TryGetValue("x-ms-continuation", out string value) ? value : null; /// The version of the REST protocol used to process the request. public string Version => _response.Headers.TryGetValue("x-ms-version", out string value) ? value : null; + /// Indicates the structured message body was accepted and mirrors back the message schema version and properties. + public string StructuredBodyType => _response.Headers.TryGetValue("x-ms-structured-body", out string value) ? value : null; } } diff --git a/sdk/storage/Azure.Storage.Files.DataLake/src/Generated/ServiceRestClient.cs b/sdk/storage/Azure.Storage.Files.DataLake/src/Generated/ServiceRestClient.cs index 118595b4d87d1..b00fa12238f4e 100644 --- a/sdk/storage/Azure.Storage.Files.DataLake/src/Generated/ServiceRestClient.cs +++ b/sdk/storage/Azure.Storage.Files.DataLake/src/Generated/ServiceRestClient.cs @@ -28,7 +28,7 @@ internal partial class ServiceRestClient /// The handler for diagnostic messaging in the client. /// The HTTP pipeline for sending and receiving REST requests and responses. /// The URL of the service account, container, or blob that is the target of the desired operation. - /// Specifies the version of the operation to use for this request. The default value is "2023-05-03". + /// Specifies the version of the operation to use for this request. The default value is "2025-01-05". /// , , or is null. public ServiceRestClient(ClientDiagnostics clientDiagnostics, HttpPipeline pipeline, string url, string version) { diff --git a/sdk/storage/Azure.Storage.Files.DataLake/src/autorest.md b/sdk/storage/Azure.Storage.Files.DataLake/src/autorest.md index 4121ebab9932e..58f5c3d055d3b 100644 --- a/sdk/storage/Azure.Storage.Files.DataLake/src/autorest.md +++ b/sdk/storage/Azure.Storage.Files.DataLake/src/autorest.md @@ -4,7 +4,7 @@ Run `dotnet build /t:GenerateCode` to generate code. ``` yaml input-file: - - https://raw.githubusercontent.com/Azure/azure-rest-api-specs/5da3c08b92d05858b728b013b69502dc93485373/specification/storage/data-plane/Azure.Storage.Files.DataLake/stable/2023-05-03/DataLakeStorage.json + - https://github.com/Azure/azure-rest-api-specs/blob/794c6178bc06c6c9dceb139e9f9d1b35b1a99701/specification/storage/data-plane/Azure.Storage.Files.DataLake/preview/2025-01-05/DataLakeStorage.json generation1-convenience-client: true modelerfour: seal-single-value-enum-by-default: true @@ -23,7 +23,7 @@ directive: if (property.includes('/{filesystem}/{path}')) { $[property]["parameters"] = $[property]["parameters"].filter(function(param) { return (typeof param['$ref'] === "undefined") || (false == param['$ref'].endsWith("#/parameters/FileSystem") && false == param['$ref'].endsWith("#/parameters/Path"))}); - } + } else if (property.includes('/{filesystem}')) { $[property]["parameters"] = $[property]["parameters"].filter(function(param) { return (typeof param['$ref'] === "undefined") || (false == param['$ref'].endsWith("#/parameters/FileSystem"))}); @@ -127,7 +127,7 @@ directive: } $[newName] = $[oldName]; delete $[oldName]; - } + } else if (property.includes('/{filesystem}')) { var oldName = property; diff --git a/sdk/storage/Azure.Storage.Files.DataLake/tests/Azure.Storage.Files.DataLake.Tests.csproj b/sdk/storage/Azure.Storage.Files.DataLake/tests/Azure.Storage.Files.DataLake.Tests.csproj index bef13bb21a1c6..1fa78690077be 100644 --- a/sdk/storage/Azure.Storage.Files.DataLake/tests/Azure.Storage.Files.DataLake.Tests.csproj +++ b/sdk/storage/Azure.Storage.Files.DataLake/tests/Azure.Storage.Files.DataLake.Tests.csproj @@ -6,6 +6,9 @@ Microsoft Azure.Storage.Files.DataLake client library tests false + + DataLakeSDK + From 5a532c6c0bb40421f086b650528d5fa3951587e5 Mon Sep 17 00:00:00 2001 From: Jocelyn <41338290+jaschrep-msft@users.noreply.github.com> Date: Tue, 21 May 2024 14:25:58 -0400 Subject: [PATCH 10/22] Retriable decode (#44155) * retriable decode * rewind mock test * bugfix * bugfix * tests --- .../Azure.Storage.Blobs/src/BlobBaseClient.cs | 2 +- ...tructuredMessageDecodingRetriableStream.cs | 178 ++++++++++++++ .../Shared/StructuredMessageDecodingStream.cs | 151 +++++++++--- .../tests/Azure.Storage.Common.Tests.csproj | 3 + .../tests/Shared/FaultyStream.cs | 13 +- ...uredMessageDecodingRetriableStreamTests.cs | 226 ++++++++++++++++++ .../StructuredMessageDecodingStreamTests.cs | 22 +- .../StructuredMessageStreamRoundtripTests.cs | 4 +- 8 files changed, 543 insertions(+), 56 deletions(-) create mode 100644 sdk/storage/Azure.Storage.Common/src/Shared/StructuredMessageDecodingRetriableStream.cs create mode 100644 sdk/storage/Azure.Storage.Common/tests/StructuredMessageDecodingRetriableStreamTests.cs diff --git a/sdk/storage/Azure.Storage.Blobs/src/BlobBaseClient.cs b/sdk/storage/Azure.Storage.Blobs/src/BlobBaseClient.cs index a0fc9a45dc47c..d9708c01e8d4f 100644 --- a/sdk/storage/Azure.Storage.Blobs/src/BlobBaseClient.cs +++ b/sdk/storage/Azure.Storage.Blobs/src/BlobBaseClient.cs @@ -1761,7 +1761,7 @@ private async ValueTask> StartDownloadAsyn if (response.GetRawResponse().Headers.TryGetValue(Constants.StructuredMessage.CrcStructuredMessageHeader, out string _) && response.GetRawResponse().Headers.TryGetValue(Constants.HeaderNames.ContentLength, out string rawContentLength)) { - result.Content = new StructuredMessageDecodingStream(result.Content, long.Parse(rawContentLength)); + (result.Content, _) = StructuredMessageDecodingStream.WrapStream(result.Content, long.Parse(rawContentLength)); } // if not null, we expected a structured message response // but we didn't find one in the above condition diff --git a/sdk/storage/Azure.Storage.Common/src/Shared/StructuredMessageDecodingRetriableStream.cs b/sdk/storage/Azure.Storage.Common/src/Shared/StructuredMessageDecodingRetriableStream.cs new file mode 100644 index 0000000000000..444fe3eb2e0a9 --- /dev/null +++ b/sdk/storage/Azure.Storage.Common/src/Shared/StructuredMessageDecodingRetriableStream.cs @@ -0,0 +1,178 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +using System; +using System.Buffers; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using Azure.Core; +using Azure.Core.Pipeline; + +namespace Azure.Storage.Shared; + +internal class StructuredMessageDecodingRetriableStream : Stream +{ + private readonly Stream _innerRetriable; + private long _decodedBytesRead; + + private readonly List _decodedDatas; + + private readonly Func _decodingStreamFactory; + private readonly Func> _decodingAsyncStreamFactory; + + public StructuredMessageDecodingRetriableStream( + Stream initialDecodingStream, + StructuredMessageDecodingStream.DecodedData initialDecodedData, + Func decodingStreamFactory, + Func> decodingAsyncStreamFactory, + ResponseClassifier responseClassifier, + int maxRetries) + { + _decodingStreamFactory = decodingStreamFactory; + _decodingAsyncStreamFactory = decodingAsyncStreamFactory; + _innerRetriable = RetriableStream.Create(initialDecodingStream, StreamFactory, StreamFactoryAsync, responseClassifier, maxRetries); + _decodedDatas = new() { initialDecodedData }; + } + + private Stream StreamFactory(long _) + { + long offset = _decodedDatas.Select(d => d.SegmentCrcs?.LastOrDefault().SegmentEnd ?? 0).Sum(); + (Stream decodingStream, StructuredMessageDecodingStream.DecodedData decodedData) = _decodingStreamFactory(offset); + _decodedDatas.Add(decodedData); + FastForwardInternal(decodingStream, _decodedBytesRead - offset, false).EnsureCompleted(); + return decodingStream; + } + + private async ValueTask StreamFactoryAsync(long _) + { + long offset = _decodedDatas.Select(d => d.SegmentCrcs?.LastOrDefault().SegmentEnd ?? 0).Sum(); + (Stream decodingStream, StructuredMessageDecodingStream.DecodedData decodedData) = await _decodingAsyncStreamFactory(offset).ConfigureAwait(false); + _decodedDatas.Add(decodedData); + await FastForwardInternal(decodingStream, _decodedBytesRead - offset, true).ConfigureAwait(false); + return decodingStream; + } + + private static async ValueTask FastForwardInternal(Stream stream, long bytes, bool async) + { + using (ArrayPool.Shared.RentDisposable(4 * Constants.KB, out byte[] buffer)) + { + if (async) + { + while (bytes > 0) + { + bytes -= await stream.ReadAsync(buffer, 0, (int)Math.Min(bytes, buffer.Length)).ConfigureAwait(false); + } + } + else + { + while (bytes > 0) + { + bytes -= stream.Read(buffer, 0, (int)Math.Min(bytes, buffer.Length)); + } + } + } + } + + protected override void Dispose(bool disposing) + { + foreach (IDisposable data in _decodedDatas) + { + data.Dispose(); + } + _decodedDatas.Clear(); + _innerRetriable.Dispose(); + } + + #region Read + public override int Read(byte[] buffer, int offset, int count) + { + int read = _innerRetriable.Read(buffer, offset, count); + _decodedBytesRead += read; + return read; + } + + public override async Task ReadAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) + { + int read = await _innerRetriable.ReadAsync(buffer, offset, count, cancellationToken).ConfigureAwait(false); + _decodedBytesRead += read; + return read; + } + +#if NETSTANDARD2_1_OR_GREATER || NETCOREAPP3_0_OR_GREATER + public override int Read(Span buffer) + { + int read = _innerRetriable.Read(buffer); + _decodedBytesRead += read; + return read; + } + + public override async ValueTask ReadAsync(Memory buffer, CancellationToken cancellationToken = default) + { + int read = await _innerRetriable.ReadAsync(buffer, cancellationToken).ConfigureAwait(false); + _decodedBytesRead += read; + return read; + } +#endif + + public override int ReadByte() + { + int val = _innerRetriable.ReadByte(); + _decodedBytesRead += 1; + return val; + } + + public override int EndRead(IAsyncResult asyncResult) + { + int read = _innerRetriable.EndRead(asyncResult); + _decodedBytesRead += read; + return read; + } + #endregion + + #region Passthru + public override bool CanRead => _innerRetriable.CanRead; + + public override bool CanSeek => _innerRetriable.CanSeek; + + public override bool CanWrite => _innerRetriable.CanWrite; + + public override bool CanTimeout => _innerRetriable.CanTimeout; + + public override long Length => _innerRetriable.Length; + + public override long Position { get => _innerRetriable.Position; set => _innerRetriable.Position = value; } + + public override void Flush() => _innerRetriable.Flush(); + + public override Task FlushAsync(CancellationToken cancellationToken) => _innerRetriable.FlushAsync(cancellationToken); + + public override long Seek(long offset, SeekOrigin origin) => _innerRetriable.Seek(offset, origin); + + public override void SetLength(long value) => _innerRetriable.SetLength(value); + + public override void Write(byte[] buffer, int offset, int count) => _innerRetriable.Write(buffer, offset, count); + + public override Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) => _innerRetriable.WriteAsync(buffer, offset, count, cancellationToken); + + public override void WriteByte(byte value) => _innerRetriable.WriteByte(value); + + public override IAsyncResult BeginWrite(byte[] buffer, int offset, int count, AsyncCallback callback, object state) => _innerRetriable.BeginWrite(buffer, offset, count, callback, state); + + public override void EndWrite(IAsyncResult asyncResult) => _innerRetriable.EndWrite(asyncResult); + + public override IAsyncResult BeginRead(byte[] buffer, int offset, int count, AsyncCallback callback, object state) => _innerRetriable.BeginRead(buffer, offset, count, callback, state); + + public override int ReadTimeout { get => _innerRetriable.ReadTimeout; set => _innerRetriable.ReadTimeout = value; } + + public override int WriteTimeout { get => _innerRetriable.WriteTimeout; set => _innerRetriable.WriteTimeout = value; } + +#if NETSTANDARD2_1_OR_GREATER || NETCOREAPP3_0_OR_GREATER + public override void Write(ReadOnlySpan buffer) => _innerRetriable.Write(buffer); + + public override ValueTask WriteAsync(ReadOnlyMemory buffer, CancellationToken cancellationToken = default) => _innerRetriable.WriteAsync(buffer, cancellationToken); +#endif + #endregion +} diff --git a/sdk/storage/Azure.Storage.Common/src/Shared/StructuredMessageDecodingStream.cs b/sdk/storage/Azure.Storage.Common/src/Shared/StructuredMessageDecodingStream.cs index db5b58ac74189..30912eab737d4 100644 --- a/sdk/storage/Azure.Storage.Common/src/Shared/StructuredMessageDecodingStream.cs +++ b/sdk/storage/Azure.Storage.Common/src/Shared/StructuredMessageDecodingStream.cs @@ -38,6 +38,57 @@ namespace Azure.Storage.Shared; /// internal class StructuredMessageDecodingStream : Stream { + internal class DecodedData : IDisposable + { + private byte[] _crcBackingArray; + + public long? InnerStreamLength { get; private set; } + public int? TotalSegments { get; private set; } + public StructuredMessage.Flags? Flags { get; private set; } + public List<(ReadOnlyMemory SegmentCrc, long SegmentEnd)> SegmentCrcs { get; private set; } + public ReadOnlyMemory TotalCrc { get; private set; } + public bool DecodeCompleted { get; private set; } + + internal void SetStreamHeaderData(int totalSegments, long innerStreamLength, StructuredMessage.Flags flags) + { + TotalSegments = totalSegments; + InnerStreamLength = innerStreamLength; + Flags = flags; + + if (flags.HasFlag(StructuredMessage.Flags.StorageCrc64)) + { + _crcBackingArray = ArrayPool.Shared.Rent((totalSegments + 1) * StructuredMessage.Crc64Length); + SegmentCrcs = new(); + } + } + + internal void ReportSegmentCrc(ReadOnlySpan crc, int segmentNum, long segmentEnd) + { + int offset = (segmentNum - 1) * StructuredMessage.Crc64Length; + crc.CopyTo(new Span(_crcBackingArray, offset, StructuredMessage.Crc64Length)); + SegmentCrcs.Add((new ReadOnlyMemory(_crcBackingArray, offset, StructuredMessage.Crc64Length), segmentEnd)); + } + + internal void ReportTotalCrc(ReadOnlySpan crc) + { + int offset = (TotalSegments.Value) * StructuredMessage.Crc64Length; + crc.CopyTo(new Span(_crcBackingArray, offset, StructuredMessage.Crc64Length)); + TotalCrc = new ReadOnlyMemory(_crcBackingArray, offset, StructuredMessage.Crc64Length); + } + internal void MarkComplete() + { + DecodeCompleted = true; + } + + public void Dispose() + { + if (_crcBackingArray is not null) + { + ArrayPool.Shared.Return(_crcBackingArray); + } + } + } + private enum SMRegion { StreamHeader, @@ -58,16 +109,16 @@ private enum SMRegion private int _segmentHeaderLength; private int _segmentFooterLength; - private int _totalSegments; - private long _innerStreamLength; + private long? _expectedInnerStreamLength; - private StructuredMessage.Flags _flags; - private bool _processedFooter = false; private bool _disposed; + private readonly DecodedData _decodedData; private StorageCrc64HashAlgorithm _totalContentCrc; private StorageCrc64HashAlgorithm _segmentCrc; + private readonly bool _validateChecksums; + public override bool CanRead => true; public override bool CanWrite => false; @@ -88,18 +139,31 @@ public override long Position set => throw new NotSupportedException(); } - public StructuredMessageDecodingStream( + public static (Stream DecodedStream, DecodedData DecodedData) WrapStream( + Stream innerStream, + long? expextedStreamLength = default) + { + DecodedData data = new(); + return (new StructuredMessageDecodingStream(innerStream, data, expextedStreamLength), data); + } + + private StructuredMessageDecodingStream( Stream innerStream, - long? expectedStreamLength = default) + DecodedData decodedData, + long? expectedStreamLength) { Argument.AssertNotNull(innerStream, nameof(innerStream)); + Argument.AssertNotNull(decodedData, nameof(decodedData)); - _innerStreamLength = expectedStreamLength ?? -1; + _expectedInnerStreamLength = expectedStreamLength; _innerBufferedStream = new BufferedStream(innerStream); + _decodedData = decodedData; // Assumes stream will be structured message 1.0. Will validate this when consuming stream. _streamHeaderLength = StructuredMessage.V1_0.StreamHeaderLength; _segmentHeaderLength = StructuredMessage.V1_0.SegmentHeaderLength; + + _validateChecksums = true; } #region Write @@ -191,14 +255,15 @@ public override async ValueTask ReadAsync(Memory buf, CancellationTok private void AssertDecodeFinished() { - if (_streamFooterLength > 0 && !_processedFooter) + if (_streamFooterLength > 0 && !_decodedData.DecodeCompleted) { throw Errors.InvalidStructuredMessage("Premature end of stream."); } - _processedFooter = true; + _decodedData.MarkComplete(); } private long _innerStreamConsumed = 0; + private long _decodedContentConsumed = 0; private SMRegion _currentRegion = SMRegion.StreamHeader; private int _currentSegmentNum = 0; private long _currentSegmentContentLength; @@ -243,6 +308,7 @@ private int Decode(Span buffer) _totalContentCrc?.Append(buffer.Slice(bufferConsumed, read)); _segmentCrc?.Append(buffer.Slice(bufferConsumed, read)); bufferConsumed += read; + _decodedContentConsumed += read; _currentSegmentContentRemaining -= read; if (_currentSegmentContentRemaining == 0) { @@ -370,24 +436,25 @@ private int ProcessStreamHeader(ReadOnlySpan span) StructuredMessage.V1_0.ReadStreamHeader( span.Slice(0, _streamHeaderLength), out long streamLength, - out _flags, - out _totalSegments); + out StructuredMessage.Flags flags, + out int totalSegments); + + _decodedData.SetStreamHeaderData(totalSegments, streamLength, flags); - if (_innerStreamLength > 0 && streamLength != _innerStreamLength) + if (_expectedInnerStreamLength.HasValue && _expectedInnerStreamLength.Value != streamLength) { throw Errors.InvalidStructuredMessage("Unexpected message size."); } - else - { - _innerStreamLength = streamLength; - } - if (_flags.HasFlag(StructuredMessage.Flags.StorageCrc64)) + if (_decodedData.Flags.Value.HasFlag(StructuredMessage.Flags.StorageCrc64)) { - _segmentFooterLength = _flags.HasFlag(StructuredMessage.Flags.StorageCrc64) ? StructuredMessage.Crc64Length : 0; - _streamFooterLength = _flags.HasFlag(StructuredMessage.Flags.StorageCrc64) ? StructuredMessage.Crc64Length : 0; - _segmentCrc = StorageCrc64HashAlgorithm.Create(); - _totalContentCrc = StorageCrc64HashAlgorithm.Create(); + _segmentFooterLength = StructuredMessage.Crc64Length; + _streamFooterLength = StructuredMessage.Crc64Length; + if (_validateChecksums) + { + _segmentCrc = StorageCrc64HashAlgorithm.Create(); + _totalContentCrc = StorageCrc64HashAlgorithm.Create(); + } } _currentRegion = SMRegion.SegmentHeader; return _streamHeaderLength; @@ -396,30 +463,34 @@ private int ProcessStreamHeader(ReadOnlySpan span) private int ProcessStreamFooter(ReadOnlySpan span) { int totalProcessed = 0; - if (_flags.HasFlag(StructuredMessage.Flags.StorageCrc64)) + if (_decodedData.Flags.Value.HasFlag(StructuredMessage.Flags.StorageCrc64)) { totalProcessed += StructuredMessage.Crc64Length; - using (ArrayPool.Shared.RentAsSpanDisposable(StructuredMessage.Crc64Length, out Span calculated)) + ReadOnlySpan expected = span.Slice(0, StructuredMessage.Crc64Length); + _decodedData.ReportTotalCrc(expected); + if (_validateChecksums) { - _totalContentCrc.GetCurrentHash(calculated); - ReadOnlySpan expected = span.Slice(0, StructuredMessage.Crc64Length); - if (!calculated.SequenceEqual(expected)) + using (ArrayPool.Shared.RentAsSpanDisposable(StructuredMessage.Crc64Length, out Span calculated)) { - throw Errors.ChecksumMismatch(calculated, expected); + _totalContentCrc.GetCurrentHash(calculated); + if (!calculated.SequenceEqual(expected)) + { + throw Errors.ChecksumMismatch(calculated, expected); + } } } } - if (_innerStreamConsumed != _innerStreamLength) + if (_innerStreamConsumed != _decodedData.InnerStreamLength) { throw Errors.InvalidStructuredMessage("Unexpected message size."); } - if (_currentSegmentNum != _totalSegments) + if (_currentSegmentNum != _decodedData.TotalSegments) { throw Errors.InvalidStructuredMessage("Missing expected message segments."); } - _processedFooter = true; + _decodedData.MarkComplete(); return totalProcessed; } @@ -442,21 +513,25 @@ private int ProcessSegmentHeader(ReadOnlySpan span) private int ProcessSegmentFooter(ReadOnlySpan span) { int totalProcessed = 0; - if (_flags.HasFlag(StructuredMessage.Flags.StorageCrc64)) + if (_decodedData.Flags.Value.HasFlag(StructuredMessage.Flags.StorageCrc64)) { totalProcessed += StructuredMessage.Crc64Length; - using (ArrayPool.Shared.RentAsSpanDisposable(StructuredMessage.Crc64Length, out Span calculated)) + ReadOnlySpan expected = span.Slice(0, StructuredMessage.Crc64Length); + if (_validateChecksums) { - _segmentCrc.GetCurrentHash(calculated); - _segmentCrc = StorageCrc64HashAlgorithm.Create(); - ReadOnlySpan expected = span.Slice(0, StructuredMessage.Crc64Length); - if (!calculated.SequenceEqual(expected)) + using (ArrayPool.Shared.RentAsSpanDisposable(StructuredMessage.Crc64Length, out Span calculated)) { - throw Errors.ChecksumMismatch(calculated, expected); + _segmentCrc.GetCurrentHash(calculated); + _segmentCrc = StorageCrc64HashAlgorithm.Create(); + if (!calculated.SequenceEqual(expected)) + { + throw Errors.ChecksumMismatch(calculated, expected); + } } } + _decodedData.ReportSegmentCrc(expected, _currentSegmentNum, _decodedContentConsumed); } - _currentRegion = _currentSegmentNum == _totalSegments ? SMRegion.StreamFooter : SMRegion.SegmentHeader; + _currentRegion = _currentSegmentNum == _decodedData.TotalSegments ? SMRegion.StreamFooter : SMRegion.SegmentHeader; return totalProcessed; } #endregion diff --git a/sdk/storage/Azure.Storage.Common/tests/Azure.Storage.Common.Tests.csproj b/sdk/storage/Azure.Storage.Common/tests/Azure.Storage.Common.Tests.csproj index bef5c97ccd50a..c4f9b715a692a 100644 --- a/sdk/storage/Azure.Storage.Common/tests/Azure.Storage.Common.Tests.csproj +++ b/sdk/storage/Azure.Storage.Common/tests/Azure.Storage.Common.Tests.csproj @@ -13,6 +13,8 @@ + + @@ -47,6 +49,7 @@ + diff --git a/sdk/storage/Azure.Storage.Common/tests/Shared/FaultyStream.cs b/sdk/storage/Azure.Storage.Common/tests/Shared/FaultyStream.cs index 7411eb1499312..f4e4b92ed73c4 100644 --- a/sdk/storage/Azure.Storage.Common/tests/Shared/FaultyStream.cs +++ b/sdk/storage/Azure.Storage.Common/tests/Shared/FaultyStream.cs @@ -15,6 +15,7 @@ internal class FaultyStream : Stream private readonly Exception _exceptionToRaise; private int _remainingExceptions; private Action _onFault; + private long _position = 0; public FaultyStream( Stream innerStream, @@ -40,7 +41,7 @@ public FaultyStream( public override long Position { - get => _innerStream.Position; + get => CanSeek ? _innerStream.Position : _position; set => _innerStream.Position = value; } @@ -53,7 +54,9 @@ public override int Read(byte[] buffer, int offset, int count) { if (_remainingExceptions == 0 || Position + count <= _raiseExceptionAt || _raiseExceptionAt >= _innerStream.Length) { - return _innerStream.Read(buffer, offset, count); + int read = _innerStream.Read(buffer, offset, count); + _position += read; + return read; } else { @@ -61,11 +64,13 @@ public override int Read(byte[] buffer, int offset, int count) } } - public override Task ReadAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) + public override async Task ReadAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) { if (_remainingExceptions == 0 || Position + count <= _raiseExceptionAt || _raiseExceptionAt >= _innerStream.Length) { - return _innerStream.ReadAsync(buffer, offset, count, cancellationToken); + int read = await _innerStream.ReadAsync(buffer, offset, count, cancellationToken); + _position += read; + return read; } else { diff --git a/sdk/storage/Azure.Storage.Common/tests/StructuredMessageDecodingRetriableStreamTests.cs b/sdk/storage/Azure.Storage.Common/tests/StructuredMessageDecodingRetriableStreamTests.cs new file mode 100644 index 0000000000000..666933e546189 --- /dev/null +++ b/sdk/storage/Azure.Storage.Common/tests/StructuredMessageDecodingRetriableStreamTests.cs @@ -0,0 +1,226 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +using System; +using System.IO; +using System.Threading; +using System.Threading.Tasks; +using Azure.Core; +using Azure.Storage.Shared; +using Azure.Storage.Test.Shared; +using Moq; +using NUnit.Framework; + +namespace Azure.Storage.Tests; + +[TestFixture(true)] +[TestFixture(false)] +public class StructuredMessageDecodingRetriableStreamTests +{ + public bool Async { get; } + + public StructuredMessageDecodingRetriableStreamTests(bool async) + { + Async = async; + } + + private Mock AllExceptionsRetry() + { + Mock mock = new(MockBehavior.Strict); + mock.Setup(rc => rc.IsRetriableException(It.IsAny())).Returns(true); + return mock; + } + + [Test] + public async ValueTask UninterruptedStream() + { + byte[] data = new Random().NextBytesInline(4 * Constants.KB).ToArray(); + byte[] dest = new byte[data.Length]; + + // mock with a simple MemoryStream rather than an actual StructuredMessageDecodingStream + using (Stream src = new MemoryStream(data)) + using (Stream retriableSrc = new StructuredMessageDecodingRetriableStream(src, new(), default, default, default, 1)) + using (Stream dst = new MemoryStream(dest)) + { + await retriableSrc.CopyToInternal(dst, Async, default); + } + + Assert.AreEqual(data, dest); + } + + [Test] + public async Task Interrupt_DataIntact([Values(true, false)] bool multipleInterrupts) + { + const int segments = 4; + const int segmentLen = Constants.KB; + const int readLen = 128; + const int interruptPos = segmentLen + (3 * readLen) + 10; + + Random r = new(); + byte[] data = r.NextBytesInline(segments * Constants.KB).ToArray(); + byte[] dest = new byte[data.Length]; + + // Mock a decoded data for the mocked StructuredMessageDecodingStream + StructuredMessageDecodingStream.DecodedData initialDecodedData = new(); + initialDecodedData.SetStreamHeaderData(segments, data.Length, StructuredMessage.Flags.StorageCrc64); + // for test purposes, initialize a DecodedData, since we are not actively decoding in this test + initialDecodedData.ReportSegmentCrc(r.NextBytesInline(StructuredMessage.Crc64Length), 1, segmentLen); + + (Stream DecodingStream, StructuredMessageDecodingStream.DecodedData DecodedData) Factory(long offset, bool faulty) + { + Stream stream = new MemoryStream(data, (int)offset, data.Length - (int)offset); + if (faulty) + { + stream = new FaultyStream(stream, interruptPos, 1, new Exception(), () => { }); + } + // Mock a decoded data for the mocked StructuredMessageDecodingStream + StructuredMessageDecodingStream.DecodedData decodedData = new(); + decodedData.SetStreamHeaderData(segments, data.Length, StructuredMessage.Flags.StorageCrc64); + // for test purposes, initialize a DecodedData, since we are not actively decoding in this test + decodedData.ReportSegmentCrc(r.NextBytesInline(StructuredMessage.Crc64Length), 1, segmentLen); + return (stream, decodedData); + } + + // mock with a simple MemoryStream rather than an actual StructuredMessageDecodingStream + using (Stream src = new MemoryStream(data)) + using (Stream faultySrc = new FaultyStream(src, interruptPos, 1, new Exception(), () => { })) + using (Stream retriableSrc = new StructuredMessageDecodingRetriableStream( + faultySrc, + initialDecodedData, + offset => Factory(offset, multipleInterrupts), + offset => new ValueTask<(Stream DecodingStream, StructuredMessageDecodingStream.DecodedData DecodedData)>(Factory(offset, multipleInterrupts)), + AllExceptionsRetry().Object, + int.MaxValue)) + using (Stream dst = new MemoryStream(dest)) + { + await retriableSrc.CopyToInternal(dst, readLen, Async, default); + } + + Assert.AreEqual(data, dest); + } + + [Test] + public async Task Interrupt_AppropriateRewind() + { + const int segments = 2; + const int segmentLen = Constants.KB; + const int dataLen = segments * segmentLen; + const int readLen = segmentLen / 4; + const int interruptOffset = 10; + const int interruptPos = segmentLen + (2 * readLen) + interruptOffset; + Random r = new(); + + // Mock a decoded data for the mocked StructuredMessageDecodingStream + StructuredMessageDecodingStream.DecodedData initialDecodedData = new(); + initialDecodedData.SetStreamHeaderData(segments, segments * segmentLen, StructuredMessage.Flags.StorageCrc64); + // By the time of interrupt, there will be one segment reported + initialDecodedData.ReportSegmentCrc(r.NextBytesInline(StructuredMessage.Crc64Length), 1, segmentLen); + + Mock mock = new(MockBehavior.Strict); + mock.SetupGet(s => s.CanRead).Returns(true); + mock.SetupGet(s => s.CanSeek).Returns(false); + if (Async) + { + mock.SetupSequence(s => s.ReadAsync(It.IsAny(), It.IsAny(), It.IsAny(), default)) + .Returns(Task.FromResult(readLen)) // start first segment + .Returns(Task.FromResult(readLen)) + .Returns(Task.FromResult(readLen)) + .Returns(Task.FromResult(readLen)) // finish first segment + .Returns(Task.FromResult(readLen)) // start second segment + .Returns(Task.FromResult(readLen)) + // faulty stream interrupt + .Returns(Task.FromResult(readLen * 2)) // restart second segment. fast-forward uses an internal 4KB buffer, so it will leap the 512 byte catchup all at once + .Returns(Task.FromResult(readLen)) + .Returns(Task.FromResult(readLen)) // end second segment + .Returns(Task.FromResult(0)) // signal end of stream + .Returns(Task.FromResult(0)) // second signal needed for stream wrapping reasons + ; + } + else + { + mock.SetupSequence(s => s.Read(It.IsAny(), It.IsAny(), It.IsAny())) + .Returns(readLen) // start first segment + .Returns(readLen) + .Returns(readLen) + .Returns(readLen) // finish first segment + .Returns(readLen) // start second segment + .Returns(readLen) + // faulty stream interrupt + .Returns(readLen * 2) // restart second segment. fast-forward uses an internal 4KB buffer, so it will leap the 512 byte catchup all at once + .Returns(readLen) + .Returns(readLen) // end second segment + .Returns(0) // signal end of stream + .Returns(0) // second signal needed for stream wrapping reasons + ; + } + Stream faultySrc = new FaultyStream(mock.Object, interruptPos, 1, new Exception(), default); + Stream retriableSrc = new StructuredMessageDecodingRetriableStream( + faultySrc, + initialDecodedData, + offset => (mock.Object, new()), + offset => new(Task.FromResult((mock.Object, new StructuredMessageDecodingStream.DecodedData()))), + AllExceptionsRetry().Object, + 1); + + int totalRead = 0; + int read = 0; + byte[] buf = new byte[readLen]; + if (Async) + { + while ((read = await retriableSrc.ReadAsync(buf, 0, buf.Length)) > 0) + { + totalRead += read; + } + } + else + { + while ((read = retriableSrc.Read(buf, 0, buf.Length)) > 0) + { + totalRead += read; + } + } + await retriableSrc.CopyToInternal(Stream.Null, readLen, Async, default); + + // Asserts we read exactly the data length, excluding the fastforward of the inner stream + Assert.That(totalRead, Is.EqualTo(dataLen)); + } + + [Test] + public async Task Interrupt_ProperDecode([Values(true, false)] bool multipleInterrupts) + { + // decoding stream inserts a buffered layer of 4 KB. use larger sizes to avoid interference from it. + const int segments = 4; + const int segmentLen = 128 * Constants.KB; + const int readLen = 8 * Constants.KB; + const int interruptPos = segmentLen + (3 * readLen) + 10; + + Random r = new(); + byte[] data = r.NextBytesInline(segments * Constants.KB).ToArray(); + byte[] dest = new byte[data.Length]; + + (Stream DecodingStream, StructuredMessageDecodingStream.DecodedData DecodedData) Factory(long offset, bool faulty) + { + Stream stream = new MemoryStream(data, (int)offset, data.Length - (int)offset); + stream = new StructuredMessageEncodingStream(stream, segmentLen, StructuredMessage.Flags.StorageCrc64); + if (faulty) + { + stream = new FaultyStream(stream, interruptPos, 1, new Exception(), () => { }); + } + return StructuredMessageDecodingStream.WrapStream(stream); + } + + (Stream decodingStream, StructuredMessageDecodingStream.DecodedData decodedData) = Factory(0, true); + using Stream retriableSrc = new StructuredMessageDecodingRetriableStream( + decodingStream, + decodedData, + offset => Factory(offset, multipleInterrupts), + offset => new ValueTask<(Stream DecodingStream, StructuredMessageDecodingStream.DecodedData DecodedData)>(Factory(offset, multipleInterrupts)), + AllExceptionsRetry().Object, + int.MaxValue); + using Stream dst = new MemoryStream(dest); + + await retriableSrc.CopyToInternal(dst, readLen, Async, default); + + Assert.AreEqual(data, dest); + } +} diff --git a/sdk/storage/Azure.Storage.Common/tests/StructuredMessageDecodingStreamTests.cs b/sdk/storage/Azure.Storage.Common/tests/StructuredMessageDecodingStreamTests.cs index f881a70c8e78f..2789672df4976 100644 --- a/sdk/storage/Azure.Storage.Common/tests/StructuredMessageDecodingStreamTests.cs +++ b/sdk/storage/Azure.Storage.Common/tests/StructuredMessageDecodingStreamTests.cs @@ -116,7 +116,7 @@ public async Task DecodesData( new Random().NextBytes(originalData); byte[] encodedData = StructuredMessageHelper.MakeEncodedData(originalData, segmentContentLength, flags); - Stream decodingStream = new StructuredMessageDecodingStream(new MemoryStream(encodedData)); + (Stream decodingStream, _) = StructuredMessageDecodingStream.WrapStream(new MemoryStream(encodedData)); byte[] decodedData; using (MemoryStream dest = new()) { @@ -136,7 +136,7 @@ public void BadStreamBadVersion() encodedData[0] = byte.MaxValue; - Stream decodingStream = new StructuredMessageDecodingStream(new MemoryStream(encodedData)); + (Stream decodingStream, _) = StructuredMessageDecodingStream.WrapStream(new MemoryStream(encodedData)); Assert.That(async () => await CopyStream(decodingStream, Stream.Null), Throws.InnerException.TypeOf()); } @@ -154,7 +154,7 @@ public async Task BadSegmentCrcThrows() encodedData[badBytePos] = (byte)~encodedData[badBytePos]; MemoryStream encodedDataStream = new(encodedData); - Stream decodingStream = new StructuredMessageDecodingStream(encodedDataStream); + (Stream decodingStream, _) = StructuredMessageDecodingStream.WrapStream(encodedDataStream); // manual try/catch to validate the proccess failed mid-stream rather than the end const int copyBufferSize = 4; @@ -183,7 +183,7 @@ public void BadStreamCrcThrows() encodedData[originalData.Length - 1] = (byte)~encodedData[originalData.Length - 1]; - Stream decodingStream = new StructuredMessageDecodingStream(new MemoryStream(encodedData)); + (Stream decodingStream, _) = StructuredMessageDecodingStream.WrapStream(new MemoryStream(encodedData)); Assert.That(async () => await CopyStream(decodingStream, Stream.Null), Throws.InnerException.TypeOf()); } @@ -196,7 +196,7 @@ public void BadStreamWrongContentLength() BinaryPrimitives.WriteInt64LittleEndian(new Span(encodedData, V1_0.StreamHeaderMessageLengthOffset, 8), 123456789L); - Stream decodingStream = new StructuredMessageDecodingStream(new MemoryStream(encodedData)); + (Stream decodingStream, _) = StructuredMessageDecodingStream.WrapStream(new MemoryStream(encodedData)); Assert.That(async () => await CopyStream(decodingStream, Stream.Null), Throws.InnerException.TypeOf()); } @@ -216,7 +216,7 @@ public void BadStreamWrongSegmentCount(int difference) BinaryPrimitives.WriteInt16LittleEndian( new Span(encodedData, V1_0.StreamHeaderSegmentCountOffset, 2), (short)(numSegments + difference)); - Stream decodingStream = new StructuredMessageDecodingStream(new MemoryStream(encodedData)); + (Stream decodingStream, _) = StructuredMessageDecodingStream.WrapStream(new MemoryStream(encodedData)); Assert.That(async () => await CopyStream(decodingStream, Stream.Null), Throws.InnerException.TypeOf()); } @@ -230,7 +230,7 @@ public void BadStreamWrongSegmentNum() BinaryPrimitives.WriteInt16LittleEndian( new Span(encodedData, V1_0.StreamHeaderLength + V1_0.SegmentHeaderNumOffset, 2), 123); - Stream decodingStream = new StructuredMessageDecodingStream(new MemoryStream(encodedData)); + (Stream decodingStream, _) = StructuredMessageDecodingStream.WrapStream(new MemoryStream(encodedData)); Assert.That(async () => await CopyStream(decodingStream, Stream.Null), Throws.InnerException.TypeOf()); } @@ -248,7 +248,7 @@ public async Task BadStreamWrongContentLength( new Span(encodedData, V1_0.StreamHeaderMessageLengthOffset, 8), encodedData.Length + difference); - Stream decodingStream = new StructuredMessageDecodingStream( + (Stream decodingStream, _) = StructuredMessageDecodingStream.WrapStream( new MemoryStream(encodedData), lengthProvided ? (long?)encodedData.Length : default); @@ -284,14 +284,14 @@ public void BadStreamMissingExpectedStreamFooter() byte[] brokenData = new byte[encodedData.Length - Crc64Length]; new Span(encodedData, 0, encodedData.Length - Crc64Length).CopyTo(brokenData); - Stream decodingStream = new StructuredMessageDecodingStream(new MemoryStream(brokenData)); + (Stream decodingStream, _) = StructuredMessageDecodingStream.WrapStream(new MemoryStream(brokenData)); Assert.That(async () => await CopyStream(decodingStream, Stream.Null), Throws.InnerException.TypeOf()); } [Test] public void NoSeek() { - StructuredMessageDecodingStream stream = new(new MemoryStream()); + (Stream stream, _) = StructuredMessageDecodingStream.WrapStream(new MemoryStream()); Assert.That(stream.CanSeek, Is.False); Assert.That(() => stream.Length, Throws.TypeOf()); @@ -303,7 +303,7 @@ public void NoSeek() [Test] public void NoWrite() { - StructuredMessageDecodingStream stream = new(new MemoryStream()); + (Stream stream, _) = StructuredMessageDecodingStream.WrapStream(new MemoryStream()); byte[] data = new byte[1024]; new Random().NextBytes(data); diff --git a/sdk/storage/Azure.Storage.Common/tests/StructuredMessageStreamRoundtripTests.cs b/sdk/storage/Azure.Storage.Common/tests/StructuredMessageStreamRoundtripTests.cs index 633233db2e73c..61583aa1ebe4e 100644 --- a/sdk/storage/Azure.Storage.Common/tests/StructuredMessageStreamRoundtripTests.cs +++ b/sdk/storage/Azure.Storage.Common/tests/StructuredMessageStreamRoundtripTests.cs @@ -113,8 +113,8 @@ public async Task RoundTrip( byte[] roundtripData; using (MemoryStream source = new(originalData)) - using (StructuredMessageEncodingStream encode = new(source, segmentLength, flags)) - using (StructuredMessageDecodingStream decode = new(encode)) + using (Stream encode = new StructuredMessageEncodingStream(source, segmentLength, flags)) + using (Stream decode = StructuredMessageDecodingStream.WrapStream(encode).DecodedStream) using (MemoryStream dest = new()) { await CopyStream(source, dest, readLen); From b40939c967a055f761555cb392cbf07b4c3d42cc Mon Sep 17 00:00:00 2001 From: Jocelyn <41338290+jaschrep-msft@users.noreply.github.com> Date: Fri, 24 May 2024 13:01:47 -0400 Subject: [PATCH 11/22] Download retriable stream structured message (#44176) * blobs retriable structured message download * test proxy * testproxy * remove commented code --- sdk/storage/Azure.Storage.Blobs/assets.json | 2 +- .../src/Azure.Storage.Blobs.csproj | 1 + .../Azure.Storage.Blobs/src/BlobBaseClient.cs | 89 ++++++++++--------- .../Shared/TransferValidationTestBase.cs | 47 ++++++++++ .../Azure.Storage.Files.DataLake/assets.json | 2 +- .../Azure.Storage.Files.Shares/assets.json | 2 +- 6 files changed, 98 insertions(+), 45 deletions(-) diff --git a/sdk/storage/Azure.Storage.Blobs/assets.json b/sdk/storage/Azure.Storage.Blobs/assets.json index 700e2c053c45b..f659bfa082944 100644 --- a/sdk/storage/Azure.Storage.Blobs/assets.json +++ b/sdk/storage/Azure.Storage.Blobs/assets.json @@ -2,5 +2,5 @@ "AssetsRepo": "Azure/azure-sdk-assets", "AssetsRepoPrefixPath": "net", "TagPrefix": "net/storage/Azure.Storage.Blobs", - "Tag": "net/storage/Azure.Storage.Blobs_efe2c4ee4f" + "Tag": "net/storage/Azure.Storage.Blobs_dcc7be748a" } diff --git a/sdk/storage/Azure.Storage.Blobs/src/Azure.Storage.Blobs.csproj b/sdk/storage/Azure.Storage.Blobs/src/Azure.Storage.Blobs.csproj index 11a4fdff8aba4..527ebfabde810 100644 --- a/sdk/storage/Azure.Storage.Blobs/src/Azure.Storage.Blobs.csproj +++ b/sdk/storage/Azure.Storage.Blobs/src/Azure.Storage.Blobs.csproj @@ -92,6 +92,7 @@ + diff --git a/sdk/storage/Azure.Storage.Blobs/src/BlobBaseClient.cs b/sdk/storage/Azure.Storage.Blobs/src/BlobBaseClient.cs index d9708c01e8d4f..bb591b8d3c0e9 100644 --- a/sdk/storage/Azure.Storage.Blobs/src/BlobBaseClient.cs +++ b/sdk/storage/Azure.Storage.Blobs/src/BlobBaseClient.cs @@ -1547,30 +1547,47 @@ internal virtual async ValueTask> Download // Wrap the response Content in a RetriableStream so we // can return it before it's finished downloading, but still // allow retrying if it fails. - Stream stream = RetriableStream.Create( - response.Value.Content, - startOffset => - StartDownloadAsync( - range, - conditionsWithEtag, - validationOptions, - startOffset, - async, - cancellationToken) - .EnsureCompleted() - .Value.Content, - async startOffset => - (await StartDownloadAsync( - range, - conditionsWithEtag, - validationOptions, - startOffset, - async, - cancellationToken) - .ConfigureAwait(false)) - .Value.Content, - ClientConfiguration.Pipeline.ResponseClassifier, - Constants.MaxReliabilityRetries); + ValueTask> Factory(long offset, bool forceStructuredMessage, bool async, CancellationToken cancellationToken) + => StartDownloadAsync( + range, + conditionsWithEtag, + validationOptions, + offset, + forceStructuredMessage, + async, + cancellationToken); + async ValueTask<(Stream DecodingStream, StructuredMessageDecodingStream.DecodedData DecodedData)> StructuredMessageFactory( + long offset, bool async, CancellationToken cancellationToken) + { + Response result = await Factory(offset, forceStructuredMessage: true, async, cancellationToken).ConfigureAwait(false); + return StructuredMessageDecodingStream.WrapStream(result.Value.Content, result.Value.Details.ContentLength); + } + Stream stream; + if (response.GetRawResponse().Headers.Contains(Constants.StructuredMessage.CrcStructuredMessageHeader)) + { + (Stream decodingStream, StructuredMessageDecodingStream.DecodedData decodedData) = StructuredMessageDecodingStream.WrapStream( + response.Value.Content, response.Value.Details.ContentLength); + stream = new StructuredMessageDecodingRetriableStream( + decodingStream, + decodedData, + startOffset => StructuredMessageFactory(startOffset, async: false, cancellationToken) + .EnsureCompleted(), + async startOffset => await StructuredMessageFactory(startOffset, async: true, cancellationToken) + .ConfigureAwait(false), + ClientConfiguration.Pipeline.ResponseClassifier, + Constants.MaxReliabilityRetries); + } + else + { + stream = RetriableStream.Create( + response.Value.Content, + startOffset => Factory(startOffset, forceStructuredMessage: false, async: false, cancellationToken) + .EnsureCompleted().Value.Content, + async startOffset => (await Factory(startOffset, forceStructuredMessage: false, async: true, cancellationToken) + .ConfigureAwait(false)).Value.Content, + ClientConfiguration.Pipeline.ResponseClassifier, + Constants.MaxReliabilityRetries); + } stream = stream.WithNoDispose().WithProgress(progressHandler); @@ -1645,6 +1662,9 @@ await ContentHasher.AssertResponseHashMatchInternal( /// /// Starting offset to request - in the event of a retry. /// + /// + /// When using transactional CRC, force the request to use structured message. + /// /// /// Whether to invoke the operation asynchronously. /// @@ -1666,6 +1686,7 @@ private async ValueTask> StartDownloadAsyn BlobRequestConditions conditions, DownloadTransferValidationOptions validationOptions, long startOffset = 0, + bool forceStructuredMessage = false, // TODO all CRC will force structured message in future bool async = true, CancellationToken cancellationToken = default) { @@ -1702,7 +1723,7 @@ private async ValueTask> StartDownloadAsyn rangeGetContentMD5 = true; break; case StorageChecksumAlgorithm.StorageCrc64: - if (pageRange?.Length <= Constants.StructuredMessage.MaxDownloadCrcWithHeader) + if (!forceStructuredMessage && pageRange?.Length <= Constants.StructuredMessage.MaxDownloadCrcWithHeader) { rangeGetContentCRC64 = true; } @@ -1757,24 +1778,8 @@ private async ValueTask> StartDownloadAsyn long length = response.IsUnavailable() ? 0 : response.Headers.ContentLength ?? 0; ClientConfiguration.Pipeline.LogTrace($"Response: {response.GetRawResponse().Status}, ContentLength: {length}"); - BlobDownloadStreamingResult result = response.ToBlobDownloadStreamingResult(); - if (response.GetRawResponse().Headers.TryGetValue(Constants.StructuredMessage.CrcStructuredMessageHeader, out string _) && - response.GetRawResponse().Headers.TryGetValue(Constants.HeaderNames.ContentLength, out string rawContentLength)) - { - (result.Content, _) = StructuredMessageDecodingStream.WrapStream(result.Content, long.Parse(rawContentLength)); - } - // if not null, we expected a structured message response - // but we didn't find one in the above condition - else if (structuredBodyType != null) - { - // okay to throw here. due to 4MB checksum limit on service downloads, and how we don't - // request structured message until we exceed that, we are not throwing on a request - // that would have otherwise succeeded and still gotten the desired checksum - throw Errors.ExpectedStructuredMessage(); - } - return Response.FromValue( - result, + response.ToBlobDownloadStreamingResult(), response.GetRawResponse()); } #endregion diff --git a/sdk/storage/Azure.Storage.Common/tests/Shared/TransferValidationTestBase.cs b/sdk/storage/Azure.Storage.Common/tests/Shared/TransferValidationTestBase.cs index 764cecb599ac5..201092978627c 100644 --- a/sdk/storage/Azure.Storage.Common/tests/Shared/TransferValidationTestBase.cs +++ b/sdk/storage/Azure.Storage.Common/tests/Shared/TransferValidationTestBase.cs @@ -2029,6 +2029,53 @@ public virtual async Task DownloadDisablesDefaultClientValidationOptions( Assert.False(response.Headers.Contains("x-ms-content-crc64")); Assert.IsTrue(dest.ToArray().SequenceEqual(data)); } + + [Test] + public virtual async Task DownloadRecoversFromInterruptWithValidation( + [ValueSource(nameof(GetValidationAlgorithms))] StorageChecksumAlgorithm algorithm) + { + using var _ = AzureEventSourceListener.CreateConsoleLogger(); + int dataLen = algorithm.ResolveAuto() switch { + StorageChecksumAlgorithm.StorageCrc64 => 5 * Constants.MB, // >4MB for multisegment + _ => Constants.KB, + }; + + await using IDisposingContainer disposingContainer = await GetDisposingContainerAsync(); + + // Arrange + var data = GetRandomBuffer(dataLen); + + TClientOptions options = ClientBuilder.GetOptions(); + options.AddPolicy(new FaultyDownloadPipelinePolicy(dataLen - 512, new IOException(), () => { }), HttpPipelinePosition.BeforeTransport); + var client = await GetResourceClientAsync( + disposingContainer.Container, + resourceLength: dataLen, + createResource: true, + options: options); + await SetupDataAsync(client, new MemoryStream(data)); + + var validationOptions = new DownloadTransferValidationOptions { ChecksumAlgorithm = algorithm }; + + // Act + var dest = new MemoryStream(); + var response = await DownloadPartitionAsync(client, dest, validationOptions, new HttpRange(length: data.Length)); + + // Assert + // no policies this time; just check response headers + switch (algorithm.ResolveAuto()) + { + case StorageChecksumAlgorithm.MD5: + Assert.True(response.Headers.Contains("Content-MD5")); + break; + case StorageChecksumAlgorithm.StorageCrc64: + Assert.True(response.Headers.Contains(Constants.StructuredMessage.CrcStructuredMessageHeader)); + break; + default: + Assert.Fail("Test can't validate given algorithm type."); + break; + } + Assert.IsTrue(dest.ToArray().SequenceEqual(data)); + } #endregion #region Auto-Algorithm Tests diff --git a/sdk/storage/Azure.Storage.Files.DataLake/assets.json b/sdk/storage/Azure.Storage.Files.DataLake/assets.json index 39ee762ad9a8a..7329a98a34f40 100644 --- a/sdk/storage/Azure.Storage.Files.DataLake/assets.json +++ b/sdk/storage/Azure.Storage.Files.DataLake/assets.json @@ -2,5 +2,5 @@ "AssetsRepo": "Azure/azure-sdk-assets", "AssetsRepoPrefixPath": "net", "TagPrefix": "net/storage/Azure.Storage.Files.DataLake", - "Tag": "net/storage/Azure.Storage.Files.DataLake_6d44446b20" + "Tag": "net/storage/Azure.Storage.Files.DataLake_9c23b9b180" } diff --git a/sdk/storage/Azure.Storage.Files.Shares/assets.json b/sdk/storage/Azure.Storage.Files.Shares/assets.json index d46664532748f..d4df7130a51d0 100644 --- a/sdk/storage/Azure.Storage.Files.Shares/assets.json +++ b/sdk/storage/Azure.Storage.Files.Shares/assets.json @@ -2,5 +2,5 @@ "AssetsRepo": "Azure/azure-sdk-assets", "AssetsRepoPrefixPath": "net", "TagPrefix": "net/storage/Azure.Storage.Files.Shares", - "Tag": "net/storage/Azure.Storage.Files.Shares_997e3d57ce" + "Tag": "net/storage/Azure.Storage.Files.Shares_5e5b51e54d" } From 47860711ef9c48baa06276a2a36fd378f1d17bf7 Mon Sep 17 00:00:00 2001 From: Jocelyn <41338290+jaschrep-msft@users.noreply.github.com> Date: Tue, 16 Jul 2024 11:50:18 -0400 Subject: [PATCH 12/22] CRC: Always Structured Message (#44955) * blockblob working * revert testing change * page/append * datalake file * testfix * bug fixes | test fixes * disable new API for presenting CRC from structured message * fix nunit * whitespace * fix/test-proxy * csproj * more csproj removeals This is building fine locally idk what's up * Trigger Fresh Build * fileshare testproxy * fix mock * Update macos image from 11 to latest (#44607) * Update macos image from 11 to latest * Update eng/pipelines/templates/jobs/ci.mgmt.yml Co-authored-by: Ben Broderick Phillips --------- Co-authored-by: Ben Broderick Phillips * Revert "Update macos image from 11 to latest (#44607)" this is causing too many problems. skipping macos tests for now. They'll run when this feature branch merges into main. This reverts commit 29e87b496fa2b60d53849afaa926a3bd0fc23529. --------- Co-authored-by: Wes Haggard Co-authored-by: Ben Broderick Phillips --- .../Azure.Storage.Blobs.Batch.Tests.csproj | 3 +- ...zure.Storage.Blobs.ChangeFeed.Tests.csproj | 3 +- sdk/storage/Azure.Storage.Blobs/assets.json | 2 +- .../src/AppendBlobClient.cs | 13 +- .../src/Azure.Storage.Blobs.csproj | 1 + .../Azure.Storage.Blobs/src/BlobBaseClient.cs | 10 +- .../src/BlockBlobClient.cs | 13 +- .../src/Models/BlobDownloadDetails.cs | 9 + .../src/Models/BlobDownloadInfo.cs | 11 + .../Azure.Storage.Blobs/src/PageBlobClient.cs | 13 +- .../src/PartitionedDownloader.cs | 50 +- .../BlobBaseClientTransferValidationTests.cs | 53 -- .../tests/PartitionedDownloaderTests.cs | 2 +- .../src/Shared/LazyLoadingReadOnlyStream.cs | 40 +- .../src/Shared/PooledMemoryStream.cs | 2 +- .../src/Shared/StreamExtensions.cs | 22 +- ...tructuredMessageDecodingRetriableStream.cs | 34 ++ ...redMessagePrecalculatedCrcWrapperStream.cs | 451 ++++++++++++++++++ .../tests/Azure.Storage.Common.Tests.csproj | 2 + .../Shared/ObserveStructuredMessagePolicy.cs | 85 ++++ .../tests/Shared/RequestExtensions.cs | 27 ++ .../Shared/TamperStreamContentsPolicy.cs | 11 +- .../Shared/TransferValidationTestBase.cs | 69 +-- ...uredMessageDecodingRetriableStreamTests.cs | 5 +- .../StructuredMessageEncodingStreamTests.cs | 25 + .../Azure.Storage.DataMovement.Blobs.csproj | 1 + ...re.Storage.DataMovement.Blobs.Tests.csproj | 5 + ...taMovement.Blobs.Files.Shares.Tests.csproj | 1 + ...age.DataMovement.Files.Shares.Tests.csproj | 1 + .../Azure.Storage.DataMovement.Tests.csproj | 1 + .../Azure.Storage.Files.DataLake/assets.json | 2 +- .../src/Azure.Storage.Files.DataLake.csproj | 1 + .../src/DataLakeFileClient.cs | 15 +- .../Azure.Storage.Files.Shares/assets.json | 2 +- .../Azure.Storage.Files.Shares.Tests.csproj | 1 + .../tests/Azure.Storage.Queues.Tests.csproj | 1 + 36 files changed, 796 insertions(+), 191 deletions(-) create mode 100644 sdk/storage/Azure.Storage.Common/src/Shared/StructuredMessagePrecalculatedCrcWrapperStream.cs create mode 100644 sdk/storage/Azure.Storage.Common/tests/Shared/ObserveStructuredMessagePolicy.cs create mode 100644 sdk/storage/Azure.Storage.Common/tests/Shared/RequestExtensions.cs diff --git a/sdk/storage/Azure.Storage.Blobs.Batch/tests/Azure.Storage.Blobs.Batch.Tests.csproj b/sdk/storage/Azure.Storage.Blobs.Batch/tests/Azure.Storage.Blobs.Batch.Tests.csproj index 2b77907e9aaac..286ab317256bf 100644 --- a/sdk/storage/Azure.Storage.Blobs.Batch/tests/Azure.Storage.Blobs.Batch.Tests.csproj +++ b/sdk/storage/Azure.Storage.Blobs.Batch/tests/Azure.Storage.Blobs.Batch.Tests.csproj @@ -23,6 +23,7 @@ + PreserveNewest @@ -42,4 +43,4 @@ - \ No newline at end of file + diff --git a/sdk/storage/Azure.Storage.Blobs.ChangeFeed/tests/Azure.Storage.Blobs.ChangeFeed.Tests.csproj b/sdk/storage/Azure.Storage.Blobs.ChangeFeed/tests/Azure.Storage.Blobs.ChangeFeed.Tests.csproj index 9682ab15ecd60..8cf13cd60744f 100644 --- a/sdk/storage/Azure.Storage.Blobs.ChangeFeed/tests/Azure.Storage.Blobs.ChangeFeed.Tests.csproj +++ b/sdk/storage/Azure.Storage.Blobs.ChangeFeed/tests/Azure.Storage.Blobs.ChangeFeed.Tests.csproj @@ -17,6 +17,7 @@ + @@ -28,4 +29,4 @@ PreserveNewest - \ No newline at end of file + diff --git a/sdk/storage/Azure.Storage.Blobs/assets.json b/sdk/storage/Azure.Storage.Blobs/assets.json index f659bfa082944..6a8a60c8c101a 100644 --- a/sdk/storage/Azure.Storage.Blobs/assets.json +++ b/sdk/storage/Azure.Storage.Blobs/assets.json @@ -2,5 +2,5 @@ "AssetsRepo": "Azure/azure-sdk-assets", "AssetsRepoPrefixPath": "net", "TagPrefix": "net/storage/Azure.Storage.Blobs", - "Tag": "net/storage/Azure.Storage.Blobs_dcc7be748a" + "Tag": "net/storage/Azure.Storage.Blobs_8b3f7ac2a4" } diff --git a/sdk/storage/Azure.Storage.Blobs/src/AppendBlobClient.cs b/sdk/storage/Azure.Storage.Blobs/src/AppendBlobClient.cs index 5a396e60a0598..9a110cf8eb13a 100644 --- a/sdk/storage/Azure.Storage.Blobs/src/AppendBlobClient.cs +++ b/sdk/storage/Azure.Storage.Blobs/src/AppendBlobClient.cs @@ -1248,7 +1248,6 @@ internal async Task> AppendBlockInternal( string structuredBodyType = null; if (validationOptions != null && validationOptions.ChecksumAlgorithm.ResolveAuto() == StorageChecksumAlgorithm.StorageCrc64 && - validationOptions.PrecalculatedChecksum.IsEmpty && ClientSideEncryption == null) // don't allow feature combination { // report progress in terms of caller bytes, not encoded bytes @@ -1256,10 +1255,14 @@ internal async Task> AppendBlockInternal( contentLength = (content?.Length - content?.Position) ?? 0; structuredBodyType = Constants.StructuredMessage.CrcStructuredMessage; content = content.WithNoDispose().WithProgress(progressHandler); - content = new StructuredMessageEncodingStream( - content, - Constants.StructuredMessage.DefaultSegmentContentLength, - StructuredMessage.Flags.StorageCrc64); + content = validationOptions.PrecalculatedChecksum.IsEmpty + ? new StructuredMessageEncodingStream( + content, + Constants.StructuredMessage.DefaultSegmentContentLength, + StructuredMessage.Flags.StorageCrc64) + : new StructuredMessagePrecalculatedCrcWrapperStream( + content, + validationOptions.PrecalculatedChecksum.Span); contentLength = (content?.Length - content?.Position) ?? 0; } else diff --git a/sdk/storage/Azure.Storage.Blobs/src/Azure.Storage.Blobs.csproj b/sdk/storage/Azure.Storage.Blobs/src/Azure.Storage.Blobs.csproj index 527ebfabde810..851474c2d0dab 100644 --- a/sdk/storage/Azure.Storage.Blobs/src/Azure.Storage.Blobs.csproj +++ b/sdk/storage/Azure.Storage.Blobs/src/Azure.Storage.Blobs.csproj @@ -95,6 +95,7 @@ + diff --git a/sdk/storage/Azure.Storage.Blobs/src/BlobBaseClient.cs b/sdk/storage/Azure.Storage.Blobs/src/BlobBaseClient.cs index bb591b8d3c0e9..d271615f6f4b7 100644 --- a/sdk/storage/Azure.Storage.Blobs/src/BlobBaseClient.cs +++ b/sdk/storage/Azure.Storage.Blobs/src/BlobBaseClient.cs @@ -1574,6 +1574,7 @@ ValueTask> Factory(long offset, bool force .EnsureCompleted(), async startOffset => await StructuredMessageFactory(startOffset, async: true, cancellationToken) .ConfigureAwait(false), + default, //decodedData => response.Value.Details.ContentCrc = decodedData.TotalCrc.ToArray(), ClientConfiguration.Pipeline.ResponseClassifier, Constants.MaxReliabilityRetries); } @@ -1723,14 +1724,7 @@ private async ValueTask> StartDownloadAsyn rangeGetContentMD5 = true; break; case StorageChecksumAlgorithm.StorageCrc64: - if (!forceStructuredMessage && pageRange?.Length <= Constants.StructuredMessage.MaxDownloadCrcWithHeader) - { - rangeGetContentCRC64 = true; - } - else - { - structuredBodyType = Constants.StructuredMessage.CrcStructuredMessage; - } + structuredBodyType = Constants.StructuredMessage.CrcStructuredMessage; break; default: break; diff --git a/sdk/storage/Azure.Storage.Blobs/src/BlockBlobClient.cs b/sdk/storage/Azure.Storage.Blobs/src/BlockBlobClient.cs index 0a90c66c851b8..fe9e6af997b6c 100644 --- a/sdk/storage/Azure.Storage.Blobs/src/BlockBlobClient.cs +++ b/sdk/storage/Azure.Storage.Blobs/src/BlockBlobClient.cs @@ -1337,7 +1337,6 @@ internal virtual async Task> StageBlockInternal( string structuredBodyType = null; if (validationOptions != null && validationOptions.ChecksumAlgorithm.ResolveAuto() == StorageChecksumAlgorithm.StorageCrc64 && - validationOptions.PrecalculatedChecksum.IsEmpty && ClientSideEncryption == null) // don't allow feature combination { // report progress in terms of caller bytes, not encoded bytes @@ -1345,10 +1344,14 @@ internal virtual async Task> StageBlockInternal( contentLength = (content?.Length - content?.Position) ?? 0; structuredBodyType = Constants.StructuredMessage.CrcStructuredMessage; content = content.WithNoDispose().WithProgress(progressHandler); - content = new StructuredMessageEncodingStream( - content, - Constants.StructuredMessage.DefaultSegmentContentLength, - StructuredMessage.Flags.StorageCrc64); + content = validationOptions.PrecalculatedChecksum.IsEmpty + ? new StructuredMessageEncodingStream( + content, + Constants.StructuredMessage.DefaultSegmentContentLength, + StructuredMessage.Flags.StorageCrc64) + : new StructuredMessagePrecalculatedCrcWrapperStream( + content, + validationOptions.PrecalculatedChecksum.Span); contentLength = (content?.Length - content?.Position) ?? 0; } else diff --git a/sdk/storage/Azure.Storage.Blobs/src/Models/BlobDownloadDetails.cs b/sdk/storage/Azure.Storage.Blobs/src/Models/BlobDownloadDetails.cs index bc119822cdc12..6104abfd9ac5f 100644 --- a/sdk/storage/Azure.Storage.Blobs/src/Models/BlobDownloadDetails.cs +++ b/sdk/storage/Azure.Storage.Blobs/src/Models/BlobDownloadDetails.cs @@ -34,6 +34,15 @@ public class BlobDownloadDetails public byte[] ContentHash { get; internal set; } #pragma warning restore CA1819 // Properties should not return arrays + // TODO enable in following PR + ///// + ///// When requested using , this value contains the CRC for the download blob range. + ///// This value may only become populated once the network stream is fully consumed. If this instance is accessed through + ///// , the network stream has already been consumed. Otherwise, consume the content stream before + ///// checking this value. + ///// + //public byte[] ContentCrc { get; internal set; } + /// /// Returns the date and time the container was last modified. Any operation that modifies the blob, including an update of the blob's metadata or properties, changes the last-modified time of the blob. /// diff --git a/sdk/storage/Azure.Storage.Blobs/src/Models/BlobDownloadInfo.cs b/sdk/storage/Azure.Storage.Blobs/src/Models/BlobDownloadInfo.cs index e034573b54b3a..1a525c718d1b4 100644 --- a/sdk/storage/Azure.Storage.Blobs/src/Models/BlobDownloadInfo.cs +++ b/sdk/storage/Azure.Storage.Blobs/src/Models/BlobDownloadInfo.cs @@ -4,6 +4,8 @@ using System; using System.ComponentModel; using System.IO; +using System.Threading.Tasks; +using Azure.Core; using Azure.Storage.Shared; namespace Azure.Storage.Blobs.Models @@ -49,6 +51,15 @@ public class BlobDownloadInfo : IDisposable, IDownloadedContent /// public BlobDownloadDetails Details { get; internal set; } + // TODO enable in following PR + ///// + ///// Indicates some contents of are mixed into the response stream. + ///// They will not be set until has been fully consumed. These details + ///// will be extracted from the content stream by the library before the calling code can + ///// encounter them. + ///// + //public bool ExpectTrailingDetails { get; internal set; } + /// /// Constructor. /// diff --git a/sdk/storage/Azure.Storage.Blobs/src/PageBlobClient.cs b/sdk/storage/Azure.Storage.Blobs/src/PageBlobClient.cs index 34befc6d4efe7..7038897531fbb 100644 --- a/sdk/storage/Azure.Storage.Blobs/src/PageBlobClient.cs +++ b/sdk/storage/Azure.Storage.Blobs/src/PageBlobClient.cs @@ -1370,7 +1370,6 @@ internal async Task> UploadPagesInternal( HttpRange range; if (validationOptions != null && validationOptions.ChecksumAlgorithm.ResolveAuto() == StorageChecksumAlgorithm.StorageCrc64 && - validationOptions.PrecalculatedChecksum.IsEmpty && ClientSideEncryption == null) // don't allow feature combination { // report progress in terms of caller bytes, not encoded bytes @@ -1379,10 +1378,14 @@ internal async Task> UploadPagesInternal( range = new HttpRange(offset, (content?.Length - content?.Position) ?? null); structuredBodyType = Constants.StructuredMessage.CrcStructuredMessage; content = content?.WithNoDispose().WithProgress(progressHandler); - content = new StructuredMessageEncodingStream( - content, - Constants.StructuredMessage.DefaultSegmentContentLength, - StructuredMessage.Flags.StorageCrc64); + content = validationOptions.PrecalculatedChecksum.IsEmpty + ? new StructuredMessageEncodingStream( + content, + Constants.StructuredMessage.DefaultSegmentContentLength, + StructuredMessage.Flags.StorageCrc64) + : new StructuredMessagePrecalculatedCrcWrapperStream( + content, + validationOptions.PrecalculatedChecksum.Span); contentLength = (content?.Length - content?.Position) ?? 0; } else diff --git a/sdk/storage/Azure.Storage.Blobs/src/PartitionedDownloader.cs b/sdk/storage/Azure.Storage.Blobs/src/PartitionedDownloader.cs index 2c52d0c256e34..276bdadb673fa 100644 --- a/sdk/storage/Azure.Storage.Blobs/src/PartitionedDownloader.cs +++ b/sdk/storage/Azure.Storage.Blobs/src/PartitionedDownloader.cs @@ -48,7 +48,8 @@ internal class PartitionedDownloader /// private readonly StorageChecksumAlgorithm _validationAlgorithm; private readonly int _checksumSize; - private bool UseMasterCrc => _validationAlgorithm.ResolveAuto() == StorageChecksumAlgorithm.StorageCrc64; + // TODO disabling master crc temporarily. segment CRCs still handled. + private bool UseMasterCrc => false; // _validationAlgorithm.ResolveAuto() == StorageChecksumAlgorithm.StorageCrc64; private StorageCrc64HashAlgorithm _masterCrcCalculator = null; /// @@ -212,8 +213,20 @@ public async Task DownloadToInternal( // If the first segment was the entire blob, we'll copy that to // the output stream and finish now - long initialLength = initialResponse.Value.Details.ContentLength; - long totalLength = ParseRangeTotalLength(initialResponse.Value.Details.ContentRange); + long initialLength; + long totalLength; + // Get blob content length downloaded from content range when available to handle transit encoding + if (string.IsNullOrWhiteSpace(initialResponse.Value.Details.ContentRange)) + { + initialLength = initialResponse.Value.Details.ContentLength; + totalLength = 0; + } + else + { + ContentRange recievedRange = ContentRange.Parse(initialResponse.Value.Details.ContentRange); + initialLength = recievedRange.End.Value - recievedRange.Start.Value + 1; + totalLength = recievedRange.Size.Value; + } if (initialLength == totalLength) { await HandleOneShotDownload(initialResponse, destination, async, cancellationToken) @@ -395,20 +408,6 @@ private async Task FinalizeDownloadInternal( } } - private static long ParseRangeTotalLength(string range) - { - if (range == null) - { - return 0; - } - int lengthSeparator = range.IndexOf("/", StringComparison.InvariantCultureIgnoreCase); - if (lengthSeparator == -1) - { - throw BlobErrors.ParsingFullHttpRangeFailed(range); - } - return long.Parse(range.Substring(lengthSeparator + 1), CultureInfo.InvariantCulture); - } - private async Task CopyToInternal( Response response, Stream destination, @@ -417,7 +416,10 @@ private async Task CopyToInternal( CancellationToken cancellationToken) { CancellationHelper.ThrowIfCancellationRequested(cancellationToken); - using IHasher hasher = ContentHasher.GetHasherFromAlgorithmId(_validationAlgorithm); + // if structured message, this crc is validated in the decoding process. don't decode it here. + using IHasher hasher = response.GetRawResponse().Headers.Contains(Constants.StructuredMessage.CrcStructuredMessageHeader) + ? null + : ContentHasher.GetHasherFromAlgorithmId(_validationAlgorithm); using Stream rawSource = response.Value.Content; using Stream source = hasher != null ? ChecksumCalculatingStream.GetReadStream(rawSource, hasher.AppendHash) @@ -432,13 +434,13 @@ await source.CopyToInternal( if (hasher != null) { hasher.GetFinalHash(checksumBuffer.Span); - (ReadOnlyMemory checksum, StorageChecksumAlgorithm _) - = ContentHasher.GetResponseChecksumOrDefault(response.GetRawResponse()); - if (!checksumBuffer.Span.SequenceEqual(checksum.Span)) - { - throw Errors.HashMismatchOnStreamedDownload(response.Value.Details.ContentRange); + (ReadOnlyMemory checksum, StorageChecksumAlgorithm _) + = ContentHasher.GetResponseChecksumOrDefault(response.GetRawResponse()); + if (!checksumBuffer.Span.SequenceEqual(checksum.Span)) + { + throw Errors.HashMismatchOnStreamedDownload(response.Value.Details.ContentRange); + } } - } } private IEnumerable GetRanges(long initialLength, long totalLength) diff --git a/sdk/storage/Azure.Storage.Blobs/tests/BlobBaseClientTransferValidationTests.cs b/sdk/storage/Azure.Storage.Blobs/tests/BlobBaseClientTransferValidationTests.cs index 5653e40af16e3..76d807835873c 100644 --- a/sdk/storage/Azure.Storage.Blobs/tests/BlobBaseClientTransferValidationTests.cs +++ b/sdk/storage/Azure.Storage.Blobs/tests/BlobBaseClientTransferValidationTests.cs @@ -95,59 +95,6 @@ public override void TestAutoResolve() } #region Added Tests - [TestCaseSource("GetValidationAlgorithms")] - public async Task ExpectedDownloadStreamingStreamTypeReturned(StorageChecksumAlgorithm algorithm) - { - await using var test = await GetDisposingContainerAsync(); - - // Arrange - var data = GetRandomBuffer(Constants.KB); - BlobClient blob = InstrumentClient(test.Container.GetBlobClient(GetNewResourceName())); - using (var stream = new MemoryStream(data)) - { - await blob.UploadAsync(stream); - } - // don't make options instance at all for no hash request - DownloadTransferValidationOptions transferValidation = algorithm == StorageChecksumAlgorithm.None - ? default - : new DownloadTransferValidationOptions { ChecksumAlgorithm = algorithm }; - - // Act - Response response = await blob.DownloadStreamingAsync(new BlobDownloadOptions - { - TransferValidation = transferValidation, - Range = new HttpRange(length: data.Length) - }); - - // Assert - // validated stream is buffered - Assert.AreEqual(typeof(MemoryStream), response.Value.Content.GetType()); - } - - [Test] - public async Task ExpectedDownloadStreamingStreamTypeReturned_None() - { - await using var test = await GetDisposingContainerAsync(); - - // Arrange - var data = GetRandomBuffer(Constants.KB); - BlobClient blob = InstrumentClient(test.Container.GetBlobClient(GetNewResourceName())); - using (var stream = new MemoryStream(data)) - { - await blob.UploadAsync(stream); - } - - // Act - Response response = await blob.DownloadStreamingAsync(new BlobDownloadOptions - { - Range = new HttpRange(length: data.Length) - }); - - // Assert - // unvalidated stream type is private; just check we didn't get back a buffered stream - Assert.AreNotEqual(typeof(MemoryStream), response.Value.Content.GetType()); - } - [Test] public virtual async Task OlderServiceVersionThrowsOnStructuredMessage() { diff --git a/sdk/storage/Azure.Storage.Blobs/tests/PartitionedDownloaderTests.cs b/sdk/storage/Azure.Storage.Blobs/tests/PartitionedDownloaderTests.cs index d8d4756a510c1..af408264c5bfa 100644 --- a/sdk/storage/Azure.Storage.Blobs/tests/PartitionedDownloaderTests.cs +++ b/sdk/storage/Azure.Storage.Blobs/tests/PartitionedDownloaderTests.cs @@ -305,7 +305,7 @@ public Response GetStream(HttpRange range, BlobRequ ContentHash = new byte[] { 1, 2, 3 }, LastModified = DateTimeOffset.Now, Metadata = new Dictionary() { { "meta", "data" } }, - ContentRange = $"bytes {range.Offset}-{range.Offset + contentLength}/{_length}", + ContentRange = $"bytes {range.Offset}-{Math.Max(1, range.Offset + contentLength - 1)}/{_length}", ETag = s_etag, ContentEncoding = "test", CacheControl = "test", diff --git a/sdk/storage/Azure.Storage.Common/src/Shared/LazyLoadingReadOnlyStream.cs b/sdk/storage/Azure.Storage.Common/src/Shared/LazyLoadingReadOnlyStream.cs index c3e9c641c3fea..fe2db427bef02 100644 --- a/sdk/storage/Azure.Storage.Common/src/Shared/LazyLoadingReadOnlyStream.cs +++ b/sdk/storage/Azure.Storage.Common/src/Shared/LazyLoadingReadOnlyStream.cs @@ -249,41 +249,9 @@ private async Task DownloadInternal(bool async, CancellationToken cancellat response = await _downloadInternalFunc(range, _validationOptions, async, cancellationToken).ConfigureAwait(false); using Stream networkStream = response.Value.Content; - - // The number of bytes we just downloaded. - long downloadSize = GetResponseRange(response.GetRawResponse()).Length.Value; - - // The number of bytes we copied in the last loop. - int copiedBytes; - - // Bytes we have copied so far. - int totalCopiedBytes = 0; - - // Bytes remaining to copy. It is save to truncate the long because we asked for a max of int _buffer size bytes. - int remainingBytes = (int)downloadSize; - - do - { - if (async) - { - copiedBytes = await networkStream.ReadAsync( - buffer: _buffer, - offset: totalCopiedBytes, - count: remainingBytes, - cancellationToken: cancellationToken).ConfigureAwait(false); - } - else - { - copiedBytes = networkStream.Read( - buffer: _buffer, - offset: totalCopiedBytes, - count: remainingBytes); - } - - totalCopiedBytes += copiedBytes; - remainingBytes -= copiedBytes; - } - while (copiedBytes != 0); + // use stream copy to ensure consumption of any trailing metadata (e.g. structured message) + // allow buffer limits to catch the error of data size mismatch + int totalCopiedBytes = (int) await networkStream.CopyToInternal(new MemoryStream(_buffer), async, cancellationToken).ConfigureAwait((false)); _bufferPosition = 0; _bufferLength = totalCopiedBytes; @@ -291,7 +259,7 @@ private async Task DownloadInternal(bool async, CancellationToken cancellat // if we deferred transactional hash validation on download, validate now // currently we always defer but that may change - if (_validationOptions != default && _validationOptions.ChecksumAlgorithm != StorageChecksumAlgorithm.None && !_validationOptions.AutoValidateChecksum) + if (_validationOptions != default && _validationOptions.ChecksumAlgorithm == StorageChecksumAlgorithm.MD5 && !_validationOptions.AutoValidateChecksum) // TODO better condition { ContentHasher.AssertResponseHashMatch(_buffer, _bufferPosition, _bufferLength, _validationOptions.ChecksumAlgorithm, response.GetRawResponse()); } diff --git a/sdk/storage/Azure.Storage.Common/src/Shared/PooledMemoryStream.cs b/sdk/storage/Azure.Storage.Common/src/Shared/PooledMemoryStream.cs index 3e218d18a90af..6070329d10d3d 100644 --- a/sdk/storage/Azure.Storage.Common/src/Shared/PooledMemoryStream.cs +++ b/sdk/storage/Azure.Storage.Common/src/Shared/PooledMemoryStream.cs @@ -251,7 +251,7 @@ public override int Read(byte[] buffer, int offset, int count) Length - Position, bufferCount - (Position - offsetOfBuffer), count - read); - Array.Copy(currentBuffer, Position - offsetOfBuffer, buffer, read, toCopy); + Array.Copy(currentBuffer, Position - offsetOfBuffer, buffer, offset + read, toCopy); read += toCopy; Position += toCopy; } diff --git a/sdk/storage/Azure.Storage.Common/src/Shared/StreamExtensions.cs b/sdk/storage/Azure.Storage.Common/src/Shared/StreamExtensions.cs index 31f121d414ea4..c8803ecf421e7 100644 --- a/sdk/storage/Azure.Storage.Common/src/Shared/StreamExtensions.cs +++ b/sdk/storage/Azure.Storage.Common/src/Shared/StreamExtensions.cs @@ -1,6 +1,8 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License. +using System; +using System.Buffers; using System.IO; using System.Threading; using System.Threading.Tasks; @@ -48,7 +50,7 @@ public static async Task WriteInternal( } } - public static Task CopyToInternal( + public static Task CopyToInternal( this Stream src, Stream dest, bool async, @@ -79,21 +81,33 @@ public static Task CopyToInternal( /// Cancellation token for the operation. /// /// - public static async Task CopyToInternal( + public static async Task CopyToInternal( this Stream src, Stream dest, int bufferSize, bool async, CancellationToken cancellationToken) { + using IDisposable _ = ArrayPool.Shared.RentDisposable(bufferSize, out byte[] buffer); + long totalRead = 0; + int read; if (async) { - await src.CopyToAsync(dest, bufferSize, cancellationToken).ConfigureAwait(false); + while (0 < (read = await src.ReadAsync(buffer, 0, buffer.Length, cancellationToken).ConfigureAwait(false))) + { + totalRead += read; + await dest.WriteAsync(buffer, 0, read, cancellationToken).ConfigureAwait(false); + } } else { - src.CopyTo(dest, bufferSize); + while (0 < (read = src.Read(buffer, 0, buffer.Length))) + { + totalRead += read; + dest.Write(buffer, 0, read); + } } + return totalRead; } } } diff --git a/sdk/storage/Azure.Storage.Common/src/Shared/StructuredMessageDecodingRetriableStream.cs b/sdk/storage/Azure.Storage.Common/src/Shared/StructuredMessageDecodingRetriableStream.cs index 444fe3eb2e0a9..fe2d6697a4621 100644 --- a/sdk/storage/Azure.Storage.Common/src/Shared/StructuredMessageDecodingRetriableStream.cs +++ b/sdk/storage/Azure.Storage.Common/src/Shared/StructuredMessageDecodingRetriableStream.cs @@ -19,6 +19,7 @@ internal class StructuredMessageDecodingRetriableStream : Stream private long _decodedBytesRead; private readonly List _decodedDatas; + private readonly Action _onComplete; private readonly Func _decodingStreamFactory; private readonly Func> _decodingAsyncStreamFactory; @@ -28,6 +29,7 @@ public StructuredMessageDecodingRetriableStream( StructuredMessageDecodingStream.DecodedData initialDecodedData, Func decodingStreamFactory, Func> decodingAsyncStreamFactory, + Action onComplete, ResponseClassifier responseClassifier, int maxRetries) { @@ -35,6 +37,7 @@ public StructuredMessageDecodingRetriableStream( _decodingAsyncStreamFactory = decodingAsyncStreamFactory; _innerRetriable = RetriableStream.Create(initialDecodingStream, StreamFactory, StreamFactoryAsync, responseClassifier, maxRetries); _decodedDatas = new() { initialDecodedData }; + _onComplete = onComplete; } private Stream StreamFactory(long _) @@ -86,11 +89,22 @@ protected override void Dispose(bool disposing) _innerRetriable.Dispose(); } + private void OnCompleted() + { + StructuredMessageDecodingStream.DecodedData final = new(); + // TODO + _onComplete?.Invoke(final); + } + #region Read public override int Read(byte[] buffer, int offset, int count) { int read = _innerRetriable.Read(buffer, offset, count); _decodedBytesRead += read; + if (read == 0) + { + OnCompleted(); + } return read; } @@ -98,6 +112,10 @@ public override async Task ReadAsync(byte[] buffer, int offset, int count, { int read = await _innerRetriable.ReadAsync(buffer, offset, count, cancellationToken).ConfigureAwait(false); _decodedBytesRead += read; + if (read == 0) + { + OnCompleted(); + } return read; } @@ -106,6 +124,10 @@ public override int Read(Span buffer) { int read = _innerRetriable.Read(buffer); _decodedBytesRead += read; + if (read == 0) + { + OnCompleted(); + } return read; } @@ -113,6 +135,10 @@ public override async ValueTask ReadAsync(Memory buffer, Cancellation { int read = await _innerRetriable.ReadAsync(buffer, cancellationToken).ConfigureAwait(false); _decodedBytesRead += read; + if (read == 0) + { + OnCompleted(); + } return read; } #endif @@ -121,6 +147,10 @@ public override int ReadByte() { int val = _innerRetriable.ReadByte(); _decodedBytesRead += 1; + if (val == -1) + { + OnCompleted(); + } return val; } @@ -128,6 +158,10 @@ public override int EndRead(IAsyncResult asyncResult) { int read = _innerRetriable.EndRead(asyncResult); _decodedBytesRead += read; + if (read == 0) + { + OnCompleted(); + } return read; } #endregion diff --git a/sdk/storage/Azure.Storage.Common/src/Shared/StructuredMessagePrecalculatedCrcWrapperStream.cs b/sdk/storage/Azure.Storage.Common/src/Shared/StructuredMessagePrecalculatedCrcWrapperStream.cs new file mode 100644 index 0000000000000..3569ef4339735 --- /dev/null +++ b/sdk/storage/Azure.Storage.Common/src/Shared/StructuredMessagePrecalculatedCrcWrapperStream.cs @@ -0,0 +1,451 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +using System; +using System.Buffers; +using System.IO; +using System.Threading; +using System.Threading.Tasks; +using Azure.Core.Pipeline; +using Azure.Storage.Common; + +namespace Azure.Storage.Shared; + +internal class StructuredMessagePrecalculatedCrcWrapperStream : Stream +{ + private readonly Stream _innerStream; + + private readonly int _streamHeaderLength; + private readonly int _streamFooterLength; + private readonly int _segmentHeaderLength; + private readonly int _segmentFooterLength; + + private bool _disposed; + + private readonly byte[] _crc; + + public override bool CanRead => true; + + public override bool CanWrite => false; + + public override bool CanSeek => _innerStream.CanSeek; + + public override bool CanTimeout => _innerStream.CanTimeout; + + public override int ReadTimeout => _innerStream.ReadTimeout; + + public override int WriteTimeout => _innerStream.WriteTimeout; + + public override long Length => + _streamHeaderLength + _streamFooterLength + + _segmentHeaderLength + _segmentFooterLength + + _innerStream.Length; + + #region Position + private enum SMRegion + { + StreamHeader, + StreamFooter, + SegmentHeader, + SegmentFooter, + SegmentContent, + } + + private SMRegion _currentRegion = SMRegion.StreamHeader; + private int _currentRegionPosition = 0; + + private long _maxSeekPosition = 0; + + public override long Position + { + get + { + return _currentRegion switch + { + SMRegion.StreamHeader => _currentRegionPosition, + SMRegion.SegmentHeader => _innerStream.Position + + _streamHeaderLength + + _currentRegionPosition, + SMRegion.SegmentContent => _streamHeaderLength + + _segmentHeaderLength + + _innerStream.Position, + SMRegion.SegmentFooter => _streamHeaderLength + + _segmentHeaderLength + + _innerStream.Length + + _currentRegionPosition, + SMRegion.StreamFooter => _streamHeaderLength + + _segmentHeaderLength + + _innerStream.Length + + _segmentFooterLength + + _currentRegionPosition, + _ => throw new InvalidDataException($"{nameof(StructuredMessageEncodingStream)} invalid state."), + }; + } + set + { + Argument.AssertInRange(value, 0, _maxSeekPosition, nameof(value)); + if (value < _streamHeaderLength) + { + _currentRegion = SMRegion.StreamHeader; + _currentRegionPosition = (int)value; + _innerStream.Position = 0; + return; + } + if (value < _streamHeaderLength + _segmentHeaderLength) + { + _currentRegion = SMRegion.SegmentHeader; + _currentRegionPosition = (int)(value - _streamHeaderLength); + _innerStream.Position = 0; + return; + } + if (value < _streamHeaderLength + _segmentHeaderLength + _innerStream.Length) + { + _currentRegion = SMRegion.SegmentContent; + _currentRegionPosition = (int)(value - _streamHeaderLength - _segmentHeaderLength); + _innerStream.Position = value - _streamHeaderLength - _segmentHeaderLength; + return; + } + if (value < _streamHeaderLength + _segmentHeaderLength + _innerStream.Length + _segmentFooterLength) + { + _currentRegion = SMRegion.SegmentFooter; + _currentRegionPosition = (int)(value - _streamHeaderLength - _segmentHeaderLength - _innerStream.Length); + _innerStream.Position = _innerStream.Length; + return; + } + + _currentRegion = SMRegion.StreamFooter; + _currentRegionPosition = (int)(value - _streamHeaderLength - _segmentHeaderLength - _innerStream.Length - _segmentFooterLength); + _innerStream.Position = _innerStream.Length; + } + } + #endregion + + public StructuredMessagePrecalculatedCrcWrapperStream( + Stream innerStream, + ReadOnlySpan precalculatedCrc) + { + Argument.AssertNotNull(innerStream, nameof(innerStream)); + if (innerStream.GetLengthOrDefault() == default) + { + throw new ArgumentException("Stream must have known length.", nameof(innerStream)); + } + if (innerStream.Position != 0) + { + throw new ArgumentException("Stream must be at starting position.", nameof(innerStream)); + } + + _streamHeaderLength = StructuredMessage.V1_0.StreamHeaderLength; + _streamFooterLength = StructuredMessage.Crc64Length; + _segmentHeaderLength = StructuredMessage.V1_0.SegmentHeaderLength; + _segmentFooterLength = StructuredMessage.Crc64Length; + + _crc = ArrayPool.Shared.Rent(StructuredMessage.Crc64Length); + precalculatedCrc.CopyTo(_crc); + + _innerStream = innerStream; + } + + #region Write + public override void Flush() => throw new NotSupportedException(); + + public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException(); + + public override void SetLength(long value) => throw new NotSupportedException(); + #endregion + + #region Read + public override int Read(byte[] buffer, int offset, int count) + => ReadInternal(buffer, offset, count, async: false, cancellationToken: default).EnsureCompleted(); + + public override async Task ReadAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) + => await ReadInternal(buffer, offset, count, async: true, cancellationToken).ConfigureAwait(false); + + private async ValueTask ReadInternal(byte[] buffer, int offset, int count, bool async, CancellationToken cancellationToken) + { + int totalRead = 0; + bool readInner = false; + while (totalRead < count && Position < Length) + { + int subreadOffset = offset + totalRead; + int subreadCount = count - totalRead; + switch (_currentRegion) + { + case SMRegion.StreamHeader: + totalRead += ReadFromStreamHeader(new Span(buffer, subreadOffset, subreadCount)); + break; + case SMRegion.StreamFooter: + totalRead += ReadFromStreamFooter(new Span(buffer, subreadOffset, subreadCount)); + break; + case SMRegion.SegmentHeader: + totalRead += ReadFromSegmentHeader(new Span(buffer, subreadOffset, subreadCount)); + break; + case SMRegion.SegmentFooter: + totalRead += ReadFromSegmentFooter(new Span(buffer, subreadOffset, subreadCount)); + break; + case SMRegion.SegmentContent: + // don't double read from stream. Allow caller to multi-read when desired. + if (readInner) + { + UpdateLatestPosition(); + return totalRead; + } + totalRead += await ReadFromInnerStreamInternal( + buffer, subreadOffset, subreadCount, async, cancellationToken).ConfigureAwait(false); + readInner = true; + break; + default: + break; + } + } + UpdateLatestPosition(); + return totalRead; + } + +#if NETSTANDARD2_1_OR_GREATER || NETCOREAPP3_0_OR_GREATER + public override int Read(Span buffer) + { + int totalRead = 0; + bool readInner = false; + while (totalRead < buffer.Length && Position < Length) + { + switch (_currentRegion) + { + case SMRegion.StreamHeader: + totalRead += ReadFromStreamHeader(buffer.Slice(totalRead)); + break; + case SMRegion.StreamFooter: + totalRead += ReadFromStreamFooter(buffer.Slice(totalRead)); + break; + case SMRegion.SegmentHeader: + totalRead += ReadFromSegmentHeader(buffer.Slice(totalRead)); + break; + case SMRegion.SegmentFooter: + totalRead += ReadFromSegmentFooter(buffer.Slice(totalRead)); + break; + case SMRegion.SegmentContent: + // don't double read from stream. Allow caller to multi-read when desired. + if (readInner) + { + UpdateLatestPosition(); + return totalRead; + } + totalRead += ReadFromInnerStream(buffer.Slice(totalRead)); + readInner = true; + break; + default: + break; + } + } + UpdateLatestPosition(); + return totalRead; + } + + public override async ValueTask ReadAsync(Memory buffer, CancellationToken cancellationToken = default) + { + int totalRead = 0; + bool readInner = false; + while (totalRead < buffer.Length && Position < Length) + { + switch (_currentRegion) + { + case SMRegion.StreamHeader: + totalRead += ReadFromStreamHeader(buffer.Slice(totalRead).Span); + break; + case SMRegion.StreamFooter: + totalRead += ReadFromStreamFooter(buffer.Slice(totalRead).Span); + break; + case SMRegion.SegmentHeader: + totalRead += ReadFromSegmentHeader(buffer.Slice(totalRead).Span); + break; + case SMRegion.SegmentFooter: + totalRead += ReadFromSegmentFooter(buffer.Slice(totalRead).Span); + break; + case SMRegion.SegmentContent: + // don't double read from stream. Allow caller to multi-read when desired. + if (readInner) + { + UpdateLatestPosition(); + return totalRead; + } + totalRead += await ReadFromInnerStreamAsync(buffer.Slice(totalRead), cancellationToken).ConfigureAwait(false); + readInner = true; + break; + default: + break; + } + } + UpdateLatestPosition(); + return totalRead; + } +#endif + + #region Read Headers/Footers + private int ReadFromStreamHeader(Span buffer) + { + int read = Math.Min(buffer.Length, _streamHeaderLength - _currentRegionPosition); + using IDisposable _ = StructuredMessage.V1_0.GetStreamHeaderBytes( + ArrayPool.Shared, + out Memory headerBytes, + Length, + StructuredMessage.Flags.StorageCrc64, + totalSegments: 1); + headerBytes.Slice(_currentRegionPosition, read).Span.CopyTo(buffer); + _currentRegionPosition += read; + + if (_currentRegionPosition == _streamHeaderLength) + { + _currentRegion = SMRegion.SegmentHeader; + _currentRegionPosition = 0; + } + + return read; + } + + private int ReadFromStreamFooter(Span buffer) + { + int read = Math.Min(buffer.Length, _segmentFooterLength - _currentRegionPosition); + if (read <= 0) + { + return 0; + } + + using IDisposable _ = StructuredMessage.V1_0.GetStreamFooterBytes( + ArrayPool.Shared, + out Memory footerBytes, + new ReadOnlySpan(_crc, 0, StructuredMessage.Crc64Length)); + footerBytes.Slice(_currentRegionPosition, read).Span.CopyTo(buffer); + _currentRegionPosition += read; + + return read; + } + + private int ReadFromSegmentHeader(Span buffer) + { + int read = Math.Min(buffer.Length, _segmentHeaderLength - _currentRegionPosition); + using IDisposable _ = StructuredMessage.V1_0.GetSegmentHeaderBytes( + ArrayPool.Shared, + out Memory headerBytes, + segmentNum: 1, + _innerStream.Length); + headerBytes.Slice(_currentRegionPosition, read).Span.CopyTo(buffer); + _currentRegionPosition += read; + + if (_currentRegionPosition == _segmentHeaderLength) + { + _currentRegion = SMRegion.SegmentContent; + _currentRegionPosition = 0; + } + + return read; + } + + private int ReadFromSegmentFooter(Span buffer) + { + int read = Math.Min(buffer.Length, _segmentFooterLength - _currentRegionPosition); + if (read < 0) + { + return 0; + } + + using IDisposable _ = StructuredMessage.V1_0.GetSegmentFooterBytes( + ArrayPool.Shared, + out Memory headerBytes, + new ReadOnlySpan(_crc, 0, StructuredMessage.Crc64Length)); + headerBytes.Slice(_currentRegionPosition, read).Span.CopyTo(buffer); + _currentRegionPosition += read; + + if (_currentRegionPosition == _segmentFooterLength) + { + _currentRegion = _innerStream.Position == _innerStream.Length + ? SMRegion.StreamFooter : SMRegion.SegmentHeader; + _currentRegionPosition = 0; + } + + return read; + } + #endregion + + #region ReadUnderlyingStream + private void CleanupContentSegment() + { + if (_innerStream.Position >= _innerStream.Length) + { + _currentRegion = SMRegion.SegmentFooter; + _currentRegionPosition = 0; + } + } + + private async ValueTask ReadFromInnerStreamInternal( + byte[] buffer, int offset, int count, bool async, CancellationToken cancellationToken) + { + int read = async + ? await _innerStream.ReadAsync(buffer, offset, count).ConfigureAwait(false) + : _innerStream.Read(buffer, offset, count); + _currentRegionPosition += read; + CleanupContentSegment(); + return read; + } + +#if NETSTANDARD2_1_OR_GREATER || NETCOREAPP3_0_OR_GREATER + private int ReadFromInnerStream(Span buffer) + { + int read = _innerStream.Read(buffer); + _currentRegionPosition += read; + CleanupContentSegment(); + return read; + } + + private async ValueTask ReadFromInnerStreamAsync(Memory buffer, CancellationToken cancellationToken) + { + int read = await _innerStream.ReadAsync(buffer, cancellationToken).ConfigureAwait(false); + _currentRegionPosition += read; + CleanupContentSegment(); + return read; + } +#endif + #endregion + + // don't allow stream to seek too far forward. track how far the stream has been naturally read. + private void UpdateLatestPosition() + { + if (_maxSeekPosition < Position) + { + _maxSeekPosition = Position; + } + } + #endregion + + public override long Seek(long offset, SeekOrigin origin) + { + switch (origin) + { + case SeekOrigin.Begin: + Position = offset; + break; + case SeekOrigin.Current: + Position += offset; + break; + case SeekOrigin.End: + Position = Length + offset; + break; + } + return Position; + } + + protected override void Dispose(bool disposing) + { + base.Dispose(disposing); + + if (_disposed) + { + return; + } + + if (disposing) + { + ArrayPool.Shared.Return(_crc); + _innerStream.Dispose(); + _disposed = true; + } + } +} diff --git a/sdk/storage/Azure.Storage.Common/tests/Azure.Storage.Common.Tests.csproj b/sdk/storage/Azure.Storage.Common/tests/Azure.Storage.Common.Tests.csproj index c4f9b715a692a..6a0dc0506be51 100644 --- a/sdk/storage/Azure.Storage.Common/tests/Azure.Storage.Common.Tests.csproj +++ b/sdk/storage/Azure.Storage.Common/tests/Azure.Storage.Common.Tests.csproj @@ -18,6 +18,7 @@ + @@ -52,6 +53,7 @@ + diff --git a/sdk/storage/Azure.Storage.Common/tests/Shared/ObserveStructuredMessagePolicy.cs b/sdk/storage/Azure.Storage.Common/tests/Shared/ObserveStructuredMessagePolicy.cs new file mode 100644 index 0000000000000..1414f4ec80076 --- /dev/null +++ b/sdk/storage/Azure.Storage.Common/tests/Shared/ObserveStructuredMessagePolicy.cs @@ -0,0 +1,85 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +using System; +using System.Collections.Generic; +using System.IO; +using Azure.Core; +using Azure.Core.Pipeline; +using Azure.Storage.Shared; + +namespace Azure.Storage.Test.Shared +{ + internal class ObserveStructuredMessagePolicy : HttpPipelineSynchronousPolicy + { + private readonly HashSet _requestScopes = new(); + + private readonly HashSet _responseScopes = new(); + + public ObserveStructuredMessagePolicy() + { + } + + public override void OnSendingRequest(HttpMessage message) + { + if (_requestScopes.Count > 0) + { + byte[] encodedContent; + byte[] underlyingContent; + StructuredMessageDecodingStream.DecodedData decodedData; + using (MemoryStream ms = new()) + { + message.Request.Content.WriteTo(ms, default); + encodedContent = ms.ToArray(); + using (MemoryStream ms2 = new()) + { + (Stream s, decodedData) = StructuredMessageDecodingStream.WrapStream(new MemoryStream(encodedContent)); + s.CopyTo(ms2); + underlyingContent = ms2.ToArray(); + } + } + } + } + + public override void OnReceivedResponse(HttpMessage message) + { + } + + public IDisposable CheckRequestScope() => CheckMessageScope.CheckRequestScope(this); + + public IDisposable CheckResponseScope() => CheckMessageScope.CheckResponseScope(this); + + private class CheckMessageScope : IDisposable + { + private bool _isRequestScope; + private ObserveStructuredMessagePolicy _policy; + + public static CheckMessageScope CheckRequestScope(ObserveStructuredMessagePolicy policy) + { + CheckMessageScope result = new() + { + _isRequestScope = true, + _policy = policy + }; + result._policy._requestScopes.Add(result); + return result; + } + + public static CheckMessageScope CheckResponseScope(ObserveStructuredMessagePolicy policy) + { + CheckMessageScope result = new() + { + _isRequestScope = false, + _policy = policy + }; + result._policy._responseScopes.Add(result); + return result; + } + + public void Dispose() + { + (_isRequestScope ? _policy._requestScopes : _policy._responseScopes).Remove(this); + } + } + } +} diff --git a/sdk/storage/Azure.Storage.Common/tests/Shared/RequestExtensions.cs b/sdk/storage/Azure.Storage.Common/tests/Shared/RequestExtensions.cs new file mode 100644 index 0000000000000..0f9cdb07773f7 --- /dev/null +++ b/sdk/storage/Azure.Storage.Common/tests/Shared/RequestExtensions.cs @@ -0,0 +1,27 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +using System.Linq; +using System.Text; +using Azure.Core; +using NUnit.Framework; + +namespace Azure.Storage; + +public static partial class RequestExtensions +{ + public static string AssertHeaderPresent(this Request request, string headerName) + { + if (request.Headers.TryGetValue(headerName, out string value)) + { + return headerName == Constants.StructuredMessage.CrcStructuredMessageHeader ? null : value; + } + StringBuilder sb = new StringBuilder() + .AppendLine($"`{headerName}` expected on request but was not found.") + .AppendLine($"{request.Method} {request.Uri}") + .AppendLine(string.Join("\n", request.Headers.Select(h => $"{h.Name}: {h.Value}s"))) + ; + Assert.Fail(sb.ToString()); + return null; + } +} diff --git a/sdk/storage/Azure.Storage.Common/tests/Shared/TamperStreamContentsPolicy.cs b/sdk/storage/Azure.Storage.Common/tests/Shared/TamperStreamContentsPolicy.cs index f4198e9dfd532..7e6c78117f53b 100644 --- a/sdk/storage/Azure.Storage.Common/tests/Shared/TamperStreamContentsPolicy.cs +++ b/sdk/storage/Azure.Storage.Common/tests/Shared/TamperStreamContentsPolicy.cs @@ -14,7 +14,7 @@ internal class TamperStreamContentsPolicy : HttpPipelineSynchronousPolicy /// /// Default tampering that changes the first byte of the stream. /// - private static readonly Func _defaultStreamTransform = stream => + private static Func GetTamperByteStreamTransform(long position) => stream => { if (stream is not MemoryStream) { @@ -23,10 +23,10 @@ internal class TamperStreamContentsPolicy : HttpPipelineSynchronousPolicy stream = buffer; } - stream.Position = 0; + stream.Position = position; var firstByte = stream.ReadByte(); - stream.Position = 0; + stream.Position = position; stream.WriteByte((byte)((firstByte + 1) % byte.MaxValue)); stream.Position = 0; @@ -37,9 +37,12 @@ internal class TamperStreamContentsPolicy : HttpPipelineSynchronousPolicy public TamperStreamContentsPolicy(Func streamTransform = default) { - _streamTransform = streamTransform ?? _defaultStreamTransform; + _streamTransform = streamTransform ?? GetTamperByteStreamTransform(0); } + public static TamperStreamContentsPolicy TamperByteAt(long position) + => new(GetTamperByteStreamTransform(position)); + public bool TransformRequestBody { get; set; } public bool TransformResponseBody { get; set; } diff --git a/sdk/storage/Azure.Storage.Common/tests/Shared/TransferValidationTestBase.cs b/sdk/storage/Azure.Storage.Common/tests/Shared/TransferValidationTestBase.cs index 201092978627c..ff06ddc4cbd92 100644 --- a/sdk/storage/Azure.Storage.Common/tests/Shared/TransferValidationTestBase.cs +++ b/sdk/storage/Azure.Storage.Common/tests/Shared/TransferValidationTestBase.cs @@ -196,18 +196,12 @@ protected string GetNewResourceName() internal static Action GetRequestChecksumHeaderAssertion(StorageChecksumAlgorithm algorithm, Func isChecksumExpected = default, byte[] expectedChecksum = default) { // action to assert a request header is as expected - void AssertChecksum(RequestHeaders headers, string headerName) + void AssertChecksum(Request req, string headerName) { - if (headers.TryGetValue(headerName, out string checksum)) - { - if (expectedChecksum != default) - { - Assert.AreEqual(Convert.ToBase64String(expectedChecksum), checksum); - } - } - else + string checksum = req.AssertHeaderPresent(headerName); + if (expectedChecksum != default) { - Assert.Fail($"{headerName} expected on request but was not found."); + Assert.AreEqual(Convert.ToBase64String(expectedChecksum), checksum); } }; @@ -222,10 +216,10 @@ void AssertChecksum(RequestHeaders headers, string headerName) switch (algorithm.ResolveAuto()) { case StorageChecksumAlgorithm.MD5: - AssertChecksum(request.Headers, "Content-MD5"); + AssertChecksum(request, "Content-MD5"); break; case StorageChecksumAlgorithm.StorageCrc64: - AssertChecksum(request.Headers, "x-ms-content-crc64"); + AssertChecksum(request, Constants.StructuredMessage.CrcStructuredMessageHeader); break; default: throw new Exception($"Bad {nameof(StorageChecksumAlgorithm)} provided to {nameof(GetRequestChecksumHeaderAssertion)}."); @@ -308,7 +302,7 @@ void AssertChecksum(ResponseHeaders headers, string headerName) AssertChecksum(response.Headers, "Content-MD5"); break; case StorageChecksumAlgorithm.StorageCrc64: - AssertChecksum(response.Headers, "x-ms-content-crc64"); + AssertChecksum(response.Headers, Constants.StructuredMessage.CrcStructuredMessageHeader); break; default: throw new Exception($"Bad {nameof(StorageChecksumAlgorithm)} provided to {nameof(GetRequestChecksumHeaderAssertion)}."); @@ -355,7 +349,7 @@ internal static void AssertWriteChecksumMismatch( var exception = ThrowsOrInconclusiveAsync(writeAction); if (expectStructuredMessage) { - Assert.That(exception.ErrorCode, Is.EqualTo("InvalidStructuredMessage")); + Assert.That(exception.ErrorCode, Is.EqualTo("Crc64Mismatch")); } else { @@ -481,7 +475,9 @@ public virtual async Task UploadPartitionUsePrecalculatedHash(StorageChecksumAlg // make pipeline assertion for checking precalculated checksum was present on upload // precalculated partition upload will never use structured message. always check header - var assertion = GetRequestChecksumHeaderAssertion(algorithm, expectedChecksum: precalculatedChecksum); + var assertion = GetRequestChecksumHeaderAssertion( + algorithm, + expectedChecksum: algorithm.ResolveAuto() == StorageChecksumAlgorithm.StorageCrc64 ? default : precalculatedChecksum); var checksumPipelineAssertion = new AssertMessageContentsPolicy(checkRequest: assertion); var clientOptions = ClientBuilder.GetOptions(); clientOptions.AddPolicy(checksumPipelineAssertion, HttpPipelinePosition.PerCall); @@ -499,7 +495,7 @@ public virtual async Task UploadPartitionUsePrecalculatedHash(StorageChecksumAlg AsyncTestDelegate operation = async () => await UploadPartitionAsync(client, stream, validationOptions); // Assert - AssertWriteChecksumMismatch(operation, algorithm); + AssertWriteChecksumMismatch(operation, algorithm, algorithm.ResolveAuto() == StorageChecksumAlgorithm.StorageCrc64); } } @@ -517,7 +513,7 @@ public virtual async Task UploadPartitionTamperedStreamThrows(StorageChecksumAlg }; // Tamper with stream contents in the pipeline to simulate silent failure in the transit layer - var streamTamperPolicy = new TamperStreamContentsPolicy(); + var streamTamperPolicy = TamperStreamContentsPolicy.TamperByteAt(100); var clientOptions = ClientBuilder.GetOptions(); clientOptions.AddPolicy(streamTamperPolicy, HttpPipelinePosition.PerCall); @@ -650,7 +646,7 @@ public virtual async Task UploadPartitionDisablesDefaultClientValidationOptions( { Assert.Fail($"Hash found when none expected."); } - if (request.Headers.Contains("x-ms-content-crc64")) + if (request.Headers.Contains(Constants.StructuredMessage.CrcStructuredMessage)) { Assert.Fail($"Hash found when none expected."); } @@ -702,7 +698,9 @@ public virtual async Task OpenWriteSuccessfulHashComputation( // make pipeline assertion for checking checksum was present on upload var checksumPipelineAssertion = new AssertMessageContentsPolicy(checkRequest: GetRequestChecksumHeaderAssertion(algorithm)); var clientOptions = ClientBuilder.GetOptions(); + //ObserveStructuredMessagePolicy observe = new(); clientOptions.AddPolicy(checksumPipelineAssertion, HttpPipelinePosition.PerCall); + //clientOptions.AddPolicy(observe, HttpPipelinePosition.BeforeTransport); var client = await GetResourceClientAsync( disposingContainer.Container, @@ -715,6 +713,7 @@ public virtual async Task OpenWriteSuccessfulHashComputation( using var writeStream = await OpenWriteAsync(client, validationOptions, streamBufferSize); // Assert + //using var obsv = observe.CheckRequestScope(); using (checksumPipelineAssertion.CheckRequestScope()) { foreach (var _ in Enumerable.Range(0, streamWrites)) @@ -743,7 +742,7 @@ public virtual async Task OpenWriteMismatchedHashThrows(StorageChecksumAlgorithm // Tamper with stream contents in the pipeline to simulate silent failure in the transit layer var clientOptions = ClientBuilder.GetOptions(); - var tamperPolicy = new TamperStreamContentsPolicy(); + var tamperPolicy = TamperStreamContentsPolicy.TamperByteAt(100); clientOptions.AddPolicy(tamperPolicy, HttpPipelinePosition.PerCall); var client = await GetResourceClientAsync( @@ -873,7 +872,7 @@ public virtual async Task OpenWriteDisablesDefaultClientValidationOptions( { Assert.Fail($"Hash found when none expected."); } - if (request.Headers.Contains("x-ms-content-crc64")) + if (request.Headers.Contains(Constants.StructuredMessage.CrcStructuredMessage)) { Assert.Fail($"Hash found when none expected."); } @@ -1236,7 +1235,7 @@ public virtual async Task ParallelUploadDisablesDefaultClientValidationOptions( { Assert.Fail($"Hash found when none expected."); } - if (request.Headers.Contains("x-ms-content-crc64")) + if (request.Headers.Contains(Constants.StructuredMessage.CrcStructuredMessage)) { Assert.Fail($"Hash found when none expected."); } @@ -1301,15 +1300,17 @@ public virtual async Task ParallelDownloadSuccessfulHashVerification( }; // Act - var dest = new MemoryStream(); + byte[] dest; + using (MemoryStream ms = new()) using (checksumPipelineAssertion.CheckRequestScope()) { - await ParallelDownloadAsync(client, dest, validationOptions, transferOptions); + await ParallelDownloadAsync(client, ms, validationOptions, transferOptions); + dest = ms.ToArray(); } // Assert // Assertion was in the pipeline and the SDK not throwing means the checksum was validated - Assert.IsTrue(dest.ToArray().SequenceEqual(data)); + Assert.IsTrue(dest.SequenceEqual(data)); } [Test] @@ -1474,7 +1475,7 @@ public virtual async Task ParallelDownloadDisablesDefaultClientValidationOptions { Assert.Fail($"Hash found when none expected."); } - if (response.Headers.Contains("x-ms-content-crc64")) + if (response.Headers.Contains(Constants.StructuredMessage.CrcStructuredMessage)) { Assert.Fail($"Hash found when none expected."); } @@ -1682,7 +1683,7 @@ public virtual async Task OpenReadDisablesDefaultClientValidationOptions( { Assert.Fail($"Hash found when none expected."); } - if (response.Headers.Contains("x-ms-content-crc64")) + if (response.Headers.Contains(Constants.StructuredMessage.CrcStructuredMessage)) { Assert.Fail($"Hash found when none expected."); } @@ -1743,7 +1744,7 @@ public virtual async Task DownloadSuccessfulHashVerification(StorageChecksumAlgo Assert.True(response.Headers.Contains("Content-MD5")); break; case StorageChecksumAlgorithm.StorageCrc64: - Assert.True(response.Headers.Contains("x-ms-content-crc64")); + Assert.True(response.Headers.Contains(Constants.StructuredMessage.CrcStructuredMessageHeader)); break; default: Assert.Fail("Test can't validate given algorithm type."); @@ -1834,7 +1835,9 @@ public virtual async Task DownloadHashMismatchThrows( // alter response contents in pipeline, forcing a checksum mismatch on verification step var clientOptions = ClientBuilder.GetOptions(); - clientOptions.AddPolicy(new TamperStreamContentsPolicy() { TransformResponseBody = true }, HttpPipelinePosition.PerCall); + var tamperPolicy = TamperStreamContentsPolicy.TamperByteAt(50); + tamperPolicy.TransformResponseBody = true; + clientOptions.AddPolicy(tamperPolicy, HttpPipelinePosition.PerCall); client = await GetResourceClientAsync( disposingContainer.Container, createResource: false, @@ -1846,7 +1849,7 @@ public virtual async Task DownloadHashMismatchThrows( AsyncTestDelegate operation = async () => await DownloadPartitionAsync(client, dest, validationOptions, new HttpRange(length: data.Length)); // Assert - if (validate) + if (validate || algorithm.ResolveAuto() == StorageChecksumAlgorithm.StorageCrc64) { // SDK responsible for finding bad checksum. Throw. ThrowsOrInconclusiveAsync(operation); @@ -1904,7 +1907,7 @@ public virtual async Task DownloadUsesDefaultClientValidationOptions( Assert.True(response.Headers.Contains("Content-MD5")); break; case StorageChecksumAlgorithm.StorageCrc64: - Assert.True(response.Headers.Contains("x-ms-content-crc64")); + Assert.True(response.Headers.Contains(Constants.StructuredMessage.CrcStructuredMessageHeader)); break; default: Assert.Fail("Test can't validate given algorithm type."); @@ -1964,7 +1967,7 @@ public virtual async Task DownloadOverwritesDefaultClientValidationOptions( Assert.True(response.Headers.Contains("Content-MD5")); break; case StorageChecksumAlgorithm.StorageCrc64: - Assert.True(response.Headers.Contains("x-ms-content-crc64")); + Assert.True(response.Headers.Contains(Constants.StructuredMessage.CrcStructuredMessageHeader)); break; default: Assert.Fail("Test can't validate given algorithm type."); @@ -2003,7 +2006,7 @@ public virtual async Task DownloadDisablesDefaultClientValidationOptions( { Assert.Fail($"Hash found when none expected."); } - if (response.Headers.Contains("x-ms-content-crc64")) + if (response.Headers.Contains(Constants.StructuredMessage.CrcStructuredMessage)) { Assert.Fail($"Hash found when none expected."); } @@ -2026,7 +2029,7 @@ public virtual async Task DownloadDisablesDefaultClientValidationOptions( // Assert // no policies this time; just check response headers Assert.False(response.Headers.Contains("Content-MD5")); - Assert.False(response.Headers.Contains("x-ms-content-crc64")); + Assert.False(response.Headers.Contains(Constants.StructuredMessage.CrcStructuredMessage)); Assert.IsTrue(dest.ToArray().SequenceEqual(data)); } diff --git a/sdk/storage/Azure.Storage.Common/tests/StructuredMessageDecodingRetriableStreamTests.cs b/sdk/storage/Azure.Storage.Common/tests/StructuredMessageDecodingRetriableStreamTests.cs index 666933e546189..39d2a5566b5ff 100644 --- a/sdk/storage/Azure.Storage.Common/tests/StructuredMessageDecodingRetriableStreamTests.cs +++ b/sdk/storage/Azure.Storage.Common/tests/StructuredMessageDecodingRetriableStreamTests.cs @@ -39,7 +39,7 @@ public async ValueTask UninterruptedStream() // mock with a simple MemoryStream rather than an actual StructuredMessageDecodingStream using (Stream src = new MemoryStream(data)) - using (Stream retriableSrc = new StructuredMessageDecodingRetriableStream(src, new(), default, default, default, 1)) + using (Stream retriableSrc = new StructuredMessageDecodingRetriableStream(src, new(), default, default, default, default, 1)) using (Stream dst = new MemoryStream(dest)) { await retriableSrc.CopyToInternal(dst, Async, default); @@ -89,6 +89,7 @@ public async Task Interrupt_DataIntact([Values(true, false)] bool multipleInterr initialDecodedData, offset => Factory(offset, multipleInterrupts), offset => new ValueTask<(Stream DecodingStream, StructuredMessageDecodingStream.DecodedData DecodedData)>(Factory(offset, multipleInterrupts)), + null, AllExceptionsRetry().Object, int.MaxValue)) using (Stream dst = new MemoryStream(dest)) @@ -159,6 +160,7 @@ public async Task Interrupt_AppropriateRewind() initialDecodedData, offset => (mock.Object, new()), offset => new(Task.FromResult((mock.Object, new StructuredMessageDecodingStream.DecodedData()))), + null, AllExceptionsRetry().Object, 1); @@ -215,6 +217,7 @@ public async Task Interrupt_ProperDecode([Values(true, false)] bool multipleInte decodedData, offset => Factory(offset, multipleInterrupts), offset => new ValueTask<(Stream DecodingStream, StructuredMessageDecodingStream.DecodedData DecodedData)>(Factory(offset, multipleInterrupts)), + null, AllExceptionsRetry().Object, int.MaxValue); using Stream dst = new MemoryStream(dest); diff --git a/sdk/storage/Azure.Storage.Common/tests/StructuredMessageEncodingStreamTests.cs b/sdk/storage/Azure.Storage.Common/tests/StructuredMessageEncodingStreamTests.cs index d4c667b937bf5..e0f91dee7de3a 100644 --- a/sdk/storage/Azure.Storage.Common/tests/StructuredMessageEncodingStreamTests.cs +++ b/sdk/storage/Azure.Storage.Common/tests/StructuredMessageEncodingStreamTests.cs @@ -218,6 +218,31 @@ public void NotSupportsFastForwardBeyondLatestRead() Assert.That(() => encodingStream.Position = 123, Throws.TypeOf()); } + [Test] + [Pairwise] + public async Task WrapperStreamCorrectData( + [Values(2048, 2005)] int dataLength, + [Values(8 * Constants.KB, 512, 530, 3)] int readLen) + { + int segmentContentLength = dataLength; + Flags flags = Flags.StorageCrc64; + + byte[] originalData = new byte[dataLength]; + new Random().NextBytes(originalData); + byte[] crc = CrcInline(originalData); + byte[] expectedEncodedData = StructuredMessageHelper.MakeEncodedData(originalData, segmentContentLength, flags); + + Stream encodingStream = new StructuredMessagePrecalculatedCrcWrapperStream(new MemoryStream(originalData), crc); + byte[] encodedData; + using (MemoryStream dest = new()) + { + await CopyStream(encodingStream, dest, readLen); + encodedData = dest.ToArray(); + } + + Assert.That(new Span(encodedData).SequenceEqual(expectedEncodedData)); + } + private static void AssertExpectedStreamHeader(ReadOnlySpan actual, int originalDataLength, Flags flags, int expectedSegments) { int expectedFooterLen = flags.HasFlag(Flags.StorageCrc64) ? Crc64Length : 0; diff --git a/sdk/storage/Azure.Storage.DataMovement.Blobs/src/Azure.Storage.DataMovement.Blobs.csproj b/sdk/storage/Azure.Storage.DataMovement.Blobs/src/Azure.Storage.DataMovement.Blobs.csproj index 6098dcd8ba33d..93e7432f186e3 100644 --- a/sdk/storage/Azure.Storage.DataMovement.Blobs/src/Azure.Storage.DataMovement.Blobs.csproj +++ b/sdk/storage/Azure.Storage.DataMovement.Blobs/src/Azure.Storage.DataMovement.Blobs.csproj @@ -37,6 +37,7 @@ + diff --git a/sdk/storage/Azure.Storage.DataMovement.Blobs/tests/Azure.Storage.DataMovement.Blobs.Tests.csproj b/sdk/storage/Azure.Storage.DataMovement.Blobs/tests/Azure.Storage.DataMovement.Blobs.Tests.csproj index f8b62d0b947e2..214903eb5f9c4 100644 --- a/sdk/storage/Azure.Storage.DataMovement.Blobs/tests/Azure.Storage.DataMovement.Blobs.Tests.csproj +++ b/sdk/storage/Azure.Storage.DataMovement.Blobs/tests/Azure.Storage.DataMovement.Blobs.Tests.csproj @@ -22,11 +22,15 @@ + + + + @@ -40,6 +44,7 @@ + diff --git a/sdk/storage/Azure.Storage.DataMovement.Files.Shares/BlobToFileSharesTests/Azure.Storage.DataMovement.Blobs.Files.Shares.Tests.csproj b/sdk/storage/Azure.Storage.DataMovement.Files.Shares/BlobToFileSharesTests/Azure.Storage.DataMovement.Blobs.Files.Shares.Tests.csproj index a6abde432473f..66a9fea0861a2 100644 --- a/sdk/storage/Azure.Storage.DataMovement.Files.Shares/BlobToFileSharesTests/Azure.Storage.DataMovement.Blobs.Files.Shares.Tests.csproj +++ b/sdk/storage/Azure.Storage.DataMovement.Files.Shares/BlobToFileSharesTests/Azure.Storage.DataMovement.Blobs.Files.Shares.Tests.csproj @@ -35,6 +35,7 @@ + diff --git a/sdk/storage/Azure.Storage.DataMovement.Files.Shares/tests/Azure.Storage.DataMovement.Files.Shares.Tests.csproj b/sdk/storage/Azure.Storage.DataMovement.Files.Shares/tests/Azure.Storage.DataMovement.Files.Shares.Tests.csproj index 8e574bca36a48..d75775beceafd 100644 --- a/sdk/storage/Azure.Storage.DataMovement.Files.Shares/tests/Azure.Storage.DataMovement.Files.Shares.Tests.csproj +++ b/sdk/storage/Azure.Storage.DataMovement.Files.Shares/tests/Azure.Storage.DataMovement.Files.Shares.Tests.csproj @@ -27,6 +27,7 @@ + diff --git a/sdk/storage/Azure.Storage.DataMovement/tests/Azure.Storage.DataMovement.Tests.csproj b/sdk/storage/Azure.Storage.DataMovement/tests/Azure.Storage.DataMovement.Tests.csproj index 8afd7735a0168..21a1ea45f92a0 100644 --- a/sdk/storage/Azure.Storage.DataMovement/tests/Azure.Storage.DataMovement.Tests.csproj +++ b/sdk/storage/Azure.Storage.DataMovement/tests/Azure.Storage.DataMovement.Tests.csproj @@ -34,6 +34,7 @@ + diff --git a/sdk/storage/Azure.Storage.Files.DataLake/assets.json b/sdk/storage/Azure.Storage.Files.DataLake/assets.json index 7329a98a34f40..e1cf4a6fa1c07 100644 --- a/sdk/storage/Azure.Storage.Files.DataLake/assets.json +++ b/sdk/storage/Azure.Storage.Files.DataLake/assets.json @@ -2,5 +2,5 @@ "AssetsRepo": "Azure/azure-sdk-assets", "AssetsRepoPrefixPath": "net", "TagPrefix": "net/storage/Azure.Storage.Files.DataLake", - "Tag": "net/storage/Azure.Storage.Files.DataLake_9c23b9b180" + "Tag": "net/storage/Azure.Storage.Files.DataLake_c7efac8f52" } diff --git a/sdk/storage/Azure.Storage.Files.DataLake/src/Azure.Storage.Files.DataLake.csproj b/sdk/storage/Azure.Storage.Files.DataLake/src/Azure.Storage.Files.DataLake.csproj index 7467863a61be0..8a2e0bcd97d46 100644 --- a/sdk/storage/Azure.Storage.Files.DataLake/src/Azure.Storage.Files.DataLake.csproj +++ b/sdk/storage/Azure.Storage.Files.DataLake/src/Azure.Storage.Files.DataLake.csproj @@ -84,6 +84,7 @@ + diff --git a/sdk/storage/Azure.Storage.Files.DataLake/src/DataLakeFileClient.cs b/sdk/storage/Azure.Storage.Files.DataLake/src/DataLakeFileClient.cs index 6088e970ec9e0..93ca4c3f9a1fd 100644 --- a/sdk/storage/Azure.Storage.Files.DataLake/src/DataLakeFileClient.cs +++ b/sdk/storage/Azure.Storage.Files.DataLake/src/DataLakeFileClient.cs @@ -2339,17 +2339,20 @@ internal virtual async Task AppendInternal( string structuredBodyType = null; if (content != null && validationOptions != null && - validationOptions.ChecksumAlgorithm.ResolveAuto() == StorageChecksumAlgorithm.StorageCrc64 && - validationOptions.PrecalculatedChecksum.IsEmpty) + validationOptions.ChecksumAlgorithm.ResolveAuto() == StorageChecksumAlgorithm.StorageCrc64) { // report progress in terms of caller bytes, not encoded bytes structuredContentLength = contentLength; structuredBodyType = Constants.StructuredMessage.CrcStructuredMessage; content = content.WithNoDispose().WithProgress(progressHandler); - content = new StructuredMessageEncodingStream( - content, - Constants.StructuredMessage.DefaultSegmentContentLength, - StructuredMessage.Flags.StorageCrc64); + content = validationOptions.PrecalculatedChecksum.IsEmpty + ? new StructuredMessageEncodingStream( + content, + Constants.StructuredMessage.DefaultSegmentContentLength, + StructuredMessage.Flags.StorageCrc64) + : new StructuredMessagePrecalculatedCrcWrapperStream( + content, + validationOptions.PrecalculatedChecksum.Span); contentLength = content.Length - content.Position; } else diff --git a/sdk/storage/Azure.Storage.Files.Shares/assets.json b/sdk/storage/Azure.Storage.Files.Shares/assets.json index d4df7130a51d0..7a34db9bef740 100644 --- a/sdk/storage/Azure.Storage.Files.Shares/assets.json +++ b/sdk/storage/Azure.Storage.Files.Shares/assets.json @@ -2,5 +2,5 @@ "AssetsRepo": "Azure/azure-sdk-assets", "AssetsRepoPrefixPath": "net", "TagPrefix": "net/storage/Azure.Storage.Files.Shares", - "Tag": "net/storage/Azure.Storage.Files.Shares_5e5b51e54d" + "Tag": "net/storage/Azure.Storage.Files.Shares_4ec8e7e485" } diff --git a/sdk/storage/Azure.Storage.Files.Shares/tests/Azure.Storage.Files.Shares.Tests.csproj b/sdk/storage/Azure.Storage.Files.Shares/tests/Azure.Storage.Files.Shares.Tests.csproj index 398a4b6367489..d09dd8fe8949f 100644 --- a/sdk/storage/Azure.Storage.Files.Shares/tests/Azure.Storage.Files.Shares.Tests.csproj +++ b/sdk/storage/Azure.Storage.Files.Shares/tests/Azure.Storage.Files.Shares.Tests.csproj @@ -17,6 +17,7 @@ + PreserveNewest diff --git a/sdk/storage/Azure.Storage.Queues/tests/Azure.Storage.Queues.Tests.csproj b/sdk/storage/Azure.Storage.Queues/tests/Azure.Storage.Queues.Tests.csproj index e0a6fab3c753b..4d0334255f041 100644 --- a/sdk/storage/Azure.Storage.Queues/tests/Azure.Storage.Queues.Tests.csproj +++ b/sdk/storage/Azure.Storage.Queues/tests/Azure.Storage.Queues.Tests.csproj @@ -21,6 +21,7 @@ + From 5e2cc227c785485f4f7b1e8f24542897c9f2cfde Mon Sep 17 00:00:00 2001 From: Jocelyn <41338290+jaschrep-msft@users.noreply.github.com> Date: Tue, 23 Jul 2024 11:10:03 -0400 Subject: [PATCH 13/22] Enable fileshare crc (#45124) * impl * testproxy * shares fix * testproxy --- .../src/Shared/StorageVersionExtensions.cs | 2 +- .../Shared/TransferValidationTestBase.cs | 7 +- .../src/Azure.Storage.Files.Shares.csproj | 4 + .../src/Generated/DirectoryRestClient.cs | 2 +- .../src/Generated/FileDownloadHeaders.cs | 4 + .../src/Generated/FileRestClient.cs | 40 +++-- .../src/Generated/FileUploadRangeHeaders.cs | 2 + .../src/Generated/ServiceRestClient.cs | 2 +- .../src/Generated/ShareRestClient.cs | 2 +- .../src/ShareErrors.cs | 15 -- .../src/ShareFileClient.cs | 162 ++++++++++++------ .../src/autorest.md | 6 +- .../ShareFileClientTransferValidationTests.cs | 4 - 13 files changed, 158 insertions(+), 94 deletions(-) diff --git a/sdk/storage/Azure.Storage.Common/src/Shared/StorageVersionExtensions.cs b/sdk/storage/Azure.Storage.Common/src/Shared/StorageVersionExtensions.cs index 2a7bd90fb82a1..44c0973ea9be1 100644 --- a/sdk/storage/Azure.Storage.Common/src/Shared/StorageVersionExtensions.cs +++ b/sdk/storage/Azure.Storage.Common/src/Shared/StorageVersionExtensions.cs @@ -46,7 +46,7 @@ internal static class StorageVersionExtensions /// public const ServiceVersion LatestVersion = #if BlobSDK || QueueSDK || FileSDK || DataLakeSDK || ChangeFeedSDK || DataMovementSDK || BlobDataMovementSDK || ShareDataMovementSDK - ServiceVersion.V2024_11_04; + ServiceVersion.V2025_01_05; #else ERROR_STORAGE_SERVICE_NOT_DEFINED; #endif diff --git a/sdk/storage/Azure.Storage.Common/tests/Shared/TransferValidationTestBase.cs b/sdk/storage/Azure.Storage.Common/tests/Shared/TransferValidationTestBase.cs index ff06ddc4cbd92..a406744f94d46 100644 --- a/sdk/storage/Azure.Storage.Common/tests/Shared/TransferValidationTestBase.cs +++ b/sdk/storage/Azure.Storage.Common/tests/Shared/TransferValidationTestBase.cs @@ -1085,7 +1085,7 @@ public virtual async Task ParallelUploadPrecalculatedComposableHashAccepted(Stor PrecalculatedChecksum = hash }; - var client = await GetResourceClientAsync(disposingContainer.Container, dataLength); + var client = await GetResourceClientAsync(disposingContainer.Container, dataLength, createResource: true); // Act await DoesNotThrowOrInconclusiveAsync( @@ -1733,7 +1733,7 @@ public virtual async Task DownloadSuccessfulHashVerification(StorageChecksumAlgo var validationOptions = new DownloadTransferValidationOptions { ChecksumAlgorithm = algorithm }; // Act - var dest = new MemoryStream(); + using var dest = new MemoryStream(); var response = await DownloadPartitionAsync(client, dest, validationOptions, new HttpRange(length: data.Length)); // Assert @@ -1750,7 +1750,8 @@ public virtual async Task DownloadSuccessfulHashVerification(StorageChecksumAlgo Assert.Fail("Test can't validate given algorithm type."); break; } - Assert.IsTrue(dest.ToArray().SequenceEqual(data)); + var result = dest.ToArray(); + Assert.IsTrue(result.SequenceEqual(data)); } #if BlobSDK || DataLakeSDK diff --git a/sdk/storage/Azure.Storage.Files.Shares/src/Azure.Storage.Files.Shares.csproj b/sdk/storage/Azure.Storage.Files.Shares/src/Azure.Storage.Files.Shares.csproj index 4b568316d32a2..a3e805eeed900 100644 --- a/sdk/storage/Azure.Storage.Files.Shares/src/Azure.Storage.Files.Shares.csproj +++ b/sdk/storage/Azure.Storage.Files.Shares/src/Azure.Storage.Files.Shares.csproj @@ -85,6 +85,10 @@ + + + + diff --git a/sdk/storage/Azure.Storage.Files.Shares/src/Generated/DirectoryRestClient.cs b/sdk/storage/Azure.Storage.Files.Shares/src/Generated/DirectoryRestClient.cs index 961c6ff47ce59..8a2edb8b99134 100644 --- a/sdk/storage/Azure.Storage.Files.Shares/src/Generated/DirectoryRestClient.cs +++ b/sdk/storage/Azure.Storage.Files.Shares/src/Generated/DirectoryRestClient.cs @@ -33,7 +33,7 @@ internal partial class DirectoryRestClient /// The handler for diagnostic messaging in the client. /// The HTTP pipeline for sending and receiving REST requests and responses. /// The URL of the service account, share, directory or file that is the target of the desired operation. - /// Specifies the version of the operation to use for this request. The default value is "2024-11-04". + /// Specifies the version of the operation to use for this request. The default value is "2025-01-05". /// If true, the trailing dot will not be trimmed from the target URI. /// Valid value is backup. /// If true, the trailing dot will not be trimmed from the source URI. diff --git a/sdk/storage/Azure.Storage.Files.Shares/src/Generated/FileDownloadHeaders.cs b/sdk/storage/Azure.Storage.Files.Shares/src/Generated/FileDownloadHeaders.cs index 61384dee810d4..c4d7056a5cfa3 100644 --- a/sdk/storage/Azure.Storage.Files.Shares/src/Generated/FileDownloadHeaders.cs +++ b/sdk/storage/Azure.Storage.Files.Shares/src/Generated/FileDownloadHeaders.cs @@ -79,5 +79,9 @@ public FileDownloadHeaders(Response response) public ShareLeaseState? LeaseState => _response.Headers.TryGetValue("x-ms-lease-state", out string value) ? value.ToShareLeaseState() : null; /// The current lease status of the file. public ShareLeaseStatus? LeaseStatus => _response.Headers.TryGetValue("x-ms-lease-status", out string value) ? value.ToShareLeaseStatus() : null; + /// Indicates the response body contains a structured message and specifies the message schema version and properties. + public string StructuredBodyType => _response.Headers.TryGetValue("x-ms-structured-body", out string value) ? value : null; + /// The length of the blob/file content inside the message body when the response body is returned as a structured message. Will always be smaller than Content-Length. + public long? StructuredContentLength => _response.Headers.TryGetValue("x-ms-structured-content-length", out long? value) ? value : null; } } diff --git a/sdk/storage/Azure.Storage.Files.Shares/src/Generated/FileRestClient.cs b/sdk/storage/Azure.Storage.Files.Shares/src/Generated/FileRestClient.cs index d4b584e6660ee..093de99705c4d 100644 --- a/sdk/storage/Azure.Storage.Files.Shares/src/Generated/FileRestClient.cs +++ b/sdk/storage/Azure.Storage.Files.Shares/src/Generated/FileRestClient.cs @@ -34,7 +34,7 @@ internal partial class FileRestClient /// The handler for diagnostic messaging in the client. /// The HTTP pipeline for sending and receiving REST requests and responses. /// The URL of the service account, share, directory or file that is the target of the desired operation. - /// Specifies the version of the operation to use for this request. The default value is "2024-11-04". + /// Specifies the version of the operation to use for this request. The default value is "2025-01-05". /// Only update is supported: - Update: Writes the bytes downloaded from the source url into the specified range. The default value is "update". /// If true, the trailing dot will not be trimmed from the target URI. /// Valid value is backup. @@ -204,7 +204,7 @@ public ResponseWithHeaders Create(long fileContentLength, str } } - internal HttpMessage CreateDownloadRequest(int? timeout, string range, bool? rangeGetContentMD5, ShareFileRequestConditions shareFileRequestConditions) + internal HttpMessage CreateDownloadRequest(int? timeout, string range, bool? rangeGetContentMD5, string structuredBodyType, ShareFileRequestConditions shareFileRequestConditions) { var message = _pipeline.CreateMessage(); var request = message.Request; @@ -230,6 +230,10 @@ internal HttpMessage CreateDownloadRequest(int? timeout, string range, bool? ran { request.Headers.Add("x-ms-range-get-content-md5", rangeGetContentMD5.Value); } + if (structuredBodyType != null) + { + request.Headers.Add("x-ms-structured-body", structuredBodyType); + } if (shareFileRequestConditions?.LeaseId != null) { request.Headers.Add("x-ms-lease-id", shareFileRequestConditions.LeaseId); @@ -246,11 +250,12 @@ internal HttpMessage CreateDownloadRequest(int? timeout, string range, bool? ran /// The timeout parameter is expressed in seconds. For more information, see <a href="https://docs.microsoft.com/en-us/rest/api/storageservices/Setting-Timeouts-for-File-Service-Operations?redirectedfrom=MSDN">Setting Timeouts for File Service Operations.</a>. /// Return file data only from the specified byte range. /// When this header is set to true and specified together with the Range header, the service returns the MD5 hash for the range, as long as the range is less than or equal to 4 MB in size. + /// Specifies the response content should be returned as a structured message and specifies the message schema version and properties. /// Parameter group. /// The cancellation token to use. - public async Task> DownloadAsync(int? timeout = null, string range = null, bool? rangeGetContentMD5 = null, ShareFileRequestConditions shareFileRequestConditions = null, CancellationToken cancellationToken = default) + public async Task> DownloadAsync(int? timeout = null, string range = null, bool? rangeGetContentMD5 = null, string structuredBodyType = null, ShareFileRequestConditions shareFileRequestConditions = null, CancellationToken cancellationToken = default) { - using var message = CreateDownloadRequest(timeout, range, rangeGetContentMD5, shareFileRequestConditions); + using var message = CreateDownloadRequest(timeout, range, rangeGetContentMD5, structuredBodyType, shareFileRequestConditions); await _pipeline.SendAsync(message, cancellationToken).ConfigureAwait(false); var headers = new FileDownloadHeaders(message.Response); switch (message.Response.Status) @@ -270,11 +275,12 @@ public async Task> DownloadAsyn /// The timeout parameter is expressed in seconds. For more information, see <a href="https://docs.microsoft.com/en-us/rest/api/storageservices/Setting-Timeouts-for-File-Service-Operations?redirectedfrom=MSDN">Setting Timeouts for File Service Operations.</a>. /// Return file data only from the specified byte range. /// When this header is set to true and specified together with the Range header, the service returns the MD5 hash for the range, as long as the range is less than or equal to 4 MB in size. + /// Specifies the response content should be returned as a structured message and specifies the message schema version and properties. /// Parameter group. /// The cancellation token to use. - public ResponseWithHeaders Download(int? timeout = null, string range = null, bool? rangeGetContentMD5 = null, ShareFileRequestConditions shareFileRequestConditions = null, CancellationToken cancellationToken = default) + public ResponseWithHeaders Download(int? timeout = null, string range = null, bool? rangeGetContentMD5 = null, string structuredBodyType = null, ShareFileRequestConditions shareFileRequestConditions = null, CancellationToken cancellationToken = default) { - using var message = CreateDownloadRequest(timeout, range, rangeGetContentMD5, shareFileRequestConditions); + using var message = CreateDownloadRequest(timeout, range, rangeGetContentMD5, structuredBodyType, shareFileRequestConditions); _pipeline.Send(message, cancellationToken); var headers = new FileDownloadHeaders(message.Response); switch (message.Response.Status) @@ -945,7 +951,7 @@ public ResponseWithHeaders BreakLease(int? timeout = null } } - internal HttpMessage CreateUploadRangeRequest(string range, ShareFileRangeWriteType fileRangeWrite, long contentLength, int? timeout, byte[] contentMD5, FileLastWrittenMode? fileLastWrittenMode, Stream optionalbody, ShareFileRequestConditions shareFileRequestConditions) + internal HttpMessage CreateUploadRangeRequest(string range, ShareFileRangeWriteType fileRangeWrite, long contentLength, int? timeout, byte[] contentMD5, FileLastWrittenMode? fileLastWrittenMode, string structuredBodyType, long? structuredContentLength, Stream optionalbody, ShareFileRequestConditions shareFileRequestConditions) { var message = _pipeline.CreateMessage(); var request = message.Request; @@ -977,6 +983,14 @@ internal HttpMessage CreateUploadRangeRequest(string range, ShareFileRangeWriteT { request.Headers.Add("x-ms-file-request-intent", _fileRequestIntent.Value.ToString()); } + if (structuredBodyType != null) + { + request.Headers.Add("x-ms-structured-body", structuredBodyType); + } + if (structuredContentLength != null) + { + request.Headers.Add("x-ms-structured-content-length", structuredContentLength.Value); + } request.Headers.Add("Accept", "application/xml"); if (optionalbody != null) { @@ -998,18 +1012,20 @@ internal HttpMessage CreateUploadRangeRequest(string range, ShareFileRangeWriteT /// The timeout parameter is expressed in seconds. For more information, see <a href="https://docs.microsoft.com/en-us/rest/api/storageservices/Setting-Timeouts-for-File-Service-Operations?redirectedfrom=MSDN">Setting Timeouts for File Service Operations.</a>. /// An MD5 hash of the content. This hash is used to verify the integrity of the data during transport. When the Content-MD5 header is specified, the File service compares the hash of the content that has arrived with the header value that was sent. If the two hashes do not match, the operation will fail with error code 400 (Bad Request). /// If the file last write time should be preserved or overwritten. + /// Required if the request body is a structured message. Specifies the message schema version and properties. + /// Required if the request body is a structured message. Specifies the length of the blob/file content inside the message body. Will always be smaller than Content-Length. /// Initial data. /// Parameter group. /// The cancellation token to use. /// is null. - public async Task> UploadRangeAsync(string range, ShareFileRangeWriteType fileRangeWrite, long contentLength, int? timeout = null, byte[] contentMD5 = null, FileLastWrittenMode? fileLastWrittenMode = null, Stream optionalbody = null, ShareFileRequestConditions shareFileRequestConditions = null, CancellationToken cancellationToken = default) + public async Task> UploadRangeAsync(string range, ShareFileRangeWriteType fileRangeWrite, long contentLength, int? timeout = null, byte[] contentMD5 = null, FileLastWrittenMode? fileLastWrittenMode = null, string structuredBodyType = null, long? structuredContentLength = null, Stream optionalbody = null, ShareFileRequestConditions shareFileRequestConditions = null, CancellationToken cancellationToken = default) { if (range == null) { throw new ArgumentNullException(nameof(range)); } - using var message = CreateUploadRangeRequest(range, fileRangeWrite, contentLength, timeout, contentMD5, fileLastWrittenMode, optionalbody, shareFileRequestConditions); + using var message = CreateUploadRangeRequest(range, fileRangeWrite, contentLength, timeout, contentMD5, fileLastWrittenMode, structuredBodyType, structuredContentLength, optionalbody, shareFileRequestConditions); await _pipeline.SendAsync(message, cancellationToken).ConfigureAwait(false); var headers = new FileUploadRangeHeaders(message.Response); switch (message.Response.Status) @@ -1028,18 +1044,20 @@ public async Task> UploadRangeAsync( /// The timeout parameter is expressed in seconds. For more information, see <a href="https://docs.microsoft.com/en-us/rest/api/storageservices/Setting-Timeouts-for-File-Service-Operations?redirectedfrom=MSDN">Setting Timeouts for File Service Operations.</a>. /// An MD5 hash of the content. This hash is used to verify the integrity of the data during transport. When the Content-MD5 header is specified, the File service compares the hash of the content that has arrived with the header value that was sent. If the two hashes do not match, the operation will fail with error code 400 (Bad Request). /// If the file last write time should be preserved or overwritten. + /// Required if the request body is a structured message. Specifies the message schema version and properties. + /// Required if the request body is a structured message. Specifies the length of the blob/file content inside the message body. Will always be smaller than Content-Length. /// Initial data. /// Parameter group. /// The cancellation token to use. /// is null. - public ResponseWithHeaders UploadRange(string range, ShareFileRangeWriteType fileRangeWrite, long contentLength, int? timeout = null, byte[] contentMD5 = null, FileLastWrittenMode? fileLastWrittenMode = null, Stream optionalbody = null, ShareFileRequestConditions shareFileRequestConditions = null, CancellationToken cancellationToken = default) + public ResponseWithHeaders UploadRange(string range, ShareFileRangeWriteType fileRangeWrite, long contentLength, int? timeout = null, byte[] contentMD5 = null, FileLastWrittenMode? fileLastWrittenMode = null, string structuredBodyType = null, long? structuredContentLength = null, Stream optionalbody = null, ShareFileRequestConditions shareFileRequestConditions = null, CancellationToken cancellationToken = default) { if (range == null) { throw new ArgumentNullException(nameof(range)); } - using var message = CreateUploadRangeRequest(range, fileRangeWrite, contentLength, timeout, contentMD5, fileLastWrittenMode, optionalbody, shareFileRequestConditions); + using var message = CreateUploadRangeRequest(range, fileRangeWrite, contentLength, timeout, contentMD5, fileLastWrittenMode, structuredBodyType, structuredContentLength, optionalbody, shareFileRequestConditions); _pipeline.Send(message, cancellationToken); var headers = new FileUploadRangeHeaders(message.Response); switch (message.Response.Status) diff --git a/sdk/storage/Azure.Storage.Files.Shares/src/Generated/FileUploadRangeHeaders.cs b/sdk/storage/Azure.Storage.Files.Shares/src/Generated/FileUploadRangeHeaders.cs index db079c2692663..322bfcd1b6d83 100644 --- a/sdk/storage/Azure.Storage.Files.Shares/src/Generated/FileUploadRangeHeaders.cs +++ b/sdk/storage/Azure.Storage.Files.Shares/src/Generated/FileUploadRangeHeaders.cs @@ -27,5 +27,7 @@ public FileUploadRangeHeaders(Response response) public bool? IsServerEncrypted => _response.Headers.TryGetValue("x-ms-request-server-encrypted", out bool? value) ? value : null; /// Last write time for the file. public DateTimeOffset? FileLastWriteTime => _response.Headers.TryGetValue("x-ms-file-last-write-time", out DateTimeOffset? value) ? value : null; + /// Indicates the structured message body was accepted and mirrors back the message schema version and properties. + public string StructuredBodyType => _response.Headers.TryGetValue("x-ms-structured-body", out string value) ? value : null; } } diff --git a/sdk/storage/Azure.Storage.Files.Shares/src/Generated/ServiceRestClient.cs b/sdk/storage/Azure.Storage.Files.Shares/src/Generated/ServiceRestClient.cs index ef4c21b9a33c7..fe5ea495a7a15 100644 --- a/sdk/storage/Azure.Storage.Files.Shares/src/Generated/ServiceRestClient.cs +++ b/sdk/storage/Azure.Storage.Files.Shares/src/Generated/ServiceRestClient.cs @@ -31,7 +31,7 @@ internal partial class ServiceRestClient /// The handler for diagnostic messaging in the client. /// The HTTP pipeline for sending and receiving REST requests and responses. /// The URL of the service account, share, directory or file that is the target of the desired operation. - /// Specifies the version of the operation to use for this request. The default value is "2024-11-04". + /// Specifies the version of the operation to use for this request. The default value is "2025-01-05". /// Valid value is backup. /// , , or is null. public ServiceRestClient(ClientDiagnostics clientDiagnostics, HttpPipeline pipeline, string url, string version, ShareTokenIntent? fileRequestIntent = null) diff --git a/sdk/storage/Azure.Storage.Files.Shares/src/Generated/ShareRestClient.cs b/sdk/storage/Azure.Storage.Files.Shares/src/Generated/ShareRestClient.cs index 599aacf2c6287..3012d3d8735b1 100644 --- a/sdk/storage/Azure.Storage.Files.Shares/src/Generated/ShareRestClient.cs +++ b/sdk/storage/Azure.Storage.Files.Shares/src/Generated/ShareRestClient.cs @@ -32,7 +32,7 @@ internal partial class ShareRestClient /// The handler for diagnostic messaging in the client. /// The HTTP pipeline for sending and receiving REST requests and responses. /// The URL of the service account, share, directory or file that is the target of the desired operation. - /// Specifies the version of the operation to use for this request. The default value is "2024-11-04". + /// Specifies the version of the operation to use for this request. The default value is "2025-01-05". /// Valid value is backup. /// , , or is null. public ShareRestClient(ClientDiagnostics clientDiagnostics, HttpPipeline pipeline, string url, string version, ShareTokenIntent? fileRequestIntent = null) diff --git a/sdk/storage/Azure.Storage.Files.Shares/src/ShareErrors.cs b/sdk/storage/Azure.Storage.Files.Shares/src/ShareErrors.cs index f776384d06add..0b27510aaa6c4 100644 --- a/sdk/storage/Azure.Storage.Files.Shares/src/ShareErrors.cs +++ b/sdk/storage/Azure.Storage.Files.Shares/src/ShareErrors.cs @@ -17,20 +17,5 @@ public static InvalidOperationException FileOrShareMissing( string fileClient, string shareClient) => new InvalidOperationException($"{leaseClient} requires either a {fileClient} or {shareClient}"); - - public static void AssertAlgorithmSupport(StorageChecksumAlgorithm? algorithm) - { - StorageChecksumAlgorithm resolved = (algorithm ?? StorageChecksumAlgorithm.None).ResolveAuto(); - switch (resolved) - { - case StorageChecksumAlgorithm.None: - case StorageChecksumAlgorithm.MD5: - return; - case StorageChecksumAlgorithm.StorageCrc64: - throw new ArgumentException("Azure File Shares do not support CRC-64."); - default: - throw new ArgumentException($"{nameof(StorageChecksumAlgorithm)} does not support value {Enum.GetName(typeof(StorageChecksumAlgorithm), resolved) ?? ((int)resolved).ToString(CultureInfo.InvariantCulture)}."); - } - } } } diff --git a/sdk/storage/Azure.Storage.Files.Shares/src/ShareFileClient.cs b/sdk/storage/Azure.Storage.Files.Shares/src/ShareFileClient.cs index 2d58482950b9a..524f9cba6db8a 100644 --- a/sdk/storage/Azure.Storage.Files.Shares/src/ShareFileClient.cs +++ b/sdk/storage/Azure.Storage.Files.Shares/src/ShareFileClient.cs @@ -2385,51 +2385,65 @@ private async Task> DownloadInternal( // Wrap the response Content in a RetriableStream so we // can return it before it's finished downloading, but still // allow retrying if it fails. - initialResponse.Value.Content = RetriableStream.Create( - stream, - startOffset => - { - (Response Response, Stream ContentStream) = StartDownloadAsync( - range, - validationOptions, - conditions, - startOffset, - async, - cancellationToken) - .EnsureCompleted(); - if (etag != Response.GetRawResponse().Headers.ETag) - { - throw new ShareFileModifiedException( - "File has been modified concurrently", - Uri, etag, Response.GetRawResponse().Headers.ETag.GetValueOrDefault(), range); - } - return ContentStream; - }, - async startOffset => + async ValueTask> Factory(long offset, bool async, CancellationToken cancellationToken) + { + (Response response, Stream contentStream) = await StartDownloadAsync( + range, + validationOptions, + conditions, + offset, + async, + cancellationToken).ConfigureAwait(false); + if (etag != response.GetRawResponse().Headers.ETag) { - (Response Response, Stream ContentStream) = await StartDownloadAsync( - range, - validationOptions, - conditions, - startOffset, - async, - cancellationToken) - .ConfigureAwait(false); - if (etag != Response.GetRawResponse().Headers.ETag) - { - throw new ShareFileModifiedException( - "File has been modified concurrently", - Uri, etag, Response.GetRawResponse().Headers.ETag.GetValueOrDefault(), range); - } - return ContentStream; - }, - ClientConfiguration.Pipeline.ResponseClassifier, - Constants.MaxReliabilityRetries); + throw new ShareFileModifiedException( + "File has been modified concurrently", + Uri, etag, response.GetRawResponse().Headers.ETag.GetValueOrDefault(), range); + } + return response; + } + async ValueTask<(Stream DecodingStream, StructuredMessageDecodingStream.DecodedData DecodedData)> StructuredMessageFactory( + long offset, bool async, CancellationToken cancellationToken) + { + Response result = await Factory(offset, async, cancellationToken).ConfigureAwait(false); + return StructuredMessageDecodingStream.WrapStream(result.Value.Content, result.Value.ContentLength); + } + + if (initialResponse.GetRawResponse().Headers.Contains(Constants.StructuredMessage.CrcStructuredMessageHeader)) + { + (Stream decodingStream, StructuredMessageDecodingStream.DecodedData decodedData) = StructuredMessageDecodingStream.WrapStream( + initialResponse.Value.Content, initialResponse.Value.ContentLength); + initialResponse.Value.Content = new StructuredMessageDecodingRetriableStream( + decodingStream, + decodedData, + startOffset => StructuredMessageFactory(startOffset, async: false, cancellationToken) + .EnsureCompleted(), + async startOffset => await StructuredMessageFactory(startOffset, async: true, cancellationToken) + .ConfigureAwait(false), + default, //decodedData => response.Value.Details.ContentCrc = decodedData.TotalCrc.ToArray(), + ClientConfiguration.Pipeline.ResponseClassifier, + Constants.MaxReliabilityRetries); + } + else + { + initialResponse.Value.Content = RetriableStream.Create( + initialResponse.Value.Content, + startOffset => Factory(startOffset, async: false, cancellationToken) + .EnsureCompleted().Value.Content, + async startOffset => (await Factory(startOffset, async: true, cancellationToken) + .ConfigureAwait(false)).Value.Content, + ClientConfiguration.Pipeline.ResponseClassifier, + Constants.MaxReliabilityRetries); + } // buffer response stream and ensure it matches the transactional hash if any // Storage will not return a hash for payload >4MB, so this buffer is capped similarly // hashing is opt-in, so this buffer is part of that opt-in - if (validationOptions != default && validationOptions.ChecksumAlgorithm != StorageChecksumAlgorithm.None && validationOptions.AutoValidateChecksum) + if (validationOptions != default && + validationOptions.ChecksumAlgorithm != StorageChecksumAlgorithm.None && + validationOptions.AutoValidateChecksum && + // structured message decoding does the validation for us + !initialResponse.GetRawResponse().Headers.Contains(Constants.StructuredMessage.CrcStructuredMessageHeader)) { // safe-buffer; transactional hash download limit well below maxInt var readDestStream = new MemoryStream((int)initialResponse.Value.ContentLength); @@ -2512,8 +2526,6 @@ await ContentHasher.AssertResponseHashMatchInternal( bool async = true, CancellationToken cancellationToken = default) { - ShareErrors.AssertAlgorithmSupport(transferValidationOverride?.ChecksumAlgorithm); - // calculation gets illegible with null coalesce; just pre-initialize var pageRange = range; pageRange = new HttpRange( @@ -2523,13 +2535,27 @@ await ContentHasher.AssertResponseHashMatchInternal( (long?)null); ClientConfiguration.Pipeline.LogTrace($"Download {Uri} with range: {pageRange}"); - ResponseWithHeaders response; + bool? rangeGetContentMD5 = null; + string structuredBodyType = null; + switch (transferValidationOverride?.ChecksumAlgorithm.ResolveAuto()) + { + case StorageChecksumAlgorithm.MD5: + rangeGetContentMD5 = true; + break; + case StorageChecksumAlgorithm.StorageCrc64: + structuredBodyType = Constants.StructuredMessage.CrcStructuredMessage; + break; + default: + break; + } + ResponseWithHeaders response; if (async) { response = await FileRestClient.DownloadAsync( range: pageRange == default ? null : pageRange.ToString(), - rangeGetContentMD5: transferValidationOverride?.ChecksumAlgorithm.ResolveAuto() == StorageChecksumAlgorithm.MD5 ? true : null, + rangeGetContentMD5: rangeGetContentMD5, + structuredBodyType: structuredBodyType, shareFileRequestConditions: conditions, cancellationToken: cancellationToken) .ConfigureAwait(false); @@ -2538,7 +2564,8 @@ await ContentHasher.AssertResponseHashMatchInternal( { response = FileRestClient.Download( range: pageRange == default ? null : pageRange.ToString(), - rangeGetContentMD5: transferValidationOverride?.ChecksumAlgorithm.ResolveAuto() == StorageChecksumAlgorithm.MD5 ? true : null, + rangeGetContentMD5: rangeGetContentMD5, + structuredBodyType: structuredBodyType, shareFileRequestConditions: conditions, cancellationToken: cancellationToken); } @@ -4612,7 +4639,6 @@ internal async Task> UploadRangeInternal( CancellationToken cancellationToken) { UploadTransferValidationOptions validationOptions = transferValidationOverride ?? ClientConfiguration.TransferValidation.Upload; - ShareErrors.AssertAlgorithmSupport(validationOptions?.ChecksumAlgorithm); using (ClientConfiguration.Pipeline.BeginLoggingScope(nameof(ShareFileClient))) { @@ -4628,14 +4654,38 @@ internal async Task> UploadRangeInternal( scope.Start(); Errors.VerifyStreamPosition(content, nameof(content)); - // compute hash BEFORE attaching progress handler - ContentHasher.GetHashResult hashResult = await ContentHasher.GetHashOrDefaultInternal( - content, - validationOptions, - async, - cancellationToken).ConfigureAwait(false); - - content = content.WithNoDispose().WithProgress(progressHandler); + ContentHasher.GetHashResult hashResult = null; + long contentLength = (content?.Length - content?.Position) ?? 0; + long? structuredContentLength = default; + string structuredBodyType = null; + if (validationOptions != null && + validationOptions.ChecksumAlgorithm.ResolveAuto() == StorageChecksumAlgorithm.StorageCrc64) + { + // report progress in terms of caller bytes, not encoded bytes + structuredContentLength = contentLength; + contentLength = (content?.Length - content?.Position) ?? 0; + structuredBodyType = Constants.StructuredMessage.CrcStructuredMessage; + content = content.WithNoDispose().WithProgress(progressHandler); + content = validationOptions.PrecalculatedChecksum.IsEmpty + ? new StructuredMessageEncodingStream( + content, + Constants.StructuredMessage.DefaultSegmentContentLength, + StructuredMessage.Flags.StorageCrc64) + : new StructuredMessagePrecalculatedCrcWrapperStream( + content, + validationOptions.PrecalculatedChecksum.Span); + contentLength = (content?.Length - content?.Position) ?? 0; + } + else + { + // compute hash BEFORE attaching progress handler + hashResult = await ContentHasher.GetHashOrDefaultInternal( + content, + validationOptions, + async, + cancellationToken).ConfigureAwait(false); + content = content.WithNoDispose().WithProgress(progressHandler); + } ResponseWithHeaders response; @@ -4648,6 +4698,8 @@ internal async Task> UploadRangeInternal( fileLastWrittenMode: fileLastWrittenMode, optionalbody: content, contentMD5: hashResult?.MD5AsArray, + structuredBodyType: structuredBodyType, + structuredContentLength: structuredContentLength, shareFileRequestConditions: conditions, cancellationToken: cancellationToken) .ConfigureAwait(false); @@ -4661,6 +4713,8 @@ internal async Task> UploadRangeInternal( fileLastWrittenMode: fileLastWrittenMode, optionalbody: content, contentMD5: hashResult?.MD5AsArray, + structuredBodyType: structuredBodyType, + structuredContentLength: structuredContentLength, shareFileRequestConditions: conditions, cancellationToken: cancellationToken); } diff --git a/sdk/storage/Azure.Storage.Files.Shares/src/autorest.md b/sdk/storage/Azure.Storage.Files.Shares/src/autorest.md index 43022bc56d1c1..d7ed8ae3216df 100644 --- a/sdk/storage/Azure.Storage.Files.Shares/src/autorest.md +++ b/sdk/storage/Azure.Storage.Files.Shares/src/autorest.md @@ -4,7 +4,7 @@ Run `dotnet build /t:GenerateCode` to generate code. ``` yaml input-file: - - https://raw.githubusercontent.com/Azure/azure-rest-api-specs/98b600498947073c18c2ac5eb7c3c658db5a1a59/specification/storage/data-plane/Microsoft.FileStorage/stable/2024-11-04/file.json + - https://raw.githubusercontent.com/Azure/azure-rest-api-specs/c8eee2dfa99d517e12e6ac8c96b14b707bb3c8eb/specification/storage/data-plane/Microsoft.FileStorage/stable/2025-01-05/file.json generation1-convenience-client: true # https://github.com/Azure/autorest/issues/4075 skip-semantics-validation: true @@ -25,7 +25,7 @@ directive: if (property.includes('/{shareName}/{directory}/{fileName}')) { $[property]["parameters"] = $[property]["parameters"].filter(function(param) { return (typeof param['$ref'] === "undefined") || (false == param['$ref'].endsWith("#/parameters/ShareName") && false == param['$ref'].endsWith("#/parameters/DirectoryPath") && false == param['$ref'].endsWith("#/parameters/FilePath"))}); - } + } else if (property.includes('/{shareName}/{directory}')) { $[property]["parameters"] = $[property]["parameters"].filter(function(param) { return (typeof param['$ref'] === "undefined") || (false == param['$ref'].endsWith("#/parameters/ShareName") && false == param['$ref'].endsWith("#/parameters/DirectoryPath"))}); @@ -46,7 +46,7 @@ directive: $.Metrics.type = "object"; ``` -### Times aren't required +### Times aren't required ``` yaml directive: - from: swagger-document diff --git a/sdk/storage/Azure.Storage.Files.Shares/tests/ShareFileClientTransferValidationTests.cs b/sdk/storage/Azure.Storage.Files.Shares/tests/ShareFileClientTransferValidationTests.cs index 3dcdb21f27b36..afe33c95847d0 100644 --- a/sdk/storage/Azure.Storage.Files.Shares/tests/ShareFileClientTransferValidationTests.cs +++ b/sdk/storage/Azure.Storage.Files.Shares/tests/ShareFileClientTransferValidationTests.cs @@ -64,10 +64,6 @@ protected override async Task GetResourceClientAsync( private void AssertSupportsHashAlgorithm(StorageChecksumAlgorithm algorithm) { - if (algorithm.ResolveAuto() == StorageChecksumAlgorithm.StorageCrc64) - { - TestHelper.AssertInconclusiveRecordingFriendly(Recording.Mode, "Azure File Share does not support CRC64."); - } } protected override async Task UploadPartitionAsync(ShareFileClient client, Stream source, UploadTransferValidationOptions transferValidation) From 212d8a459e6191fca81dbf262d937e0e372d5395 Mon Sep 17 00:00:00 2001 From: Jocelyn <41338290+jaschrep-msft@users.noreply.github.com> Date: Tue, 23 Jul 2024 12:46:42 -0400 Subject: [PATCH 14/22] block blob fix (#45129) * block blob fix * testproxy --- sdk/storage/Azure.Storage.Blobs/src/BlockBlobClient.cs | 1 - 1 file changed, 1 deletion(-) diff --git a/sdk/storage/Azure.Storage.Blobs/src/BlockBlobClient.cs b/sdk/storage/Azure.Storage.Blobs/src/BlockBlobClient.cs index fe9e6af997b6c..5e5ec82e96dca 100644 --- a/sdk/storage/Azure.Storage.Blobs/src/BlockBlobClient.cs +++ b/sdk/storage/Azure.Storage.Blobs/src/BlockBlobClient.cs @@ -882,7 +882,6 @@ internal virtual async Task> UploadInternal( if (content != null && validationOptions != null && validationOptions.ChecksumAlgorithm.ResolveAuto() == StorageChecksumAlgorithm.StorageCrc64 && - validationOptions.PrecalculatedChecksum.IsEmpty && ClientSideEncryption == null) // don't allow feature combination { // report progress in terms of caller bytes, not encoded bytes From 00a59a369d40cfa64c183779a458c386b6034183 Mon Sep 17 00:00:00 2001 From: Jocelyn <41338290+jaschrep-msft@users.noreply.github.com> Date: Fri, 26 Jul 2024 10:01:30 -0400 Subject: [PATCH 15/22] rename and validation (#45160) * rename and validation * fix --- .../Azure.Storage.Blobs/src/BlobBaseClient.cs | 4 +-- .../src/PartitionedDownloader.cs | 2 +- .../src/Shared/Constants.cs | 4 +-- .../src/Shared/Errors.Clients.cs | 8 ++++- .../StorageRequestValidationPipelinePolicy.cs | 29 +++++++++++++++++++ .../tests/Shared/RequestExtensions.cs | 2 +- .../Shared/TransferValidationTestBase.cs | 12 ++++---- .../src/ShareFileClient.cs | 4 +-- 8 files changed, 50 insertions(+), 15 deletions(-) diff --git a/sdk/storage/Azure.Storage.Blobs/src/BlobBaseClient.cs b/sdk/storage/Azure.Storage.Blobs/src/BlobBaseClient.cs index d271615f6f4b7..c92e025c0d6ab 100644 --- a/sdk/storage/Azure.Storage.Blobs/src/BlobBaseClient.cs +++ b/sdk/storage/Azure.Storage.Blobs/src/BlobBaseClient.cs @@ -1563,7 +1563,7 @@ ValueTask> Factory(long offset, bool force return StructuredMessageDecodingStream.WrapStream(result.Value.Content, result.Value.Details.ContentLength); } Stream stream; - if (response.GetRawResponse().Headers.Contains(Constants.StructuredMessage.CrcStructuredMessageHeader)) + if (response.GetRawResponse().Headers.Contains(Constants.StructuredMessage.StructuredMessageHeader)) { (Stream decodingStream, StructuredMessageDecodingStream.DecodedData decodedData) = StructuredMessageDecodingStream.WrapStream( response.Value.Content, response.Value.Details.ContentLength); @@ -1600,7 +1600,7 @@ ValueTask> Factory(long offset, bool force validationOptions.ChecksumAlgorithm != StorageChecksumAlgorithm.None && validationOptions.AutoValidateChecksum && // structured message decoding does the validation for us - !response.GetRawResponse().Headers.Contains(Constants.StructuredMessage.CrcStructuredMessageHeader)) + !response.GetRawResponse().Headers.Contains(Constants.StructuredMessage.StructuredMessageHeader)) { // safe-buffer; transactional hash download limit well below maxInt var readDestStream = new MemoryStream((int)response.Value.Details.ContentLength); diff --git a/sdk/storage/Azure.Storage.Blobs/src/PartitionedDownloader.cs b/sdk/storage/Azure.Storage.Blobs/src/PartitionedDownloader.cs index 276bdadb673fa..08a1090716f2b 100644 --- a/sdk/storage/Azure.Storage.Blobs/src/PartitionedDownloader.cs +++ b/sdk/storage/Azure.Storage.Blobs/src/PartitionedDownloader.cs @@ -417,7 +417,7 @@ private async Task CopyToInternal( { CancellationHelper.ThrowIfCancellationRequested(cancellationToken); // if structured message, this crc is validated in the decoding process. don't decode it here. - using IHasher hasher = response.GetRawResponse().Headers.Contains(Constants.StructuredMessage.CrcStructuredMessageHeader) + using IHasher hasher = response.GetRawResponse().Headers.Contains(Constants.StructuredMessage.StructuredMessageHeader) ? null : ContentHasher.GetHasherFromAlgorithmId(_validationAlgorithm); using Stream rawSource = response.Value.Content; diff --git a/sdk/storage/Azure.Storage.Common/src/Shared/Constants.cs b/sdk/storage/Azure.Storage.Common/src/Shared/Constants.cs index 0636041a65134..4893b971d6529 100644 --- a/sdk/storage/Azure.Storage.Common/src/Shared/Constants.cs +++ b/sdk/storage/Azure.Storage.Common/src/Shared/Constants.cs @@ -667,8 +667,8 @@ internal static class AccountResources internal static class StructuredMessage { - public const string CrcStructuredMessageHeader = "x-ms-structured-body"; - public const string CrcStructuredContentLength = "x-ms-structured-content-length"; + public const string StructuredMessageHeader = "x-ms-structured-body"; + public const string StructuredContentLength = "x-ms-structured-content-length"; public const string CrcStructuredMessage = "XSM/1.0; properties=crc64"; public const int DefaultSegmentContentLength = 4 * MB; public const int MaxDownloadCrcWithHeader = 4 * MB; diff --git a/sdk/storage/Azure.Storage.Common/src/Shared/Errors.Clients.cs b/sdk/storage/Azure.Storage.Common/src/Shared/Errors.Clients.cs index 22def52ec719c..4d49edeb72ecf 100644 --- a/sdk/storage/Azure.Storage.Common/src/Shared/Errors.Clients.cs +++ b/sdk/storage/Azure.Storage.Common/src/Shared/Errors.Clients.cs @@ -106,11 +106,17 @@ public static ArgumentException VersionNotSupported(string paramName) public static RequestFailedException ClientRequestIdMismatch(Response response, string echo, string original) => new RequestFailedException(response.Status, $"Response x-ms-client-request-id '{echo}' does not match the original expected request id, '{original}'.", null); + public static InvalidDataException StructuredMessageNotAcknowledgedGET(Response response) + => new InvalidDataException($"Response does not acknowledge structured message was requested. Unknown data structure in response body."); + + public static InvalidDataException StructuredMessageNotAcknowledgedPUT(Response response) + => new InvalidDataException($"Response does not acknowledge structured message was sent. Unexpected data may have been persisted to storage."); + public static ArgumentException TransactionalHashingNotSupportedWithClientSideEncryption() => new ArgumentException("Client-side encryption and transactional hashing are not supported at the same time."); public static InvalidDataException ExpectedStructuredMessage() - => new InvalidDataException($"Expected {Constants.StructuredMessage.CrcStructuredMessageHeader} in response, but found none."); + => new InvalidDataException($"Expected {Constants.StructuredMessage.StructuredMessageHeader} in response, but found none."); public static void VerifyHttpsTokenAuth(Uri uri) { diff --git a/sdk/storage/Azure.Storage.Common/src/Shared/StorageRequestValidationPipelinePolicy.cs b/sdk/storage/Azure.Storage.Common/src/Shared/StorageRequestValidationPipelinePolicy.cs index 0cef4f4d8d4ed..9f4ddb5249e82 100644 --- a/sdk/storage/Azure.Storage.Common/src/Shared/StorageRequestValidationPipelinePolicy.cs +++ b/sdk/storage/Azure.Storage.Common/src/Shared/StorageRequestValidationPipelinePolicy.cs @@ -33,6 +33,35 @@ public override void OnReceivedResponse(HttpMessage message) { throw Errors.ClientRequestIdMismatch(message.Response, echo.First(), original); } + + if (message.Request.Headers.Contains(Constants.StructuredMessage.StructuredMessageHeader) && + message.Request.Headers.Contains(Constants.StructuredMessage.StructuredContentLength)) + { + AssertStructuredMessageAcknowledgedPUT(message); + } + else if (message.Request.Headers.Contains(Constants.StructuredMessage.StructuredMessageHeader)) + { + AssertStructuredMessageAcknowledgedGET(message); + } + } + + private static void AssertStructuredMessageAcknowledgedPUT(HttpMessage message) + { + if (!message.Response.IsError && + !message.Response.Headers.Contains(Constants.StructuredMessage.StructuredMessageHeader)) + { + throw Errors.StructuredMessageNotAcknowledgedPUT(message.Response); + } + } + + private static void AssertStructuredMessageAcknowledgedGET(HttpMessage message) + { + if (!message.Response.IsError && + !(message.Response.Headers.Contains(Constants.StructuredMessage.StructuredMessageHeader) && + message.Response.Headers.Contains(Constants.StructuredMessage.StructuredContentLength))) + { + throw Errors.StructuredMessageNotAcknowledgedGET(message.Response); + } } } } diff --git a/sdk/storage/Azure.Storage.Common/tests/Shared/RequestExtensions.cs b/sdk/storage/Azure.Storage.Common/tests/Shared/RequestExtensions.cs index 0f9cdb07773f7..ad395e862f827 100644 --- a/sdk/storage/Azure.Storage.Common/tests/Shared/RequestExtensions.cs +++ b/sdk/storage/Azure.Storage.Common/tests/Shared/RequestExtensions.cs @@ -14,7 +14,7 @@ public static string AssertHeaderPresent(this Request request, string headerName { if (request.Headers.TryGetValue(headerName, out string value)) { - return headerName == Constants.StructuredMessage.CrcStructuredMessageHeader ? null : value; + return headerName == Constants.StructuredMessage.StructuredMessageHeader ? null : value; } StringBuilder sb = new StringBuilder() .AppendLine($"`{headerName}` expected on request but was not found.") diff --git a/sdk/storage/Azure.Storage.Common/tests/Shared/TransferValidationTestBase.cs b/sdk/storage/Azure.Storage.Common/tests/Shared/TransferValidationTestBase.cs index a406744f94d46..ed5651b0b0fc5 100644 --- a/sdk/storage/Azure.Storage.Common/tests/Shared/TransferValidationTestBase.cs +++ b/sdk/storage/Azure.Storage.Common/tests/Shared/TransferValidationTestBase.cs @@ -219,7 +219,7 @@ void AssertChecksum(Request req, string headerName) AssertChecksum(request, "Content-MD5"); break; case StorageChecksumAlgorithm.StorageCrc64: - AssertChecksum(request, Constants.StructuredMessage.CrcStructuredMessageHeader); + AssertChecksum(request, Constants.StructuredMessage.StructuredMessageHeader); break; default: throw new Exception($"Bad {nameof(StorageChecksumAlgorithm)} provided to {nameof(GetRequestChecksumHeaderAssertion)}."); @@ -302,7 +302,7 @@ void AssertChecksum(ResponseHeaders headers, string headerName) AssertChecksum(response.Headers, "Content-MD5"); break; case StorageChecksumAlgorithm.StorageCrc64: - AssertChecksum(response.Headers, Constants.StructuredMessage.CrcStructuredMessageHeader); + AssertChecksum(response.Headers, Constants.StructuredMessage.StructuredMessageHeader); break; default: throw new Exception($"Bad {nameof(StorageChecksumAlgorithm)} provided to {nameof(GetRequestChecksumHeaderAssertion)}."); @@ -1744,7 +1744,7 @@ public virtual async Task DownloadSuccessfulHashVerification(StorageChecksumAlgo Assert.True(response.Headers.Contains("Content-MD5")); break; case StorageChecksumAlgorithm.StorageCrc64: - Assert.True(response.Headers.Contains(Constants.StructuredMessage.CrcStructuredMessageHeader)); + Assert.True(response.Headers.Contains(Constants.StructuredMessage.StructuredMessageHeader)); break; default: Assert.Fail("Test can't validate given algorithm type."); @@ -1908,7 +1908,7 @@ public virtual async Task DownloadUsesDefaultClientValidationOptions( Assert.True(response.Headers.Contains("Content-MD5")); break; case StorageChecksumAlgorithm.StorageCrc64: - Assert.True(response.Headers.Contains(Constants.StructuredMessage.CrcStructuredMessageHeader)); + Assert.True(response.Headers.Contains(Constants.StructuredMessage.StructuredMessageHeader)); break; default: Assert.Fail("Test can't validate given algorithm type."); @@ -1968,7 +1968,7 @@ public virtual async Task DownloadOverwritesDefaultClientValidationOptions( Assert.True(response.Headers.Contains("Content-MD5")); break; case StorageChecksumAlgorithm.StorageCrc64: - Assert.True(response.Headers.Contains(Constants.StructuredMessage.CrcStructuredMessageHeader)); + Assert.True(response.Headers.Contains(Constants.StructuredMessage.StructuredMessageHeader)); break; default: Assert.Fail("Test can't validate given algorithm type."); @@ -2072,7 +2072,7 @@ public virtual async Task DownloadRecoversFromInterruptWithValidation( Assert.True(response.Headers.Contains("Content-MD5")); break; case StorageChecksumAlgorithm.StorageCrc64: - Assert.True(response.Headers.Contains(Constants.StructuredMessage.CrcStructuredMessageHeader)); + Assert.True(response.Headers.Contains(Constants.StructuredMessage.StructuredMessageHeader)); break; default: Assert.Fail("Test can't validate given algorithm type."); diff --git a/sdk/storage/Azure.Storage.Files.Shares/src/ShareFileClient.cs b/sdk/storage/Azure.Storage.Files.Shares/src/ShareFileClient.cs index 524f9cba6db8a..3f6a4890d9a89 100644 --- a/sdk/storage/Azure.Storage.Files.Shares/src/ShareFileClient.cs +++ b/sdk/storage/Azure.Storage.Files.Shares/src/ShareFileClient.cs @@ -2409,7 +2409,7 @@ async ValueTask> Factory(long offset, bool async return StructuredMessageDecodingStream.WrapStream(result.Value.Content, result.Value.ContentLength); } - if (initialResponse.GetRawResponse().Headers.Contains(Constants.StructuredMessage.CrcStructuredMessageHeader)) + if (initialResponse.GetRawResponse().Headers.Contains(Constants.StructuredMessage.StructuredMessageHeader)) { (Stream decodingStream, StructuredMessageDecodingStream.DecodedData decodedData) = StructuredMessageDecodingStream.WrapStream( initialResponse.Value.Content, initialResponse.Value.ContentLength); @@ -2443,7 +2443,7 @@ async ValueTask> Factory(long offset, bool async validationOptions.ChecksumAlgorithm != StorageChecksumAlgorithm.None && validationOptions.AutoValidateChecksum && // structured message decoding does the validation for us - !initialResponse.GetRawResponse().Headers.Contains(Constants.StructuredMessage.CrcStructuredMessageHeader)) + !initialResponse.GetRawResponse().Headers.Contains(Constants.StructuredMessage.StructuredMessageHeader)) { // safe-buffer; transactional hash download limit well below maxInt var readDestStream = new MemoryStream((int)initialResponse.Value.ContentLength); From b90be4c059cc11a80f6c8b3c882311aa78fa455b Mon Sep 17 00:00:00 2001 From: Jocelyn <41338290+jaschrep-msft@users.noreply.github.com> Date: Wed, 7 Aug 2024 12:42:07 -0400 Subject: [PATCH 16/22] crc tracking converted to longs (#45307) --- .../Azure.Storage.Blobs/src/BlobBaseClient.cs | 5 +- .../src/Shared/StorageCrc64Composer.cs | 48 ++++++-- .../src/Shared/StructuredMessage.cs | 36 +++--- ...tructuredMessageDecodingRetriableStream.cs | 88 +++++++++++--- .../Shared/StructuredMessageDecodingStream.cs | 112 +++++++----------- .../Shared/ObserveStructuredMessagePolicy.cs | 2 +- ...uredMessageDecodingRetriableStreamTests.cs | 49 +++++--- .../src/ShareFileClient.cs | 5 +- 8 files changed, 204 insertions(+), 141 deletions(-) diff --git a/sdk/storage/Azure.Storage.Blobs/src/BlobBaseClient.cs b/sdk/storage/Azure.Storage.Blobs/src/BlobBaseClient.cs index c92e025c0d6ab..1fa4f59e52e62 100644 --- a/sdk/storage/Azure.Storage.Blobs/src/BlobBaseClient.cs +++ b/sdk/storage/Azure.Storage.Blobs/src/BlobBaseClient.cs @@ -1556,7 +1556,7 @@ ValueTask> Factory(long offset, bool force forceStructuredMessage, async, cancellationToken); - async ValueTask<(Stream DecodingStream, StructuredMessageDecodingStream.DecodedData DecodedData)> StructuredMessageFactory( + async ValueTask<(Stream DecodingStream, StructuredMessageDecodingStream.RawDecodedData DecodedData)> StructuredMessageFactory( long offset, bool async, CancellationToken cancellationToken) { Response result = await Factory(offset, forceStructuredMessage: true, async, cancellationToken).ConfigureAwait(false); @@ -1565,11 +1565,12 @@ ValueTask> Factory(long offset, bool force Stream stream; if (response.GetRawResponse().Headers.Contains(Constants.StructuredMessage.StructuredMessageHeader)) { - (Stream decodingStream, StructuredMessageDecodingStream.DecodedData decodedData) = StructuredMessageDecodingStream.WrapStream( + (Stream decodingStream, StructuredMessageDecodingStream.RawDecodedData decodedData) = StructuredMessageDecodingStream.WrapStream( response.Value.Content, response.Value.Details.ContentLength); stream = new StructuredMessageDecodingRetriableStream( decodingStream, decodedData, + StructuredMessage.Flags.StorageCrc64, startOffset => StructuredMessageFactory(startOffset, async: false, cancellationToken) .EnsureCompleted(), async startOffset => await StructuredMessageFactory(startOffset, async: true, cancellationToken) diff --git a/sdk/storage/Azure.Storage.Common/src/Shared/StorageCrc64Composer.cs b/sdk/storage/Azure.Storage.Common/src/Shared/StorageCrc64Composer.cs index ab6b76d78a87e..307ff23b21144 100644 --- a/sdk/storage/Azure.Storage.Common/src/Shared/StorageCrc64Composer.cs +++ b/sdk/storage/Azure.Storage.Common/src/Shared/StorageCrc64Composer.cs @@ -12,22 +12,52 @@ namespace Azure.Storage /// internal static class StorageCrc64Composer { - public static Memory Compose(params (byte[] Crc64, long OriginalDataLength)[] partitions) + public static byte[] Compose(params (byte[] Crc64, long OriginalDataLength)[] partitions) + => Compose(partitions.AsEnumerable()); + + public static byte[] Compose(IEnumerable<(byte[] Crc64, long OriginalDataLength)> partitions) { - return Compose(partitions.AsEnumerable()); + ulong result = Compose(partitions.Select(tup => (BitConverter.ToUInt64(tup.Crc64, 0), tup.OriginalDataLength))); + return BitConverter.GetBytes(result); } - public static Memory Compose(IEnumerable<(byte[] Crc64, long OriginalDataLength)> partitions) + public static byte[] Compose(params (ReadOnlyMemory Crc64, long OriginalDataLength)[] partitions) + => Compose(partitions.AsEnumerable()); + + public static byte[] Compose(IEnumerable<(ReadOnlyMemory Crc64, long OriginalDataLength)> partitions) { - ulong result = Compose(partitions.Select(tup => (BitConverter.ToUInt64(tup.Crc64, 0), tup.OriginalDataLength))); - return new Memory(BitConverter.GetBytes(result)); +#if NETSTANDARD2_1_OR_GREATER || NETCOREAPP3_0_OR_GREATER + ulong result = Compose(partitions.Select(tup => (BitConverter.ToUInt64(tup.Crc64.Span), tup.OriginalDataLength))); +#else + ulong result = Compose(partitions.Select(tup => (System.BitConverter.ToUInt64(tup.Crc64.ToArray(), 0), tup.OriginalDataLength))); +#endif + return BitConverter.GetBytes(result); } + public static byte[] Compose( + ReadOnlySpan leftCrc64, long leftOriginalDataLength, + ReadOnlySpan rightCrc64, long rightOriginalDataLength) + { +#if NETSTANDARD2_1_OR_GREATER || NETCOREAPP3_0_OR_GREATER + ulong result = Compose( + (BitConverter.ToUInt64(leftCrc64), leftOriginalDataLength), + (BitConverter.ToUInt64(rightCrc64), rightOriginalDataLength)); +#else + ulong result = Compose( + (BitConverter.ToUInt64(leftCrc64.ToArray(), 0), leftOriginalDataLength), + (BitConverter.ToUInt64(rightCrc64.ToArray(), 0), rightOriginalDataLength)); +#endif + return BitConverter.GetBytes(result); + } + + public static ulong Compose(params (ulong Crc64, long OriginalDataLength)[] partitions) + => Compose(partitions.AsEnumerable()); + public static ulong Compose(IEnumerable<(ulong Crc64, long OriginalDataLength)> partitions) { ulong composedCrc = 0; long composedDataLength = 0; - foreach (var tup in partitions) + foreach ((ulong crc64, long originalDataLength) in partitions) { composedCrc = StorageCrc64Calculator.Concatenate( uInitialCrcAB: 0, @@ -35,9 +65,9 @@ public static ulong Compose(IEnumerable<(ulong Crc64, long OriginalDataLength)> uFinalCrcA: composedCrc, uSizeA: (ulong) composedDataLength, uInitialCrcB: 0, - uFinalCrcB: tup.Crc64, - uSizeB: (ulong)tup.OriginalDataLength); - composedDataLength += tup.OriginalDataLength; + uFinalCrcB: crc64, + uSizeB: (ulong)originalDataLength); + composedDataLength += originalDataLength; } return composedCrc; } diff --git a/sdk/storage/Azure.Storage.Common/src/Shared/StructuredMessage.cs b/sdk/storage/Azure.Storage.Common/src/Shared/StructuredMessage.cs index 1757442037f92..f7e53b6612cbc 100644 --- a/sdk/storage/Azure.Storage.Common/src/Shared/StructuredMessage.cs +++ b/sdk/storage/Azure.Storage.Common/src/Shared/StructuredMessage.cs @@ -93,22 +93,18 @@ public static IDisposable GetStreamHeaderBytes( #endregion #region StreamFooter + public static int GetStreamFooterSize(Flags flags) + => flags.HasFlag(Flags.StorageCrc64) ? Crc64Length : 0; + public static void ReadStreamFooter( ReadOnlySpan buffer, - Span crc64 = default) + Flags flags, + out ulong crc64) { - int expectedBufferSize = 0; - if (!crc64.IsEmpty) - { - Errors.AssertBufferExactSize(crc64, Crc64Length, nameof(crc64)); - expectedBufferSize += Crc64Length; - } + int expectedBufferSize = GetSegmentFooterSize(flags); Errors.AssertBufferExactSize(buffer, expectedBufferSize, nameof(buffer)); - if (!crc64.IsEmpty) - { - buffer.Slice(0, Crc64Length).CopyTo(crc64); - } + crc64 = flags.HasFlag(Flags.StorageCrc64) ? BinaryPrimitives.ReadUInt64LittleEndian(buffer) : default; } public static int WriteStreamFooter(Span buffer, ReadOnlySpan crc64 = default) @@ -193,22 +189,18 @@ public static IDisposable GetSegmentHeaderBytes( #endregion #region SegmentFooter + public static int GetSegmentFooterSize(Flags flags) + => flags.HasFlag(Flags.StorageCrc64) ? Crc64Length : 0; + public static void ReadSegmentFooter( ReadOnlySpan buffer, - Span crc64 = default) + Flags flags, + out ulong crc64) { - int expectedBufferSize = 0; - if (!crc64.IsEmpty) - { - Errors.AssertBufferExactSize(crc64, Crc64Length, nameof(crc64)); - expectedBufferSize += Crc64Length; - } + int expectedBufferSize = GetSegmentFooterSize(flags); Errors.AssertBufferExactSize(buffer, expectedBufferSize, nameof(buffer)); - if (!crc64.IsEmpty) - { - buffer.Slice(0, Crc64Length).CopyTo(crc64); - } + crc64 = flags.HasFlag(Flags.StorageCrc64) ? BinaryPrimitives.ReadUInt64LittleEndian(buffer) : default; } public static int WriteSegmentFooter(Span buffer, ReadOnlySpan crc64 = default) diff --git a/sdk/storage/Azure.Storage.Common/src/Shared/StructuredMessageDecodingRetriableStream.cs b/sdk/storage/Azure.Storage.Common/src/Shared/StructuredMessageDecodingRetriableStream.cs index fe2d6697a4621..22dfaef259972 100644 --- a/sdk/storage/Azure.Storage.Common/src/Shared/StructuredMessageDecodingRetriableStream.cs +++ b/sdk/storage/Azure.Storage.Common/src/Shared/StructuredMessageDecodingRetriableStream.cs @@ -3,6 +3,7 @@ using System; using System.Buffers; +using System.Buffers.Binary; using System.Collections.Generic; using System.IO; using System.Linq; @@ -15,21 +16,30 @@ namespace Azure.Storage.Shared; internal class StructuredMessageDecodingRetriableStream : Stream { + public class DecodedData + { + public ulong Crc { get; set; } + } + private readonly Stream _innerRetriable; private long _decodedBytesRead; - private readonly List _decodedDatas; - private readonly Action _onComplete; + private readonly StructuredMessage.Flags _expectedFlags; + private readonly List _decodedDatas; + private readonly Action _onComplete; + + private StorageCrc64HashAlgorithm _totalContentCrc; - private readonly Func _decodingStreamFactory; - private readonly Func> _decodingAsyncStreamFactory; + private readonly Func _decodingStreamFactory; + private readonly Func> _decodingAsyncStreamFactory; public StructuredMessageDecodingRetriableStream( Stream initialDecodingStream, - StructuredMessageDecodingStream.DecodedData initialDecodedData, - Func decodingStreamFactory, - Func> decodingAsyncStreamFactory, - Action onComplete, + StructuredMessageDecodingStream.RawDecodedData initialDecodedData, + StructuredMessage.Flags expectedFlags, + Func decodingStreamFactory, + Func> decodingAsyncStreamFactory, + Action onComplete, ResponseClassifier responseClassifier, int maxRetries) { @@ -37,13 +47,19 @@ public StructuredMessageDecodingRetriableStream( _decodingAsyncStreamFactory = decodingAsyncStreamFactory; _innerRetriable = RetriableStream.Create(initialDecodingStream, StreamFactory, StreamFactoryAsync, responseClassifier, maxRetries); _decodedDatas = new() { initialDecodedData }; + _expectedFlags = expectedFlags; _onComplete = onComplete; + + if (expectedFlags.HasFlag(StructuredMessage.Flags.StorageCrc64)) + { + _totalContentCrc = StorageCrc64HashAlgorithm.Create(); + } } private Stream StreamFactory(long _) { - long offset = _decodedDatas.Select(d => d.SegmentCrcs?.LastOrDefault().SegmentEnd ?? 0).Sum(); - (Stream decodingStream, StructuredMessageDecodingStream.DecodedData decodedData) = _decodingStreamFactory(offset); + long offset = _decodedDatas.SelectMany(d => d.SegmentCrcs).Select(s => s.SegmentLen).Sum(); + (Stream decodingStream, StructuredMessageDecodingStream.RawDecodedData decodedData) = _decodingStreamFactory(offset); _decodedDatas.Add(decodedData); FastForwardInternal(decodingStream, _decodedBytesRead - offset, false).EnsureCompleted(); return decodingStream; @@ -51,8 +67,8 @@ private Stream StreamFactory(long _) private async ValueTask StreamFactoryAsync(long _) { - long offset = _decodedDatas.Select(d => d.SegmentCrcs?.LastOrDefault().SegmentEnd ?? 0).Sum(); - (Stream decodingStream, StructuredMessageDecodingStream.DecodedData decodedData) = await _decodingAsyncStreamFactory(offset).ConfigureAwait(false); + long offset = _decodedDatas.SelectMany(d => d.SegmentCrcs).Select(s => s.SegmentLen).Sum(); + (Stream decodingStream, StructuredMessageDecodingStream.RawDecodedData decodedData) = await _decodingAsyncStreamFactory(offset).ConfigureAwait(false); _decodedDatas.Add(decodedData); await FastForwardInternal(decodingStream, _decodedBytesRead - offset, true).ConfigureAwait(false); return decodingStream; @@ -81,21 +97,41 @@ private static async ValueTask FastForwardInternal(Stream stream, long bytes, bo protected override void Dispose(bool disposing) { - foreach (IDisposable data in _decodedDatas) - { - data.Dispose(); - } _decodedDatas.Clear(); _innerRetriable.Dispose(); } private void OnCompleted() { - StructuredMessageDecodingStream.DecodedData final = new(); - // TODO + DecodedData final = new(); + if (_totalContentCrc != null) + { + final.Crc = ValidateCrc(); + } _onComplete?.Invoke(final); } + private ulong ValidateCrc() + { + using IDisposable _ = ArrayPool.Shared.RentDisposable(StructuredMessage.Crc64Length * 2, out byte[] buf); + Span calculatedBytes = new(buf, 0, StructuredMessage.Crc64Length); + _totalContentCrc.GetCurrentHash(calculatedBytes); + ulong calculated = BinaryPrimitives.ReadUInt64LittleEndian(calculatedBytes); + + ulong reported = _decodedDatas.Count == 1 + ? _decodedDatas.First().TotalCrc.Value + : StorageCrc64Composer.Compose(_decodedDatas.SelectMany(d => d.SegmentCrcs)); + + if (calculated != reported) + { + Span reportedBytes = new(buf, calculatedBytes.Length, StructuredMessage.Crc64Length); + BinaryPrimitives.WriteUInt64LittleEndian(reportedBytes, reported); + throw Errors.ChecksumMismatch(calculatedBytes, reportedBytes); + } + + return calculated; + } + #region Read public override int Read(byte[] buffer, int offset, int count) { @@ -105,6 +141,10 @@ public override int Read(byte[] buffer, int offset, int count) { OnCompleted(); } + else + { + _totalContentCrc?.Append(new ReadOnlySpan(buffer, offset, read)); + } return read; } @@ -116,6 +156,10 @@ public override async Task ReadAsync(byte[] buffer, int offset, int count, { OnCompleted(); } + else + { + _totalContentCrc?.Append(new ReadOnlySpan(buffer, offset, read)); + } return read; } @@ -128,6 +172,10 @@ public override int Read(Span buffer) { OnCompleted(); } + else + { + _totalContentCrc?.Append(buffer.Slice(0, read)); + } return read; } @@ -139,6 +187,10 @@ public override async ValueTask ReadAsync(Memory buffer, Cancellation { OnCompleted(); } + else + { + _totalContentCrc?.Append(buffer.Span.Slice(0, read)); + } return read; } #endif diff --git a/sdk/storage/Azure.Storage.Common/src/Shared/StructuredMessageDecodingStream.cs b/sdk/storage/Azure.Storage.Common/src/Shared/StructuredMessageDecodingStream.cs index 30912eab737d4..9957c706fe8ff 100644 --- a/sdk/storage/Azure.Storage.Common/src/Shared/StructuredMessageDecodingStream.cs +++ b/sdk/storage/Azure.Storage.Common/src/Shared/StructuredMessageDecodingStream.cs @@ -3,6 +3,7 @@ using System; using System.Buffers; +using System.Buffers.Binary; using System.Collections.Generic; using System.IO; using System.Linq; @@ -38,55 +39,14 @@ namespace Azure.Storage.Shared; /// internal class StructuredMessageDecodingStream : Stream { - internal class DecodedData : IDisposable + internal class RawDecodedData { - private byte[] _crcBackingArray; - - public long? InnerStreamLength { get; private set; } - public int? TotalSegments { get; private set; } - public StructuredMessage.Flags? Flags { get; private set; } - public List<(ReadOnlyMemory SegmentCrc, long SegmentEnd)> SegmentCrcs { get; private set; } - public ReadOnlyMemory TotalCrc { get; private set; } - public bool DecodeCompleted { get; private set; } - - internal void SetStreamHeaderData(int totalSegments, long innerStreamLength, StructuredMessage.Flags flags) - { - TotalSegments = totalSegments; - InnerStreamLength = innerStreamLength; - Flags = flags; - - if (flags.HasFlag(StructuredMessage.Flags.StorageCrc64)) - { - _crcBackingArray = ArrayPool.Shared.Rent((totalSegments + 1) * StructuredMessage.Crc64Length); - SegmentCrcs = new(); - } - } - - internal void ReportSegmentCrc(ReadOnlySpan crc, int segmentNum, long segmentEnd) - { - int offset = (segmentNum - 1) * StructuredMessage.Crc64Length; - crc.CopyTo(new Span(_crcBackingArray, offset, StructuredMessage.Crc64Length)); - SegmentCrcs.Add((new ReadOnlyMemory(_crcBackingArray, offset, StructuredMessage.Crc64Length), segmentEnd)); - } - - internal void ReportTotalCrc(ReadOnlySpan crc) - { - int offset = (TotalSegments.Value) * StructuredMessage.Crc64Length; - crc.CopyTo(new Span(_crcBackingArray, offset, StructuredMessage.Crc64Length)); - TotalCrc = new ReadOnlyMemory(_crcBackingArray, offset, StructuredMessage.Crc64Length); - } - internal void MarkComplete() - { - DecodeCompleted = true; - } - - public void Dispose() - { - if (_crcBackingArray is not null) - { - ArrayPool.Shared.Return(_crcBackingArray); - } - } + public long? InnerStreamLength { get; set; } + public int? TotalSegments { get; set; } + public StructuredMessage.Flags? Flags { get; set; } + public List<(ulong SegmentCrc, long SegmentLen)> SegmentCrcs { get; } = new(); + public ulong? TotalCrc { get; set; } + public bool DecodeCompleted { get; set; } } private enum SMRegion @@ -113,7 +73,7 @@ private enum SMRegion private bool _disposed; - private readonly DecodedData _decodedData; + private readonly RawDecodedData _decodedData; private StorageCrc64HashAlgorithm _totalContentCrc; private StorageCrc64HashAlgorithm _segmentCrc; @@ -139,17 +99,17 @@ public override long Position set => throw new NotSupportedException(); } - public static (Stream DecodedStream, DecodedData DecodedData) WrapStream( + public static (Stream DecodedStream, RawDecodedData DecodedData) WrapStream( Stream innerStream, long? expextedStreamLength = default) { - DecodedData data = new(); + RawDecodedData data = new(); return (new StructuredMessageDecodingStream(innerStream, data, expextedStreamLength), data); } private StructuredMessageDecodingStream( Stream innerStream, - DecodedData decodedData, + RawDecodedData decodedData, long? expectedStreamLength) { Argument.AssertNotNull(innerStream, nameof(innerStream)); @@ -259,7 +219,7 @@ private void AssertDecodeFinished() { throw Errors.InvalidStructuredMessage("Premature end of stream."); } - _decodedData.MarkComplete(); + _decodedData.DecodeCompleted = true; } private long _innerStreamConsumed = 0; @@ -439,7 +399,9 @@ private int ProcessStreamHeader(ReadOnlySpan span) out StructuredMessage.Flags flags, out int totalSegments); - _decodedData.SetStreamHeaderData(totalSegments, streamLength, flags); + _decodedData.InnerStreamLength = streamLength; + _decodedData.Flags = flags; + _decodedData.TotalSegments = totalSegments; if (_expectedInnerStreamLength.HasValue && _expectedInnerStreamLength.Value != streamLength) { @@ -462,20 +424,24 @@ private int ProcessStreamHeader(ReadOnlySpan span) private int ProcessStreamFooter(ReadOnlySpan span) { - int totalProcessed = 0; + int footerLen = StructuredMessage.V1_0.GetStreamFooterSize(_decodedData.Flags.Value); + StructuredMessage.V1_0.ReadStreamFooter( + span.Slice(0, footerLen), + _decodedData.Flags.Value, + out ulong reportedCrc); if (_decodedData.Flags.Value.HasFlag(StructuredMessage.Flags.StorageCrc64)) { - totalProcessed += StructuredMessage.Crc64Length; - ReadOnlySpan expected = span.Slice(0, StructuredMessage.Crc64Length); - _decodedData.ReportTotalCrc(expected); + _decodedData.TotalCrc = reportedCrc; if (_validateChecksums) { - using (ArrayPool.Shared.RentAsSpanDisposable(StructuredMessage.Crc64Length, out Span calculated)) + using (ArrayPool.Shared.RentDisposable(StructuredMessage.Crc64Length * 2, out byte[] buf)) { + Span calculated = new(buf, 0, StructuredMessage.Crc64Length); _totalContentCrc.GetCurrentHash(calculated); - if (!calculated.SequenceEqual(expected)) + if (BinaryPrimitives.ReadUInt64LittleEndian(calculated) != reportedCrc) { - throw Errors.ChecksumMismatch(calculated, expected); + Span reportedAsBytes = new(buf, calculated.Length, StructuredMessage.Crc64Length); + throw Errors.ChecksumMismatch(calculated, reportedAsBytes); } } } @@ -490,8 +456,8 @@ private int ProcessStreamFooter(ReadOnlySpan span) throw Errors.InvalidStructuredMessage("Missing expected message segments."); } - _decodedData.MarkComplete(); - return totalProcessed; + _decodedData.DecodeCompleted = true; + return footerLen; } private int ProcessSegmentHeader(ReadOnlySpan span) @@ -512,27 +478,31 @@ private int ProcessSegmentHeader(ReadOnlySpan span) private int ProcessSegmentFooter(ReadOnlySpan span) { - int totalProcessed = 0; + int footerLen = StructuredMessage.V1_0.GetSegmentFooterSize(_decodedData.Flags.Value); + StructuredMessage.V1_0.ReadSegmentFooter( + span.Slice(0, footerLen), + _decodedData.Flags.Value, + out ulong reportedCrc); if (_decodedData.Flags.Value.HasFlag(StructuredMessage.Flags.StorageCrc64)) { - totalProcessed += StructuredMessage.Crc64Length; - ReadOnlySpan expected = span.Slice(0, StructuredMessage.Crc64Length); if (_validateChecksums) { - using (ArrayPool.Shared.RentAsSpanDisposable(StructuredMessage.Crc64Length, out Span calculated)) + using (ArrayPool.Shared.RentDisposable(StructuredMessage.Crc64Length * 2, out byte[] buf)) { + Span calculated = new(buf, 0, StructuredMessage.Crc64Length); _segmentCrc.GetCurrentHash(calculated); _segmentCrc = StorageCrc64HashAlgorithm.Create(); - if (!calculated.SequenceEqual(expected)) + if (BinaryPrimitives.ReadUInt64LittleEndian(calculated) != reportedCrc) { - throw Errors.ChecksumMismatch(calculated, expected); + Span reportedAsBytes = new(buf, calculated.Length, StructuredMessage.Crc64Length); + throw Errors.ChecksumMismatch(calculated, reportedAsBytes); } } } - _decodedData.ReportSegmentCrc(expected, _currentSegmentNum, _decodedContentConsumed); + _decodedData.SegmentCrcs.Add((reportedCrc, _currentSegmentContentLength)); } _currentRegion = _currentSegmentNum == _decodedData.TotalSegments ? SMRegion.StreamFooter : SMRegion.SegmentHeader; - return totalProcessed; + return footerLen; } #endregion diff --git a/sdk/storage/Azure.Storage.Common/tests/Shared/ObserveStructuredMessagePolicy.cs b/sdk/storage/Azure.Storage.Common/tests/Shared/ObserveStructuredMessagePolicy.cs index 1414f4ec80076..828c41179bba3 100644 --- a/sdk/storage/Azure.Storage.Common/tests/Shared/ObserveStructuredMessagePolicy.cs +++ b/sdk/storage/Azure.Storage.Common/tests/Shared/ObserveStructuredMessagePolicy.cs @@ -26,7 +26,7 @@ public override void OnSendingRequest(HttpMessage message) { byte[] encodedContent; byte[] underlyingContent; - StructuredMessageDecodingStream.DecodedData decodedData; + StructuredMessageDecodingStream.RawDecodedData decodedData; using (MemoryStream ms = new()) { message.Request.Content.WriteTo(ms, default); diff --git a/sdk/storage/Azure.Storage.Common/tests/StructuredMessageDecodingRetriableStreamTests.cs b/sdk/storage/Azure.Storage.Common/tests/StructuredMessageDecodingRetriableStreamTests.cs index 39d2a5566b5ff..a0f9158040b11 100644 --- a/sdk/storage/Azure.Storage.Common/tests/StructuredMessageDecodingRetriableStreamTests.cs +++ b/sdk/storage/Azure.Storage.Common/tests/StructuredMessageDecodingRetriableStreamTests.cs @@ -2,12 +2,14 @@ // Licensed under the MIT License. using System; +using System.Buffers.Binary; using System.IO; using System.Threading; using System.Threading.Tasks; using Azure.Core; using Azure.Storage.Shared; using Azure.Storage.Test.Shared; +using Microsoft.Diagnostics.Tracing.Parsers.AspNet; using Moq; using NUnit.Framework; @@ -39,7 +41,7 @@ public async ValueTask UninterruptedStream() // mock with a simple MemoryStream rather than an actual StructuredMessageDecodingStream using (Stream src = new MemoryStream(data)) - using (Stream retriableSrc = new StructuredMessageDecodingRetriableStream(src, new(), default, default, default, default, 1)) + using (Stream retriableSrc = new StructuredMessageDecodingRetriableStream(src, new(), default, default, default, default, default, 1)) using (Stream dst = new MemoryStream(dest)) { await retriableSrc.CopyToInternal(dst, Async, default); @@ -61,12 +63,16 @@ public async Task Interrupt_DataIntact([Values(true, false)] bool multipleInterr byte[] dest = new byte[data.Length]; // Mock a decoded data for the mocked StructuredMessageDecodingStream - StructuredMessageDecodingStream.DecodedData initialDecodedData = new(); - initialDecodedData.SetStreamHeaderData(segments, data.Length, StructuredMessage.Flags.StorageCrc64); + StructuredMessageDecodingStream.RawDecodedData initialDecodedData = new() + { + TotalSegments = segments, + InnerStreamLength = data.Length, + Flags = StructuredMessage.Flags.StorageCrc64 + }; // for test purposes, initialize a DecodedData, since we are not actively decoding in this test - initialDecodedData.ReportSegmentCrc(r.NextBytesInline(StructuredMessage.Crc64Length), 1, segmentLen); + initialDecodedData.SegmentCrcs.Add((BinaryPrimitives.ReadUInt64LittleEndian(r.NextBytesInline(StructuredMessage.Crc64Length)), segmentLen)); - (Stream DecodingStream, StructuredMessageDecodingStream.DecodedData DecodedData) Factory(long offset, bool faulty) + (Stream DecodingStream, StructuredMessageDecodingStream.RawDecodedData DecodedData) Factory(long offset, bool faulty) { Stream stream = new MemoryStream(data, (int)offset, data.Length - (int)offset); if (faulty) @@ -74,10 +80,14 @@ public async Task Interrupt_DataIntact([Values(true, false)] bool multipleInterr stream = new FaultyStream(stream, interruptPos, 1, new Exception(), () => { }); } // Mock a decoded data for the mocked StructuredMessageDecodingStream - StructuredMessageDecodingStream.DecodedData decodedData = new(); - decodedData.SetStreamHeaderData(segments, data.Length, StructuredMessage.Flags.StorageCrc64); + StructuredMessageDecodingStream.RawDecodedData decodedData = new() + { + TotalSegments = segments, + InnerStreamLength = data.Length, + Flags = StructuredMessage.Flags.StorageCrc64, + }; // for test purposes, initialize a DecodedData, since we are not actively decoding in this test - decodedData.ReportSegmentCrc(r.NextBytesInline(StructuredMessage.Crc64Length), 1, segmentLen); + initialDecodedData.SegmentCrcs.Add((BinaryPrimitives.ReadUInt64LittleEndian(r.NextBytesInline(StructuredMessage.Crc64Length)), segmentLen)); return (stream, decodedData); } @@ -87,8 +97,9 @@ public async Task Interrupt_DataIntact([Values(true, false)] bool multipleInterr using (Stream retriableSrc = new StructuredMessageDecodingRetriableStream( faultySrc, initialDecodedData, + default, offset => Factory(offset, multipleInterrupts), - offset => new ValueTask<(Stream DecodingStream, StructuredMessageDecodingStream.DecodedData DecodedData)>(Factory(offset, multipleInterrupts)), + offset => new ValueTask<(Stream DecodingStream, StructuredMessageDecodingStream.RawDecodedData DecodedData)>(Factory(offset, multipleInterrupts)), null, AllExceptionsRetry().Object, int.MaxValue)) @@ -112,10 +123,14 @@ public async Task Interrupt_AppropriateRewind() Random r = new(); // Mock a decoded data for the mocked StructuredMessageDecodingStream - StructuredMessageDecodingStream.DecodedData initialDecodedData = new(); - initialDecodedData.SetStreamHeaderData(segments, segments * segmentLen, StructuredMessage.Flags.StorageCrc64); + StructuredMessageDecodingStream.RawDecodedData initialDecodedData = new() + { + TotalSegments = segments, + InnerStreamLength = segments * segmentLen, + Flags = StructuredMessage.Flags.StorageCrc64, + }; // By the time of interrupt, there will be one segment reported - initialDecodedData.ReportSegmentCrc(r.NextBytesInline(StructuredMessage.Crc64Length), 1, segmentLen); + initialDecodedData.SegmentCrcs.Add((BinaryPrimitives.ReadUInt64LittleEndian(r.NextBytesInline(StructuredMessage.Crc64Length)), segmentLen)); Mock mock = new(MockBehavior.Strict); mock.SetupGet(s => s.CanRead).Returns(true); @@ -158,8 +173,9 @@ public async Task Interrupt_AppropriateRewind() Stream retriableSrc = new StructuredMessageDecodingRetriableStream( faultySrc, initialDecodedData, + default, offset => (mock.Object, new()), - offset => new(Task.FromResult((mock.Object, new StructuredMessageDecodingStream.DecodedData()))), + offset => new(Task.FromResult((mock.Object, new StructuredMessageDecodingStream.RawDecodedData()))), null, AllExceptionsRetry().Object, 1); @@ -200,7 +216,7 @@ public async Task Interrupt_ProperDecode([Values(true, false)] bool multipleInte byte[] data = r.NextBytesInline(segments * Constants.KB).ToArray(); byte[] dest = new byte[data.Length]; - (Stream DecodingStream, StructuredMessageDecodingStream.DecodedData DecodedData) Factory(long offset, bool faulty) + (Stream DecodingStream, StructuredMessageDecodingStream.RawDecodedData DecodedData) Factory(long offset, bool faulty) { Stream stream = new MemoryStream(data, (int)offset, data.Length - (int)offset); stream = new StructuredMessageEncodingStream(stream, segmentLen, StructuredMessage.Flags.StorageCrc64); @@ -211,12 +227,13 @@ public async Task Interrupt_ProperDecode([Values(true, false)] bool multipleInte return StructuredMessageDecodingStream.WrapStream(stream); } - (Stream decodingStream, StructuredMessageDecodingStream.DecodedData decodedData) = Factory(0, true); + (Stream decodingStream, StructuredMessageDecodingStream.RawDecodedData decodedData) = Factory(0, true); using Stream retriableSrc = new StructuredMessageDecodingRetriableStream( decodingStream, decodedData, + default, offset => Factory(offset, multipleInterrupts), - offset => new ValueTask<(Stream DecodingStream, StructuredMessageDecodingStream.DecodedData DecodedData)>(Factory(offset, multipleInterrupts)), + offset => new ValueTask<(Stream DecodingStream, StructuredMessageDecodingStream.RawDecodedData DecodedData)>(Factory(offset, multipleInterrupts)), null, AllExceptionsRetry().Object, int.MaxValue); diff --git a/sdk/storage/Azure.Storage.Files.Shares/src/ShareFileClient.cs b/sdk/storage/Azure.Storage.Files.Shares/src/ShareFileClient.cs index 3f6a4890d9a89..84d2ca1e99851 100644 --- a/sdk/storage/Azure.Storage.Files.Shares/src/ShareFileClient.cs +++ b/sdk/storage/Azure.Storage.Files.Shares/src/ShareFileClient.cs @@ -2402,7 +2402,7 @@ async ValueTask> Factory(long offset, bool async } return response; } - async ValueTask<(Stream DecodingStream, StructuredMessageDecodingStream.DecodedData DecodedData)> StructuredMessageFactory( + async ValueTask<(Stream DecodingStream, StructuredMessageDecodingStream.RawDecodedData DecodedData)> StructuredMessageFactory( long offset, bool async, CancellationToken cancellationToken) { Response result = await Factory(offset, async, cancellationToken).ConfigureAwait(false); @@ -2411,11 +2411,12 @@ async ValueTask> Factory(long offset, bool async if (initialResponse.GetRawResponse().Headers.Contains(Constants.StructuredMessage.StructuredMessageHeader)) { - (Stream decodingStream, StructuredMessageDecodingStream.DecodedData decodedData) = StructuredMessageDecodingStream.WrapStream( + (Stream decodingStream, StructuredMessageDecodingStream.RawDecodedData decodedData) = StructuredMessageDecodingStream.WrapStream( initialResponse.Value.Content, initialResponse.Value.ContentLength); initialResponse.Value.Content = new StructuredMessageDecodingRetriableStream( decodingStream, decodedData, + StructuredMessage.Flags.StorageCrc64, startOffset => StructuredMessageFactory(startOffset, async: false, cancellationToken) .EnsureCompleted(), async startOffset => await StructuredMessageFactory(startOffset, async: true, cancellationToken) From 5b2c153cbae27a4e46f8279364c3b563bd2ce6ae Mon Sep 17 00:00:00 2001 From: Jocelyn <41338290+jaschrep-msft@users.noreply.github.com> Date: Mon, 12 Aug 2024 10:27:46 -0400 Subject: [PATCH 17/22] Crc reporting pt2 (#45447) * expose crc from structured message * testproxy * undo typo * exportapi --- .../api/Azure.Storage.Blobs.net6.0.cs | 1 + .../api/Azure.Storage.Blobs.netstandard2.0.cs | 1 + .../api/Azure.Storage.Blobs.netstandard2.1.cs | 1 + .../src/Azure.Storage.Blobs.csproj | 1 + .../Azure.Storage.Blobs/src/BlobBaseClient.cs | 6 +- .../src/BlobClientOptions.cs | 2 + .../src/Models/BlobDownloadDetails.cs | 15 +++-- .../BlobBaseClientTransferValidationTests.cs | 62 +++++++++++++++++++ .../src/Shared/ChecksumExtensions.cs | 22 +++++++ .../src/Shared/StructuredMessage.cs | 4 +- .../Shared/StructuredMessageDecodingStream.cs | 39 +++++------- .../tests/Azure.Storage.Common.Tests.csproj | 1 + .../src/Azure.Storage.Files.DataLake.csproj | 1 + .../api/Azure.Storage.Files.Shares.net6.0.cs | 1 + ...ure.Storage.Files.Shares.netstandard2.0.cs | 1 + .../src/Azure.Storage.Files.Shares.csproj | 3 +- .../src/Models/ShareFileDownloadInfo.cs | 6 ++ .../src/ShareFileClient.cs | 6 +- .../ShareFileClientTransferValidationTests.cs | 36 +++++++++++ 19 files changed, 174 insertions(+), 35 deletions(-) create mode 100644 sdk/storage/Azure.Storage.Common/src/Shared/ChecksumExtensions.cs diff --git a/sdk/storage/Azure.Storage.Blobs/api/Azure.Storage.Blobs.net6.0.cs b/sdk/storage/Azure.Storage.Blobs/api/Azure.Storage.Blobs.net6.0.cs index 05cdde6988050..fb52e93f85a56 100644 --- a/sdk/storage/Azure.Storage.Blobs/api/Azure.Storage.Blobs.net6.0.cs +++ b/sdk/storage/Azure.Storage.Blobs/api/Azure.Storage.Blobs.net6.0.cs @@ -516,6 +516,7 @@ public BlobDownloadDetails() { } public long BlobSequenceNumber { get { throw null; } } public Azure.Storage.Blobs.Models.BlobType BlobType { get { throw null; } } public string CacheControl { get { throw null; } } + public byte[] ContentCrc { get { throw null; } } public string ContentDisposition { get { throw null; } } public string ContentEncoding { get { throw null; } } public byte[] ContentHash { get { throw null; } } diff --git a/sdk/storage/Azure.Storage.Blobs/api/Azure.Storage.Blobs.netstandard2.0.cs b/sdk/storage/Azure.Storage.Blobs/api/Azure.Storage.Blobs.netstandard2.0.cs index 05cdde6988050..fb52e93f85a56 100644 --- a/sdk/storage/Azure.Storage.Blobs/api/Azure.Storage.Blobs.netstandard2.0.cs +++ b/sdk/storage/Azure.Storage.Blobs/api/Azure.Storage.Blobs.netstandard2.0.cs @@ -516,6 +516,7 @@ public BlobDownloadDetails() { } public long BlobSequenceNumber { get { throw null; } } public Azure.Storage.Blobs.Models.BlobType BlobType { get { throw null; } } public string CacheControl { get { throw null; } } + public byte[] ContentCrc { get { throw null; } } public string ContentDisposition { get { throw null; } } public string ContentEncoding { get { throw null; } } public byte[] ContentHash { get { throw null; } } diff --git a/sdk/storage/Azure.Storage.Blobs/api/Azure.Storage.Blobs.netstandard2.1.cs b/sdk/storage/Azure.Storage.Blobs/api/Azure.Storage.Blobs.netstandard2.1.cs index 05cdde6988050..fb52e93f85a56 100644 --- a/sdk/storage/Azure.Storage.Blobs/api/Azure.Storage.Blobs.netstandard2.1.cs +++ b/sdk/storage/Azure.Storage.Blobs/api/Azure.Storage.Blobs.netstandard2.1.cs @@ -516,6 +516,7 @@ public BlobDownloadDetails() { } public long BlobSequenceNumber { get { throw null; } } public Azure.Storage.Blobs.Models.BlobType BlobType { get { throw null; } } public string CacheControl { get { throw null; } } + public byte[] ContentCrc { get { throw null; } } public string ContentDisposition { get { throw null; } } public string ContentEncoding { get { throw null; } } public byte[] ContentHash { get { throw null; } } diff --git a/sdk/storage/Azure.Storage.Blobs/src/Azure.Storage.Blobs.csproj b/sdk/storage/Azure.Storage.Blobs/src/Azure.Storage.Blobs.csproj index 851474c2d0dab..731c7468bb7b2 100644 --- a/sdk/storage/Azure.Storage.Blobs/src/Azure.Storage.Blobs.csproj +++ b/sdk/storage/Azure.Storage.Blobs/src/Azure.Storage.Blobs.csproj @@ -52,6 +52,7 @@ + diff --git a/sdk/storage/Azure.Storage.Blobs/src/BlobBaseClient.cs b/sdk/storage/Azure.Storage.Blobs/src/BlobBaseClient.cs index 1fa4f59e52e62..e7ef0d346d0f7 100644 --- a/sdk/storage/Azure.Storage.Blobs/src/BlobBaseClient.cs +++ b/sdk/storage/Azure.Storage.Blobs/src/BlobBaseClient.cs @@ -1575,7 +1575,11 @@ ValueTask> Factory(long offset, bool force .EnsureCompleted(), async startOffset => await StructuredMessageFactory(startOffset, async: true, cancellationToken) .ConfigureAwait(false), - default, //decodedData => response.Value.Details.ContentCrc = decodedData.TotalCrc.ToArray(), + decodedData => + { + response.Value.Details.ContentCrc = new byte[StructuredMessage.Crc64Length]; + decodedData.Crc.WriteCrc64(response.Value.Details.ContentCrc); + }, ClientConfiguration.Pipeline.ResponseClassifier, Constants.MaxReliabilityRetries); } diff --git a/sdk/storage/Azure.Storage.Blobs/src/BlobClientOptions.cs b/sdk/storage/Azure.Storage.Blobs/src/BlobClientOptions.cs index b16cefc83a535..f312e621bffc4 100644 --- a/sdk/storage/Azure.Storage.Blobs/src/BlobClientOptions.cs +++ b/sdk/storage/Azure.Storage.Blobs/src/BlobClientOptions.cs @@ -318,6 +318,8 @@ private void AddHeadersAndQueryParameters() Diagnostics.LoggedHeaderNames.Add("x-ms-encryption-key-sha256"); Diagnostics.LoggedHeaderNames.Add("x-ms-copy-source-error-code"); Diagnostics.LoggedHeaderNames.Add("x-ms-copy-source-status-code"); + Diagnostics.LoggedHeaderNames.Add("x-ms-structured-body"); + Diagnostics.LoggedHeaderNames.Add("x-ms-structured-content-length"); Diagnostics.LoggedQueryParameters.Add("comp"); Diagnostics.LoggedQueryParameters.Add("maxresults"); diff --git a/sdk/storage/Azure.Storage.Blobs/src/Models/BlobDownloadDetails.cs b/sdk/storage/Azure.Storage.Blobs/src/Models/BlobDownloadDetails.cs index 6104abfd9ac5f..0490ec239798e 100644 --- a/sdk/storage/Azure.Storage.Blobs/src/Models/BlobDownloadDetails.cs +++ b/sdk/storage/Azure.Storage.Blobs/src/Models/BlobDownloadDetails.cs @@ -34,14 +34,13 @@ public class BlobDownloadDetails public byte[] ContentHash { get; internal set; } #pragma warning restore CA1819 // Properties should not return arrays - // TODO enable in following PR - ///// - ///// When requested using , this value contains the CRC for the download blob range. - ///// This value may only become populated once the network stream is fully consumed. If this instance is accessed through - ///// , the network stream has already been consumed. Otherwise, consume the content stream before - ///// checking this value. - ///// - //public byte[] ContentCrc { get; internal set; } + /// + /// When requested using , this value contains the CRC for the download blob range. + /// This value may only become populated once the network stream is fully consumed. If this instance is accessed through + /// , the network stream has already been consumed. Otherwise, consume the content stream before + /// checking this value. + /// + public byte[] ContentCrc { get; internal set; } /// /// Returns the date and time the container was last modified. Any operation that modifies the blob, including an update of the blob's metadata or properties, changes the last-modified time of the blob. diff --git a/sdk/storage/Azure.Storage.Blobs/tests/BlobBaseClientTransferValidationTests.cs b/sdk/storage/Azure.Storage.Blobs/tests/BlobBaseClientTransferValidationTests.cs index 76d807835873c..c502231087ed6 100644 --- a/sdk/storage/Azure.Storage.Blobs/tests/BlobBaseClientTransferValidationTests.cs +++ b/sdk/storage/Azure.Storage.Blobs/tests/BlobBaseClientTransferValidationTests.cs @@ -124,6 +124,68 @@ public virtual async Task OlderServiceVersionThrowsOnStructuredMessage() })).Value.Content.CopyToAsync(Stream.Null); Assert.That(operation, Throws.TypeOf()); } + + [Test] + public async Task StructuredMessagePopulatesCrcDownloadStreaming() + { + await using DisposingContainer disposingContainer = await ClientBuilder.GetTestContainerAsync( + publicAccessType: PublicAccessType.None); + + const int dataLength = Constants.KB; + byte[] data = GetRandomBuffer(dataLength); + byte[] dataCrc = new byte[8]; + StorageCrc64Calculator.ComputeSlicedSafe(data, 0L).WriteCrc64(dataCrc); + + var blob = disposingContainer.Container.GetBlobClient(GetNewResourceName()); + await blob.UploadAsync(BinaryData.FromBytes(data)); + + Response response = await blob.DownloadStreamingAsync(new() + { + TransferValidation = new DownloadTransferValidationOptions + { + ChecksumAlgorithm = StorageChecksumAlgorithm.StorageCrc64 + } + }); + + // crc is not present until response stream is consumed + Assert.That(response.Value.Details.ContentCrc, Is.Null); + + byte[] downloadedData; + using (MemoryStream ms = new()) + { + await response.Value.Content.CopyToAsync(ms); + downloadedData = ms.ToArray(); + } + + Assert.That(response.Value.Details.ContentCrc, Is.EqualTo(dataCrc)); + Assert.That(downloadedData, Is.EqualTo(data)); + } + + [Test] + public async Task StructuredMessagePopulatesCrcDownloadContent() + { + await using DisposingContainer disposingContainer = await ClientBuilder.GetTestContainerAsync( + publicAccessType: PublicAccessType.None); + + const int dataLength = Constants.KB; + byte[] data = GetRandomBuffer(dataLength); + byte[] dataCrc = new byte[8]; + StorageCrc64Calculator.ComputeSlicedSafe(data, 0L).WriteCrc64(dataCrc); + + var blob = disposingContainer.Container.GetBlobClient(GetNewResourceName()); + await blob.UploadAsync(BinaryData.FromBytes(data)); + + Response response = await blob.DownloadContentAsync(new BlobDownloadOptions() + { + TransferValidation = new DownloadTransferValidationOptions + { + ChecksumAlgorithm = StorageChecksumAlgorithm.StorageCrc64 + } + }); + + Assert.That(response.Value.Details.ContentCrc, Is.EqualTo(dataCrc)); + Assert.That(response.Value.Content.ToArray(), Is.EqualTo(data)); + } #endregion } } diff --git a/sdk/storage/Azure.Storage.Common/src/Shared/ChecksumExtensions.cs b/sdk/storage/Azure.Storage.Common/src/Shared/ChecksumExtensions.cs new file mode 100644 index 0000000000000..48304640eee43 --- /dev/null +++ b/sdk/storage/Azure.Storage.Common/src/Shared/ChecksumExtensions.cs @@ -0,0 +1,22 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +using System; +using System.Buffers.Binary; + +namespace Azure.Storage; + +internal static class ChecksumExtensions +{ + public static void WriteCrc64(this ulong crc, Span dest) + => BinaryPrimitives.WriteUInt64LittleEndian(dest, crc); + + public static bool TryWriteCrc64(this ulong crc, Span dest) + => BinaryPrimitives.TryWriteUInt64LittleEndian(dest, crc); + + public static ulong ReadCrc64(this ReadOnlySpan crc) + => BinaryPrimitives.ReadUInt64LittleEndian(crc); + + public static bool TryReadCrc64(this ReadOnlySpan crc, out ulong value) + => BinaryPrimitives.TryReadUInt64LittleEndian(crc, out value); +} diff --git a/sdk/storage/Azure.Storage.Common/src/Shared/StructuredMessage.cs b/sdk/storage/Azure.Storage.Common/src/Shared/StructuredMessage.cs index f7e53b6612cbc..a0a46837797b9 100644 --- a/sdk/storage/Azure.Storage.Common/src/Shared/StructuredMessage.cs +++ b/sdk/storage/Azure.Storage.Common/src/Shared/StructuredMessage.cs @@ -104,7 +104,7 @@ public static void ReadStreamFooter( int expectedBufferSize = GetSegmentFooterSize(flags); Errors.AssertBufferExactSize(buffer, expectedBufferSize, nameof(buffer)); - crc64 = flags.HasFlag(Flags.StorageCrc64) ? BinaryPrimitives.ReadUInt64LittleEndian(buffer) : default; + crc64 = flags.HasFlag(Flags.StorageCrc64) ? buffer.ReadCrc64() : default; } public static int WriteStreamFooter(Span buffer, ReadOnlySpan crc64 = default) @@ -200,7 +200,7 @@ public static void ReadSegmentFooter( int expectedBufferSize = GetSegmentFooterSize(flags); Errors.AssertBufferExactSize(buffer, expectedBufferSize, nameof(buffer)); - crc64 = flags.HasFlag(Flags.StorageCrc64) ? BinaryPrimitives.ReadUInt64LittleEndian(buffer) : default; + crc64 = flags.HasFlag(Flags.StorageCrc64) ? buffer.ReadCrc64() : default; } public static int WriteSegmentFooter(Span buffer, ReadOnlySpan crc64 = default) diff --git a/sdk/storage/Azure.Storage.Common/src/Shared/StructuredMessageDecodingStream.cs b/sdk/storage/Azure.Storage.Common/src/Shared/StructuredMessageDecodingStream.cs index 9957c706fe8ff..e6b193ae18260 100644 --- a/sdk/storage/Azure.Storage.Common/src/Shared/StructuredMessageDecodingStream.cs +++ b/sdk/storage/Azure.Storage.Common/src/Shared/StructuredMessageDecodingStream.cs @@ -431,20 +431,11 @@ private int ProcessStreamFooter(ReadOnlySpan span) out ulong reportedCrc); if (_decodedData.Flags.Value.HasFlag(StructuredMessage.Flags.StorageCrc64)) { - _decodedData.TotalCrc = reportedCrc; if (_validateChecksums) { - using (ArrayPool.Shared.RentDisposable(StructuredMessage.Crc64Length * 2, out byte[] buf)) - { - Span calculated = new(buf, 0, StructuredMessage.Crc64Length); - _totalContentCrc.GetCurrentHash(calculated); - if (BinaryPrimitives.ReadUInt64LittleEndian(calculated) != reportedCrc) - { - Span reportedAsBytes = new(buf, calculated.Length, StructuredMessage.Crc64Length); - throw Errors.ChecksumMismatch(calculated, reportedAsBytes); - } - } + ValidateCrc64(_totalContentCrc, reportedCrc); } + _decodedData.TotalCrc = reportedCrc; } if (_innerStreamConsumed != _decodedData.InnerStreamLength) @@ -487,23 +478,27 @@ private int ProcessSegmentFooter(ReadOnlySpan span) { if (_validateChecksums) { - using (ArrayPool.Shared.RentDisposable(StructuredMessage.Crc64Length * 2, out byte[] buf)) - { - Span calculated = new(buf, 0, StructuredMessage.Crc64Length); - _segmentCrc.GetCurrentHash(calculated); - _segmentCrc = StorageCrc64HashAlgorithm.Create(); - if (BinaryPrimitives.ReadUInt64LittleEndian(calculated) != reportedCrc) - { - Span reportedAsBytes = new(buf, calculated.Length, StructuredMessage.Crc64Length); - throw Errors.ChecksumMismatch(calculated, reportedAsBytes); - } - } + ValidateCrc64(_segmentCrc, reportedCrc); + _segmentCrc = StorageCrc64HashAlgorithm.Create(); } _decodedData.SegmentCrcs.Add((reportedCrc, _currentSegmentContentLength)); } _currentRegion = _currentSegmentNum == _decodedData.TotalSegments ? SMRegion.StreamFooter : SMRegion.SegmentHeader; return footerLen; } + + private static void ValidateCrc64(StorageCrc64HashAlgorithm calculation, ulong reported) + { + using IDisposable _ = ArrayPool.Shared.RentDisposable(StructuredMessage.Crc64Length * 2, out byte[] buf); + Span calculatedBytes = new(buf, 0, StructuredMessage.Crc64Length); + Span reportedBytes = new(buf, calculatedBytes.Length, StructuredMessage.Crc64Length); + calculation.GetCurrentHash(calculatedBytes); + reported.WriteCrc64(reportedBytes); + if (!calculatedBytes.SequenceEqual(reportedBytes)) + { + throw Errors.ChecksumMismatch(calculatedBytes, reportedBytes); + } + } #endregion public override long Seek(long offset, SeekOrigin origin) diff --git a/sdk/storage/Azure.Storage.Common/tests/Azure.Storage.Common.Tests.csproj b/sdk/storage/Azure.Storage.Common/tests/Azure.Storage.Common.Tests.csproj index 6a0dc0506be51..2863b85f6feb2 100644 --- a/sdk/storage/Azure.Storage.Common/tests/Azure.Storage.Common.Tests.csproj +++ b/sdk/storage/Azure.Storage.Common/tests/Azure.Storage.Common.Tests.csproj @@ -31,6 +31,7 @@ + diff --git a/sdk/storage/Azure.Storage.Files.DataLake/src/Azure.Storage.Files.DataLake.csproj b/sdk/storage/Azure.Storage.Files.DataLake/src/Azure.Storage.Files.DataLake.csproj index 8a2e0bcd97d46..ccd45baaff251 100644 --- a/sdk/storage/Azure.Storage.Files.DataLake/src/Azure.Storage.Files.DataLake.csproj +++ b/sdk/storage/Azure.Storage.Files.DataLake/src/Azure.Storage.Files.DataLake.csproj @@ -42,6 +42,7 @@ + diff --git a/sdk/storage/Azure.Storage.Files.Shares/api/Azure.Storage.Files.Shares.net6.0.cs b/sdk/storage/Azure.Storage.Files.Shares/api/Azure.Storage.Files.Shares.net6.0.cs index 88fbd1326e018..0cd25700dd1d7 100644 --- a/sdk/storage/Azure.Storage.Files.Shares/api/Azure.Storage.Files.Shares.net6.0.cs +++ b/sdk/storage/Azure.Storage.Files.Shares/api/Azure.Storage.Files.Shares.net6.0.cs @@ -796,6 +796,7 @@ public partial class ShareFileDownloadInfo : System.IDisposable { internal ShareFileDownloadInfo() { } public System.IO.Stream Content { get { throw null; } } + public byte[] ContentCrc { get { throw null; } } public byte[] ContentHash { get { throw null; } } public long ContentLength { get { throw null; } } public string ContentType { get { throw null; } } diff --git a/sdk/storage/Azure.Storage.Files.Shares/api/Azure.Storage.Files.Shares.netstandard2.0.cs b/sdk/storage/Azure.Storage.Files.Shares/api/Azure.Storage.Files.Shares.netstandard2.0.cs index 88fbd1326e018..0cd25700dd1d7 100644 --- a/sdk/storage/Azure.Storage.Files.Shares/api/Azure.Storage.Files.Shares.netstandard2.0.cs +++ b/sdk/storage/Azure.Storage.Files.Shares/api/Azure.Storage.Files.Shares.netstandard2.0.cs @@ -796,6 +796,7 @@ public partial class ShareFileDownloadInfo : System.IDisposable { internal ShareFileDownloadInfo() { } public System.IO.Stream Content { get { throw null; } } + public byte[] ContentCrc { get { throw null; } } public byte[] ContentHash { get { throw null; } } public long ContentLength { get { throw null; } } public string ContentType { get { throw null; } } diff --git a/sdk/storage/Azure.Storage.Files.Shares/src/Azure.Storage.Files.Shares.csproj b/sdk/storage/Azure.Storage.Files.Shares/src/Azure.Storage.Files.Shares.csproj index a3e805eeed900..547cccbd0a5c3 100644 --- a/sdk/storage/Azure.Storage.Files.Shares/src/Azure.Storage.Files.Shares.csproj +++ b/sdk/storage/Azure.Storage.Files.Shares/src/Azure.Storage.Files.Shares.csproj @@ -1,4 +1,4 @@ - + $(RequiredTargetFrameworks);net6.0 @@ -42,6 +42,7 @@ + diff --git a/sdk/storage/Azure.Storage.Files.Shares/src/Models/ShareFileDownloadInfo.cs b/sdk/storage/Azure.Storage.Files.Shares/src/Models/ShareFileDownloadInfo.cs index 0165af94435a0..4037cbdfd875e 100644 --- a/sdk/storage/Azure.Storage.Files.Shares/src/Models/ShareFileDownloadInfo.cs +++ b/sdk/storage/Azure.Storage.Files.Shares/src/Models/ShareFileDownloadInfo.cs @@ -38,6 +38,12 @@ public partial class ShareFileDownloadInfo : IDisposable, IDownloadedContent public byte[] ContentHash { get; internal set; } #pragma warning restore CA1819 // Properties should not return arrays + /// + /// When requested using , this value contains the CRC for the download blob range. + /// This value may only become populated once the network stream is fully consumed. + /// + public byte[] ContentCrc { get; internal set; } + /// /// Details returned when downloading a file /// diff --git a/sdk/storage/Azure.Storage.Files.Shares/src/ShareFileClient.cs b/sdk/storage/Azure.Storage.Files.Shares/src/ShareFileClient.cs index 84d2ca1e99851..23c5fd40d2db1 100644 --- a/sdk/storage/Azure.Storage.Files.Shares/src/ShareFileClient.cs +++ b/sdk/storage/Azure.Storage.Files.Shares/src/ShareFileClient.cs @@ -2421,7 +2421,11 @@ async ValueTask> Factory(long offset, bool async .EnsureCompleted(), async startOffset => await StructuredMessageFactory(startOffset, async: true, cancellationToken) .ConfigureAwait(false), - default, //decodedData => response.Value.Details.ContentCrc = decodedData.TotalCrc.ToArray(), + decodedData => + { + initialResponse.Value.ContentCrc = new byte[StructuredMessage.Crc64Length]; + decodedData.Crc.WriteCrc64(initialResponse.Value.ContentCrc); + }, ClientConfiguration.Pipeline.ResponseClassifier, Constants.MaxReliabilityRetries); } diff --git a/sdk/storage/Azure.Storage.Files.Shares/tests/ShareFileClientTransferValidationTests.cs b/sdk/storage/Azure.Storage.Files.Shares/tests/ShareFileClientTransferValidationTests.cs index afe33c95847d0..4cfa2b7271065 100644 --- a/sdk/storage/Azure.Storage.Files.Shares/tests/ShareFileClientTransferValidationTests.cs +++ b/sdk/storage/Azure.Storage.Files.Shares/tests/ShareFileClientTransferValidationTests.cs @@ -146,5 +146,41 @@ public override void TestAutoResolve() StorageChecksumAlgorithm.MD5, TransferValidationOptionsExtensions.ResolveAuto(StorageChecksumAlgorithm.Auto)); } + + [Test] + public async Task StructuredMessagePopulatesCrcDownloadStreaming() + { + await using DisposingShare disposingContainer = await ClientBuilder.GetTestShareAsync(); + + const int dataLength = Constants.KB; + byte[] data = GetRandomBuffer(dataLength); + byte[] dataCrc = new byte[8]; + StorageCrc64Calculator.ComputeSlicedSafe(data, 0L).WriteCrc64(dataCrc); + + ShareFileClient file = disposingContainer.Container.GetRootDirectoryClient().GetFileClient(GetNewResourceName()); + await file.CreateAsync(data.Length); + await file.UploadAsync(new MemoryStream(data)); + + Response response = await file.DownloadAsync(new ShareFileDownloadOptions() + { + TransferValidation = new DownloadTransferValidationOptions + { + ChecksumAlgorithm = StorageChecksumAlgorithm.StorageCrc64 + } + }); + + // crc is not present until response stream is consumed + Assert.That(response.Value.ContentCrc, Is.Null); + + byte[] downloadedData; + using (MemoryStream ms = new()) + { + await response.Value.Content.CopyToAsync(ms); + downloadedData = ms.ToArray(); + } + + Assert.That(response.Value.ContentCrc, Is.EqualTo(dataCrc)); + Assert.That(downloadedData, Is.EqualTo(data)); + } } } From 1fe63647390aa17b7020d30aa952ba1a857c393f Mon Sep 17 00:00:00 2001 From: Jocelyn Schreppler Date: Tue, 13 Aug 2024 14:31:12 -0400 Subject: [PATCH 18/22] testproxy --- sdk/storage/Azure.Storage.Blobs/assets.json | 2 +- sdk/storage/Azure.Storage.Files.DataLake/assets.json | 2 +- sdk/storage/Azure.Storage.Files.Shares/assets.json | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/sdk/storage/Azure.Storage.Blobs/assets.json b/sdk/storage/Azure.Storage.Blobs/assets.json index 6a8a60c8c101a..bf650c1112c2f 100644 --- a/sdk/storage/Azure.Storage.Blobs/assets.json +++ b/sdk/storage/Azure.Storage.Blobs/assets.json @@ -2,5 +2,5 @@ "AssetsRepo": "Azure/azure-sdk-assets", "AssetsRepoPrefixPath": "net", "TagPrefix": "net/storage/Azure.Storage.Blobs", - "Tag": "net/storage/Azure.Storage.Blobs_8b3f7ac2a4" + "Tag": "net/storage/Azure.Storage.Blobs_d0e3597ddc" } diff --git a/sdk/storage/Azure.Storage.Files.DataLake/assets.json b/sdk/storage/Azure.Storage.Files.DataLake/assets.json index e1cf4a6fa1c07..8949234de7a1a 100644 --- a/sdk/storage/Azure.Storage.Files.DataLake/assets.json +++ b/sdk/storage/Azure.Storage.Files.DataLake/assets.json @@ -2,5 +2,5 @@ "AssetsRepo": "Azure/azure-sdk-assets", "AssetsRepoPrefixPath": "net", "TagPrefix": "net/storage/Azure.Storage.Files.DataLake", - "Tag": "net/storage/Azure.Storage.Files.DataLake_c7efac8f52" + "Tag": "net/storage/Azure.Storage.Files.DataLake_4b543941a8" } diff --git a/sdk/storage/Azure.Storage.Files.Shares/assets.json b/sdk/storage/Azure.Storage.Files.Shares/assets.json index 7a34db9bef740..173fe0df0636c 100644 --- a/sdk/storage/Azure.Storage.Files.Shares/assets.json +++ b/sdk/storage/Azure.Storage.Files.Shares/assets.json @@ -2,5 +2,5 @@ "AssetsRepo": "Azure/azure-sdk-assets", "AssetsRepoPrefixPath": "net", "TagPrefix": "net/storage/Azure.Storage.Files.Shares", - "Tag": "net/storage/Azure.Storage.Files.Shares_4ec8e7e485" + "Tag": "net/storage/Azure.Storage.Files.Shares_dbed55c6e3" } From bd28c7f5cb18308c745741739d539fe4439f216b Mon Sep 17 00:00:00 2001 From: Jocelyn Schreppler Date: Wed, 14 Aug 2024 11:47:27 -0400 Subject: [PATCH 19/22] remove unused parameter --- .../Azure.Storage.Blobs/src/BlobBaseClient.cs | 13 ++++--------- 1 file changed, 4 insertions(+), 9 deletions(-) diff --git a/sdk/storage/Azure.Storage.Blobs/src/BlobBaseClient.cs b/sdk/storage/Azure.Storage.Blobs/src/BlobBaseClient.cs index e7ef0d346d0f7..99bec6af12cb7 100644 --- a/sdk/storage/Azure.Storage.Blobs/src/BlobBaseClient.cs +++ b/sdk/storage/Azure.Storage.Blobs/src/BlobBaseClient.cs @@ -1547,19 +1547,18 @@ internal virtual async ValueTask> Download // Wrap the response Content in a RetriableStream so we // can return it before it's finished downloading, but still // allow retrying if it fails. - ValueTask> Factory(long offset, bool forceStructuredMessage, bool async, CancellationToken cancellationToken) + ValueTask> Factory(long offset, bool async, CancellationToken cancellationToken) => StartDownloadAsync( range, conditionsWithEtag, validationOptions, offset, - forceStructuredMessage, async, cancellationToken); async ValueTask<(Stream DecodingStream, StructuredMessageDecodingStream.RawDecodedData DecodedData)> StructuredMessageFactory( long offset, bool async, CancellationToken cancellationToken) { - Response result = await Factory(offset, forceStructuredMessage: true, async, cancellationToken).ConfigureAwait(false); + Response result = await Factory(offset, async, cancellationToken).ConfigureAwait(false); return StructuredMessageDecodingStream.WrapStream(result.Value.Content, result.Value.Details.ContentLength); } Stream stream; @@ -1587,9 +1586,9 @@ ValueTask> Factory(long offset, bool force { stream = RetriableStream.Create( response.Value.Content, - startOffset => Factory(startOffset, forceStructuredMessage: false, async: false, cancellationToken) + startOffset => Factory(startOffset, async: false, cancellationToken) .EnsureCompleted().Value.Content, - async startOffset => (await Factory(startOffset, forceStructuredMessage: false, async: true, cancellationToken) + async startOffset => (await Factory(startOffset, async: true, cancellationToken) .ConfigureAwait(false)).Value.Content, ClientConfiguration.Pipeline.ResponseClassifier, Constants.MaxReliabilityRetries); @@ -1668,9 +1667,6 @@ await ContentHasher.AssertResponseHashMatchInternal( /// /// Starting offset to request - in the event of a retry. /// - /// - /// When using transactional CRC, force the request to use structured message. - /// /// /// Whether to invoke the operation asynchronously. /// @@ -1692,7 +1688,6 @@ private async ValueTask> StartDownloadAsyn BlobRequestConditions conditions, DownloadTransferValidationOptions validationOptions, long startOffset = 0, - bool forceStructuredMessage = false, // TODO all CRC will force structured message in future bool async = true, CancellationToken cancellationToken = default) { From 6c6b8f50770eec6c9645801c9283946ef2a4474c Mon Sep 17 00:00:00 2001 From: Jocelyn Schreppler Date: Wed, 14 Aug 2024 12:08:04 -0400 Subject: [PATCH 20/22] add `ExpectTrailingDetails` to download response --- .../Azure.Storage.Blobs/src/BlobBaseClient.cs | 9 ++++++--- .../src/Models/BlobDownloadInfo.cs | 15 +++++++-------- .../src/Models/BlobDownloadStreamingResult.cs | 8 ++++++++ 3 files changed, 21 insertions(+), 11 deletions(-) diff --git a/sdk/storage/Azure.Storage.Blobs/src/BlobBaseClient.cs b/sdk/storage/Azure.Storage.Blobs/src/BlobBaseClient.cs index 99bec6af12cb7..b48da27583a98 100644 --- a/sdk/storage/Azure.Storage.Blobs/src/BlobBaseClient.cs +++ b/sdk/storage/Azure.Storage.Blobs/src/BlobBaseClient.cs @@ -1031,6 +1031,7 @@ private async Task> DownloadInternal( ContentHash = blobDownloadDetails.ContentHash, ContentLength = blobDownloadDetails.ContentLength, ContentType = blobDownloadDetails.ContentType, + ExpectTrailingDetails = blobDownloadStreamingResult.ExpectTrailingDetails, }, response.GetRawResponse()); } #endregion @@ -1675,8 +1676,8 @@ await ContentHasher.AssertResponseHashMatchInternal( /// notifications that the operation should be cancelled. /// /// - /// A describing the - /// downloaded blob. contains + /// A describing the + /// downloaded blob. contains /// the blob's data. /// /// @@ -1772,9 +1773,11 @@ private async ValueTask> StartDownloadAsyn long length = response.IsUnavailable() ? 0 : response.Headers.ContentLength ?? 0; ClientConfiguration.Pipeline.LogTrace($"Response: {response.GetRawResponse().Status}, ContentLength: {length}"); - return Response.FromValue( + Response result = Response.FromValue( response.ToBlobDownloadStreamingResult(), response.GetRawResponse()); + result.Value.ExpectTrailingDetails = structuredBodyType != null; + return result; } #endregion diff --git a/sdk/storage/Azure.Storage.Blobs/src/Models/BlobDownloadInfo.cs b/sdk/storage/Azure.Storage.Blobs/src/Models/BlobDownloadInfo.cs index 1a525c718d1b4..b42801e36ab55 100644 --- a/sdk/storage/Azure.Storage.Blobs/src/Models/BlobDownloadInfo.cs +++ b/sdk/storage/Azure.Storage.Blobs/src/Models/BlobDownloadInfo.cs @@ -51,14 +51,13 @@ public class BlobDownloadInfo : IDisposable, IDownloadedContent /// public BlobDownloadDetails Details { get; internal set; } - // TODO enable in following PR - ///// - ///// Indicates some contents of are mixed into the response stream. - ///// They will not be set until has been fully consumed. These details - ///// will be extracted from the content stream by the library before the calling code can - ///// encounter them. - ///// - //public bool ExpectTrailingDetails { get; internal set; } + /// + /// Indicates some contents of are mixed into the response stream. + /// They will not be set until has been fully consumed. These details + /// will be extracted from the content stream by the library before the calling code can + /// encounter them. + /// + public bool ExpectTrailingDetails { get; internal set; } /// /// Constructor. diff --git a/sdk/storage/Azure.Storage.Blobs/src/Models/BlobDownloadStreamingResult.cs b/sdk/storage/Azure.Storage.Blobs/src/Models/BlobDownloadStreamingResult.cs index 4fbada6e67aad..9b7d4d4e00dad 100644 --- a/sdk/storage/Azure.Storage.Blobs/src/Models/BlobDownloadStreamingResult.cs +++ b/sdk/storage/Azure.Storage.Blobs/src/Models/BlobDownloadStreamingResult.cs @@ -24,6 +24,14 @@ internal BlobDownloadStreamingResult() { } /// public Stream Content { get; internal set; } + /// + /// Indicates some contents of are mixed into the response stream. + /// They will not be set until has been fully consumed. These details + /// will be extracted from the content stream by the library before the calling code can + /// encounter them. + /// + public bool ExpectTrailingDetails { get; internal set; } + /// /// Disposes the by calling Dispose on the underlying stream. /// From 64830e742e3fb2a745814c6f97d9926ad3238356 Mon Sep 17 00:00:00 2001 From: Jocelyn Schreppler Date: Wed, 14 Aug 2024 13:18:21 -0400 Subject: [PATCH 21/22] fix test inconsistency --- .../TransferValidationOptionsExtensions.cs | 7 ---- .../Shared/TransferValidationTestBase.cs | 34 ------------------- 2 files changed, 41 deletions(-) diff --git a/sdk/storage/Azure.Storage.Common/src/Shared/TransferValidationOptionsExtensions.cs b/sdk/storage/Azure.Storage.Common/src/Shared/TransferValidationOptionsExtensions.cs index af21588b4ae09..763d385240383 100644 --- a/sdk/storage/Azure.Storage.Common/src/Shared/TransferValidationOptionsExtensions.cs +++ b/sdk/storage/Azure.Storage.Common/src/Shared/TransferValidationOptionsExtensions.cs @@ -9,14 +9,7 @@ public static StorageChecksumAlgorithm ResolveAuto(this StorageChecksumAlgorithm { if (checksumAlgorithm == StorageChecksumAlgorithm.Auto) { -#if BlobSDK || DataLakeSDK || CommonSDK return StorageChecksumAlgorithm.StorageCrc64; -#elif FileSDK // file shares don't support crc64 - return StorageChecksumAlgorithm.MD5; -#else - throw new System.NotSupportedException( - $"{typeof(TransferValidationOptionsExtensions).FullName}.{nameof(ResolveAuto)} is not supported."); -#endif } return checksumAlgorithm; } diff --git a/sdk/storage/Azure.Storage.Common/tests/Shared/TransferValidationTestBase.cs b/sdk/storage/Azure.Storage.Common/tests/Shared/TransferValidationTestBase.cs index ed5651b0b0fc5..248acf8811960 100644 --- a/sdk/storage/Azure.Storage.Common/tests/Shared/TransferValidationTestBase.cs +++ b/sdk/storage/Azure.Storage.Common/tests/Shared/TransferValidationTestBase.cs @@ -227,7 +227,6 @@ void AssertChecksum(Request req, string headerName) }; } -#if BlobSDK || DataLakeSDK internal static Action GetRequestStructuredMessageAssertion( StructuredMessage.Flags flags, Func isStructuredMessageExpected = default, @@ -252,7 +251,6 @@ internal static Action GetRequestStructuredMessageAssertion( Assert.That(request.Headers.TryGetValue("x-ms-structured-content-length", out string structuredContentLength)); }; } -#endif /// /// Gets an assertion as to whether a transactional checksum appeared on a returned response. @@ -310,7 +308,6 @@ void AssertChecksum(ResponseHeaders headers, string headerName) }; } -#if BlobSDK || DataLakeSDK internal static Action GetResponseStructuredMessageAssertion( StructuredMessage.Flags flags, Func isStructuredMessageExpected = default) @@ -334,7 +331,6 @@ internal static Action GetResponseStructuredMessageAssertion( Assert.That(response.Headers.TryGetValue("x-ms-structured-content-length", out string structuredContentLength)); }; } -#endif /// /// Asserts the service returned an error that expected checksum did not match checksum on upload. @@ -417,13 +413,9 @@ public virtual async Task UploadPartitionSuccessfulHashComputation(StorageChecks }; // make pipeline assertion for checking checksum was present on upload -#if BlobSDK || DataLakeSDK var assertion = algorithm.ResolveAuto() == StorageChecksumAlgorithm.StorageCrc64 ? GetRequestStructuredMessageAssertion(StructuredMessage.Flags.StorageCrc64, null, dataLength) : GetRequestChecksumHeaderAssertion(algorithm); -#else - var assertion = GetRequestChecksumHeaderAssertion(algorithm); -#endif var checksumPipelineAssertion = new AssertMessageContentsPolicy(checkRequest: assertion); var clientOptions = ClientBuilder.GetOptions(); clientOptions.AddPolicy(checksumPipelineAssertion, HttpPipelinePosition.PerCall); @@ -530,12 +522,8 @@ public virtual async Task UploadPartitionTamperedStreamThrows(StorageChecksumAlg AsyncTestDelegate operation = async () => await UploadPartitionAsync(client, stream, validationOptions); using var listener = AzureEventSourceListener.CreateConsoleLogger(); // Assert -#if BlobSDK || DataLakeSDK AssertWriteChecksumMismatch(operation, algorithm, expectStructuredMessage: algorithm.ResolveAuto() == StorageChecksumAlgorithm.StorageCrc64); -#else - AssertWriteChecksumMismatch(operation, algorithm); -#endif } } @@ -550,13 +538,9 @@ public virtual async Task UploadPartitionUsesDefaultClientValidationOptions( var data = GetRandomBuffer(dataLength); // make pipeline assertion for checking checksum was present on upload -#if BlobSDK || DataLakeSDK var assertion = clientAlgorithm.ResolveAuto() == StorageChecksumAlgorithm.StorageCrc64 ? GetRequestStructuredMessageAssertion(StructuredMessage.Flags.StorageCrc64, null, dataLength) : GetRequestChecksumHeaderAssertion(clientAlgorithm); -#else - var assertion = GetRequestChecksumHeaderAssertion(clientAlgorithm); -#endif var checksumPipelineAssertion = new AssertMessageContentsPolicy(checkRequest: assertion); var clientOptions = ClientBuilder.GetOptions(); clientOptions.AddPolicy(checksumPipelineAssertion, HttpPipelinePosition.PerCall); @@ -596,13 +580,9 @@ public virtual async Task UploadPartitionOverwritesDefaultClientValidationOption }; // make pipeline assertion for checking checksum was present on upload -#if BlobSDK || DataLakeSDK var assertion = overrideAlgorithm.ResolveAuto() == StorageChecksumAlgorithm.StorageCrc64 ? GetRequestStructuredMessageAssertion(StructuredMessage.Flags.StorageCrc64, null, dataLength) : GetRequestChecksumHeaderAssertion(overrideAlgorithm); -#else - var assertion = GetRequestChecksumHeaderAssertion(overrideAlgorithm); -#endif var checksumPipelineAssertion = new AssertMessageContentsPolicy(checkRequest: assertion); var clientOptions = ClientBuilder.GetOptions(); clientOptions.AddPolicy(checksumPipelineAssertion, HttpPipelinePosition.PerCall); @@ -1021,13 +1001,9 @@ public virtual async Task ParallelUploadOneShotSuccessfulHashComputation(Storage }; // make pipeline assertion for checking checksum was present on upload -#if BlobSDK || DataLakeSDK var assertion = algorithm.ResolveAuto() == StorageChecksumAlgorithm.StorageCrc64 ? GetRequestStructuredMessageAssertion(StructuredMessage.Flags.StorageCrc64, ParallelUploadIsChecksumExpected, dataLength) : GetRequestChecksumHeaderAssertion(algorithm, isChecksumExpected: ParallelUploadIsChecksumExpected); -#else - var assertion = GetRequestChecksumHeaderAssertion(algorithm, isChecksumExpected: ParallelUploadIsChecksumExpected); -#endif var checksumPipelineAssertion = new AssertMessageContentsPolicy(checkRequest: assertion); var clientOptions = ClientBuilder.GetOptions(); clientOptions.AddPolicy(checksumPipelineAssertion, HttpPipelinePosition.PerCall); @@ -1115,13 +1091,9 @@ public virtual async Task ParallelUploadUsesDefaultClientValidationOptions( }; // make pipeline assertion for checking checksum was present on upload -#if BlobSDK || DataLakeSDK var assertion = clientAlgorithm.ResolveAuto() == StorageChecksumAlgorithm.StorageCrc64 && !split ? GetRequestStructuredMessageAssertion(StructuredMessage.Flags.StorageCrc64, ParallelUploadIsChecksumExpected, dataLength) : GetRequestChecksumHeaderAssertion(clientAlgorithm, isChecksumExpected: ParallelUploadIsChecksumExpected); -#else - var assertion = GetRequestChecksumHeaderAssertion(clientAlgorithm, isChecksumExpected: ParallelUploadIsChecksumExpected); -#endif var checksumPipelineAssertion = new AssertMessageContentsPolicy(checkRequest: assertion); var clientOptions = ClientBuilder.GetOptions(); clientOptions.AddPolicy(checksumPipelineAssertion, HttpPipelinePosition.PerCall); @@ -1173,13 +1145,9 @@ public virtual async Task ParallelUploadOverwritesDefaultClientValidationOptions }; // make pipeline assertion for checking checksum was present on upload -#if BlobSDK || DataLakeSDK var assertion = overrideAlgorithm.ResolveAuto() == StorageChecksumAlgorithm.StorageCrc64 && !split ? GetRequestStructuredMessageAssertion(StructuredMessage.Flags.StorageCrc64, ParallelUploadIsChecksumExpected, dataLength) : GetRequestChecksumHeaderAssertion(overrideAlgorithm, isChecksumExpected: ParallelUploadIsChecksumExpected); -#else - var assertion = GetRequestChecksumHeaderAssertion(overrideAlgorithm, isChecksumExpected: ParallelUploadIsChecksumExpected); -#endif var checksumPipelineAssertion = new AssertMessageContentsPolicy(checkRequest: assertion); var clientOptions = ClientBuilder.GetOptions(); clientOptions.AddPolicy(checksumPipelineAssertion, HttpPipelinePosition.PerCall); @@ -1754,7 +1722,6 @@ public virtual async Task DownloadSuccessfulHashVerification(StorageChecksumAlgo Assert.IsTrue(result.SequenceEqual(data)); } -#if BlobSDK || DataLakeSDK [TestCase(StorageChecksumAlgorithm.StorageCrc64, Constants.StructuredMessage.MaxDownloadCrcWithHeader, false, false)] [TestCase(StorageChecksumAlgorithm.StorageCrc64, Constants.StructuredMessage.MaxDownloadCrcWithHeader-1, false, false)] [TestCase(StorageChecksumAlgorithm.StorageCrc64, Constants.StructuredMessage.MaxDownloadCrcWithHeader+1, true, false)] @@ -1811,7 +1778,6 @@ public virtual async Task DownloadApporpriatelyUsesStructuredMessage( Assert.IsTrue(dest.ToArray().SequenceEqual(data)); } } -#endif [Test, Combinatorial] public virtual async Task DownloadHashMismatchThrows( From dc9a85ae366201043545f4b362a5d0db521f7e56 Mon Sep 17 00:00:00 2001 From: Jocelyn Schreppler Date: Fri, 16 Aug 2024 11:30:45 -0400 Subject: [PATCH 22/22] fix auto --- sdk/storage/Azure.Storage.Files.Shares/assets.json | 2 +- .../tests/ShareFileClientTransferValidationTests.cs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/sdk/storage/Azure.Storage.Files.Shares/assets.json b/sdk/storage/Azure.Storage.Files.Shares/assets.json index 173fe0df0636c..184d64e873031 100644 --- a/sdk/storage/Azure.Storage.Files.Shares/assets.json +++ b/sdk/storage/Azure.Storage.Files.Shares/assets.json @@ -2,5 +2,5 @@ "AssetsRepo": "Azure/azure-sdk-assets", "AssetsRepoPrefixPath": "net", "TagPrefix": "net/storage/Azure.Storage.Files.Shares", - "Tag": "net/storage/Azure.Storage.Files.Shares_dbed55c6e3" + "Tag": "net/storage/Azure.Storage.Files.Shares_b3158cd2dd" } diff --git a/sdk/storage/Azure.Storage.Files.Shares/tests/ShareFileClientTransferValidationTests.cs b/sdk/storage/Azure.Storage.Files.Shares/tests/ShareFileClientTransferValidationTests.cs index 4cfa2b7271065..9fd8905e388b1 100644 --- a/sdk/storage/Azure.Storage.Files.Shares/tests/ShareFileClientTransferValidationTests.cs +++ b/sdk/storage/Azure.Storage.Files.Shares/tests/ShareFileClientTransferValidationTests.cs @@ -143,7 +143,7 @@ protected override async Task SetupDataAsync(ShareFileClient client, Stream data public override void TestAutoResolve() { Assert.AreEqual( - StorageChecksumAlgorithm.MD5, + StorageChecksumAlgorithm.StorageCrc64, TransferValidationOptionsExtensions.ResolveAuto(StorageChecksumAlgorithm.Auto)); }