Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Structured message cherrypick stg96 #45496

Merged
merged 22 commits into from
Aug 20, 2024
Merged

Structured message cherrypick stg96 #45496

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
22 commits
Select commit Hold shift + click to select a range
b5b4610
Structured Message Decode Stream (#42079)
jaschrep-msft Feb 21, 2024
fc13962
Content validation update spec (#42191)
jaschrep-msft Feb 28, 2024
34df306
decode tests & bugfixes (#42256)
jaschrep-msft Feb 29, 2024
c14d574
test coverage | exception message (#42363)
jaschrep-msft Mar 6, 2024
26d8a12
Structured Message Decode: Validate Content Length (#42370)
jaschrep-msft Mar 6, 2024
305ee6f
stageblock | appendblock | putpages StructuredMessage (#42699)
jaschrep-msft Mar 21, 2024
73d5345
GET Blob Structured Message (#42959)
jaschrep-msft Mar 26, 2024
003b439
PUT Blob Structured Message (#43130)
jaschrep-msft Apr 4, 2024
d3010db
Structured Message: DataLake Append (#43275)
jaschrep-msft Apr 17, 2024
5a532c6
Retriable decode (#44155)
jaschrep-msft May 21, 2024
b40939c
Download retriable stream structured message (#44176)
jaschrep-msft May 24, 2024
4786071
CRC: Always Structured Message (#44955)
jaschrep-msft Jul 16, 2024
5e2cc22
Enable fileshare crc (#45124)
jaschrep-msft Jul 23, 2024
212d8a4
block blob fix (#45129)
jaschrep-msft Jul 23, 2024
00a59a3
rename and validation (#45160)
jaschrep-msft Jul 26, 2024
b90be4c
crc tracking converted to longs (#45307)
jaschrep-msft Aug 7, 2024
5b2c153
Crc reporting pt2 (#45447)
jaschrep-msft Aug 12, 2024
1fe6364
testproxy
jaschrep-msft Aug 13, 2024
bd28c7f
remove unused parameter
jaschrep-msft Aug 14, 2024
6c6b8f5
add `ExpectTrailingDetails` to download response
jaschrep-msft Aug 14, 2024
64830e7
fix test inconsistency
jaschrep-msft Aug 14, 2024
dc9a85a
fix auto
jaschrep-msft Aug 16, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 2 additions & 1 deletion sdk/storage/Azure.Storage.Blobs.Batch/tests/Azure.Storage.Blobs.Batch.Tests.csproj
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@
<Compile Remove="$(AzureStorageSharedTestSources)\AzuriteNUnitFixture.cs" />
<Compile Remove="$(AzureStorageSharedTestSources)\TransferValidationTestBase.cs" />
<Compile Remove="$(AzureStorageSharedTestSources)\ClientSideEncryptionTestExtensions.cs" />
<Compile Remove="$(AzureStorageSharedTestSources)\ObserveStructuredMessagePolicy.cs" />
<None Include="$(AzureStorageSharedTestSources)\*.xml">
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
</None>
Expand All @@ -42,4 +43,4 @@
<Compile Include="$(AzureStorageSharedSources)UriQueryParamsCollection.cs" LinkBase="Shared" />
<Compile Include="$(AzureStorageSharedSources)Argument.cs" LinkBase="Shared" />
</ItemGroup>
</Project>
</Project>
3 changes: 2 additions & 1 deletion sdk/storage/Azure.Storage.Blobs.ChangeFeed/tests/Azure.Storage.Blobs.ChangeFeed.Tests.csproj
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@
<Compile Remove="$(AzureStorageSharedTestSources)\TransferValidationTestBase.cs" />
<Compile Remove="$(AzureStorageSharedTestSources)\ClientSideEncryptionTestExtensions.cs" />
<Compile Remove="$(AzureStorageSharedTestSources)\RepeatingStream.cs" />
<Compile Remove="$(AzureStorageSharedTestSources)\ObserveStructuredMessagePolicy.cs" />
<Compile Remove="$(AzureStorageSharedTestSources)\StorageTestBase.SasVersion.cs" />
<Compile Remove="$(AzureStorageSharedTestSources)\Sas\*.cs" />
<None Include="$(AzureStorageSharedTestSources)\*.xml">
Expand All @@ -28,4 +29,4 @@
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
</Content>
</ItemGroup>
</Project>
</Project>
1 change: 1 addition & 0 deletions sdk/storage/Azure.Storage.Blobs/api/Azure.Storage.Blobs.net6.0.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -516,6 +516,7 @@ public BlobDownloadDetails() { }
public long BlobSequenceNumber { get { throw null; } }
public Azure.Storage.Blobs.Models.BlobType BlobType { get { throw null; } }
public string CacheControl { get { throw null; } }
public byte[] ContentCrc { get { throw null; } }
public string ContentDisposition { get { throw null; } }
public string ContentEncoding { get { throw null; } }
public byte[] ContentHash { get { throw null; } }
Expand Down
1 change: 1 addition & 0 deletions sdk/storage/Azure.Storage.Blobs/api/Azure.Storage.Blobs.netstandard2.0.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -516,6 +516,7 @@ public BlobDownloadDetails() { }
public long BlobSequenceNumber { get { throw null; } }
public Azure.Storage.Blobs.Models.BlobType BlobType { get { throw null; } }
public string CacheControl { get { throw null; } }
public byte[] ContentCrc { get { throw null; } }
public string ContentDisposition { get { throw null; } }
public string ContentEncoding { get { throw null; } }
public byte[] ContentHash { get { throw null; } }
Expand Down
1 change: 1 addition & 0 deletions sdk/storage/Azure.Storage.Blobs/api/Azure.Storage.Blobs.netstandard2.1.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -516,6 +516,7 @@ public BlobDownloadDetails() { }
public long BlobSequenceNumber { get { throw null; } }
public Azure.Storage.Blobs.Models.BlobType BlobType { get { throw null; } }
public string CacheControl { get { throw null; } }
public byte[] ContentCrc { get { throw null; } }
public string ContentDisposition { get { throw null; } }
public string ContentEncoding { get { throw null; } }
public byte[] ContentHash { get { throw null; } }
Expand Down
2 changes: 1 addition & 1 deletion sdk/storage/Azure.Storage.Blobs/assets.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,5 +2,5 @@
"AssetsRepo": "Azure/azure-sdk-assets",
"AssetsRepoPrefixPath": "net",
"TagPrefix": "net/storage/Azure.Storage.Blobs",
"Tag": "net/storage/Azure.Storage.Blobs_14eb1d6279"
"Tag": "net/storage/Azure.Storage.Blobs_d0e3597ddc"
}
45 changes: 37 additions & 8 deletions sdk/storage/Azure.Storage.Blobs/src/AppendBlobClient.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1242,14 +1242,39 @@ internal async Task<Response<BlobAppendInfo>> AppendBlockInternal(
BlobErrors.VerifyHttpsCustomerProvidedKey(Uri, ClientConfiguration.CustomerProvidedKey);
Errors.VerifyStreamPosition(content, nameof(content));

// compute hash BEFORE attaching progress handler
ContentHasher.GetHashResult hashResult = await ContentHasher.GetHashOrDefaultInternal(
content,
validationOptions,
async,
cancellationToken).ConfigureAwait(false);

content = content.WithNoDispose().WithProgress(progressHandler);
ContentHasher.GetHashResult hashResult = null;
long contentLength = (content?.Length - content?.Position) ?? 0;
long? structuredContentLength = default;
string structuredBodyType = null;
if (validationOptions != null &&
validationOptions.ChecksumAlgorithm.ResolveAuto() == StorageChecksumAlgorithm.StorageCrc64 &&
ClientSideEncryption == null) // don't allow feature combination
{
// report progress in terms of caller bytes, not encoded bytes
structuredContentLength = contentLength;
contentLength = (content?.Length - content?.Position) ?? 0;
structuredBodyType = Constants.StructuredMessage.CrcStructuredMessage;
content = content.WithNoDispose().WithProgress(progressHandler);
content = validationOptions.PrecalculatedChecksum.IsEmpty
? new StructuredMessageEncodingStream(
content,
Constants.StructuredMessage.DefaultSegmentContentLength,
StructuredMessage.Flags.StorageCrc64)
: new StructuredMessagePrecalculatedCrcWrapperStream(
content,
validationOptions.PrecalculatedChecksum.Span);
contentLength = (content?.Length - content?.Position) ?? 0;
}
else
{
// compute hash BEFORE attaching progress handler
hashResult = await ContentHasher.GetHashOrDefaultInternal(
content,
validationOptions,
async,
cancellationToken).ConfigureAwait(false);
content = content.WithNoDispose().WithProgress(progressHandler);
}

ResponseWithHeaders<AppendBlobAppendBlockHeaders> response;

Expand All @@ -1267,6 +1292,8 @@ internal async Task<Response<BlobAppendInfo>> AppendBlockInternal(
encryptionKeySha256: ClientConfiguration.CustomerProvidedKey?.EncryptionKeyHash,
encryptionAlgorithm: ClientConfiguration.CustomerProvidedKey?.EncryptionAlgorithm == null ? null : EncryptionAlgorithmTypeInternal.AES256,
encryptionScope: ClientConfiguration.EncryptionScope,
structuredBodyType: structuredBodyType,
structuredContentLength: structuredContentLength,
ifModifiedSince: conditions?.IfModifiedSince,
ifUnmodifiedSince: conditions?.IfUnmodifiedSince,
ifMatch: conditions?.IfMatch?.ToString(),
Expand All @@ -1289,6 +1316,8 @@ internal async Task<Response<BlobAppendInfo>> AppendBlockInternal(
encryptionKeySha256: ClientConfiguration.CustomerProvidedKey?.EncryptionKeyHash,
encryptionAlgorithm: ClientConfiguration.CustomerProvidedKey?.EncryptionAlgorithm == null ? null : EncryptionAlgorithmTypeInternal.AES256,
encryptionScope: ClientConfiguration.EncryptionScope,
structuredBodyType: structuredBodyType,
structuredContentLength: structuredContentLength,
ifModifiedSince: conditions?.IfModifiedSince,
ifUnmodifiedSince: conditions?.IfUnmodifiedSince,
ifMatch: conditions?.IfMatch?.ToString(),
Expand Down
6 changes: 6 additions & 0 deletions sdk/storage/Azure.Storage.Blobs/src/Azure.Storage.Blobs.csproj
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -52,6 +52,7 @@
<Compile Include="$(AzureStorageSharedSources)BufferExtensions.cs" LinkBase="Shared" />
<Compile Include="$(AzureStorageSharedSources)ChecksumCalculatingStream.cs" LinkBase="Shared" />
<Compile Include="$(AzureStorageSharedSources)ContentRange.cs" LinkBase="Shared" />
<Compile Include="$(AzureStorageSharedSources)ChecksumExtensions.cs" LinkBase="Shared" />
<Compile Include="$(AzureStorageSharedSources)Constants.cs" LinkBase="Shared" />
<Compile Include="$(AzureStorageSharedSources)CompatSwitches.cs" LinkBase="Shared" />
<Compile Include="$(AzureStorageSharedSources)ContentHasher.cs" LinkBase="Shared" />
Expand Down Expand Up @@ -91,6 +92,11 @@
<Compile Include="$(AzureStorageSharedSources)StorageServerTimeoutPolicy.cs" LinkBase="Shared" />
<Compile Include="$(AzureStorageSharedSources)StorageTelemetryPolicy.cs" LinkBase="Shared" />
<Compile Include="$(AzureStorageSharedSources)StorageVersionExtensions.cs" LinkBase="Shared" />
<Compile Include="$(AzureStorageSharedSources)StructuredMessage.cs" LinkBase="Shared" />
<Compile Include="$(AzureStorageSharedSources)StructuredMessageDecodingRetriableStream.cs" LinkBase="Shared" />
<Compile Include="$(AzureStorageSharedSources)StructuredMessageDecodingStream.cs" LinkBase="Shared" />
<Compile Include="$(AzureStorageSharedSources)StructuredMessageEncodingStream.cs" LinkBase="Shared" />
<Compile Include="$(AzureStorageSharedSources)StructuredMessagePrecalculatedCrcWrapperStream.cs" LinkBase="Shared" />
<Compile Include="$(AzureStorageSharedSources)UriExtensions.cs" LinkBase="Shared" />
<Compile Include="$(AzureStorageSharedSources)UriQueryParamsCollection.cs" LinkBase="Shared" />
<Compile Include="$(AzureStorageSharedSources)UserDelegationKeyProperties.cs" LinkBase="Shared" />
Expand Down
Loading
Loading