From e51282f0b0b9d57522374a3f10168374b1d3cf9d Mon Sep 17 00:00:00 2001 From: siminsavani-msft <77068571+siminsavani-msft@users.noreply.github.com> Date: Tue, 7 Mar 2023 16:23:33 -0500 Subject: [PATCH 1/2] Adding MD5 from UploadFileOptions --- sdk/storage/azblob/blockblob/models.go | 28 +++++----- sdk/storage/azblob/client_test.go | 74 ++++++++++++++++++++++++++ 2 files changed, 89 insertions(+), 13 deletions(-) diff --git a/sdk/storage/azblob/blockblob/models.go b/sdk/storage/azblob/blockblob/models.go index 3da15aab99df..e16cb408d4e0 100644 --- a/sdk/storage/azblob/blockblob/models.go +++ b/sdk/storage/azblob/blockblob/models.go @@ -228,24 +228,26 @@ func (o *uploadFromReaderOptions) getStageBlockOptions() *StageBlockOptions { func (o *uploadFromReaderOptions) getUploadBlockBlobOptions() *UploadOptions { return &UploadOptions{ - Tags: o.Tags, - Metadata: o.Metadata, - Tier: o.AccessTier, - HTTPHeaders: o.HTTPHeaders, - AccessConditions: o.AccessConditions, - CPKInfo: o.CPKInfo, - CPKScopeInfo: o.CPKScopeInfo, + Tags: o.Tags, + Metadata: o.Metadata, + Tier: o.AccessTier, + TransactionalContentMD5: o.TransactionalContentMD5, + HTTPHeaders: o.HTTPHeaders, + AccessConditions: o.AccessConditions, + CPKInfo: o.CPKInfo, + CPKScopeInfo: o.CPKScopeInfo, } } func (o *uploadFromReaderOptions) getCommitBlockListOptions() *CommitBlockListOptions { return &CommitBlockListOptions{ - Tags: o.Tags, - Metadata: o.Metadata, - Tier: o.AccessTier, - HTTPHeaders: o.HTTPHeaders, - CPKInfo: o.CPKInfo, - CPKScopeInfo: o.CPKScopeInfo, + Tags: o.Tags, + Metadata: o.Metadata, + Tier: o.AccessTier, + TransactionalContentMD5: o.TransactionalContentMD5, + HTTPHeaders: o.HTTPHeaders, + CPKInfo: o.CPKInfo, + CPKScopeInfo: o.CPKScopeInfo, } } diff --git a/sdk/storage/azblob/client_test.go b/sdk/storage/azblob/client_test.go index f04d5190a59a..427ded26432f 100644 --- a/sdk/storage/azblob/client_test.go +++ b/sdk/storage/azblob/client_test.go @@ -8,6 +8,8 @@ package azblob_test import ( "context" + "crypto/md5" + "encoding/hex" "errors" "fmt" "io" @@ -335,6 +337,78 @@ func (s *AZBlobUnrecordedTestsSuite) TestUploadAndDownloadFileNonZeroOffsetAndCo performUploadAndDownloadFileTest(s.T(), _require, testName, fileSize, blockSize, concurrency, downloadOffset, downloadCount) } +func (s *AZBlobUnrecordedTestsSuite) TestMD5SmallFileUpload() { + _require := require.New(s.T()) + fileSize := 1 * 1024 * 1024 // 1 MB file + + md5sum, resp, err := performUploadWithMD5Check(s.T(), _require, s.T().Name(), fileSize) + + _require.NoError(err) + + md5sumStr := hex.EncodeToString(md5sum) + respMd5 := hex.EncodeToString(resp.ContentMD5) + _require.Equal(md5sumStr, respMd5) + +} + +func (s *AZBlobUnrecordedTestsSuite) TestMD5LargeFileUpload() { + _require := require.New(s.T()) + fileSize := 257 * 1024 * 1024 // 257 MB file, 256 MB is the max for calculate md5 and service md5 won't match + + md5sum, resp, err := performUploadWithMD5Check(s.T(), _require, s.T().Name(), fileSize) + + _require.ErrorContains(err, "400 The MD5 value specified in the request did not match with the MD5 value calculated by the server.") + + md5sumStr := hex.EncodeToString(md5sum) + respMd5 := hex.EncodeToString(resp.ContentMD5) + _require.NotEqual(md5sumStr, respMd5) + +} + +func performUploadWithMD5Check(t *testing.T, _require *require.Assertions, testName string, fileSize int) ([]byte, blockblob.UploadFileResponse, error) { + // Set up file to upload + fileName := "BigFile.bin" + _ = generateFile(fileName, fileSize) + + // Open the file to upload + file, err := os.Open(fileName) + _require.NoError(err) + defer func(file *os.File) { + _ = file.Close() + }(file) + defer func(name string) { + _ = os.Remove(name) + }(fileName) + + client, err := testcommon.GetClient(t, testcommon.TestAccountDefault, nil) + _require.NoError(err) + + containerName := testcommon.GenerateContainerName(testName) + _, err = client.CreateContainer(context.Background(), containerName, nil) + _require.NoError(err) + defer func() { + _, err := client.DeleteContainer(context.Background(), containerName, nil) + _require.NoError(err) + }() + + // calculate md5 sum + h := md5.New() + if _, err := io.Copy(h, file); err != nil { + _require.NoError(err) + } + md5sum := h.Sum(nil) + // Set up test blob + blobName := testcommon.GenerateBlobName(testName) + + // Upload the file to a block blob + resp, err := client.UploadFile(context.Background(), containerName, blobName, file, + &blockblob.UploadFileOptions{ + TransactionalContentMD5: md5sum, + }) + + return md5sum, resp, err +} + func performUploadAndDownloadBufferTest(t *testing.T, _require *require.Assertions, testName string, blobSize, blockSize, concurrency, downloadOffset, downloadCount int) { // Set up buffer to upload _, bytesToUpload := testcommon.GenerateData(blobSize) From 93ffdb5523a9e057ae4c776727dc2ea6ab16c4f0 Mon Sep 17 00:00:00 2001 From: siminsavani-msft <77068571+siminsavani-msft@users.noreply.github.com> Date: Tue, 7 Mar 2023 16:29:37 -0500 Subject: [PATCH 2/2] Update CHANGELOG.md --- sdk/storage/azblob/CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/sdk/storage/azblob/CHANGELOG.md b/sdk/storage/azblob/CHANGELOG.md index b9828cdf0a93..55666a16e20a 100644 --- a/sdk/storage/azblob/CHANGELOG.md +++ b/sdk/storage/azblob/CHANGELOG.md @@ -7,6 +7,7 @@ ### Breaking Changes ### Bugs Fixed +* Added TransactionalContentMD5 to UploadOptions and CommitBlockListOptions. Fixes [20092](https://github.com/Azure/azure-sdk-for-go/issues/20092). ### Other Changes