From 18e691afe73b01d250525c7e54d57aacb595010c Mon Sep 17 00:00:00 2001 From: Jacob Lauzon <96087589+jalauzon-msft@users.noreply.github.com> Date: Mon, 4 Mar 2024 14:50:06 -0800 Subject: [PATCH] [Storage] Prepare hotfix release (#34608) --- sdk/storage/azure-storage-blob/CHANGELOG.md | 6 +++ sdk/storage/azure-storage-blob/assets.json | 2 +- .../azure/storage/blob/_download.py | 3 +- .../azure/storage/blob/_version.py | 2 +- .../azure/storage/blob/aio/_download_async.py | 3 +- .../tests/test_page_blob.py | 39 +++++++++++++++++++ .../tests/test_page_blob_async.py | 39 +++++++++++++++++++ 7 files changed, 90 insertions(+), 4 deletions(-) diff --git a/sdk/storage/azure-storage-blob/CHANGELOG.md b/sdk/storage/azure-storage-blob/CHANGELOG.md index 27b507588fcc..2771180724ed 100644 --- a/sdk/storage/azure-storage-blob/CHANGELOG.md +++ b/sdk/storage/azure-storage-blob/CHANGELOG.md @@ -1,5 +1,11 @@ # Release History +## 12.19.1 (2024-03-04) + +### Bugs Fixed +- Fixed an issue where under rare circumstances, full downloads of sparse Page Blobs could result in the +downloaded content containing up to one "chunk" of extra `\x00` at the end due to an optimization error. + ## 12.19.0 (2023-11-07) ### Features Added diff --git a/sdk/storage/azure-storage-blob/assets.json b/sdk/storage/azure-storage-blob/assets.json index 2cd230036747..7d48636bf8c0 100644 --- a/sdk/storage/azure-storage-blob/assets.json +++ b/sdk/storage/azure-storage-blob/assets.json @@ -2,5 +2,5 @@ "AssetsRepo": "Azure/azure-sdk-assets", "AssetsRepoPrefixPath": "python", "TagPrefix": "python/storage/azure-storage-blob", - "Tag": "python/storage/azure-storage-blob_12c8154ae2" + "Tag": "python/storage/azure-storage-blob_1b66da54e8" } diff --git a/sdk/storage/azure-storage-blob/azure/storage/blob/_download.py b/sdk/storage/azure-storage-blob/azure/storage/blob/_download.py index 2536c77de325..6c1d9b46bc56 100644 --- a/sdk/storage/azure-storage-blob/azure/storage/blob/_download.py +++ b/sdk/storage/azure-storage-blob/azure/storage/blob/_download.py @@ -186,7 +186,8 @@ def _download_chunk(self, chunk_start, chunk_end): # No need to download the empty chunk from server if there's no data in the chunk to be downloaded. # Do optimize and create empty chunk locally if condition is met. if self._do_optimize(download_range[0], download_range[1]): - chunk_data = b"\x00" * self.chunk_size + data_size = download_range[1] - download_range[0] + 1 + chunk_data = b"\x00" * data_size else: range_header, range_validation = validate_and_format_range_headers( download_range[0], diff --git a/sdk/storage/azure-storage-blob/azure/storage/blob/_version.py b/sdk/storage/azure-storage-blob/azure/storage/blob/_version.py index de61a38b5d71..28af23d094cb 100644 --- a/sdk/storage/azure-storage-blob/azure/storage/blob/_version.py +++ b/sdk/storage/azure-storage-blob/azure/storage/blob/_version.py @@ -4,4 +4,4 @@ # license information. # -------------------------------------------------------------------------- -VERSION = "12.19.0" +VERSION = "12.19.1" diff --git a/sdk/storage/azure-storage-blob/azure/storage/blob/aio/_download_async.py b/sdk/storage/azure-storage-blob/azure/storage/blob/aio/_download_async.py index de7f7d06f5ce..a8f68a550e92 100644 --- a/sdk/storage/azure-storage-blob/azure/storage/blob/aio/_download_async.py +++ b/sdk/storage/azure-storage-blob/azure/storage/blob/aio/_download_async.py @@ -95,7 +95,8 @@ async def _download_chunk(self, chunk_start, chunk_end): # No need to download the empty chunk from server if there's no data in the chunk to be downloaded. # Do optimize and create empty chunk locally if condition is met. if self._do_optimize(download_range[0], download_range[1]): - chunk_data = b"\x00" * self.chunk_size + data_size = download_range[1] - download_range[0] + 1 + chunk_data = b"\x00" * data_size else: range_header, range_validation = validate_and_format_range_headers( download_range[0], diff --git a/sdk/storage/azure-storage-blob/tests/test_page_blob.py b/sdk/storage/azure-storage-blob/tests/test_page_blob.py index 12cc8c58d0d7..beb40b426064 100644 --- a/sdk/storage/azure-storage-blob/tests/test_page_blob.py +++ b/sdk/storage/azure-storage-blob/tests/test_page_blob.py @@ -2269,6 +2269,45 @@ def test_download_sparse_page_blob_parallel(self, **kwargs): content = blob_client.download_blob(max_concurrency=3).readall() + @BlobPreparer() + @recorded_by_proxy + def test_download_sparse_page_blob_uneven_chunks(self, **kwargs): + storage_account_name = kwargs.pop("storage_account_name") + storage_account_key = kwargs.pop("storage_account_key") + + # Arrange + bsc = BlobServiceClient(self.account_url(storage_account_name, "blob"), credential=storage_account_key) + self._setup(bsc) + + # Choose an initial size, chunk size, and blob size, so the last chunk spills over end of blob + self.config.max_single_get_size = 4 * 1024 + self.config.max_chunk_get_size = 4 * 1024 + sparse_page_blob_size = 10 * 1024 + + blob_client = self._get_blob_reference(bsc) + blob_client.create_page_blob(sparse_page_blob_size) + + data = b'12345678' * 128 # 1024 bytes + range_start = 2 * 1024 + 512 + blob_client.upload_page(data, offset=range_start, length=len(data)) + + # Act + content = blob_client.download_blob().readall() + + # Assert + assert sparse_page_blob_size == len(content) + start = end = 0 + for r in blob_client.list_page_ranges(): + if not r.cleared: + start = r.start + end = r.end + + assert data == content[start: end + 1] + for byte in content[:start - 1]: + assert byte == 0 + for byte in content[end + 1:]: + assert byte == 0 + @BlobPreparer() @recorded_by_proxy def test_upload_progress_chunked_non_parallel(self, **kwargs): diff --git a/sdk/storage/azure-storage-blob/tests/test_page_blob_async.py b/sdk/storage/azure-storage-blob/tests/test_page_blob_async.py index 376724051ce4..27bc668bcf69 100644 --- a/sdk/storage/azure-storage-blob/tests/test_page_blob_async.py +++ b/sdk/storage/azure-storage-blob/tests/test_page_blob_async.py @@ -2238,6 +2238,45 @@ async def test_download_sparse_page_blob(self, storage_account_name, storage_acc except: assert byte == 0 + @BlobPreparer() + @recorded_by_proxy_async + async def test_download_sparse_page_blob_uneven_chunks(self, **kwargs): + storage_account_name = kwargs.pop("storage_account_name") + storage_account_key = kwargs.pop("storage_account_key") + + # Arrange + bsc = BlobServiceClient(self.account_url(storage_account_name, "blob"), credential=storage_account_key) + await self._setup(bsc) + + # Choose an initial size, chunk size, and blob size, so the last chunk spills over end of blob + self.config.max_single_get_size = 4 * 1024 + self.config.max_chunk_get_size = 4 * 1024 + sparse_page_blob_size = 10 * 1024 + + blob_client = self._get_blob_reference(bsc) + await blob_client.create_page_blob(sparse_page_blob_size) + + data = b'12345678' * 128 # 1024 bytes + range_start = 2 * 1024 + 512 + await blob_client.upload_page(data, offset=range_start, length=len(data)) + + # Act + content = await (await blob_client.download_blob()).readall() + + # Assert + assert sparse_page_blob_size == len(content) + start = end = 0 + async for r in blob_client.list_page_ranges(): + if not r.cleared: + start = r.start + end = r.end + + assert data == content[start: end + 1] + for byte in content[:start - 1]: + assert byte == 0 + for byte in content[end + 1:]: + assert byte == 0 + @BlobPreparer() @recorded_by_proxy_async async def test_upload_progress_chunked_non_parallel(self, **kwargs):