diff --git a/sdk/storage/azure-storage-blob/assets.json b/sdk/storage/azure-storage-blob/assets.json index 115f91fd3d54..84d61cd19419 100644 --- a/sdk/storage/azure-storage-blob/assets.json +++ b/sdk/storage/azure-storage-blob/assets.json @@ -2,5 +2,5 @@ "AssetsRepo": "Azure/azure-sdk-assets", "AssetsRepoPrefixPath": "python", "TagPrefix": "python/storage/azure-storage-blob", - "Tag": "python/storage/azure-storage-blob_adfb24f3d6" + "Tag": "python/storage/azure-storage-blob_fad0e99de3" } diff --git a/sdk/storage/azure-storage-blob/azure/storage/blob/_generated/_azure_blob_storage.py b/sdk/storage/azure-storage-blob/azure/storage/blob/_generated/_azure_blob_storage.py index ab930440b987..cabfed8f0666 100644 --- a/sdk/storage/azure-storage-blob/azure/storage/blob/_generated/_azure_blob_storage.py +++ b/sdk/storage/azure-storage-blob/azure/storage/blob/_generated/_azure_blob_storage.py @@ -8,6 +8,7 @@ from copy import deepcopy from typing import Any +from typing_extensions import Self from azure.core import PipelineClient from azure.core.pipeline import policies @@ -110,7 +111,7 @@ def _send_request(self, request: HttpRequest, *, stream: bool = False, **kwargs: def close(self) -> None: self._client.close() - def __enter__(self) -> "AzureBlobStorage": + def __enter__(self) -> Self: self._client.__enter__() return self diff --git a/sdk/storage/azure-storage-blob/azure/storage/blob/_generated/_serialization.py b/sdk/storage/azure-storage-blob/azure/storage/blob/_generated/_serialization.py index f0c6180722c8..8139854b97bb 100644 --- a/sdk/storage/azure-storage-blob/azure/storage/blob/_generated/_serialization.py +++ b/sdk/storage/azure-storage-blob/azure/storage/blob/_generated/_serialization.py @@ -144,6 +144,8 @@ def _json_attemp(data): # context otherwise. _LOGGER.critical("Wasn't XML not JSON, failing") raise DeserializationError("XML is invalid") from err + elif content_type.startswith("text/"): + return data_as_str raise DeserializationError("Cannot deserialize content-type: {}".format(content_type)) @classmethod diff --git a/sdk/storage/azure-storage-blob/azure/storage/blob/_generated/_vendor.py b/sdk/storage/azure-storage-blob/azure/storage/blob/_generated/_vendor.py deleted file mode 100644 index 0dafe0e287ff..000000000000 --- a/sdk/storage/azure-storage-blob/azure/storage/blob/_generated/_vendor.py +++ /dev/null @@ -1,16 +0,0 @@ -# -------------------------------------------------------------------------- -# Copyright (c) Microsoft Corporation. All rights reserved. -# Licensed under the MIT License. See License.txt in the project root for license information. -# Code generated by Microsoft (R) AutoRest Code Generator. -# Changes may cause incorrect behavior and will be lost if the code is regenerated. -# -------------------------------------------------------------------------- - -from azure.core.pipeline.transport import HttpRequest - - -def _convert_request(request, files=None): - data = request.content if not files else None - request = HttpRequest(method=request.method, url=request.url, headers=request.headers, data=data) - if files: - request.set_formdata_body(files) - return request diff --git a/sdk/storage/azure-storage-blob/azure/storage/blob/_generated/aio/_azure_blob_storage.py b/sdk/storage/azure-storage-blob/azure/storage/blob/_generated/aio/_azure_blob_storage.py index ef97d289d0c9..c76a291f3c6a 100644 --- a/sdk/storage/azure-storage-blob/azure/storage/blob/_generated/aio/_azure_blob_storage.py +++ b/sdk/storage/azure-storage-blob/azure/storage/blob/_generated/aio/_azure_blob_storage.py @@ -8,6 +8,7 @@ from copy import deepcopy from typing import Any, Awaitable +from typing_extensions import Self from azure.core import AsyncPipelineClient from azure.core.pipeline import policies @@ -112,7 +113,7 @@ def _send_request( async def close(self) -> None: await self._client.close() - async def __aenter__(self) -> "AzureBlobStorage": + async def __aenter__(self) -> Self: await self._client.__aenter__() return self diff --git a/sdk/storage/azure-storage-blob/azure/storage/blob/_generated/aio/operations/_append_blob_operations.py b/sdk/storage/azure-storage-blob/azure/storage/blob/_generated/aio/operations/_append_blob_operations.py index 2840b96ee011..70b5d865e6d2 100644 --- a/sdk/storage/azure-storage-blob/azure/storage/blob/_generated/aio/operations/_append_blob_operations.py +++ b/sdk/storage/azure-storage-blob/azure/storage/blob/_generated/aio/operations/_append_blob_operations.py @@ -19,13 +19,11 @@ map_error, ) from azure.core.pipeline import PipelineResponse -from azure.core.pipeline.transport import AsyncHttpResponse -from azure.core.rest import HttpRequest +from azure.core.rest import AsyncHttpResponse, HttpRequest from azure.core.tracing.decorator_async import distributed_trace_async from azure.core.utils import case_insensitive_dict from ... import models as _models -from ..._vendor import _convert_request from ...operations._append_blob_operations import ( build_append_block_from_url_request, build_append_block_request, @@ -207,7 +205,6 @@ async def create( # pylint: disable=inconsistent-return-statements headers=_headers, params=_params, ) - _request = _convert_request(_request) _request.url = self._client.format_url(_request.url) _stream = False @@ -372,7 +369,6 @@ async def append_block( # pylint: disable=inconsistent-return-statements headers=_headers, params=_params, ) - _request = _convert_request(_request) _request.url = self._client.format_url(_request.url) _stream = False @@ -574,7 +570,6 @@ async def append_block_from_url( # pylint: disable=inconsistent-return-statemen headers=_headers, params=_params, ) - _request = _convert_request(_request) _request.url = self._client.format_url(_request.url) _stream = False @@ -696,7 +691,6 @@ async def seal( # pylint: disable=inconsistent-return-statements headers=_headers, params=_params, ) - _request = _convert_request(_request) _request.url = self._client.format_url(_request.url) _stream = False diff --git a/sdk/storage/azure-storage-blob/azure/storage/blob/_generated/aio/operations/_blob_operations.py b/sdk/storage/azure-storage-blob/azure/storage/blob/_generated/aio/operations/_blob_operations.py index f71ffebc37b5..9cf0b47f035d 100644 --- a/sdk/storage/azure-storage-blob/azure/storage/blob/_generated/aio/operations/_blob_operations.py +++ b/sdk/storage/azure-storage-blob/azure/storage/blob/_generated/aio/operations/_blob_operations.py @@ -16,16 +16,16 @@ ResourceExistsError, ResourceNotFoundError, ResourceNotModifiedError, + StreamClosedError, + StreamConsumedError, map_error, ) from azure.core.pipeline import PipelineResponse -from azure.core.pipeline.transport import AsyncHttpResponse -from azure.core.rest import HttpRequest +from azure.core.rest import AsyncHttpResponse, HttpRequest from azure.core.tracing.decorator_async import distributed_trace_async from azure.core.utils import case_insensitive_dict from ... import models as _models -from ..._vendor import _convert_request from ...operations._blob_operations import ( build_abort_copy_from_url_request, build_acquire_lease_request, @@ -194,9 +194,9 @@ async def download( headers=_headers, params=_params, ) - _request = _convert_request(_request) _request.url = self._client.format_url(_request.url) + _decompress = kwargs.pop("decompress", True) _stream = True pipeline_response: PipelineResponse = await self._client._pipeline.run( # pylint: disable=protected-access _request, stream=_stream, **kwargs @@ -205,6 +205,10 @@ async def download( response = pipeline_response.http_response if response.status_code not in [200, 206]: + try: + await response.read() # Load the body in memory and close the socket + except (StreamConsumedError, StreamClosedError): + pass map_error(status_code=response.status_code, response=response, error_map=error_map) error = self._deserialize.failsafe_deserialize(_models.StorageError, pipeline_response) raise HttpResponseError(response=response, model=error) @@ -289,8 +293,6 @@ async def download( ) response_headers["x-ms-legal-hold"] = self._deserialize("bool", response.headers.get("x-ms-legal-hold")) - deserialized = response.stream_download(self._client._pipeline) - if response.status_code == 206: response_headers["Last-Modified"] = self._deserialize("rfc-1123", response.headers.get("Last-Modified")) response_headers["x-ms-creation-time"] = self._deserialize( @@ -373,7 +375,7 @@ async def download( ) response_headers["x-ms-legal-hold"] = self._deserialize("bool", response.headers.get("x-ms-legal-hold")) - deserialized = response.stream_download(self._client._pipeline) + deserialized = response.stream_download(self._client._pipeline, decompress=_decompress) if cls: return cls(pipeline_response, deserialized, response_headers) # type: ignore @@ -478,7 +480,6 @@ async def get_properties( # pylint: disable=inconsistent-return-statements headers=_headers, params=_params, ) - _request = _convert_request(_request) _request.url = self._client.format_url(_request.url) _stream = False @@ -691,7 +692,6 @@ async def delete( # pylint: disable=inconsistent-return-statements headers=_headers, params=_params, ) - _request = _convert_request(_request) _request.url = self._client.format_url(_request.url) _stream = False @@ -759,7 +759,6 @@ async def undelete( # pylint: disable=inconsistent-return-statements headers=_headers, params=_params, ) - _request = _convert_request(_request) _request.url = self._client.format_url(_request.url) _stream = False @@ -839,7 +838,6 @@ async def set_expiry( # pylint: disable=inconsistent-return-statements headers=_headers, params=_params, ) - _request = _convert_request(_request) _request.url = self._client.format_url(_request.url) _stream = False @@ -961,7 +959,6 @@ async def set_http_headers( # pylint: disable=inconsistent-return-statements headers=_headers, params=_params, ) - _request = _convert_request(_request) _request.url = self._client.format_url(_request.url) _stream = False @@ -1055,7 +1052,6 @@ async def set_immutability_policy( # pylint: disable=inconsistent-return-statem headers=_headers, params=_params, ) - _request = _convert_request(_request) _request.url = self._client.format_url(_request.url) _stream = False @@ -1129,7 +1125,6 @@ async def delete_immutability_policy( # pylint: disable=inconsistent-return-sta headers=_headers, params=_params, ) - _request = _convert_request(_request) _request.url = self._client.format_url(_request.url) _stream = False @@ -1200,7 +1195,6 @@ async def set_legal_hold( # pylint: disable=inconsistent-return-statements headers=_headers, params=_params, ) - _request = _convert_request(_request) _request.url = self._client.format_url(_request.url) _stream = False @@ -1330,7 +1324,6 @@ async def set_metadata( # pylint: disable=inconsistent-return-statements headers=_headers, params=_params, ) - _request = _convert_request(_request) _request.url = self._client.format_url(_request.url) _stream = False @@ -1448,7 +1441,6 @@ async def acquire_lease( # pylint: disable=inconsistent-return-statements headers=_headers, params=_params, ) - _request = _convert_request(_request) _request.url = self._client.format_url(_request.url) _stream = False @@ -1549,7 +1541,6 @@ async def release_lease( # pylint: disable=inconsistent-return-statements headers=_headers, params=_params, ) - _request = _convert_request(_request) _request.url = self._client.format_url(_request.url) _stream = False @@ -1649,7 +1640,6 @@ async def renew_lease( # pylint: disable=inconsistent-return-statements headers=_headers, params=_params, ) - _request = _convert_request(_request) _request.url = self._client.format_url(_request.url) _stream = False @@ -1756,7 +1746,6 @@ async def change_lease( # pylint: disable=inconsistent-return-statements headers=_headers, params=_params, ) - _request = _convert_request(_request) _request.url = self._client.format_url(_request.url) _stream = False @@ -1863,7 +1852,6 @@ async def break_lease( # pylint: disable=inconsistent-return-statements headers=_headers, params=_params, ) - _request = _convert_request(_request) _request.url = self._client.format_url(_request.url) _stream = False @@ -1994,7 +1982,6 @@ async def create_snapshot( # pylint: disable=inconsistent-return-statements headers=_headers, params=_params, ) - _request = _convert_request(_request) _request.url = self._client.format_url(_request.url) _stream = False @@ -2169,7 +2156,6 @@ async def start_copy_from_url( # pylint: disable=inconsistent-return-statements headers=_headers, params=_params, ) - _request = _convert_request(_request) _request.url = self._client.format_url(_request.url) _stream = False @@ -2357,7 +2343,6 @@ async def copy_from_url( # pylint: disable=inconsistent-return-statements headers=_headers, params=_params, ) - _request = _convert_request(_request) _request.url = self._client.format_url(_request.url) _stream = False @@ -2458,7 +2443,6 @@ async def abort_copy_from_url( # pylint: disable=inconsistent-return-statements headers=_headers, params=_params, ) - _request = _convert_request(_request) _request.url = self._client.format_url(_request.url) _stream = False @@ -2573,7 +2557,6 @@ async def set_tier( # pylint: disable=inconsistent-return-statements headers=_headers, params=_params, ) - _request = _convert_request(_request) _request.url = self._client.format_url(_request.url) _stream = False @@ -2589,19 +2572,11 @@ async def set_tier( # pylint: disable=inconsistent-return-statements raise HttpResponseError(response=response, model=error) response_headers = {} - if response.status_code == 200: - response_headers["x-ms-client-request-id"] = self._deserialize( - "str", response.headers.get("x-ms-client-request-id") - ) - response_headers["x-ms-request-id"] = self._deserialize("str", response.headers.get("x-ms-request-id")) - response_headers["x-ms-version"] = self._deserialize("str", response.headers.get("x-ms-version")) - - if response.status_code == 202: - response_headers["x-ms-client-request-id"] = self._deserialize( - "str", response.headers.get("x-ms-client-request-id") - ) - response_headers["x-ms-request-id"] = self._deserialize("str", response.headers.get("x-ms-request-id")) - response_headers["x-ms-version"] = self._deserialize("str", response.headers.get("x-ms-version")) + response_headers["x-ms-client-request-id"] = self._deserialize( + "str", response.headers.get("x-ms-client-request-id") + ) + response_headers["x-ms-request-id"] = self._deserialize("str", response.headers.get("x-ms-request-id")) + response_headers["x-ms-version"] = self._deserialize("str", response.headers.get("x-ms-version")) if cls: return cls(pipeline_response, None, response_headers) # type: ignore @@ -2650,7 +2625,6 @@ async def get_account_info( # pylint: disable=inconsistent-return-statements headers=_headers, params=_params, ) - _request = _convert_request(_request) _request.url = self._client.format_url(_request.url) _stream = False @@ -2783,9 +2757,9 @@ async def query( headers=_headers, params=_params, ) - _request = _convert_request(_request) _request.url = self._client.format_url(_request.url) + _decompress = kwargs.pop("decompress", True) _stream = True pipeline_response: PipelineResponse = await self._client._pipeline.run( # pylint: disable=protected-access _request, stream=_stream, **kwargs @@ -2794,6 +2768,10 @@ async def query( response = pipeline_response.http_response if response.status_code not in [200, 206]: + try: + await response.read() # Load the body in memory and close the socket + except (StreamConsumedError, StreamClosedError): + pass map_error(status_code=response.status_code, response=response, error_map=error_map) error = self._deserialize.failsafe_deserialize(_models.StorageError, pipeline_response) raise HttpResponseError(response=response, model=error) @@ -2857,8 +2835,6 @@ async def query( "bytearray", response.headers.get("x-ms-blob-content-md5") ) - deserialized = response.stream_download(self._client._pipeline) - if response.status_code == 206: response_headers["Last-Modified"] = self._deserialize("rfc-1123", response.headers.get("Last-Modified")) response_headers["x-ms-meta"] = self._deserialize("{str}", response.headers.get("x-ms-meta")) @@ -2920,7 +2896,7 @@ async def query( "bytearray", response.headers.get("x-ms-blob-content-md5") ) - deserialized = response.stream_download(self._client._pipeline) + deserialized = response.stream_download(self._client._pipeline, decompress=_decompress) if cls: return cls(pipeline_response, deserialized, response_headers) # type: ignore @@ -3001,7 +2977,6 @@ async def get_tags( headers=_headers, params=_params, ) - _request = _convert_request(_request) _request.url = self._client.format_url(_request.url) _stream = False @@ -3024,7 +2999,7 @@ async def get_tags( response_headers["x-ms-version"] = self._deserialize("str", response.headers.get("x-ms-version")) response_headers["Date"] = self._deserialize("rfc-1123", response.headers.get("Date")) - deserialized = self._deserialize("BlobTags", pipeline_response) + deserialized = self._deserialize("BlobTags", pipeline_response.http_response) if cls: return cls(pipeline_response, deserialized, response_headers) # type: ignore @@ -3117,7 +3092,6 @@ async def set_tags( # pylint: disable=inconsistent-return-statements headers=_headers, params=_params, ) - _request = _convert_request(_request) _request.url = self._client.format_url(_request.url) _stream = False diff --git a/sdk/storage/azure-storage-blob/azure/storage/blob/_generated/aio/operations/_block_blob_operations.py b/sdk/storage/azure-storage-blob/azure/storage/blob/_generated/aio/operations/_block_blob_operations.py index 5123996799f5..d833c25c0eec 100644 --- a/sdk/storage/azure-storage-blob/azure/storage/blob/_generated/aio/operations/_block_blob_operations.py +++ b/sdk/storage/azure-storage-blob/azure/storage/blob/_generated/aio/operations/_block_blob_operations.py @@ -19,13 +19,11 @@ map_error, ) from azure.core.pipeline import PipelineResponse -from azure.core.pipeline.transport import AsyncHttpResponse -from azure.core.rest import HttpRequest +from azure.core.rest import AsyncHttpResponse, HttpRequest from azure.core.tracing.decorator_async import distributed_trace_async from azure.core.utils import case_insensitive_dict from ... import models as _models -from ..._vendor import _convert_request from ...operations._block_blob_operations import ( build_commit_block_list_request, build_get_block_list_request, @@ -236,7 +234,6 @@ async def upload( # pylint: disable=inconsistent-return-statements headers=_headers, params=_params, ) - _request = _convert_request(_request) _request.url = self._client.format_url(_request.url) _stream = False @@ -471,7 +468,6 @@ async def put_blob_from_url( # pylint: disable=inconsistent-return-statements headers=_headers, params=_params, ) - _request = _convert_request(_request) _request.url = self._client.format_url(_request.url) _stream = False @@ -610,7 +606,6 @@ async def stage_block( # pylint: disable=inconsistent-return-statements headers=_headers, params=_params, ) - _request = _convert_request(_request) _request.url = self._client.format_url(_request.url) _stream = False @@ -773,7 +768,6 @@ async def stage_block_from_url( # pylint: disable=inconsistent-return-statement headers=_headers, params=_params, ) - _request = _convert_request(_request) _request.url = self._client.format_url(_request.url) _stream = False @@ -984,7 +978,6 @@ async def commit_block_list( # pylint: disable=inconsistent-return-statements headers=_headers, params=_params, ) - _request = _convert_request(_request) _request.url = self._client.format_url(_request.url) _stream = False @@ -1101,7 +1094,6 @@ async def get_block_list( headers=_headers, params=_params, ) - _request = _convert_request(_request) _request.url = self._client.format_url(_request.url) _stream = False @@ -1130,7 +1122,7 @@ async def get_block_list( response_headers["x-ms-version"] = self._deserialize("str", response.headers.get("x-ms-version")) response_headers["Date"] = self._deserialize("rfc-1123", response.headers.get("Date")) - deserialized = self._deserialize("BlockList", pipeline_response) + deserialized = self._deserialize("BlockList", pipeline_response.http_response) if cls: return cls(pipeline_response, deserialized, response_headers) # type: ignore diff --git a/sdk/storage/azure-storage-blob/azure/storage/blob/_generated/aio/operations/_container_operations.py b/sdk/storage/azure-storage-blob/azure/storage/blob/_generated/aio/operations/_container_operations.py index 9a5197df8829..48a1a14749c7 100644 --- a/sdk/storage/azure-storage-blob/azure/storage/blob/_generated/aio/operations/_container_operations.py +++ b/sdk/storage/azure-storage-blob/azure/storage/blob/_generated/aio/operations/_container_operations.py @@ -15,16 +15,16 @@ ResourceExistsError, ResourceNotFoundError, ResourceNotModifiedError, + StreamClosedError, + StreamConsumedError, map_error, ) from azure.core.pipeline import PipelineResponse -from azure.core.pipeline.transport import AsyncHttpResponse -from azure.core.rest import HttpRequest +from azure.core.rest import AsyncHttpResponse, HttpRequest from azure.core.tracing.decorator_async import distributed_trace_async from azure.core.utils import case_insensitive_dict from ... import models as _models -from ..._vendor import _convert_request from ...operations._container_operations import ( build_acquire_lease_request, build_break_lease_request, @@ -145,7 +145,6 @@ async def create( # pylint: disable=inconsistent-return-statements headers=_headers, params=_params, ) - _request = _convert_request(_request) _request.url = self._client.format_url(_request.url) _stream = False @@ -227,7 +226,6 @@ async def get_properties( # pylint: disable=inconsistent-return-statements headers=_headers, params=_params, ) - _request = _convert_request(_request) _request.url = self._client.format_url(_request.url) _stream = False @@ -339,7 +337,6 @@ async def delete( # pylint: disable=inconsistent-return-statements headers=_headers, params=_params, ) - _request = _convert_request(_request) _request.url = self._client.format_url(_request.url) _stream = False @@ -437,7 +434,6 @@ async def set_metadata( # pylint: disable=inconsistent-return-statements headers=_headers, params=_params, ) - _request = _convert_request(_request) _request.url = self._client.format_url(_request.url) _stream = False @@ -521,7 +517,6 @@ async def get_access_policy( headers=_headers, params=_params, ) - _request = _convert_request(_request) _request.url = self._client.format_url(_request.url) _stream = False @@ -549,7 +544,7 @@ async def get_access_policy( response_headers["x-ms-version"] = self._deserialize("str", response.headers.get("x-ms-version")) response_headers["Date"] = self._deserialize("rfc-1123", response.headers.get("Date")) - deserialized = self._deserialize("[SignedIdentifier]", pipeline_response) + deserialized = self._deserialize("[SignedIdentifier]", pipeline_response.http_response) if cls: return cls(pipeline_response, deserialized, response_headers) # type: ignore @@ -640,7 +635,6 @@ async def set_access_policy( # pylint: disable=inconsistent-return-statements headers=_headers, params=_params, ) - _request = _convert_request(_request) _request.url = self._client.format_url(_request.url) _stream = False @@ -725,7 +719,6 @@ async def restore( # pylint: disable=inconsistent-return-statements headers=_headers, params=_params, ) - _request = _convert_request(_request) _request.url = self._client.format_url(_request.url) _stream = False @@ -808,7 +801,6 @@ async def rename( # pylint: disable=inconsistent-return-statements headers=_headers, params=_params, ) - _request = _convert_request(_request) _request.url = self._client.format_url(_request.url) _stream = False @@ -895,9 +887,9 @@ async def submit_batch( headers=_headers, params=_params, ) - _request = _convert_request(_request) _request.url = self._client.format_url(_request.url) + _decompress = kwargs.pop("decompress", True) _stream = True pipeline_response: PipelineResponse = await self._client._pipeline.run( # pylint: disable=protected-access _request, stream=_stream, **kwargs @@ -906,6 +898,10 @@ async def submit_batch( response = pipeline_response.http_response if response.status_code not in [202]: + try: + await response.read() # Load the body in memory and close the socket + except (StreamConsumedError, StreamClosedError): + pass map_error(status_code=response.status_code, response=response, error_map=error_map) error = self._deserialize.failsafe_deserialize(_models.StorageError, pipeline_response) raise HttpResponseError(response=response, model=error) @@ -915,7 +911,7 @@ async def submit_batch( response_headers["x-ms-request-id"] = self._deserialize("str", response.headers.get("x-ms-request-id")) response_headers["x-ms-version"] = self._deserialize("str", response.headers.get("x-ms-version")) - deserialized = response.stream_download(self._client._pipeline) + deserialized = response.stream_download(self._client._pipeline, decompress=_decompress) if cls: return cls(pipeline_response, deserialized, response_headers) # type: ignore @@ -998,7 +994,6 @@ async def filter_blobs( headers=_headers, params=_params, ) - _request = _convert_request(_request) _request.url = self._client.format_url(_request.url) _stream = False @@ -1021,7 +1016,7 @@ async def filter_blobs( response_headers["x-ms-version"] = self._deserialize("str", response.headers.get("x-ms-version")) response_headers["Date"] = self._deserialize("rfc-1123", response.headers.get("Date")) - deserialized = self._deserialize("FilterBlobSegment", pipeline_response) + deserialized = self._deserialize("FilterBlobSegment", pipeline_response.http_response) if cls: return cls(pipeline_response, deserialized, response_headers) # type: ignore @@ -1101,7 +1096,6 @@ async def acquire_lease( # pylint: disable=inconsistent-return-statements headers=_headers, params=_params, ) - _request = _convert_request(_request) _request.url = self._client.format_url(_request.url) _stream = False @@ -1195,7 +1189,6 @@ async def release_lease( # pylint: disable=inconsistent-return-statements headers=_headers, params=_params, ) - _request = _convert_request(_request) _request.url = self._client.format_url(_request.url) _stream = False @@ -1288,7 +1281,6 @@ async def renew_lease( # pylint: disable=inconsistent-return-statements headers=_headers, params=_params, ) - _request = _convert_request(_request) _request.url = self._client.format_url(_request.url) _stream = False @@ -1388,7 +1380,6 @@ async def break_lease( # pylint: disable=inconsistent-return-statements headers=_headers, params=_params, ) - _request = _convert_request(_request) _request.url = self._client.format_url(_request.url) _stream = False @@ -1488,7 +1479,6 @@ async def change_lease( # pylint: disable=inconsistent-return-statements headers=_headers, params=_params, ) - _request = _convert_request(_request) _request.url = self._client.format_url(_request.url) _stream = False @@ -1592,7 +1582,6 @@ async def list_blob_flat_segment( headers=_headers, params=_params, ) - _request = _convert_request(_request) _request.url = self._client.format_url(_request.url) _stream = False @@ -1616,7 +1605,7 @@ async def list_blob_flat_segment( response_headers["x-ms-version"] = self._deserialize("str", response.headers.get("x-ms-version")) response_headers["Date"] = self._deserialize("rfc-1123", response.headers.get("Date")) - deserialized = self._deserialize("ListBlobsFlatSegmentResponse", pipeline_response) + deserialized = self._deserialize("ListBlobsFlatSegmentResponse", pipeline_response.http_response) if cls: return cls(pipeline_response, deserialized, response_headers) # type: ignore @@ -1705,7 +1694,6 @@ async def list_blob_hierarchy_segment( headers=_headers, params=_params, ) - _request = _convert_request(_request) _request.url = self._client.format_url(_request.url) _stream = False @@ -1729,7 +1717,7 @@ async def list_blob_hierarchy_segment( response_headers["x-ms-version"] = self._deserialize("str", response.headers.get("x-ms-version")) response_headers["Date"] = self._deserialize("rfc-1123", response.headers.get("Date")) - deserialized = self._deserialize("ListBlobsHierarchySegmentResponse", pipeline_response) + deserialized = self._deserialize("ListBlobsHierarchySegmentResponse", pipeline_response.http_response) if cls: return cls(pipeline_response, deserialized, response_headers) # type: ignore @@ -1780,7 +1768,6 @@ async def get_account_info( # pylint: disable=inconsistent-return-statements headers=_headers, params=_params, ) - _request = _convert_request(_request) _request.url = self._client.format_url(_request.url) _stream = False diff --git a/sdk/storage/azure-storage-blob/azure/storage/blob/_generated/aio/operations/_page_blob_operations.py b/sdk/storage/azure-storage-blob/azure/storage/blob/_generated/aio/operations/_page_blob_operations.py index 45d5c64de5fb..bf77639fe40e 100644 --- a/sdk/storage/azure-storage-blob/azure/storage/blob/_generated/aio/operations/_page_blob_operations.py +++ b/sdk/storage/azure-storage-blob/azure/storage/blob/_generated/aio/operations/_page_blob_operations.py @@ -19,13 +19,11 @@ map_error, ) from azure.core.pipeline import PipelineResponse -from azure.core.pipeline.transport import AsyncHttpResponse -from azure.core.rest import HttpRequest +from azure.core.rest import AsyncHttpResponse, HttpRequest from azure.core.tracing.decorator_async import distributed_trace_async from azure.core.utils import case_insensitive_dict from ... import models as _models -from ..._vendor import _convert_request from ...operations._page_blob_operations import ( build_clear_pages_request, build_copy_incremental_request, @@ -228,7 +226,6 @@ async def create( # pylint: disable=inconsistent-return-statements headers=_headers, params=_params, ) - _request = _convert_request(_request) _request.url = self._client.format_url(_request.url) _stream = False @@ -402,7 +399,6 @@ async def upload_pages( # pylint: disable=inconsistent-return-statements headers=_headers, params=_params, ) - _request = _convert_request(_request) _request.url = self._client.format_url(_request.url) _stream = False @@ -564,7 +560,6 @@ async def clear_pages( # pylint: disable=inconsistent-return-statements headers=_headers, params=_params, ) - _request = _convert_request(_request) _request.url = self._client.format_url(_request.url) _stream = False @@ -760,7 +755,6 @@ async def upload_pages_from_url( # pylint: disable=inconsistent-return-statemen headers=_headers, params=_params, ) - _request = _convert_request(_request) _request.url = self._client.format_url(_request.url) _stream = False @@ -904,7 +898,6 @@ async def get_page_ranges( headers=_headers, params=_params, ) - _request = _convert_request(_request) _request.url = self._client.format_url(_request.url) _stream = False @@ -932,7 +925,7 @@ async def get_page_ranges( response_headers["x-ms-version"] = self._deserialize("str", response.headers.get("x-ms-version")) response_headers["Date"] = self._deserialize("rfc-1123", response.headers.get("Date")) - deserialized = self._deserialize("PageList", pipeline_response) + deserialized = self._deserialize("PageList", pipeline_response.http_response) if cls: return cls(pipeline_response, deserialized, response_headers) # type: ignore @@ -1058,7 +1051,6 @@ async def get_page_ranges_diff( headers=_headers, params=_params, ) - _request = _convert_request(_request) _request.url = self._client.format_url(_request.url) _stream = False @@ -1086,7 +1078,7 @@ async def get_page_ranges_diff( response_headers["x-ms-version"] = self._deserialize("str", response.headers.get("x-ms-version")) response_headers["Date"] = self._deserialize("rfc-1123", response.headers.get("Date")) - deserialized = self._deserialize("PageList", pipeline_response) + deserialized = self._deserialize("PageList", pipeline_response.http_response) if cls: return cls(pipeline_response, deserialized, response_headers) # type: ignore @@ -1190,7 +1182,6 @@ async def resize( # pylint: disable=inconsistent-return-statements headers=_headers, params=_params, ) - _request = _convert_request(_request) _request.url = self._client.format_url(_request.url) _stream = False @@ -1306,7 +1297,6 @@ async def update_sequence_number( # pylint: disable=inconsistent-return-stateme headers=_headers, params=_params, ) - _request = _convert_request(_request) _request.url = self._client.format_url(_request.url) _stream = False @@ -1413,7 +1403,6 @@ async def copy_incremental( # pylint: disable=inconsistent-return-statements headers=_headers, params=_params, ) - _request = _convert_request(_request) _request.url = self._client.format_url(_request.url) _stream = False diff --git a/sdk/storage/azure-storage-blob/azure/storage/blob/_generated/aio/operations/_service_operations.py b/sdk/storage/azure-storage-blob/azure/storage/blob/_generated/aio/operations/_service_operations.py index 7a3c62fec53d..f9f8ff0be86c 100644 --- a/sdk/storage/azure-storage-blob/azure/storage/blob/_generated/aio/operations/_service_operations.py +++ b/sdk/storage/azure-storage-blob/azure/storage/blob/_generated/aio/operations/_service_operations.py @@ -15,16 +15,16 @@ ResourceExistsError, ResourceNotFoundError, ResourceNotModifiedError, + StreamClosedError, + StreamConsumedError, map_error, ) from azure.core.pipeline import PipelineResponse -from azure.core.pipeline.transport import AsyncHttpResponse -from azure.core.rest import HttpRequest +from azure.core.rest import AsyncHttpResponse, HttpRequest from azure.core.tracing.decorator_async import distributed_trace_async from azure.core.utils import case_insensitive_dict from ... import models as _models -from ..._vendor import _convert_request from ...operations._service_operations import ( build_filter_blobs_request, build_get_account_info_request, @@ -119,7 +119,6 @@ async def set_properties( # pylint: disable=inconsistent-return-statements headers=_headers, params=_params, ) - _request = _convert_request(_request) _request.url = self._client.format_url(_request.url) _stream = False @@ -189,7 +188,6 @@ async def get_properties( headers=_headers, params=_params, ) - _request = _convert_request(_request) _request.url = self._client.format_url(_request.url) _stream = False @@ -211,7 +209,7 @@ async def get_properties( response_headers["x-ms-request-id"] = self._deserialize("str", response.headers.get("x-ms-request-id")) response_headers["x-ms-version"] = self._deserialize("str", response.headers.get("x-ms-version")) - deserialized = self._deserialize("StorageServiceProperties", pipeline_response) + deserialized = self._deserialize("StorageServiceProperties", pipeline_response.http_response) if cls: return cls(pipeline_response, deserialized, response_headers) # type: ignore @@ -264,7 +262,6 @@ async def get_statistics( headers=_headers, params=_params, ) - _request = _convert_request(_request) _request.url = self._client.format_url(_request.url) _stream = False @@ -287,7 +284,7 @@ async def get_statistics( response_headers["x-ms-version"] = self._deserialize("str", response.headers.get("x-ms-version")) response_headers["Date"] = self._deserialize("rfc-1123", response.headers.get("Date")) - deserialized = self._deserialize("StorageServiceStats", pipeline_response) + deserialized = self._deserialize("StorageServiceStats", pipeline_response.http_response) if cls: return cls(pipeline_response, deserialized, response_headers) # type: ignore @@ -368,7 +365,6 @@ async def list_containers_segment( headers=_headers, params=_params, ) - _request = _convert_request(_request) _request.url = self._client.format_url(_request.url) _stream = False @@ -390,7 +386,7 @@ async def list_containers_segment( response_headers["x-ms-request-id"] = self._deserialize("str", response.headers.get("x-ms-request-id")) response_headers["x-ms-version"] = self._deserialize("str", response.headers.get("x-ms-version")) - deserialized = self._deserialize("ListContainersSegmentResponse", pipeline_response) + deserialized = self._deserialize("ListContainersSegmentResponse", pipeline_response.http_response) if cls: return cls(pipeline_response, deserialized, response_headers) # type: ignore @@ -453,7 +449,6 @@ async def get_user_delegation_key( headers=_headers, params=_params, ) - _request = _convert_request(_request) _request.url = self._client.format_url(_request.url) _stream = False @@ -476,7 +471,7 @@ async def get_user_delegation_key( response_headers["x-ms-version"] = self._deserialize("str", response.headers.get("x-ms-version")) response_headers["Date"] = self._deserialize("rfc-1123", response.headers.get("Date")) - deserialized = self._deserialize("UserDelegationKey", pipeline_response) + deserialized = self._deserialize("UserDelegationKey", pipeline_response.http_response) if cls: return cls(pipeline_response, deserialized, response_headers) # type: ignore @@ -527,7 +522,6 @@ async def get_account_info( # pylint: disable=inconsistent-return-statements headers=_headers, params=_params, ) - _request = _convert_request(_request) _request.url = self._client.format_url(_request.url) _stream = False @@ -615,9 +609,9 @@ async def submit_batch( headers=_headers, params=_params, ) - _request = _convert_request(_request) _request.url = self._client.format_url(_request.url) + _decompress = kwargs.pop("decompress", True) _stream = True pipeline_response: PipelineResponse = await self._client._pipeline.run( # pylint: disable=protected-access _request, stream=_stream, **kwargs @@ -626,6 +620,10 @@ async def submit_batch( response = pipeline_response.http_response if response.status_code not in [200]: + try: + await response.read() # Load the body in memory and close the socket + except (StreamConsumedError, StreamClosedError): + pass map_error(status_code=response.status_code, response=response, error_map=error_map) error = self._deserialize.failsafe_deserialize(_models.StorageError, pipeline_response) raise HttpResponseError(response=response, model=error) @@ -635,7 +633,7 @@ async def submit_batch( response_headers["x-ms-request-id"] = self._deserialize("str", response.headers.get("x-ms-request-id")) response_headers["x-ms-version"] = self._deserialize("str", response.headers.get("x-ms-version")) - deserialized = response.stream_download(self._client._pipeline) + deserialized = response.stream_download(self._client._pipeline, decompress=_decompress) if cls: return cls(pipeline_response, deserialized, response_headers) # type: ignore @@ -717,7 +715,6 @@ async def filter_blobs( headers=_headers, params=_params, ) - _request = _convert_request(_request) _request.url = self._client.format_url(_request.url) _stream = False @@ -740,7 +737,7 @@ async def filter_blobs( response_headers["x-ms-version"] = self._deserialize("str", response.headers.get("x-ms-version")) response_headers["Date"] = self._deserialize("rfc-1123", response.headers.get("Date")) - deserialized = self._deserialize("FilterBlobSegment", pipeline_response) + deserialized = self._deserialize("FilterBlobSegment", pipeline_response.http_response) if cls: return cls(pipeline_response, deserialized, response_headers) # type: ignore diff --git a/sdk/storage/azure-storage-blob/azure/storage/blob/_generated/operations/_append_blob_operations.py b/sdk/storage/azure-storage-blob/azure/storage/blob/_generated/operations/_append_blob_operations.py index f950ba4053bc..91b8dd56c7f7 100644 --- a/sdk/storage/azure-storage-blob/azure/storage/blob/_generated/operations/_append_blob_operations.py +++ b/sdk/storage/azure-storage-blob/azure/storage/blob/_generated/operations/_append_blob_operations.py @@ -19,14 +19,12 @@ map_error, ) from azure.core.pipeline import PipelineResponse -from azure.core.pipeline.transport import HttpResponse -from azure.core.rest import HttpRequest +from azure.core.rest import HttpRequest, HttpResponse from azure.core.tracing.decorator import distributed_trace from azure.core.utils import case_insensitive_dict from .. import models as _models from .._serialization import Serializer -from .._vendor import _convert_request if sys.version_info >= (3, 9): from collections.abc import MutableMapping @@ -577,7 +575,6 @@ def create( # pylint: disable=inconsistent-return-statements headers=_headers, params=_params, ) - _request = _convert_request(_request) _request.url = self._client.format_url(_request.url) _stream = False @@ -742,7 +739,6 @@ def append_block( # pylint: disable=inconsistent-return-statements headers=_headers, params=_params, ) - _request = _convert_request(_request) _request.url = self._client.format_url(_request.url) _stream = False @@ -944,7 +940,6 @@ def append_block_from_url( # pylint: disable=inconsistent-return-statements headers=_headers, params=_params, ) - _request = _convert_request(_request) _request.url = self._client.format_url(_request.url) _stream = False @@ -1066,7 +1061,6 @@ def seal( # pylint: disable=inconsistent-return-statements headers=_headers, params=_params, ) - _request = _convert_request(_request) _request.url = self._client.format_url(_request.url) _stream = False diff --git a/sdk/storage/azure-storage-blob/azure/storage/blob/_generated/operations/_blob_operations.py b/sdk/storage/azure-storage-blob/azure/storage/blob/_generated/operations/_blob_operations.py index 385821f836a3..83033f6b9f4c 100644 --- a/sdk/storage/azure-storage-blob/azure/storage/blob/_generated/operations/_blob_operations.py +++ b/sdk/storage/azure-storage-blob/azure/storage/blob/_generated/operations/_blob_operations.py @@ -16,17 +16,17 @@ ResourceExistsError, ResourceNotFoundError, ResourceNotModifiedError, + StreamClosedError, + StreamConsumedError, map_error, ) from azure.core.pipeline import PipelineResponse -from azure.core.pipeline.transport import HttpResponse -from azure.core.rest import HttpRequest +from azure.core.rest import HttpRequest, HttpResponse from azure.core.tracing.decorator import distributed_trace from azure.core.utils import case_insensitive_dict from .. import models as _models from .._serialization import Serializer -from .._vendor import _convert_request if sys.version_info >= (3, 9): from collections.abc import MutableMapping @@ -1601,9 +1601,9 @@ def download( headers=_headers, params=_params, ) - _request = _convert_request(_request) _request.url = self._client.format_url(_request.url) + _decompress = kwargs.pop("decompress", True) _stream = True pipeline_response: PipelineResponse = self._client._pipeline.run( # pylint: disable=protected-access _request, stream=_stream, **kwargs @@ -1612,6 +1612,10 @@ def download( response = pipeline_response.http_response if response.status_code not in [200, 206]: + try: + response.read() # Load the body in memory and close the socket + except (StreamConsumedError, StreamClosedError): + pass map_error(status_code=response.status_code, response=response, error_map=error_map) error = self._deserialize.failsafe_deserialize(_models.StorageError, pipeline_response) raise HttpResponseError(response=response, model=error) @@ -1696,8 +1700,6 @@ def download( ) response_headers["x-ms-legal-hold"] = self._deserialize("bool", response.headers.get("x-ms-legal-hold")) - deserialized = response.stream_download(self._client._pipeline) - if response.status_code == 206: response_headers["Last-Modified"] = self._deserialize("rfc-1123", response.headers.get("Last-Modified")) response_headers["x-ms-creation-time"] = self._deserialize( @@ -1780,7 +1782,7 @@ def download( ) response_headers["x-ms-legal-hold"] = self._deserialize("bool", response.headers.get("x-ms-legal-hold")) - deserialized = response.stream_download(self._client._pipeline) + deserialized = response.stream_download(self._client._pipeline, decompress=_decompress) if cls: return cls(pipeline_response, deserialized, response_headers) # type: ignore @@ -1885,7 +1887,6 @@ def get_properties( # pylint: disable=inconsistent-return-statements headers=_headers, params=_params, ) - _request = _convert_request(_request) _request.url = self._client.format_url(_request.url) _stream = False @@ -2098,7 +2099,6 @@ def delete( # pylint: disable=inconsistent-return-statements headers=_headers, params=_params, ) - _request = _convert_request(_request) _request.url = self._client.format_url(_request.url) _stream = False @@ -2166,7 +2166,6 @@ def undelete( # pylint: disable=inconsistent-return-statements headers=_headers, params=_params, ) - _request = _convert_request(_request) _request.url = self._client.format_url(_request.url) _stream = False @@ -2246,7 +2245,6 @@ def set_expiry( # pylint: disable=inconsistent-return-statements headers=_headers, params=_params, ) - _request = _convert_request(_request) _request.url = self._client.format_url(_request.url) _stream = False @@ -2368,7 +2366,6 @@ def set_http_headers( # pylint: disable=inconsistent-return-statements headers=_headers, params=_params, ) - _request = _convert_request(_request) _request.url = self._client.format_url(_request.url) _stream = False @@ -2462,7 +2459,6 @@ def set_immutability_policy( # pylint: disable=inconsistent-return-statements headers=_headers, params=_params, ) - _request = _convert_request(_request) _request.url = self._client.format_url(_request.url) _stream = False @@ -2536,7 +2532,6 @@ def delete_immutability_policy( # pylint: disable=inconsistent-return-statement headers=_headers, params=_params, ) - _request = _convert_request(_request) _request.url = self._client.format_url(_request.url) _stream = False @@ -2607,7 +2602,6 @@ def set_legal_hold( # pylint: disable=inconsistent-return-statements headers=_headers, params=_params, ) - _request = _convert_request(_request) _request.url = self._client.format_url(_request.url) _stream = False @@ -2737,7 +2731,6 @@ def set_metadata( # pylint: disable=inconsistent-return-statements headers=_headers, params=_params, ) - _request = _convert_request(_request) _request.url = self._client.format_url(_request.url) _stream = False @@ -2855,7 +2848,6 @@ def acquire_lease( # pylint: disable=inconsistent-return-statements headers=_headers, params=_params, ) - _request = _convert_request(_request) _request.url = self._client.format_url(_request.url) _stream = False @@ -2956,7 +2948,6 @@ def release_lease( # pylint: disable=inconsistent-return-statements headers=_headers, params=_params, ) - _request = _convert_request(_request) _request.url = self._client.format_url(_request.url) _stream = False @@ -3056,7 +3047,6 @@ def renew_lease( # pylint: disable=inconsistent-return-statements headers=_headers, params=_params, ) - _request = _convert_request(_request) _request.url = self._client.format_url(_request.url) _stream = False @@ -3163,7 +3153,6 @@ def change_lease( # pylint: disable=inconsistent-return-statements headers=_headers, params=_params, ) - _request = _convert_request(_request) _request.url = self._client.format_url(_request.url) _stream = False @@ -3270,7 +3259,6 @@ def break_lease( # pylint: disable=inconsistent-return-statements headers=_headers, params=_params, ) - _request = _convert_request(_request) _request.url = self._client.format_url(_request.url) _stream = False @@ -3401,7 +3389,6 @@ def create_snapshot( # pylint: disable=inconsistent-return-statements headers=_headers, params=_params, ) - _request = _convert_request(_request) _request.url = self._client.format_url(_request.url) _stream = False @@ -3576,7 +3563,6 @@ def start_copy_from_url( # pylint: disable=inconsistent-return-statements headers=_headers, params=_params, ) - _request = _convert_request(_request) _request.url = self._client.format_url(_request.url) _stream = False @@ -3764,7 +3750,6 @@ def copy_from_url( # pylint: disable=inconsistent-return-statements headers=_headers, params=_params, ) - _request = _convert_request(_request) _request.url = self._client.format_url(_request.url) _stream = False @@ -3865,7 +3850,6 @@ def abort_copy_from_url( # pylint: disable=inconsistent-return-statements headers=_headers, params=_params, ) - _request = _convert_request(_request) _request.url = self._client.format_url(_request.url) _stream = False @@ -3980,7 +3964,6 @@ def set_tier( # pylint: disable=inconsistent-return-statements headers=_headers, params=_params, ) - _request = _convert_request(_request) _request.url = self._client.format_url(_request.url) _stream = False @@ -3996,19 +3979,11 @@ def set_tier( # pylint: disable=inconsistent-return-statements raise HttpResponseError(response=response, model=error) response_headers = {} - if response.status_code == 200: - response_headers["x-ms-client-request-id"] = self._deserialize( - "str", response.headers.get("x-ms-client-request-id") - ) - response_headers["x-ms-request-id"] = self._deserialize("str", response.headers.get("x-ms-request-id")) - response_headers["x-ms-version"] = self._deserialize("str", response.headers.get("x-ms-version")) - - if response.status_code == 202: - response_headers["x-ms-client-request-id"] = self._deserialize( - "str", response.headers.get("x-ms-client-request-id") - ) - response_headers["x-ms-request-id"] = self._deserialize("str", response.headers.get("x-ms-request-id")) - response_headers["x-ms-version"] = self._deserialize("str", response.headers.get("x-ms-version")) + response_headers["x-ms-client-request-id"] = self._deserialize( + "str", response.headers.get("x-ms-client-request-id") + ) + response_headers["x-ms-request-id"] = self._deserialize("str", response.headers.get("x-ms-request-id")) + response_headers["x-ms-version"] = self._deserialize("str", response.headers.get("x-ms-version")) if cls: return cls(pipeline_response, None, response_headers) # type: ignore @@ -4057,7 +4032,6 @@ def get_account_info( # pylint: disable=inconsistent-return-statements headers=_headers, params=_params, ) - _request = _convert_request(_request) _request.url = self._client.format_url(_request.url) _stream = False @@ -4190,9 +4164,9 @@ def query( headers=_headers, params=_params, ) - _request = _convert_request(_request) _request.url = self._client.format_url(_request.url) + _decompress = kwargs.pop("decompress", True) _stream = True pipeline_response: PipelineResponse = self._client._pipeline.run( # pylint: disable=protected-access _request, stream=_stream, **kwargs @@ -4201,6 +4175,10 @@ def query( response = pipeline_response.http_response if response.status_code not in [200, 206]: + try: + response.read() # Load the body in memory and close the socket + except (StreamConsumedError, StreamClosedError): + pass map_error(status_code=response.status_code, response=response, error_map=error_map) error = self._deserialize.failsafe_deserialize(_models.StorageError, pipeline_response) raise HttpResponseError(response=response, model=error) @@ -4264,8 +4242,6 @@ def query( "bytearray", response.headers.get("x-ms-blob-content-md5") ) - deserialized = response.stream_download(self._client._pipeline) - if response.status_code == 206: response_headers["Last-Modified"] = self._deserialize("rfc-1123", response.headers.get("Last-Modified")) response_headers["x-ms-meta"] = self._deserialize("{str}", response.headers.get("x-ms-meta")) @@ -4327,7 +4303,7 @@ def query( "bytearray", response.headers.get("x-ms-blob-content-md5") ) - deserialized = response.stream_download(self._client._pipeline) + deserialized = response.stream_download(self._client._pipeline, decompress=_decompress) if cls: return cls(pipeline_response, deserialized, response_headers) # type: ignore @@ -4408,7 +4384,6 @@ def get_tags( headers=_headers, params=_params, ) - _request = _convert_request(_request) _request.url = self._client.format_url(_request.url) _stream = False @@ -4431,7 +4406,7 @@ def get_tags( response_headers["x-ms-version"] = self._deserialize("str", response.headers.get("x-ms-version")) response_headers["Date"] = self._deserialize("rfc-1123", response.headers.get("Date")) - deserialized = self._deserialize("BlobTags", pipeline_response) + deserialized = self._deserialize("BlobTags", pipeline_response.http_response) if cls: return cls(pipeline_response, deserialized, response_headers) # type: ignore @@ -4524,7 +4499,6 @@ def set_tags( # pylint: disable=inconsistent-return-statements headers=_headers, params=_params, ) - _request = _convert_request(_request) _request.url = self._client.format_url(_request.url) _stream = False diff --git a/sdk/storage/azure-storage-blob/azure/storage/blob/_generated/operations/_block_blob_operations.py b/sdk/storage/azure-storage-blob/azure/storage/blob/_generated/operations/_block_blob_operations.py index 0a3083b16a7b..ab161913ceb0 100644 --- a/sdk/storage/azure-storage-blob/azure/storage/blob/_generated/operations/_block_blob_operations.py +++ b/sdk/storage/azure-storage-blob/azure/storage/blob/_generated/operations/_block_blob_operations.py @@ -19,14 +19,12 @@ map_error, ) from azure.core.pipeline import PipelineResponse -from azure.core.pipeline.transport import HttpResponse -from azure.core.rest import HttpRequest +from azure.core.rest import HttpRequest, HttpResponse from azure.core.tracing.decorator import distributed_trace from azure.core.utils import case_insensitive_dict from .. import models as _models from .._serialization import Serializer -from .._vendor import _convert_request if sys.version_info >= (3, 9): from collections.abc import MutableMapping @@ -844,7 +842,6 @@ def upload( # pylint: disable=inconsistent-return-statements headers=_headers, params=_params, ) - _request = _convert_request(_request) _request.url = self._client.format_url(_request.url) _stream = False @@ -1079,7 +1076,6 @@ def put_blob_from_url( # pylint: disable=inconsistent-return-statements headers=_headers, params=_params, ) - _request = _convert_request(_request) _request.url = self._client.format_url(_request.url) _stream = False @@ -1218,7 +1214,6 @@ def stage_block( # pylint: disable=inconsistent-return-statements headers=_headers, params=_params, ) - _request = _convert_request(_request) _request.url = self._client.format_url(_request.url) _stream = False @@ -1381,7 +1376,6 @@ def stage_block_from_url( # pylint: disable=inconsistent-return-statements headers=_headers, params=_params, ) - _request = _convert_request(_request) _request.url = self._client.format_url(_request.url) _stream = False @@ -1592,7 +1586,6 @@ def commit_block_list( # pylint: disable=inconsistent-return-statements headers=_headers, params=_params, ) - _request = _convert_request(_request) _request.url = self._client.format_url(_request.url) _stream = False @@ -1709,7 +1702,6 @@ def get_block_list( headers=_headers, params=_params, ) - _request = _convert_request(_request) _request.url = self._client.format_url(_request.url) _stream = False @@ -1738,7 +1730,7 @@ def get_block_list( response_headers["x-ms-version"] = self._deserialize("str", response.headers.get("x-ms-version")) response_headers["Date"] = self._deserialize("rfc-1123", response.headers.get("Date")) - deserialized = self._deserialize("BlockList", pipeline_response) + deserialized = self._deserialize("BlockList", pipeline_response.http_response) if cls: return cls(pipeline_response, deserialized, response_headers) # type: ignore diff --git a/sdk/storage/azure-storage-blob/azure/storage/blob/_generated/operations/_container_operations.py b/sdk/storage/azure-storage-blob/azure/storage/blob/_generated/operations/_container_operations.py index 37fe1f75556b..719059977673 100644 --- a/sdk/storage/azure-storage-blob/azure/storage/blob/_generated/operations/_container_operations.py +++ b/sdk/storage/azure-storage-blob/azure/storage/blob/_generated/operations/_container_operations.py @@ -16,17 +16,17 @@ ResourceExistsError, ResourceNotFoundError, ResourceNotModifiedError, + StreamClosedError, + StreamConsumedError, map_error, ) from azure.core.pipeline import PipelineResponse -from azure.core.pipeline.transport import HttpResponse -from azure.core.rest import HttpRequest +from azure.core.rest import HttpRequest, HttpResponse from azure.core.tracing.decorator import distributed_trace from azure.core.utils import case_insensitive_dict from .. import models as _models from .._serialization import Serializer -from .._vendor import _convert_request if sys.version_info >= (3, 9): from collections.abc import MutableMapping @@ -976,7 +976,6 @@ def create( # pylint: disable=inconsistent-return-statements headers=_headers, params=_params, ) - _request = _convert_request(_request) _request.url = self._client.format_url(_request.url) _stream = False @@ -1058,7 +1057,6 @@ def get_properties( # pylint: disable=inconsistent-return-statements headers=_headers, params=_params, ) - _request = _convert_request(_request) _request.url = self._client.format_url(_request.url) _stream = False @@ -1170,7 +1168,6 @@ def delete( # pylint: disable=inconsistent-return-statements headers=_headers, params=_params, ) - _request = _convert_request(_request) _request.url = self._client.format_url(_request.url) _stream = False @@ -1268,7 +1265,6 @@ def set_metadata( # pylint: disable=inconsistent-return-statements headers=_headers, params=_params, ) - _request = _convert_request(_request) _request.url = self._client.format_url(_request.url) _stream = False @@ -1352,7 +1348,6 @@ def get_access_policy( headers=_headers, params=_params, ) - _request = _convert_request(_request) _request.url = self._client.format_url(_request.url) _stream = False @@ -1380,7 +1375,7 @@ def get_access_policy( response_headers["x-ms-version"] = self._deserialize("str", response.headers.get("x-ms-version")) response_headers["Date"] = self._deserialize("rfc-1123", response.headers.get("Date")) - deserialized = self._deserialize("[SignedIdentifier]", pipeline_response) + deserialized = self._deserialize("[SignedIdentifier]", pipeline_response.http_response) if cls: return cls(pipeline_response, deserialized, response_headers) # type: ignore @@ -1471,7 +1466,6 @@ def set_access_policy( # pylint: disable=inconsistent-return-statements headers=_headers, params=_params, ) - _request = _convert_request(_request) _request.url = self._client.format_url(_request.url) _stream = False @@ -1556,7 +1550,6 @@ def restore( # pylint: disable=inconsistent-return-statements headers=_headers, params=_params, ) - _request = _convert_request(_request) _request.url = self._client.format_url(_request.url) _stream = False @@ -1639,7 +1632,6 @@ def rename( # pylint: disable=inconsistent-return-statements headers=_headers, params=_params, ) - _request = _convert_request(_request) _request.url = self._client.format_url(_request.url) _stream = False @@ -1726,9 +1718,9 @@ def submit_batch( headers=_headers, params=_params, ) - _request = _convert_request(_request) _request.url = self._client.format_url(_request.url) + _decompress = kwargs.pop("decompress", True) _stream = True pipeline_response: PipelineResponse = self._client._pipeline.run( # pylint: disable=protected-access _request, stream=_stream, **kwargs @@ -1737,6 +1729,10 @@ def submit_batch( response = pipeline_response.http_response if response.status_code not in [202]: + try: + response.read() # Load the body in memory and close the socket + except (StreamConsumedError, StreamClosedError): + pass map_error(status_code=response.status_code, response=response, error_map=error_map) error = self._deserialize.failsafe_deserialize(_models.StorageError, pipeline_response) raise HttpResponseError(response=response, model=error) @@ -1746,7 +1742,7 @@ def submit_batch( response_headers["x-ms-request-id"] = self._deserialize("str", response.headers.get("x-ms-request-id")) response_headers["x-ms-version"] = self._deserialize("str", response.headers.get("x-ms-version")) - deserialized = response.stream_download(self._client._pipeline) + deserialized = response.stream_download(self._client._pipeline, decompress=_decompress) if cls: return cls(pipeline_response, deserialized, response_headers) # type: ignore @@ -1829,7 +1825,6 @@ def filter_blobs( headers=_headers, params=_params, ) - _request = _convert_request(_request) _request.url = self._client.format_url(_request.url) _stream = False @@ -1852,7 +1847,7 @@ def filter_blobs( response_headers["x-ms-version"] = self._deserialize("str", response.headers.get("x-ms-version")) response_headers["Date"] = self._deserialize("rfc-1123", response.headers.get("Date")) - deserialized = self._deserialize("FilterBlobSegment", pipeline_response) + deserialized = self._deserialize("FilterBlobSegment", pipeline_response.http_response) if cls: return cls(pipeline_response, deserialized, response_headers) # type: ignore @@ -1932,7 +1927,6 @@ def acquire_lease( # pylint: disable=inconsistent-return-statements headers=_headers, params=_params, ) - _request = _convert_request(_request) _request.url = self._client.format_url(_request.url) _stream = False @@ -2026,7 +2020,6 @@ def release_lease( # pylint: disable=inconsistent-return-statements headers=_headers, params=_params, ) - _request = _convert_request(_request) _request.url = self._client.format_url(_request.url) _stream = False @@ -2119,7 +2112,6 @@ def renew_lease( # pylint: disable=inconsistent-return-statements headers=_headers, params=_params, ) - _request = _convert_request(_request) _request.url = self._client.format_url(_request.url) _stream = False @@ -2219,7 +2211,6 @@ def break_lease( # pylint: disable=inconsistent-return-statements headers=_headers, params=_params, ) - _request = _convert_request(_request) _request.url = self._client.format_url(_request.url) _stream = False @@ -2319,7 +2310,6 @@ def change_lease( # pylint: disable=inconsistent-return-statements headers=_headers, params=_params, ) - _request = _convert_request(_request) _request.url = self._client.format_url(_request.url) _stream = False @@ -2423,7 +2413,6 @@ def list_blob_flat_segment( headers=_headers, params=_params, ) - _request = _convert_request(_request) _request.url = self._client.format_url(_request.url) _stream = False @@ -2447,7 +2436,7 @@ def list_blob_flat_segment( response_headers["x-ms-version"] = self._deserialize("str", response.headers.get("x-ms-version")) response_headers["Date"] = self._deserialize("rfc-1123", response.headers.get("Date")) - deserialized = self._deserialize("ListBlobsFlatSegmentResponse", pipeline_response) + deserialized = self._deserialize("ListBlobsFlatSegmentResponse", pipeline_response.http_response) if cls: return cls(pipeline_response, deserialized, response_headers) # type: ignore @@ -2536,7 +2525,6 @@ def list_blob_hierarchy_segment( headers=_headers, params=_params, ) - _request = _convert_request(_request) _request.url = self._client.format_url(_request.url) _stream = False @@ -2560,7 +2548,7 @@ def list_blob_hierarchy_segment( response_headers["x-ms-version"] = self._deserialize("str", response.headers.get("x-ms-version")) response_headers["Date"] = self._deserialize("rfc-1123", response.headers.get("Date")) - deserialized = self._deserialize("ListBlobsHierarchySegmentResponse", pipeline_response) + deserialized = self._deserialize("ListBlobsHierarchySegmentResponse", pipeline_response.http_response) if cls: return cls(pipeline_response, deserialized, response_headers) # type: ignore @@ -2611,7 +2599,6 @@ def get_account_info( # pylint: disable=inconsistent-return-statements headers=_headers, params=_params, ) - _request = _convert_request(_request) _request.url = self._client.format_url(_request.url) _stream = False diff --git a/sdk/storage/azure-storage-blob/azure/storage/blob/_generated/operations/_page_blob_operations.py b/sdk/storage/azure-storage-blob/azure/storage/blob/_generated/operations/_page_blob_operations.py index e388063fc7fa..a280a9f3048d 100644 --- a/sdk/storage/azure-storage-blob/azure/storage/blob/_generated/operations/_page_blob_operations.py +++ b/sdk/storage/azure-storage-blob/azure/storage/blob/_generated/operations/_page_blob_operations.py @@ -19,14 +19,12 @@ map_error, ) from azure.core.pipeline import PipelineResponse -from azure.core.pipeline.transport import HttpResponse -from azure.core.rest import HttpRequest +from azure.core.rest import HttpRequest, HttpResponse from azure.core.tracing.decorator import distributed_trace from azure.core.utils import case_insensitive_dict from .. import models as _models from .._serialization import Serializer -from .._vendor import _convert_request if sys.version_info >= (3, 9): from collections.abc import MutableMapping @@ -980,7 +978,6 @@ def create( # pylint: disable=inconsistent-return-statements headers=_headers, params=_params, ) - _request = _convert_request(_request) _request.url = self._client.format_url(_request.url) _stream = False @@ -1154,7 +1151,6 @@ def upload_pages( # pylint: disable=inconsistent-return-statements headers=_headers, params=_params, ) - _request = _convert_request(_request) _request.url = self._client.format_url(_request.url) _stream = False @@ -1316,7 +1312,6 @@ def clear_pages( # pylint: disable=inconsistent-return-statements headers=_headers, params=_params, ) - _request = _convert_request(_request) _request.url = self._client.format_url(_request.url) _stream = False @@ -1512,7 +1507,6 @@ def upload_pages_from_url( # pylint: disable=inconsistent-return-statements headers=_headers, params=_params, ) - _request = _convert_request(_request) _request.url = self._client.format_url(_request.url) _stream = False @@ -1656,7 +1650,6 @@ def get_page_ranges( headers=_headers, params=_params, ) - _request = _convert_request(_request) _request.url = self._client.format_url(_request.url) _stream = False @@ -1684,7 +1677,7 @@ def get_page_ranges( response_headers["x-ms-version"] = self._deserialize("str", response.headers.get("x-ms-version")) response_headers["Date"] = self._deserialize("rfc-1123", response.headers.get("Date")) - deserialized = self._deserialize("PageList", pipeline_response) + deserialized = self._deserialize("PageList", pipeline_response.http_response) if cls: return cls(pipeline_response, deserialized, response_headers) # type: ignore @@ -1810,7 +1803,6 @@ def get_page_ranges_diff( headers=_headers, params=_params, ) - _request = _convert_request(_request) _request.url = self._client.format_url(_request.url) _stream = False @@ -1838,7 +1830,7 @@ def get_page_ranges_diff( response_headers["x-ms-version"] = self._deserialize("str", response.headers.get("x-ms-version")) response_headers["Date"] = self._deserialize("rfc-1123", response.headers.get("Date")) - deserialized = self._deserialize("PageList", pipeline_response) + deserialized = self._deserialize("PageList", pipeline_response.http_response) if cls: return cls(pipeline_response, deserialized, response_headers) # type: ignore @@ -1942,7 +1934,6 @@ def resize( # pylint: disable=inconsistent-return-statements headers=_headers, params=_params, ) - _request = _convert_request(_request) _request.url = self._client.format_url(_request.url) _stream = False @@ -2058,7 +2049,6 @@ def update_sequence_number( # pylint: disable=inconsistent-return-statements headers=_headers, params=_params, ) - _request = _convert_request(_request) _request.url = self._client.format_url(_request.url) _stream = False @@ -2165,7 +2155,6 @@ def copy_incremental( # pylint: disable=inconsistent-return-statements headers=_headers, params=_params, ) - _request = _convert_request(_request) _request.url = self._client.format_url(_request.url) _stream = False diff --git a/sdk/storage/azure-storage-blob/azure/storage/blob/_generated/operations/_service_operations.py b/sdk/storage/azure-storage-blob/azure/storage/blob/_generated/operations/_service_operations.py index 0d1bc1509661..2e2a84dc524d 100644 --- a/sdk/storage/azure-storage-blob/azure/storage/blob/_generated/operations/_service_operations.py +++ b/sdk/storage/azure-storage-blob/azure/storage/blob/_generated/operations/_service_operations.py @@ -15,17 +15,17 @@ ResourceExistsError, ResourceNotFoundError, ResourceNotModifiedError, + StreamClosedError, + StreamConsumedError, map_error, ) from azure.core.pipeline import PipelineResponse -from azure.core.pipeline.transport import HttpResponse -from azure.core.rest import HttpRequest +from azure.core.rest import HttpRequest, HttpResponse from azure.core.tracing.decorator import distributed_trace from azure.core.utils import case_insensitive_dict from .. import models as _models from .._serialization import Serializer -from .._vendor import _convert_request if sys.version_info >= (3, 9): from collections.abc import MutableMapping @@ -427,7 +427,6 @@ def set_properties( # pylint: disable=inconsistent-return-statements headers=_headers, params=_params, ) - _request = _convert_request(_request) _request.url = self._client.format_url(_request.url) _stream = False @@ -497,7 +496,6 @@ def get_properties( headers=_headers, params=_params, ) - _request = _convert_request(_request) _request.url = self._client.format_url(_request.url) _stream = False @@ -519,7 +517,7 @@ def get_properties( response_headers["x-ms-request-id"] = self._deserialize("str", response.headers.get("x-ms-request-id")) response_headers["x-ms-version"] = self._deserialize("str", response.headers.get("x-ms-version")) - deserialized = self._deserialize("StorageServiceProperties", pipeline_response) + deserialized = self._deserialize("StorageServiceProperties", pipeline_response.http_response) if cls: return cls(pipeline_response, deserialized, response_headers) # type: ignore @@ -572,7 +570,6 @@ def get_statistics( headers=_headers, params=_params, ) - _request = _convert_request(_request) _request.url = self._client.format_url(_request.url) _stream = False @@ -595,7 +592,7 @@ def get_statistics( response_headers["x-ms-version"] = self._deserialize("str", response.headers.get("x-ms-version")) response_headers["Date"] = self._deserialize("rfc-1123", response.headers.get("Date")) - deserialized = self._deserialize("StorageServiceStats", pipeline_response) + deserialized = self._deserialize("StorageServiceStats", pipeline_response.http_response) if cls: return cls(pipeline_response, deserialized, response_headers) # type: ignore @@ -676,7 +673,6 @@ def list_containers_segment( headers=_headers, params=_params, ) - _request = _convert_request(_request) _request.url = self._client.format_url(_request.url) _stream = False @@ -698,7 +694,7 @@ def list_containers_segment( response_headers["x-ms-request-id"] = self._deserialize("str", response.headers.get("x-ms-request-id")) response_headers["x-ms-version"] = self._deserialize("str", response.headers.get("x-ms-version")) - deserialized = self._deserialize("ListContainersSegmentResponse", pipeline_response) + deserialized = self._deserialize("ListContainersSegmentResponse", pipeline_response.http_response) if cls: return cls(pipeline_response, deserialized, response_headers) # type: ignore @@ -761,7 +757,6 @@ def get_user_delegation_key( headers=_headers, params=_params, ) - _request = _convert_request(_request) _request.url = self._client.format_url(_request.url) _stream = False @@ -784,7 +779,7 @@ def get_user_delegation_key( response_headers["x-ms-version"] = self._deserialize("str", response.headers.get("x-ms-version")) response_headers["Date"] = self._deserialize("rfc-1123", response.headers.get("Date")) - deserialized = self._deserialize("UserDelegationKey", pipeline_response) + deserialized = self._deserialize("UserDelegationKey", pipeline_response.http_response) if cls: return cls(pipeline_response, deserialized, response_headers) # type: ignore @@ -835,7 +830,6 @@ def get_account_info( # pylint: disable=inconsistent-return-statements headers=_headers, params=_params, ) - _request = _convert_request(_request) _request.url = self._client.format_url(_request.url) _stream = False @@ -923,9 +917,9 @@ def submit_batch( headers=_headers, params=_params, ) - _request = _convert_request(_request) _request.url = self._client.format_url(_request.url) + _decompress = kwargs.pop("decompress", True) _stream = True pipeline_response: PipelineResponse = self._client._pipeline.run( # pylint: disable=protected-access _request, stream=_stream, **kwargs @@ -934,6 +928,10 @@ def submit_batch( response = pipeline_response.http_response if response.status_code not in [200]: + try: + response.read() # Load the body in memory and close the socket + except (StreamConsumedError, StreamClosedError): + pass map_error(status_code=response.status_code, response=response, error_map=error_map) error = self._deserialize.failsafe_deserialize(_models.StorageError, pipeline_response) raise HttpResponseError(response=response, model=error) @@ -943,7 +941,7 @@ def submit_batch( response_headers["x-ms-request-id"] = self._deserialize("str", response.headers.get("x-ms-request-id")) response_headers["x-ms-version"] = self._deserialize("str", response.headers.get("x-ms-version")) - deserialized = response.stream_download(self._client._pipeline) + deserialized = response.stream_download(self._client._pipeline, decompress=_decompress) if cls: return cls(pipeline_response, deserialized, response_headers) # type: ignore @@ -1025,7 +1023,6 @@ def filter_blobs( headers=_headers, params=_params, ) - _request = _convert_request(_request) _request.url = self._client.format_url(_request.url) _stream = False @@ -1048,7 +1045,7 @@ def filter_blobs( response_headers["x-ms-version"] = self._deserialize("str", response.headers.get("x-ms-version")) response_headers["Date"] = self._deserialize("rfc-1123", response.headers.get("Date")) - deserialized = self._deserialize("FilterBlobSegment", pipeline_response) + deserialized = self._deserialize("FilterBlobSegment", pipeline_response.http_response) if cls: return cls(pipeline_response, deserialized, response_headers) # type: ignore diff --git a/sdk/storage/azure-storage-blob/azure/storage/blob/_shared/policies.py b/sdk/storage/azure-storage-blob/azure/storage/blob/_shared/policies.py index 803515717797..07de6ad5579e 100644 --- a/sdk/storage/azure-storage-blob/azure/storage/blob/_shared/policies.py +++ b/sdk/storage/azure-storage-blob/azure/storage/blob/_shared/policies.py @@ -35,11 +35,6 @@ from .constants import DEFAULT_OAUTH_SCOPE from .models import LocationMode -try: - _unicode_type = unicode # type: ignore -except NameError: - _unicode_type = str - if TYPE_CHECKING: from azure.core.credentials import TokenCredential from azure.core.pipeline.transport import ( # pylint: disable=non-abstract-transport-import @@ -52,7 +47,7 @@ def encode_base64(data): - if isinstance(data, _unicode_type): + if isinstance(data, str): data = data.encode('utf-8') encoded = base64.b64encode(data) return encoded.decode('utf-8') @@ -95,10 +90,14 @@ def is_retry(response, mode): # pylint: disable=too-many-return-statements if status in [501, 505]: return False return True + return False + + +def is_checksum_retry(response): # retry if invalid content md5 if response.context.get('validate_content', False) and response.http_response.headers.get('content-md5'): computed_md5 = response.http_request.headers.get('content-md5', None) or \ - encode_base64(StorageContentValidation.get_content_md5(response.http_response.body())) + encode_base64(StorageContentValidation.get_content_md5(response.http_response.body())) if response.http_response.headers['content-md5'] != computed_md5: return True return False @@ -301,7 +300,7 @@ def send(self, request: "PipelineRequest") -> "PipelineResponse": response = self.next.send(request) - will_retry = is_retry(response, request.context.options.get('mode')) + will_retry = is_retry(response, request.context.options.get('mode')) or is_checksum_retry(response) # Auth error could come from Bearer challenge, in which case this request will be made again is_auth_error = response.http_response.status_code == 401 should_update_counts = not (will_retry or is_auth_error) @@ -527,7 +526,7 @@ def send(self, request): while retries_remaining: try: response = self.next.send(request) - if is_retry(response, retry_settings['mode']): + if is_retry(response, retry_settings['mode']) or is_checksum_retry(response): retries_remaining = self.increment( retry_settings, request=request.http_request, diff --git a/sdk/storage/azure-storage-blob/azure/storage/blob/_shared/policies_async.py b/sdk/storage/azure-storage-blob/azure/storage/blob/_shared/policies_async.py index 047168f3f8cd..67987a090f33 100644 --- a/sdk/storage/azure-storage-blob/azure/storage/blob/_shared/policies_async.py +++ b/sdk/storage/azure-storage-blob/azure/storage/blob/_shared/policies_async.py @@ -10,12 +10,12 @@ import random from typing import Any, Dict, TYPE_CHECKING -from azure.core.exceptions import AzureError +from azure.core.exceptions import AzureError, StreamClosedError, StreamConsumedError from azure.core.pipeline.policies import AsyncBearerTokenCredentialPolicy, AsyncHTTPPolicy from .authentication import AzureSigningError, StorageHttpChallenge from .constants import DEFAULT_OAUTH_SCOPE -from .policies import is_retry, StorageRetryPolicy +from .policies import encode_base64, is_retry, StorageContentValidation, StorageRetryPolicy if TYPE_CHECKING: from azure.core.credentials_async import AsyncTokenCredential @@ -42,6 +42,20 @@ async def retry_hook(settings, **kwargs): **kwargs) +async def is_checksum_retry(response): + # retry if invalid content md5 + if response.context.get('validate_content', False) and response.http_response.headers.get('content-md5'): + try: + await response.http_response.read() # Load the body in memory and close the socket + except (StreamClosedError, StreamConsumedError): + pass + computed_md5 = response.http_request.headers.get('content-md5', None) or \ + encode_base64(StorageContentValidation.get_content_md5(response.http_response.content)) + if response.http_response.headers['content-md5'] != computed_md5: + return True + return False + + class AsyncStorageResponseHook(AsyncHTTPPolicy): def __init__(self, **kwargs): # pylint: disable=unused-argument @@ -64,9 +78,8 @@ async def send(self, request: "PipelineRequest") -> "PipelineResponse": request.context.options.pop('raw_response_hook', self._response_callback) response = await self.next.send(request) - await response.http_response.load_body() + will_retry = is_retry(response, request.context.options.get('mode')) or await is_checksum_retry(response) - will_retry = is_retry(response, request.context.options.get('mode')) # Auth error could come from Bearer challenge, in which case this request will be made again is_auth_error = response.http_response.status_code == 401 should_update_counts = not (will_retry or is_auth_error) @@ -112,7 +125,7 @@ async def send(self, request): while retries_remaining: try: response = await self.next.send(request) - if is_retry(response, retry_settings['mode']): + if is_retry(response, retry_settings['mode']) or await is_checksum_retry(response): retries_remaining = self.increment( retry_settings, request=request.http_request, diff --git a/sdk/storage/azure-storage-blob/azure/storage/blob/aio/_download_async.py b/sdk/storage/azure-storage-blob/azure/storage/blob/aio/_download_async.py index 399e5ef0fb0c..dab5afdca85d 100644 --- a/sdk/storage/azure-storage-blob/azure/storage/blob/aio/_download_async.py +++ b/sdk/storage/azure-storage-blob/azure/storage/blob/aio/_download_async.py @@ -19,7 +19,7 @@ Tuple, TypeVar, Union, TYPE_CHECKING ) -from azure.core.exceptions import HttpResponseError +from azure.core.exceptions import DecodeError, HttpResponseError, IncompleteReadError from .._shared.request_handlers import validate_and_format_range_headers from .._shared.response_handlers import parse_length_from_content_range, process_storage_error @@ -46,7 +46,8 @@ async def process_content(data: Any, start_offset: int, end_offset: int, encryption: Dict[str, Any]) -> bytes: if data is None: raise ValueError("Response cannot be None.") - content = cast(bytes, data.response.body()) + await data.response.read() + content = cast(bytes, data.response.content) if encryption.get('key') is not None or encryption.get('resolver') is not None: try: return decrypt_blob( @@ -120,20 +121,30 @@ async def _download_chunk(self, chunk_start: int, chunk_end: int) -> Tuple[bytes download_range[1], check_content_md5=self.validate_content ) - try: - _, response = await cast(Awaitable[Any], self.client.download( - range=range_header, - range_get_content_md5=range_validation, - validate_content=self.validate_content, - data_stream_total=self.total_size, - download_stream_current=self.progress_total, - **self.request_options - )) - except HttpResponseError as error: - process_storage_error(error) + retry_active = True + retry_total = 3 + while retry_active: + try: + _, response = await cast(Awaitable[Any], self.client.download( + range=range_header, + range_get_content_md5=range_validation, + validate_content=self.validate_content, + data_stream_total=self.total_size, + download_stream_current=self.progress_total, + **self.request_options + )) + except HttpResponseError as error: + process_storage_error(error) - chunk_data = await process_content(response, offset[0], offset[1], self.encryption_options) + try: + chunk_data = await process_content(response, offset[0], offset[1], self.encryption_options) + retry_active = False + except (IncompleteReadError, HttpResponseError, DecodeError) as error: + retry_total -= 1 + if retry_total <= 0: + raise HttpResponseError(error, error=error) from error + await asyncio.sleep(1) content_length = response.content_length # This makes sure that if_match is set so that we can validate @@ -342,66 +353,78 @@ async def _initial_request(self): self._initial_range[1], start_range_required=False, end_range_required=False, - check_content_md5=self._validate_content) + check_content_md5=self._validate_content + ) - try: - location_mode, response = cast(Tuple[Optional[str], Any], await self._clients.blob.download( - range=range_header, - range_get_content_md5=range_validation, - validate_content=self._validate_content, - data_stream_total=None, - download_stream_current=0, - **self._request_options)) - - # Check the location we read from to ensure we use the same one - # for subsequent requests. - self._location_mode = location_mode - - # Parse the total file size and adjust the download size if ranges - # were specified - self._file_size = parse_length_from_content_range(response.properties.content_range) - if self._file_size is None: - raise ValueError("Required Content-Range response header is missing or malformed.") - # Remove any extra encryption data size from blob size - self._file_size = adjust_blob_size_for_encryption(self._file_size, self._encryption_data) - - if self._end_range is not None and self._start_range is not None: - # Use the length unless it is over the end of the file - self.size = min(self._file_size - self._start_range, self._end_range - self._start_range + 1) - elif self._start_range is not None: - self.size = self._file_size - self._start_range - else: - self.size = self._file_size + retry_active = True + retry_total = 3 + while retry_active: + try: + location_mode, response = cast(Tuple[Optional[str], Any], await self._clients.blob.download( + range=range_header, + range_get_content_md5=range_validation, + validate_content=self._validate_content, + data_stream_total=None, + download_stream_current=0, + **self._request_options + )) - except HttpResponseError as error: - if self._start_range is None and error.response and error.status_code == 416: - # Get range will fail on an empty file. If the user did not - # request a range, do a regular get request in order to get - # any properties. - try: - _, response = cast(Tuple[Optional[Any], Any], await self._clients.blob.download( - validate_content=self._validate_content, - data_stream_total=0, - download_stream_current=0, - **self._request_options)) - except HttpResponseError as e: - process_storage_error(e) - - # Set the download size to empty - self.size = 0 - self._file_size = 0 - else: - process_storage_error(error) + # Check the location we read from to ensure we use the same one + # for subsequent requests. + self._location_mode = location_mode + + # Parse the total file size and adjust the download size if ranges + # were specified + self._file_size = parse_length_from_content_range(response.properties.content_range) + if self._file_size is None: + raise ValueError("Required Content-Range response header is missing or malformed.") + # Remove any extra encryption data size from blob size + self._file_size = adjust_blob_size_for_encryption(self._file_size, self._encryption_data) + + if self._end_range is not None and self._start_range is not None: + # Use the length unless it is over the end of the file + self.size = min(self._file_size - self._start_range, self._end_range - self._start_range + 1) + elif self._start_range is not None: + self.size = self._file_size - self._start_range + else: + self.size = self._file_size - if self.size == 0: - self._current_content = b"" - else: - self._current_content = await process_content( - response, - self._initial_offset[0], - self._initial_offset[1], - self._encryption_options - ) + except HttpResponseError as error: + if self._start_range is None and error.response and error.status_code == 416: + # Get range will fail on an empty file. If the user did not + # request a range, do a regular get request in order to get + # any properties. + try: + _, response = cast(Tuple[Optional[Any], Any], await self._clients.blob.download( + validate_content=self._validate_content, + data_stream_total=0, + download_stream_current=0, + **self._request_options)) + except HttpResponseError as e: + process_storage_error(e) + + # Set the download size to empty + self.size = 0 + self._file_size = 0 + else: + process_storage_error(error) + + try: + if self.size == 0: + self._current_content = b"" + else: + self._current_content = await process_content( + response, + self._initial_offset[0], + self._initial_offset[1], + self._encryption_options + ) + retry_active = False + except (IncompleteReadError, HttpResponseError, DecodeError) as error: + retry_total -= 1 + if retry_total <= 0: + raise HttpResponseError(error, error=error) from error + await asyncio.sleep(1) self._download_offset += len(self._current_content) self._raw_download_offset += response.content_length diff --git a/sdk/storage/azure-storage-blob/tests/test_blob_encryption.py b/sdk/storage/azure-storage-blob/tests/test_blob_encryption.py index 600f28efc470..f7eeadde11de 100644 --- a/sdk/storage/azure-storage-blob/tests/test_blob_encryption.py +++ b/sdk/storage/azure-storage-blob/tests/test_blob_encryption.py @@ -174,7 +174,7 @@ def test_invalid_value_kek_unwrap(self, **kwargs): self._setup(storage_account_name, storage_account_key) self.bsc.require_encryption = True self.bsc.key_encryption_key = KeyWrapper('key1') - blob = self._create_small_blob(BlobType.BlockBlob) + blob = self._create_small_blob(BlobType.BLOCKBLOB) # Act blob.key_encryption_key = KeyWrapper('key1') @@ -250,7 +250,7 @@ def test_get_blob_nonmatching_kid(self, **kwargs): self._setup(storage_account_name, storage_account_key) self.bsc.require_encryption = True self.bsc.key_encryption_key = KeyWrapper('key1') - blob = self._create_small_blob(BlobType.BlockBlob) + blob = self._create_small_blob(BlobType.BLOCKBLOB) # Act self.bsc.key_encryption_key.kid = 'Invalid' diff --git a/sdk/storage/azure-storage-blob/tests/test_blob_encryption_async.py b/sdk/storage/azure-storage-blob/tests/test_blob_encryption_async.py index c155b9cd97fd..d715060ef6e8 100644 --- a/sdk/storage/azure-storage-blob/tests/test_blob_encryption_async.py +++ b/sdk/storage/azure-storage-blob/tests/test_blob_encryption_async.py @@ -178,7 +178,7 @@ async def test_invalid_value_kek_unwrap(self, **kwargs): await self._setup(storage_account_name, storage_account_key) self.bsc.require_encryption = True self.bsc.key_encryption_key = KeyWrapper('key1') - blob = await self._create_small_blob(BlobType.BlockBlob) + blob = await self._create_small_blob(BlobType.BLOCKBLOB) # Act blob.key_encryption_key = KeyWrapper('key1') @@ -257,7 +257,7 @@ async def test_get_blob_nonmatching_kid(self, **kwargs): await self._setup(storage_account_name, storage_account_key) self.bsc.require_encryption = True self.bsc.key_encryption_key = KeyWrapper('key1') - blob = await self._create_small_blob(BlobType.BlockBlob) + blob = await self._create_small_blob(BlobType.BLOCKBLOB) # Act self.bsc.key_encryption_key.kid = 'Invalid' diff --git a/sdk/storage/azure-storage-blob/tests/test_retry_async.py b/sdk/storage/azure-storage-blob/tests/test_retry_async.py index fe782be5c7be..8cf71b3393aa 100644 --- a/sdk/storage/azure-storage-blob/tests/test_retry_async.py +++ b/sdk/storage/azure-storage-blob/tests/test_retry_async.py @@ -8,11 +8,12 @@ import pytest from unittest import mock -from aiohttp.client_exceptions import ClientPayloadError, ServerTimeoutError +from aiohttp.client_exceptions import ServerTimeoutError from aiohttp.streams import StreamReader from azure.core.exceptions import ( AzureError, ClientAuthenticationError, + IncompleteReadError, HttpResponseError, ResourceExistsError, ServiceResponseError @@ -521,14 +522,14 @@ async def test_streaming_retry(self, **kwargs): stream_reader_read_mock = mock.MagicMock() future = asyncio.Future() - future.set_exception(ClientPayloadError()) + future.set_exception(IncompleteReadError()) stream_reader_read_mock.return_value = future with mock.patch.object(StreamReader, "read", stream_reader_read_mock), pytest.raises(HttpResponseError): blob = container.get_blob_client(blob=blob_name) count = [0] blob._pipeline._transport.send = self._count_wrapper(count, blob._pipeline._transport.send) await blob.download_blob() - assert stream_reader_read_mock.call_count == count[0] == 4 + assert stream_reader_read_mock.call_count == count[0] == 3 @BlobPreparer() async def test_invalid_storage_account_key(self, **kwargs): diff --git a/sdk/storage/azure-storage-file-datalake/azure/storage/filedatalake/_generated/_azure_data_lake_storage_restapi.py b/sdk/storage/azure-storage-file-datalake/azure/storage/filedatalake/_generated/_azure_data_lake_storage_restapi.py index b266181aabec..e1dec2bc80cd 100644 --- a/sdk/storage/azure-storage-file-datalake/azure/storage/filedatalake/_generated/_azure_data_lake_storage_restapi.py +++ b/sdk/storage/azure-storage-file-datalake/azure/storage/filedatalake/_generated/_azure_data_lake_storage_restapi.py @@ -8,6 +8,7 @@ from copy import deepcopy from typing import Any, Optional +from typing_extensions import Self from azure.core import PipelineClient from azure.core.pipeline import policies @@ -103,7 +104,7 @@ def _send_request(self, request: HttpRequest, *, stream: bool = False, **kwargs: def close(self) -> None: self._client.close() - def __enter__(self) -> "AzureDataLakeStorageRESTAPI": + def __enter__(self) -> Self: self._client.__enter__() return self diff --git a/sdk/storage/azure-storage-file-datalake/azure/storage/filedatalake/_generated/_serialization.py b/sdk/storage/azure-storage-file-datalake/azure/storage/filedatalake/_generated/_serialization.py index f0c6180722c8..8139854b97bb 100644 --- a/sdk/storage/azure-storage-file-datalake/azure/storage/filedatalake/_generated/_serialization.py +++ b/sdk/storage/azure-storage-file-datalake/azure/storage/filedatalake/_generated/_serialization.py @@ -144,6 +144,8 @@ def _json_attemp(data): # context otherwise. _LOGGER.critical("Wasn't XML not JSON, failing") raise DeserializationError("XML is invalid") from err + elif content_type.startswith("text/"): + return data_as_str raise DeserializationError("Cannot deserialize content-type: {}".format(content_type)) @classmethod diff --git a/sdk/storage/azure-storage-file-datalake/azure/storage/filedatalake/_generated/_vendor.py b/sdk/storage/azure-storage-file-datalake/azure/storage/filedatalake/_generated/_vendor.py deleted file mode 100644 index 0dafe0e287ff..000000000000 --- a/sdk/storage/azure-storage-file-datalake/azure/storage/filedatalake/_generated/_vendor.py +++ /dev/null @@ -1,16 +0,0 @@ -# -------------------------------------------------------------------------- -# Copyright (c) Microsoft Corporation. All rights reserved. -# Licensed under the MIT License. See License.txt in the project root for license information. -# Code generated by Microsoft (R) AutoRest Code Generator. -# Changes may cause incorrect behavior and will be lost if the code is regenerated. -# -------------------------------------------------------------------------- - -from azure.core.pipeline.transport import HttpRequest - - -def _convert_request(request, files=None): - data = request.content if not files else None - request = HttpRequest(method=request.method, url=request.url, headers=request.headers, data=data) - if files: - request.set_formdata_body(files) - return request diff --git a/sdk/storage/azure-storage-file-datalake/azure/storage/filedatalake/_generated/aio/_azure_data_lake_storage_restapi.py b/sdk/storage/azure-storage-file-datalake/azure/storage/filedatalake/_generated/aio/_azure_data_lake_storage_restapi.py index 74c4f0620bce..a16a5be74366 100644 --- a/sdk/storage/azure-storage-file-datalake/azure/storage/filedatalake/_generated/aio/_azure_data_lake_storage_restapi.py +++ b/sdk/storage/azure-storage-file-datalake/azure/storage/filedatalake/_generated/aio/_azure_data_lake_storage_restapi.py @@ -8,6 +8,7 @@ from copy import deepcopy from typing import Any, Awaitable, Optional +from typing_extensions import Self from azure.core import AsyncPipelineClient from azure.core.pipeline import policies @@ -105,7 +106,7 @@ def _send_request( async def close(self) -> None: await self._client.close() - async def __aenter__(self) -> "AzureDataLakeStorageRESTAPI": + async def __aenter__(self) -> Self: await self._client.__aenter__() return self diff --git a/sdk/storage/azure-storage-file-datalake/azure/storage/filedatalake/_generated/aio/operations/_file_system_operations.py b/sdk/storage/azure-storage-file-datalake/azure/storage/filedatalake/_generated/aio/operations/_file_system_operations.py index f5c11c835fb5..43db73b3c9b5 100644 --- a/sdk/storage/azure-storage-file-datalake/azure/storage/filedatalake/_generated/aio/operations/_file_system_operations.py +++ b/sdk/storage/azure-storage-file-datalake/azure/storage/filedatalake/_generated/aio/operations/_file_system_operations.py @@ -18,13 +18,11 @@ map_error, ) from azure.core.pipeline import PipelineResponse -from azure.core.pipeline.transport import AsyncHttpResponse -from azure.core.rest import HttpRequest +from azure.core.rest import AsyncHttpResponse, HttpRequest from azure.core.tracing.decorator_async import distributed_trace_async from azure.core.utils import case_insensitive_dict from ... import models as _models -from ..._vendor import _convert_request from ...operations._file_system_operations import ( build_create_request, build_delete_request, @@ -119,7 +117,6 @@ async def create( # pylint: disable=inconsistent-return-statements headers=_headers, params=_params, ) - _request = _convert_request(_request) _request.url = self._client.format_url(_request.url) _stream = False @@ -217,7 +214,6 @@ async def set_properties( # pylint: disable=inconsistent-return-statements headers=_headers, params=_params, ) - _request = _convert_request(_request) _request.url = self._client.format_url(_request.url) _stream = False @@ -285,7 +281,6 @@ async def get_properties( # pylint: disable=inconsistent-return-statements headers=_headers, params=_params, ) - _request = _convert_request(_request) _request.url = self._client.format_url(_request.url) _stream = False @@ -379,7 +374,6 @@ async def delete( # pylint: disable=inconsistent-return-statements headers=_headers, params=_params, ) - _request = _convert_request(_request) _request.url = self._client.format_url(_request.url) _stream = False @@ -480,7 +474,6 @@ async def list_paths( headers=_headers, params=_params, ) - _request = _convert_request(_request) _request.url = self._client.format_url(_request.url) _stream = False @@ -503,7 +496,7 @@ async def list_paths( response_headers["x-ms-version"] = self._deserialize("str", response.headers.get("x-ms-version")) response_headers["x-ms-continuation"] = self._deserialize("str", response.headers.get("x-ms-continuation")) - deserialized = self._deserialize("PathList", pipeline_response) + deserialized = self._deserialize("PathList", pipeline_response.http_response) if cls: return cls(pipeline_response, deserialized, response_headers) # type: ignore @@ -594,7 +587,6 @@ async def list_blob_hierarchy_segment( headers=_headers, params=_params, ) - _request = _convert_request(_request) _request.url = self._client.format_url(_request.url) _stream = False @@ -618,7 +610,7 @@ async def list_blob_hierarchy_segment( response_headers["x-ms-version"] = self._deserialize("str", response.headers.get("x-ms-version")) response_headers["Date"] = self._deserialize("rfc-1123", response.headers.get("Date")) - deserialized = self._deserialize("ListBlobsHierarchySegmentResponse", pipeline_response) + deserialized = self._deserialize("ListBlobsHierarchySegmentResponse", pipeline_response.http_response) if cls: return cls(pipeline_response, deserialized, response_headers) # type: ignore diff --git a/sdk/storage/azure-storage-file-datalake/azure/storage/filedatalake/_generated/aio/operations/_path_operations.py b/sdk/storage/azure-storage-file-datalake/azure/storage/filedatalake/_generated/aio/operations/_path_operations.py index 89ac80e699c6..ad9a7a7294ea 100644 --- a/sdk/storage/azure-storage-file-datalake/azure/storage/filedatalake/_generated/aio/operations/_path_operations.py +++ b/sdk/storage/azure-storage-file-datalake/azure/storage/filedatalake/_generated/aio/operations/_path_operations.py @@ -15,16 +15,16 @@ ResourceExistsError, ResourceNotFoundError, ResourceNotModifiedError, + StreamClosedError, + StreamConsumedError, map_error, ) from azure.core.pipeline import PipelineResponse -from azure.core.pipeline.transport import AsyncHttpResponse -from azure.core.rest import HttpRequest +from azure.core.rest import AsyncHttpResponse, HttpRequest from azure.core.tracing.decorator_async import distributed_trace_async from azure.core.utils import case_insensitive_dict from ... import models as _models -from ..._vendor import _convert_request from ...operations._path_operations import ( build_append_data_request, build_create_request, @@ -292,7 +292,6 @@ async def create( # pylint: disable=inconsistent-return-statements headers=_headers, params=_params, ) - _request = _convert_request(_request) _request.url = self._client.format_url(_request.url) _stream = False @@ -541,7 +540,6 @@ async def update( headers=_headers, params=_params, ) - _request = _convert_request(_request) _request.url = self._client.format_url(_request.url) _stream = False @@ -578,7 +576,7 @@ async def update( response_headers["x-ms-request-id"] = self._deserialize("str", response.headers.get("x-ms-request-id")) response_headers["x-ms-version"] = self._deserialize("str", response.headers.get("x-ms-version")) - deserialized = self._deserialize("SetAccessControlRecursiveResponse", pipeline_response) + deserialized = self._deserialize("SetAccessControlRecursiveResponse", pipeline_response.http_response) if response.status_code == 202: response_headers["Content-MD5"] = self._deserialize("str", response.headers.get("Content-MD5")) @@ -619,7 +617,7 @@ async def lease( # pylint: disable=inconsistent-return-statements the current lease ID in "x-ms-lease-id" and the new lease ID in "x-ms-proposed-lease-id" to change the lease ID of an active lease. Use "renew" and specify the "x-ms-lease-id" to renew an existing lease. Use "release" and specify the "x-ms-lease-id" to release a lease. Known values - are: "acquire", "break", "change", "renew", and "release". Required. + are: "acquire", "break", "change", "renew", "release", and "break". Required. :type x_ms_lease_action: str or ~azure.storage.filedatalake.models.PathLeaseAction :param request_id_parameter: Provides a client-generated, opaque value with a 1 KB character limit that is recorded in the analytics logs when storage analytics logging is enabled. Default @@ -689,7 +687,6 @@ async def lease( # pylint: disable=inconsistent-return-statements headers=_headers, params=_params, ) - _request = _convert_request(_request) _request.url = self._client.format_url(_request.url) _stream = False @@ -830,9 +827,9 @@ async def read( headers=_headers, params=_params, ) - _request = _convert_request(_request) _request.url = self._client.format_url(_request.url) + _decompress = kwargs.pop("decompress", True) _stream = True pipeline_response: PipelineResponse = await self._client._pipeline.run( # pylint: disable=protected-access _request, stream=_stream, **kwargs @@ -841,6 +838,10 @@ async def read( response = pipeline_response.http_response if response.status_code not in [200, 206]: + try: + await response.read() # Load the body in memory and close the socket + except (StreamConsumedError, StreamClosedError): + pass map_error(status_code=response.status_code, response=response, error_map=error_map) error = self._deserialize.failsafe_deserialize(_models.StorageError, pipeline_response) raise HttpResponseError(response=response, model=error) @@ -879,8 +880,6 @@ async def read( "str", response.headers.get("x-ms-encryption-key-sha256") ) - deserialized = response.stream_download(self._client._pipeline) - if response.status_code == 206: response_headers["Accept-Ranges"] = self._deserialize("str", response.headers.get("Accept-Ranges")) response_headers["Cache-Control"] = self._deserialize("str", response.headers.get("Cache-Control")) @@ -915,7 +914,7 @@ async def read( "str", response.headers.get("x-ms-encryption-key-sha256") ) - deserialized = response.stream_download(self._client._pipeline) + deserialized = response.stream_download(self._client._pipeline, decompress=_decompress) if cls: return cls(pipeline_response, deserialized, response_headers) # type: ignore @@ -1012,7 +1011,6 @@ async def get_properties( # pylint: disable=inconsistent-return-statements headers=_headers, params=_params, ) - _request = _convert_request(_request) _request.url = self._client.format_url(_request.url) _stream = False @@ -1146,7 +1144,6 @@ async def delete( # pylint: disable=inconsistent-return-statements headers=_headers, params=_params, ) - _request = _convert_request(_request) _request.url = self._client.format_url(_request.url) _stream = False @@ -1270,7 +1267,6 @@ async def set_access_control( # pylint: disable=inconsistent-return-statements headers=_headers, params=_params, ) - _request = _convert_request(_request) _request.url = self._client.format_url(_request.url) _stream = False @@ -1381,7 +1377,6 @@ async def set_access_control_recursive( headers=_headers, params=_params, ) - _request = _convert_request(_request) _request.url = self._client.format_url(_request.url) _stream = False @@ -1405,7 +1400,7 @@ async def set_access_control_recursive( response_headers["x-ms-request-id"] = self._deserialize("str", response.headers.get("x-ms-request-id")) response_headers["x-ms-version"] = self._deserialize("str", response.headers.get("x-ms-version")) - deserialized = self._deserialize("SetAccessControlRecursiveResponse", pipeline_response) + deserialized = self._deserialize("SetAccessControlRecursiveResponse", pipeline_response.http_response) if cls: return cls(pipeline_response, deserialized, response_headers) # type: ignore @@ -1573,7 +1568,6 @@ async def flush_data( # pylint: disable=inconsistent-return-statements headers=_headers, params=_params, ) - _request = _convert_request(_request) _request.url = self._client.format_url(_request.url) _stream = False @@ -1734,7 +1728,6 @@ async def append_data( # pylint: disable=inconsistent-return-statements headers=_headers, params=_params, ) - _request = _convert_request(_request) _request.url = self._client.format_url(_request.url) _stream = False @@ -1826,7 +1819,6 @@ async def set_expiry( # pylint: disable=inconsistent-return-statements headers=_headers, params=_params, ) - _request = _convert_request(_request) _request.url = self._client.format_url(_request.url) _stream = False @@ -1904,7 +1896,6 @@ async def undelete( # pylint: disable=inconsistent-return-statements headers=_headers, params=_params, ) - _request = _convert_request(_request) _request.url = self._client.format_url(_request.url) _stream = False diff --git a/sdk/storage/azure-storage-file-datalake/azure/storage/filedatalake/_generated/aio/operations/_service_operations.py b/sdk/storage/azure-storage-file-datalake/azure/storage/filedatalake/_generated/aio/operations/_service_operations.py index 57032bf8b7dc..a6da031a1483 100644 --- a/sdk/storage/azure-storage-file-datalake/azure/storage/filedatalake/_generated/aio/operations/_service_operations.py +++ b/sdk/storage/azure-storage-file-datalake/azure/storage/filedatalake/_generated/aio/operations/_service_operations.py @@ -19,13 +19,11 @@ map_error, ) from azure.core.pipeline import PipelineResponse -from azure.core.pipeline.transport import AsyncHttpResponse -from azure.core.rest import HttpRequest +from azure.core.rest import AsyncHttpResponse, HttpRequest from azure.core.tracing.decorator import distributed_trace from azure.core.utils import case_insensitive_dict from ... import models as _models -from ..._vendor import _convert_request from ...operations._service_operations import build_list_file_systems_request if sys.version_info >= (3, 9): @@ -124,12 +122,10 @@ def prepare_request(next_link=None): headers=_headers, params=_params, ) - _request = _convert_request(_request) _request.url = self._client.format_url(_request.url) else: _request = HttpRequest("GET", next_link) - _request = _convert_request(_request) _request.url = self._client.format_url(_request.url) _request.method = "GET" return _request diff --git a/sdk/storage/azure-storage-file-datalake/azure/storage/filedatalake/_generated/models/_azure_data_lake_storage_restapi_enums.py b/sdk/storage/azure-storage-file-datalake/azure/storage/filedatalake/_generated/models/_azure_data_lake_storage_restapi_enums.py index acaa735596d2..c9bb43b5e4a0 100644 --- a/sdk/storage/azure-storage-file-datalake/azure/storage/filedatalake/_generated/models/_azure_data_lake_storage_restapi_enums.py +++ b/sdk/storage/azure-storage-file-datalake/azure/storage/filedatalake/_generated/models/_azure_data_lake_storage_restapi_enums.py @@ -51,10 +51,11 @@ class PathLeaseAction(str, Enum, metaclass=CaseInsensitiveEnumMeta): """PathLeaseAction.""" ACQUIRE = "acquire" - BREAK_ENUM = "break" + BREAK = "break" CHANGE = "change" RENEW = "renew" RELEASE = "release" + BREAK_ENUM = "break" class PathRenameMode(str, Enum, metaclass=CaseInsensitiveEnumMeta): diff --git a/sdk/storage/azure-storage-file-datalake/azure/storage/filedatalake/_generated/operations/_file_system_operations.py b/sdk/storage/azure-storage-file-datalake/azure/storage/filedatalake/_generated/operations/_file_system_operations.py index 69efc186e612..35dbe9f3738b 100644 --- a/sdk/storage/azure-storage-file-datalake/azure/storage/filedatalake/_generated/operations/_file_system_operations.py +++ b/sdk/storage/azure-storage-file-datalake/azure/storage/filedatalake/_generated/operations/_file_system_operations.py @@ -19,14 +19,12 @@ map_error, ) from azure.core.pipeline import PipelineResponse -from azure.core.pipeline.transport import HttpResponse -from azure.core.rest import HttpRequest +from azure.core.rest import HttpRequest, HttpResponse from azure.core.tracing.decorator import distributed_trace from azure.core.utils import case_insensitive_dict from .. import models as _models from .._serialization import Serializer -from .._vendor import _convert_request if sys.version_info >= (3, 9): from collections.abc import MutableMapping @@ -380,7 +378,6 @@ def create( # pylint: disable=inconsistent-return-statements headers=_headers, params=_params, ) - _request = _convert_request(_request) _request.url = self._client.format_url(_request.url) _stream = False @@ -478,7 +475,6 @@ def set_properties( # pylint: disable=inconsistent-return-statements headers=_headers, params=_params, ) - _request = _convert_request(_request) _request.url = self._client.format_url(_request.url) _stream = False @@ -546,7 +542,6 @@ def get_properties( # pylint: disable=inconsistent-return-statements headers=_headers, params=_params, ) - _request = _convert_request(_request) _request.url = self._client.format_url(_request.url) _stream = False @@ -640,7 +635,6 @@ def delete( # pylint: disable=inconsistent-return-statements headers=_headers, params=_params, ) - _request = _convert_request(_request) _request.url = self._client.format_url(_request.url) _stream = False @@ -741,7 +735,6 @@ def list_paths( headers=_headers, params=_params, ) - _request = _convert_request(_request) _request.url = self._client.format_url(_request.url) _stream = False @@ -764,7 +757,7 @@ def list_paths( response_headers["x-ms-version"] = self._deserialize("str", response.headers.get("x-ms-version")) response_headers["x-ms-continuation"] = self._deserialize("str", response.headers.get("x-ms-continuation")) - deserialized = self._deserialize("PathList", pipeline_response) + deserialized = self._deserialize("PathList", pipeline_response.http_response) if cls: return cls(pipeline_response, deserialized, response_headers) # type: ignore @@ -855,7 +848,6 @@ def list_blob_hierarchy_segment( headers=_headers, params=_params, ) - _request = _convert_request(_request) _request.url = self._client.format_url(_request.url) _stream = False @@ -879,7 +871,7 @@ def list_blob_hierarchy_segment( response_headers["x-ms-version"] = self._deserialize("str", response.headers.get("x-ms-version")) response_headers["Date"] = self._deserialize("rfc-1123", response.headers.get("Date")) - deserialized = self._deserialize("ListBlobsHierarchySegmentResponse", pipeline_response) + deserialized = self._deserialize("ListBlobsHierarchySegmentResponse", pipeline_response.http_response) if cls: return cls(pipeline_response, deserialized, response_headers) # type: ignore diff --git a/sdk/storage/azure-storage-file-datalake/azure/storage/filedatalake/_generated/operations/_path_operations.py b/sdk/storage/azure-storage-file-datalake/azure/storage/filedatalake/_generated/operations/_path_operations.py index e3d158497721..eb2d08acfa37 100644 --- a/sdk/storage/azure-storage-file-datalake/azure/storage/filedatalake/_generated/operations/_path_operations.py +++ b/sdk/storage/azure-storage-file-datalake/azure/storage/filedatalake/_generated/operations/_path_operations.py @@ -16,17 +16,17 @@ ResourceExistsError, ResourceNotFoundError, ResourceNotModifiedError, + StreamClosedError, + StreamConsumedError, map_error, ) from azure.core.pipeline import PipelineResponse -from azure.core.pipeline.transport import HttpResponse -from azure.core.rest import HttpRequest +from azure.core.rest import HttpRequest, HttpResponse from azure.core.tracing.decorator import distributed_trace from azure.core.utils import case_insensitive_dict from .. import models as _models from .._serialization import Serializer -from .._vendor import _convert_request if sys.version_info >= (3, 9): from collections.abc import MutableMapping @@ -1154,7 +1154,6 @@ def create( # pylint: disable=inconsistent-return-statements headers=_headers, params=_params, ) - _request = _convert_request(_request) _request.url = self._client.format_url(_request.url) _stream = False @@ -1403,7 +1402,6 @@ def update( headers=_headers, params=_params, ) - _request = _convert_request(_request) _request.url = self._client.format_url(_request.url) _stream = False @@ -1440,7 +1438,7 @@ def update( response_headers["x-ms-request-id"] = self._deserialize("str", response.headers.get("x-ms-request-id")) response_headers["x-ms-version"] = self._deserialize("str", response.headers.get("x-ms-version")) - deserialized = self._deserialize("SetAccessControlRecursiveResponse", pipeline_response) + deserialized = self._deserialize("SetAccessControlRecursiveResponse", pipeline_response.http_response) if response.status_code == 202: response_headers["Content-MD5"] = self._deserialize("str", response.headers.get("Content-MD5")) @@ -1481,7 +1479,7 @@ def lease( # pylint: disable=inconsistent-return-statements the current lease ID in "x-ms-lease-id" and the new lease ID in "x-ms-proposed-lease-id" to change the lease ID of an active lease. Use "renew" and specify the "x-ms-lease-id" to renew an existing lease. Use "release" and specify the "x-ms-lease-id" to release a lease. Known values - are: "acquire", "break", "change", "renew", and "release". Required. + are: "acquire", "break", "change", "renew", "release", and "break". Required. :type x_ms_lease_action: str or ~azure.storage.filedatalake.models.PathLeaseAction :param request_id_parameter: Provides a client-generated, opaque value with a 1 KB character limit that is recorded in the analytics logs when storage analytics logging is enabled. Default @@ -1551,7 +1549,6 @@ def lease( # pylint: disable=inconsistent-return-statements headers=_headers, params=_params, ) - _request = _convert_request(_request) _request.url = self._client.format_url(_request.url) _stream = False @@ -1692,9 +1689,9 @@ def read( headers=_headers, params=_params, ) - _request = _convert_request(_request) _request.url = self._client.format_url(_request.url) + _decompress = kwargs.pop("decompress", True) _stream = True pipeline_response: PipelineResponse = self._client._pipeline.run( # pylint: disable=protected-access _request, stream=_stream, **kwargs @@ -1703,6 +1700,10 @@ def read( response = pipeline_response.http_response if response.status_code not in [200, 206]: + try: + response.read() # Load the body in memory and close the socket + except (StreamConsumedError, StreamClosedError): + pass map_error(status_code=response.status_code, response=response, error_map=error_map) error = self._deserialize.failsafe_deserialize(_models.StorageError, pipeline_response) raise HttpResponseError(response=response, model=error) @@ -1741,8 +1742,6 @@ def read( "str", response.headers.get("x-ms-encryption-key-sha256") ) - deserialized = response.stream_download(self._client._pipeline) - if response.status_code == 206: response_headers["Accept-Ranges"] = self._deserialize("str", response.headers.get("Accept-Ranges")) response_headers["Cache-Control"] = self._deserialize("str", response.headers.get("Cache-Control")) @@ -1777,7 +1776,7 @@ def read( "str", response.headers.get("x-ms-encryption-key-sha256") ) - deserialized = response.stream_download(self._client._pipeline) + deserialized = response.stream_download(self._client._pipeline, decompress=_decompress) if cls: return cls(pipeline_response, deserialized, response_headers) # type: ignore @@ -1874,7 +1873,6 @@ def get_properties( # pylint: disable=inconsistent-return-statements headers=_headers, params=_params, ) - _request = _convert_request(_request) _request.url = self._client.format_url(_request.url) _stream = False @@ -2008,7 +2006,6 @@ def delete( # pylint: disable=inconsistent-return-statements headers=_headers, params=_params, ) - _request = _convert_request(_request) _request.url = self._client.format_url(_request.url) _stream = False @@ -2132,7 +2129,6 @@ def set_access_control( # pylint: disable=inconsistent-return-statements headers=_headers, params=_params, ) - _request = _convert_request(_request) _request.url = self._client.format_url(_request.url) _stream = False @@ -2243,7 +2239,6 @@ def set_access_control_recursive( headers=_headers, params=_params, ) - _request = _convert_request(_request) _request.url = self._client.format_url(_request.url) _stream = False @@ -2267,7 +2262,7 @@ def set_access_control_recursive( response_headers["x-ms-request-id"] = self._deserialize("str", response.headers.get("x-ms-request-id")) response_headers["x-ms-version"] = self._deserialize("str", response.headers.get("x-ms-version")) - deserialized = self._deserialize("SetAccessControlRecursiveResponse", pipeline_response) + deserialized = self._deserialize("SetAccessControlRecursiveResponse", pipeline_response.http_response) if cls: return cls(pipeline_response, deserialized, response_headers) # type: ignore @@ -2435,7 +2430,6 @@ def flush_data( # pylint: disable=inconsistent-return-statements headers=_headers, params=_params, ) - _request = _convert_request(_request) _request.url = self._client.format_url(_request.url) _stream = False @@ -2596,7 +2590,6 @@ def append_data( # pylint: disable=inconsistent-return-statements headers=_headers, params=_params, ) - _request = _convert_request(_request) _request.url = self._client.format_url(_request.url) _stream = False @@ -2688,7 +2681,6 @@ def set_expiry( # pylint: disable=inconsistent-return-statements headers=_headers, params=_params, ) - _request = _convert_request(_request) _request.url = self._client.format_url(_request.url) _stream = False @@ -2766,7 +2758,6 @@ def undelete( # pylint: disable=inconsistent-return-statements headers=_headers, params=_params, ) - _request = _convert_request(_request) _request.url = self._client.format_url(_request.url) _stream = False diff --git a/sdk/storage/azure-storage-file-datalake/azure/storage/filedatalake/_generated/operations/_service_operations.py b/sdk/storage/azure-storage-file-datalake/azure/storage/filedatalake/_generated/operations/_service_operations.py index 5c3b01d7576c..e9bb654ff33a 100644 --- a/sdk/storage/azure-storage-file-datalake/azure/storage/filedatalake/_generated/operations/_service_operations.py +++ b/sdk/storage/azure-storage-file-datalake/azure/storage/filedatalake/_generated/operations/_service_operations.py @@ -19,14 +19,12 @@ ) from azure.core.paging import ItemPaged from azure.core.pipeline import PipelineResponse -from azure.core.pipeline.transport import HttpResponse -from azure.core.rest import HttpRequest +from azure.core.rest import HttpRequest, HttpResponse from azure.core.tracing.decorator import distributed_trace from azure.core.utils import case_insensitive_dict from .. import models as _models from .._serialization import Serializer -from .._vendor import _convert_request if sys.version_info >= (3, 9): from collections.abc import MutableMapping @@ -172,12 +170,10 @@ def prepare_request(next_link=None): headers=_headers, params=_params, ) - _request = _convert_request(_request) _request.url = self._client.format_url(_request.url) else: _request = HttpRequest("GET", next_link) - _request = _convert_request(_request) _request.url = self._client.format_url(_request.url) _request.method = "GET" return _request diff --git a/sdk/storage/azure-storage-file-datalake/azure/storage/filedatalake/_shared/policies.py b/sdk/storage/azure-storage-file-datalake/azure/storage/filedatalake/_shared/policies.py index 09600f0bafb1..6e07fae56e70 100644 --- a/sdk/storage/azure-storage-file-datalake/azure/storage/filedatalake/_shared/policies.py +++ b/sdk/storage/azure-storage-file-datalake/azure/storage/filedatalake/_shared/policies.py @@ -35,11 +35,6 @@ from .constants import DEFAULT_OAUTH_SCOPE from .models import LocationMode -try: - _unicode_type = unicode # type: ignore -except NameError: - _unicode_type = str - if TYPE_CHECKING: from azure.core.credentials import TokenCredential from azure.core.pipeline import ( # pylint: disable=non-abstract-transport-import @@ -52,7 +47,7 @@ def encode_base64(data): - if isinstance(data, _unicode_type): + if isinstance(data, str): data = data.encode('utf-8') encoded = base64.b64encode(data) return encoded.decode('utf-8') @@ -95,10 +90,14 @@ def is_retry(response, mode): # pylint: disable=too-many-return-statements if status in [501, 505]: return False return True + return False + + +def is_checksum_retry(response): # retry if invalid content md5 if response.context.get('validate_content', False) and response.http_response.headers.get('content-md5'): computed_md5 = response.http_request.headers.get('content-md5', None) or \ - encode_base64(StorageContentValidation.get_content_md5(response.http_response.body())) + encode_base64(StorageContentValidation.get_content_md5(response.http_response.body())) if response.http_response.headers['content-md5'] != computed_md5: return True return False @@ -301,7 +300,7 @@ def send(self, request: "PipelineRequest") -> "PipelineResponse": response = self.next.send(request) - will_retry = is_retry(response, request.context.options.get('mode')) + will_retry = is_retry(response, request.context.options.get('mode')) or is_checksum_retry(response) # Auth error could come from Bearer challenge, in which case this request will be made again is_auth_error = response.http_response.status_code == 401 should_update_counts = not (will_retry or is_auth_error) @@ -529,7 +528,7 @@ def send(self, request): while retries_remaining: try: response = self.next.send(request) - if is_retry(response, retry_settings['mode']): + if is_retry(response, retry_settings['mode']) or is_checksum_retry(response): retries_remaining = self.increment( retry_settings, request=request.http_request, diff --git a/sdk/storage/azure-storage-file-datalake/azure/storage/filedatalake/_shared/policies_async.py b/sdk/storage/azure-storage-file-datalake/azure/storage/filedatalake/_shared/policies_async.py index ffb9c2f4afd3..66faaf7ce894 100644 --- a/sdk/storage/azure-storage-file-datalake/azure/storage/filedatalake/_shared/policies_async.py +++ b/sdk/storage/azure-storage-file-datalake/azure/storage/filedatalake/_shared/policies_async.py @@ -10,12 +10,12 @@ import random from typing import Any, Dict, TYPE_CHECKING -from azure.core.exceptions import AzureError +from azure.core.exceptions import AzureError, StreamClosedError, StreamConsumedError from azure.core.pipeline.policies import AsyncBearerTokenCredentialPolicy, AsyncHTTPPolicy from .authentication import AzureSigningError, StorageHttpChallenge from .constants import DEFAULT_OAUTH_SCOPE -from .policies import is_retry, StorageRetryPolicy +from .policies import encode_base64, is_retry, StorageContentValidation, StorageRetryPolicy if TYPE_CHECKING: from azure.core.credentials_async import AsyncTokenCredential @@ -42,6 +42,20 @@ async def retry_hook(settings, **kwargs): **kwargs) +async def is_checksum_retry(response): + # retry if invalid content md5 + if response.context.get('validate_content', False) and response.http_response.headers.get('content-md5'): + try: + await response.http_response.read() # Load the body in memory and close the socket + except (StreamClosedError, StreamConsumedError): + pass + computed_md5 = response.http_request.headers.get('content-md5', None) or \ + encode_base64(StorageContentValidation.get_content_md5(response.http_response.content)) + if response.http_response.headers['content-md5'] != computed_md5: + return True + return False + + class AsyncStorageResponseHook(AsyncHTTPPolicy): def __init__(self, **kwargs): # pylint: disable=unused-argument @@ -64,9 +78,8 @@ async def send(self, request: "PipelineRequest") -> "PipelineResponse": request.context.options.pop('raw_response_hook', self._response_callback) response = await self.next.send(request) - await response.http_response.load_body() - will_retry = is_retry(response, request.context.options.get('mode')) + will_retry = is_retry(response, request.context.options.get('mode')) or await is_checksum_retry(response) # Auth error could come from Bearer challenge, in which case this request will be made again is_auth_error = response.http_response.status_code == 401 should_update_counts = not (will_retry or is_auth_error) @@ -112,7 +125,7 @@ async def send(self, request): while retries_remaining: try: response = await self.next.send(request) - if is_retry(response, retry_settings['mode']): + if is_retry(response, retry_settings['mode']) or await is_checksum_retry(response): retries_remaining = self.increment( retry_settings, request=request.http_request, diff --git a/sdk/storage/azure-storage-file-share/azure/storage/fileshare/_generated/_azure_file_storage.py b/sdk/storage/azure-storage-file-share/azure/storage/fileshare/_generated/_azure_file_storage.py index f8ac46283536..66a4dbd98ceb 100644 --- a/sdk/storage/azure-storage-file-share/azure/storage/fileshare/_generated/_azure_file_storage.py +++ b/sdk/storage/azure-storage-file-share/azure/storage/fileshare/_generated/_azure_file_storage.py @@ -8,6 +8,7 @@ from copy import deepcopy from typing import Any, Optional, Union +from typing_extensions import Self from azure.core import PipelineClient from azure.core.pipeline import policies @@ -121,7 +122,7 @@ def _send_request(self, request: HttpRequest, *, stream: bool = False, **kwargs: def close(self) -> None: self._client.close() - def __enter__(self) -> "AzureFileStorage": + def __enter__(self) -> Self: self._client.__enter__() return self diff --git a/sdk/storage/azure-storage-file-share/azure/storage/fileshare/_generated/_serialization.py b/sdk/storage/azure-storage-file-share/azure/storage/fileshare/_generated/_serialization.py index f0c6180722c8..8139854b97bb 100644 --- a/sdk/storage/azure-storage-file-share/azure/storage/fileshare/_generated/_serialization.py +++ b/sdk/storage/azure-storage-file-share/azure/storage/fileshare/_generated/_serialization.py @@ -144,6 +144,8 @@ def _json_attemp(data): # context otherwise. _LOGGER.critical("Wasn't XML not JSON, failing") raise DeserializationError("XML is invalid") from err + elif content_type.startswith("text/"): + return data_as_str raise DeserializationError("Cannot deserialize content-type: {}".format(content_type)) @classmethod diff --git a/sdk/storage/azure-storage-file-share/azure/storage/fileshare/_generated/_vendor.py b/sdk/storage/azure-storage-file-share/azure/storage/fileshare/_generated/_vendor.py deleted file mode 100644 index 0dafe0e287ff..000000000000 --- a/sdk/storage/azure-storage-file-share/azure/storage/fileshare/_generated/_vendor.py +++ /dev/null @@ -1,16 +0,0 @@ -# -------------------------------------------------------------------------- -# Copyright (c) Microsoft Corporation. All rights reserved. -# Licensed under the MIT License. See License.txt in the project root for license information. -# Code generated by Microsoft (R) AutoRest Code Generator. -# Changes may cause incorrect behavior and will be lost if the code is regenerated. -# -------------------------------------------------------------------------- - -from azure.core.pipeline.transport import HttpRequest - - -def _convert_request(request, files=None): - data = request.content if not files else None - request = HttpRequest(method=request.method, url=request.url, headers=request.headers, data=data) - if files: - request.set_formdata_body(files) - return request diff --git a/sdk/storage/azure-storage-file-share/azure/storage/fileshare/_generated/aio/_azure_file_storage.py b/sdk/storage/azure-storage-file-share/azure/storage/fileshare/_generated/aio/_azure_file_storage.py index 8dd1300ab7ff..94784b9052bc 100644 --- a/sdk/storage/azure-storage-file-share/azure/storage/fileshare/_generated/aio/_azure_file_storage.py +++ b/sdk/storage/azure-storage-file-share/azure/storage/fileshare/_generated/aio/_azure_file_storage.py @@ -8,6 +8,7 @@ from copy import deepcopy from typing import Any, Awaitable, Optional, Union +from typing_extensions import Self from azure.core import AsyncPipelineClient from azure.core.pipeline import policies @@ -123,7 +124,7 @@ def _send_request( async def close(self) -> None: await self._client.close() - async def __aenter__(self) -> "AzureFileStorage": + async def __aenter__(self) -> Self: await self._client.__aenter__() return self diff --git a/sdk/storage/azure-storage-file-share/azure/storage/fileshare/_generated/aio/operations/_directory_operations.py b/sdk/storage/azure-storage-file-share/azure/storage/fileshare/_generated/aio/operations/_directory_operations.py index 538d7f3b0d97..80d6896d2f11 100644 --- a/sdk/storage/azure-storage-file-share/azure/storage/fileshare/_generated/aio/operations/_directory_operations.py +++ b/sdk/storage/azure-storage-file-share/azure/storage/fileshare/_generated/aio/operations/_directory_operations.py @@ -18,13 +18,11 @@ map_error, ) from azure.core.pipeline import PipelineResponse -from azure.core.pipeline.transport import AsyncHttpResponse -from azure.core.rest import HttpRequest +from azure.core.rest import AsyncHttpResponse, HttpRequest from azure.core.tracing.decorator_async import distributed_trace_async from azure.core.utils import case_insensitive_dict from ... import models as _models -from ..._vendor import _convert_request from ...operations._directory_operations import ( build_create_request, build_delete_request, @@ -154,7 +152,6 @@ async def create( # pylint: disable=inconsistent-return-statements headers=_headers, params=_params, ) - _request = _convert_request(_request) _request.url = self._client.format_url(_request.url) _stream = False @@ -244,7 +241,6 @@ async def get_properties( # pylint: disable=inconsistent-return-statements headers=_headers, params=_params, ) - _request = _convert_request(_request) _request.url = self._client.format_url(_request.url) _stream = False @@ -330,7 +326,6 @@ async def delete( # pylint: disable=inconsistent-return-statements headers=_headers, params=_params, ) - _request = _convert_request(_request) _request.url = self._client.format_url(_request.url) _stream = False @@ -440,7 +435,6 @@ async def set_properties( # pylint: disable=inconsistent-return-statements headers=_headers, params=_params, ) - _request = _convert_request(_request) _request.url = self._client.format_url(_request.url) _stream = False @@ -530,7 +524,6 @@ async def set_metadata( # pylint: disable=inconsistent-return-statements headers=_headers, params=_params, ) - _request = _convert_request(_request) _request.url = self._client.format_url(_request.url) _stream = False @@ -633,7 +626,6 @@ async def list_files_and_directories_segment( headers=_headers, params=_params, ) - _request = _convert_request(_request) _request.url = self._client.format_url(_request.url) _stream = False @@ -654,7 +646,7 @@ async def list_files_and_directories_segment( response_headers["x-ms-version"] = self._deserialize("str", response.headers.get("x-ms-version")) response_headers["Date"] = self._deserialize("rfc-1123", response.headers.get("Date")) - deserialized = self._deserialize("ListFilesAndDirectoriesSegmentResponse", pipeline_response) + deserialized = self._deserialize("ListFilesAndDirectoriesSegmentResponse", pipeline_response.http_response) if cls: return cls(pipeline_response, deserialized, response_headers) # type: ignore @@ -725,7 +717,6 @@ async def list_handles( headers=_headers, params=_params, ) - _request = _convert_request(_request) _request.url = self._client.format_url(_request.url) _stream = False @@ -746,7 +737,7 @@ async def list_handles( response_headers["x-ms-version"] = self._deserialize("str", response.headers.get("x-ms-version")) response_headers["Date"] = self._deserialize("rfc-1123", response.headers.get("Date")) - deserialized = self._deserialize("ListHandlesResponse", pipeline_response) + deserialized = self._deserialize("ListHandlesResponse", pipeline_response.http_response) if cls: return cls(pipeline_response, deserialized, response_headers) # type: ignore @@ -816,7 +807,6 @@ async def force_close_handles( # pylint: disable=inconsistent-return-statements headers=_headers, params=_params, ) - _request = _convert_request(_request) _request.url = self._client.format_url(_request.url) _stream = False @@ -972,7 +962,6 @@ async def rename( # pylint: disable=inconsistent-return-statements headers=_headers, params=_params, ) - _request = _convert_request(_request) _request.url = self._client.format_url(_request.url) _stream = False diff --git a/sdk/storage/azure-storage-file-share/azure/storage/fileshare/_generated/aio/operations/_file_operations.py b/sdk/storage/azure-storage-file-share/azure/storage/fileshare/_generated/aio/operations/_file_operations.py index 2be57bdc00d6..35bc561a6c07 100644 --- a/sdk/storage/azure-storage-file-share/azure/storage/fileshare/_generated/aio/operations/_file_operations.py +++ b/sdk/storage/azure-storage-file-share/azure/storage/fileshare/_generated/aio/operations/_file_operations.py @@ -15,16 +15,16 @@ ResourceExistsError, ResourceNotFoundError, ResourceNotModifiedError, + StreamClosedError, + StreamConsumedError, map_error, ) from azure.core.pipeline import PipelineResponse -from azure.core.pipeline.transport import AsyncHttpResponse -from azure.core.rest import HttpRequest +from azure.core.rest import AsyncHttpResponse, HttpRequest from azure.core.tracing.decorator_async import distributed_trace_async from azure.core.utils import case_insensitive_dict from ... import models as _models -from ..._vendor import _convert_request from ...operations._file_operations import ( build_abort_copy_request, build_acquire_lease_request, @@ -197,7 +197,6 @@ async def create( # pylint: disable=inconsistent-return-statements headers=_headers, params=_params, ) - _request = _convert_request(_request) _request.url = self._client.format_url(_request.url) _stream = False @@ -299,9 +298,9 @@ async def download( headers=_headers, params=_params, ) - _request = _convert_request(_request) _request.url = self._client.format_url(_request.url) + _decompress = kwargs.pop("decompress", True) _stream = True pipeline_response: PipelineResponse = await self._client._pipeline.run( # pylint: disable=protected-access _request, stream=_stream, **kwargs @@ -310,136 +309,66 @@ async def download( response = pipeline_response.http_response if response.status_code not in [200, 206]: + try: + await response.read() # Load the body in memory and close the socket + except (StreamConsumedError, StreamClosedError): + pass map_error(status_code=response.status_code, response=response, error_map=error_map) error = self._deserialize.failsafe_deserialize(_models.StorageError, pipeline_response) raise HttpResponseError(response=response, model=error) response_headers = {} - if response.status_code == 200: - response_headers["Last-Modified"] = self._deserialize("rfc-1123", response.headers.get("Last-Modified")) - response_headers["x-ms-meta"] = self._deserialize("{str}", response.headers.get("x-ms-meta")) - response_headers["Content-Length"] = self._deserialize("int", response.headers.get("Content-Length")) - response_headers["Content-Type"] = self._deserialize("str", response.headers.get("Content-Type")) - response_headers["Content-Range"] = self._deserialize("str", response.headers.get("Content-Range")) - response_headers["ETag"] = self._deserialize("str", response.headers.get("ETag")) - response_headers["Content-MD5"] = self._deserialize("bytearray", response.headers.get("Content-MD5")) - response_headers["Content-Encoding"] = self._deserialize("str", response.headers.get("Content-Encoding")) - response_headers["Cache-Control"] = self._deserialize("str", response.headers.get("Cache-Control")) - response_headers["Content-Disposition"] = self._deserialize( - "str", response.headers.get("Content-Disposition") - ) - response_headers["Content-Language"] = self._deserialize("str", response.headers.get("Content-Language")) - response_headers["x-ms-request-id"] = self._deserialize("str", response.headers.get("x-ms-request-id")) - response_headers["x-ms-version"] = self._deserialize("str", response.headers.get("x-ms-version")) - response_headers["Accept-Ranges"] = self._deserialize("str", response.headers.get("Accept-Ranges")) - response_headers["Date"] = self._deserialize("rfc-1123", response.headers.get("Date")) - response_headers["x-ms-copy-completion-time"] = self._deserialize( - "rfc-1123", response.headers.get("x-ms-copy-completion-time") - ) - response_headers["x-ms-copy-status-description"] = self._deserialize( - "str", response.headers.get("x-ms-copy-status-description") - ) - response_headers["x-ms-copy-id"] = self._deserialize("str", response.headers.get("x-ms-copy-id")) - response_headers["x-ms-copy-progress"] = self._deserialize( - "str", response.headers.get("x-ms-copy-progress") - ) - response_headers["x-ms-copy-source"] = self._deserialize("str", response.headers.get("x-ms-copy-source")) - response_headers["x-ms-copy-status"] = self._deserialize("str", response.headers.get("x-ms-copy-status")) - response_headers["x-ms-content-md5"] = self._deserialize( - "bytearray", response.headers.get("x-ms-content-md5") - ) - response_headers["x-ms-server-encrypted"] = self._deserialize( - "bool", response.headers.get("x-ms-server-encrypted") - ) - response_headers["x-ms-file-attributes"] = self._deserialize( - "str", response.headers.get("x-ms-file-attributes") - ) - response_headers["x-ms-file-creation-time"] = self._deserialize( - "str", response.headers.get("x-ms-file-creation-time") - ) - response_headers["x-ms-file-last-write-time"] = self._deserialize( - "str", response.headers.get("x-ms-file-last-write-time") - ) - response_headers["x-ms-file-change-time"] = self._deserialize( - "str", response.headers.get("x-ms-file-change-time") - ) - response_headers["x-ms-file-permission-key"] = self._deserialize( - "str", response.headers.get("x-ms-file-permission-key") - ) - response_headers["x-ms-file-id"] = self._deserialize("str", response.headers.get("x-ms-file-id")) - response_headers["x-ms-file-parent-id"] = self._deserialize( - "str", response.headers.get("x-ms-file-parent-id") - ) - response_headers["x-ms-lease-duration"] = self._deserialize( - "str", response.headers.get("x-ms-lease-duration") - ) - response_headers["x-ms-lease-state"] = self._deserialize("str", response.headers.get("x-ms-lease-state")) - response_headers["x-ms-lease-status"] = self._deserialize("str", response.headers.get("x-ms-lease-status")) - - deserialized = response.stream_download(self._client._pipeline) - - if response.status_code == 206: - response_headers["Last-Modified"] = self._deserialize("rfc-1123", response.headers.get("Last-Modified")) - response_headers["x-ms-meta"] = self._deserialize("{str}", response.headers.get("x-ms-meta")) - response_headers["Content-Length"] = self._deserialize("int", response.headers.get("Content-Length")) - response_headers["Content-Type"] = self._deserialize("str", response.headers.get("Content-Type")) - response_headers["Content-Range"] = self._deserialize("str", response.headers.get("Content-Range")) - response_headers["ETag"] = self._deserialize("str", response.headers.get("ETag")) - response_headers["Content-MD5"] = self._deserialize("bytearray", response.headers.get("Content-MD5")) - response_headers["Content-Encoding"] = self._deserialize("str", response.headers.get("Content-Encoding")) - response_headers["Cache-Control"] = self._deserialize("str", response.headers.get("Cache-Control")) - response_headers["Content-Disposition"] = self._deserialize( - "str", response.headers.get("Content-Disposition") - ) - response_headers["Content-Language"] = self._deserialize("str", response.headers.get("Content-Language")) - response_headers["x-ms-request-id"] = self._deserialize("str", response.headers.get("x-ms-request-id")) - response_headers["x-ms-version"] = self._deserialize("str", response.headers.get("x-ms-version")) - response_headers["Accept-Ranges"] = self._deserialize("str", response.headers.get("Accept-Ranges")) - response_headers["Date"] = self._deserialize("rfc-1123", response.headers.get("Date")) - response_headers["x-ms-copy-completion-time"] = self._deserialize( - "rfc-1123", response.headers.get("x-ms-copy-completion-time") - ) - response_headers["x-ms-copy-status-description"] = self._deserialize( - "str", response.headers.get("x-ms-copy-status-description") - ) - response_headers["x-ms-copy-id"] = self._deserialize("str", response.headers.get("x-ms-copy-id")) - response_headers["x-ms-copy-progress"] = self._deserialize( - "str", response.headers.get("x-ms-copy-progress") - ) - response_headers["x-ms-copy-source"] = self._deserialize("str", response.headers.get("x-ms-copy-source")) - response_headers["x-ms-copy-status"] = self._deserialize("str", response.headers.get("x-ms-copy-status")) - response_headers["x-ms-content-md5"] = self._deserialize( - "bytearray", response.headers.get("x-ms-content-md5") - ) - response_headers["x-ms-server-encrypted"] = self._deserialize( - "bool", response.headers.get("x-ms-server-encrypted") - ) - response_headers["x-ms-file-attributes"] = self._deserialize( - "str", response.headers.get("x-ms-file-attributes") - ) - response_headers["x-ms-file-creation-time"] = self._deserialize( - "str", response.headers.get("x-ms-file-creation-time") - ) - response_headers["x-ms-file-last-write-time"] = self._deserialize( - "str", response.headers.get("x-ms-file-last-write-time") - ) - response_headers["x-ms-file-change-time"] = self._deserialize( - "str", response.headers.get("x-ms-file-change-time") - ) - response_headers["x-ms-file-permission-key"] = self._deserialize( - "str", response.headers.get("x-ms-file-permission-key") - ) - response_headers["x-ms-file-id"] = self._deserialize("str", response.headers.get("x-ms-file-id")) - response_headers["x-ms-file-parent-id"] = self._deserialize( - "str", response.headers.get("x-ms-file-parent-id") - ) - response_headers["x-ms-lease-duration"] = self._deserialize( - "str", response.headers.get("x-ms-lease-duration") - ) - response_headers["x-ms-lease-state"] = self._deserialize("str", response.headers.get("x-ms-lease-state")) - response_headers["x-ms-lease-status"] = self._deserialize("str", response.headers.get("x-ms-lease-status")) - - deserialized = response.stream_download(self._client._pipeline) + response_headers["Last-Modified"] = self._deserialize("rfc-1123", response.headers.get("Last-Modified")) + response_headers["x-ms-meta"] = self._deserialize("{str}", response.headers.get("x-ms-meta")) + response_headers["Content-Length"] = self._deserialize("int", response.headers.get("Content-Length")) + response_headers["Content-Type"] = self._deserialize("str", response.headers.get("Content-Type")) + response_headers["Content-Range"] = self._deserialize("str", response.headers.get("Content-Range")) + response_headers["ETag"] = self._deserialize("str", response.headers.get("ETag")) + response_headers["Content-MD5"] = self._deserialize("bytearray", response.headers.get("Content-MD5")) + response_headers["Content-Encoding"] = self._deserialize("str", response.headers.get("Content-Encoding")) + response_headers["Cache-Control"] = self._deserialize("str", response.headers.get("Cache-Control")) + response_headers["Content-Disposition"] = self._deserialize("str", response.headers.get("Content-Disposition")) + response_headers["Content-Language"] = self._deserialize("str", response.headers.get("Content-Language")) + response_headers["x-ms-request-id"] = self._deserialize("str", response.headers.get("x-ms-request-id")) + response_headers["x-ms-version"] = self._deserialize("str", response.headers.get("x-ms-version")) + response_headers["Accept-Ranges"] = self._deserialize("str", response.headers.get("Accept-Ranges")) + response_headers["Date"] = self._deserialize("rfc-1123", response.headers.get("Date")) + response_headers["x-ms-copy-completion-time"] = self._deserialize( + "rfc-1123", response.headers.get("x-ms-copy-completion-time") + ) + response_headers["x-ms-copy-status-description"] = self._deserialize( + "str", response.headers.get("x-ms-copy-status-description") + ) + response_headers["x-ms-copy-id"] = self._deserialize("str", response.headers.get("x-ms-copy-id")) + response_headers["x-ms-copy-progress"] = self._deserialize("str", response.headers.get("x-ms-copy-progress")) + response_headers["x-ms-copy-source"] = self._deserialize("str", response.headers.get("x-ms-copy-source")) + response_headers["x-ms-copy-status"] = self._deserialize("str", response.headers.get("x-ms-copy-status")) + response_headers["x-ms-content-md5"] = self._deserialize("bytearray", response.headers.get("x-ms-content-md5")) + response_headers["x-ms-server-encrypted"] = self._deserialize( + "bool", response.headers.get("x-ms-server-encrypted") + ) + response_headers["x-ms-file-attributes"] = self._deserialize( + "str", response.headers.get("x-ms-file-attributes") + ) + response_headers["x-ms-file-creation-time"] = self._deserialize( + "str", response.headers.get("x-ms-file-creation-time") + ) + response_headers["x-ms-file-last-write-time"] = self._deserialize( + "str", response.headers.get("x-ms-file-last-write-time") + ) + response_headers["x-ms-file-change-time"] = self._deserialize( + "str", response.headers.get("x-ms-file-change-time") + ) + response_headers["x-ms-file-permission-key"] = self._deserialize( + "str", response.headers.get("x-ms-file-permission-key") + ) + response_headers["x-ms-file-id"] = self._deserialize("str", response.headers.get("x-ms-file-id")) + response_headers["x-ms-file-parent-id"] = self._deserialize("str", response.headers.get("x-ms-file-parent-id")) + response_headers["x-ms-lease-duration"] = self._deserialize("str", response.headers.get("x-ms-lease-duration")) + response_headers["x-ms-lease-state"] = self._deserialize("str", response.headers.get("x-ms-lease-state")) + response_headers["x-ms-lease-status"] = self._deserialize("str", response.headers.get("x-ms-lease-status")) + + deserialized = response.stream_download(self._client._pipeline, decompress=_decompress) if cls: return cls(pipeline_response, deserialized, response_headers) # type: ignore @@ -499,7 +428,6 @@ async def get_properties( # pylint: disable=inconsistent-return-statements headers=_headers, params=_params, ) - _request = _convert_request(_request) _request.url = self._client.format_url(_request.url) _stream = False @@ -613,7 +541,6 @@ async def delete( # pylint: disable=inconsistent-return-statements headers=_headers, params=_params, ) - _request = _convert_request(_request) _request.url = self._client.format_url(_request.url) _stream = False @@ -757,7 +684,6 @@ async def set_http_headers( # pylint: disable=inconsistent-return-statements headers=_headers, params=_params, ) - _request = _convert_request(_request) _request.url = self._client.format_url(_request.url) _stream = False @@ -856,7 +782,6 @@ async def set_metadata( # pylint: disable=inconsistent-return-statements headers=_headers, params=_params, ) - _request = _convert_request(_request) _request.url = self._client.format_url(_request.url) _stream = False @@ -945,7 +870,6 @@ async def acquire_lease( # pylint: disable=inconsistent-return-statements headers=_headers, params=_params, ) - _request = _convert_request(_request) _request.url = self._client.format_url(_request.url) _stream = False @@ -1024,7 +948,6 @@ async def release_lease( # pylint: disable=inconsistent-return-statements headers=_headers, params=_params, ) - _request = _convert_request(_request) _request.url = self._client.format_url(_request.url) _stream = False @@ -1112,7 +1035,6 @@ async def change_lease( # pylint: disable=inconsistent-return-statements headers=_headers, params=_params, ) - _request = _convert_request(_request) _request.url = self._client.format_url(_request.url) _stream = False @@ -1199,7 +1121,6 @@ async def break_lease( # pylint: disable=inconsistent-return-statements headers=_headers, params=_params, ) - _request = _convert_request(_request) _request.url = self._client.format_url(_request.url) _stream = False @@ -1319,7 +1240,6 @@ async def upload_range( # pylint: disable=inconsistent-return-statements headers=_headers, params=_params, ) - _request = _convert_request(_request) _request.url = self._client.format_url(_request.url) _stream = False @@ -1452,7 +1372,6 @@ async def upload_range_from_url( # pylint: disable=inconsistent-return-statemen headers=_headers, params=_params, ) - _request = _convert_request(_request) _request.url = self._client.format_url(_request.url) _stream = False @@ -1559,7 +1478,6 @@ async def get_range_list( headers=_headers, params=_params, ) - _request = _convert_request(_request) _request.url = self._client.format_url(_request.url) _stream = False @@ -1582,7 +1500,7 @@ async def get_range_list( response_headers["x-ms-version"] = self._deserialize("str", response.headers.get("x-ms-version")) response_headers["Date"] = self._deserialize("rfc-1123", response.headers.get("Date")) - deserialized = self._deserialize("ShareFileRangeList", pipeline_response) + deserialized = self._deserialize("ShareFileRangeList", pipeline_response.http_response) if cls: return cls(pipeline_response, deserialized, response_headers) # type: ignore @@ -1691,7 +1609,6 @@ async def start_copy( # pylint: disable=inconsistent-return-statements headers=_headers, params=_params, ) - _request = _convert_request(_request) _request.url = self._client.format_url(_request.url) _stream = False @@ -1777,7 +1694,6 @@ async def abort_copy( # pylint: disable=inconsistent-return-statements headers=_headers, params=_params, ) - _request = _convert_request(_request) _request.url = self._client.format_url(_request.url) _stream = False @@ -1859,7 +1775,6 @@ async def list_handles( headers=_headers, params=_params, ) - _request = _convert_request(_request) _request.url = self._client.format_url(_request.url) _stream = False @@ -1880,7 +1795,7 @@ async def list_handles( response_headers["x-ms-version"] = self._deserialize("str", response.headers.get("x-ms-version")) response_headers["Date"] = self._deserialize("rfc-1123", response.headers.get("Date")) - deserialized = self._deserialize("ListHandlesResponse", pipeline_response) + deserialized = self._deserialize("ListHandlesResponse", pipeline_response.http_response) if cls: return cls(pipeline_response, deserialized, response_headers) # type: ignore @@ -1945,7 +1860,6 @@ async def force_close_handles( # pylint: disable=inconsistent-return-statements headers=_headers, params=_params, ) - _request = _convert_request(_request) _request.url = self._client.format_url(_request.url) _stream = False @@ -2106,7 +2020,6 @@ async def rename( # pylint: disable=inconsistent-return-statements headers=_headers, params=_params, ) - _request = _convert_request(_request) _request.url = self._client.format_url(_request.url) _stream = False diff --git a/sdk/storage/azure-storage-file-share/azure/storage/fileshare/_generated/aio/operations/_service_operations.py b/sdk/storage/azure-storage-file-share/azure/storage/fileshare/_generated/aio/operations/_service_operations.py index e6cc22728183..b393d21685ba 100644 --- a/sdk/storage/azure-storage-file-share/azure/storage/fileshare/_generated/aio/operations/_service_operations.py +++ b/sdk/storage/azure-storage-file-share/azure/storage/fileshare/_generated/aio/operations/_service_operations.py @@ -18,13 +18,11 @@ map_error, ) from azure.core.pipeline import PipelineResponse -from azure.core.pipeline.transport import AsyncHttpResponse -from azure.core.rest import HttpRequest +from azure.core.rest import AsyncHttpResponse, HttpRequest from azure.core.tracing.decorator_async import distributed_trace_async from azure.core.utils import case_insensitive_dict from ... import models as _models -from ..._vendor import _convert_request from ...operations._service_operations import ( build_get_properties_request, build_list_shares_segment_request, @@ -106,7 +104,6 @@ async def set_properties( # pylint: disable=inconsistent-return-statements headers=_headers, params=_params, ) - _request = _convert_request(_request) _request.url = self._client.format_url(_request.url) _stream = False @@ -167,7 +164,6 @@ async def get_properties(self, timeout: Optional[int] = None, **kwargs: Any) -> headers=_headers, params=_params, ) - _request = _convert_request(_request) _request.url = self._client.format_url(_request.url) _stream = False @@ -186,7 +182,7 @@ async def get_properties(self, timeout: Optional[int] = None, **kwargs: Any) -> response_headers["x-ms-request-id"] = self._deserialize("str", response.headers.get("x-ms-request-id")) response_headers["x-ms-version"] = self._deserialize("str", response.headers.get("x-ms-version")) - deserialized = self._deserialize("StorageServiceProperties", pipeline_response) + deserialized = self._deserialize("StorageServiceProperties", pipeline_response.http_response) if cls: return cls(pipeline_response, deserialized, response_headers) # type: ignore @@ -257,7 +253,6 @@ async def list_shares_segment( headers=_headers, params=_params, ) - _request = _convert_request(_request) _request.url = self._client.format_url(_request.url) _stream = False @@ -276,7 +271,7 @@ async def list_shares_segment( response_headers["x-ms-request-id"] = self._deserialize("str", response.headers.get("x-ms-request-id")) response_headers["x-ms-version"] = self._deserialize("str", response.headers.get("x-ms-version")) - deserialized = self._deserialize("ListSharesResponse", pipeline_response) + deserialized = self._deserialize("ListSharesResponse", pipeline_response.http_response) if cls: return cls(pipeline_response, deserialized, response_headers) # type: ignore diff --git a/sdk/storage/azure-storage-file-share/azure/storage/fileshare/_generated/aio/operations/_share_operations.py b/sdk/storage/azure-storage-file-share/azure/storage/fileshare/_generated/aio/operations/_share_operations.py index 4540e926b6c3..46d2c38171cc 100644 --- a/sdk/storage/azure-storage-file-share/azure/storage/fileshare/_generated/aio/operations/_share_operations.py +++ b/sdk/storage/azure-storage-file-share/azure/storage/fileshare/_generated/aio/operations/_share_operations.py @@ -19,13 +19,11 @@ map_error, ) from azure.core.pipeline import PipelineResponse -from azure.core.pipeline.transport import AsyncHttpResponse -from azure.core.rest import HttpRequest +from azure.core.rest import AsyncHttpResponse, HttpRequest from azure.core.tracing.decorator_async import distributed_trace_async from azure.core.utils import case_insensitive_dict from ... import models as _models -from ..._vendor import _convert_request from ...operations._share_operations import ( build_acquire_lease_request, build_break_lease_request, @@ -158,7 +156,6 @@ async def create( # pylint: disable=inconsistent-return-statements headers=_headers, params=_params, ) - _request = _convert_request(_request) _request.url = self._client.format_url(_request.url) _stream = False @@ -237,7 +234,6 @@ async def get_properties( # pylint: disable=inconsistent-return-statements headers=_headers, params=_params, ) - _request = _convert_request(_request) _request.url = self._client.format_url(_request.url) _stream = False @@ -364,7 +360,6 @@ async def delete( # pylint: disable=inconsistent-return-statements headers=_headers, params=_params, ) - _request = _convert_request(_request) _request.url = self._client.format_url(_request.url) _stream = False @@ -455,7 +450,6 @@ async def acquire_lease( # pylint: disable=inconsistent-return-statements headers=_headers, params=_params, ) - _request = _convert_request(_request) _request.url = self._client.format_url(_request.url) _stream = False @@ -544,7 +538,6 @@ async def release_lease( # pylint: disable=inconsistent-return-statements headers=_headers, params=_params, ) - _request = _convert_request(_request) _request.url = self._client.format_url(_request.url) _stream = False @@ -638,7 +631,6 @@ async def change_lease( # pylint: disable=inconsistent-return-statements headers=_headers, params=_params, ) - _request = _convert_request(_request) _request.url = self._client.format_url(_request.url) _stream = False @@ -727,7 +719,6 @@ async def renew_lease( # pylint: disable=inconsistent-return-statements headers=_headers, params=_params, ) - _request = _convert_request(_request) _request.url = self._client.format_url(_request.url) _stream = False @@ -830,7 +821,6 @@ async def break_lease( # pylint: disable=inconsistent-return-statements headers=_headers, params=_params, ) - _request = _convert_request(_request) _request.url = self._client.format_url(_request.url) _stream = False @@ -904,7 +894,6 @@ async def create_snapshot( # pylint: disable=inconsistent-return-statements headers=_headers, params=_params, ) - _request = _convert_request(_request) _request.url = self._client.format_url(_request.url) _stream = False @@ -1037,7 +1026,6 @@ async def create_permission( # pylint: disable=inconsistent-return-statements headers=_headers, params=_params, ) - _request = _convert_request(_request) _request.url = self._client.format_url(_request.url) _stream = False @@ -1118,7 +1106,6 @@ async def get_permission( headers=_headers, params=_params, ) - _request = _convert_request(_request) _request.url = self._client.format_url(_request.url) _stream = False @@ -1138,7 +1125,7 @@ async def get_permission( response_headers["x-ms-version"] = self._deserialize("str", response.headers.get("x-ms-version")) response_headers["Date"] = self._deserialize("rfc-1123", response.headers.get("Date")) - deserialized = self._deserialize("SharePermission", pipeline_response) + deserialized = self._deserialize("SharePermission", pipeline_response.http_response) if cls: return cls(pipeline_response, deserialized, response_headers) # type: ignore @@ -1230,7 +1217,6 @@ async def set_properties( # pylint: disable=inconsistent-return-statements headers=_headers, params=_params, ) - _request = _convert_request(_request) _request.url = self._client.format_url(_request.url) _stream = False @@ -1310,7 +1296,6 @@ async def set_metadata( # pylint: disable=inconsistent-return-statements headers=_headers, params=_params, ) - _request = _convert_request(_request) _request.url = self._client.format_url(_request.url) _stream = False @@ -1385,7 +1370,6 @@ async def get_access_policy( headers=_headers, params=_params, ) - _request = _convert_request(_request) _request.url = self._client.format_url(_request.url) _stream = False @@ -1407,7 +1391,7 @@ async def get_access_policy( response_headers["x-ms-version"] = self._deserialize("str", response.headers.get("x-ms-version")) response_headers["Date"] = self._deserialize("rfc-1123", response.headers.get("Date")) - deserialized = self._deserialize("[SignedIdentifier]", pipeline_response) + deserialized = self._deserialize("[SignedIdentifier]", pipeline_response.http_response) if cls: return cls(pipeline_response, deserialized, response_headers) # type: ignore @@ -1477,7 +1461,6 @@ async def set_access_policy( # pylint: disable=inconsistent-return-statements headers=_headers, params=_params, ) - _request = _convert_request(_request) _request.url = self._client.format_url(_request.url) _stream = False @@ -1552,7 +1535,6 @@ async def get_statistics( headers=_headers, params=_params, ) - _request = _convert_request(_request) _request.url = self._client.format_url(_request.url) _stream = False @@ -1574,7 +1556,7 @@ async def get_statistics( response_headers["x-ms-version"] = self._deserialize("str", response.headers.get("x-ms-version")) response_headers["Date"] = self._deserialize("rfc-1123", response.headers.get("Date")) - deserialized = self._deserialize("ShareStats", pipeline_response) + deserialized = self._deserialize("ShareStats", pipeline_response.http_response) if cls: return cls(pipeline_response, deserialized, response_headers) # type: ignore @@ -1639,7 +1621,6 @@ async def restore( # pylint: disable=inconsistent-return-statements headers=_headers, params=_params, ) - _request = _convert_request(_request) _request.url = self._client.format_url(_request.url) _stream = False diff --git a/sdk/storage/azure-storage-file-share/azure/storage/fileshare/_generated/operations/_directory_operations.py b/sdk/storage/azure-storage-file-share/azure/storage/fileshare/_generated/operations/_directory_operations.py index 070a3c027837..98a8faa7d456 100644 --- a/sdk/storage/azure-storage-file-share/azure/storage/fileshare/_generated/operations/_directory_operations.py +++ b/sdk/storage/azure-storage-file-share/azure/storage/fileshare/_generated/operations/_directory_operations.py @@ -18,14 +18,12 @@ map_error, ) from azure.core.pipeline import PipelineResponse -from azure.core.pipeline.transport import HttpResponse -from azure.core.rest import HttpRequest +from azure.core.rest import HttpRequest, HttpResponse from azure.core.tracing.decorator import distributed_trace from azure.core.utils import case_insensitive_dict from .. import models as _models from .._serialization import Serializer -from .._vendor import _convert_request if sys.version_info >= (3, 9): from collections.abc import MutableMapping @@ -647,7 +645,6 @@ def create( # pylint: disable=inconsistent-return-statements headers=_headers, params=_params, ) - _request = _convert_request(_request) _request.url = self._client.format_url(_request.url) _stream = False @@ -737,7 +734,6 @@ def get_properties( # pylint: disable=inconsistent-return-statements headers=_headers, params=_params, ) - _request = _convert_request(_request) _request.url = self._client.format_url(_request.url) _stream = False @@ -823,7 +819,6 @@ def delete( # pylint: disable=inconsistent-return-statements headers=_headers, params=_params, ) - _request = _convert_request(_request) _request.url = self._client.format_url(_request.url) _stream = False @@ -933,7 +928,6 @@ def set_properties( # pylint: disable=inconsistent-return-statements headers=_headers, params=_params, ) - _request = _convert_request(_request) _request.url = self._client.format_url(_request.url) _stream = False @@ -1023,7 +1017,6 @@ def set_metadata( # pylint: disable=inconsistent-return-statements headers=_headers, params=_params, ) - _request = _convert_request(_request) _request.url = self._client.format_url(_request.url) _stream = False @@ -1126,7 +1119,6 @@ def list_files_and_directories_segment( headers=_headers, params=_params, ) - _request = _convert_request(_request) _request.url = self._client.format_url(_request.url) _stream = False @@ -1147,7 +1139,7 @@ def list_files_and_directories_segment( response_headers["x-ms-version"] = self._deserialize("str", response.headers.get("x-ms-version")) response_headers["Date"] = self._deserialize("rfc-1123", response.headers.get("Date")) - deserialized = self._deserialize("ListFilesAndDirectoriesSegmentResponse", pipeline_response) + deserialized = self._deserialize("ListFilesAndDirectoriesSegmentResponse", pipeline_response.http_response) if cls: return cls(pipeline_response, deserialized, response_headers) # type: ignore @@ -1218,7 +1210,6 @@ def list_handles( headers=_headers, params=_params, ) - _request = _convert_request(_request) _request.url = self._client.format_url(_request.url) _stream = False @@ -1239,7 +1230,7 @@ def list_handles( response_headers["x-ms-version"] = self._deserialize("str", response.headers.get("x-ms-version")) response_headers["Date"] = self._deserialize("rfc-1123", response.headers.get("Date")) - deserialized = self._deserialize("ListHandlesResponse", pipeline_response) + deserialized = self._deserialize("ListHandlesResponse", pipeline_response.http_response) if cls: return cls(pipeline_response, deserialized, response_headers) # type: ignore @@ -1309,7 +1300,6 @@ def force_close_handles( # pylint: disable=inconsistent-return-statements headers=_headers, params=_params, ) - _request = _convert_request(_request) _request.url = self._client.format_url(_request.url) _stream = False @@ -1465,7 +1455,6 @@ def rename( # pylint: disable=inconsistent-return-statements headers=_headers, params=_params, ) - _request = _convert_request(_request) _request.url = self._client.format_url(_request.url) _stream = False diff --git a/sdk/storage/azure-storage-file-share/azure/storage/fileshare/_generated/operations/_file_operations.py b/sdk/storage/azure-storage-file-share/azure/storage/fileshare/_generated/operations/_file_operations.py index 94a72d63c923..c07fbd141e66 100644 --- a/sdk/storage/azure-storage-file-share/azure/storage/fileshare/_generated/operations/_file_operations.py +++ b/sdk/storage/azure-storage-file-share/azure/storage/fileshare/_generated/operations/_file_operations.py @@ -15,17 +15,17 @@ ResourceExistsError, ResourceNotFoundError, ResourceNotModifiedError, + StreamClosedError, + StreamConsumedError, map_error, ) from azure.core.pipeline import PipelineResponse -from azure.core.pipeline.transport import HttpResponse -from azure.core.rest import HttpRequest +from azure.core.rest import HttpRequest, HttpResponse from azure.core.tracing.decorator import distributed_trace from azure.core.utils import case_insensitive_dict from .. import models as _models from .._serialization import Serializer -from .._vendor import _convert_request if sys.version_info >= (3, 9): from collections.abc import MutableMapping @@ -1237,7 +1237,6 @@ def create( # pylint: disable=inconsistent-return-statements headers=_headers, params=_params, ) - _request = _convert_request(_request) _request.url = self._client.format_url(_request.url) _stream = False @@ -1339,9 +1338,9 @@ def download( headers=_headers, params=_params, ) - _request = _convert_request(_request) _request.url = self._client.format_url(_request.url) + _decompress = kwargs.pop("decompress", True) _stream = True pipeline_response: PipelineResponse = self._client._pipeline.run( # pylint: disable=protected-access _request, stream=_stream, **kwargs @@ -1350,136 +1349,66 @@ def download( response = pipeline_response.http_response if response.status_code not in [200, 206]: + try: + response.read() # Load the body in memory and close the socket + except (StreamConsumedError, StreamClosedError): + pass map_error(status_code=response.status_code, response=response, error_map=error_map) error = self._deserialize.failsafe_deserialize(_models.StorageError, pipeline_response) raise HttpResponseError(response=response, model=error) response_headers = {} - if response.status_code == 200: - response_headers["Last-Modified"] = self._deserialize("rfc-1123", response.headers.get("Last-Modified")) - response_headers["x-ms-meta"] = self._deserialize("{str}", response.headers.get("x-ms-meta")) - response_headers["Content-Length"] = self._deserialize("int", response.headers.get("Content-Length")) - response_headers["Content-Type"] = self._deserialize("str", response.headers.get("Content-Type")) - response_headers["Content-Range"] = self._deserialize("str", response.headers.get("Content-Range")) - response_headers["ETag"] = self._deserialize("str", response.headers.get("ETag")) - response_headers["Content-MD5"] = self._deserialize("bytearray", response.headers.get("Content-MD5")) - response_headers["Content-Encoding"] = self._deserialize("str", response.headers.get("Content-Encoding")) - response_headers["Cache-Control"] = self._deserialize("str", response.headers.get("Cache-Control")) - response_headers["Content-Disposition"] = self._deserialize( - "str", response.headers.get("Content-Disposition") - ) - response_headers["Content-Language"] = self._deserialize("str", response.headers.get("Content-Language")) - response_headers["x-ms-request-id"] = self._deserialize("str", response.headers.get("x-ms-request-id")) - response_headers["x-ms-version"] = self._deserialize("str", response.headers.get("x-ms-version")) - response_headers["Accept-Ranges"] = self._deserialize("str", response.headers.get("Accept-Ranges")) - response_headers["Date"] = self._deserialize("rfc-1123", response.headers.get("Date")) - response_headers["x-ms-copy-completion-time"] = self._deserialize( - "rfc-1123", response.headers.get("x-ms-copy-completion-time") - ) - response_headers["x-ms-copy-status-description"] = self._deserialize( - "str", response.headers.get("x-ms-copy-status-description") - ) - response_headers["x-ms-copy-id"] = self._deserialize("str", response.headers.get("x-ms-copy-id")) - response_headers["x-ms-copy-progress"] = self._deserialize( - "str", response.headers.get("x-ms-copy-progress") - ) - response_headers["x-ms-copy-source"] = self._deserialize("str", response.headers.get("x-ms-copy-source")) - response_headers["x-ms-copy-status"] = self._deserialize("str", response.headers.get("x-ms-copy-status")) - response_headers["x-ms-content-md5"] = self._deserialize( - "bytearray", response.headers.get("x-ms-content-md5") - ) - response_headers["x-ms-server-encrypted"] = self._deserialize( - "bool", response.headers.get("x-ms-server-encrypted") - ) - response_headers["x-ms-file-attributes"] = self._deserialize( - "str", response.headers.get("x-ms-file-attributes") - ) - response_headers["x-ms-file-creation-time"] = self._deserialize( - "str", response.headers.get("x-ms-file-creation-time") - ) - response_headers["x-ms-file-last-write-time"] = self._deserialize( - "str", response.headers.get("x-ms-file-last-write-time") - ) - response_headers["x-ms-file-change-time"] = self._deserialize( - "str", response.headers.get("x-ms-file-change-time") - ) - response_headers["x-ms-file-permission-key"] = self._deserialize( - "str", response.headers.get("x-ms-file-permission-key") - ) - response_headers["x-ms-file-id"] = self._deserialize("str", response.headers.get("x-ms-file-id")) - response_headers["x-ms-file-parent-id"] = self._deserialize( - "str", response.headers.get("x-ms-file-parent-id") - ) - response_headers["x-ms-lease-duration"] = self._deserialize( - "str", response.headers.get("x-ms-lease-duration") - ) - response_headers["x-ms-lease-state"] = self._deserialize("str", response.headers.get("x-ms-lease-state")) - response_headers["x-ms-lease-status"] = self._deserialize("str", response.headers.get("x-ms-lease-status")) - - deserialized = response.stream_download(self._client._pipeline) - - if response.status_code == 206: - response_headers["Last-Modified"] = self._deserialize("rfc-1123", response.headers.get("Last-Modified")) - response_headers["x-ms-meta"] = self._deserialize("{str}", response.headers.get("x-ms-meta")) - response_headers["Content-Length"] = self._deserialize("int", response.headers.get("Content-Length")) - response_headers["Content-Type"] = self._deserialize("str", response.headers.get("Content-Type")) - response_headers["Content-Range"] = self._deserialize("str", response.headers.get("Content-Range")) - response_headers["ETag"] = self._deserialize("str", response.headers.get("ETag")) - response_headers["Content-MD5"] = self._deserialize("bytearray", response.headers.get("Content-MD5")) - response_headers["Content-Encoding"] = self._deserialize("str", response.headers.get("Content-Encoding")) - response_headers["Cache-Control"] = self._deserialize("str", response.headers.get("Cache-Control")) - response_headers["Content-Disposition"] = self._deserialize( - "str", response.headers.get("Content-Disposition") - ) - response_headers["Content-Language"] = self._deserialize("str", response.headers.get("Content-Language")) - response_headers["x-ms-request-id"] = self._deserialize("str", response.headers.get("x-ms-request-id")) - response_headers["x-ms-version"] = self._deserialize("str", response.headers.get("x-ms-version")) - response_headers["Accept-Ranges"] = self._deserialize("str", response.headers.get("Accept-Ranges")) - response_headers["Date"] = self._deserialize("rfc-1123", response.headers.get("Date")) - response_headers["x-ms-copy-completion-time"] = self._deserialize( - "rfc-1123", response.headers.get("x-ms-copy-completion-time") - ) - response_headers["x-ms-copy-status-description"] = self._deserialize( - "str", response.headers.get("x-ms-copy-status-description") - ) - response_headers["x-ms-copy-id"] = self._deserialize("str", response.headers.get("x-ms-copy-id")) - response_headers["x-ms-copy-progress"] = self._deserialize( - "str", response.headers.get("x-ms-copy-progress") - ) - response_headers["x-ms-copy-source"] = self._deserialize("str", response.headers.get("x-ms-copy-source")) - response_headers["x-ms-copy-status"] = self._deserialize("str", response.headers.get("x-ms-copy-status")) - response_headers["x-ms-content-md5"] = self._deserialize( - "bytearray", response.headers.get("x-ms-content-md5") - ) - response_headers["x-ms-server-encrypted"] = self._deserialize( - "bool", response.headers.get("x-ms-server-encrypted") - ) - response_headers["x-ms-file-attributes"] = self._deserialize( - "str", response.headers.get("x-ms-file-attributes") - ) - response_headers["x-ms-file-creation-time"] = self._deserialize( - "str", response.headers.get("x-ms-file-creation-time") - ) - response_headers["x-ms-file-last-write-time"] = self._deserialize( - "str", response.headers.get("x-ms-file-last-write-time") - ) - response_headers["x-ms-file-change-time"] = self._deserialize( - "str", response.headers.get("x-ms-file-change-time") - ) - response_headers["x-ms-file-permission-key"] = self._deserialize( - "str", response.headers.get("x-ms-file-permission-key") - ) - response_headers["x-ms-file-id"] = self._deserialize("str", response.headers.get("x-ms-file-id")) - response_headers["x-ms-file-parent-id"] = self._deserialize( - "str", response.headers.get("x-ms-file-parent-id") - ) - response_headers["x-ms-lease-duration"] = self._deserialize( - "str", response.headers.get("x-ms-lease-duration") - ) - response_headers["x-ms-lease-state"] = self._deserialize("str", response.headers.get("x-ms-lease-state")) - response_headers["x-ms-lease-status"] = self._deserialize("str", response.headers.get("x-ms-lease-status")) - - deserialized = response.stream_download(self._client._pipeline) + response_headers["Last-Modified"] = self._deserialize("rfc-1123", response.headers.get("Last-Modified")) + response_headers["x-ms-meta"] = self._deserialize("{str}", response.headers.get("x-ms-meta")) + response_headers["Content-Length"] = self._deserialize("int", response.headers.get("Content-Length")) + response_headers["Content-Type"] = self._deserialize("str", response.headers.get("Content-Type")) + response_headers["Content-Range"] = self._deserialize("str", response.headers.get("Content-Range")) + response_headers["ETag"] = self._deserialize("str", response.headers.get("ETag")) + response_headers["Content-MD5"] = self._deserialize("bytearray", response.headers.get("Content-MD5")) + response_headers["Content-Encoding"] = self._deserialize("str", response.headers.get("Content-Encoding")) + response_headers["Cache-Control"] = self._deserialize("str", response.headers.get("Cache-Control")) + response_headers["Content-Disposition"] = self._deserialize("str", response.headers.get("Content-Disposition")) + response_headers["Content-Language"] = self._deserialize("str", response.headers.get("Content-Language")) + response_headers["x-ms-request-id"] = self._deserialize("str", response.headers.get("x-ms-request-id")) + response_headers["x-ms-version"] = self._deserialize("str", response.headers.get("x-ms-version")) + response_headers["Accept-Ranges"] = self._deserialize("str", response.headers.get("Accept-Ranges")) + response_headers["Date"] = self._deserialize("rfc-1123", response.headers.get("Date")) + response_headers["x-ms-copy-completion-time"] = self._deserialize( + "rfc-1123", response.headers.get("x-ms-copy-completion-time") + ) + response_headers["x-ms-copy-status-description"] = self._deserialize( + "str", response.headers.get("x-ms-copy-status-description") + ) + response_headers["x-ms-copy-id"] = self._deserialize("str", response.headers.get("x-ms-copy-id")) + response_headers["x-ms-copy-progress"] = self._deserialize("str", response.headers.get("x-ms-copy-progress")) + response_headers["x-ms-copy-source"] = self._deserialize("str", response.headers.get("x-ms-copy-source")) + response_headers["x-ms-copy-status"] = self._deserialize("str", response.headers.get("x-ms-copy-status")) + response_headers["x-ms-content-md5"] = self._deserialize("bytearray", response.headers.get("x-ms-content-md5")) + response_headers["x-ms-server-encrypted"] = self._deserialize( + "bool", response.headers.get("x-ms-server-encrypted") + ) + response_headers["x-ms-file-attributes"] = self._deserialize( + "str", response.headers.get("x-ms-file-attributes") + ) + response_headers["x-ms-file-creation-time"] = self._deserialize( + "str", response.headers.get("x-ms-file-creation-time") + ) + response_headers["x-ms-file-last-write-time"] = self._deserialize( + "str", response.headers.get("x-ms-file-last-write-time") + ) + response_headers["x-ms-file-change-time"] = self._deserialize( + "str", response.headers.get("x-ms-file-change-time") + ) + response_headers["x-ms-file-permission-key"] = self._deserialize( + "str", response.headers.get("x-ms-file-permission-key") + ) + response_headers["x-ms-file-id"] = self._deserialize("str", response.headers.get("x-ms-file-id")) + response_headers["x-ms-file-parent-id"] = self._deserialize("str", response.headers.get("x-ms-file-parent-id")) + response_headers["x-ms-lease-duration"] = self._deserialize("str", response.headers.get("x-ms-lease-duration")) + response_headers["x-ms-lease-state"] = self._deserialize("str", response.headers.get("x-ms-lease-state")) + response_headers["x-ms-lease-status"] = self._deserialize("str", response.headers.get("x-ms-lease-status")) + + deserialized = response.stream_download(self._client._pipeline, decompress=_decompress) if cls: return cls(pipeline_response, deserialized, response_headers) # type: ignore @@ -1539,7 +1468,6 @@ def get_properties( # pylint: disable=inconsistent-return-statements headers=_headers, params=_params, ) - _request = _convert_request(_request) _request.url = self._client.format_url(_request.url) _stream = False @@ -1653,7 +1581,6 @@ def delete( # pylint: disable=inconsistent-return-statements headers=_headers, params=_params, ) - _request = _convert_request(_request) _request.url = self._client.format_url(_request.url) _stream = False @@ -1797,7 +1724,6 @@ def set_http_headers( # pylint: disable=inconsistent-return-statements headers=_headers, params=_params, ) - _request = _convert_request(_request) _request.url = self._client.format_url(_request.url) _stream = False @@ -1896,7 +1822,6 @@ def set_metadata( # pylint: disable=inconsistent-return-statements headers=_headers, params=_params, ) - _request = _convert_request(_request) _request.url = self._client.format_url(_request.url) _stream = False @@ -1985,7 +1910,6 @@ def acquire_lease( # pylint: disable=inconsistent-return-statements headers=_headers, params=_params, ) - _request = _convert_request(_request) _request.url = self._client.format_url(_request.url) _stream = False @@ -2064,7 +1988,6 @@ def release_lease( # pylint: disable=inconsistent-return-statements headers=_headers, params=_params, ) - _request = _convert_request(_request) _request.url = self._client.format_url(_request.url) _stream = False @@ -2152,7 +2075,6 @@ def change_lease( # pylint: disable=inconsistent-return-statements headers=_headers, params=_params, ) - _request = _convert_request(_request) _request.url = self._client.format_url(_request.url) _stream = False @@ -2239,7 +2161,6 @@ def break_lease( # pylint: disable=inconsistent-return-statements headers=_headers, params=_params, ) - _request = _convert_request(_request) _request.url = self._client.format_url(_request.url) _stream = False @@ -2359,7 +2280,6 @@ def upload_range( # pylint: disable=inconsistent-return-statements headers=_headers, params=_params, ) - _request = _convert_request(_request) _request.url = self._client.format_url(_request.url) _stream = False @@ -2492,7 +2412,6 @@ def upload_range_from_url( # pylint: disable=inconsistent-return-statements headers=_headers, params=_params, ) - _request = _convert_request(_request) _request.url = self._client.format_url(_request.url) _stream = False @@ -2599,7 +2518,6 @@ def get_range_list( headers=_headers, params=_params, ) - _request = _convert_request(_request) _request.url = self._client.format_url(_request.url) _stream = False @@ -2622,7 +2540,7 @@ def get_range_list( response_headers["x-ms-version"] = self._deserialize("str", response.headers.get("x-ms-version")) response_headers["Date"] = self._deserialize("rfc-1123", response.headers.get("Date")) - deserialized = self._deserialize("ShareFileRangeList", pipeline_response) + deserialized = self._deserialize("ShareFileRangeList", pipeline_response.http_response) if cls: return cls(pipeline_response, deserialized, response_headers) # type: ignore @@ -2731,7 +2649,6 @@ def start_copy( # pylint: disable=inconsistent-return-statements headers=_headers, params=_params, ) - _request = _convert_request(_request) _request.url = self._client.format_url(_request.url) _stream = False @@ -2817,7 +2734,6 @@ def abort_copy( # pylint: disable=inconsistent-return-statements headers=_headers, params=_params, ) - _request = _convert_request(_request) _request.url = self._client.format_url(_request.url) _stream = False @@ -2899,7 +2815,6 @@ def list_handles( headers=_headers, params=_params, ) - _request = _convert_request(_request) _request.url = self._client.format_url(_request.url) _stream = False @@ -2920,7 +2835,7 @@ def list_handles( response_headers["x-ms-version"] = self._deserialize("str", response.headers.get("x-ms-version")) response_headers["Date"] = self._deserialize("rfc-1123", response.headers.get("Date")) - deserialized = self._deserialize("ListHandlesResponse", pipeline_response) + deserialized = self._deserialize("ListHandlesResponse", pipeline_response.http_response) if cls: return cls(pipeline_response, deserialized, response_headers) # type: ignore @@ -2985,7 +2900,6 @@ def force_close_handles( # pylint: disable=inconsistent-return-statements headers=_headers, params=_params, ) - _request = _convert_request(_request) _request.url = self._client.format_url(_request.url) _stream = False @@ -3146,7 +3060,6 @@ def rename( # pylint: disable=inconsistent-return-statements headers=_headers, params=_params, ) - _request = _convert_request(_request) _request.url = self._client.format_url(_request.url) _stream = False diff --git a/sdk/storage/azure-storage-file-share/azure/storage/fileshare/_generated/operations/_service_operations.py b/sdk/storage/azure-storage-file-share/azure/storage/fileshare/_generated/operations/_service_operations.py index 217612094666..7ff801a48b09 100644 --- a/sdk/storage/azure-storage-file-share/azure/storage/fileshare/_generated/operations/_service_operations.py +++ b/sdk/storage/azure-storage-file-share/azure/storage/fileshare/_generated/operations/_service_operations.py @@ -18,14 +18,12 @@ map_error, ) from azure.core.pipeline import PipelineResponse -from azure.core.pipeline.transport import HttpResponse -from azure.core.rest import HttpRequest +from azure.core.rest import HttpRequest, HttpResponse from azure.core.tracing.decorator import distributed_trace from azure.core.utils import case_insensitive_dict from .. import models as _models from .._serialization import Serializer -from .._vendor import _convert_request if sys.version_info >= (3, 9): from collections.abc import MutableMapping @@ -233,7 +231,6 @@ def set_properties( # pylint: disable=inconsistent-return-statements headers=_headers, params=_params, ) - _request = _convert_request(_request) _request.url = self._client.format_url(_request.url) _stream = False @@ -294,7 +291,6 @@ def get_properties(self, timeout: Optional[int] = None, **kwargs: Any) -> _model headers=_headers, params=_params, ) - _request = _convert_request(_request) _request.url = self._client.format_url(_request.url) _stream = False @@ -313,7 +309,7 @@ def get_properties(self, timeout: Optional[int] = None, **kwargs: Any) -> _model response_headers["x-ms-request-id"] = self._deserialize("str", response.headers.get("x-ms-request-id")) response_headers["x-ms-version"] = self._deserialize("str", response.headers.get("x-ms-version")) - deserialized = self._deserialize("StorageServiceProperties", pipeline_response) + deserialized = self._deserialize("StorageServiceProperties", pipeline_response.http_response) if cls: return cls(pipeline_response, deserialized, response_headers) # type: ignore @@ -384,7 +380,6 @@ def list_shares_segment( headers=_headers, params=_params, ) - _request = _convert_request(_request) _request.url = self._client.format_url(_request.url) _stream = False @@ -403,7 +398,7 @@ def list_shares_segment( response_headers["x-ms-request-id"] = self._deserialize("str", response.headers.get("x-ms-request-id")) response_headers["x-ms-version"] = self._deserialize("str", response.headers.get("x-ms-version")) - deserialized = self._deserialize("ListSharesResponse", pipeline_response) + deserialized = self._deserialize("ListSharesResponse", pipeline_response.http_response) if cls: return cls(pipeline_response, deserialized, response_headers) # type: ignore diff --git a/sdk/storage/azure-storage-file-share/azure/storage/fileshare/_generated/operations/_share_operations.py b/sdk/storage/azure-storage-file-share/azure/storage/fileshare/_generated/operations/_share_operations.py index 025ca1df7d94..8639994b70b7 100644 --- a/sdk/storage/azure-storage-file-share/azure/storage/fileshare/_generated/operations/_share_operations.py +++ b/sdk/storage/azure-storage-file-share/azure/storage/fileshare/_generated/operations/_share_operations.py @@ -19,14 +19,12 @@ map_error, ) from azure.core.pipeline import PipelineResponse -from azure.core.pipeline.transport import HttpResponse -from azure.core.rest import HttpRequest +from azure.core.rest import HttpRequest, HttpResponse from azure.core.tracing.decorator import distributed_trace from azure.core.utils import case_insensitive_dict from .. import models as _models from .._serialization import Serializer -from .._vendor import _convert_request if sys.version_info >= (3, 9): from collections.abc import MutableMapping @@ -969,7 +967,6 @@ def create( # pylint: disable=inconsistent-return-statements headers=_headers, params=_params, ) - _request = _convert_request(_request) _request.url = self._client.format_url(_request.url) _stream = False @@ -1048,7 +1045,6 @@ def get_properties( # pylint: disable=inconsistent-return-statements headers=_headers, params=_params, ) - _request = _convert_request(_request) _request.url = self._client.format_url(_request.url) _stream = False @@ -1175,7 +1171,6 @@ def delete( # pylint: disable=inconsistent-return-statements headers=_headers, params=_params, ) - _request = _convert_request(_request) _request.url = self._client.format_url(_request.url) _stream = False @@ -1266,7 +1261,6 @@ def acquire_lease( # pylint: disable=inconsistent-return-statements headers=_headers, params=_params, ) - _request = _convert_request(_request) _request.url = self._client.format_url(_request.url) _stream = False @@ -1355,7 +1349,6 @@ def release_lease( # pylint: disable=inconsistent-return-statements headers=_headers, params=_params, ) - _request = _convert_request(_request) _request.url = self._client.format_url(_request.url) _stream = False @@ -1449,7 +1442,6 @@ def change_lease( # pylint: disable=inconsistent-return-statements headers=_headers, params=_params, ) - _request = _convert_request(_request) _request.url = self._client.format_url(_request.url) _stream = False @@ -1538,7 +1530,6 @@ def renew_lease( # pylint: disable=inconsistent-return-statements headers=_headers, params=_params, ) - _request = _convert_request(_request) _request.url = self._client.format_url(_request.url) _stream = False @@ -1641,7 +1632,6 @@ def break_lease( # pylint: disable=inconsistent-return-statements headers=_headers, params=_params, ) - _request = _convert_request(_request) _request.url = self._client.format_url(_request.url) _stream = False @@ -1715,7 +1705,6 @@ def create_snapshot( # pylint: disable=inconsistent-return-statements headers=_headers, params=_params, ) - _request = _convert_request(_request) _request.url = self._client.format_url(_request.url) _stream = False @@ -1848,7 +1837,6 @@ def create_permission( # pylint: disable=inconsistent-return-statements headers=_headers, params=_params, ) - _request = _convert_request(_request) _request.url = self._client.format_url(_request.url) _stream = False @@ -1929,7 +1917,6 @@ def get_permission( headers=_headers, params=_params, ) - _request = _convert_request(_request) _request.url = self._client.format_url(_request.url) _stream = False @@ -1949,7 +1936,7 @@ def get_permission( response_headers["x-ms-version"] = self._deserialize("str", response.headers.get("x-ms-version")) response_headers["Date"] = self._deserialize("rfc-1123", response.headers.get("Date")) - deserialized = self._deserialize("SharePermission", pipeline_response) + deserialized = self._deserialize("SharePermission", pipeline_response.http_response) if cls: return cls(pipeline_response, deserialized, response_headers) # type: ignore @@ -2041,7 +2028,6 @@ def set_properties( # pylint: disable=inconsistent-return-statements headers=_headers, params=_params, ) - _request = _convert_request(_request) _request.url = self._client.format_url(_request.url) _stream = False @@ -2121,7 +2107,6 @@ def set_metadata( # pylint: disable=inconsistent-return-statements headers=_headers, params=_params, ) - _request = _convert_request(_request) _request.url = self._client.format_url(_request.url) _stream = False @@ -2196,7 +2181,6 @@ def get_access_policy( headers=_headers, params=_params, ) - _request = _convert_request(_request) _request.url = self._client.format_url(_request.url) _stream = False @@ -2218,7 +2202,7 @@ def get_access_policy( response_headers["x-ms-version"] = self._deserialize("str", response.headers.get("x-ms-version")) response_headers["Date"] = self._deserialize("rfc-1123", response.headers.get("Date")) - deserialized = self._deserialize("[SignedIdentifier]", pipeline_response) + deserialized = self._deserialize("[SignedIdentifier]", pipeline_response.http_response) if cls: return cls(pipeline_response, deserialized, response_headers) # type: ignore @@ -2288,7 +2272,6 @@ def set_access_policy( # pylint: disable=inconsistent-return-statements headers=_headers, params=_params, ) - _request = _convert_request(_request) _request.url = self._client.format_url(_request.url) _stream = False @@ -2363,7 +2346,6 @@ def get_statistics( headers=_headers, params=_params, ) - _request = _convert_request(_request) _request.url = self._client.format_url(_request.url) _stream = False @@ -2385,7 +2367,7 @@ def get_statistics( response_headers["x-ms-version"] = self._deserialize("str", response.headers.get("x-ms-version")) response_headers["Date"] = self._deserialize("rfc-1123", response.headers.get("Date")) - deserialized = self._deserialize("ShareStats", pipeline_response) + deserialized = self._deserialize("ShareStats", pipeline_response.http_response) if cls: return cls(pipeline_response, deserialized, response_headers) # type: ignore @@ -2450,7 +2432,6 @@ def restore( # pylint: disable=inconsistent-return-statements headers=_headers, params=_params, ) - _request = _convert_request(_request) _request.url = self._client.format_url(_request.url) _stream = False diff --git a/sdk/storage/azure-storage-file-share/azure/storage/fileshare/_shared/policies.py b/sdk/storage/azure-storage-file-share/azure/storage/fileshare/_shared/policies.py index 01383d8af468..dc04463a5835 100644 --- a/sdk/storage/azure-storage-file-share/azure/storage/fileshare/_shared/policies.py +++ b/sdk/storage/azure-storage-file-share/azure/storage/fileshare/_shared/policies.py @@ -35,11 +35,6 @@ from .constants import DEFAULT_OAUTH_SCOPE from .models import LocationMode -try: - _unicode_type = unicode # type: ignore -except NameError: - _unicode_type = str - if TYPE_CHECKING: from azure.core.credentials import TokenCredential from azure.core.pipeline.transport import ( # pylint: disable=non-abstract-transport-import @@ -52,7 +47,7 @@ def encode_base64(data): - if isinstance(data, _unicode_type): + if isinstance(data, str): data = data.encode('utf-8') encoded = base64.b64encode(data) return encoded.decode('utf-8') @@ -95,10 +90,14 @@ def is_retry(response, mode): # pylint: disable=too-many-return-statements if status in [501, 505]: return False return True + return False + + +def is_checksum_retry(response): # retry if invalid content md5 if response.context.get('validate_content', False) and response.http_response.headers.get('content-md5'): computed_md5 = response.http_request.headers.get('content-md5', None) or \ - encode_base64(StorageContentValidation.get_content_md5(response.http_response.body())) + encode_base64(StorageContentValidation.get_content_md5(response.http_response.body())) if response.http_response.headers['content-md5'] != computed_md5: return True return False @@ -301,7 +300,7 @@ def send(self, request: "PipelineRequest") -> "PipelineResponse": response = self.next.send(request) - will_retry = is_retry(response, request.context.options.get('mode')) + will_retry = is_retry(response, request.context.options.get('mode')) or is_checksum_retry(response) # Auth error could come from Bearer challenge, in which case this request will be made again is_auth_error = response.http_response.status_code == 401 should_update_counts = not (will_retry or is_auth_error) @@ -526,7 +525,7 @@ def send(self, request): while retries_remaining: try: response = self.next.send(request) - if is_retry(response, retry_settings['mode']): + if is_retry(response, retry_settings['mode']) or is_checksum_retry(response): retries_remaining = self.increment( retry_settings, request=request.http_request, diff --git a/sdk/storage/azure-storage-file-share/azure/storage/fileshare/_shared/policies_async.py b/sdk/storage/azure-storage-file-share/azure/storage/fileshare/_shared/policies_async.py index 047168f3f8cd..374077bdd4d9 100644 --- a/sdk/storage/azure-storage-file-share/azure/storage/fileshare/_shared/policies_async.py +++ b/sdk/storage/azure-storage-file-share/azure/storage/fileshare/_shared/policies_async.py @@ -10,12 +10,12 @@ import random from typing import Any, Dict, TYPE_CHECKING -from azure.core.exceptions import AzureError +from azure.core.exceptions import AzureError, StreamClosedError, StreamConsumedError from azure.core.pipeline.policies import AsyncBearerTokenCredentialPolicy, AsyncHTTPPolicy from .authentication import AzureSigningError, StorageHttpChallenge from .constants import DEFAULT_OAUTH_SCOPE -from .policies import is_retry, StorageRetryPolicy +from .policies import encode_base64, is_retry, StorageContentValidation, StorageRetryPolicy if TYPE_CHECKING: from azure.core.credentials_async import AsyncTokenCredential @@ -42,6 +42,20 @@ async def retry_hook(settings, **kwargs): **kwargs) +async def is_checksum_retry(response): + # retry if invalid content md5 + if response.context.get('validate_content', False) and response.http_response.headers.get('content-md5'): + try: + await response.http_response.read() # Load the body in memory and close the socket + except (StreamClosedError, StreamConsumedError): + pass + computed_md5 = response.http_request.headers.get('content-md5', None) or \ + encode_base64(StorageContentValidation.get_content_md5(response.http_response.content)) + if response.http_response.headers['content-md5'] != computed_md5: + return True + return False + + class AsyncStorageResponseHook(AsyncHTTPPolicy): def __init__(self, **kwargs): # pylint: disable=unused-argument @@ -64,9 +78,8 @@ async def send(self, request: "PipelineRequest") -> "PipelineResponse": request.context.options.pop('raw_response_hook', self._response_callback) response = await self.next.send(request) - await response.http_response.load_body() - will_retry = is_retry(response, request.context.options.get('mode')) + will_retry = is_retry(response, request.context.options.get('mode')) or await is_checksum_retry(response) # Auth error could come from Bearer challenge, in which case this request will be made again is_auth_error = response.http_response.status_code == 401 should_update_counts = not (will_retry or is_auth_error) @@ -112,7 +125,7 @@ async def send(self, request): while retries_remaining: try: response = await self.next.send(request) - if is_retry(response, retry_settings['mode']): + if is_retry(response, retry_settings['mode']) or await is_checksum_retry(response): retries_remaining = self.increment( retry_settings, request=request.http_request, diff --git a/sdk/storage/azure-storage-file-share/azure/storage/fileshare/aio/_download_async.py b/sdk/storage/azure-storage-file-share/azure/storage/fileshare/aio/_download_async.py index 4bead0ebf43f..22b5121b13cb 100644 --- a/sdk/storage/azure-storage-file-share/azure/storage/fileshare/aio/_download_async.py +++ b/sdk/storage/azure-storage-file-share/azure/storage/fileshare/aio/_download_async.py @@ -22,7 +22,8 @@ async def process_content(data): raise ValueError("Response cannot be None.") try: - return data.response.body() + await data.response.read() + return data.response.content except Exception as error: raise HttpResponseError(message="Download stream interrupted.", response=data.response, error=error) from error diff --git a/sdk/storage/azure-storage-queue/azure/storage/queue/_generated/_azure_queue_storage.py b/sdk/storage/azure-storage-queue/azure/storage/queue/_generated/_azure_queue_storage.py index 9b9522e6e364..ce3d7d165b43 100644 --- a/sdk/storage/azure-storage-queue/azure/storage/queue/_generated/_azure_queue_storage.py +++ b/sdk/storage/azure-storage-queue/azure/storage/queue/_generated/_azure_queue_storage.py @@ -8,6 +8,7 @@ from copy import deepcopy from typing import Any +from typing_extensions import Self from azure.core import PipelineClient from azure.core.pipeline import policies @@ -97,7 +98,7 @@ def _send_request(self, request: HttpRequest, *, stream: bool = False, **kwargs: def close(self) -> None: self._client.close() - def __enter__(self) -> "AzureQueueStorage": + def __enter__(self) -> Self: self._client.__enter__() return self diff --git a/sdk/storage/azure-storage-queue/azure/storage/queue/_generated/_serialization.py b/sdk/storage/azure-storage-queue/azure/storage/queue/_generated/_serialization.py index f0c6180722c8..8139854b97bb 100644 --- a/sdk/storage/azure-storage-queue/azure/storage/queue/_generated/_serialization.py +++ b/sdk/storage/azure-storage-queue/azure/storage/queue/_generated/_serialization.py @@ -144,6 +144,8 @@ def _json_attemp(data): # context otherwise. _LOGGER.critical("Wasn't XML not JSON, failing") raise DeserializationError("XML is invalid") from err + elif content_type.startswith("text/"): + return data_as_str raise DeserializationError("Cannot deserialize content-type: {}".format(content_type)) @classmethod diff --git a/sdk/storage/azure-storage-queue/azure/storage/queue/_generated/_vendor.py b/sdk/storage/azure-storage-queue/azure/storage/queue/_generated/_vendor.py deleted file mode 100644 index 0dafe0e287ff..000000000000 --- a/sdk/storage/azure-storage-queue/azure/storage/queue/_generated/_vendor.py +++ /dev/null @@ -1,16 +0,0 @@ -# -------------------------------------------------------------------------- -# Copyright (c) Microsoft Corporation. All rights reserved. -# Licensed under the MIT License. See License.txt in the project root for license information. -# Code generated by Microsoft (R) AutoRest Code Generator. -# Changes may cause incorrect behavior and will be lost if the code is regenerated. -# -------------------------------------------------------------------------- - -from azure.core.pipeline.transport import HttpRequest - - -def _convert_request(request, files=None): - data = request.content if not files else None - request = HttpRequest(method=request.method, url=request.url, headers=request.headers, data=data) - if files: - request.set_formdata_body(files) - return request diff --git a/sdk/storage/azure-storage-queue/azure/storage/queue/_generated/aio/_azure_queue_storage.py b/sdk/storage/azure-storage-queue/azure/storage/queue/_generated/aio/_azure_queue_storage.py index a6e4460ecb1f..586e4c7d28cd 100644 --- a/sdk/storage/azure-storage-queue/azure/storage/queue/_generated/aio/_azure_queue_storage.py +++ b/sdk/storage/azure-storage-queue/azure/storage/queue/_generated/aio/_azure_queue_storage.py @@ -8,6 +8,7 @@ from copy import deepcopy from typing import Any, Awaitable +from typing_extensions import Self from azure.core import AsyncPipelineClient from azure.core.pipeline import policies @@ -99,7 +100,7 @@ def _send_request( async def close(self) -> None: await self._client.close() - async def __aenter__(self) -> "AzureQueueStorage": + async def __aenter__(self) -> Self: await self._client.__aenter__() return self diff --git a/sdk/storage/azure-storage-queue/azure/storage/queue/_generated/aio/operations/_message_id_operations.py b/sdk/storage/azure-storage-queue/azure/storage/queue/_generated/aio/operations/_message_id_operations.py index dddee6d2daf2..fcd95f055e5c 100644 --- a/sdk/storage/azure-storage-queue/azure/storage/queue/_generated/aio/operations/_message_id_operations.py +++ b/sdk/storage/azure-storage-queue/azure/storage/queue/_generated/aio/operations/_message_id_operations.py @@ -18,13 +18,11 @@ map_error, ) from azure.core.pipeline import PipelineResponse -from azure.core.pipeline.transport import AsyncHttpResponse -from azure.core.rest import HttpRequest +from azure.core.rest import AsyncHttpResponse, HttpRequest from azure.core.tracing.decorator_async import distributed_trace_async from azure.core.utils import case_insensitive_dict from ... import models as _models -from ..._vendor import _convert_request from ...operations._message_id_operations import build_delete_request, build_update_request if sys.version_info >= (3, 9): @@ -124,7 +122,6 @@ async def update( # pylint: disable=inconsistent-return-statements headers=_headers, params=_params, ) - _request = _convert_request(_request) _request.url = self._client.format_url(_request.url) _stream = False @@ -194,7 +191,6 @@ async def delete( # pylint: disable=inconsistent-return-statements headers=_headers, params=_params, ) - _request = _convert_request(_request) _request.url = self._client.format_url(_request.url) _stream = False diff --git a/sdk/storage/azure-storage-queue/azure/storage/queue/_generated/aio/operations/_messages_operations.py b/sdk/storage/azure-storage-queue/azure/storage/queue/_generated/aio/operations/_messages_operations.py index 7be183ba87d7..72fbd1738bf2 100644 --- a/sdk/storage/azure-storage-queue/azure/storage/queue/_generated/aio/operations/_messages_operations.py +++ b/sdk/storage/azure-storage-queue/azure/storage/queue/_generated/aio/operations/_messages_operations.py @@ -18,13 +18,11 @@ map_error, ) from azure.core.pipeline import PipelineResponse -from azure.core.pipeline.transport import AsyncHttpResponse -from azure.core.rest import HttpRequest +from azure.core.rest import AsyncHttpResponse, HttpRequest from azure.core.tracing.decorator_async import distributed_trace_async from azure.core.utils import case_insensitive_dict from ... import models as _models -from ..._vendor import _convert_request from ...operations._messages_operations import ( build_clear_request, build_dequeue_request, @@ -116,7 +114,6 @@ async def dequeue( headers=_headers, params=_params, ) - _request = _convert_request(_request) _request.url = self._client.format_url(_request.url) _stream = False @@ -136,7 +133,7 @@ async def dequeue( response_headers["x-ms-version"] = self._deserialize("str", response.headers.get("x-ms-version")) response_headers["Date"] = self._deserialize("rfc-1123", response.headers.get("Date")) - deserialized = self._deserialize("[DequeuedMessageItem]", pipeline_response) + deserialized = self._deserialize("[DequeuedMessageItem]", pipeline_response.http_response) if cls: return cls(pipeline_response, deserialized, response_headers) # type: ignore @@ -182,7 +179,6 @@ async def clear( # pylint: disable=inconsistent-return-statements headers=_headers, params=_params, ) - _request = _convert_request(_request) _request.url = self._client.format_url(_request.url) _stream = False @@ -276,7 +272,6 @@ async def enqueue( headers=_headers, params=_params, ) - _request = _convert_request(_request) _request.url = self._client.format_url(_request.url) _stream = False @@ -296,7 +291,7 @@ async def enqueue( response_headers["x-ms-version"] = self._deserialize("str", response.headers.get("x-ms-version")) response_headers["Date"] = self._deserialize("rfc-1123", response.headers.get("Date")) - deserialized = self._deserialize("[EnqueuedMessage]", pipeline_response) + deserialized = self._deserialize("[EnqueuedMessage]", pipeline_response.http_response) if cls: return cls(pipeline_response, deserialized, response_headers) # type: ignore @@ -355,7 +350,6 @@ async def peek( headers=_headers, params=_params, ) - _request = _convert_request(_request) _request.url = self._client.format_url(_request.url) _stream = False @@ -375,7 +369,7 @@ async def peek( response_headers["x-ms-version"] = self._deserialize("str", response.headers.get("x-ms-version")) response_headers["Date"] = self._deserialize("rfc-1123", response.headers.get("Date")) - deserialized = self._deserialize("[PeekedMessageItem]", pipeline_response) + deserialized = self._deserialize("[PeekedMessageItem]", pipeline_response.http_response) if cls: return cls(pipeline_response, deserialized, response_headers) # type: ignore diff --git a/sdk/storage/azure-storage-queue/azure/storage/queue/_generated/aio/operations/_queue_operations.py b/sdk/storage/azure-storage-queue/azure/storage/queue/_generated/aio/operations/_queue_operations.py index 796f71282102..981c09a9943c 100644 --- a/sdk/storage/azure-storage-queue/azure/storage/queue/_generated/aio/operations/_queue_operations.py +++ b/sdk/storage/azure-storage-queue/azure/storage/queue/_generated/aio/operations/_queue_operations.py @@ -18,13 +18,11 @@ map_error, ) from azure.core.pipeline import PipelineResponse -from azure.core.pipeline.transport import AsyncHttpResponse -from azure.core.rest import HttpRequest +from azure.core.rest import AsyncHttpResponse, HttpRequest from azure.core.tracing.decorator_async import distributed_trace_async from azure.core.utils import case_insensitive_dict from ... import models as _models -from ..._vendor import _convert_request from ...operations._queue_operations import ( build_create_request, build_delete_request, @@ -111,7 +109,6 @@ async def create( # pylint: disable=inconsistent-return-statements headers=_headers, params=_params, ) - _request = _convert_request(_request) _request.url = self._client.format_url(_request.url) _stream = False @@ -127,15 +124,9 @@ async def create( # pylint: disable=inconsistent-return-statements raise HttpResponseError(response=response, model=error) response_headers = {} - if response.status_code == 201: - response_headers["x-ms-request-id"] = self._deserialize("str", response.headers.get("x-ms-request-id")) - response_headers["x-ms-version"] = self._deserialize("str", response.headers.get("x-ms-version")) - response_headers["Date"] = self._deserialize("rfc-1123", response.headers.get("Date")) - - if response.status_code == 204: - response_headers["x-ms-request-id"] = self._deserialize("str", response.headers.get("x-ms-request-id")) - response_headers["x-ms-version"] = self._deserialize("str", response.headers.get("x-ms-version")) - response_headers["Date"] = self._deserialize("rfc-1123", response.headers.get("Date")) + response_headers["x-ms-request-id"] = self._deserialize("str", response.headers.get("x-ms-request-id")) + response_headers["x-ms-version"] = self._deserialize("str", response.headers.get("x-ms-version")) + response_headers["Date"] = self._deserialize("rfc-1123", response.headers.get("Date")) if cls: return cls(pipeline_response, None, response_headers) # type: ignore @@ -179,7 +170,6 @@ async def delete( # pylint: disable=inconsistent-return-statements headers=_headers, params=_params, ) - _request = _convert_request(_request) _request.url = self._client.format_url(_request.url) _stream = False @@ -244,7 +234,6 @@ async def get_properties( # pylint: disable=inconsistent-return-statements headers=_headers, params=_params, ) - _request = _convert_request(_request) _request.url = self._client.format_url(_request.url) _stream = False @@ -324,7 +313,6 @@ async def set_metadata( # pylint: disable=inconsistent-return-statements headers=_headers, params=_params, ) - _request = _convert_request(_request) _request.url = self._client.format_url(_request.url) _stream = False @@ -389,7 +377,6 @@ async def get_access_policy( headers=_headers, params=_params, ) - _request = _convert_request(_request) _request.url = self._client.format_url(_request.url) _stream = False @@ -409,7 +396,7 @@ async def get_access_policy( response_headers["x-ms-version"] = self._deserialize("str", response.headers.get("x-ms-version")) response_headers["Date"] = self._deserialize("rfc-1123", response.headers.get("Date")) - deserialized = self._deserialize("[SignedIdentifier]", pipeline_response) + deserialized = self._deserialize("[SignedIdentifier]", pipeline_response.http_response) if cls: return cls(pipeline_response, deserialized, response_headers) # type: ignore @@ -474,7 +461,6 @@ async def set_access_policy( # pylint: disable=inconsistent-return-statements headers=_headers, params=_params, ) - _request = _convert_request(_request) _request.url = self._client.format_url(_request.url) _stream = False diff --git a/sdk/storage/azure-storage-queue/azure/storage/queue/_generated/aio/operations/_service_operations.py b/sdk/storage/azure-storage-queue/azure/storage/queue/_generated/aio/operations/_service_operations.py index ff256d797a87..809294124f93 100644 --- a/sdk/storage/azure-storage-queue/azure/storage/queue/_generated/aio/operations/_service_operations.py +++ b/sdk/storage/azure-storage-queue/azure/storage/queue/_generated/aio/operations/_service_operations.py @@ -18,13 +18,11 @@ map_error, ) from azure.core.pipeline import PipelineResponse -from azure.core.pipeline.transport import AsyncHttpResponse -from azure.core.rest import HttpRequest +from azure.core.rest import AsyncHttpResponse, HttpRequest from azure.core.tracing.decorator_async import distributed_trace_async from azure.core.utils import case_insensitive_dict from ... import models as _models -from ..._vendor import _convert_request from ...operations._service_operations import ( build_get_properties_request, build_get_statistics_request, @@ -114,7 +112,6 @@ async def set_properties( # pylint: disable=inconsistent-return-statements headers=_headers, params=_params, ) - _request = _convert_request(_request) _request.url = self._client.format_url(_request.url) _stream = False @@ -180,7 +177,6 @@ async def get_properties( headers=_headers, params=_params, ) - _request = _convert_request(_request) _request.url = self._client.format_url(_request.url) _stream = False @@ -199,7 +195,7 @@ async def get_properties( response_headers["x-ms-request-id"] = self._deserialize("str", response.headers.get("x-ms-request-id")) response_headers["x-ms-version"] = self._deserialize("str", response.headers.get("x-ms-version")) - deserialized = self._deserialize("StorageServiceProperties", pipeline_response) + deserialized = self._deserialize("StorageServiceProperties", pipeline_response.http_response) if cls: return cls(pipeline_response, deserialized, response_headers) # type: ignore @@ -251,7 +247,6 @@ async def get_statistics( headers=_headers, params=_params, ) - _request = _convert_request(_request) _request.url = self._client.format_url(_request.url) _stream = False @@ -271,7 +266,7 @@ async def get_statistics( response_headers["x-ms-version"] = self._deserialize("str", response.headers.get("x-ms-version")) response_headers["Date"] = self._deserialize("rfc-1123", response.headers.get("Date")) - deserialized = self._deserialize("StorageServiceStats", pipeline_response) + deserialized = self._deserialize("StorageServiceStats", pipeline_response.http_response) if cls: return cls(pipeline_response, deserialized, response_headers) # type: ignore @@ -350,7 +345,6 @@ async def list_queues_segment( headers=_headers, params=_params, ) - _request = _convert_request(_request) _request.url = self._client.format_url(_request.url) _stream = False @@ -370,7 +364,7 @@ async def list_queues_segment( response_headers["x-ms-version"] = self._deserialize("str", response.headers.get("x-ms-version")) response_headers["Date"] = self._deserialize("rfc-1123", response.headers.get("Date")) - deserialized = self._deserialize("ListQueuesSegmentResponse", pipeline_response) + deserialized = self._deserialize("ListQueuesSegmentResponse", pipeline_response.http_response) if cls: return cls(pipeline_response, deserialized, response_headers) # type: ignore diff --git a/sdk/storage/azure-storage-queue/azure/storage/queue/_generated/operations/_message_id_operations.py b/sdk/storage/azure-storage-queue/azure/storage/queue/_generated/operations/_message_id_operations.py index 32e90d88ef22..b4db4ab66666 100644 --- a/sdk/storage/azure-storage-queue/azure/storage/queue/_generated/operations/_message_id_operations.py +++ b/sdk/storage/azure-storage-queue/azure/storage/queue/_generated/operations/_message_id_operations.py @@ -18,14 +18,12 @@ map_error, ) from azure.core.pipeline import PipelineResponse -from azure.core.pipeline.transport import HttpResponse -from azure.core.rest import HttpRequest +from azure.core.rest import HttpRequest, HttpResponse from azure.core.tracing.decorator import distributed_trace from azure.core.utils import case_insensitive_dict from .. import models as _models from .._serialization import Serializer -from .._vendor import _convert_request if sys.version_info >= (3, 9): from collections.abc import MutableMapping @@ -207,7 +205,6 @@ def update( # pylint: disable=inconsistent-return-statements headers=_headers, params=_params, ) - _request = _convert_request(_request) _request.url = self._client.format_url(_request.url) _stream = False @@ -277,7 +274,6 @@ def delete( # pylint: disable=inconsistent-return-statements headers=_headers, params=_params, ) - _request = _convert_request(_request) _request.url = self._client.format_url(_request.url) _stream = False diff --git a/sdk/storage/azure-storage-queue/azure/storage/queue/_generated/operations/_messages_operations.py b/sdk/storage/azure-storage-queue/azure/storage/queue/_generated/operations/_messages_operations.py index 96bfb1fb774c..61a581285efe 100644 --- a/sdk/storage/azure-storage-queue/azure/storage/queue/_generated/operations/_messages_operations.py +++ b/sdk/storage/azure-storage-queue/azure/storage/queue/_generated/operations/_messages_operations.py @@ -18,14 +18,12 @@ map_error, ) from azure.core.pipeline import PipelineResponse -from azure.core.pipeline.transport import HttpResponse -from azure.core.rest import HttpRequest +from azure.core.rest import HttpRequest, HttpResponse from azure.core.tracing.decorator import distributed_trace from azure.core.utils import case_insensitive_dict from .. import models as _models from .._serialization import Serializer -from .._vendor import _convert_request if sys.version_info >= (3, 9): from collections.abc import MutableMapping @@ -271,7 +269,6 @@ def dequeue( headers=_headers, params=_params, ) - _request = _convert_request(_request) _request.url = self._client.format_url(_request.url) _stream = False @@ -291,7 +288,7 @@ def dequeue( response_headers["x-ms-version"] = self._deserialize("str", response.headers.get("x-ms-version")) response_headers["Date"] = self._deserialize("rfc-1123", response.headers.get("Date")) - deserialized = self._deserialize("[DequeuedMessageItem]", pipeline_response) + deserialized = self._deserialize("[DequeuedMessageItem]", pipeline_response.http_response) if cls: return cls(pipeline_response, deserialized, response_headers) # type: ignore @@ -337,7 +334,6 @@ def clear( # pylint: disable=inconsistent-return-statements headers=_headers, params=_params, ) - _request = _convert_request(_request) _request.url = self._client.format_url(_request.url) _stream = False @@ -431,7 +427,6 @@ def enqueue( headers=_headers, params=_params, ) - _request = _convert_request(_request) _request.url = self._client.format_url(_request.url) _stream = False @@ -451,7 +446,7 @@ def enqueue( response_headers["x-ms-version"] = self._deserialize("str", response.headers.get("x-ms-version")) response_headers["Date"] = self._deserialize("rfc-1123", response.headers.get("Date")) - deserialized = self._deserialize("[EnqueuedMessage]", pipeline_response) + deserialized = self._deserialize("[EnqueuedMessage]", pipeline_response.http_response) if cls: return cls(pipeline_response, deserialized, response_headers) # type: ignore @@ -510,7 +505,6 @@ def peek( headers=_headers, params=_params, ) - _request = _convert_request(_request) _request.url = self._client.format_url(_request.url) _stream = False @@ -530,7 +524,7 @@ def peek( response_headers["x-ms-version"] = self._deserialize("str", response.headers.get("x-ms-version")) response_headers["Date"] = self._deserialize("rfc-1123", response.headers.get("Date")) - deserialized = self._deserialize("[PeekedMessageItem]", pipeline_response) + deserialized = self._deserialize("[PeekedMessageItem]", pipeline_response.http_response) if cls: return cls(pipeline_response, deserialized, response_headers) # type: ignore diff --git a/sdk/storage/azure-storage-queue/azure/storage/queue/_generated/operations/_queue_operations.py b/sdk/storage/azure-storage-queue/azure/storage/queue/_generated/operations/_queue_operations.py index 8de6bf6679f3..56069f4acb6a 100644 --- a/sdk/storage/azure-storage-queue/azure/storage/queue/_generated/operations/_queue_operations.py +++ b/sdk/storage/azure-storage-queue/azure/storage/queue/_generated/operations/_queue_operations.py @@ -18,14 +18,12 @@ map_error, ) from azure.core.pipeline import PipelineResponse -from azure.core.pipeline.transport import HttpResponse -from azure.core.rest import HttpRequest +from azure.core.rest import HttpRequest, HttpResponse from azure.core.tracing.decorator import distributed_trace from azure.core.utils import case_insensitive_dict from .. import models as _models from .._serialization import Serializer -from .._vendor import _convert_request if sys.version_info >= (3, 9): from collections.abc import MutableMapping @@ -317,7 +315,6 @@ def create( # pylint: disable=inconsistent-return-statements headers=_headers, params=_params, ) - _request = _convert_request(_request) _request.url = self._client.format_url(_request.url) _stream = False @@ -333,15 +330,9 @@ def create( # pylint: disable=inconsistent-return-statements raise HttpResponseError(response=response, model=error) response_headers = {} - if response.status_code == 201: - response_headers["x-ms-request-id"] = self._deserialize("str", response.headers.get("x-ms-request-id")) - response_headers["x-ms-version"] = self._deserialize("str", response.headers.get("x-ms-version")) - response_headers["Date"] = self._deserialize("rfc-1123", response.headers.get("Date")) - - if response.status_code == 204: - response_headers["x-ms-request-id"] = self._deserialize("str", response.headers.get("x-ms-request-id")) - response_headers["x-ms-version"] = self._deserialize("str", response.headers.get("x-ms-version")) - response_headers["Date"] = self._deserialize("rfc-1123", response.headers.get("Date")) + response_headers["x-ms-request-id"] = self._deserialize("str", response.headers.get("x-ms-request-id")) + response_headers["x-ms-version"] = self._deserialize("str", response.headers.get("x-ms-version")) + response_headers["Date"] = self._deserialize("rfc-1123", response.headers.get("Date")) if cls: return cls(pipeline_response, None, response_headers) # type: ignore @@ -385,7 +376,6 @@ def delete( # pylint: disable=inconsistent-return-statements headers=_headers, params=_params, ) - _request = _convert_request(_request) _request.url = self._client.format_url(_request.url) _stream = False @@ -450,7 +440,6 @@ def get_properties( # pylint: disable=inconsistent-return-statements headers=_headers, params=_params, ) - _request = _convert_request(_request) _request.url = self._client.format_url(_request.url) _stream = False @@ -530,7 +519,6 @@ def set_metadata( # pylint: disable=inconsistent-return-statements headers=_headers, params=_params, ) - _request = _convert_request(_request) _request.url = self._client.format_url(_request.url) _stream = False @@ -595,7 +583,6 @@ def get_access_policy( headers=_headers, params=_params, ) - _request = _convert_request(_request) _request.url = self._client.format_url(_request.url) _stream = False @@ -615,7 +602,7 @@ def get_access_policy( response_headers["x-ms-version"] = self._deserialize("str", response.headers.get("x-ms-version")) response_headers["Date"] = self._deserialize("rfc-1123", response.headers.get("Date")) - deserialized = self._deserialize("[SignedIdentifier]", pipeline_response) + deserialized = self._deserialize("[SignedIdentifier]", pipeline_response.http_response) if cls: return cls(pipeline_response, deserialized, response_headers) # type: ignore @@ -680,7 +667,6 @@ def set_access_policy( # pylint: disable=inconsistent-return-statements headers=_headers, params=_params, ) - _request = _convert_request(_request) _request.url = self._client.format_url(_request.url) _stream = False diff --git a/sdk/storage/azure-storage-queue/azure/storage/queue/_generated/operations/_service_operations.py b/sdk/storage/azure-storage-queue/azure/storage/queue/_generated/operations/_service_operations.py index f9eea8cde861..99f097bebcf5 100644 --- a/sdk/storage/azure-storage-queue/azure/storage/queue/_generated/operations/_service_operations.py +++ b/sdk/storage/azure-storage-queue/azure/storage/queue/_generated/operations/_service_operations.py @@ -18,14 +18,12 @@ map_error, ) from azure.core.pipeline import PipelineResponse -from azure.core.pipeline.transport import HttpResponse -from azure.core.rest import HttpRequest +from azure.core.rest import HttpRequest, HttpResponse from azure.core.tracing.decorator import distributed_trace from azure.core.utils import case_insensitive_dict from .. import models as _models from .._serialization import Serializer -from .._vendor import _convert_request if sys.version_info >= (3, 9): from collections.abc import MutableMapping @@ -265,7 +263,6 @@ def set_properties( # pylint: disable=inconsistent-return-statements headers=_headers, params=_params, ) - _request = _convert_request(_request) _request.url = self._client.format_url(_request.url) _stream = False @@ -331,7 +328,6 @@ def get_properties( headers=_headers, params=_params, ) - _request = _convert_request(_request) _request.url = self._client.format_url(_request.url) _stream = False @@ -350,7 +346,7 @@ def get_properties( response_headers["x-ms-request-id"] = self._deserialize("str", response.headers.get("x-ms-request-id")) response_headers["x-ms-version"] = self._deserialize("str", response.headers.get("x-ms-version")) - deserialized = self._deserialize("StorageServiceProperties", pipeline_response) + deserialized = self._deserialize("StorageServiceProperties", pipeline_response.http_response) if cls: return cls(pipeline_response, deserialized, response_headers) # type: ignore @@ -402,7 +398,6 @@ def get_statistics( headers=_headers, params=_params, ) - _request = _convert_request(_request) _request.url = self._client.format_url(_request.url) _stream = False @@ -422,7 +417,7 @@ def get_statistics( response_headers["x-ms-version"] = self._deserialize("str", response.headers.get("x-ms-version")) response_headers["Date"] = self._deserialize("rfc-1123", response.headers.get("Date")) - deserialized = self._deserialize("StorageServiceStats", pipeline_response) + deserialized = self._deserialize("StorageServiceStats", pipeline_response.http_response) if cls: return cls(pipeline_response, deserialized, response_headers) # type: ignore @@ -501,7 +496,6 @@ def list_queues_segment( headers=_headers, params=_params, ) - _request = _convert_request(_request) _request.url = self._client.format_url(_request.url) _stream = False @@ -521,7 +515,7 @@ def list_queues_segment( response_headers["x-ms-version"] = self._deserialize("str", response.headers.get("x-ms-version")) response_headers["Date"] = self._deserialize("rfc-1123", response.headers.get("Date")) - deserialized = self._deserialize("ListQueuesSegmentResponse", pipeline_response) + deserialized = self._deserialize("ListQueuesSegmentResponse", pipeline_response.http_response) if cls: return cls(pipeline_response, deserialized, response_headers) # type: ignore diff --git a/sdk/storage/azure-storage-queue/azure/storage/queue/_shared/policies.py b/sdk/storage/azure-storage-queue/azure/storage/queue/_shared/policies.py index 4f22c181bb2e..d6f3daab5c7a 100644 --- a/sdk/storage/azure-storage-queue/azure/storage/queue/_shared/policies.py +++ b/sdk/storage/azure-storage-queue/azure/storage/queue/_shared/policies.py @@ -35,11 +35,6 @@ from .constants import DEFAULT_OAUTH_SCOPE from .models import LocationMode -try: - _unicode_type = unicode # type: ignore -except NameError: - _unicode_type = str - if TYPE_CHECKING: from azure.core.credentials import TokenCredential from azure.core.pipeline.transport import ( # pylint: disable=non-abstract-transport-import @@ -52,7 +47,7 @@ def encode_base64(data): - if isinstance(data, _unicode_type): + if isinstance(data, str): data = data.encode('utf-8') encoded = base64.b64encode(data) return encoded.decode('utf-8') @@ -95,10 +90,13 @@ def is_retry(response, mode): # pylint: disable=too-many-return-statements if status in [501, 505]: return False return True + return False + +def is_checksum_retry(response): # retry if invalid content md5 if response.context.get('validate_content', False) and response.http_response.headers.get('content-md5'): computed_md5 = response.http_request.headers.get('content-md5', None) or \ - encode_base64(StorageContentValidation.get_content_md5(response.http_response.body())) + encode_base64(StorageContentValidation.get_content_md5(response.http_response.body())) if response.http_response.headers['content-md5'] != computed_md5: return True return False @@ -307,7 +305,7 @@ def send(self, request: "PipelineRequest") -> "PipelineResponse": response = self.next.send(request) - will_retry = is_retry(response, request.context.options.get('mode')) + will_retry = is_retry(response, request.context.options.get('mode')) or is_checksum_retry(response) # Auth error could come from Bearer challenge, in which case this request will be made again is_auth_error = response.http_response.status_code == 401 should_update_counts = not (will_retry or is_auth_error) @@ -532,7 +530,7 @@ def send(self, request): while retries_remaining: try: response = self.next.send(request) - if is_retry(response, retry_settings['mode']): + if is_retry(response, retry_settings['mode']) or is_checksum_retry(response): retries_remaining = self.increment( retry_settings, request=request.http_request, diff --git a/sdk/storage/azure-storage-queue/azure/storage/queue/_shared/policies_async.py b/sdk/storage/azure-storage-queue/azure/storage/queue/_shared/policies_async.py index 047168f3f8cd..374077bdd4d9 100644 --- a/sdk/storage/azure-storage-queue/azure/storage/queue/_shared/policies_async.py +++ b/sdk/storage/azure-storage-queue/azure/storage/queue/_shared/policies_async.py @@ -10,12 +10,12 @@ import random from typing import Any, Dict, TYPE_CHECKING -from azure.core.exceptions import AzureError +from azure.core.exceptions import AzureError, StreamClosedError, StreamConsumedError from azure.core.pipeline.policies import AsyncBearerTokenCredentialPolicy, AsyncHTTPPolicy from .authentication import AzureSigningError, StorageHttpChallenge from .constants import DEFAULT_OAUTH_SCOPE -from .policies import is_retry, StorageRetryPolicy +from .policies import encode_base64, is_retry, StorageContentValidation, StorageRetryPolicy if TYPE_CHECKING: from azure.core.credentials_async import AsyncTokenCredential @@ -42,6 +42,20 @@ async def retry_hook(settings, **kwargs): **kwargs) +async def is_checksum_retry(response): + # retry if invalid content md5 + if response.context.get('validate_content', False) and response.http_response.headers.get('content-md5'): + try: + await response.http_response.read() # Load the body in memory and close the socket + except (StreamClosedError, StreamConsumedError): + pass + computed_md5 = response.http_request.headers.get('content-md5', None) or \ + encode_base64(StorageContentValidation.get_content_md5(response.http_response.content)) + if response.http_response.headers['content-md5'] != computed_md5: + return True + return False + + class AsyncStorageResponseHook(AsyncHTTPPolicy): def __init__(self, **kwargs): # pylint: disable=unused-argument @@ -64,9 +78,8 @@ async def send(self, request: "PipelineRequest") -> "PipelineResponse": request.context.options.pop('raw_response_hook', self._response_callback) response = await self.next.send(request) - await response.http_response.load_body() - will_retry = is_retry(response, request.context.options.get('mode')) + will_retry = is_retry(response, request.context.options.get('mode')) or await is_checksum_retry(response) # Auth error could come from Bearer challenge, in which case this request will be made again is_auth_error = response.http_response.status_code == 401 should_update_counts = not (will_retry or is_auth_error) @@ -112,7 +125,7 @@ async def send(self, request): while retries_remaining: try: response = await self.next.send(request) - if is_retry(response, retry_settings['mode']): + if is_retry(response, retry_settings['mode']) or await is_checksum_retry(response): retries_remaining = self.increment( retry_settings, request=request.http_request,