Skip to content

Commit

Permalink
[Storage] Consolidate offset and range parameters (#7598)
Browse files Browse the repository at this point in the history 
* Updated blob sync tests

* Offset refactor APIS (#19)

* rename to offset and length

* fix

* more changes

* Updated blob sync tests

* Temp block async tests

* Fix upload page behaviour

* Fix clear page behaviour

* Update from_url offset behaviour

* Update page ranges

* Fix download blob behaviour

* Some cleanup

* Fixed page size

* More test fixes

* Some more fixes

* Fixed page tests

* Fixed encryption tests

* Fix common test

* append anf page blob async

* some more test fixes

* Fix live tests

* more changes async

* Fix sparse blob test

* Last tests

* update docstrings

* pylint
  • Loading branch information
@annatisch
annatisch authored and Rakshith Bhyravabhotla committed Oct 5, 2019
1 parent 716c141 commit b969b5e
Show file tree
Hide file tree
Showing 24 changed files with 5,321 additions and 5,760 deletions.
124 changes: 59 additions & 65 deletions sdk/storage/azure-storage-blob/azure/storage/blob/aio/blob_client_async.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1296,29 +1296,27 @@ async def set_premium_page_blob_tier(self, premium_page_blob_tier, **kwargs):

@distributed_trace_async
async def get_page_ranges( # type: ignore
self, start_range=None, # type: Optional[int]
end_range=None, # type: Optional[int]
self, offset=None, # type: Optional[int]
length=None, # type: Optional[int]
previous_snapshot_diff=None, # type: Optional[Union[str, Dict[str, Any]]]
**kwargs
):
# type: (...) -> List[dict[str, int]]
"""Returns the list of valid page ranges for a Page Blob or snapshot
of a page blob.
:param int start_range:
Start of byte range to use for getting valid page ranges.
If no end_range is given, all bytes after the start_range will be searched.
:param int offset:
Start of byte range to use for downloading a section of the blob.
Must be set if length is provided.
Pages must be aligned with 512-byte boundaries, the start offset
must be a modulus of 512 and the end offset must be a modulus of
512-1. Examples of valid byte ranges are 0-511, 512-, etc.
:param int end_range:
End of byte range to use for getting valid page ranges.
If end_range is given, start_range must be provided.
This range will return valid page ranges for from the offset start up to
offset end.
must be a modulus of 512 and the length must be a modulus of
512.
:param int length:
Number of bytes to use for getting valid page ranges.
If length is given, offset must be provided.
Pages must be aligned with 512-byte boundaries, the start offset
must be a modulus of 512 and the end offset must be a modulus of
512-1. Examples of valid byte ranges are 0-511, 512-, etc.
must be a modulus of 512 and the length must be a modulus of
512.
:param lease:
Required if the blob has an active lease. Value can be a LeaseClient object
or the lease ID as a string.
Expand Down Expand Up @@ -1356,8 +1354,8 @@ async def get_page_ranges( # type: ignore
:rtype: tuple(list(dict(str, str), list(dict(str, str))
"""
options = self._get_page_ranges_options(
start_range=start_range,
end_range=end_range,
offset=offset,
length=length,
previous_snapshot_diff=previous_snapshot_diff,
**kwargs)
try:
Expand Down Expand Up @@ -1475,28 +1473,26 @@ async def resize_blob(self, size, **kwargs):
@distributed_trace_async
async def upload_page( # type: ignore
self, page, # type: bytes
start_range, # type: int
end_range, # type: int
length=None, # type: Optional[int]
offset, # type: int
length, # type: int
**kwargs
):
# type: (...) -> Dict[str, Union[str, datetime]]
"""The Upload Pages operation writes a range of pages to a page blob.
:param bytes page:
Content of the page.
:param int start_range:
Start of byte range to use for writing to a section of the blob.
Pages must be aligned with 512-byte boundaries, the start offset
must be a modulus of 512 and the end offset must be a modulus of
512-1. Examples of valid byte ranges are 0-511, 512-1023, etc.
:param int end_range:
End of byte range to use for writing to a section of the blob.
:param int offset:
Start of byte range to use for downloading a section of the blob.
Must be set if length is provided.
Pages must be aligned with 512-byte boundaries, the start offset
must be a modulus of 512 and the end offset must be a modulus of
512-1. Examples of valid byte ranges are 0-511, 512-1023, etc.
must be a modulus of 512 and the length must be a modulus of
512.
:param int length:
Length of the page
Number of bytes to use for writing to a section of the blob.
Pages must be aligned with 512-byte boundaries, the start offset
must be a modulus of 512 and the length must be a modulus of
512.
:param lease:
Required if the blob has an active lease. Value can be a LeaseClient object
or the lease ID as a string.
Expand Down Expand Up @@ -1552,8 +1548,7 @@ async def upload_page( # type: ignore
"""
options = self._upload_page_options(
page=page,
start_range=start_range,
end_range=end_range,
offset=offset,
length=length,
**kwargs)
try:
Expand All @@ -1563,9 +1558,9 @@ async def upload_page( # type: ignore

@distributed_trace_async
async def upload_pages_from_url(self, source_url, # type: str
range_start, # type: int
range_end, # type: int
source_range_start, # type: int
offset, # type: int
length, # type: int
source_offset, # type: int
**kwargs
):
# type: (...) -> Dict[str, Any]
Expand All @@ -1575,19 +1570,17 @@ async def upload_pages_from_url(self, source_url, # type: str
:param str source_url:
The URL of the source data. It can point to any Azure Blob or File, that is either public or has a
shared access signature attached.
:param int range_start:
Start of byte range to use for writing to a section of the blob.
Pages must be aligned with 512-byte boundaries, the start offset
must be a modulus of 512 and the end offset must be a modulus of
512-1. Examples of valid byte ranges are 0-511, 512-1023, etc.
:param int range_end:
End of byte range to use for writing to a section of the blob.
:param int offset:
Start of byte range to use for downloading a section of the blob.
Must be set if length is provided.
:param int length:
Number of bytes to use for writing to a section of the blob.
Pages must be aligned with 512-byte boundaries, the start offset
must be a modulus of 512 and the end offset must be a modulus of
512-1. Examples of valid byte ranges are 0-511, 512-1023, etc.
:param int source_range_start:
must be a modulus of 512 and the length must be a modulus of
512.
:param int source_offset:
This indicates the start of the range of bytes(inclusive) that has to be taken from the copy source.
The service will read the same number of bytes as the destination range (end_range-start_range).
The service will read the same number of bytes as the destination range (length-offset).
:param bytes source_content_md5:
If given, the service will calculate the MD5 hash of the block content and compare against this value.
:param ~datetime.datetime source_if_modified_since:
Expand Down Expand Up @@ -1654,9 +1647,9 @@ async def upload_pages_from_url(self, source_url, # type: str

options = self._upload_pages_from_url_options(
source_url=source_url,
range_start=range_start,
range_end=range_end,
source_range_start=source_range_start,
offset=offset,
length=length,
source_offset=source_offset,
**kwargs
)
try:
Expand All @@ -1665,20 +1658,20 @@ async def upload_pages_from_url(self, source_url, # type: str
process_storage_error(error)

@distributed_trace_async
async def clear_page(self, start_range, end_range, **kwargs):
async def clear_page(self, offset, length, **kwargs):
# type: (int, int) -> Dict[str, Union[str, datetime]]
"""Clears a range of pages.
:param int start_range:
Start of byte range to use for writing to a section of the blob.
:param int offset:
Start of byte range to use for downloading a section of the blob.
Must be set if length is provided.
Pages must be aligned with 512-byte boundaries, the start offset
must be a modulus of 512 and the end offset must be a modulus of
512-1. Examples of valid byte ranges are 0-511, 512-1023, etc.
:param int end_range:
End of byte range to use for writing to a section of the blob.
must be a modulus of 512.
:param int length:
Number of bytes to use for writing to a section of the blob.
Pages must be aligned with 512-byte boundaries, the start offset
must be a modulus of 512 and the end offset must be a modulus of
512-1. Examples of valid byte ranges are 0-511, 512-1023, etc.
must be a modulus of 512 and the length must be a modulus of
512.
:param lease:
Required if the blob has an active lease. Value can be a LeaseClient object
or the lease ID as a string.
Expand Down Expand Up @@ -1722,7 +1715,7 @@ async def clear_page(self, start_range, end_range, **kwargs):
:returns: Blob-updated property dict (Etag and last modified).
:rtype: dict(str, Any)
"""
options = self._clear_page_options(start_range, end_range, **kwargs)
options = self._clear_page_options(offset, length, **kwargs)
try:
return await self._client.page_blob.clear_pages(**options) # type: ignore
except StorageErrorException as error:
Expand Down Expand Up @@ -1809,8 +1802,8 @@ async def append_block( # type: ignore

@distributed_trace_async()
async def append_block_from_url(self, copy_source_url, # type: str
source_range_start=None, # type Optional[int]
source_range_end=None, # type Optional[int]
source_offset=None, # type Optional[int]
source_length=None, # type Optional[int]
**kwargs):
# type: (...) -> Dict[str, Union[str, datetime, int]]
"""
Expand All @@ -1819,10 +1812,10 @@ async def append_block_from_url(self, copy_source_url, # type: str
:param str copy_source_url:
The URL of the source data. It can point to any Azure Blob or File, that is either public or has a
shared access signature attached.
:param int source_range_start:
:param int source_offset:
This indicates the start of the range of bytes(inclusive) that has to be taken from the copy source.
:param int source_range_end:
This indicates the end of the range of bytes(inclusive) that has to be taken from the copy source.
:param int source_length:
This indicates the end of the range of bytes that has to be taken from the copy source.
:param bytearray source_content_md5:
If given, the service will calculate the MD5 hash of the block content and compare against this value.
:param int maxsize_condition:
Expand Down Expand Up @@ -1884,6 +1877,7 @@ async def append_block_from_url(self, copy_source_url, # type: str
the operation only if the source resource does not exist, and fail the
operation if it does exist.
:param ~azure.storage.blob.CustomerProvidedEncryptionKey cpk:
Encrypts the data on the service-side with the given key.
Use of customer-provided keys must be done over HTTPS.
As the encryption key itself is provided in the request,
Expand All @@ -1893,8 +1887,8 @@ async def append_block_from_url(self, copy_source_url, # type: str
"""
options = self._append_block_from_url_options(
copy_source_url,
source_range_start=source_range_start,
source_range_end=source_range_end,
source_offset=source_offset,
source_length=source_length,
**kwargs
)
try:
Expand Down
Loading

0 comments on commit b969b5e

Please sign in to comment.