Merge STG75 into master (#15598)

* [Blob]Generate STG75 Blob Tags (#14986)

* [Blob]Generate STG75 Blob Tags

* update api version

* generator problem

* Fixed tags value

* Removed tag value altogether

Co-authored-by: tasherif-msft <[email protected]>

* [Storage][STG75]Lease on Get/Set Tags & Find Blob by Tags (#15090)

* [Storage][STG75]Lease on Get/Set Tags & Find Blob by Tags

* pylint

* [Fileshare] Share enable protocol share squash root (#15048)

* generated root squash swagger

* added api version

* added new params

* fixed valueerror

* rerecorded

* docstring fix

* fixed docstrings

* added sync tests

* added async tests

* fixed docstrings

* renamed params and class

* renamed params

* renamed enabled protocols to protocols

* fixed tests

* fixed the validation issue

* changed models to support list[str]

* added tests

* changed models type

* [Blob] Added upload blob from url feature (#15027)

* fixed swagger and added put blob

* added method signitures

* added str

* added more kwargs docstrings

* finished adding kwargs

* return type

* implemented new api

* BlockBlob client fix

* renamed param

* renamed param

* added content length

* test experiment

* added tests sync

* added all tests

* attempting to fix ci

* added if tags match cond keyword

* added more tests

* added extra header

* added more tests and removed keywords

* recorded sync tests

* added async tests

* fixed import issue

* rerecorded test

* changed docstring

* added overwrite and tests

* rerecorded

* cleaned up options method

* cleaned up options method some more

* fixed merge conflict

* marked playback only for fileshares

Co-authored-by: Xiaoxi Fu <[email protected]>

sdk/storage/azure-storage-blob/azure/storage/blob/_blob_client.py

@@ -52,7 +52,7 @@
 from ._upload_helpers import (
     upload_block_blob,
     upload_append_blob,
-    upload_page_blob)
+    upload_page_blob, _any_conditions)
 from ._models import BlobType, BlobBlock, BlobProperties, BlobQueryError
 from ._download import StorageStreamDownloader
 from ._lease import BlobLeaseClient
@@ -406,6 +406,146 @@ def _upload_blob_options(  # pylint:disable=too-many-statements
             raise ValueError("Unsupported BlobType: {}".format(blob_type))
         return kwargs
 
+    def _upload_blob_from_url_options(self, source_url, **kwargs):
+        # type: (...) -> Dict[str, Any]
+        tier = kwargs.pop('standard_blob_tier', None)
+        overwrite = kwargs.pop('overwrite', False)
+        content_settings = kwargs.pop('content_settings', None)
+        if content_settings:
+            kwargs['blob_http_headers'] = BlobHTTPHeaders(
+                blob_cache_control=content_settings.cache_control,
+                blob_content_type=content_settings.content_type,
+                blob_content_md5=None,
+                blob_content_encoding=content_settings.content_encoding,
+                blob_content_language=content_settings.content_language,
+                blob_content_disposition=content_settings.content_disposition
+            )
+        cpk = kwargs.pop('cpk', None)
+        cpk_info = None
+        if cpk:
+            if self.scheme.lower() != 'https':
+                raise ValueError("Customer provided encryption key must be used over HTTPS.")
+            cpk_info = CpkInfo(encryption_key=cpk.key_value, encryption_key_sha256=cpk.key_hash,
+                               encryption_algorithm=cpk.algorithm)
+
+        options = {
+            'content_length': 0,
+            'copy_source_blob_properties': kwargs.pop('include_source_blob_properties', True),
+            'source_content_md5': kwargs.pop('source_content_md5', None),
+            'copy_source': source_url,
+            'modified_access_conditions': get_modify_conditions(kwargs),
+            'blob_tags_string': serialize_blob_tags_header(kwargs.pop('tags', None)),
+            'cls': return_response_headers,
+            'lease_access_conditions': get_access_conditions(kwargs.pop('destination_lease', None)),
+            'tier': tier.value if tier else None,
+            'source_modified_access_conditions': get_source_conditions(kwargs),
+            'cpk_info': cpk_info,
+            'cpk_scope_info': get_cpk_scope_info(kwargs)
+        }
+        options.update(kwargs)
+        if not overwrite and not _any_conditions(**options): # pylint: disable=protected-access
+            options['modified_access_conditions'].if_none_match = '*'
+        return options
+
+    @distributed_trace
+    def upload_blob_from_url(self, source_url, **kwargs):
+        # type: (str, Any) -> Dict[str, Any]
+        """
+        Creates a new Block Blob where the content of the blob is read from a given URL.
+        The content of an existing blob is overwritten with the new blob.
+
+        :param str source_url:
+            A URL of up to 2 KB in length that specifies a file or blob.
+            The value should be URL-encoded as it would appear in a request URI.
+            If the source is in another account, the source must either be public
+            or must be authenticated via a shared access signature. If the source
+            is public, no authentication is required.
+            Examples:
+            https://myaccount.blob.core.windows.net/mycontainer/myblob
+
+            https://myaccount.blob.core.windows.net/mycontainer/myblob?snapshot=<DateTime>
+
+            https://otheraccount.blob.core.windows.net/mycontainer/myblob?sastoken
+        :keyword bool overwrite: Whether the blob to be uploaded should overwrite the current data.
+            If True, upload_blob will overwrite the existing data. If set to False, the
+            operation will fail with ResourceExistsError.
+        :keyword bool include_source_blob_properties:
+            Indicates if properties from the source blob should be copied. Defaults to True.
+        :keyword tags:
+            Name-value pairs associated with the blob as tag. Tags are case-sensitive.
+            The tag set may contain at most 10 tags.  Tag keys must be between 1 and 128 characters,
+            and tag values must be between 0 and 256 characters.
+            Valid tag key and value characters include: lowercase and uppercase letters, digits (0-9),
+            space (` `), plus (+), minus (-), period (.), solidus (/), colon (:), equals (=), underscore (_)
+        :paramtype tags: dict(str, str)
+        :keyword bytearray source_content_md5:
+            Specify the md5 that is used to verify the integrity of the source bytes.
+        :keyword ~datetime.datetime source_if_modified_since:
+            A DateTime value. Azure expects the date value passed in to be UTC.
+            If timezone is included, any non-UTC datetimes will be converted to UTC.
+            If a date is passed in without timezone info, it is assumed to be UTC.
+            Specify this header to perform the operation only
+            if the source resource has been modified since the specified time.
+        :keyword ~datetime.datetime source_if_unmodified_since:
+            A DateTime value. Azure expects the date value passed in to be UTC.
+            If timezone is included, any non-UTC datetimes will be converted to UTC.
+            If a date is passed in without timezone info, it is assumed to be UTC.
+            Specify this header to perform the operation only if
+            the source resource has not been modified since the specified date/time.
+        :keyword str source_etag:
+            The source ETag value, or the wildcard character (*). Used to check if the resource has changed,
+            and act according to the condition specified by the `match_condition` parameter.
+        :keyword ~azure.core.MatchConditions source_match_condition:
+            The source match condition to use upon the etag.
+        :keyword ~datetime.datetime if_modified_since:
+            A DateTime value. Azure expects the date value passed in to be UTC.
+            If timezone is included, any non-UTC datetimes will be converted to UTC.
+            If a date is passed in without timezone info, it is assumed to be UTC.
+            Specify this header to perform the operation only
+            if the resource has been modified since the specified time.
+        :keyword ~datetime.datetime if_unmodified_since:
+            A DateTime value. Azure expects the date value passed in to be UTC.
+            If timezone is included, any non-UTC datetimes will be converted to UTC.
+            If a date is passed in without timezone info, it is assumed to be UTC.
+            Specify this header to perform the operation only if
+            the resource has not been modified since the specified date/time.
+        :keyword str etag:
+            The destination ETag value, or the wildcard character (*). Used to check if the resource has changed,
+            and act according to the condition specified by the `match_condition` parameter.
+        :keyword ~azure.core.MatchConditions match_condition:
+            The destination match condition to use upon the etag.
+        :keyword destination_lease:
+            The lease ID specified for this header must match the lease ID of the
+            destination blob. If the request does not include the lease ID or it is not
+            valid, the operation fails with status code 412 (Precondition Failed).
+        :paramtype destination_lease: ~azure.storage.blob.BlobLeaseClient or str
+        :keyword int timeout:
+            The timeout parameter is expressed in seconds.
+        :keyword ~azure.storage.blob.ContentSettings content_settings:
+            ContentSettings object used to set blob properties. Used to set content type, encoding,
+            language, disposition, md5, and cache control.
+        :keyword ~azure.storage.blob.CustomerProvidedEncryptionKey cpk:
+            Encrypts the data on the service-side with the given key.
+            Use of customer-provided keys must be done over HTTPS.
+            As the encryption key itself is provided in the request,
+            a secure connection must be established to transfer the key.
+        :keyword str encryption_scope:
+            A predefined encryption scope used to encrypt the data on the service. An encryption
+            scope can be created using the Management API and referenced here by name. If a default
+            encryption scope has been defined at the container, this value will override it if the
+            container-level scope is configured to allow overrides. Otherwise an error will be raised.
+        :keyword ~azure.storage.blob.StandardBlobTier standard_blob_tier:
+            A standard blob tier value to set the blob to. For this version of the library,
+            this is only applicable to block blobs on standard storage accounts.
+        """
+        options = self._upload_blob_from_url_options(
+            source_url=self._encode_source_url(source_url),
+            **kwargs)
+        try:
+            return self._client.block_blob.put_blob_from_url(**options)
+        except StorageErrorException as error:
+            process_storage_error(error)
+
     @distributed_trace
     def upload_blob(  # pylint: disable=too-many-locals
             self, data,  # type: Union[Iterable[AnyStr], IO[AnyStr]]
@@ -2372,10 +2512,12 @@ def set_premium_page_blob_tier(self, premium_page_blob_tier, **kwargs):
     def _set_blob_tags_options(self, tags=None, **kwargs):
         # type: (Optional[Dict[str, str]], **Any) -> Dict[str, Any]
         tags = serialize_blob_tags(tags)
+        access_conditions = get_access_conditions(kwargs.pop('lease', None))
         mod_conditions = get_modify_conditions(kwargs)
 
         options = {
             'tags': tags,
+            'lease_access_conditions': access_conditions,
             'modified_access_conditions': mod_conditions,
             'cls': return_response_headers}
         options.update(kwargs)
@@ -2410,6 +2552,11 @@ def set_blob_tags(self, tags=None, **kwargs):
             blob.
         :keyword str if_tags_match_condition:
             Specify a SQL where clause on blob tags to operate only on destination blob with a matching value.
+            eg. ``\"\\\"tagname\\\"='my tag'\"``
+        :keyword lease:
+            Required if the blob has an active lease. Value can be a BlobLeaseClient object
+            or the lease ID as a string.
+        :paramtype lease: ~azure.storage.blob.BlobLeaseClient or str
         :keyword int timeout:
             The timeout parameter is expressed in seconds.
         :returns: Blob-updated property dict (Etag and last modified)
@@ -2423,11 +2570,13 @@ def set_blob_tags(self, tags=None, **kwargs):
 
     def _get_blob_tags_options(self, **kwargs):
         # type: (**Any) -> Dict[str, str]
+        access_conditions = get_access_conditions(kwargs.pop('lease', None))
         mod_conditions = get_modify_conditions(kwargs)
 
         options = {
             'version_id': kwargs.pop('version_id', None),
             'snapshot': self.snapshot,
+            'lease_access_conditions': access_conditions,
             'modified_access_conditions': mod_conditions,
             'timeout': kwargs.pop('timeout', None),
             'cls': return_headers_and_deserialized}
@@ -2446,6 +2595,11 @@ def get_blob_tags(self, **kwargs):
             value that, when present, specifies the version of the blob to add tags to.
         :keyword str if_tags_match_condition:
             Specify a SQL where clause on blob tags to operate only on destination blob with a matching value.
+            eg. ``\"\\\"tagname\\\"='my tag'\"``
+        :keyword lease:
+            Required if the blob has an active lease. Value can be a BlobLeaseClient object
+            or the lease ID as a string.
+        :paramtype lease: ~azure.storage.blob.BlobLeaseClient or str
         :keyword int timeout:
             The timeout parameter is expressed in seconds.
         :returns: Key value pairs of blob tags.

sdk/storage/azure-storage-blob/azure/storage/blob/_blob_service_client.py

@@ -28,7 +28,8 @@
 from ._generated.models import StorageErrorException, StorageServiceProperties, KeyInfo
 from ._container_client import ContainerClient
 from ._blob_client import BlobClient
-from ._models import ContainerPropertiesPaged, FilteredBlobPaged
+from ._models import ContainerPropertiesPaged
+from ._list_blobs_helper import FilteredBlobPaged
 from ._serialize import get_api_version
 from ._deserialize import service_stats_deserialize, service_properties_deserialize
 
@@ -47,6 +48,7 @@
         CorsRule,
         RetentionPolicy,
         StaticWebsite,
+        FilteredBlob
     )
 
 

sdk/storage/azure-storage-blob/azure/storage/blob/_generated/_azure_blob_storage.py

@@ -55,7 +55,7 @@ def __init__(self, url, **kwargs):
         self._client = PipelineClient(base_url=base_url, config=self._config, **kwargs)
 
         client_models = {k: v for k, v in models.__dict__.items() if isinstance(v, type)}
-        self.api_version = '2020-02-10'
+        self.api_version = '2020-04-08'
         self._serialize = Serializer(client_models)
         self._deserialize = Deserializer(client_models)
 

sdk/storage/azure-storage-blob/azure/storage/blob/_generated/_configuration.py

@@ -40,7 +40,7 @@ def __init__(self, url, **kwargs):
         self.generate_client_request_id = True
 
         self.url = url
-        self.version = "2020-02-10"
+        self.version = "2020-04-08"
 
     def _configure(self, **kwargs):
         self.user_agent_policy = kwargs.get('user_agent_policy') or policies.UserAgentPolicy(**kwargs)

sdk/storage/azure-storage-blob/azure/storage/blob/_generated/aio/_azure_blob_storage_async.py

@@ -56,7 +56,7 @@ def __init__(
         self._client = AsyncPipelineClient(base_url=base_url, config=self._config, **kwargs)
 
         client_models = {k: v for k, v in models.__dict__.items() if isinstance(v, type)}
-        self.api_version = '2020-02-10'
+        self.api_version = '2020-04-08'
         self._serialize = Serializer(client_models)
         self._deserialize = Deserializer(client_models)
 

sdk/storage/azure-storage-blob/azure/storage/blob/_generated/aio/_configuration_async.py

@@ -41,7 +41,7 @@ def __init__(self, url, **kwargs):
         self.accept_language = None
 
         self.url = url
-        self.version = "2020-02-10"
+        self.version = "2020-04-08"
 
     def _configure(self, **kwargs):
         self.user_agent_policy = kwargs.get('user_agent_policy') or policies.UserAgentPolicy(**kwargs)

sdk/storage/azure-storage-blob/azure/storage/blob/_generated/aio/operations_async/_blob_operations_async.py

@@ -1119,7 +1119,7 @@ async def set_expiry(self, expiry_options, timeout=None, request_id=None, expire
             header_parameters['x-ms-client-request-id'] = self._serialize.header("request_id", request_id, 'str')
         header_parameters['x-ms-expiry-option'] = self._serialize.header("expiry_options", expiry_options, 'str')
         if expires_on is not None:
-            header_parameters['x-ms-expiry-time'] = self._serialize.header("expires_on", expires_on, 'rfc-1123')
+            header_parameters['x-ms-expiry-time'] = self._serialize.header("expires_on", expires_on, 'str')
 
         # Construct and send request
         request = self._client.put(url, query_parameters, header_parameters)
@@ -2876,7 +2876,7 @@ async def query(self, query_request=None, snapshot=None, timeout=None, request_i
         return deserialized
     query.metadata = {'url': '/{containerName}/{blob}'}
 
-    async def get_tags(self, timeout=None, request_id=None, snapshot=None, version_id=None, modified_access_conditions=None, *, cls=None, **kwargs):
+    async def get_tags(self, timeout=None, request_id=None, snapshot=None, version_id=None, modified_access_conditions=None, lease_access_conditions=None, *, cls=None, **kwargs):
         """The Get Tags operation enables users to get the tags associated with a
         blob.
 
@@ -2903,6 +2903,10 @@ async def get_tags(self, timeout=None, request_id=None, snapshot=None, version_i
          operation
         :type modified_access_conditions:
          ~azure.storage.blob.models.ModifiedAccessConditions
+        :param lease_access_conditions: Additional parameters for the
+         operation
+        :type lease_access_conditions:
+         ~azure.storage.blob.models.LeaseAccessConditions
         :param callable cls: A custom type or function that will be passed the
          direct response
         :return: BlobTags or the result of cls(response)
@@ -2914,6 +2918,9 @@ async def get_tags(self, timeout=None, request_id=None, snapshot=None, version_i
         if_tags = None
         if modified_access_conditions is not None:
             if_tags = modified_access_conditions.if_tags
+        lease_id = None
+        if lease_access_conditions is not None:
+            lease_id = lease_access_conditions.lease_id
 
         comp = "tags"
 
@@ -2942,6 +2949,8 @@ async def get_tags(self, timeout=None, request_id=None, snapshot=None, version_i
             header_parameters['x-ms-client-request-id'] = self._serialize.header("request_id", request_id, 'str')
         if if_tags is not None:
             header_parameters['x-ms-if-tags'] = self._serialize.header("if_tags", if_tags, 'str')
+        if lease_id is not None:
+            header_parameters['x-ms-lease-id'] = self._serialize.header("lease_id", lease_id, 'str')
 
         # Construct and send request
         request = self._client.get(url, query_parameters, header_parameters)
@@ -2970,7 +2979,7 @@ async def get_tags(self, timeout=None, request_id=None, snapshot=None, version_i
         return deserialized
     get_tags.metadata = {'url': '/{containerName}/{blob}'}
 
-    async def set_tags(self, timeout=None, version_id=None, transactional_content_md5=None, transactional_content_crc64=None, request_id=None, tags=None, modified_access_conditions=None, *, cls=None, **kwargs):
+    async def set_tags(self, timeout=None, version_id=None, transactional_content_md5=None, transactional_content_crc64=None, request_id=None, tags=None, modified_access_conditions=None, lease_access_conditions=None, *, cls=None, **kwargs):
         """The Set Tags operation enables users to set tags on a blob.
 
         :param timeout: The timeout parameter is expressed in seconds. For
@@ -2998,6 +3007,10 @@ async def set_tags(self, timeout=None, version_id=None, transactional_content_md
          operation
         :type modified_access_conditions:
          ~azure.storage.blob.models.ModifiedAccessConditions
+        :param lease_access_conditions: Additional parameters for the
+         operation
+        :type lease_access_conditions:
+         ~azure.storage.blob.models.LeaseAccessConditions
         :param callable cls: A custom type or function that will be passed the
          direct response
         :return: None or the result of cls(response)
@@ -3009,6 +3022,9 @@ async def set_tags(self, timeout=None, version_id=None, transactional_content_md
         if_tags = None
         if modified_access_conditions is not None:
             if_tags = modified_access_conditions.if_tags
+        lease_id = None
+        if lease_access_conditions is not None:
+            lease_id = lease_access_conditions.lease_id
 
         comp = "tags"
 
@@ -3039,6 +3055,8 @@ async def set_tags(self, timeout=None, version_id=None, transactional_content_md
             header_parameters['x-ms-client-request-id'] = self._serialize.header("request_id", request_id, 'str')
         if if_tags is not None:
             header_parameters['x-ms-if-tags'] = self._serialize.header("if_tags", if_tags, 'str')
+        if lease_id is not None:
+            header_parameters['x-ms-lease-id'] = self._serialize.header("lease_id", lease_id, 'str')
 
         # Construct body
         if tags is not None: