diff --git a/sdk/eventhub/azure-eventhubs/azure/eventhub/__init__.py b/sdk/eventhub/azure-eventhubs/azure/eventhub/__init__.py index e2bcc43ed877..9766b6816ab8 100644 --- a/sdk/eventhub/azure-eventhubs/azure/eventhub/__init__.py +++ b/sdk/eventhub/azure-eventhubs/azure/eventhub/__init__.py @@ -5,21 +5,29 @@ __version__ = "1.3.1" -from azure.eventhub.common import EventData, EventHubError, EventPosition +from azure.eventhub.common import EventData, EventPosition +from azure.eventhub.error import EventHubError, EventDataError, ConnectError, AuthenticationError from azure.eventhub.client import EventHubClient from azure.eventhub.sender import Sender from azure.eventhub.receiver import Receiver -from uamqp.constants import MessageSendResult -from uamqp.constants import TransportType +from .constants import MessageSendResult +from .constants import TransportType +from .common import FIRST_AVAILABLE, NEW_EVENTS_ONLY, SharedKeyCredentials, SASTokenCredentials __all__ = [ + "__version__", "EventData", "EventHubError", + "ConnectError", + "EventDataError", + "AuthenticationError", "EventPosition", "EventHubClient", "Sender", "Receiver", "MessageSendResult", "TransportType", + "FIRST_AVAILABLE", "NEW_EVENTS_ONLY", + "SharedKeyCredentials", + "SASTokenCredentials", ] - diff --git a/sdk/eventhub/azure-eventhubs/azure/eventhub/aio/event_hubs_client_async.py b/sdk/eventhub/azure-eventhubs/azure/eventhub/aio/event_hubs_client_async.py index 275f76f6ee62..d88461c98d0c 100644 --- a/sdk/eventhub/azure-eventhubs/azure/eventhub/aio/event_hubs_client_async.py +++ b/sdk/eventhub/azure-eventhubs/azure/eventhub/aio/event_hubs_client_async.py @@ -15,7 +15,7 @@ AMQPClientAsync, ) -from azure.eventhub.common import parse_sas_token +from azure.eventhub.common import parse_sas_token, SharedKeyCredentials, SASTokenCredentials from azure.eventhub import ( EventHubError) from ..client_abstract import EventHubClientAbstract @@ -55,17 +55,19 @@ def _create_auth(self, username=None, password=None): http_proxy = self.config.http_proxy transport_type = self.config.transport_type auth_timeout = self.config.auth_timeout - if self.aad_credential and self.sas_token: - raise ValueError("Can't have both sas_token and aad_credential") - elif self.aad_credential: - get_jwt_token = functools.partial(self.aad_credential.get_token, ['https://eventhubs.azure.net//.default']) - # TODO: should use async aad_credential.get_token. Check with Charles for async identity api - return authentication.JWTTokenAsync(self.auth_uri, self.auth_uri, - get_jwt_token, http_proxy=http_proxy, - transport_type=transport_type) - elif self.sas_token: - token = self.sas_token() if callable(self.sas_token) else self.sas_token + if isinstance(self.credential, SharedKeyCredentials): + username = username or self._auth_config['username'] + password = password or self._auth_config['password'] + if "@sas.root" in username: + return authentication.SASLPlain( + self.host, username, password, http_proxy=http_proxy, transport_type=transport_type) + return authentication.SASTokenAsync.from_shared_access_key( + self.auth_uri, username, password, timeout=auth_timeout, http_proxy=http_proxy, + transport_type=transport_type) + + elif isinstance(self.credential, SASTokenCredentials): + token = self.credential.get_sas_token() try: expiry = int(parse_sas_token(token)['se']) except (KeyError, TypeError, IndexError): @@ -77,15 +79,14 @@ def _create_auth(self, username=None, password=None): http_proxy=http_proxy, transport_type=transport_type) - username = username or self._auth_config['username'] - password = password or self._auth_config['password'] - if "@sas.root" in username: - return authentication.SASLPlain( - self.address.hostname, username, password, http_proxy=http_proxy, transport_type=transport_type) - return authentication.SASTokenAsync.from_shared_access_key( - self.auth_uri, username, password, timeout=auth_timeout, http_proxy=http_proxy, transport_type=transport_type) + else: + get_jwt_token = functools.partial(self.credential.get_token, ['https://eventhubs.azure.net//.default']) + return authentication.JWTTokenAsync(self.auth_uri, self.auth_uri, + get_jwt_token, http_proxy=http_proxy, + transport_type=transport_type) + - async def get_eventhub_information(self): + async def get_properties(self): """ Get details on the specified EventHub async. @@ -108,18 +109,67 @@ async def get_eventhub_information(self): eh_info = response.get_data() output = {} if eh_info: - output['name'] = eh_info[b'name'].decode('utf-8') - output['type'] = eh_info[b'type'].decode('utf-8') - output['created_at'] = datetime.datetime.fromtimestamp(float(eh_info[b'created_at'])/1000) - output['partition_count'] = eh_info[b'partition_count'] + output['path'] = eh_info[b'name'].decode('utf-8') + output['created_at'] = datetime.datetime.utcfromtimestamp(float(eh_info[b'created_at'])/1000) output['partition_ids'] = [p.decode('utf-8') for p in eh_info[b'partition_ids']] return output finally: await mgmt_client.close_async() + + async def get_partition_ids(self): + return (await self.get_properties())['partition_ids'] + + async def get_partition_properties(self, partition): + """ + Get information on the specified partition async. + Keys in the details dictionary include: + + -'name' + -'type' + -'partition' + -'begin_sequence_number' + -'last_enqueued_sequence_number' + -'last_enqueued_offset' + -'last_enqueued_time_utc' + -'is_partition_empty' + + :param partition: The target partition id. + :type partition: str + :rtype: dict + """ + alt_creds = { + "username": self._auth_config.get("iot_username"), + "password": self._auth_config.get("iot_password")} + try: + mgmt_auth = self._create_auth(**alt_creds) + mgmt_client = AMQPClientAsync(self.mgmt_target, auth=mgmt_auth, debug=self.debug) + await mgmt_client.open_async() + mgmt_msg = Message(application_properties={'name': self.eh_name, + 'partition': partition}) + response = await mgmt_client.mgmt_request_async( + mgmt_msg, + constants.READ_OPERATION, + op_type=b'com.microsoft:partition', + status_code_field=b'status-code', + description_fields=b'status-description') + partition_info = response.get_data() + output = {} + if partition_info: + output['event_hub_path'] = partition_info[b'name'].decode('utf-8') + output['id'] = partition_info[b'partition'].decode('utf-8') + output['beginning_sequence_number'] = partition_info[b'begin_sequence_number'] + output['last_enqueued_sequence_number'] = partition_info[b'last_enqueued_sequence_number'] + output['last_enqueued_offset'] = partition_info[b'last_enqueued_offset'].decode('utf-8') + output['last_enqueued_time_utc'] = datetime.datetime.utcfromtimestamp( + float(partition_info[b'last_enqueued_time_utc'] / 1000)) + output['is_empty'] = partition_info[b'is_partition_empty'] + return output + finally: + await mgmt_client.close_async() def create_receiver( - self, consumer_group, partition, offset=None, epoch=None, operation=None, - prefetch=None, keep_alive=None, auto_reconnect=None, loop=None): + self, partition_id, consumer_group="$Default", event_position=None, exclusive_receiver_priority=None, operation=None, + prefetch=None, loop=None): """ Add an async receiver to the client for a particular consumer group and partition. @@ -127,8 +177,8 @@ def create_receiver( :type consumer_group: str :param partition: The ID of the partition. :type partition: str - :param offset: The offset from which to start receiving. - :type offset: ~azure.eventhub.common.Offset + :param event_position: The position from which to start receiving. + :type event_position: ~azure.eventhub.common.EventPosition :param prefetch: The message prefetch count of the receiver. Default is 300. :type prefetch: int :operation: An optional operation to be appended to the hostname in the source URL. @@ -145,53 +195,18 @@ def create_receiver( :caption: Add an async receiver to the client for a particular consumer group and partition. """ - keep_alive = self.config.keep_alive if keep_alive is None else keep_alive - auto_reconnect = self.config.auto_reconnect if auto_reconnect is None else auto_reconnect prefetch = self.config.prefetch if prefetch is None else prefetch path = self.address.path + operation if operation else self.address.path source_url = "amqps://{}{}/ConsumerGroups/{}/Partitions/{}".format( - self.address.hostname, path, consumer_group, partition) + self.address.hostname, path, consumer_group, partition_id) handler = Receiver( - self, source_url, offset=offset, epoch=epoch, prefetch=prefetch, keep_alive=keep_alive, - auto_reconnect=auto_reconnect, loop=loop) + self, source_url, offset=event_position, exclusive_receiver_priority=exclusive_receiver_priority, + prefetch=prefetch, loop=loop) return handler - def create_epoch_receiver( - self, consumer_group, partition, epoch, prefetch=300, operation=None, loop=None): - """ - Add an async receiver to the client with an epoch value. Only a single epoch receiver - can connect to a partition at any given time - additional epoch receivers must have - a higher epoch value or they will be rejected. If a 2nd epoch receiver has - connected, the first will be closed. - - :param consumer_group: The name of the consumer group. - :type consumer_group: str - :param partition: The ID of the partition. - :type partition: str - :param epoch: The epoch value for the receiver. - :type epoch: int - :param prefetch: The message prefetch count of the receiver. Default is 300. - :type prefetch: int - :operation: An optional operation to be appended to the hostname in the source URL. - The value must start with `/` character. - :type operation: str - :rtype: ~azure.eventhub.aio.receiver_async.ReceiverAsync - - Example: - .. literalinclude:: ../examples/async_examples/test_examples_eventhub_async.py - :start-after: [START create_eventhub_client_async_epoch_receiver] - :end-before: [END create_eventhub_client_async_epoch_receiver] - :language: python - :dedent: 4 - :caption: Add an async receiver to the client with an epoch value. - - """ - return self.create_receiver(consumer_group, partition, epoch=epoch, prefetch=prefetch, - operation=operation, loop=loop) - def create_sender( - self, partition=None, operation=None, send_timeout=None, keep_alive=None, auto_reconnect=None, loop=None): + self, partition_id=None, operation=None, send_timeout=None, loop=None): """ Add an async sender to the client to send ~azure.eventhub.common.EventData object to an EventHub. @@ -229,10 +244,7 @@ def create_sender( if operation: target = target + operation send_timeout = self.config.send_timeout if send_timeout is None else send_timeout - keep_alive = self.config.keep_alive if keep_alive is None else keep_alive - auto_reconnect = self.config.auto_reconnect if auto_reconnect is None else auto_reconnect handler = Sender( - self, target, partition=partition, send_timeout=send_timeout, - keep_alive=keep_alive, auto_reconnect=auto_reconnect, loop=loop) + self, target, partition=partition_id, send_timeout=send_timeout, loop=loop) return handler diff --git a/sdk/eventhub/azure-eventhubs/azure/eventhub/aio/receiver_async.py b/sdk/eventhub/azure-eventhubs/azure/eventhub/aio/receiver_async.py index aafe4c8dcd20..6614001dc93d 100644 --- a/sdk/eventhub/azure-eventhubs/azure/eventhub/aio/receiver_async.py +++ b/sdk/eventhub/azure-eventhubs/azure/eventhub/aio/receiver_async.py @@ -11,7 +11,7 @@ from uamqp import ReceiveClientAsync, Source from azure.eventhub import EventHubError, EventData -from azure.eventhub.common import _error_handler +from azure.eventhub.error import EventHubError, AuthenticationError, ConnectError, _error_handler log = logging.getLogger(__name__) @@ -33,8 +33,8 @@ class Receiver(object): _epoch = b'com.microsoft:epoch' def __init__( # pylint: disable=super-init-not-called - self, client, source, offset=None, prefetch=300, epoch=None, - keep_alive=None, auto_reconnect=False, loop=None): + self, client, source, offset=None, prefetch=300, exclusive_receiver_priority=None, + keep_alive=None, auto_reconnect=True, loop=None): """ Instantiate an async receiver. @@ -54,8 +54,9 @@ def __init__( # pylint: disable=super-init-not-called self.client = client self.source = source self.offset = offset + self.messages_iter = None self.prefetch = prefetch - self.epoch = epoch + self.exclusive_receiver_priority = exclusive_receiver_priority self.keep_alive = keep_alive self.auto_reconnect = auto_reconnect self.retry_policy = errors.ErrorPolicy(max_retries=self.client.config.max_retries, on_error=_error_handler) @@ -68,8 +69,8 @@ def __init__( # pylint: disable=super-init-not-called source = Source(self.source) if self.offset is not None: source.set_filter(self.offset.selector()) - if epoch: - self.properties = {types.AMQPSymbol(self._epoch): types.AMQPLong(int(epoch))} + if exclusive_receiver_priority: + self.properties = {types.AMQPSymbol(self._epoch): types.AMQPLong(int(exclusive_receiver_priority))} self._handler = ReceiveClientAsync( source, auth=self.client.get_auth(), @@ -80,7 +81,7 @@ def __init__( # pylint: disable=super-init-not-called error_policy=self.retry_policy, keep_alive_interval=self.keep_alive, client_name=self.name, - properties=self.client.create_properties(), + properties=self.client.create_properties(self.client.config.user_agent), loop=self.loop) async def __aenter__(self): @@ -90,12 +91,14 @@ async def __aexit__(self, exc_type, exc_val, exc_tb): await self.close(exc_val) def __aiter__(self): - self.messages_iter = self._handler.receive_messages_iter_async() return self async def __anext__(self): + await self._open() while True: try: + if not self.messages_iter: + self.messages_iter = self._handler.receive_messages_iter_async() message = await self.messages_iter.__anext__() event_data = EventData(message=message) self.offset = event_data.offset @@ -107,25 +110,32 @@ async def __anext__(self): if shutdown.action.retry and self.auto_reconnect: log.info("Receiver detached. Attempting reconnect.") await self.reconnect() - log.info("Receiver detached. Shutting down.") - error = EventHubError(str(shutdown), shutdown) - await self.close(exception=error) - raise error + else: + log.info("Receiver detached. Shutting down.") + error = ConnectError(str(shutdown), shutdown) + await self.close(exception=error) + raise error except errors.MessageHandlerError as shutdown: if self.auto_reconnect: log.info("Receiver detached. Attempting reconnect.") await self.reconnect() - log.info("Receiver detached. Shutting down.") - error = EventHubError(str(shutdown), shutdown) - await self.close(exception=error) - raise error + else: + log.info("Receiver detached. Shutting down.") + error = ConnectError(str(shutdown), shutdown) + await self.close(exception=error) + raise error + except StopAsyncIteration: + raise + except asyncio.CancelledError: + # TODO: stop self.message_iter + raise except Exception as e: log.info("Unexpected error occurred (%r). Shutting down.", e) error = EventHubError("Receive failed: {}".format(e)) await self.close(exception=error) raise error - async def open(self): + async def _open(self): """ Open the Receiver using the supplied conneciton. If the handler has previously been redirected, the redirect @@ -144,7 +154,6 @@ async def open(self): """ # pylint: disable=protected-access - self.running = True if self.redirected: self.source = self.redirected.address source = Source(self.source) @@ -156,18 +165,46 @@ async def open(self): self._handler = ReceiveClientAsync( source, auth=self.client.get_auth(**alt_creds), - debug=self.client.debug, + debug=self.client.config.network_tracing, prefetch=self.prefetch, link_properties=self.properties, timeout=self.timeout, error_policy=self.retry_policy, keep_alive_interval=self.keep_alive, client_name=self.name, - properties=self.client.create_properties(), + properties=self.client.create_properties(self.client.config.user_agent), loop=self.loop) - await self._handler.open_async() - while not await self._handler.client_ready_async(): - await asyncio.sleep(0.05) + + if not self.running: + try: + await self._handler.open_async() + self.running = True + while not await self._handler.client_ready_async(): + await asyncio.sleep(0.05) + except errors.AuthenticationException: + log.info("Receiver failed authentication. Retrying...") + await self.reconnect() + except (errors.LinkDetach, errors.ConnectionClose) as shutdown: + if shutdown.action.retry and self.auto_reconnect: + log.info("Receiver detached. Attempting reconnect.") + await self.reconnect() + else: + log.info("Receiver detached. Failed to connect") + error = ConnectError(str(shutdown), shutdown) + raise error + except errors.AMQPConnectionError as shutdown: + if str(shutdown).startswith("Unable to open authentication session") and self.auto_reconnect: + log.info("Receiver couldn't authenticate (%r).", shutdown) + error = AuthenticationError(str(shutdown)) + raise error + else: + log.info("Receiver connection error (%r).", shutdown) + error = ConnectError(str(shutdown)) + raise error + except Exception as e: + log.info("Unexpected error occurred (%r)", e) + error = EventHubError("Receiver connect failed: {}".format(e)) + raise error async def _reconnect(self): # pylint: disable=too-many-statements # pylint: disable=protected-access @@ -181,23 +218,24 @@ async def _reconnect(self): # pylint: disable=too-many-statements self._handler = ReceiveClientAsync( source, auth=self.client.get_auth(**alt_creds), - debug=self.client.debug, + debug=self.client.config.network_tracing, prefetch=self.prefetch, link_properties=self.properties, timeout=self.timeout, error_policy=self.retry_policy, keep_alive_interval=self.keep_alive, client_name=self.name, - properties=self.client.create_properties(), + properties=self.client.create_properties(self.client.config.user_agent), loop=self.loop) + self.messages_iter = None try: await self._handler.open_async() while not await self._handler.client_ready_async(): await asyncio.sleep(0.05) return True - except errors.TokenExpired as shutdown: + except errors.AuthenticationException as shutdown: log.info("AsyncReceiver disconnected due to token expiry. Shutting down.") - error = EventHubError(str(shutdown), shutdown) + error = AuthenticationError(str(shutdown), shutdown) await self.close(exception=error) raise error except (errors.LinkDetach, errors.ConnectionClose) as shutdown: @@ -205,7 +243,7 @@ async def _reconnect(self): # pylint: disable=too-many-statements log.info("AsyncReceiver detached. Attempting reconnect.") return False log.info("AsyncReceiver detached. Shutting down.") - error = EventHubError(str(shutdown), shutdown) + error = ConnectError(str(shutdown), shutdown) await self.close(exception=error) raise error except errors.MessageHandlerError as shutdown: @@ -213,7 +251,7 @@ async def _reconnect(self): # pylint: disable=too-many-statements log.info("AsyncReceiver detached. Attempting reconnect.") return False log.info("AsyncReceiver detached. Shutting down.") - error = EventHubError(str(shutdown), shutdown) + error = ConnectError(str(shutdown), shutdown) await self.close(exception=error) raise error except errors.AMQPConnectionError as shutdown: @@ -221,7 +259,7 @@ async def _reconnect(self): # pylint: disable=too-many-statements log.info("AsyncReceiver couldn't authenticate. Attempting reconnect.") return False log.info("AsyncReceiver connection error (%r). Shutting down.", shutdown) - error = EventHubError(str(shutdown)) + error = ConnectError(str(shutdown)) await self.close(exception=error) raise error except Exception as e: @@ -233,7 +271,7 @@ async def _reconnect(self): # pylint: disable=too-many-statements async def reconnect(self): """If the Receiver was disconnected from the service with a retryable error - attempt to reconnect.""" - while not await self._reconnect_async(): + while not await self._reconnect(): await asyncio.sleep(self.reconnect_backoff) async def close(self, exception=None): @@ -263,13 +301,25 @@ async def close(self, exception=None): elif isinstance(exception, EventHubError): self.error = exception elif isinstance(exception, (errors.LinkDetach, errors.ConnectionClose)): - self.error = EventHubError(str(exception), exception) + self.error = ConnectError(str(exception), exception) elif exception: self.error = EventHubError(str(exception)) else: self.error = EventHubError("This receive handler is now closed.") await self._handler.close_async() + @property + def queue_size(self): + """ + The current size of the unprocessed Event queue. + + :rtype: int + """ + # pylint: disable=protected-access + if self._handler._received_messages: + return self._handler._received_messages.qsize() + return 0 + async def receive(self, max_batch_size=None, timeout=None): """ Receive events asynchronously from the EventHub. @@ -294,83 +344,41 @@ async def receive(self, max_batch_size=None, timeout=None): """ if self.error: raise self.error - if not self.running: - await self.open() - data_batch = [] - try: - timeout_ms = 1000 * timeout if timeout else 0 - message_batch = await self._handler.receive_message_batch_async( - max_batch_size=max_batch_size, - timeout=timeout_ms) - for message in message_batch: - event_data = EventData(message=message) - self.offset = event_data.offset - data_batch.append(event_data) - return data_batch - except (errors.TokenExpired, errors.AuthenticationException): - log.info("AsyncReceiver disconnected due to token error. Attempting reconnect.") - await self.reconnect() - return data_batch - except (errors.LinkDetach, errors.ConnectionClose) as shutdown: - if shutdown.action.retry and self.auto_reconnect: - log.info("AsyncReceiver detached. Attempting reconnect.") - await self.reconnect() - return data_batch - log.info("AsyncReceiver detached. Shutting down.") - error = EventHubError(str(shutdown), shutdown) - await self.close(exception=error) - raise error - except errors.MessageHandlerError as shutdown: - if self.auto_reconnect: - log.info("AsyncReceiver detached. Attempting reconnect.") - await self.reconnect() - return data_batch - log.info("AsyncReceiver detached. Shutting down.") - error = EventHubError(str(shutdown), shutdown) - await self.close(exception=error) - raise error - except Exception as e: - log.info("Unexpected error occurred (%r). Shutting down.", e) - error = EventHubError("Receive failed: {}".format(e)) - await self.close(exception=error) - raise error - - async def __aenter__(self): - return self - - async def __aexit__(self, exc_type, exc_val, exc_tb): - await self.close(exc_val) + await self._open() - def __aiter__(self): - self.messages_iter = self._handler.receive_messages_iter_async() - return self - - async def __anext__(self): + data_batch = [] while True: try: - message = await self.messages_iter.__anext__() - event_data = EventData(message=message) - self.offset = event_data.offset - return event_data + timeout_ms = 1000 * timeout if timeout else 0 + message_batch = await self._handler.receive_message_batch_async( + max_batch_size=max_batch_size, + timeout=timeout_ms) + for message in message_batch: + event_data = EventData(message=message) + self.offset = event_data.offset + data_batch.append(event_data) + return data_batch except (errors.TokenExpired, errors.AuthenticationException): - log.info("Receiver disconnected due to token error. Attempting reconnect.") + log.info("AsyncReceiver disconnected due to token error. Attempting reconnect.") await self.reconnect() except (errors.LinkDetach, errors.ConnectionClose) as shutdown: if shutdown.action.retry and self.auto_reconnect: - log.info("Receiver detached. Attempting reconnect.") + log.info("AsyncReceiver detached. Attempting reconnect.") await self.reconnect() - log.info("Receiver detached. Shutting down.") - error = EventHubError(str(shutdown), shutdown) - await self.close(exception=error) - raise error + else: + log.info("AsyncReceiver detached. Shutting down.") + error = ConnectError(str(shutdown), shutdown) + await self.close(exception=error) + raise error except errors.MessageHandlerError as shutdown: if self.auto_reconnect: - log.info("Receiver detached. Attempting reconnect.") + log.info("AsyncReceiver detached. Attempting reconnect.") await self.reconnect() - log.info("Receiver detached. Shutting down.") - error = EventHubError(str(shutdown), shutdown) - await self.close(exception=error) - raise error + else: + log.info("AsyncReceiver detached. Shutting down.") + error = ConnectError(str(shutdown), shutdown) + await self.close(exception=error) + raise error except Exception as e: log.info("Unexpected error occurred (%r). Shutting down.", e) error = EventHubError("Receive failed: {}".format(e)) diff --git a/sdk/eventhub/azure-eventhubs/azure/eventhub/aio/sender_async.py b/sdk/eventhub/azure-eventhubs/azure/eventhub/aio/sender_async.py index 0ef46d519579..e263131ff859 100644 --- a/sdk/eventhub/azure-eventhubs/azure/eventhub/aio/sender_async.py +++ b/sdk/eventhub/azure-eventhubs/azure/eventhub/aio/sender_async.py @@ -12,7 +12,9 @@ from azure.eventhub import MessageSendResult from azure.eventhub import EventHubError -from azure.eventhub.common import _error_handler, _BatchSendEventData +from azure.eventhub.common import EventData, _BatchSendEventData +from azure.eventhub.error import EventHubError, ConnectError, \ + AuthenticationError, EventDataError, _error_handler log = logging.getLogger(__name__) @@ -33,7 +35,7 @@ class Sender(object): def __init__( # pylint: disable=super-init-not-called self, client, target, partition=None, send_timeout=60, - keep_alive=None, auto_reconnect=False, loop=None): + keep_alive=None, auto_reconnect=True, loop=None): """ Instantiate an EventHub event SenderAsync handler. @@ -74,12 +76,12 @@ def __init__( # pylint: disable=super-init-not-called self._handler = SendClientAsync( self.target, auth=self.client.get_auth(), - debug=self.client.debug, + debug=self.client.config.network_tracing, msg_timeout=self.timeout, error_policy=self.retry_policy, keep_alive_interval=self.keep_alive, client_name=self.name, - properties=self.client.create_properties(), + properties=self.client.create_properties(self.client.config.user_agent), loop=self.loop) self._outcome = None self._condition = None @@ -90,7 +92,7 @@ async def __aenter__(self): async def __aexit__(self, exc_type, exc_val, exc_tb): await self.close(exc_val) - async def open(self): + async def _open(self): """ Open the Sender using the supplied conneciton. If the handler has previously been redirected, the redirect @@ -108,22 +110,48 @@ async def open(self): :caption: Open the Sender using the supplied conneciton. """ - self.running = True if self.redirected: self.target = self.redirected.address self._handler = SendClientAsync( self.target, auth=self.client.get_auth(), - debug=self.client.debug, + debug=self.client.config.network_tracing, msg_timeout=self.timeout, error_policy=self.retry_policy, keep_alive_interval=self.keep_alive, client_name=self.name, - properties=self.client.create_properties(), + properties=self.client.create_properties(self.client.config.user_agent), loop=self.loop) - await self._handler.open_async() - while not await self._handler.client_ready_async(): - await asyncio.sleep(0.05) + if not self.running: + try: + await self._handler.open_async() + self.running = True + while not await self._handler.client_ready_async(): + await asyncio.sleep(0.05) + except errors.AuthenticationException: + log.info("Sender failed authentication. Retrying...") + await self.reconnect() + except (errors.LinkDetach, errors.ConnectionClose) as shutdown: + if shutdown.action.retry and self.auto_reconnect: + log.info("Sender detached. Attempting reconnect.") + await self.reconnect() + else: + log.info("Sender detached. Failed to connect") + error = ConnectError(str(shutdown), shutdown) + raise error + except errors.AMQPConnectionError as shutdown: + if str(shutdown).startswith("Unable to open authentication session") and self.auto_reconnect: + log.info("Sender couldn't authenticate.", shutdown) + error = AuthenticationError(str(shutdown)) + raise error + else: + log.info("Sender connection error (%r).", shutdown) + error = ConnectError(str(shutdown)) + raise error + except Exception as e: + log.info("Unexpected error occurred (%r)", e) + error = EventHubError("Sender connect failed: {}".format(e)) + raise error async def _reconnect(self): await self._handler.close_async() @@ -131,21 +159,23 @@ async def _reconnect(self): self._handler = SendClientAsync( self.target, auth=self.client.get_auth(), - debug=self.client.debug, + debug=self.client.config.network_tracing, msg_timeout=self.timeout, error_policy=self.retry_policy, keep_alive_interval=self.keep_alive, client_name=self.name, - properties=self.client.create_properties(), + properties=self.client.create_properties(self.client.config.user_agent), loop=self.loop) try: await self._handler.open_async() + while not await self._handler.client_ready_async(): + await asyncio.sleep(0.05) self._handler.queue_message(*unsent_events) await self._handler.wait_async() return True - except errors.TokenExpired as shutdown: + except errors.AuthenticationException as shutdown: log.info("AsyncSender disconnected due to token expiry. Shutting down.") - error = EventHubError(str(shutdown), shutdown) + error = AuthenticationError(str(shutdown), shutdown) await self.close(exception=error) raise error except (errors.LinkDetach, errors.ConnectionClose) as shutdown: @@ -153,7 +183,7 @@ async def _reconnect(self): log.info("AsyncSender detached. Attempting reconnect.") return False log.info("AsyncSender reconnect failed. Shutting down.") - error = EventHubError(str(shutdown), shutdown) + error = ConnectError(str(shutdown), shutdown) await self.close(exception=error) raise error except errors.MessageHandlerError as shutdown: @@ -161,7 +191,7 @@ async def _reconnect(self): log.info("AsyncSender detached. Attempting reconnect.") return False log.info("AsyncSender reconnect failed. Shutting down.") - error = EventHubError(str(shutdown), shutdown) + error = ConnectError(str(shutdown), shutdown) await self.close(exception=error) raise error except errors.AMQPConnectionError as shutdown: @@ -169,7 +199,7 @@ async def _reconnect(self): log.info("AsyncSender couldn't authenticate. Attempting reconnect.") return False log.info("AsyncSender connection error (%r). Shutting down.", shutdown) - error = EventHubError(str(shutdown)) + error = ConnectError(str(shutdown)) await self.close(exception=error) raise error except Exception as e: @@ -211,7 +241,7 @@ async def close(self, exception=None): elif isinstance(exception, EventHubError): self.error = exception elif isinstance(exception, (errors.LinkDetach, errors.ConnectionClose)): - self.error = EventHubError(str(exception), exception) + self.error = ConnectError(str(exception), exception) elif exception: self.error = EventHubError(str(exception)) else: @@ -219,14 +249,13 @@ async def close(self, exception=None): await self._handler.close_async() async def _send_event_data(self, event_data): - if not self.running: - await self.open() + await self._open() try: self._handler.send_message(event_data.message) if self._outcome != MessageSendResult.Ok: raise Sender._error(self._outcome, self._condition) except errors.MessageException as failed: - error = EventHubError(str(failed), failed) + error = EventDataError(str(failed), failed) await self.close(exception=error) raise error except (errors.TokenExpired, errors.AuthenticationException): @@ -238,7 +267,7 @@ async def _send_event_data(self, event_data): await self.reconnect() else: log.info("Sender detached. Shutting down.") - error = EventHubError(str(shutdown), shutdown) + error = ConnectError(str(shutdown), shutdown) await self.close(exception=error) raise error except errors.MessageHandlerError as shutdown: @@ -247,7 +276,7 @@ async def _send_event_data(self, event_data): await self.reconnect() else: log.info("Sender detached. Shutting down.") - error = EventHubError(str(shutdown), shutdown) + error = ConnectError(str(shutdown), shutdown) await self.close(exception=error) raise error except Exception as e: @@ -258,34 +287,14 @@ async def _send_event_data(self, event_data): else: return self._outcome - async def send(self, event_data): - """ - Sends an event data and asynchronously waits until - acknowledgement is received or operation times out. - - :param event_data: The event to be sent. - :type event_data: ~azure.eventhub.common.EventData - :raises: ~azure.eventhub.common.EventHubError if the message fails to - send. - - Example: - .. literalinclude:: ../examples/async_examples/test_examples_eventhub_async.py - :start-after: [START eventhub_client_async_send] - :end-before: [END eventhub_client_async_send] - :language: python - :dedent: 4 - :caption: Sends an event data and asynchronously waits - until acknowledgement is received or operation times out. - - """ - if self.error: - raise self.error - if event_data.partition_key and self.partition: - raise ValueError("EventData partition key cannot be used with a partition sender.") - event_data.message.on_send_complete = self._on_outcome - await self._send_event_data(event_data) + @staticmethod + def _set_batching_label(event_datas, batching_label): + ed_iter = iter(event_datas) + for ed in ed_iter: + ed._batching_label = batching_label + yield ed - async def send_batch(self, batch_event_data): + async def send(self, event_data, batching_label=None): """ Sends an event data and blocks until acknowledgement is received or operation times out. @@ -308,87 +317,16 @@ async def send_batch(self, batch_event_data): """ if self.error: raise self.error - - def verify_partition(event_datas): - ed_iter = iter(event_datas) - try: - ed = next(ed_iter) - partition_key = ed.partition_key - yield ed - except StopIteration: - raise ValueError("batch_event_data must not be empty") - for ed in ed_iter: - if ed.partition_key != partition_key: - raise ValueError("partition key of all EventData must be the same if being sent in a batch") - yield ed - - wrapper_event_data = _BatchSendEventData(verify_partition(batch_event_data)) + if isinstance(event_data, EventData): + if batching_label: + event_data._batching_label = batching_label + wrapper_event_data = event_data + else: + wrapper_event_data = _BatchSendEventData( + self._set_batching_label(event_data, batching_label), + batching_label=batching_label) if batching_label else _BatchSendEventData(event_data) wrapper_event_data.message.on_send_complete = self._on_outcome - return await self._send_event_data(wrapper_event_data) - - def queue_message(self, event_data, callback=None): - """ - Transfers an event data and notifies the callback when the operation is done. - - :param event_data: The event to be sent. - :type event_data: ~azure.eventhub.common.EventData - :param callback: Callback to be run once the message has been send. - This must be a function that accepts two arguments. - :type callback: callable[~uamqp.constants.MessageSendResult, ~azure.eventhub.common.EventHubError] - - Example: - .. literalinclude:: ../examples/test_examples_eventhub.py - :start-after: [START eventhub_client_transfer] - :end-before: [END eventhub_client_transfer] - :language: python - :dedent: 4 - :caption: Transfers an event data and notifies the callback when the operation is done. - - """ - if self.error: - raise self.error - if not self.running: - self.open() - if event_data.partition_key and self.partition: - raise ValueError("EventData partition key cannot be used with a partition sender.") - if callback: - event_data.message.on_send_complete = lambda o, c: callback(o, Sender._error(o, c)) - self._handler.queue_message(event_data.message) - - async def send_pending_messages(self): - """ - Wait until all transferred events have been sent. - """ - if self.error: - raise self.error - if not self.running: - raise ValueError("Unable to send until client has been started.") - try: - await self._handler.wait_async() - except (errors.TokenExpired, errors.AuthenticationException): - log.info("AsyncSender disconnected due to token error. Attempting reconnect.") - await self.reconnect() - except (errors.LinkDetach, errors.ConnectionClose) as shutdown: - if shutdown.action.retry and self.auto_reconnect: - log.info("AsyncSender detached. Attempting reconnect.") - await self.reconnect() - else: - log.info("AsyncSender detached. Shutting down.") - error = EventHubError(str(shutdown), shutdown) - await self.close(exception=error) - raise error - except errors.MessageHandlerError as shutdown: - if self.auto_reconnect: - log.info("AsyncSender detached. Attempting reconnect.") - await self.reconnect() - else: - log.info("AsyncSender detached. Shutting down.") - error = EventHubError(str(shutdown), shutdown) - await self.close(exception=error) - raise error - except Exception as e: - log.info("Unexpected error occurred (%r).", e) - raise EventHubError("Send failed: {}".format(e)) + await self._send_event_data(wrapper_event_data) def _on_outcome(self, outcome, condition): """ diff --git a/sdk/eventhub/azure-eventhubs/azure/eventhub/client.py b/sdk/eventhub/azure-eventhubs/azure/eventhub/client.py index 58cd975e4baf..8fb7940850e9 100644 --- a/sdk/eventhub/azure-eventhubs/azure/eventhub/client.py +++ b/sdk/eventhub/azure-eventhubs/azure/eventhub/client.py @@ -17,15 +17,17 @@ from urllib.parse import urlparse, unquote_plus, urlencode, quote_plus import uamqp -from uamqp import Message +from uamqp import Message, AMQPClient from uamqp import authentication from uamqp import constants from azure.eventhub import __version__ from azure.eventhub.sender import Sender from azure.eventhub.receiver import Receiver -from azure.eventhub.common import EventHubError, parse_sas_token +from azure.eventhub.common import parse_sas_token +from azure.eventhub.error import EventHubError from .client_abstract import EventHubClientAbstract +from .common import SASTokenCredentials, SharedKeyCredentials log = logging.getLogger(__name__) @@ -59,16 +61,20 @@ def _create_auth(self, username=None, password=None): http_proxy = self.config.http_proxy transport_type = self.config.transport_type auth_timeout = self.config.auth_timeout - if self.aad_credential and self.sas_token: - raise ValueError("Can't have both sas_token and aad_credential") - elif self.aad_credential: - get_jwt_token = functools.partial(self.aad_credential.get_token, ['https://eventhubs.azure.net//.default']) - return authentication.JWTTokenAuth(self.auth_uri, self.auth_uri, - get_jwt_token, http_proxy=http_proxy, - transport_type=transport_type) - elif self.sas_token: - token = self.sas_token() if callable(self.sas_token) else self.sas_token + # TODO: the following code can be refactored to create auth from classes directly instead of using if-else + if isinstance(self.credential, SharedKeyCredentials): + username = username or self._auth_config['username'] + password = password or self._auth_config['password'] + if "@sas.root" in username: + return authentication.SASLPlain( + self.host, username, password, http_proxy=http_proxy, transport_type=transport_type) + return authentication.SASTokenAuth.from_shared_access_key( + self.auth_uri, username, password, timeout=auth_timeout, http_proxy=http_proxy, + transport_type=transport_type) + + elif isinstance(self.credential, SASTokenCredentials): + token = self.credential.get_sas_token() try: expiry = int(parse_sas_token(token)['se']) except (KeyError, TypeError, IndexError): @@ -80,15 +86,15 @@ def _create_auth(self, username=None, password=None): http_proxy=http_proxy, transport_type=transport_type) - username = username or self._auth_config['username'] - password = password or self._auth_config['password'] - if "@sas.root" in username: - return authentication.SASLPlain( - self.address.hostname, username, password, http_proxy=http_proxy, transport_type=transport_type) - return authentication.SASTokenAuth.from_shared_access_key( - self.auth_uri, username, password, timeout=auth_timeout, http_proxy=http_proxy, transport_type=transport_type) + else: # Azure credential + get_jwt_token = functools.partial(self.credential.get_token, + ['https://eventhubs.azure.net//.default']) + return authentication.JWTTokenAuth(self.auth_uri, self.auth_uri, + get_jwt_token, http_proxy=http_proxy, + transport_type=transport_type) + - def get_eventhub_information(self): + def get_properties(self): """ Get details on the specified EventHub. Keys in the details dictionary include: @@ -118,20 +124,68 @@ def get_eventhub_information(self): eh_info = response.get_data() output = {} if eh_info: - output['name'] = eh_info[b'name'].decode('utf-8') - output['type'] = eh_info[b'type'].decode('utf-8') - output['created_at'] = datetime.datetime.fromtimestamp(float(eh_info[b'created_at'])/1000) - output['partition_count'] = eh_info[b'partition_count'] + output['path'] = eh_info[b'name'].decode('utf-8') + output['created_at'] = datetime.datetime.utcfromtimestamp(float(eh_info[b'created_at'])/1000) output['partition_ids'] = [p.decode('utf-8') for p in eh_info[b'partition_ids']] return output finally: mgmt_client.close() + def get_partition_ids(self): + return self.get_properties()['partition_ids'] + + def get_partition_properties(self, partition): + """ + Get information on the specified partition async. + Keys in the details dictionary include: + + -'name' + -'type' + -'partition' + -'begin_sequence_number' + -'last_enqueued_sequence_number' + -'last_enqueued_offset' + -'last_enqueued_time_utc' + -'is_partition_empty' + + :param partition: The target partition id. + :type partition: str + :rtype: dict + """ + alt_creds = { + "username": self._auth_config.get("iot_username"), + "password": self._auth_config.get("iot_password")} + try: + mgmt_auth = self._create_auth(**alt_creds) + mgmt_client = AMQPClient(self.mgmt_target, auth=mgmt_auth, debug=self.debug) + mgmt_client.open() + mgmt_msg = Message(application_properties={'name': self.eh_name, + 'partition': partition}) + response = mgmt_client.mgmt_request( + mgmt_msg, + constants.READ_OPERATION, + op_type=b'com.microsoft:partition', + status_code_field=b'status-code', + description_fields=b'status-description') + partition_info = response.get_data() + output = {} + if partition_info: + output['event_hub_path'] = partition_info[b'name'].decode('utf-8') + # output['type'] = partition_info[b'type'].decode('utf-8') + output['id'] = partition_info[b'partition'].decode('utf-8') + output['beginning_sequence_number'] = partition_info[b'begin_sequence_number'] + output['last_enqueued_sequence_number'] = partition_info[b'last_enqueued_sequence_number'] + output['last_enqueued_offset'] = partition_info[b'last_enqueued_offset'].decode('utf-8') + output['last_enqueued_time_utc'] = datetime.datetime.utcfromtimestamp( + float(partition_info[b'last_enqueued_time_utc'] / 1000)) + output['is_empty'] = partition_info[b'is_partition_empty'] + return output + finally: + mgmt_client.close() + def create_receiver( - self, consumer_group, partition, offset=None, epoch=None, operation=None, + self, partition_id, consumer_group="$Default", event_position=None, exclusive_receiver_priority=None, operation=None, prefetch=None, - keep_alive=None, - auto_reconnect=None, ): """ Add a receiver to the client for a particular consumer group and partition. @@ -140,8 +194,8 @@ def create_receiver( :type consumer_group: str :param partition: The ID of the partition. :type partition: str - :param offset: The offset from which to start receiving. - :type offset: ~azure.eventhub.common.Offset + :param event_position: The position from which to start receiving. + :type event_position: ~azure.eventhub.common.EventPosition :param prefetch: The message prefetch count of the receiver. Default is 300. :type prefetch: int :operation: An optional operation to be appended to the hostname in the source URL. @@ -158,23 +212,17 @@ def create_receiver( :caption: Add a receiver to the client for a particular consumer group and partition. """ - keep_alive = self.config.keep_alive if keep_alive is None else keep_alive - auto_reconnect = self.config.auto_reconnect if auto_reconnect is None else auto_reconnect prefetch = self.config.prefetch if prefetch is None else prefetch path = self.address.path + operation if operation else self.address.path source_url = "amqps://{}{}/ConsumerGroups/{}/Partitions/{}".format( - self.address.hostname, path, consumer_group, partition) + self.address.hostname, path, consumer_group, partition_id) handler = Receiver( - self, source_url, offset=offset, epoch=epoch, prefetch=prefetch, keep_alive=keep_alive, auto_reconnect=auto_reconnect) + self, source_url, event_position=event_position, exclusive_receiver_priority=exclusive_receiver_priority, + prefetch=prefetch) return handler - def create_epoch_receiver( - self, consumer_group, partition, epoch, prefetch=300, - operation=None): - return self.create_receiver(consumer_group, partition, epoch=epoch, prefetch=prefetch, operation=operation) - - def create_sender(self, partition=None, operation=None, send_timeout=None, keep_alive=None, auto_reconnect=None): + def create_sender(self, partition_id=None, operation=None, send_timeout=None): """ Add a sender to the client to send EventData object to an EventHub. @@ -209,9 +257,7 @@ def create_sender(self, partition=None, operation=None, send_timeout=None, keep_ if operation: target = target + operation send_timeout = self.config.send_timeout if send_timeout is None else send_timeout - keep_alive = self.config.keep_alive if keep_alive is None else keep_alive - auto_reconnect = self.config.auto_reconnect if auto_reconnect is None else auto_reconnect handler = Sender( - self, target, partition=partition, send_timeout=send_timeout, keep_alive=keep_alive, auto_reconnect=auto_reconnect) + self, target, partition=partition_id, send_timeout=send_timeout) return handler diff --git a/sdk/eventhub/azure-eventhubs/azure/eventhub/client_abstract.py b/sdk/eventhub/azure-eventhubs/azure/eventhub/client_abstract.py index 1435d15bd2be..26435fd93635 100644 --- a/sdk/eventhub/azure-eventhubs/azure/eventhub/client_abstract.py +++ b/sdk/eventhub/azure-eventhubs/azure/eventhub/client_abstract.py @@ -5,7 +5,6 @@ from __future__ import unicode_literals import logging -import datetime import sys import uuid import time @@ -17,16 +16,11 @@ except ImportError: from urllib.parse import urlparse, unquote_plus, urlencode, quote_plus -import uamqp -from uamqp import Message -from uamqp import authentication -from uamqp import constants from azure.eventhub import __version__ -from azure.eventhub.sender import Sender -from azure.eventhub.receiver import Receiver -from azure.eventhub.common import EventHubError, parse_sas_token from azure.eventhub.configuration import Configuration +from azure.eventhub import constants +from .common import SASTokenCredentials, SharedKeyCredentials, Address log = logging.getLogger(__name__) @@ -101,8 +95,7 @@ class EventHubClientAbstract(object): """ - def __init__( - self, address, username=None, password=None, sas_token=None, aad_credential=None, **kwargs): + def __init__(self, host, event_hub_path, credential, **kwargs): """ Constructs a new EventHubClient with the given address URL. @@ -131,67 +124,29 @@ def __init__( :type sas_token: str or callable """ self.container_id = "eventhub.pysdk-" + str(uuid.uuid4())[:8] - self.sas_token = sas_token - self.address = urlparse(address) - self.aad_credential = aad_credential - self.eh_name = self.address.path.lstrip('/') - # self.http_proxy = kwargs.get("http_proxy") + self.address = Address() + self.address.hostname = host + self.address.path = "/" + event_hub_path if event_hub_path else "" + self._auth_config = {} + self.credential = credential + if isinstance(credential, SharedKeyCredentials): + self.username = credential.policy + self.password = credential.key + self._auth_config['username'] = self.username + self._auth_config['password'] = self.password + + self.host = host + self.eh_name = event_hub_path self.keep_alive = kwargs.get("keep_alive", 30) self.auto_reconnect = kwargs.get("auto_reconnect", True) - self.mgmt_target = "amqps://{}/{}".format(self.address.hostname, self.eh_name) - url_username = unquote_plus(self.address.username) if self.address.username else None - username = username or url_username - url_password = unquote_plus(self.address.password) if self.address.password else None - password = password or url_password - if (not username or not password) and not sas_token: - raise ValueError("Please supply either username and password, or a SAS token") + self.mgmt_target = "amqps://{}/{}".format(self.host, self.eh_name) self.auth_uri = "sb://{}{}".format(self.address.hostname, self.address.path) - self._auth_config = {'username': username, 'password': password} self.get_auth = functools.partial(self._create_auth) - # self.debug = kwargs.get("debug", False) # debug - #self.auth_timeout = auth_timeout - - self.stopped = False self.config = Configuration(**kwargs) self.debug = self.config.network_tracing log.info("%r: Created the Event Hub client", self.container_id) - @classmethod - def from_sas_token(cls, address, sas_token, eventhub=None, **kwargs): - """Create an EventHubClient from an existing auth token or token generator. - - :param address: The Event Hub address URL - :type address: str - :param sas_token: A SAS token or function that returns a SAS token. If a function is supplied, - it will be used to retrieve subsequent tokens in the case of token expiry. The function should - take no arguments. - :type sas_token: str or callable - :param eventhub: The name of the EventHub, if not already included in the address URL. - :type eventhub: str - :param debug: Whether to output network trace logs to the logger. Default - is `False`. - :type debug: bool - :param http_proxy: HTTP proxy settings. This must be a dictionary with the following - keys: 'proxy_hostname' (str value) and 'proxy_port' (int value). - Additionally the following keys may also be present: 'username', 'password'. - :type http_proxy: dict[str, Any] - :param auth_timeout: The time in seconds to wait for a token to be authorized by the service. - The default value is 60 seconds. If set to 0, no timeout will be enforced from the client. - :type auth_timeout: int - - Example: - .. literalinclude:: ../examples/test_examples_eventhub.py - :start-after: [START create_eventhub_client_sas_token] - :end-before: [END create_eventhub_client_sas_token] - :language: python - :dedent: 4 - :caption: Create an EventHubClient from an existing auth token or token generator. - - """ - address = _build_uri(address, eventhub) - return cls(address, sas_token=sas_token, **kwargs) - @classmethod def from_connection_string(cls, conn_str, eventhub=None, **kwargs): """Create an EventHubClient from a connection string. @@ -223,8 +178,12 @@ def from_connection_string(cls, conn_str, eventhub=None, **kwargs): """ address, policy, key, entity = _parse_conn_str(conn_str) entity = eventhub or entity - address = _build_uri(address, entity) - return cls(address, username=policy, password=key, **kwargs) + left_slash_pos = address.find("//") + if left_slash_pos != -1: + host = address[left_slash_pos + 2:] + else: + host = address + return cls(host, entity, SharedKeyCredentials(policy, key), **kwargs) @classmethod def from_iothub_connection_string(cls, conn_str, **kwargs): @@ -257,7 +216,12 @@ def from_iothub_connection_string(cls, conn_str, **kwargs): hub_name = address.split('.')[0] username = "{}@sas.root.{}".format(policy, hub_name) password = _generate_sas_token(address, policy, key) - client = cls("amqps://" + address, username=username, password=password, **kwargs) + left_slash_pos = address.find("//") + if left_slash_pos != -1: + host = address[left_slash_pos + 2:] + else: + host = address + client = cls(host, "", SharedKeyCredentials(username, password), **kwargs) client._auth_config = { # pylint: disable=protected-access 'iot_username': policy, 'iot_password': key, @@ -265,16 +229,11 @@ def from_iothub_connection_string(cls, conn_str, **kwargs): 'password': password} return client - @classmethod - def from_aad_credential(cls, address, aad_credential, eventhub=None, **kwargs): - address = _build_uri(address, eventhub) - return cls(address, aad_credential=aad_credential, **kwargs) - @abstractmethod def _create_auth(self, username=None, password=None): pass - def create_properties(self): # pylint: disable=no-self-use + def create_properties(self, user_agent=None): # pylint: disable=no-self-use """ Format the properties with which to instantiate the connection. This acts like a user agent over HTTP. @@ -286,20 +245,29 @@ def create_properties(self): # pylint: disable=no-self-use properties["version"] = __version__ properties["framework"] = "Python {}.{}.{}".format(*sys.version_info[0:3]) properties["platform"] = sys.platform + + final_user_agent = 'azsdk-python-eventhub/{} ({}; {})'.format( + __version__, properties["framework"], sys.platform) + if user_agent: + final_user_agent = '{}, {}'.format(final_user_agent, user_agent) + + if len(final_user_agent) > constants.MAX_USER_AGENT_LENGTH: + raise ValueError("The user-agent string cannot be more than {} in length." + "Current user_agent string is: {} with length: {}".format( + constants.MAX_USER_AGENT_LENGTH, final_user_agent, len(final_user_agent))) + + properties["user-agent"] = final_user_agent return properties def _process_redirect_uri(self, redirect): redirect_uri = redirect.address.decode('utf-8') auth_uri, _, _ = redirect_uri.partition("/ConsumerGroups") self.address = urlparse(auth_uri) + self.host = self.address.hostname self.auth_uri = "sb://{}{}".format(self.address.hostname, self.address.path) self.eh_name = self.address.path.lstrip('/') self.mgmt_target = redirect_uri - @abstractmethod - def get_eventhub_information(self): - pass - @abstractmethod def create_receiver( self, consumer_group, partition, epoch=None, offset=None, prefetch=300, diff --git a/sdk/eventhub/azure-eventhubs/azure/eventhub/common.py b/sdk/eventhub/azure-eventhubs/azure/eventhub/common.py index 03a602616c4d..3af21e5d2e86 100644 --- a/sdk/eventhub/azure-eventhubs/azure/eventhub/common.py +++ b/sdk/eventhub/azure-eventhubs/azure/eventhub/common.py @@ -11,41 +11,11 @@ import six -from uamqp import Message, BatchMessage +import uamqp +from uamqp import BatchMessage from uamqp import types, constants, errors from uamqp.message import MessageHeader, MessageProperties -_NO_RETRY_ERRORS = ( - b"com.microsoft:argument-out-of-range", - b"com.microsoft:entity-disabled", - b"com.microsoft:auth-failed", - b"com.microsoft:precondition-failed", - b"com.microsoft:argument-error" -) - -def _error_handler(error): - """ - Called internally when an event has failed to send so we - can parse the error to determine whether we should attempt - to retry sending the event again. - Returns the action to take according to error type. - - :param error: The error received in the send attempt. - :type error: Exception - :rtype: ~uamqp.errors.ErrorAction - """ - if error.condition == b'com.microsoft:server-busy': - return errors.ErrorAction(retry=True, backoff=4) - if error.condition == b'com.microsoft:timeout': - return errors.ErrorAction(retry=True, backoff=2) - if error.condition == b'com.microsoft:operation-cancelled': - return errors.ErrorAction(retry=True) - if error.condition == b"com.microsoft:container-close": - return errors.ErrorAction(retry=True, backoff=4) - if error.condition in _NO_RETRY_ERRORS: - return errors.ErrorAction(retry=False) - return errors.ErrorAction(retry=True) - def parse_sas_token(sas_token): """Parse a SAS token into its components. @@ -63,6 +33,9 @@ def parse_sas_token(sas_token): return sas_data +Message = uamqp.Message + + class EventData(object): """ The EventData class is a holder of event content. @@ -118,6 +91,26 @@ def __init__(self, body=None, to_device=None, message=None): else: self.message = Message(body, properties=self.msg_properties) + def __str__(self): + dic = { + 'body': self.body_as_str(), + 'application_properties': str(self.application_properties) + } + + if self.sequence_number: + dic['sequence_number'] = str(self.sequence_number) + if self.offset: + dic['offset'] = str(self.offset) + if self.enqueued_time: + dic['enqueued_time'] = str(self.enqueued_time) + if self.device_id: + dic['device_id'] = str(self.device_id) + if self._batching_label: + dic['_batching_label'] = str(self._batching_label) + + + return str(dic) + @property def sequence_number(self): """ @@ -130,9 +123,9 @@ def sequence_number(self): @property def offset(self): """ - The offset of the event data object. + The position of the event data object. - :rtype: ~azure.eventhub.common.Offset + :rtype: ~azure.eventhub.common.EventPosition """ try: return EventPosition(self._annotations[EventData.PROP_OFFSET].decode('UTF-8')) @@ -162,7 +155,7 @@ def device_id(self): return self._annotations.get(EventData.PROP_DEVICE_ID, None) @property - def partition_key(self): + def _batching_label(self): """ The partition key of the event data object. @@ -173,8 +166,8 @@ def partition_key(self): except KeyError: return self._annotations.get(EventData.PROP_PARTITION_KEY, None) - @partition_key.setter - def partition_key(self, value): + @_batching_label.setter + def _batching_label(self, value): """ Set the partition key of the event data object. @@ -189,6 +182,7 @@ def partition_key(self, value): self.message.header = header self._annotations = annotations + @property def application_properties(self): """ @@ -262,9 +256,20 @@ def encode_message(self): class _BatchSendEventData(EventData): - def __init__(self, batch_event_data): - # TODO: rethink if to_device should be included in - self.message = BatchMessage(data=batch_event_data, multi_messages=True, properties=None) + def __init__(self, batch_event_data, batching_label=None): + self.message = BatchMessage(data=batch_event_data, multi_messages=False, properties=None) + self.set_batching_label(batching_label) + + def set_batching_label(self, value): + if value: + annotations = self.message.annotations + if annotations is None: + annotations = dict() + annotations[types.AMQPSymbol(EventData.PROP_PARTITION_KEY)] = value + header = MessageHeader() + header.durable = True + self.message.annotations = annotations + self.message.header = header class EventPosition(object): @@ -294,9 +299,12 @@ def __init__(self, value, inclusive=False): :param inclusive: Whether to include the supplied value as the start point. :type inclusive: bool """ - self.value = value + self.value = value if value is not None else "-1" self.inclusive = inclusive + def __str__(self): + return str(self.value) + def selector(self): """ Creates a selector expression of the offset. @@ -312,12 +320,12 @@ def selector(self): return ("amqp.annotation.x-opt-offset {} '{}'".format(operator, self.value)).encode('utf-8') @staticmethod - def from_start_of_stream(): - return EventPosition("-1") + def first_available(): + return FIRST_AVAILABLE - @staticmethod - def from_end_of_stream(): - return EventPosition("@latest") + @classmethod + def new_events_only(cls): + return NEW_EVENTS_ONLY @staticmethod def from_offset(offset, inclusive=False): @@ -332,53 +340,29 @@ def from_enqueued_time(enqueued_time, inclusive=False): return EventPosition(enqueued_time, inclusive) -class EventHubError(Exception): - """ - Represents an error happened in the client. - - :ivar message: The error message. - :vartype message: str - :ivar error: The error condition, if available. - :vartype error: str - :ivar details: The error details, if included in the - service response. - :vartype details: dict[str, str] - """ +FIRST_AVAILABLE = EventPosition("-1") +NEW_EVENTS_ONLY = EventPosition("@latest") + + +# TODO: move some behaviors to these two classes. +class SASTokenCredentials(object): + def __init__(self, token): + self.token = token + + def get_sas_token(self): + if callable(self.token): + return self.token() + else: + return self.token + + +class SharedKeyCredentials(object): + def __init__(self, policy, key): + self.policy = policy + self.key = key + - def __init__(self, message, details=None): - self.error = None - self.message = message - self.details = details - if isinstance(message, constants.MessageSendResult): - self.message = "Message send failed with result: {}".format(message) - if details and isinstance(details, Exception): - try: - condition = details.condition.value.decode('UTF-8') - except AttributeError: - condition = details.condition.decode('UTF-8') - _, _, self.error = condition.partition(':') - self.message += "\nError: {}".format(self.error) - try: - self._parse_error(details.description) - for detail in self.details: - self.message += "\n{}".format(detail) - except: # pylint: disable=bare-except - self.message += "\n{}".format(details) - super(EventHubError, self).__init__(self.message) - - def _parse_error(self, error_list): - details = [] - self.message = error_list if isinstance(error_list, six.text_type) else error_list.decode('UTF-8') - details_index = self.message.find(" Reference:") - if details_index >= 0: - details_msg = self.message[details_index + 1:] - self.message = self.message[0:details_index] - - tracking_index = details_msg.index(", TrackingId:") - system_index = details_msg.index(", SystemTracker:") - timestamp_index = details_msg.index(", Timestamp:") - details.append(details_msg[:tracking_index]) - details.append(details_msg[tracking_index + 2: system_index]) - details.append(details_msg[system_index + 2: timestamp_index]) - details.append(details_msg[timestamp_index + 2:]) - self.details = details +class Address(object): + def __init__(self, hostname=None, path=None): + self.hostname = hostname + self.path = path diff --git a/sdk/eventhub/azure-eventhubs/azure/eventhub/configuration.py b/sdk/eventhub/azure-eventhubs/azure/eventhub/configuration.py index 2d7a7be57638..b6e030c9e3a6 100644 --- a/sdk/eventhub/azure-eventhubs/azure/eventhub/configuration.py +++ b/sdk/eventhub/azure-eventhubs/azure/eventhub/configuration.py @@ -3,19 +3,19 @@ # Licensed under the MIT License. See License.txt in the project root for license information. # -------------------------------------------------------------------------------------------- -from uamqp.constants import TransportType +from .constants import TransportType class Configuration(object): def __init__(self, **kwargs): self.user_agent = kwargs.get("user_agent") self.max_retries = kwargs.get("max_retries", 3) - self.network_tracing = kwargs.get("debug", False) + self.network_tracing = kwargs.get("network_tracing", False) self.http_proxy = kwargs.get("http_proxy") - self.auto_reconnect = kwargs.get("auto_reconnect", False) - self.keep_alive = kwargs.get("keep_alive", 0) self.transport_type = TransportType.AmqpOverWebsocket if self.http_proxy \ else kwargs.get("transport_type", TransportType.Amqp) self.auth_timeout = kwargs.get("auth_timeout", 60) - self.prefetch = kwargs.get("prefetch") + self.prefetch = kwargs.get("prefetch", 300) + self.max_batch_size = kwargs.get("max_batch_size") + self.receive_timeout = kwargs.get("receive_timeout", 0) self.send_timeout = kwargs.get("send_timeout", 60) diff --git a/sdk/eventhub/azure-eventhubs/azure/eventhub/constants.py b/sdk/eventhub/azure-eventhubs/azure/eventhub/constants.py new file mode 100644 index 000000000000..e71d3815f48f --- /dev/null +++ b/sdk/eventhub/azure-eventhubs/azure/eventhub/constants.py @@ -0,0 +1,11 @@ +#------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for +# license information. +#-------------------------------------------------------------------------- + +from uamqp import constants + +MAX_USER_AGENT_LENGTH = 512 +TransportType = constants.TransportType +MessageSendResult = constants.MessageSendResult diff --git a/sdk/eventhub/azure-eventhubs/azure/eventhub/error.py b/sdk/eventhub/azure-eventhubs/azure/eventhub/error.py new file mode 100644 index 000000000000..69aaa701496b --- /dev/null +++ b/sdk/eventhub/azure-eventhubs/azure/eventhub/error.py @@ -0,0 +1,108 @@ +# -------------------------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for license information. +# -------------------------------------------------------------------------------------------- + +from uamqp import types, constants, errors +import six +from azure.core import AzureError + +_NO_RETRY_ERRORS = ( + b"com.microsoft:argument-out-of-range", + b"com.microsoft:entity-disabled", + b"com.microsoft:auth-failed", + b"com.microsoft:precondition-failed", + b"com.microsoft:argument-error" +) + +def _error_handler(error): + """ + Called internally when an event has failed to send so we + can parse the error to determine whether we should attempt + to retry sending the event again. + Returns the action to take according to error type. + + :param error: The error received in the send attempt. + :type error: Exception + :rtype: ~uamqp.errors.ErrorAction + """ + if error.condition == b'com.microsoft:server-busy': + return errors.ErrorAction(retry=True, backoff=4) + if error.condition == b'com.microsoft:timeout': + return errors.ErrorAction(retry=True, backoff=2) + if error.condition == b'com.microsoft:operation-cancelled': + return errors.ErrorAction(retry=True) + if error.condition == b"com.microsoft:container-close": + return errors.ErrorAction(retry=True, backoff=4) + if error.condition in _NO_RETRY_ERRORS: + return errors.ErrorAction(retry=False) + return errors.ErrorAction(retry=True) + + +class EventHubError(AzureError): + """ + Represents an error happened in the client. + + :ivar message: The error message. + :vartype message: str + :ivar error: The error condition, if available. + :vartype error: str + :ivar details: The error details, if included in the + service response. + :vartype details: dict[str, str] + """ + + def __init__(self, message, details=None): + self.error = None + self.message = message + self.details = details + if isinstance(message, constants.MessageSendResult): + self.message = "Message send failed with result: {}".format(message) + if details and isinstance(details, Exception): + try: + condition = details.condition.value.decode('UTF-8') + except AttributeError: + try: + condition = details.condition.decode('UTF-8') + except AttributeError: + condition = None + if condition: + _, _, self.error = condition.partition(':') + self.message += "\nError: {}".format(self.error) + try: + self._parse_error(details.description) + for detail in self.details: + self.message += "\n{}".format(detail) + except: # pylint: disable=bare-except + self.message += "\n{}".format(details) + super(EventHubError, self).__init__(self.message) + + def _parse_error(self, error_list): + details = [] + self.message = error_list if isinstance(error_list, six.text_type) else error_list.decode('UTF-8') + details_index = self.message.find(" Reference:") + if details_index >= 0: + details_msg = self.message[details_index + 1:] + self.message = self.message[0:details_index] + + tracking_index = details_msg.index(", TrackingId:") + system_index = details_msg.index(", SystemTracker:") + timestamp_index = details_msg.index(", Timestamp:") + details.append(details_msg[:tracking_index]) + details.append(details_msg[tracking_index + 2: system_index]) + details.append(details_msg[system_index + 2: timestamp_index]) + details.append(details_msg[timestamp_index + 2:]) + self.details = details + + +class AuthenticationError(EventHubError): + pass + + +class ConnectError(EventHubError): + pass + + +class EventDataError(EventHubError): + pass + diff --git a/sdk/eventhub/azure-eventhubs/azure/eventhub/receiver.py b/sdk/eventhub/azure-eventhubs/azure/eventhub/receiver.py index 4577d0332af5..4643cb29419f 100644 --- a/sdk/eventhub/azure-eventhubs/azure/eventhub/receiver.py +++ b/sdk/eventhub/azure-eventhubs/azure/eventhub/receiver.py @@ -11,7 +11,8 @@ from uamqp import types, errors from uamqp import ReceiveClient, Source -from azure.eventhub.common import EventHubError, EventData, _error_handler +from azure.eventhub.common import EventData +from azure.eventhub.error import EventHubError, AuthenticationError, ConnectError, _error_handler log = logging.getLogger(__name__) @@ -33,7 +34,7 @@ class Receiver(object): timeout = 0 _epoch = b'com.microsoft:epoch' - def __init__(self, client, source, offset=None, prefetch=300, epoch=None, keep_alive=None, auto_reconnect=True): + def __init__(self, client, source, event_position=None, prefetch=300, exclusive_receiver_priority=None, keep_alive=None, auto_reconnect=True): """ Instantiate a receiver. @@ -50,10 +51,10 @@ def __init__(self, client, source, offset=None, prefetch=300, epoch=None, keep_a self.running = False self.client = client self.source = source - self.offset = offset - self.iter_started = False + self.offset = event_position + self.messages_iter = None self.prefetch = prefetch - self.epoch = epoch + self.exclusive_receiver_priority = exclusive_receiver_priority self.keep_alive = keep_alive self.auto_reconnect = auto_reconnect self.retry_policy = errors.ErrorPolicy(max_retries=self.client.config.max_retries, on_error=_error_handler) @@ -66,19 +67,19 @@ def __init__(self, client, source, offset=None, prefetch=300, epoch=None, keep_a source = Source(self.source) if self.offset is not None: source.set_filter(self.offset.selector()) - if epoch: - self.properties = {types.AMQPSymbol(self._epoch): types.AMQPLong(int(epoch))} + if exclusive_receiver_priority: + self.properties = {types.AMQPSymbol(self._epoch): types.AMQPLong(int(exclusive_receiver_priority))} self._handler = ReceiveClient( source, auth=self.client.get_auth(), - debug=self.client.debug, + debug=self.client.config.network_tracing, prefetch=self.prefetch, link_properties=self.properties, timeout=self.timeout, error_policy=self.retry_policy, keep_alive_interval=self.keep_alive, client_name=self.name, - properties=self.client.create_properties()) + properties=self.client.create_properties(self.client.config.user_agent)) def __enter__(self): return self @@ -87,16 +88,14 @@ def __exit__(self, exc_type, exc_val, exc_tb): self.close(exc_val) def __iter__(self): - if not self.running: - self.open() - if not self.iter_started: - self.iter_started = True - self.messages_iter = self._handler.receive_messages_iter() return self def __next__(self): + self._open() while True: try: + if not self.messages_iter: + self.messages_iter = self._handler.receive_messages_iter() message = next(self.messages_iter) event_data = EventData(message=message) self.offset = event_data.offset @@ -108,25 +107,29 @@ def __next__(self): if shutdown.action.retry and self.auto_reconnect: log.info("Receiver detached. Attempting reconnect.") self.reconnect() - log.info("Receiver detached. Shutting down.") - error = EventHubError(str(shutdown), shutdown) - self.close(exception=error) - raise error + else: + log.info("Receiver detached. Shutting down.") + error = EventHubError(str(shutdown), shutdown) + self.close(exception=error) + raise error except errors.MessageHandlerError as shutdown: if self.auto_reconnect: log.info("Receiver detached. Attempting reconnect.") self.reconnect() - log.info("Receiver detached. Shutting down.") - error = EventHubError(str(shutdown), shutdown) - self.close(exception=error) - raise error + else: + log.info("Receiver detached. Shutting down.") + error = EventHubError(str(shutdown), shutdown) + self.close(exception=error) + raise error + except StopIteration: + raise except Exception as e: log.info("Unexpected error occurred (%r). Shutting down.", e) error = EventHubError("Receive failed: {}".format(e)) self.close(exception=error) raise error - def open(self): + def _open(self): """ Open the Receiver using the supplied conneciton. If the handler has previously been redirected, the redirect @@ -145,7 +148,6 @@ def open(self): """ # pylint: disable=protected-access - self.running = True if self.redirected: self.source = self.redirected.address source = Source(self.source) @@ -157,17 +159,45 @@ def open(self): self._handler = ReceiveClient( source, auth=self.client.get_auth(**alt_creds), - debug=self.client.debug, + debug=self.client.config.network_tracing, prefetch=self.prefetch, link_properties=self.properties, timeout=self.timeout, error_policy=self.retry_policy, keep_alive_interval=self.keep_alive, client_name=self.name, - properties=self.client.create_properties()) - self._handler.open() - while not self._handler.client_ready(): - time.sleep(0.05) + properties=self.client.create_properties(self.client.config.user_agent)) + if not self.running: + try: + self._handler.open() + self.running = True + while not self._handler.client_ready(): + time.sleep(0.05) + + except errors.AuthenticationException: + log.info("Receiver failed authentication. Retrying...") + self.reconnect() + except (errors.LinkDetach, errors.ConnectionClose) as shutdown: + if shutdown.action.retry and self.auto_reconnect: + log.info("Receiver detached. Attempting reconnect.") + self.reconnect() + else: + log.info("Receiver detached. Failed to connect") + error = ConnectError(str(shutdown), shutdown) + raise error + except errors.AMQPConnectionError as shutdown: + if str(shutdown).startswith("Unable to open authentication session") and self.auto_reconnect: + log.info("Receiver couldn't authenticate (%r).", shutdown) + error = AuthenticationError(str(shutdown), shutdown) + raise error + else: + log.info("Receiver connection error (%r).", shutdown) + error = ConnectError(str(shutdown), shutdown) + raise error + except Exception as e: + log.info("Unexpected error occurred (%r)", e) + error = EventHubError("Receiver connect failed: {}".format(e)) + raise error def _reconnect(self): # pylint: disable=too-many-statements # pylint: disable=protected-access @@ -181,22 +211,23 @@ def _reconnect(self): # pylint: disable=too-many-statements self._handler = ReceiveClient( source, auth=self.client.get_auth(**alt_creds), - debug=self.client.debug, + debug=self.client.config.network_tracing, prefetch=self.prefetch, link_properties=self.properties, timeout=self.timeout, error_policy=self.retry_policy, keep_alive_interval=self.keep_alive, client_name=self.name, - properties=self.client.create_properties()) + properties=self.client.create_properties(self.client.config.user_agent)) + self.messages_iter = None try: self._handler.open() while not self._handler.client_ready(): time.sleep(0.05) return True - except errors.TokenExpired as shutdown: + except errors.AuthenticationException as shutdown: log.info("Receiver disconnected due to token expiry. Shutting down.") - error = EventHubError(str(shutdown), shutdown) + error = AuthenticationError(str(shutdown), shutdown) self.close(exception=error) raise error except (errors.LinkDetach, errors.ConnectionClose) as shutdown: @@ -204,7 +235,7 @@ def _reconnect(self): # pylint: disable=too-many-statements log.info("Receiver detached. Attempting reconnect.") return False log.info("Receiver detached. Shutting down.") - error = EventHubError(str(shutdown), shutdown) + error = ConnectError(str(shutdown), shutdown) self.close(exception=error) raise error except errors.MessageHandlerError as shutdown: @@ -212,7 +243,7 @@ def _reconnect(self): # pylint: disable=too-many-statements log.info("Receiver detached. Attempting reconnect.") return False log.info("Receiver detached. Shutting down.") - error = EventHubError(str(shutdown), shutdown) + error = ConnectError(str(shutdown), shutdown) self.close(exception=error) raise error except errors.AMQPConnectionError as shutdown: @@ -220,7 +251,7 @@ def _reconnect(self): # pylint: disable=too-many-statements log.info("Receiver couldn't authenticate. Attempting reconnect.") return False log.info("Receiver connection error (%r). Shutting down.", shutdown) - error = EventHubError(str(shutdown)) + error = ConnectError(str(shutdown), shutdown) self.close(exception=error) raise error except Exception as e: @@ -235,38 +266,6 @@ def reconnect(self): while not self._reconnect(): time.sleep(self.reconnect_backoff) - def get_handler_state(self): - """ - Get the state of the underlying handler with regards to start - up processes. - - :rtype: ~uamqp.constants.MessageReceiverState - """ - # pylint: disable=protected-access - return self._handler._message_receiver.get_state() - - def has_started(self): - """ - Whether the handler has completed all start up processes such as - establishing the connection, session, link and authentication, and - is not ready to process messages. - **This function is now deprecated and will be removed in v2.0+.** - - :rtype: bool - """ - # pylint: disable=protected-access - timeout = False - auth_in_progress = False - if self._handler._connection.cbs: - timeout, auth_in_progress = self._handler._auth.handle_token() - if timeout: - raise EventHubError("Authorization timeout.") - if auth_in_progress: - return False - if not self._handler._client_ready(): - return False - return True - def close(self, exception=None): """ Close down the handler. If the handler has already closed, @@ -334,43 +333,43 @@ def receive(self, max_batch_size=None, timeout=None): """ if self.error: raise self.error - if not self.running: - self.open() + self._open() + data_batch = [] - try: - timeout_ms = 1000 * timeout if timeout else 0 - message_batch = self._handler.receive_message_batch( - max_batch_size=max_batch_size, - timeout=timeout_ms) - for message in message_batch: - event_data = EventData(message=message) - self.offset = event_data.offset - data_batch.append(event_data) - return data_batch - except (errors.TokenExpired, errors.AuthenticationException): - log.info("Receiver disconnected due to token error. Attempting reconnect.") - self.reconnect() - return data_batch - except (errors.LinkDetach, errors.ConnectionClose) as shutdown: - if shutdown.action.retry and self.auto_reconnect: - log.info("Receiver detached. Attempting reconnect.") - self.reconnect() + while True: + try: + timeout_ms = 1000 * timeout if timeout else 0 + message_batch = self._handler.receive_message_batch( + max_batch_size=max_batch_size, + timeout=timeout_ms) + for message in message_batch: + event_data = EventData(message=message) + self.offset = event_data.offset + data_batch.append(event_data) return data_batch - log.info("Receiver detached. Shutting down.") - error = EventHubError(str(shutdown), shutdown) - self.close(exception=error) - raise error - except errors.MessageHandlerError as shutdown: - if self.auto_reconnect: - log.info("Receiver detached. Attempting reconnect.") + except (errors.TokenExpired, errors.AuthenticationException): + log.info("Receiver disconnected due to token error. Attempting reconnect.") self.reconnect() - return data_batch - log.info("Receiver detached. Shutting down.") - error = EventHubError(str(shutdown), shutdown) - self.close(exception=error) - raise error - except Exception as e: - log.info("Unexpected error occurred (%r). Shutting down.", e) - error = EventHubError("Receive failed: {}".format(e)) - self.close(exception=error) - raise error + except (errors.LinkDetach, errors.ConnectionClose) as shutdown: + if shutdown.action.retry and self.auto_reconnect: + log.info("Receiver detached. Attempting reconnect.") + self.reconnect() + else: + log.info("Receiver detached. Shutting down.") + error = ConnectError(str(shutdown), shutdown) + self.close(exception=error) + raise error + except errors.MessageHandlerError as shutdown: + if self.auto_reconnect: + log.info("Receiver detached. Attempting reconnect.") + self.reconnect() + else: + log.info("Receiver detached. Shutting down.") + error = ConnectError(str(shutdown), shutdown) + self.close(exception=error) + raise error + except Exception as e: + log.info("Unexpected error occurred (%r). Shutting down.", e) + error = EventHubError("Receive failed: {}".format(e)) + self.close(exception=error) + raise error diff --git a/sdk/eventhub/azure-eventhubs/azure/eventhub/sender.py b/sdk/eventhub/azure-eventhubs/azure/eventhub/sender.py index ab113eac1c28..ccb193835c20 100644 --- a/sdk/eventhub/azure-eventhubs/azure/eventhub/sender.py +++ b/sdk/eventhub/azure-eventhubs/azure/eventhub/sender.py @@ -12,7 +12,9 @@ from uamqp import SendClient from uamqp.constants import MessageSendResult -from azure.eventhub.common import EventHubError, EventData, _BatchSendEventData, _error_handler +from azure.eventhub.common import EventData, _BatchSendEventData +from azure.eventhub.error import EventHubError, ConnectError, \ + AuthenticationError, EventDataError, _error_handler log = logging.getLogger(__name__) @@ -70,12 +72,12 @@ def __init__(self, client, target, partition=None, send_timeout=60, keep_alive=N self._handler = SendClient( self.target, auth=self.client.get_auth(), - debug=self.client.debug, + debug=self.client.config.network_tracing, msg_timeout=self.timeout, error_policy=self.retry_policy, keep_alive_interval=self.keep_alive, client_name=self.name, - properties=self.client.create_properties()) + properties=self.client.create_properties(self.client.config.user_agent)) self._outcome = None self._condition = None @@ -85,7 +87,7 @@ def __enter__(self): def __exit__(self, exc_type, exc_val, exc_tb): self.close(exc_val) - def open(self): + def _open(self): """ Open the Sender using the supplied conneciton. If the handler has previously been redirected, the redirect @@ -103,21 +105,47 @@ def open(self): :caption: Open the Sender using the supplied conneciton. """ - self.running = True if self.redirected: self.target = self.redirected.address self._handler = SendClient( self.target, auth=self.client.get_auth(), - debug=self.client.debug, + debug=self.client.config.network_tracing, msg_timeout=self.timeout, error_policy=self.retry_policy, keep_alive_interval=self.keep_alive, client_name=self.name, - properties=self.client.create_properties()) - self._handler.open() - while not self._handler.client_ready(): - time.sleep(0.05) + properties=self.client.create_properties(self.client.config.user_agent)) + if not self.running: + try: + self._handler.open() + self.running = True + while not self._handler.client_ready(): + time.sleep(0.05) + except errors.AuthenticationException: + log.info("Sender failed authentication. Retrying...") + self.reconnect() + except (errors.LinkDetach, errors.ConnectionClose) as shutdown: + if shutdown.action.retry and self.auto_reconnect: + log.info("Sender detached. Attempting reconnect.") + self.reconnect() + else: + log.info("Sender detached. Failed to connect") + error = ConnectError(str(shutdown), shutdown) + raise error + except errors.AMQPConnectionError as shutdown: + if str(shutdown).startswith("Unable to open authentication session") and self.auto_reconnect: + log.info("Sender couldn't authenticate.", shutdown) + error = AuthenticationError(str(shutdown), shutdown) + raise error + else: + log.info("Sender connection error (%r).", shutdown) + error = ConnectError(str(shutdown), shutdown) + raise error + except Exception as e: + log.info("Unexpected error occurred (%r)", e) + error = EventHubError("Sender connect failed: {}".format(e)) + raise error def _reconnect(self): # pylint: disable=protected-access @@ -126,20 +154,22 @@ def _reconnect(self): self._handler = SendClient( self.target, auth=self.client.get_auth(), - debug=self.client.debug, + debug=self.client.config.network_tracing, msg_timeout=self.timeout, error_policy=self.retry_policy, keep_alive_interval=self.keep_alive, client_name=self.name, - properties=self.client.create_properties()) + properties=self.client.create_properties(self.client.config.user_agent)) try: self._handler.open() + while not self._handler.client_ready(): + time.sleep(0.05) self._handler.queue_message(*unsent_events) self._handler.wait() return True - except errors.TokenExpired as shutdown: + except errors.AuthenticationException as shutdown: log.info("Sender disconnected due to token expiry. Shutting down.") - error = EventHubError(str(shutdown), shutdown) + error = AuthenticationError(str(shutdown), shutdown) self.close(exception=error) raise error except (errors.LinkDetach, errors.ConnectionClose) as shutdown: @@ -147,7 +177,7 @@ def _reconnect(self): log.info("Sender detached. Attempting reconnect.") return False log.info("Sender reconnect failed. Shutting down.") - error = EventHubError(str(shutdown), shutdown) + error = ConnectError(str(shutdown), shutdown) self.close(exception=error) raise error except errors.MessageHandlerError as shutdown: @@ -155,7 +185,7 @@ def _reconnect(self): log.info("Sender detached. Attempting reconnect.") return False log.info("Sender reconnect failed. Shutting down.") - error = EventHubError(str(shutdown), shutdown) + error = ConnectError(str(shutdown), shutdown) self.close(exception=error) raise error except errors.AMQPConnectionError as shutdown: @@ -163,7 +193,7 @@ def _reconnect(self): log.info("Sender couldn't authenticate. Attempting reconnect.") return False log.info("Sender connection error (%r). Shutting down.", shutdown) - error = EventHubError(str(shutdown)) + error = ConnectError(str(shutdown), shutdown) self.close(exception=error) raise error except Exception as e: @@ -178,16 +208,6 @@ def reconnect(self): while not self._reconnect(): time.sleep(self.reconnect_backoff) - def get_handler_state(self): - """ - Get the state of the underlying handler with regards to start - up processes. - - :rtype: ~uamqp.constants.MessageSenderState - """ - # pylint: disable=protected-access - return self._handler._message_sender.get_state() - def close(self, exception=None): """ Close down the handler. If the handler has already closed, @@ -221,14 +241,14 @@ def close(self, exception=None): self._handler.close() def _send_event_data(self, event_data): - if not self.running: - self.open() + self._open() + try: self._handler.send_message(event_data.message) if self._outcome != MessageSendResult.Ok: raise Sender._error(self._outcome, self._condition) except errors.MessageException as failed: - error = EventHubError(str(failed), failed) + error = EventDataError(str(failed), failed) self.close(exception=error) raise error except (errors.TokenExpired, errors.AuthenticationException): @@ -240,7 +260,7 @@ def _send_event_data(self, event_data): self.reconnect() else: log.info("Sender detached. Shutting down.") - error = EventHubError(str(shutdown), shutdown) + error = ConnectError(str(shutdown), shutdown) self.close(exception=error) raise error except errors.MessageHandlerError as shutdown: @@ -249,7 +269,7 @@ def _send_event_data(self, event_data): self.reconnect() else: log.info("Sender detached. Shutting down.") - error = EventHubError(str(shutdown), shutdown) + error = ConnectError(str(shutdown), shutdown) self.close(exception=error) raise error except Exception as e: @@ -260,35 +280,14 @@ def _send_event_data(self, event_data): else: return self._outcome - def send(self, event_data): - """ - Sends an event data and blocks until acknowledgement is - received or operation times out. - - :param event_data: The event to be sent. - :type event_data: ~azure.eventhub.common.EventData - :raises: ~azure.eventhub.common.EventHubError if the message fails to - send. - :return: The outcome of the message send. - :rtype: ~uamqp.constants.MessageSendResult - - Example: - .. literalinclude:: ../examples/test_examples_eventhub.py - :start-after: [START eventhub_client_sync_send] - :end-before: [END eventhub_client_sync_send] - :language: python - :dedent: 4 - :caption: Sends an event data and blocks until acknowledgement is received or operation times out. - - """ - if self.error: - raise self.error - if event_data.partition_key and self.partition: - raise ValueError("EventData partition key cannot be used with a partition sender.") - event_data.message.on_send_complete = self._on_outcome - return self._send_event_data(event_data) + @staticmethod + def _set_batching_label(event_datas, batching_label): + ed_iter = iter(event_datas) + for ed in ed_iter: + ed._batching_label = batching_label + yield ed - def send_batch(self, batch_event_data): + def send(self, event_data, batching_label=None): """ Sends an event data and blocks until acknowledgement is received or operation times out. @@ -311,96 +310,16 @@ def send_batch(self, batch_event_data): """ if self.error: raise self.error - - def verify_partition(event_datas): - ed_iter = iter(event_datas) - try: - ed = next(ed_iter) - partition_key = ed.partition_key - yield ed - except StopIteration: - raise ValueError("batch_event_data must not be empty") - for ed in ed_iter: - if ed.partition_key != partition_key: - raise ValueError("partition key of all EventData must be the same if being sent in a batch") - yield ed - - wrapper_event_data = _BatchSendEventData(verify_partition(batch_event_data)) + if isinstance(event_data, EventData): + if batching_label: + event_data._batching_label = batching_label + wrapper_event_data = event_data + else: + wrapper_event_data = _BatchSendEventData( + self._set_batching_label(event_data, batching_label), + batching_label=batching_label) if batching_label else _BatchSendEventData(event_data) wrapper_event_data.message.on_send_complete = self._on_outcome - return self._send_event_data(wrapper_event_data) - - def queue_message(self, event_data, callback=None): - """ - Transfers an event data and notifies the callback when the operation is done. - - :param event_data: The event to be sent. - :type event_data: ~azure.eventhub.common.EventData - :param callback: Callback to be run once the message has been send. - This must be a function that accepts two arguments. - :type callback: callable[~uamqp.constants.MessageSendResult, ~azure.eventhub.common.EventHubError] - - Example: - .. literalinclude:: ../examples/test_examples_eventhub.py - :start-after: [START eventhub_client_transfer] - :end-before: [END eventhub_client_transfer] - :language: python - :dedent: 4 - :caption: Transfers an event data and notifies the callback when the operation is done. - - """ - if self.error: - raise self.error - if not self.running: - self.open() - if event_data.partition_key and self.partition: - raise ValueError("EventData partition key cannot be used with a partition sender.") - if callback: - event_data.message.on_send_complete = lambda o, c: callback(o, Sender._error(o, c)) - self._handler.queue_message(event_data.message) - - def send_pending_messages(self): - """ - Wait until all transferred events have been sent. - - Example: - .. literalinclude:: ../examples/test_examples_eventhub.py - :start-after: [START eventhub_client_transfer] - :end-before: [END eventhub_client_transfer] - :language: python - :dedent: 4 - :caption: Wait until all transferred events have been sent. - - """ - if self.error: - raise self.error - if not self.running: - self.open() - try: - self._handler.wait() - except (errors.TokenExpired, errors.AuthenticationException): - log.info("Sender disconnected due to token error. Attempting reconnect.") - self.reconnect() - except (errors.LinkDetach, errors.ConnectionClose) as shutdown: - if shutdown.action.retry and self.auto_reconnect: - log.info("Sender detached. Attempting reconnect.") - self.reconnect() - else: - log.info("Sender detached. Shutting down.") - error = EventHubError(str(shutdown), shutdown) - self.close(exception=error) - raise error - except errors.MessageHandlerError as shutdown: - if self.auto_reconnect: - log.info("Sender detached. Attempting reconnect.") - self.reconnect() - else: - log.info("Sender detached. Shutting down.") - error = EventHubError(str(shutdown), shutdown) - self.close(exception=error) - raise error - except Exception as e: - log.info("Unexpected error occurred (%r).", e) - raise EventHubError("Send failed: {}".format(e)) + self._send_event_data(wrapper_event_data) def _on_outcome(self, outcome, condition): """ diff --git a/sdk/eventhub/azure-eventhubs/conftest.py b/sdk/eventhub/azure-eventhubs/conftest.py index ce6f83adc6af..68a211917f4c 100644 --- a/sdk/eventhub/azure-eventhubs/conftest.py +++ b/sdk/eventhub/azure-eventhubs/conftest.py @@ -165,13 +165,11 @@ def device_id(): @pytest.fixture() def connstr_receivers(connection_str): client = EventHubClient.from_connection_string(connection_str, debug=False) - eh_hub_info = client.get_eventhub_information() - partitions = eh_hub_info["partition_ids"] - - recv_offset = EventPosition("@latest") + partitions = client.get_partition_ids() receivers = [] for p in partitions: - receiver = client.create_receiver("$default", p, prefetch=500, offset=EventPosition("@latest")) + #receiver = client.create_receiver(partition_id=p, prefetch=500, event_position=EventPosition("@latest")) + receiver = client.create_receiver(partition_id=p, prefetch=500, event_position=EventPosition("-1")) receivers.append(receiver) receiver.receive(timeout=1) yield connection_str, receivers @@ -183,12 +181,11 @@ def connstr_receivers(connection_str): @pytest.fixture() def connstr_senders(connection_str): client = EventHubClient.from_connection_string(connection_str, debug=True) - eh_hub_info = client.get_eventhub_information() - partitions = eh_hub_info["partition_ids"] + partitions = client.get_partition_ids() senders = [] for p in partitions: - sender = client.create_sender(partition=p) + sender = client.create_sender(partition_id=p) senders.append(sender) yield connection_str, senders for s in senders: diff --git a/sdk/eventhub/azure-eventhubs/examples/proxy.py b/sdk/eventhub/azure-eventhubs/examples/proxy.py new file mode 100644 index 000000000000..b4a2d51b0411 --- /dev/null +++ b/sdk/eventhub/azure-eventhubs/examples/proxy.py @@ -0,0 +1,57 @@ +#!/usr/bin/env python + +""" +An example to show sending and receiving events behind a proxy +""" +import os +import logging + +from azure.eventhub import EventHubClient, EventPosition, EventData + +import examples +logger = examples.get_logger(logging.INFO) + + +# Address can be in either of these formats: +# "amqps://:@.servicebus.windows.net/myeventhub" +# "amqps://.servicebus.windows.net/myeventhub" +ADDRESS = os.environ.get('EVENT_HUB_ADDRESS') + +# SAS policy and key are not required if they are encoded in the URL +USER = os.environ.get('EVENT_HUB_SAS_POLICY') +KEY = os.environ.get('EVENT_HUB_SAS_KEY') +CONSUMER_GROUP = "$default" +EVENT_POSITION = EventPosition.first_available() +PARTITION = "0" +HTTP_PROXY = { + 'proxy_hostname': '127.0.0.1', # proxy hostname + 'proxy_port': 3128, # proxy port + 'username': 'admin', # username used for proxy authentication if needed + 'password': '123456' # password used for proxy authentication if needed +} + + +if not ADDRESS: + raise ValueError("No EventHubs URL supplied.") + +client = EventHubClient(ADDRESS, debug=False, username=USER, password=KEY, http_proxy=HTTP_PROXY) +sender = client.create_sender(partition=PARTITION) +receiver = client.create_receiver(consumer_group=CONSUMER_GROUP, partition=PARTITION, event_position=EVENT_POSITION) +try: + event_list = [] + for i in range(20): + event_list.append(EventData("Event Number {}".format(i))) + + print('Start sending events behind a proxy.') + + with sender: + sender.send(list) + + print('Start receiving events behind a proxy.') + + with receiver: + received = receiver.receive(max_batch_size=50, timeout=5) + +except KeyboardInterrupt: + pass + diff --git a/sdk/eventhub/azure-eventhubs/setup.py b/sdk/eventhub/azure-eventhubs/setup.py index 1fdb12ec33d8..6f6cf3399021 100644 --- a/sdk/eventhub/azure-eventhubs/setup.py +++ b/sdk/eventhub/azure-eventhubs/setup.py @@ -8,6 +8,7 @@ import re import os.path +import sys from io import open from setuptools import find_packages, setup @@ -34,6 +35,22 @@ with open('HISTORY.rst') as f: history = f.read() +exclude_packages = [ + 'tests', + "tests.asynctests", + 'examples', + # Exclude packages that will be covered by PEP420 or nspkg + 'azure', + '*.eventprocessorhost', + '*.eventprocessorhost.*' + ] + +if sys.version_info[0] < 3 or (sys.version_info[0] == 3 and sys.version_info[1] < 5): + exclude_packages.extend([ + '*.aio', + '*.aio.*' + ]) + setup( name=PACKAGE_NAME, version=version, @@ -44,28 +61,25 @@ author_email='azpysdkhelp@microsoft.com', url='https://github.com/Azure/azure-sdk-for-python', classifiers=[ - 'Development Status :: 5 - Production/Stable', + 'Development Status :: 3 - Alpha', 'Programming Language :: Python', 'Programming Language :: Python :: 2', 'Programming Language :: Python :: 2.7', 'Programming Language :: Python :: 3', - 'Programming Language :: Python :: 3.4', + # 'Programming Language :: Python :: 3.4', 'Programming Language :: Python :: 3.5', 'Programming Language :: Python :: 3.6', 'Programming Language :: Python :: 3.7', 'License :: OSI Approved :: MIT License', ], zip_safe=False, - packages=find_packages(exclude=[ - "azure", - "examples", - "tests", - "tests.asynctests"]), + packages=find_packages(exclude=exclude_packages), install_requires=[ 'uamqp~=1.2.0', 'msrestazure>=0.4.32,<2.0.0', 'azure-common~=1.1', - 'azure-storage-blob~=1.3' + 'azure-storage-blob~=1.3', + 'azure-core~=1.0', ], extras_require={ ":python_version<'3.0'": ['azure-nspkg'], diff --git a/sdk/eventhub/azure-eventhubs/tests/asynctests/test_iothub_receive_async.py b/sdk/eventhub/azure-eventhubs/tests/asynctests/test_iothub_receive_async.py index a8bc39757d87..fdb5c1ffea35 100644 --- a/sdk/eventhub/azure-eventhubs/tests/asynctests/test_iothub_receive_async.py +++ b/sdk/eventhub/azure-eventhubs/tests/asynctests/test_iothub_receive_async.py @@ -9,43 +9,37 @@ import pytest import time -from azure import eventhub -from azure.eventhub import EventData, Offset, EventHubError, EventHubClientAsync +from azure.eventhub.aio import EventHubClient +from azure.eventhub import EventData, EventPosition, EventHubError async def pump(receiver, sleep=None): messages = 0 if sleep: await asyncio.sleep(sleep) - batch = await receiver.receive(timeout=1) - messages += len(batch) + async with receiver: + batch = await receiver.receive(timeout=1) + messages += len(batch) return messages async def get_partitions(iot_connection_str): - try: - client = EventHubClientAsync.from_iothub_connection_string(iot_connection_str, debug=True) - client.add_async_receiver("$default", "0", prefetch=1000, operation='/messages/events') - await client.run_async() - partitions = await client.get_eventhub_info_async() + client = EventHubClient.from_iothub_connection_string(iot_connection_str, network_tracing=True) + receiver = client.create_receiver(partition_id="0", prefetch=1000, operation='/messages/events') + async with receiver: + partitions = await client.get_properties() return partitions["partition_ids"] - finally: - await client.stop_async() @pytest.mark.liveTest @pytest.mark.asyncio async def test_iothub_receive_multiple_async(iot_connection_str): partitions = await get_partitions(iot_connection_str) - client = EventHubClientAsync.from_iothub_connection_string(iot_connection_str, debug=True) - try: - receivers = [] - for p in partitions: - receivers.append(client.add_async_receiver("$default", p, prefetch=10, operation='/messages/events')) - await client.run_async() - outputs = await asyncio.gather(*[pump(r) for r in receivers]) - - assert isinstance(outputs[0], int) and outputs[0] <= 10 - assert isinstance(outputs[1], int) and outputs[1] <= 10 - finally: - await client.stop_async() + client = EventHubClient.from_iothub_connection_string(iot_connection_str, network_tracing=True) + receivers = [] + for p in partitions: + receivers.append(client.create_receiver(partition_id=p, prefetch=10, operation='/messages/events')) + outputs = await asyncio.gather(*[pump(r) for r in receivers]) + + assert isinstance(outputs[0], int) and outputs[0] <= 10 + assert isinstance(outputs[1], int) and outputs[1] <= 10 diff --git a/sdk/eventhub/azure-eventhubs/tests/asynctests/test_longrunning_eph.py b/sdk/eventhub/azure-eventhubs/tests/asynctests/test_longrunning_eph.py index 9a51d067e312..78611b3bf2ef 100644 --- a/sdk/eventhub/azure-eventhubs/tests/asynctests/test_longrunning_eph.py +++ b/sdk/eventhub/azure-eventhubs/tests/asynctests/test_longrunning_eph.py @@ -13,7 +13,8 @@ import pytest from logging.handlers import RotatingFileHandler -from azure.eventhub import EventHubClientAsync, EventData +from azure.eventhub.aio import EventHubClient +from azure.eventhub import EventData from azure.eventprocessorhost import ( AbstractEventProcessor, AzureStorageCheckpointLeaseManager, @@ -123,13 +124,14 @@ async def pump(pid, sender, duration): total = 0 try: - while time.time() < deadline: - data = EventData(body=b"D" * 512) - sender.transfer(data) - total += 1 - if total % 100 == 0: - await sender.wait_async() - #logger.info("{}: Send total {}".format(pid, total)) + async with sender: + while time.time() < deadline: + data = EventData(body=b"D" * 512) + sender.queue_message(data) + total += 1 + if total % 100 == 0: + await sender.send_pending_messages() + #logger.info("{}: Send total {}".format(pid, total)) except Exception as err: logger.error("{}: Send failed {}".format(pid, err)) raise @@ -164,14 +166,13 @@ def test_long_running_eph(live_eventhub): live_eventhub['key_name'], live_eventhub['access_key'], live_eventhub['event_hub']) - send_client = EventHubClientAsync.from_connection_string(conn_str) + send_client = EventHubClient.from_connection_string(conn_str) pumps = [] for pid in ["0", "1"]: - sender = send_client.add_async_sender(partition=pid, send_timeout=0, keep_alive=False) + sender = send_client.create_sender(partition_id=pid, send_timeout=0, keep_alive=False) pumps.append(pump(pid, sender, 15)) - loop.run_until_complete(send_client.run_async()) results = loop.run_until_complete(asyncio.gather(*pumps, return_exceptions=True)) - loop.run_until_complete(send_client.stop_async()) + assert not any(results) # Eventhub config and storage manager @@ -198,7 +199,7 @@ def test_long_running_eph(live_eventhub): EventProcessor, eh_config, storage_manager, - ep_params=["param1","param2"], + ep_params=["param1", "param2"], eph_options=eh_options, loop=loop) diff --git a/sdk/eventhub/azure-eventhubs/tests/asynctests/test_longrunning_eph_with_context.py b/sdk/eventhub/azure-eventhubs/tests/asynctests/test_longrunning_eph_with_context.py index 3c926dd77470..7b4a9021db1d 100644 --- a/sdk/eventhub/azure-eventhubs/tests/asynctests/test_longrunning_eph_with_context.py +++ b/sdk/eventhub/azure-eventhubs/tests/asynctests/test_longrunning_eph_with_context.py @@ -13,7 +13,8 @@ import pytest from logging.handlers import RotatingFileHandler -from azure.eventhub import EventHubClientAsync, EventData +from azure.eventhub.aio import EventHubClient +from azure.eventhub import EventData from azure.eventprocessorhost import ( AbstractEventProcessor, AzureStorageCheckpointLeaseManager, @@ -128,13 +129,14 @@ async def pump(pid, sender, duration): total = 0 try: - while time.time() < deadline: - data = EventData(body=b"D" * 512) - sender.transfer(data) - total += 1 - if total % 100 == 0: - await sender.wait_async() - #logger.info("{}: Send total {}".format(pid, total)) + async with sender: + while time.time() < deadline: + data = EventData(body=b"D" * 512) + sender.queue_message(data) + total += 1 + if total % 100 == 0: + await sender.send_pending_messages() + #logger.info("{}: Send total {}".format(pid, total)) except Exception as err: logger.error("{}: Send failed {}".format(pid, err)) raise @@ -169,14 +171,12 @@ def test_long_running_context_eph(live_eventhub): live_eventhub['key_name'], live_eventhub['access_key'], live_eventhub['event_hub']) - send_client = EventHubClientAsync.from_connection_string(conn_str) + send_client = EventHubClient.from_connection_string(conn_str) pumps = [] for pid in ["0", "1"]: - sender = send_client.add_async_sender(partition=pid, send_timeout=0, keep_alive=False) + sender = send_client.add_async_sender(partition_id=pid, send_timeout=0) pumps.append(pump(pid, sender, 15)) - loop.run_until_complete(send_client.run_async()) results = loop.run_until_complete(asyncio.gather(*pumps, return_exceptions=True)) - loop.run_until_complete(send_client.stop_async()) assert not any(results) # Eventhub config and storage manager @@ -223,4 +223,4 @@ def test_long_running_context_eph(live_eventhub): config['namespace'] = os.environ['EVENT_HUB_NAMESPACE'] config['consumer_group'] = "$Default" config['partition'] = "0" - test_long_running_eph(config) + test_long_running_context_eph(config) diff --git a/sdk/eventhub/azure-eventhubs/tests/asynctests/test_longrunning_receive_async.py b/sdk/eventhub/azure-eventhubs/tests/asynctests/test_longrunning_receive_async.py index b3e7dca8a2dc..1f50144674b8 100644 --- a/sdk/eventhub/azure-eventhubs/tests/asynctests/test_longrunning_receive_async.py +++ b/sdk/eventhub/azure-eventhubs/tests/asynctests/test_longrunning_receive_async.py @@ -18,8 +18,8 @@ import pytest from logging.handlers import RotatingFileHandler -from azure.eventhub import Offset -from azure.eventhub import EventHubClientAsync +from azure.eventhub import EventPosition +from azure.eventhub.aio import EventHubClient def get_logger(filename, level=logging.INFO): @@ -48,7 +48,7 @@ def get_logger(filename, level=logging.INFO): async def get_partitions(client): - eh_data = await client.get_eventhub_info_async() + eh_data = await client.get_properties() return eh_data["partition_ids"] @@ -56,34 +56,37 @@ async def pump(_pid, receiver, _args, _dl): total = 0 iteration = 0 deadline = time.time() + _dl + try: - while time.time() < deadline: - batch = await receiver.receive(timeout=1) - size = len(batch) - total += size - iteration += 1 - if size == 0: - print("{}: No events received, queue size {}, delivered {}".format( - _pid, - receiver.queue_size, - total)) - elif iteration >= 5: - iteration = 0 - print("{}: total received {}, last sn={}, last offset={}".format( - _pid, - total, - batch[-1].sequence_number, - batch[-1].offset.value)) - print("{}: total received {}".format( - _pid, - total)) + async with receiver: + while time.time() < deadline: + batch = await receiver.receive(timeout=1) + size = len(batch) + total += size + iteration += 1 + if size == 0: + print("{}: No events received, queue size {}, delivered {}".format( + _pid, + receiver.queue_size, + total)) + elif iteration >= 5: + iteration = 0 + print("{}: total received {}, last sn={}, last offset={}".format( + _pid, + total, + batch[-1].sequence_number, + batch[-1].offset.value)) + print("{}: total received {}".format( + _pid, + total)) except Exception as e: print("Partition {} receiver failed: {}".format(_pid, e)) raise @pytest.mark.liveTest -def test_long_running_receive_async(connection_str): +@pytest.mark.asyncio +async def test_long_running_receive_async(connection_str): parser = argparse.ArgumentParser() parser.add_argument("--duration", help="Duration in seconds of the test", type=int, default=30) parser.add_argument("--consumer", help="Consumer group name", default="$default") @@ -98,11 +101,11 @@ def test_long_running_receive_async(connection_str): loop = asyncio.get_event_loop() args, _ = parser.parse_known_args() if args.conn_str: - client = EventHubClientAsync.from_connection_string( + client = EventHubClient.from_connection_string( args.conn_str, - eventhub=args.eventhub, auth_timeout=240, debug=False) + eventhub=args.eventhub, auth_timeout=240, network_tracing=False) elif args.address: - client = EventHubClientAsync( + client = EventHubClient( args.address, auth_timeout=240, username=args.sas_policy, @@ -116,22 +119,21 @@ def test_long_running_receive_async(connection_str): try: if not args.partitions: - partitions = loop.run_until_complete(get_partitions(client)) + partitions = await client.get_partition_ids() else: partitions = args.partitions.split(",") pumps = [] for pid in partitions: - receiver = client.add_async_receiver( - consumer_group=args.consumer, - partition=pid, - offset=Offset(args.offset), - prefetch=50) + receiver = client.create_receiver( + partition_id=pid, + event_position=EventPosition(args.offset), + prefetch=50, + loop=loop) pumps.append(pump(pid, receiver, args, args.duration)) - loop.run_until_complete(client.run_async()) - loop.run_until_complete(asyncio.gather(*pumps)) + await asyncio.gather(*pumps) finally: - loop.run_until_complete(client.stop_async()) + pass if __name__ == '__main__': - test_long_running_receive_async(os.environ.get('EVENT_HUB_CONNECTION_STR')) + asyncio.run(test_long_running_receive_async(os.environ.get('EVENT_HUB_CONNECTION_STR'))) diff --git a/sdk/eventhub/azure-eventhubs/tests/asynctests/test_longrunning_send_async.py b/sdk/eventhub/azure-eventhubs/tests/asynctests/test_longrunning_send_async.py index 56832f87a87d..dd87e5324558 100644 --- a/sdk/eventhub/azure-eventhubs/tests/asynctests/test_longrunning_send_async.py +++ b/sdk/eventhub/azure-eventhubs/tests/asynctests/test_longrunning_send_async.py @@ -13,7 +13,8 @@ import pytest from logging.handlers import RotatingFileHandler -from azure.eventhub import EventHubClientAsync, EventData +from azure.eventhub import EventData +from azure.eventhub.aio import EventHubClient def get_logger(filename, level=logging.INFO): @@ -47,7 +48,7 @@ def check_send_successful(outcome, condition): async def get_partitions(args): - eh_data = await args.get_eventhub_info_async() + eh_data = await args.get_properties() return eh_data["partition_ids"] @@ -65,16 +66,17 @@ def data_generator(): logger.info("{}: Sending single messages".format(pid)) try: - while time.time() < deadline: - if args.batch > 1: - data = EventData(batch=data_generator()) - else: - data = EventData(body=b"D" * args.payload) - sender.transfer(data, callback=check_send_successful) - total += args.batch - if total % 100 == 0: - await sender.wait_async() - logger.info("{}: Send total {}".format(pid, total)) + async with sender: + while time.time() < deadline: + if args.batch > 1: + data = EventData(body=data_generator()) + else: + data = EventData(body=b"D" * args.payload) + sender.queue_message(data, callback=check_send_successful) + total += args.batch + if total % 100 == 0: + await sender.send_pending_messages() + logger.info("{}: Send total {}".format(pid, total)) except Exception as err: logger.error("{}: Send failed {}".format(pid, err)) raise @@ -82,7 +84,8 @@ def data_generator(): @pytest.mark.liveTest -def test_long_running_partition_send_async(connection_str): +@pytest.mark.asyncio +async def test_long_running_partition_send_async(connection_str): parser = argparse.ArgumentParser() parser.add_argument("--duration", help="Duration in seconds of the test", type=int, default=30) parser.add_argument("--payload", help="payload size", type=int, default=1024) @@ -99,11 +102,11 @@ def test_long_running_partition_send_async(connection_str): args, _ = parser.parse_known_args() if args.conn_str: - client = EventHubClientAsync.from_connection_string( + client = EventHubClient.from_connection_string( args.conn_str, - eventhub=args.eventhub, debug=True) + eventhub=args.eventhub, network_tracing=True) elif args.address: - client = EventHubClientAsync( + client = EventHubClient( args.address, username=args.sas_policy, password=args.sas_key, @@ -117,7 +120,7 @@ def test_long_running_partition_send_async(connection_str): try: if not args.partitions: - partitions = loop.run_until_complete(get_partitions(client)) + partitions = await client.get_partition_ids() else: pid_range = args.partitions.split("-") if len(pid_range) > 1: @@ -126,16 +129,15 @@ def test_long_running_partition_send_async(connection_str): partitions = args.partitions.split(",") pumps = [] for pid in partitions: - sender = client.add_async_sender(partition=pid, send_timeout=0, keep_alive=False) + sender = client.create_sender(partition_id=pid, send_timeout=0) pumps.append(pump(pid, sender, args, args.duration)) - loop.run_until_complete(client.run_async()) - results = loop.run_until_complete(asyncio.gather(*pumps, return_exceptions=True)) + results = await asyncio.gather(*pumps, return_exceptions=True) assert not results except Exception as e: logger.error("Sender failed: {}".format(e)) finally: - logger.info("Shutting down sender") - loop.run_until_complete(client.stop_async()) + pass + if __name__ == '__main__': - test_long_running_partition_send_async(os.environ.get('EVENT_HUB_CONNECTION_STR')) + asyncio.run(test_long_running_partition_send_async(os.environ.get('EVENT_HUB_CONNECTION_STR'))) diff --git a/sdk/eventhub/azure-eventhubs/tests/asynctests/test_negative_async.py b/sdk/eventhub/azure-eventhubs/tests/asynctests/test_negative_async.py index 4b2e8b0a367b..4e904d19453f 100644 --- a/sdk/eventhub/azure-eventhubs/tests/asynctests/test_negative_async.py +++ b/sdk/eventhub/azure-eventhubs/tests/asynctests/test_negative_async.py @@ -10,102 +10,104 @@ import time import sys -from azure import eventhub from azure.eventhub import ( - EventHubClientAsync, EventData, - Offset, - EventHubError) - + EventPosition, + EventHubError, + ConnectError, + AuthenticationError, + EventDataError, +) +from azure.eventhub.aio import EventHubClient @pytest.mark.liveTest @pytest.mark.asyncio async def test_send_with_invalid_hostname_async(invalid_hostname, connstr_receivers): _, receivers = connstr_receivers - client = EventHubClientAsync.from_connection_string(invalid_hostname, debug=True) - sender = client.add_async_sender() - with pytest.raises(EventHubError): - await client.run_async() + client = EventHubClient.from_connection_string(invalid_hostname, network_tracing=True) + sender = client.create_sender() + with pytest.raises(AuthenticationError): + await sender._open() @pytest.mark.liveTest @pytest.mark.asyncio async def test_receive_with_invalid_hostname_async(invalid_hostname): - client = EventHubClientAsync.from_connection_string(invalid_hostname, debug=True) - sender = client.add_async_receiver("$default", "0") - with pytest.raises(EventHubError): - await client.run_async() + client = EventHubClient.from_connection_string(invalid_hostname, network_tracing=True) + sender = client.create_receiver(partition_id="0") + with pytest.raises(AuthenticationError): + await sender._open() @pytest.mark.liveTest @pytest.mark.asyncio async def test_send_with_invalid_key_async(invalid_key, connstr_receivers): _, receivers = connstr_receivers - client = EventHubClientAsync.from_connection_string(invalid_key, debug=False) - sender = client.add_async_sender() - with pytest.raises(EventHubError): - await client.run_async() + client = EventHubClient.from_connection_string(invalid_key, network_tracing=False) + sender = client.create_sender() + with pytest.raises(AuthenticationError): + await sender._open() @pytest.mark.liveTest @pytest.mark.asyncio async def test_receive_with_invalid_key_async(invalid_key): - client = EventHubClientAsync.from_connection_string(invalid_key, debug=True) - sender = client.add_async_receiver("$default", "0") - with pytest.raises(EventHubError): - await client.run_async() + client = EventHubClient.from_connection_string(invalid_key, network_tracing=True) + sender = client.create_receiver(partition_id="0") + with pytest.raises(AuthenticationError): + await sender._open() @pytest.mark.liveTest @pytest.mark.asyncio async def test_send_with_invalid_policy_async(invalid_policy, connstr_receivers): _, receivers = connstr_receivers - client = EventHubClientAsync.from_connection_string(invalid_policy, debug=False) - sender = client.add_async_sender() - with pytest.raises(EventHubError): - await client.run_async() + client = EventHubClient.from_connection_string(invalid_policy, network_tracing=False) + sender = client.create_sender() + with pytest.raises(AuthenticationError): + await sender._open() @pytest.mark.liveTest @pytest.mark.asyncio async def test_receive_with_invalid_policy_async(invalid_policy): - client = EventHubClientAsync.from_connection_string(invalid_policy, debug=True) - sender = client.add_async_receiver("$default", "0") - with pytest.raises(EventHubError): - await client.run_async() + client = EventHubClient.from_connection_string(invalid_policy, network_tracing=True) + sender = client.create_receiver(partition_id="0") + with pytest.raises(AuthenticationError): + await sender._open() @pytest.mark.liveTest @pytest.mark.asyncio async def test_send_partition_key_with_partition_async(connection_str): - client = EventHubClientAsync.from_connection_string(connection_str, debug=True) - sender = client.add_async_sender(partition="1") + pytest.skip("Skipped tentatively. Confirm whether to throw ValueError or just warn users") + client = EventHubClient.from_connection_string(connection_str, network_tracing=True) + sender = client.create_sender(partition_id="1") try: - await client.run_async() data = EventData(b"Data") data.partition_key = b"PKey" with pytest.raises(ValueError): await sender.send(data) finally: - await client.stop_async() + await sender.close() @pytest.mark.liveTest @pytest.mark.asyncio async def test_non_existing_entity_sender_async(connection_str): - client = EventHubClientAsync.from_connection_string(connection_str, eventhub="nemo", debug=False) - sender = client.add_async_sender(partition="1") - with pytest.raises(EventHubError): - await client.run_async() + client = EventHubClient.from_connection_string(connection_str, eventhub="nemo", network_tracing=False) + sender = client.create_sender(partition_id="1") + with pytest.raises(AuthenticationError): + await sender._open() @pytest.mark.liveTest @pytest.mark.asyncio async def test_non_existing_entity_receiver_async(connection_str): - client = EventHubClientAsync.from_connection_string(connection_str, eventhub="nemo", debug=False) - receiver = client.add_async_receiver("$default", "0") - with pytest.raises(EventHubError): - await client.run_async() + client = EventHubClient.from_connection_string(connection_str, eventhub="nemo", network_tracing=False) + receiver = client.create_receiver(partition_id="0") + with pytest.raises(AuthenticationError): + await receiver._open() @pytest.mark.liveTest @@ -113,14 +115,13 @@ async def test_non_existing_entity_receiver_async(connection_str): async def test_receive_from_invalid_partitions_async(connection_str): partitions = ["XYZ", "-1", "1000", "-" ] for p in partitions: - client = EventHubClientAsync.from_connection_string(connection_str, debug=True) - receiver = client.add_async_receiver("$default", p) + client = EventHubClient.from_connection_string(connection_str, network_tracing=True) + receiver = client.create_receiver(partition_id=p) try: - with pytest.raises(EventHubError): - await client.run_async() + with pytest.raises(ConnectError): await receiver.receive(timeout=10) finally: - await client.stop_async() + await receiver.close() @pytest.mark.liveTest @@ -128,13 +129,13 @@ async def test_receive_from_invalid_partitions_async(connection_str): async def test_send_to_invalid_partitions_async(connection_str): partitions = ["XYZ", "-1", "1000", "-" ] for p in partitions: - client = EventHubClientAsync.from_connection_string(connection_str, debug=False) - sender = client.add_async_sender(partition=p) + client = EventHubClient.from_connection_string(connection_str, network_tracing=False) + sender = client.create_sender(partition_id=p) try: - with pytest.raises(EventHubError): - await client.run_async() + with pytest.raises(ConnectError): + await sender._open() finally: - await client.stop_async() + await sender.close() @pytest.mark.liveTest @@ -142,63 +143,59 @@ async def test_send_to_invalid_partitions_async(connection_str): async def test_send_too_large_message_async(connection_str): if sys.platform.startswith('darwin'): pytest.skip("Skipping on OSX - open issue regarding message size") - client = EventHubClientAsync.from_connection_string(connection_str, debug=False) - sender = client.add_async_sender() + client = EventHubClient.from_connection_string(connection_str, network_tracing=False) + sender = client.create_sender() try: - await client.run_async() - data = EventData(b"A" * 300000) - with pytest.raises(EventHubError): + data = EventData(b"A" * 1100000) + with pytest.raises(EventDataError): await sender.send(data) finally: - await client.stop_async() + await sender.close() @pytest.mark.liveTest @pytest.mark.asyncio async def test_send_null_body_async(connection_str): - client = EventHubClientAsync.from_connection_string(connection_str, debug=False) - sender = client.add_async_sender() + client = EventHubClient.from_connection_string(connection_str, network_tracing=False) + sender = client.create_sender() try: - await client.run_async() with pytest.raises(ValueError): data = EventData(None) await sender.send(data) finally: - await client.stop_async() + await sender.close() async def pump(receiver): - messages = 0 - count = 0 - batch = await receiver.receive(timeout=10) - while batch and count <= 5: - count += 1 - messages += len(batch) + async with receiver: + messages = 0 + count = 0 batch = await receiver.receive(timeout=10) - return messages + while batch and count <= 5: + count += 1 + messages += len(batch) + batch = await receiver.receive(timeout=10) + return messages @pytest.mark.liveTest @pytest.mark.asyncio async def test_max_receivers_async(connstr_senders): connection_str, senders = connstr_senders - client = EventHubClientAsync.from_connection_string(connection_str, debug=True) + client = EventHubClient.from_connection_string(connection_str, network_tracing=True) receivers = [] for i in range(6): - receivers.append(client.add_async_receiver("$default", "0", prefetch=1000, offset=Offset('@latest'))) - try: - await client.run_async() - outputs = await asyncio.gather( - pump(receivers[0]), - pump(receivers[1]), - pump(receivers[2]), - pump(receivers[3]), - pump(receivers[4]), - pump(receivers[5]), - return_exceptions=True) - print(outputs) - failed = [o for o in outputs if isinstance(o, EventHubError)] - assert len(failed) == 1 - print(failed[0].message) - finally: - await client.stop_async() + receivers.append(client.create_receiver(partition_id="0", prefetch=1000, event_position=EventPosition('@latest'))) + + outputs = await asyncio.gather( + pump(receivers[0]), + pump(receivers[1]), + pump(receivers[2]), + pump(receivers[3]), + pump(receivers[4]), + pump(receivers[5]), + return_exceptions=True) + print(outputs) + failed = [o for o in outputs if isinstance(o, EventHubError)] + assert len(failed) == 1 + print(failed[0].message) diff --git a/sdk/eventhub/azure-eventhubs/tests/asynctests/test_properties_async.py b/sdk/eventhub/azure-eventhubs/tests/asynctests/test_properties_async.py new file mode 100644 index 000000000000..20641033e5bb --- /dev/null +++ b/sdk/eventhub/azure-eventhubs/tests/asynctests/test_properties_async.py @@ -0,0 +1,45 @@ +#------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for +# license information. +#-------------------------------------------------------------------------- + +import pytest +from azure.eventhub import SharedKeyCredentials +from azure.eventhub.aio import EventHubClient + + +@pytest.mark.liveTest +@pytest.mark.asyncio +async def test_get_properties(live_eventhub): + client = EventHubClient(live_eventhub['hostname'], live_eventhub['event_hub'], + SharedKeyCredentials(live_eventhub['key_name'], live_eventhub['access_key']) + ) + properties = await client.get_properties() + assert properties['path'] == live_eventhub['event_hub'] and properties['partition_ids'] == ['0', '1'] + + +@pytest.mark.liveTest +@pytest.mark.asyncio +async def test_get_partition_ids(live_eventhub): + client = EventHubClient(live_eventhub['hostname'], live_eventhub['event_hub'], + SharedKeyCredentials(live_eventhub['key_name'], live_eventhub['access_key']) + ) + partition_ids = await client.get_partition_ids() + assert partition_ids == ['0', '1'] + + +@pytest.mark.liveTest +@pytest.mark.asyncio +async def test_get_partition_properties(live_eventhub): + client = EventHubClient(live_eventhub['hostname'], live_eventhub['event_hub'], + SharedKeyCredentials(live_eventhub['key_name'], live_eventhub['access_key']) + ) + properties = await client.get_partition_properties('0') + assert properties['event_hub_path'] == live_eventhub['event_hub'] \ + and properties['id'] == '0' \ + and 'beginning_sequence_number' in properties \ + and 'last_enqueued_sequence_number' in properties \ + and 'last_enqueued_offset' in properties \ + and 'last_enqueued_time_utc' in properties \ + and 'is_empty' in properties diff --git a/sdk/eventhub/azure-eventhubs/tests/asynctests/test_receive_async.py b/sdk/eventhub/azure-eventhubs/tests/asynctests/test_receive_async.py index 1be11107dae0..18db98649264 100644 --- a/sdk/eventhub/azure-eventhubs/tests/asynctests/test_receive_async.py +++ b/sdk/eventhub/azure-eventhubs/tests/asynctests/test_receive_async.py @@ -9,7 +9,7 @@ import pytest import time -from azure.eventhub import EventData, EventPosition, EventHubError +from azure.eventhub import EventData, EventPosition, EventHubError, TransportType from azure.eventhub.aio import EventHubClient @@ -17,8 +17,8 @@ @pytest.mark.asyncio async def test_receive_end_of_stream_async(connstr_senders): connection_str, senders = connstr_senders - client = EventHubClient.from_connection_string(connection_str, debug=False) - receiver = client.create_receiver("$default", "0", offset=EventPosition('@latest')) + client = EventHubClient.from_connection_string(connection_str, network_tracing=False) + receiver = client.create_receiver(partition_id="0", event_position=EventPosition('@latest')) async with receiver: received = await receiver.receive(timeout=5) assert len(received) == 0 @@ -33,8 +33,8 @@ async def test_receive_end_of_stream_async(connstr_senders): @pytest.mark.asyncio async def test_receive_with_offset_async(connstr_senders): connection_str, senders = connstr_senders - client = EventHubClient.from_connection_string(connection_str, debug=False) - receiver = client.create_receiver("$default", "0", offset=EventPosition('@latest')) + client = EventHubClient.from_connection_string(connection_str, network_tracing=False) + receiver = client.create_receiver(partition_id="0", event_position=EventPosition('@latest')) async with receiver: received = await receiver.receive(timeout=5) assert len(received) == 0 @@ -44,7 +44,7 @@ async def test_receive_with_offset_async(connstr_senders): assert len(received) == 1 offset = received[0].offset - offset_receiver = client.create_receiver("$default", "0", offset=offset) + offset_receiver = client.create_receiver(partition_id="0", event_position=offset) async with offset_receiver: received = await offset_receiver.receive(timeout=5) assert len(received) == 0 @@ -57,8 +57,8 @@ async def test_receive_with_offset_async(connstr_senders): @pytest.mark.asyncio async def test_receive_with_inclusive_offset_async(connstr_senders): connection_str, senders = connstr_senders - client = EventHubClient.from_connection_string(connection_str, debug=False) - receiver = client.create_receiver("$default", "0", offset=EventPosition('@latest')) + client = EventHubClient.from_connection_string(connection_str, network_tracing=False) + receiver = client.create_receiver(partition_id="0", event_position=EventPosition('@latest')) async with receiver: received = await receiver.receive(timeout=5) assert len(received) == 0 @@ -68,7 +68,7 @@ async def test_receive_with_inclusive_offset_async(connstr_senders): assert len(received) == 1 offset = received[0].offset - offset_receiver = client.create_receiver("$default", "0", offset=EventPosition(offset.value, inclusive=True)) + offset_receiver = client.create_receiver(partition_id="0", event_position=EventPosition(offset.value, inclusive=True)) async with offset_receiver: received = await offset_receiver.receive(timeout=5) assert len(received) == 1 @@ -78,8 +78,8 @@ async def test_receive_with_inclusive_offset_async(connstr_senders): @pytest.mark.asyncio async def test_receive_with_datetime_async(connstr_senders): connection_str, senders = connstr_senders - client = EventHubClient.from_connection_string(connection_str, debug=False) - receiver = client.create_receiver("$default", "0", offset=EventPosition('@latest')) + client = EventHubClient.from_connection_string(connection_str, network_tracing=False) + receiver = client.create_receiver(partition_id="0", event_position=EventPosition('@latest')) async with receiver: received = await receiver.receive(timeout=5) assert len(received) == 0 @@ -88,7 +88,7 @@ async def test_receive_with_datetime_async(connstr_senders): assert len(received) == 1 offset = received[0].enqueued_time - offset_receiver = client.create_receiver("$default", "0", offset=EventPosition(offset)) + offset_receiver = client.create_receiver(partition_id="0", event_position=EventPosition(offset)) async with offset_receiver: received = await offset_receiver.receive(timeout=5) assert len(received) == 0 @@ -101,9 +101,10 @@ async def test_receive_with_datetime_async(connstr_senders): @pytest.mark.liveTest @pytest.mark.asyncio async def test_receive_with_sequence_no_async(connstr_senders): + # TODO: sampe problem as the sync version connection_str, senders = connstr_senders - client = EventHubClient.from_connection_string(connection_str, debug=False) - receiver = client.create_receiver("$default", "0", offset=EventPosition('@latest')) + client = EventHubClient.from_connection_string(connection_str, network_tracing=False) + receiver = client.create_receiver(partition_id="0", event_position=EventPosition('@latest')) async with receiver: received = await receiver.receive(timeout=5) assert len(received) == 0 @@ -112,7 +113,7 @@ async def test_receive_with_sequence_no_async(connstr_senders): assert len(received) == 1 offset = received[0].sequence_number - offset_receiver = client.create_receiver("$default", "0", offset=EventPosition(offset)) + offset_receiver = client.create_receiver(partition_id="0", event_position=EventPosition(offset)) async with offset_receiver: received = await offset_receiver.receive(timeout=5) assert len(received) == 0 @@ -126,8 +127,8 @@ async def test_receive_with_sequence_no_async(connstr_senders): @pytest.mark.asyncio async def test_receive_with_inclusive_sequence_no_async(connstr_senders): connection_str, senders = connstr_senders - client = EventHubClient.from_connection_string(connection_str, debug=False) - receiver = client.create_receiver("$default", "0", offset=EventPosition('@latest')) + client = EventHubClient.from_connection_string(connection_str, network_tracing=False) + receiver = client.create_receiver(partition_id="0", event_position=EventPosition('@latest')) async with receiver: received = await receiver.receive(timeout=5) assert len(received) == 0 @@ -136,7 +137,7 @@ async def test_receive_with_inclusive_sequence_no_async(connstr_senders): assert len(received) == 1 offset = received[0].sequence_number - offset_receiver = client.create_receiver("$default", "0", offset=EventPosition(offset, inclusive=True)) + offset_receiver = client.create_receiver(partition_id="0", event_position=EventPosition(offset, inclusive=True)) async with offset_receiver: received = await offset_receiver.receive(timeout=5) assert len(received) == 1 @@ -146,8 +147,8 @@ async def test_receive_with_inclusive_sequence_no_async(connstr_senders): @pytest.mark.asyncio async def test_receive_batch_async(connstr_senders): connection_str, senders = connstr_senders - client = EventHubClient.from_connection_string(connection_str, debug=False) - receiver = client.create_receiver("$default", "0", prefetch=500, offset=EventPosition('@latest')) + client = EventHubClient.from_connection_string(connection_str, network_tracing=False) + receiver = client.create_receiver(partition_id="0", prefetch=500, event_position=EventPosition('@latest')) async with receiver: received = await receiver.receive(timeout=5) assert len(received) == 0 @@ -174,19 +175,19 @@ async def pump(receiver, sleep=None): @pytest.mark.liveTest @pytest.mark.asyncio -async def test_epoch_receiver_async(connstr_senders): +async def test_exclusive_receiver_async(connstr_senders): connection_str, senders = connstr_senders senders[0].send(EventData(b"Receiving only a single event")) - client = EventHubClient.from_connection_string(connection_str, debug=False) + client = EventHubClient.from_connection_string(connection_str, network_tracing=False) receivers = [] - for epoch in [10, 20]: - receivers.append(client.create_epoch_receiver("$default", "0", epoch, prefetch=5)) + for exclusive_receiver_priority in [10, 20]: + receivers.append(client.create_receiver(partition_id="0", exclusive_receiver_priority=exclusive_receiver_priority, prefetch=5)) outputs = await asyncio.gather( pump(receivers[0]), pump(receivers[1]), return_exceptions=True) - assert isinstance(outputs[0], EventHubError) + assert isinstance(outputs[0], EventHubError) # TODO; it's LinkDetach error assert outputs[1] == 1 @@ -196,14 +197,14 @@ async def test_multiple_receiver_async(connstr_senders): connection_str, senders = connstr_senders senders[0].send(EventData(b"Receiving only a single event")) - client = EventHubClient.from_connection_string(connection_str, debug=True) - partitions = await client.get_eventhub_information() + client = EventHubClient.from_connection_string(connection_str, network_tracing=True) + partitions = await client.get_properties() assert partitions["partition_ids"] == ["0", "1"] receivers = [] for i in range(2): - receivers.append(client.create_receiver("$default", "0", prefetch=10)) + receivers.append(client.create_receiver(partition_id="0", prefetch=10)) try: - more_partitions = await client.get_eventhub_information() + more_partitions = await client.get_properties() assert more_partitions["partition_ids"] == ["0", "1"] outputs = await asyncio.gather( pump(receivers[0]), @@ -218,14 +219,14 @@ async def test_multiple_receiver_async(connstr_senders): @pytest.mark.liveTest @pytest.mark.asyncio -async def test_epoch_receiver_after_non_epoch_receiver_async(connstr_senders): +async def test_exclusive_receiver_after_non_exclusive_receiver_async(connstr_senders): connection_str, senders = connstr_senders senders[0].send(EventData(b"Receiving only a single event")) - client = EventHubClient.from_connection_string(connection_str, debug=False) + client = EventHubClient.from_connection_string(connection_str, network_tracing=False) receivers = [] - receivers.append(client.create_receiver("$default", "0", prefetch=10)) - receivers.append(client.create_epoch_receiver("$default", "0", 15, prefetch=10)) + receivers.append(client.create_receiver(partition_id="0", prefetch=10)) + receivers.append(client.create_receiver(partition_id="0", exclusive_receiver_priority=15, prefetch=10)) try: outputs = await asyncio.gather( pump(receivers[0]), @@ -240,14 +241,14 @@ async def test_epoch_receiver_after_non_epoch_receiver_async(connstr_senders): @pytest.mark.liveTest @pytest.mark.asyncio -async def test_non_epoch_receiver_after_epoch_receiver_async(connstr_senders): +async def test_non_exclusive_receiver_after_exclusive_receiver_async(connstr_senders): connection_str, senders = connstr_senders senders[0].send(EventData(b"Receiving only a single event")) - client = EventHubClient.from_connection_string(connection_str, debug=False) + client = EventHubClient.from_connection_string(connection_str, network_tracing=False) receivers = [] - receivers.append(client.create_epoch_receiver("$default", "0", 15, prefetch=10)) - receivers.append(client.create_receiver("$default", "0", prefetch=10)) + receivers.append(client.create_receiver(partition_id="0", exclusive_receiver_priority=15, prefetch=10)) + receivers.append(client.create_receiver(partition_id="0", prefetch=10)) try: outputs = await asyncio.gather( pump(receivers[0]), @@ -263,7 +264,6 @@ async def test_non_epoch_receiver_after_epoch_receiver_async(connstr_senders): @pytest.mark.liveTest @pytest.mark.asyncio async def test_receive_batch_with_app_prop_async(connstr_senders): - #pytest.skip("Waiting on uAMQP release") connection_str, senders = connstr_senders app_prop_key = "raw_prop" app_prop_value = "raw_value" @@ -279,13 +279,13 @@ def batched(): ed.application_properties = app_prop yield ed - client = EventHubClient.from_connection_string(connection_str, debug=False) - receiver = client.create_receiver("$default", "0", prefetch=500, offset=EventPosition('@latest')) + client = EventHubClient.from_connection_string(connection_str, network_tracing=False) + receiver = client.create_receiver(partition_id="0", prefetch=500, event_position=EventPosition('@latest')) async with receiver: received = await receiver.receive(timeout=5) assert len(received) == 0 - senders[0].send_batch(batched()) + senders[0].send(batched()) await asyncio.sleep(1) @@ -296,3 +296,27 @@ def batched(): assert list(message.body)[0] == "Event Data {}".format(index).encode('utf-8') assert (app_prop_key.encode('utf-8') in message.application_properties) \ and (dict(message.application_properties)[app_prop_key.encode('utf-8')] == app_prop_value.encode('utf-8')) + + +@pytest.mark.liveTest +@pytest.mark.asyncio +async def test_receive_over_websocket_async(connstr_senders): + connection_str, senders = connstr_senders + client = EventHubClient.from_connection_string(connection_str, transport_type=TransportType.AmqpOverWebsocket, network_tracing=False) + receiver = client.create_receiver(partition_id="0", prefetch=500, event_position=EventPosition('@latest')) + + event_list = [] + for i in range(20): + event_list.append(EventData("Event Number {}".format(i))) + + async with receiver: + received = await receiver.receive(timeout=5) + assert len(received) == 0 + + with senders[0]: + senders[0].send(event_list) + + time.sleep(1) + + received = await receiver.receive(max_batch_size=50, timeout=5) + assert len(received) == 20 diff --git a/sdk/eventhub/azure-eventhubs/tests/asynctests/test_reconnect_async.py b/sdk/eventhub/azure-eventhubs/tests/asynctests/test_reconnect_async.py index 9fafc0dc0069..ebbace8c8a05 100644 --- a/sdk/eventhub/azure-eventhubs/tests/asynctests/test_reconnect_async.py +++ b/sdk/eventhub/azure-eventhubs/tests/asynctests/test_reconnect_async.py @@ -9,43 +9,44 @@ import asyncio import pytest -from azure import eventhub from azure.eventhub import ( - EventHubClientAsync, EventData, - Offset, + EventPosition, EventHubError) +from azure.eventhub.aio import EventHubClient @pytest.mark.liveTest @pytest.mark.asyncio async def test_send_with_long_interval_async(connstr_receivers): connection_str, receivers = connstr_receivers - client = EventHubClientAsync.from_connection_string(connection_str, debug=True) - sender = client.add_async_sender() + client = EventHubClient.from_connection_string(connection_str, network_tracing=True) + sender = client.create_sender() try: - await client.run_async() await sender.send(EventData(b"A single event")) - for _ in range(2): - await asyncio.sleep(300) + for _ in range(1): + #await asyncio.sleep(300) + sender._handler._connection._conn.destroy() await sender.send(EventData(b"A single event")) finally: - await client.stop_async() + await sender.close() received = [] for r in receivers: - received.extend(r.receive(timeout=1)) - assert len(received) == 3 + r._handler._connection._conn.destroy() + received.extend(r.receive(timeout=1)) + assert len(received) == 2 assert list(received[0].body)[0] == b"A single event" def pump(receiver): messages = [] - batch = receiver.receive(timeout=1) - messages.extend(batch) - while batch: + with receiver: batch = receiver.receive(timeout=1) messages.extend(batch) + while batch: + batch = receiver.receive(timeout=1) + messages.extend(batch) return messages @@ -53,10 +54,9 @@ def pump(receiver): @pytest.mark.asyncio async def test_send_with_forced_conn_close_async(connstr_receivers): connection_str, receivers = connstr_receivers - client = EventHubClientAsync.from_connection_string(connection_str, debug=True) - sender = client.add_async_sender() + client = EventHubClient.from_connection_string(connection_str, network_tracing=True) + sender = client.create_sender() try: - await client.run_async() await sender.send(EventData(b"A single event")) sender._handler._message_sender.destroy() await asyncio.sleep(300) @@ -67,28 +67,10 @@ async def test_send_with_forced_conn_close_async(connstr_receivers): await sender.send(EventData(b"A single event")) await sender.send(EventData(b"A single event")) finally: - await client.stop_async() + await sender.close() received = [] for r in receivers: received.extend(pump(r)) assert len(received) == 5 assert list(received[0].body)[0] == b"A single event" - - -# def test_send_with_forced_link_detach(connstr_receivers): -# connection_str, receivers = connstr_receivers -# client = EventHubClient.from_connection_string(connection_str, debug=True) -# sender = client.add_sender() -# size = 20 * 1024 -# try: -# client.run() -# for i in range(1000): -# sender.transfer(EventData([b"A"*size, b"B"*size, b"C"*size, b"D"*size, b"A"*size, b"B"*size, b"C"*size, b"D"*size, b"A"*size, b"B"*size, b"C"*size, b"D"*size])) -# sender.wait() -# finally: -# client.stop() - -# received = [] -# for r in receivers: -# received.extend(r.receive(timeout=10)) diff --git a/sdk/eventhub/azure-eventhubs/tests/asynctests/test_send_async.py b/sdk/eventhub/azure-eventhubs/tests/asynctests/test_send_async.py index b17dad9cae2c..9883be044345 100644 --- a/sdk/eventhub/azure-eventhubs/tests/asynctests/test_send_async.py +++ b/sdk/eventhub/azure-eventhubs/tests/asynctests/test_send_async.py @@ -11,7 +11,7 @@ import time import json -from azure.eventhub import EventData +from azure.eventhub import EventData, TransportType from azure.eventhub.aio import EventHubClient @@ -19,34 +19,35 @@ @pytest.mark.asyncio async def test_send_with_partition_key_async(connstr_receivers): connection_str, receivers = connstr_receivers - client = EventHubClient.from_connection_string(connection_str, debug=False) + client = EventHubClient.from_connection_string(connection_str, network_tracing=False) sender = client.create_sender() - data_val = 0 - for partition in [b"a", b"b", b"c", b"d", b"e", b"f"]: - partition_key = b"test_partition_" + partition - for i in range(50): - data = EventData(str(data_val)) - data.partition_key = partition_key - data_val += 1 - await sender.send(data) + async with sender: + data_val = 0 + for partition in [b"a", b"b", b"c", b"d", b"e", b"f"]: + partition_key = b"test_partition_" + partition + for i in range(50): + data = EventData(str(data_val)) + # data.partition_key = partition_key + data_val += 1 + await sender.send(data, batching_label=partition_key) found_partition_keys = {} for index, partition in enumerate(receivers): received = partition.receive(timeout=5) for message in received: try: - existing = found_partition_keys[message.partition_key] + existing = found_partition_keys[message._batching_label] assert existing == index except KeyError: - found_partition_keys[message.partition_key] = index + found_partition_keys[message._batching_label] = index @pytest.mark.liveTest @pytest.mark.asyncio async def test_send_and_receive_zero_length_body_async(connstr_receivers): connection_str, receivers = connstr_receivers - client = EventHubClient.from_connection_string(connection_str, debug=False) + client = EventHubClient.from_connection_string(connection_str, network_tracing=False) sender = client.create_sender() async with sender: await sender.send(EventData("")) @@ -63,7 +64,7 @@ async def test_send_and_receive_zero_length_body_async(connstr_receivers): @pytest.mark.asyncio async def test_send_single_event_async(connstr_receivers): connection_str, receivers = connstr_receivers - client = EventHubClient.from_connection_string(connection_str, debug=False) + client = EventHubClient.from_connection_string(connection_str, network_tracing=False) sender = client.create_sender() async with sender: await sender.send(EventData(b"A single event")) @@ -80,14 +81,15 @@ async def test_send_single_event_async(connstr_receivers): @pytest.mark.asyncio async def test_send_batch_async(connstr_receivers): connection_str, receivers = connstr_receivers + def batched(): for i in range(10): yield EventData("Event number {}".format(i)) - client = EventHubClient.from_connection_string(connection_str, debug=False) + client = EventHubClient.from_connection_string(connection_str, network_tracing=False) sender = client.create_sender() async with sender: - await sender.send_batch(batched()) + await sender.send(batched()) time.sleep(1) received = [] @@ -103,8 +105,8 @@ def batched(): @pytest.mark.asyncio async def test_send_partition_async(connstr_receivers): connection_str, receivers = connstr_receivers - client = EventHubClient.from_connection_string(connection_str, debug=False) - sender = client.create_sender(partition="1") + client = EventHubClient.from_connection_string(connection_str, network_tracing=False) + sender = client.create_sender(partition_id="1") async with sender: await sender.send(EventData(b"Data")) @@ -118,8 +120,8 @@ async def test_send_partition_async(connstr_receivers): @pytest.mark.asyncio async def test_send_non_ascii_async(connstr_receivers): connection_str, receivers = connstr_receivers - client = EventHubClient.from_connection_string(connection_str, debug=False) - sender = client.create_sender(partition="0") + client = EventHubClient.from_connection_string(connection_str, network_tracing=False) + sender = client.create_sender(partition_id="0") async with sender: await sender.send(EventData("é,è,à,ù,â,ê,î,ô,û")) await sender.send(EventData(json.dumps({"foo": "漢字"}))) @@ -139,10 +141,10 @@ def batched(): for i in range(10): yield EventData("Event number {}".format(i)) - client = EventHubClient.from_connection_string(connection_str, debug=False) - sender = client.create_sender(partition="1") + client = EventHubClient.from_connection_string(connection_str, network_tracing=False) + sender = client.create_sender(partition_id="1") async with sender: - await sender.send_batch(batched()) + await sender.send(batched()) partition_0 = receivers[0].receive(timeout=2) assert len(partition_0) == 0 @@ -154,7 +156,7 @@ def batched(): @pytest.mark.asyncio async def test_send_array_async(connstr_receivers): connection_str, receivers = connstr_receivers - client = EventHubClient.from_connection_string(connection_str, debug=False) + client = EventHubClient.from_connection_string(connection_str, network_tracing=False) sender = client.create_sender() async with sender: await sender.send(EventData([b"A", b"B", b"C"])) @@ -171,9 +173,9 @@ async def test_send_array_async(connstr_receivers): @pytest.mark.asyncio async def test_send_multiple_clients_async(connstr_receivers): connection_str, receivers = connstr_receivers - client = EventHubClient.from_connection_string(connection_str, debug=False) - sender_0 = client.create_sender(partition="0") - sender_1 = client.create_sender(partition="1") + client = EventHubClient.from_connection_string(connection_str, network_tracing=False) + sender_0 = client.create_sender(partition_id="0") + sender_1 = client.create_sender(partition_id="1") async with sender_0 and sender_1: await sender_0.send(EventData(b"Message 0")) await sender_1.send(EventData(b"Message 1")) @@ -187,7 +189,6 @@ async def test_send_multiple_clients_async(connstr_receivers): @pytest.mark.liveTest @pytest.mark.asyncio async def test_send_batch_with_app_prop_async(connstr_receivers): - # pytest.skip("Waiting on uAMQP release") connection_str, receivers = connstr_receivers app_prop_key = "raw_prop" app_prop_value = "raw_value" @@ -197,16 +198,16 @@ def batched(): for i in range(10): ed = EventData("Event number {}".format(i)) ed.application_properties = app_prop - yield "Event number {}".format(i) + yield ed for i in range(10, 20): ed = EventData("Event number {}".format(i)) ed.application_properties = app_prop - yield "Event number {}".format(i) + yield ed - client = EventHubClient.from_connection_string(connection_str, debug=False) + client = EventHubClient.from_connection_string(connection_str, network_tracing=False) sender = client.create_sender() async with sender: - await sender.send_batch(batched()) + await sender.send(batched()) time.sleep(1) @@ -219,3 +220,25 @@ def batched(): assert list(message.body)[0] == "Event number {}".format(index).encode('utf-8') assert (app_prop_key.encode('utf-8') in message.application_properties) \ and (dict(message.application_properties)[app_prop_key.encode('utf-8')] == app_prop_value.encode('utf-8')) + + +@pytest.mark.liveTest +@pytest.mark.asyncio +async def test_send_over_websocket_async(connstr_receivers): + connection_str, receivers = connstr_receivers + client = EventHubClient.from_connection_string(connection_str, transport_type=TransportType.AmqpOverWebsocket, network_tracing=False) + sender = client.create_sender() + + event_list = [] + for i in range(20): + event_list.append(EventData("Event Number {}".format(i))) + + async with sender: + await sender.send(event_list) + + time.sleep(1) + received = [] + for r in receivers: + received.extend(r.receive(timeout=3)) + + assert len(received) == 20 diff --git a/sdk/eventhub/azure-eventhubs/tests/test_iothub_receive.py b/sdk/eventhub/azure-eventhubs/tests/test_iothub_receive.py index ce3db34940e8..82add37ef868 100644 --- a/sdk/eventhub/azure-eventhubs/tests/test_iothub_receive.py +++ b/sdk/eventhub/azure-eventhubs/tests/test_iothub_receive.py @@ -8,19 +8,19 @@ import pytest import time -from azure import eventhub -from azure.eventhub import EventData, EventHubClient, Offset +from azure.eventhub import EventData, EventHubClient @pytest.mark.liveTest def test_iothub_receive_sync(iot_connection_str, device_id): - client = EventHubClient.from_iothub_connection_string(iot_connection_str, debug=True) - receiver = client.add_receiver("$default", "0", operation='/messages/events') + pytest.skip("current code will cause ErrorCodes.LinkRedirect") + client = EventHubClient.from_iothub_connection_string(iot_connection_str, network_tracing=True) + receiver = client.create_receiver(partition_id="0", operation='/messages/events') + receiver._open() try: - client.run() - partitions = client.get_eventhub_info() + partitions = client.get_properties() assert partitions["partition_ids"] == ["0", "1", "2", "3"] received = receiver.receive(timeout=5) assert len(received) == 0 finally: - client.stop() \ No newline at end of file + receiver.close() diff --git a/sdk/eventhub/azure-eventhubs/tests/test_iothub_send.py b/sdk/eventhub/azure-eventhubs/tests/test_iothub_send.py index 96d4adaa4cf1..df7a184a227e 100644 --- a/sdk/eventhub/azure-eventhubs/tests/test_iothub_send.py +++ b/sdk/eventhub/azure-eventhubs/tests/test_iothub_send.py @@ -11,19 +11,16 @@ from uamqp.message import MessageProperties -from azure import eventhub from azure.eventhub import EventData, EventHubClient @pytest.mark.liveTest def test_iothub_send_single_event(iot_connection_str, device_id): - client = EventHubClient.from_iothub_connection_string(iot_connection_str, debug=True) - sender = client.add_sender(operation='/messages/devicebound') + client = EventHubClient.from_iothub_connection_string(iot_connection_str, network_tracing=True) + sender = client.create_sender(operation='/messages/devicebound') try: - client.run() - outcome = sender.send(EventData(b"A single event", to_device=device_id)) - assert outcome.value == 0 + sender.send(EventData(b"A single event", to_device=device_id)) except: raise finally: - client.stop() + sender.close() diff --git a/sdk/eventhub/azure-eventhubs/tests/test_longrunning_receive.py b/sdk/eventhub/azure-eventhubs/tests/test_longrunning_receive.py index 1afbd9c05103..7ae53a0b2496 100644 --- a/sdk/eventhub/azure-eventhubs/tests/test_longrunning_receive.py +++ b/sdk/eventhub/azure-eventhubs/tests/test_longrunning_receive.py @@ -18,7 +18,7 @@ from logging.handlers import RotatingFileHandler -from azure.eventhub import Offset +from azure.eventhub import EventPosition from azure.eventhub import EventHubClient def get_logger(filename, level=logging.INFO): @@ -47,7 +47,7 @@ def get_logger(filename, level=logging.INFO): def get_partitions(args): - eh_data = args.get_eventhub_info() + eh_data = args.get_properties() return eh_data["partition_ids"] @@ -97,7 +97,7 @@ def test_long_running_receive(connection_str): if args.conn_str: client = EventHubClient.from_connection_string( args.conn_str, - eventhub=args.eventhub, debug=False) + eventhub=args.eventhub, network_tracing=False) elif args.address: client = EventHubClient( args.address, @@ -117,15 +117,14 @@ def test_long_running_receive(connection_str): partitions = args.partitions.split(",") pumps = {} for pid in partitions: - pumps[pid] = client.add_receiver( - consumer_group=args.consumer, - partition=pid, - offset=Offset(args.offset), + pumps[pid] = client.create_receiver( + partition_id=pid, + event_position=EventPosition(args.offset), prefetch=50) - client.run() pump(pumps, args.duration) finally: - client.stop() + for pid in partitions: + pumps[pid].close() if __name__ == '__main__': diff --git a/sdk/eventhub/azure-eventhubs/tests/test_longrunning_send.py b/sdk/eventhub/azure-eventhubs/tests/test_longrunning_send.py index 31744d8550dd..90c6d0dc3cf9 100644 --- a/sdk/eventhub/azure-eventhubs/tests/test_longrunning_send.py +++ b/sdk/eventhub/azure-eventhubs/tests/test_longrunning_send.py @@ -51,8 +51,7 @@ def check_send_successful(outcome, condition): def main(client, args): - sender = client.add_sender() - client.run() + sender = client.create_sender() deadline = time.time() + args.duration total = 0 @@ -70,16 +69,16 @@ def data_generator(): if args.batch > 1: data = EventData(batch=data_generator()) else: - data = EventData(body=b"D" * args.payload) - sender.transfer(data, callback=check_send_successful) + data = EventData(batch=b"D" * args.payload) + sender.queue_message(data, callback=check_send_successful) total += args.batch if total % 10000 == 0: - sender.wait() - print("Send total {}".format(total)) + sender.send_pending_messages() + print("Send total {}".format(total)) except Exception as err: print("Send failed {}".format(err)) finally: - client.stop() + sender.close() print("Sent total {}".format(total)) diff --git a/sdk/eventhub/azure-eventhubs/tests/test_negative.py b/sdk/eventhub/azure-eventhubs/tests/test_negative.py index 28fd7493ef13..206c5b415002 100644 --- a/sdk/eventhub/azure-eventhubs/tests/test_negative.py +++ b/sdk/eventhub/azure-eventhubs/tests/test_negative.py @@ -9,159 +9,156 @@ import time import sys -from azure import eventhub from azure.eventhub import ( EventData, - Offset, + EventPosition, EventHubError, + AuthenticationError, + ConnectError, + EventDataError, EventHubClient) @pytest.mark.liveTest def test_send_with_invalid_hostname(invalid_hostname, connstr_receivers): _, receivers = connstr_receivers - client = EventHubClient.from_connection_string(invalid_hostname, debug=False) - sender = client.add_sender() - with pytest.raises(EventHubError): - client.run() + client = EventHubClient.from_connection_string(invalid_hostname, network_tracing=False) + sender = client.create_sender() + with pytest.raises(AuthenticationError): + sender._open() @pytest.mark.liveTest def test_receive_with_invalid_hostname_sync(invalid_hostname): - client = EventHubClient.from_connection_string(invalid_hostname, debug=True) - receiver = client.add_receiver("$default", "0") - with pytest.raises(EventHubError): - client.run() + client = EventHubClient.from_connection_string(invalid_hostname, network_tracing=True) + receiver = client.create_receiver(partition_id="0") + with pytest.raises(AuthenticationError): + receiver._open() @pytest.mark.liveTest def test_send_with_invalid_key(invalid_key, connstr_receivers): _, receivers = connstr_receivers - client = EventHubClient.from_connection_string(invalid_key, debug=False) - sender = client.add_sender() - with pytest.raises(EventHubError): - client.run() + client = EventHubClient.from_connection_string(invalid_key, network_tracing=False) + sender = client.create_sender() + with pytest.raises(AuthenticationError): + sender._open() @pytest.mark.liveTest def test_receive_with_invalid_key_sync(invalid_key): - client = EventHubClient.from_connection_string(invalid_key, debug=True) - receiver = client.add_receiver("$default", "0") - with pytest.raises(EventHubError): - client.run() + client = EventHubClient.from_connection_string(invalid_key, network_tracing=True) + receiver = client.create_receiver(partition_id="0") + with pytest.raises(AuthenticationError): + receiver._open() @pytest.mark.liveTest def test_send_with_invalid_policy(invalid_policy, connstr_receivers): _, receivers = connstr_receivers - client = EventHubClient.from_connection_string(invalid_policy, debug=False) - sender = client.add_sender() - with pytest.raises(EventHubError): - client.run() + client = EventHubClient.from_connection_string(invalid_policy, network_tracing=False) + sender = client.create_sender() + with pytest.raises(AuthenticationError): + sender._open() @pytest.mark.liveTest def test_receive_with_invalid_policy_sync(invalid_policy): - client = EventHubClient.from_connection_string(invalid_policy, debug=True) - receiver = client.add_receiver("$default", "0") - with pytest.raises(EventHubError): - client.run() + client = EventHubClient.from_connection_string(invalid_policy, network_tracing=True) + receiver = client.create_receiver(partition_id="0") + with pytest.raises(AuthenticationError): + receiver._open() @pytest.mark.liveTest def test_send_partition_key_with_partition_sync(connection_str): - client = EventHubClient.from_connection_string(connection_str, debug=True) - sender = client.add_sender(partition="1") + pytest.skip("Skipped tentatively. Confirm whether to throw ValueError or just warn users") + client = EventHubClient.from_connection_string(connection_str, network_tracing=True) + sender = client.create_sender(partition_id="1") try: - client.run() data = EventData(b"Data") data.partition_key = b"PKey" with pytest.raises(ValueError): sender.send(data) finally: - client.stop() + sender.close() @pytest.mark.liveTest def test_non_existing_entity_sender(connection_str): - client = EventHubClient.from_connection_string(connection_str, eventhub="nemo", debug=False) - sender = client.add_sender(partition="1") - with pytest.raises(EventHubError): - client.run() + client = EventHubClient.from_connection_string(connection_str, eventhub="nemo", network_tracing=False) + sender = client.create_sender(partition_id="1") + with pytest.raises(AuthenticationError): + sender._open() @pytest.mark.liveTest def test_non_existing_entity_receiver(connection_str): - client = EventHubClient.from_connection_string(connection_str, eventhub="nemo", debug=False) - receiver = client.add_receiver("$default", "0") - with pytest.raises(EventHubError): - client.run() + client = EventHubClient.from_connection_string(connection_str, eventhub="nemo", network_tracing=False) + receiver = client.create_receiver(partition_id="0") + with pytest.raises(AuthenticationError): + receiver._open() @pytest.mark.liveTest def test_receive_from_invalid_partitions_sync(connection_str): partitions = ["XYZ", "-1", "1000", "-" ] for p in partitions: - client = EventHubClient.from_connection_string(connection_str, debug=True) - receiver = client.add_receiver("$default", p) + client = EventHubClient.from_connection_string(connection_str, network_tracing=True) + receiver = client.create_receiver(partition_id=p) try: - with pytest.raises(EventHubError): - client.run() + with pytest.raises(ConnectError): receiver.receive(timeout=10) finally: - client.stop() + receiver.close() @pytest.mark.liveTest def test_send_to_invalid_partitions(connection_str): partitions = ["XYZ", "-1", "1000", "-" ] for p in partitions: - client = EventHubClient.from_connection_string(connection_str, debug=False) - sender = client.add_sender(partition=p) + client = EventHubClient.from_connection_string(connection_str, network_tracing=False) + sender = client.create_sender(partition_id=p) try: - with pytest.raises(EventHubError): - client.run() + with pytest.raises(ConnectError): + sender._open() finally: - client.stop() + sender.close() @pytest.mark.liveTest def test_send_too_large_message(connection_str): if sys.platform.startswith('darwin'): pytest.skip("Skipping on OSX - open issue regarding message size") - client = EventHubClient.from_connection_string(connection_str, debug=True) - sender = client.add_sender() + client = EventHubClient.from_connection_string(connection_str, network_tracing=True) + sender = client.create_sender() try: - client.run() - data = EventData(b"A" * 300000) - with pytest.raises(EventHubError): + data = EventData(b"A" * 1100000) + with pytest.raises(EventDataError): sender.send(data) finally: - client.stop() + sender.close() @pytest.mark.liveTest def test_send_null_body(connection_str): - partitions = ["XYZ", "-1", "1000", "-" ] - client = EventHubClient.from_connection_string(connection_str, debug=False) - sender = client.add_sender() + partitions = ["XYZ", "-1", "1000", "-"] + client = EventHubClient.from_connection_string(connection_str, network_tracing=False) + sender = client.create_sender() try: - client.run() with pytest.raises(ValueError): data = EventData(None) sender.send(data) finally: - client.stop() + sender.close() @pytest.mark.liveTest def test_message_body_types(connstr_senders): connection_str, senders = connstr_senders - client = EventHubClient.from_connection_string(connection_str, debug=False) - receiver = client.add_receiver("$default", "0", offset=Offset('@latest')) + client = EventHubClient.from_connection_string(connection_str, network_tracing=False) + receiver = client.create_receiver(partition_id="0", event_position=EventPosition('@latest')) try: - client.run() - received = receiver.receive(timeout=5) assert len(received) == 0 senders[0].send(EventData(b"Bytes Data")) @@ -207,4 +204,4 @@ def test_message_body_types(connstr_senders): except: raise finally: - client.stop() \ No newline at end of file + receiver.close() diff --git a/sdk/eventhub/azure-eventhubs/tests/test_properties.py b/sdk/eventhub/azure-eventhubs/tests/test_properties.py new file mode 100644 index 000000000000..b1889bdcf179 --- /dev/null +++ b/sdk/eventhub/azure-eventhubs/tests/test_properties.py @@ -0,0 +1,41 @@ +#------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for +# license information. +#-------------------------------------------------------------------------- + +import pytest +from azure.eventhub import EventHubClient, SharedKeyCredentials + + +@pytest.mark.liveTest +def test_get_properties(live_eventhub): + client = EventHubClient(live_eventhub['hostname'], live_eventhub['event_hub'], + SharedKeyCredentials(live_eventhub['key_name'], live_eventhub['access_key']) + ) + properties = client.get_properties() + assert properties['path'] == live_eventhub['event_hub'] and properties['partition_ids'] == ['0', '1'] + + +@pytest.mark.liveTest +def test_get_partition_ids(live_eventhub): + client = EventHubClient(live_eventhub['hostname'], live_eventhub['event_hub'], + SharedKeyCredentials(live_eventhub['key_name'], live_eventhub['access_key']) + ) + partition_ids = client.get_partition_ids() + assert partition_ids == ['0', '1'] + + +@pytest.mark.liveTest +def test_get_partition_properties(live_eventhub): + client = EventHubClient(live_eventhub['hostname'], live_eventhub['event_hub'], + SharedKeyCredentials(live_eventhub['key_name'], live_eventhub['access_key']) + ) + properties = client.get_partition_properties('0') + assert properties['event_hub_path'] == live_eventhub['event_hub'] \ + and properties['id'] == '0' \ + and 'beginning_sequence_number' in properties \ + and 'last_enqueued_sequence_number' in properties \ + and 'last_enqueued_offset' in properties \ + and 'last_enqueued_time_utc' in properties \ + and 'is_empty' in properties diff --git a/sdk/eventhub/azure-eventhubs/tests/test_receive.py b/sdk/eventhub/azure-eventhubs/tests/test_receive.py index 51fbb3a6079a..38944e553bdf 100644 --- a/sdk/eventhub/azure-eventhubs/tests/test_receive.py +++ b/sdk/eventhub/azure-eventhubs/tests/test_receive.py @@ -9,13 +9,13 @@ import time import datetime -from azure.eventhub import EventData, EventHubClient, EventPosition +from azure.eventhub import EventData, EventHubClient, EventPosition, TransportType # def test_receive_without_events(connstr_senders): # connection_str, senders = connstr_senders -# client = EventHubClient.from_connection_string(connection_str, debug=True) -# receiver = client.create_receiver("$default", "0", offset=EventPosition('@latest')) +# client = EventHubClient.from_connection_string(connection_str, network_tracing=True) +# receiver = client.create_receiver("$default", "0", event_position=EventPosition('@latest')) # finish = datetime.datetime.now() + datetime.timedelta(seconds=240) # count = 0 # try: @@ -36,8 +36,8 @@ @pytest.mark.liveTest def test_receive_end_of_stream(connstr_senders): connection_str, senders = connstr_senders - client = EventHubClient.from_connection_string(connection_str, debug=False) - receiver = client.create_receiver("$default", "0", offset=EventPosition('@latest')) + client = EventHubClient.from_connection_string(connection_str, network_tracing=False) + receiver = client.create_receiver(partition_id="0", event_position=EventPosition('@latest')) with receiver: received = receiver.receive(timeout=5) assert len(received) == 0 @@ -52,12 +52,12 @@ def test_receive_end_of_stream(connstr_senders): @pytest.mark.liveTest def test_receive_with_offset_sync(connstr_senders): connection_str, senders = connstr_senders - client = EventHubClient.from_connection_string(connection_str, debug=False) - partitions = client.get_eventhub_information() + client = EventHubClient.from_connection_string(connection_str, network_tracing=False) + partitions = client.get_properties() assert partitions["partition_ids"] == ["0", "1"] - receiver = client.create_receiver("$default", "0", offset=EventPosition('@latest')) + receiver = client.create_receiver(partition_id="0", event_position=EventPosition('@latest')) with receiver: - more_partitions = client.get_eventhub_information() + more_partitions = client.get_properties() assert more_partitions["partition_ids"] == ["0", "1"] received = receiver.receive(timeout=5) @@ -70,7 +70,7 @@ def test_receive_with_offset_sync(connstr_senders): assert list(received[0].body) == [b'Data'] assert received[0].body_as_str() == "Data" - offset_receiver = client.create_receiver("$default", "0", offset=offset) + offset_receiver = client.create_receiver(partition_id="0", event_position=offset) with offset_receiver: received = offset_receiver.receive(timeout=5) assert len(received) == 0 @@ -82,8 +82,8 @@ def test_receive_with_offset_sync(connstr_senders): @pytest.mark.liveTest def test_receive_with_inclusive_offset(connstr_senders): connection_str, senders = connstr_senders - client = EventHubClient.from_connection_string(connection_str, debug=False) - receiver = client.create_receiver("$default", "0", offset=EventPosition('@latest')) + client = EventHubClient.from_connection_string(connection_str, network_tracing=False) + receiver = client.create_receiver(partition_id="0", event_position=EventPosition('@latest')) with receiver: received = receiver.receive(timeout=5) @@ -97,7 +97,7 @@ def test_receive_with_inclusive_offset(connstr_senders): assert list(received[0].body) == [b'Data'] assert received[0].body_as_str() == "Data" - offset_receiver = client.create_receiver("$default", "0", offset=EventPosition(offset.value, inclusive=True)) + offset_receiver = client.create_receiver(partition_id="0", event_position=EventPosition(offset.value, inclusive=True)) with offset_receiver: received = offset_receiver.receive(timeout=5) assert len(received) == 1 @@ -106,12 +106,12 @@ def test_receive_with_inclusive_offset(connstr_senders): @pytest.mark.liveTest def test_receive_with_datetime_sync(connstr_senders): connection_str, senders = connstr_senders - client = EventHubClient.from_connection_string(connection_str, debug=False) - partitions = client.get_eventhub_information() + client = EventHubClient.from_connection_string(connection_str, network_tracing=False) + partitions = client.get_properties() assert partitions["partition_ids"] == ["0", "1"] - receiver = client.create_receiver("$default", "0", offset=EventPosition('@latest')) + receiver = client.create_receiver(partition_id="0", event_position=EventPosition('@latest')) with receiver: - more_partitions = client.get_eventhub_information() + more_partitions = client.get_properties() assert more_partitions["partition_ids"] == ["0", "1"] received = receiver.receive(timeout=5) assert len(received) == 0 @@ -123,7 +123,7 @@ def test_receive_with_datetime_sync(connstr_senders): assert list(received[0].body) == [b'Data'] assert received[0].body_as_str() == "Data" - offset_receiver = client.create_receiver("$default", "0", offset=EventPosition(offset)) + offset_receiver = client.create_receiver(partition_id="0", event_position=EventPosition(offset)) with offset_receiver: received = offset_receiver.receive(timeout=5) assert len(received) == 0 @@ -135,7 +135,7 @@ def test_receive_with_datetime_sync(connstr_senders): @pytest.mark.liveTest def test_receive_with_custom_datetime_sync(connstr_senders): connection_str, senders = connstr_senders - client = EventHubClient.from_connection_string(connection_str, debug=False) + client = EventHubClient.from_connection_string(connection_str, network_tracing=False) for i in range(5): senders[0].send(EventData(b"Message before timestamp")) time.sleep(60) @@ -145,7 +145,7 @@ def test_receive_with_custom_datetime_sync(connstr_senders): for i in range(5): senders[0].send(EventData(b"Message after timestamp")) - receiver = client.create_receiver("$default", "0", offset=EventPosition(offset)) + receiver = client.create_receiver(partition_id="0", event_position=EventPosition(offset)) with receiver: all_received = [] received = receiver.receive(timeout=1) @@ -162,8 +162,9 @@ def test_receive_with_custom_datetime_sync(connstr_senders): @pytest.mark.liveTest def test_receive_with_sequence_no(connstr_senders): connection_str, senders = connstr_senders - client = EventHubClient.from_connection_string(connection_str, debug=False) - receiver = client.create_receiver("$default", "0", offset=EventPosition('@latest')) + client = EventHubClient.from_connection_string(connection_str, network_tracing=False) + receiver = client.create_receiver(partition_id="0", event_position=EventPosition('@latest')) + with receiver: received = receiver.receive(timeout=5) assert len(received) == 0 @@ -173,7 +174,7 @@ def test_receive_with_sequence_no(connstr_senders): assert len(received) == 1 offset = received[0].sequence_number - offset_receiver = client.create_receiver("$default", "0", offset=EventPosition(offset)) + offset_receiver = client.create_receiver(partition_id="0", event_position=EventPosition(offset, False)) with offset_receiver: received = offset_receiver.receive(timeout=5) assert len(received) == 0 @@ -182,12 +183,11 @@ def test_receive_with_sequence_no(connstr_senders): received = offset_receiver.receive(timeout=5) assert len(received) == 1 - @pytest.mark.liveTest def test_receive_with_inclusive_sequence_no(connstr_senders): connection_str, senders = connstr_senders - client = EventHubClient.from_connection_string(connection_str, debug=False) - receiver = client.create_receiver("$default", "0", offset=EventPosition('@latest')) + client = EventHubClient.from_connection_string(connection_str, network_tracing=False) + receiver = client.create_receiver(partition_id="0", event_position=EventPosition('@latest')) with receiver: received = receiver.receive(timeout=5) assert len(received) == 0 @@ -195,7 +195,7 @@ def test_receive_with_inclusive_sequence_no(connstr_senders): received = receiver.receive(timeout=5) assert len(received) == 1 offset = received[0].sequence_number - offset_receiver = client.create_receiver("$default", "0", offset=EventPosition(offset, inclusive=True)) + offset_receiver = client.create_receiver(partition_id="0", event_position=EventPosition(offset, inclusive=True)) with offset_receiver: received = offset_receiver.receive(timeout=5) assert len(received) == 1 @@ -204,8 +204,8 @@ def test_receive_with_inclusive_sequence_no(connstr_senders): @pytest.mark.liveTest def test_receive_batch(connstr_senders): connection_str, senders = connstr_senders - client = EventHubClient.from_connection_string(connection_str, debug=False) - receiver = client.create_receiver("$default", "0", prefetch=500, offset=EventPosition('@latest')) + client = EventHubClient.from_connection_string(connection_str, network_tracing=False) + receiver = client.create_receiver(partition_id="0", prefetch=500, event_position=EventPosition('@latest')) with receiver: received = receiver.receive(timeout=5) assert len(received) == 0 @@ -233,13 +233,13 @@ def batched(): ed.application_properties = batch_app_prop yield ed - client = EventHubClient.from_connection_string(connection_str, debug=False) - receiver = client.create_receiver("$default", "0", prefetch=500, offset=EventPosition('@latest')) + client = EventHubClient.from_connection_string(connection_str, network_tracing=False) + receiver = client.create_receiver(partition_id="0", prefetch=500, event_position=EventPosition('@latest')) with receiver: received = receiver.receive(timeout=5) assert len(received) == 0 - senders[0].send_batch(batched()) + senders[0].send(batched()) time.sleep(1) @@ -251,3 +251,25 @@ def batched(): assert (app_prop_key.encode('utf-8') in message.application_properties) \ and (dict(message.application_properties)[app_prop_key.encode('utf-8')] == app_prop_value.encode('utf-8')) + +@pytest.mark.liveTest +def test_receive_over_websocket_sync(connstr_senders): + connection_str, senders = connstr_senders + client = EventHubClient.from_connection_string(connection_str, transport_type=TransportType.AmqpOverWebsocket, network_tracing=False) + receiver = client.create_receiver(partition_id="0", prefetch=500, event_position=EventPosition('@latest')) + + event_list = [] + for i in range(20): + event_list.append(EventData("Event Number {}".format(i))) + + with receiver: + received = receiver.receive(timeout=5) + assert len(received) == 0 + + with senders[0] as sender: + sender.send(event_list) + + time.sleep(1) + + received = receiver.receive(max_batch_size=50, timeout=5) + assert len(received) == 20 diff --git a/sdk/eventhub/azure-eventhubs/tests/test_reconnect.py b/sdk/eventhub/azure-eventhubs/tests/test_reconnect.py index d44fb77106bb..b24cca267c82 100644 --- a/sdk/eventhub/azure-eventhubs/tests/test_reconnect.py +++ b/sdk/eventhub/azure-eventhubs/tests/test_reconnect.py @@ -8,10 +8,9 @@ import time import pytest -from azure import eventhub from azure.eventhub import ( EventData, - Offset, + EventPosition, EventHubError, EventHubClient) @@ -19,64 +18,40 @@ @pytest.mark.liveTest def test_send_with_long_interval_sync(connstr_receivers): connection_str, receivers = connstr_receivers - client = EventHubClient.from_connection_string(connection_str, debug=True) - sender = client.add_sender() - try: - client.run() + client = EventHubClient.from_connection_string(connection_str, network_tracing=True) + sender = client.create_sender() + with sender: sender.send(EventData(b"A single event")) - for _ in range(2): + for _ in range(1): time.sleep(300) sender.send(EventData(b"A single event")) - finally: - client.stop() received = [] for r in receivers: received.extend(r.receive(timeout=1)) - assert len(received) == 3 + assert len(received) == 2 assert list(received[0].body)[0] == b"A single event" @pytest.mark.liveTest def test_send_with_forced_conn_close_sync(connstr_receivers): connection_str, receivers = connstr_receivers - client = EventHubClient.from_connection_string(connection_str, debug=True) - sender = client.add_sender() - try: - client.run() + client = EventHubClient.from_connection_string(connection_str, network_tracing=True) + sender = client.create_sender() + with sender: sender.send(EventData(b"A single event")) - sender._handler._message_sender.destroy() + sender._handler._connection._conn.destroy() time.sleep(300) sender.send(EventData(b"A single event")) sender.send(EventData(b"A single event")) - sender._handler._message_sender.destroy() + sender._handler._connection._conn.destroy() time.sleep(300) sender.send(EventData(b"A single event")) sender.send(EventData(b"A single event")) - finally: - client.stop() received = [] for r in receivers: received.extend(r.receive(timeout=1)) assert len(received) == 5 assert list(received[0].body)[0] == b"A single event" - - -# def test_send_with_forced_link_detach(connstr_receivers): -# connection_str, receivers = connstr_receivers -# client = EventHubClient.from_connection_string(connection_str, debug=True) -# sender = client.add_sender() -# size = 20 * 1024 -# try: -# client.run() -# for i in range(1000): -# sender.transfer(EventData([b"A"*size, b"B"*size, b"C"*size, b"D"*size, b"A"*size, b"B"*size, b"C"*size, b"D"*size, b"A"*size, b"B"*size, b"C"*size, b"D"*size])) -# sender.wait() -# finally: -# client.stop() - -# received = [] -# for r in receivers: -# received.extend(r.receive(timeout=10)) diff --git a/sdk/eventhub/azure-eventhubs/tests/test_send.py b/sdk/eventhub/azure-eventhubs/tests/test_send.py index cdf1f0ebc6d0..3af0cbed2ef2 100644 --- a/sdk/eventhub/azure-eventhubs/tests/test_send.py +++ b/sdk/eventhub/azure-eventhubs/tests/test_send.py @@ -10,13 +10,13 @@ import json import sys -from azure.eventhub import EventData, EventHubClient +from azure.eventhub import EventData, EventHubClient, TransportType @pytest.mark.liveTest def test_send_with_partition_key(connstr_receivers): connection_str, receivers = connstr_receivers - client = EventHubClient.from_connection_string(connection_str, debug=False) + client = EventHubClient.from_connection_string(connection_str, network_tracing=False) sender = client.create_sender() with sender: data_val = 0 @@ -24,19 +24,19 @@ def test_send_with_partition_key(connstr_receivers): partition_key = b"test_partition_" + partition for i in range(50): data = EventData(str(data_val)) - data.partition_key = partition_key + #data.partition_key = partition_key data_val += 1 - sender.send(data) + sender.send(data, batching_label=partition_key) found_partition_keys = {} for index, partition in enumerate(receivers): received = partition.receive(timeout=5) for message in received: try: - existing = found_partition_keys[message.partition_key] + existing = found_partition_keys[message._batching_label] assert existing == index except KeyError: - found_partition_keys[message.partition_key] = index + found_partition_keys[message._batching_label] = index @pytest.mark.liveTest @@ -44,7 +44,7 @@ def test_send_and_receive_large_body_size(connstr_receivers): if sys.platform.startswith('darwin'): pytest.skip("Skipping on OSX - open issue regarding message size") connection_str, receivers = connstr_receivers - client = EventHubClient.from_connection_string(connection_str, debug=False) + client = EventHubClient.from_connection_string(connection_str, network_tracing=False) sender = client.create_sender() with sender: payload = 250 * 1024 @@ -61,7 +61,7 @@ def test_send_and_receive_large_body_size(connstr_receivers): @pytest.mark.liveTest def test_send_and_receive_zero_length_body(connstr_receivers): connection_str, receivers = connstr_receivers - client = EventHubClient.from_connection_string(connection_str, debug=False) + client = EventHubClient.from_connection_string(connection_str, network_tracing=False) sender = client.create_sender() with sender: sender.send(EventData("")) @@ -77,7 +77,7 @@ def test_send_and_receive_zero_length_body(connstr_receivers): @pytest.mark.liveTest def test_send_single_event(connstr_receivers): connection_str, receivers = connstr_receivers - client = EventHubClient.from_connection_string(connection_str, debug=False) + client = EventHubClient.from_connection_string(connection_str, network_tracing=False) sender = client.create_sender() with sender: sender.send(EventData(b"A single event")) @@ -93,14 +93,15 @@ def test_send_single_event(connstr_receivers): @pytest.mark.liveTest def test_send_batch_sync(connstr_receivers): connection_str, receivers = connstr_receivers + def batched(): for i in range(10): yield EventData("Event number {}".format(i)) - client = EventHubClient.from_connection_string(connection_str, debug=False) + client = EventHubClient.from_connection_string(connection_str, network_tracing=False) sender = client.create_sender() with sender: - sender.send_batch(batched()) + sender.send(batched()) time.sleep(1) received = [] @@ -115,8 +116,8 @@ def batched(): @pytest.mark.liveTest def test_send_partition(connstr_receivers): connection_str, receivers = connstr_receivers - client = EventHubClient.from_connection_string(connection_str, debug=False) - sender = client.create_sender(partition="1") + client = EventHubClient.from_connection_string(connection_str, network_tracing=False) + sender = client.create_sender(partition_id="1") with sender: sender.send(EventData(b"Data")) @@ -129,8 +130,8 @@ def test_send_partition(connstr_receivers): @pytest.mark.liveTest def test_send_non_ascii(connstr_receivers): connection_str, receivers = connstr_receivers - client = EventHubClient.from_connection_string(connection_str, debug=False) - sender = client.create_sender(partition="0") + client = EventHubClient.from_connection_string(connection_str, network_tracing=False) + sender = client.create_sender(partition_id="0") with sender: sender.send(EventData(u"é,è,à,ù,â,ê,î,ô,û")) sender.send(EventData(json.dumps({"foo": u"漢字"}))) @@ -144,14 +145,15 @@ def test_send_non_ascii(connstr_receivers): @pytest.mark.liveTest def test_send_partition_batch(connstr_receivers): connection_str, receivers = connstr_receivers + def batched(): for i in range(10): yield EventData("Event number {}".format(i)) - client = EventHubClient.from_connection_string(connection_str, debug=False) - sender = client.create_sender(partition="1") + client = EventHubClient.from_connection_string(connection_str, network_tracing=False) + sender = client.create_sender(partition_id="1") with sender: - sender.send_batch(batched()) + sender.send(batched()) time.sleep(1) partition_0 = receivers[0].receive(timeout=2) @@ -163,7 +165,7 @@ def batched(): @pytest.mark.liveTest def test_send_array_sync(connstr_receivers): connection_str, receivers = connstr_receivers - client = EventHubClient.from_connection_string(connection_str, debug=True) + client = EventHubClient.from_connection_string(connection_str, network_tracing=True) sender = client.create_sender() with sender: sender.send(EventData([b"A", b"B", b"C"])) @@ -179,9 +181,9 @@ def test_send_array_sync(connstr_receivers): @pytest.mark.liveTest def test_send_multiple_clients(connstr_receivers): connection_str, receivers = connstr_receivers - client = EventHubClient.from_connection_string(connection_str, debug=False) - sender_0 = client.create_sender(partition="0") - sender_1 = client.create_sender(partition="1") + client = EventHubClient.from_connection_string(connection_str, network_tracing=False) + sender_0 = client.create_sender(partition_id="0") + sender_1 = client.create_sender(partition_id="1") with sender_0: sender_0.send(EventData(b"Message 0")) with sender_1: @@ -195,7 +197,6 @@ def test_send_multiple_clients(connstr_receivers): @pytest.mark.liveTest def test_send_batch_with_app_prop_sync(connstr_receivers): - #pytest.skip("Waiting on uAMQP release") connection_str, receivers = connstr_receivers app_prop_key = "raw_prop" app_prop_value = "raw_value" @@ -211,11 +212,13 @@ def batched(): ed.application_properties = app_prop yield ed - client = EventHubClient.from_connection_string(connection_str, debug=False) + client = EventHubClient.from_connection_string(connection_str, network_tracing=False) sender = client.create_sender() with sender: - sender.send_batch(batched()) + sender.send(batched()) + time.sleep(1) + received = [] for r in receivers: received.extend(r.receive(timeout=3)) @@ -225,3 +228,24 @@ def batched(): assert list(message.body)[0] == "Event number {}".format(index).encode('utf-8') assert (app_prop_key.encode('utf-8') in message.application_properties) \ and (dict(message.application_properties)[app_prop_key.encode('utf-8')] == app_prop_value.encode('utf-8')) + + +@pytest.mark.liveTest +def test_send_over_websocket_sync(connstr_receivers): + connection_str, receivers = connstr_receivers + client = EventHubClient.from_connection_string(connection_str, transport_type=TransportType.AmqpOverWebsocket, network_tracing=False) + sender = client.create_sender() + + event_list = [] + for i in range(20): + event_list.append(EventData("Event Number {}".format(i))) + + with sender: + sender.send(event_list) + + time.sleep(1) + received = [] + for r in receivers: + received.extend(r.receive(timeout=3)) + + assert len(received) == 20