From a8fd00bb50687494841b3caee0519dd39ca7046c Mon Sep 17 00:00:00 2001 From: Rakshith Bhyravabhotla Date: Mon, 26 Jul 2021 16:51:41 -0700 Subject: [PATCH] Remove responses and return a list (#19872) * Remove responses and return a list * lint * Sample to fetch response in key-value form * fix test * Update sdk/monitor/azure-monitor-query/tests/test_logs_client.py * Update sdk/monitor/azure-monitor-query/tests/test_logs_client.py * flay * skip https://github.com/Azure/azure-sdk-for-python/issues/19917 --- sdk/monitor/azure-monitor-query/CHANGELOG.md | 2 + sdk/monitor/azure-monitor-query/README.md | 4 +- .../azure/monitor/query/__init__.py | 2 - .../azure/monitor/query/_log_query_client.py | 16 +++--- .../azure/monitor/query/_models.py | 25 +-------- .../query/aio/_log_query_client_async.py | 17 ++++--- .../samples/sample_batch_query.py | 6 +-- .../sample_logs_query_key_value_form.py | 51 +++++++++++++++++++ .../tests/async/test_logs_client_async.py | 8 +-- .../tests/test_logs_client.py | 15 +++--- 10 files changed, 90 insertions(+), 56 deletions(-) create mode 100644 sdk/monitor/azure-monitor-query/samples/sample_logs_query_key_value_form.py diff --git a/sdk/monitor/azure-monitor-query/CHANGELOG.md b/sdk/monitor/azure-monitor-query/CHANGELOG.md index 0e67e299e216..96b6f1deb205 100644 --- a/sdk/monitor/azure-monitor-query/CHANGELOG.md +++ b/sdk/monitor/azure-monitor-query/CHANGELOG.md @@ -9,6 +9,8 @@ ### Breaking Changes - `aggregation` param in the query API is renamed to `aggregations` +- `batch_query` API now returns a list of responses. +- `LogsBatchResults` model is now removed. ### Bugs Fixed diff --git a/sdk/monitor/azure-monitor-query/README.md b/sdk/monitor/azure-monitor-query/README.md index 42e61e8eee5b..e7acaf4e1305 100644 --- a/sdk/monitor/azure-monitor-query/README.md +++ b/sdk/monitor/azure-monitor-query/README.md @@ -189,8 +189,8 @@ requests = [ ] response = client.batch_query(requests) -for response in response.responses: - body = response.body +for rsp in response: + body = rsp.body if not body.tables: print("Something is wrong") else: diff --git a/sdk/monitor/azure-monitor-query/azure/monitor/query/__init__.py b/sdk/monitor/azure-monitor-query/azure/monitor/query/__init__.py index d680a6ec0641..09c1dd8bb86d 100644 --- a/sdk/monitor/azure-monitor-query/azure/monitor/query/__init__.py +++ b/sdk/monitor/azure-monitor-query/azure/monitor/query/__init__.py @@ -15,7 +15,6 @@ MetricsResult, LogsBatchResultError, LogsQueryRequest, - LogsBatchResults, MetricNamespace, MetricDefinition, MetricsMetadataValue, @@ -30,7 +29,6 @@ __all__ = [ "AggregationType", "LogsQueryClient", - "LogsBatchResults", "LogsBatchResultError", "LogsQueryResults", "LogsQueryResultColumn", diff --git a/sdk/monitor/azure-monitor-query/azure/monitor/query/_log_query_client.py b/sdk/monitor/azure-monitor-query/azure/monitor/query/_log_query_client.py index 63f19cda2837..861dcff276cf 100644 --- a/sdk/monitor/azure-monitor-query/azure/monitor/query/_log_query_client.py +++ b/sdk/monitor/azure-monitor-query/azure/monitor/query/_log_query_client.py @@ -11,8 +11,8 @@ from ._generated._monitor_query_client import MonitorQueryClient from ._generated.models import BatchRequest, QueryBody as LogsQueryBody -from ._helpers import get_authentication_policy, process_error, construct_iso8601 -from ._models import LogsQueryResults, LogsQueryRequest, LogsBatchResults +from ._helpers import get_authentication_policy, process_error, construct_iso8601, order_results +from ._models import LogsQueryResults, LogsQueryRequest, LogsQueryResult if TYPE_CHECKING: from azure.core.credentials import TokenCredential @@ -131,7 +131,7 @@ def query(self, workspace_id, query, duration=None, **kwargs): process_error(e) def batch_query(self, queries, **kwargs): - # type: (Union[Sequence[Dict], Sequence[LogsQueryRequest]], Any) -> LogsBatchResults + # type: (Union[Sequence[Dict], Sequence[LogsQueryRequest]], Any) -> Sequence[LogsQueryResult] """Execute a list of analytics queries. Each request can be either a LogQueryRequest object or an equivalent serialized model. @@ -140,7 +140,7 @@ def batch_query(self, queries, **kwargs): :param queries: The list of queries that should be processed :type queries: list[dict] or list[~azure.monitor.query.LogsQueryRequest] :return: BatchResponse, or the result of cls(response) - :rtype: ~azure.monitor.query.LogsBatchResults + :rtype: ~list[~azure.monitor.query.LogsQueryResult] :raises: ~azure.core.exceptions.HttpResponseError .. admonition:: Example: @@ -162,9 +162,11 @@ def batch_query(self, queries, **kwargs): request_order = [req['id'] for req in queries] batch = BatchRequest(requests=queries) generated = self._query_op.batch(batch, **kwargs) - return LogsBatchResults._from_generated( # pylint: disable=protected-access - generated, request_order - ) + return order_results( + request_order, + [ + LogsQueryResult._from_generated(rsp) for rsp in generated.responses # pylint: disable=protected-access + ]) def close(self): # type: () -> None diff --git a/sdk/monitor/azure-monitor-query/azure/monitor/query/_models.py b/sdk/monitor/azure-monitor-query/azure/monitor/query/_models.py index f05b05332df4..dc22c7e5908b 100644 --- a/sdk/monitor/azure-monitor-query/azure/monitor/query/_models.py +++ b/sdk/monitor/azure-monitor-query/azure/monitor/query/_models.py @@ -9,7 +9,7 @@ import uuid from typing import Any, Optional, List -from ._helpers import order_results, construct_iso8601 +from ._helpers import construct_iso8601 from ._generated.models import ( Column as InternalColumn, BatchQueryRequest as InternalLogQueryRequest, @@ -234,29 +234,6 @@ def _from_generated(cls, generated): body=LogsQueryResults._from_generated(generated.body) # pylint: disable=protected-access ) -class LogsBatchResults(object): - """Response to a batch. - - :keyword responses: An array of responses corresponding to each individual request in a batch. - :paramtype responses: list[azure.monitor.query.LogsQueryResult] - :keyword error: Error response for a batch request. - :paramtype error: ~azure.monitor.query.LogsBatchResultError - """ - def __init__(self, **kwargs): - # type: (Any) -> None - self.responses = kwargs.get("responses", None) - self.error = kwargs.get("error", None) - - @classmethod - def _from_generated(cls, generated, request_order): - if not generated: - return cls() - return cls( - responses=order_results(request_order, [ - LogsQueryResult._from_generated(rsp) for rsp in generated.responses # pylint: disable=protected-access - ]) - ) - class LogsBatchResultError(object): """Error response for a batch request. diff --git a/sdk/monitor/azure-monitor-query/azure/monitor/query/aio/_log_query_client_async.py b/sdk/monitor/azure-monitor-query/azure/monitor/query/aio/_log_query_client_async.py index f840472815dd..bafa51be1efb 100644 --- a/sdk/monitor/azure-monitor-query/azure/monitor/query/aio/_log_query_client_async.py +++ b/sdk/monitor/azure-monitor-query/azure/monitor/query/aio/_log_query_client_async.py @@ -11,8 +11,8 @@ from .._generated.aio._monitor_query_client import MonitorQueryClient from .._generated.models import BatchRequest, QueryBody as LogsQueryBody -from .._helpers import process_error, construct_iso8601 -from .._models import LogsQueryResults, LogsQueryRequest, LogsBatchResults +from .._helpers import process_error, construct_iso8601, order_results +from .._models import LogsQueryResults, LogsQueryRequest, LogsQueryResult from ._helpers_asyc import get_authentication_policy if TYPE_CHECKING: @@ -118,7 +118,7 @@ async def batch_query( self, queries: Union[Sequence[Dict], Sequence[LogsQueryRequest]], **kwargs: Any - ) -> LogsBatchResults: + ) -> Sequence[LogsQueryResult]: """Execute a list of analytics queries. Each request can be either a LogQueryRequest object or an equivalent serialized model. @@ -127,7 +127,7 @@ async def batch_query( :param queries: The list of queries that should be processed :type queries: list[dict] or list[~azure.monitor.query.LogsQueryRequest] :return: BatchResponse, or the result of cls(response) - :rtype: ~azure.monitor.query.LogsBatchResults + :rtype: ~list[~azure.monitor.query.LogsQueryResult] :raises: ~azure.core.exceptions.HttpResponseError """ try: @@ -139,9 +139,12 @@ async def batch_query( except AttributeError: request_order = [req['id'] for req in queries] batch = BatchRequest(requests=queries) - return LogsBatchResults._from_generated( # pylint: disable=protected-access - await self._query_op.batch(batch, **kwargs), request_order - ) + generated = await self._query_op.batch(batch, **kwargs) + return order_results( + request_order, + [ + LogsQueryResult._from_generated(rsp) for rsp in generated.responses # pylint: disable=protected-access + ]) async def __aenter__(self) -> "LogsQueryClient": await self._client.__aenter__() diff --git a/sdk/monitor/azure-monitor-query/samples/sample_batch_query.py b/sdk/monitor/azure-monitor-query/samples/sample_batch_query.py index 71a7ec331489..3afc3a8d351c 100644 --- a/sdk/monitor/azure-monitor-query/samples/sample_batch_query.py +++ b/sdk/monitor/azure-monitor-query/samples/sample_batch_query.py @@ -27,14 +27,14 @@ workspace_id= os.environ['LOG_WORKSPACE_ID'] ), LogsQueryRequest( - query= "AppRequests", + query= "AppRequests | take 5", workspace_id= os.environ['LOG_WORKSPACE_ID'], include_statistics=True ), ] -response = client.batch_query(requests) +responses = client.batch_query(requests) -for response in response.responses: +for response in responses: body = response.body print(response.id) if not body.tables: diff --git a/sdk/monitor/azure-monitor-query/samples/sample_logs_query_key_value_form.py b/sdk/monitor/azure-monitor-query/samples/sample_logs_query_key_value_form.py new file mode 100644 index 000000000000..bff5aa4b12ce --- /dev/null +++ b/sdk/monitor/azure-monitor-query/samples/sample_logs_query_key_value_form.py @@ -0,0 +1,51 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. + +import os +import pandas as pd +from datetime import datetime, timedelta +from msrest.serialization import UTC +from azure.monitor.query import LogsQueryClient +from azure.identity import DefaultAzureCredential + +credential = DefaultAzureCredential() +client = LogsQueryClient(credential) + +# Response time trend +# request duration over the last 12 hours. +query = """AppRequests | +summarize avgRequestDuration=avg(DurationMs) by bin(TimeGenerated, 10m), _ResourceId""" + +end_time = datetime.now(UTC()) + +# returns LogsQueryResults +response = client.query(os.environ['LOG_WORKSPACE_ID'], query, duration=timedelta(days=1), end_time=end_time) + +if not response.tables: + print("No results for the query") + +for table in response.tables: + pd.json_normalize + df = pd.DataFrame(table.rows, columns=[col.name for col in table.columns]) + key_value = df.to_dict(orient='records') + print(key_value) + +""" +[ + { + 'TimeGenerated': '2021-07-21T04:40:00Z', + '_ResourceId': '/subscriptions/faa080af....', + 'avgRequestDuration': 19.7987 + }, + { + 'TimeGenerated': '2021-07-21T04:50:00Z', + '_ResourceId': '/subscriptions/faa08....', + 'avgRequestDuration': 33.9654 + }, + { + 'TimeGenerated': '2021-07-21T05:00:00Z', + '_ResourceId': '/subscriptions/faa080....', + 'avgRequestDuration': 44.13115 + } +] +""" diff --git a/sdk/monitor/azure-monitor-query/tests/async/test_logs_client_async.py b/sdk/monitor/azure-monitor-query/tests/async/test_logs_client_async.py index 6044cd756e37..ab089a004e86 100644 --- a/sdk/monitor/azure-monitor-query/tests/async/test_logs_client_async.py +++ b/sdk/monitor/azure-monitor-query/tests/async/test_logs_client_async.py @@ -27,10 +27,10 @@ async def test_logs_auth(): assert response is not None assert response.tables is not None +@pytest.mark.skip("https://github.com/Azure/azure-sdk-for-python/issues/19917") @pytest.mark.live_test_only async def test_logs_server_timeout(): client = LogsQueryClient(_credential()) - with pytest.raises(HttpResponseError) as e: response = await client.query( os.environ['LOG_WORKSPACE_ID'], @@ -62,7 +62,7 @@ async def test_logs_batch_query(): ] response = await client.batch_query(requests) - assert len(response.responses) == 3 + assert len(response) == 3 @pytest.mark.skip('https://github.com/Azure/azure-sdk-for-python/issues/19382') @pytest.mark.live_test_only @@ -110,7 +110,7 @@ async def test_logs_batch_query_additional_workspaces(): ] response = await client.batch_query(requests) - assert len(response.responses) == 3 + assert len(response) == 3 - for resp in response.responses: + for resp in response: assert len(resp.body.tables[0].rows) == 2 diff --git a/sdk/monitor/azure-monitor-query/tests/test_logs_client.py b/sdk/monitor/azure-monitor-query/tests/test_logs_client.py index 340a90f7af8f..8dc945f4b380 100644 --- a/sdk/monitor/azure-monitor-query/tests/test_logs_client.py +++ b/sdk/monitor/azure-monitor-query/tests/test_logs_client.py @@ -48,6 +48,7 @@ def test_logs_single_query_with_partial_success(): assert response is not None +@pytest.mark.skip("https://github.com/Azure/azure-sdk-for-python/issues/19917") @pytest.mark.live_test_only def test_logs_server_timeout(): client = LogsQueryClient(_credential()) @@ -55,10 +56,10 @@ def test_logs_server_timeout(): with pytest.raises(HttpResponseError) as e: response = client.query( os.environ['LOG_WORKSPACE_ID'], - "range x from 1 to 10000000000 step 1 | count", + "range x from 1 to 1000000000000000 step 1 | count", server_timeout=1, ) - assert e.message.contains('Gateway timeout') + assert 'Gateway timeout' in e.value.message @pytest.mark.live_test_only def test_logs_batch_query(): @@ -83,7 +84,7 @@ def test_logs_batch_query(): ] response = client.batch_query(requests) - assert len(response.responses) == 3 + assert len(response) == 3 @pytest.mark.live_test_only def test_logs_single_query_with_statistics(): @@ -121,9 +122,9 @@ def test_logs_batch_query_with_statistics_in_some(): ] response = client.batch_query(requests) - assert len(response.responses) == 3 - assert response.responses[0].body.statistics is None - assert response.responses[2].body.statistics is not None + assert len(response) == 3 + assert response[0].body.statistics is None + assert response[2].body.statistics is not None @pytest.mark.skip('https://github.com/Azure/azure-sdk-for-python/issues/19382') @pytest.mark.live_test_only @@ -169,5 +170,5 @@ def test_logs_batch_query_additional_workspaces(): ] response = client.batch_query(requests) - for resp in response.responses: + for resp in response: assert len(resp.body.tables[0].rows) == 2