From 8b822d76d39487b1a09d3c4e60e237c0eeb8f7de Mon Sep 17 00:00:00 2001 From: Xiang Yan Date: Thu, 18 Feb 2021 10:14:51 -0800 Subject: [PATCH 1/4] add perf test for app config --- .../dev_requirements.txt | 1 + .../tests/perfstress_tests/README.md | 44 +++++++++++++++++++ .../tests/perfstress_tests/__init__.py | 0 .../tests/perfstress_tests/get_set.py | 42 ++++++++++++++++++ 4 files changed, 87 insertions(+) create mode 100644 sdk/appconfiguration/azure-appconfiguration/tests/perfstress_tests/README.md create mode 100644 sdk/appconfiguration/azure-appconfiguration/tests/perfstress_tests/__init__.py create mode 100644 sdk/appconfiguration/azure-appconfiguration/tests/perfstress_tests/get_set.py diff --git a/sdk/appconfiguration/azure-appconfiguration/dev_requirements.txt b/sdk/appconfiguration/azure-appconfiguration/dev_requirements.txt index 58d8fe68650b..b004f340451e 100644 --- a/sdk/appconfiguration/azure-appconfiguration/dev_requirements.txt +++ b/sdk/appconfiguration/azure-appconfiguration/dev_requirements.txt @@ -1,3 +1,4 @@ +-e ../../../tools/azure-devtools ../../core/azure-core -e ../../identity/azure-identity aiohttp>=3.0; python_version >= '3.5' diff --git a/sdk/appconfiguration/azure-appconfiguration/tests/perfstress_tests/README.md b/sdk/appconfiguration/azure-appconfiguration/tests/perfstress_tests/README.md new file mode 100644 index 000000000000..4d00e3cd1d93 --- /dev/null +++ b/sdk/appconfiguration/azure-appconfiguration/tests/perfstress_tests/README.md @@ -0,0 +1,44 @@ +# Blob Performance Tests + +In order to run the performance tests, the `azure-devtools` package must be installed. This is done as part of the `dev_requirements`. +Start be creating a new virtual environment for your perf tests. This will need to be a Python 3 environment, preferably >=3.7. +Note that tests for T1 and T2 SDKs cannot be run from the same environment, and will need to be setup separately. + +### Setup for test resources + +These tests will run against a pre-configured application configuration service. The following environment variable will need to be set for the tests to access the live resources: +``` +AZURE_APP_CONFIG_CONNECTION_STRING= +``` + +### Setup for perf test runs + +```cmd +(env) ~/azure-appconfiguration> pip install -r dev_requirements.txt +(env) ~/azure-appconfiguration> pip install -e . +``` + +## Test commands + +When `azure-devtools` is installed, you will have access to the `perfstress` command line tool, which will scan the current module for runable perf tests. Only a specific test can be run at a time (i.e. there is no "run all" feature). + +```cmd +(env) ~/azure-appconfiguration> cd tests +(env) ~/azure-appconfiguration/tests> perfstress +``` +Using the `perfstress` command alone will list the available perf tests found. + +### Common perf command line options +These options are available for all perf tests: +- `--duration=10` Number of seconds to run as many operations (the "run" function) as possible. Default is 10. +- `--iterations=1` Number of test iterations to run. Default is 1. +- `--parallel=1` Number of tests to run in parallel. Default is 1. +- `--no-client-share` Whether each parallel test instance should share a single client, or use their own. Default is False (sharing). +- `--warm-up=5` Number of seconds to spend warming up the connection before measuring begins. Default is 5. +- `--sync` Whether to run the tests in sync or async. Default is False (async). This flag must be used for Storage legacy tests, which do not support async. +- `--no-cleanup` Whether to keep newly created resources after test run. Default is False (resources will be deleted). + +## Example command +```cmd +(env) ~/azure-appconfiguration/tests> perfstress GetSetTest +``` diff --git a/sdk/appconfiguration/azure-appconfiguration/tests/perfstress_tests/__init__.py b/sdk/appconfiguration/azure-appconfiguration/tests/perfstress_tests/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/sdk/appconfiguration/azure-appconfiguration/tests/perfstress_tests/get_set.py b/sdk/appconfiguration/azure-appconfiguration/tests/perfstress_tests/get_set.py new file mode 100644 index 000000000000..f9c8eca40507 --- /dev/null +++ b/sdk/appconfiguration/azure-appconfiguration/tests/perfstress_tests/get_set.py @@ -0,0 +1,42 @@ +# -------------------------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for license information. +# -------------------------------------------------------------------------------------------- + +import os + +from azure_devtools.perfstress_tests import PerfStressTest + +from azure.appconfiguration import ConfigurationSetting, AzureAppConfigurationClient as SyncAppConfigClient +from azure.appconfiguration.aio import AzureAppConfigurationClient as AsyncAppConfigClient + + +class GetSetTest(PerfStressTest): + service_client = None + async_service_client = None + + def __init__(self, arguments): + super().__init__(arguments) + connection_string = self.get_from_env("AZURE_APP_CONFIG_CONNECTION_STRING") + self.service_client = SyncAppConfigClient.from_connection_string(connection_string=connection_string) + self.async_service_client = AsyncAppConfigClient.from_connection_string(connection_string=connection_string) + + async def close(self): + # await self.async_service_client.close() + await super().close() + + def run_sync(self): + kv = ConfigurationSetting( + key="KEY", + value="VALUE", + ) + self.service_client.set_configuration_setting(kv) + self.service_client.get_configuration_setting(key=kv.key) + + async def run_async(self): + kv = ConfigurationSetting( + key="KEY", + value="VALUE", + ) + await self.async_service_client.set_configuration_setting(kv) + await self.async_service_client.get_configuration_setting(key=kv.key) From 1faa17d08c97002d5d0fc24ff432070e1589478b Mon Sep 17 00:00:00 2001 From: Xiang Yan Date: Thu, 18 Feb 2021 10:16:30 -0800 Subject: [PATCH 2/4] update readme --- .../azure-appconfiguration/tests/perfstress_tests/README.md | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/sdk/appconfiguration/azure-appconfiguration/tests/perfstress_tests/README.md b/sdk/appconfiguration/azure-appconfiguration/tests/perfstress_tests/README.md index 4d00e3cd1d93..252a06cb65c8 100644 --- a/sdk/appconfiguration/azure-appconfiguration/tests/perfstress_tests/README.md +++ b/sdk/appconfiguration/azure-appconfiguration/tests/perfstress_tests/README.md @@ -1,8 +1,7 @@ -# Blob Performance Tests +# App config Performance Tests In order to run the performance tests, the `azure-devtools` package must be installed. This is done as part of the `dev_requirements`. Start be creating a new virtual environment for your perf tests. This will need to be a Python 3 environment, preferably >=3.7. -Note that tests for T1 and T2 SDKs cannot be run from the same environment, and will need to be setup separately. ### Setup for test resources From dcf7057a7b08db1800acdfbd8593ac7c1e93dffb Mon Sep 17 00:00:00 2001 From: Xiang Yan Date: Tue, 23 Feb 2021 12:41:52 -0800 Subject: [PATCH 3/4] updates --- .../tests/perfstress_tests/README.md | 3 +- .../tests/perfstress_tests/get.py | 38 +++++++++++++++++++ .../perfstress_tests/{get_set.py => set.py} | 14 +++---- 3 files changed, 46 insertions(+), 9 deletions(-) create mode 100644 sdk/appconfiguration/azure-appconfiguration/tests/perfstress_tests/get.py rename sdk/appconfiguration/azure-appconfiguration/tests/perfstress_tests/{get_set.py => set.py} (83%) diff --git a/sdk/appconfiguration/azure-appconfiguration/tests/perfstress_tests/README.md b/sdk/appconfiguration/azure-appconfiguration/tests/perfstress_tests/README.md index 252a06cb65c8..a29db0d81af8 100644 --- a/sdk/appconfiguration/azure-appconfiguration/tests/perfstress_tests/README.md +++ b/sdk/appconfiguration/azure-appconfiguration/tests/perfstress_tests/README.md @@ -32,12 +32,11 @@ These options are available for all perf tests: - `--duration=10` Number of seconds to run as many operations (the "run" function) as possible. Default is 10. - `--iterations=1` Number of test iterations to run. Default is 1. - `--parallel=1` Number of tests to run in parallel. Default is 1. -- `--no-client-share` Whether each parallel test instance should share a single client, or use their own. Default is False (sharing). - `--warm-up=5` Number of seconds to spend warming up the connection before measuring begins. Default is 5. - `--sync` Whether to run the tests in sync or async. Default is False (async). This flag must be used for Storage legacy tests, which do not support async. - `--no-cleanup` Whether to keep newly created resources after test run. Default is False (resources will be deleted). ## Example command ```cmd -(env) ~/azure-appconfiguration/tests> perfstress GetSetTest +(env) ~/azure-appconfiguration/tests> perfstress GetTest ``` diff --git a/sdk/appconfiguration/azure-appconfiguration/tests/perfstress_tests/get.py b/sdk/appconfiguration/azure-appconfiguration/tests/perfstress_tests/get.py new file mode 100644 index 000000000000..fdecfb1c8cdc --- /dev/null +++ b/sdk/appconfiguration/azure-appconfiguration/tests/perfstress_tests/get.py @@ -0,0 +1,38 @@ +# -------------------------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for license information. +# -------------------------------------------------------------------------------------------- + +import os + +from azure_devtools.perfstress_tests import PerfStressTest + +from azure.appconfiguration import ConfigurationSetting, AzureAppConfigurationClient as SyncAppConfigClient +from azure.appconfiguration.aio import AzureAppConfigurationClient as AsyncAppConfigClient + + +class GetTest(PerfStressTest): + def __init__(self, arguments): + super().__init__(arguments) + connection_string = self.get_from_env("AZURE_APP_CONFIG_CONNECTION_STRING") + self.key = "KEY" + self.service_client = SyncAppConfigClient.from_connection_string(connection_string=connection_string) + self.async_service_client = AsyncAppConfigClient.from_connection_string(connection_string=connection_string) + + async def global_setup(self): + await super().global_setup() + kv = ConfigurationSetting( + key=self.key, + value="VALUE", + ) + await self.async_service_client.set_configuration_setting(kv) + + async def close(self): + # await self.async_service_client.close() + await super().close() + + def run_sync(self): + self.service_client.get_configuration_setting(key=self.key) + + async def run_async(self): + await self.async_service_client.get_configuration_setting(key=self.key) diff --git a/sdk/appconfiguration/azure-appconfiguration/tests/perfstress_tests/get_set.py b/sdk/appconfiguration/azure-appconfiguration/tests/perfstress_tests/set.py similarity index 83% rename from sdk/appconfiguration/azure-appconfiguration/tests/perfstress_tests/get_set.py rename to sdk/appconfiguration/azure-appconfiguration/tests/perfstress_tests/set.py index f9c8eca40507..2c5148ef1a4d 100644 --- a/sdk/appconfiguration/azure-appconfiguration/tests/perfstress_tests/get_set.py +++ b/sdk/appconfiguration/azure-appconfiguration/tests/perfstress_tests/set.py @@ -4,6 +4,7 @@ # -------------------------------------------------------------------------------------------- import os +import uuid from azure_devtools.perfstress_tests import PerfStressTest @@ -11,13 +12,14 @@ from azure.appconfiguration.aio import AzureAppConfigurationClient as AsyncAppConfigClient -class GetSetTest(PerfStressTest): +class SetTest(PerfStressTest): service_client = None async_service_client = None def __init__(self, arguments): super().__init__(arguments) connection_string = self.get_from_env("AZURE_APP_CONFIG_CONNECTION_STRING") + self.key = "KEY" self.service_client = SyncAppConfigClient.from_connection_string(connection_string=connection_string) self.async_service_client = AsyncAppConfigClient.from_connection_string(connection_string=connection_string) @@ -27,16 +29,14 @@ async def close(self): def run_sync(self): kv = ConfigurationSetting( - key="KEY", - value="VALUE", + key=self.key, + value="VALUE" + str(uuid.uuid4()), ) self.service_client.set_configuration_setting(kv) - self.service_client.get_configuration_setting(key=kv.key) async def run_async(self): kv = ConfigurationSetting( - key="KEY", - value="VALUE", + key=self.key, + value="VALUE" + str(uuid.uuid4()), ) await self.async_service_client.set_configuration_setting(kv) - await self.async_service_client.get_configuration_setting(key=kv.key) From 09a2fea490f8a13dbc36af7740af51aed7f24a47 Mon Sep 17 00:00:00 2001 From: Xiang Yan Date: Mon, 1 Mar 2021 08:48:59 -0800 Subject: [PATCH 4/4] move uuid.uuid4() into ctor --- .../azure-appconfiguration/tests/perfstress_tests/set.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/sdk/appconfiguration/azure-appconfiguration/tests/perfstress_tests/set.py b/sdk/appconfiguration/azure-appconfiguration/tests/perfstress_tests/set.py index 2c5148ef1a4d..5e8f5e1a8f29 100644 --- a/sdk/appconfiguration/azure-appconfiguration/tests/perfstress_tests/set.py +++ b/sdk/appconfiguration/azure-appconfiguration/tests/perfstress_tests/set.py @@ -20,6 +20,7 @@ def __init__(self, arguments): super().__init__(arguments) connection_string = self.get_from_env("AZURE_APP_CONFIG_CONNECTION_STRING") self.key = "KEY" + self.value = str(uuid.uuid4()) self.service_client = SyncAppConfigClient.from_connection_string(connection_string=connection_string) self.async_service_client = AsyncAppConfigClient.from_connection_string(connection_string=connection_string) @@ -30,13 +31,13 @@ async def close(self): def run_sync(self): kv = ConfigurationSetting( key=self.key, - value="VALUE" + str(uuid.uuid4()), + value="VALUE" + self.value, ) self.service_client.set_configuration_setting(kv) async def run_async(self): kv = ConfigurationSetting( key=self.key, - value="VALUE" + str(uuid.uuid4()), + value="VALUE" + self.value, ) await self.async_service_client.set_configuration_setting(kv)