pulled dev

.github/CODEOWNERS

@@ -1,60 +1,60 @@
 # See for instructions on this file https://help.github.com/articles/about-codeowners/
 
-*help.py @msyyc @jiasli @jsntcy
-*help.yaml @msyyc @jiasli @jsntcy
-*help.yml @msyyc @jiasli @jsntcy
+*help.py @kairu-ms @jiasli @jsntcy
+*help.yaml @kairu-ms @jiasli @jsntcy
+*help.yml @kairu-ms @jiasli @jsntcy
 
 /linter_exclusions.yml @jsntcy @kairu-ms
 
-/doc/ @jiasli @qwordy @dbradish-microsoft
-/tools/ @fengzhou-msft @jiasli
-/scripts/ @fengzhou-msft @jiasli
-/scripts/live_test @qwordy
-/src/azure-cli-testsdk/ @jsntcy @jiasli @kairu-ms @qwordy
-/src/azure-cli-core/ @jiasli @Juliehzl @fengzhou-msft @evelyn-ys @jsntcy @houk-ms @kairu-ms
-/src/azure-cli-core/azure/cli/core/extension/ @msyyc @fengzhou-msft
+/doc/ @jiasli @kairu-ms @jsntcy @dbradish-microsoft
+/tools/ @kairu-ms @jiasli
+/scripts/ @kairu-ms @jiasli
+/scripts/live_test @Juliehzl @kairu-ms
+/src/azure-cli-testsdk/ @jsntcy @jiasli @kairu-ms
+/src/azure-cli-core/ @jiasli @Juliehzl @evelyn-ys @jsntcy @kairu-ms
+/src/azure-cli-core/azure/cli/core/extension/ @jsntcy @kairu-ms
 /src/azure-cli-core/azure/cli/core/_profile.py @jiasli @evelyn-ys
 /src/azure-cli-core/azure/cli/core/adal_authentication.py @jiasli @evelyn-ys
 /src/azure-cli-core/azure/cli/core/msal_authentication.py @jiasli @evelyn-ys
 /src/azure-cli-core/azure/cli/core/style.py @jiasli @evelyn-ys @zhoxing-ms
-/src/azure-cli/azure/cli/command_modules/acr/ @fengzhou-msft @yungezz @northtyphoon @rosanch
-/src/azure-cli/azure/cli/command_modules/acs/ @fengzhou-msft @zqingqing1 @gtracer @xizhamsft @andyliuliming
+/src/azure-cli/azure/cli/command_modules/acr/ @Juliehzl @zhoxing-ms @northtyphoon @rosanch
+/src/azure-cli/azure/cli/command_modules/acs/ @Juliehzl @zhoxing-ms @zqingqing1 @gtracer @xizhamsft @andyliuliming
 /src/azure-cli/azure/cli/command_modules/advisor/ @Prasanna-Padmanabhan
 /src/azure-cli/azure/cli/command_modules/apim/ @kevinhillinger @jonlester
-/src/azure-cli/azure/cli/command_modules/appconfig/ @shenmuxiaosen @avanigupta @qwordy @zhoxing-ms
+/src/azure-cli/azure/cli/command_modules/appconfig/ @shenmuxiaosen @avanigupta @Juliehzl @zhoxing-ms
 /src/azure-cli/azure/cli/command_modules/appservice/ @Juliehzl @panchagnula
 /src/azure-cli/azure/cli/command_modules/aro/ @mjudeikis @jim-minter
-/src/azure-cli/azure/cli/command_modules/backup/ @dragonfly91 @fengzhou-msft @qwordy @akshayneema
-/src/azure-cli/azure/cli/command_modules/batch/ @bgklein @gingi @dpwatrous @paterasMSFT @qwordy @zhoxing-ms
+/src/azure-cli/azure/cli/command_modules/backup/ @dragonfly91 @Juliehzl @zhoxing-ms @akshayneema
+/src/azure-cli/azure/cli/command_modules/batch/ @bgklein @gingi @dpwatrous @paterasMSFT @Juliehzl @zhoxing-ms
 /src/azure-cli/azure/cli/command_modules/batchai/ @AlexanderYukhanov
-/src/azure-cli/azure/cli/command_modules/cloud/ @jiasli @fengzhou-msft @Juliehzl
+/src/azure-cli/azure/cli/command_modules/cloud/ @jiasli @Juliehzl
 /src/azure-cli/azure/cli/command_modules/consumption/ @sandeepnl
-/src/azure-cli/azure/cli/command_modules/container/ @zhoxing-ms @fengzhou-msft
-/src/azure-cli/azure/cli/command_modules/cosmosdb/ @dmakwana @kristynhamasaki @Juliehzl
+/src/azure-cli/azure/cli/command_modules/container/ @Juliehzl @zhoxing-ms
+/src/azure-cli/azure/cli/command_modules/cosmosdb/ @dmakwana @kristynhamasaki @Juliehzl @evelyn-ys
 /src/azure-cli/azure/cli/command_modules/databoxedge/ @evelyn-ys @Juliehzl
 /src/azure-cli/azure/cli/command_modules/dls/ @akharit @rahuldutta90 @Juliehzl @jsntcy
 /src/azure-cli/azure/cli/command_modules/dms/ @temandr @binuj
 /src/azure-cli/azure/cli/command_modules/eventhubs/ @v-ajnava
 /src/azure-cli/azure/cli/command_modules/eventgrid/ @Juliehzl @evelyn-ys @VidyaKukke
-/src/azure-cli/azure/cli/command_modules/extension/ @fengzhou-msft @msyyc
-/src/azure-cli/azure/cli/command_modules/feedback/ @fengzhou-msft @jiasli
+/src/azure-cli/azure/cli/command_modules/extension/ @jsntcy @kairu-ms
+/src/azure-cli/azure/cli/command_modules/feedback/ @jiasli @kairu-ms
 /src/azure-cli/azure/cli/command_modules/hdinsight/ @aim-for-better @Juliehzl @kairu-ms
-/src/azure-cli/azure/cli/command_modules/iot/ @digimaun
-/src/azure-cli/azure/cli/command_modules/keyvault/ @evelyn-ys @jiasli @houk-ms
-/src/azure-cli/azure/cli/command_modules/monitor/ @msyyc @jsntcy @kairu-ms
-/src/azure-cli/azure/cli/command_modules/natgateway/ @khannarheams @jsntcy @kairu-ms @msyyc
-/src/azure-cli/azure/cli/command_modules/network/ @jsntcy @kairu-ms @msyyc
+/src/azure-cli/azure/cli/command_modules/iot/ @digimaun @zhoxing-ms
+/src/azure-cli/azure/cli/command_modules/keyvault/ @evelyn-ys @jiasli
+/src/azure-cli/azure/cli/command_modules/monitor/ @jsntcy @kairu-ms
+/src/azure-cli/azure/cli/command_modules/natgateway/ @khannarheams @jsntcy @kairu-ms
+/src/azure-cli/azure/cli/command_modules/network/ @jsntcy @kairu-ms
 /src/azure-cli/azure/cli/command_modules/policyinsights/ @cheggert
-/src/azure-cli/azure/cli/command_modules/privatedns/ @jsntcy @kairu-ms @msyyc
-/src/azure-cli/azure/cli/command_modules/profile/ @jiasli @evelyn-ys @fengzhou-msft
+/src/azure-cli/azure/cli/command_modules/privatedns/ @jsntcy @kairu-ms
+/src/azure-cli/azure/cli/command_modules/profile/ @jiasli @evelyn-ys
 /src/azure-cli/azure/cli/command_modules/rdbms/ @arde0708 @Juliehzl @evelyn-ys @DaeunYim
 /src/azure-cli/azure/cli/command_modules/resource/ @Juliehzl @zhoxing-ms
-/src/azure-cli/azure/cli/command_modules/role/ @jiasli @evelyn-ys @jsntcy @fengzhou-msft
+/src/azure-cli/azure/cli/command_modules/role/ @jiasli @evelyn-ys
 /src/azure-cli/azure/cli/command_modules/search/ @huangbolun @kairu-ms
-/src/azure-cli/azure/cli/command_modules/servicebus/ @v-ajnava @fengzhou-msft
-/src/azure-cli/azure/cli/command_modules/servicefabric/ @QingChenmsft @qwordy @Juliehzl
+/src/azure-cli/azure/cli/command_modules/servicebus/ @v-ajnava @Juliehzl @zhoxing-ms
+/src/azure-cli/azure/cli/command_modules/servicefabric/ @QingChenmsft @Juliehzl @zhoxing-ms
 /src/azure-cli/azure/cli/command_modules/sql/ @jaredmoo @Juliehzl @evelyn-ys
 /src/azure-cli/azure/cli/command_modules/storage/ @Juliehzl @jsntcy @zhoxing-ms @evelyn-ys
 /src/azure-cli/azure/cli/command_modules/synapse/ @idear1203 @sunsw1994 @aim-for-better
 /src/azure-cli/azure/cli/command_modules/util/ @jiasli @Juliehzl @zhoxing-ms @evelyn-ys
-/src/azure-cli/azure/cli/command_modules/vm/ @qwordy @houk-ms @yungezz @Juliehzl @zhoxing-ms
+/src/azure-cli/azure/cli/command_modules/vm/ @Juliehzl @zhoxing-ms

pylintrc

@@ -12,7 +12,6 @@ reports=no
 disable=
     arguments-out-of-order,
     bad-option-value,
-    consider-using-generator,
     consider-using-with,
     cyclic-import,
     duplicate-code,
@@ -32,7 +31,6 @@ disable=
     too-many-arguments,
     too-many-function-args,
     too-many-lines,
-    use-a-generator,
     using-constant-test,
     wrong-import-order,
 

scripts/release/rpm/test_rpm_package.py

@@ -14,7 +14,7 @@
 pytest_parallel_cmd = '{} -n auto'.format(pytest_base_cmd)
 
 for mod_name in mod_list:
-    if mod_name in ['botservice', 'network', 'cloud']:
+    if mod_name in ['botservice', 'network', 'cloud', 'appservice']:
         exit_code = subprocess.call(['{} --junit-xml /azure_cli_test_result/{}.xml --pyargs azure.cli.command_modules.{}'.format(pytest_base_cmd, mod_name, mod_name)], shell=True)
     else:
         exit_code = subprocess.call(['{} --junit-xml /azure_cli_test_result/{}.xml --pyargs azure.cli.command_modules.{}'.format(pytest_parallel_cmd, mod_name, mod_name)], shell=True)

src/azure-cli-core/HISTORY.rst

@@ -3,6 +3,10 @@
 Release History
 ===============
 
+2.25.0
+++++++
+* Fix #18360 and #18262: Write to stderr issue (#18414)
+
 2.24.2
 ++++++
 * No changes

src/azure-cli-core/README.rst

@@ -1,2 +1,7 @@
 Microsoft Azure CLI Core Module
 ==================================
+
+Release History
+===============
+
+See `Release History on GitHub <https://github.com/Azure/azure-cli/blob/dev/src/azure-cli-core/HISTORY.rst>`__.

src/azure-cli-core/azure/cli/core/__init__.py

@@ -4,7 +4,7 @@
 # --------------------------------------------------------------------------------------------
 # pylint: disable=line-too-long
 
-__version__ = "2.24.2"
+__version__ = "2.25.0"
 
 import os
 import sys
@@ -887,6 +887,7 @@ def get_op_handler(self, operation, operation_group=None):
                 operation = operation.replace(rt.import_prefix,
                                               get_versioned_sdk_path(self.cli_ctx.cloud.profile, rt,
                                                                      operation_group=operation_group))
+                break
 
         try:
             mod_to_import, attr_path = operation.split('#')

src/azure-cli-core/azure/cli/core/_profile.py

@@ -358,7 +358,7 @@ def find_subscriptions_in_vm_with_msi(self, identity_id=None, allow_no_subscript
         token_entry = msi_creds.token
         token = token_entry['access_token']
         logger.info('MSI: token was retrieved. Now trying to initialize local accounts...')
-        decode = jwt.decode(token, verify=False, algorithms=['RS256'])
+        decode = jwt.decode(token, algorithms=['RS256'], options={"verify_signature": False})
         tenant = decode['tid']
 
         subscription_finder = SubscriptionFinder(self.cli_ctx, self.auth_ctx_factory, None)
@@ -382,7 +382,7 @@ def find_subscriptions_in_cloud_console(self):
 
         _, token, _ = self._get_token_from_cloud_shell(self.cli_ctx.cloud.endpoints.active_directory_resource_id)
         logger.info('MSI: token was retrieved. Now trying to initialize local accounts...')
-        decode = jwt.decode(token, verify=False, algorithms=['RS256'])
+        decode = jwt.decode(token, algorithms=['RS256'], options={"verify_signature": False})
         tenant = decode['tid']
 
         subscription_finder = SubscriptionFinder(self.cli_ctx, self.auth_ctx_factory, None)

src/azure-cli-core/azure/cli/core/commands/__init__.py

@@ -107,7 +107,7 @@ def _expand_file_prefix(arg):
         except IndexError:
             return _maybe_load_file(arg_split[0])
 
-    return list([_expand_file_prefix(arg) for arg in args])
+    return [_expand_file_prefix(arg) for arg in args]
 
 
 def _pre_command_table_create(cli_ctx, args):

src/azure-cli-core/azure/cli/core/extension/operations.py

@@ -81,7 +81,7 @@ def _validate_whl_extension(ext_file):
     zip_ref.extractall(tmp_dir)
     zip_ref.close()
     azext_metadata = WheelExtension.get_azext_metadata(tmp_dir)
-    shutil.rmtree(tmp_dir)
+    rmtree_with_retry(tmp_dir)
     check_version_compatibility(azext_metadata)
 
 
@@ -169,7 +169,7 @@ def _add_whl_ext(cli_ctx, source, ext_sha256=None, pip_extra_index_urls=None, pi
         pip_status_code = _run_pip(pip_args, extension_path)
     if pip_status_code > 0:
         logger.debug('Pip failed so deleting anything we might have installed at %s', extension_path)
-        shutil.rmtree(extension_path, ignore_errors=True)
+        rmtree_with_retry(extension_path)
         raise CLIError('An error occurred. Pip failed with status code {}. '
                        'Use --debug for more information.'.format(pip_status_code))
     # Save the whl we used to install the extension in the extension dir.
@@ -351,10 +351,6 @@ def add_extension(cmd=None, source=None, extension_name=None, index_url=None, ye
 
 
 def remove_extension(extension_name):
-    def log_err(func, path, exc_info):
-        logger.warning("Error occurred attempting to delete item from the extension '%s'.", extension_name)
-        logger.warning("%s: %s - %s", func, path, exc_info)
-
     try:
         # Get the extension and it will raise an error if it doesn't exist
         ext = get_extension(extension_name)
@@ -364,7 +360,7 @@ def log_err(func, path, exc_info):
                 "`azdev extension remove {name}`".format(name=extension_name))
         # We call this just before we remove the extension so we can get the metadata before it is gone
         _augment_telemetry_with_ext_info(extension_name, ext)
-        shutil.rmtree(ext.path, onerror=log_err)
+        rmtree_with_retry(ext.path)
         CommandIndex().invalidate()
     except ExtensionNotInstalledException as e:
         raise CLIError(e)
@@ -408,13 +404,13 @@ def update_extension(cmd=None, extension_name=None, index_url=None, pip_extra_in
         logger.debug('Backing up the current extension: %s to %s', extension_path, backup_dir)
         shutil.copytree(extension_path, backup_dir)
         # Remove current version of the extension
-        shutil.rmtree(extension_path)
+        rmtree_with_retry(extension_path)
         # Install newer version
         try:
             _add_whl_ext(cli_ctx=cmd_cli_ctx, source=download_url, ext_sha256=ext_sha256,
                          pip_extra_index_urls=pip_extra_index_urls, pip_proxy=pip_proxy)
             logger.debug('Deleting backup of old extension at %s', backup_dir)
-            shutil.rmtree(backup_dir)
+            rmtree_with_retry(backup_dir)
         except Exception as err:
             logger.error('An error occurred whilst updating.')
             logger.error(err)
@@ -563,3 +559,22 @@ def check_distro_consistency():
                      "for your distribution or change the above file accordingly.")
         logger.debug("Linux distro check: %s has '%s', current distro is '%s'",
                      LIST_FILE_PATH, stored_linux_dist_name, current_linux_dist_name)
+
+
+def rmtree_with_retry(path):
+    # A workaround for https://bugs.python.org/issue33240
+    # Retry shutil.rmtree several times, but even if it fails after several retries, don't block the command execution.
+    retry_num = 3
+    import time
+    while True:
+        try:
+            shutil.rmtree(path)
+            return
+        except OSError as err:
+            if retry_num > 0:
+                logger.warning("Failed to delete '%s': %s. Retrying ...", path, err)
+                retry_num -= 1
+                time.sleep(1)
+            else:
+                logger.warning("Failed to delete '%s': %s. You may try to delete it manually.", path, err)
+                break

src/azure-cli-core/azure/cli/core/profiles/_shared.py

@@ -60,12 +60,12 @@ class ResourceType(Enum):  # pylint: disable=too-few-public-methods
     MGMT_ARO = ('azure.mgmt.redhatopenshift', 'AzureRedHatOpenShiftClient')
     MGMT_DATABOXEDGE = ('azure.mgmt.databoxedge', 'DataBoxEdgeManagementClient')
     MGMT_CUSTOMLOCATION = ('azure.mgmt.extendedlocation', 'CustomLocations')
+    MGMT_CONTAINERSERVICE = ('azure.mgmt.containerservice', 'ContainerServiceClient')
     # the "None" below will stay till a command module fills in the type so "get_mgmt_service_client"
     # can be provided with "ResourceType.XXX" to initialize the client object. This usually happens
     # when related commands start to support Multi-API
 
     DATA_COSMOS_TABLE = ('azure.multiapi.cosmosdb', None)
-    MGMT_CONTAINERSERVICE = ('azure.mgmt.containerservice', None)
     MGMT_ADVISOR = ('azure.mgmt.advisor', None)
     MGMT_MEDIA = ('azure.mgmt.media', None)
     MGMT_BACKUP = ('azure.mgmt.recoveryservicesbackup', None)
@@ -213,7 +213,11 @@ def default_api_version(self):
         ResourceType.MGMT_IOTHUB: '2021-03-31',
         ResourceType.MGMT_ARO: '2020-04-30',
         ResourceType.MGMT_DATABOXEDGE: '2019-08-01',
-        ResourceType.MGMT_CUSTOMLOCATION: '2021-03-15-preview'
+        ResourceType.MGMT_CUSTOMLOCATION: '2021-03-15-preview',
+        ResourceType.MGMT_CONTAINERSERVICE: SDKProfile('2021-05-01', {
+            'container_services': '2017-07-01',
+            'open_shift_managed_clusters': '2019-09-30-preview'
+        })
     },
     '2020-09-01-hybrid': {
         ResourceType.MGMT_STORAGE: '2019-06-01',
@@ -253,7 +257,12 @@ def default_api_version(self):
         ResourceType.MGMT_APPSERVICE: '2018-02-01',
         ResourceType.MGMT_EVENTHUB: '2018-01-01-preview',
         ResourceType.MGMT_IOTHUB: '2019-07-01-preview',
-        ResourceType.MGMT_DATABOXEDGE: '2019-08-01'
+        ResourceType.MGMT_DATABOXEDGE: '2019-08-01',
+        ResourceType.MGMT_CONTAINERREGISTRY: '2019-05-01',
+        ResourceType.MGMT_CONTAINERSERVICE: SDKProfile('2020-11-01', {
+            'container_services': '2017-07-01',
+            'open_shift_managed_clusters': '2019-09-30-preview'
+        })
     },
     '2019-03-01-hybrid': {
         ResourceType.MGMT_STORAGE: '2017-10-01',
@@ -577,7 +586,7 @@ def get_versioned_sdk_path(api_profile, resource_type, operation_group=None):
     """
     api_version = get_api_version(api_profile, resource_type)
     if api_version is None:
-        return resource_type
+        return resource_type.import_prefix
     if isinstance(api_version, _ApiVersions):
         if operation_group is None:
             raise ValueError("operation_group is required for resource type '{}'".format(resource_type))

src/azure-cli-core/setup.py

@@ -8,7 +8,7 @@
 from codecs import open
 from setuptools import setup, find_packages
 
-VERSION = "2.24.2"
+VERSION = "2.25.0"
 
 # If we have source, validate that our version numbers match
 # This should prevent uploading releases with mismatched versions.
@@ -55,9 +55,9 @@
     'msal>=1.10.0,<2.0.0',
     'paramiko>=2.0.8,<3.0.0',
     'pkginfo>=1.5.0.1',
-    'PyJWT==1.7.1',
+    'PyJWT>=2.1.0',
     'pyopenssl>=17.1.0',  # https://github.com/pyca/pyopenssl/pull/612
-    'requests~=2.22',
+    'requests~=2.25.1',
     'six~=1.12',
     'urllib3[secure]>=1.26.5',
 ]
@@ -72,14 +72,12 @@
 
 with open('README.rst', 'r', encoding='utf-8') as f:
     README = f.read()
-with open('HISTORY.rst', 'r', encoding='utf-8') as f:
-    HISTORY = f.read()
 
 setup(
     name='azure-cli-core',
     version=VERSION,
     description='Microsoft Azure Command-Line Tools Core Module',
-    long_description=README + '\n\n' + HISTORY,
+    long_description=README,
     license='MIT',
     author='Microsoft Corporation',
     author_email='[email protected]',