[Please ignore] [side branch for azurestack] Bhsantha/temp aks (#18547)

* Adding ACR and AKS for azurestack hybrid profile

* Change managed cluster SDK profile for acs

* Container service resource type to managed cluster sdk

* Removing hardcoded api-version to import models

* Removing versioned models import

* Removing versioned models import

* Fixing style and models

* Changing Azure stack API version for container service

* Adding operation-group to container service

* Changing container service test_validator

* Changing container service test_validator

* Modifying operation_group to managed_clusters

* Adding managed_clusters operation group to aks run command

* Container service test code changes

* Revert "Container service test code changes"

This reverts commit c551f3b.

* Adding mock cmd to acs test

* Fix resource tyope in acs test

* Fix resource tyope in acs test

* Removing resource type in mock cmd

* Changing acs get_models

* Adding operation group to get_models

* Adding magicmock to acs test

* Ading magicmock to acs test

* Modifying mock cmd

* Modifying acs mock cmd

* Adding operation_group to acs mock

* Adding mod=models for acs mock cmd

* Changing hybrid authorization api-version

* Adding test for acs and acr hybrid profiles

* removing redundant operation_group properties

* Adding operation_group to acs loadbalancer

* {Packaging} Bump PyJWT to 2.1.0 (#18506)

* {Packaging} Be more strict on `requests` (#18518)

* {Misc.} Fix typos in HISTORY.rst (#18504)

* {Pylint} Fix `use-a-generator` (#18523)

* {Core} `az extension`: Retry `shutil.rmtree` (#18061)

* Remove preview out of security module (#18529)

Co-authored-by: Ahmad Abas <[email protected]>

* {AppService} Make `appservice` tests serial (#18534)

* ACR check_health changes

* Style check changes

* Removing unsed models

* Mergin dev changes

* Changes to hybrid profile test

* Acs hybrid test changes

Co-authored-by: Bhuvaneswari Santharam <[email protected]>
Co-authored-by: Jiashuo Li <[email protected]>
Co-authored-by: Ahmad Abbas <[email protected]>
Co-authored-by: Ahmad Abas <[email protected]>

pylintrc

@@ -12,7 +12,6 @@ reports=no
 disable=
     arguments-out-of-order,
     bad-option-value,
-    consider-using-generator,
     consider-using-with,
     cyclic-import,
     duplicate-code,
@@ -32,7 +31,6 @@ disable=
     too-many-arguments,
     too-many-function-args,
     too-many-lines,
-    use-a-generator,
     using-constant-test,
     wrong-import-order,
 

scripts/release/rpm/test_rpm_package.py

@@ -14,7 +14,7 @@
 pytest_parallel_cmd = '{} -n auto'.format(pytest_base_cmd)
 
 for mod_name in mod_list:
-    if mod_name in ['botservice', 'network', 'cloud']:
+    if mod_name in ['botservice', 'network', 'cloud', 'appservice']:
         exit_code = subprocess.call(['{} --junit-xml /azure_cli_test_result/{}.xml --pyargs azure.cli.command_modules.{}'.format(pytest_base_cmd, mod_name, mod_name)], shell=True)
     else:
         exit_code = subprocess.call(['{} --junit-xml /azure_cli_test_result/{}.xml --pyargs azure.cli.command_modules.{}'.format(pytest_parallel_cmd, mod_name, mod_name)], shell=True)

src/azure-cli-core/azure/cli/core/_profile.py

@@ -358,7 +358,7 @@ def find_subscriptions_in_vm_with_msi(self, identity_id=None, allow_no_subscript
         token_entry = msi_creds.token
         token = token_entry['access_token']
         logger.info('MSI: token was retrieved. Now trying to initialize local accounts...')
-        decode = jwt.decode(token, verify=False, algorithms=['RS256'])
+        decode = jwt.decode(token, algorithms=['RS256'], options={"verify_signature": False})
         tenant = decode['tid']
 
         subscription_finder = SubscriptionFinder(self.cli_ctx, self.auth_ctx_factory, None)
@@ -382,7 +382,7 @@ def find_subscriptions_in_cloud_console(self):
 
         _, token, _ = self._get_token_from_cloud_shell(self.cli_ctx.cloud.endpoints.active_directory_resource_id)
         logger.info('MSI: token was retrieved. Now trying to initialize local accounts...')
-        decode = jwt.decode(token, verify=False, algorithms=['RS256'])
+        decode = jwt.decode(token, algorithms=['RS256'], options={"verify_signature": False})
         tenant = decode['tid']
 
         subscription_finder = SubscriptionFinder(self.cli_ctx, self.auth_ctx_factory, None)

src/azure-cli-core/azure/cli/core/commands/__init__.py

@@ -107,7 +107,7 @@ def _expand_file_prefix(arg):
         except IndexError:
             return _maybe_load_file(arg_split[0])
 
-    return list([_expand_file_prefix(arg) for arg in args])
+    return [_expand_file_prefix(arg) for arg in args]
 
 
 def _pre_command_table_create(cli_ctx, args):

src/azure-cli-core/azure/cli/core/extension/operations.py

@@ -81,7 +81,7 @@ def _validate_whl_extension(ext_file):
     zip_ref.extractall(tmp_dir)
     zip_ref.close()
     azext_metadata = WheelExtension.get_azext_metadata(tmp_dir)
-    shutil.rmtree(tmp_dir)
+    rmtree_with_retry(tmp_dir)
     check_version_compatibility(azext_metadata)
 
 
@@ -169,7 +169,7 @@ def _add_whl_ext(cli_ctx, source, ext_sha256=None, pip_extra_index_urls=None, pi
         pip_status_code = _run_pip(pip_args, extension_path)
     if pip_status_code > 0:
         logger.debug('Pip failed so deleting anything we might have installed at %s', extension_path)
-        shutil.rmtree(extension_path, ignore_errors=True)
+        rmtree_with_retry(extension_path)
         raise CLIError('An error occurred. Pip failed with status code {}. '
                        'Use --debug for more information.'.format(pip_status_code))
     # Save the whl we used to install the extension in the extension dir.
@@ -351,10 +351,6 @@ def add_extension(cmd=None, source=None, extension_name=None, index_url=None, ye
 
 
 def remove_extension(extension_name):
-    def log_err(func, path, exc_info):
-        logger.warning("Error occurred attempting to delete item from the extension '%s'.", extension_name)
-        logger.warning("%s: %s - %s", func, path, exc_info)
-
     try:
         # Get the extension and it will raise an error if it doesn't exist
         ext = get_extension(extension_name)
@@ -364,7 +360,7 @@ def log_err(func, path, exc_info):
                 "`azdev extension remove {name}`".format(name=extension_name))
         # We call this just before we remove the extension so we can get the metadata before it is gone
         _augment_telemetry_with_ext_info(extension_name, ext)
-        shutil.rmtree(ext.path, onerror=log_err)
+        rmtree_with_retry(ext.path)
         CommandIndex().invalidate()
     except ExtensionNotInstalledException as e:
         raise CLIError(e)
@@ -408,13 +404,13 @@ def update_extension(cmd=None, extension_name=None, index_url=None, pip_extra_in
         logger.debug('Backing up the current extension: %s to %s', extension_path, backup_dir)
         shutil.copytree(extension_path, backup_dir)
         # Remove current version of the extension
-        shutil.rmtree(extension_path)
+        rmtree_with_retry(extension_path)
         # Install newer version
         try:
             _add_whl_ext(cli_ctx=cmd_cli_ctx, source=download_url, ext_sha256=ext_sha256,
                          pip_extra_index_urls=pip_extra_index_urls, pip_proxy=pip_proxy)
             logger.debug('Deleting backup of old extension at %s', backup_dir)
-            shutil.rmtree(backup_dir)
+            rmtree_with_retry(backup_dir)
         except Exception as err:
             logger.error('An error occurred whilst updating.')
             logger.error(err)
@@ -563,3 +559,22 @@ def check_distro_consistency():
                      "for your distribution or change the above file accordingly.")
         logger.debug("Linux distro check: %s has '%s', current distro is '%s'",
                      LIST_FILE_PATH, stored_linux_dist_name, current_linux_dist_name)
+
+
+def rmtree_with_retry(path):
+    # A workaround for https://bugs.python.org/issue33240
+    # Retry shutil.rmtree several times, but even if it fails after several retries, don't block the command execution.
+    retry_num = 3
+    import time
+    while True:
+        try:
+            shutil.rmtree(path)
+            return
+        except OSError as err:
+            if retry_num > 0:
+                logger.warning("Failed to delete '%s': %s. Retrying ...", path, err)
+                retry_num -= 1
+                time.sleep(1)
+            else:
+                logger.warning("Failed to delete '%s': %s. You may try to delete it manually.", path, err)
+                break

src/azure-cli-core/azure/cli/core/profiles/_shared.py

@@ -60,12 +60,12 @@ class ResourceType(Enum):  # pylint: disable=too-few-public-methods
     MGMT_ARO = ('azure.mgmt.redhatopenshift', 'AzureRedHatOpenShiftClient')
     MGMT_DATABOXEDGE = ('azure.mgmt.databoxedge', 'DataBoxEdgeManagementClient')
     MGMT_CUSTOMLOCATION = ('azure.mgmt.extendedlocation', 'CustomLocations')
+    MGMT_CONTAINERSERVICE = ('azure.mgmt.containerservice', 'ContainerServiceClient')
     # the "None" below will stay till a command module fills in the type so "get_mgmt_service_client"
     # can be provided with "ResourceType.XXX" to initialize the client object. This usually happens
     # when related commands start to support Multi-API
 
     DATA_COSMOS_TABLE = ('azure.multiapi.cosmosdb', None)
-    MGMT_CONTAINERSERVICE = ('azure.mgmt.containerservice', None)
     MGMT_ADVISOR = ('azure.mgmt.advisor', None)
     MGMT_MEDIA = ('azure.mgmt.media', None)
     MGMT_BACKUP = ('azure.mgmt.recoveryservicesbackup', None)
@@ -212,7 +212,11 @@ def default_api_version(self):
         ResourceType.MGMT_IOTHUB: '2021-03-31',
         ResourceType.MGMT_ARO: '2020-04-30',
         ResourceType.MGMT_DATABOXEDGE: '2019-08-01',
-        ResourceType.MGMT_CUSTOMLOCATION: '2021-03-15-preview'
+        ResourceType.MGMT_CUSTOMLOCATION: '2021-03-15-preview',
+        ResourceType.MGMT_CONTAINERSERVICE: SDKProfile('2021-03-01', {
+            'container_services': '2017-07-01',
+            'open_shift_managed_clusters': '2019-10-27-preview'
+        })
     },
     '2020-09-01-hybrid': {
         ResourceType.MGMT_STORAGE: '2019-06-01',
@@ -237,7 +241,7 @@ def default_api_version(self):
         ResourceType.MGMT_RESOURCE_SUBSCRIPTIONS: '2016-06-01',
         ResourceType.MGMT_RESOURCE_TEMPLATESPECS: '2015-01-01',
         ResourceType.MGMT_NETWORK_DNS: '2016-04-01',
-        ResourceType.MGMT_AUTHORIZATION: SDKProfile('2016-09-01', {
+        ResourceType.MGMT_AUTHORIZATION: SDKProfile('2015-07-01', {
             'classic_administrators': '2015-06-01',
             'policy_assignments': '2016-12-01',
             'policy_definitions': '2016-12-01'
@@ -252,7 +256,12 @@ def default_api_version(self):
         ResourceType.MGMT_APPSERVICE: '2018-02-01',
         ResourceType.MGMT_EVENTHUB: '2018-01-01-preview',
         ResourceType.MGMT_IOTHUB: '2019-07-01-preview',
-        ResourceType.MGMT_DATABOXEDGE: '2019-08-01'
+        ResourceType.MGMT_DATABOXEDGE: '2019-08-01',
+        ResourceType.MGMT_CONTAINERREGISTRY: '2019-05-01',
+        ResourceType.MGMT_CONTAINERSERVICE: SDKProfile('2020-11-01', {
+            'container_services': '2017-07-01',
+            'open_shift_managed_clusters': '2019-10-27-preview'
+        })
     },
     '2019-03-01-hybrid': {
         ResourceType.MGMT_STORAGE: '2017-10-01',

src/azure-cli-core/setup.py

@@ -55,9 +55,9 @@
     'msal>=1.10.0,<2.0.0',
     'paramiko>=2.0.8,<3.0.0',
     'pkginfo>=1.5.0.1',
-    'PyJWT==1.7.1',
+    'PyJWT>=2.1.0',
     'pyopenssl>=17.1.0',  # https://github.com/pyca/pyopenssl/pull/612
-    'requests~=2.22',
+    'requests~=2.25.1',
     'six~=1.12',
     'urllib3[secure]>=1.26.5',
 ]