[k8s-extension] Update extension CLI to v1.4.3 (Azure#6683)

src/k8s-extension/HISTORY.rst

@@ -3,6 +3,14 @@
 Release History
 ===============
 
+1.4.3
+++++++++++++++++++
+* microsoft.azuremonitor.containers: Extend ContainerInsights Extension dataCollectionSettings with streams and containerlogv2 field. Also, add a kind tag in DCR creation for the ContainerInsights extension.
+* microsoft.dapr: Use semver instead of packaging
+* microsoft.azuremonitor.containers.metrics: update logic to sanitize cluster name for dc* objects
+* microsoft.openservicemesh: Fix osm-arc version check for CI tags
+* add support to skip provisioning of prerequisites for Azure Monitor K8s extensions
+
 1.4.2
 ++++++++++++++++++
 * microsoft.azuremonitor.containers: ContainerInsights Extension Managed Identity Auth Enabled by default

src/k8s-extension/azext_k8s_extension/partner_extensions/AzureMonitorMetrics.py

@@ -9,7 +9,7 @@
 import json
 import re
 
-from ..utils import get_cluster_rp_api_version
+from ..utils import get_cluster_rp_api_version, is_skip_prerequisites_specified
 
 from knack.log import get_logger
 
@@ -48,16 +48,19 @@ def Create(self, cmd, client, resource_group_name, cluster_name, name, cluster_t
         if release_train is None:
             release_train = 'stable'
 
-        cluster_subscription = get_subscription_id(cmd.cli_ctx)
-        ensure_azure_monitor_profile_prerequisites(
-            cmd,
-            cluster_rp,
-            cluster_subscription,
-            resource_group_name,
-            cluster_name,
-            configuration_settings,
-            cluster_type
-        )
+        if not is_skip_prerequisites_specified(configuration_settings):
+            cluster_subscription = get_subscription_id(cmd.cli_ctx)
+            ensure_azure_monitor_profile_prerequisites(
+                cmd,
+                cluster_rp,
+                cluster_subscription,
+                resource_group_name,
+                cluster_name,
+                configuration_settings,
+                cluster_type
+            )
+        else:
+            logger.info("Provisioning of prerequisites is skipped")
 
         create_identity = True
         extension = Extension(
@@ -72,6 +75,16 @@ def Create(self, cmd, client, resource_group_name, cluster_name, name, cluster_t
         return extension, name, create_identity
 
     def Delete(self, cmd, client, resource_group_name, cluster_name, name, cluster_type, cluster_rp, yes):
-        # cluster_rp, _ = get_cluster_rp_api_version(cluster_type=cluster_type, cluster_rp=cluster_rp)
+        cluster_rp, _ = get_cluster_rp_api_version(cluster_type=cluster_type, cluster_rp=cluster_rp)
+        try:
+            extension = client.get(resource_group_name, cluster_rp, cluster_type, cluster_name, name)
+        except Exception:
+            pass  # its OK to ignore the exception since MSI auth in preview
+
+        if (extension is not None) and (extension.configuration_settings is not None):
+            if is_skip_prerequisites_specified(extension.configuration_settings):
+                logger.info("Deprovisioning of prerequisites is skipped")
+                return
+
         cluster_subscription = get_subscription_id(cmd.cli_ctx)
         unlink_azure_monitor_profile_artifacts(cmd, cluster_subscription, resource_group_name, cluster_name)

src/k8s-extension/azext_k8s_extension/partner_extensions/ContainerInsights.py

@@ -9,7 +9,7 @@
 import json
 import re
 
-from ..utils import get_cluster_rp_api_version
+from ..utils import get_cluster_rp_api_version, is_skip_prerequisites_specified
 from .. import consts
 
 from knack.log import get_logger
@@ -31,6 +31,7 @@
     cf_resources, cf_resource_groups, cf_log_analytics)
 
 logger = get_logger(__name__)
+DCR_API_VERSION = "2022-06-01"
 
 
 class ContainerInsights(DefaultExtension):
@@ -59,8 +60,11 @@ def Create(self, cmd, client, resource_group_name, cluster_name, name, cluster_t
                        'only supports cluster scope and single instance of this extension.', extension_type)
         logger.warning("Defaulting to extension name '%s' and release-namespace '%s'", name, release_namespace)
 
-        _get_container_insights_settings(cmd, resource_group_name, cluster_rp, cluster_type, cluster_name, configuration_settings,
-                                         configuration_protected_settings, is_ci_extension_type)
+        if not is_skip_prerequisites_specified(configuration_settings):
+            _get_container_insights_settings(cmd, resource_group_name, cluster_rp, cluster_type, cluster_name, configuration_settings,
+                                             configuration_protected_settings, is_ci_extension_type)
+        else:
+            logger.info("Provisioning of prerequisites is skipped")
 
         # NOTE-2: Return a valid Extension object, Instance name and flag for Identity
         create_identity = True
@@ -85,6 +89,11 @@ def Delete(self, cmd, client, resource_group_name, cluster_name, name, cluster_t
         except Exception:
             pass  # its OK to ignore the exception since MSI auth in preview
 
+        if (extension is not None) and (extension.configuration_settings is not None):
+            if is_skip_prerequisites_specified(extension.configuration_settings):
+                logger.info("Deprovisioning of prerequisites is skipped")
+                return
+
         subscription_id = get_subscription_id(cmd.cli_ctx)
         # handle cluster type here
         cluster_resource_id = '/subscriptions/{0}/resourceGroups/{1}/providers/{2}/{3}/{4}'.format(subscription_id, resource_group_name, cluster_rp, cluster_type, cluster_name)
@@ -100,7 +109,7 @@ def Delete(self, cmd, client, resource_group_name, cluster_name, name, cluster_t
                 if (isinstance(useAADAuthSetting, str) and str(useAADAuthSetting).lower() == "true") or (isinstance(useAADAuthSetting, bool) and useAADAuthSetting):
                     useAADAuth = True
         if useAADAuth:
-            association_url = cmd.cli_ctx.cloud.endpoints.resource_manager + f"{cluster_resource_id}/providers/Microsoft.Insights/dataCollectionRuleAssociations/ContainerInsightsExtension?api-version=2021-04-01"
+            association_url = cmd.cli_ctx.cloud.endpoints.resource_manager + f"{cluster_resource_id}/providers/Microsoft.Insights/dataCollectionRuleAssociations/ContainerInsightsExtension?api-version={DCR_API_VERSION}"
             for _ in range(3):
                 try:
                     send_raw_request(cmd.cli_ctx, "GET", association_url,)
@@ -114,7 +123,7 @@ def Delete(self, cmd, client, resource_group_name, cluster_name, name, cluster_t
                     pass  # its OK to ignore the exception since MSI auth in preview
 
         if isDCRAExists:
-            association_url = cmd.cli_ctx.cloud.endpoints.resource_manager + f"{cluster_resource_id}/providers/Microsoft.Insights/dataCollectionRuleAssociations/ContainerInsightsExtension?api-version=2021-04-01"
+            association_url = cmd.cli_ctx.cloud.endpoints.resource_manager + f"{cluster_resource_id}/providers/Microsoft.Insights/dataCollectionRuleAssociations/ContainerInsightsExtension?api-version={DCR_API_VERSION}"
             for _ in range(3):
                 try:
                     send_raw_request(cmd.cli_ctx, "DELETE", association_url,)
@@ -455,6 +464,8 @@ def _get_container_insights_settings(cmd, cluster_resource_group_name, cluster_r
     subscription_id = get_subscription_id(cmd.cli_ctx)
     workspace_resource_id = ''
     useAADAuth = True
+    if 'amalogs.useAADAuth' not in configuration_settings:
+        configuration_settings['amalogs.useAADAuth'] = "true"
     extensionSettings = {}
 
     if configuration_settings is not None:
@@ -471,11 +482,15 @@ def _get_container_insights_settings(cmd, cluster_resource_group_name, cluster_r
             logger.info("provided useAADAuth flag is : %s", useAADAuthSetting)
             if (isinstance(useAADAuthSetting, str) and str(useAADAuthSetting).lower() == "true") or (isinstance(useAADAuthSetting, bool) and useAADAuthSetting):
                 useAADAuth = True
+            else:
+                useAADAuth = False
         elif 'amalogs.useAADAuth' in configuration_settings:
             useAADAuthSetting = configuration_settings['amalogs.useAADAuth']
             logger.info("provided useAADAuth flag is : %s", useAADAuthSetting)
             if (isinstance(useAADAuthSetting, str) and str(useAADAuthSetting).lower() == "true") or (isinstance(useAADAuthSetting, bool) and useAADAuthSetting):
                 useAADAuth = True
+            else:
+                useAADAuth = False
         if useAADAuth and ('dataCollectionSettings' in configuration_settings):
             dataCollectionSettingsString = configuration_settings["dataCollectionSettings"]
             logger.info("provided dataCollectionSettings  is : %s", dataCollectionSettingsString)
@@ -495,6 +510,14 @@ def _get_container_insights_settings(cmd, cluster_resource_group_name, cluster_r
                 namspaces = dataCollectionSettings["namespaces"]
                 if isinstance(namspaces, list) is False:
                     raise InvalidArgumentValueError('namespaces must be an array type')
+            if 'enableContainerLogV2' in dataCollectionSettings.keys():
+                enableContainerLogV2Value = dataCollectionSettings["enableContainerLogV2"]
+                if not isinstance(enableContainerLogV2Value, bool):
+                    raise InvalidArgumentValueError('enableContainerLogV2Value value MUST be either true or false')
+            if 'streams' in dataCollectionSettings.keys():
+                streams = dataCollectionSettings["streams"]
+                if isinstance(streams, list) is False:
+                    raise InvalidArgumentValueError('streams must be an array type')
             extensionSettings["dataCollectionSettings"] = dataCollectionSettings
 
     workspace_resource_id = workspace_resource_id.strip()
@@ -673,34 +696,35 @@ def _ensure_container_insights_dcr_for_monitoring(cmd, subscription_id, cluster_
             if (cluster_region not in region_ids):
                 raise ClientRequestError(f"Data Collection Rule Associations are not supported for cluster region {cluster_region}")
 
-    dcr_url = cmd.cli_ctx.cloud.endpoints.resource_manager + f"{dcr_resource_id}?api-version=2021-04-01"
+    dcr_url = cmd.cli_ctx.cloud.endpoints.resource_manager + f"{dcr_resource_id}?api-version={DCR_API_VERSION}"
     # get existing tags on the container insights extension DCR if the customer added any
     existing_tags = get_existing_container_insights_extension_dcr_tags(cmd, dcr_url)
+    streams = ["Microsoft-ContainerInsights-Group-Default"]
+    if extensionSettings is not None and 'dataCollectionSettings' in extensionSettings.keys():
+        dataCollectionSettings = extensionSettings["dataCollectionSettings"]
+        if dataCollectionSettings is not None and 'streams' in dataCollectionSettings.keys():
+            streams = dataCollectionSettings["streams"]
 
     # create the DCR
     dcr_creation_body = json.dumps(
         {
             "location": workspace_region,
             "tags": existing_tags,
+            "kind": "Linux",
             "properties": {
                 "dataSources": {
                     "extensions": [
                         {
                             "name": "ContainerInsightsExtension",
-                            "streams": [
-                                "Microsoft-ContainerInsights-Group-Default"
-                            ],
+                            "streams": streams,
                             "extensionName": "ContainerInsights",
                             "extensionSettings": extensionSettings
                         }
                     ]
                 },
                 "dataFlows": [
                     {
-                        "streams": [
-                            "Microsoft-ContainerInsights-Group-Default"
-
-                        ],
+                        "streams": streams,
                         "destinations": ["la-workspace"],
                     }
                 ],
@@ -735,7 +759,7 @@ def _ensure_container_insights_dcr_for_monitoring(cmd, subscription_id, cluster_
             },
         }
     )
-    association_url = cmd.cli_ctx.cloud.endpoints.resource_manager + f"{cluster_resource_id}/providers/Microsoft.Insights/dataCollectionRuleAssociations/ContainerInsightsExtension?api-version=2021-04-01"
+    association_url = cmd.cli_ctx.cloud.endpoints.resource_manager + f"{cluster_resource_id}/providers/Microsoft.Insights/dataCollectionRuleAssociations/ContainerInsightsExtension?api-version={DCR_API_VERSION}"
     for _ in range(3):
         try:
             send_raw_request(cmd.cli_ctx, "PUT", association_url, body=association_body,)

src/k8s-extension/azext_k8s_extension/partner_extensions/Dapr.py

@@ -13,7 +13,7 @@
 from copy import deepcopy
 from knack.log import get_logger
 from knack.prompting import prompt, prompt_y_n
-from packaging import version as packaging_version
+from semver import VersionInfo
 
 from ..vendored_sdks.models import Extension, PatchExtension, Scope, ScopeCluster
 from .DefaultExtension import DefaultExtension
@@ -204,7 +204,7 @@ def _is_downgrade(v1: str, v2: str) -> bool:
         Returns True if version v1 is less than version v2.
         """
         try:
-            return packaging_version.Version(v1) < packaging_version.Version(v2)
-        except packaging_version.InvalidVersion:
+            return VersionInfo.parse(v1) < VersionInfo.parse(v2)
+        except ValueError:
             logger.debug("Warning: Unable to compare versions %s and %s.", v1, v2)
             return True  # This will cause the apply-CRDs hook to be disabled, which is safe.

src/k8s-extension/azext_k8s_extension/partner_extensions/OpenServiceMesh.py

@@ -71,7 +71,6 @@ def Create(self, cmd, client, resource_group_name, cluster_name, name, cluster_t
 
 def _validate_tested_distro(cmd, cluster_resource_group_name, cluster_name, extension_version, extension_release_train):
 
-    logger.warning("Running validated distros...")
     field_unavailable_error = '\"testedDistros\" field unavailable for version {0} of microsoft.openservicemesh, ' \
         'cannot determine if this Kubernetes distribution has been properly tested'.format(extension_version)
 
@@ -104,7 +103,10 @@ def _validate_tested_distro(cmd, cluster_resource_group_name, cluster_name, exte
 
         extension_version = tags[len(tags) - 1]
 
-    if version.parse(str(extension_version)) <= version.parse("0.8.3"):
+    ext_str = str(extension_version)
+
+    # Don't parse version for test and CI tags
+    if "pr" not in ext_str and "release" not in ext_str and version.parse(ext_str) <= version.parse("0.8.3"):
         logger.warning(field_unavailable_error)
         return
 

src/k8s-extension/azext_k8s_extension/partner_extensions/azuremonitormetrics/dc/defaults.py

@@ -13,10 +13,10 @@
 # All DC* object names should end only in alpha numeric (after `length` trim)
 # DCE remove underscore from cluster name
 def sanitize_name(name, objtype, length):
+    name = name[0:length]
     length = length - 1
     if objtype == DC_TYPE.DCE:
-        name = name.replace("_", "")
-    name = name[0:length]
+        name = ''.join(char for char in name if char.isalnum() or char == '-')
     lastIndexAlphaNumeric = len(name) - 1
     while ((name[lastIndexAlphaNumeric].isalnum() is False) and lastIndexAlphaNumeric > -1):
         lastIndexAlphaNumeric = lastIndexAlphaNumeric - 1

src/k8s-extension/azext_k8s_extension/utils.py

@@ -54,3 +54,17 @@ def is_dogfood_cluster(cmd):
         urlparse(cmd.cli_ctx.cloud.endpoints.resource_manager).hostname
         == consts.DF_RM_HOSTNAME
     )
+
+
+def is_skip_prerequisites_specified(configuration_settings):
+    # Determine if provisioning to prerequisites should be skipped by a configuration setting named skipPrerequisites.
+    SKIP_PREQUISITES = 'skipPrerequisites'
+
+    has_skip_prerequisites_set = False
+
+    if SKIP_PREQUISITES in configuration_settings:
+        skip_prerequisites_configuration_setting = configuration_settings[SKIP_PREQUISITES]
+        if (isinstance(skip_prerequisites_configuration_setting, str) and str(skip_prerequisites_configuration_setting).lower() == "true") or (isinstance(skip_prerequisites_configuration_setting, bool) and skip_prerequisites_configuration_setting):
+            has_skip_prerequisites_set = True
+
+    return has_skip_prerequisites_set

src/k8s-extension/setup.py

@@ -33,7 +33,7 @@
 # TODO: Add any additional SDK dependencies here
 DEPENDENCIES = []
 
-VERSION = "1.4.2"
+VERSION = "1.4.3"
 
 with open("README.rst", "r", encoding="utf-8") as f:
     README = f.read()