Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[AKS] az aks create/update: Add support for KEDA workload auto-scaler #24698

Closed
wants to merge 5 commits into from
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
15 changes: 15 additions & 0 deletions src/azure-cli/azure/cli/command_modules/acs/_help.py
Original file line number Diff line number Diff line change
Expand Up @@ -433,6 +433,9 @@
- name: --enable-oidc-issuer
type: bool
short-summary: Enable OIDC issuer.
- name: --enable-keda
type: bool
short-summary: Enable KEDA workload auto-scaler.

examples:
- name: Create a Kubernetes cluster with an existing SSH public key.
Expand Down Expand Up @@ -501,6 +504,8 @@
text: az aks create -g MyResourceGroup -n MyMC --kubernetes-version 1.20.13 --location westus2 --host-group-id /subscriptions/00000/resourceGroups/AnotherResourceGroup/providers/Microsoft.ContainerService/hostGroups/myHostGroup --node-vm-size VMSize --enable-managed-identity --assign-identity <user_assigned_identity_resource_id>
- name: Create a kubernetes cluster with no CNI installed.
text: az aks create -g MyResourceGroup -n MyManagedCluster --network-plugin none
- name: Create a kubernetes cluster with KEDA workload autoscaler enabled.
text: az aks create -g MyResourceGroup -n MyManagedCluster --enable-keda
"""

helps['aks update'] = """
Expand Down Expand Up @@ -709,6 +714,12 @@
- name: --enable-oidc-issuer
type: bool
short-summary: Enable OIDC issuer.
- name: --enable-keda
type: bool
short-summary: Enable KEDA workload auto-scaler.
- name: --disable-keda
type: bool
short-summary: Disable KEDA workload auto-scaler.

examples:
- name: Reconcile the cluster back to its current state.
Expand Down Expand Up @@ -759,6 +770,10 @@
text: az aks update -g MyResourceGroup -n MyManagedCluster --enable-windows-gmsa
- name: Enable Windows gmsa for a kubernetes cluster without setting DNS server in the vnet used by the cluster.
text: az aks update -g MyResourceGroup -n MyManagedCluster --enable-windows-gmsa --gmsa-dns-server "10.240.0.4" --gmsa-root-domain-name "contoso.com"
- name: Enable KEDA workload autoscaler for an existing kubernetes cluster.
text: az aks update -g MyResourceGroup -n MyManagedCluster --enable-keda
- name: Disable KEDA workload autoscaler for an existing kubernetes cluster.
text: az aks update -g MyResourceGroup -n MyManagedCluster --disable-keda
"""

helps['aks delete'] = """
Expand Down
3 changes: 3 additions & 0 deletions src/azure-cli/azure/cli/command_modules/acs/_params.py
Original file line number Diff line number Diff line change
Expand Up @@ -217,6 +217,7 @@ def load_arguments(self, _):
c.argument('azure_keyvault_kms_key_vault_network_access', arg_type=get_enum_type(keyvault_network_access_types))
c.argument('azure_keyvault_kms_key_vault_resource_id', validator=validate_azure_keyvault_kms_key_vault_resource_id)
c.argument('http_proxy_config')
c.argument('enable_keda', action='store_true')
# addons
c.argument('enable_addons', options_list=['--enable-addons', '-a'])
c.argument('workspace_resource_id')
Expand Down Expand Up @@ -318,6 +319,8 @@ def load_arguments(self, _):
c.argument('azure_keyvault_kms_key_vault_network_access', arg_type=get_enum_type(keyvault_network_access_types))
c.argument('azure_keyvault_kms_key_vault_resource_id', validator=validate_azure_keyvault_kms_key_vault_resource_id)
c.argument('http_proxy_config')
c.argument('enable_keda', action='store_true')
c.argument('disable_keda', action='store_true')
# addons
c.argument('enable_secret_rotation', action='store_true')
c.argument('disable_secret_rotation', action='store_true', validator=validate_keyvault_secrets_provider_disable_and_enable_parameters)
Expand Down
3 changes: 3 additions & 0 deletions src/azure-cli/azure/cli/command_modules/acs/custom.py
Original file line number Diff line number Diff line change
Expand Up @@ -435,6 +435,7 @@ def aks_create(
azure_keyvault_kms_key_id=None,
azure_keyvault_kms_key_vault_network_access=None,
azure_keyvault_kms_key_vault_resource_id=None,
enable_keda=False,
# addons
enable_addons=None,
workspace_resource_id=None,
Expand Down Expand Up @@ -561,6 +562,8 @@ def aks_update(
azure_keyvault_kms_key_vault_network_access=None,
azure_keyvault_kms_key_vault_resource_id=None,
http_proxy_config=None,
enable_keda=False,
disable_keda=False,
# addons
enable_secret_rotation=False,
disable_secret_rotation=False,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -107,6 +107,8 @@
ManagedClusterStorageProfileFileCSIDriver = TypeVar('ManagedClusterStorageProfileFileCSIDriver')
ManagedClusterStorageProfileBlobCSIDriver = TypeVar('ManagedClusterStorageProfileBlobCSIDriver')
ManagedClusterStorageProfileSnapshotController = TypeVar('ManagedClusterStorageProfileSnapshotController')
ManagedClusterWorkloadAutoscalerProfile = TypeVar('ManagedClusterWorkloadAutoscalerProfile')
ManagedClusterWorkloadAutoscalerProfileKeda = TypeVar('ManagedClusterWorkloadAutoscalerProfileKeda')

# TODO
# 1. remove enable_rbac related implementation
Expand Down Expand Up @@ -635,6 +637,76 @@ def get_blob_driver(self) -> Optional[ManagedClusterStorageProfileBlobCSIDriver]

return profile

def get_enable_keda(self) -> bool:
"""Obtain the value of enable_keda.

This function will verify the parameter by default. If both enable_keda and disable_keda are specified, raise a
MutuallyExclusiveArgumentError.

:return: bool
"""
return self._get_enable_keda(enable_validation=True)

def _get_enable_keda(self, enable_validation: bool = False) -> bool:
"""Internal function to obtain the value of enable_keda.

This function supports the option of enable_validation. When enabled, if both enable_keda and disable_keda are
specified, raise a MutuallyExclusiveArgumentError.

:return: bool
"""
# Read the original value passed by the command.
enable_keda = self.raw_param.get("enable_keda")

# In create mode, try to read the property value corresponding to the parameter from the `mc` object.
if self.decorator_mode == DecoratorMode.CREATE:
if (
self.mc and
self.mc.workload_auto_scaler_profile and
self.mc.workload_auto_scaler_profile.keda
):
enable_keda = self.mc.workload_auto_scaler_profile.keda.enabled

# This parameter does not need dynamic completion.
if enable_validation:
if enable_keda and self._get_disable_keda(enable_validation=False):
raise MutuallyExclusiveArgumentError(
"Cannot specify --enable-keda and --disable-keda at the same time."
)

return enable_keda

def get_disable_keda(self) -> bool:
"""Obtain the value of disable_keda.

This function will verify the parameter by default. If both enable_keda and disable_keda are specified, raise a
MutuallyExclusiveArgumentError.

:return: bool
"""
return self._get_disable_keda(enable_validation=True)

def _get_disable_keda(self, enable_validation: bool = False) -> bool:
"""Internal function to obtain the value of disable_keda.

This function supports the option of enable_validation. When enabled, if both enable_keda and disable_keda are
specified, raise a MutuallyExclusiveArgumentError.

:return: bool
"""
# Read the original value passed by the command.
disable_keda = self.raw_param.get("disable_keda")

# This option is not supported in create mode, hence we do not read the property value from the `mc` object.
# This parameter does not need dynamic completion.
if enable_validation:
if disable_keda and self._get_enable_keda(enable_validation=False):
raise MutuallyExclusiveArgumentError(
"Cannot specify --enable-keda and --disable-keda at the same time."
)

return disable_keda

def get_snapshot_controller(self) -> Optional[ManagedClusterStorageProfileSnapshotController]:
"""Obtain the value of storage_profile.snapshot_controller

Expand Down Expand Up @@ -5160,6 +5232,39 @@ def set_up_oidc_issuer_profile(self, mc: ManagedCluster) -> ManagedCluster:

return mc

def set_up_workload_auto_scaler_profile(self, mc: ManagedCluster) -> ManagedCluster:
"""Set up workload auto-scaler profile for the ManagedCluster object.

:return: the ManagedCluster object
"""
self._ensure_mc(mc)

if self.context.get_enable_keda():
if mc.workload_auto_scaler_profile is None:
mc.workload_auto_scaler_profile = self.models.ManagedClusterWorkloadAutoscalerProfile()
mc.workload_auto_scaler_profile.keda = ManagedClusterWorkloadAutoscalerProfileKeda(enabled=True)

return mc

def update_workload_auto_scaler_profile(self, mc: ManagedCluster) -> ManagedCluster:
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

You should move this function to the other class AKSManagedClusterUpdateDecorator. The function to construct the updated profile (update_mc_profile_default) is also in this class.

"""Update workload auto-scaler profile for the ManagedCluster object.

:return: the ManagedCluster object
"""
self._ensure_mc(mc)

if self.context.get_enable_keda():
if mc.workload_auto_scaler_profile is None:
mc.workload_auto_scaler_profile = self.models.ManagedClusterWorkloadAutoscalerProfile()
mc.workload_auto_scaler_profile.keda = ManagedClusterWorkloadAutoscalerProfileKeda(enabled=True)

if self.context.get_disable_keda():
if mc.workload_auto_scaler_profile is None:
mc.workload_auto_scaler_profile = self.models.ManagedClusterWorkloadAutoscalerProfile()
mc.workload_auto_scaler_profile.keda = ManagedClusterWorkloadAutoscalerProfileKeda(enabled=False)

return mc

def set_up_api_server_access_profile(self, mc: ManagedCluster) -> ManagedCluster:
"""Set up api server access profile and fqdn subdomain for the ManagedCluster object.

Expand Down Expand Up @@ -5375,6 +5480,8 @@ def construct_mc_profile_default(self, bypass_restore_defaults: bool = False) ->
mc = self.set_up_azure_keyvault_kms(mc)
# set up http proxy config
mc = self.set_up_http_proxy_config(mc)
# set up workload autoscaler profile
mc = self.set_up_workload_auto_scaler_profile(mc)

# DO NOT MOVE: keep this at the bottom, restore defaults
if not bypass_restore_defaults:
Expand Down Expand Up @@ -6319,6 +6426,8 @@ def update_mc_profile_default(self) -> ManagedCluster:
mc = self.update_identity_profile(mc)
# set up http proxy config
mc = self.update_http_proxy_config(mc)
# update workload autoscaler profile
mc = self.update_workload_auto_scaler_profile(mc)
return mc

def check_is_postprocessing_required(self, mc: ManagedCluster) -> bool:
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -7938,3 +7938,70 @@ def test_aks_nodepool_add_with_gpu_instance_profile(self, resource_group, resour
# delete
self.cmd(
'aks delete -g {resource_group} -n {name} --yes --no-wait', checks=[self.is_empty()])

@AllowLargeResponse()
@AKSCustomResourceGroupPreparer(random_name_length=17, name_prefix='clitest', location='westus2')
def test_aks_create_with_keda(self, resource_group, resource_group_location):
aks_name = self.create_random_name('cliakstest', 16)
self.kwargs.update({
'resource_group': resource_group,
'name': aks_name,
'location': resource_group_location,
'ssh_key_value': self.generate_ssh_keys(),
})

# create: enable-keda
create_cmd = 'aks create --resource-group={resource_group} --name={name} --location={location} --ssh-key-value={ssh_key_value} --output=json ' \
'--aks-custom-headers=AKSHTTPCustomFeatures=Microsoft.ContainerService/AKS-KedaPreview ' \
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Does this feature still need the feature flag? If not, please remove this custom header

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Currently, it does. Once all other changes are done for GA, we will remove the feature flag. I'll send in another PR to change that here as well.

'--enable-keda'
self.cmd(create_cmd, checks=[
self.check('provisioningState', 'Succeeded'),
self.check('workloadAutoScalerProfile.keda.enabled', True),
])

# delete
delete_cmd = 'aks delete --resource-group={resource_group} --name={name} --yes --no-wait'
self.cmd(delete_cmd, checks=[
self.is_empty(),
])

@AllowLargeResponse()
@AKSCustomResourceGroupPreparer(random_name_length=17, name_prefix='clitest', location='westus2')
def test_aks_update_with_keda(self, resource_group, resource_group_location):
aks_name = self.create_random_name('cliakstest', 16)
self.kwargs.update({
'resource_group': resource_group,
'name': aks_name,
'location': resource_group_location,
'ssh_key_value': self.generate_ssh_keys(),
})

# create: without enable-keda
create_cmd = 'aks create --resource-group={resource_group} --name={name} --location={location} --ssh-key-value={ssh_key_value} --output=json'
self.cmd(create_cmd, checks=[
self.check('provisioningState', 'Succeeded'),
self.not_exists('workloadAutoScalerProfile.keda'),
])

# update: enable-keda
update_cmd = 'aks update --resource-group={resource_group} --name={name} --yes --output=json ' \
'--aks-custom-headers=AKSHTTPCustomFeatures=Microsoft.ContainerService/AKS-KedaPreview ' \
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Also remove this header if the feature is not protected by AFEC anymore.

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Please refer to my reply above.

'--enable-keda'
self.cmd(update_cmd, checks=[
self.check('provisioningState', 'Succeeded'),
self.check('workloadAutoScalerProfile.keda.enabled', True),
])

# update: disable-keda
update_cmd = 'aks update --resource-group={resource_group} --name={name} --yes --output=json ' \
'--disable-keda'
self.cmd(update_cmd, checks=[
self.check('provisioningState', 'Succeeded'),
self.check('workloadAutoScalerProfile.keda.enabled', False),
])

# delete
cmd = 'aks delete --resource-group={resource_group} --name={name} --yes --no-wait'
self.cmd(cmd, checks=[
self.is_empty(),
])