Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[HDInsight] Migrate to track2 python sdk 7.0.0 #18237

Merged
merged 32 commits into from
Jun 1, 2021
Merged
Show file tree
Hide file tree
Changes from 30 commits
Commits
Show all changes
32 commits
Select commit Hold shift + click to select a range
00b9235
{Docs} Remove stale reference in README to closed issue about extensi…
dkmiller Apr 27, 2020
ab16df0
Merge remote-tracking branch 'upstream/release'
azclibot Apr 28, 2020
03436bc
Merge remote-tracking branch 'upstream/release'
azclibot Apr 30, 2020
7d98b1f
Merge remote-tracking branch 'upstream/release'
azclibot May 19, 2020
06f6ec5
Merge remote-tracking branch 'upstream/release'
azclibot Jun 2, 2020
adab2d2
Merge remote-tracking branch 'upstream/release'
azclibot Jun 23, 2020
c7b9f98
Merge remote-tracking branch 'upstream/release'
azclibot Jul 14, 2020
18e8b8d
Merge remote-tracking branch 'upstream/release'
azclibot Jul 16, 2020
1eb5074
Merge remote-tracking branch 'upstream/release'
azclibot Aug 4, 2020
cc3a159
Merge remote-tracking branch 'upstream/release' into master
azclibot Aug 6, 2020
af8f98e
Merge remote-tracking branch 'upstream/release' into master
azclibot Aug 26, 2020
0f904b9
Merge remote-tracking branch 'upstream/release' into master
azclibot Aug 28, 2020
d17a993
Merge remote-tracking branch 'upstream/release' into master
azclibot Sep 22, 2020
331b406
Merge branch 'master' of github.com:Azure/azure-cli into dev
Sep 24, 2020
c98c195
Merge branch 'dev' of github.com:Azure/azure-cli into dev
Sep 25, 2020
ee648a0
Merge branch 'dev' of github.com:Azure/azure-cli into dev
Nov 5, 2020
d2faa9f
Merge branch 'dev' of github.com:Azure/azure-cli into dev
Nov 9, 2020
64d0de4
Merge branch 'dev' of github.com:Azure/azure-cli into dev
Nov 10, 2020
7b250f5
Merge branch 'dev' of github.com:Azure/azure-cli into dev
Nov 18, 2020
03c8785
Merge branch 'dev' of github.com:Azure/azure-cli into dev
Dec 1, 2020
2f814b1
Merge branch 'dev' of github.com:Azure/azure-cli into dev
Dec 7, 2020
82ce0cc
Merge branch 'dev' of github.com:Azure/azure-cli into dev
Jan 29, 2021
3731883
Merge branch 'dev' of github.com:Azure/azure-cli into dev
Mar 18, 2021
1e48381
Merge branch 'dev' of github.com:Azure/azure-cli into dev
Apr 2, 2021
b50a530
Merge branch 'dev' of github.com:Azure/azure-cli into dev
Apr 25, 2021
15a1cef
Merge branch 'dev' of github.com:Azure/azure-cli into dev
Apr 26, 2021
a7fb6f7
Merge branch 'dev' of github.com:Azure/azure-cli into dev
May 11, 2021
0ce596f
Merge branch 'dev' of github.com:Azure/azure-cli into dev
May 24, 2021
7f9eed4
hdinsight migrate to track2 sdk and update session records
May 26, 2021
5fcc130
Add missing help
May 27, 2021
cb7cdb5
Remove duplicate check no_wait logic
May 28, 2021
87c6a61
Fix style error
May 31, 2021
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
21 changes: 19 additions & 2 deletions src/azure-cli/azure/cli/command_modules/hdinsight/_help.py
Original file line number Diff line number Diff line change
Expand Up @@ -56,7 +56,6 @@
helps['hdinsight create'] = """
type: command
short-summary: Create a new cluster.
long-summary: 'Create a new cluster. Incoming BREAKING CHANGE: The default value "large" of parameters `--workernode-size` and `--headnode-size` will be removed in the next CLI version.'
examples:
- name: Create a cluster with an existing storage account.
text: |-
Expand Down Expand Up @@ -158,6 +157,24 @@
--enable-compute-isolation --workernode-size "Standard_E8S_V3" --headnode-size "Standard_E8S_V3"
"""

helps['hdinsight resize'] = """
type: command
short-summary: Resize the specified HDInsight cluster to the specified size.
examples:
- name: Resize the cluster's workernode.
text: |-
az hdinsight resize --name MyCluster --resource-group rg --workernode-count 5
"""

helps['hdinsight update'] = """
type: command
short-summary: Update the tags of the specified HDInsight cluster.
examples:
- name: Update the tags.
text: |-
az hdinsight update --name MyCluster --resource-group rg --tags key=value
"""

helps['hdinsight list'] = """
type: command
short-summary: List HDInsight clusters in a resource group or subscription.
Expand Down Expand Up @@ -313,7 +330,7 @@
examples:
- name: Update a schedule condition.
text: |-
az hdinsight autoscale condition update --resource-group MyResourceGroup --cluster-name MyCluster --index 0\\
az hdinsight autoscale condition update --resource-group MyResourceGroup --cluster-name MyCluster --index 0 \\
--time 10:00 --workernode-count 5
"""

Expand Down
12 changes: 6 additions & 6 deletions src/azure-cli/azure/cli/command_modules/hdinsight/commands.py
Original file line number Diff line number Diff line change
Expand Up @@ -52,13 +52,13 @@ def load_command_table(self, _): # pylint: disable=too-many-statements
# cluster operations
with self.command_group('hdinsight', hdinsight_clusters_sdk, client_factory=cf_hdinsight_clusters) as g:
g.custom_command('create', 'create_cluster', supports_no_wait=True)
g.command('resize', 'resize', supports_no_wait=True)
g.custom_command('resize', 'resize_cluster', supports_no_wait=True)
g.show_command('show', 'get')
g.custom_command('list', 'list_clusters')
g.wait_command('wait')
g.command('delete', 'delete', confirmation=True, supports_no_wait=True)
g.command('delete', 'begin_delete', confirmation=True, supports_no_wait=True)
g.custom_command('rotate-disk-encryption-key', 'rotate_hdi_cluster_key', supports_no_wait=True)
g.command('update', 'update', supports_no_wait=True)
g.custom_command('update', 'update_cluster', supports_no_wait=True)

# usage operations
with self.command_group('hdinsight', hdinsight_locations_sdk, client_factory=cf_hdinsight_locations) as g:
Expand Down Expand Up @@ -91,19 +91,19 @@ def load_command_table(self, _): # pylint: disable=too-many-statements
g.show_command('show', 'get')
g.command('list', 'list_by_cluster')
g.wait_command('wait')
g.command('delete', 'delete', confirmation=True, supports_no_wait=True)
g.command('delete', 'begin_delete', confirmation=True, supports_no_wait=True)

# Monitoring operations
with self.command_group('hdinsight monitor', hdinsight_extensions_sdk, client_factory=cf_hdinsight_extensions) as g:
g.show_command('show', 'get_monitoring_status')
g.custom_command('enable', 'enable_hdi_monitoring')
g.command('disable', 'disable_monitoring')
g.command('disable', 'begin_disable_monitoring')

# VirtualMachine operations
with self.command_group('hdinsight host', hdinsight_virtual_machines_sdk,
client_factory=cf_hdinsight_virtual_machines) as g:
g.command('list', 'list_hosts')
g.command('restart', 'restart_hosts', confirmation=True)
g.command('restart', 'begin_restart_hosts', confirmation=True)

# Autoscale operations
with self.command_group('hdinsight autoscale', hdinsight_clusters_sdk, client_factory=cf_hdinsight_clusters) as g:
Expand Down
114 changes: 85 additions & 29 deletions src/azure-cli/azure/cli/command_modules/hdinsight/custom.py
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@
# Copyright (c) Microsoft Corporation. All rights reserved.
# Licensed under the MIT License. See License.txt in the project root for license information.
# --------------------------------------------------------------------------------------------

from knack.log import get_logger
from knack.prompting import prompt_pass, NoTTYException
from knack.util import CLIError
Expand Down Expand Up @@ -43,7 +44,7 @@ def create_cluster(cmd, client, cluster_name, resource_group_name, cluster_type,
from azure.mgmt.hdinsight.models import ClusterCreateParametersExtended, ClusterCreateProperties, OSType, \
ClusterDefinition, ComputeProfile, HardwareProfile, Role, OsProfile, LinuxOperatingSystemProfile, \
StorageProfile, StorageAccount, DataDisksGroups, SecurityProfile, \
DirectoryType, DiskEncryptionProperties, Tier, SshProfile, SshPublicKey, \
DiskEncryptionProperties, Tier, SshProfile, SshPublicKey, \
KafkaRestProperties, ClientGroupInfo, EncryptionInTransitProperties, \
Autoscale, AutoscaleCapacity, AutoscaleRecurrence, AutoscaleSchedule, AutoscaleTimeAndCapacity, \
NetworkProperties, PrivateLink, ComputeIsolationProperties
Expand Down Expand Up @@ -319,7 +320,7 @@ def create_cluster(cmd, client, cluster_name, resource_group_name, cluster_type,
ldaps_urls = ['ldaps://{}:636'.format(domain_name)]

security_profile = domain and SecurityProfile(
directory_type=DirectoryType.active_directory,
directory_type=SecurityProfile.directory_type,
domain=domain_name,
ldaps_urls=ldaps_urls,
domain_username=cluster_admin_account,
Expand Down Expand Up @@ -396,9 +397,9 @@ def create_cluster(cmd, client, cluster_name, resource_group_name, cluster_type,
)

if no_wait:
return sdk_no_wait(no_wait, client.create, resource_group_name, cluster_name, create_params)
return sdk_no_wait(no_wait, client.begin_create, resource_group_name, cluster_name, create_params)
aim-for-better marked this conversation as resolved.
Show resolved Hide resolved

return client.create(resource_group_name, cluster_name, create_params)
return client.begin_create(resource_group_name, cluster_name, create_params)


def list_clusters(cmd, client, resource_group_name=None): # pylint: disable=unused-argument
Expand All @@ -408,6 +409,16 @@ def list_clusters(cmd, client, resource_group_name=None): # pylint: disable=unu
return list(clusters_list)


def update_cluster(cmd, client, cluster_name, resource_group_name, tags=None, no_wait=False):
from azure.mgmt.hdinsight.models import ClusterPatchParameters
cluster_patch_parameters = ClusterPatchParameters(
tags=tags
)
if no_wait:
return sdk_no_wait(no_wait, client.update, resource_group_name, cluster_name, cluster_patch_parameters)
return client.update(resource_group_name, cluster_name, cluster_patch_parameters)


# pylint: disable=unused-argument
def rotate_hdi_cluster_key(cmd, client, resource_group_name, cluster_name,
encryption_vault_uri, encryption_key_name, encryption_key_version, no_wait=False):
Expand All @@ -419,9 +430,22 @@ def rotate_hdi_cluster_key(cmd, client, resource_group_name, cluster_name,
)

if no_wait:
return sdk_no_wait(no_wait, client.rotate_disk_encryption_key, resource_group_name, cluster_name, rotate_params)
return sdk_no_wait(no_wait, client.begin_rotate_disk_encryption_key, resource_group_name, cluster_name,
rotate_params)

return client.begin_rotate_disk_encryption_key(resource_group_name, cluster_name, rotate_params)


def resize_cluster(cmd, client, cluster_name, resource_group_name, target_instance_count, no_wait=False):
from azure.mgmt.hdinsight.models import ClusterResizeParameters, RoleName
resize_cluster_parameters = ClusterResizeParameters(
target_instance_count=target_instance_count
)
if no_wait:
return sdk_no_wait(no_wait, client.begin_resize, resource_group_name, cluster_name, RoleName.WORKERNODE,
resize_cluster_parameters)

return client.rotate_disk_encryption_key(resource_group_name, cluster_name, rotate_params)
return client.begin_resize(resource_group_name, cluster_name, RoleName.WORKERNODE, resize_cluster_parameters)


def _all_or_none(*params):
Expand Down Expand Up @@ -508,12 +532,13 @@ def create_hdi_application(cmd, client, resource_group_name, cluster_name, appli
properties=application_properties
)

return client.create(resource_group_name, cluster_name, application_name, create_params)
return client.begin_create(resource_group_name, cluster_name, application_name, create_params)


# pylint: disable=unused-argument
def enable_hdi_monitoring(cmd, client, resource_group_name, cluster_name, workspace,
primary_key=None, workspace_type='resource_id', no_validation_timeout=False):
from azure.mgmt.hdinsight.models import ClusterMonitoringRequest
from msrestazure.tools import parse_resource_id
from ._client_factory import cf_log_analytics

Expand Down Expand Up @@ -542,18 +567,21 @@ def enable_hdi_monitoring(cmd, client, resource_group_name, cluster_name, worksp

workspace_id = log_analytics_workspace.customer_id

return client.enable_monitoring(
monitor_request_parameter = ClusterMonitoringRequest(
workspace_id=workspace_id,
primary_key=primary_key
)
return client.begin_enable_monitoring(
resource_group_name,
cluster_name,
workspace_id,
primary_key)
monitor_request_parameter)


# pylint: disable=unused-argument
def execute_hdi_script_action(cmd, client, resource_group_name, cluster_name,
script_uri, script_action_name, roles, script_parameters=None,
persist_on_success=False):
from azure.mgmt.hdinsight.models import RuntimeScriptAction
from azure.mgmt.hdinsight.models import RuntimeScriptAction, ExecuteScriptActionParameters

script_actions_params = [
RuntimeScriptAction(
Expand All @@ -563,16 +591,19 @@ def execute_hdi_script_action(cmd, client, resource_group_name, cluster_name,
roles=roles
)
]

return client.execute_script_actions(resource_group_name, cluster_name, persist_on_success, script_actions_params)
execute_script_action_parameters = ExecuteScriptActionParameters(
persist_on_success=persist_on_success,
script_actions=script_actions_params
)
return client.begin_execute_script_actions(resource_group_name, cluster_name, execute_script_action_parameters)


# pylint: disable=redefined-builtin
def create_autoscale(cmd, client, resource_group_name, cluster_name, type, min_workernode_count=None,
max_workernode_count=None, timezone=None, days=None, time=None, workernode_count=None,
no_wait=False, yes=False):
from azure.mgmt.hdinsight.models import Autoscale, AutoscaleCapacity, AutoscaleRecurrence, AutoscaleSchedule, \
AutoscaleTimeAndCapacity
AutoscaleTimeAndCapacity, AutoscaleConfigurationUpdateParameter, RoleName
load_based_type = "Load"
schedule_based_type = "Schedule"
autoscale_configuration = None
Expand Down Expand Up @@ -612,13 +643,17 @@ def create_autoscale(cmd, client, resource_group_name, cluster_name, type, min_w
"This operation will override previous autoscale configuration. "
"Are you sure you want to perform this operation", yes)

return sdk_no_wait(no_wait, client.update_auto_scale_configuration, resource_group_name, cluster_name,
autoscale_configuration)
autoscale_configuration_update_parameter = AutoscaleConfigurationUpdateParameter(
autoscale=autoscale_configuration
)
return sdk_no_wait(no_wait, client.begin_update_auto_scale_configuration, resource_group_name, cluster_name,
RoleName.WORKERNODE, autoscale_configuration_update_parameter)


def update_autoscale(cmd, client, resource_group_name, cluster_name, min_workernode_count=None,
max_workernode_count=None, timezone=None, no_wait=False):
from azure.mgmt.hdinsight.models import AutoscaleCapacity, AutoscaleRecurrence
from azure.mgmt.hdinsight.models import AutoscaleCapacity, AutoscaleRecurrence, \
AutoscaleConfigurationUpdateParameter, RoleName
cluster = client.get(resource_group_name, cluster_name)
autoscale_configuration = _extract_and_validate_autoscale_configuration(cluster)

Expand All @@ -644,8 +679,11 @@ def update_autoscale(cmd, client, resource_group_name, cluster_name, min_workern
elif not autoscale_configuration.recurrence:
autoscale_configuration.recurrence = AutoscaleRecurrence(time_zone=timezone)

return sdk_no_wait(no_wait, client.update_auto_scale_configuration, resource_group_name, cluster_name,
autoscale_configuration)
autoscale_configuration_update_parameter = AutoscaleConfigurationUpdateParameter(
autoscale=autoscale_configuration
)
return sdk_no_wait(no_wait, client.begin_update_auto_scale_configuration, resource_group_name, cluster_name,
RoleName.WORKERNODE, autoscale_configuration_update_parameter)


def show_autoscale(cmd, client, resource_group_name, cluster_name):
Expand All @@ -655,12 +693,17 @@ def show_autoscale(cmd, client, resource_group_name, cluster_name):


def delete_autoscale(cmd, client, resource_group_name, cluster_name, no_wait=False):
from azure.mgmt.hdinsight.models import AutoscaleConfigurationUpdateParameter, RoleName

cluster = client.get(resource_group_name, cluster_name)
_extract_and_validate_autoscale_configuration(cluster)
autoscale_configuration = None

return sdk_no_wait(no_wait, client.update_auto_scale_configuration, resource_group_name, cluster_name,
autoscale_configuration)
autoscale_configuration_update_parameter = AutoscaleConfigurationUpdateParameter(
autoscale=autoscale_configuration
)
return sdk_no_wait(no_wait, client.begin_update_auto_scale_configuration, resource_group_name, cluster_name,
RoleName.WORKERNODE, autoscale_configuration_update_parameter)


def list_timezones(cmd, client):
Expand All @@ -670,7 +713,8 @@ def list_timezones(cmd, client):

def create_autoscale_condition(cmd, client, resource_group_name, cluster_name, days, time, workernode_count,
no_wait=False):
from azure.mgmt.hdinsight.models import AutoscaleRecurrence, AutoscaleSchedule, AutoscaleTimeAndCapacity
from azure.mgmt.hdinsight.models import AutoscaleRecurrence, AutoscaleSchedule, AutoscaleTimeAndCapacity, \
AutoscaleConfigurationUpdateParameter, RoleName
cluster = client.get(resource_group_name, cluster_name)
autoscale_configuration = _extract_and_validate_autoscale_configuration(cluster)

Expand All @@ -688,13 +732,17 @@ def create_autoscale_condition(cmd, client, resource_group_name, cluster_name, d
autoscale_configuration.recurrence = AutoscaleRecurrence(
schedule=[condition]
)

return sdk_no_wait(no_wait, client.update_auto_scale_configuration, resource_group_name, cluster_name,
autoscale_configuration)
autoscale_configuration_update_parameter = AutoscaleConfigurationUpdateParameter(
autoscale=autoscale_configuration
)
return sdk_no_wait(no_wait, client.begin_update_auto_scale_configuration, resource_group_name, cluster_name,
RoleName.WORKERNODE, autoscale_configuration_update_parameter)


def update_autoscale_condition(cmd, client, resource_group_name, cluster_name, index, days=None, time=None,
workernode_count=None, no_wait=False):
from azure.mgmt.hdinsight.models import AutoscaleConfigurationUpdateParameter, RoleName

cluster = client.get(resource_group_name, cluster_name)
autoscale_configuration = _extract_and_validate_autoscale_configuration(cluster)
_validate_schedule_configuration(autoscale_configuration)
Expand All @@ -713,11 +761,16 @@ def update_autoscale_condition(cmd, client, resource_group_name, cluster_name, i
autoscale_configuration.recurrence.schedule[index].time_and_capacity.min_instance_count = workernode_count
autoscale_configuration.recurrence.schedule[index].time_and_capacity.max_instance_count = workernode_count

return sdk_no_wait(no_wait, client.update_auto_scale_configuration, resource_group_name, cluster_name,
autoscale_configuration)
autoscale_configuration_update_parameter = AutoscaleConfigurationUpdateParameter(
autoscale=autoscale_configuration
)
return sdk_no_wait(no_wait, client.begin_update_auto_scale_configuration, resource_group_name, cluster_name,
RoleName.WORKERNODE, autoscale_configuration_update_parameter)


def delete_autoscale_condition(cmd, client, resource_group_name, cluster_name, index, no_wait=False):
from azure.mgmt.hdinsight.models import AutoscaleConfigurationUpdateParameter, RoleName

cluster = client.get(resource_group_name, cluster_name)
autoscale_configuration = _extract_and_validate_autoscale_configuration(cluster)
_validate_schedule_configuration(autoscale_configuration)
Expand All @@ -728,8 +781,11 @@ def delete_autoscale_condition(cmd, client, resource_group_name, cluster_name, i
'If you want to disable autoscale please use `az hdinsight autoscale delete`.')
autoscale_configuration.recurrence.schedule = [autoscale_configuration.recurrence.schedule[i] for i in
range(conditions_count) if i not in index]
return sdk_no_wait(no_wait, client.update_auto_scale_configuration, resource_group_name, cluster_name,
autoscale_configuration)
autoscale_configuration_update_parameter = AutoscaleConfigurationUpdateParameter(
autoscale=autoscale_configuration
)
return sdk_no_wait(no_wait, client.begin_update_auto_scale_configuration, resource_group_name, cluster_name,
RoleName.WORKERNODE, autoscale_configuration_update_parameter)


def list_autoscale_condition(cmd, client, resource_group_name, cluster_name):
Expand Down
Loading