[rdbms] az postgres flexible-server migration: Add customer facing feature to migrate postgres db servers from Sterling to Meru platform

src/azure-cli/azure/cli/command_modules/rdbms/_helptext_pg.py

@@ -176,6 +176,58 @@
     text: az postgres flexible-server list --resource-group testGroup
 """
 
+helps['postgres flexible-server migration create'] = """
+type: command
+short-summary: Create a new migration workflow for a flexible server.
+examples:
+  - name: Start a migration workflow on the target server identified by the parameters. The configurations of the migration should be specified in the migrationConfig.json file.
+    text: az postgres flexible-server migration create --subscription-id xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx --resource-group testGroup --name testServer --body @"migrationConfig.json"
+"""
+
+helps['postgres flexible-server migration list'] = """
+type: command
+short-summary: List the migrations of a flexible server.
+examples:
+  - name: List the currently active migrations of a target flexible server.
+    text: az postgres flexible-server migration list --subscription-id xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx --resource-group testGroup --name testServer --filter Active"
+  - name: List all (Active/Completed) migrations of a target flexible server.
+    text: az postgres flexible-server migration list --subscription-id xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx --resource-group testGroup --name testServer --filter All"
+"""
+
+helps['postgres flexible-server migration show'] = """
+type: command
+short-summary: Get the details of a specific migration.
+examples:
+  - name: Get the details of a specific migration of a target flexible server.
+    text: az postgres flexible-server migration show --subscription-id xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx --resource-group testGroup --name testServer --migration-id xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx"
+"""
+
+helps['postgres flexible-server migration update'] = """
+type: command
+short-summary: Update a specific migration.
+examples:
+  - name: Allow the migration workflow to setup logical replication on the source. Note that this command will restart the source server.
+    text: az postgres flexible-server migration update --subscription-id xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx --resource-group testGroup --name testServer --migration-id xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx --setup-logical-replication"
+  - name: Specify the list of DBs to migrate. A minimum of 1 and a maximum of 8 DBs can be specified. You can migrate additional DBs concurrently using new migrations. Note that each additional DB affects the performance of the source server.
+    text: az postgres flexible-server migration update --subscription-id xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx --resource-group testGroup --name testServer --migration-id xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx --db1 dbName1 --db2 dbName2 --db3 dbName3"
+  - name: Allow the migration workflow to overwrite the DB on the target.
+    text: az postgres flexible-server migration update --subscription-id xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx --resource-group testGroup --name testServer --migration-id xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx --overwrite-dbs"
+  - name: Specify the start time for the data migration to start. This should be within 2 weeks from the current time.
+    text: az postgres flexible-server migration update --subscription-id xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx --resource-group testGroup --name testServer --migration-id xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx --start-time-utc '2021-12-28T17:06:03.4669999-07:00'"
+  - name: Start the data migration now, rather than wait for the migration window start time.
+    text: az postgres flexible-server migration update --subscription-id xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx --resource-group testGroup --name testServer --migration-id xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx --initiate-data-migration"
+  - name: Cutover the data migration. After this is complete, subsequent updates to the source DB will not be migrated to the target.
+    text: az postgres flexible-server migration update --subscription-id xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx --resource-group testGroup --name testServer --migration-id xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx --cutover"
+"""
+
+helps['postgres flexible-server migration delete'] = """
+type: command
+short-summary: Delete a specific migration.
+examples:
+  - name: Cancel/delete the migration workflow. The migration workflows can be canceled/deleted at any point.
+    text: az postgres flexible-server migration delete --subscription-id xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx --resource-group testGroup --name testServer --migration-id xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx"
+"""
+
 helps['postgres flexible-server parameter'] = """
 type: group
 short-summary: Commands for managing server parameter values for flexible server.

src/azure-cli/azure/cli/command_modules/rdbms/_params.py

@@ -228,6 +228,14 @@ def _flexible_server_params(command_group):
                                                help="Name of the server. The name can contain only lowercase letters, numbers, and the hyphen (-) character. Minimum 3 characters and maximum 63 characters.",
                                                local_context_attribute=LocalContextAttribute(name='server_name', actions=[LocalContextAction.SET, LocalContextAction.GET], scopes=['{} flexible-server'.format(command_group)]))
 
+        subscription_arg_type = CLIArgumentType(metavar='NAME',
+                                                help="ID of subscription. You can configure the default subscription using `az account set -s NAME_OR_ID`",
+                                                local_context_attribute=LocalContextAttribute(name='subscription_id', actions=[LocalContextAction.SET, LocalContextAction.GET], scopes=['{} flexible-server'.format(command_group)]))
+
+        migration_id_arg_type = CLIArgumentType(metavar='NAME',
+                                                help="ID of the migration.",
+                                                local_context_attribute=LocalContextAttribute(name='migration_id', actions=[LocalContextAction.SET, LocalContextAction.GET], scopes=['{} flexible-server'.format(command_group)]))
+
         administrator_login_setter_arg_type = CLIArgumentType(metavar='NAME',
                                                               local_context_attribute=LocalContextAttribute(name='administrator_login', actions=[LocalContextAction.SET], scopes=['{} flexible-server'.format(command_group)]))
 
@@ -400,6 +408,8 @@ def _flexible_server_params(command_group):
                 else:
                     c.argument('server_name', id_part='name', options_list=['--name', '-n'], arg_type=server_name_arg_type)
 
+        handle_migration_parameters(command_group, server_name_arg_type, subscription_arg_type, migration_id_arg_type)
+
         for scope in ['create', 'delete', 'show', 'update']:
             argument_context_string = '{} flexible-server firewall-rule {}'.format(command_group, scope)
             with self.argument_context(argument_context_string) as c:
@@ -465,5 +475,53 @@ def _flexible_server_params(command_group):
         with self.argument_context('{} flexible-server replica stop-replication'.format(command_group)) as c:
             c.argument('server_name', options_list=['--name', '-n'], help='Name of the replica server.')
 
+    def handle_migration_parameters(command_group, server_name_arg_type, subscription_arg_type, migration_id_arg_type):
+        for scope in ['create', 'show', 'list', 'update', 'delete']:
+            argument_context_string = '{} flexible-server migration {}'.format(command_group, scope)
+            with self.argument_context(argument_context_string) as c:
+                c.argument('subscription_id', arg_type=subscription_arg_type, options_list=['--subscription-id'],
+                           help='Subscription ID of the migration target server.')
+                c.argument('resource_group_name', arg_type=resource_group_name_type,
+                           help='Resource Group Name of the migration target server.')
+                c.argument('server_name', id_part='name', options_list=['--name', '-n'], arg_type=server_name_arg_type,
+                           help='Migration target server name.')
+                if scope == "create":
+                    c.argument('body', options_list=['--body', '-b'],
+                               help='Request body. Use @{file} to load from a file. For quoting issues in different terminals, '
+                               'see https://github.com/Azure/azure-cli/blob/dev/doc/use_cli_effectively.md#quoting-issues')
+                    c.argument('migration_id', arg_type=migration_id_arg_type, options_list=['--migration-id'],
+                               help='Name or ID of the migration.')
+                elif scope == "show":
+                    c.argument('migration_id', arg_type=migration_id_arg_type, options_list=['--migration-id'],
+                               help='Name or ID of the migration.')
+                elif scope == "list":
+                    c.argument('migration_filter', options_list=['--filter'], required=False,
+                               help='Indicates whether all the migrations or just the Active migrations are returned. Active is the default. Valid values are: Active, All.')
+                elif scope == "update":
+                    c.argument('migration_id', arg_type=migration_id_arg_type, options_list=['--migration-id'],
+                               help='Name or ID of the migration.')
+                    c.argument('setup-logical_replication', options_list=['--setup-logical-replication'], action='store_true', required=False,
+                               help='Allows the migration workflow to setup logical replication on the source. Note that this command will restart the source server.')
+                    c.argument('db1', options_list=['--db1', '--db'], required=False,
+                               help='Specifies the first DB to migrate. A minimum of 1 and a maximum of 8 DBs can be specified using --db1, --db2, --db3... You can migrate additional DBs concurrently using new migrations. Note that each additional DB affects the performance of the source server.')
+                    c.argument('db2', options_list=['--db2'], required=False)
+                    c.argument('db3', options_list=['--db3'], required=False)
+                    c.argument('db4', options_list=['--db4'], required=False)
+                    c.argument('db5', options_list=['--db5'], required=False)
+                    c.argument('db6', options_list=['--db6'], required=False)
+                    c.argument('db7', options_list=['--db7'], required=False)
+                    c.argument('db8', options_list=['--db8'], required=False)
+                    c.argument('overwrite_dbs', options_list=['--overwrite-dbs'], action='store_true', required=False,
+                               help='Allows the migration workflow to overwrite the DB on the target.')
+                    # c.argument('start_time_utc', options_list=['--start-time-utc'], required=False,
+                    #            help='Specifies the start time for the data migration to start. This should be within 2 weeks from the current time.')
+                    # c.argument('initiate_data_migration', options_list=['--initiate-data-migration'], action='store_true', required=False,
+                    #            help='Starts the data migration now, rather than wait for the migration window start time.')
+                    c.argument('cutover', options_list=['--cutover'], action='store_true', required=False,
+                               help='Cuts over the data migration. After this is complete, subsequent updates to the source DB will not be migrated to the target.')
+                elif scope == "delete":
+                    c.argument('migration_id', arg_type=migration_id_arg_type, options_list=['--migration-id'],
+                               help='Name or ID of the migration.')
+
     _flexible_server_params('postgres')
     _flexible_server_params('mysql')

src/azure-cli/azure/cli/command_modules/rdbms/flexible_server_commands.py

@@ -128,6 +128,16 @@ def load_flexibleserver_command_table(self, _):
                                  custom_func_name='flexible_firewall_rule_update_custom_func',
                                  custom_func_type=flexible_server_custom_common)
 
+    with self.command_group('postgres flexible-server migration', postgres_flexible_firewall_rule_sdk,
+                            custom_command_type=flexible_servers_custom_postgres,
+                            client_factory=cf_postgres_flexible_firewall_rules,
+                            is_preview=True) as g:
+        g.custom_command('create', 'migration_create_func', custom_command_type=flexible_server_custom_common)
+        g.custom_command('show', 'migration_show_func', custom_command_type=flexible_server_custom_common)
+        g.custom_command('list', 'migration_list_func', custom_command_type=flexible_server_custom_common)
+        g.custom_command('update', 'migration_update_func', custom_command_type=flexible_server_custom_common)
+        g.custom_command('delete', 'migration_delete_func', custom_command_type=flexible_server_custom_common)
+
     with self.command_group('postgres flexible-server parameter', postgres_flexible_config_sdk,
                             custom_command_type=flexible_servers_custom_postgres,
                             client_factory=cf_postgres_flexible_config,

src/azure-cli/azure/cli/command_modules/rdbms/flexible_server_custom_common.py

@@ -4,9 +4,14 @@
 # --------------------------------------------------------------------------------------------
 
 # pylint: disable=unused-argument, line-too-long
+
+import uuid
 from datetime import datetime
 from knack.log import get_logger
 from knack.util import CLIError
+from azure.cli.core.azclierror import RequiredArgumentMissingError
+from azure.cli.core.azclierror import MutuallyExclusiveArgumentError
+from azure.cli.core.util import send_raw_request
 from azure.cli.core.util import user_confirmation
 
 logger = get_logger(__name__)
@@ -71,6 +76,104 @@ def firewall_rule_create_func(client, resource_group_name, server_name, firewall
         parameters)
 
 
+def migration_create_func(cmd, client, subscription_id, resource_group_name, server_name, body, migration_id=None):
+
+    if migration_id is None:
+        # Convert a UUID to a string of hex digits in standard form
+        migration_id = str(uuid.uuid4())
+
+    r = send_raw_request(cmd.cli_ctx, "put", "https://management.azure.com/subscriptions/{}/resourceGroups/{}/providers/Microsoft.DBforPostgreSQL/flexibleServers/{}/migrations/{}?api-version=2020-02-14-privatepreview".format(subscription_id, resource_group_name, server_name, migration_id), None, None, body)
+
+    return r.json()
+
+
+def migration_show_func(cmd, client, subscription_id, resource_group_name, server_name, migration_id, level="Default"):
+
+    r = send_raw_request(cmd.cli_ctx, "get", "https://management.azure.com/subscriptions/{}/resourceGroups/{}/providers/Microsoft.DBforPostgreSQL/flexibleServers/{}/migrations/{}?level={}&api-version=2020-02-14-privatepreview".format(subscription_id, resource_group_name, server_name, migration_id, level))
+
+    return r.json()
+
+
+def migration_list_func(cmd, client, subscription_id, resource_group_name, server_name, migration_filter="Active"):
+
+    r = send_raw_request(cmd.cli_ctx, "get", "https://management.azure.com/subscriptions/{}/resourceGroups/{}/providers/Microsoft.DBforPostgreSQL/flexibleServers/{}/migrations?migrationListFilter={}&api-version=2020-02-14-privatepreview".format(subscription_id, resource_group_name, server_name, migration_filter))
+
+    return r.json()
+
+
+def migration_update_func(cmd, client, subscription_id, resource_group_name, server_name, migration_id, setup_logical_replication=None, db1=None, db2=None, db3=None, db4=None, db5=None, db6=None, db7=None, db8=None, overwrite_dbs=None, start_time_utc=None, initiate_data_migration=None, cutover=None, cancel=None):
+
+    operationSpecified = False
+    if setup_logical_replication is True:
+        operationSpecified = True
+        body = "{\"properties\": {\"setupLogicalReplicationOnSourceDBIfNeeded\": \"true\"} }"
+
+    db_names = None
+    db_names = db_names_concat_func(db_names, db1)
+    db_names = db_names_concat_func(db_names, db2)
+    db_names = db_names_concat_func(db_names, db3)
+    db_names = db_names_concat_func(db_names, db4)
+    db_names = db_names_concat_func(db_names, db5)
+    db_names = db_names_concat_func(db_names, db6)
+    db_names = db_names_concat_func(db_names, db7)
+    db_names = db_names_concat_func(db_names, db8)
+
+    if db_names is not None:
+        if operationSpecified is True:
+            raise MutuallyExclusiveArgumentError("Incorrect Usage: Can only specify one update operation.")
+        operationSpecified = True
+        prefix = "{ \"properties\": { \"dBsToMigrate\": ["
+        suffix = "] } }"
+        body = prefix + db_names + suffix
+
+    if overwrite_dbs is True:
+        if operationSpecified is True:
+            raise MutuallyExclusiveArgumentError("Incorrect Usage: Can only specify one update operation.")
+        operationSpecified = True
+        body = "{\"properties\": {\"overwriteDBsInTarget\": \"true\"} }"
+
+    # if start_time_utc is not None:
+    #     if operationSpecified is True:
+    #         raise MutuallyExclusiveArgumentError("Incorrect Usage: Can only specify one update operation.")
+    #     operationSpecified = True
+    #     body = "{\"properties\": {\"MigrationWindowStartTimeInUtc\": \"{}\"} }".format(start_time_utc)
+
+    # if initiate_data_migration is True:
+    #     if operationSpecified is True:
+    #         raise MutuallyExclusiveArgumentError("Incorrect Usage: Can only specify one update operation.")
+    #     operationSpecified = True
+    #     body = "{\"properties\": {\"startDataMigration\": \"true\"} }"
+
+    if cutover is True:
+        if operationSpecified is True:
+            raise MutuallyExclusiveArgumentError("Incorrect Usage: Can only specify one update operation.")
+        operationSpecified = True
+        body = "{\"properties\": {\"triggerCutover\": \"true\"} }"
+
+    if operationSpecified is False:
+        raise RequiredArgumentMissingError("Incorrect Usage: Atleast one update operation needs to be specified.")
+
+    send_raw_request(cmd.cli_ctx, "patch", "https://management.azure.com/subscriptions/{}/resourceGroups/{}/providers/Microsoft.DBforPostgreSQL/flexibleServers/{}/migrations/{}?api-version=2020-02-14-privatepreview".format(subscription_id, resource_group_name, server_name, migration_id), None, None, body)
+
+    return migration_id
+
+
+def migration_delete_func(cmd, client, subscription_id, resource_group_name, server_name, migration_id):
+
+    r = send_raw_request(cmd.cli_ctx, "delete", "https://management.azure.com/subscriptions/{}/resourceGroups/{}/providers/Microsoft.DBforPostgreSQL/flexibleServers/{}/migrations/{}?api-version=2020-02-14-privatepreview".format(subscription_id, resource_group_name, server_name, migration_id))
+
+    return r.json()
+
+
+def db_names_concat_func(current_db_names, new_db_name):
+
+    if new_db_name is not None:
+        db = "\"{}\"".format(new_db_name)
+        current_db_names = db if current_db_names is None else current_db_names + ", " + db
+
+    return current_db_names
+
+
 def firewall_rule_delete_func(client, resource_group_name, server_name, firewall_rule_name, yes=None):
     result = None
     if not yes:

src/azure-cli/azure/cli/command_modules/rdbms/tests/latest/migrationPublic.json

@@ -0,0 +1,30 @@
+{
+    "properties": {
+        "SourceDBServerResourceId": "subscriptions/6a37df99-a9de-48c4-91e5-7e6ab00b2362/resourceGroups/raganesa-s-s-pg-1/providers/Microsoft.DBforPostgreSQL/servers/raganesa-s-s-pg-1",
+        "SecretParameters":
+        {
+            "AdminCredentials":
+            {
+                "SourceServerPassword": "xxxx",
+                "TargetServerPassword": "xxxx"
+            },
+            "AADApp":
+            {
+                "ClientId": "xxxxxx",
+                "TenantId": "72f988bf-86f1-41af-91ab-2d7cd011db47",
+                "AadSecret": "xxxxx"
+            }
+        },
+        "DBsToMigrate": [
+            "dvdrental"
+        ],
+        "MigrationResourceGroup":
+        {
+            "ResourceId": "subscriptions/6a37df99-a9de-48c4-91e5-7e6ab00b2362/resourceGroups/raganesa-DMSBuddy-Demo",
+            "Location": "West US 2"
+        },
+        "SetupLogicalReplicationOnSourceDBIfNeeded": "true",
+        "OverwriteDBsinTarget": "true",
+        "TriggerCutover": "true"
+    }
+}