Merge branch 'k8s-extension/public' into huayang/fe-migration

scripts/ci/avail-ext-doc/list-template.md

@@ -1,6 +1,6 @@
 ---
-title: Available extensions for the Azure CLI
-description: A complete list of the officially supported extensions for the Azure CLI.
+title: List of available Azure CLI extensions | Microsoft Docs
+description: A complete list of officially supported Azure Command-Line Interface (CLI) extensions that are provided and maintained by Microsoft.
 author: haroldrandom
 ms.author: jianzen
 manager: yonzhan,yungezz
@@ -10,18 +10,19 @@ ms.prod: azure
 ms.technology: azure-cli
 ms.devlang: azure-cli
 ms.custom: devx-track-azurecli
+keywords: az extension, azure cli extensions, azure extensions
 ---
 
-# Available extensions for the Azure CLI
+# Available Azure CLI extensions
 
-This article is a complete list of the available extensions for the Azure CLI which are supported by Microsoft.
-
-The list of extensions is also available from the CLI. To get it, run [az extension list-available](/cli/azure/extension#az_extension_list_available):
+This article is a complete list of the available extensions for the Azure CLI which are supported by Microsoft.  The list of extensions is also available from the CLI. To get it, run [az extension list-available](/cli/azure/extension#az_extension_list_available):
 
 ```azurecli-interactive
 az extension list-available --output table
 ```
 
+You will be prompted to install an extension on first use.  
+
 | Extension | Required Minimum CLI Version | Description | Status | Release Notes |
 |----|-----------------|-------------|---------|---------------|{% for extension in extensions %}
 |[{{ extension.name }}]({{ extension.project_url }}) | {{ extension.min_cli_core_version }} | {{ extension.desc }} | {{ extension.status }} | [{{extension.version}}]({{extension.history}}) |{% endfor %}

src/aks-preview/HISTORY.md

@@ -2,6 +2,15 @@
 
 Release History
 ===============
+
+0.5.25
++++++
+* Add support for http proxy
+
+0.5.24
++++++
+* * Add "--aks-custom-headers" for "az aks nodepool upgrade"
+
 0.5.23
 +++++
 * Fix issue that `maintenanceconfiguration add` subcommand cannot work

src/aks-preview/azcli_aks_live_test/scripts/test_cli_live.sh

@@ -24,6 +24,12 @@ source azEnv/bin/activate
 source ./scripts/setup_venv.sh
 removeAKSPreview
 
+# login before the recording test to avoid the newly added test case does not include a corresponding
+# recording file and fall back to live mode, otherwise it will prompt `az login`.
+az login --service-principal -u "${AZCLI_ALT_CLIENT_ID}" -p "${AZCLI_ALT_CLIENT_SECRET}" -t "${TENANT_ID}"
+az account set -s "${AZCLI_ALT_SUBSCRIPTION_ID}"
+az account show
+
 # prepare running options
 # base options
 base_options="-c --no-exitfirst --report-path ./reports --reruns 3 --capture=sys"
@@ -50,35 +56,56 @@ if [[ -n ${CLI_TEST_COVERAGE} ]]; then
 fi
 
 # recording test
-if [[ ${TEST_MODE} == "record" || ${TEST_MODE} == "all" ]]; then
+if [[ ${TEST_MODE} == "recording" || ${TEST_MODE} == "all" ]]; then
     echo "Test in recording mode!"
     coverage_options=" -cc test_aks_commands.AzureKubernetesServiceScenarioTest"
     recording_options="${base_options}${coverage_options}"
     recording_options+=" --json-report-file=cli_recording_report.json"
     recording_options+=" --xml-file=cli_recording_result.xml"
-    echo "recording options: ${recording_options}"
-    azaks ${recording_options}
+    echo "Recording test options: ${recording_options}"
+    test_result=0
+    azaks ${recording_options} || test_result=$?
+    if [[ ${test_result} -ne 0 ]]; then
+        echo "Recording test failed!"
+        echo "Please refer to this wiki (https://dev.azure.com/msazure/CloudNativeCompute/_wiki/wikis/CloudNativeCompute.wiki/156735/AZCLI-AKS-Live-Unit-Test-Pipeline) for troubleshooting guidelines."
+        exit ${test_result}
+    else
+        echo -e "Recording test passed!\n\n"
+    fi
 fi
 
 # live test
 if [[ ${TEST_MODE} == "live" || ${TEST_MODE} == "all" ]]; then
     echo "Test in live mode!"
-    az login --service-principal -u "${AZCLI_ALT_CLIENT_ID}" -p "${AZCLI_ALT_CLIENT_SECRET}" -t "${TENANT_ID}"
-    az account set -s "${AZCLI_ALT_SUBSCRIPTION_ID}"
-    az account show
     live_options="${base_options}${filter_options}"
     live_options+=" -l --json-report-file=cli_live_report.json"
     live_options+=" --xml-file=cli_live_result.xml"
-    echo "live options: ${live_options}"
-    azaks ${live_options}
+    echo "Live test options: ${live_options}"
+    test_result=0
+    azaks ${live_options} || test_result=$?
+    if [[ ${test_result} -ne 0 ]]; then
+        echo "Live test failed!"
+        echo "Please refer to this wiki (https://dev.azure.com/msazure/CloudNativeCompute/_wiki/wikis/CloudNativeCompute.wiki/156735/AZCLI-AKS-Live-Unit-Test-Pipeline) for troubleshooting guidelines."
+        exit ${test_result}
+    else
+        echo -e "Live test passed!\n\n"
+    fi
 fi
 
-# re-recording test
+# live-replay test
 if [[ ${TEST_MODE} == "live" || ${TEST_MODE} == "all" ]]; then
-    echo "Test in re-recording mode(after live test)!"
-    re_recording_options="${base_options}${filter_options}"
-    re_recording_options+=" --json-report-file=cli_re_recording_report.json"
-    re_recording_options+=" --xml-file=cli_re_recording_result.xml"
-    echo "re-recording options: ${re_recording_options}"
-    azaks ${re_recording_options}
+    echo "Test in live-replay mode!"
+    live_replay_options="${base_options}${filter_options}"
+    live_replay_options+=" --json-report-file=cli_live_replay_report.json"
+    live_replay_options+=" --xml-file=cli_live_replay_result.xml"
+    echo "Live-replay test options: ${live_replay_options}"
+    test_result=0
+    azaks ${live_replay_options} || test_result=$?
+    if [[ ${test_result} -ne 0 ]]; then
+        echo "Live-replay test failed!"
+        echo "Please refer to this wiki (https://dev.azure.com/msazure/CloudNativeCompute/_wiki/wikis/CloudNativeCompute.wiki/156735/AZCLI-AKS-Live-Unit-Test-Pipeline) for troubleshooting guidelines."
+        exit ${test_result}
+    else
+        echo -e "Live-replay test passed!\n\n"
+    fi
 fi

src/aks-preview/azcli_aks_live_test/scripts/test_cli_unit.sh

@@ -36,5 +36,6 @@ mkdir -p reports/ && cp ${acs_base_dir}/coverage_acs.json reports/
 
 if [[ ${acs_unit_test_failed} == "true" ]]; then
     echo "Unit test failed!"
+    echo "Please refer to this wiki (https://dev.azure.com/msazure/CloudNativeCompute/_wiki/wikis/CloudNativeCompute.wiki/156735/AZCLI-AKS-Live-Unit-Test-Pipeline) for troubleshooting guidelines."
     exit 1
 fi

src/aks-preview/azcli_aks_live_test/scripts/test_ext_live.sh

@@ -23,6 +23,12 @@ source azEnv/bin/activate
 # setup aks-preview
 ./scripts/setup_venv.sh setup-akspreview
 
+# login before the recording test to avoid the newly added test case does not include a corresponding
+# recording file and fall back to live mode, otherwise it will prompt `az login`.
+az login --service-principal -u "${AZCLI_ALT_CLIENT_ID}" -p "${AZCLI_ALT_CLIENT_SECRET}" -t "${TENANT_ID}"
+az account set -s "${AZCLI_ALT_SUBSCRIPTION_ID}"
+az account show
+
 # prepare running options
 # base options
 base_options="-e --no-exitfirst --report-path ./reports --reruns 3 --capture=sys"
@@ -49,35 +55,56 @@ if [[ -n ${EXT_TEST_COVERAGE} ]]; then
 fi
 
 # recording test
-if [[ ${TEST_MODE} == "record" || ${TEST_MODE} == "all" ]]; then
+if [[ ${TEST_MODE} == "recording" || ${TEST_MODE} == "all" ]]; then
     echo "Test in recording mode!"
     coverage_options=" -ec test_aks_commands.AzureKubernetesServiceScenarioTest"
     recording_options="${base_options}${coverage_options}"
     recording_options+=" --json-report-file=ext_recording_report.json"
     recording_options+=" --xml-file=ext_recording_result.xml"
-    echo "recording options: ${recording_options}"
-    azaks ${recording_options}
+    echo "Recording test options: ${recording_options}"
+    test_result=0
+    azaks ${recording_options} || test_result=$?
+    if [[ ${test_result} -ne 0 ]]; then
+        echo "Recording test failed!"
+        echo "Please refer to this wiki (https://dev.azure.com/msazure/CloudNativeCompute/_wiki/wikis/CloudNativeCompute.wiki/156735/AZCLI-AKS-Live-Unit-Test-Pipeline) for troubleshooting guidelines."
+        exit ${test_result}
+    else
+        echo -e "Recording test passed!\n\n"
+    fi
 fi
 
 # live test
 if [[ ${TEST_MODE} == "live" || ${TEST_MODE} == "all" ]]; then
     echo "Test in live mode!"
-    az login --service-principal -u "${AZCLI_ALT_CLIENT_ID}" -p "${AZCLI_ALT_CLIENT_SECRET}" -t "${TENANT_ID}"
-    az account set -s "${AZCLI_ALT_SUBSCRIPTION_ID}"
-    az account show
     live_options="${base_options}${filter_options}"
     live_options+=" -l --json-report-file=ext_live_report.json"
     live_options+=" --xml-file=ext_live_result.xml"
-    echo "live options: ${live_options}"
-    azaks ${live_options}
+    echo "Live test options: ${live_options}"
+    test_result=0
+    azaks ${live_options} || test_result=$?
+    if [[ ${test_result} -ne 0 ]]; then
+        echo "Live test failed!"
+        echo "Please refer to this wiki (https://dev.azure.com/msazure/CloudNativeCompute/_wiki/wikis/CloudNativeCompute.wiki/156735/AZCLI-AKS-Live-Unit-Test-Pipeline) for troubleshooting guidelines."
+        exit ${test_result}
+    else
+        echo -e "Live test passed!\n\n"
+    fi
 fi
 
-# re-recording test
+# live-replay test
 if [[ ${TEST_MODE} == "live" || ${TEST_MODE} == "all" ]]; then
-    echo "Test in re-recording mode(after live test)!"
-    re_recording_options="${base_options}${filter_options}"
-    re_recording_options+=" --json-report-file=ext_re_recording_report.json"
-    re_recording_options+=" --xml-file=ext_re_recording_result.xml"
-    echo "re-recording options: ${re_recording_options}"
-    azaks ${re_recording_options}
+    echo "Test in live-replay mode!"
+    live_replay_options="${base_options}${filter_options}"
+    live_replay_options+=" --json-report-file=ext_live_replay_report.json"
+    live_replay_options+=" --xml-file=ext_live_replay_result.xml"
+    echo "Live-replay test options: ${live_replay_options}"
+    test_result=0
+    azaks ${live_replay_options} || test_result=$?
+    if [[ ${test_result} -ne 0 ]]; then
+        echo "Live-replay test failed!"
+        echo "Please refer to this wiki (https://dev.azure.com/msazure/CloudNativeCompute/_wiki/wikis/CloudNativeCompute.wiki/156735/AZCLI-AKS-Live-Unit-Test-Pipeline) for troubleshooting guidelines."
+        exit ${test_result}
+    else
+        echo -e "Live-replay test passed!\n\n"
+    fi
 fi

src/aks-preview/azcli_aks_live_test/scripts/test_ext_unit.sh

@@ -35,5 +35,6 @@ mkdir -p reports/ && cp ${aks_preview_base_dir}/coverage_azext_aks_preview.json
 
 if [[ ${azext_aks_preview_unit_test_failed} == "true" ]]; then
     echo "Unit test failed!"
+    echo "Please refer to this wiki (https://dev.azure.com/msazure/CloudNativeCompute/_wiki/wikis/CloudNativeCompute.wiki/156735/AZCLI-AKS-Live-Unit-Test-Pipeline) for troubleshooting guidelines."
     exit 1
 fi

src/aks-preview/azcli_aks_live_test/vsts-azcli-aks-live-test.yaml

@@ -18,6 +18,8 @@ variables:
     value: "azure-cli-extensions"
   - name: LIVE_TEST_BASE_DIR
     value: "azure-cli-extensions/src/aks-preview/azcli_aks_live_test"
+  - name: ACS_BASE_DIR
+    value: "azure-cli/src/azure-cli/azure/cli/command_modules/acs"
   - name: AKS_PREVIEW_BASE_DIR
     value: "azure-cli-extensions/src/aks-preview/azext_aks_preview"
 
@@ -70,16 +72,23 @@ jobs:
         targetFolder: $(Build.ArtifactStagingDirectory)
       condition: succeededOrFailed()
       displayName: "Copy Reports & Logs"
-    - bash: mkdir -p $(Build.ArtifactStagingDirectory)/recordings
+    - bash: mkdir -p $(Build.ArtifactStagingDirectory)/cli-recordings $(Build.ArtifactStagingDirectory)/ext-recordings
       condition: succeededOrFailed()
       displayName: "Create Dir for Recordings"
+    - task: CopyFiles@2
+      inputs:
+        contents: '$(ACS_BASE_DIR)/tests/latest/recordings/**'
+        flattenFolders: true
+        targetFolder: $(Build.ArtifactStagingDirectory)/cli-recordings
+      condition: and(succeededOrFailed(), in(variables['COVERAGE'], 'cli', 'all'))
+      displayName: "Copy CLI Recordings"
     - task: CopyFiles@2
       inputs:
         contents: '$(AKS_PREVIEW_BASE_DIR)/tests/latest/recordings/**'
         flattenFolders: true
-        targetFolder: $(Build.ArtifactStagingDirectory)/recordings
-      condition: succeededOrFailed()
-      displayName: "Copy Recordings"
+        targetFolder: $(Build.ArtifactStagingDirectory)/ext-recordings
+      condition: and(succeededOrFailed(), in(variables['COVERAGE'], 'ext', 'all'))
+      displayName: "Copy EXT Recordings"
     - task: PublishBuildArtifacts@1
       inputs:
         pathToPublish: $(Build.ArtifactStagingDirectory)

src/aks-preview/azext_aks_preview/_help.py

@@ -317,6 +317,9 @@
         - name: --linux-os-config
           type: string
           short-summary: OS configurations for Linux agent nodes.
+        - name: --http-proxy-config
+          type: string
+          short-summary: Http Proxy configuration for this cluster.
         - name: --enable-pod-identity
           type: bool
           short-summary: (PREVIEW) Enable pod identity addon.
@@ -967,6 +970,9 @@
         - name: --max-surge
           type: string
           short-summary: Extra nodes used to speed upgrade. When specified, it represents the number or percent used, eg. 5 or 33%
+        - name: --aks-custom-headers
+          type: string
+          short-summary: Send custom headers. When specified, format should be Key1=Value1,Key2=Value2
 """
 
 helps['aks nodepool update'] = """

src/aks-preview/azext_aks_preview/_params.py

@@ -121,6 +121,7 @@ def load_arguments(self, _):
         c.argument('auto_upgrade_channel', arg_type=get_enum_type([CONST_RAPID_UPGRADE_CHANNEL, CONST_STABLE_UPGRADE_CHANNEL, CONST_PATCH_UPGRADE_CHANNEL, CONST_NODE_IMAGE_UPGRADE_CHANNEL, CONST_NONE_UPGRADE_CHANNEL]))
         c.argument('kubelet_config', type=str)
         c.argument('linux_os_config', type=str)
+        c.argument('http_proxy_config', options_list=['--http-proxy-config'], type=str)
         c.argument('enable_pod_identity', action='store_true')
         c.argument('appgw_name', options_list=['--appgw-name'], arg_group='Application Gateway')
         c.argument('appgw_subnet_prefix', options_list=['--appgw-subnet-prefix'], arg_group='Application Gateway', deprecate_info=c.deprecate(redirect='--appgw-subnet-cidr', hide=True))
@@ -238,6 +239,7 @@ def load_arguments(self, _):
 
     with self.argument_context('aks nodepool upgrade') as c:
         c.argument('max_surge', type=str, validator=validate_max_surge)
+        c.argument('aks_custom_headers')
 
     with self.argument_context('aks nodepool update') as c:
         c.argument('enable_cluster_autoscaler', options_list=["--enable-cluster-autoscaler", "-e"], action='store_true')

src/aks-preview/azext_aks_preview/custom.py

@@ -77,6 +77,7 @@
                                                                       ManagedClusterAutoUpgradeProfile,
                                                                       KubeletConfig,
                                                                       LinuxOSConfig,
+                                                                      ManagedClusterHTTPProxyConfig,
                                                                       SysctlConfig,
                                                                       ManagedClusterPodIdentityProfile,
                                                                       ManagedClusterPodIdentity,
@@ -1028,6 +1029,7 @@ def aks_create(cmd,     # pylint: disable=too-many-locals,too-many-statements,to
                enable_sgxquotehelper=False,
                kubelet_config=None,
                linux_os_config=None,
+               http_proxy_config=None,
                assign_identity=None,
                auto_upgrade_channel=None,
                enable_pod_identity=False,
@@ -1440,6 +1442,9 @@ def aks_create(cmd,     # pylint: disable=too-many-locals,too-many-statements,to
                 "--fqdn-subdomain should only be used for private cluster with custom private dns zone")
         mc.fqdn_subdomain = fqdn_subdomain
 
+    if http_proxy_config:
+        mc.http_proxy_config = _get_http_proxy_config(http_proxy_config)
+
     if uptime_sla:
         mc.sku = ManagedClusterSKU(
             name="Basic",
@@ -3308,7 +3313,8 @@ def aks_agentpool_upgrade(cmd,  # pylint: disable=unused-argument
                           kubernetes_version='',
                           no_wait=False,
                           node_image_only=False,
-                          max_surge=None):
+                          max_surge=None,
+                          aks_custom_headers=None):
     if kubernetes_version != '' and node_image_only:
         raise CLIError('Conflicting flags. Upgrading the Kubernetes version will also upgrade node image version.'
                        'If you only want to upgrade the node version please use the "--node-image-only" option only.')
@@ -3329,7 +3335,9 @@ def aks_agentpool_upgrade(cmd,  # pylint: disable=unused-argument
     if max_surge:
         instance.upgrade_settings.max_surge = max_surge
 
-    return sdk_no_wait(no_wait, client.begin_create_or_update, resource_group_name, cluster_name, nodepool_name, instance)
+    headers = get_aks_custom_headers(aks_custom_headers)
+
+    return sdk_no_wait(no_wait, client.begin_create_or_update, resource_group_name, cluster_name, nodepool_name, instance, headers=headers)
 
 
 def aks_agentpool_get_upgrade_profile(cmd,   # pylint: disable=unused-argument
@@ -4069,6 +4077,8 @@ def _is_msi_cluster(managed_cluster):
 
 
 def _get_kubelet_config(file_path):
+    if not os.path.isfile(file_path):
+        raise CLIError("{} is not valid file, or not accessable.".format(file_path))
     kubelet_config = get_file_json(file_path)
     if not isinstance(kubelet_config, dict):
         raise CLIError(
@@ -4097,6 +4107,8 @@ def _get_kubelet_config(file_path):
 
 
 def _get_linux_os_config(file_path):
+    if not os.path.isfile(file_path):
+        raise CLIError("{} is not valid file, or not accessable.".format(file_path))
     os_config = get_file_json(file_path)
     if not isinstance(os_config, dict):
         raise CLIError(
@@ -4168,6 +4180,22 @@ def _get_linux_os_config(file_path):
     return config_object
 
 
+def _get_http_proxy_config(file_path):
+    if not os.path.isfile(file_path):
+        raise CLIError("{} is not valid file, or not accessable.".format(file_path))
+    hp_config = get_file_json(file_path)
+    if not isinstance(hp_config, dict):
+        raise CLIError(
+            "Error reading Http Proxy Config at {}. Please see https://aka.ms/HttpProxyConfig for correct format.".format(file_path))
+    config_object = ManagedClusterHTTPProxyConfig()
+    config_object.http_proxy = hp_config.get("httpProxy", None)
+    config_object.https_proxy = hp_config.get("httpsProxy", None)
+    config_object.no_proxy = hp_config.get("noProxy", None)
+    config_object.trusted_ca = hp_config.get("trustedCa", None)
+
+    return config_object
+
+
 def _is_pod_identity_addon_enabled(instance):
     if not instance:
         return False