Integrate Availability Zone from private repository (#4523)

* Initial support for VM and single zone VMSS (#14)

* wire up the initial zone support work

* add output

* add tests

* disable package verifications

* use a private copy of network sdk with zone support (#15)

* Support for zoned public IP. Make global zone_type and zones_type. (#16)

* Support for zoned public IP. Make global zone_type and zones_type.

* Code review feedback.

* apply api version range on vm zone test (#19)

* show zone in the table output (#20)

* install: support to build out a msi installer from local sources (#17)

* doc:add command examples using availability zones (#21)

* Installer: build debian bundle from a local clone (#28)

* skip a few expeneive travis builds

* test: update tests to work with new azure-mgmt-compute with zone support

* network: support zone in network lb create (#37)

* VNet peering examples. (#51)

* Add BrazilUS and Dogfood cloud config files. (#50)

* undo all changes specifically for private repository

* fix help per review feedback

* fix lint error

* use newer nrp sdk version

packaged_releases/debian/debian_build.sh

@@ -7,7 +7,6 @@
 set -ex
 
 : "${CLI_VERSION:?CLI_VERSION environment variable not set.}"
-: "${CLI_DOWNLOAD_SHA256:?CLI_DOWNLOAD_SHA256 environment variable not set.}"
 : "${BUILD_ARTIFACT_DIR:?BUILD_ARTIFACT_DIR environment variable not set.}"
 
 if [ -z "$1" ]
@@ -16,24 +15,47 @@ if [ -z "$1" ]
     exit 1
 fi
 
+local_repo=$2 
+if [ -z "$local_repo" ]
+  then
+    : "${CLI_DOWNLOAD_SHA256:?CLI_DOWNLOAD_SHA256 environment variable not set.}"
+fi
+
 sudo apt-get update
 
 debian_directory_creator=$1
 
 # Install dependencies for the build
 sudo apt-get install -y libssl-dev libffi-dev python3-dev debhelper
 # Download, Extract, Patch, Build CLI
+tmp_pkg_dir=$(mktemp -d)
 working_dir=$(mktemp -d)
-source_archive=$working_dir/azure-cli-${CLI_VERSION}.tar.gz
-source_dir=$working_dir/azure-cli-${CLI_VERSION}
 cd $working_dir
-wget https://azurecliprod.blob.core.windows.net/releases/azure-cli_packaged_${CLI_VERSION}.tar.gz -qO $source_archive
-echo "$CLI_DOWNLOAD_SHA256  $source_archive" | sha256sum -c -
-mkdir $source_dir
-# Extract archive
-archive_extract_dir=$(mktemp -d)
-tar -xvzf $source_archive -C $archive_extract_dir
-cp -r $archive_extract_dir/azure-cli_packaged_${CLI_VERSION}/* $source_dir
+if [ -z "$local_repo" ]
+  then
+    source_archive=$working_dir/azure-cli-${CLI_VERSION}.tar.gz
+    source_dir=$working_dir/azure-cli-${CLI_VERSION}
+    deb_file=$working_dir/azure-cli_${CLI_VERSION}-1_all.deb
+    az_completion_file=$source_dir/az.completion
+    wget https://azurecliprod.blob.core.windows.net/releases/azure-cli_packaged_${CLI_VERSION}.tar.gz -qO $source_archive
+    echo "$CLI_DOWNLOAD_SHA256  $source_archive" | sha256sum -c -
+    mkdir $source_dir
+    # Extract archive
+    archive_extract_dir=$(mktemp -d)
+    tar -xvzf $source_archive -C $archive_extract_dir
+    cp -r $archive_extract_dir/azure-cli_packaged_${CLI_VERSION}/* $source_dir
+  else
+    source_dir=$local_repo
+    deb_file=$local_repo/../azure-cli_${CLI_VERSION}-1_all.deb
+    az_completion_file=$source_dir/packaged_releases/az.completion
+    # clean up old build output
+    if [ -d "$source_dir/debian" ]
+      then
+        rm -rf $source_dir/debian
+    fi
+    cp $local_repo/privates/*.whl $tmp_pkg_dir
+fi
+
 # Build Python from source and include
 python_dir=$(mktemp -d)
 python_archive=$(mktemp)
@@ -47,7 +69,6 @@ make
 #  required to run the 'make install'
 sudo apt-get install -y zlib1g-dev
 make install
-tmp_pkg_dir=$(mktemp -d)
 
 # note: This installation step could happen in debian/rules but was unable to escape $ char.
 # It does not affect the built .deb file though.
@@ -63,11 +84,11 @@ mkdir $source_dir/debian
 # Create temp dir for the debian/ directory used for CLI build.
 cli_debian_dir_tmp=$(mktemp -d)
 
-$debian_directory_creator $cli_debian_dir_tmp $source_dir/az.completion $source_dir
+$debian_directory_creator $cli_debian_dir_tmp $az_completion_file $source_dir
 cp -r $cli_debian_dir_tmp/* $source_dir/debian
 cd $source_dir
 dpkg-buildpackage -us -uc
 echo "The archive is available at $working_dir/azure-cli_${CLI_VERSION}-1_all.deb"
-cp $working_dir/azure-cli_${CLI_VERSION}-1_all.deb ${BUILD_ARTIFACT_DIR}
+cp $deb_file ${BUILD_ARTIFACT_DIR}
 echo "The archive has also been copied to ${BUILD_ARTIFACT_DIR}"
 echo "Done."

packaged_releases/windows/scripts/build-packages.py

@@ -0,0 +1,66 @@
+# --------------------------------------------------------------------------------------------
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# Licensed under the MIT License. See License.txt in the project root for license information.
+# --------------------------------------------------------------------------------------------
+
+"""
+Script to build all command modules that can be used to install a fully self-contained instance of the CLI.
+"""
+
+from __future__ import print_function
+
+import glob
+import os
+import sys
+import tempfile
+import subprocess
+
+def _error_exit(msg):
+    print('ERROR: '+msg, file=sys.stderr)
+    sys.exit(1)
+
+def _print_status(msg=''):
+    print('-- '+msg)
+
+def _get_tmp_dir():
+    return tempfile.mkdtemp()
+
+def _get_tmp_file():
+    return tempfile.mkstemp()[1]
+
+def _exec_command(command_list, cwd=None, stdout=None):
+    """Returns True in the command was executed successfully"""
+    try:
+        _print_status('Executing {}'.format(command_list))
+        subprocess.check_call(command_list, stdout=stdout, cwd=cwd)
+        return True
+    except subprocess.CalledProcessError as err:
+        print(err, file=sys.stderr)
+        return False
+
+def _build_package(path_to_package, dist_dir):
+    cmd_success = _exec_command(['python', 'setup.py', 'bdist_wheel', '-d', dist_dir], cwd=path_to_package)
+    if not cmd_success:
+        _error_exit('Error building {}.'.format(path_to_package))
+
+def build_packages(clone_root, dist_dir):
+    packages_to_build = [
+        os.path.join(clone_root, 'src', 'azure-cli'),
+        os.path.join(clone_root, 'src', 'azure-cli-core'),
+        os.path.join(clone_root, 'src', 'azure-cli-nspkg'),
+        os.path.join(clone_root, 'src', 'azure-cli-command_modules-nspkg'),
+    ]
+
+    packages_to_build.extend(glob.glob(os.path.join(clone_root, 'src', 'command_modules', 'azure-cli-*')))
+    for p in packages_to_build:
+        if os.path.isfile(os.path.join(p, 'setup.py')):
+            _build_package(p, dist_dir)
+
+if __name__ == '__main__':
+    if len(sys.argv) == 1:
+        raise ValueError('Please provide temporary path for local built packages')
+    dist_dir = sys.argv[1]
+    clone_root = sys.argv[2]
+    build_packages(clone_root, dist_dir)
+    print("package were built to {}".format(dist_dir))
+    print("Done.")

packaged_releases/windows/scripts/build_local.cmd

@@ -0,0 +1,121 @@
+@echo off
+SetLocal EnableDelayedExpansion
+echo build a msi installer using local cli sources and python executables. You need to have curl.exe, unzip.exe and msbuild.exe available under PATH
+echo.
+
+set "PATH=%PATH%;%ProgramFiles%\Git\bin;%ProgramFiles%\Git\usr\bin"
+
+if "%CLIVERSION%"=="" (
+    echo Please set the CLIVERSION environment variable, e.g. 2.0.13
+    goto ERROR
+)
+set PYTHON_VERSION=3.6.1
+
+set WIX_DOWNLOAD_URL="https://azurecliprod.blob.core.windows.net/msi/wix310-binaries-mirror.zip"
+
+:: Set up the output directory and temp. directories
+echo Cleaning previous build artifacts...
+set OUTPUT_DIR=%~dp0..\out
+if exist %OUTPUT_DIR% rmdir /s /q %OUTPUT_DIR%
+mkdir %OUTPUT_DIR%
+
+set TEMP_SCRATCH_FOLDER=%HOMEDRIVE%%HOMEPATH%\zcli_scratch
+set BUILDING_DIR=%HOMEDRIVE%%HOMEPATH%\zcli
+set WIX_DIR=%HOMEDRIVE%%HOMEPATH%\zwix
+set REPO_ROOT=%~dp0..\..\..
+
+:: look for python 3.x so we can build into the installer
+if not "%1"=="" (
+   set PYTHON_DIR=%1
+   set PYTHON_EXE=%1\python.exe
+   goto PYTHON_FOUND
+)
+
+FOR /f %%i IN ('where python') DO (
+  set PY_FILE_DRIVE=%%~di
+  set PY_FILE_PATH=%%~pi
+  set PY_FILE_NAME=%%~ni
+  set PYTHON_EXE=!PY_FILE_DRIVE!!PY_FILE_PATH!!PY_FILE_NAME!.exe
+  set PYTHON_DIR=!PY_FILE_DRIVE!!PY_FILE_PATH!
+  FOR /F "delims=" %%j IN ('!PYTHON_EXE! --version') DO (
+    set PYTHON_VER=%%j
+    echo.!PYTHON_VER!|findstr /C:"%PYTHON_VERSION%" >nul 2>&1
+    if not errorlevel 1 (
+       goto PYTHON_FOUND
+    )
+  )
+)
+echo python %PYTHON_VERSION% is needed to create installer.
+exit /b 1
+:PYTHON_FOUND
+echo Python Executables: %PYTHON_DIR%, %PYTHON_EXE%
+
+::reset working folders
+if exist %BUILDING_DIR% rmdir /s /q %BUILDING_DIR%
+::rmdir always returns 0, so check folder's existence 
+if exist %BUILDING_DIR% (
+    echo Failed to delete %BUILDING_DIR%.
+    goto ERROR
+)
+mkdir %BUILDING_DIR%
+
+if exist %TEMP_SCRATCH_FOLDER% rmdir /s /q %TEMP_SCRATCH_FOLDER%
+if exist %TEMP_SCRATCH_FOLDER% (
+    echo Failed to delete %TEMP_SCRATCH_FOLDER%.
+    goto ERROR
+)
+mkdir %TEMP_SCRATCH_FOLDER%
+
+copy %REPO_ROOT%\privates\*.whl %TEMP_SCRATCH_FOLDER%
+
+::ensure wix is available
+if exist %WIX_DIR% (
+    echo Using existing Wix at %WIX_DIR%
+)
+if not exist %WIX_DIR% (
+    mkdir %WIX_DIR%
+    pushd %WIX_DIR%
+    echo Downloading Wix.
+    curl -o wix-archive.zip %WIX_DOWNLOAD_URL% -k
+    unzip -q wix-archive.zip
+    if %errorlevel% neq 0 goto ERROR
+    del wix-archive.zip
+    echo Wix downloaded and extracted successfully.
+    popd
+)
+
+:: Use the Python version on the machine that creates the MSI
+robocopy %PYTHON_DIR% %BUILDING_DIR% /s /NFL /NDL
+
+:: Build & install all the packages with bdist_wheel
+%BUILDING_DIR%\python %~dp0build-packages.py %TEMP_SCRATCH_FOLDER% %REPO_ROOT%
+if %errorlevel% neq 0 goto ERROR
+:: Install them to the temp folder so to be packaged 
+%BUILDING_DIR%\python.exe -m pip install -f %TEMP_SCRATCH_FOLDER% --no-cache-dir azure-cli
+%BUILDING_DIR%\python.exe -m pip install --force-reinstall --upgrade azure-nspkg azure-mgmt-nspkg
+
+echo Creating the wbin (Windows binaries) folder that will be added to the path...
+mkdir %BUILDING_DIR%\wbin
+copy %REPO_ROOT%\packaged_releases\windows\scripts\az.cmd %BUILDING_DIR%\wbin\
+if %errorlevel% neq 0 goto ERROR
+copy %REPO_ROOT%\packaged_releases\windows\resources\CLI_LICENSE.rtf %BUILDING_DIR%
+copy %REPO_ROOT%\packaged_releases\windows\resources\ThirdPartyNotices.txt %BUILDING_DIR%
+del %BUILDING_DIR%\Scripts\pip.exe
+del %BUILDING_DIR%\Scripts\pip3.exe
+del %BUILDING_DIR%\Scripts\pip3.6.exe
+if %errorlevel% neq 0 goto ERROR
+
+echo Building MSI...
+msbuild /t:rebuild /p:Configuration=Release %REPO_ROOT%\packaged_releases\windows\azure-cli.wixproj
+
+start %OUTPUT_DIR%
+
+goto END
+
+:ERROR
+echo Error occurred, please check the output for details.
+exit /b 1
+
+:END
+exit /b 0
+popd

packaged_releases/windows/scripts/prepareBuild.cmd

@@ -10,6 +10,7 @@ if "%CLIVERSION%"=="" (
     goto ERROR
 )
 
+::when change to a later version, please update ones in build_local.cmd 
 set PYTHON_VERSION=3.6.1
 
 pushd %~dp0..\

src/azure-cli-core/azure/cli/core/commands/parameters.py

@@ -225,6 +225,20 @@ def __call__(self, parser, namespace, values, option_string=None):
     action='store_true'
 )
 
+zones_type = CliArgumentType(
+    options_list=['--zones', '-z'],
+    nargs='+',
+    help='Space separated list of availability zones into which to provision the resource.',
+    choices=['1', '2', '3']
+)
+
+zone_type = CliArgumentType(
+    options_list=['--zone', '-z'],
+    help='Availability zone into which to provision the resource.',
+    choices=['1', '2', '3'],
+    nargs=1
+)
+
 register_cli_argument('', 'resource_group_name', resource_group_name_type)
 register_cli_argument('', 'location', location_type)
 register_cli_argument('', 'deployment_name', deployment_name_type)

src/azure-cli-core/azure/cli/core/profiles/_shared.py

@@ -28,7 +28,7 @@ class ResourceType(Enum):  # pylint: disable=too-few-public-methods
 
     MGMT_STORAGE = ('azure.mgmt.storage',
                     'StorageManagementClient')
-    MGMT_COMPUTE = ('azure.mgmt.compute.compute',
+    MGMT_COMPUTE = ('azure.mgmt.compute',
                     'ComputeManagementClient')
     MGMT_NETWORK = ('azure.mgmt.network',
                     'NetworkManagementClient')

src/command_modules/azure-cli-acs/setup.py

@@ -31,7 +31,7 @@
 
 DEPENDENCIES = [
     'azure-mgmt-authorization==0.30.0',
-    'azure-mgmt-compute==2.1.0',
+    'azure-mgmt-compute==3.0.0rc1',
     'azure-mgmt-containerservice==1.0.0',
     'azure-graphrbac==0.31.0',
     'azure-cli-core',

src/command_modules/azure-cli-network/HISTORY.rst

@@ -5,6 +5,7 @@ Release History
 
 unreleased
 +++++++++++++++++++
+* `lb/public-ip`: Add availability zone support.
 * `express-route`: Add support for IPv6 Microsoft Peering
 * Add `asg` application security group commands.
 * `nic create`: Added `--application-security-groups` support.