Merge branch 'dev' of https://github.com/Azure/azure-cli into cli-mgm…

…t-grp

.github/CODEOWNERS

@@ -22,6 +22,7 @@
 /src/command_modules/azure-cli-keyvault/ @tjprescott
 /src/command_modules/azure-cli-monitor/ @troydai
 /src/command_modules/azure-cli-network/ @tjprescott
+/src/command_modules/azure-cli-policyinsights/ @bulentelmaci
 /src/command_modules/azure-cli-profile/ @yugangw-msft
 /src/command_modules/azure-cli-storage/ @troydai
 /src/command_modules/azure-cli-servicefabric/ @QingChenmsft

.gitignore

@@ -5,6 +5,7 @@ __pycache__/
 # Virtual environment
 env/
 env27/
+venv/
 .python-version
 
 # PTVS analysis

azure-cli.pyproj

@@ -423,6 +423,7 @@
     <Compile Include="command_modules\azure-cli-iot\azure\cli\command_modules\iot\tests\latest\test_iot_dps_commands.py" />
     <Compile Include="command_modules\azure-cli-iot\azure\cli\command_modules\iot\tests\latest\test_sas_token_auth.py" />
     <Compile Include="command_modules\azure-cli-iot\azure\cli\command_modules\iot\tests\latest\_test_utils.py" />
+    <Compile Include="command_modules\azure-cli-iot\azure\cli\command_modules\iot\tests\latest\__init__.py" />
     <Compile Include="command_modules\azure-cli-iot\azure\cli\command_modules\iot\_client_factory.py" />
     <Compile Include="command_modules\azure-cli-iot\azure\cli\command_modules\iot\_completers.py" />
     <Compile Include="command_modules\azure-cli-iot\azure\cli\command_modules\iot\_constants.py" />

azure-cli2017.pyproj

@@ -424,6 +424,7 @@
     <Compile Include="command_modules\azure-cli-iot\azure\cli\command_modules\iot\tests\latest\test_iot_dps_commands.py" />
     <Compile Include="command_modules\azure-cli-iot\azure\cli\command_modules\iot\tests\latest\test_sas_token_auth.py" />
     <Compile Include="command_modules\azure-cli-iot\azure\cli\command_modules\iot\tests\latest\_test_utils.py" />
+    <Compile Include="command_modules\azure-cli-iot\azure\cli\command_modules\iot\tests\latest\__init__.py" />
     <Compile Include="command_modules\azure-cli-iot\azure\cli\command_modules\iot\_constants.py" />
     <Compile Include="command_modules\azure-cli-iot\azure\cli\command_modules\iot\_client_factory.py" />
     <Compile Include="command_modules\azure-cli-iot\azure\cli\command_modules\iot\_completers.py" />

doc/extensions/README.md

@@ -12,6 +12,8 @@ What is an Extension?
 - Currently, we support one extension type, a [Python Wheel](http://pythonwheels.com/).
 - All extension documentation here refers to this type of extension.
 
+> Extensions should be built with wheel `0.29.0` or `0.30.0` until [#6441](https://github.com/Azure/azure-cli/issues/6441) is resolved
+
 
 What an Extension is not
 ------------------------

doc/sphinx/azhelpgen/doc_source_map.json

@@ -36,6 +36,7 @@
     "monitor": "src/command_modules/azure-cli-monitor/azure/cli/command_modules/monitor/_help.py",
     "network": "src/command_modules/azure-cli-network/azure/cli/command_modules/network/_help.py",
     "policy": "src/command_modules/azure-cli-resource/azure/cli/command_modules/resource/_help.py",
+    "policyinsights": "src/command_modules/azure-cli-policyinsights/azure/cli/command_modules/policyinsights/_help.py",
     "provider": "src/command_modules/azure-cli-resource/azure/cli/command_modules/resource/_help.py",
     "rdbms": "src/command_modules/azure-cli-rdbms/azure/cli/command_modules/rdbms/_help.py",
     "redis": "src/command_modules/azure-cli-redis/azure/cli/command_modules/redis/_help.py",

scripts/ci/a01/docker_app/metadata.yml

@@ -12,6 +12,9 @@ environments:
   - name: A01_SP_TENANT
     type: secret
     value: sp.tenant
+  - name: A01_EVENTGRID_ENDPOINT
+    type: secret
+    value: feature.eventgrid.endpoint
   - name: AZURE_TEST_RUN_LIVE
     type: argument-switch-live
     value: "True"

scripts/curl_install_pypi/install.py

@@ -216,7 +216,7 @@ def _default_rc_file_creation_step():
 
 def _find_line_in_file(file_path, search_pattern):
     try:
-        with open(file_path, 'r') as search_file:
+        with open(file_path, 'r', encoding="utf-8") as search_file:
             for line in search_file:
                 if search_pattern in line:
                     return True
@@ -226,7 +226,7 @@ def _find_line_in_file(file_path, search_pattern):
 
 def _modify_rc(rc_file_path, line_to_add):
     if not _find_line_in_file(rc_file_path, line_to_add):
-        with open(rc_file_path, 'a') as rc_file:
+        with open(rc_file_path, 'a', encoding="utf-8") as rc_file:
             rc_file.write('\n'+line_to_add+'\n')
 
 def create_tab_completion_file(filename):

src/azure-cli-core/HISTORY.rst

@@ -2,10 +2,12 @@
 
 Release History
 ===============
-
 2.0.34
 ++++++
-* Minor fixes.
+* core: support cross tenant resource referencing
+* Improve telemetry upload reliability
+  1. Remove retry. Once failed stop uploading.
+  2. Update the process start configuration to prevent upload process from blocking the CLI process.
 
 2.0.33
 ++++++

src/azure-cli-core/azure/cli/core/_profile.py

@@ -71,7 +71,9 @@ def _authentication_context_factory(cli_ctx, tenant, cache):
     import adal
     authority_url = cli_ctx.cloud.endpoints.active_directory
     is_adfs = bool(re.match('.+(/adfs|/adfs/)$', authority_url, re.I))
-    if not is_adfs:
+    if is_adfs:
+        authority_url = authority_url.rstrip('/')  # workaround: ADAL is known to reject auth urls with trailing /
+    else:
         authority_url = authority_url + '/' + (tenant or _COMMON_TENANT)
     return adal.AuthenticationContext(authority_url, cache=cache, api_version=None, validate_authority=(not is_adfs))
 
@@ -437,14 +439,21 @@ def _try_parse_msi_account_name(account):
                 return parts[0], (None if len(parts) <= 1 else parts[1])
         return None, None
 
-    def get_login_credentials(self, resource=None,
-                              subscription_id=None):
+    def get_login_credentials(self, resource=None, subscription_id=None, aux_subscriptions=None):
         account = self.get_subscription(subscription_id)
         user_type = account[_USER_ENTITY][_USER_TYPE]
         username_or_sp_id = account[_USER_ENTITY][_USER_NAME]
         resource = resource or self.cli_ctx.cloud.endpoints.active_directory_resource_id
 
         identity_type, identity_id = Profile._try_parse_msi_account_name(account)
+
+        external_tenants_info = []
+        ext_subs = [aux_sub for aux_sub in (aux_subscriptions or []) if aux_sub != subscription_id]
+        for ext_sub in ext_subs:
+            sub = self.get_subscription(ext_sub)
+            if sub[_TENANT_ID] != account[_TENANT_ID]:
+                external_tenants_info.append((sub[_USER_ENTITY][_USER_NAME], sub[_TENANT_ID]))
+
         if identity_type is None:
             def _retrieve_token():
                 if in_cloud_console() and account[_USER_ENTITY].get(_CLOUD_SHELL_ID):
@@ -453,8 +462,16 @@ def _retrieve_token():
                     return self._creds_cache.retrieve_token_for_user(username_or_sp_id,
                                                                      account[_TENANT_ID], resource)
                 return self._creds_cache.retrieve_token_for_service_principal(username_or_sp_id, resource)
+
+            def _retrieve_tokens_from_external_tenants():
+                external_tokens = []
+                for u, t in external_tenants_info:
+                    external_tokens.append(self._creds_cache.retrieve_token_for_user(u, t, resource))
+                return external_tokens
+
             from azure.cli.core.adal_authentication import AdalAuthentication
-            auth_object = AdalAuthentication(_retrieve_token)
+            auth_object = AdalAuthentication(_retrieve_token,
+                                             _retrieve_tokens_from_external_tenants if external_tenants_info else None)
         else:
             if self._msi_creds is None:
                 self._msi_creds = MsiAccountTypes.msi_auth_factory(identity_type, identity_id, resource)

src/azure-cli-core/azure/cli/core/adal_authentication.py

@@ -14,14 +14,17 @@
 
 class AdalAuthentication(Authentication):  # pylint: disable=too-few-public-methods
 
-    def __init__(self, token_retriever):
+    def __init__(self, token_retriever, external_tenant_token_retriever=None):
         self._token_retriever = token_retriever
+        self._external_tenant_token_retriever = external_tenant_token_retriever
 
     def signed_session(self, session=None):
         session = session or super(AdalAuthentication, self).signed_session()
-
+        external_tenant_tokens = None
         try:
             scheme, token, _ = self._token_retriever()
+            if self._external_tenant_token_retriever:
+                external_tenant_tokens = self._external_tenant_token_retriever()
         except CLIError as err:
             if in_cloud_console():
                 AdalAuthentication._log_hostname()
@@ -40,6 +43,9 @@ def signed_session(self, session=None):
 
         header = "{} {}".format(scheme, token)
         session.headers['Authorization'] = header
+        if external_tenant_tokens:
+            aux_tokens = ';'.join(['{} {}'.format(scheme2, tokens2) for scheme2, tokens2, _ in external_tenant_tokens])
+            session.headers['x-ms-authorization-auxiliary'] = aux_tokens
         return session
 
     @staticmethod

src/azure-cli-core/azure/cli/core/commands/client_factory.py

@@ -44,7 +44,11 @@ def resolve_client_arg_name(operation, kwargs):
 
 
 def get_mgmt_service_client(cli_ctx, client_or_resource_type, subscription_id=None, api_version=None,
-                            **kwargs):
+                            aux_subscriptions=None, **kwargs):
+    """
+     :params subscription_id: the current account's subscription
+     :param aux_subscriptions: mainly for cross tenant scenarios, say vnet peering.
+    """
     sdk_profile = None
     if isinstance(client_or_resource_type, (ResourceType, CustomResourceType)):
         # Get the versioned client
@@ -57,7 +61,9 @@ def get_mgmt_service_client(cli_ctx, client_or_resource_type, subscription_id=No
         # Get the non-versioned client
         client_type = client_or_resource_type
     client, _ = _get_mgmt_service_client(cli_ctx, client_type, subscription_id=subscription_id,
-                                         api_version=api_version, sdk_profile=sdk_profile, **kwargs)
+                                         api_version=api_version, sdk_profile=sdk_profile,
+                                         aux_subscriptions=aux_subscriptions,
+                                         **kwargs)
     return client
 
 
@@ -104,12 +110,14 @@ def _get_mgmt_service_client(cli_ctx,
                              base_url_bound=True,
                              resource=None,
                              sdk_profile=None,
+                             aux_subscriptions=None,
                              **kwargs):
     from azure.cli.core._profile import Profile
     logger.debug('Getting management service client client_type=%s', client_type.__name__)
     resource = resource or cli_ctx.cloud.endpoints.active_directory_resource_id
     profile = Profile(cli_ctx=cli_ctx)
-    cred, subscription_id, _ = profile.get_login_credentials(subscription_id=subscription_id, resource=resource)
+    cred, subscription_id, _ = profile.get_login_credentials(subscription_id=subscription_id, resource=resource,
+                                                             aux_subscriptions=aux_subscriptions)
 
     client_kwargs = {}
     if base_url_bound:

src/azure-cli-core/azure/cli/core/parser.py

@@ -172,6 +172,10 @@ def _add_argument(obj, arg):
         options_list = arg.options_list
         argparse_options = {name: value for name, value in arg.options.items() if name in ARGPARSE_SUPPORTED_KWARGS}
         if options_list:
+            for opt in options_list:
+                if not opt.startswith('--') and len(opt) != 2:
+                    raise CLIError("command authoring error: multi-character short option '{}' is not allowed. "
+                                   "Use a single character or convert to a long-option.".format(opt))
             return obj.add_argument(*options_list, **argparse_options)
         else:
             if 'required' in argparse_options:

src/azure-cli-core/azure/cli/core/telemetry.py

@@ -227,7 +227,6 @@ def flush():
 
     payload = _session.generate_payload()
     if payload:
-
         subprocess.Popen([sys.executable, os.path.realpath(telemetry_core.__file__), payload])
 
     # reset session fields, retaining correlation id and application
@@ -241,7 +240,28 @@ def conclude():
 
     payload = _session.generate_payload()
     if payload:
-        subprocess.Popen([sys.executable, os.path.realpath(telemetry_core.__file__), payload])
+        kwargs = {
+            'args': [sys.executable, os.path.realpath(telemetry_core.__file__), payload]
+        }
+        if os.name == 'nt':
+            # Windows process creation flag to not reuse the parent console.
+            # Without this, the background service is associated with the
+            # starting process's console, and will block that console from
+            # exiting until the background service self-terminates.
+            # Elsewhere, fork just does the right thing.
+            kwargs['creationflags'] = 0x00000010  # CREATE_NEW_CONSOLE
+
+            startupinfo = subprocess.STARTUPINFO()
+            startupinfo.dwFlags |= subprocess.STARTF_USESHOWWINDOW
+            startupinfo.wShowWindow = subprocess.SW_HIDE
+            kwargs['startupinfo'] = startupinfo
+        else:
+            if sys.version_info >= (3, 3):
+                kwargs['stdin'] = subprocess.DEVNULL
+                if not telemetry_core.in_diagnostic_mode:
+                    kwargs['stdout'] = subprocess.DEVNULL
+                    kwargs['stderr'] = subprocess.STDOUT
+        subprocess.Popen(**kwargs)
 
 
 @decorators.suppress_all_exceptions(raise_in_diagnostics=True)

src/azure-cli-core/azure/cli/core/telemetry_upload.py

@@ -29,7 +29,6 @@
 class LimitedRetrySender(SynchronousSender):
     def __init__(self):
         super(LimitedRetrySender, self).__init__()
-        self.retry = 0
 
     def send(self, data_to_send):
         """ Override the default resend mechanism in SenderBase. Stop resend when it fails."""
@@ -38,22 +37,17 @@ def send(self, data_to_send):
         request = HTTPClient.Request(self._service_endpoint_uri, bytearray(request_payload, 'utf-8'),
                                      {'Accept': 'application/json', 'Content-Type': 'application/json; charset=utf-8'})
         try:
-            response = HTTPClient.urlopen(request, timeout=10)
-            status_code = response.getcode()
-            if 200 <= status_code < 300:
-                return
+            HTTPClient.urlopen(request, timeout=10)
         except HTTPError as e:
-            if e.getcode() == 400:
-                return
-        except Exception:  # pylint: disable=broad-except
-            if self.retry < 3:
-                self.retry = self.retry + 1
-            else:
-                return
-
-        # Add our unsent data back on to the queue
-        for data in data_to_send:
-            self._queue.put(data)
+            if in_diagnostic_mode():
+                sys.stdout.write('\nUpload failed. HTTPError: {}\n'.format(e))
+        except OSError as e:  # socket timeout
+            # stop retry during socket timeout
+            if in_diagnostic_mode():
+                sys.stdout.write('\nUpload failed. OSError: {}.\n'.format(e))
+        except Exception as e:  # pylint: disable=broad-except
+            if in_diagnostic_mode():
+                sys.stdout.write('\nUnexpected exception: {}\n'.format(e))
 
 
 def in_diagnostic_mode():
@@ -68,7 +62,6 @@ def in_diagnostic_mode():
 def upload(data_to_save):
     if in_diagnostic_mode():
         sys.stdout.write('Telemetry upload begins\n')
-        sys.stdout.write('Got data {}\n'.format(json.dumps(json.loads(data_to_save), indent=2)))
 
     try:
         data_to_save = json.loads(data_to_save.replace("'", '"'))