Implement a new command to output the CRMint UI url

This command is necessary to separate the concerns between our `crmint cloud setup`, `crmint cloud migrate` and other commands. We also implemented a waiting loop that will ping the UI url to notify the user when it is ready to be opened in the browser. The Cloud Shell tutorial has been updated accordingly. PiperOrigin-RevId: 495408716
google-marketing-solutions · Dec 14, 2022 · 0e6aa28 · 0e6aa28
1 parent aadd14b
commit 0e6aa28
Show file tree

Hide file tree

Showing 8 changed files with 287 additions and 44 deletions.
diff --git a/cli/commands/bundle.py b/cli/commands/bundle.py
@@ -37,6 +37,7 @@ def install(ctx: click.Context, use_vpc: bool, debug: bool) -> None:
   ctx.invoke(cloud.checklist, debug=debug)
   ctx.invoke(cloud.setup, debug=debug)
   ctx.invoke(cloud.migrate, debug=debug)
+  ctx.invoke(cloud.url, debug=debug)
 
 
 @cli.command('update')
@@ -48,6 +49,7 @@ def update(ctx: click.Context, version: str, debug: bool):
   ctx.invoke(stages.update, version=version, debug=debug)
   ctx.invoke(cloud.setup, debug=debug)
   ctx.invoke(cloud.migrate, debug=debug)
+  ctx.invoke(cloud.url, debug=debug)
 
 
 @cli.command('allow-users')
@@ -58,6 +60,7 @@ def allow_users(ctx: click.Context, user_emails: str, debug: bool):
   """Allow a list of user emails to access CRMint and Setup."""
   ctx.invoke(stages.allow_users, user_emails=user_emails, debug=debug)
   ctx.invoke(cloud.setup, debug=debug)
+  ctx.invoke(cloud.url, debug=debug)
 
 
 if __name__ == '__main__':

diff --git a/cli/commands/bundle_tests.py b/cli/commands/bundle_tests.py
@@ -52,6 +52,8 @@ def setUp(self):
         'migrate_sql_conn_name': {'value': 'project:region:db_instance'},
         'cloud_db_uri': {'value': 'mysql://db:3306/name'},
         'cloud_build_worker_pool': {'value': 'my_worker_pool'},
+        'secured_url': {'value': 'https://secured.com'},
+        'unsecured_url': {'value': 'https://temporary.com'},
     }
     self.enter_context(
         mock.patch.object(constants, 'STAGE_DIR', tmp_stage_dir.full_path))
@@ -73,6 +75,12 @@ def setUp(self):
             'get_region',
             autospec=True,
             return_value='us-central1'))
+    self.enter_context(
+        mock.patch.object(
+            shared,
+            'wait_for_frontend',
+            autospec=True,
+            return_value='https://secured.com'))
     self.enter_context(
         mock.patch.object(
             cloud,
@@ -110,6 +118,7 @@ def test_can_run_install_with_existing_stage_file(self):
     self.assertIn('>>>> Checklist', result.output)
     self.assertIn('>>>> Setup', result.output)
     self.assertIn('>>>> Sync database', result.output)
+    self.assertIn('>>>> CRMint UI', result.output)
 
   def test_cannot_run_update_without_stage_file(self):
     runner = testing.CliRunner()
@@ -134,6 +143,7 @@ def test_can_run_update_with_stage_file(self):
     self.assertRegex(result.output, 'Stage updated to version: 3.3')
     self.assertIn('>>>> Setup', result.output)
     self.assertIn('>>>> Sync database', result.output)
+    self.assertIn('>>>> CRMint UI', result.output)
 
   def test_can_allow_new_users_and_setup(self):
     shutil.copyfile(
@@ -148,6 +158,7 @@ def test_can_allow_new_users_and_setup(self):
     self.assertIn('>>>> Allow new users', result.output)
     self.assertIn('>>>> Setup', result.output)
     self.assertNotIn('>>>> Sync database', result.output)
+    self.assertIn('>>>> CRMint UI', result.output)
 
 
 if __name__ == '__main__':

diff --git a/cli/commands/cloud.py b/cli/commands/cloud.py
@@ -165,7 +165,7 @@ def terraform_apply(debug: bool = False) -> bool:
   # NB: No need to set `-var-file` when applying a saved plan.
   cmd = 'terraform apply -auto-approve /tmp/tfplan'
   shared.execute_command(
-      'Apply Terraform plan', cmd, cwd='./terraform', debug=debug)
+      'Apply Terraform plan (~10min)', cmd, cwd='./terraform', debug=debug)
 
 
 def terraform_show_plan(debug: bool = False) -> str:
@@ -220,26 +220,6 @@ def configuration_summary_from_plan(debug: bool = False) -> bool:
         textwrap.indent(f'{resource_type_cleaned} ({count})', _INDENT_PREFIX))
 
 
-def display_frontend_url(debug: bool = False):
-  """Displays CRMint UI urls."""
-  cmd = 'terraform output secured_url'
-  _, out, _ = shared.execute_command(
-      'CRMint UI',
-      cmd,
-      cwd='./terraform',
-      debug_uses_std_out=False,
-      debug=debug)
-  click.echo(textwrap.indent(out.strip(), _INDENT_PREFIX))
-  cmd = 'terraform output unsecured_url'
-  _, out, _ = shared.execute_command(
-      'CRMint UI (unsecured, temporarily)',
-      cmd,
-      cwd='./terraform',
-      debug_uses_std_out=False,
-      debug=debug)
-  click.echo(textwrap.indent(out.strip(), _INDENT_PREFIX))
-
-
 def terraform_outputs(debug: bool = False):
   """Runs the Terraform output command."""
   cmd = 'terraform output -json'
@@ -383,9 +363,6 @@ def setup(stage_path: Union[None, str], debug: bool) -> None:
   terraform_plan(stage, debug=debug)
   configuration_summary_from_plan(debug=debug)
   terraform_apply(debug=debug)
-
-  # Displays the frontend url to improve the user experience.
-  display_frontend_url(debug=debug)
   click.echo(click.style('Done.', fg='magenta', bold=True))
 
 
@@ -421,17 +398,14 @@ def _run_command(section_name: str,
 
   # Resets the state of pipelines and jobs.
   trigger_command(cmd, outputs, debug=debug)
-
-  # Displays the frontend url to improve the user experience.
-  display_frontend_url(debug=debug)
   click.echo(click.style('Done.', fg='magenta', bold=True))
 
 
 @cli.command('migrate')
 @click.option('--stage_path', type=str, default=None)
 @click.option('--debug/--no-debug', default=False)
 def migrate(stage_path: Union[None, str], debug: bool):
-  """Reset pipeline statuses."""
+  """Migrate the database to the latest schema."""
   _run_command(
       '>>>> Sync database',
       'python -m flask db upgrade; python -m flask db-seeds;',
@@ -451,5 +425,44 @@ def reset(stage_path: Union[None, str], debug: bool):
       debug=debug)
 
 
+@cli.command('url')
+@click.option('--stage_path', type=str, default=None)
+@click.option('--debug/--no-debug', default=False)
+def url(stage_path: Union[None, str], debug: bool):
+  """Retrieve the frontend URL to access the UI."""
+  click.echo(click.style('>>>> CRMint UI', fg='magenta', bold=True))
+
+  if stage_path is not None:
+    stage_path = pathlib.Path(stage_path)
+
+  try:
+    stage = shared.fetch_stage_or_default(stage_path, debug=debug)
+  except shared.CannotFetchStageError:
+    sys.exit(1)
+
+  # Switches workspace.
+  terraform_init(debug=debug)
+  terraform_switch_workspace(stage, debug=debug)
+
+  # Retrieves outputs from the current Terraform state.
+  outputs_json_raw = terraform_outputs(debug=debug)
+  outputs = json.loads(outputs_json_raw)
+
+  if not outputs:
+    click.secho(f'No state found in current workspace: {stage.project_id}',
+                fg='red',
+                bold=True)
+    click.secho('Fix this by running: $ crmint cloud setup', fg='green')
+    sys.exit(1)
+
+  secured_url = outputs['secured_url']['value']
+  available_url = shared.wait_for_frontend(secured_url, debug=debug)
+  if available_url:
+    click.secho(f'Secured url: {available_url}', fg='green', bold=True)
+  else:
+    # No available url yet.
+    sys.exit(1)
+
+
 if __name__ == '__main__':
   cli()
diff --git a/cli/commands/cloud_tests.py b/cli/commands/cloud_tests.py
@@ -1,5 +1,6 @@
 """Tests for cli.commands.cloud."""
 
+import json
 import os
 import pathlib
 import shutil
@@ -10,7 +11,6 @@
 
 from absl.testing import absltest
 from absl.testing import parameterized
-import click
 from click import testing
 
 from cli.commands import cloud
@@ -31,9 +31,9 @@ class CloudChecklistTest(parameterized.TestCase):
       ('User not project owner', 'roles/editor', 1),
       ('User is project owner', 'roles/owner', 0),
       ('User has other role is project owner', 'roles/viewer\nroles/owner', 0),
-      ('User is project editor with missing roles', 'roles/editor\nroles/viewer', 1),
-      ('User is project editor with one missing role', 'roles/editor\nroles/iap.admin\nroles/run.admin\nroles/compute.networkAdmin', 1),
-      ('User is project editor with all extra roles', 'roles/editor\nroles/iap.admin\nroles/run.admin\nroles/compute.networkAdmin\nroles/resourcemanager.projectIamAdmin\nroles/secretmanager.admin', 0),
+      ('User is project editor with missing roles', 'roles/editor\nroles/viewer', 1),  # pylint: disable=line-too-long
+      ('User is project editor with one missing role', 'roles/editor\nroles/iap.admin\nroles/run.admin\nroles/compute.networkAdmin', 1),  # pylint: disable=line-too-long
+      ('User is project editor with all extra roles', 'roles/editor\nroles/iap.admin\nroles/run.admin\nroles/compute.networkAdmin\nroles/resourcemanager.projectIamAdmin\nroles/secretmanager.admin', 0),  # pylint: disable=line-too-long
   )
   def test_user_with_different_roles(self, user_role, exit_code):
     side_effect_run = test_helpers.mock_subprocess_result_side_effect(
@@ -100,7 +100,7 @@ def test_validates_stdout(self):
     )
 
 
-class CloudSetupTest(parameterized.TestCase):
+class CloudTestBase(parameterized.TestCase):
 
   def setUp(self):
     super().setUp()
@@ -123,10 +123,12 @@ def setUp(self):
         mock.patch.object(constants, 'STAGE_DIR', tmp_stage_dir.full_path))
     shutil.copyfile(
         _datafile('dummy_project_with_vpc.tfvars.json'),
-        pathlib.Path(constants.STAGE_DIR, 'dummy_project_with_vpc.tfvars.json'))
+        pathlib.Path(constants.STAGE_DIR,
+                     'dummy_project_with_vpc.tfvars.json'))
     shutil.copyfile(
         _datafile('dummy_project_without_vpc.tfvars.json'),
-        pathlib.Path(constants.STAGE_DIR, 'dummy_project_without_vpc.tfvars.json'))
+        pathlib.Path(constants.STAGE_DIR,
+                     'dummy_project_without_vpc.tfvars.json'))
     # Uses a temporary directory we can keep a reference to.
     self.tmp_workdir = self.create_tempdir('workdir')
     self.enter_context(
@@ -136,12 +138,15 @@ def setUp(self):
             autospec=True,
             return_value=self.tmp_workdir.full_path))
 
+
+class CloudSetupTest(CloudTestBase):
+
   def test_fetch_existing_stage(self):
     """Should not raise an exception if stage file exists."""
     stage_path = pathlib.Path(
         constants.STAGE_DIR, 'dummy_project_with_vpc.tfvars.json')
     try:
-      stage = shared.fetch_stage_or_default(stage_path)
+      _ = shared.fetch_stage_or_default(stage_path)
     except shared.CannotFetchStageError:
       self.fail('Should not raise an exception')
 
@@ -209,11 +214,7 @@ def test_validates_stdout_without_vpc(self):
                  Cloud Run Service \\(3\\)
                  Cloud Run Service IAM Member \\(3\\)
                  (.|\\n)*
-            ---> Apply Terraform plan ✓
-            ---> CRMint UI ✓
-                 output
-            ---> CRMint UI \\(unsecured, temporarily\\) ✓
-                 output
+            ---> Apply Terraform plan \\(~10min\\) ✓
             Done.
             """)
     )
@@ -235,5 +236,31 @@ def test_validates_stdout_with_vpc(self):
     self.assertIn('VPC Access Connector (1)', result.output)
 
 
+class CloudUrlTest(CloudTestBase):
+
+  def setUp(self):
+    super().setUp()
+    tf_outputs = json.dumps(
+        {
+            'secured_url': {'value': 'https://secured.com'},
+            'unsecured_url': {'value': 'https://temporary.com'},
+        })
+    self.enter_context(
+        mock.patch.object(
+            cloud, 'terraform_outputs', autospec=True, return_value=tf_outputs))
+
+  def test_frontend_url_available(self):
+    self.enter_context(
+        mock.patch.object(
+            shared,
+            'wait_for_frontend',
+            autospec=True,
+            return_value='https://secured.com'))
+    runner = testing.CliRunner()
+    result = runner.invoke(cloud.url, catch_exceptions=False)
+    self.assertEqual(result.exit_code, 0, msg=result.output)
+    self.assertIn('Secured url: https://secured.com', result.output)
+
+
 if __name__ == '__main__':
   absltest.main()
diff --git a/cli/utils/shared.py b/cli/utils/shared.py
@@ -14,16 +14,20 @@
 
 """Package for shared methods among the commands."""
 
+import http
 import json
 import os
 import pathlib
 import re
 import subprocess
 import textwrap
+import time
 import types
 from typing import Callable, NewType, Tuple, Union
 
 import click
+import requests
+from requests import exceptions
 
 from cli.utils import constants
 from cli.utils import settings
@@ -372,3 +376,64 @@ def list_available_tags(image_uri: str, debug: bool = False):
 def filter_versions_from_tags(tags: list[str]) -> list[str]:
   """Filters a list of tags to return a list of versions."""
   return [tag for tag in tags if re.fullmatch(r'[\d\.]+', tag)]
+
+
+def wait_for_frontend(url: str,
+                      max_attempts: int = 720,
+                      attempt_delay: int = 10,
+                      debug: bool = False) -> Union[str, None]:
+  """Waiting loop to detect when the frontend is accessible.
+
+  This is needed, because an Global External HTTPS Load Balancer can take
+  some time before exposing itself to the public IP address.
+
+  Args:
+    url: Frontend URL to ping.
+    max_attempts: Maximum number of attempts to get a ready url.
+        Defaults to 720 attempts.
+    attempt_delay: Number of seconds between each attempt. Defaults to 10.
+    debug: Enables the debug mode on requests calls.
+  Returns:
+    The first available url or None if we reached the max number of attempts.
+  """
+  click.secho(
+      '---> Waiting for the UI readiness (~15min on average, up to 2h max)',
+      fg='blue',
+      bold=True,
+      nl=debug)
+  if debug:
+    click.echo('')
+  available_url = None
+  ping_attempts = 0
+  with spinner.spinner(disable=debug, color='blue', bold=True):
+    while ping_attempts < max_attempts:
+      try:
+        response = requests.head(url, verify=True)
+      except (exceptions.SSLError, exceptions.ConnectionError) as inst:
+        response = None
+        if debug:
+          click.echo(f'Failed to connect: {str(inst)}')
+          click.echo('')
+      if debug and response:
+        status_code_explained = http.client.responses[response.status_code]
+        headers_formatted = '\n'.join(
+            f'{k}: {v}' for k, v in response.headers.items())
+        click.echo(f'{response.request.method} {response.request.url}')
+        click.echo(f'HTTP/1 {response.status_code} {status_code_explained}')
+        click.echo(headers_formatted)
+        click.echo('')
+      if response and response.ok:
+        available_url = url
+        break
+      ping_attempts += 1
+      time.sleep(attempt_delay)
+  if not debug:
+    click.echo('')
+  if ping_attempts >= max_attempts:
+    click.echo(textwrap.indent(
+        f'Failed after {ping_attempts} attempts. Retry: $ crmint cloud url',
+        _INDENT_PREFIX))
+    return None
+  else:
+    click.echo(textwrap.indent('Ready', _INDENT_PREFIX))
+    return available_url