Merge branch 'master' into fix_column_families

CHANGELOG.md

@@ -8,6 +8,7 @@ All notable changes to this project will be documented in this file.
 
 ### BLUEPRINTS
 
+- [[#1063](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/1063)] Network dashboard: PSA ranges support, starting with Cloud SQL ([aurelienlegrand](https://github.com/aurelienlegrand)) <!-- 2022-12-22 12:14:42+00:00 -->
 - [[#1062](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/1062)] Fixes for GKE ([wiktorn](https://github.com/wiktorn)) <!-- 2022-12-21 22:14:52+00:00 -->
 - [[#1060](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/1060)] Update src/README.md for Network Dashboard ([aurelienlegrand](https://github.com/aurelienlegrand)) <!-- 2022-12-21 15:30:10+00:00 -->
 - [[#1020](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/1020)] Networking dashboard and discovery tool refactor ([ludoo](https://github.com/ludoo)) <!-- 2022-12-18 09:07:24+00:00 -->
@@ -19,6 +20,8 @@ All notable changes to this project will be documented in this file.
 
 ### MODULES
 
+- [[#1067](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/1067)] Corrected load balancing scheme in backend service ([apichick](https://github.com/apichick)) <!-- 2022-12-22 11:41:06+00:00 -->
+- [[#1066](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/1066)] Refactor GCS module and tests for Terraform 1.3 ([ludoo](https://github.com/ludoo)) <!-- 2022-12-22 11:27:09+00:00 -->
 - [[#1062](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/1062)] Fixes for GKE ([wiktorn](https://github.com/wiktorn)) <!-- 2022-12-21 22:14:52+00:00 -->
 - [[#1061](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/1061)] **incompatible change:** Allow using dynamically generated address in LB modules NEGs ([ludoo](https://github.com/ludoo)) <!-- 2022-12-21 16:04:56+00:00 -->
 - [[#1059](https://github.com/GoogleCloudPlatform/cloud-foundation-fabric/pull/1059)] Read ranges from correct fields in firewall factory ([juliocc](https://github.com/juliocc)) <!-- 2022-12-20 09:13:54+00:00 -->

blueprints/cloud-operations/network-dashboard/src/plugins/discover-cai-compute.py → blueprints/cloud-operations/network-dashboard/src/plugins/discover-cai.py

@@ -24,21 +24,22 @@
 from . import HTTPRequest, Level, Resource, register_init, register_discovery
 from .utils import parse_cai_results
 
-
 CAI_URL = ('https://content-cloudasset.googleapis.com/v1'
            '/{root}/assets'
            '?contentType=RESOURCE&{asset_types}&pageSize=500')
 LOGGER = logging.getLogger('net-dash.discovery.cai-compute')
 TYPES = {
-    'addresses': 'Address',
-    'firewall_policies': 'FirewallPolicy',
-    'firewall_rules': 'Firewall',
-    'forwarding_rules': 'ForwardingRule',
-    'instances': 'Instance',
-    'networks': 'Network',
-    'subnetworks': 'Subnetwork',
-    'routers': 'Router',
-    'routes': 'Route',
+    'addresses': 'compute.googleapis.com/Address',
+    'global_addresses': 'compute.googleapis.com/GlobalAddress',
+    'firewall_policies': 'compute.googleapis.com/FirewallPolicy',
+    'firewall_rules': 'compute.googleapis.com/Firewall',
+    'forwarding_rules': 'compute.googleapis.com/ForwardingRule',
+    'instances': 'compute.googleapis.com/Instance',
+    'networks': 'compute.googleapis.com/Network',
+    'subnetworks': 'compute.googleapis.com/Subnetwork',
+    'routers': 'compute.googleapis.com/Router',
+    'routes': 'compute.googleapis.com/Route',
+    'sql_instances': 'sqladmin.googleapis.com/Instance'
 }
 NAMES = {v: k for k, v in TYPES.items()}
 
@@ -61,7 +62,8 @@ def _handle_discovery(resources, response, data):
   'Processes the asset API response and returns parsed resources or next URL.'
   LOGGER.info('discovery handle request')
   for result in parse_cai_results(data, 'cai-compute', method='list'):
-    resource = _handle_resource(resources, result['resource'])
+    resource = _handle_resource(
+        resources, result['assetType'], result['resource'])
     if not resource:
       continue
     yield resource
@@ -72,15 +74,18 @@ def _handle_discovery(resources, response, data):
     yield HTTPRequest(f'{url}&pageToken={page_token}', {}, None)
 
 
-def _handle_resource(resources, data):
+def _handle_resource(resources, asset_type, data):
   'Parses and returns a single resource. Calls resource-level handler.'
-  attrs = data['data']
   # general attributes shared by all resource types
-  resource_name = NAMES[data['discoveryName']]
+  attrs = data['data']
+  # we use the asset type as the discovery name sometimes does not match
+  # e.g. assetType = GlobalAddress but discoveryName = Address
+  resource_name = NAMES[asset_type]
   resource = {
-      'id': attrs['id'],
+      'id': attrs.get('id'),
       'name': attrs['name'],
-      'self_link': _self_link(attrs['selfLink'])
+      'self_link': _self_link(attrs['selfLink']),
+      'assetType': asset_type
   }
   # derive parent type and id and skip if parent is not within scope
   parent_data = _get_parent(data['parent'], resources)
@@ -145,6 +150,19 @@ def _handle_forwarding_rules(resource, data):
   }
 
 
+def _handle_global_addresses(resource, data):
+  'Handles GlobalAddress type resource data (ex: PSA ranges).'
+  network = data.get('network')
+  return {
+      'address': data['address'],
+      'prefixLength': data.get('prefixLength') or None,
+      'internal': data.get('addressType') == 'INTERNAL',
+      'purpose': data.get('purpose', ''),
+      'status': data.get('status', ''),
+      'network': None if not network else _self_link(network),
+  }
+
+
 def _handle_instances(resource, data):
   'Handles instance type resource data.'
   if data['status'] != 'RUNNING':
@@ -184,6 +202,18 @@ def _handle_routes(resource, data):
   return {'next_hop_type': hop[0], 'network': _self_link(data['network'])}
 
 
+def _handle_sql_instances(resource, data):
+  'Handles cloud sql instance type resource data.'
+  return {
+      'name': data['name'],
+      'self_link': _self_link(data['selfLink']),
+      'ipAddresses': [
+          i['ipAddress'] for i in data['ipAddresses'] if i['type'] == 'PRIVATE'
+      ],
+      'region': data['region'],
+      'availabilityType': data['settings']['availabilityType'],
+  }
+
 def _handle_subnetworks(resource, data):
   'Handles subnetwork type resource data.'
   secondary_ranges = [{
@@ -201,14 +231,14 @@ def _handle_subnetworks(resource, data):
 
 def _self_link(s):
   'Removes initial part from self links.'
-  return s.removeprefix('https://www.googleapis.com/compute/v1/')
+  return '/'.join(s.split('/')[5:])
 
 
 def _url(resources):
   'Returns discovery URL'
   discovery_root = resources['config:discovery_root']
   asset_types = '&'.join(
-      f'assetTypes=compute.googleapis.com/{t}' for t in TYPES.values())
+      f'assetTypes={t}' for t in TYPES.values())
   return CAI_URL.format(root=discovery_root, asset_types=asset_types)
 
 

blueprints/cloud-operations/network-dashboard/src/plugins/series-psa.py

@@ -0,0 +1,71 @@
+# Copyright 2022 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+'Prepares descriptors and timeseries for subnetwork-level metrics.'
+
+import collections
+import ipaddress
+import itertools
+import logging
+
+from . import MetricDescriptor, TimeSeries, register_timeseries
+
+DESCRIPTOR_ATTRS = {
+    'addresses_available': 'Address limit per psa range',
+    'addresses_used': 'Addresses used per psa range',
+    'addresses_used_ratio': 'Addresses used ratio per psa range'
+}
+LOGGER = logging.getLogger('net-dash.timeseries.psa')
+
+
+def _sql_addresses(sql_instances):
+  'Returns counts of Cloud SQL instances per PSA range.'
+  for v in sql_instances.values():
+    if not v['ipAddresses']:
+      continue
+    # 1 IP for the instance + 1 IP for the ILB + 1 IP if HA
+    yield v['ipAddresses'][0], 2 if v['availabilityType'] != 'REGIONAL' else 3
+
+
+@register_timeseries
+def timeseries(resources):
+  'Returns used/available/ratio timeseries for addresses by PSA ranges.'
+  LOGGER.info('timeseries')
+  for dtype, name in DESCRIPTOR_ATTRS.items():
+    yield MetricDescriptor(f'network/psa/{dtype}', name,
+                           ('project', 'network', 'subnetwork'),
+                           dtype.endswith('ratio'))
+  psa_nets = {
+      k: ipaddress.ip_network('{}/{}'.format(v['address'], v['prefixLength']))
+      for k, v in resources['global_addresses'].items() if v['prefixLength']
+  }
+  psa_counts = {}
+  for address, ip_count in _sql_addresses(resources.get('sql_instances', {})):
+    ip_address = ipaddress.ip_address(address)
+    for k, v in psa_nets.items():
+      if ip_address in v:
+        psa_counts[k] = psa_counts.get(k, 0) + ip_count
+        break
+
+  for k, v in psa_counts.items():
+    max_ips = psa_nets[k].num_addresses - 4
+    psa_range = resources['global_addresses'][k]
+    labels = {
+        'network': psa_range['network'],
+        'project': psa_range['project_id'],
+        'psa_range': psa_range['name']
+    }
+    yield TimeSeries('network/psa/addresses_available', max_ips, labels)
+    yield TimeSeries('network/psa/addresses_used', v, labels)
+    yield TimeSeries('network/psa/addresses_used_ratio',
+                     0 if v == 0 else v / max_ips, labels)

fast/stages/FAQ.md

@@ -0,0 +1,23 @@
+
+## 00-bootstrap
+1. How to handle requests where automation, logging and/or billing export projects are not under organization but in different folders.
+    - Run bootstrap stage and let automation, logging and/or billing projects be created under organization.
+    - Run resource manager stage or any other custom stage which creates the folders where these projects will reside. 
+    - Once folders are created add folder ids to varibale "project_parent_ids" in bootstrap stage and run bootstrap stage.
+    - This step will move the projects from organization to the parent folders specificed.
+
+## cicd
+1. Why do we need two seperate ServiceAccounts when configuring cicd pipelines (cicd SA and IaC SA)
+    - Having seperate service accounts helps shutdown the pipeline incase of any issues and still keep IaC SA and ability to run terraform plan/apply manually.
+    - A pipeline can only generate a token that can get access to an SA. It cannot directly call a provider file to impersonate IaC SA.
+    - Having providers file that allows impersonation to IaC SA allows flexibility to run terraform manually or from CICD Pipelines.
+    <p align="center">
+        <img src="IaC_SA.png" alt="CICD SA and IaC SA">
+    </p>
+
+
+
+
+
+
+

modules/apigee/README.md

@@ -180,12 +180,13 @@ module "apigee" {
 
 | name | description | sensitive |
 |---|---|:---:|
-| [envgroups](outputs.tf#L17) | Environment groups. |  |
-| [environments](outputs.tf#L22) | Environment. |  |
-| [instances](outputs.tf#L27) | Instances. |  |
-| [org_id](outputs.tf#L32) | Organization ID. |  |
-| [org_name](outputs.tf#L37) | Organization name. |  |
-| [organization](outputs.tf#L42) | Organization. |  |
-| [service_attachments](outputs.tf#L47) | Service attachments. |  |
+| [endpoint_attachment_hosts](outputs.tf#L17) | Endpoint hosts. |  |
+| [envgroups](outputs.tf#L22) | Environment groups. |  |
+| [environments](outputs.tf#L27) | Environment. |  |
+| [instances](outputs.tf#L32) | Instances. |  |
+| [org_id](outputs.tf#L37) | Organization ID. |  |
+| [org_name](outputs.tf#L42) | Organization name. |  |
+| [organization](outputs.tf#L47) | Organization. |  |
+| [service_attachments](outputs.tf#L52) | Service attachments. |  |
 
 <!-- END TFDOC -->

modules/apigee/outputs.tf

@@ -14,6 +14,11 @@
  * limitations under the License.
  */
 
+output "endpoint_attachment_hosts" {
+  description = "Endpoint hosts."
+  value       = { for k, v in google_apigee_endpoint_attachment.endpoint_attachments : k => v.host }
+}
+
 output "envgroups" {
   description = "Environment groups."
   value       = try(google_apigee_envgroup.envgroups, null)