From 7f5c177cfec1769b6fd91bbb748548862fb34d1c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aur=C3=A9lien=20Legrand?= Date: Wed, 25 Jan 2023 16:02:30 +0100 Subject: [PATCH] Network Dashboard: PSA support for Filestore and Memorystore (#1106) * Support for Filestore and Memorystore PSA ranges Co-authored-by: Ludovico Magnocavallo --- .../dashboards/quotas-utilization.json | 143 ++++++++++-------- .../deploy-cloud-function/README.md | 2 +- .../deploy-cloud-function/variables.tf | 2 +- .../src/plugins/discover-cai.py | 50 +++++- .../src/plugins/series-psa.py | 43 +++++- 5 files changed, 167 insertions(+), 73 deletions(-) diff --git a/blueprints/cloud-operations/network-dashboard/dashboards/quotas-utilization.json b/blueprints/cloud-operations/network-dashboard/dashboards/quotas-utilization.json index 1c11bdb7af..361eb8214d 100644 --- a/blueprints/cloud-operations/network-dashboard/dashboards/quotas-utilization.json +++ b/blueprints/cloud-operations/network-dashboard/dashboards/quotas-utilization.json @@ -1,5 +1,4 @@ { - "category": "CUSTOM", "displayName": "quotas_utilization", "mosaicLayout": { "columns": 12, @@ -18,7 +17,6 @@ "plotType": "LINE", "targetAxis": "Y1", "timeSeriesQuery": { - "apiSource": "DEFAULT_CLOUD", "timeSeriesFilter": { "aggregation": { "alignmentPeriod": "3600s", @@ -40,9 +38,7 @@ } } }, - "width": 6, - "xPos": 0, - "yPos": 0 + "width": 6 }, { "height": 4, @@ -58,7 +54,6 @@ "plotType": "LINE", "targetAxis": "Y1", "timeSeriesQuery": { - "apiSource": "DEFAULT_CLOUD", "timeSeriesFilter": { "aggregation": { "alignmentPeriod": "3600s", @@ -81,7 +76,6 @@ } }, "width": 6, - "xPos": 0, "yPos": 12 }, { @@ -98,7 +92,6 @@ "plotType": "LINE", "targetAxis": "Y1", "timeSeriesQuery": { - "apiSource": "DEFAULT_CLOUD", "timeSeriesFilter": { "aggregation": { "alignmentPeriod": "3600s", @@ -121,7 +114,6 @@ } }, "width": 6, - "xPos": 0, "yPos": 8 }, { @@ -138,7 +130,6 @@ "plotType": "LINE", "targetAxis": "Y1", "timeSeriesQuery": { - "apiSource": "DEFAULT_CLOUD", "timeSeriesFilter": { "aggregation": { "alignmentPeriod": "3600s", @@ -178,7 +169,6 @@ "plotType": "LINE", "targetAxis": "Y1", "timeSeriesQuery": { - "apiSource": "DEFAULT_CLOUD", "timeSeriesFilter": { "aggregation": { "alignmentPeriod": "3600s", @@ -201,7 +191,6 @@ } }, "width": 6, - "xPos": 0, "yPos": 4 }, { @@ -218,7 +207,6 @@ "plotType": "LINE", "targetAxis": "Y1", "timeSeriesQuery": { - "apiSource": "DEFAULT_CLOUD", "timeSeriesFilter": { "aggregation": { "alignmentPeriod": "3600s", @@ -241,8 +229,7 @@ } }, "width": 6, - "xPos": 6, - "yPos": 0 + "xPos": 6 }, { "height": 4, @@ -258,7 +245,6 @@ "plotType": "LINE", "targetAxis": "Y1", "timeSeriesQuery": { - "apiSource": "DEFAULT_CLOUD", "timeSeriesFilter": { "aggregation": { "alignmentPeriod": "3600s", @@ -298,7 +284,6 @@ "plotType": "LINE", "targetAxis": "Y1", "timeSeriesQuery": { - "apiSource": "DEFAULT_CLOUD", "timeSeriesFilter": { "aggregation": { "alignmentPeriod": "3600s", @@ -330,17 +315,19 @@ }, "dataSets": [ { - "minAlignmentPeriod": "60s", + "minAlignmentPeriod": "3600s", "plotType": "LINE", "targetAxis": "Y1", "timeSeriesQuery": { - "apiSource": "DEFAULT_CLOUD", "timeSeriesFilter": { "aggregation": { - "alignmentPeriod": "60s", - "perSeriesAligner": "ALIGN_MEAN" + "alignmentPeriod": "3600s", + "perSeriesAligner": "ALIGN_NEXT_OLDER" }, - "filter": "metric.type=\"custom.googleapis.com/netmon/peering_group/routes_dynamic_used_ratio\" resource.type=\"global\"" + "filter": "metric.type=\"custom.googleapis.com/netmon/peering_group/routes_dynamic_used_ratio\" resource.type=\"global\"", + "secondaryAggregation": { + "alignmentPeriod": "60s" + } } } } @@ -353,7 +340,6 @@ } }, "width": 6, - "xPos": 0, "yPos": 20 }, { @@ -366,21 +352,23 @@ }, "dataSets": [ { - "minAlignmentPeriod": "60s", + "minAlignmentPeriod": "3600s", "plotType": "LINE", "targetAxis": "Y1", "timeSeriesQuery": { - "apiSource": "DEFAULT_CLOUD", "timeSeriesFilter": { "aggregation": { - "alignmentPeriod": "60s", + "alignmentPeriod": "3600s", "crossSeriesReducer": "REDUCE_SUM", "groupByFields": [ "metric.label.\"project\"" ], - "perSeriesAligner": "ALIGN_MEAN" + "perSeriesAligner": "ALIGN_NEXT_OLDER" }, - "filter": "metric.type=\"custom.googleapis.com/netmon/project/firewall_rules_used_ratio\" resource.type=\"global\"" + "filter": "metric.type=\"custom.googleapis.com/netmon/project/firewall_rules_used_ratio\" resource.type=\"global\"", + "secondaryAggregation": { + "alignmentPeriod": "60s" + } } } } @@ -393,8 +381,7 @@ } }, "width": 6, - "xPos": 0, - "yPos": 32 + "yPos": 28 }, { "height": 4, @@ -406,17 +393,19 @@ }, "dataSets": [ { - "minAlignmentPeriod": "60s", + "minAlignmentPeriod": "3600s", "plotType": "LINE", "targetAxis": "Y1", "timeSeriesQuery": { - "apiSource": "DEFAULT_CLOUD", "timeSeriesFilter": { "aggregation": { - "alignmentPeriod": "60s", - "perSeriesAligner": "ALIGN_MEAN" + "alignmentPeriod": "3600s", + "perSeriesAligner": "ALIGN_NEXT_OLDER" }, - "filter": "metric.type=\"custom.googleapis.com/netmon/firewall_policy/tuples_used_ratio\" resource.type=\"global\"" + "filter": "metric.type=\"custom.googleapis.com/netmon/firewall_policy/tuples_used_ratio\" resource.type=\"global\"", + "secondaryAggregation": { + "alignmentPeriod": "60s" + } } } } @@ -430,7 +419,7 @@ }, "width": 6, "xPos": 6, - "yPos": 28 + "yPos": 24 }, { "height": 4, @@ -442,17 +431,19 @@ }, "dataSets": [ { - "minAlignmentPeriod": "60s", + "minAlignmentPeriod": "3600s", "plotType": "LINE", "targetAxis": "Y1", "timeSeriesQuery": { - "apiSource": "DEFAULT_CLOUD", "timeSeriesFilter": { "aggregation": { - "alignmentPeriod": "60s", - "perSeriesAligner": "ALIGN_MEAN" + "alignmentPeriod": "3600s", + "perSeriesAligner": "ALIGN_NEXT_OLDER" }, - "filter": "metric.type=\"custom.googleapis.com/netmon/subnetwork/addresses_used_ratio\" resource.type=\"global\"" + "filter": "metric.type=\"custom.googleapis.com/netmon/subnetwork/addresses_used_ratio\" resource.type=\"global\"", + "secondaryAggregation": { + "alignmentPeriod": "60s" + } } } } @@ -465,7 +456,6 @@ } }, "width": 6, - "xPos": 6, "yPos": 16 }, { @@ -478,30 +468,27 @@ }, "dataSets": [ { - "minAlignmentPeriod": "60s", + "minAlignmentPeriod": "3600s", "plotType": "LINE", "targetAxis": "Y1", "timeSeriesQuery": { - "apiSource": "DEFAULT_CLOUD", "timeSeriesFilter": { "aggregation": { - "alignmentPeriod": "60s", + "alignmentPeriod": "3600s", "crossSeriesReducer": "REDUCE_SUM", "groupByFields": [ "metric.label.\"project\"" ], - "perSeriesAligner": "ALIGN_MEAN" + "perSeriesAligner": "ALIGN_NEXT_OLDER" }, "filter": "metric.type=\"custom.googleapis.com/netmon/project/routes_static_used_ratio\" resource.type=\"global\"", "secondaryAggregation": { - "alignmentPeriod": "60s", - "perSeriesAligner": "ALIGN_NONE" + "alignmentPeriod": "60s" } } } } ], - "thresholds": [], "timeshiftDuration": "0s", "yAxis": { "label": "y1Axis", @@ -510,8 +497,8 @@ } }, "width": 6, - "xPos": 0, - "yPos": 24 + "xPos": 6, + "yPos": 20 }, { "height": 4, @@ -523,22 +510,23 @@ }, "dataSets": [ { - "minAlignmentPeriod": "60s", + "minAlignmentPeriod": "3600s", "plotType": "LINE", "targetAxis": "Y1", "timeSeriesQuery": { - "apiSource": "DEFAULT_CLOUD", "timeSeriesFilter": { "aggregation": { - "alignmentPeriod": "60s", - "perSeriesAligner": "ALIGN_MEAN" + "alignmentPeriod": "3600s", + "perSeriesAligner": "ALIGN_NEXT_OLDER" }, - "filter": "metric.type=\"custom.googleapis.com/netmon/peering_group/routes_static_used_ratio\" resource.type=\"global\"" + "filter": "metric.type=\"custom.googleapis.com/netmon/peering_group/routes_static_used_ratio\" resource.type=\"global\"", + "secondaryAggregation": { + "alignmentPeriod": "60s" + } } } } ], - "thresholds": [], "timeshiftDuration": "0s", "yAxis": { "label": "y1Axis", @@ -547,8 +535,45 @@ } }, "width": 6, - "xPos": 0, - "yPos": 28 + "yPos": 24 + }, + { + "height": 4, + "widget": { + "title": "Addresses used ratio per psa range [NEXT OLDER]", + "xyChart": { + "chartOptions": { + "mode": "COLOR" + }, + "dataSets": [ + { + "minAlignmentPeriod": "3600s", + "plotType": "LINE", + "targetAxis": "Y1", + "timeSeriesQuery": { + "timeSeriesFilter": { + "aggregation": { + "alignmentPeriod": "3600s", + "perSeriesAligner": "ALIGN_NEXT_OLDER" + }, + "filter": "metric.type=\"custom.googleapis.com/netmon/network/psa/addresses_used_ratio\" resource.type=\"global\"", + "secondaryAggregation": { + "alignmentPeriod": "60s" + } + } + } + } + ], + "timeshiftDuration": "0s", + "yAxis": { + "label": "y1Axis", + "scale": "LINEAR" + } + } + }, + "width": 6, + "xPos": 6, + "yPos": 16 } ] } diff --git a/blueprints/cloud-operations/network-dashboard/deploy-cloud-function/README.md b/blueprints/cloud-operations/network-dashboard/deploy-cloud-function/README.md index bf1d7b5cf7..aa0bdf42e1 100644 --- a/blueprints/cloud-operations/network-dashboard/deploy-cloud-function/README.md +++ b/blueprints/cloud-operations/network-dashboard/deploy-cloud-function/README.md @@ -74,7 +74,7 @@ dashboard_json_path = "../dashboards/quotas-utilization.json" | [name](variables.tf#L75) | Name used to create Cloud Function related resources. | string | | "net-dash" | | [project_create_config](variables.tf#L81) | Optional configuration if project creation is required. | object({…}) | | null | | [region](variables.tf#L95) | Compute region where the Cloud Function will be deployed. | string | | "europe-west1" | -| [schedule_config](variables.tf#L101) | Schedule timer configuration in crontab format. | string | | "0/30 * * * *" | +| [schedule_config](variables.tf#L101) | Schedule timer configuration in crontab format. | string | | "*/30 * * * *" | ## Outputs diff --git a/blueprints/cloud-operations/network-dashboard/deploy-cloud-function/variables.tf b/blueprints/cloud-operations/network-dashboard/deploy-cloud-function/variables.tf index ab59f91f52..680b689dd8 100644 --- a/blueprints/cloud-operations/network-dashboard/deploy-cloud-function/variables.tf +++ b/blueprints/cloud-operations/network-dashboard/deploy-cloud-function/variables.tf @@ -101,5 +101,5 @@ variable "region" { variable "schedule_config" { description = "Schedule timer configuration in crontab format." type = string - default = "0/30 * * * *" + default = "*/30 * * * *" } diff --git a/blueprints/cloud-operations/network-dashboard/src/plugins/discover-cai.py b/blueprints/cloud-operations/network-dashboard/src/plugins/discover-cai.py index 3041df38a0..1ac62d0664 100644 --- a/blueprints/cloud-operations/network-dashboard/src/plugins/discover-cai.py +++ b/blueprints/cloud-operations/network-dashboard/src/plugins/discover-cai.py @@ -39,7 +39,9 @@ 'subnetworks': 'compute.googleapis.com/Subnetwork', 'routers': 'compute.googleapis.com/Router', 'routes': 'compute.googleapis.com/Route', - 'sql_instances': 'sqladmin.googleapis.com/Instance' + 'sql_instances': 'sqladmin.googleapis.com/Instance', + 'filestore_instances': 'file.googleapis.com/Instance', + 'memorystore_instances': 'redis.googleapis.com/Instance', } NAMES = {v: k for k, v in TYPES.items()} @@ -82,10 +84,16 @@ def _handle_resource(resources, asset_type, data): # e.g. assetType = GlobalAddress but discoveryName = Address resource_name = NAMES[asset_type] resource = { - 'id': attrs.get('id'), - 'name': attrs['name'], - 'self_link': _self_link(attrs['selfLink']), - 'assetType': asset_type + 'id': + attrs.get('id'), + 'name': + attrs['name'], + # Some resources (ex: Filestore) don't have a self_link, using parent + name in that case + 'self_link': + f'{data["parent"]}/{attrs["name"]}' + if not 'selfLink' in attrs else _self_link(attrs['selfLink']), + 'assetType': + asset_type } # derive parent type and id and skip if parent is not within scope parent_data = _get_parent(data['parent'], resources) @@ -212,6 +220,38 @@ def _handle_sql_instances(resource, data): ], 'region': data['region'], 'availabilityType': data['settings']['availabilityType'], + 'network': data['settings']['ipConfiguration']['privateNetwork'] + } + + +def _handle_filestore_instances(resource, data): + 'Handles filestore instance type resource data.' + return { + # Getting only the instance name, removing the rest + 'name': data['name'].split('/')[-1], + # Is a list but for now, only one network is supported for Filestore + 'network': data['networks'][0]['network'], + 'reservedIpRange': data['networks'][0]['reservedIpRange'], + 'ipAddresses': data['networks'][0]['ipAddresses'] + } + + +def _handle_memorystore_instances(resource, data): + 'Handles Memorystore (Redis) instance type resource data.' + return { + # Getting only the instance name, removing the rest + 'name': + data['name'].split('/')[-1], + 'locationId': + data['locationId'], + 'replicaCount': + 0 if not 'replicaCount' in data else data['replicaCount'], + 'network': + data['authorizedNetwork'], + 'reservedIpRange': + '' if not 'reservedIpRange' in data else data['reservedIpRange'], + 'host': + '' if not 'host' in data else data['host'], } diff --git a/blueprints/cloud-operations/network-dashboard/src/plugins/series-psa.py b/blueprints/cloud-operations/network-dashboard/src/plugins/series-psa.py index e9993e0745..82e06009d8 100644 --- a/blueprints/cloud-operations/network-dashboard/src/plugins/series-psa.py +++ b/blueprints/cloud-operations/network-dashboard/src/plugins/series-psa.py @@ -34,7 +34,28 @@ def _sql_addresses(sql_instances): if not v['ipAddresses']: continue # 1 IP for the instance + 1 IP for the ILB + 1 IP if HA - yield v['ipAddresses'][0], 2 if v['availabilityType'] != 'REGIONAL' else 3 + yield v['ipAddresses'][ + 0], 2 if v['availabilityType'] != 'REGIONAL' else 3, v['network'] + + +def _filestore_addresses(filestore_instances): + 'Returns counts of Filestore instances per PSA range.' + for v in filestore_instances.values(): + if not v['ipAddresses'] or not v['reservedIpRange']: + continue + # Subnet size (reservedIpRange) can be /29, /26 or /24 + yield v['ipAddresses'][0], ipaddress.ip_network( + v['reservedIpRange']).num_addresses, v['network'] + + +def _memorystore_addresses(memorystore_instances): + 'Returns counts of Memorystore (Redis) instances per PSA range.' + for v in memorystore_instances.values(): + if not v['reservedIpRange'] or v['reservedIpRange'] == '': + continue + # Subnet size (reservedIpRange) can be minimum /28 or /29 + yield v['host'], ipaddress.ip_network( + v['reservedIpRange']).num_addresses, v['network'] @register_timeseries @@ -46,26 +67,34 @@ def timeseries(resources): ('project', 'network', 'subnetwork'), dtype.endswith('ratio')) psa_nets = { - k: ipaddress.ip_network('{}/{}'.format(v['address'], v['prefixLength'])) - for k, v in resources['global_addresses'].items() - if v['prefixLength'] + k: { + 'network_link': + v['network'], + 'network_prefix': + ipaddress.ip_network('{}/{}'.format(v['address'], + v['prefixLength'])) + } for k, v in resources['global_addresses'].items() if v['prefixLength'] } psa_counts = {} - for address, ip_count in _sql_addresses(resources.get('sql_instances', {})): + for address, ip_count, network in itertools.chain( + _sql_addresses(resources.get('sql_instances', {})), + _filestore_addresses(resources.get('filestore_instances', {})), + _memorystore_addresses(resources.get('memorystore_instances', {}))): ip_address = ipaddress.ip_address(address) for k, v in psa_nets.items(): - if ip_address in v: + if network == v['network_link'] and ip_address in v['network_prefix']: psa_counts[k] = psa_counts.get(k, 0) + ip_count break for k, v in psa_counts.items(): - max_ips = psa_nets[k].num_addresses - 4 + max_ips = psa_nets[k]['network_prefix'].num_addresses - 4 psa_range = resources['global_addresses'][k] labels = { 'network': psa_range['network'], 'project': psa_range['project_id'], 'psa_range': psa_range['name'] } + yield TimeSeries('network/psa/addresses_available', max_ips, labels) yield TimeSeries('network/psa/addresses_used', v, labels) yield TimeSeries('network/psa/addresses_used_ratio',