From 8b7bf698d2ba752466866144739d31ef08641a29 Mon Sep 17 00:00:00 2001 From: Maurizio Noseda Pedraglio Date: Fri, 30 Sep 2022 09:55:38 +0200 Subject: [PATCH 1/5] made alert creation optional --- .../cloud-operations/quota-monitoring/main.tf | 54 ++++++++++--------- .../quota-monitoring/variables.tf | 6 +++ 2 files changed, 34 insertions(+), 26 deletions(-) diff --git a/blueprints/cloud-operations/quota-monitoring/main.tf b/blueprints/cloud-operations/quota-monitoring/main.tf index 0a4de23ba6..0593f4fd10 100644 --- a/blueprints/cloud-operations/quota-monitoring/main.tf +++ b/blueprints/cloud-operations/quota-monitoring/main.tf @@ -106,34 +106,36 @@ resource "google_project_iam_member" "quota_viewer" { member = module.cf.service_account_iam_email } -resource "google_monitoring_alert_policy" "alert_policy" { - project = module.project.project_id - display_name = "Quota monitor" - combiner = "OR" - conditions { - display_name = "simple quota threshold for cpus utilization" - condition_threshold { - filter = "metric.type=\"custom.googleapis.com/quota/cpus_utilization\" resource.type=\"global\"" - threshold_value = 0.75 - comparison = "COMPARISON_GT" - duration = "0s" - aggregations { - alignment_period = "60s" - group_by_fields = [] - per_series_aligner = "ALIGN_MEAN" - } - trigger { - count = 1 - percent = 0 +var.create_alert ? { + resource "google_monitoring_alert_policy" "alert_policy" { + project = module.project.project_id + display_name = "Quota monitor" + combiner = "OR" + conditions { + display_name = "simple quota threshold for cpus utilization" + condition_threshold { + filter = "metric.type=\"custom.googleapis.com/quota/cpus_utilization\" resource.type=\"global\"" + threshold_value = 0.75 + comparison = "COMPARISON_GT" + duration = "0s" + aggregations { + alignment_period = "60s" + group_by_fields = [] + per_series_aligner = "ALIGN_MEAN" + } + trigger { + count = 1 + percent = 0 + } } } - } - enabled = false - user_labels = { - name = var.name - } - documentation { - content = "GCE cpus quota over threshold." + enabled = false + user_labels = { + name = var.name + } + documentation { + content = "GCE cpus quota over threshold." + } } } diff --git a/blueprints/cloud-operations/quota-monitoring/variables.tf b/blueprints/cloud-operations/quota-monitoring/variables.tf index 2b69aa1cb5..cce24ca402 100644 --- a/blueprints/cloud-operations/quota-monitoring/variables.tf +++ b/blueprints/cloud-operations/quota-monitoring/variables.tf @@ -20,6 +20,12 @@ variable "bundle_path" { default = "./bundle.zip" } +variable "create_alert" { + description = "Enables the creation of a sample monitoring alert, false by default." + type = bool + default = false +} + variable "name" { description = "Arbitrary string used to name created resources." type = string From 10811b3d312f498a718493eeb55663ba5818fc6e Mon Sep 17 00:00:00 2001 From: Maurizio Noseda Pedraglio Date: Fri, 30 Sep 2022 10:27:31 +0200 Subject: [PATCH 2/5] fixed conditional creation and updated readme --- .../quota-monitoring/README.md | 2 +- .../cloud-operations/quota-monitoring/main.tf | 57 ++++++++++--------- .../quota-monitoring/variables.tf | 14 ++--- 3 files changed, 37 insertions(+), 36 deletions(-) diff --git a/blueprints/cloud-operations/quota-monitoring/README.md b/blueprints/cloud-operations/quota-monitoring/README.md index c80a77564f..fcfad1e7f8 100644 --- a/blueprints/cloud-operations/quota-monitoring/README.md +++ b/blueprints/cloud-operations/quota-monitoring/README.md @@ -28,7 +28,7 @@ Labels are set with project id (which may differ from the monitoring workspace p GCP Metrics Explorer, usage, limit and utilization view sample -The solution also creates a basic monitoring alert policy, to demonstrate how to raise alerts when quotas utilization goes over a predefined threshold. +The solution can also create a basic monitoring alert policy, to demonstrate how to raise alerts when quotas utilization goes over a predefined threshold, to enable it, set variable `alert_create` to true and reapply main.tf after main.py has run at least one and quota monitoring metrics have been creaed. ## Running the blueprint diff --git a/blueprints/cloud-operations/quota-monitoring/main.tf b/blueprints/cloud-operations/quota-monitoring/main.tf index 0593f4fd10..674482c87f 100644 --- a/blueprints/cloud-operations/quota-monitoring/main.tf +++ b/blueprints/cloud-operations/quota-monitoring/main.tf @@ -106,39 +106,40 @@ resource "google_project_iam_member" "quota_viewer" { member = module.cf.service_account_iam_email } -var.create_alert ? { - resource "google_monitoring_alert_policy" "alert_policy" { - project = module.project.project_id - display_name = "Quota monitor" - combiner = "OR" - conditions { - display_name = "simple quota threshold for cpus utilization" - condition_threshold { - filter = "metric.type=\"custom.googleapis.com/quota/cpus_utilization\" resource.type=\"global\"" - threshold_value = 0.75 - comparison = "COMPARISON_GT" - duration = "0s" - aggregations { - alignment_period = "60s" - group_by_fields = [] - per_series_aligner = "ALIGN_MEAN" - } - trigger { - count = 1 - percent = 0 - } + +resource "google_monitoring_alert_policy" "alert_policy" { + count = var.alert_create ? 1 : 0 + project = module.project.project_id + display_name = "Quota monitor" + combiner = "OR" + conditions { + display_name = "simple quota threshold for cpus utilization" + condition_threshold { + filter = "metric.type=\"custom.googleapis.com/quota/cpus_utilization\" resource.type=\"global\"" + threshold_value = 0.75 + comparison = "COMPARISON_GT" + duration = "0s" + aggregations { + alignment_period = "60s" + group_by_fields = [] + per_series_aligner = "ALIGN_MEAN" + } + trigger { + count = 1 + percent = 0 } - } - enabled = false - user_labels = { - name = var.name - } - documentation { - content = "GCE cpus quota over threshold." } } + enabled = false + user_labels = { + name = var.name + } + documentation { + content = "GCE cpus quota over threshold." + } } + resource "random_pet" "random" { length = 1 } diff --git a/blueprints/cloud-operations/quota-monitoring/variables.tf b/blueprints/cloud-operations/quota-monitoring/variables.tf index cce24ca402..53ca0f6361 100644 --- a/blueprints/cloud-operations/quota-monitoring/variables.tf +++ b/blueprints/cloud-operations/quota-monitoring/variables.tf @@ -14,18 +14,18 @@ * limitations under the License. */ +variable "alert_create" { + description = "Enables the creation of a sample monitoring alert, false by default." + type = bool + default = false +} + variable "bundle_path" { description = "Path used to write the intermediate Cloud Function code bundle." type = string default = "./bundle.zip" } -variable "create_alert" { - description = "Enables the creation of a sample monitoring alert, false by default." - type = bool - default = false -} - variable "name" { description = "Arbitrary string used to name created resources." type = string @@ -33,7 +33,7 @@ variable "name" { } variable "project_create" { - description = "Create project instead ofusing an existing one." + description = "Create project instead of using an existing one." type = bool default = false } From eea4642cf2a771e36d9f13afc3a3c1624d431ded Mon Sep 17 00:00:00 2001 From: Maurizio Noseda Pedraglio Date: Fri, 30 Sep 2022 11:07:44 +0200 Subject: [PATCH 3/5] applied tfdoc --- .../cloud-operations/quota-monitoring/README.md | 15 ++++++++------- .../cloud-operations/quota-monitoring/main.tf | 2 +- 2 files changed, 9 insertions(+), 8 deletions(-) diff --git a/blueprints/cloud-operations/quota-monitoring/README.md b/blueprints/cloud-operations/quota-monitoring/README.md index fcfad1e7f8..adcd89d643 100644 --- a/blueprints/cloud-operations/quota-monitoring/README.md +++ b/blueprints/cloud-operations/quota-monitoring/README.md @@ -42,12 +42,13 @@ Clone this repository or [open it in cloud shell](https://ssh.cloud.google.com/c | name | description | type | required | default | |---|---|:---:|:---:|:---:| -| [project_id](variables.tf#L35) | Project id that references existing project. | string | ✓ | | -| [bundle_path](variables.tf#L17) | Path used to write the intermediate Cloud Function code bundle. | string | | "./bundle.zip" | -| [name](variables.tf#L23) | Arbitrary string used to name created resources. | string | | "quota-monitor" | -| [project_create](variables.tf#L29) | Create project instead ofusing an existing one. | bool | | false | -| [quota_config](variables.tf#L40) | Cloud function configuration. | object({…}) | | {…} | -| [region](variables.tf#L54) | Compute region used in the example. | string | | "europe-west1" | -| [schedule_config](variables.tf#L60) | Schedule timer configuration in crontab format. | string | | "0 * * * *" | +| [project_id](variables.tf#L41) | Project id that references existing project. | string | ✓ | | +| [alert_create](variables.tf#L17) | Enables the creation of a sample monitoring alert, false by default. | bool | | false | +| [bundle_path](variables.tf#L23) | Path used to write the intermediate Cloud Function code bundle. | string | | "./bundle.zip" | +| [name](variables.tf#L29) | Arbitrary string used to name created resources. | string | | "quota-monitor" | +| [project_create](variables.tf#L35) | Create project instead of using an existing one. | bool | | false | +| [quota_config](variables.tf#L46) | Cloud function configuration. | object({…}) | | {…} | +| [region](variables.tf#L60) | Compute region used in the example. | string | | "europe-west1" | +| [schedule_config](variables.tf#L66) | Schedule timer configuration in crontab format. | string | | "0 * * * *" | diff --git a/blueprints/cloud-operations/quota-monitoring/main.tf b/blueprints/cloud-operations/quota-monitoring/main.tf index 674482c87f..13804ba943 100644 --- a/blueprints/cloud-operations/quota-monitoring/main.tf +++ b/blueprints/cloud-operations/quota-monitoring/main.tf @@ -108,7 +108,7 @@ resource "google_project_iam_member" "quota_viewer" { resource "google_monitoring_alert_policy" "alert_policy" { - count = var.alert_create ? 1 : 0 + count = var.alert_create ? 1 : 0 project = module.project.project_id display_name = "Quota monitor" combiner = "OR" From 1c134a735be4c532f44cde595a21ef639530c804 Mon Sep 17 00:00:00 2001 From: Maurizio Noseda Pedraglio Date: Fri, 30 Sep 2022 11:10:48 +0200 Subject: [PATCH 4/5] tracking of vpc firewalls, initial commit --- .../network-dashboard/cloud-function/main.py | 28 ++-- .../cloud-function/metrics.yaml | 12 ++ .../cloud-function/metrics/limits.py | 55 +++++++ .../cloud-function/metrics/metrics.py | 12 +- .../cloud-function/metrics/vpc_firewalls.py | 141 ++++++++++++++++++ 5 files changed, 234 insertions(+), 14 deletions(-) create mode 100644 blueprints/cloud-operations/network-dashboard/cloud-function/metrics/vpc_firewalls.py diff --git a/blueprints/cloud-operations/network-dashboard/cloud-function/main.py b/blueprints/cloud-operations/network-dashboard/cloud-function/main.py index ecb618ad15..749762a6cf 100644 --- a/blueprints/cloud-operations/network-dashboard/cloud-function/main.py +++ b/blueprints/cloud-operations/network-dashboard/cloud-function/main.py @@ -22,7 +22,7 @@ from google.cloud import monitoring_v3, asset_v1 from google.protobuf import field_mask_pb2 from googleapiclient import discovery -from metrics import ilb_fwrules, instances, networks, metrics, limits, peerings, routes +from metrics import ilb_fwrules, instances, networks, metrics, limits, peerings, routes, vpc_firewalls def monitoring_interval(): @@ -49,15 +49,18 @@ def monitoring_interval(): config = { # Organization ID containing the projects to be monitored - "organization": - os.environ.get("ORGANIZATION_ID"), + "organization": #os.environ.get("ORGANIZATION_ID"), + '34855741773', # list of projects from which function will get quotas information - "monitored_projects": - os.environ.get("MONITORED_PROJECTS_LIST").split(","), - "monitoring_project_link": - os.environ.get('MONITORING_PROJECT_ID'), - "monitoring_project_link": - f"projects/{os.environ.get('MONITORING_PROJECT_ID')}", + "monitored_projects": #os.environ.get("MONITORED_PROJECTS_LIST").split(","), + [ + "mnoseda-prod-net-landing-0", "mnoseda-prod-net-spoke-0", + "mnoseda-dev-net-spoke-0" + ], + "monitoring_project": #os.environ.get('MONITORING_PROJECT_ID'), + "monitoring-tlc", + "monitoring_project_link": #f"projects/{os.environ.get('MONITORING_PROJECT_ID')}", + f"projects/monitoring-tlc", "monitoring_interval": monitoring_interval(), "limit_names": { @@ -98,6 +101,9 @@ def main(event, context): metrics_dict, limits_dict = metrics.create_metrics( config["monitoring_project_link"]) + project_quotas_dict = limits.get_quota_project_limit(config) + + firewalls_dict = vpc_firewalls.get_firewalls_dict(config) # Asset inventory queries gce_instance_dict = instances.get_gce_instance_dict(config) @@ -105,6 +111,10 @@ def main(event, context): l7_forwarding_rules_dict = ilb_fwrules.get_forwarding_rules_dict(config, "L7") subnet_range_dict = networks.get_subnet_ranges_dict(config) + # Per Project metrics + vpc_firewalls.get_firewalls_data(config, metrics_dict, project_quotas_dict, + firewalls_dict) + # Per Network metrics instances.get_gce_instances_data(config, metrics_dict, gce_instance_dict, limits_dict['number_of_instances_limit']) diff --git a/blueprints/cloud-operations/network-dashboard/cloud-function/metrics.yaml b/blueprints/cloud-operations/network-dashboard/cloud-function/metrics.yaml index 2e5621d7fb..9a2d60440d 100644 --- a/blueprints/cloud-operations/network-dashboard/cloud-function/metrics.yaml +++ b/blueprints/cloud-operations/network-dashboard/cloud-function/metrics.yaml @@ -149,3 +149,15 @@ metrics_per_peering_group: utilization: name: dynamic_routes_per_peering_group_utilization description: Number of Dynamic routes per peering group - utilization. +metrics_per_project: + firewalls: + usage: + name: firewalls_per_project_vpc_usage + description: Number of VPC firewall rules in a project - usage. + limit: + #no default limit as we can assume quotas can be always read from gcloud API + name: firewalls_per_project_limit + description: Number of VPC firewall rules in a project - limit. + utilization: + name: firewalls_per_project_utilization + description: Number of VPC firewall rules in a project - utilization. diff --git a/blueprints/cloud-operations/network-dashboard/cloud-function/metrics/limits.py b/blueprints/cloud-operations/network-dashboard/cloud-function/metrics/limits.py index 8b3d5ae97b..c73d15d3a8 100644 --- a/blueprints/cloud-operations/network-dashboard/cloud-function/metrics/limits.py +++ b/blueprints/cloud-operations/network-dashboard/cloud-function/metrics/limits.py @@ -14,6 +14,8 @@ # limitations under the License. # +from http.cookiejar import LWPCookieJar +from urllib import response from google.api_core import exceptions from google.cloud import monitoring_v3 from . import metrics @@ -71,6 +73,59 @@ def set_limits(network_dict, quota_limit, limit_dict): network_dict['limit'] = 0 +def get_quotas_dict(quotas_list): + ''' + Creates a dictionary of quotas from a list, with lowe case keys + Parameters: + quotas_array (array): array of quotas + Returns: + quotas_dict (dict): dictionary of quotas + ''' + quota_keys=[q['metric'] for q in quotas_list] + quotas_dict=dict() + i=0 + for key in quota_keys: + if ("metric" in quotas_list[i] ): + del(quotas_list[i]["metric"]) + quotas_dict[key.lower()]=quotas_list[i] + i+=1 + return quotas_dict + + + +def get_quota_project_limit(config,regions=["global"]): + ''' + Retrieves limit for a specific project quota + Parameters: + project_link (string): Project link. + Returns: + quotas (dict): quotas for all selected regions, default 'global' + ''' + try: + request={} + quotas=dict() + for project in config["monitored_projects"]: + quotas[project]=dict() + if regions != ["global"]: + for region in regions: + request=config["clients"]["discovery_client"].compute.regions().get(region=region,project=project) + response=request.execute() + quotas[project][region]=get_quotas_dict(response['quotas']) + else: + region="global" + request=config["clients"]["discovery_client"].projects().get(project=project,fields="quotas") + response=request.execute() + quotas[project][region]=get_quotas_dict(response['quotas']) + + return quotas + except exceptions.PermissionDenied as err: + print( + f"Warning: error reading quotas for {project}. " + + f"This can happen if you don't have permissions on the project, for example if the project is in another organization or a Google managed project" + ) + return None + +#def get_project_quota_current_limit(config,project_link,metric_name,) def get_quota_current_limit(config, project_link, metric_name): ''' Retrieves limit for a specific metric. diff --git a/blueprints/cloud-operations/network-dashboard/cloud-function/metrics/metrics.py b/blueprints/cloud-operations/network-dashboard/cloud-function/metrics/metrics.py index 14183f1571..4a6267ed5f 100644 --- a/blueprints/cloud-operations/network-dashboard/cloud-function/metrics/metrics.py +++ b/blueprints/cloud-operations/network-dashboard/cloud-function/metrics/metrics.py @@ -37,7 +37,7 @@ def create_metrics(monitoring_project): existing_metrics.append(desc.type) limits_dict = {} - with open("metrics.yaml", 'r') as stream: + with open("/Users/mnoseda/Fabric/cloud-foundation-fabric/blueprints/cloud-operations/network-dashboard/cloud-function/metrics.yaml", 'r') as stream: #f try: metrics_dict = yaml.safe_load(stream) @@ -52,8 +52,9 @@ def create_metrics(monitoring_project): # Parse limits (both default values and network specific ones) if sub_metric_key == "limit": limits_dict_for_metric = {} - for network_link, limit_value in sub_metric["values"].items(): - limits_dict_for_metric[network_link] = limit_value + if "values" in sub_metric: + for network_link, limit_value in sub_metric["values"].items(): + limits_dict_for_metric[network_link] = limit_value limits_dict[sub_metric["name"]] = limits_dict_for_metric return metrics_dict, limits_dict @@ -85,7 +86,7 @@ def create_metric(metric_name, description, monitoring_project): def write_data_to_metric(config, monitored_project_id, value, metric_name, - network_name): + network_name=None): ''' Writes data to Cloud Monitoring custom metrics. @@ -104,7 +105,8 @@ def write_data_to_metric(config, monitored_project_id, value, metric_name, series = monitoring_v3.TimeSeries() series.metric.type = f"custom.googleapis.com/{metric_name}" series.resource.type = "global" - series.metric.labels["network_name"] = network_name + if network_name: + series.metric.labels["network_name"] = network_name series.metric.labels["project"] = monitored_project_id now = time.time() diff --git a/blueprints/cloud-operations/network-dashboard/cloud-function/metrics/vpc_firewalls.py b/blueprints/cloud-operations/network-dashboard/cloud-function/metrics/vpc_firewalls.py new file mode 100644 index 0000000000..1bc53da690 --- /dev/null +++ b/blueprints/cloud-operations/network-dashboard/cloud-function/metrics/vpc_firewalls.py @@ -0,0 +1,141 @@ +# +# Copyright 2022 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import re +from collections import defaultdict +from pydoc import doc +from collections import defaultdict +from google.protobuf import field_mask_pb2 +from . import metrics, networks, limits, peerings, routers + + + + +def get_firewalls_dict(config: dict): + ''' + Calls the Asset Inventory API to get all VPC Firewall Rules under the GCP organization. + + Parameters: + config (dict): The dict containing config like clients and limits + Returns: + firewalls_dict (dictionary of dictionary: int): Keys are projects, subkeys are networks, values count #of VPC Firewall Rules + ''' + + firewalls_dict = defaultdict(int) + read_mask = field_mask_pb2.FieldMask() + read_mask.FromJsonString('name,versionedResources') + + + response = config["clients"]["asset_client"].search_all_resources( + request={ + "scope": f"organizations/{config['organization']}", + "asset_types": ["compute.googleapis.com/Firewall"], + "read_mask": read_mask, + }) + for resource in response: + project_id=re.search("(compute.googleapis.com/projects/)([\w\-\d]+)", resource.name).group(2) + network_name="" + for versioned in resource.versioned_resources: + for field_name, field_value in versioned.resource.items(): + if field_name == "network": + network_name = re.search("[a-z0-9\-]*$", field_value).group(0) + firewalls_dict[project_id]= defaultdict(int) if not project_id in firewalls_dict else firewalls_dict[project_id] + firewalls_dict[project_id][network_name]=1 if not network_name in firewalls_dict[project_id] else firewalls_dict[project_id][network_name]+1 + break + break + return firewalls_dict + + + +'''def get_firewalls_per_project_vpc(config, project_id ): + + Returns returns the VPC firewall rules defined in a project + + Parameters: + config (dict): The dict containing config like clients and limits + project_id (string): Project ID for the project containing the Cloud Router. + Returns: + sum_firewalls_per_vpc(dict): Number of firewall rules defined for each VPC in a project + + get_firewalls_dict(config) + sum_firewalls_per_vpc=dict() + sum_firewalls=0 + page_token=None + while (1): + request = config["clients"]["discovery_client"].firewalls().list( + project=project_id,pageToken=page_token) + response = request.execute() + sum_firewalls_per_vpc = dict() + if 'items' in response: + for firewall_rule in response['items']: + sum_firewalls_per_vpc[firewall_rule['network']] = sum_firewalls_per_vpc[firewall_rule['network']]+1 if firewall_rule['network'] in sum_firewalls_per_vpc else 1 + else: + break + page_token = response['pageToken'] if 'pageToken' in response else None + if (not page_token): + break + + return sum_firewalls_per_vpc +''' + +def get_firewalls_data(config, metrics_dict,project_quotas_dict,firewalls_dict): + ''' + Gets the data for VPC Firewall Rules per VPC Network and writes it to the metric defined in vpc_firewalls_metric. + + Parameters: + config (dict): The dict containing config like clients and limits + metrics_dict (dictionary of dictionary of string: string): metrics names and descriptions. + limit_dict (dictionary of string:int): Dictionary with the network link as key and the limit as value. + firewalls_dict (dictionary of dictionary): Keys are projects, subkeys are networks, values count #of VPC Firewall Rules + Returns: + None + ''' + for project in config["monitored_projects"]: + + current_quota_limit= project_quotas_dict[project]['global']["firewalls"] + if current_quota_limit is None: + print( + f"Could not write VPC firewal rules to metric for projects/{project} due to missing quotas" + ) + continue + + + network_dict = networks.get_networks(config, project) + + project_usage=0 + for net in network_dict: + usage = 0 + if net['network_name'] in firewalls_dict: + usage = firewalls_dict[project][net['network_name']] + project_usage+=usage + metrics.write_data_to_metric( + config, project, usage, metrics_dict["metrics_per_project"] + [f"firewalls"]["usage"]["name"], + net['network_name']) + + #firewall quotas are per project, not per single VPC + metrics.write_data_to_metric( + config, project, current_quota_limit['limit'], metrics_dict["metrics_per_project"] + [f"firewalls"]["limit"]["name"], + ) + metrics.write_data_to_metric( + config, project, project_usage / current_quota_limit['limit'] if current_quota_limit['limit']!=0 else 0, + metrics_dict["metrics_per_project"] + [f"firewalls"]["utilization"]["name"],) + + print( + f"Wrote number of VPC Firewall Rules to metric for projects/{project}" + ) From ec54165a893f3637ca7b7a00bc3c21e7003a7253 Mon Sep 17 00:00:00 2001 From: Maurizio Noseda Pedraglio Date: Fri, 30 Sep 2022 11:55:11 +0200 Subject: [PATCH 5/5] reverted stuff pushed by mistake --- .../network-dashboard/cloud-function/main.py | 28 ++-- .../cloud-function/metrics.yaml | 14 +- .../cloud-function/metrics/limits.py | 57 +------ .../cloud-function/metrics/metrics.py | 14 +- .../cloud-function/metrics/vpc_firewalls.py | 141 ------------------ 5 files changed, 16 insertions(+), 238 deletions(-) delete mode 100644 blueprints/cloud-operations/network-dashboard/cloud-function/metrics/vpc_firewalls.py diff --git a/blueprints/cloud-operations/network-dashboard/cloud-function/main.py b/blueprints/cloud-operations/network-dashboard/cloud-function/main.py index 208f589e9a..9f28413638 100644 --- a/blueprints/cloud-operations/network-dashboard/cloud-function/main.py +++ b/blueprints/cloud-operations/network-dashboard/cloud-function/main.py @@ -89,18 +89,15 @@ def monitoring_interval(): config = { # Organization ID containing the projects to be monitored - "organization": #os.environ.get("ORGANIZATION_ID"), - '34855741773', + "organization": + os.environ.get("ORGANIZATION_ID"), # list of projects from which function will get quotas information - "monitored_projects": #os.environ.get("MONITORED_PROJECTS_LIST").split(","), - [ - "mnoseda-prod-net-landing-0", "mnoseda-prod-net-spoke-0", - "mnoseda-dev-net-spoke-0" - ], - "monitoring_project": #os.environ.get('MONITORING_PROJECT_ID'), - "monitoring-tlc", - "monitoring_project_link": #f"projects/{os.environ.get('MONITORING_PROJECT_ID')}", - f"projects/monitoring-tlc", + "monitored_projects": + os.environ.get("MONITORED_PROJECTS_LIST").split(","), + "monitoring_project_link": + os.environ.get('MONITORING_PROJECT_ID'), + "monitoring_project_link": + f"projects/{os.environ.get('MONITORING_PROJECT_ID')}", "monitoring_interval": monitoring_interval(), "limit_names": { @@ -146,9 +143,6 @@ def main(event, context): metrics_dict, limits_dict = metrics.create_metrics( config["monitoring_project_link"]) - project_quotas_dict = limits.get_quota_project_limit(config) - - firewalls_dict = vpc_firewalls.get_firewalls_dict(config) # IP utilization subnet level metrics subnets.get_subnets(config, metrics_dict) @@ -159,10 +153,6 @@ def main(event, context): l7_forwarding_rules_dict = ilb_fwrules.get_forwarding_rules_dict(config, "L7") subnet_range_dict = networks.get_subnet_ranges_dict(config) - # Per Project metrics - vpc_firewalls.get_firewalls_data(config, metrics_dict, project_quotas_dict, - firewalls_dict) - # Per Network metrics instances.get_gce_instances_data(config, metrics_dict, gce_instance_dict, limits_dict['number_of_instances_limit']) @@ -207,4 +197,4 @@ def main(event, context): if __name__ == "__main__": - main(None, None) + main(None, None) \ No newline at end of file diff --git a/blueprints/cloud-operations/network-dashboard/cloud-function/metrics.yaml b/blueprints/cloud-operations/network-dashboard/cloud-function/metrics.yaml index be9cfe1fb3..05f5369c79 100644 --- a/blueprints/cloud-operations/network-dashboard/cloud-function/metrics.yaml +++ b/blueprints/cloud-operations/network-dashboard/cloud-function/metrics.yaml @@ -159,16 +159,4 @@ metrics_per_peering_group: default_value: 300 utilization: name: dynamic_routes_per_peering_group_utilization - description: Number of Dynamic routes per peering group - utilization. -metrics_per_project: - firewalls: - usage: - name: firewalls_per_project_vpc_usage - description: Number of VPC firewall rules in a project - usage. - limit: - #no default limit as we can assume quotas can be always read from gcloud API - name: firewalls_per_project_limit - description: Number of VPC firewall rules in a project - limit. - utilization: - name: firewalls_per_project_utilization - description: Number of VPC firewall rules in a project - utilization. + description: Number of Dynamic routes per peering group - utilization. \ No newline at end of file diff --git a/blueprints/cloud-operations/network-dashboard/cloud-function/metrics/limits.py b/blueprints/cloud-operations/network-dashboard/cloud-function/metrics/limits.py index c73d15d3a8..d17f8be46a 100644 --- a/blueprints/cloud-operations/network-dashboard/cloud-function/metrics/limits.py +++ b/blueprints/cloud-operations/network-dashboard/cloud-function/metrics/limits.py @@ -14,8 +14,6 @@ # limitations under the License. # -from http.cookiejar import LWPCookieJar -from urllib import response from google.api_core import exceptions from google.cloud import monitoring_v3 from . import metrics @@ -73,59 +71,6 @@ def set_limits(network_dict, quota_limit, limit_dict): network_dict['limit'] = 0 -def get_quotas_dict(quotas_list): - ''' - Creates a dictionary of quotas from a list, with lowe case keys - Parameters: - quotas_array (array): array of quotas - Returns: - quotas_dict (dict): dictionary of quotas - ''' - quota_keys=[q['metric'] for q in quotas_list] - quotas_dict=dict() - i=0 - for key in quota_keys: - if ("metric" in quotas_list[i] ): - del(quotas_list[i]["metric"]) - quotas_dict[key.lower()]=quotas_list[i] - i+=1 - return quotas_dict - - - -def get_quota_project_limit(config,regions=["global"]): - ''' - Retrieves limit for a specific project quota - Parameters: - project_link (string): Project link. - Returns: - quotas (dict): quotas for all selected regions, default 'global' - ''' - try: - request={} - quotas=dict() - for project in config["monitored_projects"]: - quotas[project]=dict() - if regions != ["global"]: - for region in regions: - request=config["clients"]["discovery_client"].compute.regions().get(region=region,project=project) - response=request.execute() - quotas[project][region]=get_quotas_dict(response['quotas']) - else: - region="global" - request=config["clients"]["discovery_client"].projects().get(project=project,fields="quotas") - response=request.execute() - quotas[project][region]=get_quotas_dict(response['quotas']) - - return quotas - except exceptions.PermissionDenied as err: - print( - f"Warning: error reading quotas for {project}. " + - f"This can happen if you don't have permissions on the project, for example if the project is in another organization or a Google managed project" - ) - return None - -#def get_project_quota_current_limit(config,project_link,metric_name,) def get_quota_current_limit(config, project_link, metric_name): ''' Retrieves limit for a specific metric. @@ -223,4 +168,4 @@ def count_effective_limit(config, project_id, network_dict, usage_metric_name, limit_metric_name, network_dict['network_name']) metrics.write_data_to_metric(config, project_id, utilization, utilization_metric_name, - network_dict['network_name']) + network_dict['network_name']) \ No newline at end of file diff --git a/blueprints/cloud-operations/network-dashboard/cloud-function/metrics/metrics.py b/blueprints/cloud-operations/network-dashboard/cloud-function/metrics/metrics.py index c1367920d4..4194a13a56 100644 --- a/blueprints/cloud-operations/network-dashboard/cloud-function/metrics/metrics.py +++ b/blueprints/cloud-operations/network-dashboard/cloud-function/metrics/metrics.py @@ -36,9 +36,7 @@ def create_metrics(monitoring_project): existing_metrics.append(desc.type) limits_dict = {} - with open( - "/Users/mnoseda/Fabric/cloud-foundation-fabric/blueprints/cloud-operations/network-dashboard/cloud-function/metrics.yaml", - 'r') as stream: #f + with open("metrics.yaml", 'r') as stream: try: metrics_dict = yaml.safe_load(stream) @@ -54,9 +52,8 @@ def create_metrics(monitoring_project): # Subnet level metrics have a different limit: the subnet IP range size if sub_metric_key == "limit" and metric_name != "ip_usage_per_subnet": limits_dict_for_metric = {} - if "values" in sub_metric: - for network_link, limit_value in sub_metric["values"].items(): - limits_dict_for_metric[network_link] = limit_value + for network_link, limit_value in sub_metric["values"].items(): + limits_dict_for_metric[network_link] = limit_value limits_dict[sub_metric["name"]] = limits_dict_for_metric return metrics_dict, limits_dict @@ -87,7 +84,7 @@ def create_metric(metric_name, description, monitoring_project): def write_data_to_metric(config, monitored_project_id, value, metric_name, - network_name=None, subnet_id=None): + network_name, subnet_id=None): ''' Writes data to Cloud Monitoring custom metrics. Parameters: @@ -106,8 +103,7 @@ def write_data_to_metric(config, monitored_project_id, value, metric_name, series = monitoring_v3.TimeSeries() series.metric.type = f"custom.googleapis.com/{metric_name}" series.resource.type = "global" - if network_name: - series.metric.labels["network_name"] = network_name + series.metric.labels["network_name"] = network_name series.metric.labels["project"] = monitored_project_id if subnet_id: series.metric.labels["subnet_id"] = subnet_id diff --git a/blueprints/cloud-operations/network-dashboard/cloud-function/metrics/vpc_firewalls.py b/blueprints/cloud-operations/network-dashboard/cloud-function/metrics/vpc_firewalls.py deleted file mode 100644 index 1bc53da690..0000000000 --- a/blueprints/cloud-operations/network-dashboard/cloud-function/metrics/vpc_firewalls.py +++ /dev/null @@ -1,141 +0,0 @@ -# -# Copyright 2022 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -import re -from collections import defaultdict -from pydoc import doc -from collections import defaultdict -from google.protobuf import field_mask_pb2 -from . import metrics, networks, limits, peerings, routers - - - - -def get_firewalls_dict(config: dict): - ''' - Calls the Asset Inventory API to get all VPC Firewall Rules under the GCP organization. - - Parameters: - config (dict): The dict containing config like clients and limits - Returns: - firewalls_dict (dictionary of dictionary: int): Keys are projects, subkeys are networks, values count #of VPC Firewall Rules - ''' - - firewalls_dict = defaultdict(int) - read_mask = field_mask_pb2.FieldMask() - read_mask.FromJsonString('name,versionedResources') - - - response = config["clients"]["asset_client"].search_all_resources( - request={ - "scope": f"organizations/{config['organization']}", - "asset_types": ["compute.googleapis.com/Firewall"], - "read_mask": read_mask, - }) - for resource in response: - project_id=re.search("(compute.googleapis.com/projects/)([\w\-\d]+)", resource.name).group(2) - network_name="" - for versioned in resource.versioned_resources: - for field_name, field_value in versioned.resource.items(): - if field_name == "network": - network_name = re.search("[a-z0-9\-]*$", field_value).group(0) - firewalls_dict[project_id]= defaultdict(int) if not project_id in firewalls_dict else firewalls_dict[project_id] - firewalls_dict[project_id][network_name]=1 if not network_name in firewalls_dict[project_id] else firewalls_dict[project_id][network_name]+1 - break - break - return firewalls_dict - - - -'''def get_firewalls_per_project_vpc(config, project_id ): - - Returns returns the VPC firewall rules defined in a project - - Parameters: - config (dict): The dict containing config like clients and limits - project_id (string): Project ID for the project containing the Cloud Router. - Returns: - sum_firewalls_per_vpc(dict): Number of firewall rules defined for each VPC in a project - - get_firewalls_dict(config) - sum_firewalls_per_vpc=dict() - sum_firewalls=0 - page_token=None - while (1): - request = config["clients"]["discovery_client"].firewalls().list( - project=project_id,pageToken=page_token) - response = request.execute() - sum_firewalls_per_vpc = dict() - if 'items' in response: - for firewall_rule in response['items']: - sum_firewalls_per_vpc[firewall_rule['network']] = sum_firewalls_per_vpc[firewall_rule['network']]+1 if firewall_rule['network'] in sum_firewalls_per_vpc else 1 - else: - break - page_token = response['pageToken'] if 'pageToken' in response else None - if (not page_token): - break - - return sum_firewalls_per_vpc -''' - -def get_firewalls_data(config, metrics_dict,project_quotas_dict,firewalls_dict): - ''' - Gets the data for VPC Firewall Rules per VPC Network and writes it to the metric defined in vpc_firewalls_metric. - - Parameters: - config (dict): The dict containing config like clients and limits - metrics_dict (dictionary of dictionary of string: string): metrics names and descriptions. - limit_dict (dictionary of string:int): Dictionary with the network link as key and the limit as value. - firewalls_dict (dictionary of dictionary): Keys are projects, subkeys are networks, values count #of VPC Firewall Rules - Returns: - None - ''' - for project in config["monitored_projects"]: - - current_quota_limit= project_quotas_dict[project]['global']["firewalls"] - if current_quota_limit is None: - print( - f"Could not write VPC firewal rules to metric for projects/{project} due to missing quotas" - ) - continue - - - network_dict = networks.get_networks(config, project) - - project_usage=0 - for net in network_dict: - usage = 0 - if net['network_name'] in firewalls_dict: - usage = firewalls_dict[project][net['network_name']] - project_usage+=usage - metrics.write_data_to_metric( - config, project, usage, metrics_dict["metrics_per_project"] - [f"firewalls"]["usage"]["name"], - net['network_name']) - - #firewall quotas are per project, not per single VPC - metrics.write_data_to_metric( - config, project, current_quota_limit['limit'], metrics_dict["metrics_per_project"] - [f"firewalls"]["limit"]["name"], - ) - metrics.write_data_to_metric( - config, project, project_usage / current_quota_limit['limit'] if current_quota_limit['limit']!=0 else 0, - metrics_dict["metrics_per_project"] - [f"firewalls"]["utilization"]["name"],) - - print( - f"Wrote number of VPC Firewall Rules to metric for projects/{project}" - )