Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

[GCP] Enable TSDB #7555

Open
2 of 4 tasks
constanca-m opened this issue Aug 28, 2023 · 1 comment
Open
2 of 4 tasks

[GCP] Enable TSDB #7555

constanca-m opened this issue Aug 28, 2023 · 1 comment
Assignees
@gpop63 @constanca-m
Labels
Integration:gcp Google Cloud Platform

Comments

@constanca-m
Copy link
Contributor

constanca-m commented Aug 28, 2023 •
edited by gpop63
Loading

Add metric type:

  1. [GCP][CloudSQL] Set metric type #7556
  2. [GCP] Set metric type for Cloud Run, Compute, Dataproc and Firestore data streams #7560
  3. [GCP] Set metric type for GKE, Load Balancing, PubSub, Redis and Storage data streams #7569

Set dimension fields:

Firestore blocked by:

Enable TSDB by default:
@constanca-m constanca-m self-assigned this Aug 28, 2023
@constanca-m constanca-m mentioned this issue Aug 28, 2023
Closed
45 tasks
@constanca-m constanca-m added the Integration:gcp Google Cloud Platform label Aug 28, 2023
This was referenced Aug 28, 2023
Merged
Merged
Merged
@constanca-m constanca-m mentioned this issue Oct 27, 2023
Merged
16 tasks
@gpop63
Copy link
Contributor

gpop63 commented Nov 28, 2023

I created a table so we can see which data streams have the required fields for cloud packages.

Data stream cloud.account.id cloud.region cloud.availability_zone agent.id
compute ✓ ✓ ✓ ✓
gke ✓ ✗ ✗ ✓
loadbalancing_metrics ✓ ✗ ✗ ✓
pubsub ✓ ✗ ✗ ✓
redis ✓ ✗ ✗ ✓
storage ✓ ✗ ✗ ✓
cloudrun_metrics ✓ ✗ ✗ ✓
cloudsql_mysql ✓ ✗ ✗ ✓
cloudsql_postgresql ✓ ✗ ✗ ✓
cloudsql_sqlserver ✓ ✗ ✗ ✓
firestore ✓ ✗ ✗ ✓
dataproc ✓ ✗ ✗ ✓

I will add actual documents for each data stream so we can check them at a later time if needed.

compute 

{
    "cloud": {
      "availability_zone": "us-central1-c",
      "instance": {
        "name": "gke-miguel-kubecon-default-pool-8d586473-uj1d",
        "id": "7442613444431818544"
      },
      "provider": "gcp",
      "machine": {
        "type": "e2-medium"
      },
      "region": "us-central1",
      "account": {
        "name": "elastic-obs-integrations-dev",
        "id": "elastic-obs-integrations-dev"
      }
    },
    "agent": {
      "name": "docker-fleet-agent",
      "id": "8442dd66-0f0b-4b54-93f6-6dcf25f45296",
      "ephemeral_id": "1de59397-2d84-4e9e-8ec2-8becfa2dccd3",
      "type": "metricbeat",
      "version": "8.12.0"
    },
    "@timestamp": "2023-11-28T11:13:00.000Z",
    "ecs": {
      "version": "8.0.0"
    },
    "gcp": {
      "compute": {
        "instance": {
          "network": {
            "ingress": {
              "bytes": 9414,
              "packets": {
                "count": 134
              }
            },
            "egress": {
              "bytes": 11485,
              "packets": {
                "count": 89
              }
            }
          }
        }
      },
      "labels": {
        "metrics": {
          "loadbalanced": "true"
        },
        "user": {
          "division": "engineering",
          "goog-k8s-cluster-name": "miguel-kubecon",
          "org": "obs",
          "goog-k8s-node-pool-name": "default-pool",
          "goog-k8s-cluster-location": "us-central1-c",
          "project": "miguelluna",
          "team": "cloud-native-monitoring",
          "goog-gke-node": ""
        }
      }
    },
    "service": {
      "type": "gcp"
    },
    "data_stream": {
      "namespace": "default",
      "type": "metrics",
      "dataset": "gcp.compute"
    },
    "host": {
      "hostname": "docker-fleet-agent",
      "os": {
        "kernel": "5.10.102.1-microsoft-standard-WSL2",
        "codename": "focal",
        "name": "Ubuntu",
        "type": "linux",
        "family": "debian",
        "version": "20.04.6 LTS (Focal Fossa)",
        "platform": "ubuntu"
      },
      "containerized": true,
      "ip": [
        "172.22.0.7"
      ],
      "name": "docker-fleet-agent",
      "id": "d03b51e638e64b05b5cf16c41d2058c0",
      "mac": [
        "02-42-AC-16-00-07"
      ],
      "architecture": "x86_64"
    },
    "elastic_agent": {
      "id": "8442dd66-0f0b-4b54-93f6-6dcf25f45296",
      "version": "8.12.0",
      "snapshot": true
    },
    "metricset": {
      "period": 60000,
      "name": "metrics"
    },
    "event": {
      "duration": 3126038318,
      "agent_id_status": "verified",
      "ingested": "2023-11-28T11:17:09Z",
      "module": "gcp",
      "dataset": "gcp.compute"
    }
  }

gke 

{
    "cloud": {
      "provider": "gcp",
      "account": {
        "name": "elastic-obs-integrations-dev",
        "id": "elastic-obs-integrations-dev"
      }
    },
    "agent": {
      "name": "docker-fleet-agent",
      "id": "8442dd66-0f0b-4b54-93f6-6dcf25f45296",
      "ephemeral_id": "1de59397-2d84-4e9e-8ec2-8becfa2dccd3",
      "type": "metricbeat",
      "version": "8.12.0"
    },
    "@timestamp": "2023-11-28T11:14:05.736Z",
    "ecs": {
      "version": "8.0.0"
    },
    "gcp": {
      "gke": {
        "container": {
          "memory": {
            "request": {
              "bytes": 10485760
            },
            "limit": {
              "bytes": 52428800
            }
          },
          "restart": {
            "count": 0
          },
          "cpu": {
            "request_cores": {
              "value": 0.005
            },
            "limit_cores": {
              "value": 0
            }
          },
          "ephemeral_storage": {
            "request": {
              "bytes": 0
            },
            "limit": {
              "bytes": 0
            }
          }
        }
      },
      "labels": {
        "resource": {
          "cluster_name": "tetiana-prometheus",
          "container_name": "csi-driver-registrar",
          "location": "europe-west1",
          "pod_name": "pdcsi-node-7b6hq",
          "namespace_name": "kube-system"
        }
      }
    },
    "service": {
      "type": "gcp"
    },
    "data_stream": {
      "namespace": "default",
      "type": "metrics",
      "dataset": "gcp.gke"
    },
    "host": {
      "hostname": "docker-fleet-agent",
      "os": {
        "kernel": "5.10.102.1-microsoft-standard-WSL2",
        "codename": "focal",
        "name": "Ubuntu",
        "type": "linux",
        "family": "debian",
        "version": "20.04.6 LTS (Focal Fossa)",
        "platform": "ubuntu"
      },
      "containerized": true,
      "ip": [
        "172.22.0.7"
      ],
      "name": "docker-fleet-agent",
      "id": "d03b51e638e64b05b5cf16c41d2058c0",
      "mac": [
        "02-42-AC-16-00-07"
      ],
      "architecture": "x86_64"
    },
    "elastic_agent": {
      "id": "8442dd66-0f0b-4b54-93f6-6dcf25f45296",
      "version": "8.12.0",
      "snapshot": true
    },
    "metricset": {
      "period": 60000,
      "name": "metrics"
    },
    "event": {
      "duration": 1244946929,
      "agent_id_status": "verified",
      "ingested": "2023-11-28T11:18:19Z",
      "module": "gcp",
      "dataset": "gcp.gke"
    }
  }

loadbalancing_metrics 

{
    "cloud": {
      "provider": "gcp",
      "account": {
        "name": "elastic-obs-integrations-dev",
        "id": "elastic-obs-integrations-dev"
      }
    },
    "agent": {
      "name": "docker-fleet-agent",
      "id": "8442dd66-0f0b-4b54-93f6-6dcf25f45296",
      "ephemeral_id": "1de59397-2d84-4e9e-8ec2-8becfa2dccd3",
      "type": "metricbeat",
      "version": "8.12.0"
    },
    "@timestamp": "2023-11-28T11:11:00.000Z",
    "ecs": {
      "version": "8.0.0"
    },
    "gcp": {
      "loadbalancing_metrics": {
        "https": {
          "backend_request": {
            "bytes": 417,
            "count": 1
          },
          "backend_response": {
            "bytes": 488
          }
        }
      },
      "labels": {
        "resource": {
          "backend_scope_type": "INVALID_BACKEND",
          "matched_url_path_rule": "UNMATCHED",
          "backend_target_name": "tas-demo-elastic-http-lb",
          "backend_type": "INVALID_BACKEND",
          "backend_scope": "INVALID_BACKEND",
          "target_proxy_name": "tas-demo-elastic-http-lb",
          "forwarding_rule_name": "tas-demo-elastic-http-lb",
          "backend_name": "INVALID_BACKEND",
          "url_map_name": "tas-demo-elastic-https-lb",
          "backend_target_type": "BACKEND_SERVICE",
          "region": "global"
        },
        "metrics": {
          "response_code": "502",
          "proxy_continent": "America",
          "cache_result": "DISABLED",
          "response_code_class": "500"
        }
      }
    },
    "data_stream": {
      "namespace": "default",
      "type": "metrics",
      "dataset": "gcp.loadbalancing_metrics"
    },
    "service": {
      "type": "gcp"
    },
    "elastic_agent": {
      "id": "8442dd66-0f0b-4b54-93f6-6dcf25f45296",
      "version": "8.12.0",
      "snapshot": true
    },
    "host": {
      "hostname": "docker-fleet-agent",
      "os": {
        "kernel": "5.10.102.1-microsoft-standard-WSL2",
        "codename": "focal",
        "name": "Ubuntu",
        "family": "debian",
        "type": "linux",
        "version": "20.04.6 LTS (Focal Fossa)",
        "platform": "ubuntu"
      },
      "containerized": true,
      "ip": [
        "172.22.0.7"
      ],
      "name": "docker-fleet-agent",
      "id": "d03b51e638e64b05b5cf16c41d2058c0",
      "mac": [
        "02-42-AC-16-00-07"
      ],
      "architecture": "x86_64"
    },
    "metricset": {
      "period": 60000,
      "name": "metrics"
    },
    "event": {
      "duration": 476084876,
      "agent_id_status": "verified",
      "ingested": "2023-11-28T11:15:18Z",
      "module": "gcp",
      "dataset": "gcp.loadbalancing_metrics"
    }
  }

pubsub 

{
    "cloud": {
      "provider": "gcp",
      "account": {
        "name": "elastic-obs-integrations-dev",
        "id": "elastic-obs-integrations-dev"
      }
    },
    "agent": {
      "name": "docker-fleet-agent",
      "id": "8442dd66-0f0b-4b54-93f6-6dcf25f45296",
      "ephemeral_id": "1de59397-2d84-4e9e-8ec2-8becfa2dccd3",
      "type": "metricbeat",
      "version": "8.12.0"
    },
    "@timestamp": "2023-11-28T11:15:00.000Z",
    "ecs": {
      "version": "8.0.0"
    },
    "gcp": {
      "pubsub": {
        "subscription": {
          "retained_acked_bytes_by_region": {
            "bytes": 0
          },
          "unacked_bytes_by_region": {
            "bytes": 0
          },
          "oldest_unacked_message_age_by_region": {
            "value": 0
          },
          "oldest_retained_acked_message_age_by_region": {
            "value": 0
          }
        }
      },
      "labels": {
        "resource": {
          "subscription_id": "filebeat-gcp-audit"
        },
        "metrics": {
          "region": "northamerica-northeast2"
        }
      }
    },
    "data_stream": {
      "namespace": "default",
      "type": "metrics",
      "dataset": "gcp.pubsub"
    },
    "service": {
      "type": "gcp"
    },
    "elastic_agent": {
      "id": "8442dd66-0f0b-4b54-93f6-6dcf25f45296",
      "version": "8.12.0",
      "snapshot": true
    },
    "host": {
      "hostname": "docker-fleet-agent",
      "os": {
        "kernel": "5.10.102.1-microsoft-standard-WSL2",
        "codename": "focal",
        "name": "Ubuntu",
        "family": "debian",
        "type": "linux",
        "version": "20.04.6 LTS (Focal Fossa)",
        "platform": "ubuntu"
      },
      "containerized": true,
      "ip": [
        "172.22.0.7"
      ],
      "name": "docker-fleet-agent",
      "id": "d03b51e638e64b05b5cf16c41d2058c0",
      "mac": [
        "02-42-AC-16-00-07"
      ],
      "architecture": "x86_64"
    },
    "metricset": {
      "period": 60000,
      "name": "metrics"
    },
    "event": {
      "duration": 434512489,
      "agent_id_status": "verified",
      "ingested": "2023-11-28T11:19:18Z",
      "module": "gcp",
      "dataset": "gcp.pubsub"
    }
  }

redis 

{
    "cloud": {
      "instance": {
        "name": "redis1",
        "id": "projects/elastic-obs-integrations-dev/locations/us-central1/instances/redis1"
      },
      "provider": "gcp",
      "machine": {
        "type": "BASIC"
      },
      "account": {
        "name": "elastic-obs-integrations-dev",
        "id": "elastic-obs-integrations-dev"
      }
    },
    "agent": {
      "name": "docker-fleet-agent",
      "id": "8442dd66-0f0b-4b54-93f6-6dcf25f45296",
      "type": "metricbeat",
      "ephemeral_id": "1de59397-2d84-4e9e-8ec2-8becfa2dccd3",
      "version": "8.12.0"
    },
    "@timestamp": "2023-11-28T11:15:26.810Z",
    "ecs": {
      "version": "8.0.0"
    },
    "gcp": {
      "metrics": {
        "stats": {
          "reject_connections_count": {}
        }
      },
      "redis": {
        "stats": {
          "reject_connections": {
            "count": 0
          }
        }
      },
      "labels": {
        "resource": {
          "region": "us-central1",
          "node_id": "node-0"
        },
        "metrics": {
          "role": "primary"
        }
      }
    },
    "service": {
      "type": "gcp"
    },
    "data_stream": {
      "namespace": "default",
      "type": "metrics",
      "dataset": "gcp.redis"
    },
    "elastic_agent": {
      "id": "8442dd66-0f0b-4b54-93f6-6dcf25f45296",
      "version": "8.12.0",
      "snapshot": true
    },
    "host": {
      "hostname": "docker-fleet-agent",
      "os": {
        "kernel": "5.10.102.1-microsoft-standard-WSL2",
        "codename": "focal",
        "name": "Ubuntu",
        "family": "debian",
        "type": "linux",
        "version": "20.04.6 LTS (Focal Fossa)",
        "platform": "ubuntu"
      },
      "containerized": true,
      "ip": [
        "172.22.0.7"
      ],
      "name": "docker-fleet-agent",
      "id": "d03b51e638e64b05b5cf16c41d2058c0",
      "mac": [
        "02-42-AC-16-00-07"
      ],
      "architecture": "x86_64"
    },
    "metricset": {
      "period": 60000,
      "name": "metrics"
    },
    "event": {
      "duration": 1381430449,
      "agent_id_status": "verified",
      "ingested": "2023-11-28T11:19:39Z",
      "module": "gcp",
      "dataset": "gcp.redis"
    }
  }

storage 

{
    "cloud": {
      "provider": "gcp",
      "account": {
        "name": "elastic-obs-integrations-dev",
        "id": "elastic-obs-integrations-dev"
      }
    },
    "agent": {
      "name": "docker-fleet-agent",
      "id": "8442dd66-0f0b-4b54-93f6-6dcf25f45296",
      "type": "metricbeat",
      "ephemeral_id": "1de59397-2d84-4e9e-8ec2-8becfa2dccd3",
      "version": "8.12.0"
    },
    "@timestamp": "2023-11-28T11:10:00.000Z",
    "ecs": {
      "version": "8.0.0"
    },
    "gcp": {
      "storage": {
        "storage": {
          "object": {
            "count": 6
          }
        }
      },
      "labels": {
        "resource": {
          "bucket_name": "dataproc-temp-us-central1-774712120909-e8nuxv73",
          "location": "us-central1"
        },
        "metrics": {
          "storage_class": "REGIONAL"
        }
      }
    },
    "data_stream": {
      "namespace": "default",
      "type": "metrics",
      "dataset": "gcp.storage"
    },
    "service": {
      "type": "gcp"
    },
    "host": {
      "hostname": "docker-fleet-agent",
      "os": {
        "kernel": "5.10.102.1-microsoft-standard-WSL2",
        "codename": "focal",
        "name": "Ubuntu",
        "type": "linux",
        "family": "debian",
        "version": "20.04.6 LTS (Focal Fossa)",
        "platform": "ubuntu"
      },
      "containerized": true,
      "ip": [
        "172.22.0.7"
      ],
      "name": "docker-fleet-agent",
      "id": "d03b51e638e64b05b5cf16c41d2058c0",
      "mac": [
        "02-42-AC-16-00-07"
      ],
      "architecture": "x86_64"
    },
    "elastic_agent": {
      "id": "8442dd66-0f0b-4b54-93f6-6dcf25f45296",
      "version": "8.12.0",
      "snapshot": true
    },
    "metricset": {
      "period": 60000,
      "name": "metrics"
    },
    "event": {
      "duration": 815694657,
      "agent_id_status": "verified",
      "ingested": "2023-11-28T11:20:18Z",
      "module": "gcp",
      "dataset": "gcp.storage"
    }
  }

cloudrun_metrics 

{
    "cloud": {
      "provider": "gcp",
      "account": {
        "name": "elastic-obs-integrations-dev",
        "id": "elastic-obs-integrations-dev"
      }
    },
    "agent": {
      "name": "docker-fleet-agent",
      "id": "8442dd66-0f0b-4b54-93f6-6dcf25f45296",
      "type": "metricbeat",
      "ephemeral_id": "1de59397-2d84-4e9e-8ec2-8becfa2dccd3",
      "version": "8.12.0"
    },
    "@timestamp": "2023-11-28T11:18:21.590Z",
    "ecs": {
      "version": "8.0.0"
    },
    "gcp": {
      "cloudrun_metrics": {
        "container": {
          "billable_instance_time": 2.7
        }
      },
      "labels": {
        "resource": {
          "revision_name": "damien-test-hello-00001-ped",
          "service_name": "damien-test-hello",
          "location": "us-central1",
          "configuration_name": "damien-test-hello"
        }
      }
    },
    "data_stream": {
      "namespace": "default",
      "type": "metrics",
      "dataset": "gcp.cloudrun_metrics"
    },
    "service": {
      "type": "gcp"
    },
    "host": {
      "hostname": "docker-fleet-agent",
      "os": {
        "kernel": "5.10.102.1-microsoft-standard-WSL2",
        "codename": "focal",
        "name": "Ubuntu",
        "type": "linux",
        "family": "debian",
        "version": "20.04.6 LTS (Focal Fossa)",
        "platform": "ubuntu"
      },
      "ip": [
        "172.22.0.7"
      ],
      "containerized": true,
      "name": "docker-fleet-agent",
      "id": "d03b51e638e64b05b5cf16c41d2058c0",
      "mac": [
        "02-42-AC-16-00-07"
      ],
      "architecture": "x86_64"
    },
    "elastic_agent": {
      "id": "8442dd66-0f0b-4b54-93f6-6dcf25f45296",
      "version": "8.12.0",
      "snapshot": true
    },
    "metricset": {
      "period": 60000,
      "name": "metrics"
    },
    "event": {
      "duration": 503983018,
      "agent_id_status": "verified",
      "ingested": "2023-11-28T11:21:34Z",
      "module": "gcp",
      "dataset": "gcp.cloudrun_metrics"
    }
  }

cloudsql_mysql 

{
    "cloud": {
      "provider": "gcp",
      "account": {
        "name": "elastic-observability",
        "id": "elastic-observability"
      }
    },
    "agent": {
      "name": "docker-fleet-agent",
      "id": "8442dd66-0f0b-4b54-93f6-6dcf25f45296",
      "ephemeral_id": "1de59397-2d84-4e9e-8ec2-8becfa2dccd3",
      "type": "metricbeat",
      "version": "8.12.0"
    },
    "@timestamp": "2023-11-28T11:20:00.000Z",
    "ecs": {
      "version": "8.0.0"
    },
    "gcp": {
      "cloudsql_mysql": {
        "database": {
          "instance_state": false
        }
      },
      "labels": {
        "resource": {
          "database_id": "elastic-observability:mysql",
          "region": "us-central"
        },
        "cloudsql": {
          "name": "mysql",
          "version": "8.0.31"
        },
        "metrics": {
          "state": "RUNNABLE"
        }
      }
    },
    "service": {
      "type": "gcp"
    },
    "data_stream": {
      "namespace": "default",
      "type": "metrics",
      "dataset": "gcp.cloudsql_mysql"
    },
    "elastic_agent": {
      "id": "8442dd66-0f0b-4b54-93f6-6dcf25f45296",
      "version": "8.12.0",
      "snapshot": true
    },
    "host": {
      "hostname": "docker-fleet-agent",
      "os": {
        "kernel": "5.10.102.1-microsoft-standard-WSL2",
        "codename": "focal",
        "name": "Ubuntu",
        "family": "debian",
        "type": "linux",
        "version": "20.04.6 LTS (Focal Fossa)",
        "platform": "ubuntu"
      },
      "containerized": true,
      "ip": [
        "172.22.0.7"
      ],
      "name": "docker-fleet-agent",
      "id": "d03b51e638e64b05b5cf16c41d2058c0",
      "mac": [
        "02-42-AC-16-00-07"
      ],
      "architecture": "x86_64"
    },
    "metricset": {
      "period": 60000,
      "name": "metrics"
    },
    "event": {
      "duration": 1074356670,
      "agent_id_status": "verified",
      "ingested": "2023-11-28T11:23:29Z",
      "module": "gcp",
      "dataset": "gcp.cloudsql_mysql"
    }
  }

cloudsql_postgresql 

{
    "cloud": {
      "provider": "gcp",
      "account": {
        "name": "elastic-observability",
        "id": "elastic-observability"
      }
    },
    "agent": {
      "name": "docker-fleet-agent",
      "id": "8442dd66-0f0b-4b54-93f6-6dcf25f45296",
      "ephemeral_id": "1de59397-2d84-4e9e-8ec2-8becfa2dccd3",
      "type": "metricbeat",
      "version": "8.12.0"
    },
    "@timestamp": "2023-11-28T11:20:00.000Z",
    "ecs": {
      "version": "8.0.0"
    },
    "gcp": {
      "cloudsql_postgresql": {
        "database": {
          "num_backends": {
            "count": 2
          }
        }
      },
      "labels": {
        "resource": {
          "database_id": "elastic-observability:postgres",
          "region": "us-central"
        },
        "cloudsql": {
          "name": "postgres",
          "version": "14"
        },
        "metrics": {
          "database": "cloudsqladmin"
        }
      }
    },
    "data_stream": {
      "namespace": "default",
      "type": "metrics",
      "dataset": "gcp.cloudsql_postgresql"
    },
    "service": {
      "type": "gcp"
    },
    "elastic_agent": {
      "id": "8442dd66-0f0b-4b54-93f6-6dcf25f45296",
      "version": "8.12.0",
      "snapshot": true
    },
    "host": {
      "hostname": "docker-fleet-agent",
      "os": {
        "kernel": "5.10.102.1-microsoft-standard-WSL2",
        "codename": "focal",
        "name": "Ubuntu",
        "type": "linux",
        "family": "debian",
        "version": "20.04.6 LTS (Focal Fossa)",
        "platform": "ubuntu"
      },
      "containerized": true,
      "ip": [
        "172.22.0.7"
      ],
      "name": "docker-fleet-agent",
      "id": "d03b51e638e64b05b5cf16c41d2058c0",
      "mac": [
        "02-42-AC-16-00-07"
      ],
      "architecture": "x86_64"
    },
    "metricset": {
      "period": 60000,
      "name": "metrics"
    },
    "event": {
      "duration": 1140922692,
      "agent_id_status": "verified",
      "ingested": "2023-11-28T11:23:18Z",
      "module": "gcp",
      "dataset": "gcp.cloudsql_postgresql"
    }
  }

cloudsql_sqlserver 

{
    "cloud": {
      "provider": "gcp",
      "account": {
        "name": "elastic-observability",
        "id": "elastic-observability"
      }
    },
    "agent": {
      "name": "docker-fleet-agent",
      "id": "8442dd66-0f0b-4b54-93f6-6dcf25f45296",
      "type": "metricbeat",
      "ephemeral_id": "1de59397-2d84-4e9e-8ec2-8becfa2dccd3",
      "version": "8.12.0"
    },
    "@timestamp": "2023-11-28T11:20:00.000Z",
    "ecs": {
      "version": "8.0.0"
    },
    "gcp": {
      "cloudsql_sqlserver": {
        "database": {
          "disk": {
            "write_ops": {
              "count": 110
            },
            "bytes_used": {
              "bytes": 651927552
            },
            "read_ops": {
              "count": 0
            },
            "quota": {
              "bytes": 105089261568
            },
            "utilization": {
              "pct": 0.006203560119015185
            }
          },
          "memory": {
            "quota": {
              "bytes": 27331235840
            },
            "total_usage": {
              "bytes": 3081568256
            },
            "usage": {
              "bytes": 2311217152
            },
            "utilization": {
              "pct": 0.08456321424798038
            }
          },
          "cpu": {
            "usage_time": {
              "sec": 11.019461885036435
            },
            "utilization": {
              "pct": 0.045281727500696436
            },
            "reserved_cores": {
              "count": 4
            }
          },
          "up": 1,
          "network": {
            "received_bytes": {
              "count": 315087
            },
            "connections": {
              "count": 9
            },
            "sent_bytes": {
              "count": 3681782
            }
          },
          "uptime": {
            "sec": 60
          }
        }
      },
      "labels": {
        "resource": {
          "database_id": "elastic-observability:ms-sql",
          "region": "us-west1"
        },
        "cloudsql": {
          "name": "sqlserver",
          "version": "2019_standard"
        }
      }
    },
    "service": {
      "type": "gcp"
    },
    "data_stream": {
      "namespace": "default",
      "type": "metrics",
      "dataset": "gcp.cloudsql_sqlserver"
    },
    "elastic_agent": {
      "id": "8442dd66-0f0b-4b54-93f6-6dcf25f45296",
      "version": "8.12.0",
      "snapshot": true
    },
    "host": {
      "hostname": "docker-fleet-agent",
      "os": {
        "kernel": "5.10.102.1-microsoft-standard-WSL2",
        "codename": "focal",
        "name": "Ubuntu",
        "type": "linux",
        "family": "debian",
        "version": "20.04.6 LTS (Focal Fossa)",
        "platform": "ubuntu"
      },
      "ip": [
        "172.22.0.7"
      ],
      "containerized": true,
      "name": "docker-fleet-agent",
      "id": "d03b51e638e64b05b5cf16c41d2058c0",
      "mac": [
        "02-42-AC-16-00-07"
      ],
      "architecture": "x86_64"
    },
    "metricset": {
      "period": 60000,
      "name": "metrics"
    },
    "event": {
      "duration": 1086857363,
      "agent_id_status": "verified",
      "ingested": "2023-11-28T11:23:29Z",
      "module": "gcp",
      "dataset": "gcp.cloudsql_sqlserver"
    }
  }

firestore 

{
    "cloud": {
      "provider": "gcp",
      "account": {
        "name": "robust-catalyst-399814",
        "id": "robust-catalyst-399814"
      }
    },
    "agent": {
      "name": "docker-fleet-agent",
      "id": "8442dd66-0f0b-4b54-93f6-6dcf25f45296",
      "type": "metricbeat",
      "ephemeral_id": "1de59397-2d84-4e9e-8ec2-8becfa2dccd3",
      "version": "8.12.0"
    },
    "@timestamp": "2023-11-28T11:28:00.000Z",
    "ecs": {
      "version": "8.0.0"
    },
    "gcp": {
      "firestore": {
        "document": {
          "delete": {
            "count": 18
          }
        }
      },
      "labels": {
        "metrics": {
          "module": "__unknown__",
          "version": "__unknown__"
        }
      }
    },
    "data_stream": {
      "namespace": "default",
      "type": "metrics",
      "dataset": "gcp.firestore"
    },
    "service": {
      "type": "gcp"
    },
    "elastic_agent": {
      "id": "8442dd66-0f0b-4b54-93f6-6dcf25f45296",
      "version": "8.12.0",
      "snapshot": true
    },
    "host": {
      "hostname": "docker-fleet-agent",
      "os": {
        "kernel": "5.10.102.1-microsoft-standard-WSL2",
        "codename": "focal",
        "name": "Ubuntu",
        "type": "linux",
        "family": "debian",
        "version": "20.04.6 LTS (Focal Fossa)",
        "platform": "ubuntu"
      },
      "containerized": true,
      "ip": [
        "172.22.0.7"
      ],
      "name": "docker-fleet-agent",
      "id": "d03b51e638e64b05b5cf16c41d2058c0",
      "mac": [
        "02-42-AC-16-00-07"
      ],
      "architecture": "x86_64"
    },
    "metricset": {
      "period": 60000,
      "name": "metrics"
    },
    "event": {
      "duration": 397651706,
      "agent_id_status": "verified",
      "ingested": "2023-11-28T11:32:39Z",
      "module": "gcp",
      "dataset": "gcp.firestore"
    }
  }

dataproc 

  {
    "cloud": {
      "provider": "gcp",
      "account": {
        "name": "elastic-obs-integrations-dev",
        "id": "elastic-obs-integrations-dev"
      }
    },
    "agent": {
      "name": "docker-fleet-agent",
      "id": "8442dd66-0f0b-4b54-93f6-6dcf25f45296",
      "ephemeral_id": "1f9a913c-dd2f-4347-a5fc-5746e4b9f819",
      "type": "metricbeat",
      "version": "8.12.0"
    },
    "@timestamp": "2023-11-28T11:47:00.000Z",
    "ecs": {
      "version": "8.0.0"
    },
    "gcp": {
      "labels": {
        "resource": {
          "cluster_name": "cluster-aa34",
          "cluster_uuid": "89f8d591-cb18-4778-9c14-8be71fb578f1",
          "region": "us-central1"
        },
        "metrics": {
          "operation_type": "START_CLUSTER",
          "error_type": "UNKNOWN_ERROR"
        }
      },
      "dataproc": {
        "cluster": {
          "operation": {
            "failed": {
              "count": 0
            }
          }
        }
      }
    },
    "service": {
      "type": "gcp"
    },
    "data_stream": {
      "namespace": "default",
      "type": "metrics",
      "dataset": "gcp.dataproc"
    },
    "elastic_agent": {
      "id": "8442dd66-0f0b-4b54-93f6-6dcf25f45296",
      "version": "8.12.0",
      "snapshot": true
    },
    "host": {
      "hostname": "docker-fleet-agent",
      "os": {
        "kernel": "5.10.102.1-microsoft-standard-WSL2",
        "codename": "focal",
        "name": "Ubuntu",
        "type": "linux",
        "family": "debian",
        "version": "20.04.6 LTS (Focal Fossa)",
        "platform": "ubuntu"
      },
      "containerized": true,
      "ip": [
        "172.22.0.7"
      ],
      "name": "docker-fleet-agent",
      "id": "d03b51e638e64b05b5cf16c41d2058c0",
      "mac": [
        "02-42-AC-16-00-07"
      ],
      "architecture": "x86_64"
    },
    "metricset": {
      "period": 60000,
      "name": "metrics"
    },
    "event": {
      "duration": 539443435,
      "agent_id_status": "verified",
      "ingested": "2023-11-28T11:50:03Z",
      "module": "gcp",
      "dataset": "gcp.dataproc"
    }
  }

Some data streams have the region/AZ under gcp.labels.

@lalit-satapathy lalit-satapathy mentioned this issue Dec 1, 2023
Closed
11 tasks
@gpop63 gpop63 mentioned this issue Dec 6, 2023
Merged
16 tasks
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment
Assignees

@gpop63 gpop63

@constanca-m constanca-m

Labels
Integration:gcp Google Cloud Platform
Projects
None yet
Milestone
No milestone
Development

No branches or pull requests
2 participants
@gpop63 @constanca-m