Add GCP test package (#701)

.ci/Jenkinsfile

@@ -15,10 +15,14 @@ pipeline {
     HOME = "${env.WORKSPACE}"
     KIND_VERSION = 'v0.11.1'
     K8S_VERSION = 'v1.23.0'
+
     JOB_GCS_BUCKET = 'beats-ci-temp'
     JOB_GCS_BUCKET_INTERNAL = 'beats-ci-temp-internal'
     JOB_GCS_CREDENTIALS = 'beats-ci-gcs-plugin'
     JOB_GCS_EXT_CREDENTIALS = 'beats-ci-gcs-plugin-file-credentials'
+    ELASTIC_PACKAGE_GCP_SECRET = 'secret/observability-team/ci/service-account/elastic-package-gcp'
+    ELASTIC_OBSERVABILITY_PROJECT_ID = 'elastic-observability'
+
     JOB_SIGNING_CREDENTIALS = 'sign-artifacts-with-gpg-job'
     INTERNAL_CI_JOB_GCS_CREDENTIALS = 'internal-ci-gcs-plugin'
 
@@ -358,6 +362,12 @@ def withCloudTestEnv(Closure body) {
     [var: "AWS_ACCESS_KEY_ID", password: aws.access_key],
     [var: "AWS_SECRET_ACCESS_KEY", password: aws.secret_key],
   ])
+  // GCP
+  withGCPEnv(secret: env.ELASTIC_PACKAGE_GCP_SECRET) {
+    maskedVars.add([var: 'GOOGLE_CREDENTIALS', password: readFile(file: env.GOOGLE_APPLICATION_CREDENTIALS)]);
+    maskedVars.add([var: 'GCP_PROJECT_ID', password: env.ELASTIC_OBSERVABILITY_PROJECT_ID])
+  }
+  // Masking
   withEnvMask(vars: maskedVars) {
     body()
   }

test/packages/parallel/gcp/_dev/build/build.yml

@@ -0,0 +1,3 @@
+dependencies:
+  ecs:
+    reference: [email protected]

test/packages/parallel/gcp/_dev/build/docs/README.md

@@ -0,0 +1,22 @@
+# Google Cloud Platform Integration
+
+This integration is used to fetches logs and metrics from 
+[Google Cloud Platform](https://cloud.google.com/).
+
+## GCP Credentials
+GCP credentials are required for running GCP integration. 
+
+### Configuration parameters
+* *project_id*: ID of the GCP project.
+* *credentials_file*: Path to JSON file with GCP credentials. Required when not using `credentials_json`.
+* *credentials_json*: Raw JSON text of GCP Credentials. Required when not using `credentials_file`.
+
+#### Data stream specific configuration parameters
+* *period*: How often the data stream is executed.
+* *region*: Specify which GCP regions to query metrics from. If the `region`
+is not set in the config, then by default, the integration will query metrics
+from all available GCP regions. If both `region` and `zone` is set, `region` takes precedent.
+* *zone*: Specify which GCP zones to query metrics from. If the `zone`
+is not set in the config, then by default, the integration will query metrics
+from all available GCP zone. If both `region` and `zone` is set, `region` takes precedent.
+* *exclude_labels*: Exclude additional labels from metrics.  Defaults to false.

test/packages/parallel/gcp/_dev/build/docs/compute.md

@@ -0,0 +1,9 @@
+# Compute
+
+## Metrics
+
+This is the `compute` dataset.
+
+{{event "compute"}}
+
+{{fields "compute"}}

test/packages/parallel/gcp/changelog.yml

@@ -0,0 +1,80 @@
+# newer versions go on top
+- version: "999.999.999"
+  changes:
+    - description: "version override for testing"
+      type: enhancement
+      link: https://github.com/elastic/elastic-package/pull/701
+- version: "1.2.0"
+  changes:
+    - description: Add 8.0.0 version constraint
+      type: enhancement
+      link: https://github.com/elastic/integrations/pull/2251
+    - description: Add GCP Billing Metricset
+      type: enhancement
+      link: https://github.com/elastic/integrations/pull/2141
+    - description: Add GCP Compute Metricset
+      type: enhancement
+      link: https://github.com/elastic/integrations/pull/2301
+- version: "1.1.2"
+  changes:
+    - description: Update Title and Description.
+      type: enhancement
+      link: https://github.com/elastic/integrations/pull/1965
+- version: "1.1.1"
+  changes:
+    - description: Fix logic that checks for the 'forwarded' tag
+      type: bugfix
+      link: https://github.com/elastic/integrations/pull/1818
+- version: "1.1.0"
+  changes:
+    - description: Update to ECS 1.12.0
+      type: enhancement
+      link: https://github.com/elastic/integrations/pull/1661
+- version: "1.0.0"
+  changes:
+    - description: Move from experimental to GA
+      type: enhancement
+      link: https://github.com/elastic/integrations/pull/1568
+    - description: remove experimental from data_sets
+      type: enhancement
+      link: https://github.com/elastic/integrations/pull/1717
+- version: "0.3.3"
+  changes:
+    - description: Convert to generated ECS fields
+      type: enhancement
+      link: https://github.com/elastic/integrations/pull/1478
+- version: '0.3.2'
+  changes:
+    - description: update to ECS 1.11.0
+      type: enhancement
+      link: https://github.com/elastic/integrations/pull/1385
+- version: "0.3.1"
+  changes:
+    - description: Escape special characters in docs
+      type: enhancement
+      link: https://github.com/elastic/integrations/pull/1405
+- version: "0.3.0"
+  changes:
+    - description: Update integration description
+      type: enhancement
+      link: https://github.com/elastic/integrations/pull/1364
+- version: "0.2.0"
+  changes:
+    - description: Set "event.module" and "event.dataset"
+      type: enhancement
+      link: https://github.com/elastic/integrations/pull/1240
+- version: "0.1.0"
+  changes:
+    - description: update to ECS 1.10.0 and adding event.original options
+      type: enhancement
+      link: https://github.com/elastic/integrations/pull/1045
+- version: "0.0.2"
+  changes:
+    - description: update to ECS 1.9.0
+      type: enhancement
+      link: https://github.com/elastic/integrations/pull/846
+- version: "0.0.1"
+  changes:
+    - description: initial release
+      type: enhancement # can be one of: enhancement, bugfix, breaking-change
+      link: https://github.com/elastic/integrations/pull/459

test/packages/parallel/gcp/data_stream/compute/_dev/deploy/tf/compute.tf

@@ -0,0 +1,33 @@
+data "google_compute_image" "default" {
+  # https://cloud.google.com/compute/docs/images
+  family  = "ubuntu-minimal-2004-lts"
+  project = "ubuntu-os-cloud"
+}
+
+resource "google_compute_instance" "default" {
+  name = "elastic-package-system-test-${var.TEST_RUN_ID}"
+  // NOTE: e2 instance type is required to collect instance/memory/balloon/* 
+  // metrics, available only on those instances.
+  // https://cloud.google.com/monitoring/api/metrics_gcp
+  machine_type = "e2-micro"
+  zone         = var.zone
+
+  labels = {
+    team   = "integrations"
+    run_id = var.TEST_RUN_ID
+  }
+
+  boot_disk {
+    initialize_params {
+      image = data.google_compute_image.default.self_link
+    }
+  }
+
+  network_interface {
+    network = "default"
+
+    access_config {
+      // Ephemeral public IP
+    }
+  }
+}

test/packages/parallel/gcp/data_stream/compute/_dev/deploy/tf/env.yml

@@ -0,0 +1,10 @@
+version: '2.3'
+services:
+  terraform:
+    environment:
+      - GCP_PROJECT_ID=${GCP_PROJECT_ID}
+      - GCP_ZONE=${GCP_ZONE:-us-central1-a}
+      - GOOGLE_CREDENTIALS=${GOOGLE_CREDENTIALS}
+      # pass project id to Terraform (NOTE: they are not passed to system test)
+      - TF_VAR_gcp_project_id=${GCP_PROJECT_ID}
+      - TF_VAR_zone=${GCP_ZONE:-us-central1-a}

test/packages/parallel/gcp/data_stream/compute/_dev/deploy/tf/main.tf

@@ -0,0 +1,4 @@
+provider "google" {
+  project = var.gcp_project_id
+}
+

test/packages/parallel/gcp/data_stream/compute/_dev/deploy/tf/vars.tf

@@ -0,0 +1,16 @@
+variable "TEST_RUN_ID" {
+  default = "detached"
+}
+
+variable "gcp_project_id" {
+  type = string
+}
+
+variable "zone" {
+  type = string
+  // NOTE: if you change this value you **must** change it also for test
+  // configuration, otherwise the tests will not be able to find metrics in
+  // the specified region
+  default = "us-central1-a"
+  # https://cloud.google.com/compute/docs/regions-zones#available
+}

test/packages/parallel/gcp/data_stream/compute/_dev/test/system/test-default-config.yml

@@ -0,0 +1,7 @@
+# wait_for_data_timeout: 10m
+vars:
+  project_id: "{{GCP_PROJECT_ID}}"
+  zone: "{{GCP_ZONE}}"
+  credentials_json: '{{{GOOGLE_CREDENTIALS}}}'
+data_streams:
+  vars: ~

test/packages/parallel/gcp/data_stream/compute/agent/stream/stream.yml.hbs

@@ -0,0 +1,16 @@
+metricsets: ["compute"]
+period: {{period}}
+project_id: {{project_id}}
+{{#if credentials_file}}
+credentials_file_path: {{credentials_file}}
+{{/if}}
+{{#if credentials_json}}
+credentials_json: '{{credentials_json}}'
+{{/if}}
+{{#if region}}
+region: {{region}}
+{{/if}}
+{{#if zone}}
+zone: {{zone}}
+{{/if}}
+exclude_labels: {{exclude_labels}}