From 5a8256e1b51db60b9cfc7b617915c1852ade2363 Mon Sep 17 00:00:00 2001 From: Miren Esnaola Date: Tue, 28 Feb 2023 21:52:13 +0100 Subject: [PATCH] Added autopilot blueprint --- .gitignore | 5 + blueprints/gke/autopilot/README.md | 94 +++++ blueprints/gke/autopilot/ansible.tf | 37 ++ blueprints/gke/autopilot/ansible/ansible.cfg | 8 + .../gke/autopilot/ansible/inventory/hosts.ini | 1 + .../gke/autopilot/ansible/playbook.yaml | 128 +++++++ .../gke/autopilot/bundle/app/hpa.yaml.j2 | 37 ++ .../gke/autopilot/bundle/app/ingress.yaml | 42 +++ .../autopilot/bundle/app/kustomization.yaml | 18 + .../gke/autopilot/bundle/app/namespace.yaml | 18 + .../gke/autopilot/bundle/app/nginx.yaml | 127 +++++++ .../autopilot/bundle/locust/image/Dockerfile | 21 ++ .../bundle/locust/image/locust-files/test.py | 66 ++++ .../gke/autopilot/bundle/locust/image/run.sh | 26 ++ .../gke/autopilot/bundle/locust/ingress.yaml | 42 +++ .../bundle/locust/kustomization.yaml.j2 | 66 ++++ .../gke/autopilot/bundle/locust/master.yaml | 128 +++++++ .../autopilot/bundle/locust/namespace.yaml | 18 + .../gke/autopilot/bundle/locust/workers.yaml | 51 +++ .../custom-stackdriver-metrics-adapter.yaml | 184 ++++++++++ .../monitoring/dashboards/k8s-global.json | 1 + .../monitoring/dashboards/k8s-namespaces.json | 1 + .../monitoring/dashboards/k8s-nodes.json | 1 + .../monitoring/dashboards/k8s-pods.json | 1 + .../bundle/monitoring/dashboards/locust.json | 1 + .../bundle/monitoring/dashboards/nginx.json | 1 + .../autopilot/bundle/monitoring/frontend.yaml | 79 ++++ .../autopilot/bundle/monitoring/grafana.yaml | 184 ++++++++++ .../autopilot/bundle/monitoring/ingress.yaml | 43 +++ .../bundle/monitoring/kube-state-metrics.yaml | 342 ++++++++++++++++++ .../bundle/monitoring/kustomization.yaml.j2 | 72 ++++ .../bundle/monitoring/namespace.yaml | 18 + blueprints/gke/autopilot/cluster.tf | 54 +++ blueprints/gke/autopilot/diagram.png | Bin 0 -> 33543 bytes blueprints/gke/autopilot/glbs.tf | 25 ++ blueprints/gke/autopilot/main.tf | 65 ++++ blueprints/gke/autopilot/mgmt.tf | 35 ++ blueprints/gke/autopilot/outputs.tf | 20 + .../gke/autopilot/templates/gssh.sh.tpl | 30 ++ blueprints/gke/autopilot/test.py | 25 ++ blueprints/gke/autopilot/variables.tf | 83 +++++ blueprints/gke/autopilot/vpc.tf | 45 +++ 42 files changed, 2243 insertions(+) create mode 100644 blueprints/gke/autopilot/README.md create mode 100644 blueprints/gke/autopilot/ansible.tf create mode 100644 blueprints/gke/autopilot/ansible/ansible.cfg create mode 100644 blueprints/gke/autopilot/ansible/inventory/hosts.ini create mode 100644 blueprints/gke/autopilot/ansible/playbook.yaml create mode 100644 blueprints/gke/autopilot/bundle/app/hpa.yaml.j2 create mode 100644 blueprints/gke/autopilot/bundle/app/ingress.yaml create mode 100644 blueprints/gke/autopilot/bundle/app/kustomization.yaml create mode 100644 blueprints/gke/autopilot/bundle/app/namespace.yaml create mode 100644 blueprints/gke/autopilot/bundle/app/nginx.yaml create mode 100644 blueprints/gke/autopilot/bundle/locust/image/Dockerfile create mode 100644 blueprints/gke/autopilot/bundle/locust/image/locust-files/test.py create mode 100755 blueprints/gke/autopilot/bundle/locust/image/run.sh create mode 100644 blueprints/gke/autopilot/bundle/locust/ingress.yaml create mode 100755 blueprints/gke/autopilot/bundle/locust/kustomization.yaml.j2 create mode 100644 blueprints/gke/autopilot/bundle/locust/master.yaml create mode 100644 blueprints/gke/autopilot/bundle/locust/namespace.yaml create mode 100644 blueprints/gke/autopilot/bundle/locust/workers.yaml create mode 100644 blueprints/gke/autopilot/bundle/monitoring/custom-stackdriver-metrics-adapter.yaml create mode 100644 blueprints/gke/autopilot/bundle/monitoring/dashboards/k8s-global.json create mode 100644 blueprints/gke/autopilot/bundle/monitoring/dashboards/k8s-namespaces.json create mode 100644 blueprints/gke/autopilot/bundle/monitoring/dashboards/k8s-nodes.json create mode 100644 blueprints/gke/autopilot/bundle/monitoring/dashboards/k8s-pods.json create mode 100644 blueprints/gke/autopilot/bundle/monitoring/dashboards/locust.json create mode 100644 blueprints/gke/autopilot/bundle/monitoring/dashboards/nginx.json create mode 100644 blueprints/gke/autopilot/bundle/monitoring/frontend.yaml create mode 100644 blueprints/gke/autopilot/bundle/monitoring/grafana.yaml create mode 100644 blueprints/gke/autopilot/bundle/monitoring/ingress.yaml create mode 100644 blueprints/gke/autopilot/bundle/monitoring/kube-state-metrics.yaml create mode 100755 blueprints/gke/autopilot/bundle/monitoring/kustomization.yaml.j2 create mode 100644 blueprints/gke/autopilot/bundle/monitoring/namespace.yaml create mode 100644 blueprints/gke/autopilot/cluster.tf create mode 100644 blueprints/gke/autopilot/diagram.png create mode 100644 blueprints/gke/autopilot/glbs.tf create mode 100644 blueprints/gke/autopilot/main.tf create mode 100644 blueprints/gke/autopilot/mgmt.tf create mode 100644 blueprints/gke/autopilot/outputs.tf create mode 100644 blueprints/gke/autopilot/templates/gssh.sh.tpl create mode 100644 blueprints/gke/autopilot/test.py create mode 100644 blueprints/gke/autopilot/variables.tf create mode 100644 blueprints/gke/autopilot/vpc.tf diff --git a/.gitignore b/.gitignore index 91778178cf..ed88b00d05 100644 --- a/.gitignore +++ b/.gitignore @@ -49,3 +49,8 @@ blueprints/apigee/hybrid-gke/apiproxy.zip blueprints/apigee/hybrid-gke/deploy-apiproxy.sh blueprints/apigee/hybrid-gke/ansible/gssh.sh blueprints/apigee/hybrid-gke/ansible/vars/vars.yaml +blueprints/gke/autopilot/ansible/gssh.sh +blueprints/gke/autopilot/ansible/vars/vars.yaml +blueprints/gke/autopilot/bundle/monitoring/kustomization.yaml +blueprints/gke/autopilot/bundle/locust/kustomization.yaml +blueprints/gke/autopilot/bundle.tar.gz \ No newline at end of file diff --git a/blueprints/gke/autopilot/README.md b/blueprints/gke/autopilot/README.md new file mode 100644 index 0000000000..62d8cfaf41 --- /dev/null +++ b/blueprints/gke/autopilot/README.md @@ -0,0 +1,94 @@ +# Load testing an application running on an autopilot cluster + +This blueprint creates an Autopilot cluster with Google-managed Prometheus enabled and install an application that scales as the traffic that is hitting the load balancer exposing it grows. It also installs the tooling required to distributed load test with [locust](https://locust.io) on that application and the monitoring tooling required to observe how things evolve in the cluster during the load test. Ansible is used to install the application and all the tooling on a management VM. + +The diagram below depicts the architecture. + +![Diagram](./diagram.png) + +## Running the blueprint + +1. Clone this repository or [open it in cloud shell](https://ssh.cloud.google.com/cloudshell/editor?cloudshell_git_repo=https%3A%2F%2Fgithub.com%2Fterraform-google-modules%2Fcloud-foundation-fabric&cloudshell_print=cloud-shell-readme.txt&cloudshell_working_dir=blueprints%2Fgke%2Fautopilot), then go through the following steps to create resources: + +2. Initialize the terraform configuration + + ``` + terraform init + ``` + +3. Apply the terraform configuration + + ``` + terraform apply -var project_id=my-project-id + ``` + +4. Copy the IP addresses for grafana, the locust master. + +4. Change to the ansible directory and run the following command + + ``` + ansible-playbook -v playbook.yaml + ``` + +5. Open to the locust master web interface url in your browser and start the load test + + +6. SSH to the management VM + + ``` + gcloud compute ssh mgmt --project my-project + ``` + +7. Run the following command to check that the application pods are running on different nodes than the load testing and monitoring tooling. + + ``` + kubectl get pods -A -o wide + ``` + +8. Run the following command to see how the application pods scale + + ``` + kubectl get hpa -n sample -w + ``` + +9. Run the following command to see how the cluster nodes scale + + ``` + kubectl get nodes -n + ``` + +Alternatively you can also check all the above using the dashboards available in grafana. + + +## Variables + +| name | description | type | required | default | +|---|---|:---:|:---:|:---:| +| [project_id](variables.tf#L68) | Project ID. | string | ✓ | | +| [cluster_network_config](variables.tf#L17) | Cluster network configuration. | object({…}) | | {…} | +| [mgmt_server_config](variables.tf#L37) | Management server configuration. | object({…}) | | {…} | +| [mgmt_subnet_cidr_block](variables.tf#L53) | Management subnet IP CIDR range. | string | | "10.0.2.0/24" | +| [project_create](variables.tf#L59) | Parameters for the creation of the new project. | object({…}) | | null | +| [region](variables.tf#L73) | Region. | string | | "europe-west1" | +| [zone](variables.tf#L79) | Zone. | string | | "europe-west1-c" | + +## Outputs + +| name | description | sensitive | +|---|---|:---:| +| [urls](outputs.tf#L17) | | | + + +## Test + +```hcl +module "test" { + source = "./fabric/blueprints/gke/autopilot" + project_create = { + billing_account_id = "12345-12345-12345" + parent = "folders/123456789" + } + project_id = "my-project" +} +# tftest modules=10 resources=30 +``` \ No newline at end of file diff --git a/blueprints/gke/autopilot/ansible.tf b/blueprints/gke/autopilot/ansible.tf new file mode 100644 index 0000000000..9ee2170af9 --- /dev/null +++ b/blueprints/gke/autopilot/ansible.tf @@ -0,0 +1,37 @@ +/** + * Copyright 2023 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +# tfdoc:file:description Ansible generated files. + +resource "local_file" "vars_file" { + content = yamlencode({ + cluster = module.cluster.name + region = var.region + project_id = module.project.project_id + app_url = local.urls["app"] + }) + filename = "${path.module}/ansible/vars/vars.yaml" + file_permission = "0666" +} + +resource "local_file" "gssh_file" { + content = templatefile("${path.module}/templates/gssh.sh.tpl", { + project_id = module.project.project_id + zone = var.zone + }) + filename = "${path.module}/ansible/gssh.sh" + file_permission = "0777" +} diff --git a/blueprints/gke/autopilot/ansible/ansible.cfg b/blueprints/gke/autopilot/ansible/ansible.cfg new file mode 100644 index 0000000000..654f1729dc --- /dev/null +++ b/blueprints/gke/autopilot/ansible/ansible.cfg @@ -0,0 +1,8 @@ +[defaults] +inventory = inventory/hosts.ini +timeout = 900 + +[ssh_connection] +pipelining = True +ssh_executable = ./gssh.sh +transfer_method = piped \ No newline at end of file diff --git a/blueprints/gke/autopilot/ansible/inventory/hosts.ini b/blueprints/gke/autopilot/ansible/inventory/hosts.ini new file mode 100644 index 0000000000..842da83f43 --- /dev/null +++ b/blueprints/gke/autopilot/ansible/inventory/hosts.ini @@ -0,0 +1 @@ +mgmt \ No newline at end of file diff --git a/blueprints/gke/autopilot/ansible/playbook.yaml b/blueprints/gke/autopilot/ansible/playbook.yaml new file mode 100644 index 0000000000..ad30859ca1 --- /dev/null +++ b/blueprints/gke/autopilot/ansible/playbook.yaml @@ -0,0 +1,128 @@ +# Copyright 2022 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- hosts: mgmt + gather_facts: "no" + vars_files: + - vars/vars.yaml + environment: + USE_GKE_GCLOUD_AUTH_PLUGIN: True + tasks: + - name: Download the Google Cloud SDK package repository signing key + get_url: + url: https://packages.cloud.google.com/apt/doc/apt-key.gpg + dest: /usr/share/keyrings/cloud.google.gpg + force: yes + become: true + become_user: root + - name: Add Google Cloud SDK package repository source + apt_repository: + filename: google-cloud-sdk + repo: "deb [signed-by=/usr/share/keyrings/cloud.google.gpg] https://packages.cloud.google.com/apt cloud-sdk main" + state: present + update_cache: yes + become: true + become_user: root + - name: Install dependencies + apt: + pkg: + - google-cloud-sdk-gke-gcloud-auth-plugin + - kubectl + state: present + become: true + become_user: root + - name: Enable bash completion for kubectl + shell: + cmd: kubectl completion bash > /etc/bash_completion.d/kubectl + creates: /etc/bash_completion.d/kubectl + become: true + become_user: root + - name: Get cluster credentials + shell: > + gcloud container clusters get-credentials {{ cluster }} + --region {{ region }} + --project {{ project_id }} + --internal-ip + - name: Render templates + template: + src: ../bundle/{{ item }}/kustomization.yaml.j2 + dest: ../bundle/{{ item }}/kustomization.yaml + delegate_to: localhost + with_items: + - monitoring + - locust + - name: Remove bundle locally + local_action: + module: file + path: ../bundle.tar.gz + state: absent + - name: Archive bundle locally + archive: + path: ../bundle + dest: ../bundle.tar.gz + delegate_to: localhost + - name: Unarchive bundle remotely + unarchive: + src: ../bundle.tar.gz + dest: ~/ + - name: Build locust image + shell: > + gcloud builds submit --tag {{ region }}-docker.pkg.dev/{{ project_id }}/registry/load-test:latest \ + --project {{ project_id }} . + args: + chdir: ~/bundle/locust/image + - name: Enable scraping of kubelet and cAdvisor metrics + shell: > + kubectl patch operatorconfig config + -n gmp-public + --type=merge + -p '{"collection":{"kubeletScraping":{"interval": "30s"}}}' + - name: Deploy monitoring tooling + shell: > + kubectl apply -k . + args: + chdir: ~/bundle/monitoring + - name: Deploy app + shell: > + kubectl apply -k . + args: + chdir: ~/bundle/app + - name: Get forwarding rule name + shell: > + while true; do + forwarding_rule_name=$(kubectl get ingress -n sample -o=jsonpath='{.items[0].metadata.annotations.ingress\.kubernetes\.io\/forwarding-rule}') + if [ -n "$forwarding_rule_name" ]; then + echo $forwarding_rule_name + break + fi + sleep 10 + done + register: forwarding_rule_name_output + - name: Set fact forwarding_url_name + set_fact: + forwarding_rule_name: "{{ forwarding_rule_name_output.stdout }}" + - name: Render template (HPA) + template: + src: ../bundle/app/hpa.yaml.j2 + dest: ~/bundle/app/hpa.yaml + - name: Apply HPA manifest + shell: > + kubectl apply -f hpa.yaml + args: + chdir: ~/bundle/app + - name: Deploy locust + shell: > + kubectl apply -k . + args: + chdir: ~/bundle/locust diff --git a/blueprints/gke/autopilot/bundle/app/hpa.yaml.j2 b/blueprints/gke/autopilot/bundle/app/hpa.yaml.j2 new file mode 100644 index 0000000000..54a12b4b5b --- /dev/null +++ b/blueprints/gke/autopilot/bundle/app/hpa.yaml.j2 @@ -0,0 +1,37 @@ +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: autoscaling/v2 +kind: HorizontalPodAutoscaler +metadata: + name: nginx + namespace: sample +spec: + scaleTargetRef: + apiVersion: apps/v1 + kind: Deployment + name: nginx + minReplicas: 1 + maxReplicas: 50 + metrics: + - type: External + external: + metric: + name: loadbalancing.googleapis.com|https|request_count + selector: + matchLabels: + resource.labels.forwarding_rule_name: {{ forwarding_rule_name }} + target: + type: AverageValue + averageValue: 5 \ No newline at end of file diff --git a/blueprints/gke/autopilot/bundle/app/ingress.yaml b/blueprints/gke/autopilot/bundle/app/ingress.yaml new file mode 100644 index 0000000000..0591024074 --- /dev/null +++ b/blueprints/gke/autopilot/bundle/app/ingress.yaml @@ -0,0 +1,42 @@ +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: cloud.google.com/v1 +kind: BackendConfig +metadata: + name: backendconfig + namespace: sample +spec: + healthCheck: + requestPath: / + port: 80 + type: HTTP + logging: + enable: true + sampleRate: 0.5 +--- +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + annotations: + kubernetes.io/ingress.global-static-ip-name: "app" + kubernetes.io/ingress.allow-http: "true" + name: ingress + namespace: sample +spec: + defaultBackend: + service: + name: nginx + port: + name: web \ No newline at end of file diff --git a/blueprints/gke/autopilot/bundle/app/kustomization.yaml b/blueprints/gke/autopilot/bundle/app/kustomization.yaml new file mode 100644 index 0000000000..cb074c018f --- /dev/null +++ b/blueprints/gke/autopilot/bundle/app/kustomization.yaml @@ -0,0 +1,18 @@ +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https:#www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +resources: + - namespace.yaml + - nginx.yaml + - ingress.yaml diff --git a/blueprints/gke/autopilot/bundle/app/namespace.yaml b/blueprints/gke/autopilot/bundle/app/namespace.yaml new file mode 100644 index 0000000000..60510a40b3 --- /dev/null +++ b/blueprints/gke/autopilot/bundle/app/namespace.yaml @@ -0,0 +1,18 @@ +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: v1 +kind: Namespace +metadata: + name: sample \ No newline at end of file diff --git a/blueprints/gke/autopilot/bundle/app/nginx.yaml b/blueprints/gke/autopilot/bundle/app/nginx.yaml new file mode 100644 index 0000000000..48a9d2c156 --- /dev/null +++ b/blueprints/gke/autopilot/bundle/app/nginx.yaml @@ -0,0 +1,127 @@ +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: v1 +kind: ConfigMap +metadata: + name: nginx-config + namespace: sample +data: + nginx.conf: | + events {} + http { + server { + listen 80; + root /var/www/html; + location / { + return 200 'Hello World!'; + } + } + server { + listen 8080; + location /stub_status { + stub_status on; + } + } + } +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: nginx + namespace: sample +spec: + replicas: 1 + selector: + matchLabels: + app: nginx + template: + metadata: + labels: + app: nginx + spec: + containers: + - name: nginx + image: nginx:latest + ports: + - containerPort: 80 + name: web + - containerPort: 8080 + name: status + volumeMounts: + - name: nginx-config + mountPath: /etc/nginx/nginx.conf + subPath: nginx.conf + readinessProbe: + httpGet: + path: /stub_status + port: 8080 + initialDelaySeconds: 2 + periodSeconds: 2 + failureThreshold: 1 + requests: + cpu: 10m + memory: 10Mi + limits: + memory: 10Mi + - name: nginx-prometheus-exporter + image: nginx/nginx-prometheus-exporter:0.10.0 + ports: + - containerPort: 9113 + name: metrics + env: + - name: SCRAPE_URI + value: http://localhost:8080/stub_status + requests: + cpu: 5m + memory: 5Mi + limits: + memory: 5Mi + volumes: + - name: nginx-config + configMap: + name: nginx-config +--- +apiVersion: v1 +kind: Service +metadata: + name: nginx + namespace: sample + annotations: + annotations: + cloud.google.com/neg: '{"ingress": true}' + cloud.google.com/app-protocols: '{"web":"HTTP"}' + cloud.google.com/backend-config: '{"default": "backendconfig"}' + labels: + app: nginx +spec: + ports: + - name: web + port: 80 + protocol: TCP + selector: + app: nginx +--- +apiVersion: monitoring.googleapis.com/v1 +kind: ClusterPodMonitoring +metadata: + name: nginx + namespace: sample +spec: + selector: + matchLabels: + app: nginx + endpoints: + - port: metrics + interval: 30s \ No newline at end of file diff --git a/blueprints/gke/autopilot/bundle/locust/image/Dockerfile b/blueprints/gke/autopilot/bundle/locust/image/Dockerfile new file mode 100644 index 0000000000..ce38e10e26 --- /dev/null +++ b/blueprints/gke/autopilot/bundle/locust/image/Dockerfile @@ -0,0 +1,21 @@ +# Copyright 2023 Google Inc. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +FROM locustio/locust:latest + +ADD locust-files /home/locust/locust-files + +ADD run.sh /home/locust/run.sh + +ENTRYPOINT ["/home/locust/run.sh"] \ No newline at end of file diff --git a/blueprints/gke/autopilot/bundle/locust/image/locust-files/test.py b/blueprints/gke/autopilot/bundle/locust/image/locust-files/test.py new file mode 100644 index 0000000000..5309d5af36 --- /dev/null +++ b/blueprints/gke/autopilot/bundle/locust/image/locust-files/test.py @@ -0,0 +1,66 @@ + +# Copyright 2023 Google Inc. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import logging +import os +from locust import HttpUser, LoadTestShape, task, between + + +class TestUser(HttpUser): + + host = os.getenv("URL", "http://nginx.sample.svc.cluster.local") + + wait_time = between(int(os.getenv('MIN_WAIT_TIME'), 1), + int(os.getenv('MAX_WAIT_TIME'), 2)) + + @task + def home(self): + with self.client.get("/", catch_response=True) as response: + if response.status_code == 200: + response.success() + else: + logging.info('Response code is ' + str(response.status_code)) + + +class CustomLoadShape(LoadTestShape): + + stages = [] + + num_stages = int(os.getenv('NUM_STAGES', 20)) + stage_duration = int(os.getenv('STAGE_DURATION', 60)) + spawn_rate = int(os.getenv('SPAWN_RATE', 1)) + new_users_per_stage = int(os.getenv('NEW_USERS_PER_STAGE', 10)) + + for i in range(1, num_stages + 1): + stages.append({ + 'duration': stage_duration * i, + 'users': new_users_per_stage * i, + 'spawn_rate': spawn_rate + }) + + for i in range(1, num_stages): + stages.append({ + 'duration': stage_duration * (num_stages + i), + 'users': new_users_per_stage * (num_stages - i), + 'spawn_rate': spawn_rate + }) + + def tick(self): + run_time = self.get_run_time() + for stage in self.stages: + if run_time < stage['duration']: + tick_data = (stage['users'], stage['spawn_rate']) + return tick_data + return None diff --git a/blueprints/gke/autopilot/bundle/locust/image/run.sh b/blueprints/gke/autopilot/bundle/locust/image/run.sh new file mode 100755 index 0000000000..6e43970762 --- /dev/null +++ b/blueprints/gke/autopilot/bundle/locust/image/run.sh @@ -0,0 +1,26 @@ +#!/bin/bash + +# Copyright 2023 Google Inc. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +LOCUS_OPTS="-f /home/locust/locust-files" +LOCUST_MODE=${LOCUST_MODE:-standalone} + +if [[ "$LOCUST_MODE" = "master" ]]; then + LOCUS_OPTS="$LOCUS_OPTS --master" +elif [[ "$LOCUST_MODE" = "worker" ]]; then + LOCUS_OPTS="$LOCUS_OPTS --worker --master-host=$LOCUST_MASTER" +fi + +locust $LOCUS_OPTS \ No newline at end of file diff --git a/blueprints/gke/autopilot/bundle/locust/ingress.yaml b/blueprints/gke/autopilot/bundle/locust/ingress.yaml new file mode 100644 index 0000000000..8db7a1a4bf --- /dev/null +++ b/blueprints/gke/autopilot/bundle/locust/ingress.yaml @@ -0,0 +1,42 @@ +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: cloud.google.com/v1 +kind: BackendConfig +metadata: + name: backendconfig + namespace: locust +spec: + healthCheck: + requestPath: / + port: 8089 + type: HTTP + logging: + enable: true + sampleRate: 0.5 +--- +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: ingress + namespace: locust + annotations: + kubernetes.io/ingress.global-static-ip-name: "locust" + kubernetes.io/ingress.allow-http: "true" +spec: + defaultBackend: + service: + name: locust-master-web + port: + name: loc-master-web \ No newline at end of file diff --git a/blueprints/gke/autopilot/bundle/locust/kustomization.yaml.j2 b/blueprints/gke/autopilot/bundle/locust/kustomization.yaml.j2 new file mode 100755 index 0000000000..878c92a0c3 --- /dev/null +++ b/blueprints/gke/autopilot/bundle/locust/kustomization.yaml.j2 @@ -0,0 +1,66 @@ +# Copyright 2023 Google Inc. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +resources: + - namespace.yaml + - master.yaml + - workers.yaml + - ingress.yaml +patches: + - target: + group: apps + version: v1 + kind: Deployment + name: locust-master + namespace: locust + patch: |- + apiVersion: apps/v1 + kind: Deployment + metadata: + name: locust-master + namespace: locust + spec: + template: + spec: + containers: + - name: locust-master + image: load-test-image + env: + - name: URL + value: {{ app_url }} + - target: + group: apps + version: v1 + kind: Deployment + name: locust-worker + namespace: locust + patch: |- + apiVersion: apps/v1 + kind: Deployment + metadata: + name: locust-worker + namespace: locust + spec: + template: + spec: + containers: + - name: locust-master + image: load-test-image + env: + - name: URL + value: {{ app_url }} +images: + - name: load-test-image + newName: {{ region }}-docker.pkg.dev/{{ project_id}}/registry/load-test + newTag: latest \ No newline at end of file diff --git a/blueprints/gke/autopilot/bundle/locust/master.yaml b/blueprints/gke/autopilot/bundle/locust/master.yaml new file mode 100644 index 0000000000..89bc3c0299 --- /dev/null +++ b/blueprints/gke/autopilot/bundle/locust/master.yaml @@ -0,0 +1,128 @@ +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: "apps/v1" +kind: "Deployment" +metadata: + name: locust-master + namespace: locust + labels: + name: locust-master +spec: + replicas: 1 + selector: + matchLabels: + app: locust-master + template: + metadata: + labels: + app: locust-master + spec: + tolerations: + - key: group + operator: Equal + value: "locust" + effect: NoSchedule + nodeSelector: + group: "locust" + containers: + - name: locust-master + image: load-test-image + env: + - name: LOCUST_MODE + value: master + ports: + - name: loc-master-web + containerPort: 8089 + protocol: TCP + - name: loc-master-p1 + containerPort: 5557 + protocol: TCP + - name: loc-master-p2 + containerPort: 5558 + protocol: TCP + resources: + requests: + cpu: 50m + memory: 50Mi + limits: + memory: 50Mi + - name: locust-prometheus-exporter + image: containersol/locust_exporter + ports: + - name: metrics + containerPort: 9646 + resources: + requests: + cpu: 5m + memory: 5Mi + limits: + memory: 5Mi +--- +kind: Service +apiVersion: v1 +metadata: + name: locust-master + namespace: locust + labels: + app: locust-master +spec: + ports: + - port: 5557 + targetPort: loc-master-p1 + protocol: TCP + name: loc-master-p1 + - port: 5558 + targetPort: loc-master-p2 + protocol: TCP + name: loc-master-p2 + - port: 9646 + targetPort: metrics + protocol: TCP + name: metrics + selector: + app: locust-master +--- +kind: Service +apiVersion: v1 +metadata: + name: locust-master-web + namespace: locust + annotations: + cloud.google.com/neg: '{"ingress": true}' + cloud.google.com/app-protocols: '{"loc-master-web":"HTTP"}' + cloud.google.com/backend-config: '{"default": "backendconfig"}' + labels: + app: locust-master +spec: + ports: + - port: 8089 + targetPort: loc-master-web + protocol: TCP + name: loc-master-web + selector: + app: locust-master +--- +apiVersion: monitoring.googleapis.com/v1 +kind: ClusterPodMonitoring +metadata: + name: locust-master + namespace: locust +spec: + selector: + matchLabels: + app: locust-master + endpoints: + - port: metrics + interval: 30s \ No newline at end of file diff --git a/blueprints/gke/autopilot/bundle/locust/namespace.yaml b/blueprints/gke/autopilot/bundle/locust/namespace.yaml new file mode 100644 index 0000000000..16aa6e1853 --- /dev/null +++ b/blueprints/gke/autopilot/bundle/locust/namespace.yaml @@ -0,0 +1,18 @@ +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: v1 +kind: Namespace +metadata: + name: locust \ No newline at end of file diff --git a/blueprints/gke/autopilot/bundle/locust/workers.yaml b/blueprints/gke/autopilot/bundle/locust/workers.yaml new file mode 100644 index 0000000000..7d6b41fe56 --- /dev/null +++ b/blueprints/gke/autopilot/bundle/locust/workers.yaml @@ -0,0 +1,51 @@ +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: "apps/v1" +kind: "Deployment" +metadata: + name: locust-worker + namespace: locust + labels: + name: locust-worker +spec: + replicas: 5 + selector: + matchLabels: + app: locust-worker + template: + metadata: + labels: + app: locust-worker + spec: + tolerations: + - key: group + operator: Equal + value: "locust" + effect: NoSchedule + nodeSelector: + group: "locust" + containers: + - name: locust-worker + image: load-test-image + env: + - name: LOCUST_MODE + value: worker + - name: LOCUST_MASTER + value: locust-master + requests: + cpu: 20m + memory: 50Mi + limits: + memory: 50Mi \ No newline at end of file diff --git a/blueprints/gke/autopilot/bundle/monitoring/custom-stackdriver-metrics-adapter.yaml b/blueprints/gke/autopilot/bundle/monitoring/custom-stackdriver-metrics-adapter.yaml new file mode 100644 index 0000000000..8e159213fe --- /dev/null +++ b/blueprints/gke/autopilot/bundle/monitoring/custom-stackdriver-metrics-adapter.yaml @@ -0,0 +1,184 @@ +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https:#www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: v1 +kind: ServiceAccount +metadata: + name: custom-metrics-stackdriver-adapter + namespace: monitoring +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: custom-metrics:system:auth-delegator +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: system:auth-delegator +subjects: +- kind: ServiceAccount + name: custom-metrics-stackdriver-adapter + namespace: monitoring +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: custom-metrics-auth-reader + namespace: kube-system +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: extension-apiserver-authentication-reader +subjects: +- kind: ServiceAccount + name: custom-metrics-stackdriver-adapter + namespace: monitoring +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: custom-metrics-resource-reader +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: view +subjects: +- kind: ServiceAccount + name: custom-metrics-stackdriver-adapter + namespace: monitoring +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: custom-metrics-stackdriver-adapter + namespace: monitoring + labels: + run: custom-metrics-stackdriver-adapter + k8s-app: custom-metrics-stackdriver-adapter +spec: + replicas: 1 + selector: + matchLabels: + run: custom-metrics-stackdriver-adapter + k8s-app: custom-metrics-stackdriver-adapter + template: + metadata: + labels: + run: custom-metrics-stackdriver-adapter + k8s-app: custom-metrics-stackdriver-adapter + kubernetes.io/cluster-service: "true" + spec: + serviceAccountName: custom-metrics-stackdriver-adapter + containers: + - image: gcr.io/gke-release/custom-metrics-stackdriver-adapter:v0.13.1-gke.0 + imagePullPolicy: Always + name: pod-custom-metrics-stackdriver-adapter + command: + - /adapter + - --use-new-resource-model=false + resources: + limits: + cpu: 100m + memory: 150Mi + requests: + memory: 150Mi +--- +apiVersion: v1 +kind: Service +metadata: + labels: + run: custom-metrics-stackdriver-adapter + k8s-app: custom-metrics-stackdriver-adapter + kubernetes.io/cluster-service: 'true' + kubernetes.io/name: Adapter + name: custom-metrics-stackdriver-adapter + namespace: monitoring +spec: + ports: + - port: 443 + protocol: TCP + targetPort: 443 + selector: + run: custom-metrics-stackdriver-adapter + k8s-app: custom-metrics-stackdriver-adapter + type: ClusterIP +--- +apiVersion: apiregistration.k8s.io/v1 +kind: APIService +metadata: + name: v1beta1.custom.metrics.k8s.io +spec: + insecureSkipTLSVerify: true + group: custom.metrics.k8s.io + groupPriorityMinimum: 100 + versionPriority: 100 + service: + name: custom-metrics-stackdriver-adapter + namespace: monitoring + version: v1beta1 +--- +apiVersion: apiregistration.k8s.io/v1 +kind: APIService +metadata: + name: v1beta2.custom.metrics.k8s.io +spec: + insecureSkipTLSVerify: true + group: custom.metrics.k8s.io + groupPriorityMinimum: 100 + versionPriority: 200 + service: + name: custom-metrics-stackdriver-adapter + namespace: monitoring + version: v1beta2 +--- +apiVersion: apiregistration.k8s.io/v1 +kind: APIService +metadata: + name: v1beta1.external.metrics.k8s.io +spec: + insecureSkipTLSVerify: true + group: external.metrics.k8s.io + groupPriorityMinimum: 100 + versionPriority: 100 + service: + name: custom-metrics-stackdriver-adapter + namespace: monitoring + version: v1beta1 +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: external-metrics-reader +rules: +- apiGroups: + - "external.metrics.k8s.io" + resources: + - "*" + verbs: + - list + - get + - watch© +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: external-metrics-reader +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: external-metrics-reader +subjects: +- kind: ServiceAccount + name: horizontal-pod-autoscaler + namespace: kube-system \ No newline at end of file diff --git a/blueprints/gke/autopilot/bundle/monitoring/dashboards/k8s-global.json b/blueprints/gke/autopilot/bundle/monitoring/dashboards/k8s-global.json new file mode 100644 index 0000000000..13d3eb4ca4 --- /dev/null +++ b/blueprints/gke/autopilot/bundle/monitoring/dashboards/k8s-global.json @@ -0,0 +1 @@ +{"annotations":{"list":[{"builtIn":1,"datasource":{"type":"datasource","uid":"grafana"},"enable":true,"hide":true,"iconColor":"rgba(0, 211, 255, 1)","name":"Annotations & Alerts","target":{"limit":100,"matchAny":false,"tags":[],"type":"dashboard"},"type":"dashboard"}]},"description":"This is a modern 'Global View' dashboard for your Kubernetes cluster(s). Made for kube-prometheus-stack and take advantage of the latest Grafana features. GitHub repository: https://github.com/dotdc/grafana-dashboards-kubernetes","editable":true,"fiscalYearStartMonth":0,"gnetId":15757,"graphTooltip":1,"id":11,"iteration":1677857459220,"links":[],"liveNow":false,"panels":[{"collapsed":false,"datasource":{"type":"prometheus","uid":"prometheus"},"gridPos":{"h":1,"w":24,"x":0,"y":0},"id":67,"panels":[],"title":"Overview","type":"row"},{"datasource":{"type":"prometheus","uid":"${datasource}"},"fieldConfig":{"defaults":{"color":{"mode":"continuous-GrYlRd"},"mappings":[],"max":1,"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"green","value":null},{"color":"red","value":80}]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":6,"x":0,"y":1},"id":77,"options":{"displayMode":"lcd","minVizHeight":10,"minVizWidth":0,"orientation":"horizontal","reduceOptions":{"calcs":["lastNotNull"],"fields":"","values":false},"showUnfilled":true},"pluginVersion":"8.3.4","targets":[{"datasource":{"type":"prometheus","uid":"${datasource}"},"editorMode":"code","exemplar":false,"expr":"sum(kube_pod_container_resource_requests{unit=\"core\"}) / sum(machine_cpu_cores)","hide":false,"interval":"","legendFormat":"Requests","range":true,"refId":"B"},{"datasource":{"type":"prometheus","uid":"${datasource}"},"editorMode":"code","exemplar":false,"expr":"sum(kube_pod_container_resource_limits{unit=\"core\"}) / sum(machine_cpu_cores)","hide":false,"interval":"","legendFormat":"Limits","range":true,"refId":"C"}],"title":"Global CPU Usage","type":"bargauge"},{"datasource":{"type":"prometheus","uid":"${datasource}"},"fieldConfig":{"defaults":{"color":{"mode":"continuous-GrYlRd"},"decimals":2,"mappings":[],"max":1,"min":0,"thresholds":{"mode":"percentage","steps":[{"color":"green","value":null},{"color":"red","value":80}]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":6,"x":6,"y":1},"id":78,"options":{"displayMode":"lcd","minVizHeight":10,"minVizWidth":0,"orientation":"horizontal","reduceOptions":{"calcs":["lastNotNull"],"fields":"","values":false},"showUnfilled":true,"text":{}},"pluginVersion":"8.3.4","targets":[{"datasource":{"type":"prometheus","uid":"${datasource}"},"editorMode":"code","exemplar":false,"expr":"sum(kube_pod_container_resource_requests{unit=\"byte\"}) / sum(machine_memory_bytes)","hide":false,"interval":"","legendFormat":"Requests","range":true,"refId":"B"},{"datasource":{"type":"prometheus","uid":"${datasource}"},"editorMode":"code","exemplar":false,"expr":"sum(kube_pod_container_resource_limits{unit=\"byte\"}) / sum(machine_memory_bytes)","hide":false,"interval":"","legendFormat":"Limits","range":true,"refId":"C"}],"title":"Global RAM Usage","type":"bargauge"},{"datasource":{"type":"prometheus","uid":"${datasource}"},"fieldConfig":{"defaults":{"mappings":[],"noValue":"0","thresholds":{"mode":"absolute","steps":[{"color":"blue","value":null}]}},"overrides":[]},"gridPos":{"h":4,"w":2,"x":12,"y":1},"id":59,"options":{"colorMode":"value","graphMode":"none","justifyMode":"auto","orientation":"auto","reduceOptions":{"calcs":["last"],"fields":"","values":false},"text":{},"textMode":"value"},"pluginVersion":"8.3.4","targets":[{"datasource":{"type":"prometheus","uid":"${datasource}"},"expr":"count(kube_namespace_created)","interval":"","legendFormat":"","refId":"A"}],"title":"Namespaces","type":"stat"},{"datasource":{"type":"prometheus","uid":"${datasource}"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"NB","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":25,"gradientMode":"opacity","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"smooth","lineWidth":2,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":true,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green","value":null},{"color":"red","value":80}]},"unit":"short"},"overrides":[]},"gridPos":{"h":12,"w":10,"x":14,"y":1},"id":52,"options":{"legend":{"calcs":["min","max","mean"],"displayMode":"table","placement":"right","showLegend":true},"tooltip":{"mode":"multi","sort":"none"}},"pluginVersion":"8.3.3","targets":[{"datasource":{"type":"prometheus","uid":"${datasource}"},"exemplar":true,"expr":"sum(kube_namespace_labels)","interval":"","legendFormat":"Namespaces","refId":"A"},{"datasource":{"type":"prometheus","uid":"${datasource}"},"expr":"sum(kube_pod_container_status_running)","interval":"","legendFormat":"Running Containers","refId":"B"},{"datasource":{"type":"prometheus","uid":"${datasource}"},"expr":"sum(kube_pod_status_phase{phase='Running'})","interval":"","legendFormat":"Running Pods","refId":"O"},{"datasource":{"type":"prometheus","uid":"${datasource}"},"expr":"sum(kube_service_info)","interval":"","legendFormat":"Services","refId":"C"},{"datasource":{"type":"prometheus","uid":"${datasource}"},"expr":"sum(kube_endpoint_info)","interval":"","legendFormat":"Endpoints","refId":"D"},{"datasource":{"type":"prometheus","uid":"${datasource}"},"expr":"sum(kube_ingress_info)","interval":"","legendFormat":"Ingresses","refId":"E"},{"datasource":{"type":"prometheus","uid":"${datasource}"},"expr":"sum(kube_deployment_labels)","interval":"","legendFormat":"Deployments","refId":"F"},{"datasource":{"type":"prometheus","uid":"${datasource}"},"expr":"sum(kube_statefulset_labels)","interval":"","legendFormat":"Statefulsets","refId":"G"},{"datasource":{"type":"prometheus","uid":"${datasource}"},"expr":"sum(kube_daemonset_labels)","interval":"","legendFormat":"Daemonsets","refId":"H"},{"datasource":{"type":"prometheus","uid":"${datasource}"},"expr":"sum(kube_persistentvolumeclaim_info)","interval":"","legendFormat":"Persistent Volume Claims","refId":"I"},{"datasource":{"type":"prometheus","uid":"${datasource}"},"expr":"sum(kube_hpa_labels)","interval":"","legendFormat":"Horizontal Pod Autoscalers","refId":"J"},{"datasource":{"type":"prometheus","uid":"${datasource}"},"expr":"sum(kube_configmap_info)","interval":"","legendFormat":"Configmaps","refId":"K"},{"datasource":{"type":"prometheus","uid":"${datasource}"},"expr":"sum(kube_secret_info)","interval":"","legendFormat":"Secrets","refId":"L"},{"datasource":{"type":"prometheus","uid":"${datasource}"},"expr":"sum(kube_networkpolicy_labels)","interval":"","legendFormat":"Network Policies","refId":"M"},{"datasource":{"type":"prometheus","uid":"${datasource}"},"exemplar":true,"expr":"count(up{job=\"node-exporter\"})","hide":false,"interval":"","legendFormat":"Nodes","refId":"N"}],"title":"Kubernetes Resource Count","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"${datasource}"},"fieldConfig":{"defaults":{"mappings":[],"noValue":"0","thresholds":{"mode":"absolute","steps":[{"color":"blue","value":null}]}},"overrides":[]},"gridPos":{"h":4,"w":2,"x":12,"y":5},"id":62,"options":{"colorMode":"value","graphMode":"none","justifyMode":"auto","orientation":"auto","reduceOptions":{"calcs":["last"],"fields":"","values":false},"text":{},"textMode":"value"},"pluginVersion":"8.3.4","targets":[{"datasource":{"type":"prometheus","uid":"${datasource}"},"expr":"sum(kube_pod_status_phase{phase='Running'})","interval":"","legendFormat":"","refId":"A"}],"title":"Running Pods","type":"stat"},{"datasource":{"type":"prometheus","uid":"${datasource}"},"fieldConfig":{"defaults":{"mappings":[],"noValue":"0","thresholds":{"mode":"absolute","steps":[{"color":"rgb(255, 255, 255)","value":null}]},"unit":"none"},"overrides":[]},"gridPos":{"h":4,"w":6,"x":0,"y":9},"id":37,"options":{"colorMode":"none","graphMode":"none","justifyMode":"center","orientation":"auto","reduceOptions":{"calcs":["mean"],"fields":"","values":false},"text":{},"textMode":"auto"},"pluginVersion":"8.3.4","targets":[{"datasource":{"type":"prometheus","uid":"${datasource}"},"editorMode":"code","exemplar":false,"expr":"sum(kube_pod_container_resource_requests{unit=\"core\"})","hide":false,"interval":"","legendFormat":"Requests","range":true,"refId":"B"},{"datasource":{"type":"prometheus","uid":"${datasource}"},"editorMode":"code","exemplar":false,"expr":"sum(kube_pod_container_resource_limits{unit=\"core\"})","hide":false,"interval":"","legendFormat":"Limits","range":true,"refId":"C"},{"datasource":{"type":"prometheus","uid":"${datasource}"},"editorMode":"code","exemplar":false,"expr":"sum(machine_cpu_cores)","hide":false,"interval":"","legendFormat":"Total","range":true,"refId":"D"}],"title":"CPU Usage","type":"stat"},{"datasource":{"type":"prometheus","uid":"${datasource}"},"fieldConfig":{"defaults":{"mappings":[],"noValue":"0","thresholds":{"mode":"absolute","steps":[{"color":"rgb(255, 255, 255)","value":null}]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":4,"w":8,"x":6,"y":9},"id":39,"options":{"colorMode":"none","graphMode":"none","justifyMode":"auto","orientation":"auto","reduceOptions":{"calcs":["mean"],"fields":"","values":false},"text":{},"textMode":"auto"},"pluginVersion":"8.3.4","targets":[{"datasource":{"type":"prometheus","uid":"${datasource}"},"editorMode":"code","exemplar":false,"expr":"sum(kube_pod_container_resource_requests{unit=\"byte\"})","hide":false,"interval":"","legendFormat":"Requests","range":true,"refId":"B"},{"datasource":{"type":"prometheus","uid":"${datasource}"},"editorMode":"code","exemplar":false,"expr":"sum(kube_pod_container_resource_limits{unit=\"byte\"})","hide":false,"interval":"","legendFormat":"Limits","range":true,"refId":"C"},{"datasource":{"type":"prometheus","uid":"${datasource}"},"editorMode":"code","exemplar":false,"expr":"sum(machine_memory_bytes)","hide":false,"interval":"","legendFormat":"Total","range":true,"refId":"D"}],"title":"RAM Usage","type":"stat"},{"collapsed":false,"datasource":{"type":"prometheus","uid":"prometheus"},"gridPos":{"h":1,"w":24,"x":0,"y":13},"id":71,"panels":[],"title":"Resources","type":"row"},{"datasource":{"type":"prometheus","uid":"${datasource}"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"CPU Cores","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":25,"gradientMode":"opacity","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"smooth","lineStyle":{"fill":"solid"},"lineWidth":2,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":true,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"decimals":2,"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green","value":null},{"color":"red","value":80}]},"unit":"none"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":14},"id":46,"options":{"legend":{"calcs":["min","max","mean"],"displayMode":"table","placement":"right","showLegend":true,"sortBy":"Max","sortDesc":true},"tooltip":{"mode":"single","sort":"none"}},"pluginVersion":"8.3.3","targets":[{"datasource":{"type":"prometheus","uid":"${datasource}"},"exemplar":false,"expr":"sum(rate(container_cpu_usage_seconds_total{image!=\"\"}[$__rate_interval])) by (namespace)","interval":"$resolution","legendFormat":"{{ namespace }}","refId":"A"}],"title":"CPU Utilization by namespace","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"${datasource}"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"MEMORY","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":25,"gradientMode":"opacity","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"smooth","lineWidth":2,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":true,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green","value":null},{"color":"red","value":80}]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":14},"id":50,"options":{"legend":{"calcs":["min","max","mean"],"displayMode":"table","placement":"right","showLegend":true,"sortBy":"Max","sortDesc":true},"tooltip":{"mode":"single","sort":"none"}},"pluginVersion":"8.3.3","targets":[{"datasource":{"type":"prometheus","uid":"${datasource}"},"exemplar":true,"expr":"sum(container_memory_working_set_bytes{image!=\"\"}) by (namespace)","interval":"$resolution","legendFormat":"{{ namespace }}","refId":"A"}],"title":"Memory Utilization by namespace","type":"timeseries"},{"collapsed":false,"datasource":{"type":"prometheus","uid":"prometheus"},"gridPos":{"h":1,"w":24,"x":0,"y":22},"id":69,"panels":[],"title":"Network","type":"row"},{"datasource":{"type":"prometheus","uid":"${datasource}"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"BANDWIDTH","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":25,"gradientMode":"opacity","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"smooth","lineWidth":2,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":true,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green","value":null},{"color":"red","value":80}]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":23},"id":79,"options":{"legend":{"calcs":[],"displayMode":"list","placement":"bottom","showLegend":false},"tooltip":{"mode":"multi","sort":"desc"}},"pluginVersion":"8.3.3","targets":[{"datasource":{"type":"prometheus","uid":"${datasource}"},"editorMode":"code","exemplar":true,"expr":"sum(rate(container_network_receive_bytes_total[$__rate_interval])) by (namespace)","interval":"$resolution","legendFormat":"Received bytes in {{ namespace }}","range":true,"refId":"A"},{"datasource":{"type":"prometheus","uid":"${datasource}"},"editorMode":"code","expr":"- sum(rate(container_network_transmit_bytes_total[$__rate_interval])) by (namespace)","hide":false,"interval":"$resolution","legendFormat":"Transmitted bytes in {{ namespace }}","range":true,"refId":"B"}],"title":"Network Received by namespace","type":"timeseries"}],"refresh":"30s","schemaVersion":34,"style":"dark","tags":["Kubernetes","Prometheus"],"templating":{"list":[{"current":{"selected":false,"text":"Prometheus","value":"Prometheus"},"hide":0,"includeAll":false,"multi":false,"name":"datasource","options":[],"query":"prometheus","queryValue":"","refresh":1,"regex":"","skipUrlSync":false,"type":"datasource"},{"current":{"selected":true,"text":"30s","value":"30s"},"hide":0,"includeAll":false,"multi":false,"name":"resolution","options":[{"selected":false,"text":"1s","value":"1s"},{"selected":false,"text":"15s","value":"15s"},{"selected":true,"text":"30s","value":"30s"},{"selected":false,"text":"1m","value":"1m"},{"selected":false,"text":"3m","value":"3m"},{"selected":false,"text":"5m","value":"5m"}],"query":"1s, 15s, 30s, 1m, 3m, 5m","queryValue":"","skipUrlSync":false,"type":"custom"}]},"time":{"from":"now-1h","to":"now"},"timepicker":{},"timezone":"","title":"k8s / view / global","uid":"k8s_view_global","version":3,"weekStart":""} \ No newline at end of file diff --git a/blueprints/gke/autopilot/bundle/monitoring/dashboards/k8s-namespaces.json b/blueprints/gke/autopilot/bundle/monitoring/dashboards/k8s-namespaces.json new file mode 100644 index 0000000000..815e9e1a92 --- /dev/null +++ b/blueprints/gke/autopilot/bundle/monitoring/dashboards/k8s-namespaces.json @@ -0,0 +1 @@ +{"annotations":{"list":[{"builtIn":1,"datasource":{"type":"datasource","uid":"grafana"},"enable":true,"hide":true,"iconColor":"rgba(0, 211, 255, 1)","name":"Annotations & Alerts","target":{"limit":100,"matchAny":false,"tags":[],"type":"dashboard"},"type":"dashboard"}]},"description":"This is a modern 'Namespaces View' dashboard for your Kubernetes cluster(s). Made for kube-prometheus-stack and take advantage of the latest Grafana features. GitHub repository: https://github.com/dotdc/grafana-dashboards-kubernetes","editable":true,"fiscalYearStartMonth":0,"gnetId":15758,"graphTooltip":1,"id":10,"iteration":1677857471900,"links":[],"liveNow":false,"panels":[{"collapsed":false,"datasource":{"type":"datasource","uid":"grafana"},"gridPos":{"h":1,"w":24,"x":0,"y":0},"id":38,"panels":[],"title":"Overview","type":"row"},{"datasource":{"type":"prometheus","uid":"${datasource}"},"fieldConfig":{"defaults":{"decimals":2,"mappings":[],"max":1,"min":0,"thresholds":{"mode":"percentage","steps":[{"color":"green","value":null},{"color":"orange","value":50},{"color":"red","value":70}]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":7,"w":6,"x":0,"y":1},"id":46,"options":{"orientation":"auto","reduceOptions":{"calcs":["mean"],"fields":"","values":false},"showThresholdLabels":false,"showThresholdMarkers":true,"text":{}},"pluginVersion":"8.3.4","targets":[{"datasource":{"type":"prometheus","uid":"${datasource}"},"editorMode":"code","exemplar":false,"expr":"sum(rate(container_cpu_usage_seconds_total{namespace=~\"$namespace\", image!=\"\"}[$__rate_interval])) / sum(machine_cpu_cores)","instant":true,"interval":"","legendFormat":"","range":false,"refId":"A"}],"title":"Namespace(s) usage on total cluster CPU in %","type":"gauge"},{"datasource":{"type":"prometheus","uid":"${datasource}"},"fieldConfig":{"defaults":{"decimals":2,"mappings":[],"max":1,"min":0,"thresholds":{"mode":"percentage","steps":[{"color":"green","value":null},{"color":"orange","value":50},{"color":"red","value":70}]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":7,"w":6,"x":6,"y":1},"id":48,"options":{"orientation":"auto","reduceOptions":{"calcs":["lastNotNull"],"fields":"","values":false},"showThresholdLabels":false,"showThresholdMarkers":true,"text":{}},"pluginVersion":"8.3.4","targets":[{"datasource":{"type":"prometheus","uid":"${datasource}"},"exemplar":true,"expr":"sum(container_memory_working_set_bytes{namespace=~\"$namespace\"}) / sum(machine_memory_bytes)","interval":"","legendFormat":"","refId":"A"}],"title":"Namespace(s) usage on total cluster RAM in %","type":"gauge"},{"datasource":{"type":"prometheus","uid":"${datasource}"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":25,"gradientMode":"opacity","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"smooth","lineWidth":2,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":true,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green","value":null},{"color":"red","value":80}]},"unit":"short"},"overrides":[]},"gridPos":{"h":11,"w":12,"x":12,"y":1},"id":32,"options":{"legend":{"calcs":["min","max","mean"],"displayMode":"table","placement":"right","showLegend":true,"sortBy":"Max","sortDesc":true},"tooltip":{"mode":"multi","sort":"desc"}},"pluginVersion":"8.3.3","targets":[{"datasource":{"type":"prometheus","uid":"${datasource}"},"expr":"sum(kube_pod_container_status_running{namespace=~\"$namespace\"})","interval":"","legendFormat":"Running Pods","refId":"A"},{"datasource":{"type":"prometheus","uid":"${datasource}"},"expr":"sum(kube_service_info{namespace=~\"$namespace\"})","interval":"","legendFormat":"Services","refId":"B"},{"datasource":{"type":"prometheus","uid":"${datasource}"},"expr":"sum(kube_ingress_info{namespace=~\"$namespace\"})","interval":"","legendFormat":"Ingresses","refId":"C"},{"datasource":{"type":"prometheus","uid":"${datasource}"},"expr":"sum(kube_deployment_labels{namespace=~\"$namespace\"})","interval":"","legendFormat":"Deployments","refId":"D"},{"datasource":{"type":"prometheus","uid":"${datasource}"},"expr":"sum(kube_statefulset_labels{namespace=~\"$namespace\"})","interval":"","legendFormat":"Statefulsets","refId":"E"},{"datasource":{"type":"prometheus","uid":"${datasource}"},"expr":"sum(kube_daemonset_labels{namespace=~\"$namespace\"})","interval":"","legendFormat":"Daemonsets","refId":"F"},{"datasource":{"type":"prometheus","uid":"${datasource}"},"expr":"sum(kube_persistentvolumeclaim_info{namespace=~\"$namespace\"})","interval":"","legendFormat":"Persistent Volume Claims","refId":"G"},{"datasource":{"type":"prometheus","uid":"${datasource}"},"expr":"sum(kube_hpa_labels{namespace=~\"$namespace\"})","interval":"","legendFormat":"Horizontal Pod Autoscalers","refId":"H"},{"datasource":{"type":"prometheus","uid":"${datasource}"},"expr":"sum(kube_configmap_info{namespace=~\"$namespace\"})","interval":"","legendFormat":"Configmaps","refId":"I"},{"datasource":{"type":"prometheus","uid":"${datasource}"},"expr":"sum(kube_secret_info{namespace=~\"$namespace\"})","interval":"","legendFormat":"Secrets","refId":"J"},{"datasource":{"type":"prometheus","uid":"${datasource}"},"expr":"sum(kube_networkpolicy_labels{namespace=~\"$namespace\"})","interval":"","legendFormat":"Network Policies","refId":"K"}],"title":"Kubernetes Resource Count","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"${datasource}"},"fieldConfig":{"defaults":{"mappings":[],"noValue":"0","thresholds":{"mode":"absolute","steps":[{"color":"rgb(255, 255, 255)","value":null}]},"unit":"none"},"overrides":[]},"gridPos":{"h":4,"w":6,"x":0,"y":8},"id":62,"options":{"colorMode":"none","graphMode":"none","justifyMode":"center","orientation":"auto","reduceOptions":{"calcs":["mean"],"fields":"","values":false},"text":{},"textMode":"auto"},"pluginVersion":"8.3.4","targets":[{"datasource":{"type":"prometheus","uid":"${datasource}"},"editorMode":"code","exemplar":true,"expr":"sum(rate(container_cpu_usage_seconds_total{namespace=~\"$namespace\"}[$__rate_interval]))","interval":"","legendFormat":"Real","range":true,"refId":"A"},{"datasource":{"type":"prometheus","uid":"${datasource}"},"editorMode":"code","expr":"sum(kube_pod_container_resource_requests{namespace=~\"$namespace\", unit=\"core\"})","hide":false,"legendFormat":"Requests","range":true,"refId":"B"},{"datasource":{"type":"prometheus","uid":"${datasource}"},"editorMode":"code","expr":"sum(kube_pod_container_resource_limits{namespace=~\"$namespace\", unit=\"core\"})","hide":false,"legendFormat":"Limits","range":true,"refId":"C"},{"datasource":{"type":"prometheus","uid":"${datasource}"},"editorMode":"code","expr":"sum(machine_cpu_cores)","hide":false,"legendFormat":"Cluster Total","range":true,"refId":"D"}],"title":"Namespace(s) CPU Usage in cores","type":"stat"},{"datasource":{"type":"prometheus","uid":"${datasource}"},"fieldConfig":{"defaults":{"mappings":[],"noValue":"0","thresholds":{"mode":"absolute","steps":[{"color":"rgb(255, 255, 255)","value":null}]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":4,"w":6,"x":6,"y":8},"id":64,"options":{"colorMode":"none","graphMode":"none","justifyMode":"auto","orientation":"auto","reduceOptions":{"calcs":["mean"],"fields":"","values":false},"text":{},"textMode":"auto"},"pluginVersion":"8.3.4","targets":[{"datasource":{"type":"prometheus","uid":"${datasource}"},"editorMode":"code","exemplar":true,"expr":"sum(container_memory_working_set_bytes{namespace=~\"$namespace\"})","interval":"","legendFormat":"Real","range":true,"refId":"A"},{"datasource":{"type":"prometheus","uid":"${datasource}"},"editorMode":"code","expr":"sum(kube_pod_container_resource_requests{namespace=~\"$namespace\", unit=\"byte\"})","hide":false,"legendFormat":"Requests","range":true,"refId":"B"},{"datasource":{"type":"prometheus","uid":"${datasource}"},"editorMode":"code","expr":"sum(kube_pod_container_resource_limits{namespace=~\"$namespace\", unit=\"byte\"})","hide":false,"legendFormat":"Limits","range":true,"refId":"C"},{"datasource":{"type":"prometheus","uid":"${datasource}"},"editorMode":"code","expr":"sum(machine_memory_bytes)","hide":false,"legendFormat":"Cluster Total","range":true,"refId":"D"}],"title":"Namespace(s) RAM Usage in bytes","type":"stat"},{"collapsed":false,"datasource":{"type":"datasource","uid":"grafana"},"gridPos":{"h":1,"w":24,"x":0,"y":12},"id":40,"panels":[],"title":"Resources","type":"row"},{"datasource":{"type":"prometheus","uid":"${datasource}"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"CPU Cores","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":25,"gradientMode":"opacity","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"smooth","lineWidth":2,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":true,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green","value":null},{"color":"red","value":80}]},"unit":"none"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":13},"id":29,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"right","showLegend":true},"tooltip":{"mode":"single","sort":"none"}},"pluginVersion":"8.3.3","targets":[{"datasource":{"type":"prometheus","uid":"${datasource}"},"exemplar":true,"expr":"sum(rate(container_cpu_usage_seconds_total{namespace=~\"$namespace\", image!=\"\"}[$__rate_interval])) by (pod)","interval":"$resolution","legendFormat":"{{ pod }}","refId":"A"}],"title":"CPU usage by Pod","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"${datasource}"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":25,"gradientMode":"opacity","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"smooth","lineWidth":2,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":true,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green","value":null},{"color":"red","value":80}]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":13},"id":30,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"right","showLegend":true},"tooltip":{"mode":"single","sort":"none"}},"pluginVersion":"8.3.3","targets":[{"datasource":{"type":"prometheus","uid":"${datasource}"},"exemplar":true,"expr":"sum(container_memory_working_set_bytes{namespace=~\"$namespace\", image!=\"\"}) by (pod)","interval":"$resolution","legendFormat":"{{ pod }}","refId":"A"}],"title":"Memory usage by Pod","type":"timeseries"},{"collapsed":false,"datasource":{"type":"datasource","uid":"grafana"},"gridPos":{"h":1,"w":24,"x":0,"y":21},"id":44,"panels":[],"title":"Kubernetes Resources","type":"row"},{"datasource":{"type":"prometheus","uid":"${datasource}"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":25,"gradientMode":"opacity","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"smooth","lineWidth":2,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":true,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"decimals":0,"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green","value":null},{"color":"red","value":80}]},"unit":"none"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":22},"id":5,"options":{"legend":{"calcs":[],"displayMode":"list","placement":"bottom","showLegend":false},"tooltip":{"mode":"multi","sort":"desc"}},"pluginVersion":"8.3.3","targets":[{"datasource":{"type":"prometheus","uid":"${datasource}"},"expr":"sum(kube_pod_container_status_ready{namespace=~\"$namespace\"})","interval":"","legendFormat":"Ready","refId":"A"},{"datasource":{"type":"prometheus","uid":"${datasource}"},"expr":"sum(kube_pod_container_status_running{namespace=~\"$namespace\"})","interval":"","legendFormat":"Running","refId":"B"},{"datasource":{"type":"prometheus","uid":"${datasource}"},"expr":"sum(kube_pod_container_status_waiting{namespace=~\"$namespace\"})","interval":"","legendFormat":"Waiting","refId":"C"},{"datasource":{"type":"prometheus","uid":"${datasource}"},"expr":"sum(kube_pod_container_status_restarts_total{namespace=~\"$namespace\"})","interval":"","legendFormat":"Restarts Total","refId":"D"},{"datasource":{"type":"prometheus","uid":"${datasource}"},"expr":"sum(kube_pod_container_status_terminated{namespace=~\"$namespace\"})","interval":"","legendFormat":"Terminated","refId":"E"}],"title":"Nb of pods by state","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"${datasource}"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":25,"gradientMode":"opacity","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"smooth","lineWidth":2,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":true,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"decimals":0,"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green","value":null},{"color":"red","value":80}]},"unit":"none"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":22},"id":2,"options":{"legend":{"calcs":[],"displayMode":"list","placement":"bottom","showLegend":false},"tooltip":{"mode":"multi","sort":"none"}},"pluginVersion":"8.3.3","targets":[{"datasource":{"type":"prometheus","uid":"${datasource}"},"expr":"sum(kube_pod_container_info{namespace=~\"$namespace\"}) by (pod)","interval":"","legendFormat":"{{ pod }}","refId":"A"}],"title":"Nb of containers by pod","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"${datasource}"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":25,"gradientMode":"opacity","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"smooth","lineWidth":2,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green","value":null},{"color":"red","value":80}]},"unit":"short"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":30},"id":7,"options":{"legend":{"calcs":[],"displayMode":"list","placement":"bottom","showLegend":false},"tooltip":{"mode":"multi","sort":"none"}},"pluginVersion":"8.3.3","targets":[{"datasource":{"type":"prometheus","uid":"${datasource}"},"exemplar":true,"expr":"sum(kube_deployment_status_replicas_available{namespace=~\"$namespace\"}) by (deployment)","interval":"","legendFormat":"{{ deployment }}","refId":"A"}],"title":"Replicas available by deployment","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"${datasource}"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":25,"gradientMode":"opacity","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"smooth","lineWidth":2,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green","value":null},{"color":"red","value":80}]},"unit":"short"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":30},"id":8,"options":{"legend":{"calcs":[],"displayMode":"list","placement":"bottom","showLegend":false},"tooltip":{"mode":"multi","sort":"none"}},"pluginVersion":"8.3.3","targets":[{"datasource":{"type":"prometheus","uid":"${datasource}"},"expr":"sum(kube_deployment_status_replicas_unavailable{namespace=~\"$namespace\"}) by (deployment)","interval":"","legendFormat":"{{ deployment }}","refId":"A"}],"title":"Replicas unavailable by deployment","type":"timeseries"},{"collapsed":false,"datasource":{"type":"datasource","uid":"grafana"},"gridPos":{"h":1,"w":24,"x":0,"y":38},"id":42,"panels":[],"title":"Kubernetes Storage","type":"row"},{"datasource":{"type":"prometheus","uid":"${datasource}"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":25,"gradientMode":"opacity","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"smooth","lineWidth":2,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":true,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"decimals":2,"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green","value":null},{"color":"red","value":80}]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":39},"id":12,"options":{"legend":{"calcs":[],"displayMode":"list","placement":"bottom","showLegend":false},"tooltip":{"mode":"multi","sort":"desc"}},"pluginVersion":"8.3.3","targets":[{"datasource":{"type":"prometheus","uid":"${datasource}"},"exemplar":true,"expr":"sum(kubelet_volume_stats_used_bytes{namespace=~\"$namespace\"}) by (persistentvolumeclaim) / sum(kubelet_volume_stats_capacity_bytes{namespace=~\"$namespace\"}) by (persistentvolumeclaim)","interval":"","legendFormat":"{{ persistentvolumeclaim }}","refId":"A"}],"title":"Persistent Volumes - Capacity","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"${datasource}"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":25,"gradientMode":"opacity","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"smooth","lineWidth":2,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":true,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"decimals":2,"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green","value":null},{"color":"red","value":80}]},"unit":"percent"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":39},"id":27,"options":{"legend":{"calcs":[],"displayMode":"list","placement":"bottom","showLegend":false},"tooltip":{"mode":"multi","sort":"desc"}},"pluginVersion":"8.3.3","targets":[{"datasource":{"type":"prometheus","uid":"${datasource}"},"exemplar":true,"expr":"sum(kubelet_volume_stats_inodes_used{namespace=~\"$namespace\"}) by (persistentvolumeclaim) / sum(kubelet_volume_stats_inodes{namespace=~\"$namespace\"}) by (persistentvolumeclaim) * 100","interval":"","legendFormat":"{{ persistentvolumeclaim }}","refId":"A"}],"title":"Persistent Volumes - Inodes","type":"timeseries"}],"refresh":"30s","schemaVersion":34,"style":"dark","tags":["Kubernetes","Prometheus"],"templating":{"list":[{"current":{"selected":false,"text":"Prometheus","value":"Prometheus"},"hide":0,"includeAll":false,"multi":false,"name":"datasource","options":[],"query":"prometheus","queryValue":"","refresh":1,"regex":"","skipUrlSync":false,"type":"datasource"},{"current":{"selected":true,"text":["kube-system"],"value":["kube-system"]},"datasource":{"type":"prometheus","uid":"${datasource}"},"definition":"label_values(kube_pod_info, namespace)","hide":0,"includeAll":true,"multi":true,"name":"namespace","options":[],"query":{"query":"label_values(kube_pod_info, namespace)","refId":"StandardVariableQuery"},"refresh":1,"regex":"","skipUrlSync":false,"sort":1,"tagValuesQuery":"","tagsQuery":"","type":"query","useTags":false},{"current":{"selected":true,"text":"30s","value":"30s"},"hide":0,"includeAll":false,"multi":false,"name":"resolution","options":[{"selected":false,"text":"1s","value":"1s"},{"selected":false,"text":"15s","value":"15s"},{"selected":true,"text":"30s","value":"30s"},{"selected":false,"text":"1m","value":"1m"},{"selected":false,"text":"3m","value":"3m"},{"selected":false,"text":"5m","value":"5m"}],"query":"1s, 15s, 30s, 1m, 3m, 5m","queryValue":"","skipUrlSync":false,"type":"custom"}]},"time":{"from":"now-1h","to":"now"},"timepicker":{},"timezone":"","title":"k8s / view / namespaces","uid":"k8s_view_ns","version":1,"weekStart":""} \ No newline at end of file diff --git a/blueprints/gke/autopilot/bundle/monitoring/dashboards/k8s-nodes.json b/blueprints/gke/autopilot/bundle/monitoring/dashboards/k8s-nodes.json new file mode 100644 index 0000000000..9c8fee93fa --- /dev/null +++ b/blueprints/gke/autopilot/bundle/monitoring/dashboards/k8s-nodes.json @@ -0,0 +1 @@ +{"annotations":{"list":[{"builtIn":1,"datasource":{"type":"datasource","uid":"grafana"},"enable":true,"hide":true,"iconColor":"rgba(0, 211, 255, 1)","name":"Annotations & Alerts","target":{"limit":100,"matchAny":false,"tags":[],"type":"dashboard"},"type":"dashboard"}]},"description":"This is a modern 'Nodes View' dashboard for your Kubernetes cluster(s). Made for kube-prometheus-stack and take advantage of the latest Grafana features. GitHub repository: https://github.com/dotdc/grafana-dashboards-kubernetes","editable":true,"fiscalYearStartMonth":0,"gnetId":15759,"graphTooltip":1,"id":9,"iteration":1677857491220,"links":[],"liveNow":false,"panels":[{"collapsed":false,"datasource":{"type":"prometheus","uid":"prometheus"},"gridPos":{"h":1,"w":24,"x":0,"y":0},"id":40,"panels":[],"title":"Overview","type":"row"},{"datasource":{"type":"prometheus","uid":"${datasource}"},"fieldConfig":{"defaults":{"mappings":[],"noValue":"0","thresholds":{"mode":"absolute","steps":[{"color":"rgb(255, 255, 255)","value":null}]},"unit":"none"},"overrides":[]},"gridPos":{"h":8,"w":3,"x":0,"y":1},"id":11,"options":{"colorMode":"none","graphMode":"none","justifyMode":"auto","orientation":"auto","reduceOptions":{"calcs":["mean"],"fields":"","values":false},"text":{},"textMode":"auto"},"pluginVersion":"8.3.4","targets":[{"datasource":{"type":"prometheus","uid":"${datasource}"},"exemplar":true,"expr":"sum(machine_cpu_cores{node=\"$node\"})","interval":"$resolution","legendFormat":"","refId":"A"}],"title":"CPU Total","type":"stat"},{"datasource":{"type":"prometheus","uid":"${datasource}"},"fieldConfig":{"defaults":{"mappings":[],"noValue":"0","thresholds":{"mode":"absolute","steps":[{"color":"rgb(255, 255, 255)","value":null}]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":5,"x":3,"y":1},"id":17,"options":{"colorMode":"none","graphMode":"none","justifyMode":"auto","orientation":"auto","reduceOptions":{"calcs":["mean"],"fields":"","values":false},"text":{},"textMode":"auto"},"pluginVersion":"8.3.4","targets":[{"datasource":{"type":"prometheus","uid":"${datasource}"},"exemplar":true,"expr":"machine_memory_bytes{node=\"$node\"}","instant":false,"interval":"","legendFormat":"","refId":"A"}],"title":"RAM Total","type":"stat"},{"datasource":{"type":"prometheus","uid":"${datasource}"},"fieldConfig":{"defaults":{"mappings":[],"noValue":"0","thresholds":{"mode":"absolute","steps":[{"color":"blue","value":null}]}},"overrides":[]},"gridPos":{"h":8,"w":4,"x":8,"y":1},"id":24,"options":{"colorMode":"value","graphMode":"none","justifyMode":"auto","orientation":"auto","reduceOptions":{"calcs":["last"],"fields":"","values":false},"text":{},"textMode":"value"},"pluginVersion":"8.3.4","targets":[{"datasource":{"type":"prometheus","uid":"${datasource}"},"exemplar":true,"expr":"sum(kube_pod_info{node=\"$node\"})","interval":"","legendFormat":"","refId":"A"}],"title":"Pods on node","type":"stat"},{"datasource":{"type":"prometheus","uid":"${datasource}"},"fieldConfig":{"defaults":{"color":{"mode":"thresholds"},"custom":{"align":"auto","displayMode":"auto"},"links":[{"targetBlank":true,"title":"Pod details","url":"/d/k8s_views_pods/kubernetes-views-pods?${datasource:queryparam}&var-namespace=${__data.fields.namespace}&var-pod=${__data.fields.pod}&${resolution:queryparam}&${__url_time_range}"}],"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green","value":null},{"color":"red","value":80}]}},"overrides":[{"matcher":{"id":"byName","options":"pod"},"properties":[{"id":"custom.width","value":416}]},{"matcher":{"id":"byName","options":"priority_class"},"properties":[{"id":"custom.width","value":176}]},{"matcher":{"id":"byName","options":"pod_ip"},"properties":[{"id":"custom.width","value":157}]},{"matcher":{"id":"byName","options":"created_by_kind"},"properties":[{"id":"custom.width","value":205}]},{"matcher":{"id":"byName","options":"namespace"},"properties":[{"id":"custom.width","value":263}]}]},"gridPos":{"h":8,"w":12,"x":12,"y":1},"id":5,"options":{"footer":{"fields":"","reducer":["sum"],"show":false},"showHeader":true,"sortBy":[]},"pluginVersion":"8.3.4","targets":[{"datasource":{"type":"prometheus","uid":"${datasource}"},"exemplar":true,"expr":"kube_pod_info{node=\"$node\"}","format":"table","interval":"","legendFormat":"","refId":"A"}],"title":"List of pods on node ($node)","transformations":[{"id":"organize","options":{"excludeByName":{"Time":true,"Value":true,"__name__":true,"container":true,"created_by_kind":false,"created_by_name":true,"endpoint":true,"env":true,"host_ip":true,"host_network":true,"instance":true,"job":true,"node":true,"project":true,"prometheus_replica":true,"service":true,"uid":true},"indexByName":{"Time":6,"Value":20,"__name__":7,"container":8,"created_by_kind":2,"created_by_name":9,"endpoint":10,"env":11,"host_ip":5,"host_network":12,"instance":13,"job":14,"namespace":1,"node":15,"pod":0,"pod_ip":3,"priority_class":4,"project":16,"prometheus_replica":17,"service":18,"uid":19},"renameByName":{}}},{"id":"groupBy","options":{"fields":{"created_by_kind":{"aggregations":[],"operation":"groupby"},"host_ip":{"aggregations":[],"operation":"groupby"},"namespace":{"aggregations":["last"],"operation":"groupby"},"pod":{"aggregations":[],"operation":"groupby"},"pod_ip":{"aggregations":[],"operation":"groupby"},"priority_class":{"aggregations":[],"operation":"groupby"}}}}],"type":"table"},{"collapsed":false,"datasource":{"type":"prometheus","uid":"prometheus"},"gridPos":{"h":1,"w":24,"x":0,"y":9},"id":38,"panels":[],"title":"Resources","type":"row"},{"datasource":{"type":"prometheus","uid":"${datasource}"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"CPU Cores","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":25,"gradientMode":"opacity","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"smooth","lineWidth":2,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":true,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green","value":null},{"color":"red","value":80}]},"unit":"none"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":10},"id":26,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"right","showLegend":true},"tooltip":{"mode":"single","sort":"none"}},"pluginVersion":"8.3.3","targets":[{"datasource":{"type":"prometheus","uid":"${datasource}"},"exemplar":true,"expr":"sum(rate(container_cpu_usage_seconds_total{node=\"$node\", image!=\"\"}[$__rate_interval])) by (pod)","interval":"$resolution","legendFormat":"{{ pod }}","refId":"A"}],"title":"CPU usage by Pod","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"${datasource}"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":25,"gradientMode":"opacity","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"smooth","lineWidth":2,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":true,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green","value":null},{"color":"red","value":80}]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":10},"id":28,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"right","showLegend":true},"tooltip":{"mode":"single","sort":"none"}},"pluginVersion":"8.3.3","targets":[{"datasource":{"type":"prometheus","uid":"${datasource}"},"exemplar":true,"expr":"sum(container_memory_working_set_bytes{node=\"$node\", image!=\"\"}) by (pod)","interval":"$resolution","legendFormat":"{{ pod }}","refId":"A"}],"title":"Memory usage by Pod","type":"timeseries"}],"refresh":"30s","schemaVersion":34,"style":"dark","tags":["Kubernetes","Prometheus"],"templating":{"list":[{"current":{"isNone":true,"selected":false,"text":"None","value":""},"datasource":{"type":"prometheus","uid":"${datasource}"},"definition":"label_values(node_uname_info, job)","hide":2,"includeAll":false,"multi":false,"name":"job","options":[],"query":{"query":"label_values(node_uname_info, job)","refId":"StandardVariableQuery"},"refresh":1,"regex":"","skipUrlSync":false,"sort":1,"type":"query"},{"current":{"selected":false,"text":"Prometheus","value":"Prometheus"},"hide":0,"includeAll":false,"multi":false,"name":"datasource","options":[],"query":"prometheus","queryValue":"","refresh":1,"regex":"","skipUrlSync":false,"type":"datasource"},{"current":{"selected":true,"text":"30s","value":"30s"},"hide":0,"includeAll":false,"multi":false,"name":"resolution","options":[{"selected":false,"text":"1s","value":"1s"},{"selected":false,"text":"15s","value":"15s"},{"selected":true,"text":"30s","value":"30s"},{"selected":false,"text":"1m","value":"1m"},{"selected":false,"text":"3m","value":"3m"},{"selected":false,"text":"5m","value":"5m"}],"query":"1s, 15s, 30s, 1m, 3m, 5m","queryValue":"","skipUrlSync":false,"type":"custom"},{"current":{"selected":false,"text":"gk3-cluster-default-pool-30d45773-8vk4","value":"gk3-cluster-default-pool-30d45773-8vk4"},"datasource":{"type":"prometheus","uid":"${datasource}"},"definition":"label_values(kube_node_info, node)","hide":0,"includeAll":false,"multi":false,"name":"node","options":[],"query":{"query":"label_values(kube_node_info, node)","refId":"StandardVariableQuery"},"refresh":2,"regex":"","skipUrlSync":false,"sort":1,"type":"query"},{"current":{"isNone":true,"selected":false,"text":"None","value":""},"datasource":{"type":"prometheus","uid":"${datasource}"},"definition":"label_values(node_uname_info{nodename=~\"(?i:($node))\"}, instance)","hide":2,"includeAll":false,"multi":false,"name":"instance","options":[],"query":{"query":"label_values(node_uname_info{nodename=~\"(?i:($node))\"}, instance)","refId":"StandardVariableQuery"},"refresh":2,"regex":"","skipUrlSync":false,"sort":1,"type":"query"}]},"time":{"from":"now-1h","to":"now"},"timepicker":{},"timezone":"","title":"k8s / view / nodes","uid":"k8s_view_nodes","version":4,"weekStart":""} \ No newline at end of file diff --git a/blueprints/gke/autopilot/bundle/monitoring/dashboards/k8s-pods.json b/blueprints/gke/autopilot/bundle/monitoring/dashboards/k8s-pods.json new file mode 100644 index 0000000000..af3f914567 --- /dev/null +++ b/blueprints/gke/autopilot/bundle/monitoring/dashboards/k8s-pods.json @@ -0,0 +1 @@ +{"annotations":{"list":[{"builtIn":1,"datasource":{"type":"datasource","uid":"grafana"},"enable":true,"hide":true,"iconColor":"rgba(0, 211, 255, 1)","name":"Annotations & Alerts","target":{"limit":100,"matchAny":false,"tags":[],"type":"dashboard"},"type":"dashboard"}]},"description":"This is a modern 'Pods View' dashboard for your Kubernetes cluster(s). Made for kube-prometheus-stack and take advantage of the latest Grafana features. GitHub repository: https://github.com/dotdc/grafana-dashboards-kubernetes","editable":true,"fiscalYearStartMonth":0,"gnetId":15760,"graphTooltip":1,"id":8,"iteration":1677857508072,"links":[],"liveNow":false,"panels":[{"collapsed":false,"datasource":{"type":"datasource","uid":"grafana"},"gridPos":{"h":1,"w":24,"x":0,"y":0},"id":43,"panels":[],"targets":[{"datasource":{"type":"datasource","uid":"grafana"},"refId":"A"}],"title":"Information","type":"row"},{"datasource":{"type":"prometheus","uid":"${datasource}"},"fieldConfig":{"defaults":{"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"rgb(255, 255, 255)","value":null}]},"unit":"none"},"overrides":[]},"gridPos":{"h":2,"w":12,"x":0,"y":1},"id":2,"options":{"colorMode":"none","graphMode":"none","justifyMode":"auto","orientation":"auto","reduceOptions":{"calcs":["mean"],"fields":"","values":false},"textMode":"name"},"pluginVersion":"8.3.4","targets":[{"datasource":{"type":"prometheus","uid":"${datasource}"},"exemplar":true,"expr":"kube_pod_info{namespace=\"$namespace\", pod=\"$pod\"}","interval":"","legendFormat":"{{ created_by_kind }}: {{ created_by_name }}","refId":"A"}],"title":"Created by","type":"stat"},{"datasource":{"type":"prometheus","uid":"${datasource}"},"fieldConfig":{"defaults":{"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"rgb(255, 255, 255)","value":null}]},"unit":"none"},"overrides":[]},"gridPos":{"h":2,"w":6,"x":12,"y":1},"id":33,"options":{"colorMode":"none","graphMode":"none","justifyMode":"auto","orientation":"auto","reduceOptions":{"calcs":["mean"],"fields":"","values":false},"textMode":"name"},"pluginVersion":"8.3.4","targets":[{"datasource":{"type":"prometheus","uid":"${datasource}"},"expr":"kube_pod_info{namespace=\"$namespace\", pod=\"$pod\"}","interval":"","legendFormat":"{{ node }}","refId":"A"}],"title":"Running on","type":"stat"},{"datasource":{"type":"prometheus","uid":"${datasource}"},"fieldConfig":{"defaults":{"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"rgb(255, 255, 255)","value":null}]},"unit":"none"},"overrides":[]},"gridPos":{"h":2,"w":6,"x":18,"y":1},"id":41,"options":{"colorMode":"none","graphMode":"none","justifyMode":"auto","orientation":"auto","reduceOptions":{"calcs":["mean"],"fields":"","values":false},"textMode":"name"},"pluginVersion":"8.3.4","targets":[{"datasource":{"type":"prometheus","uid":"${datasource}"},"expr":"kube_pod_info{namespace=\"$namespace\", pod=\"$pod\"}","interval":"","legendFormat":"{{ pod_ip }}","refId":"A"}],"title":"Pod IP","type":"stat"},{"collapsed":false,"datasource":{"type":"datasource","uid":"grafana"},"gridPos":{"h":1,"w":24,"x":0,"y":3},"id":47,"panels":[],"targets":[{"datasource":{"type":"datasource","uid":"grafana"},"refId":"A"}],"title":"Resources","type":"row"},{"datasource":{"type":"prometheus","uid":"${datasource}"},"fieldConfig":{"defaults":{"color":{"fixedColor":"blue","mode":"fixed"},"decimals":2,"mappings":[],"max":1,"min":0,"thresholds":{"mode":"percentage","steps":[{"color":"green","value":null},{"color":"#EAB839","value":60},{"color":"red","value":75}]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":3,"x":0,"y":4},"id":39,"options":{"orientation":"auto","reduceOptions":{"calcs":["last"],"fields":"","values":false},"showThresholdLabels":false,"showThresholdMarkers":true},"pluginVersion":"8.3.4","targets":[{"datasource":{"type":"prometheus","uid":"${datasource}"},"exemplar":false,"expr":"sum(rate(container_cpu_usage_seconds_total{namespace=\"$namespace\", pod=\"$pod\", image!=\"\"}[$__rate_interval])) / sum(kube_pod_container_resource_requests{namespace=\"$namespace\", pod=\"$pod\", unit=\"core\"})","instant":true,"interval":"$resolution","legendFormat":"Requests","refId":"A"}],"title":"Total pod CPU Requests usage","type":"gauge"},{"datasource":{"type":"prometheus","uid":"${datasource}"},"fieldConfig":{"defaults":{"decimals":2,"mappings":[],"max":1,"min":0,"thresholds":{"mode":"percentage","steps":[{"color":"green","value":null},{"color":"#EAB839","value":60},{"color":"red","value":75}]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":3,"x":3,"y":4},"id":48,"options":{"orientation":"auto","reduceOptions":{"calcs":["last"],"fields":"","values":false},"showThresholdLabels":false,"showThresholdMarkers":true},"pluginVersion":"8.3.4","targets":[{"datasource":{"type":"prometheus","uid":"${datasource}"},"exemplar":false,"expr":"sum(rate(container_cpu_usage_seconds_total{namespace=\"$namespace\", pod=\"$pod\", image!=\"\"}[$__rate_interval])) / sum(kube_pod_container_resource_limits{namespace=\"$namespace\", pod=\"$pod\", unit=\"core\"})","instant":true,"interval":"$resolution","legendFormat":"Limits","refId":"A"}],"title":"Total pod CPU Limits usage","type":"gauge"},{"datasource":{"type":"prometheus","uid":"${datasource}"},"fieldConfig":{"defaults":{"color":{"fixedColor":"blue","mode":"fixed"},"decimals":2,"mappings":[],"max":1,"min":0,"thresholds":{"mode":"percentage","steps":[{"color":"blue","value":null},{"color":"#EAB839","value":80},{"color":"red","value":99}]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":3,"x":6,"y":4},"id":40,"options":{"orientation":"auto","reduceOptions":{"calcs":["last"],"fields":"","values":false},"showThresholdLabels":false,"showThresholdMarkers":true},"pluginVersion":"8.3.4","targets":[{"datasource":{"type":"prometheus","uid":"${datasource}"},"exemplar":false,"expr":"sum(container_memory_working_set_bytes{namespace=\"$namespace\", pod=\"$pod\", image!=\"\"}) / sum(kube_pod_container_resource_requests{namespace=\"$namespace\", pod=\"$pod\", unit=\"byte\"})","instant":true,"interval":"$resolution","legendFormat":"Requests","refId":"A"}],"title":"Total pod RAM Requests usage","type":"gauge"},{"datasource":{"type":"prometheus","uid":"${datasource}"},"fieldConfig":{"defaults":{"decimals":2,"mappings":[],"max":1,"min":0,"thresholds":{"mode":"percentage","steps":[{"color":"green","value":null},{"color":"#EAB839","value":60},{"color":"red","value":75}]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":3,"x":9,"y":4},"id":49,"options":{"orientation":"auto","reduceOptions":{"calcs":["last"],"fields":"","values":false},"showThresholdLabels":false,"showThresholdMarkers":true},"pluginVersion":"8.3.4","targets":[{"datasource":{"type":"prometheus","uid":"${datasource}"},"exemplar":false,"expr":"sum(container_memory_working_set_bytes{namespace=\"$namespace\", pod=\"$pod\", image!=\"\"}) / sum(kube_pod_container_resource_limits{namespace=\"$namespace\", pod=\"$pod\", unit=\"byte\"}) ","instant":true,"interval":"$resolution","legendFormat":"Limits","refId":"B"}],"title":"Total pod RAM Limits usage","type":"gauge"},{"datasource":{"type":"prometheus","uid":"${datasource}"},"fieldConfig":{"defaults":{"custom":{"align":"auto","displayMode":"auto","filterable":false},"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"blue","value":null}]},"unit":"none"},"overrides":[{"matcher":{"id":"byName","options":"Memory Requests"},"properties":[{"id":"unit","value":"bytes"}]},{"matcher":{"id":"byName","options":"Memory Limits"},"properties":[{"id":"unit","value":"bytes"}]},{"matcher":{"id":"byName","options":"Memory Used"},"properties":[{"id":"unit","value":"bytes"}]}]},"gridPos":{"h":8,"w":12,"x":12,"y":4},"id":38,"options":{"footer":{"fields":"","reducer":["sum"],"show":false},"showHeader":true,"sortBy":[]},"pluginVersion":"8.3.4","targets":[{"datasource":{"type":"prometheus","uid":"${datasource}"},"exemplar":false,"expr":"sum(kube_pod_container_resource_requests{namespace=\"$namespace\", pod=\"$pod\", unit=\"core\"}) by (container)","format":"table","instant":true,"interval":"","intervalFactor":1,"legendFormat":"","refId":"A"},{"datasource":{"type":"prometheus","uid":"${datasource}"},"exemplar":false,"expr":"sum(kube_pod_container_resource_limits{namespace=\"$namespace\", pod=\"$pod\", unit=\"core\"}) by (container)","format":"table","instant":true,"interval":"","intervalFactor":1,"legendFormat":"","refId":"B"},{"datasource":{"type":"prometheus","uid":"${datasource}"},"exemplar":false,"expr":"sum(kube_pod_container_resource_requests{namespace=\"$namespace\", pod=\"$pod\", unit=\"byte\"}) by (container)","format":"table","instant":true,"interval":"","legendFormat":"","refId":"C"},{"datasource":{"type":"prometheus","uid":"${datasource}"},"exemplar":false,"expr":"sum(kube_pod_container_resource_limits{namespace=\"$namespace\", pod=\"$pod\", unit=\"byte\"}) by (container)","format":"table","instant":true,"interval":"","legendFormat":"","refId":"D"},{"datasource":{"type":"prometheus","uid":"${datasource}"},"editorMode":"code","exemplar":false,"expr":"sum(rate(container_cpu_usage_seconds_total{namespace=\"$namespace\", pod=\"$pod\", image!=\"\", container!=\"POD\"}[$__rate_interval])) by (container)","format":"table","hide":false,"instant":true,"legendFormat":"__auto","range":false,"refId":"E"},{"datasource":{"type":"prometheus","uid":"${datasource}"},"editorMode":"code","exemplar":false,"expr":"sum(container_memory_working_set_bytes{namespace=\"$namespace\", pod=\"$pod\", image!=\"\", container!=\"POD\"}) by (container)","format":"table","hide":false,"instant":true,"range":false,"refId":"F"}],"title":"Resources by container","transformations":[{"id":"seriesToColumns","options":{"byField":"container"}},{"id":"organize","options":{"excludeByName":{"Time":true,"Time 1":true,"Time 2":true,"Time 4":true,"__name__":true,"__name__ 1":true,"__name__ 2":true,"__name__ 3":true,"__name__ 4":true,"container":false,"endpoint":true,"endpoint 2":true,"endpoint 3":true,"endpoint 4":true,"instance":true,"instance 2":true,"instance 3":true,"instance 4":true,"job":true,"job 2":true,"job 3":true,"job 4":true,"namespace":true,"namespace 2":true,"namespace 3":true,"namespace 4":true,"node":true,"node 2":true,"node 3":true,"node 4":true,"pod":true,"pod 2":true,"pod 3":true,"pod 4":true,"resource 1":true,"resource 2":true,"resource 3":true,"resource 4":true,"service":true,"service 2":true,"service 3":true,"service 4":true,"uid 1":true,"uid 2":true,"uid 3":true,"uid 4":true,"unit 1":true,"unit 2":true,"unit 3":true,"unit 4":true},"indexByName":{"Time 1":7,"Time 2":8,"Time 3":9,"Time 4":10,"Time 5":11,"Time 6":12,"Value #A":2,"Value #B":3,"Value #C":5,"Value #D":6,"Value #E":1,"Value #F":4,"container":0},"renameByName":{"Value #A":"CPU Requests","Value #B":"CPU Limits","Value #C":"Memory Requests","Value #D":"Memory Limits","Value #E":"CPU Used","Value #F":"Memory Used","container":"Container"}}}],"type":"table"},{"datasource":{"type":"prometheus","uid":"${datasource}"},"fieldConfig":{"defaults":{"color":{"mode":"thresholds"},"custom":{"axisLabel":"Percent","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":25,"gradientMode":"opacity","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"smooth","lineWidth":2,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":true,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"area"}},"mappings":[],"max":1,"min":0,"thresholds":{"mode":"percentage","steps":[{"color":"red","value":null},{"color":"yellow","value":20},{"color":"green","value":30},{"color":"yellow","value":70},{"color":"red","value":80}]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":12},"id":50,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"right","showLegend":true},"tooltip":{"mode":"multi","sort":"desc"}},"pluginVersion":"8.3.3","targets":[{"datasource":{"type":"prometheus","uid":"${datasource}"},"editorMode":"code","exemplar":true,"expr":"sum(rate(container_cpu_usage_seconds_total{namespace=\"$namespace\", pod=\"$pod\", image!=\"\"}[$__rate_interval])) by (container) / sum(kube_pod_container_resource_requests{namespace=\"$namespace\", pod=\"$pod\", unit=\"core\"}) by (container)","interval":"$resolution","legendFormat":"{{ container }} REQUESTS","range":true,"refId":"A"},{"datasource":{"type":"prometheus","uid":"${datasource}"},"editorMode":"code","expr":"sum(rate(container_cpu_usage_seconds_total{namespace=\"$namespace\", pod=\"$pod\", image!=\"\"}[$__rate_interval])) by (container) / sum(kube_pod_container_resource_limits{namespace=\"$namespace\", pod=\"$pod\", unit=\"core\"}) by (container)","hide":false,"legendFormat":"{{ container }} LIMITS","range":true,"refId":"B"}],"title":"CPU Usage / Requests & Limits by container","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"${datasource}"},"fieldConfig":{"defaults":{"color":{"fixedColor":"blue","mode":"thresholds"},"custom":{"axisLabel":"Percent","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":25,"gradientMode":"opacity","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"smooth","lineStyle":{"fill":"solid"},"lineWidth":2,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":true,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"area"}},"mappings":[],"max":1,"min":0,"thresholds":{"mode":"percentage","steps":[{"color":"red","value":null},{"color":"yellow","value":20},{"color":"green","value":30},{"color":"#EAB839","value":70},{"color":"red","value":80}]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":12},"id":30,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"right","showLegend":true},"tooltip":{"mode":"multi","sort":"desc"}},"pluginVersion":"8.3.3","targets":[{"datasource":{"type":"prometheus","uid":"${datasource}"},"editorMode":"code","exemplar":true,"expr":"sum(container_memory_working_set_bytes{namespace=\"$namespace\", pod=\"$pod\", image!=\"\"}) by (container) / sum(kube_pod_container_resource_requests{namespace=\"$namespace\", pod=\"$pod\", unit=\"byte\"}) by (container)","interval":"","legendFormat":"{{ container }} REQUESTS","range":true,"refId":"A"},{"datasource":{"type":"prometheus","uid":"${datasource}"},"editorMode":"code","expr":"sum(container_memory_working_set_bytes{namespace=\"$namespace\", pod=\"$pod\", image!=\"\"}) by (container) / sum(kube_pod_container_resource_limits{namespace=\"$namespace\", pod=\"$pod\", unit=\"byte\"}) by (container)","hide":false,"legendFormat":"{{ container }} LIMITS","range":true,"refId":"B"}],"title":"Memory Usage / Requests & Limits by container","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"${datasource}"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"CPU Cores","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":25,"gradientMode":"opacity","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"smooth","lineWidth":2,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":true,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green","value":null},{"color":"red","value":80}]},"unit":"none"},"overrides":[{"matcher":{"id":"byName","options":"limit"},"properties":[{"id":"color","value":{"fixedColor":"#F2495C","mode":"fixed"}},{"id":"custom.fillOpacity","value":0}]}]},"gridPos":{"h":8,"w":12,"x":0,"y":20},"id":29,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"right","showLegend":true},"tooltip":{"mode":"multi","sort":"desc"}},"pluginVersion":"8.3.3","targets":[{"datasource":{"type":"prometheus","uid":"${datasource}"},"exemplar":true,"expr":"sum(rate(container_cpu_usage_seconds_total{namespace=\"$namespace\", pod=\"$pod\", image!=\"\", container!=\"POD\"}[$__rate_interval])) by (container)","interval":"$resolution","legendFormat":"{{ container }}","refId":"A"}],"title":"CPU Usage by container","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"${datasource}"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"Bytes","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":25,"gradientMode":"opacity","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"smooth","lineStyle":{"fill":"solid"},"lineWidth":2,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":true,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green","value":null},{"color":"red","value":80}]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":20},"id":51,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"right","showLegend":true},"tooltip":{"mode":"multi","sort":"desc"}},"pluginVersion":"8.3.3","targets":[{"datasource":{"type":"prometheus","uid":"${datasource}"},"exemplar":true,"expr":"sum(container_memory_working_set_bytes{namespace=\"$namespace\", pod=\"$pod\", image!=\"\", container!=\"POD\"}) by (container)","interval":"","legendFormat":"{{ container }}","refId":"A"}],"title":"Memory Usage by container","type":"timeseries"},{"collapsed":false,"datasource":{"type":"datasource","uid":"grafana"},"gridPos":{"h":1,"w":24,"x":0,"y":28},"id":45,"panels":[],"targets":[{"datasource":{"type":"datasource","uid":"grafana"},"refId":"A"}],"title":"Network","type":"row"},{"datasource":{"type":"prometheus","uid":"${datasource}"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":25,"gradientMode":"opacity","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"smooth","lineWidth":2,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":true,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green","value":null},{"color":"red","value":80}]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":29},"id":31,"options":{"legend":{"calcs":[],"displayMode":"list","placement":"bottom","showLegend":true},"tooltip":{"mode":"multi","sort":"none"}},"pluginVersion":"8.3.3","targets":[{"datasource":{"type":"prometheus","uid":"${datasource}"},"exemplar":true,"expr":"sum(rate(container_network_receive_bytes_total{namespace=\"$namespace\", pod=\"$pod\"}[$__rate_interval]))","interval":"$resolution","legendFormat":"Received","refId":"A"},{"datasource":{"type":"prometheus","uid":"${datasource}"},"exemplar":true,"expr":"- sum(rate(container_network_transmit_bytes_total{namespace=\"$namespace\", pod=\"$pod\"}[$__rate_interval]))","interval":"$resolution","legendFormat":"Transmitted","refId":"B"}],"title":"Network - Bandwidth","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"${datasource}"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":25,"gradientMode":"opacity","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"smooth","lineWidth":2,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":true,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green","value":null},{"color":"red","value":80}]},"unit":"pps"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":29},"id":34,"options":{"legend":{"calcs":[],"displayMode":"list","placement":"bottom","showLegend":true},"tooltip":{"mode":"multi","sort":"none"}},"pluginVersion":"8.3.3","targets":[{"datasource":{"type":"prometheus","uid":"${datasource}"},"exemplar":true,"expr":"sum(rate(container_network_receive_packets_total{namespace=\"$namespace\", pod=\"$pod\"}[$__rate_interval]))","interval":"$resolution","legendFormat":"Received","refId":"A"},{"datasource":{"type":"prometheus","uid":"${datasource}"},"exemplar":true,"expr":"- sum(rate(container_network_transmit_packets_total{namespace=\"$namespace\", pod=\"$pod\"}[$__rate_interval]))","interval":"$resolution","legendFormat":"Transmitted","refId":"B"}],"title":"Network - Packets Rate","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"${datasource}"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":25,"gradientMode":"opacity","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"smooth","lineWidth":2,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":true,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green","value":null},{"color":"red","value":80}]},"unit":"pps"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":37},"id":36,"options":{"legend":{"calcs":[],"displayMode":"list","placement":"bottom","showLegend":true},"tooltip":{"mode":"multi","sort":"none"}},"pluginVersion":"8.3.3","targets":[{"datasource":{"type":"prometheus","uid":"${datasource}"},"exemplar":true,"expr":"sum(rate(container_network_receive_packets_dropped_total{namespace=\"$namespace\", pod=\"$pod\"}[$__rate_interval]))","interval":"$resolution","legendFormat":"Received","refId":"A"},{"datasource":{"type":"prometheus","uid":"${datasource}"},"exemplar":true,"expr":"- sum(rate(container_network_transmit_packets_dropped_total{namespace=\"$namespace\", pod=\"$pod\"}[$__rate_interval]))","interval":"$resolution","legendFormat":"Transmitted","refId":"B"}],"title":"Network - Packets Dropped","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"${datasource}"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":25,"gradientMode":"opacity","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"smooth","lineWidth":2,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"never","spanNulls":true,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green","value":null},{"color":"red","value":80}]},"unit":"pps"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":37},"id":37,"options":{"legend":{"calcs":[],"displayMode":"list","placement":"bottom","showLegend":true},"tooltip":{"mode":"multi","sort":"none"}},"pluginVersion":"8.3.3","targets":[{"datasource":{"type":"prometheus","uid":"${datasource}"},"exemplar":true,"expr":"sum(rate(container_network_receive_errors_total{namespace=\"$namespace\", pod=\"$pod\"}[$__rate_interval]))","interval":"$resolution","legendFormat":"Received","refId":"A"},{"datasource":{"type":"prometheus","uid":"${datasource}"},"exemplar":true,"expr":"- sum(rate(container_network_transmit_errors_total{namespace=\"$namespace\", pod=\"$pod\"}[$__rate_interval]))","interval":"$resolution","legendFormat":"Transmitted","refId":"B"}],"title":"Network - Errors","type":"timeseries"}],"refresh":"30s","schemaVersion":34,"style":"dark","tags":["Kubernetes","Prometheus"],"templating":{"list":[{"current":{"selected":false,"text":"Prometheus","value":"Prometheus"},"hide":0,"includeAll":false,"multi":false,"name":"datasource","options":[],"query":"prometheus","queryValue":"","refresh":1,"regex":"","skipUrlSync":false,"type":"datasource"},{"current":{"selected":false,"text":"monitoring","value":"monitoring"},"datasource":{"type":"prometheus","uid":"${datasource}"},"definition":"label_values(kube_pod_info, namespace)","hide":0,"includeAll":false,"multi":false,"name":"namespace","options":[],"query":{"query":"label_values(kube_pod_info, namespace)","refId":"Prometheus-namespace-Variable-Query"},"refresh":1,"regex":"","skipUrlSync":false,"sort":1,"tagValuesQuery":"","tagsQuery":"","type":"query","useTags":false},{"current":{"selected":false,"text":"custom-metrics-stackdriver-adapter-5d95c47588-h2xlx","value":"custom-metrics-stackdriver-adapter-5d95c47588-h2xlx"},"datasource":{"type":"prometheus","uid":"${datasource}"},"definition":"label_values(kube_pod_info{namespace=\"$namespace\"}, pod)","hide":0,"includeAll":false,"multi":false,"name":"pod","options":[],"query":{"query":"label_values(kube_pod_info{namespace=\"$namespace\"}, pod)","refId":"Prometheus-pod-Variable-Query"},"refresh":2,"regex":"","skipUrlSync":false,"sort":1,"tagValuesQuery":"","tagsQuery":"","type":"query","useTags":false},{"current":{"selected":true,"text":"30s","value":"30s"},"hide":0,"includeAll":false,"multi":false,"name":"resolution","options":[{"selected":false,"text":"1s","value":"1s"},{"selected":false,"text":"15s","value":"15s"},{"selected":true,"text":"30s","value":"30s"},{"selected":false,"text":"1m","value":"1m"},{"selected":false,"text":"3m","value":"3m"},{"selected":false,"text":"5m","value":"5m"}],"query":"1s, 15s, 30s, 1m, 3m, 5m","queryValue":"","skipUrlSync":false,"type":"custom"}]},"time":{"from":"now-1h","to":"now"},"timepicker":{},"timezone":"","title":"k8s / view / pods","uid":"k8s_view_pods","version":1,"weekStart":""} \ No newline at end of file diff --git a/blueprints/gke/autopilot/bundle/monitoring/dashboards/locust.json b/blueprints/gke/autopilot/bundle/monitoring/dashboards/locust.json new file mode 100644 index 0000000000..ba95c09953 --- /dev/null +++ b/blueprints/gke/autopilot/bundle/monitoring/dashboards/locust.json @@ -0,0 +1 @@ +{"annotations":{"list":[{"builtIn":1,"datasource":"-- Grafana --","enable":true,"hide":true,"iconColor":"rgba(0, 211, 255, 1)","limit":100,"name":"Annotations & Alerts","showIn":0,"target":{"limit":100,"matchAny":false,"tags":[],"type":"dashboard"},"type":"dashboard"}]},"description":"Locust Exporter - Container Solutions","editable":true,"fiscalYearStartMonth":0,"graphTooltip":0,"id":8,"links":[],"liveNow":false,"panels":[{"datasource":{"type":"prometheus","uid":"PBFA97CFB590B2093"},"fieldConfig":{"defaults":{"color":{"mode":"thresholds"},"mappings":[{"options":{"0":{"text":"Stop"},"1":{"text":"Hatching"},"2":{"text":"Running"}},"type":"value"}],"thresholds":{"mode":"absolute","steps":[{"color":"rgba(245, 54, 54, 0.9)","value":null},{"color":"rgba(237, 129, 40, 0.89)","value":0},{"color":"rgba(50, 172, 45, 0.97)","value":2}]},"unit":"none"},"overrides":[]},"gridPos":{"h":7,"w":4,"x":0,"y":0},"id":1,"links":[],"maxDataPoints":100,"options":{"colorMode":"value","graphMode":"area","justifyMode":"auto","orientation":"horizontal","reduceOptions":{"calcs":["mean"],"fields":"","values":false},"textMode":"auto"},"pluginVersion":"8.3.4","targets":[{"expr":"locust_running","interval":"","intervalFactor":2,"legendFormat":"","metric":"","refId":"A","step":20}],"title":"Locust Status","type":"stat"},{"datasource":{"type":"prometheus","uid":"PBFA97CFB590B2093"},"fieldConfig":{"defaults":{"color":{"fixedColor":"rgb(31, 120, 193)","mode":"fixed"},"mappings":[{"options":{"match":"null","result":{"text":"N/A"}},"type":"special"}],"thresholds":{"mode":"absolute","steps":[{"color":"green","value":null},{"color":"red","value":80}]},"unit":"none"},"overrides":[]},"gridPos":{"h":4,"w":4,"x":4,"y":0},"id":2,"links":[],"maxDataPoints":100,"options":{"colorMode":"none","graphMode":"area","justifyMode":"auto","orientation":"horizontal","reduceOptions":{"calcs":["mean"],"fields":"","values":false},"textMode":"auto"},"pluginVersion":"8.3.4","targets":[{"expr":"locust_users","interval":"","intervalFactor":2,"legendFormat":"","refId":"A","step":20}],"title":"Swarmed users","type":"stat"},{"fieldConfig":{"defaults":{"color":{"fixedColor":"rgb(31, 120, 193)","mode":"fixed"},"decimals":2,"mappings":[{"options":{"match":"null","result":{"text":"N/A"}},"type":"special"}],"thresholds":{"mode":"absolute","steps":[{"color":"green","value":null},{"color":"red","value":80}]},"unit":"short"},"overrides":[]},"gridPos":{"h":7,"w":6,"x":8,"y":0},"id":4,"links":[],"maxDataPoints":100,"options":{"colorMode":"none","graphMode":"area","justifyMode":"auto","orientation":"horizontal","reduceOptions":{"calcs":["mean"],"fields":"","values":false},"textMode":"auto"},"pluginVersion":"8.3.4","targets":[{"datasource":{"type":"prometheus","uid":"PBFA97CFB590B2093"},"exemplar":true,"expr":"locust_requests_current_rps{method=~\"\"}","interval":"","intervalFactor":2,"legendFormat":"","refId":"A","step":20}],"title":"Current RPS","type":"stat"},{"fieldConfig":{"defaults":{"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green","value":null}]}},"overrides":[]},"gridPos":{"h":4,"w":4,"x":14,"y":0},"id":18,"options":{"colorMode":"value","graphMode":"area","justifyMode":"auto","orientation":"auto","reduceOptions":{"calcs":["mean"],"fields":"","values":false},"textMode":"auto"},"pluginVersion":"8.3.4","targets":[{"datasource":{"type":"prometheus","uid":"PBFA97CFB590B2093"},"exemplar":true,"expr":"locust_requests_num_requests{method=~\"\"}","interval":"","legendFormat":"","refId":"A"}],"title":"Requests","type":"stat"},{"fieldConfig":{"defaults":{"mappings":[],"max":1,"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"semi-dark-red","value":null}]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":7,"w":6,"x":18,"y":0},"id":21,"options":{"orientation":"auto","reduceOptions":{"calcs":["last"],"fields":"","values":false},"showThresholdLabels":false,"showThresholdMarkers":true},"pluginVersion":"8.3.4","targets":[{"datasource":{"type":"prometheus","uid":"PBFA97CFB590B2093"},"exemplar":true,"expr":"locust_requests_fail_ratio","interval":"","legendFormat":"","refId":"A"}],"title":"Fails","type":"gauge"},{"datasource":{"type":"prometheus","uid":"PBFA97CFB590B2093"},"fieldConfig":{"defaults":{"color":{"fixedColor":"rgb(31, 120, 193)","mode":"fixed"},"mappings":[{"options":{"match":"null","result":{"text":"N/A"}},"type":"special"}],"thresholds":{"mode":"absolute","steps":[{"color":"green","value":null},{"color":"red","value":80}]},"unit":"none"},"overrides":[]},"gridPos":{"h":3,"w":4,"x":4,"y":4},"id":3,"links":[],"maxDataPoints":100,"options":{"colorMode":"none","graphMode":"area","justifyMode":"auto","orientation":"horizontal","reduceOptions":{"calcs":["mean"],"fields":"","values":false},"textMode":"auto"},"pluginVersion":"8.3.4","targets":[{"expr":"locust_workers_count","interval":"","intervalFactor":2,"legendFormat":"","refId":"A","step":20}],"title":"Connected workers","type":"stat"},{"fieldConfig":{"defaults":{"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"red","value":null}]}},"overrides":[]},"gridPos":{"h":3,"w":4,"x":14,"y":4},"id":19,"options":{"colorMode":"value","graphMode":"area","justifyMode":"auto","orientation":"auto","reduceOptions":{"calcs":["mean"],"fields":"","values":false},"textMode":"auto"},"pluginVersion":"8.3.4","targets":[{"datasource":{"type":"prometheus","uid":"PBFA97CFB590B2093"},"exemplar":true,"expr":"locust_requests_num_failures{method=~\"\"}","interval":"","legendFormat":"","refId":"A"}],"title":"Failers","type":"stat"},{"columns":[],"fontSize":"100%","gridPos":{"h":6,"w":24,"x":0,"y":7},"id":14,"showHeader":true,"sort":{"col":0,"desc":true},"styles":[{"alias":"Time","align":"auto","dateFormat":"YYYY-MM-DD HH:mm:ss","pattern":"Time","type":"hidden"},{"alias":"Method","align":"auto","colors":["rgba(245, 54, 54, 0.9)","rgba(237, 129, 40, 0.89)","rgba(50, 172, 45, 0.97)"],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"mappingType":1,"pattern":"method","preserveFormat":false,"thresholds":[],"type":"string","unit":"short"},{"alias":"URL","align":"auto","colors":["rgba(245, 54, 54, 0.9)","rgba(237, 129, 40, 0.89)","rgba(50, 172, 45, 0.97)"],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"mappingType":1,"pattern":"name","thresholds":[],"type":"string","unit":"short"},{"alias":"MIN RT","align":"","colors":["rgba(245, 54, 54, 0.9)","rgba(237, 129, 40, 0.89)","rgba(50, 172, 45, 0.97)"],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":0,"mappingType":1,"pattern":"Value #A","thresholds":[],"type":"number","unit":"ms"},{"alias":"Errors","align":"","colors":["rgba(245, 54, 54, 0.9)","rgba(237, 129, 40, 0.89)","rgba(50, 172, 45, 0.97)"],"decimals":0,"pattern":"Value #B","thresholds":[],"type":"number","unit":"none"},{"alias":"MAX RT","align":"","colors":["rgba(245, 54, 54, 0.9)","rgba(237, 129, 40, 0.89)","rgba(50, 172, 45, 0.97)"],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":0,"mappingType":1,"pattern":"Value #C","thresholds":[],"type":"number","unit":"ms"},{"alias":"MEDIAN RT","align":"auto","colors":["rgba(245, 54, 54, 0.9)","rgba(237, 129, 40, 0.89)","rgba(50, 172, 45, 0.97)"],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":0,"mappingType":1,"pattern":"Value #D","thresholds":[],"type":"number","unit":"ms"},{"alias":"AVG RT","align":"auto","colors":["rgba(245, 54, 54, 0.9)","rgba(237, 129, 40, 0.89)","rgba(50, 172, 45, 0.97)"],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"mappingType":1,"pattern":"Value #E","thresholds":[],"type":"number","unit":"ms"},{"alias":"Errors Ratio","align":"auto","colors":["rgba(245, 54, 54, 0.9)","rgba(237, 129, 40, 0.89)","rgba(50, 172, 45, 0.97)"],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"mappingType":1,"pattern":"Value #H","thresholds":[],"type":"number","unit":"reqps"},{"alias":"Requests","align":"auto","colors":["rgba(245, 54, 54, 0.9)","rgba(237, 129, 40, 0.89)","rgba(50, 172, 45, 0.97)"],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":0,"mappingType":1,"pattern":"Value #F","thresholds":[],"type":"number","unit":"none"},{"alias":"Requests Ratio","align":"auto","colors":["rgba(245, 54, 54, 0.9)","rgba(237, 129, 40, 0.89)","rgba(50, 172, 45, 0.97)"],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"mappingType":1,"pattern":"Value #G","thresholds":[],"type":"number","unit":"reqps"},{"alias":"Content","align":"auto","colors":["rgba(245, 54, 54, 0.9)","rgba(237, 129, 40, 0.89)","rgba(50, 172, 45, 0.97)"],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"mappingType":1,"pattern":"Value #I","thresholds":[],"type":"number","unit":"bytes"}],"targets":[{"datasource":{"type":"prometheus","uid":"PBFA97CFB590B2093"},"exemplar":false,"expr":"sum(locust_requests_avg_content_length{method=~\".+\"}) by (method, name)","format":"table","instant":true,"interval":"","legendFormat":"","refId":"I"},{"datasource":{"type":"prometheus","uid":"PBFA97CFB590B2093"},"exemplar":false,"expr":"sum(locust_requests_min_response_time{method=~\".+\"}) by (method, name)","format":"table","instant":true,"interval":"","legendFormat":"","refId":"A"},{"datasource":{"type":"prometheus","uid":"PBFA97CFB590B2093"},"exemplar":false,"expr":"sum(locust_requests_max_response_time{method=~\".+\"}) by (method, name)","format":"table","instant":true,"interval":"","legendFormat":"","refId":"C"},{"datasource":{"type":"prometheus","uid":"PBFA97CFB590B2093"},"exemplar":false,"expr":"sum(locust_requests_avg_response_time{method=~\".+\"}) by (method, name)","format":"table","instant":true,"interval":"","legendFormat":"","refId":"E"},{"datasource":{"type":"prometheus","uid":"PBFA97CFB590B2093"},"exemplar":false,"expr":"sum(locust_requests_median_response_time{method=~\".+\"}) by (method, name)","format":"table","instant":true,"interval":"","legendFormat":"","refId":"D"},{"datasource":{"type":"prometheus","uid":"PBFA97CFB590B2093"},"exemplar":false,"expr":"sum(locust_requests_num_failures{method=~\".+\"}) by (method, name)","format":"table","instant":true,"interval":"","legendFormat":"","refId":"B"},{"datasource":{"type":"prometheus","uid":"PBFA97CFB590B2093"},"exemplar":false,"expr":"sum(locust_requests_current_fail_per_sec{method=~\".+\"}) by (method, name)","format":"table","instant":true,"interval":"","legendFormat":"","refId":"H"},{"datasource":{"type":"prometheus","uid":"PBFA97CFB590B2093"},"exemplar":false,"expr":"sum(locust_requests_num_requests{method=~\".+\"}) by (method, name)","format":"table","instant":true,"interval":"","legendFormat":"","refId":"F"},{"datasource":{"type":"prometheus","uid":"PBFA97CFB590B2093"},"exemplar":false,"expr":"sum(locust_requests_current_rps{method=~\".+\"}) by (method, name)","format":"table","instant":true,"interval":"","legendFormat":"","refId":"G"}],"title":"Endpoints","transform":"table","type":"table-old"},{"columns":[],"fontSize":"100%","gridPos":{"h":3,"w":24,"x":0,"y":13},"id":16,"showHeader":true,"sort":{"col":0,"desc":true},"styles":[{"alias":"Time","align":"auto","dateFormat":"YYYY-MM-DD HH:mm:ss","pattern":"Time","type":"hidden"},{"alias":"Requests","align":"auto","colors":["rgba(245, 54, 54, 0.9)","rgba(237, 129, 40, 0.89)","rgba(50, 172, 45, 0.97)"],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":0,"mappingType":1,"pattern":"Value","thresholds":[],"type":"number","unit":"none"},{"alias":"Method","align":"auto","colors":["rgba(245, 54, 54, 0.9)","rgba(237, 129, 40, 0.89)","rgba(50, 172, 45, 0.97)"],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"mappingType":1,"pattern":"method","thresholds":[],"type":"string","unit":"short"},{"alias":"URL","align":"auto","colors":["rgba(245, 54, 54, 0.9)","rgba(237, 129, 40, 0.89)","rgba(50, 172, 45, 0.97)"],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"mappingType":1,"pattern":"name","thresholds":[],"type":"string","unit":"short"},{"alias":"Error","align":"auto","colors":["rgba(245, 54, 54, 0.9)","rgba(237, 129, 40, 0.89)","rgba(50, 172, 45, 0.97)"],"dateFormat":"YYYY-MM-DD HH:mm:ss","decimals":2,"link":false,"mappingType":1,"pattern":"error","thresholds":[],"type":"string","unit":"short"},{"alias":"","align":"left","colors":["rgba(245, 54, 54, 0.9)","rgba(237, 129, 40, 0.89)","rgba(50, 172, 45, 0.97)"],"decimals":2,"pattern":"/.*/","thresholds":[],"type":"number","unit":"short"}],"targets":[{"expr":"sum(locust_errors) by (method, name, error)","format":"table","hide":false,"instant":true,"interval":"","intervalFactor":1,"legendFormat":"","refId":"A"}],"title":"Errors","transform":"table","type":"table-old"},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"decimals":0,"editable":true,"error":false,"fieldConfig":{"defaults":{"links":[]},"overrides":[]},"fill":1,"fillGradient":0,"gridPos":{"h":8,"w":17,"x":0,"y":16},"hiddenSeries":false,"id":6,"legend":{"avg":false,"current":false,"max":false,"min":false,"show":true,"total":false,"values":false},"lines":true,"linewidth":1,"links":[],"nullPointMode":"connected","options":{"alertThreshold":true},"paceLength":10,"percentage":false,"pluginVersion":"8.3.4","pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[{"alias":"AVG MAX","yaxis":2}],"spaceLength":10,"stack":false,"steppedLine":false,"targets":[{"datasource":{"type":"prometheus","uid":"PBFA97CFB590B2093"},"exemplar":true,"expr":"avg(locust_requests_max_response_time{method=~\".+\"})","interval":"","intervalFactor":2,"legendFormat":"AVG MAX","refId":"A","step":2},{"datasource":{"type":"prometheus","uid":"PBFA97CFB590B2093"},"exemplar":true,"expr":"avg(locust_requests_min_response_time{method=~\".+\"})","interval":"","intervalFactor":2,"legendFormat":"AVG MIN","metric":"","refId":"B","step":2},{"datasource":{"type":"prometheus","uid":"PBFA97CFB590B2093"},"exemplar":true,"expr":"avg(locust_requests_avg_response_time{method=~\".+\"})","interval":"","intervalFactor":2,"legendFormat":"AVG AVG","metric":"","refId":"C","step":2},{"datasource":{"type":"prometheus","uid":"PBFA97CFB590B2093"},"exemplar":true,"expr":"avg(locust_requests_median_response_time{method=~\".+\"})","interval":"","intervalFactor":2,"legendFormat":"AVG MEDIAN","refId":"D","step":2}],"thresholds":[],"timeRegions":[],"title":"Response Times","tooltip":{"msResolution":false,"shared":true,"sort":0,"value_type":"individual"},"type":"graph","xaxis":{"mode":"time","show":true,"values":[]},"yaxes":[{"decimals":0,"format":"ms","logBase":1,"show":true},{"decimals":0,"format":"ms","logBase":1,"show":true}],"yaxis":{"align":false}},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"decimals":0,"editable":true,"error":false,"fieldConfig":{"defaults":{"links":[]},"overrides":[]},"fill":1,"fillGradient":0,"gridPos":{"h":8,"w":7,"x":17,"y":16},"hiddenSeries":false,"id":15,"interval":"","legend":{"avg":false,"current":false,"max":false,"min":false,"show":true,"total":false,"values":false},"lines":true,"linewidth":1,"links":[],"nullPointMode":"connected","options":{"alertThreshold":true},"paceLength":10,"percentage":false,"pluginVersion":"8.3.4","pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[{"alias":"AVG MAX","yaxis":2}],"spaceLength":10,"stack":false,"steppedLine":false,"targets":[{"datasource":{"type":"prometheus","uid":"PBFA97CFB590B2093"},"exemplar":true,"expr":"locust_requests_current_response_time_percentile_95","interval":"","intervalFactor":2,"legendFormat":"P95","refId":"D","step":2},{"datasource":{"type":"prometheus","uid":"PBFA97CFB590B2093"},"expr":"locust_requests_current_response_time_percentile_50","interval":"","legendFormat":"P50","refId":"A"}],"thresholds":[],"timeRegions":[],"title":"Response Times","tooltip":{"msResolution":false,"shared":true,"sort":0,"value_type":"individual"},"type":"graph","xaxis":{"mode":"time","show":true,"values":[]},"yaxes":[{"decimals":0,"format":"ms","logBase":1,"show":true},{"decimals":0,"format":"ms","logBase":1,"show":true}],"yaxis":{"align":false}},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":{"type":"prometheus","uid":"PBFA97CFB590B2093"},"decimals":0,"fieldConfig":{"defaults":{"links":[]},"overrides":[]},"fill":1,"fillGradient":0,"gridPos":{"h":8,"w":12,"x":0,"y":24},"hiddenSeries":false,"id":9,"legend":{"avg":false,"current":false,"max":false,"min":false,"show":false,"total":false,"values":false},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null","options":{"alertThreshold":true},"paceLength":10,"percentage":false,"pluginVersion":"8.3.4","pointradius":2,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"stack":false,"steppedLine":false,"targets":[{"expr":"locust_users","format":"time_series","interval":"","intervalFactor":1,"legendFormat":"Users","refId":"A"}],"thresholds":[],"timeRegions":[],"title":"Users","tooltip":{"shared":true,"sort":0,"value_type":"individual"},"type":"graph","xaxis":{"mode":"time","show":true,"values":[]},"yaxes":[{"format":"short","logBase":1,"show":true},{"format":"short","logBase":1,"show":true}],"yaxis":{"align":false}},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":{"type":"prometheus","uid":"PBFA97CFB590B2093"},"decimals":0,"fieldConfig":{"defaults":{"links":[]},"overrides":[]},"fill":1,"fillGradient":0,"gridPos":{"h":8,"w":12,"x":12,"y":24},"hiddenSeries":false,"id":12,"interval":"","legend":{"avg":false,"current":false,"max":false,"min":false,"show":false,"total":false,"values":false},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null","options":{"alertThreshold":true},"paceLength":10,"percentage":false,"pluginVersion":"8.3.4","pointradius":2,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"stack":false,"steppedLine":false,"targets":[{"expr":"locust_workers_count","format":"time_series","instant":false,"interval":"","intervalFactor":1,"legendFormat":"Workers","refId":"A"}],"thresholds":[],"timeRegions":[],"title":"Workers","tooltip":{"shared":true,"sort":0,"value_type":"individual"},"type":"graph","xaxis":{"mode":"time","show":true,"values":[]},"yaxes":[{"decimals":0,"format":"short","logBase":1,"show":true},{"decimals":0,"format":"short","logBase":1,"show":false}],"yaxis":{"align":false}},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":{"type":"prometheus","uid":"PBFA97CFB590B2093"},"editable":true,"error":false,"fieldConfig":{"defaults":{"links":[]},"overrides":[]},"fill":1,"fillGradient":0,"gridPos":{"h":7,"w":24,"x":0,"y":32},"hiddenSeries":false,"id":7,"legend":{"avg":false,"current":false,"max":false,"min":false,"show":true,"total":false,"values":false},"lines":true,"linewidth":1,"links":[],"nullPointMode":"connected","paceLength":10,"percentage":false,"pluginVersion":"8.3.4","pointradius":5,"points":false,"renderer":"flot","seriesOverrides":[{"alias":"fail_ratio [%]","yaxis":2}],"spaceLength":10,"stack":false,"steppedLine":false,"targets":[{"expr":"rate(locust_requests_num_requests{method=~\".+\"}[1m])","format":"time_series","interval":"","intervalFactor":2,"legendFormat":"{{ method}} - {{name}}","refId":"A","step":2},{"expr":" sum(rate(locust_requests_num_requests{method=~\".+\"}[1m]))","format":"time_series","interval":"","intervalFactor":1,"legendFormat":"total","refId":"C"},{"expr":"locust_requests_fail_ratio * 100","format":"time_series","interval":"","intervalFactor":2,"legendFormat":"fail_ratio [%]","refId":"B","step":2}],"thresholds":[],"timeRegions":[],"title":"Requests per endpoint / s","tooltip":{"msResolution":false,"shared":true,"sort":0,"value_type":"individual"},"type":"graph","xaxis":{"mode":"time","show":true,"values":[]},"yaxes":[{"format":"reqps","logBase":1,"show":true},{"format":"percent","logBase":1,"min":"0","show":true}],"yaxis":{"align":false}}],"refresh":"10s","schemaVersion":34,"style":"dark","tags":["locust"],"templating":{"list":[]},"time":{"from":"now-15m","to":"now"},"timepicker":{"refresh_intervals":["5s","10s","30s","1m","5m","15m","30m","1h","2h","1d"],"time_options":["5m","15m","1h","6h","12h","24h","2d","7d","30d"]},"timezone":"browser","title":"locust / view","uid":"0WllLp6mz","version":4,"weekStart":""} \ No newline at end of file diff --git a/blueprints/gke/autopilot/bundle/monitoring/dashboards/nginx.json b/blueprints/gke/autopilot/bundle/monitoring/dashboards/nginx.json new file mode 100644 index 0000000000..50f743621b --- /dev/null +++ b/blueprints/gke/autopilot/bundle/monitoring/dashboards/nginx.json @@ -0,0 +1 @@ +{"__inputs":[{"description":"","label":"Prometheus","name":"DS_PROMETHEUS","pluginId":"prometheus","pluginName":"Prometheus","type":"datasource"}],"__requires":[{"id":"grafana","name":"Grafana","type":"grafana","version":"5.0.0"},{"id":"graph","name":"Graph","type":"panel","version":""},{"id":"prometheus","name":"Prometheus","type":"datasource","version":"1.0.0"},{"id":"singlestat","name":"Singlestat","type":"panel","version":""}],"annotations":{"list":[{"builtIn":1,"datasource":"-- Grafana --","enable":true,"hide":true,"iconColor":"rgba(0, 211, 255, 1)","name":"Annotations & Alerts","type":"dashboard"}]},"description":"Official dashboard for NGINX Prometheus exporter","editable":true,"gnetId":null,"graphTooltip":0,"id":null,"iteration":1562682051068,"links":[],"panels":[{"collapsed":false,"datasource":"${DS_PROMETHEUS}","gridPos":{"h":1,"w":24,"x":0,"y":0},"id":4,"panels":[],"title":"Status","type":"row"},{"cacheTimeout":null,"colorBackground":true,"colorPostfix":false,"colorPrefix":false,"colorValue":false,"colors":["#E02F44","#FF9830","#299c46"],"datasource":"${DS_PROMETHEUS}","decimals":null,"description":"","format":"none","gauge":{"maxValue":100,"minValue":0,"show":false,"thresholdLabels":false,"thresholdMarkers":true},"gridPos":{"h":3,"w":12,"x":0,"y":1},"id":8,"interval":null,"links":[],"mappingType":1,"mappingTypes":[{"name":"value to text","value":1},{"name":"range to text","value":2}],"maxDataPoints":100,"nullPointMode":"connected","nullText":null,"options":{},"postfix":"","postfixFontSize":"50%","prefix":"","prefixFontSize":"50%","rangeMaps":[{"from":"null","text":"N/A","to":"null"}],"repeat":"instance","repeatDirection":"h","sparkline":{"fillColor":"rgba(31, 118, 189, 0.18)","full":false,"lineColor":"rgb(31, 120, 193)","show":false},"tableColumn":"","targets":[{"expr":"nginx_up{instance=~\"$instance\"}","format":"time_series","instant":false,"intervalFactor":1,"refId":"A"}],"thresholds":"1,1","timeFrom":null,"timeShift":null,"title":"NGINX Status for $instance","type":"singlestat","valueFontSize":"100%","valueMaps":[{"op":"=","text":"Down","value":"0"},{"op":"=","text":"Up","value":"1"}],"valueName":"current"},{"collapsed":false,"datasource":"${DS_PROMETHEUS}","gridPos":{"h":1,"w":24,"x":0,"y":4},"id":6,"panels":[],"title":"Metrics","type":"row"},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"${DS_PROMETHEUS}","decimals":null,"description":"","fill":1,"gridPos":{"h":10,"w":12,"x":0,"y":5},"id":10,"legend":{"alignAsTable":false,"avg":false,"current":false,"hideEmpty":false,"max":false,"min":false,"rightSide":false,"show":true,"total":false,"values":false},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null","options":{},"percentage":false,"pointradius":2,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"stack":false,"steppedLine":false,"targets":[{"expr":"irate(nginx_connections_accepted{instance=~\"$instance\"}[5m])","format":"time_series","instant":false,"intervalFactor":1,"legendFormat":"{{instance}} accepted","refId":"A"},{"expr":"irate(nginx_connections_handled{instance=~\"$instance\"}[5m])","format":"time_series","instant":false,"intervalFactor":1,"legendFormat":"{{instance}} handled","refId":"B"}],"thresholds":[],"timeFrom":null,"timeRegions":[],"timeShift":null,"title":"Processed connections","tooltip":{"shared":true,"sort":0,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"decimals":1,"format":"short","label":"Connections (rate)","logBase":1,"max":null,"min":null,"show":true},{"format":"short","label":"","logBase":1,"max":null,"min":null,"show":true}],"yaxis":{"align":false,"alignLevel":null}},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"${DS_PROMETHEUS}","decimals":0,"fill":1,"gridPos":{"h":10,"w":12,"x":12,"y":5},"id":12,"legend":{"alignAsTable":false,"avg":false,"current":false,"max":false,"min":false,"rightSide":false,"show":true,"total":false,"values":false},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null","options":{},"percentage":false,"pointradius":2,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"stack":false,"steppedLine":false,"targets":[{"expr":"nginx_connections_active{instance=~\"$instance\"}","format":"time_series","intervalFactor":1,"legendFormat":"{{instance}} active","refId":"A"},{"expr":"nginx_connections_reading{instance=~\"$instance\"}","format":"time_series","intervalFactor":1,"legendFormat":"{{instance}} reading","refId":"B"},{"expr":"nginx_connections_waiting{instance=~\"$instance\"}","format":"time_series","intervalFactor":1,"legendFormat":"{{instance}} waiting","refId":"C"},{"expr":"nginx_connections_writing{instance=~\"$instance\"}","format":"time_series","intervalFactor":1,"legendFormat":"{{instance}} writing","refId":"D"}],"thresholds":[],"timeFrom":null,"timeRegions":[],"timeShift":null,"title":"Active Connections","tooltip":{"shared":true,"sort":0,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"decimals":0,"format":"short","label":"Connections","logBase":1,"max":null,"min":null,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":true}],"yaxis":{"align":false,"alignLevel":null}},{"aliasColors":{},"bars":false,"dashLength":10,"dashes":false,"datasource":"${DS_PROMETHEUS}","fill":1,"gridPos":{"h":8,"w":24,"x":0,"y":15},"id":15,"legend":{"avg":false,"current":false,"max":false,"min":false,"show":true,"total":false,"values":false},"lines":true,"linewidth":1,"links":[],"nullPointMode":"null","options":{},"percentage":false,"pointradius":2,"points":false,"renderer":"flot","seriesOverrides":[],"spaceLength":10,"stack":false,"steppedLine":false,"targets":[{"expr":"irate(nginx_http_requests_total{instance=~\"$instance\"}[5m])","format":"time_series","intervalFactor":1,"legendFormat":"{{instance}} total requests","refId":"A"}],"thresholds":[],"timeFrom":null,"timeRegions":[],"timeShift":null,"title":"Total requests","tooltip":{"shared":true,"sort":0,"value_type":"individual"},"type":"graph","xaxis":{"buckets":null,"mode":"time","name":null,"show":true,"values":[]},"yaxes":[{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":true},{"format":"short","label":null,"logBase":1,"max":null,"min":null,"show":true}],"yaxis":{"align":false,"alignLevel":null}}],"refresh":"5s","schemaVersion":18,"style":"dark","tags":["nginx","prometheus","nginx prometheus exporter"],"templating":{"list":[{"current":{"selected":false,"tags":[],"text":"default","value":"default"},"hide":0,"includeAll":false,"label":"datasource","multi":false,"name":"DS_PROMETHEUS","options":[],"query":"prometheus","refresh":1,"regex":"","skipUrlSync":false,"type":"datasource"},{"allValue":null,"current":{},"datasource":"${DS_PROMETHEUS}","definition":"label_values(nginx_up, instance)","hide":0,"includeAll":true,"label":"","multi":true,"name":"instance","options":[],"query":"label_values(nginx_up, instance)","refresh":1,"regex":"","skipUrlSync":false,"sort":0,"tagValuesQuery":"","tags":[],"tagsQuery":"","type":"query","useTags":false}]},"time":{"from":"now-15m","to":"now"},"timepicker":{"refresh_intervals":["5s","10s","30s","1m","5m","15m","30m","1h","2h","1d"],"time_options":["5m","15m","1h","6h","12h","24h","2d","7d","30d"]},"timezone":"","title":"nginx / view","uid":"nginx_view","version":1} \ No newline at end of file diff --git a/blueprints/gke/autopilot/bundle/monitoring/frontend.yaml b/blueprints/gke/autopilot/bundle/monitoring/frontend.yaml new file mode 100644 index 0000000000..d37b2e0f5e --- /dev/null +++ b/blueprints/gke/autopilot/bundle/monitoring/frontend.yaml @@ -0,0 +1,79 @@ +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https:#www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: v1 +kind: ServiceAccount +metadata: + name: frontend + namespace: monitoring +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: frontend + namespace: monitoring +spec: + replicas: 1 + selector: + matchLabels: + app: frontend + template: + metadata: + labels: + app: frontend + spec: + serviceAccountName: frontend + tolerations: + - key: group + operator: Equal + value: monitoring + effect: NoSchedule + nodeSelector: + group: monitoring + automountServiceAccountToken: true + containers: + - name: frontend + image: "gke.gcr.io/prometheus-engine/frontend:v0.5.0-gke.0" + args: + - "--web.listen-address=:9090" + ports: + - name: web + containerPort: 9090 + resources: + requests: + cpu: 10m + memory: 15Mi + limits: + memory: 15Mi + readinessProbe: + httpGet: + path: /-/ready + port: web + livenessProbe: + httpGet: + path: /-/healthy + port: web +--- +apiVersion: v1 +kind: Service +metadata: + name: frontend + namespace: monitoring +spec: + clusterIP: None + selector: + app: frontend + ports: + - name: web + port: 9090 \ No newline at end of file diff --git a/blueprints/gke/autopilot/bundle/monitoring/grafana.yaml b/blueprints/gke/autopilot/bundle/monitoring/grafana.yaml new file mode 100644 index 0000000000..dd4bfb03cd --- /dev/null +++ b/blueprints/gke/autopilot/bundle/monitoring/grafana.yaml @@ -0,0 +1,184 @@ +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: v1 +kind: ConfigMap +metadata: + name: grafana + namespace: monitoring +data: + allow-snippet-annotations: "false" + grafana.ini: | + [analytics] + check_for_updates = true + [grafana_net] + url = https://grafana.net + [log] + mode = console + [paths] + data = /var/lib/grafana/ + logs = /var/log/grafana + plugins = /var/lib/grafana/plugins + provisioning = /etc/grafana/provisioning + datasources.yaml: | + apiVersion: 1 + datasources: + - access: proxy + editable: true + isDefault: true + jsonData: + timeInterval: 5s + name: Prometheus + orgId: 1 + type: prometheus + url: http://frontend.monitoring.svc.cluster.local:9090 + dashboardproviders.yaml: | + apiVersion: 1 + providers: + - disableDeletion: false + folder: k8s + name: k8s + options: + path: /var/lib/grafana/dashboards/k8s + orgId: 1 + type: file + - disableDeletion: false + folder: locust + name: locust + options: + path: /var/lib/grafana/dashboards/locust + orgId: 1 + type: file + - disableDeletion: false + folder: nginx + name: nginx + options: + path: /var/lib/grafana/dashboards/nginx + orgId: 1 + type: file +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: grafana + namespace: monitoring +spec: + replicas: 1 + selector: + matchLabels: + app: grafana + template: + metadata: + labels: + app: grafana + spec: + tolerations: + - key: group + operator: Equal + value: monitoring + effect: NoSchedule + nodeSelector: + group: monitoring + containers: + - name: grafana + image: grafana/grafana:8.3.4 + ports: + - name: web + containerPort: 3000 + env: + - name: GF_PATHS_DATA + value: /var/lib/grafana/ + - name: GF_PATHS_LOGS + value: /var/log/grafana + - name: GF_PATHS_PLUGINS + value: /var/lib/grafana/plugins + - name: GF_PATHS_PROVISIONING + value: /etc/grafana/provisioning + - name: "GF_AUTH_ANONYMOUS_ENABLED" + value: "true" + - name: "GF_AUTH_ANONYMOUS_ORG_ROLE" + value: "Admin" + - name: "GF_AUTH_BASIC_ENABLED" + value: "false" + - name: "GF_SECURITY_ADMIN_PASSWORD" + value: "-" + - name: "GF_SECURITY_ADMIN_USER" + value: "-" + volumeMounts: + - name: config + mountPath: "/etc/grafana/grafana.ini" + subPath: grafana.ini + - name: storage + mountPath: "/var/lib/grafana" + - name: k8s-grafana-dashboards + mountPath: "/var/lib/grafana/dashboards/k8s" + - name: locust-grafana-dashboards + mountPath: "/var/lib/grafana/dashboards/locust" + - name: nginx-grafana-dashboards + mountPath: "/var/lib/grafana/dashboards/nginx" + - name: config + mountPath: "/etc/grafana/provisioning/datasources/datasources.yaml" + subPath: "datasources.yaml" + - name: config + mountPath: "/etc/grafana/provisioning/dashboards/dashboardproviders.yaml" + subPath: "dashboardproviders.yaml" + resources: + requests: + cpu: 30m + memory: 100Mi + limits: + memory: 100Mi + livenessProbe: + failureThreshold: 10 + httpGet: + path: /api/health + port: 3000 + initialDelaySeconds: 60 + timeoutSeconds: 30 + readinessProbe: + httpGet: + path: /api/health + port: 3000 + volumes: + - name: config + configMap: + name: grafana + - name: k8s-grafana-dashboards + configMap: + name: k8s-grafana-dashboards + - name: locust-grafana-dashboards + configMap: + name: locust-grafana-dashboards + - name: nginx-grafana-dashboards + configMap: + name: nginx-grafana-dashboards + - name: storage + emptyDir: {} +--- +apiVersion: v1 +kind: Service +metadata: + name: grafana + namespace: monitoring + annotations: + cloud.google.com/neg: '{"ingress": true}' + cloud.google.com/app-protocols: '{"web":"HTTP"}' + cloud.google.com/backend-config: '{"default": "backendconfig"}' +spec: + clusterIP: None + selector: + app: grafana + ports: + - name: web + port: 3000 \ No newline at end of file diff --git a/blueprints/gke/autopilot/bundle/monitoring/ingress.yaml b/blueprints/gke/autopilot/bundle/monitoring/ingress.yaml new file mode 100644 index 0000000000..12b810a29b --- /dev/null +++ b/blueprints/gke/autopilot/bundle/monitoring/ingress.yaml @@ -0,0 +1,43 @@ +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- +apiVersion: cloud.google.com/v1 +kind: BackendConfig +metadata: + name: backendconfig + namespace: monitoring +spec: + healthCheck: + requestPath: /api/health + port: 3000 + type: HTTP + logging: + enable: true + sampleRate: 0.5 +--- +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: ingress + namespace: monitoring + annotations: + kubernetes.io/ingress.global-static-ip-name: "grafana" + kubernetes.io/ingress.allow-http: "true" +spec: + defaultBackend: + service: + name: grafana + port: + name: web \ No newline at end of file diff --git a/blueprints/gke/autopilot/bundle/monitoring/kube-state-metrics.yaml b/blueprints/gke/autopilot/bundle/monitoring/kube-state-metrics.yaml new file mode 100644 index 0000000000..d74bdb75f4 --- /dev/null +++ b/blueprints/gke/autopilot/bundle/monitoring/kube-state-metrics.yaml @@ -0,0 +1,342 @@ +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: apps/v1 +kind: StatefulSet +metadata: + labels: + app.kubernetes.io/name: kube-state-metrics + app.kubernetes.io/version: 2.3.0 + namespace: gmp-public + name: kube-state-metrics +spec: + replicas: 1 + selector: + matchLabels: + app.kubernetes.io/name: kube-state-metrics + serviceName: kube-state-metrics + template: + metadata: + labels: + app.kubernetes.io/name: kube-state-metrics + app.kubernetes.io/version: 2.3.0 + spec: + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: kubernetes.io/arch + operator: In + values: + - arm64 + - amd64 + - key: kubernetes.io/os + operator: In + values: + - linux + containers: + - name: kube-state-metric + image: k8s.gcr.io/kube-state-metrics/kube-state-metrics:v2.3.0 + env: + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + args: + - --pod=$(POD_NAME) + - --pod-namespace=$(POD_NAMESPACE) + - --port=8080 + - --telemetry-port=8081 + ports: + - name: metrics + containerPort: 8080 + - name: metrics-self + containerPort: 8081 + resources: + requests: + cpu: 10m + memory: 50Mi + limits: + memory: 50Mi + securityContext: + allowPrivilegeEscalation: false + privileged: false + capabilities: + drop: + - all + runAsUser: 1000 + runAsGroup: 1000 + livenessProbe: + httpGet: + path: /healthz + port: 8080 + initialDelaySeconds: 5 + timeoutSeconds: 5 + readinessProbe: + httpGet: + path: / + port: 8081 + initialDelaySeconds: 5 + timeoutSeconds: 5 + serviceAccountName: kube-state-metrics +--- +apiVersion: v1 +kind: Service +metadata: + labels: + app.kubernetes.io/name: kube-state-metrics + app.kubernetes.io/version: 2.3.0 + namespace: gmp-public + name: kube-state-metrics +spec: + clusterIP: None + ports: + - name: metrics + port: 8080 + targetPort: metrics + - name: metrics-self + port: 8081 + targetPort: metrics-self + selector: + app.kubernetes.io/name: kube-state-metrics +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + namespace: gmp-public + name: kube-state-metrics + labels: + app.kubernetes.io/name: kube-state-metrics + app.kubernetes.io/version: 2.3.0 +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: gmp-public:kube-state-metrics + labels: + app.kubernetes.io/name: kube-state-metrics + app.kubernetes.io/version: 2.3.0 +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: gmp-public:kube-state-metrics +subjects: +- kind: ServiceAccount + namespace: gmp-public + name: kube-state-metrics +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: gmp-public:kube-state-metrics + labels: + app.kubernetes.io/name: kube-state-metrics + app.kubernetes.io/version: 2.3.0 +rules: +- apiGroups: + - "" + resources: + - configmaps + - secrets + - nodes + - pods + - services + - resourcequotas + - replicationcontrollers + - limitranges + - persistentvolumeclaims + - persistentvolumes + - namespaces + - endpoints + verbs: + - list + - watch +- apiGroups: + - "" + resources: + - pods + verbs: + - get +- apiGroups: + - extensions + resources: + - daemonsets + - deployments + - replicasets + - ingresses + verbs: + - list + - watch +- apiGroups: + - apps + resources: + - statefulsets + - daemonsets + - deployments + - replicasets + verbs: + - list + - watch +- apiGroups: + - apps + resources: + - statefulsets + verbs: + - get +- apiGroups: + - batch + resources: + - cronjobs + - jobs + verbs: + - list + - watch +- apiGroups: + - autoscaling + resources: + - horizontalpodautoscalers + verbs: + - list + - watch +- apiGroups: + - authentication.k8s.io + resources: + - tokenreviews + verbs: + - create +- apiGroups: + - authorization.k8s.io + resources: + - subjectaccessreviews + verbs: + - create +- apiGroups: + - policy + resources: + - poddisruptionbudgets + verbs: + - list + - watch +- apiGroups: + - certificates.k8s.io + resources: + - certificatesigningrequests + verbs: + - list + - watch +- apiGroups: + - storage.k8s.io + resources: + - storageclasses + - volumeattachments + verbs: + - list + - watch +- apiGroups: + - admissionregistration.k8s.io + resources: + - mutatingwebhookconfigurations + - validatingwebhookconfigurations + verbs: + - list + - watch +- apiGroups: + - networking.k8s.io + resources: + - networkpolicies + - ingresses + verbs: + - list + - watch +- apiGroups: + - coordination.k8s.io + resources: + - leases + verbs: + - list + - watch +--- +# TODO(pintohutch): bump to autoscaling/v2 when 1.23 is the default in the GKE +# stable release channel. +apiVersion: autoscaling/v2 +kind: HorizontalPodAutoscaler +metadata: + name: kube-state-metrics + namespace: gmp-public +spec: + maxReplicas: 10 + minReplicas: 1 + scaleTargetRef: + apiVersion: apps/v1 + kind: StatefulSet + name: kube-state-metrics + metrics: + - type: Resource + resource: + name: memory + target: + type: Utilization + averageUtilization: 60 + behavior: + scaleDown: + policies: + - type: Pods + value: 1 + # Under-utilization needs to persist for `periodSeconds` before any action can be taken. + # Current supported max from https://kubernetes.io/docs/reference/kubernetes-api/workload-resources/horizontal-pod-autoscaler-v2beta2/. + periodSeconds: 1800 + # Current supported max from https://kubernetes.io/docs/reference/kubernetes-api/workload-resources/horizontal-pod-autoscaler-v2beta2/. + stabilizationWindowSeconds: 3600 +--- +apiVersion: monitoring.googleapis.com/v1 +kind: ClusterPodMonitoring +metadata: + name: kube-state-metrics + labels: + app.kubernetes.io/name: kube-state-metrics + app.kubernetes.io/part-of: google-cloud-managed-prometheus +spec: + selector: + matchLabels: + app.kubernetes.io/name: kube-state-metrics + endpoints: + - port: metrics + interval: 30s + metricRelabeling: + - action: keep + regex: kube_(daemonset|deployment|pod|namespace|node|statefulset)_.+ + sourceLabels: [__name__] + targetLabels: + metadata: [] # explicitly empty so the metric labels are respected +--- +apiVersion: monitoring.googleapis.com/v1 +kind: PodMonitoring +metadata: + namespace: gmp-public + name: kube-state-metrics + labels: + app.kubernetes.io/name: kube-state-metrics + app.kubernetes.io/part-of: google-cloud-managed-prometheus +spec: + selector: + matchLabels: + app.kubernetes.io/name: kube-state-metrics + endpoints: + - port: metrics-self + interval: 30s \ No newline at end of file diff --git a/blueprints/gke/autopilot/bundle/monitoring/kustomization.yaml.j2 b/blueprints/gke/autopilot/bundle/monitoring/kustomization.yaml.j2 new file mode 100755 index 0000000000..6a738f53d8 --- /dev/null +++ b/blueprints/gke/autopilot/bundle/monitoring/kustomization.yaml.j2 @@ -0,0 +1,72 @@ +# Copyright 2023 Google Inc. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +resources: + - namespace.yaml + - frontend.yaml + - grafana.yaml + - ingress.yaml + - custom-stackdriver-metrics-adapter.yaml + - kube-state-metrics.yaml +configMapGenerator: +- name: k8s-grafana-dashboards + namespace: monitoring + options: + disableNameSuffixHash: true + files: + - dashboards/k8s-global.json + - dashboards/k8s-namespaces.json + - dashboards/k8s-nodes.json + - dashboards/k8s-pods.json +- name: locust-grafana-dashboards + namespace: monitoring + options: + disableNameSuffixHash: true + files: + - dashboards/locust.json +- name: nginx-grafana-dashboards + namespace: monitoring + options: + disableNameSuffixHash: true + files: + - dashboards/nginx.json +patches: + - target: + version: v1 + kind: ServiceAccount + name: frontend + namespace: monitoring + patch: |- + - op: add + path: /metadata/annotations/iam.gke.io~1gcp-service-account + value: sa-monitoring@{{ project_id }}.iam.gserviceaccount.com + - target: + version: v1 + kind: ServiceAccount + name: custom-metrics-stackdriver-adapter + namespace: monitoring + patch: |- + - op: add + path: /metadata/annotations/iam.gke.io~1gcp-service-account + value: sa-monitoring@{{ project_id }}.iam.gserviceaccount.com + - target: + group: apps + version: v1 + kind: Deployment + name: frontend + namespace: monitoring + patch: |- + - op: add + path: /spec/template/spec/containers/0/args/- + value: "--query.project-id={{ project_id }}" \ No newline at end of file diff --git a/blueprints/gke/autopilot/bundle/monitoring/namespace.yaml b/blueprints/gke/autopilot/bundle/monitoring/namespace.yaml new file mode 100644 index 0000000000..38e4ef69cc --- /dev/null +++ b/blueprints/gke/autopilot/bundle/monitoring/namespace.yaml @@ -0,0 +1,18 @@ +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https:#www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: v1 +kind: Namespace +metadata: + name: monitoring \ No newline at end of file diff --git a/blueprints/gke/autopilot/cluster.tf b/blueprints/gke/autopilot/cluster.tf new file mode 100644 index 0000000000..2ded1f637e --- /dev/null +++ b/blueprints/gke/autopilot/cluster.tf @@ -0,0 +1,54 @@ +/** + * Copyright 2023 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +module "cluster" { + source = "../../../modules/gke-cluster" + project_id = module.project.project_id + name = "cluster" + location = var.region + vpc_config = { + network = module.vpc.self_link + subnetwork = module.vpc.subnet_self_links["${var.region}/subnet-cluster"] + secondary_range_names = { + pods = "pods" + services = "services" + } + master_authorized_ranges = var.cluster_network_config.master_authorized_cidr_blocks + master_ipv4_cidr_block = var.cluster_network_config.master_cidr_block + } + enable_features = { + autopilot = true + } + monitoring_config = { + enenable_components = ["SYSTEM_COMPONENTS"] + managed_prometheus = true + } + cluster_autoscaling = { + auto_provisioning_defaults = { + service_account = module.node_sa.email + } + } + release_channel = "RAPID" + depends_on = [ + module.project + ] +} + +module "node_sa" { + source = "../../../modules/iam-service-account" + project_id = module.project.project_id + name = "sa-node" +} \ No newline at end of file diff --git a/blueprints/gke/autopilot/diagram.png b/blueprints/gke/autopilot/diagram.png new file mode 100644 index 0000000000000000000000000000000000000000..121f115a3d7ae0b40ba2267055bb694b1a8d1385 GIT binary patch literal 33543 zcmeFZcT|&G_bw{Mih!bmN>gbH1XL8HV*x~(f`AapR(c7&g-~s%C`b!R2`Ej3fYKpA zRFo)U8;|%=dP^tzId2D{>=rrz9u)#k}$xxID{QaE-0NiC@`e z-zUETlHY(5{gItd|1wR2Z@fILvyVBcR&Z&33a#6ql1#V!esXb#!&90a3mAe@%|^8U zR`4TGn=gqD{N1<12K@Vuk^V6Fb22D!8~CGfQ$`!RHAH z9t>N)cxr2DP5n4_6v*i?{o!5U^#5-^AD&Dy;p5wPk67hsdPvsKLUiNRaHEH{S_Q(_ z7|Xro-O3J|fm~&VvE;eSnFCVZMs=8i?$Y&tt)P#mA*mX9d#|auc~5- zt(8LzF1)X-4oiHDjah$#Mw?(w&gAD8T3KUfr=~pbua0c_vZ$nV5y+L#iUEK2^l8+B z@0{ZLrSfjM)sVy|CepIu;gYBgyO|G@1lmF(?*_Fr5oWs)@d| zODNxt-5>-H_)(~2%^rQ)oB5C&eRSWOWe=Y;|LF&*75;gi!yRR}F9h?Yc-4$f9*BuD zDff3H232BKc+Mp_E656Y*?8Y@tP>Z#;+N3h*O!b9gO?B&MjuXg(!XOnmh@r|E^u!i z-K>M%!S02I8XC!|s$wINdw?cx?h-JXRpZXeQq0ZL+4amm35zM6oQp;#e$-rdiC;{o z#19CSa_u80i1NenU2XY}t%c$hrXJ%2>iw#!mC(?Rj>5-TGu#+bbQE%uxaX~PlUc2~ zCANW^le2Yz-NNB>xI6KE6-1VWO9tsZ9V%BjEHvB+>-C(iLT6_a5F}od+PMK-R|+-` zUKB|XipRV%W7?wr;M>}2Y|Xd7^|5d6o!>kuowzYQ8H^o=FCkFlv}7S7YkDk8Nj0Sa zCK#XHV%4cwjp0wS+-P;NRw-ys-x(aN8k#7#7&qWY8q%Bk^5yAtbr{c;K2LYg`z^&* z&v#8!YolPjrHjqO)(s6128KHfckcA{$)B*DG?XsM6>Ue%j65$TZ=?2UL?|vVa~5gG zYvBgA{5>+pd^hi0N|4Zaz;ea9Q<*KTodTy0_~ZoylumgO<`rO-8+%D5`D=k2t>uko zX3iC8swor-7CJK~ z+HtnYbl_=3EGib+-`4B9Om2uwZ+acPB`{U>n`*z$9Dt2;PindHM3SwR^<$q*pmu zv)JY9>m|fa7}>KO?_`hz#oA)-=67I#rWQF4KtwF{sc#0q44R{1uoldnI~Rd2k#AD1 zXymh;g$`lRJb$;wp0ePuL_0cpIXT}&&lX@TBOSz!tbts<>^HA0276i#8A`j_yQod3 zD>%NQFU<+~LVnT%_4+ug=zYeK_yR4vayhboUroLEIk%PDcMA&r4B_|IPoyje2@3Le zdlBW|Ac*Hn>Vqn)N3<^~G2Afy?lgRqw552L(rDA~ppCQwsd^f6(Vfb!-JG~OU9x0N zc;Q-PUvWA?%uBb3g4@?=#yfAe6S7ETmED)8=z6re>)Az_)lM`2Md7oWP>DhJEAIYS z&&b0kzLc$hfU1!!@0A}IG8dQu-toX*R9ZBOVAnETY3 zsVx*A`K7{Rygz0jtHSr}+x)C!LQ+n{$OP=_GO@+fnqSfPzGf`*o#yq^uVav2iT*ZAQt9x06OgXcHGo$fGMGIol~L zJkeMNn;s7Kn2)ovQK)bMHAu7XD47}F2<)WEEX|=S_oC`>?yz}xd4g{C z1XUYPX+X^kB-$y+NOzo;?ms2{RHEJ2!gbY8-o8qx7v@J8k?YhZZR@|=KTSI`_9pdQa7c>q`M^4F&}s9udttRZ|y6!ke#6j@?n!GA@lhE&$-{85UBaSI&x_ocJh*-y7;JN9eI za0h7l?cAas_T4Kpy1(b89>mcoBhC*u#*VPP@JJwF6{Z5VleJfhH>3yo=w?}bR|uXh zn>_&Xry3k$$e4-L|J!tZ|~JiV7h&8l&ADD1zBY}rE5c@_N=^$ z(I2$b5Re{wl8pDrY7Tb2urPZAbCy%msRv?`z18K4Q7QlOhd>4A)#8-G6t`B zw2^G`E$8C?L9Rq+&GmDQJJX>JUBy@wtHKIxKg-Su#Q`GTcbt!?gJE#G;N|X_AdMI- zGV>e9&J1o;0`DkpnkpENDYohx9VnuG863_9HlmI|AkN=(SubG%HjjRt z9d0e)6@DWobfChXIGoR3K??y`i*b=HXXo#}v%q8_jYeX_g|^WM?U}UMq6~ZgMW4)AaZ}U{ zk36qsj~L0!Y>dC#ds_GT*^74r{64O&w-OCy6B+@!ps@0&5BulY&61`i(Hl1_i!wVC z&Lvzm($6k0=xuFvTf;LOmN*VecK_8^g$Ci<+}zxC8dK)`u#?#7XoR)xDH_O;g_E;9 zx!nxBNTZn#GXv(vm+{QdBEo}6#+X3lXR2cZxz}s0+|N%NRnhgvk^)Yqv<2M2jsnwQ zU|5W~X~V@5d}gVeDW+()myeG%AKsJA5FD0Tp+)39c)67H#r?xtRaF{+fRgo!OpQ*B zv&`#5uvT1Ai!`w%>Nw#`)p<-}p=|K_mA>cQ(tJS!P%-<$nFo;wZp8!QUf~^PtGF$1 zK7Pl*OMxh^jD_nPA$-dulp=+i*hiVtL%{)f_w8+D6Al)eog(YemIQ*u(5QrwsYd8(-HE9-X2Eaq6ArQ{3pCcza6K#P6aO zLAE3#>CFv7s#wvh94ApxVKqOhpj6J0wdTkH{{mr=aiDcjLwM>)4sgob(XX<6BA-KK zAtywL)6=XQ4?Tv5hg~_~HgNsiOaMyZ=VqG=NO1*;a(%6>P#q&dN^MPD#;dFj{}YcV zrl!P1g^|X{als4|U1%p~F&<%K^QtsWU$XIYUHDe`>jnsZKoGBfsbD*x?!o zWNjspXFW9=#Y|5YXOSVvyS6u&ah9E%>;x{+_Lc|trwh4u?(S`at2L@*em7MH>_KI8 zc$d7-`gP#T=jPswEKJ^A#A}b^1ajq`%Jn`Q9u9j|gI*|Jn7}tRtRAOjbNQE=Z1vw= zv%N_(c~%!yGJlcbUCWVgSa~1aY);SH?DzN-yftSoE3GxNKoPNmj>RI1$IZmtX>#u~ z8pIoAuw^5U5MO+*PuNYZ1QT0C#7>yx`L}j(yQDY9%LDgCtT1t#uUih{4Y;~(i&F4U zRNF)2d1GI5%D89XdY4Ifr(T*MHaYE0?X|Xvj^EEL4-~8EiMGu4U z{=B5ECGO-A_GH?C-s$Le(v^U-sx6;PQAKNZ|Bz!i3b&yBP$-U+NifOPV1;5G+kriqs-0V1-3z5BFC zucxy+4-m4?lb-L7xs;u?9>5~2%0&p?N`RJLISh`V63@byTbDk$$2=|+lpa8r$B52iW>M2U_SwMqN1>4G ze#YmpPds;4--l|IEE}b%Z*@0Gr{u6mljPiMec`q6q0~(48fxaER(IIPrPaH%imwm; zo~1*Z>5Yz^JL;ie=VXN$(lxe(5X?|+E6kM6b?S)#C3QV@sJ(&*v%6pVG;dR}5v3Qu z`|)R|tL+2@^5+7U=CeK+2$xjUw|9(h#4sAbGi8f90zW)b)J~^(b+=i> z$m`jqoDymWc5F_UcHnT|MWWX+EvzdVGf6a8n@gzOXs~FQDDIhz9cTheJHvZ9^zP@p5vC zH#Slh-a*|l}Wquu+XDfbO%qhXF z5$!=;^B)J6=MVH#z`P<5aldzMc>{%nAcEVd3NO97u%nA?om4q5yS4ScJo{e7zVUTT zWaHU~jTqvX!L3hkht8;f^Wy5fI4YH@si`@op5AJB?b@JgiJhIDF#Xijltn5&35|yE z@g-+vX`pvjc6Hsji`bIY(M&|Xq>_RY0uZ0!L!-W(GJ?Iyp{zwA5o0_CT4+qM?M4t` zq|_hvrI%Ebh)0+6I_`p=KG$&n$xor)30ahpl{LsQl|Oarg`c60PA-DvXm}&fssRlx ziinKVPLL_^nC=J)qBC4uoi{JhgPvd~R7YiJXU9o7O?CQzuBo|v;lgcy$R60RmY4sX zX@}Tp%B7f&H!XFC^g_y#9k}6adb5rh*0vJ{sf&6h&0~&JgAlv8-$!9iArZyJ#5$1T zre)6}B8FHP-ObDr3l|nf>mwR<%BMSXyw0r?u_eaY+_Ik6uU}s;W(|O=hge%#y*8=6 z;O6G0ua6WB3k|LCqeuz~S$lhTan}re_;3LxF$)2#lilp5M6X}xwmhQ0>M1fAcyDh0(QLcWRg>V+1i))3=hp~?LBhdzdv7lYk4$?0mL4K_}JJ|`_AlO zR^GbUa$q0*<&@@R)t2`*zxzFQeoU{bg`I`=R0C z(b~&W;Ngand`UO&4W8f3rKaeqIo8mF$Z>q$; zHGiJOuBxI^LEAtCSB57h=7@d18slt?@q~dln3$Nf2W*S50ZbSe!E z4O3H7@v&7^DjrX&f$MQ}bkx!cEmYZ>4+e=7+u!VYoF*zJhNQj{;-d^+yn8a$9!IO! zJX1&wKddhTbIHL>I!us_Y3u4@QLR~qKi?}(=fS-6@_uk|qCCQLebt4^tn{%efiP{D zO>_RZMy7@VCuh^9*QutKBlqp5TvrAe-~@}LQWEKL-QCZlqT2fVv$1p!G!R%y;;_lI zqfFcyjE`5?&YilvH+{YRLv4`ZwQorRh!=%Jh>*a(Vq-$D%&z%iJh?YB2UAftGUFTY zg8gOfqgK&xQTfJxoW5`A);AAB>AckP`~_~;4nP2`sZv;I-gYVIK)T^ioS-Y*)2k>; zC`(K?V>Q3HkvJP_ll#-3GoIN~y1orCTOMf!Q=R75?Aa0wBuaV(Vt=ffr)3(0@V+g3>xP#FGFK^iY+Kd%ouCLrQ{oxx?t)pTNBk0!S zHnUXUfacGSzH1wB00#!3jL7;1ht47c6WaqRybd4#oZt=iX{XYSKL2quZdb|q9-Wu` zn5qD^Gspgpx`3(b;;ohrU>h%^B7H(*{lbj7nraMx3E3wF>8PKSISdFY*R;?Sj6E0b z^53UJ=<$z276UEk$0;S=e04J9J{oanx<5qYZy~X7b>Cgz)TAABRQeJ*nf){;PrrPp z$Z{r9@BI0ojaQH=N7ZG$D^n|(1>}Ny;Rp_x=y!KQkIy2LeW>R@1|L3rcu5sv;o;#y z?t@4rYGbY7XC)=4!tl4eo5g;;Wpg-02%Zs=`|7gM$NO|JDaQN*$KcIk96v+*SMV^& z9+#xden2f`4=&_b5_(TSa^Y796$OiQkgiY5+W)b!G1J}LsX2&C6r9d+B|y{X;Gnl;RF} zhJ0>plnGQ(qDLawj^WgcL8zT<;LYzNRN+DDtn~mpgoHla~iiszI}86O{?D*Xy^z)W9X&>n=>#i3731?TnkJF1>= zhH=XZ1{bBIj8NrcdUdv)5jT>}`)S3S3d)307L<33PROgh#{~n{~h@!L0d~vM7j@kmP3mj9Xl>DvpmbQB{ zN&yI((BD>c*~kcw!(H|9AwS`iVs|KR0}>eBHNng&InlwmJ)g^2y6n@ZPeR%#<2Qpk z&=YjjwKAJORpx`^-V6SeE;lV-*-N#sh&6`m2 zZb7sLvz>EmzV9Kfb6?JvOinZ>OG`@=iDlmJPkrYv0YH2Tf8^ zqw~_e8@8PB@-Yfjzab0LVU4C&QHf5@z2np+*%S#02_5eiUR_Y5RU?D|t5fSKGZs=3hOwlgOki&5zAB28{t!xh+f>gecL zyaz$p1k;hE3?k_8(9qA-XHuJ0Q=+YBVP3nb1oQ~W`H|0@IH4B$+H>&z+}vC`eQ9#? zMLTGrZ@_qJK@?s29&sbgZ-zNo@W8FEQ(CS^@{FS8tc0qWMt~daG{#>AJ#%n{SypjP zETePUZ+&(C#l>K)LbFZ*8woQ>RO{j(C*pE;L z(S%s(LSkKq)S)`DSEf0MM!cUr6EVDp6|5sHCjSLf1&y5Hqa5 z$zh`ann1nf-_N}Cq5mGQ|ZZU&$@9ms!NQ~&F+lc?wZB(8d zLlImW+)J!r3aK8NTIouArGwKlSWfjN`}4k z+pFH~%$o13AsM9!-<_(0l9|b~Y>ujz_g_7Sok4zHe(V4)Z*TQT1aD zk&7c}Aq$##`^PrA<-^joQmZZ~bxOu6lzo5qhe?m73GD~<-`Oyb4d)F&xm*#qY1ms$ zr+SV_yP!J@1n;slJg@|R|`ZE$)DfXTI@~AryxhG>5AZ zaCfW2@o%v2ZlI*B71ZMMXubVHzxW)N-yrRix2mZI7WNFTKh^G^(PLoHH$I?Y;MvH} z;nP_Dd}CIiC`%MZ^MwNGs82=bH`EM1(T+n_V!(=FRHVxmu@C3!MI5DPmgP4p5fE!~ zNIik*q%n3DJyVHc1V-Wc5G)h(-c523KO<@$QiIKE4%`#4<89^Ot5>fkh{Vo;zusOp zvVb$-hv&V^J!#L`M2;t=r=?x+MT=oBo92hJ(lK)$0}T=Y#&w4*?wC-9ejIhRw6->2 zw6L<0NzkIQpSQ>h3JOXU|M2xWGUDReEIIz@hODGO$>n60b64iAX-e3( zo7RT+z)^)S!q%-Gfn+T^TMSl`oSdAWf1O=WP;ex``_7#^3W?eqdUdy_qMMtW0i(do z%G&h#^XIMp2jV~Gld;LYQ}f;yPsq`2x!X%}W{xft?dG!u{T1?~&+QLA3r~*G8I7() z(slG}&9XXuUQ4^F9yAbeMm*O8%7_^=l&YZuT`*vM+YI3sKw5yp7 znDfF;@g($jxXhT}{ndyRI-{IE&vM4kc|2L&R_)h#VqkTo_d#o?>=-S`zEW!-r5YO~ z>K?jkP@1mTat!#$_e2486>CdNz3bO6Lq|qN+S=Qrx7KK)(P(kA3I_*=cofRf-W~wl z7|Ro^!Iejg(o#|`hfAF~laijUeotM#ew&`2UVD2x$n&DbJy^xUfM|?IvQ-<=VSSdbU_w$*)x)%#<(hPE zfMV!>mx)G49}BmH?77F7&tFi;h5Bmusv%sbYR^hv0qn}cRJ;BarTP$E_HEm@FD@*M z(bJ*JS%s_3HtOE~wO!qMC%uf3I~^qW?Iw+YJc7hFKY8*bBe}1)S58LeFc*%k7au#n z#qcRsM1?B5Wpn6>bHEyHCHMbKKjWzGOaQ}2m$e+iQ$Btj*Y@Z*koyZv3JVMK7XTpU zC@dxhO8C*v%`0hr*UiiZ@c7q|8c8-0KtXi$r?No#A-Tq26qj8~M2Tw`yfgl+;%;w6R;U43PPBYQVK2N-~|iq@*AWjSj!p zbTVo5a@`9Yy-|eLwA=w4uuVLj0UgBTRlkhMEfJ|CP`E6xCymp`9IQRhbNH~ZsAyY9 z&3UA5P?m#}QzCQ^ZXq|u5h)S9j<5x4%a;^jv2uoU*SAd9a4%hg>Gg0u)65D{+gW%# z%0%j1__OsJaU9ZH8_Zkr=qG#xN={2ozu&5yY)*qTXIaq>>k^|&FTvb(b1&}70aGn2 z^K}GNN?Hv%At3=Ti^Fqz2q!HrE*7YtO}U(ZrfqBf$16TQr4C2T>NKY+p@R=olmXwC z`fxNXN1-FFd_Xi{XLGiAOkm`mw%1xLNfz;0;i!)O<0d{LB1!FL>HuwzT7p)-zP2`g zv$}e{V>eiGxzu1!Ryq&NcG-!$LXt+!f;>NuT7vSY1jlN9il+;9WBwS(?IahDCf&RU zN*{nAkVLqkT$IueM9ubM%>3Xh_y$f^R8-+In(W^Y`MBuxd`ym0cSYW(LEFO#_%gro zhG6a15RovHbeyxmDQ~B}jjjbX#z$uGcsxHhe{2EiS#WT0J8jOQq@=`92C8!rGgN}X zKrw0+QLMR8?1x5-!pa-rd9A4EZ|q_5&l2Ja-FB0`(Ul<})>n z)6+qqb%&Sx0#l1_(;Jz-GLpBalP3+UzXj}Jc~zNd&id%3j#mb`Tu|u`pq`6AkBliT zGaj|f+2B@#*KY$Z?q^KgoN4{JB&d*8O0)>(ew(ZEcoO3hSu?I(Qm5-sYI|)?i$Gez z*_PG;Uy4cJmrHXje9uSi!VmCAup1%+H}-q%41Os#ctZt5ErX|<@k-m94z~5Du0Y)8 zty&V&#VyYZOB3GRG`w*Tr_(+y1*=8dD)Cw(maG=;ngD71Y`JeL2q~qC z{^xdo%RN;T7|%F703{MY##mvCifxoQBt0GaN}%joW@Y46O)bQcBrKWCd17pzwfGfz zdnY;G5Jl1g?B-lw^0{w$EOcwW{9c^?gArxUwYwZREA6}&--#2DhtH%#*_mxXf!6cD zSueC9Cfmg#SAGvpF>Byzv-ELY#gVo`wc?1?BWEx62^5v8`4yiuy7+tU3>(>JsuTw} z#oOOFMg6NvLh|a}Z>LihEb6p1yY#j zKThwk>DmK&WkFWo;O`V37%K29J@T)fw|>{IiKW{u9JWplXH~-zcbk?xjx@Y+*thpW ze7-LW@mQpZ1$hoo818_dQBfEKIOwPL-VF>5T3gNu;4X`J*x~GrJ{UUn;kyZMo_e7y z%FU8ci7y&`D~Q?x6sF*bZ9K^aDS)Sx!=aNX<3v>c6ynJAH#?qW%?a^?ZKTong1ZQ zac6E-WzA!e*~}aHahx=bN8D0NB!i zvbuQwC)01p=+1vJgblZQ(U6%qEgt``!H~Q5Qn_+z zFUqtiu6az~)pf3}$=auvz!D?1s?0X%Y`ONw%NgBwYq&#Gs^ur2O9g}|B zOgLPlEl1X<`Ck_dJoSRo^jQC$p7$b(MGt+HBsK+JNdKe4`q*)YjiC1q8}%O}*?b9! zeWwj&H(djxb{m6C>42o+@?umF^_GLLS_T@q^RIT$)XODHtvqkmXdV40zlI+x>52<&*z-~GHn|vO- zC}ANrPc0m#Gnycudy8!=laeL8g>4g?ai&3yV=A^s|o?9ZqkCvn{;_H5@Pg6&9HDoDDtGd3;Eehp;rn-NStbk-)Fd%eSCVdKG>sS zbOERHb^+(=Wr*yvBf8bEaOA{C_VrhIU96Zngpw=6Pzz})FhJZhEkodl$GXIM6PLJ%y<&L5lZ93TC3go zTba$-n-*hR{^<*vVd2UQMr?~79qJji#qyXG*;GS;W20%*?W8NsgwUSuE>i(Bo^$!y zSFXR7fh;OtgFCL5=;r2@gtiSB~zm(i8f@frakBTf)(8`Pa>LRSPN;KiH?7v$W8J!v?D~IrAZ@&-9e) z6?Iz@ZKn&EuS<#Vo%hj{3-=L=(N;pV~pBPlhv`8QYE#8M-dv{sWOZT+8S32(Hp&9YM)NeJwhv^&4uzg>n-6fdFd1EYYo?Q zZh75QifP|(T-tN!jokL--nPr^>d^XDq+-ZNchvxM z6ZL!RpOi3ql1MCP4ewQCV!B*J<#a&Y+CDl6tB^;9^Y|(0uS(^W|A$g-BBs5#aNwEw zKL&a-W_LeDM&qImMn`XFnT=f1Xz~{@Lt&rIRpQSIC#9Zd<>?QYoLbuyA#12MqdQEu z`;*AH#Z6pu3GH2feWw_gwCC}nle+X?^&Oh7ksrw)0+D433C=^k_q@AHJj3%Dm+q-= z6g@9EjJ%J;)VM{KL)JyjT|UXq!p0Wx7oxGb0kw`LIdYcNr1p16fee&TS+D( zv#TeQELrjGf^X%ovD#P-0tcx7qp}s-{KQ2`{TPX*$hMxroSQ8+REb!)+-A@FW_rSY zhQ}^^883^;eL_~_c8&Cv#`X@pg5t&|<}2|EmUskb+e+@fHCLOLP>gG-b+KNb@=6ja zf-&ELPg|w$wV(75`HK+ZaVY}95O%ByKA2s)EpD{dzTP#m_nJJNA+F!16WQ*w1`S$y zThqluk8k-UKcJK=)8iG)xo{y``h;m~{xRxJB(s@r|9H0Z&fu>@xJUV82s`Mw8NLPo%HSQ`jQO7~ zDplSGWyrseUsRIaVH5IeF#ln*UlD_>=nmj%|ASH74eVgF$OB6V~7 z(la;b;wJgt`Ytb(YSr7cPxnqU=BIv_?bk^lLKyS?1lI$3cz23PR_RNe9@??N!BO{r zL3j7yO4UlExyoGch5G&HsXyWB{}L?%0t4a)EdBpt_TU|0Rh#}{#TS*T=emUct7(Gi z4YIL(g=tJAgFj#zfb9T(g0_TGti7?Ix8L1;wZS_Oc;wo%iTi`YjTeuvmE6r0?Jf2ca_sD)j}uHKgQtGeSOVQ*m5D;@VuVbq3cHMWK@-m+rf!PxZ@Plo|LJ$y)3UW2LanJTcIUUF&bE~N zJ}?W&vV2SNIGdQ-RXcl0SZ}WLdHPa*Y$>+&tvMz8{_B-`j+_DP;^hurS=ZeBBo!mW z`m2VCYh)8kwaVwZ*6ob~H%V2qo4Xiny3y<%5uc2{)SYSfRJ(0ozu;V&dCr|qp356! zl47^3FplhgA9k*Ls~a<9RDq{}ZRx>&d^PR9G zX4!LZ0}kA>9*jwxsd?)X&~xW6PezuaxC-8br6{o6IZ+XM%41*Bu=YYj6Sd0YSXW)>^tT3RH@0u-hUdiP$-AOKYp#THw zEA+senZJY39B9A4RV`fo9|v`w~no7J4nk6d14YUjp(VsrI6VY}QqA*_CQ zDihz*DBsaIumIG{R^8Dn=4bdrEh#U{u0SxB22<97K#gq;p7+x+_A~oHQctk?m!$OA*WABm7xTp6JEF?Y4fOGFZGLPW zCbdGf+yEB*R{NhkgTAAZLpvT_?Wgt>%^1ndQkZ&nCATo-#ZC>_I7G>ff0K=$%V21mFp@Tpg|J#D2 zdQP15p{*yx%_O!(CPMcE29oRFrfCIdD1DitMLYZ{`0y0xoiOMF;Lbw{MVkM@+yHVJ zFo|D@sUBkoMt67dUb8<4rXMQ?b7OR$Vg63i{Xj#7()2Pb_rKSWf9J$tVv+!3b#kNs zfS15aZ}Na&q4VZ7{{q^3xahbM|FV3AC0zhA{}*b8F~8xH3RvV1)IW4*lX3+K3@ALW ztc|IdRCqm+C}HkD6{B6T&b0^kR_eFRB>cc$Ttr=CM#)0xb0GoLcI6GY&eRE~#OUZ# ziOs(V;z!+kQ--$|1Ca`-rO#(qn$oU&^InnR*d~a>EsSRFa=J*Ii4sCe$ow~vyoD*L zvuQVd_SoHI)dA<_pkUT)Gm7583zOh!NtS8F9#z=fn#JOeafZ{gT=Vd<7gp|0PQ(|1 ze5#>mq@?O&W5msnF){TQKmqXKKA{w0wtf5dv9V<{Mny+YdcD1Si|?P1du8{;^X?M= zU~Dw#bYO^{sf9h8wL7&zYz9KtUNav9Gy;Uhx)Ot{~_X1ARhX)t2X#s{8+uq`bYU6jtIp$}N#2To(37 zo;5_?$*yenq5>)3$+LsEL(fE_y-z}6pf}5@ady5|^U&Gc)By96kY1P1sRM%-Kx2DP zky9r+DzNeM=U$9g&dU8I<0DC+eJGe7Ng96icJF^p>l_Z^;7>N{C3x83I+RR4{1utUmkN3&3$J0{pqP;vFZ`BFKrRuGTM<8X#BpWA4#qU;ES3%>u#fuj& z&SNfI+(>>JXbO7tK^=9oaM{40qz&4dU`HCL@5PY%|H-5tFf+)M<0&a^P@J5^&=S0C z(TS#o7>}j+0da(>mxYCt=i*MEoZgv|>u_pz`AO;6t~l$F(40&;-WkHhg~WIFnK^wG z2HM)%aJZ+Upn>7CQhG{CenG)(e>t??QfzRCf5xTL&3?_0oyIInNFD{GO6sIZxzAa7c@y@L z)MC$Bqgn0=u-=A|;o&ZmEveGG{IaZ!PuH_l$tzHwl&)^g4m1OSn8@;lFu3@5TjBYl5q5O3`tIhKP6PI*- zpTuaNfKsmtl%?aCPgNh|lIm}XqeLliUQ)|V=Xy8H;#?=X+nmJxPAAdaa~6B-9lQP8 z&UUt8Q|M;FH*HL5=C)WWC`J4oPrkcR8_HvO2C~{i+PDM%C_H;y1AClATZl?vO>?KC zMxGlNZ^DRM`#4rCbNR}62zr1k1M08R;|HBnQd1i~`>uPjvoJGDgRUd|k=)3ts;ZKA z+V8=4N>&pFr$~oY0FzYyJJsCbNN*L z3TR6$C@2sR5NK%yO|m*wpdE&7Xo|4Byu4ruVhixKmgAriVr^|rEM~eon0ep7#pbH} zbdcuw&I!Q!UVAHquG8&T;G3Lb&d=Sym}1>1FF(H1*msa7aL+vN0JlU0K6gG(hlo*e zqFnu}RXdWtL#+I`uy6}K;GUbd>J($KW{;$0WuZ{<$zJe16B_uGNtAeM#PrmZJ6*!_ z=NVL$k7HwFp`oGRV+8Og!(gXRi8fA%NLwiTPRYad~()MlJEDSktc0w>Z#xsiwkMu_imB zm?C5uL{nQrBDWVsFwHmjY0m^`JM_CT12-9sZXk0jG(Dz;R;3(VSXjunoRzmqY!aeH zEmu}n9y)Z$Ey$QKH5Ea-VWDn#Ldw>_^70ueNy)nMl*be@*`3bB*jQ8~Jv}|OF@X-W z;p9ecW#wV{1)I?Wo?4#4i0{-43N_7zg4oShh?3+)7s`&4*d?3x7d0w~FYFZqm{K_4= z@pxe~Ee#DD-iuMu3)*4Tx&{V9MRoo83H3fZ=XrUBHU8Ds6Bvw!7aKG+?3|>VPdB%ybmY z>wvb(%T6tr?JSQt@q2h#JyAk?>=pe5dJ9i7d8O~9~8&se=ie$tLOmP$lTw&>GwhjGaYv&n6le1Kw+~7<3Bt~ zvG2krS^q7c`&~>sY*eIv=S%MfZc4g(?w4%yM|S<|fPU|;5&gOhUvK)OM7>inDWUTH zr>ZVH71iqhnTMVUL>`LcaxUI}h@yqV=I+_7C;ktvNH8MM>egl%?{}H)!TqD~53KcH zDE`d`i%nr^fGHw3*}1=q`V<6r$p4kT$CG`(h{rk~>$*8Pu|A+D35B!(Yo6z9L ziJx`TLw(;z^@A<{8P7L67d9&u|I#ynSJ>yN6ehhrn9^HW{@Bmj-IssW%=q|bz2^5S z&OU#ourNjrPV>w|hg=wGAxxHkltRDvNPN?-Uxj}uI9}g10JK9Y%FCk%hFZZcWe|av z>>;gS+h#|HAXCK_Y6GvGgToAn;E9(w_yV}7uURv+i>@0OczJoT?%oX&C%@v_?Yrqq zZQX^>V$XP&&w~!Bk&ZLd`h}s0J4NdgcPmC!$47fUbUb|iWm)9b-^#jp^umotfhz8o z>9dUrrfMUF)YQ~K#G39vPtVM>IW6t14&}Co!P;{QG~jT!mDQN>7Z(m@QdHFbZqop^ zvu!Ocu7qA`DxNh%f|9Bp1A;i~di??Tj*VuVWQI7{wu$i~n!@0q zMcMw+Y<+)!KN#fD(5;=CuqV4FGBQqNwAaz(F4+Xfs4PgVae|JIq1)^yy=Wr{hnfqXk$~64XFX;y^i~v#aa)v?YVb zeX!v?S`oAU&;o8_a~YK)_ed_^Gqcx(;4y(aI;#%)CJ7793zjA(2{ZbkC{Y{E8t&0& z7NZ=UOHZN;m$%Y#EN?2+Z)?h141t!mqltD8RtW^7KX=-s2c*x4N#CjD5y)pVo3s?Sn`y}1t$$3WrfRg$@x!_snww0F^S3Mf3^3OQBi(h+c=UE5(3hp zfJmuyk4lILNH>@a-JK#T!bnMj!XVASkkVlxNDf^B(%s$98Bl-v`~TPT>3u)EYrU+c zEIrK4efGWgKKr`%IoI9idXUyk{_*j`0Z@!mxZ~|PyFc+ceRP7oY$=H%{jqd`RC>X6 z*6qgOgT*1ZfYWeRLfA%p@psqd@zsSrWY0eO%KYnt`k@wsa_7ZUHy>6LCr>! zD1SO^(tjqQgTI4cvIFJ~(2W|hA@_0SMD>%>Cv+29qfbYJ>#O}TnD^M3;ROw8*(U7c zxNEPeZ63=$(2(;cCTO1$P5fN8G8sfS^IDjR)kfxUIa$PFF??3ofy+WV#cp;#ReMRw za^SE~&y!ool5~1DSV~pj%%Id+-N1nIEQ93csAkPPm(-y6f*C`cC6gOp-RqUlTa;5M z^95QWo_Z?XXhRu9m+9-lJ(64gYuX)!TnU*{@j^D&5do+DpyK@b1Vs$YXvB5I$l5*9 zX()4^SqFQpwTv1#NAEs*0qog-_o}L@Dl7hcPH!H$X`9l@EzQ=J7Ai9mTtC)`cOmUo z&dyqAc`IIqWEF2ueEjTxkO^igtMriLcP)LwL)COqECIr;yUq?JvFV7@d z{qiLwHDq0bk_XMn$!QLebXhde*B=0murWiNnaznNARu^GpLNM+7SJ3GWE*ElXD+nz zs3C@HeE5ht8*Vl?H^W*JHlB4use1(C@QZpEjJ^9&aMlLajvW2B{x@h$ul`tFweMPMh~xstn%hY` zyq16XmQK!=vfRNg2PYnQ2+3sNT{NZ+zEs1Mr?nVls&fr1iqoKt;TjYfi7`iu)Q* z!QSkkzfycjhC{oXrXVZ2o$8ww48e3R!h4ozy{u8Y@OEOxgToGLLkfq2idUTerQzzf z!1ZqVD*MFb$PZL4Jt3b4&Z8+y;5ZvO*EgZ^c&+)3P2-j{Gy03~b;R4)UhFtF`g>d$ zE65;9Ec@(vWNzOMES=fjF71>hPBqlk1vyx0;^XM^ELLxtYFxctxD*vNHGD9T2}w5+ z(CEFlP(n;hJW_X!%+##$2Uz3*z+MgA&JTbBMM{%LkLsVrgoZL#z^xjWumJ3C!!I&L z(cfaETsc^&W7-h(NkAKlc<;_|ya$=JA}}y;d`ZSTjP$IvrDa6bI21#?+DVXEAc!G6 z#C^ix+B^qQMyGRHsOCzjs&Mt@H_>emLmIP!>Ba51xU^;523o}zC#g)RBAM>oxh;~w z?Zs3x3l)zV}XrOz1pOu+N4nVuAnzKqFT(Pp`K+Dm><)voq`MAtrzY-ZTak;N`an+r16cB!UHHiRn5xL zpGMttyBV)zzgNFk@8K?CHsjg9X@12~^`4Pt(5*bZ8$4kbUW6D73v-9iZbp)@HduXL z`Z6QVF&Vd8*Lm~(;OSND6c)TlDJMz4)n{dA9Ut|_dlWKGYDP%-Y;{xaOi#ege1Z;@ zgFQuDCim|xJu9lih0z)nA$RvwOXycds^$}-{YDPtchd{_3GJDv6{nWVCBD_g?g*;H z6OBnDGdygCk;xP9QAF#nkj>PsapSFB-Mu-pgM4`F_^bSD-of`XoQNO`P0Z*YP*T(F z_Lorj9Xg~1OIBP$0*dd90kzbI(zDSoAi;qo2IvPo5_+^tCj9j!m^0ZlI0Fuj5f^2l z%z6^MZbyKyyu`#dM;?z|zNx#GpAzoWs?R7{FZtG1rv~4>d$$}9l~-7YugS^FYgZ?*b5)>RF0Fb!_uT2dh~Bb>O+Jp!D899@r|R;x zdCy#35UdDBeO@6O?2LCMzk*c*BS#8gu0yeomXZxrm1lSpO$U}A!*z* zHi*F!{=$xKQz>logZkn6;$HbIgQ&U4J}h&q+SZbMT6i#dv^1JGCE#vW17F4FGhrD- zytz1dUL^CY}&uPc6ZrQy?r%ab?q&)Z$KbaQDhU;6|e z{e=T{`|OBue%6D6Ol1l9B9rJF)AK%6Pybx)Pc=ai0W3lUMYp|_^k@4Br_}jWa@f%3 z1a%AXXD{DQfM4S3s&+eQS>qt4ZZ#IQ1Idd~Ng}~+t@=8m!kFY4AK_K}_3?WDOf={6 z>L{%I^S3e1EFOrYlmK!|Q!=t*?j`i>`YXi?TXuHVT5THjd0hchndm;#u*rT{vw9MY z)!#bXE7p6Vc$Ty8pmAuqt@|oSdBBvXK_!ArMxm?${tt@lF4qb>wI9jH;^Cx%RH8Cq^n5t?;T6Zd zsTVyL!VVPH^A!+Gu*ofF66B`JKry`2+!y%7mPWl}WW7DE3UwLoG#dnZ?A2E}{*k z1ZfuqY4@2MaZ)9QG5QG`S5#;Z1|QCfz9PB9Bw^co<%c__V0b`b8fE(=2Lw!)9S|i* zNJyB*5#!=QmX`Le>Ei4zk9=lDV^gy}C)GBO_C4s;_xbKFd|_>9;f@>o(op&O(@EgBZhdTfn~OdJ z(g)MmQPoQRjsq32#C2_0>0Y_qIP_f`OgXioFp@;+J;UhYt{cjR)D*@pDIF)^XdEiKAnV^5zxef^p|c0Q#Y9K!4AiH?reHZnpj@>r3Ya%yR5 zH88hZiLOVEfvTzi2!aVnOG^t1Mn*(Lynem9yJ5kD25MVmNO0s+)0pcLo|TOaA2)Zz z=#L+W1al4!j>d)t4RH<*c|!{wKz>}g!VHn*=XW^V-vL@*FXiA+MZ6t#3GaPZM@Nki zgox|%JwyE|6>;Y*{hEzCWoudHx6R`_m`tLDi!z1Jo7GQ#{!}nz)_N}ns<=3Q?t3Gb z+Hn3moZX4(qE4H~6LLdM!7+90zEQwc0_BkEW$H;zp9}id?C%a@p{mNXi*zo7=q=#f z=#r}o)#GNVfGf;Zw7nPJaIB$6`@sQD?P6`oJ zF2|b2{DCW~#^VmXo@`$kqkJ%zUj~d-Irp(4Zcjx`sSY{iOx*Xd4BH&z!SnZ$=X47BHl3`@a*an~|@$B%{ODdQb}EWU?k zb;=vn_9Okxm05ACFNzI4HWl!~Og4yp^bq_7_L{AJ0SZAjLzjJ9xKW^v{@zin{kn4) zHMraBFB49D8LghQBo|Pb_;q(uX)8lYd*K-~y5)`~q?y+AIB2hct`|Uj;y#C--rk34 zO7RgGu{WfA>l;x(A96r=1~u5OUbS;{6fi*S?d^q!hx;(ux$X27!mg#o*f~3If?Wcv zIC?Lv%*<57Ghf+4B9ybpHY$CPK}IY#aB)$BhYHLb$k z-CL^ZsemuS!{81a>aku%aurm^7BSqT6ji1SglKt*k$9l(sx*jC|$c zcOUZB@K9sp+(Ym7_DpQOS^40%8iW!TCxRBjmUPC|`${27hXN-R_`oR$F)iltVqhvm zp~#%@u;+yWE(lLG7iG1;SMa*pRmaBuI$WRHMQ08ZxnqR z#s`Cgg8;Qzv!Pk56G9*u<+to?Zo$~Vej!1@Sp1@*vp@=Id-=W@lcc*wKj>hAA2QU| zhVi0-D4{f{2B*fX=4GNgtfgO`n{F?sA~Wrqe`~Z|vQ$lHi}N08T&e?#l8Bl3!@#N&>Nk2 z@Qs?+fRv7^1eKkA4oTIg3E>&V=!67~mjD*W!l5UEqK zaP3Vd<$BjQ!rLpXpAOIZ0*pnWlJcf1g8i&QYrPBiY9=$)cekO2)k5+4Z>=yLkwqs|}V1x-m!2grP_ZES?~ z9M0nxg#*e8;Npj1I)cYZx8E7!i=`+c#0$#Uu2v8iQ`o-F-1v@_pqeNaYnD5W3Ef@A zANFOeHXoBhch}bisL8~r;<5W@yLj-696&)#g)L?TaJK$iljKvrxWteHw!1xKa$=5l zcKMZ@_Pc%G0KY+s-Jj2lUIAGEER+Y&Nzc+TGNORx`1q*9c|p+zUf!~_q;1Sf9E@{p zIPfjF=NSD>^VPMtzwViP8xuns&d9_xUe8uXw)y&Ei5bmxK0_nxp!c2dm5n92QSxfc0Z5W zdn1pGQLe?qMy7Nm*wImSCQk>hQ5_04J7j*MBSNj0t#ob;3buCZ6WXx*eRPd=C7pI9 zh7glr`xX=C-*-y|S};HgoSTcQ(RSya$f_w2Jpln$DdIQVOib?w{9~g~7P}w#9|IwT zAr}N@j&fn!$sOb5!`xDx934Sn)jQpo2z3YT4=`&7#(q^$#%kqSgPip*3*id@htStC zFqmIlJnW24_dGMh*TTc6I%b>ajd@IUpi;MGF0Gt4_CC6wiRh}?DTi$xJdgT_sH>3J zmMUEv8jYGuXrweMadB%ad2`rQvN8Fo)rG3O?y1@ePShAQne5l}4)?i%?uWASW}WJEa*cskp>M-|KmhEx?mQwv)8ihI>8>avO()HM4mEc-qcep=$3$o zIAWv*-r`je1M?o8KN_>QB#C5c3plS@(uzrVOdREz(%XLah1OXB7=i4t6gZ}DrKLe! z7Z-O=*Q&1V@2thf()_w6#*%nv9Ov#bN8EE#VMc!B;I!US+*4W0<_H~_r}l5L*pyt@ z>{&;RPt?mDU=soF{q0$Y`=x`6U;|0EPG)XyVthP5$(cE9h!T+<0&3H!6T0u{0d%5P zj20RSg1SF`cxp4F!oQ@8nqHSSUp~;AJ(z+gfY&=k?c3WVC&1-?#qI7|VNubzeHoA_ zj?eCBYr85g-WVkE<{yUvS3hc`3>i1>knmvQ{_Zu>Xtn7i<3iOpXN;QHBb)A!h5NWP z90S;WuA&>Wm6?|H1-TA6Dbd)C+_?^8@$M`}iZqi>uqiWf{?SE>Y3%IA; zu>`65p^x%n2yI?|B(z;TxO(nA&XA2KG75Y@l7Zjm+$6w6w>2#Y?8iJ#v=_6pVEPRA`e()?3c$XA;v!cVbX}ZAKH+plNq5Hb!zzuIKzS1$ z9i1L1m1qOnS}u#<+bU!TK=7U{1c`}ZVPW0fS~uG!?7zYL?h>V!m&aM)F#yf9pKF2V zDc>j!!HT0sf}Tw}lzgj}QojsBU9s}Dv0{~(isxB#X-@Qm*5UNr-nC(RJIUQ-=y1j= zqkB=_-q3B{NV+9_z=?kZI=Zo6QU-^IeP%m3+0c@blJX&plyu69iVhWnU~?5{21?qE zegR!LuvrVpS%Yn^vca_pvL6$reNb|s_U;E;UCGI2jE#-?yh!}-M*GwwZ<2tA`1WVA z#dyhdYilHz_V~e9Tb3XD`}-?nb)UI#-90?0sfA~2m+W4FDMh-HPork;jrE!vHDfCU zw(~A-A_anCDHBPsuhDv5ZpA&#Ui;4aDE3-z;44zyrX@Wdlj7Y?DLoCh?}r{M(5N}v z?)f4A!jSm%2nk8cn10WP%4`MOJ}o|_hJ|Y9e5;K{51G;pFPAiEsa!#h4zuODE~I|y zdSJdd%jWg^$g#=kC}2PaakLO8Uof-#$+xL?8IJ!MXIDAM2IRFG$!f8*>xfn=kE1=B zEgzBD*ZVQn8qa`irG};^013DC$4iTgwFuUMdgV}!&(hjXir*c(cWpd8JZdTm3So!s zq$QZTVT7m%(?hnu5T5dixq{2L8g#flJSrqGjSRPQzU*SZJ4Dj_s}KrRZy*y1gQaJU zPfP%|avtpEy^WB~wfSBe78-ijz`%23R@3cPRY$#e!}@wrrB!{!%SzGIw6xIz6-;age;vhGP)ToSr^6?5oA>ntx9ALXmDJsM)V;!&A|y7HtC5Rd zx#YU@=ik!D>t_P)irBr}U7KH`^s-gf_HHxN9p$&ahv^Pnw-3@`h;VxqS~7sd4L-MaM6X_D zafpcd0wf(Z@M6D5KmmEq>wggdI(b^wkJl*V&J?|}1p=BtX)QH9y)g}c9dx#Vx@PJC z*jxy7)O-8;1ECZQC$&hd>XezBoE)!lHFrF~4WQ0uy$?$J+XxW1nMTh*OFJPpc4fyC zY+8saKVMksGY(Su_;`JqsUt`^Om&CI?Yukzp$b~F_;aa)*GTWuBnonE2IIl|_b+=< zAWNF{3NLvW=B{QI5uuF6YD`viU7&qWWct}_KWQC>2vci}>x9+5N$VQ*(;Jpfa6n3A zuh9e^Ekx{n?dTePpEdkkg>qtev1GN&!Ph5ow0!M*PFmYQ>-VFzY{lBU`!!P_!ga## zX*QvZ`!w2WJDao%%8q=+SsD$8Q09zg7zbE!4OS#@Y?B}1BB7{W9^Ig}z&B38sXk92 zlkmlGt;u>j;6AwbngmZ@HhQfdXd|H#D__;uL#Xo@6ZWo~%@5}wzLAyDv^lPl@;#qu zErxz(^mD7jG~W-6Ba!PWsA@~WYZl*HU%euV?FJHIkJHF~qoQ9l8O|mbgB+v?;dvus zH~Q&0p_i9eBa81)iA|!wBb|I$A1L%j{=pe0;N$=yAtBfgP+~K@=v-l3=?b*jYv_xg zJp>!Iv(i^h?gPx%cpro6IO4(PK=*G5%`furac=8V%FU;uIC98i75<>KBP54ySDm+Tn8H!jMV}%*%2ubUoGl^B)^Z(WS|fgTNDe-Bw0xT1 ztQH*-vS7%&M0C+4x4$q@WPSLpy|nedAGA?Xvqpeb6#`xq$6d48cqGS$R{Pe}bdH~@ z;{b@m-@(>1`2>`$-o(3>u!qt*V2S3cw7PlS=ZA~0 zcju2*>-8ytobjEyVGlu&xwDdZ9Z9euS>VM8VDIR-YHatu_Yvqdji+bLy=X02jo;ZL zb%$OTd+C5IKEDR2AN%PEc0X=^>n}+t-6HIN)~0a4L19RTO`w7YUS3eiL@(*SzBpJB z&1-lAAbtAIEne9>01g6&!)M&)@P%Ek9aYiL`9ZL*QSl`S#^;Hj3h3sD}0f5r8q50f-re`nK4tZ z4$QZLO3~UtwU&JB^Os*qxXh$JR?g)9JP}9Dmb$w&J42cDnuU*R=&!6T^lspitqRxv^wyfLQlus z4~Oa>Sd#fVD}%x)@JIXVm+xwiqigb-j}R>`sqG#)4O~)&?Tf88D~kG;ODwd6*j&uD z#0U^~R1*dt84Utx#t}?8otpZ1YqWh?W4X(8d696qL|bye zhEKzB7jFnB9uG2L?N(h;Y+G)sDug$T!x!N$Y`kSz@qYOoA9G|tS)^bG9mBui-AP$x zmkqTrkyI?n(yk_-W@;&|4e|Rp;k)$_l&XhY0Vux^s?$kxZ&b$Jj~>3Bt4S*rmxE-H zLM(Ye~R}t9QUEZ_0n%gAyePuQ9J@?v7@52s*{!nxDt*IkW+V(gwN~E zdS*85wNgtA8?3Cqx`3|1JLw7hF6K|nfH6V~wP~(T8H_uotJGd_WIKC%ZFT>X+I2Wz zMO)9ZNo_g~+1z&%pX+d+{`V=D?XXT@)&dp=%l@2fHPq+*ANT&^e}iS9iv=L2yA$DG zAoE{^FsW(XQvg0rHa$F{0de&ccK_)bhHw1A-Ty@Epn~W!6QU(GMKhyMO!YUqBl$V< zZ?H{#@1MKX<@5DEOyT9Lh=~CkEUd2LHN$v0%)$ zniE#|I7(+>@ME%}&hDhq@tc}B3F2Q|#D5#cjyn%bjrrU!+hULi>9h#R?*{V;)`72q zNn`(}i2j^1u!-j1gMUOm{-5mNpXUC_8U4wP{2ZR8I)?Pel+xeB7|8M$*L2ck#CN2} z`1db=!91f{xBtx|n6yS-0d#CxgMvgZ^Ya&m9oe=4&Oz{ZC3!Q#!gtJzb&uu2=E+#oAx*kep>#&44+v3c;IAp`e}KA z03y)1_%D|KZ&Uucpge{C9z6XstU%N}dT`2@{p%}N*XRA`PdY|_#3s!uHV5@&I+Y2& zDvhzN?4dME;?@6*c?_pRz+!CmCu@PZPXd-lUjN+npa1&BmHp?ZFe6W)6gR5-e^V95 zq5LcqbmlPx)k!AX?PG-~;qRFLcVotpu6F&)E3o58Z*&X)3f{l}h2afHCBXmv$ES`h zC%`g}{Ci!!fv-FKr>i9Lu%e%Z<_i74X&Qh0^$0v)_Gg|!G@INs9ABpMeo?F-n*8yL z8W(8n@GcDN;j>(<=AT&k_fN!Qi@m6IvkrmvFL+IDlHI893Ei?DegkV0p_Ef1>X=V_ z`g0ovyDu)*6JosMw#x@1Z1AG{bN?h>aVdRlEUX}g4(^}h!1TdLeJm{K!#V-@$rVn6 z3-rG1z1qx*+17~huv1t1||F{g*)G4edUxUr4P8LcKUc^{ffCD{QM(_o&urB?x zS~+_b=lBMBVqrl(Dac&cz{kR(6;IRsXUSpEF2VZqYPjHHd1Uzy<76@1H-@{f7- zlWRYL{95k8B+mSoIe=9c3-SLy@PFD$FQI%8EG%wb_bQ{Ycx+FwodXN=MTK!V@WuTE zJa_P=1FA91_dG0&lVM@qJB4v-EUXU@jGqC|!67xpe8(dEzkY!aM|e5BMkQL1m(-YF Os35B>gSv6=>Hh$TrNvtS literal 0 HcmV?d00001 diff --git a/blueprints/gke/autopilot/glbs.tf b/blueprints/gke/autopilot/glbs.tf new file mode 100644 index 0000000000..fbcfd7b30d --- /dev/null +++ b/blueprints/gke/autopilot/glbs.tf @@ -0,0 +1,25 @@ +/** + * Copyright 2023 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +locals { + urls = {for k, v in module.addresses.global_addresses: k => "http://${v.address}"} +} + +module "addresses" { + source = "../../../modules/net-address" + project_id = module.project.project_id + global_addresses = ["grafana", "locust", "app"] +} diff --git a/blueprints/gke/autopilot/main.tf b/blueprints/gke/autopilot/main.tf new file mode 100644 index 0000000000..9856dfaa23 --- /dev/null +++ b/blueprints/gke/autopilot/main.tf @@ -0,0 +1,65 @@ +/** + * Copyright 2023 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +module "project" { + source = "../../../modules/project" + billing_account = (var.project_create != null + ? var.project_create.billing_account_id + : null + ) + parent = (var.project_create != null + ? var.project_create.parent + : null + ) + project_create = var.project_create != null + name = var.project_id + services = [ + "artifactregistry.googleapis.com", + "cloudbuild.googleapis.com", + "container.googleapis.com" + ] + iam = { + "roles/monitoring.viewer" = [module.monitoring_sa.iam_email] + "roles/container.nodeServiceAccount" = [module.node_sa.iam_email] + "roles/container.admin" = [module.mgmt_server.service_account_iam_email] + "roles/storage.admin" = [module.mgmt_server.service_account_iam_email] + "roles/cloudbuild.builds.editor" = [module.mgmt_server.service_account_iam_email] + "roles/viewer" = [module.mgmt_server.service_account_iam_email] + } +} + +module "monitoring_sa" { + source = "../../../modules/iam-service-account" + project_id = module.project.project_id + name = "sa-monitoring" + iam = { + "roles/iam.workloadIdentityUser" = [ + "serviceAccount:${module.cluster.workload_identity_pool}[monitoring/frontend]", + "serviceAccount:${module.cluster.workload_identity_pool}[monitoring/custom-metrics-stackdriver-adapter]" + ] + } +} + +module "docker_artifact_registry" { + source = "../../../modules/artifact-registry" + project_id = module.project.project_id + location = var.region + format = "DOCKER" + id = "registry" + iam = { + "roles/artifactregistry.reader" = [module.node_sa.iam_email] + } +} diff --git a/blueprints/gke/autopilot/mgmt.tf b/blueprints/gke/autopilot/mgmt.tf new file mode 100644 index 0000000000..82010dcf95 --- /dev/null +++ b/blueprints/gke/autopilot/mgmt.tf @@ -0,0 +1,35 @@ +/** + * Copyright 2023 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +module "mgmt_server" { + source = "../../../modules/compute-vm" + project_id = module.project.project_id + zone = var.zone + name = "mgmt" + instance_type = var.mgmt_server_config.instance_type + network_interfaces = [{ + network = module.vpc.self_link + subnetwork = module.vpc.subnet_self_links["${var.region}/subnet-mgmt"] + nat = false + addresses = null + }] + service_account_create = true + boot_disk = { + image = var.mgmt_server_config.image + type = var.mgmt_server_config.disk_type + size = var.mgmt_server_config.disk_size + } +} \ No newline at end of file diff --git a/blueprints/gke/autopilot/outputs.tf b/blueprints/gke/autopilot/outputs.tf new file mode 100644 index 0000000000..042ae8679a --- /dev/null +++ b/blueprints/gke/autopilot/outputs.tf @@ -0,0 +1,20 @@ +/** + * Copyright 2023 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + output "urls" { + description = "Grafanam, locust and application URLs." + value = local.urls + } \ No newline at end of file diff --git a/blueprints/gke/autopilot/templates/gssh.sh.tpl b/blueprints/gke/autopilot/templates/gssh.sh.tpl new file mode 100644 index 0000000000..b366231d48 --- /dev/null +++ b/blueprints/gke/autopilot/templates/gssh.sh.tpl @@ -0,0 +1,30 @@ +#!/bin/bash +# +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +host="$${@: -2: 1}" +cmd="$${@: -1: 1}" + +gcloud_args=" +--tunnel-through-iap +--zone=${zone} +--project=${project_id} +--quiet +--no-user-output-enabled +-- +-C +" + +exec gcloud compute ssh "$host" $gcloud_args "$cmd" \ No newline at end of file diff --git a/blueprints/gke/autopilot/test.py b/blueprints/gke/autopilot/test.py new file mode 100644 index 0000000000..bc6ac52383 --- /dev/null +++ b/blueprints/gke/autopilot/test.py @@ -0,0 +1,25 @@ +import os +import json + +stages = [] + +num_stages = int(os.getenv('NUM_STAGES', 20)) +stage_duration = int(os.getenv('STAGE_DURATION', 60)) +spawn_rate = int(os.getenv('SPAWN_RATE', 1)) +new_users_per_stage = int(os.getenv('NEW_USERS_PER_STAGE', 10)) + +for i in range (1, num_stages + 1): + stages.append({ + 'duration': stage_duration * i, + 'users': new_users_per_stage * i, + 'spawn_rate': spawn_rate + }) + +for i in range(1, num_stages): + stages.append({ + 'duration': stage_duration * (num_stages + i), + 'users': new_users_per_stage * (num_stages - i), + 'spawn_rate': spawn_rate + }) + +print(json.dumps(stages)) \ No newline at end of file diff --git a/blueprints/gke/autopilot/variables.tf b/blueprints/gke/autopilot/variables.tf new file mode 100644 index 0000000000..47cbb33eb3 --- /dev/null +++ b/blueprints/gke/autopilot/variables.tf @@ -0,0 +1,83 @@ +/** + * Copyright 2023 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +variable "cluster_network_config" { + description = "Cluster network configuration." + type = object({ + nodes_cidr_block = string + pods_cidr_block = string + services_cidr_block = string + master_authorized_cidr_blocks = map(string) + master_cidr_block = string + }) + default = { + nodes_cidr_block = "10.0.1.0/24" + pods_cidr_block = "172.16.0.0/20" + services_cidr_block = "192.168.0.0/24" + master_authorized_cidr_blocks = { + internal = "10.0.0.0/8" + } + master_cidr_block = "10.0.0.0/28" + } +} + +variable "mgmt_server_config" { + description = "Management server configuration." + type = object({ + disk_size = number + disk_type = string + image = string + instance_type = string + }) + default = { + disk_size = 50 + disk_type = "pd-ssd" + image = "projects/ubuntu-os-cloud/global/images/family/ubuntu-2204-lts" + instance_type = "n1-standard-2" + } +} + +variable "mgmt_subnet_cidr_block" { + description = "Management subnet IP CIDR range." + type = string + default = "10.0.2.0/24" +} + +variable "project_create" { + description = "Parameters for the creation of the new project." + type = object({ + billing_account_id = string + parent = string + }) + default = null +} + +variable "project_id" { + description = "Project ID." + type = string +} + +variable "region" { + description = "Region." + type = string + default = "europe-west1" +} + +variable "zone" { + description = "Zone." + type = string + default = "europe-west1-c" +} diff --git a/blueprints/gke/autopilot/vpc.tf b/blueprints/gke/autopilot/vpc.tf new file mode 100644 index 0000000000..7115f375ff --- /dev/null +++ b/blueprints/gke/autopilot/vpc.tf @@ -0,0 +1,45 @@ +/** + * Copyright 2023 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +module "vpc" { + source = "../../../modules/net-vpc" + project_id = module.project.project_id + name = "vpc" + subnets = [ + { + ip_cidr_range = var.mgmt_subnet_cidr_block + name = "subnet-mgmt" + region = var.region + }, + { + ip_cidr_range = var.cluster_network_config.nodes_cidr_block + name = "subnet-cluster" + region = var.region + secondary_ip_ranges = { + pods = var.cluster_network_config.pods_cidr_block + services = var.cluster_network_config.services_cidr_block + } + } + ] +} + +module "nat" { + source = "../../../modules/net-cloudnat" + project_id = module.project.project_id + region = var.region + name = "nat" + router_network = module.vpc.name +} \ No newline at end of file