Presto Mixin (#1087)

* finished * dakota 1/n * caleb 1/n * added links * added jvm gauges + updated titles * alert fixes * k8 support + emily + gabe feedback * query shortening due to JMX config update * update alerts metrics * Emily 2/n * sample app revisions * screenshots added --------- Co-authored-by: Emily <[email protected]>
grafana · Nov 2, 2023 · 6acda51 · 6acda51
1 parent 356d4fb
commit 6acda51
Show file tree

Hide file tree

Showing 12 changed files with 3,729 additions and 0 deletions.
diff --git a/presto-mixin/.lint b/presto-mixin/.lint
@@ -0,0 +1,17 @@
+exclusions:
+  template-job-rule:
+    reason: "Prometheus datasource variable is being named as prometheus_datasource now while linter expects 'datasource'"
+  panel-datasource-rule:
+    reason: "Loki datasource variable is being named as loki_datasource now while linter expects 'datasource'"
+  template-datasource-rule:
+    reason: "Based on new convention we are using variable names prometheus_datasource and loki_datasource where as linter expects 'datasource'"
+  template-instance-rule:
+    reason: "Based on new convention we are using variable names prometheus_datasource and loki_datasource where as linter expects 'datasource'"
+  target-instance-rule:
+    reason: "The dashboard is a 'cluster' dashboard where the instance refers to nodes, this dashboard focuses only on the cluster view."
+    entries:
+      - dashboard: "Presto overview"
+  panel-title-description-rule:
+    reason: "Not required for logs volume"
+  panel-units-rule:
+    reason: "Logs volume has no unit"
diff --git a/presto-mixin/Makefile b/presto-mixin/Makefile
@@ -0,0 +1,34 @@
+JSONNET_FMT := jsonnetfmt -n 2 --max-blank-lines 1 --string-style s --comment-style s
+
+.PHONY: all
+all: build dashboards_out prometheus_alerts.yaml
+
+vendor: jsonnetfile.json
+	jb install
+
+.PHONY: build
+build: vendor
+
+.PHONY: fmt
+fmt:
+	find . -name 'vendor' -prune -o -name '*.libsonnet' -print -o -name '*.jsonnet' -print | \
+		xargs -n 1 -- $(JSONNET_FMT) -i
+
+.PHONY: lint
+lint: build
+	find . -name 'vendor' -prune -o -name '*.libsonnet' -print -o -name '*.jsonnet' -print | \
+		while read f; do \
+			$(JSONNET_FMT) "$$f" | diff -u "$$f" -; \
+		done
+	mixtool lint mixin.libsonnet
+
+dashboards_out: mixin.libsonnet config.libsonnet $(wildcard dashboards/*)
+	@mkdir -p dashboards_out
+	mixtool generate dashboards mixin.libsonnet -d dashboards_out
+
+prometheus_alerts.yaml: mixin.libsonnet alerts/*.libsonnet
+	mixtool generate alerts mixin.libsonnet -a prometheus_alerts.yaml
+
+.PHONY: clean
+clean:
+	rm -rf dashboards_out prometheus_alerts.yaml
diff --git a/presto-mixin/README.md b/presto-mixin/README.md
@@ -0,0 +1,132 @@
+# Presto mixin
+
+The Presto mixin is a set of configurable Grafana dashboards and alerts.
+
+The Presto mixin contains the following dashboards:
+
+- Presto overview
+- Presto coordinator
+- Presto worker
+- Presto logs
+
+and the following alerts:
+
+- PrestoHighInsufficientResources
+- PrestoHighTaskFailuresWarning
+- PrestoHighTaskFailuresCritical
+- PrestoHighQueuedTaskCount
+- PrestoHighBlockedNodes
+- PrestoHighFailedQueriesWarning
+- PrestoHighFailedQueriesCritical
+
+## Presto overview
+
+The Presto overview dashboard provides details on integration status/alerts, workers/coordinators, error failures, data throughput, blocked nodes, and distributed bytes.
+![Presto overview dashboard (queries)](https://storage.googleapis.com/grafanalabs-integration-assets/presto/screenshots/presto_overview_1.png)
+![Presto overview dashboard (processing)](https://storage.googleapis.com/grafanalabs-integration-assets/presto/screenshots/presto_overview_1.png)
+
+## Presto coordinator overview
+
+The Presto coordinator overview dashboard provides details on various query counts and rates, query execution time, CPU time consumed, CPU input throughput, error failures, JVM metrics, and memory pool information.
+![Presto coordinator dashboard (queries)](https://storage.googleapis.com/grafanalabs-integration-assets/presto/screenshots/presto_coordinator_1.png)
+![Presto coordinator dashboard (JVM)](https://storage.googleapis.com/grafanalabs-integration-assets/presto/screenshots/presto_coordinator_2.png)
+
+## Presto worker overview
+
+The Presto worker overview dashboard provides details on various task rates, pool sizes, output positions, data throughput, JVM metrics, and memory pool information.
+![Presto worker dashboard (tasks)](https://storage.googleapis.com/grafanalabs-integration-assets/presto/screenshots/presto_worker_1.png)
+![Presto worker dashboard (JVM)](https://storage.googleapis.com/grafanalabs-integration-assets/presto/screenshots/presto_worker_2.png)
+
+## Presto logs
+
+The Presto logs dashboard provides details on incoming system logs.
+![Presto logs dashboard](https://storage.googleapis.com/grafanalabs-integration-assets/presto/screenshots/presto_logs_overview.png)
+
+Presto system logs are enabled by default in the `config.libsonnet` and can be removed by setting `enableLokiLogs` to `false`. Then run `make` again to regenerate the dashboard:
+
+```
+{
+  _config+:: {
+    enableLokiLogs: false,
+  },
+}
+```
+
+In order for the selectors to properly work for system logs ingested into your logs datasource, please also include the matching `instance`, `job`, and `presto_cluster` labels onto the [scrape configs](https://grafana.com/docs/loki/latest/clients/promtail/configuration/#scrape_configs) as to match the labels for ingested metrics.
+
+```yaml
+scrape_configs:
+  - job_name: integrations/presto
+    static_configs:
+      - targets: [localhost]
+        labels:
+          job: integrations/presto
+          instance: "<your-instance-name>"
+          presto_cluster: "<your-cluster-name>"
+          __path__: /var/presto/logs/*.log
+    pipeline_stages:
+      - multiline:
+          firstline: '\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{3}'
+      - regex:
+          expression: '\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{3}Z\s+(?P<level>\w+)(?P<message>.+)'
+      - labels:
+          level:
+```
+
+## Alerts overview
+
+- PrestoHighInsufficientFailures: The amount of failures that are occurring due to insufficient resources are scaling, causing saturation in the system.
+- PrestoHighTaskFailuresWarning: The amount of tasks that are failing is increasing, this might affect query processing and could result in incomplete or incorrect results.
+- PrestoHighTaskFailuresCritical: The amount of tasks that are failing has reached a critical level. This might affect query processing and could result in incomplete or incorrect results.
+- PrestoHighQueuedTaskCount: The amount of tasks that are being put in queue is increasing. A high number of queued tasks can lead to increased query latencies and degraded system performance.
+- PrestoHighBlockedNodes: The amount of nodes that are blocked due to memory restrictions is increasing. Blocked nodes can cause performance degradation and resource starvation.
+- PrestoHighFailedQueriesWarning: The amount of queries failing is increasing. Failed queries can prevent users from accessing data, disrupt analytics processes, and might indicate underlying issues with the system or data.
+- PrestoHighFailedQueriesCritical: The amount of queries failing has increased to critical levels. Failed queries can prevent users from accessing data, disrupt analytics processes, and might indicate underlying issues with the system or data.
+
+Default thresholds can be configured in `config.libsonnet`.
+
+```js
+{
+    _configs+:: {
+
+        // alerts thresholds
+        alertsHighInsufficientResourceErrors: 0, // count
+        alertsHighTaskFailuresWarning: 0, // count
+        alertsHighTaskFailuresCritical: 30, // percent
+        alertsHighQueuedTaskCount: 5, // count
+        alertsHighBlockedNodesCount: 0, // count
+        alertsHighFailedQueryCountWarning: 0, // count
+        alertsHighFailedQueryCountCritical: 30, // percent
+    }
+}
+```
+
+## Install tools
+
+```bash
+go install github.com/jsonnet-bundler/jsonnet-bundler/cmd/jb@latest
+go install github.com/monitoring-mixins/mixtool/cmd/mixtool@latest
+```
+
+For linting and formatting, you would also need `jsonnetfmt` installed. If you
+have a working Go development environment, it's easiest to run the following:
+
+```bash
+go install github.com/google/go-jsonnet/cmd/jsonnetfmt@latest
+```
+
+The files in `dashboards_out` need to be imported
+into your Grafana server. The exact details will be depending on your environment.
+
+`prometheus_alerts.yaml` needs to be imported into Prometheus.
+
+## Generate dashboards and alerts
+
+Edit `config.libsonnet` if required and then build JSON dashboard files for Grafana:
+
+```bash
+make
+```
+
+For more advanced uses of mixins, see
+https://github.com/monitoring-mixins/docs.
diff --git a/presto-mixin/alerts/alerts.libsonnet b/presto-mixin/alerts/alerts.libsonnet
@@ -0,0 +1,130 @@
+{
+  prometheusAlerts+:: {
+    groups+: [
+      {
+        name: 'presto-alerts',
+        rules: [
+          {
+            alert: 'PrestoHighInsufficientResources',
+            expr: |||
+              increase(presto_QueryManager_InsufficientResourcesFailures_TotalCount[5m]) > %(alertsHighInsufficientResourceErrors)s
+            ||| % $._config,
+            'for': '5m',
+            labels: {
+              severity: 'critical',
+            },
+            annotations: {
+              summary: 'The amount of failures that are occurring due to insufficient resources are scaling, causing saturation in the system.',
+              description:
+                (
+                  'The number of insufficient resource failures on {{$labels.instance}} is {{ printf "%%.0f" $value }} which is greater than the threshold of %(alertsHighInsufficientResourceErrors)s.'
+                ) % $._config,
+            },
+          },
+          {
+            alert: 'PrestoHighTaskFailuresWarning',
+            expr: |||
+              increase(presto_TaskManager_FailedTasks_TotalCount[5m]) > %(alertsHighTaskFailuresWarning)s
+            ||| % $._config,
+            'for': '5m',
+            labels: {
+              severity: 'warning',
+            },
+            annotations: {
+              summary: 'The amount of tasks that are failing is increasing, this might affect query processing and could result in incomplete or incorrect results.',
+              description:
+                (
+                  'The number of task failures on {{$labels.instance}} is {{ printf "%%.0f" $value }} which is above the threshold of %(alertsHighTaskFailuresWarning)s.'
+                ) % $._config,
+            },
+          },
+          {
+            alert: 'PrestoHighTaskFailuresCritical',
+            expr: |||
+              increase(presto_TaskManager_FailedTasks_TotalCount[5m]) / clamp_min(increase(presto_TaskManager_FailedTasks_TotalCount[10m]), 1) * 100 > %(alertsHighTaskFailuresCritical)s
+            ||| % $._config,
+            'for': '5m',
+            labels: {
+              severity: 'critical',
+            },
+            annotations: {
+              summary: 'The amount of tasks that are failing has reached a critical level. This might affect query processing and could result in incomplete or incorrect results.',
+              description:
+                (
+                  'The number of task failures on {{$labels.instance}} is {{ printf "%%.0f" $value }} which is above the threshold of %(alertsHighTaskFailuresCritical)s%%s.'
+                ) % $._config,
+            },
+          },
+          {
+            alert: 'PrestoHighQueuedTaskCount',
+            expr: |||
+              increase(presto_QueryExecution_Executor_QueuedTaskCount[5m]) > %(alertsHighQueuedTaskCount)s
+            ||| % $._config,
+            'for': '5m',
+            labels: {
+              severity: 'warning',
+            },
+            annotations: {
+              summary: 'The amount of tasks that are being put in queue is increasing. A high number of queued tasks can lead to increased query latencies and degraded system performance.',
+              description:
+                (
+                  'The number of queued tasks on {{$labels.instance}} is {{ printf "%%.0f" $value }} which is greater than the threshold of %(alertsHighQueuedTaskCount)s'
+                ) % $._config,
+            },
+          },
+          {
+            alert: 'PrestoHighBlockedNodes',
+            expr: |||
+              increase(presto_ClusterMemoryPool_general_BlockedNodes[5m]) > %(alertsHighBlockedNodesCount)s
+            ||| % $._config,
+            'for': '5m',
+            labels: {
+              severity: 'critical',
+            },
+            annotations: {
+              summary: 'The amount of nodes that are blocked due to memory restrictions is increasing. Blocked nodes can cause performance degradation and resource starvation.',
+              description:
+                (
+                  'The number of blocked nodes on {{$labels.instance}} is {{ printf "%%.0f" $value }} which is greater than the threshold of %(alertsHighBlockedNodesCount)s'
+                ) % $._config,
+            },
+          },
+          {
+            alert: 'PrestoHighFailedQueriesWarning',
+            expr: |||
+              increase(presto_QueryManager_FailedQueries_TotalCount[5m]) > %(alertsHighFailedQueryCountWarning)s
+            ||| % $._config,
+            'for': '5m',
+            labels: {
+              severity: 'warning',
+            },
+            annotations: {
+              summary: 'The amount of queries failing is increasing. Failed queries can prevent users from accessing data, disrupt analytics processes, and might indicate underlying issues with the system or data.',
+              description:
+                (
+                  'The number of failed queries on {{$labels.instance}} is {{ printf "%%.0f" $value }} which is greater than the threshold of %(alertsHighFailedQueryCountWarning)s'
+                ) % $._config,
+            },
+          },
+          {
+            alert: 'PrestoHighFailedQueriesCritical',
+            expr: |||
+              increase(presto_QueryManager_FailedQueries_TotalCount[5m]) / clamp_min(increase(presto_QueryManager_FailedQueries_TotalCount[10m]), 1) * 100 > %(alertsHighFailedQueryCountCritical)s
+            ||| % $._config,
+            'for': '5m',
+            labels: {
+              severity: 'critical',
+            },
+            annotations: {
+              summary: 'The amount of queries failing has increased to critical levels. Failed queries can prevent users from accessing data, disrupt analytics processes, and might indicate underlying issues with the system or data.',
+              description:
+                (
+                  'The number of failed queries on {{$labels.instance}} is {{ printf "%%.0f" $value }} which is greater than the threshold of %(alertsHighFailedQueryCountCritical)s%%s.'
+                ) % $._config,
+            },
+          },
+        ],
+      },
+    ],
+  },
+}
diff --git a/presto-mixin/config.libsonnet b/presto-mixin/config.libsonnet
@@ -0,0 +1,26 @@
+{
+  _config+:: {
+    enableMultiCluster: false,
+    prestoOverviewSelector: if self.enableMultiCluster then 'job=~"$job", cluster=~"$cluster"' else 'job=~"$job"',
+    prestoSelector: if self.enableMultiCluster then 'job=~"$job", instance=~"$instance", cluster=~"$cluster"' else 'job=~"$job", instance=~"$instance"',
+    prestoAlertSelector: if self.enableMultiCluster then 'job=~"${job:regex}", cluster=~"${cluster:regex}"' else 'job=~"${job:regex}"',
+    prestoOverviewLegendSelector: if self.enableMultiCluster then '{{cluster}} - {{presto_cluster}}' else '{{presto_cluster}}',
+    prestoLegendSelector: if self.enableMultiCluster then '{{cluster}} - {{instance}}' else '{{instance}}',
+    filterSelector: 'job=~"integrations/presto"',
+
+    dashboardTags: ['presto-mixin'],
+    dashboardPeriod: 'now-30m',
+    dashboardTimezone: 'default',
+    dashboardRefresh: '1m',
+
+    // alerts thresholds
+    alertsHighInsufficientResourceErrors: 0,  // count
+    alertsHighTaskFailuresWarning: 0,  // count
+    alertsHighTaskFailuresCritical: 30,  // percent
+    alertsHighQueuedTaskCount: 5,  // count
+    alertsHighBlockedNodesCount: 0,  // count
+    alertsHighFailedQueryCountWarning: 0,  // count
+    alertsHighFailedQueryCountCritical: 30,  // percent
+    enableLokiLogs: true,
+  },
+}
diff --git a/presto-mixin/dashboards/dashboards.libsonnet b/presto-mixin/dashboards/dashboards.libsonnet
@@ -0,0 +1,4 @@
+(import 'presto-overview.libsonnet') +
+(import 'presto-coordinator.libsonnet') +
+(import 'presto-worker.libsonnet') +
+(import 'presto-logs-overview.libsonnet')