From 2c5b0dc6569edecafe78cd53c44b8875926b6a38 Mon Sep 17 00:00:00 2001 From: Charlie Le Date: Fri, 25 Oct 2024 14:20:14 -0700 Subject: [PATCH] update cortex dashboards pulls in latest changes from cortex-jsonnet to update the dashboards in Grafana. also updates the alerting and recording rules. Signed-off-by: Charlie Le --- docs/getting-started/alerts.yaml | 4 +- .../dashboards/alertmanager.json | 2789 ++++----- .../cortex-compactor-resources.json | 615 +- .../dashboards/cortex-compactor.json | 1551 ++--- .../dashboards/cortex-config.json | 186 +- .../dashboards/cortex-object-store.json | 880 ++- .../dashboards/cortex-queries.json | 3042 ++++------ .../dashboards/cortex-reads.json | 5148 +++++++++-------- .../dashboards/cortex-rollout-progress.json | 770 +-- .../dashboards/cortex-scaling.json | 94 +- .../dashboards/cortex-slow-queries.json | 2 +- .../dashboards/cortex-writes.json | 2944 +++++----- docs/getting-started/dashboards/ruler.json | 3196 +++++----- docs/getting-started/rules.yaml | 4 +- 14 files changed, 9431 insertions(+), 11794 deletions(-) diff --git a/docs/getting-started/alerts.yaml b/docs/getting-started/alerts.yaml index cddd1f57da..bfb33b0345 100644 --- a/docs/getting-started/alerts.yaml +++ b/docs/getting-started/alerts.yaml @@ -245,7 +245,7 @@ groups: message: | The number of in-memory series per ingester in {{ $labels.cluster }}/{{ $labels.namespace }} is too high. expr: | - avg by (cluster, namespace) (cortex_ingester_memory_series) > 1.6e6 + avg by (cluster, namespace) (cortex_ingester_memory_series) > 3.2e6 for: 2h labels: severity: warning @@ -254,7 +254,7 @@ groups: message: | Ingesters in {{ $labels.cluster }}/{{ $labels.namespace }} ingest too many samples per second. expr: | - avg by (cluster, namespace) (rate(cortex_ingester_ingested_samples_total[1m])) > 80e3 + avg by (cluster, namespace) (rate(cortex_ingester_ingested_samples_total[1m])) > 160e3 for: 15m labels: severity: warning diff --git a/docs/getting-started/dashboards/alertmanager.json b/docs/getting-started/dashboards/alertmanager.json index b9bf3cd133..3d61215caa 100644 --- a/docs/getting-started/dashboards/alertmanager.json +++ b/docs/getting-started/dashboards/alertmanager.json @@ -27,232 +27,151 @@ "height": "100px", "panels": [ { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", - "fill": 1, + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 1, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "short" + }, + "overrides": [ ] + }, "format": "short", "id": 1, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, "links": [ ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 4, - "stack": false, - "steppedLine": false, "targets": [ { "expr": "sum(cluster_job_pod:cortex_alertmanager_alerts:sum{job=~\"$job\"})", "format": "time_series", "instant": true, - "intervalFactor": 2, "refId": "A" } ], "thresholds": "70,80", - "timeFrom": null, - "timeShift": null, "title": "Total Alerts", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "singlestat", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] + "type": "singlestat" }, { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", - "fill": 1, + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 1, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "short" + }, + "overrides": [ ] + }, "format": "short", "id": 2, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, "links": [ ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 4, - "stack": false, - "steppedLine": false, "targets": [ { "expr": "sum(cluster_job_pod:cortex_alertmanager_silences:sum{job=~\"$job\"})", "format": "time_series", "instant": true, - "intervalFactor": 2, "refId": "A" } ], "thresholds": "70,80", - "timeFrom": null, - "timeShift": null, "title": "Total Silences", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "singlestat", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] + "type": "singlestat" }, { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", - "fill": 1, + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 1, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "short" + }, + "overrides": [ ] + }, "format": "short", "id": 3, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, "links": [ ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 4, - "stack": false, - "steppedLine": false, "targets": [ { "expr": "max(cortex_alertmanager_tenants_discovered{job=~\"$job\"})", "format": "time_series", "instant": true, - "intervalFactor": 2, "refId": "A" } ], "thresholds": "70,80", - "timeFrom": null, - "timeShift": null, "title": "Tenants", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "singlestat", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] + "type": "singlestat" } ], "repeat": null, @@ -267,90 +186,59 @@ "height": "250px", "panels": [ { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", - "fill": 1, - "id": 4, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 1, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "short" + }, + "overrides": [ ] }, - "lines": true, - "linewidth": 1, + "id": 4, "links": [ ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 12, - "stack": false, - "steppedLine": false, "targets": [ { "expr": "sum(cluster_job:cortex_alertmanager_alerts_received_total:rate5m{job=~\"$job\"})\n-\nsum(cluster_job:cortex_alertmanager_alerts_invalid_total:rate5m{job=~\"$job\"})\n", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "success", - "legendLink": null, - "step": 10 + "legendLink": null }, { "expr": "sum(cluster_job:cortex_alertmanager_alerts_invalid_total:rate5m{job=~\"$job\"})", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "failed", - "legendLink": null, - "step": 10 + "legendLink": null } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, "title": "APS", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] + "type": "timeseries" } ], "repeat": null, @@ -365,253 +253,177 @@ "height": "250px", "panels": [ { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", - "fill": 1, - "id": 5, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 1, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "short" + }, + "overrides": [ ] }, - "lines": true, - "linewidth": 1, + "id": 5, "links": [ ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 4, - "stack": false, - "steppedLine": false, "targets": [ { "expr": "sum(cluster_job_integration:cortex_alertmanager_notifications_total:rate5m{job=~\"$job\"})\n-\nsum(cluster_job_integration:cortex_alertmanager_notifications_failed_total:rate5m{job=~\"$job\"})\n", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "success", - "legendLink": null, - "step": 10 + "legendLink": null }, { "expr": "sum(cluster_job_integration:cortex_alertmanager_notifications_failed_total:rate5m{job=~\"$job\"})", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "failed", - "legendLink": null, - "step": 10 + "legendLink": null } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, "title": "NPS", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] + "type": "timeseries" }, { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", - "fill": 1, - "id": 6, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 1, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "short" + }, + "overrides": [ ] }, - "lines": true, - "linewidth": 1, + "id": 6, "links": [ ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 4, - "stack": false, - "steppedLine": false, "targets": [ { "expr": "(\nsum(cluster_job_integration:cortex_alertmanager_notifications_total:rate5m{job=~\"$job\"}) by(integration)\n-\nsum(cluster_job_integration:cortex_alertmanager_notifications_failed_total:rate5m{job=~\"$job\"}) by(integration)\n) > 0\nor on () vector(0)\n", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "success - {{ integration }}", - "legendLink": null, - "step": 10 + "legendLink": null }, { "expr": "sum(cluster_job_integration:cortex_alertmanager_notifications_failed_total:rate5m{job=~\"$job\"}) by(integration)", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "failed - {{ integration }}", - "legendLink": null, - "step": 10 + "legendLink": null } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, "title": "NPS by integration", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] + "type": "timeseries" }, { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", - "fill": 1, - "id": 7, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 1, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "ms" + }, + "overrides": [ ] }, - "lines": true, - "linewidth": 1, + "id": 7, "links": [ ], "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 4, - "stack": false, - "steppedLine": false, "targets": [ { "expr": "histogram_quantile(0.99, sum(rate(cortex_alertmanager_notification_latency_seconds_bucket{job=~\"$job\"}[$__rate_interval])) by (le)) * 1e3", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "99th Percentile", - "refId": "A", - "step": 10 + "refId": "A" }, { "expr": "histogram_quantile(0.50, sum(rate(cortex_alertmanager_notification_latency_seconds_bucket{job=~\"$job\"}[$__rate_interval])) by (le)) * 1e3", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "50th Percentile", - "refId": "B", - "step": 10 + "refId": "B" }, { "expr": "sum(rate(cortex_alertmanager_notification_latency_seconds_sum{job=~\"$job\"}[$__rate_interval])) * 1e3 / sum(rate(cortex_alertmanager_notification_latency_seconds_count{job=~\"$job\"}[$__rate_interval]))", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "Average", - "refId": "C", - "step": 10 + "refId": "C" } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, "title": "Latency", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, + "type": "timeseries", "yaxes": [ { "format": "ms", @@ -650,157 +462,257 @@ "3xx": "#6ED0E0", "4xx": "#EF843C", "5xx": "#E24D42", + "OK": "#7EB26D", + "cancel": "#A9A9A9", "error": "#E24D42", "success": "#7EB26D" }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 100, + "lineWidth": 0, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + } + }, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "reqps" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "1xx" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EAB839", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "2xx" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#7EB26D", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "3xx" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#6ED0E0", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "4xx" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EF843C", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "5xx" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E24D42", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "OK" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#7EB26D", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "cancel" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#A9A9A9", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "error" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E24D42", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "success" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#7EB26D", + "mode": "fixed" + } + } + ] + } + ] + }, "fill": 10, "id": 8, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, "linewidth": 0, "links": [ ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 6, "stack": true, - "steppedLine": false, "targets": [ { - "expr": "sum by (status) (\n label_replace(label_replace(rate(cortex_request_duration_seconds_count{job=~\"$job\", route=~\"api_v1_alerts|alertmanager\"}[$__rate_interval]),\n \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"),\n \"status\", \"${1}\", \"status_code\", \"([a-z]+)\"))\n", + "expr": "sum by (status) (\n label_replace(label_replace(rate(cortex_request_duration_seconds_count{job=~\"$job\", route=~\"api_v1_alerts|alertmanager\"}[$__rate_interval]),\n \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"),\n \"status\", \"${1}\", \"status_code\", \"([a-zA-Z]+)\"))\n", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "{{status}}", - "refId": "A", - "step": 10 + "refId": "A" } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, "title": "QPS", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] + "type": "timeseries" }, { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", - "fill": 1, - "id": 9, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 1, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "ms" + }, + "overrides": [ ] }, - "lines": true, - "linewidth": 1, + "id": 9, "links": [ ], "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 6, - "stack": false, - "steppedLine": false, "targets": [ { "expr": "histogram_quantile(0.99, sum by (le) (cluster_job_route:cortex_request_duration_seconds_bucket:sum_rate{job=~\"$job\", route=~\"api_v1_alerts|alertmanager\"})) * 1e3", "format": "time_series", - "intervalFactor": 2, - "legendFormat": "99th Percentile", - "refId": "A", - "step": 10 + "legendFormat": "99th percentile", + "refId": "A" }, { "expr": "histogram_quantile(0.50, sum by (le) (cluster_job_route:cortex_request_duration_seconds_bucket:sum_rate{job=~\"$job\", route=~\"api_v1_alerts|alertmanager\"})) * 1e3", "format": "time_series", - "intervalFactor": 2, - "legendFormat": "50th Percentile", - "refId": "B", - "step": 10 + "legendFormat": "50th percentile", + "refId": "B" }, { "expr": "1e3 * sum(cluster_job_route:cortex_request_duration_seconds_sum:sum_rate{job=~\"$job\", route=~\"api_v1_alerts|alertmanager\"}) / sum(cluster_job_route:cortex_request_duration_seconds_count:sum_rate{job=~\"$job\", route=~\"api_v1_alerts|alertmanager\"})", "format": "time_series", - "intervalFactor": 2, "legendFormat": "Average", - "refId": "C", - "step": 10 + "refId": "C" } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, "title": "Latency", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, + "type": "timeseries", "yaxes": [ { "format": "ms", @@ -833,140 +745,103 @@ "height": "250px", "panels": [ { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 1, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "rps" + }, + "overrides": [ ] + }, "fill": 10, "id": 10, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, "linewidth": 0, "links": [ ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 3, "stack": true, - "steppedLine": false, "targets": [ { "expr": "sum by(operation) (rate(thanos_objstore_bucket_operations_total{job=~\"$job\",component=\"alertmanager-storage\"}[$__rate_interval]))", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "{{operation}}", - "legendLink": null, - "step": 10 + "legendLink": null } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, "title": "Operations / sec", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, - "yaxes": [ - { - "format": "rps", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] + "type": "timeseries" }, { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", - "fill": 1, - "id": 11, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 1, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "percentunit" + }, + "overrides": [ ] }, - "lines": true, - "linewidth": 1, + "id": 11, "links": [ ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 3, - "stack": false, - "steppedLine": false, "targets": [ { "expr": "sum by(operation) (rate(thanos_objstore_bucket_operation_failures_total{job=~\"$job\",component=\"alertmanager-storage\"}[$__rate_interval])) / sum by(operation) (rate(thanos_objstore_bucket_operations_total{job=~\"$job\",component=\"alertmanager-storage\"}[$__rate_interval]))", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "{{operation}}", - "legendLink": null, - "step": 10 + "legendLink": null } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, "title": "Error rate", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, + "type": "timeseries", "yaxes": [ { "format": "percentunit", @@ -987,81 +862,67 @@ ] }, { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", - "fill": 1, - "id": 12, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 1, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "short" + }, + "overrides": [ ] }, - "lines": true, - "linewidth": 1, + "id": 12, "links": [ ], "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 3, - "stack": false, - "steppedLine": false, "targets": [ { "expr": "histogram_quantile(0.99, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{job=~\"$job\",component=\"alertmanager-storage\",operation=\"attributes\"}[$__rate_interval])) by (le)) * 1e3", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "99th Percentile", - "refId": "A", - "step": 10 + "refId": "A" }, { "expr": "histogram_quantile(0.50, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{job=~\"$job\",component=\"alertmanager-storage\",operation=\"attributes\"}[$__rate_interval])) by (le)) * 1e3", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "50th Percentile", - "refId": "B", - "step": 10 + "refId": "B" }, { "expr": "sum(rate(thanos_objstore_bucket_operation_duration_seconds_sum{job=~\"$job\",component=\"alertmanager-storage\",operation=\"attributes\"}[$__rate_interval])) * 1e3 / sum(rate(thanos_objstore_bucket_operation_duration_seconds_count{job=~\"$job\",component=\"alertmanager-storage\",operation=\"attributes\"}[$__rate_interval]))", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "Average", - "refId": "C", - "step": 10 + "refId": "C" } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, "title": "Latency of Op: Attributes", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, + "type": "timeseries", "yaxes": [ { "format": "ms", @@ -1082,81 +943,67 @@ ] }, { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", - "fill": 1, - "id": 13, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 1, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "short" + }, + "overrides": [ ] }, - "lines": true, - "linewidth": 1, + "id": 13, "links": [ ], "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 3, - "stack": false, - "steppedLine": false, "targets": [ { "expr": "histogram_quantile(0.99, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{job=~\"$job\",component=\"alertmanager-storage\",operation=\"exists\"}[$__rate_interval])) by (le)) * 1e3", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "99th Percentile", - "refId": "A", - "step": 10 + "refId": "A" }, { "expr": "histogram_quantile(0.50, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{job=~\"$job\",component=\"alertmanager-storage\",operation=\"exists\"}[$__rate_interval])) by (le)) * 1e3", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "50th Percentile", - "refId": "B", - "step": 10 + "refId": "B" }, { "expr": "sum(rate(thanos_objstore_bucket_operation_duration_seconds_sum{job=~\"$job\",component=\"alertmanager-storage\",operation=\"exists\"}[$__rate_interval])) * 1e3 / sum(rate(thanos_objstore_bucket_operation_duration_seconds_count{job=~\"$job\",component=\"alertmanager-storage\",operation=\"exists\"}[$__rate_interval]))", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "Average", - "refId": "C", - "step": 10 + "refId": "C" } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, "title": "Latency of Op: Exists", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, + "type": "timeseries", "yaxes": [ { "format": "ms", @@ -1189,81 +1036,67 @@ "height": "250px", "panels": [ { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", - "fill": 1, - "id": 14, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 1, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "short" + }, + "overrides": [ ] }, - "lines": true, - "linewidth": 1, + "id": 14, "links": [ ], "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 3, - "stack": false, - "steppedLine": false, "targets": [ { "expr": "histogram_quantile(0.99, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{job=~\"$job\",component=\"alertmanager-storage\",operation=\"get\"}[$__rate_interval])) by (le)) * 1e3", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "99th Percentile", - "refId": "A", - "step": 10 + "refId": "A" }, { "expr": "histogram_quantile(0.50, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{job=~\"$job\",component=\"alertmanager-storage\",operation=\"get\"}[$__rate_interval])) by (le)) * 1e3", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "50th Percentile", - "refId": "B", - "step": 10 + "refId": "B" }, { "expr": "sum(rate(thanos_objstore_bucket_operation_duration_seconds_sum{job=~\"$job\",component=\"alertmanager-storage\",operation=\"get\"}[$__rate_interval])) * 1e3 / sum(rate(thanos_objstore_bucket_operation_duration_seconds_count{job=~\"$job\",component=\"alertmanager-storage\",operation=\"get\"}[$__rate_interval]))", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "Average", - "refId": "C", - "step": 10 + "refId": "C" } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, "title": "Latency of Op: Get", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, + "type": "timeseries", "yaxes": [ { "format": "ms", @@ -1284,81 +1117,67 @@ ] }, { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", - "fill": 1, - "id": 15, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 1, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "short" + }, + "overrides": [ ] }, - "lines": true, - "linewidth": 1, + "id": 15, "links": [ ], "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 3, - "stack": false, - "steppedLine": false, "targets": [ { "expr": "histogram_quantile(0.99, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{job=~\"$job\",component=\"alertmanager-storage\",operation=\"get_range\"}[$__rate_interval])) by (le)) * 1e3", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "99th Percentile", - "refId": "A", - "step": 10 + "refId": "A" }, { "expr": "histogram_quantile(0.50, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{job=~\"$job\",component=\"alertmanager-storage\",operation=\"get_range\"}[$__rate_interval])) by (le)) * 1e3", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "50th Percentile", - "refId": "B", - "step": 10 + "refId": "B" }, { "expr": "sum(rate(thanos_objstore_bucket_operation_duration_seconds_sum{job=~\"$job\",component=\"alertmanager-storage\",operation=\"get_range\"}[$__rate_interval])) * 1e3 / sum(rate(thanos_objstore_bucket_operation_duration_seconds_count{job=~\"$job\",component=\"alertmanager-storage\",operation=\"get_range\"}[$__rate_interval]))", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "Average", - "refId": "C", - "step": 10 + "refId": "C" } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, "title": "Latency of Op: GetRange", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, + "type": "timeseries", "yaxes": [ { "format": "ms", @@ -1379,81 +1198,67 @@ ] }, { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", - "fill": 1, - "id": 16, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 1, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "short" + }, + "overrides": [ ] }, - "lines": true, - "linewidth": 1, + "id": 16, "links": [ ], "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 3, - "stack": false, - "steppedLine": false, "targets": [ { "expr": "histogram_quantile(0.99, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{job=~\"$job\",component=\"alertmanager-storage\",operation=\"upload\"}[$__rate_interval])) by (le)) * 1e3", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "99th Percentile", - "refId": "A", - "step": 10 + "refId": "A" }, { "expr": "histogram_quantile(0.50, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{job=~\"$job\",component=\"alertmanager-storage\",operation=\"upload\"}[$__rate_interval])) by (le)) * 1e3", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "50th Percentile", - "refId": "B", - "step": 10 + "refId": "B" }, { "expr": "sum(rate(thanos_objstore_bucket_operation_duration_seconds_sum{job=~\"$job\",component=\"alertmanager-storage\",operation=\"upload\"}[$__rate_interval])) * 1e3 / sum(rate(thanos_objstore_bucket_operation_duration_seconds_count{job=~\"$job\",component=\"alertmanager-storage\",operation=\"upload\"}[$__rate_interval]))", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "Average", - "refId": "C", - "step": 10 + "refId": "C" } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, "title": "Latency of Op: Upload", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, + "type": "timeseries", "yaxes": [ { "format": "ms", @@ -1474,81 +1279,67 @@ ] }, { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", - "fill": 1, - "id": 17, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 1, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "short" + }, + "overrides": [ ] }, - "lines": true, - "linewidth": 1, + "id": 17, "links": [ ], "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 3, - "stack": false, - "steppedLine": false, "targets": [ { "expr": "histogram_quantile(0.99, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{job=~\"$job\",component=\"alertmanager-storage\",operation=\"delete\"}[$__rate_interval])) by (le)) * 1e3", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "99th Percentile", - "refId": "A", - "step": 10 + "refId": "A" }, { "expr": "histogram_quantile(0.50, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{job=~\"$job\",component=\"alertmanager-storage\",operation=\"delete\"}[$__rate_interval])) by (le)) * 1e3", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "50th Percentile", - "refId": "B", - "step": 10 + "refId": "B" }, { "expr": "sum(rate(thanos_objstore_bucket_operation_duration_seconds_sum{job=~\"$job\",component=\"alertmanager-storage\",operation=\"delete\"}[$__rate_interval])) * 1e3 / sum(rate(thanos_objstore_bucket_operation_duration_seconds_count{job=~\"$job\",component=\"alertmanager-storage\",operation=\"delete\"}[$__rate_interval]))", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "Average", - "refId": "C", - "step": 10 + "refId": "C" } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, "title": "Latency of Op: Delete", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, + "type": "timeseries", "yaxes": [ { "format": "ms", @@ -1581,235 +1372,157 @@ "height": "250px", "panels": [ { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 1, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "short" + }, + "overrides": [ ] + }, "fill": 10, "id": 18, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, "linewidth": 0, "links": [ ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 4, "stack": true, - "steppedLine": false, "targets": [ { "expr": "max by(pod) (cortex_alertmanager_tenants_owned{job=~\"$job\"})", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "{{pod}}", - "legendLink": null, - "step": 10 + "legendLink": null } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, "title": "Per pod Tenants", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] + "type": "timeseries" }, { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 1, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "short" + }, + "overrides": [ ] + }, "fill": 10, "id": 19, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, "linewidth": 0, "links": [ ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 4, "stack": true, - "steppedLine": false, "targets": [ { "expr": "sum by(pod) (cluster_job_pod:cortex_alertmanager_alerts:sum{job=~\"$job\"})", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "{{pod}}", - "legendLink": null, - "step": 10 + "legendLink": null } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, "title": "Per pod Alerts", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] + "type": "timeseries" }, { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 1, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "short" + }, + "overrides": [ ] + }, "fill": 10, "id": 20, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, "linewidth": 0, "links": [ ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 4, "stack": true, - "steppedLine": false, "targets": [ { "expr": "sum by(pod) (cluster_job_pod:cortex_alertmanager_silences:sum{job=~\"$job\"})", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "{{pod}}", - "legendLink": null, - "step": 10 + "legendLink": null } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, "title": "Per pod Silences", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] + "type": "timeseries" } ], "repeat": null, @@ -1824,244 +1537,155 @@ "height": "250px", "panels": [ { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", - "fill": 1, - "id": 21, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 1, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "short" + }, + "overrides": [ ] }, - "lines": true, - "linewidth": 1, + "id": 21, "links": [ ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 4, - "stack": false, - "steppedLine": false, "targets": [ { "expr": "sum(rate(cortex_alertmanager_sync_configs_total{job=~\"$job\"}[$__rate_interval]))\n-\nsum(rate(cortex_alertmanager_sync_configs_failed_total{job=~\"$job\"}[$__rate_interval]))\n", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "success", - "legendLink": null, - "step": 10 + "legendLink": null }, { "expr": "sum(rate(cortex_alertmanager_sync_configs_failed_total{job=~\"$job\"}[$__rate_interval]))", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "failed", - "legendLink": null, - "step": 10 + "legendLink": null } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, "title": "Syncs/sec", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] + "type": "timeseries" }, { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", - "fill": 1, - "id": 22, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 1, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "short" + }, + "overrides": [ ] }, - "lines": true, - "linewidth": 1, + "id": 22, "links": [ ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 4, - "stack": false, - "steppedLine": false, "targets": [ { "expr": "sum by(reason) (rate(cortex_alertmanager_sync_configs_total{job=~\"$job\"}[$__rate_interval]))", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "{{reason}}", - "legendLink": null, - "step": 10 + "legendLink": null } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, "title": "Syncs/sec (By Reason)", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] + "type": "timeseries" }, { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", - "fill": 1, - "id": 23, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 1, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "short" + }, + "overrides": [ ] }, - "lines": true, - "linewidth": 1, + "id": 23, "links": [ ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 4, - "stack": false, - "steppedLine": false, "targets": [ { "expr": "sum (rate(cortex_alertmanager_ring_check_errors_total{job=~\"$job\"}[$__rate_interval]))", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "errors", - "legendLink": null, - "step": 10 + "legendLink": null } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, "title": "Ring Check Errors/sec", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] + "type": "timeseries" } ], "repeat": null, @@ -2076,158 +1700,115 @@ "height": "250px", "panels": [ { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", - "fill": 1, - "id": 24, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 1, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "short" + }, + "overrides": [ ] }, - "lines": true, - "linewidth": 1, + "id": 24, "links": [ ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 4, - "stack": false, - "steppedLine": false, "targets": [ { "expr": "sum by(outcome) (rate(cortex_alertmanager_state_initial_sync_completed_total{job=~\"$job\"}[$__rate_interval]))", "format": "time_series", "interval": "1m", - "intervalFactor": 2, "legendFormat": "{{outcome}}", - "legendLink": null, - "step": 10 + "legendLink": null } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, "title": "Initial syncs /sec", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] + "type": "timeseries" }, { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", - "fill": 1, - "id": 25, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 1, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "s" + }, + "overrides": [ ] }, - "lines": true, - "linewidth": 1, + "id": 25, "links": [ ], "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 4, - "stack": false, - "steppedLine": false, "targets": [ { "expr": "histogram_quantile(0.99, sum(rate(cortex_alertmanager_state_initial_sync_duration_seconds_bucket{job=~\"$job\"}[$__rate_interval])) by (le)) * 1e3", "format": "time_series", "interval": "1m", - "intervalFactor": 2, "legendFormat": "99th Percentile", - "refId": "A", - "step": 10 + "refId": "A" }, { "expr": "histogram_quantile(0.50, sum(rate(cortex_alertmanager_state_initial_sync_duration_seconds_bucket{job=~\"$job\"}[$__rate_interval])) by (le)) * 1e3", "format": "time_series", "interval": "1m", - "intervalFactor": 2, "legendFormat": "50th Percentile", - "refId": "B", - "step": 10 + "refId": "B" }, { "expr": "sum(rate(cortex_alertmanager_state_initial_sync_duration_seconds_sum{job=~\"$job\"}[$__rate_interval])) * 1e3 / sum(rate(cortex_alertmanager_state_initial_sync_duration_seconds_count{job=~\"$job\"}[$__rate_interval]))", "format": "time_series", "interval": "1m", - "intervalFactor": 2, "legendFormat": "Average", - "refId": "C", - "step": 10 + "refId": "C" } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, "title": "Initial sync duration", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, + "type": "timeseries", "yaxes": [ { "format": "ms", @@ -2248,90 +1829,59 @@ ] }, { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", - "fill": 1, - "id": 26, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 1, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "short" + }, + "overrides": [ ] }, - "lines": true, - "linewidth": 1, + "id": 26, "links": [ ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 4, - "stack": false, - "steppedLine": false, "targets": [ { "expr": "sum(rate(cortex_alertmanager_state_fetch_replica_state_total{job=~\"$job\"}[$__rate_interval]))\n-\nsum(rate(cortex_alertmanager_state_fetch_replica_state_failed_total{job=~\"$job\"}[$__rate_interval]))\n", "format": "time_series", "interval": "1m", - "intervalFactor": 2, "legendFormat": "success", - "legendLink": null, - "step": 10 + "legendLink": null }, { "expr": "sum(rate(cortex_alertmanager_state_fetch_replica_state_failed_total{job=~\"$job\"}[$__rate_interval]))", "format": "time_series", "interval": "1m", - "intervalFactor": 2, "legendFormat": "failed", - "legendLink": null, - "step": 10 + "legendLink": null } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, "title": "Fetch state from other alertmanagers /sec", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] + "type": "timeseries" } ], "repeat": null, @@ -2346,262 +1896,169 @@ "height": "250px", "panels": [ { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", - "fill": 1, - "id": 27, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 1, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "short" + }, + "overrides": [ ] }, - "lines": true, - "linewidth": 1, + "id": 27, "links": [ ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 4, - "stack": false, - "steppedLine": false, "targets": [ { "expr": "sum(cluster_job:cortex_alertmanager_state_replication_total:rate5m{job=~\"$job\"})\n-\nsum(cluster_job:cortex_alertmanager_state_replication_failed_total:rate5m{job=~\"$job\"})\n", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "success", - "legendLink": null, - "step": 10 + "legendLink": null }, { "expr": "sum(cluster_job:cortex_alertmanager_state_replication_failed_total:rate5m{job=~\"$job\"})", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "failed", - "legendLink": null, - "step": 10 + "legendLink": null } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, "title": "Replicate state to other alertmanagers /sec", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] + "type": "timeseries" }, { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", - "fill": 1, - "id": 28, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 1, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "short" + }, + "overrides": [ ] }, - "lines": true, - "linewidth": 1, + "id": 28, "links": [ ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 4, - "stack": false, - "steppedLine": false, "targets": [ { "expr": "sum(cluster_job:cortex_alertmanager_partial_state_merges_total:rate5m{job=~\"$job\"})\n-\nsum(cluster_job:cortex_alertmanager_partial_state_merges_failed_total:rate5m{job=~\"$job\"})\n", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "success", - "legendLink": null, - "step": 10 + "legendLink": null }, { "expr": "sum(cluster_job:cortex_alertmanager_partial_state_merges_failed_total:rate5m{job=~\"$job\"})", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "failed", - "legendLink": null, - "step": 10 + "legendLink": null } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, "title": "Merge state from other alertmanagers /sec", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] + "type": "timeseries" }, { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", - "fill": 1, - "id": 29, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 1, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "short" + }, + "overrides": [ ] }, - "lines": true, - "linewidth": 1, + "id": 29, "links": [ ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 4, - "stack": false, - "steppedLine": false, "targets": [ { "expr": "sum(rate(cortex_alertmanager_state_persist_total{job=~\"$job\"}[$__rate_interval]))\n-\nsum(rate(cortex_alertmanager_state_persist_failed_total{job=~\"$job\"}[$__rate_interval]))\n", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "success", - "legendLink": null, - "step": 10 + "legendLink": null }, { "expr": "sum(rate(cortex_alertmanager_state_persist_failed_total{job=~\"$job\"}[$__rate_interval]))", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "failed", - "legendLink": null, - "step": 10 + "legendLink": null } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, "title": "Persist state to remote storage /sec", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] + "type": "timeseries" } ], "repeat": null, @@ -2625,7 +2082,7 @@ "value": "Cortex" }, "hide": 0, - "label": null, + "label": "Data source", "name": "datasource", "options": [ ], "query": "prometheus", diff --git a/docs/getting-started/dashboards/cortex-compactor-resources.json b/docs/getting-started/dashboards/cortex-compactor-resources.json index dfaf7c870b..6c36a6b597 100644 --- a/docs/getting-started/dashboards/cortex-compactor-resources.json +++ b/docs/getting-started/dashboards/cortex-compactor-resources.json @@ -27,30 +27,40 @@ "height": "250px", "panels": [ { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", - "fill": 1, - "id": 1, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 1, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "short" + }, + "overrides": [ ] }, - "lines": true, - "linewidth": 1, + "id": 1, "links": [ ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "seriesOverrides": [ { "alias": "limit", @@ -58,89 +68,64 @@ "fill": 0 } ], - "spaceLength": 10, "span": 4, - "stack": false, - "steppedLine": false, "targets": [ { "expr": "sum by(pod) (rate(container_cpu_usage_seconds_total{job=~\"$job\",container=~\"compactor\"}[$__rate_interval]))", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "{{pod}}", - "legendLink": null, - "step": 10 + "legendLink": null }, { "expr": "min(container_spec_cpu_quota{job=~\"$job\",container=~\"compactor\"} / container_spec_cpu_period{job=~\"$job\",container=~\"compactor\"})", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "limit", - "legendLink": null, - "step": 10 + "legendLink": null } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, "title": "CPU", "tooltip": { "sort": 2 }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] + "type": "timeseries" }, { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", - "fill": 1, - "id": 2, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 1, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "bytes" + }, + "overrides": [ ] }, - "lines": true, - "linewidth": 1, + "id": 2, "links": [ ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "seriesOverrides": [ { "alias": "limit", @@ -148,45 +133,28 @@ "fill": 0 } ], - "spaceLength": 10, "span": 4, - "stack": false, - "steppedLine": false, "targets": [ { "expr": "max by(pod) (container_memory_working_set_bytes{job=~\"$job\",container=~\"compactor\"})", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "{{pod}}", - "legendLink": null, - "step": 10 + "legendLink": null }, { "expr": "min(container_spec_memory_limit_bytes{job=~\"$job\",container=~\"compactor\"} > 0)", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "limit", - "legendLink": null, - "step": 10 + "legendLink": null } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, "title": "Memory (workingset)", "tooltip": { "sort": 2 }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, + "type": "timeseries", "yaxes": [ { "format": "bytes", @@ -207,61 +175,55 @@ ] }, { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", - "fill": 1, - "id": 3, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 1, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "bytes" + }, + "overrides": [ ] }, - "lines": true, - "linewidth": 1, + "id": 3, "links": [ ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 4, - "stack": false, - "steppedLine": false, "targets": [ { "expr": "sum by(pod) (go_memstats_heap_inuse_bytes{job=~\"$job\"})", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "{{pod}}", - "legendLink": null, - "step": 10 + "legendLink": null } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, "title": "Memory (go heap inuse)", "tooltip": { "sort": 2 }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, + "type": "timeseries", "yaxes": [ { "format": "bytes", @@ -294,63 +256,55 @@ "height": "250px", "panels": [ { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 1, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "Bps" + }, + "overrides": [ ] + }, "fill": 10, "id": 4, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, "linewidth": 0, "links": [ ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 6, "stack": true, - "steppedLine": false, "targets": [ { "expr": "sum by(pod) (rate(container_network_receive_bytes_total{job=~\"$job\",pod=~\"compactor.*\"}[$__rate_interval]))", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "{{pod}}", - "legendLink": null, - "step": 10 + "legendLink": null } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, "title": "Receive Bandwidth", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, + "type": "timeseries", "yaxes": [ { "format": "Bps", @@ -371,63 +325,55 @@ ] }, { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 1, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "Bps" + }, + "overrides": [ ] + }, "fill": 10, "id": 5, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, "linewidth": 0, "links": [ ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 6, "stack": true, - "steppedLine": false, "targets": [ { "expr": "sum by(pod) (rate(container_network_transmit_bytes_total{job=~\"$job\",pod=~\"compactor.*\"}[$__rate_interval]))", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "{{pod}}", - "legendLink": null, - "step": 10 + "legendLink": null } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, "title": "Transmit Bandwidth", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, + "type": "timeseries", "yaxes": [ { "format": "Bps", @@ -460,63 +406,55 @@ "height": "250px", "panels": [ { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 1, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "Bps" + }, + "overrides": [ ] + }, "fill": 10, "id": 6, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, "linewidth": 0, "links": [ ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 4, "stack": true, - "steppedLine": false, "targets": [ { "expr": "sum by(instance, pod, device) (\n rate(\n node_disk_written_bytes_total[$__rate_interval]\n )\n)\n+\nignoring(pod) group_right() (\n label_replace(\n count by(\n instance,\n pod,\n device\n )\n (\n container_fs_writes_bytes_total{\n job=~\"$job\",\n container=\"compactor\",\n device!~\".*sda.*\"\n }\n ),\n \"device\",\n \"$1\",\n \"device\",\n \"/dev/(.*)\"\n ) * 0\n)\n\n", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "{{pod}} - {{device}}", - "legendLink": null, - "step": 10 + "legendLink": null } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, "title": "Disk Writes", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, + "type": "timeseries", "yaxes": [ { "format": "Bps", @@ -537,63 +475,55 @@ ] }, { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 1, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "Bps" + }, + "overrides": [ ] + }, "fill": 10, "id": 7, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, "linewidth": 0, "links": [ ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 4, "stack": true, - "steppedLine": false, "targets": [ { "expr": "sum by(instance, pod, device) (\n rate(\n node_disk_read_bytes_total[$__rate_interval]\n )\n) + ignoring(pod) group_right() (\n label_replace(\n count by(\n instance,\n pod,\n device\n )\n (\n container_fs_writes_bytes_total{\n job=~\"$job\",\n container=\"compactor\",\n device!~\".*sda.*\"\n }\n ),\n \"device\",\n \"$1\",\n \"device\",\n \"/dev/(.*)\"\n ) * 0\n)\n\n", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "{{pod}} - {{device}}", - "legendLink": null, - "step": 10 + "legendLink": null } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, "title": "Disk Reads", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, + "type": "timeseries", "yaxes": [ { "format": "Bps", @@ -614,63 +544,52 @@ ] }, { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", - "fill": 1, - "id": 8, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 1, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "percentunit" + }, + "overrides": [ ] }, - "lines": true, - "linewidth": 1, + "id": 8, "links": [ ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 4, - "stack": false, - "steppedLine": false, "targets": [ { "expr": "max by(persistentvolumeclaim) (\n kubelet_volume_stats_used_bytes{job=~\"$job\"} /\n kubelet_volume_stats_capacity_bytes{job=~\"$job\"}\n)\nand\ncount by(persistentvolumeclaim) (\n kube_persistentvolumeclaim_labels{\n job=~\"$job\",\n label_name=\"compactor\"\n }\n)\n", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "{{persistentvolumeclaim}}", - "legendLink": null, - "step": 10 + "legendLink": null } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, "title": "Disk Space Utilization", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, + "type": "timeseries", "yaxes": [ { "format": "percentunit", @@ -712,7 +631,7 @@ "value": "Cortex" }, "hide": 0, - "label": null, + "label": "Data source", "name": "datasource", "options": [ ], "query": "prometheus", diff --git a/docs/getting-started/dashboards/cortex-compactor.json b/docs/getting-started/dashboards/cortex-compactor.json index f1ae6789b9..9dfb797e2f 100644 --- a/docs/getting-started/dashboards/cortex-compactor.json +++ b/docs/getting-started/dashboards/cortex-compactor.json @@ -33,158 +33,120 @@ "started": "#34CCEB" }, "bars": true, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", "description": "### Per-instance runs\nNumber of times a compactor instance triggers a compaction across all tenants that it manages.\n\n", + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 1, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "ops" + }, + "overrides": [ ] + }, "fill": 10, "id": 1, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, "lines": false, "linewidth": 0, "links": [ ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 6, "stack": true, - "steppedLine": false, "targets": [ { "expr": "sum(rate(cortex_compactor_runs_started_total{job=~\"$job\"}[$__rate_interval]))", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "started", - "legendLink": null, - "step": 10 + "legendLink": null }, { "expr": "sum(rate(cortex_compactor_runs_completed_total{job=~\"$job\"}[$__rate_interval]))", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "completed", - "legendLink": null, - "step": 10 + "legendLink": null }, { "expr": "sum(rate(cortex_compactor_runs_failed_total{job=~\"$job\"}[$__rate_interval]))", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "failed", - "legendLink": null, - "step": 10 + "legendLink": null } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, "title": "Per-instance runs / sec", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, - "yaxes": [ - { - "format": "ops", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] + "type": "timeseries" }, { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", "description": "### Tenants compaction progress\nIn a multi-tenant cluster, display the progress of tenants that are compacted while compaction is running.\nReset to 0 after the compaction run is completed for all tenants in the shard.\n\n", - "fill": 1, - "id": 2, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 1, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "short" + }, + "overrides": [ ] }, - "lines": true, - "linewidth": 1, + "id": 2, "links": [ ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 6, - "stack": false, - "steppedLine": false, "targets": [ { "expr": "(\n cortex_compactor_tenants_processing_succeeded{job=~\"$job\"} +\n cortex_compactor_tenants_processing_failed{job=~\"$job\"} +\n cortex_compactor_tenants_skipped{job=~\"$job\"}\n) / cortex_compactor_tenants_discovered{job=~\"$job\"}\n", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "{{pod}}", - "legendLink": null, - "step": 10 + "legendLink": null } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, "title": "Tenants compaction progress", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, + "type": "timeseries", "yaxes": [ { "format": "percentunit", @@ -217,160 +179,117 @@ "height": "250px", "panels": [ { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", "description": "### Compacted blocks / sec\nRate of blocks that are generated as a result of a compaction operation.\n\n", - "fill": 1, - "id": 3, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 1, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "ops" + }, + "overrides": [ ] }, - "lines": true, - "linewidth": 1, + "id": 3, "links": [ ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 6, - "stack": false, - "steppedLine": false, "targets": [ { "expr": "sum(rate(prometheus_tsdb_compactions_total{job=~\"$job\"}[$__rate_interval]))", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "blocks", - "legendLink": null, - "step": 10 + "legendLink": null } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, "title": "Compacted blocks / sec", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, - "yaxes": [ - { - "format": "ops", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] + "type": "timeseries" }, { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", "description": "### Per-block compaction duration\nDisplay the amount of time that it has taken to generate a single compacted block.\n\n", - "fill": 1, - "id": 4, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 1, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "s" + }, + "overrides": [ ] }, - "lines": true, - "linewidth": 1, + "id": 4, "links": [ ], "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 6, - "stack": false, - "steppedLine": false, "targets": [ { "expr": "histogram_quantile(0.99, sum(rate(prometheus_tsdb_compaction_duration_seconds_bucket{job=~\"$job\"}[$__rate_interval])) by (le)) * 1e3", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "99th Percentile", - "refId": "A", - "step": 10 + "refId": "A" }, { "expr": "histogram_quantile(0.50, sum(rate(prometheus_tsdb_compaction_duration_seconds_bucket{job=~\"$job\"}[$__rate_interval])) by (le)) * 1e3", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "50th Percentile", - "refId": "B", - "step": 10 + "refId": "B" }, { "expr": "sum(rate(prometheus_tsdb_compaction_duration_seconds_sum{job=~\"$job\"}[$__rate_interval])) * 1e3 / sum(rate(prometheus_tsdb_compaction_duration_seconds_count{job=~\"$job\"}[$__rate_interval]))", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "Average", - "refId": "C", - "step": 10 + "refId": "C" } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, "title": "Per-block compaction duration", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, + "type": "timeseries", "yaxes": [ { "format": "ms", @@ -403,159 +322,101 @@ "height": "250px", "panels": [ { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", - "fill": 1, - "id": 5, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 1, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "short" + }, + "overrides": [ ] }, - "lines": true, - "linewidth": 1, + "id": 5, "links": [ ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 6, - "stack": false, - "steppedLine": false, "targets": [ { "expr": "avg(max by(user) (cortex_bucket_blocks_count{job=~\"$job\"}))", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "avg", - "legendLink": null, - "step": 10 + "legendLink": null } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, "title": "Average blocks / tenant", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] + "type": "timeseries" }, { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", "description": "### Tenants with largest number of blocks\nThe 10 tenants with the largest number of blocks.\n\n", - "fill": 1, - "id": 6, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 1, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "short" + }, + "overrides": [ ] }, - "lines": true, - "linewidth": 1, + "id": 6, "links": [ ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 6, - "stack": false, - "steppedLine": false, "targets": [ { "expr": "topk(10, max by(user) (cortex_bucket_blocks_count{job=~\"$job\"}))", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "{{user}}", - "legendLink": null, - "step": 10 + "legendLink": null } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, "title": "Tenants with largest number of blocks", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] + "type": "timeseries" } ], "repeat": null, @@ -570,152 +431,114 @@ "height": "250px", "panels": [ { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", - "fill": 1, - "id": 7, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 1, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "ops" + }, + "overrides": [ ] }, - "lines": true, - "linewidth": 1, + "id": 7, "links": [ ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 6, - "stack": false, - "steppedLine": false, "targets": [ { "expr": "sum(rate(cortex_compactor_blocks_marked_for_deletion_total{job=~\"$job\"}[$__rate_interval]))", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "blocks", - "legendLink": null, - "step": 10 + "legendLink": null } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, "title": "Blocks marked for deletion / sec", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, - "yaxes": [ - { - "format": "ops", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] + "type": "timeseries" }, { "aliasColors": { "failed": "#E24D42", "successful": "#7EB26D" }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 1, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "short" + }, + "overrides": [ ] + }, "fill": 10, "id": 8, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, "linewidth": 0, "links": [ ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 6, "stack": true, - "steppedLine": false, "targets": [ { "expr": "sum(rate(cortex_compactor_blocks_cleaned_total{job=~\"$job\"}[$__rate_interval]))", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "successful", - "legendLink": null, - "step": 10 + "legendLink": null }, { "expr": "sum(rate(cortex_compactor_block_cleanup_failures_total{job=~\"$job\"}[$__rate_interval]))", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "failed", - "legendLink": null, - "step": 10 + "legendLink": null } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, "title": "Blocks deletions / sec", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, + "type": "timeseries", "yaxes": [ { "format": "ops", @@ -752,71 +575,62 @@ "failed": "#E24D42", "successful": "#7EB26D" }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 1, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "short" + }, + "overrides": [ ] + }, "fill": 10, "id": 9, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, "linewidth": 0, "links": [ ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 6, "stack": true, - "steppedLine": false, "targets": [ { "expr": "sum(rate(cortex_compactor_meta_syncs_total{job=~\"$job\"}[$__rate_interval])) - sum(rate(cortex_compactor_meta_sync_failures_total{job=~\"$job\"}[$__rate_interval]))", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "successful", - "legendLink": null, - "step": 10 + "legendLink": null }, { "expr": "sum(rate(cortex_compactor_meta_sync_failures_total{job=~\"$job\"}[$__rate_interval]))", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "failed", - "legendLink": null, - "step": 10 + "legendLink": null } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, "title": "Metadata Syncs / sec", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, + "type": "timeseries", "yaxes": [ { "format": "ops", @@ -837,81 +651,67 @@ ] }, { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", - "fill": 1, - "id": 10, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 1, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "ms" + }, + "overrides": [ ] }, - "lines": true, - "linewidth": 1, + "id": 10, "links": [ ], "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 6, - "stack": false, - "steppedLine": false, "targets": [ { "expr": "histogram_quantile(0.99, sum(rate(cortex_compactor_meta_sync_duration_seconds_bucket{job=~\"$job\"}[$__rate_interval])) by (le)) * 1e3", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "99th Percentile", - "refId": "A", - "step": 10 + "refId": "A" }, { "expr": "histogram_quantile(0.50, sum(rate(cortex_compactor_meta_sync_duration_seconds_bucket{job=~\"$job\"}[$__rate_interval])) by (le)) * 1e3", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "50th Percentile", - "refId": "B", - "step": 10 + "refId": "B" }, { "expr": "sum(rate(cortex_compactor_meta_sync_duration_seconds_sum{job=~\"$job\"}[$__rate_interval])) * 1e3 / sum(rate(cortex_compactor_meta_sync_duration_seconds_count{job=~\"$job\"}[$__rate_interval]))", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "Average", - "refId": "C", - "step": 10 + "refId": "C" } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, "title": "Metadata Sync Duration", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, + "type": "timeseries", "yaxes": [ { "format": "ms", @@ -944,140 +744,103 @@ "height": "250px", "panels": [ { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 1, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "rps" + }, + "overrides": [ ] + }, "fill": 10, "id": 11, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, "linewidth": 0, "links": [ ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 3, "stack": true, - "steppedLine": false, "targets": [ { "expr": "sum by(operation) (rate(thanos_objstore_bucket_operations_total{job=~\"$job\",component=\"compactor\"}[$__rate_interval]))", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "{{operation}}", - "legendLink": null, - "step": 10 + "legendLink": null } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, "title": "Operations / sec", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, - "yaxes": [ - { - "format": "rps", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] + "type": "timeseries" }, { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", - "fill": 1, - "id": 12, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 1, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "percentunit" + }, + "overrides": [ ] }, - "lines": true, - "linewidth": 1, + "id": 12, "links": [ ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 3, - "stack": false, - "steppedLine": false, "targets": [ { "expr": "sum by(operation) (rate(thanos_objstore_bucket_operation_failures_total{job=~\"$job\",component=\"compactor\"}[$__rate_interval])) / sum by(operation) (rate(thanos_objstore_bucket_operations_total{job=~\"$job\",component=\"compactor\"}[$__rate_interval]))", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "{{operation}}", - "legendLink": null, - "step": 10 + "legendLink": null } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, "title": "Error rate", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, + "type": "timeseries", "yaxes": [ { "format": "percentunit", @@ -1098,81 +861,67 @@ ] }, { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", - "fill": 1, - "id": 13, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 1, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "short" + }, + "overrides": [ ] }, - "lines": true, - "linewidth": 1, + "id": 13, "links": [ ], "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 3, - "stack": false, - "steppedLine": false, "targets": [ { "expr": "histogram_quantile(0.99, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{job=~\"$job\",component=\"compactor\",operation=\"attributes\"}[$__rate_interval])) by (le)) * 1e3", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "99th Percentile", - "refId": "A", - "step": 10 + "refId": "A" }, { "expr": "histogram_quantile(0.50, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{job=~\"$job\",component=\"compactor\",operation=\"attributes\"}[$__rate_interval])) by (le)) * 1e3", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "50th Percentile", - "refId": "B", - "step": 10 + "refId": "B" }, { "expr": "sum(rate(thanos_objstore_bucket_operation_duration_seconds_sum{job=~\"$job\",component=\"compactor\",operation=\"attributes\"}[$__rate_interval])) * 1e3 / sum(rate(thanos_objstore_bucket_operation_duration_seconds_count{job=~\"$job\",component=\"compactor\",operation=\"attributes\"}[$__rate_interval]))", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "Average", - "refId": "C", - "step": 10 + "refId": "C" } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, "title": "Latency of Op: Attributes", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, + "type": "timeseries", "yaxes": [ { "format": "ms", @@ -1193,81 +942,67 @@ ] }, { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", - "fill": 1, - "id": 14, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 1, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "short" + }, + "overrides": [ ] }, - "lines": true, - "linewidth": 1, + "id": 14, "links": [ ], "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 3, - "stack": false, - "steppedLine": false, "targets": [ { "expr": "histogram_quantile(0.99, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{job=~\"$job\",component=\"compactor\",operation=\"exists\"}[$__rate_interval])) by (le)) * 1e3", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "99th Percentile", - "refId": "A", - "step": 10 + "refId": "A" }, { "expr": "histogram_quantile(0.50, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{job=~\"$job\",component=\"compactor\",operation=\"exists\"}[$__rate_interval])) by (le)) * 1e3", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "50th Percentile", - "refId": "B", - "step": 10 + "refId": "B" }, { "expr": "sum(rate(thanos_objstore_bucket_operation_duration_seconds_sum{job=~\"$job\",component=\"compactor\",operation=\"exists\"}[$__rate_interval])) * 1e3 / sum(rate(thanos_objstore_bucket_operation_duration_seconds_count{job=~\"$job\",component=\"compactor\",operation=\"exists\"}[$__rate_interval]))", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "Average", - "refId": "C", - "step": 10 + "refId": "C" } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, "title": "Latency of Op: Exists", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, + "type": "timeseries", "yaxes": [ { "format": "ms", @@ -1300,81 +1035,67 @@ "height": "250px", "panels": [ { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", - "fill": 1, - "id": 15, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 1, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "short" + }, + "overrides": [ ] }, - "lines": true, - "linewidth": 1, + "id": 15, "links": [ ], "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 3, - "stack": false, - "steppedLine": false, "targets": [ { "expr": "histogram_quantile(0.99, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{job=~\"$job\",component=\"compactor\",operation=\"get\"}[$__rate_interval])) by (le)) * 1e3", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "99th Percentile", - "refId": "A", - "step": 10 + "refId": "A" }, { "expr": "histogram_quantile(0.50, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{job=~\"$job\",component=\"compactor\",operation=\"get\"}[$__rate_interval])) by (le)) * 1e3", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "50th Percentile", - "refId": "B", - "step": 10 + "refId": "B" }, { "expr": "sum(rate(thanos_objstore_bucket_operation_duration_seconds_sum{job=~\"$job\",component=\"compactor\",operation=\"get\"}[$__rate_interval])) * 1e3 / sum(rate(thanos_objstore_bucket_operation_duration_seconds_count{job=~\"$job\",component=\"compactor\",operation=\"get\"}[$__rate_interval]))", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "Average", - "refId": "C", - "step": 10 + "refId": "C" } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, "title": "Latency of Op: Get", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, + "type": "timeseries", "yaxes": [ { "format": "ms", @@ -1395,81 +1116,67 @@ ] }, { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", - "fill": 1, - "id": 16, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 1, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "short" + }, + "overrides": [ ] }, - "lines": true, - "linewidth": 1, + "id": 16, "links": [ ], "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 3, - "stack": false, - "steppedLine": false, "targets": [ { "expr": "histogram_quantile(0.99, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{job=~\"$job\",component=\"compactor\",operation=\"get_range\"}[$__rate_interval])) by (le)) * 1e3", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "99th Percentile", - "refId": "A", - "step": 10 + "refId": "A" }, { "expr": "histogram_quantile(0.50, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{job=~\"$job\",component=\"compactor\",operation=\"get_range\"}[$__rate_interval])) by (le)) * 1e3", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "50th Percentile", - "refId": "B", - "step": 10 + "refId": "B" }, { "expr": "sum(rate(thanos_objstore_bucket_operation_duration_seconds_sum{job=~\"$job\",component=\"compactor\",operation=\"get_range\"}[$__rate_interval])) * 1e3 / sum(rate(thanos_objstore_bucket_operation_duration_seconds_count{job=~\"$job\",component=\"compactor\",operation=\"get_range\"}[$__rate_interval]))", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "Average", - "refId": "C", - "step": 10 + "refId": "C" } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, "title": "Latency of Op: GetRange", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, + "type": "timeseries", "yaxes": [ { "format": "ms", @@ -1490,81 +1197,67 @@ ] }, { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", - "fill": 1, - "id": 17, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 1, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "short" + }, + "overrides": [ ] }, - "lines": true, - "linewidth": 1, + "id": 17, "links": [ ], "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 3, - "stack": false, - "steppedLine": false, "targets": [ { "expr": "histogram_quantile(0.99, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{job=~\"$job\",component=\"compactor\",operation=\"upload\"}[$__rate_interval])) by (le)) * 1e3", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "99th Percentile", - "refId": "A", - "step": 10 + "refId": "A" }, { "expr": "histogram_quantile(0.50, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{job=~\"$job\",component=\"compactor\",operation=\"upload\"}[$__rate_interval])) by (le)) * 1e3", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "50th Percentile", - "refId": "B", - "step": 10 + "refId": "B" }, { "expr": "sum(rate(thanos_objstore_bucket_operation_duration_seconds_sum{job=~\"$job\",component=\"compactor\",operation=\"upload\"}[$__rate_interval])) * 1e3 / sum(rate(thanos_objstore_bucket_operation_duration_seconds_count{job=~\"$job\",component=\"compactor\",operation=\"upload\"}[$__rate_interval]))", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "Average", - "refId": "C", - "step": 10 + "refId": "C" } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, "title": "Latency of Op: Upload", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, + "type": "timeseries", "yaxes": [ { "format": "ms", @@ -1585,81 +1278,67 @@ ] }, { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", - "fill": 1, - "id": 18, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 1, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "short" + }, + "overrides": [ ] }, - "lines": true, - "linewidth": 1, + "id": 18, "links": [ ], "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 3, - "stack": false, - "steppedLine": false, "targets": [ { "expr": "histogram_quantile(0.99, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{job=~\"$job\",component=\"compactor\",operation=\"delete\"}[$__rate_interval])) by (le)) * 1e3", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "99th Percentile", - "refId": "A", - "step": 10 + "refId": "A" }, { "expr": "histogram_quantile(0.50, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{job=~\"$job\",component=\"compactor\",operation=\"delete\"}[$__rate_interval])) by (le)) * 1e3", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "50th Percentile", - "refId": "B", - "step": 10 + "refId": "B" }, { "expr": "sum(rate(thanos_objstore_bucket_operation_duration_seconds_sum{job=~\"$job\",component=\"compactor\",operation=\"delete\"}[$__rate_interval])) * 1e3 / sum(rate(thanos_objstore_bucket_operation_duration_seconds_count{job=~\"$job\",component=\"compactor\",operation=\"delete\"}[$__rate_interval]))", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "Average", - "refId": "C", - "step": 10 + "refId": "C" } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, "title": "Latency of Op: Delete", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, + "type": "timeseries", "yaxes": [ { "format": "ms", @@ -1701,7 +1380,7 @@ "value": "Cortex" }, "hide": 0, - "label": null, + "label": "Data source", "name": "datasource", "options": [ ], "query": "prometheus", diff --git a/docs/getting-started/dashboards/cortex-config.json b/docs/getting-started/dashboards/cortex-config.json index 20026709e7..8592fc7f25 100644 --- a/docs/getting-started/dashboards/cortex-config.json +++ b/docs/getting-started/dashboards/cortex-config.json @@ -27,81 +27,55 @@ "height": "250px", "panels": [ { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 1, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "instances" + }, + "overrides": [ ] + }, "fill": 10, "id": 1, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, "linewidth": 0, "links": [ ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 12, "stack": true, - "steppedLine": false, "targets": [ { "expr": "count(cortex_config_hash{job=~\"$job\"}) by (sha256)", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "sha256:{{sha256}}", - "legendLink": null, - "step": 10 + "legendLink": null } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, "title": "Startup config file hashes", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, - "yaxes": [ - { - "format": "instances", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] + "type": "timeseries" } ], "repeat": null, @@ -116,81 +90,55 @@ "height": "250px", "panels": [ { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 1, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "instances" + }, + "overrides": [ ] + }, "fill": 10, "id": 2, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, "linewidth": 0, "links": [ ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 12, "stack": true, - "steppedLine": false, "targets": [ { "expr": "count(cortex_runtime_config_hash{job=~\"$job\"}) by (sha256)", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "sha256:{{sha256}}", - "legendLink": null, - "step": 10 + "legendLink": null } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, "title": "Runtime config file hashes", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, - "yaxes": [ - { - "format": "instances", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] + "type": "timeseries" } ], "repeat": null, @@ -214,7 +162,7 @@ "value": "Cortex" }, "hide": 0, - "label": null, + "label": "Data source", "name": "datasource", "options": [ ], "query": "prometheus", diff --git a/docs/getting-started/dashboards/cortex-object-store.json b/docs/getting-started/dashboards/cortex-object-store.json index 20d96b2650..aca2a08207 100644 --- a/docs/getting-started/dashboards/cortex-object-store.json +++ b/docs/getting-started/dashboards/cortex-object-store.json @@ -27,158 +27,103 @@ "height": "250px", "panels": [ { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 1, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "rps" + }, + "overrides": [ ] + }, "fill": 10, "id": 1, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, "linewidth": 0, "links": [ ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 6, "stack": true, - "steppedLine": false, "targets": [ { "expr": "sum by(component) (rate(thanos_objstore_bucket_operations_total{job=~\"$job\"}[$__rate_interval]))", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "{{component}}", - "legendLink": null, - "step": 10 + "legendLink": null } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, "title": "RPS / component", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, - "yaxes": [ - { - "format": "rps", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] + "type": "timeseries" }, { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", - "fill": 1, - "id": 2, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 1, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "percentunit" + }, + "overrides": [ ] }, - "lines": true, - "linewidth": 1, + "id": 2, "links": [ ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 6, - "stack": false, - "steppedLine": false, "targets": [ { "expr": "sum by(component) (rate(thanos_objstore_bucket_operation_failures_total{job=~\"$job\"}[$__rate_interval])) / sum by(component) (rate(thanos_objstore_bucket_operations_total{job=~\"$job\"}[$__rate_interval]))", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "{{component}}", - "legendLink": null, - "step": 10 + "legendLink": null } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, "title": "Error rate / component", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, - "yaxes": [ - { - "format": "percentunit", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] + "type": "timeseries" } ], "repeat": null, @@ -193,158 +138,103 @@ "height": "250px", "panels": [ { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 1, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "rps" + }, + "overrides": [ ] + }, "fill": 10, "id": 3, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, "linewidth": 0, "links": [ ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 6, "stack": true, - "steppedLine": false, "targets": [ { "expr": "sum by(operation) (rate(thanos_objstore_bucket_operations_total{job=~\"$job\"}[$__rate_interval]))", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "{{operation}}", - "legendLink": null, - "step": 10 + "legendLink": null } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, "title": "RPS / operation", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, - "yaxes": [ - { - "format": "rps", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] + "type": "timeseries" }, { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", - "fill": 1, - "id": 4, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 1, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "percentunit" + }, + "overrides": [ ] }, - "lines": true, - "linewidth": 1, + "id": 4, "links": [ ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 6, - "stack": false, - "steppedLine": false, "targets": [ { "expr": "sum by(operation) (rate(thanos_objstore_bucket_operation_failures_total{job=~\"$job\"}[$__rate_interval])) / sum by(operation) (rate(thanos_objstore_bucket_operations_total{job=~\"$job\"}[$__rate_interval]))", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "{{operation}}", - "legendLink": null, - "step": 10 + "legendLink": null } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, "title": "Error rate / operation", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, - "yaxes": [ - { - "format": "percentunit", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] + "type": "timeseries" } ], "repeat": null, @@ -359,81 +249,67 @@ "height": "250px", "panels": [ { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", - "fill": 1, - "id": 5, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 1, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "short" + }, + "overrides": [ ] }, - "lines": true, - "linewidth": 1, + "id": 5, "links": [ ], "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 4, - "stack": false, - "steppedLine": false, "targets": [ { "expr": "histogram_quantile(0.99, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{job=~\"$job\",operation=\"get\"}[$__rate_interval])) by (le)) * 1e3", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "99th Percentile", - "refId": "A", - "step": 10 + "refId": "A" }, { "expr": "histogram_quantile(0.50, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{job=~\"$job\",operation=\"get\"}[$__rate_interval])) by (le)) * 1e3", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "50th Percentile", - "refId": "B", - "step": 10 + "refId": "B" }, { "expr": "sum(rate(thanos_objstore_bucket_operation_duration_seconds_sum{job=~\"$job\",operation=\"get\"}[$__rate_interval])) * 1e3 / sum(rate(thanos_objstore_bucket_operation_duration_seconds_count{job=~\"$job\",operation=\"get\"}[$__rate_interval]))", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "Average", - "refId": "C", - "step": 10 + "refId": "C" } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, "title": "Op: Get", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, + "type": "timeseries", "yaxes": [ { "format": "ms", @@ -454,81 +330,67 @@ ] }, { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", - "fill": 1, - "id": 6, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 1, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "short" + }, + "overrides": [ ] }, - "lines": true, - "linewidth": 1, + "id": 6, "links": [ ], "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 4, - "stack": false, - "steppedLine": false, "targets": [ { "expr": "histogram_quantile(0.99, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{job=~\"$job\",operation=\"get_range\"}[$__rate_interval])) by (le)) * 1e3", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "99th Percentile", - "refId": "A", - "step": 10 + "refId": "A" }, { "expr": "histogram_quantile(0.50, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{job=~\"$job\",operation=\"get_range\"}[$__rate_interval])) by (le)) * 1e3", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "50th Percentile", - "refId": "B", - "step": 10 + "refId": "B" }, { "expr": "sum(rate(thanos_objstore_bucket_operation_duration_seconds_sum{job=~\"$job\",operation=\"get_range\"}[$__rate_interval])) * 1e3 / sum(rate(thanos_objstore_bucket_operation_duration_seconds_count{job=~\"$job\",operation=\"get_range\"}[$__rate_interval]))", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "Average", - "refId": "C", - "step": 10 + "refId": "C" } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, "title": "Op: GetRange", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, + "type": "timeseries", "yaxes": [ { "format": "ms", @@ -549,81 +411,67 @@ ] }, { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", - "fill": 1, - "id": 7, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 1, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "short" + }, + "overrides": [ ] }, - "lines": true, - "linewidth": 1, + "id": 7, "links": [ ], "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 4, - "stack": false, - "steppedLine": false, "targets": [ { "expr": "histogram_quantile(0.99, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{job=~\"$job\",operation=\"exists\"}[$__rate_interval])) by (le)) * 1e3", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "99th Percentile", - "refId": "A", - "step": 10 + "refId": "A" }, { "expr": "histogram_quantile(0.50, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{job=~\"$job\",operation=\"exists\"}[$__rate_interval])) by (le)) * 1e3", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "50th Percentile", - "refId": "B", - "step": 10 + "refId": "B" }, { "expr": "sum(rate(thanos_objstore_bucket_operation_duration_seconds_sum{job=~\"$job\",operation=\"exists\"}[$__rate_interval])) * 1e3 / sum(rate(thanos_objstore_bucket_operation_duration_seconds_count{job=~\"$job\",operation=\"exists\"}[$__rate_interval]))", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "Average", - "refId": "C", - "step": 10 + "refId": "C" } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, "title": "Op: Exists", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, + "type": "timeseries", "yaxes": [ { "format": "ms", @@ -656,81 +504,67 @@ "height": "250px", "panels": [ { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", - "fill": 1, - "id": 8, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 1, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "short" + }, + "overrides": [ ] }, - "lines": true, - "linewidth": 1, + "id": 8, "links": [ ], "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 4, - "stack": false, - "steppedLine": false, "targets": [ { "expr": "histogram_quantile(0.99, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{job=~\"$job\",operation=\"attributes\"}[$__rate_interval])) by (le)) * 1e3", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "99th Percentile", - "refId": "A", - "step": 10 + "refId": "A" }, { "expr": "histogram_quantile(0.50, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{job=~\"$job\",operation=\"attributes\"}[$__rate_interval])) by (le)) * 1e3", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "50th Percentile", - "refId": "B", - "step": 10 + "refId": "B" }, { "expr": "sum(rate(thanos_objstore_bucket_operation_duration_seconds_sum{job=~\"$job\",operation=\"attributes\"}[$__rate_interval])) * 1e3 / sum(rate(thanos_objstore_bucket_operation_duration_seconds_count{job=~\"$job\",operation=\"attributes\"}[$__rate_interval]))", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "Average", - "refId": "C", - "step": 10 + "refId": "C" } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, "title": "Op: Attributes", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, + "type": "timeseries", "yaxes": [ { "format": "ms", @@ -751,81 +585,67 @@ ] }, { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", - "fill": 1, - "id": 9, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 1, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "short" + }, + "overrides": [ ] }, - "lines": true, - "linewidth": 1, + "id": 9, "links": [ ], "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 4, - "stack": false, - "steppedLine": false, "targets": [ { "expr": "histogram_quantile(0.99, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{job=~\"$job\",operation=\"upload\"}[$__rate_interval])) by (le)) * 1e3", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "99th Percentile", - "refId": "A", - "step": 10 + "refId": "A" }, { "expr": "histogram_quantile(0.50, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{job=~\"$job\",operation=\"upload\"}[$__rate_interval])) by (le)) * 1e3", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "50th Percentile", - "refId": "B", - "step": 10 + "refId": "B" }, { "expr": "sum(rate(thanos_objstore_bucket_operation_duration_seconds_sum{job=~\"$job\",operation=\"upload\"}[$__rate_interval])) * 1e3 / sum(rate(thanos_objstore_bucket_operation_duration_seconds_count{job=~\"$job\",operation=\"upload\"}[$__rate_interval]))", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "Average", - "refId": "C", - "step": 10 + "refId": "C" } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, "title": "Op: Upload", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, + "type": "timeseries", "yaxes": [ { "format": "ms", @@ -846,81 +666,67 @@ ] }, { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", - "fill": 1, - "id": 10, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 1, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "short" + }, + "overrides": [ ] }, - "lines": true, - "linewidth": 1, + "id": 10, "links": [ ], "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 4, - "stack": false, - "steppedLine": false, "targets": [ { "expr": "histogram_quantile(0.99, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{job=~\"$job\",operation=\"delete\"}[$__rate_interval])) by (le)) * 1e3", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "99th Percentile", - "refId": "A", - "step": 10 + "refId": "A" }, { "expr": "histogram_quantile(0.50, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{job=~\"$job\",operation=\"delete\"}[$__rate_interval])) by (le)) * 1e3", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "50th Percentile", - "refId": "B", - "step": 10 + "refId": "B" }, { "expr": "sum(rate(thanos_objstore_bucket_operation_duration_seconds_sum{job=~\"$job\",operation=\"delete\"}[$__rate_interval])) * 1e3 / sum(rate(thanos_objstore_bucket_operation_duration_seconds_count{job=~\"$job\",operation=\"delete\"}[$__rate_interval]))", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "Average", - "refId": "C", - "step": 10 + "refId": "C" } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, "title": "Op: Delete", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, + "type": "timeseries", "yaxes": [ { "format": "ms", @@ -962,7 +768,7 @@ "value": "Cortex" }, "hide": 0, - "label": null, + "label": "Data source", "name": "datasource", "options": [ ], "query": "prometheus", diff --git a/docs/getting-started/dashboards/cortex-queries.json b/docs/getting-started/dashboards/cortex-queries.json index d2a38c2248..63cb39ee33 100644 --- a/docs/getting-started/dashboards/cortex-queries.json +++ b/docs/getting-started/dashboards/cortex-queries.json @@ -27,81 +27,67 @@ "height": "250px", "panels": [ { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", - "fill": 1, + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 1, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "ms" + }, + "overrides": [ ] + }, "id": 1, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, "links": [ ], "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 4, - "stack": false, - "steppedLine": false, "targets": [ { "expr": "histogram_quantile(0.99, sum(rate(cortex_query_frontend_queue_duration_seconds_bucket{job=~\"$job\"}[$__rate_interval])) by (le)) * 1e3", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "99th Percentile", - "refId": "A", - "step": 10 + "refId": "A" }, { "expr": "histogram_quantile(0.50, sum(rate(cortex_query_frontend_queue_duration_seconds_bucket{job=~\"$job\"}[$__rate_interval])) by (le)) * 1e3", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "50th Percentile", - "refId": "B", - "step": 10 + "refId": "B" }, { "expr": "sum(rate(cortex_query_frontend_queue_duration_seconds_sum{job=~\"$job\"}[$__rate_interval])) * 1e3 / sum(rate(cortex_query_frontend_queue_duration_seconds_count{job=~\"$job\"}[$__rate_interval]))", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "Average", - "refId": "C", - "step": 10 + "refId": "C" } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, "title": "Queue Duration", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, + "type": "timeseries", "yaxes": [ { "format": "ms", @@ -122,84 +108,70 @@ ] }, { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", - "fill": 1, + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 1, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "short" + }, + "overrides": [ ] + }, "id": 2, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, "links": [ ], "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 4, - "stack": false, - "steppedLine": false, "targets": [ { "expr": "histogram_quantile(0.99, sum(rate(cortex_query_frontend_retries_bucket{job=~\"$job\"}[$__rate_interval])) by (le)) * 1", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "99th Percentile", - "refId": "A", - "step": 10 + "refId": "A" }, { "expr": "histogram_quantile(0.50, sum(rate(cortex_query_frontend_retries_bucket{job=~\"$job\"}[$__rate_interval])) by (le)) * 1", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "50th Percentile", - "refId": "B", - "step": 10 + "refId": "B" }, { "expr": "sum(rate(cortex_query_frontend_retries_sum{job=~\"$job\"}[$__rate_interval])) * 1 / sum(rate(cortex_query_frontend_retries_count{job=~\"$job\"}[$__rate_interval]))", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "Average", - "refId": "C", - "step": 10 + "refId": "C" } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, "title": "Retries", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, + "type": "timeseries", "yaxes": [ { - "format": "short", + "format": "ms", "label": null, "logBase": 1, "max": null, @@ -217,81 +189,52 @@ ] }, { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", - "fill": 1, + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 1, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "short" + }, + "overrides": [ ] + }, "id": 3, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, "links": [ ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 4, - "stack": false, - "steppedLine": false, "targets": [ { "expr": "cortex_query_frontend_queue_length{job=~\"$job\"}", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "{{cluster}} / {{namespace}} / {{pod}}", - "legendLink": null, - "step": 10 + "legendLink": null } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, "title": "Queue Length", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] + "type": "timeseries" } ], "repeat": null, @@ -306,81 +249,67 @@ "height": "250px", "panels": [ { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", - "fill": 1, + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 1, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "ms" + }, + "overrides": [ ] + }, "id": 4, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, "links": [ ], "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 6, - "stack": false, - "steppedLine": false, "targets": [ { "expr": "histogram_quantile(0.99, sum(rate(cortex_query_scheduler_queue_duration_seconds_bucket{job=~\"$job\"}[$__rate_interval])) by (le)) * 1e3", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "99th Percentile", - "refId": "A", - "step": 10 + "refId": "A" }, { "expr": "histogram_quantile(0.50, sum(rate(cortex_query_scheduler_queue_duration_seconds_bucket{job=~\"$job\"}[$__rate_interval])) by (le)) * 1e3", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "50th Percentile", - "refId": "B", - "step": 10 + "refId": "B" }, { "expr": "sum(rate(cortex_query_scheduler_queue_duration_seconds_sum{job=~\"$job\"}[$__rate_interval])) * 1e3 / sum(rate(cortex_query_scheduler_queue_duration_seconds_count{job=~\"$job\"}[$__rate_interval]))", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "Average", - "refId": "C", - "step": 10 + "refId": "C" } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, "title": "Queue Duration", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, + "type": "timeseries", "yaxes": [ { "format": "ms", @@ -401,81 +330,52 @@ ] }, { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", - "fill": 1, + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 1, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "short" + }, + "overrides": [ ] + }, "id": 5, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, "links": [ ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 6, - "stack": false, - "steppedLine": false, "targets": [ { "expr": "cortex_query_scheduler_queue_length{job=~\"$job\"}", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "{{cluster}} / {{namespace}} / {{pod}}", - "legendLink": null, - "step": 10 + "legendLink": null } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, "title": "Queue Length", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] + "type": "timeseries" } ], "repeat": null, @@ -490,141 +390,101 @@ "height": "250px", "panels": [ { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", "description": "### Intervals per Query\nThe average number of splitted queries (partitioned by time) executed a single input query.\n\n", - "fill": 1, + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 1, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "short" + }, + "overrides": [ ] + }, "id": 6, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, "links": [ ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 4, - "stack": false, - "steppedLine": false, "targets": [ { "expr": "sum(rate(cortex_frontend_split_queries_total{job=~\"$job\"}[1m])) / sum(rate(cortex_frontend_query_range_duration_seconds_count{job=~\"$job\", method=\"split_by_interval\"}[1m]))", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "splitting rate", - "legendLink": null, - "step": 10 + "legendLink": null } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, "title": "Intervals per Query", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] + "type": "timeseries" }, { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", - "fill": 1, + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 1, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "short" + }, + "overrides": [ ] + }, "id": 7, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, "links": [ ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 4, - "stack": false, - "steppedLine": false, "targets": [ { "expr": "sum(rate(cortex_cache_hits{name=~\"frontend.+\", job=~\"$job\"}[1m])) / sum(rate(cortex_cache_fetched_keys{name=~\"frontend.+\", job=~\"$job\"}[1m])) or\nsum(rate(cortex_cache_hits_total{name=~\"frontend.+\", job=~\"$job\"}[1m])) / sum(rate(cortex_cache_fetched_keys_total{name=~\"frontend.+\", job=~\"$job\"}[1m]))\n", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "Hit Rate", - "legendLink": null, - "step": 10 + "legendLink": null } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, "title": "Results Cache Hit %", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, + "type": "timeseries", "yaxes": [ { "format": "percentunit", @@ -645,81 +505,52 @@ ] }, { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", - "fill": 1, + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 1, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "short" + }, + "overrides": [ ] + }, "id": 8, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, "links": [ ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 4, - "stack": false, - "steppedLine": false, "targets": [ { "expr": "sum(rate(cortex_cache_fetched_keys{name=~\"frontend.+\", job=~\"$job\"}[1m])) - sum(rate(cortex_cache_hits{name=~\"frontend.+\", job=~\"$job\"}[1m])) or\nsum(rate(cortex_cache_fetched_keys_total{name=~\"frontend.+\", job=~\"$job\"}[1m])) - sum(rate(cortex_cache_hits_total{name=~\"frontend.+\", job=~\"$job\"}[1m]))\n", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "Miss Rate", - "legendLink": null, - "step": 10 + "legendLink": null } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, "title": "Results Cache misses", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] + "type": "timeseries" } ], "repeat": null, @@ -734,64 +565,53 @@ "height": "250px", "panels": [ { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", "description": "### Sharded Queries Ratio\nThe % of queries that have been successfully rewritten and executed in a shardable way.\nThis panel takes in account only type of queries which are supported by query sharding (eg. range queries).\n\n", - "fill": 1, + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 1, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "short" + }, + "overrides": [ ] + }, "id": 9, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, "links": [ ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 6, - "stack": false, - "steppedLine": false, "targets": [ { "expr": "sum(rate(cortex_frontend_query_sharding_rewrites_succeeded_total{job=~\"$job\"}[$__rate_interval])) /\nsum(rate(cortex_frontend_query_sharding_rewrites_attempted_total{job=~\"$job\"}[$__rate_interval]))\n", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "sharded queries ratio", - "legendLink": null, - "step": 10 + "legendLink": null } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, "title": "Sharded Queries Ratio", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, + "type": "timeseries", "yaxes": [ { "format": "percentunit", @@ -812,85 +632,71 @@ ] }, { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", "description": "### Number of Sharded Queries per Query\nHow many sharded queries have been executed for a single input query. It tracks only queries which have\nbeen successfully rewritten in a shardable way.\n\n", - "fill": 1, + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 1, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "short" + }, + "overrides": [ ] + }, "id": 10, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, "links": [ ], "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 6, - "stack": false, - "steppedLine": false, "targets": [ { "expr": "histogram_quantile(0.99, sum(rate(cortex_frontend_sharded_queries_per_query_bucket{job=~\"$job\"}[$__rate_interval])) by (le)) * 1", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "99th Percentile", - "refId": "A", - "step": 10 + "refId": "A" }, { "expr": "histogram_quantile(0.50, sum(rate(cortex_frontend_sharded_queries_per_query_bucket{job=~\"$job\"}[$__rate_interval])) by (le)) * 1", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "50th Percentile", - "refId": "B", - "step": 10 + "refId": "B" }, { "expr": "sum(rate(cortex_frontend_sharded_queries_per_query_sum{job=~\"$job\"}[$__rate_interval])) * 1 / sum(rate(cortex_frontend_sharded_queries_per_query_count{job=~\"$job\"}[$__rate_interval]))", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "Average", - "refId": "C", - "step": 10 + "refId": "C" } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, "title": "Number of Sharded Queries per Query", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, + "type": "timeseries", "yaxes": [ { - "format": "short", + "format": "ms", "label": null, "logBase": 1, "max": null, @@ -920,235 +726,151 @@ "height": "250px", "panels": [ { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 1, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "ms" + }, + "overrides": [ ] + }, "fill": 10, "id": 11, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, "linewidth": 0, "links": [ ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 4, "stack": true, - "steppedLine": false, "targets": [ { "expr": "max by (slice) (prometheus_engine_query_duration_seconds{quantile=\"0.9\",job=~\"$job\"}) * 1e3", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "{{slice}}", - "legendLink": null, - "step": 10 + "legendLink": null } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, "title": "Stages", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, - "yaxes": [ - { - "format": "ms", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] + "type": "timeseries" }, { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", - "fill": 1, + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 1, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "short" + }, + "overrides": [ ] + }, "id": 12, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, "links": [ ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 4, - "stack": false, - "steppedLine": false, "targets": [ { "expr": "sum(rate(cortex_cache_fetched_keys{job=~\"$job\",name=\"chunksmemcache\"}[1m])) - sum(rate(cortex_cache_hits{job=~\"$job\",name=\"chunksmemcache\"}[1m])) or\nsum(rate(cortex_cache_fetched_keys_total{job=~\"$job\",name=\"chunksmemcache\"}[1m])) - sum(rate(cortex_cache_hits_total{job=~\"$job\",name=\"chunksmemcache\"}[1m]))\n", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "Hit rate", - "legendLink": null, - "step": 10 + "legendLink": null } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, "title": "Chunk cache misses", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] + "type": "timeseries" }, { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", - "fill": 1, + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 1, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "short" + }, + "overrides": [ ] + }, "id": 13, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, "links": [ ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 4, - "stack": false, - "steppedLine": false, "targets": [ { "expr": "sum(rate(cortex_cache_corrupt_chunks_total{job=~\"$job\"}[1m]))", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "Corrupt chunks", - "legendLink": null, - "step": 10 + "legendLink": null } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, "title": "Chunk cache corruptions", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] + "type": "timeseries" } ], "repeat": null, @@ -1163,81 +885,67 @@ "height": "250px", "panels": [ { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", - "fill": 1, + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 1, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "short" + }, + "overrides": [ ] + }, "id": 14, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, "links": [ ], "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 4, - "stack": false, - "steppedLine": false, "targets": [ { "expr": "histogram_quantile(0.99, sum by (le) (cluster_job:cortex_ingester_queried_series_bucket:sum_rate{job=~\"$job\"})) * 1", "format": "time_series", - "intervalFactor": 2, - "legendFormat": "99th Percentile", - "refId": "A", - "step": 10 + "legendFormat": "99th percentile", + "refId": "A" }, { "expr": "histogram_quantile(0.50, sum by (le) (cluster_job:cortex_ingester_queried_series_bucket:sum_rate{job=~\"$job\"})) * 1", "format": "time_series", - "intervalFactor": 2, - "legendFormat": "50th Percentile", - "refId": "B", - "step": 10 + "legendFormat": "50th percentile", + "refId": "B" }, { "expr": "1 * sum(cluster_job:cortex_ingester_queried_series_sum:sum_rate{job=~\"$job\"}) / sum(cluster_job:cortex_ingester_queried_series_count:sum_rate{job=~\"$job\"})", "format": "time_series", - "intervalFactor": 2, "legendFormat": "Average", - "refId": "C", - "step": 10 + "refId": "C" } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, "title": "Series per Query", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, + "type": "timeseries", "yaxes": [ { - "format": "short", + "format": "ms", "label": null, "logBase": 1, "max": null, @@ -1255,81 +963,67 @@ ] }, { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", - "fill": 1, + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 1, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "short" + }, + "overrides": [ ] + }, "id": 15, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, "links": [ ], "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 4, - "stack": false, - "steppedLine": false, "targets": [ { "expr": "histogram_quantile(0.99, sum by (le) (cluster_job:cortex_ingester_queried_chunks_bucket:sum_rate{job=~\"$job\"})) * 1", "format": "time_series", - "intervalFactor": 2, - "legendFormat": "99th Percentile", - "refId": "A", - "step": 10 + "legendFormat": "99th percentile", + "refId": "A" }, { "expr": "histogram_quantile(0.50, sum by (le) (cluster_job:cortex_ingester_queried_chunks_bucket:sum_rate{job=~\"$job\"})) * 1", "format": "time_series", - "intervalFactor": 2, - "legendFormat": "50th Percentile", - "refId": "B", - "step": 10 + "legendFormat": "50th percentile", + "refId": "B" }, { "expr": "1 * sum(cluster_job:cortex_ingester_queried_chunks_sum:sum_rate{job=~\"$job\"}) / sum(cluster_job:cortex_ingester_queried_chunks_count:sum_rate{job=~\"$job\"})", "format": "time_series", - "intervalFactor": 2, "legendFormat": "Average", - "refId": "C", - "step": 10 + "refId": "C" } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, "title": "Chunks per Query", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, + "type": "timeseries", "yaxes": [ { - "format": "short", + "format": "ms", "label": null, "logBase": 1, "max": null, @@ -1347,81 +1041,67 @@ ] }, { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", - "fill": 1, + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 1, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "short" + }, + "overrides": [ ] + }, "id": 16, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, "links": [ ], "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 4, - "stack": false, - "steppedLine": false, "targets": [ { "expr": "histogram_quantile(0.99, sum by (le) (cluster_job:cortex_ingester_queried_samples_bucket:sum_rate{job=~\"$job\"})) * 1", "format": "time_series", - "intervalFactor": 2, - "legendFormat": "99th Percentile", - "refId": "A", - "step": 10 + "legendFormat": "99th percentile", + "refId": "A" }, { "expr": "histogram_quantile(0.50, sum by (le) (cluster_job:cortex_ingester_queried_samples_bucket:sum_rate{job=~\"$job\"})) * 1", "format": "time_series", - "intervalFactor": 2, - "legendFormat": "50th Percentile", - "refId": "B", - "step": 10 + "legendFormat": "50th percentile", + "refId": "B" }, { "expr": "1 * sum(cluster_job:cortex_ingester_queried_samples_sum:sum_rate{job=~\"$job\"}) / sum(cluster_job:cortex_ingester_queried_samples_count:sum_rate{job=~\"$job\"})", "format": "time_series", - "intervalFactor": 2, "legendFormat": "Average", - "refId": "C", - "step": 10 + "refId": "C" } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, "title": "Samples per Query", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, + "type": "timeseries", "yaxes": [ { - "format": "short", + "format": "ms", "label": null, "logBase": 1, "max": null, @@ -1451,84 +1131,70 @@ "height": "250px", "panels": [ { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", - "fill": 1, + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 1, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "short" + }, + "overrides": [ ] + }, "id": 17, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, "links": [ ], "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 4, - "stack": false, - "steppedLine": false, "targets": [ { "expr": "histogram_quantile(0.99, sum(rate(cortex_querier_storegateway_instances_hit_per_query_bucket{job=~\"$job\"}[$__rate_interval])) by (le)) * 1", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "99th Percentile", - "refId": "A", - "step": 10 + "refId": "A" }, { "expr": "histogram_quantile(0.50, sum(rate(cortex_querier_storegateway_instances_hit_per_query_bucket{job=~\"$job\"}[$__rate_interval])) by (le)) * 1", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "50th Percentile", - "refId": "B", - "step": 10 + "refId": "B" }, { "expr": "sum(rate(cortex_querier_storegateway_instances_hit_per_query_sum{job=~\"$job\"}[$__rate_interval])) * 1 / sum(rate(cortex_querier_storegateway_instances_hit_per_query_count{job=~\"$job\"}[$__rate_interval]))", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "Average", - "refId": "C", - "step": 10 + "refId": "C" } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, "title": "Number of store-gateways hit per Query", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, + "type": "timeseries", "yaxes": [ { - "format": "short", + "format": "ms", "label": null, "logBase": 1, "max": null, @@ -1546,84 +1212,70 @@ ] }, { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", - "fill": 1, + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 1, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "short" + }, + "overrides": [ ] + }, "id": 18, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, "links": [ ], "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 4, - "stack": false, - "steppedLine": false, "targets": [ { "expr": "histogram_quantile(0.99, sum(rate(cortex_querier_storegateway_refetches_per_query_bucket{job=~\"$job\"}[$__rate_interval])) by (le)) * 1", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "99th Percentile", - "refId": "A", - "step": 10 + "refId": "A" }, { "expr": "histogram_quantile(0.50, sum(rate(cortex_querier_storegateway_refetches_per_query_bucket{job=~\"$job\"}[$__rate_interval])) by (le)) * 1", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "50th Percentile", - "refId": "B", - "step": 10 + "refId": "B" }, { "expr": "sum(rate(cortex_querier_storegateway_refetches_per_query_sum{job=~\"$job\"}[$__rate_interval])) * 1 / sum(rate(cortex_querier_storegateway_refetches_per_query_count{job=~\"$job\"}[$__rate_interval]))", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "Average", - "refId": "C", - "step": 10 + "refId": "C" } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, "title": "Refetches of missing blocks per Query", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, + "type": "timeseries", "yaxes": [ { - "format": "short", + "format": "ms", "label": null, "logBase": 1, "max": null, @@ -1641,63 +1293,52 @@ ] }, { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", - "fill": 1, + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 1, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "short" + }, + "overrides": [ ] + }, "id": 19, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, "links": [ ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 4, - "stack": false, - "steppedLine": false, "targets": [ { "expr": "sum(rate(cortex_querier_blocks_consistency_checks_failed_total{job=~\"$job\"}[1m])) / sum(rate(cortex_querier_blocks_consistency_checks_total{job=~\"$job\"}[1m]))", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "Failure Rate", - "legendLink": null, - "step": 10 + "legendLink": null } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, "title": "Consistency checks failed", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, + "type": "timeseries", "yaxes": [ { "format": "percentunit", @@ -1730,265 +1371,191 @@ "height": "250px", "panels": [ { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", - "fill": 1, + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 1, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "short" + }, + "overrides": [ ] + }, "id": 20, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, "links": [ ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 4, - "stack": false, - "steppedLine": false, "targets": [ { "expr": "max(cortex_bucket_index_loaded{job=~\"$job\"})", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "Max", - "legendLink": null, - "step": 10 + "legendLink": null }, { "expr": "min(cortex_bucket_index_loaded{job=~\"$job\"})", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "Min", - "legendLink": null, - "step": 10 + "legendLink": null }, { "expr": "avg(cortex_bucket_index_loaded{job=~\"$job\"})", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "Average", - "legendLink": null, - "step": 10 + "legendLink": null } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, "title": "Bucket indexes loaded (per querier)", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] + "type": "timeseries" }, { "aliasColors": { "failed": "#E24D42", "successful": "#7EB26D" }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 1, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "short" + }, + "overrides": [ ] + }, "fill": 10, "id": 21, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, "linewidth": 0, "links": [ ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 4, "stack": true, - "steppedLine": false, "targets": [ { "expr": "sum(rate(cortex_bucket_index_loads_total{job=~\"$job\"}[$__rate_interval])) - sum(rate(cortex_bucket_index_load_failures_total{job=~\"$job\"}[$__rate_interval]))", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "successful", - "legendLink": null, - "step": 10 + "legendLink": null }, { "expr": "sum(rate(cortex_bucket_index_load_failures_total{job=~\"$job\"}[$__rate_interval]))", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "failed", - "legendLink": null, - "step": 10 + "legendLink": null } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, "title": "Bucket indexes load / sec", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] + "type": "timeseries" }, { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", - "fill": 1, + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 1, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "ms" + }, + "overrides": [ ] + }, "id": 22, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, "links": [ ], "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 4, - "stack": false, - "steppedLine": false, "targets": [ { "expr": "histogram_quantile(0.99, sum(rate(cortex_bucket_index_load_duration_seconds_bucket{job=~\"$job\"}[$__rate_interval])) by (le)) * 1e3", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "99th Percentile", - "refId": "A", - "step": 10 + "refId": "A" }, { "expr": "histogram_quantile(0.50, sum(rate(cortex_bucket_index_load_duration_seconds_bucket{job=~\"$job\"}[$__rate_interval])) by (le)) * 1e3", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "50th Percentile", - "refId": "B", - "step": 10 + "refId": "B" }, { "expr": "sum(rate(cortex_bucket_index_load_duration_seconds_sum{job=~\"$job\"}[$__rate_interval])) * 1e3 / sum(rate(cortex_bucket_index_load_duration_seconds_count{job=~\"$job\"}[$__rate_interval]))", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "Average", - "refId": "C", - "step": 10 + "refId": "C" } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, "title": "Bucket indexes load latency", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, + "type": "timeseries", "yaxes": [ { "format": "ms", @@ -2021,235 +1588,154 @@ "height": "250px", "panels": [ { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", - "fill": 1, + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 1, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "ops" + }, + "overrides": [ ] + }, "id": 23, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, "links": [ ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 4, - "stack": false, - "steppedLine": false, "targets": [ { "expr": "sum(rate(cortex_bucket_store_series_blocks_queried_sum{component=\"store-gateway\",job=~\"$job\"}[$__rate_interval]))", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "blocks", - "legendLink": null, - "step": 10 + "legendLink": null } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, "title": "Blocks queried / sec", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, - "yaxes": [ - { - "format": "ops", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] + "type": "timeseries" }, { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 1, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "ops" + }, + "overrides": [ ] + }, "fill": 10, "id": 24, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, "linewidth": 0, "links": [ ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 4, "stack": true, - "steppedLine": false, "targets": [ { "expr": "sum by(data_type) (rate(cortex_bucket_store_series_data_fetched_sum{component=\"store-gateway\",job=~\"$job\"}[$__rate_interval]))", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "{{data_type}}", - "legendLink": null, - "step": 10 + "legendLink": null } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, "title": "Data fetched / sec", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, - "yaxes": [ - { - "format": "ops", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] + "type": "timeseries" }, { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 1, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "ops" + }, + "overrides": [ ] + }, "fill": 10, "id": 25, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, "linewidth": 0, "links": [ ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 4, "stack": true, - "steppedLine": false, "targets": [ { "expr": "sum by(data_type) (rate(cortex_bucket_store_series_data_touched_sum{component=\"store-gateway\",job=~\"$job\"}[$__rate_interval]))", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "{{data_type}}", - "legendLink": null, - "step": 10 + "legendLink": null } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, "title": "Data touched / sec", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, - "yaxes": [ - { - "format": "ops", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] + "type": "timeseries" } ], "repeat": null, @@ -2264,81 +1750,67 @@ "height": "250px", "panels": [ { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", - "fill": 1, + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 1, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "short" + }, + "overrides": [ ] + }, "id": 26, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, "links": [ ], "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 4, - "stack": false, - "steppedLine": false, "targets": [ { "expr": "histogram_quantile(0.99, sum(rate(cortex_bucket_store_series_get_all_duration_seconds_bucket{component=\"store-gateway\",job=~\"$job\"}[$__rate_interval])) by (le)) * 1e3", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "99th Percentile", - "refId": "A", - "step": 10 + "refId": "A" }, { "expr": "histogram_quantile(0.50, sum(rate(cortex_bucket_store_series_get_all_duration_seconds_bucket{component=\"store-gateway\",job=~\"$job\"}[$__rate_interval])) by (le)) * 1e3", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "50th Percentile", - "refId": "B", - "step": 10 + "refId": "B" }, { "expr": "sum(rate(cortex_bucket_store_series_get_all_duration_seconds_sum{component=\"store-gateway\",job=~\"$job\"}[$__rate_interval])) * 1e3 / sum(rate(cortex_bucket_store_series_get_all_duration_seconds_count{component=\"store-gateway\",job=~\"$job\"}[$__rate_interval]))", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "Average", - "refId": "C", - "step": 10 + "refId": "C" } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, "title": "Series fetch duration (per request)", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, + "type": "timeseries", "yaxes": [ { "format": "ms", @@ -2359,81 +1831,67 @@ ] }, { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", - "fill": 1, + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 1, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "short" + }, + "overrides": [ ] + }, "id": 27, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, "links": [ ], "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 4, - "stack": false, - "steppedLine": false, "targets": [ { "expr": "histogram_quantile(0.99, sum(rate(cortex_bucket_store_series_merge_duration_seconds_bucket{component=\"store-gateway\",job=~\"$job\"}[$__rate_interval])) by (le)) * 1e3", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "99th Percentile", - "refId": "A", - "step": 10 + "refId": "A" }, { "expr": "histogram_quantile(0.50, sum(rate(cortex_bucket_store_series_merge_duration_seconds_bucket{component=\"store-gateway\",job=~\"$job\"}[$__rate_interval])) by (le)) * 1e3", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "50th Percentile", - "refId": "B", - "step": 10 + "refId": "B" }, { "expr": "sum(rate(cortex_bucket_store_series_merge_duration_seconds_sum{component=\"store-gateway\",job=~\"$job\"}[$__rate_interval])) * 1e3 / sum(rate(cortex_bucket_store_series_merge_duration_seconds_count{component=\"store-gateway\",job=~\"$job\"}[$__rate_interval]))", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "Average", - "refId": "C", - "step": 10 + "refId": "C" } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, "title": "Series merge duration (per request)", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, + "type": "timeseries", "yaxes": [ { "format": "ms", @@ -2454,81 +1912,52 @@ ] }, { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", - "fill": 1, + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 1, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "short" + }, + "overrides": [ ] + }, "id": 28, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, "links": [ ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 4, - "stack": false, - "steppedLine": false, "targets": [ { "expr": "sum(rate(cortex_bucket_store_series_result_series_sum{component=\"store-gateway\",job=~\"$job\"}[$__rate_interval])) / sum(rate(cortex_bucket_store_series_result_series_count{component=\"store-gateway\",job=~\"$job\"}[$__rate_interval]))", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "avg series returned", - "legendLink": null, - "step": 10 + "legendLink": null } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, "title": "Series returned (per request)", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] + "type": "timeseries" } ], "repeat": null, @@ -2543,259 +1972,176 @@ "height": "250px", "panels": [ { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", - "fill": 1, + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 1, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "short" + }, + "overrides": [ ] + }, "id": 29, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, "links": [ ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 4, - "stack": false, - "steppedLine": false, "targets": [ { "expr": "sum(cortex_bucket_store_blocks_loaded{component=\"store-gateway\",job=~\"$job\"}) without (user)", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "{{pod}}", - "legendLink": null, - "step": 10 + "legendLink": null } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, "title": "Blocks currently loaded", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] + "type": "timeseries" }, { "aliasColors": { "failed": "#E24D42", "successful": "#7EB26D" }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 1, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "short" + }, + "overrides": [ ] + }, "fill": 10, "id": 30, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, "linewidth": 0, "links": [ ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 4, "stack": true, - "steppedLine": false, "targets": [ { "expr": "sum(rate(cortex_bucket_store_block_loads_total{component=\"store-gateway\",job=~\"$job\"}[$__rate_interval])) - sum(rate(cortex_bucket_store_block_load_failures_total{component=\"store-gateway\",job=~\"$job\"}[$__rate_interval]))", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "successful", - "legendLink": null, - "step": 10 + "legendLink": null }, { "expr": "sum(rate(cortex_bucket_store_block_load_failures_total{component=\"store-gateway\",job=~\"$job\"}[$__rate_interval]))", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "failed", - "legendLink": null, - "step": 10 + "legendLink": null } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, "title": "Blocks loaded / sec", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] + "type": "timeseries" }, { "aliasColors": { "failed": "#E24D42", "successful": "#7EB26D" }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 1, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "short" + }, + "overrides": [ ] + }, "fill": 10, "id": 31, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, "linewidth": 0, "links": [ ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 4, "stack": true, - "steppedLine": false, "targets": [ { "expr": "sum(rate(cortex_bucket_store_block_drops_total{component=\"store-gateway\",job=~\"$job\"}[$__rate_interval])) - sum(rate(cortex_bucket_store_block_drop_failures_total{component=\"store-gateway\",job=~\"$job\"}[$__rate_interval]))", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "successful", - "legendLink": null, - "step": 10 + "legendLink": null }, { "expr": "sum(rate(cortex_bucket_store_block_drop_failures_total{component=\"store-gateway\",job=~\"$job\"}[$__rate_interval]))", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "failed", - "legendLink": null, - "step": 10 + "legendLink": null } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, "title": "Blocks dropped / sec", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] + "type": "timeseries" } ], "repeat": null, @@ -2810,158 +2156,115 @@ "height": "250px", "panels": [ { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", - "fill": 1, + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 1, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "short" + }, + "overrides": [ ] + }, "id": 32, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, "links": [ ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 4, - "stack": false, - "steppedLine": false, "targets": [ { "expr": "cortex_bucket_store_indexheader_lazy_load_total{job=~\"$job\"} - cortex_bucket_store_indexheader_lazy_unload_total{job=~\"$job\"}", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "{{pod}}", - "legendLink": null, - "step": 10 + "legendLink": null } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, "title": "Lazy loaded index-headers", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] + "type": "timeseries" }, { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", - "fill": 1, + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 1, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "ms" + }, + "overrides": [ ] + }, "id": 33, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, "links": [ ], "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 4, - "stack": false, - "steppedLine": false, "targets": [ { "expr": "histogram_quantile(0.99, sum(rate(cortex_bucket_store_indexheader_lazy_load_duration_seconds_bucket{job=~\"$job\"}[$__rate_interval])) by (le)) * 1e3", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "99th Percentile", - "refId": "A", - "step": 10 + "refId": "A" }, { "expr": "histogram_quantile(0.50, sum(rate(cortex_bucket_store_indexheader_lazy_load_duration_seconds_bucket{job=~\"$job\"}[$__rate_interval])) by (le)) * 1e3", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "50th Percentile", - "refId": "B", - "step": 10 + "refId": "B" }, { "expr": "sum(rate(cortex_bucket_store_indexheader_lazy_load_duration_seconds_sum{job=~\"$job\"}[$__rate_interval])) * 1e3 / sum(rate(cortex_bucket_store_indexheader_lazy_load_duration_seconds_count{job=~\"$job\"}[$__rate_interval]))", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "Average", - "refId": "C", - "step": 10 + "refId": "C" } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, "title": "Index-header lazy load duration", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, + "type": "timeseries", "yaxes": [ { "format": "ms", @@ -2982,63 +2285,52 @@ ] }, { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", - "fill": 1, + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 1, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "short" + }, + "overrides": [ ] + }, "id": 34, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, "links": [ ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 4, - "stack": false, - "steppedLine": false, "targets": [ { "expr": "sum(rate(cortex_bucket_store_series_hash_cache_hits_total{job=~\"$job\"}[$__rate_interval]))\n/\nsum(rate(cortex_bucket_store_series_hash_cache_requests_total{job=~\"$job\"}[$__rate_interval]))\n", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "hit ratio", - "legendLink": null, - "step": 10 + "legendLink": null } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, "title": "Series hash cache hit ratio", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, + "type": "timeseries", "yaxes": [ { "format": "percentunit", @@ -3080,7 +2372,7 @@ "value": "Cortex" }, "hide": 0, - "label": null, + "label": "Data source", "name": "datasource", "options": [ ], "query": "prometheus", diff --git a/docs/getting-started/dashboards/cortex-reads.json b/docs/getting-started/dashboards/cortex-reads.json index a02f9c7aa0..558982943f 100644 --- a/docs/getting-started/dashboards/cortex-reads.json +++ b/docs/getting-started/dashboards/cortex-reads.json @@ -50,158 +50,104 @@ "height": "100px", "panels": [ { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", "description": "### Instant queries per second\nRate of instant queries per second being made to the system.\nIncludes both queries made to the /prometheus API as\nwell as queries from the ruler.\n\n", - "fill": 1, + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 1, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "short" + }, + "overrides": [ ] + }, "format": "reqps", "id": 2, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, "links": [ ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 6, - "stack": false, - "steppedLine": false, "targets": [ { "expr": "sum(\n rate(\n cortex_request_duration_seconds_count{\n job=~\"$job\",\n route=~\"(prometheus|api_prom)_api_v1_query\"\n }[$__rate_interval]\n )\n) +\nsum(\n rate(\n cortex_prometheus_rule_evaluations_total{\n job=~\"$job\"\n }[$__rate_interval]\n )\n)\n", "format": "time_series", "instant": true, - "intervalFactor": 2, "refId": "A" } ], "thresholds": "70,80", - "timeFrom": null, - "timeShift": null, "title": "Instant queries / sec", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "singlestat", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] + "type": "singlestat" }, { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", "description": "### Range queries per second\nRate of range queries per second being made to\nCortex via the /prometheus API.\n\n", - "fill": 1, + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 1, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "short" + }, + "overrides": [ ] + }, "format": "reqps", "id": 3, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, "links": [ ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 6, - "stack": false, - "steppedLine": false, "targets": [ { "expr": "sum(\n rate(\n cortex_request_duration_seconds_count{\n job=~\"$job\",\n route=~\"(prometheus|api_prom)_api_v1_query_range\"\n }[$__rate_interval]\n )\n)\n", "format": "time_series", "instant": true, - "intervalFactor": 2, "refId": "A" } ], "thresholds": "70,80", - "timeFrom": null, - "timeShift": null, "title": "Range queries / sec", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "singlestat", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] + "type": "singlestat" } ], "repeat": null, @@ -222,157 +168,257 @@ "3xx": "#6ED0E0", "4xx": "#EF843C", "5xx": "#E24D42", + "OK": "#7EB26D", + "cancel": "#A9A9A9", "error": "#E24D42", "success": "#7EB26D" }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 100, + "lineWidth": 0, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + } + }, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "reqps" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "1xx" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EAB839", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "2xx" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#7EB26D", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "3xx" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#6ED0E0", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "4xx" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EF843C", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "5xx" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E24D42", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "OK" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#7EB26D", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "cancel" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#A9A9A9", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "error" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E24D42", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "success" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#7EB26D", + "mode": "fixed" + } + } + ] + } + ] + }, "fill": 10, "id": 4, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, "linewidth": 0, "links": [ ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 4, "stack": true, - "steppedLine": false, "targets": [ { - "expr": "sum by (status) (\n label_replace(label_replace(rate(cortex_request_duration_seconds_count{job=~\"$job\", route=~\"(prometheus|api_prom)_api_v1_.+\"}[$__rate_interval]),\n \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"),\n \"status\", \"${1}\", \"status_code\", \"([a-z]+)\"))\n", + "expr": "sum by (status) (\n label_replace(label_replace(rate(cortex_request_duration_seconds_count{job=~\"$job\", route=~\"(prometheus|api_prom)_api_v1_.+\"}[$__rate_interval]),\n \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"),\n \"status\", \"${1}\", \"status_code\", \"([a-zA-Z]+)\"))\n", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "{{status}}", - "refId": "A", - "step": 10 + "refId": "A" } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, "title": "Requests / sec", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] + "type": "timeseries" }, { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", - "fill": 1, - "id": 5, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 1, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "ms" + }, + "overrides": [ ] }, - "lines": true, - "linewidth": 1, + "id": 5, "links": [ ], "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 4, - "stack": false, - "steppedLine": false, "targets": [ { "expr": "histogram_quantile(0.99, sum by (le) (cluster_job_route:cortex_request_duration_seconds_bucket:sum_rate{job=~\"$job\", route=~\"(prometheus|api_prom)_api_v1_.+\"})) * 1e3", "format": "time_series", - "intervalFactor": 2, - "legendFormat": "99th Percentile", - "refId": "A", - "step": 10 + "legendFormat": "99th percentile", + "refId": "A" }, { "expr": "histogram_quantile(0.50, sum by (le) (cluster_job_route:cortex_request_duration_seconds_bucket:sum_rate{job=~\"$job\", route=~\"(prometheus|api_prom)_api_v1_.+\"})) * 1e3", "format": "time_series", - "intervalFactor": 2, - "legendFormat": "50th Percentile", - "refId": "B", - "step": 10 + "legendFormat": "50th percentile", + "refId": "B" }, { "expr": "1e3 * sum(cluster_job_route:cortex_request_duration_seconds_sum:sum_rate{job=~\"$job\", route=~\"(prometheus|api_prom)_api_v1_.+\"}) / sum(cluster_job_route:cortex_request_duration_seconds_count:sum_rate{job=~\"$job\", route=~\"(prometheus|api_prom)_api_v1_.+\"})", "format": "time_series", - "intervalFactor": 2, "legendFormat": "Average", - "refId": "C", - "step": 10 + "refId": "C" } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, "title": "Latency", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, + "type": "timeseries", "yaxes": [ { "format": "ms", @@ -393,73 +439,59 @@ ] }, { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 1, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "s" + }, + "overrides": [ ] + }, "fill": 0, "id": 6, "legend": { "show": false }, - "lines": true, - "linewidth": 1, "links": [ ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 4, - "stack": false, - "steppedLine": false, "targets": [ { "expr": "histogram_quantile(0.99, sum by(le, pod) (rate(cortex_request_duration_seconds_bucket{job=~\"$job\", route=~\"(prometheus|api_prom)_api_v1_.+\"}[$__rate_interval])))", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "", - "legendLink": null, - "step": 10 + "legendLink": null } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, "title": "Per pod p99 Latency", "tooltip": { "sort": 2 }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, - "yaxes": [ - { - "format": "s", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] + "type": "timeseries" } ], "repeat": null, @@ -480,157 +512,257 @@ "3xx": "#6ED0E0", "4xx": "#EF843C", "5xx": "#E24D42", + "OK": "#7EB26D", + "cancel": "#A9A9A9", "error": "#E24D42", "success": "#7EB26D" }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 100, + "lineWidth": 0, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + } + }, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "reqps" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "1xx" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EAB839", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "2xx" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#7EB26D", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "3xx" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#6ED0E0", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "4xx" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EF843C", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "5xx" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E24D42", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "OK" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#7EB26D", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "cancel" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#A9A9A9", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "error" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E24D42", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "success" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#7EB26D", + "mode": "fixed" + } + } + ] + } + ] + }, "fill": 10, "id": 7, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, "linewidth": 0, "links": [ ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 4, "stack": true, - "steppedLine": false, "targets": [ { - "expr": "sum by (status) (\n label_replace(label_replace(rate(cortex_request_duration_seconds_count{job=~\"$job\", route=~\"(prometheus|api_prom)_api_v1_.+\"}[$__rate_interval]),\n \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"),\n \"status\", \"${1}\", \"status_code\", \"([a-z]+)\"))\n", + "expr": "sum by (status) (\n label_replace(label_replace(rate(cortex_request_duration_seconds_count{job=~\"$job\", route=~\"(prometheus|api_prom)_api_v1_.+\"}[$__rate_interval]),\n \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"),\n \"status\", \"${1}\", \"status_code\", \"([a-zA-Z]+)\"))\n", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "{{status}}", - "refId": "A", - "step": 10 + "refId": "A" } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, "title": "Requests / sec", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] + "type": "timeseries" }, { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", - "fill": 1, - "id": 8, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 1, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "ms" + }, + "overrides": [ ] }, - "lines": true, - "linewidth": 1, + "id": 8, "links": [ ], "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 4, - "stack": false, - "steppedLine": false, "targets": [ { "expr": "histogram_quantile(0.99, sum by (le) (cluster_job_route:cortex_request_duration_seconds_bucket:sum_rate{job=~\"$job\", route=~\"(prometheus|api_prom)_api_v1_.+\"})) * 1e3", "format": "time_series", - "intervalFactor": 2, - "legendFormat": "99th Percentile", - "refId": "A", - "step": 10 + "legendFormat": "99th percentile", + "refId": "A" }, { "expr": "histogram_quantile(0.50, sum by (le) (cluster_job_route:cortex_request_duration_seconds_bucket:sum_rate{job=~\"$job\", route=~\"(prometheus|api_prom)_api_v1_.+\"})) * 1e3", "format": "time_series", - "intervalFactor": 2, - "legendFormat": "50th Percentile", - "refId": "B", - "step": 10 + "legendFormat": "50th percentile", + "refId": "B" }, { "expr": "1e3 * sum(cluster_job_route:cortex_request_duration_seconds_sum:sum_rate{job=~\"$job\", route=~\"(prometheus|api_prom)_api_v1_.+\"}) / sum(cluster_job_route:cortex_request_duration_seconds_count:sum_rate{job=~\"$job\", route=~\"(prometheus|api_prom)_api_v1_.+\"})", "format": "time_series", - "intervalFactor": 2, "legendFormat": "Average", - "refId": "C", - "step": 10 + "refId": "C" } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, "title": "Latency", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, + "type": "timeseries", "yaxes": [ { "format": "ms", @@ -651,73 +783,59 @@ ] }, { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 1, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "s" + }, + "overrides": [ ] + }, "fill": 0, "id": 9, "legend": { "show": false }, - "lines": true, - "linewidth": 1, "links": [ ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 4, - "stack": false, - "steppedLine": false, "targets": [ { "expr": "histogram_quantile(0.99, sum by(le, pod) (rate(cortex_request_duration_seconds_bucket{job=~\"$job\", route=~\"(prometheus|api_prom)_api_v1_.+\"}[$__rate_interval])))", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "", - "legendLink": null, - "step": 10 + "legendLink": null } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, "title": "Per pod p99 Latency", "tooltip": { "sort": 2 }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, - "yaxes": [ - { - "format": "s", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] + "type": "timeseries" } ], "repeat": null, @@ -749,160 +867,260 @@ "3xx": "#6ED0E0", "4xx": "#EF843C", "5xx": "#E24D42", + "OK": "#7EB26D", + "cancel": "#A9A9A9", "error": "#E24D42", "success": "#7EB26D" }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 100, + "lineWidth": 0, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + } + }, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "reqps" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "1xx" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EAB839", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "2xx" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#7EB26D", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "3xx" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#6ED0E0", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "4xx" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EF843C", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "5xx" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E24D42", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "OK" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#7EB26D", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "cancel" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#A9A9A9", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "error" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E24D42", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "success" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#7EB26D", + "mode": "fixed" + } + } + ] + } + ] + }, "fill": 10, "id": 11, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, "linewidth": 0, "links": [ ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 4, "stack": true, - "steppedLine": false, "targets": [ { - "expr": "sum by (status) (\n label_replace(label_replace(rate(cortex_query_scheduler_queue_duration_seconds_count{job=~\"$job\"}[$__rate_interval]),\n \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"),\n \"status\", \"${1}\", \"status_code\", \"([a-z]+)\"))\n", + "expr": "sum by (status) (\n label_replace(label_replace(rate(cortex_query_scheduler_queue_duration_seconds_count{job=~\"$job\"}[$__rate_interval]),\n \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"),\n \"status\", \"${1}\", \"status_code\", \"([a-zA-Z]+)\"))\n", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "{{status}}", - "refId": "A", - "step": 10 + "refId": "A" } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, "title": "Requests / sec", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] + "type": "timeseries" }, { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", - "fill": 1, - "id": 12, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 1, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "short" + }, + "overrides": [ ] }, - "lines": true, - "linewidth": 1, + "id": 12, "links": [ ], "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 4, - "stack": false, - "steppedLine": false, "targets": [ { "expr": "histogram_quantile(0.99, sum(rate(cortex_query_scheduler_queue_duration_seconds_bucket{job=~\"$job\"}[$__rate_interval])) by (le)) * 1e3", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "99th Percentile", - "refId": "A", - "step": 10 + "refId": "A" }, { "expr": "histogram_quantile(0.50, sum(rate(cortex_query_scheduler_queue_duration_seconds_bucket{job=~\"$job\"}[$__rate_interval])) by (le)) * 1e3", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "50th Percentile", - "refId": "B", - "step": 10 + "refId": "B" }, { "expr": "sum(rate(cortex_query_scheduler_queue_duration_seconds_sum{job=~\"$job\"}[$__rate_interval])) * 1e3 / sum(rate(cortex_query_scheduler_queue_duration_seconds_count{job=~\"$job\"}[$__rate_interval]))", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "Average", - "refId": "C", - "step": 10 + "refId": "C" } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, "title": "Latency (Time in Queue)", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, + "type": "timeseries", "yaxes": [ { "format": "ms", @@ -941,157 +1159,257 @@ "3xx": "#6ED0E0", "4xx": "#EF843C", "5xx": "#E24D42", + "OK": "#7EB26D", + "cancel": "#A9A9A9", "error": "#E24D42", "success": "#7EB26D" }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 100, + "lineWidth": 0, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + } + }, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "reqps" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "1xx" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EAB839", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "2xx" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#7EB26D", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "3xx" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#6ED0E0", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "4xx" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EF843C", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "5xx" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E24D42", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "OK" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#7EB26D", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "cancel" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#A9A9A9", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "error" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E24D42", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "success" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#7EB26D", + "mode": "fixed" + } + } + ] + } + ] + }, "fill": 10, "id": 13, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, "linewidth": 0, "links": [ ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 6, "stack": true, - "steppedLine": false, "targets": [ { - "expr": "sum by (status) (\n label_replace(label_replace(rate(cortex_cache_request_duration_seconds_count{method=~\"frontend.+\", job=~\"$job\"}[$__rate_interval]),\n \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"),\n \"status\", \"${1}\", \"status_code\", \"([a-z]+)\"))\n", + "expr": "sum by (status) (\n label_replace(label_replace(rate(cortex_cache_request_duration_seconds_count{method=~\"frontend.+\", job=~\"$job\"}[$__rate_interval]),\n \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"),\n \"status\", \"${1}\", \"status_code\", \"([a-zA-Z]+)\"))\n", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "{{status}}", - "refId": "A", - "step": 10 + "refId": "A" } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, "title": "Requests / sec", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] + "type": "timeseries" }, { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", - "fill": 1, - "id": 14, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 1, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "ms" + }, + "overrides": [ ] }, - "lines": true, - "linewidth": 1, + "id": 14, "links": [ ], "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 6, - "stack": false, - "steppedLine": false, "targets": [ { "expr": "histogram_quantile(0.99, sum by (le) (cluster_job_method:cortex_cache_request_duration_seconds_bucket:sum_rate{job=~\"$job\", method=~\"frontend.+\"})) * 1e3", "format": "time_series", - "intervalFactor": 2, - "legendFormat": "99th Percentile", - "refId": "A", - "step": 10 + "legendFormat": "99th percentile", + "refId": "A" }, { "expr": "histogram_quantile(0.50, sum by (le) (cluster_job_method:cortex_cache_request_duration_seconds_bucket:sum_rate{job=~\"$job\", method=~\"frontend.+\"})) * 1e3", "format": "time_series", - "intervalFactor": 2, - "legendFormat": "50th Percentile", - "refId": "B", - "step": 10 + "legendFormat": "50th percentile", + "refId": "B" }, { "expr": "1e3 * sum(cluster_job_method:cortex_cache_request_duration_seconds_sum:sum_rate{job=~\"$job\", method=~\"frontend.+\"}) / sum(cluster_job_method:cortex_cache_request_duration_seconds_count:sum_rate{job=~\"$job\", method=~\"frontend.+\"})", "format": "time_series", - "intervalFactor": 2, "legendFormat": "Average", - "refId": "C", - "step": 10 + "refId": "C" } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, "title": "Latency", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, + "type": "timeseries", "yaxes": [ { "format": "ms", @@ -1130,157 +1448,257 @@ "3xx": "#6ED0E0", "4xx": "#EF843C", "5xx": "#E24D42", + "OK": "#7EB26D", + "cancel": "#A9A9A9", "error": "#E24D42", "success": "#7EB26D" }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 100, + "lineWidth": 0, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + } + }, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "reqps" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "1xx" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EAB839", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "2xx" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#7EB26D", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "3xx" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#6ED0E0", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "4xx" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EF843C", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "5xx" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E24D42", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "OK" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#7EB26D", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "cancel" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#A9A9A9", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "error" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E24D42", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "success" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#7EB26D", + "mode": "fixed" + } + } + ] + } + ] + }, "fill": 10, "id": 15, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, "linewidth": 0, "links": [ ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 4, "stack": true, - "steppedLine": false, "targets": [ { - "expr": "sum by (status) (\n label_replace(label_replace(rate(cortex_querier_request_duration_seconds_count{job=~\"$job\", route=~\"(prometheus|api_prom)_api_v1_.+\"}[$__rate_interval]),\n \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"),\n \"status\", \"${1}\", \"status_code\", \"([a-z]+)\"))\n", + "expr": "sum by (status) (\n label_replace(label_replace(rate(cortex_querier_request_duration_seconds_count{job=~\"$job\", route=~\"(prometheus|api_prom)_api_v1_.+\"}[$__rate_interval]),\n \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"),\n \"status\", \"${1}\", \"status_code\", \"([a-zA-Z]+)\"))\n", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "{{status}}", - "refId": "A", - "step": 10 + "refId": "A" } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, "title": "Requests / sec", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] + "type": "timeseries" }, { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", - "fill": 1, - "id": 16, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 1, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "ms" + }, + "overrides": [ ] }, - "lines": true, - "linewidth": 1, + "id": 16, "links": [ ], "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 4, - "stack": false, - "steppedLine": false, "targets": [ { "expr": "histogram_quantile(0.99, sum by (le) (cluster_job_route:cortex_querier_request_duration_seconds_bucket:sum_rate{job=~\"$job\", route=~\"(prometheus|api_prom)_api_v1_.+\"})) * 1e3", "format": "time_series", - "intervalFactor": 2, - "legendFormat": "99th Percentile", - "refId": "A", - "step": 10 + "legendFormat": "99th percentile", + "refId": "A" }, { "expr": "histogram_quantile(0.50, sum by (le) (cluster_job_route:cortex_querier_request_duration_seconds_bucket:sum_rate{job=~\"$job\", route=~\"(prometheus|api_prom)_api_v1_.+\"})) * 1e3", "format": "time_series", - "intervalFactor": 2, - "legendFormat": "50th Percentile", - "refId": "B", - "step": 10 + "legendFormat": "50th percentile", + "refId": "B" }, { "expr": "1e3 * sum(cluster_job_route:cortex_querier_request_duration_seconds_sum:sum_rate{job=~\"$job\", route=~\"(prometheus|api_prom)_api_v1_.+\"}) / sum(cluster_job_route:cortex_querier_request_duration_seconds_count:sum_rate{job=~\"$job\", route=~\"(prometheus|api_prom)_api_v1_.+\"})", "format": "time_series", - "intervalFactor": 2, "legendFormat": "Average", - "refId": "C", - "step": 10 + "refId": "C" } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, "title": "Latency", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, + "type": "timeseries", "yaxes": [ { "format": "ms", @@ -1301,73 +1719,59 @@ ] }, { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 1, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "s" + }, + "overrides": [ ] + }, "fill": 0, "id": 17, "legend": { "show": false }, - "lines": true, - "linewidth": 1, "links": [ ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, - "span": 4, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "histogram_quantile(0.99, sum by(le, pod) (rate(cortex_querier_request_duration_seconds_bucket{job=~\"$job\", route=~\"(prometheus|api_prom)_api_v1_.+\"}[$__rate_interval])))", - "format": "time_series", - "interval": "15s", - "intervalFactor": 2, - "legendFormat": "", - "legendLink": null, - "step": 10 + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" } - ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, - "title": "Per pod p99 Latency", - "tooltip": { - "sort": 2 - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] }, - "yaxes": [ - { - "format": "s", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false + "span": 4, + "targets": [ + { + "expr": "histogram_quantile(0.99, sum by(le, pod) (rate(cortex_querier_request_duration_seconds_bucket{job=~\"$job\", route=~\"(prometheus|api_prom)_api_v1_.+\"}[$__rate_interval])))", + "format": "time_series", + "interval": "15s", + "legendFormat": "", + "legendLink": null } - ] + ], + "title": "Per pod p99 Latency", + "tooltip": { + "sort": 2 + }, + "type": "timeseries" } ], "repeat": null, @@ -1388,157 +1792,257 @@ "3xx": "#6ED0E0", "4xx": "#EF843C", "5xx": "#E24D42", + "OK": "#7EB26D", + "cancel": "#A9A9A9", "error": "#E24D42", "success": "#7EB26D" }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 100, + "lineWidth": 0, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + } + }, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "reqps" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "1xx" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EAB839", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "2xx" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#7EB26D", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "3xx" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#6ED0E0", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "4xx" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EF843C", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "5xx" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E24D42", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "OK" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#7EB26D", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "cancel" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#A9A9A9", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "error" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E24D42", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "success" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#7EB26D", + "mode": "fixed" + } + } + ] + } + ] + }, "fill": 10, "id": 18, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, "linewidth": 0, "links": [ ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 4, "stack": true, - "steppedLine": false, "targets": [ { - "expr": "sum by (status) (\n label_replace(label_replace(rate(cortex_request_duration_seconds_count{job=~\"$job\",route=~\"/cortex.Ingester/Query(Stream)?|/cortex.Ingester/MetricsForLabelMatchers|/cortex.Ingester/LabelValues|/cortex.Ingester/MetricsMetadata\"}[$__rate_interval]),\n \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"),\n \"status\", \"${1}\", \"status_code\", \"([a-z]+)\"))\n", + "expr": "sum by (status) (\n label_replace(label_replace(rate(cortex_request_duration_seconds_count{job=~\"$job\",route=~\"/cortex.Ingester/Query(Stream)?|/cortex.Ingester/MetricsForLabelMatchers|/cortex.Ingester/LabelValues|/cortex.Ingester/MetricsMetadata\"}[$__rate_interval]),\n \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"),\n \"status\", \"${1}\", \"status_code\", \"([a-zA-Z]+)\"))\n", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "{{status}}", - "refId": "A", - "step": 10 + "refId": "A" } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, "title": "Requests / sec", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] + "type": "timeseries" }, { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", - "fill": 1, - "id": 19, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 1, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "ms" + }, + "overrides": [ ] }, - "lines": true, - "linewidth": 1, + "id": 19, "links": [ ], "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 4, - "stack": false, - "steppedLine": false, "targets": [ { "expr": "histogram_quantile(0.99, sum by (le) (cluster_job_route:cortex_request_duration_seconds_bucket:sum_rate{job=~\"$job\", route=~\"/cortex.Ingester/Query(Stream)?|/cortex.Ingester/MetricsForLabelMatchers|/cortex.Ingester/LabelValues|/cortex.Ingester/MetricsMetadata\"})) * 1e3", "format": "time_series", - "intervalFactor": 2, - "legendFormat": "99th Percentile", - "refId": "A", - "step": 10 + "legendFormat": "99th percentile", + "refId": "A" }, { "expr": "histogram_quantile(0.50, sum by (le) (cluster_job_route:cortex_request_duration_seconds_bucket:sum_rate{job=~\"$job\", route=~\"/cortex.Ingester/Query(Stream)?|/cortex.Ingester/MetricsForLabelMatchers|/cortex.Ingester/LabelValues|/cortex.Ingester/MetricsMetadata\"})) * 1e3", "format": "time_series", - "intervalFactor": 2, - "legendFormat": "50th Percentile", - "refId": "B", - "step": 10 + "legendFormat": "50th percentile", + "refId": "B" }, { "expr": "1e3 * sum(cluster_job_route:cortex_request_duration_seconds_sum:sum_rate{job=~\"$job\", route=~\"/cortex.Ingester/Query(Stream)?|/cortex.Ingester/MetricsForLabelMatchers|/cortex.Ingester/LabelValues|/cortex.Ingester/MetricsMetadata\"}) / sum(cluster_job_route:cortex_request_duration_seconds_count:sum_rate{job=~\"$job\", route=~\"/cortex.Ingester/Query(Stream)?|/cortex.Ingester/MetricsForLabelMatchers|/cortex.Ingester/LabelValues|/cortex.Ingester/MetricsMetadata\"})", "format": "time_series", - "intervalFactor": 2, "legendFormat": "Average", - "refId": "C", - "step": 10 + "refId": "C" } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, "title": "Latency", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, + "type": "timeseries", "yaxes": [ { "format": "ms", @@ -1559,73 +2063,59 @@ ] }, { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 1, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "s" + }, + "overrides": [ ] + }, "fill": 0, "id": 20, "legend": { "show": false }, - "lines": true, - "linewidth": 1, "links": [ ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 4, - "stack": false, - "steppedLine": false, "targets": [ { "expr": "histogram_quantile(0.99, sum by(le, pod) (rate(cortex_request_duration_seconds_bucket{job=~\"$job\", route=~\"/cortex.Ingester/Query(Stream)?|/cortex.Ingester/MetricsForLabelMatchers|/cortex.Ingester/LabelValues|/cortex.Ingester/MetricsMetadata\"}[$__rate_interval])))", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "", - "legendLink": null, - "step": 10 + "legendLink": null } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, "title": "Per pod p99 Latency", "tooltip": { "sort": 2 }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, - "yaxes": [ - { - "format": "s", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] + "type": "timeseries" } ], "repeat": null, @@ -1646,157 +2136,257 @@ "3xx": "#6ED0E0", "4xx": "#EF843C", "5xx": "#E24D42", + "OK": "#7EB26D", + "cancel": "#A9A9A9", "error": "#E24D42", "success": "#7EB26D" }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 100, + "lineWidth": 0, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + } + }, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "reqps" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "1xx" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EAB839", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "2xx" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#7EB26D", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "3xx" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#6ED0E0", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "4xx" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EF843C", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "5xx" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E24D42", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "OK" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#7EB26D", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "cancel" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#A9A9A9", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "error" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E24D42", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "success" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#7EB26D", + "mode": "fixed" + } + } + ] + } + ] + }, "fill": 10, "id": 21, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, "linewidth": 0, "links": [ ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 4, "stack": true, - "steppedLine": false, "targets": [ { - "expr": "sum by (status) (\n label_replace(label_replace(rate(cortex_request_duration_seconds_count{job=~\"$job\",route=~\"/gatewaypb.StoreGateway/.*\"}[$__rate_interval]),\n \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"),\n \"status\", \"${1}\", \"status_code\", \"([a-z]+)\"))\n", + "expr": "sum by (status) (\n label_replace(label_replace(rate(cortex_request_duration_seconds_count{job=~\"$job\",route=~\"/gatewaypb.StoreGateway/.*\"}[$__rate_interval]),\n \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"),\n \"status\", \"${1}\", \"status_code\", \"([a-zA-Z]+)\"))\n", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "{{status}}", - "refId": "A", - "step": 10 + "refId": "A" } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, "title": "Requests / sec", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] + "type": "timeseries" }, { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", - "fill": 1, - "id": 22, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 1, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "ms" + }, + "overrides": [ ] }, - "lines": true, - "linewidth": 1, + "id": 22, "links": [ ], "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 4, - "stack": false, - "steppedLine": false, "targets": [ { "expr": "histogram_quantile(0.99, sum by (le) (cluster_job_route:cortex_request_duration_seconds_bucket:sum_rate{job=~\"$job\", route=~\"/gatewaypb.StoreGateway/.*\"})) * 1e3", "format": "time_series", - "intervalFactor": 2, - "legendFormat": "99th Percentile", - "refId": "A", - "step": 10 + "legendFormat": "99th percentile", + "refId": "A" }, { "expr": "histogram_quantile(0.50, sum by (le) (cluster_job_route:cortex_request_duration_seconds_bucket:sum_rate{job=~\"$job\", route=~\"/gatewaypb.StoreGateway/.*\"})) * 1e3", "format": "time_series", - "intervalFactor": 2, - "legendFormat": "50th Percentile", - "refId": "B", - "step": 10 + "legendFormat": "50th percentile", + "refId": "B" }, { "expr": "1e3 * sum(cluster_job_route:cortex_request_duration_seconds_sum:sum_rate{job=~\"$job\", route=~\"/gatewaypb.StoreGateway/.*\"}) / sum(cluster_job_route:cortex_request_duration_seconds_count:sum_rate{job=~\"$job\", route=~\"/gatewaypb.StoreGateway/.*\"})", "format": "time_series", - "intervalFactor": 2, "legendFormat": "Average", - "refId": "C", - "step": 10 + "refId": "C" } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, "title": "Latency", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, + "type": "timeseries", "yaxes": [ { "format": "ms", @@ -1817,73 +2407,59 @@ ] }, { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 1, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "s" + }, + "overrides": [ ] + }, "fill": 0, "id": 23, "legend": { "show": false }, - "lines": true, - "linewidth": 1, "links": [ ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 4, - "stack": false, - "steppedLine": false, "targets": [ { "expr": "histogram_quantile(0.99, sum by(le, pod) (rate(cortex_request_duration_seconds_bucket{job=~\"$job\", route=~\"/gatewaypb.StoreGateway/.*\"}[$__rate_interval])))", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "", - "legendLink": null, - "step": 10 + "legendLink": null } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, "title": "Per pod p99 Latency", "tooltip": { "sort": 2 }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, - "yaxes": [ - { - "format": "s", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] + "type": "timeseries" } ], "repeat": null, @@ -1898,158 +2474,118 @@ "height": "250px", "panels": [ { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 1, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "ops" + }, + "overrides": [ ] + }, "fill": 10, "id": 24, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, "linewidth": 0, "links": [ ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 4, "stack": true, - "steppedLine": false, "targets": [ { "expr": "sum by(operation) (\n rate(\n thanos_memcached_operations_total{\n component=\"store-gateway\",\n name=\"index-cache\",\n job=~\"$job\"\n }[$__rate_interval]\n )\n)\n", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "{{operation}}", - "legendLink": null, - "step": 10 + "legendLink": null } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, "title": "Requests / sec", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, - "yaxes": [ - { - "format": "ops", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] + "type": "timeseries" }, { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", - "fill": 1, - "id": 25, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 1, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "short" + }, + "overrides": [ ] }, - "lines": true, - "linewidth": 1, + "id": 25, "links": [ ], "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 4, - "stack": false, - "steppedLine": false, "targets": [ { "expr": "histogram_quantile(0.99, sum(rate(thanos_memcached_operation_duration_seconds_bucket{\n job=~\"$job\",\n operation=\"getmulti\",\n component=\"store-gateway\",\n name=\"index-cache\"\n}\n[$__rate_interval])) by (le)) * 1e3", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "99th Percentile", - "refId": "A", - "step": 10 + "refId": "A" }, { "expr": "histogram_quantile(0.50, sum(rate(thanos_memcached_operation_duration_seconds_bucket{\n job=~\"$job\",\n operation=\"getmulti\",\n component=\"store-gateway\",\n name=\"index-cache\"\n}\n[$__rate_interval])) by (le)) * 1e3", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "50th Percentile", - "refId": "B", - "step": 10 + "refId": "B" }, { "expr": "sum(rate(thanos_memcached_operation_duration_seconds_sum{\n job=~\"$job\",\n operation=\"getmulti\",\n component=\"store-gateway\",\n name=\"index-cache\"\n}\n[$__rate_interval])) * 1e3 / sum(rate(thanos_memcached_operation_duration_seconds_count{\n job=~\"$job\",\n operation=\"getmulti\",\n component=\"store-gateway\",\n name=\"index-cache\"\n}\n[$__rate_interval]))", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "Average", - "refId": "C", - "step": 10 + "refId": "C" } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, "title": "Latency (getmulti)", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, + "type": "timeseries", "yaxes": [ { "format": "ms", @@ -2070,64 +2606,53 @@ ] }, { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", "description": "### Hit Ratio\nEven if you do not set up memcached for the blocks index cache, you will still see data in this panel because Cortex by default has an\nin-memory blocks index cache.\n\n", - "fill": 1, - "id": 26, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 1, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "short" + }, + "overrides": [ ] }, - "lines": true, - "linewidth": 1, + "id": 26, "links": [ ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 4, - "stack": false, - "steppedLine": false, "targets": [ { "expr": "sum by(item_type) (\n rate(\n thanos_store_index_cache_hits_total{\n component=\"store-gateway\",\n job=~\"$job\"\n }[$__rate_interval]\n )\n)\n/\nsum by(item_type) (\n rate(\n thanos_store_index_cache_requests_total{\n component=\"store-gateway\",\n job=~\"$job\"\n }[$__rate_interval]\n )\n)\n", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "{{item_type}}", - "legendLink": null, - "step": 10 + "legendLink": null } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, "title": "Hit ratio", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, + "type": "timeseries", "yaxes": [ { "format": "percentunit", @@ -2160,63 +2685,55 @@ "height": "250px", "panels": [ { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 1, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "ops" + }, + "overrides": [ ] + }, "fill": 10, "id": 27, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, "linewidth": 0, "links": [ ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 4, "stack": true, - "steppedLine": false, "targets": [ { "expr": "sum by(operation) (\n rate(\n thanos_memcached_operations_total{\n job=~\"$job\",\n component=\"store-gateway\",\n name=\"chunks-cache\"\n }[$__rate_interval]\n )\n)\n", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "{{operation}}", - "legendLink": null, - "step": 10 + "legendLink": null } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, "title": "Requests / sec", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, + "type": "timeseries", "yaxes": [ { "format": "ops", @@ -2237,81 +2754,67 @@ ] }, { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", - "fill": 1, - "id": 28, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 1, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "short" + }, + "overrides": [ ] }, - "lines": true, - "linewidth": 1, + "id": 28, "links": [ ], "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 4, - "stack": false, - "steppedLine": false, "targets": [ { "expr": "histogram_quantile(0.99, sum(rate(thanos_memcached_operation_duration_seconds_bucket{\n job=~\"$job\",\n operation=\"getmulti\",\n component=\"store-gateway\",\n name=\"chunks-cache\"\n}\n[$__rate_interval])) by (le)) * 1e3", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "99th Percentile", - "refId": "A", - "step": 10 + "refId": "A" }, { "expr": "histogram_quantile(0.50, sum(rate(thanos_memcached_operation_duration_seconds_bucket{\n job=~\"$job\",\n operation=\"getmulti\",\n component=\"store-gateway\",\n name=\"chunks-cache\"\n}\n[$__rate_interval])) by (le)) * 1e3", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "50th Percentile", - "refId": "B", - "step": 10 + "refId": "B" }, { "expr": "sum(rate(thanos_memcached_operation_duration_seconds_sum{\n job=~\"$job\",\n operation=\"getmulti\",\n component=\"store-gateway\",\n name=\"chunks-cache\"\n}\n[$__rate_interval])) * 1e3 / sum(rate(thanos_memcached_operation_duration_seconds_count{\n job=~\"$job\",\n operation=\"getmulti\",\n component=\"store-gateway\",\n name=\"chunks-cache\"\n}\n[$__rate_interval]))", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "Average", - "refId": "C", - "step": 10 + "refId": "C" } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, "title": "Latency (getmulti)", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, + "type": "timeseries", "yaxes": [ { "format": "ms", @@ -2332,63 +2835,52 @@ ] }, { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", - "fill": 1, - "id": 29, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 1, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "percentunit" + }, + "overrides": [ ] }, - "lines": true, - "linewidth": 1, + "id": 29, "links": [ ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 4, - "stack": false, - "steppedLine": false, "targets": [ { "expr": "sum(\n rate(\n thanos_cache_memcached_hits_total{\n job=~\"$job\",\n component=\"store-gateway\",\n name=\"chunks-cache\"\n }[$__rate_interval]\n )\n)\n/\nsum(\n rate(\n thanos_cache_memcached_requests_total{\n job=~\"$job\",\n component=\"store-gateway\",\n name=\"chunks-cache\"\n }[$__rate_interval]\n )\n)\n", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "items", - "legendLink": null, - "step": 10 + "legendLink": null } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, "title": "Hit ratio", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, + "type": "timeseries", "yaxes": [ { "format": "percentunit", @@ -2421,63 +2913,55 @@ "height": "250px", "panels": [ { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 1, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "ops" + }, + "overrides": [ ] + }, "fill": 10, "id": 30, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, "linewidth": 0, "links": [ ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 4, "stack": true, - "steppedLine": false, "targets": [ { "expr": "sum by(operation) (\n rate(\n thanos_memcached_operations_total{\n job=~\"$job\",\n component=\"store-gateway\",\n name=\"metadata-cache\"\n }[$__rate_interval]\n )\n)\n", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "{{operation}}", - "legendLink": null, - "step": 10 + "legendLink": null } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, "title": "Requests / sec", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, + "type": "timeseries", "yaxes": [ { "format": "ops", @@ -2498,81 +2982,67 @@ ] }, { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", - "fill": 1, - "id": 31, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 1, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "short" + }, + "overrides": [ ] }, - "lines": true, - "linewidth": 1, + "id": 31, "links": [ ], "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 4, - "stack": false, - "steppedLine": false, "targets": [ { "expr": "histogram_quantile(0.99, sum(rate(thanos_memcached_operation_duration_seconds_bucket{\n job=~\"$job\",\n operation=\"getmulti\",\n component=\"store-gateway\",\n name=\"metadata-cache\"\n}\n[$__rate_interval])) by (le)) * 1e3", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "99th Percentile", - "refId": "A", - "step": 10 + "refId": "A" }, { "expr": "histogram_quantile(0.50, sum(rate(thanos_memcached_operation_duration_seconds_bucket{\n job=~\"$job\",\n operation=\"getmulti\",\n component=\"store-gateway\",\n name=\"metadata-cache\"\n}\n[$__rate_interval])) by (le)) * 1e3", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "50th Percentile", - "refId": "B", - "step": 10 + "refId": "B" }, { "expr": "sum(rate(thanos_memcached_operation_duration_seconds_sum{\n job=~\"$job\",\n operation=\"getmulti\",\n component=\"store-gateway\",\n name=\"metadata-cache\"\n}\n[$__rate_interval])) * 1e3 / sum(rate(thanos_memcached_operation_duration_seconds_count{\n job=~\"$job\",\n operation=\"getmulti\",\n component=\"store-gateway\",\n name=\"metadata-cache\"\n}\n[$__rate_interval]))", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "Average", - "refId": "C", - "step": 10 + "refId": "C" } - ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, - "title": "Latency (getmulti)", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, + ], + "title": "Latency (getmulti)", + "type": "timeseries", "yaxes": [ { "format": "ms", @@ -2593,63 +3063,52 @@ ] }, { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", - "fill": 1, - "id": 32, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 1, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "percentunit" + }, + "overrides": [ ] }, - "lines": true, - "linewidth": 1, + "id": 32, "links": [ ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 4, - "stack": false, - "steppedLine": false, "targets": [ { "expr": "sum(\n rate(\n thanos_cache_memcached_hits_total{\n job=~\"$job\",\n component=\"store-gateway\",\n name=\"metadata-cache\"\n }[$__rate_interval]\n )\n)\n/\nsum(\n rate(\n thanos_cache_memcached_requests_total{\n job=~\"$job\",\n component=\"store-gateway\",\n name=\"metadata-cache\"\n }[$__rate_interval]\n )\n)\n", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "items", - "legendLink": null, - "step": 10 + "legendLink": null } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, "title": "Hit ratio", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, + "type": "timeseries", "yaxes": [ { "format": "percentunit", @@ -2682,63 +3141,55 @@ "height": "250px", "panels": [ { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 1, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "ops" + }, + "overrides": [ ] + }, "fill": 10, "id": 33, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, "linewidth": 0, "links": [ ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 4, "stack": true, - "steppedLine": false, "targets": [ { "expr": "sum by(operation) (\n rate(\n thanos_memcached_operations_total{\n job=~\"$job\",\n component=\"querier\",\n name=\"metadata-cache\"\n }[$__rate_interval]\n )\n)\n", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "{{operation}}", - "legendLink": null, - "step": 10 + "legendLink": null } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, "title": "Requests / sec", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, + "type": "timeseries", "yaxes": [ { "format": "ops", @@ -2759,81 +3210,67 @@ ] }, { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", - "fill": 1, - "id": 34, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 1, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "short" + }, + "overrides": [ ] }, - "lines": true, - "linewidth": 1, + "id": 34, "links": [ ], "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 4, - "stack": false, - "steppedLine": false, "targets": [ { "expr": "histogram_quantile(0.99, sum(rate(thanos_memcached_operation_duration_seconds_bucket{\n job=~\"$job\",\n operation=\"getmulti\",\n component=\"querier\",\n name=\"metadata-cache\"\n}\n[$__rate_interval])) by (le)) * 1e3", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "99th Percentile", - "refId": "A", - "step": 10 + "refId": "A" }, { "expr": "histogram_quantile(0.50, sum(rate(thanos_memcached_operation_duration_seconds_bucket{\n job=~\"$job\",\n operation=\"getmulti\",\n component=\"querier\",\n name=\"metadata-cache\"\n}\n[$__rate_interval])) by (le)) * 1e3", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "50th Percentile", - "refId": "B", - "step": 10 + "refId": "B" }, { "expr": "sum(rate(thanos_memcached_operation_duration_seconds_sum{\n job=~\"$job\",\n operation=\"getmulti\",\n component=\"querier\",\n name=\"metadata-cache\"\n}\n[$__rate_interval])) * 1e3 / sum(rate(thanos_memcached_operation_duration_seconds_count{\n job=~\"$job\",\n operation=\"getmulti\",\n component=\"querier\",\n name=\"metadata-cache\"\n}\n[$__rate_interval]))", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "Average", - "refId": "C", - "step": 10 + "refId": "C" } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, "title": "Latency (getmulti)", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, + "type": "timeseries", "yaxes": [ { "format": "ms", @@ -2854,63 +3291,52 @@ ] }, { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", - "fill": 1, - "id": 35, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 1, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "percentunit" + }, + "overrides": [ ] }, - "lines": true, - "linewidth": 1, + "id": 35, "links": [ ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 4, - "stack": false, - "steppedLine": false, "targets": [ { "expr": "sum(\n rate(\n thanos_cache_memcached_hits_total{\n job=~\"$job\",\n component=\"querier\",\n name=\"metadata-cache\"\n }[$__rate_interval]\n )\n)\n/\nsum(\n rate(\n thanos_cache_memcached_requests_total{\n job=~\"$job\",\n component=\"querier\",\n name=\"metadata-cache\"\n }[$__rate_interval]\n )\n)\n", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "items", - "legendLink": null, - "step": 10 + "legendLink": null } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, "title": "Hit ratio", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, + "type": "timeseries", "yaxes": [ { "format": "percentunit", @@ -2943,140 +3369,103 @@ "height": "250px", "panels": [ { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 1, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "rps" + }, + "overrides": [ ] + }, "fill": 10, "id": 36, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, "linewidth": 0, "links": [ ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 3, "stack": true, - "steppedLine": false, "targets": [ { "expr": "sum by(operation) (rate(thanos_objstore_bucket_operations_total{job=~\"$job\",component=\"store-gateway\"}[$__rate_interval]))", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "{{operation}}", - "legendLink": null, - "step": 10 + "legendLink": null } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, "title": "Operations / sec", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, - "yaxes": [ - { - "format": "rps", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] + "type": "timeseries" }, { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", - "fill": 1, - "id": 37, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 1, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "percentunit" + }, + "overrides": [ ] }, - "lines": true, - "linewidth": 1, + "id": 37, "links": [ ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 3, - "stack": false, - "steppedLine": false, "targets": [ { "expr": "sum by(operation) (rate(thanos_objstore_bucket_operation_failures_total{job=~\"$job\",component=\"store-gateway\"}[$__rate_interval])) / sum by(operation) (rate(thanos_objstore_bucket_operations_total{job=~\"$job\",component=\"store-gateway\"}[$__rate_interval]))", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "{{operation}}", - "legendLink": null, - "step": 10 + "legendLink": null } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, "title": "Error rate", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, + "type": "timeseries", "yaxes": [ { "format": "percentunit", @@ -3097,81 +3486,67 @@ ] }, { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", - "fill": 1, - "id": 38, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 1, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "short" + }, + "overrides": [ ] }, - "lines": true, - "linewidth": 1, + "id": 38, "links": [ ], "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 3, - "stack": false, - "steppedLine": false, "targets": [ { "expr": "histogram_quantile(0.99, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{job=~\"$job\",component=\"store-gateway\",operation=\"attributes\"}[$__rate_interval])) by (le)) * 1e3", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "99th Percentile", - "refId": "A", - "step": 10 + "refId": "A" }, { "expr": "histogram_quantile(0.50, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{job=~\"$job\",component=\"store-gateway\",operation=\"attributes\"}[$__rate_interval])) by (le)) * 1e3", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "50th Percentile", - "refId": "B", - "step": 10 + "refId": "B" }, { "expr": "sum(rate(thanos_objstore_bucket_operation_duration_seconds_sum{job=~\"$job\",component=\"store-gateway\",operation=\"attributes\"}[$__rate_interval])) * 1e3 / sum(rate(thanos_objstore_bucket_operation_duration_seconds_count{job=~\"$job\",component=\"store-gateway\",operation=\"attributes\"}[$__rate_interval]))", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "Average", - "refId": "C", - "step": 10 + "refId": "C" } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, "title": "Latency of Op: Attributes", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, + "type": "timeseries", "yaxes": [ { "format": "ms", @@ -3192,81 +3567,67 @@ ] }, { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", - "fill": 1, - "id": 39, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 1, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "short" + }, + "overrides": [ ] }, - "lines": true, - "linewidth": 1, + "id": 39, "links": [ ], "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 3, - "stack": false, - "steppedLine": false, "targets": [ { "expr": "histogram_quantile(0.99, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{job=~\"$job\",component=\"store-gateway\",operation=\"exists\"}[$__rate_interval])) by (le)) * 1e3", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "99th Percentile", - "refId": "A", - "step": 10 + "refId": "A" }, { "expr": "histogram_quantile(0.50, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{job=~\"$job\",component=\"store-gateway\",operation=\"exists\"}[$__rate_interval])) by (le)) * 1e3", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "50th Percentile", - "refId": "B", - "step": 10 + "refId": "B" }, { "expr": "sum(rate(thanos_objstore_bucket_operation_duration_seconds_sum{job=~\"$job\",component=\"store-gateway\",operation=\"exists\"}[$__rate_interval])) * 1e3 / sum(rate(thanos_objstore_bucket_operation_duration_seconds_count{job=~\"$job\",component=\"store-gateway\",operation=\"exists\"}[$__rate_interval]))", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "Average", - "refId": "C", - "step": 10 + "refId": "C" } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, "title": "Latency of Op: Exists", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, + "type": "timeseries", "yaxes": [ { "format": "ms", @@ -3299,81 +3660,67 @@ "height": "250px", "panels": [ { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", - "fill": 1, - "id": 40, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 1, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "short" + }, + "overrides": [ ] }, - "lines": true, - "linewidth": 1, + "id": 40, "links": [ ], "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 3, - "stack": false, - "steppedLine": false, "targets": [ { "expr": "histogram_quantile(0.99, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{job=~\"$job\",component=\"store-gateway\",operation=\"get\"}[$__rate_interval])) by (le)) * 1e3", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "99th Percentile", - "refId": "A", - "step": 10 + "refId": "A" }, { "expr": "histogram_quantile(0.50, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{job=~\"$job\",component=\"store-gateway\",operation=\"get\"}[$__rate_interval])) by (le)) * 1e3", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "50th Percentile", - "refId": "B", - "step": 10 + "refId": "B" }, { "expr": "sum(rate(thanos_objstore_bucket_operation_duration_seconds_sum{job=~\"$job\",component=\"store-gateway\",operation=\"get\"}[$__rate_interval])) * 1e3 / sum(rate(thanos_objstore_bucket_operation_duration_seconds_count{job=~\"$job\",component=\"store-gateway\",operation=\"get\"}[$__rate_interval]))", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "Average", - "refId": "C", - "step": 10 + "refId": "C" } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, "title": "Latency of Op: Get", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, + "type": "timeseries", "yaxes": [ { "format": "ms", @@ -3394,81 +3741,67 @@ ] }, { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", - "fill": 1, - "id": 41, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 1, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "short" + }, + "overrides": [ ] }, - "lines": true, - "linewidth": 1, + "id": 41, "links": [ ], "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 3, - "stack": false, - "steppedLine": false, "targets": [ { "expr": "histogram_quantile(0.99, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{job=~\"$job\",component=\"store-gateway\",operation=\"get_range\"}[$__rate_interval])) by (le)) * 1e3", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "99th Percentile", - "refId": "A", - "step": 10 + "refId": "A" }, { "expr": "histogram_quantile(0.50, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{job=~\"$job\",component=\"store-gateway\",operation=\"get_range\"}[$__rate_interval])) by (le)) * 1e3", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "50th Percentile", - "refId": "B", - "step": 10 + "refId": "B" }, { "expr": "sum(rate(thanos_objstore_bucket_operation_duration_seconds_sum{job=~\"$job\",component=\"store-gateway\",operation=\"get_range\"}[$__rate_interval])) * 1e3 / sum(rate(thanos_objstore_bucket_operation_duration_seconds_count{job=~\"$job\",component=\"store-gateway\",operation=\"get_range\"}[$__rate_interval]))", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "Average", - "refId": "C", - "step": 10 + "refId": "C" } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, "title": "Latency of Op: GetRange", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, + "type": "timeseries", "yaxes": [ { "format": "ms", @@ -3489,81 +3822,67 @@ ] }, { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", - "fill": 1, - "id": 42, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 1, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "short" + }, + "overrides": [ ] }, - "lines": true, - "linewidth": 1, + "id": 42, "links": [ ], "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 3, - "stack": false, - "steppedLine": false, "targets": [ { "expr": "histogram_quantile(0.99, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{job=~\"$job\",component=\"store-gateway\",operation=\"upload\"}[$__rate_interval])) by (le)) * 1e3", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "99th Percentile", - "refId": "A", - "step": 10 + "refId": "A" }, { "expr": "histogram_quantile(0.50, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{job=~\"$job\",component=\"store-gateway\",operation=\"upload\"}[$__rate_interval])) by (le)) * 1e3", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "50th Percentile", - "refId": "B", - "step": 10 + "refId": "B" }, { "expr": "sum(rate(thanos_objstore_bucket_operation_duration_seconds_sum{job=~\"$job\",component=\"store-gateway\",operation=\"upload\"}[$__rate_interval])) * 1e3 / sum(rate(thanos_objstore_bucket_operation_duration_seconds_count{job=~\"$job\",component=\"store-gateway\",operation=\"upload\"}[$__rate_interval]))", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "Average", - "refId": "C", - "step": 10 + "refId": "C" } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, "title": "Latency of Op: Upload", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, + "type": "timeseries", "yaxes": [ { "format": "ms", @@ -3584,81 +3903,67 @@ ] }, { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", - "fill": 1, - "id": 43, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 1, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "short" + }, + "overrides": [ ] }, - "lines": true, - "linewidth": 1, + "id": 43, "links": [ ], "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 3, - "stack": false, - "steppedLine": false, "targets": [ { "expr": "histogram_quantile(0.99, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{job=~\"$job\",component=\"store-gateway\",operation=\"delete\"}[$__rate_interval])) by (le)) * 1e3", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "99th Percentile", - "refId": "A", - "step": 10 + "refId": "A" }, { "expr": "histogram_quantile(0.50, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{job=~\"$job\",component=\"store-gateway\",operation=\"delete\"}[$__rate_interval])) by (le)) * 1e3", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "50th Percentile", - "refId": "B", - "step": 10 + "refId": "B" }, { "expr": "sum(rate(thanos_objstore_bucket_operation_duration_seconds_sum{job=~\"$job\",component=\"store-gateway\",operation=\"delete\"}[$__rate_interval])) * 1e3 / sum(rate(thanos_objstore_bucket_operation_duration_seconds_count{job=~\"$job\",component=\"store-gateway\",operation=\"delete\"}[$__rate_interval]))", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "Average", - "refId": "C", - "step": 10 + "refId": "C" } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, "title": "Latency of Op: Delete", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, + "type": "timeseries", "yaxes": [ { "format": "ms", @@ -3691,140 +3996,103 @@ "height": "250px", "panels": [ { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 1, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "rps" + }, + "overrides": [ ] + }, "fill": 10, "id": 44, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, "linewidth": 0, "links": [ ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 3, "stack": true, - "steppedLine": false, "targets": [ { "expr": "sum by(operation) (rate(thanos_objstore_bucket_operations_total{job=~\"$job\",component=\"querier\"}[$__rate_interval]))", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "{{operation}}", - "legendLink": null, - "step": 10 + "legendLink": null } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, "title": "Operations / sec", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, - "yaxes": [ - { - "format": "rps", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] + "type": "timeseries" }, { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", - "fill": 1, - "id": 45, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 1, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "percentunit" + }, + "overrides": [ ] }, - "lines": true, - "linewidth": 1, + "id": 45, "links": [ ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 3, - "stack": false, - "steppedLine": false, "targets": [ { "expr": "sum by(operation) (rate(thanos_objstore_bucket_operation_failures_total{job=~\"$job\",component=\"querier\"}[$__rate_interval])) / sum by(operation) (rate(thanos_objstore_bucket_operations_total{job=~\"$job\",component=\"querier\"}[$__rate_interval]))", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "{{operation}}", - "legendLink": null, - "step": 10 + "legendLink": null } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, "title": "Error rate", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, + "type": "timeseries", "yaxes": [ { "format": "percentunit", @@ -3845,81 +4113,67 @@ ] }, { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", - "fill": 1, - "id": 46, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 1, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "short" + }, + "overrides": [ ] }, - "lines": true, - "linewidth": 1, + "id": 46, "links": [ ], "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 3, - "stack": false, - "steppedLine": false, "targets": [ { "expr": "histogram_quantile(0.99, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{job=~\"$job\",component=\"querier\",operation=\"attributes\"}[$__rate_interval])) by (le)) * 1e3", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "99th Percentile", - "refId": "A", - "step": 10 + "refId": "A" }, { "expr": "histogram_quantile(0.50, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{job=~\"$job\",component=\"querier\",operation=\"attributes\"}[$__rate_interval])) by (le)) * 1e3", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "50th Percentile", - "refId": "B", - "step": 10 + "refId": "B" }, { "expr": "sum(rate(thanos_objstore_bucket_operation_duration_seconds_sum{job=~\"$job\",component=\"querier\",operation=\"attributes\"}[$__rate_interval])) * 1e3 / sum(rate(thanos_objstore_bucket_operation_duration_seconds_count{job=~\"$job\",component=\"querier\",operation=\"attributes\"}[$__rate_interval]))", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "Average", - "refId": "C", - "step": 10 + "refId": "C" } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, "title": "Latency of Op: Attributes", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, + "type": "timeseries", "yaxes": [ { "format": "ms", @@ -3940,81 +4194,67 @@ ] }, { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", - "fill": 1, - "id": 47, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 1, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "short" + }, + "overrides": [ ] }, - "lines": true, - "linewidth": 1, + "id": 47, "links": [ ], "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 3, - "stack": false, - "steppedLine": false, "targets": [ { "expr": "histogram_quantile(0.99, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{job=~\"$job\",component=\"querier\",operation=\"exists\"}[$__rate_interval])) by (le)) * 1e3", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "99th Percentile", - "refId": "A", - "step": 10 + "refId": "A" }, { "expr": "histogram_quantile(0.50, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{job=~\"$job\",component=\"querier\",operation=\"exists\"}[$__rate_interval])) by (le)) * 1e3", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "50th Percentile", - "refId": "B", - "step": 10 + "refId": "B" }, { "expr": "sum(rate(thanos_objstore_bucket_operation_duration_seconds_sum{job=~\"$job\",component=\"querier\",operation=\"exists\"}[$__rate_interval])) * 1e3 / sum(rate(thanos_objstore_bucket_operation_duration_seconds_count{job=~\"$job\",component=\"querier\",operation=\"exists\"}[$__rate_interval]))", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "Average", - "refId": "C", - "step": 10 + "refId": "C" } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, "title": "Latency of Op: Exists", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, + "type": "timeseries", "yaxes": [ { "format": "ms", @@ -4047,81 +4287,67 @@ "height": "250px", "panels": [ { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", - "fill": 1, - "id": 48, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 1, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "short" + }, + "overrides": [ ] }, - "lines": true, - "linewidth": 1, + "id": 48, "links": [ ], "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 3, - "stack": false, - "steppedLine": false, "targets": [ { "expr": "histogram_quantile(0.99, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{job=~\"$job\",component=\"querier\",operation=\"get\"}[$__rate_interval])) by (le)) * 1e3", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "99th Percentile", - "refId": "A", - "step": 10 + "refId": "A" }, { "expr": "histogram_quantile(0.50, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{job=~\"$job\",component=\"querier\",operation=\"get\"}[$__rate_interval])) by (le)) * 1e3", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "50th Percentile", - "refId": "B", - "step": 10 + "refId": "B" }, { "expr": "sum(rate(thanos_objstore_bucket_operation_duration_seconds_sum{job=~\"$job\",component=\"querier\",operation=\"get\"}[$__rate_interval])) * 1e3 / sum(rate(thanos_objstore_bucket_operation_duration_seconds_count{job=~\"$job\",component=\"querier\",operation=\"get\"}[$__rate_interval]))", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "Average", - "refId": "C", - "step": 10 + "refId": "C" } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, "title": "Latency of Op: Get", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, + "type": "timeseries", "yaxes": [ { "format": "ms", @@ -4142,81 +4368,67 @@ ] }, { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", - "fill": 1, - "id": 49, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 1, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "short" + }, + "overrides": [ ] }, - "lines": true, - "linewidth": 1, + "id": 49, "links": [ ], "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 3, - "stack": false, - "steppedLine": false, "targets": [ { "expr": "histogram_quantile(0.99, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{job=~\"$job\",component=\"querier\",operation=\"get_range\"}[$__rate_interval])) by (le)) * 1e3", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "99th Percentile", - "refId": "A", - "step": 10 + "refId": "A" }, { "expr": "histogram_quantile(0.50, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{job=~\"$job\",component=\"querier\",operation=\"get_range\"}[$__rate_interval])) by (le)) * 1e3", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "50th Percentile", - "refId": "B", - "step": 10 + "refId": "B" }, { "expr": "sum(rate(thanos_objstore_bucket_operation_duration_seconds_sum{job=~\"$job\",component=\"querier\",operation=\"get_range\"}[$__rate_interval])) * 1e3 / sum(rate(thanos_objstore_bucket_operation_duration_seconds_count{job=~\"$job\",component=\"querier\",operation=\"get_range\"}[$__rate_interval]))", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "Average", - "refId": "C", - "step": 10 + "refId": "C" } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, "title": "Latency of Op: GetRange", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, + "type": "timeseries", "yaxes": [ { "format": "ms", @@ -4237,81 +4449,67 @@ ] }, { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", - "fill": 1, - "id": 50, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 1, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "short" + }, + "overrides": [ ] }, - "lines": true, - "linewidth": 1, + "id": 50, "links": [ ], "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 3, - "stack": false, - "steppedLine": false, "targets": [ { "expr": "histogram_quantile(0.99, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{job=~\"$job\",component=\"querier\",operation=\"upload\"}[$__rate_interval])) by (le)) * 1e3", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "99th Percentile", - "refId": "A", - "step": 10 + "refId": "A" }, { "expr": "histogram_quantile(0.50, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{job=~\"$job\",component=\"querier\",operation=\"upload\"}[$__rate_interval])) by (le)) * 1e3", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "50th Percentile", - "refId": "B", - "step": 10 + "refId": "B" }, { "expr": "sum(rate(thanos_objstore_bucket_operation_duration_seconds_sum{job=~\"$job\",component=\"querier\",operation=\"upload\"}[$__rate_interval])) * 1e3 / sum(rate(thanos_objstore_bucket_operation_duration_seconds_count{job=~\"$job\",component=\"querier\",operation=\"upload\"}[$__rate_interval]))", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "Average", - "refId": "C", - "step": 10 + "refId": "C" } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, "title": "Latency of Op: Upload", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, + "type": "timeseries", "yaxes": [ { "format": "ms", @@ -4332,81 +4530,67 @@ ] }, { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", - "fill": 1, - "id": 51, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 1, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "short" + }, + "overrides": [ ] }, - "lines": true, - "linewidth": 1, + "id": 51, "links": [ ], "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 3, - "stack": false, - "steppedLine": false, "targets": [ { "expr": "histogram_quantile(0.99, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{job=~\"$job\",component=\"querier\",operation=\"delete\"}[$__rate_interval])) by (le)) * 1e3", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "99th Percentile", - "refId": "A", - "step": 10 + "refId": "A" }, { "expr": "histogram_quantile(0.50, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{job=~\"$job\",component=\"querier\",operation=\"delete\"}[$__rate_interval])) by (le)) * 1e3", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "50th Percentile", - "refId": "B", - "step": 10 + "refId": "B" }, { "expr": "sum(rate(thanos_objstore_bucket_operation_duration_seconds_sum{job=~\"$job\",component=\"querier\",operation=\"delete\"}[$__rate_interval])) * 1e3 / sum(rate(thanos_objstore_bucket_operation_duration_seconds_count{job=~\"$job\",component=\"querier\",operation=\"delete\"}[$__rate_interval]))", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "Average", - "refId": "C", - "step": 10 + "refId": "C" } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, "title": "Latency of Op: Delete", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, + "type": "timeseries", "yaxes": [ { "format": "ms", @@ -4448,7 +4632,7 @@ "value": "Cortex" }, "hide": 0, - "label": null, + "label": "Data source", "name": "datasource", "options": [ ], "query": "prometheus", diff --git a/docs/getting-started/dashboards/cortex-rollout-progress.json b/docs/getting-started/dashboards/cortex-rollout-progress.json index fe24526292..9bd20e9e95 100644 --- a/docs/getting-started/dashboards/cortex-rollout-progress.json +++ b/docs/getting-started/dashboards/cortex-rollout-progress.json @@ -22,10 +22,6 @@ ], "panels": [ { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", "fieldConfig": { "defaults": { @@ -55,7 +51,6 @@ "unit": "percentunit" } }, - "fill": 1, "gridPos": { "h": 8, "w": 10, @@ -63,19 +58,7 @@ "y": 0 }, "id": 1, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, "links": [ ], - "nullPointMode": "null as zero", "options": { "displayMode": "basic", "orientation": "horizontal", @@ -87,15 +70,6 @@ "values": false } }, - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, - "span": 6, - "stack": false, - "steppedLine": false, "targets": [ { "expr": "(\n sum by(cortex_service) (\n label_replace(\n kube_statefulset_status_replicas_updated{job=~\"$job\",statefulset=~\"cortex-gw|distributor|ingester.*|query-frontend.*|query-scheduler.*|querier.*|compactor|store-gateway|ruler|alertmanager\"},\n \"cortex_service\", \"$1\", \"statefulset\", \"(.*?)(?:-zone-[a-z])?\"\n )\n )\n /\n sum by(cortex_service) (\n label_replace(\n kube_statefulset_replicas{job=~\"$job\"},\n \"cortex_service\", \"$1\", \"statefulset\", \"(.*?)(?:-zone-[a-z])?\"\n )\n )\n) and (\n sum by(cortex_service) (\n label_replace(\n kube_statefulset_replicas{job=~\"$job\"},\n \"cortex_service\", \"$1\", \"statefulset\", \"(.*?)(?:-zone-[a-z])?\"\n )\n )\n > 0\n)\n", @@ -114,47 +88,10 @@ "step": null } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, "title": "Rollout progress", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "bargauge", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] + "type": "bargauge" }, { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", "fieldConfig": { "defaults": { @@ -176,7 +113,6 @@ }, "overrides": [ ] }, - "fill": 1, "gridPos": { "h": 4, "w": 2, @@ -184,28 +120,16 @@ "y": 0 }, "id": 2, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, "links": [ ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, - "span": 6, - "stack": false, - "steppedLine": false, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "targets": [ { "expr": "sum(rate(cortex_request_duration_seconds_count{job=~\"$job\", route=~\"api_(v1|prom)_push\",status_code=~\"2.+\"}[$__rate_interval])) /\nsum(rate(cortex_request_duration_seconds_count{job=~\"$job\", route=~\"api_(v1|prom)_push\"}[$__rate_interval]))\n", @@ -218,47 +142,10 @@ "step": null } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, "title": "Writes - 2xx", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "stat", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] + "type": "stat" }, { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", "fieldConfig": { "defaults": { @@ -288,7 +175,6 @@ }, "overrides": [ ] }, - "fill": 1, "gridPos": { "h": 4, "w": 2, @@ -296,28 +182,16 @@ "y": 0 }, "id": 3, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, "links": [ ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, - "span": 6, - "stack": false, - "steppedLine": false, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "targets": [ { "expr": "sum(rate(cortex_request_duration_seconds_count{job=~\"$job\", route=~\"api_(v1|prom)_push\",status_code=~\"4.+\"}[$__rate_interval])) /\nsum(rate(cortex_request_duration_seconds_count{job=~\"$job\", route=~\"api_(v1|prom)_push\"}[$__rate_interval]))\n", @@ -330,47 +204,10 @@ "step": null } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, "title": "Writes - 4xx", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "stat", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] + "type": "stat" }, { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", "fieldConfig": { "defaults": { @@ -396,7 +233,6 @@ }, "overrides": [ ] }, - "fill": 1, "gridPos": { "h": 4, "w": 2, @@ -404,28 +240,16 @@ "y": 0 }, "id": 4, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, "links": [ ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, - "span": 6, - "stack": false, - "steppedLine": false, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "targets": [ { "expr": "sum(rate(cortex_request_duration_seconds_count{job=~\"$job\", route=~\"api_(v1|prom)_push\",status_code=~\"5.+\"}[$__rate_interval])) /\nsum(rate(cortex_request_duration_seconds_count{job=~\"$job\", route=~\"api_(v1|prom)_push\"}[$__rate_interval]))\n", @@ -438,47 +262,10 @@ "step": null } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, "title": "Writes - 5xx", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "stat", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] + "type": "stat" }, { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", "fieldConfig": { "defaults": { @@ -508,7 +295,6 @@ }, "overrides": [ ] }, - "fill": 1, "gridPos": { "h": 4, "w": 8, @@ -516,28 +302,16 @@ "y": 0 }, "id": 5, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, "links": [ ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, - "span": 6, - "stack": false, - "steppedLine": false, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "targets": [ { "expr": "histogram_quantile(0.99, sum by (le) (cluster_job_route:cortex_request_duration_seconds_bucket:sum_rate{job=~\"$job\", route=~\"api_(v1|prom)_push\"}))\n", @@ -550,47 +324,10 @@ "step": null } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, - "title": "Writes 99th Latency", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "stat", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] + "title": "Writes 99th latency", + "type": "stat" }, { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", "fieldConfig": { "defaults": { @@ -612,7 +349,6 @@ }, "overrides": [ ] }, - "fill": 1, "gridPos": { "h": 4, "w": 2, @@ -620,28 +356,16 @@ "y": 4 }, "id": 6, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, "links": [ ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, - "span": 6, - "stack": false, - "steppedLine": false, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "targets": [ { "expr": "sum(rate(cortex_request_duration_seconds_count{job=~\"$job\", route=~\"(prometheus|api_prom)_api_v1_.+\",status_code=~\"2.+\"}[$__rate_interval])) /\nsum(rate(cortex_request_duration_seconds_count{job=~\"$job\", route=~\"(prometheus|api_prom)_api_v1_.+\"}[$__rate_interval]))\n", @@ -654,47 +378,10 @@ "step": null } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, "title": "Reads - 2xx", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "stat", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] + "type": "stat" }, { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", "fieldConfig": { "defaults": { @@ -724,7 +411,6 @@ }, "overrides": [ ] }, - "fill": 1, "gridPos": { "h": 4, "w": 2, @@ -732,28 +418,16 @@ "y": 4 }, "id": 7, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, "links": [ ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, - "span": 6, - "stack": false, - "steppedLine": false, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "targets": [ { "expr": "sum(rate(cortex_request_duration_seconds_count{job=~\"$job\", route=~\"(prometheus|api_prom)_api_v1_.+\",status_code=~\"4.+\"}[$__rate_interval])) /\nsum(rate(cortex_request_duration_seconds_count{job=~\"$job\", route=~\"(prometheus|api_prom)_api_v1_.+\"}[$__rate_interval]))\n", @@ -766,47 +440,10 @@ "step": null } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, "title": "Reads - 4xx", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "stat", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] + "type": "stat" }, { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", "fieldConfig": { "defaults": { @@ -832,7 +469,6 @@ }, "overrides": [ ] }, - "fill": 1, "gridPos": { "h": 4, "w": 2, @@ -840,28 +476,16 @@ "y": 4 }, "id": 8, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, "links": [ ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, - "span": 6, - "stack": false, - "steppedLine": false, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "targets": [ { "expr": "sum(rate(cortex_request_duration_seconds_count{job=~\"$job\", route=~\"(prometheus|api_prom)_api_v1_.+\",status_code=~\"5.+\"}[$__rate_interval])) /\nsum(rate(cortex_request_duration_seconds_count{job=~\"$job\", route=~\"(prometheus|api_prom)_api_v1_.+\"}[$__rate_interval]))\n", @@ -874,47 +498,10 @@ "step": null } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, "title": "Reads - 5xx", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "stat", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] + "type": "stat" }, { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", "fieldConfig": { "defaults": { @@ -944,7 +531,6 @@ }, "overrides": [ ] }, - "fill": 1, "gridPos": { "h": 4, "w": 8, @@ -952,28 +538,16 @@ "y": 4 }, "id": 9, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, "links": [ ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, - "span": 6, - "stack": false, - "steppedLine": false, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "targets": [ { "expr": "histogram_quantile(0.99, sum by (le) (cluster_job_route:cortex_request_duration_seconds_bucket:sum_rate{job=~\"$job\", route=~\"(prometheus|api_prom)_api_v1_.+\"}))\n", @@ -986,47 +560,10 @@ "step": null } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, - "title": "Reads 99th Latency", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "stat", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] + "title": "Reads 99th latency", + "type": "stat" }, { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", "fieldConfig": { "defaults": { @@ -1056,7 +593,6 @@ }, "overrides": [ ] }, - "fill": 1, "gridPos": { "h": 8, "w": 10, @@ -1064,34 +600,13 @@ "y": 8 }, "id": 10, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, "links": [ ], - "nullPointMode": "null as zero", "options": { "text": { "titleSize": 14, "valueSize": 14 } }, - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, - "span": 6, - "stack": false, - "steppedLine": false, "targets": [ { "expr": "kube_deployment_status_replicas_unavailable{job=~\"$job\", deployment=~\"cortex-gw|distributor|ingester.*|query-frontend.*|query-scheduler.*|querier.*|compactor|store-gateway|ruler|alertmanager\"}\n> 0\n", @@ -1114,41 +629,8 @@ "step": null } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, "title": "Unhealthy pods", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "stat", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] + "type": "stat" }, { "datasource": "$datasource", @@ -1214,12 +696,29 @@ "type": "table" }, { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", - "fill": 1, + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 1, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "short" + }, + "overrides": [ ] + }, "gridPos": { "h": 8, "w": 8, @@ -1227,65 +726,34 @@ "y": 8 }, "id": 12, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, "links": [ ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, - "span": 6, - "stack": false, - "steppedLine": false, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "targets": [ { "expr": "1 - (\n avg_over_time(histogram_quantile(0.99, sum by (le) (cluster_job_route:cortex_request_duration_seconds_bucket:sum_rate{job=~\"$job\", route=~\"api_(v1|prom)_push\"} offset 24h))[1h:])\n /\n avg_over_time(histogram_quantile(0.99, sum by (le) (cluster_job_route:cortex_request_duration_seconds_bucket:sum_rate{job=~\"$job\", route=~\"api_(v1|prom)_push\"}))[1h:])\n)\n", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "writes", - "legendLink": null, - "step": 10 + "legendLink": null }, { "expr": "1 - (\n avg_over_time(histogram_quantile(0.99, sum by (le) (cluster_job_route:cortex_request_duration_seconds_bucket:sum_rate{job=~\"$job\", route=~\"(prometheus|api_prom)_api_v1_.+\"} offset 24h))[1h:])\n /\n avg_over_time(histogram_quantile(0.99, sum by (le) (cluster_job_route:cortex_request_duration_seconds_bucket:sum_rate{job=~\"$job\", route=~\"(prometheus|api_prom)_api_v1_.+\"}))[1h:])\n)\n", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "reads", - "legendLink": null, - "step": 10 + "legendLink": null } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, "title": "Latency vs 24h ago", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, + "type": "timeseries", "yaxes": [ { "format": "percentunit", @@ -1321,7 +789,7 @@ "value": "Cortex" }, "hide": 0, - "label": null, + "label": "Data source", "name": "datasource", "options": [ ], "query": "prometheus", diff --git a/docs/getting-started/dashboards/cortex-scaling.json b/docs/getting-started/dashboards/cortex-scaling.json index f481f25bdd..fcadcc3bd1 100644 --- a/docs/getting-started/dashboards/cortex-scaling.json +++ b/docs/getting-started/dashboards/cortex-scaling.json @@ -49,12 +49,27 @@ "height": "400px", "panels": [ { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 1, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "short" + }, "overrides": [ { "matcher": { @@ -142,81 +157,34 @@ } ] }, - "fill": 1, "id": 2, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, "links": [ ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "sort": { "col": 0, "desc": false }, - "spaceLength": 10, "span": 12, - "stack": false, - "steppedLine": false, "targets": [ { "expr": "sort_desc(\n cluster_namespace_deployment_reason:required_replicas:count{cluster=~\"$cluster\", namespace=~\"$namespace\"}\n > ignoring(reason) group_left\n cluster_namespace_deployment:actual_replicas:count{cluster=~\"$cluster\", namespace=~\"$namespace\"}\n)\n", "format": "table", "instant": true, - "intervalFactor": 2, "legendFormat": "", - "refId": "A", - "step": 10 + "refId": "A" } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, "title": "Workload-based scaling", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, "transform": "table", - "type": "table", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] + "type": "table" } ], "repeat": null, @@ -240,7 +208,7 @@ "value": "Cortex" }, "hide": 0, - "label": null, + "label": "Data source", "name": "datasource", "options": [ ], "query": "prometheus", diff --git a/docs/getting-started/dashboards/cortex-slow-queries.json b/docs/getting-started/dashboards/cortex-slow-queries.json index cf1ac17c49..d2853710a9 100644 --- a/docs/getting-started/dashboards/cortex-slow-queries.json +++ b/docs/getting-started/dashboards/cortex-slow-queries.json @@ -171,7 +171,7 @@ "value": "Cortex" }, "hide": 0, - "label": null, + "label": "Data source", "name": "datasource", "options": [ ], "query": "prometheus", diff --git a/docs/getting-started/dashboards/cortex-writes.json b/docs/getting-started/dashboards/cortex-writes.json index f7b3070e9d..296a65a655 100644 --- a/docs/getting-started/dashboards/cortex-writes.json +++ b/docs/getting-started/dashboards/cortex-writes.json @@ -50,308 +50,200 @@ "height": "100px", "panels": [ { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", - "fill": 1, + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 1, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "short" + }, + "overrides": [ ] + }, "format": "short", "id": 2, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, "links": [ ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 3, - "stack": false, - "steppedLine": false, "targets": [ { "expr": "sum(cluster_namespace_job:cortex_distributor_received_samples:rate5m{job=~\"$job\"})", "format": "time_series", "instant": true, - "intervalFactor": 2, "refId": "A" } ], "thresholds": "70,80", - "timeFrom": null, - "timeShift": null, "title": "Samples / sec", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "singlestat", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] + "type": "singlestat" }, { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", - "fill": 1, + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 1, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "short" + }, + "overrides": [ ] + }, "format": "short", "id": 3, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, "links": [ ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 3, - "stack": false, - "steppedLine": false, "targets": [ { "expr": "sum(cortex_ingester_memory_series{job=~\"$job\"}\n/ on(cluster, namespace) group_left\nmax by (cluster, namespace) (cortex_distributor_replication_factor{job=~\"$job\"}))\n", "format": "time_series", "instant": true, - "intervalFactor": 2, "refId": "A" } ], "thresholds": "70,80", - "timeFrom": null, - "timeShift": null, "title": "Active Series", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "singlestat", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] + "type": "singlestat" }, { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", - "fill": 1, + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 1, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "short" + }, + "overrides": [ ] + }, "format": "short", "id": 4, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, "links": [ ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 3, - "stack": false, - "steppedLine": false, "targets": [ { "expr": "count(count by(user) (cortex_ingester_active_series{job=~\"$job\"}))", "format": "time_series", "instant": true, - "intervalFactor": 2, "refId": "A" } ], "thresholds": "70,80", - "timeFrom": null, - "timeShift": null, "title": "Tenants", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "singlestat", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] + "type": "singlestat" }, { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", - "fill": 1, + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 1, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "short" + }, + "overrides": [ ] + }, "format": "reqps", "id": 5, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, "links": [ ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 3, - "stack": false, - "steppedLine": false, "targets": [ { "expr": "sum(rate(cortex_request_duration_seconds_count{job=~\"$job\", route=~\"api_(v1|prom)_push\"}[5m]))", "format": "time_series", "instant": true, - "intervalFactor": 2, "refId": "A" } ], "thresholds": "70,80", - "timeFrom": null, - "timeShift": null, "title": "Requests / sec", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "singlestat", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] + "type": "singlestat" } ], "repeat": null, @@ -372,157 +264,257 @@ "3xx": "#6ED0E0", "4xx": "#EF843C", "5xx": "#E24D42", + "OK": "#7EB26D", + "cancel": "#A9A9A9", "error": "#E24D42", "success": "#7EB26D" }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 100, + "lineWidth": 0, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + } + }, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "reqps" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "1xx" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EAB839", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "2xx" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#7EB26D", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "3xx" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#6ED0E0", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "4xx" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EF843C", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "5xx" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E24D42", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "OK" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#7EB26D", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "cancel" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#A9A9A9", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "error" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E24D42", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "success" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#7EB26D", + "mode": "fixed" + } + } + ] + } + ] + }, "fill": 10, "id": 6, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, "linewidth": 0, "links": [ ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 4, "stack": true, - "steppedLine": false, "targets": [ { - "expr": "sum by (status) (\n label_replace(label_replace(rate(cortex_request_duration_seconds_count{job=~\"$job\", route=~\"api_(v1|prom)_push\"}[$__rate_interval]),\n \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"),\n \"status\", \"${1}\", \"status_code\", \"([a-z]+)\"))\n", + "expr": "sum by (status) (\n label_replace(label_replace(rate(cortex_request_duration_seconds_count{job=~\"$job\", route=~\"api_(v1|prom)_push\"}[$__rate_interval]),\n \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"),\n \"status\", \"${1}\", \"status_code\", \"([a-zA-Z]+)\"))\n", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "{{status}}", - "refId": "A", - "step": 10 + "refId": "A" } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, "title": "Requests / sec", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] + "type": "timeseries" }, { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", - "fill": 1, + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 1, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "ms" + }, + "overrides": [ ] + }, "id": 7, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, "links": [ ], "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 4, - "stack": false, - "steppedLine": false, "targets": [ { "expr": "histogram_quantile(0.99, sum by (le) (cluster_job_route:cortex_request_duration_seconds_bucket:sum_rate{job=~\"$job\", route=~\"api_(v1|prom)_push\"})) * 1e3", "format": "time_series", - "intervalFactor": 2, - "legendFormat": "99th Percentile", - "refId": "A", - "step": 10 + "legendFormat": "99th percentile", + "refId": "A" }, { "expr": "histogram_quantile(0.50, sum by (le) (cluster_job_route:cortex_request_duration_seconds_bucket:sum_rate{job=~\"$job\", route=~\"api_(v1|prom)_push\"})) * 1e3", "format": "time_series", - "intervalFactor": 2, - "legendFormat": "50th Percentile", - "refId": "B", - "step": 10 + "legendFormat": "50th percentile", + "refId": "B" }, { "expr": "1e3 * sum(cluster_job_route:cortex_request_duration_seconds_sum:sum_rate{job=~\"$job\", route=~\"api_(v1|prom)_push\"}) / sum(cluster_job_route:cortex_request_duration_seconds_count:sum_rate{job=~\"$job\", route=~\"api_(v1|prom)_push\"})", "format": "time_series", - "intervalFactor": 2, "legendFormat": "Average", - "refId": "C", - "step": 10 + "refId": "C" } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, "title": "Latency", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, + "type": "timeseries", "yaxes": [ { "format": "ms", @@ -543,73 +535,59 @@ ] }, { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 1, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "s" + }, + "overrides": [ ] + }, "fill": 0, "id": 8, "legend": { "show": false }, - "lines": true, - "linewidth": 1, "links": [ ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 4, - "stack": false, - "steppedLine": false, "targets": [ { "expr": "histogram_quantile(0.99, sum by(le, pod) (rate(cortex_request_duration_seconds_bucket{job=~\"$job\", route=~\"api_(v1|prom)_push\"}[$__rate_interval])))", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "", - "legendLink": null, - "step": 10 + "legendLink": null } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, "title": "Per pod p99 Latency", "tooltip": { "sort": 2 }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, - "yaxes": [ - { - "format": "s", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] + "type": "timeseries" } ], "repeat": null, @@ -630,157 +608,257 @@ "3xx": "#6ED0E0", "4xx": "#EF843C", "5xx": "#E24D42", + "OK": "#7EB26D", + "cancel": "#A9A9A9", "error": "#E24D42", "success": "#7EB26D" }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 100, + "lineWidth": 0, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + } + }, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "reqps" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "1xx" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EAB839", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "2xx" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#7EB26D", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "3xx" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#6ED0E0", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "4xx" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EF843C", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "5xx" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E24D42", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "OK" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#7EB26D", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "cancel" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#A9A9A9", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "error" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E24D42", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "success" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#7EB26D", + "mode": "fixed" + } + } + ] + } + ] + }, "fill": 10, "id": 9, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, "linewidth": 0, "links": [ ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 4, "stack": true, - "steppedLine": false, "targets": [ { - "expr": "sum by (status) (\n label_replace(label_replace(rate(cortex_request_duration_seconds_count{job=~\"$job\", route=~\"/distributor.Distributor/Push|/httpgrpc.*|api_(v1|prom)_push\"}[$__rate_interval]),\n \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"),\n \"status\", \"${1}\", \"status_code\", \"([a-z]+)\"))\n", + "expr": "sum by (status) (\n label_replace(label_replace(rate(cortex_request_duration_seconds_count{job=~\"$job\", route=~\"/distributor.Distributor/Push|/httpgrpc.*|api_(v1|prom)_push\"}[$__rate_interval]),\n \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"),\n \"status\", \"${1}\", \"status_code\", \"([a-zA-Z]+)\"))\n", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "{{status}}", - "refId": "A", - "step": 10 + "refId": "A" } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, "title": "Requests / sec", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] + "type": "timeseries" }, { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", - "fill": 1, + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 1, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "ms" + }, + "overrides": [ ] + }, "id": 10, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, "links": [ ], "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 4, - "stack": false, - "steppedLine": false, "targets": [ { "expr": "histogram_quantile(0.99, sum by (le) (cluster_job_route:cortex_request_duration_seconds_bucket:sum_rate{job=~\"$job\", route=~\"/distributor.Distributor/Push|/httpgrpc.*|api_(v1|prom)_push\"})) * 1e3", "format": "time_series", - "intervalFactor": 2, - "legendFormat": "99th Percentile", - "refId": "A", - "step": 10 + "legendFormat": "99th percentile", + "refId": "A" }, { "expr": "histogram_quantile(0.50, sum by (le) (cluster_job_route:cortex_request_duration_seconds_bucket:sum_rate{job=~\"$job\", route=~\"/distributor.Distributor/Push|/httpgrpc.*|api_(v1|prom)_push\"})) * 1e3", "format": "time_series", - "intervalFactor": 2, - "legendFormat": "50th Percentile", - "refId": "B", - "step": 10 + "legendFormat": "50th percentile", + "refId": "B" }, { "expr": "1e3 * sum(cluster_job_route:cortex_request_duration_seconds_sum:sum_rate{job=~\"$job\", route=~\"/distributor.Distributor/Push|/httpgrpc.*|api_(v1|prom)_push\"}) / sum(cluster_job_route:cortex_request_duration_seconds_count:sum_rate{job=~\"$job\", route=~\"/distributor.Distributor/Push|/httpgrpc.*|api_(v1|prom)_push\"})", "format": "time_series", - "intervalFactor": 2, "legendFormat": "Average", - "refId": "C", - "step": 10 + "refId": "C" } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, "title": "Latency", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, + "type": "timeseries", "yaxes": [ { "format": "ms", @@ -801,73 +879,59 @@ ] }, { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 1, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "s" + }, + "overrides": [ ] + }, "fill": 0, "id": 11, "legend": { "show": false }, - "lines": true, - "linewidth": 1, "links": [ ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 4, - "stack": false, - "steppedLine": false, "targets": [ { "expr": "histogram_quantile(0.99, sum by(le, pod) (rate(cortex_request_duration_seconds_bucket{job=~\"$job\", route=~\"/distributor.Distributor/Push|/httpgrpc.*|api_(v1|prom)_push\"}[$__rate_interval])))", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "", - "legendLink": null, - "step": 10 + "legendLink": null } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, "title": "Per pod p99 Latency", "tooltip": { "sort": 2 }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, - "yaxes": [ - { - "format": "s", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] + "type": "timeseries" } ], "repeat": null, @@ -888,157 +952,257 @@ "3xx": "#6ED0E0", "4xx": "#EF843C", "5xx": "#E24D42", + "OK": "#7EB26D", + "cancel": "#A9A9A9", "error": "#E24D42", "success": "#7EB26D" }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 100, + "lineWidth": 0, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + } + }, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "reqps" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "1xx" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EAB839", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "2xx" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#7EB26D", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "3xx" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#6ED0E0", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "4xx" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EF843C", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "5xx" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E24D42", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "OK" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#7EB26D", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "cancel" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#A9A9A9", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "error" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E24D42", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "success" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#7EB26D", + "mode": "fixed" + } + } + ] + } + ] + }, "fill": 10, "id": 12, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, "linewidth": 0, "links": [ ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 6, "stack": true, - "steppedLine": false, "targets": [ { - "expr": "sum by (status) (\n label_replace(label_replace(rate(cortex_kv_request_duration_seconds_count{job=~\"$job\"}[$__rate_interval]),\n \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"),\n \"status\", \"${1}\", \"status_code\", \"([a-z]+)\"))\n", + "expr": "sum by (status) (\n label_replace(label_replace(rate(cortex_kv_request_duration_seconds_count{job=~\"$job\"}[$__rate_interval]),\n \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"),\n \"status\", \"${1}\", \"status_code\", \"([a-zA-Z]+)\"))\n", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "{{status}}", - "refId": "A", - "step": 10 + "refId": "A" } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, "title": "Requests / sec", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] + "type": "timeseries" }, { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", - "fill": 1, + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 1, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "ms" + }, + "overrides": [ ] + }, "id": 13, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, "links": [ ], "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 6, - "stack": false, - "steppedLine": false, "targets": [ { "expr": "histogram_quantile(0.99, sum by (le) (cluster_job:cortex_kv_request_duration_seconds_bucket:sum_rate{job=~\"$job\"})) * 1e3", "format": "time_series", - "intervalFactor": 2, - "legendFormat": "99th Percentile", - "refId": "A", - "step": 10 + "legendFormat": "99th percentile", + "refId": "A" }, { "expr": "histogram_quantile(0.50, sum by (le) (cluster_job:cortex_kv_request_duration_seconds_bucket:sum_rate{job=~\"$job\"})) * 1e3", "format": "time_series", - "intervalFactor": 2, - "legendFormat": "50th Percentile", - "refId": "B", - "step": 10 + "legendFormat": "50th percentile", + "refId": "B" }, { "expr": "1e3 * sum(cluster_job:cortex_kv_request_duration_seconds_sum:sum_rate{job=~\"$job\"}) / sum(cluster_job:cortex_kv_request_duration_seconds_count:sum_rate{job=~\"$job\"})", "format": "time_series", - "intervalFactor": 2, "legendFormat": "Average", - "refId": "C", - "step": 10 + "refId": "C" } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, "title": "Latency", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, + "type": "timeseries", "yaxes": [ { "format": "ms", @@ -1077,157 +1241,257 @@ "3xx": "#6ED0E0", "4xx": "#EF843C", "5xx": "#E24D42", + "OK": "#7EB26D", + "cancel": "#A9A9A9", "error": "#E24D42", "success": "#7EB26D" }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 100, + "lineWidth": 0, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + } + }, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "reqps" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "1xx" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EAB839", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "2xx" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#7EB26D", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "3xx" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#6ED0E0", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "4xx" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EF843C", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "5xx" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E24D42", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "OK" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#7EB26D", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "cancel" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#A9A9A9", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "error" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E24D42", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "success" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#7EB26D", + "mode": "fixed" + } + } + ] + } + ] + }, "fill": 10, "id": 14, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 0, - "links": [ ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, - "span": 4, - "stack": true, - "steppedLine": false, - "targets": [ - { - "expr": "sum by (status) (\n label_replace(label_replace(rate(cortex_request_duration_seconds_count{job=~\"$job\",route=\"/cortex.Ingester/Push\"}[$__rate_interval]),\n \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"),\n \"status\", \"${1}\", \"status_code\", \"([a-z]+)\"))\n", - "format": "time_series", - "interval": "15s", - "intervalFactor": 2, - "legendFormat": "{{status}}", - "refId": "A", - "step": 10 - } - ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, - "title": "Requests / sec", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true + "linewidth": 0, + "links": [ ], + "options": { + "legend": { + "showLegend": true }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "span": 4, + "stack": true, + "targets": [ { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false + "expr": "sum by (status) (\n label_replace(label_replace(rate(cortex_request_duration_seconds_count{job=~\"$job\",route=\"/cortex.Ingester/Push\"}[$__rate_interval]),\n \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"),\n \"status\", \"${1}\", \"status_code\", \"([a-zA-Z]+)\"))\n", + "format": "time_series", + "interval": "15s", + "legendFormat": "{{status}}", + "refId": "A" } - ] + ], + "title": "Requests / sec", + "type": "timeseries" }, { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", - "fill": 1, + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 1, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "ms" + }, + "overrides": [ ] + }, "id": 15, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, "links": [ ], "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 4, - "stack": false, - "steppedLine": false, "targets": [ { "expr": "histogram_quantile(0.99, sum by (le) (cluster_job_route:cortex_request_duration_seconds_bucket:sum_rate{job=~\"$job\", route=\"/cortex.Ingester/Push\"})) * 1e3", "format": "time_series", - "intervalFactor": 2, - "legendFormat": "99th Percentile", - "refId": "A", - "step": 10 + "legendFormat": "99th percentile", + "refId": "A" }, { "expr": "histogram_quantile(0.50, sum by (le) (cluster_job_route:cortex_request_duration_seconds_bucket:sum_rate{job=~\"$job\", route=\"/cortex.Ingester/Push\"})) * 1e3", "format": "time_series", - "intervalFactor": 2, - "legendFormat": "50th Percentile", - "refId": "B", - "step": 10 + "legendFormat": "50th percentile", + "refId": "B" }, { "expr": "1e3 * sum(cluster_job_route:cortex_request_duration_seconds_sum:sum_rate{job=~\"$job\", route=\"/cortex.Ingester/Push\"}) / sum(cluster_job_route:cortex_request_duration_seconds_count:sum_rate{job=~\"$job\", route=\"/cortex.Ingester/Push\"})", "format": "time_series", - "intervalFactor": 2, "legendFormat": "Average", - "refId": "C", - "step": 10 + "refId": "C" } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, "title": "Latency", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, + "type": "timeseries", "yaxes": [ { "format": "ms", @@ -1248,73 +1512,59 @@ ] }, { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 1, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "s" + }, + "overrides": [ ] + }, "fill": 0, "id": 16, "legend": { "show": false }, - "lines": true, - "linewidth": 1, "links": [ ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 4, - "stack": false, - "steppedLine": false, "targets": [ { "expr": "histogram_quantile(0.99, sum by(le, pod) (rate(cortex_request_duration_seconds_bucket{job=~\"$job\", route=\"/cortex.Ingester/Push\"}[$__rate_interval])))", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "", - "legendLink": null, - "step": 10 + "legendLink": null } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, "title": "Per pod p99 Latency", "tooltip": { "sort": 2 }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, - "yaxes": [ - { - "format": "s", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] + "type": "timeseries" } ], "repeat": null, @@ -1335,157 +1585,257 @@ "3xx": "#6ED0E0", "4xx": "#EF843C", "5xx": "#E24D42", + "OK": "#7EB26D", + "cancel": "#A9A9A9", "error": "#E24D42", "success": "#7EB26D" }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 100, + "lineWidth": 0, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + } + }, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "reqps" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "1xx" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EAB839", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "2xx" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#7EB26D", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "3xx" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#6ED0E0", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "4xx" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EF843C", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "5xx" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E24D42", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "OK" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#7EB26D", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "cancel" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#A9A9A9", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "error" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E24D42", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "success" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#7EB26D", + "mode": "fixed" + } + } + ] + } + ] + }, "fill": 10, "id": 17, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, "linewidth": 0, "links": [ ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 6, "stack": true, - "steppedLine": false, "targets": [ { - "expr": "sum by (status) (\n label_replace(label_replace(rate(cortex_kv_request_duration_seconds_count{job=~\"$job\"}[$__rate_interval]),\n \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"),\n \"status\", \"${1}\", \"status_code\", \"([a-z]+)\"))\n", + "expr": "sum by (status) (\n label_replace(label_replace(rate(cortex_kv_request_duration_seconds_count{job=~\"$job\"}[$__rate_interval]),\n \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"),\n \"status\", \"${1}\", \"status_code\", \"([a-zA-Z]+)\"))\n", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "{{status}}", - "refId": "A", - "step": 10 + "refId": "A" } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, "title": "Requests / sec", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] + "type": "timeseries" }, { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", - "fill": 1, + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 1, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "ms" + }, + "overrides": [ ] + }, "id": 18, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, "links": [ ], "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 6, - "stack": false, - "steppedLine": false, "targets": [ { "expr": "histogram_quantile(0.99, sum by (le) (cluster_job:cortex_kv_request_duration_seconds_bucket:sum_rate{job=~\"$job\"})) * 1e3", "format": "time_series", - "intervalFactor": 2, - "legendFormat": "99th Percentile", - "refId": "A", - "step": 10 + "legendFormat": "99th percentile", + "refId": "A" }, { "expr": "histogram_quantile(0.50, sum by (le) (cluster_job:cortex_kv_request_duration_seconds_bucket:sum_rate{job=~\"$job\"})) * 1e3", "format": "time_series", - "intervalFactor": 2, - "legendFormat": "50th Percentile", - "refId": "B", - "step": 10 + "legendFormat": "50th percentile", + "refId": "B" }, { "expr": "1e3 * sum(cluster_job:cortex_kv_request_duration_seconds_sum:sum_rate{job=~\"$job\"}) / sum(cluster_job:cortex_kv_request_duration_seconds_count:sum_rate{job=~\"$job\"})", "format": "time_series", - "intervalFactor": 2, "legendFormat": "Average", - "refId": "C", - "step": 10 + "refId": "C" } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, "title": "Latency", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, + "type": "timeseries", "yaxes": [ { "format": "ms", @@ -1522,168 +1872,127 @@ "failed": "#E24D42", "successful": "#7EB26D" }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", "description": "### Uploaded blocks / sec\nThe rate of blocks being uploaded from the ingesters\nto object storage.\n\n", + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 1, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "short" + }, + "overrides": [ ] + }, "fill": 10, "id": 19, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, "linewidth": 0, "links": [ ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 6, "stack": true, - "steppedLine": false, "targets": [ { "expr": "sum(rate(cortex_ingester_shipper_uploads_total{job=~\"$job\"}[$__rate_interval])) - sum(rate(cortex_ingester_shipper_upload_failures_total{job=~\"$job\"}[$__rate_interval]))", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "successful", - "legendLink": null, - "step": 10 + "legendLink": null }, { "expr": "sum(rate(cortex_ingester_shipper_upload_failures_total{job=~\"$job\"}[$__rate_interval]))", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "failed", - "legendLink": null, - "step": 10 + "legendLink": null } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, "title": "Uploaded blocks / sec", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] + "type": "timeseries" }, { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", "description": "### Upload latency\nThe average, median (50th percentile), and 99th percentile time\nthe ingesters take to upload blocks to object storage.\n\n", - "fill": 1, + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 1, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "ms" + }, + "overrides": [ ] + }, "id": 20, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, "links": [ ], "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 6, - "stack": false, - "steppedLine": false, "targets": [ { "expr": "histogram_quantile(0.99, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{job=~\"$job\",component=\"ingester\",operation=\"upload\"}[$__rate_interval])) by (le)) * 1e3", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "99th Percentile", - "refId": "A", - "step": 10 + "refId": "A" }, { "expr": "histogram_quantile(0.50, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{job=~\"$job\",component=\"ingester\",operation=\"upload\"}[$__rate_interval])) by (le)) * 1e3", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "50th Percentile", - "refId": "B", - "step": 10 + "refId": "B" }, { "expr": "sum(rate(thanos_objstore_bucket_operation_duration_seconds_sum{job=~\"$job\",component=\"ingester\",operation=\"upload\"}[$__rate_interval])) * 1e3 / sum(rate(thanos_objstore_bucket_operation_duration_seconds_count{job=~\"$job\",component=\"ingester\",operation=\"upload\"}[$__rate_interval]))", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "Average", - "refId": "C", - "step": 10 + "refId": "C" } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, "title": "Upload latency", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, + "type": "timeseries", "yaxes": [ { "format": "ms", @@ -1720,168 +2029,127 @@ "failed": "#E24D42", "successful": "#7EB26D" }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", "description": "### Compactions per second\nIngesters maintain a local TSDB per-tenant on disk. Each TSDB maintains a head block for each\nactive time series; these blocks get periodically compacted (by default, every 2h).\nThis panel shows the rate of compaction operations across all TSDBs on all ingesters.\n\n", + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 1, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "short" + }, + "overrides": [ ] + }, "fill": 10, "id": 21, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, "linewidth": 0, "links": [ ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 6, "stack": true, - "steppedLine": false, "targets": [ { "expr": "sum(rate(cortex_ingester_tsdb_compactions_total{job=~\"$job\"}[$__rate_interval]))", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "successful", - "legendLink": null, - "step": 10 + "legendLink": null }, { "expr": "sum(rate(cortex_ingester_tsdb_compactions_failed_total{job=~\"$job\"}[$__rate_interval]))", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "failed", - "legendLink": null, - "step": 10 + "legendLink": null } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, "title": "Compactions / sec", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] + "type": "timeseries" }, { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", "description": "### Compaction latency\nThe average, median (50th percentile), and 99th percentile time ingesters take to compact TSDB head blocks\non the local filesystem.\n\n", - "fill": 1, + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 1, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "ms" + }, + "overrides": [ ] + }, "id": 22, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, "links": [ ], "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 6, - "stack": false, - "steppedLine": false, "targets": [ { "expr": "histogram_quantile(0.99, sum(rate(cortex_ingester_tsdb_compaction_duration_seconds_bucket{job=~\"$job\"}[$__rate_interval])) by (le)) * 1e3", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "99th Percentile", - "refId": "A", - "step": 10 + "refId": "A" }, { "expr": "histogram_quantile(0.50, sum(rate(cortex_ingester_tsdb_compaction_duration_seconds_bucket{job=~\"$job\"}[$__rate_interval])) by (le)) * 1e3", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "50th Percentile", - "refId": "B", - "step": 10 + "refId": "B" }, { "expr": "sum(rate(cortex_ingester_tsdb_compaction_duration_seconds_sum{job=~\"$job\"}[$__rate_interval])) * 1e3 / sum(rate(cortex_ingester_tsdb_compaction_duration_seconds_count{job=~\"$job\"}[$__rate_interval]))", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "Average", - "refId": "C", - "step": 10 + "refId": "C" } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, "title": "Compactions latency", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, + "type": "timeseries", "yaxes": [ { "format": "ms", @@ -1918,329 +2186,237 @@ "failed": "#E24D42", "successful": "#7EB26D" }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", "description": "### WAL truncations per second\nThe WAL is truncated each time a new TSDB block is written. This panel measures the rate of\ntruncations.\n\n", + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 1, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "short" + }, + "overrides": [ ] + }, "fill": 10, "id": 23, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, "linewidth": 0, "links": [ ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 3, "stack": true, - "steppedLine": false, "targets": [ { "expr": "sum(rate(cortex_ingester_tsdb_wal_truncations_total{job=~\"$job\"}[$__rate_interval])) - sum(rate(cortex_ingester_tsdb_wal_truncations_failed_total{job=~\"$job\"}[$__rate_interval]))", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "successful", - "legendLink": null, - "step": 10 + "legendLink": null }, { "expr": "sum(rate(cortex_ingester_tsdb_wal_truncations_failed_total{job=~\"$job\"}[$__rate_interval]))", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "failed", - "legendLink": null, - "step": 10 + "legendLink": null } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, "title": "WAL truncations / sec", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] + "type": "timeseries" }, { "aliasColors": { "failed": "#E24D42", "successful": "#7EB26D" }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", "description": "### Checkpoints created per second\nCheckpoints are created as part of the WAL truncation process.\nThis metric measures the rate of checkpoint creation.\n\n", + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 1, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "short" + }, + "overrides": [ ] + }, "fill": 10, "id": 24, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, "linewidth": 0, "links": [ ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 3, "stack": true, - "steppedLine": false, "targets": [ { "expr": "sum(rate(cortex_ingester_tsdb_checkpoint_creations_total{job=~\"$job\"}[$__rate_interval])) - sum(rate(cortex_ingester_tsdb_checkpoint_creations_failed_total{job=~\"$job\"}[$__rate_interval]))", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "successful", - "legendLink": null, - "step": 10 + "legendLink": null }, { "expr": "sum(rate(cortex_ingester_tsdb_checkpoint_creations_failed_total{job=~\"$job\"}[$__rate_interval]))", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "failed", - "legendLink": null, - "step": 10 + "legendLink": null } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, "title": "Checkpoints created / sec", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] + "type": "timeseries" }, { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", "description": "### WAL truncations latency (including checkpointing)\nAverage time taken to perform a full WAL truncation,\nincluding the time taken for the checkpointing to complete.\n\n", - "fill": 1, + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 1, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "s" + }, + "overrides": [ ] + }, "id": 25, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, "links": [ ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 3, - "stack": false, - "steppedLine": false, "targets": [ { "expr": "sum(rate(cortex_ingester_tsdb_wal_truncate_duration_seconds_sum{job=~\"$job\"}[$__rate_interval])) / sum(rate(cortex_ingester_tsdb_wal_truncate_duration_seconds_count{job=~\"$job\"}[$__rate_interval]))", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "avg", - "legendLink": null, - "step": 10 + "legendLink": null } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, "title": "WAL truncations latency (includes checkpointing)", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, - "yaxes": [ - { - "format": "s", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] + "type": "timeseries" }, { "aliasColors": { "WAL": "#E24D42", "mmap-ed chunks": "#E28A42" }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 1, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "ops" + }, + "overrides": [ ] + }, "fill": 10, "id": 26, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, "linewidth": 0, "links": [ ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 3, "stack": true, - "steppedLine": false, "targets": [ { "expr": "sum(rate(cortex_ingester_wal_corruptions_total{job=~\"$job\"}[$__rate_interval]))", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "WAL", - "legendLink": null, - "step": 10 + "legendLink": null }, { "expr": "sum(rate(cortex_ingester_tsdb_mmap_chunk_corruptions_total{job=~\"$job\"}[$__rate_interval]))", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "mmap-ed chunks", - "legendLink": null, - "step": 10 + "legendLink": null } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, "title": "Corruptions / sec", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, + "type": "timeseries", "yaxes": [ { "format": "ops", @@ -2282,7 +2458,7 @@ "value": "Cortex" }, "hide": 0, - "label": null, + "label": "Data source", "name": "datasource", "options": [ ], "query": "prometheus", diff --git a/docs/getting-started/dashboards/ruler.json b/docs/getting-started/dashboards/ruler.json index d17fcad627..e48964da33 100644 --- a/docs/getting-started/dashboards/ruler.json +++ b/docs/getting-started/dashboards/ruler.json @@ -27,308 +27,200 @@ "height": "100px", "panels": [ { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", - "fill": 1, + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 1, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "short" + }, + "overrides": [ ] + }, "format": "short", "id": 1, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, "links": [ ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 3, - "stack": false, - "steppedLine": false, "targets": [ { "expr": "sum(cortex_ruler_managers_total{job=~\"$job\"})", "format": "time_series", "instant": true, - "intervalFactor": 2, "refId": "A" } ], "thresholds": "70,80", - "timeFrom": null, - "timeShift": null, "title": "Active Configurations", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "singlestat", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] + "type": "singlestat" }, { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", - "fill": 1, + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 1, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "short" + }, + "overrides": [ ] + }, "format": "short", "id": 2, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, "links": [ ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 3, - "stack": false, - "steppedLine": false, "targets": [ { "expr": "sum(cortex_prometheus_rule_group_rules{job=~\"$job\"})", "format": "time_series", "instant": true, - "intervalFactor": 2, "refId": "A" } ], "thresholds": "70,80", - "timeFrom": null, - "timeShift": null, "title": "Total Rules", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "singlestat", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] + "type": "singlestat" }, { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", - "fill": 1, + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 1, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "short" + }, + "overrides": [ ] + }, "format": "reqps", "id": 3, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, "links": [ ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 3, - "stack": false, - "steppedLine": false, "targets": [ { "expr": "sum(rate(cortex_ingester_client_request_duration_seconds_count{job=~\"$job\", operation=\"/cortex.Ingester/QueryStream\"}[5m]))", "format": "time_series", "instant": true, - "intervalFactor": 2, "refId": "A" } ], "thresholds": "70,80", - "timeFrom": null, - "timeShift": null, "title": "Read from Ingesters - QPS", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "singlestat", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] + "type": "singlestat" }, { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", - "fill": 1, + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 1, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "short" + }, + "overrides": [ ] + }, "format": "reqps", "id": 4, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, "links": [ ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 3, - "stack": false, - "steppedLine": false, "targets": [ { "expr": "sum(rate(cortex_ingester_client_request_duration_seconds_count{job=~\"$job\", operation=\"/cortex.Ingester/Push\"}[5m]))", "format": "time_series", "instant": true, - "intervalFactor": 2, "refId": "A" } ], "thresholds": "70,80", - "timeFrom": null, - "timeShift": null, "title": "Write to Ingesters - QPS", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "singlestat", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] + "type": "singlestat" } ], "repeat": null, @@ -343,167 +235,107 @@ "height": "250px", "panels": [ { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", - "fill": 1, + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 1, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "short" + }, + "overrides": [ ] + }, "id": 5, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, "links": [ ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 6, - "stack": false, - "steppedLine": false, "targets": [ { "expr": "sum(rate(cortex_prometheus_rule_evaluations_total{job=~\"$job\"}[$__rate_interval]))\n-\nsum(rate(cortex_prometheus_rule_evaluation_failures_total{job=~\"$job\"}[$__rate_interval]))\n", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "success", - "legendLink": null, - "step": 10 + "legendLink": null }, { "expr": "sum(rate(cortex_prometheus_rule_evaluation_failures_total{job=~\"$job\"}[$__rate_interval]))", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "failed", - "legendLink": null, - "step": 10 + "legendLink": null } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, "title": "EPS", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] + "type": "timeseries" }, { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", - "fill": 1, + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 1, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "s" + }, + "overrides": [ ] + }, "id": 6, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, "links": [ ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 6, - "stack": false, - "steppedLine": false, "targets": [ { "expr": "sum (rate(cortex_prometheus_rule_evaluation_duration_seconds_sum{job=~\"$job\"}[$__rate_interval]))\n /\nsum (rate(cortex_prometheus_rule_evaluation_duration_seconds_count{job=~\"$job\"}[$__rate_interval]))\n", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "average", - "legendLink": null, - "step": 10 + "legendLink": null } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, "title": "Latency", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] + "type": "timeseries" } ], "repeat": null, @@ -524,157 +356,257 @@ "3xx": "#6ED0E0", "4xx": "#EF843C", "5xx": "#E24D42", + "OK": "#7EB26D", + "cancel": "#A9A9A9", "error": "#E24D42", "success": "#7EB26D" }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 100, + "lineWidth": 0, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + } + }, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "reqps" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "1xx" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EAB839", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "2xx" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#7EB26D", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "3xx" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#6ED0E0", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "4xx" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EF843C", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "5xx" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E24D42", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "OK" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#7EB26D", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "cancel" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#A9A9A9", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "error" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E24D42", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "success" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#7EB26D", + "mode": "fixed" + } + } + ] + } + ] + }, "fill": 10, "id": 7, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, "linewidth": 0, "links": [ ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 4, "stack": true, - "steppedLine": false, "targets": [ { - "expr": "sum by (status) (\n label_replace(label_replace(rate(cortex_request_duration_seconds_count{job=~\"$job\", route=~\"api_prom_rules.*|api_prom_api_v1_(rules|alerts)\"}[$__rate_interval]),\n \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"),\n \"status\", \"${1}\", \"status_code\", \"([a-z]+)\"))\n", + "expr": "sum by (status) (\n label_replace(label_replace(rate(cortex_request_duration_seconds_count{job=~\"$job\", route=~\"api_prom_rules.*|api_prom_api_v1_(rules|alerts)\"}[$__rate_interval]),\n \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"),\n \"status\", \"${1}\", \"status_code\", \"([a-zA-Z]+)\"))\n", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "{{status}}", - "refId": "A", - "step": 10 + "refId": "A" } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, "title": "QPS", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] + "type": "timeseries" }, { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", - "fill": 1, + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 1, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "ms" + }, + "overrides": [ ] + }, "id": 8, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, "links": [ ], "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 4, - "stack": false, - "steppedLine": false, "targets": [ { "expr": "histogram_quantile(0.99, sum by (le) (cluster_job_route:cortex_request_duration_seconds_bucket:sum_rate{job=~\"$job\", route=~\"api_prom_rules.*|api_prom_api_v1_(rules|alerts)\"})) * 1e3", "format": "time_series", - "intervalFactor": 2, - "legendFormat": "99th Percentile", - "refId": "A", - "step": 10 + "legendFormat": "99th percentile", + "refId": "A" }, { "expr": "histogram_quantile(0.50, sum by (le) (cluster_job_route:cortex_request_duration_seconds_bucket:sum_rate{job=~\"$job\", route=~\"api_prom_rules.*|api_prom_api_v1_(rules|alerts)\"})) * 1e3", "format": "time_series", - "intervalFactor": 2, - "legendFormat": "50th Percentile", - "refId": "B", - "step": 10 + "legendFormat": "50th percentile", + "refId": "B" }, { "expr": "1e3 * sum(cluster_job_route:cortex_request_duration_seconds_sum:sum_rate{job=~\"$job\", route=~\"api_prom_rules.*|api_prom_api_v1_(rules|alerts)\"}) / sum(cluster_job_route:cortex_request_duration_seconds_count:sum_rate{job=~\"$job\", route=~\"api_prom_rules.*|api_prom_api_v1_(rules|alerts)\"})", "format": "time_series", - "intervalFactor": 2, "legendFormat": "Average", - "refId": "C", - "step": 10 + "refId": "C" } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, "title": "Latency", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, + "type": "timeseries", "yaxes": [ { "format": "ms", @@ -695,81 +627,52 @@ ] }, { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", - "fill": 1, + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 1, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "s" + }, + "overrides": [ ] + }, "id": 9, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, "links": [ ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 4, - "stack": false, - "steppedLine": false, "targets": [ { "expr": "histogram_quantile(0.99, sum by (route, le) (cluster_job_route:cortex_request_duration_seconds_bucket:sum_rate{job=~\"$job\", route=~\"api_prom_rules.*|api_prom_api_v1_(rules|alerts)\"}))", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "{{ route }}", - "legendLink": null, - "step": 10 + "legendLink": null } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, - "title": "Per route p99 Latency", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, - "yaxes": [ - { - "format": "s", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] + "title": "Per route p99 latency", + "type": "timeseries" } ], "repeat": null, @@ -790,160 +693,260 @@ "3xx": "#6ED0E0", "4xx": "#EF843C", "5xx": "#E24D42", + "OK": "#7EB26D", + "cancel": "#A9A9A9", "error": "#E24D42", "success": "#7EB26D" }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 100, + "lineWidth": 0, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + } + }, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "reqps" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "1xx" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EAB839", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "2xx" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#7EB26D", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "3xx" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#6ED0E0", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "4xx" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EF843C", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "5xx" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E24D42", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "OK" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#7EB26D", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "cancel" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#A9A9A9", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "error" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E24D42", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "success" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#7EB26D", + "mode": "fixed" + } + } + ] + } + ] + }, "fill": 10, "id": 10, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, "linewidth": 0, "links": [ ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 6, "stack": true, - "steppedLine": false, "targets": [ { - "expr": "sum by (status) (\n label_replace(label_replace(rate(cortex_ingester_client_request_duration_seconds_count{job=~\"$job\", operation=\"/cortex.Ingester/Push\"}[$__rate_interval]),\n \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"),\n \"status\", \"${1}\", \"status_code\", \"([a-z]+)\"))\n", + "expr": "sum by (status) (\n label_replace(label_replace(rate(cortex_ingester_client_request_duration_seconds_count{job=~\"$job\", operation=\"/cortex.Ingester/Push\"}[$__rate_interval]),\n \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"),\n \"status\", \"${1}\", \"status_code\", \"([a-zA-Z]+)\"))\n", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "{{status}}", - "refId": "A", - "step": 10 + "refId": "A" } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, "title": "QPS", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] + "type": "timeseries" }, { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", - "fill": 1, + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 1, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "ms" + }, + "overrides": [ ] + }, "id": 11, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, "links": [ ], "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 6, - "stack": false, - "steppedLine": false, "targets": [ { "expr": "histogram_quantile(0.99, sum(rate(cortex_ingester_client_request_duration_seconds_bucket{job=~\"$job\", operation=\"/cortex.Ingester/Push\"}[$__rate_interval])) by (le)) * 1e3", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "99th Percentile", - "refId": "A", - "step": 10 + "refId": "A" }, { "expr": "histogram_quantile(0.50, sum(rate(cortex_ingester_client_request_duration_seconds_bucket{job=~\"$job\", operation=\"/cortex.Ingester/Push\"}[$__rate_interval])) by (le)) * 1e3", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "50th Percentile", - "refId": "B", - "step": 10 + "refId": "B" }, { "expr": "sum(rate(cortex_ingester_client_request_duration_seconds_sum{job=~\"$job\", operation=\"/cortex.Ingester/Push\"}[$__rate_interval])) * 1e3 / sum(rate(cortex_ingester_client_request_duration_seconds_count{job=~\"$job\", operation=\"/cortex.Ingester/Push\"}[$__rate_interval]))", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "Average", - "refId": "C", - "step": 10 + "refId": "C" } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, "title": "Latency", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, + "type": "timeseries", "yaxes": [ { "format": "ms", @@ -982,160 +985,260 @@ "3xx": "#6ED0E0", "4xx": "#EF843C", "5xx": "#E24D42", + "OK": "#7EB26D", + "cancel": "#A9A9A9", "error": "#E24D42", "success": "#7EB26D" }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 100, + "lineWidth": 0, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + } + }, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "reqps" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "1xx" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EAB839", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "2xx" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#7EB26D", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "3xx" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#6ED0E0", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "4xx" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EF843C", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "5xx" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E24D42", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "OK" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#7EB26D", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "cancel" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#A9A9A9", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "error" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E24D42", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "success" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#7EB26D", + "mode": "fixed" + } + } + ] + } + ] + }, "fill": 10, "id": 12, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, "linewidth": 0, "links": [ ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 6, "stack": true, - "steppedLine": false, "targets": [ { - "expr": "sum by (status) (\n label_replace(label_replace(rate(cortex_ingester_client_request_duration_seconds_count{job=~\"$job\", operation=\"/cortex.Ingester/QueryStream\"}[$__rate_interval]),\n \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"),\n \"status\", \"${1}\", \"status_code\", \"([a-z]+)\"))\n", + "expr": "sum by (status) (\n label_replace(label_replace(rate(cortex_ingester_client_request_duration_seconds_count{job=~\"$job\", operation=\"/cortex.Ingester/QueryStream\"}[$__rate_interval]),\n \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"),\n \"status\", \"${1}\", \"status_code\", \"([a-zA-Z]+)\"))\n", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "{{status}}", - "refId": "A", - "step": 10 + "refId": "A" } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, "title": "QPS", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] + "type": "timeseries" }, { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", - "fill": 1, + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 1, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "ms" + }, + "overrides": [ ] + }, "id": 13, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, "links": [ ], "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 6, - "stack": false, - "steppedLine": false, "targets": [ { "expr": "histogram_quantile(0.99, sum(rate(cortex_ingester_client_request_duration_seconds_bucket{job=~\"$job\", operation=\"/cortex.Ingester/QueryStream\"}[$__rate_interval])) by (le)) * 1e3", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "99th Percentile", - "refId": "A", - "step": 10 + "refId": "A" }, { "expr": "histogram_quantile(0.50, sum(rate(cortex_ingester_client_request_duration_seconds_bucket{job=~\"$job\", operation=\"/cortex.Ingester/QueryStream\"}[$__rate_interval])) by (le)) * 1e3", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "50th Percentile", - "refId": "B", - "step": 10 + "refId": "B" }, { "expr": "sum(rate(cortex_ingester_client_request_duration_seconds_sum{job=~\"$job\", operation=\"/cortex.Ingester/QueryStream\"}[$__rate_interval])) * 1e3 / sum(rate(cortex_ingester_client_request_duration_seconds_count{job=~\"$job\", operation=\"/cortex.Ingester/QueryStream\"}[$__rate_interval]))", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "Average", - "refId": "C", - "step": 10 + "refId": "C" } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, "title": "Latency", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, + "type": "timeseries", "yaxes": [ { "format": "ms", @@ -1168,84 +1271,70 @@ "height": "250px", "panels": [ { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", - "fill": 1, + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 1, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "short" + }, + "overrides": [ ] + }, "id": 14, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, "links": [ ], "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 4, - "stack": false, - "steppedLine": false, "targets": [ { "expr": "histogram_quantile(0.99, sum(rate(cortex_querier_storegateway_instances_hit_per_query_bucket{job=~\"$job\"}[$__rate_interval])) by (le)) * 1", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "99th Percentile", - "refId": "A", - "step": 10 + "refId": "A" }, { "expr": "histogram_quantile(0.50, sum(rate(cortex_querier_storegateway_instances_hit_per_query_bucket{job=~\"$job\"}[$__rate_interval])) by (le)) * 1", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "50th Percentile", - "refId": "B", - "step": 10 + "refId": "B" }, { "expr": "sum(rate(cortex_querier_storegateway_instances_hit_per_query_sum{job=~\"$job\"}[$__rate_interval])) * 1 / sum(rate(cortex_querier_storegateway_instances_hit_per_query_count{job=~\"$job\"}[$__rate_interval]))", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "Average", - "refId": "C", - "step": 10 + "refId": "C" } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, "title": "Number of store-gateways hit per Query", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, + "type": "timeseries", "yaxes": [ { - "format": "short", + "format": "ms", "label": null, "logBase": 1, "max": null, @@ -1263,84 +1352,70 @@ ] }, { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", - "fill": 1, + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 1, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "short" + }, + "overrides": [ ] + }, "id": 15, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, "links": [ ], "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 4, - "stack": false, - "steppedLine": false, "targets": [ { "expr": "histogram_quantile(0.99, sum(rate(cortex_querier_storegateway_refetches_per_query_bucket{job=~\"$job\"}[$__rate_interval])) by (le)) * 1", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "99th Percentile", - "refId": "A", - "step": 10 + "refId": "A" }, { "expr": "histogram_quantile(0.50, sum(rate(cortex_querier_storegateway_refetches_per_query_bucket{job=~\"$job\"}[$__rate_interval])) by (le)) * 1", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "50th Percentile", - "refId": "B", - "step": 10 + "refId": "B" }, { "expr": "sum(rate(cortex_querier_storegateway_refetches_per_query_sum{job=~\"$job\"}[$__rate_interval])) * 1 / sum(rate(cortex_querier_storegateway_refetches_per_query_count{job=~\"$job\"}[$__rate_interval]))", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "Average", - "refId": "C", - "step": 10 + "refId": "C" } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, "title": "Refetches of missing blocks per Query", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, + "type": "timeseries", "yaxes": [ { - "format": "short", + "format": "ms", "label": null, "logBase": 1, "max": null, @@ -1358,63 +1433,52 @@ ] }, { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", - "fill": 1, + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 1, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "short" + }, + "overrides": [ ] + }, "id": 16, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, "links": [ ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 4, - "stack": false, - "steppedLine": false, "targets": [ { "expr": "sum(rate(cortex_querier_blocks_consistency_checks_failed_total{job=~\"$job\"}[1m])) / sum(rate(cortex_querier_blocks_consistency_checks_total{job=~\"$job\"}[1m]))", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "Failure Rate", - "legendLink": null, - "step": 10 + "legendLink": null } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, "title": "Consistency checks failed", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, + "type": "timeseries", "yaxes": [ { "format": "percentunit", @@ -1447,235 +1511,148 @@ "height": "250px", "panels": [ { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", - "fill": 1, + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 1, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "short" + }, + "overrides": [ ] + }, "id": 17, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, "links": [ ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 4, - "stack": false, - "steppedLine": false, "targets": [ { "expr": "sum by(user) (rate(cortex_prometheus_notifications_errors_total{job=~\"$job\"}[$__rate_interval]))\n /\nsum by(user) (rate(cortex_prometheus_notifications_sent_total{job=~\"$job\"}[$__rate_interval]))\n> 0\n", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "{{ user }}", - "legendLink": null, - "step": 10 + "legendLink": null } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, "title": "Delivery Errors", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] + "type": "timeseries" }, { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", - "fill": 1, + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 1, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "short" + }, + "overrides": [ ] + }, "id": 18, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, "links": [ ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 4, - "stack": false, - "steppedLine": false, "targets": [ { "expr": "sum by(user) (rate(cortex_prometheus_notifications_queue_length{job=~\"$job\"}[$__rate_interval]))\n /\nsum by(user) (rate(cortex_prometheus_notifications_queue_capacity{job=~\"$job\"}[$__rate_interval])) > 0\n", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "{{ user }}", - "legendLink": null, - "step": 10 + "legendLink": null } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, "title": "Queue Length", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] + "type": "timeseries" }, { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", - "fill": 1, + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 1, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "short" + }, + "overrides": [ ] + }, "id": 19, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, "links": [ ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 4, - "stack": false, - "steppedLine": false, "targets": [ { "expr": "sum by (user) (increase(cortex_prometheus_notifications_dropped_total{job=~\"$job\"}[$__rate_interval])) > 0\n", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "{{ user }}", - "legendLink": null, - "step": 10 + "legendLink": null } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, "title": "Dropped", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] + "type": "timeseries" } ], "repeat": null, @@ -1690,235 +1667,148 @@ "height": "250px", "panels": [ { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", - "fill": 1, + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 1, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "short" + }, + "overrides": [ ] + }, "id": 20, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, "links": [ ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 4, - "stack": false, - "steppedLine": false, "targets": [ { "expr": "sum by(user) (rate(cortex_prometheus_rule_group_iterations_missed_total{job=~\"$job\"}[$__rate_interval])) > 0", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "{{ user }}", - "legendLink": null, - "step": 10 + "legendLink": null } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, "title": "Missed Iterations", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] + "type": "timeseries" }, { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", - "fill": 1, + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 1, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "s" + }, + "overrides": [ ] + }, "id": 21, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, "links": [ ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 4, - "stack": false, - "steppedLine": false, "targets": [ { "expr": "rate(cortex_prometheus_rule_group_duration_seconds_sum{job=~\"$job\"}[$__rate_interval])\n /\nrate(cortex_prometheus_rule_group_duration_seconds_count{job=~\"$job\"}[$__rate_interval])\n", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "{{ user }}", - "legendLink": null, - "step": 10 + "legendLink": null } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, "title": "Latency", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] + "type": "timeseries" }, { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", - "fill": 1, + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 1, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "short" + }, + "overrides": [ ] + }, "id": 22, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, "links": [ ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 4, - "stack": false, - "steppedLine": false, "targets": [ { "expr": "sum by(rule_group) (rate(cortex_prometheus_rule_evaluation_failures_total{job=~\"$job\"}[$__rate_interval])) > 0", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "{{ rule_group }}", - "legendLink": null, - "step": 10 + "legendLink": null } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, "title": "Failures", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] + "type": "timeseries" } ], "repeat": null, @@ -1933,81 +1823,52 @@ "height": "250px", "panels": [ { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", - "fill": 1, + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 1, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "s" + }, + "overrides": [ ] + }, "id": 23, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, "links": [ ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 12, - "stack": false, - "steppedLine": false, "targets": [ { "expr": "sum by(user) (rate(cortex_prometheus_rule_evaluation_duration_seconds_sum{job=~\"$job\"}[$__rate_interval]))\n /\nsum by(user) (rate(cortex_prometheus_rule_evaluation_duration_seconds_count{job=~\"$job\"}[$__rate_interval]))\n", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "{{ user }}", - "legendLink": null, - "step": 10 + "legendLink": null } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, "title": "Latency", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] + "type": "timeseries" } ], "repeat": null, @@ -2022,140 +1883,103 @@ "height": "250px", "panels": [ { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 1, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "rps" + }, + "overrides": [ ] + }, "fill": 10, "id": 24, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, "linewidth": 0, "links": [ ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 3, "stack": true, - "steppedLine": false, "targets": [ { "expr": "sum by(operation) (rate(thanos_objstore_bucket_operations_total{job=~\"$job\",component=\"ruler-storage\"}[$__rate_interval]))", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "{{operation}}", - "legendLink": null, - "step": 10 + "legendLink": null } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, "title": "Operations / sec", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, - "yaxes": [ - { - "format": "rps", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] + "type": "timeseries" }, { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", - "fill": 1, + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 1, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "percentunit" + }, + "overrides": [ ] + }, "id": 25, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, "links": [ ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 3, - "stack": false, - "steppedLine": false, "targets": [ { "expr": "sum by(operation) (rate(thanos_objstore_bucket_operation_failures_total{job=~\"$job\",component=\"ruler-storage\"}[$__rate_interval])) / sum by(operation) (rate(thanos_objstore_bucket_operations_total{job=~\"$job\",component=\"ruler-storage\"}[$__rate_interval]))", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "{{operation}}", - "legendLink": null, - "step": 10 + "legendLink": null } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, "title": "Error rate", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, + "type": "timeseries", "yaxes": [ { "format": "percentunit", @@ -2176,81 +2000,67 @@ ] }, { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", - "fill": 1, + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 1, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "short" + }, + "overrides": [ ] + }, "id": 26, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, "links": [ ], "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 3, - "stack": false, - "steppedLine": false, "targets": [ { "expr": "histogram_quantile(0.99, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{job=~\"$job\",component=\"ruler-storage\",operation=\"attributes\"}[$__rate_interval])) by (le)) * 1e3", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "99th Percentile", - "refId": "A", - "step": 10 + "refId": "A" }, { "expr": "histogram_quantile(0.50, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{job=~\"$job\",component=\"ruler-storage\",operation=\"attributes\"}[$__rate_interval])) by (le)) * 1e3", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "50th Percentile", - "refId": "B", - "step": 10 + "refId": "B" }, { "expr": "sum(rate(thanos_objstore_bucket_operation_duration_seconds_sum{job=~\"$job\",component=\"ruler-storage\",operation=\"attributes\"}[$__rate_interval])) * 1e3 / sum(rate(thanos_objstore_bucket_operation_duration_seconds_count{job=~\"$job\",component=\"ruler-storage\",operation=\"attributes\"}[$__rate_interval]))", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "Average", - "refId": "C", - "step": 10 + "refId": "C" } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, "title": "Latency of Op: Attributes", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, + "type": "timeseries", "yaxes": [ { "format": "ms", @@ -2271,81 +2081,67 @@ ] }, { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", - "fill": 1, + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 1, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "short" + }, + "overrides": [ ] + }, "id": 27, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, "links": [ ], "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 3, - "stack": false, - "steppedLine": false, "targets": [ { "expr": "histogram_quantile(0.99, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{job=~\"$job\",component=\"ruler-storage\",operation=\"exists\"}[$__rate_interval])) by (le)) * 1e3", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "99th Percentile", - "refId": "A", - "step": 10 + "refId": "A" }, { "expr": "histogram_quantile(0.50, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{job=~\"$job\",component=\"ruler-storage\",operation=\"exists\"}[$__rate_interval])) by (le)) * 1e3", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "50th Percentile", - "refId": "B", - "step": 10 + "refId": "B" }, { "expr": "sum(rate(thanos_objstore_bucket_operation_duration_seconds_sum{job=~\"$job\",component=\"ruler-storage\",operation=\"exists\"}[$__rate_interval])) * 1e3 / sum(rate(thanos_objstore_bucket_operation_duration_seconds_count{job=~\"$job\",component=\"ruler-storage\",operation=\"exists\"}[$__rate_interval]))", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "Average", - "refId": "C", - "step": 10 + "refId": "C" } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, "title": "Latency of Op: Exists", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, + "type": "timeseries", "yaxes": [ { "format": "ms", @@ -2378,81 +2174,67 @@ "height": "250px", "panels": [ { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", - "fill": 1, + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 1, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "short" + }, + "overrides": [ ] + }, "id": 28, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, "links": [ ], "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 3, - "stack": false, - "steppedLine": false, "targets": [ { "expr": "histogram_quantile(0.99, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{job=~\"$job\",component=\"ruler-storage\",operation=\"get\"}[$__rate_interval])) by (le)) * 1e3", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "99th Percentile", - "refId": "A", - "step": 10 + "refId": "A" }, { "expr": "histogram_quantile(0.50, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{job=~\"$job\",component=\"ruler-storage\",operation=\"get\"}[$__rate_interval])) by (le)) * 1e3", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "50th Percentile", - "refId": "B", - "step": 10 + "refId": "B" }, { "expr": "sum(rate(thanos_objstore_bucket_operation_duration_seconds_sum{job=~\"$job\",component=\"ruler-storage\",operation=\"get\"}[$__rate_interval])) * 1e3 / sum(rate(thanos_objstore_bucket_operation_duration_seconds_count{job=~\"$job\",component=\"ruler-storage\",operation=\"get\"}[$__rate_interval]))", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "Average", - "refId": "C", - "step": 10 + "refId": "C" } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, "title": "Latency of Op: Get", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, + "type": "timeseries", "yaxes": [ { "format": "ms", @@ -2473,81 +2255,67 @@ ] }, { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", - "fill": 1, + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 1, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "short" + }, + "overrides": [ ] + }, "id": 29, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, "links": [ ], "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 3, - "stack": false, - "steppedLine": false, "targets": [ { "expr": "histogram_quantile(0.99, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{job=~\"$job\",component=\"ruler-storage\",operation=\"get_range\"}[$__rate_interval])) by (le)) * 1e3", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "99th Percentile", - "refId": "A", - "step": 10 + "refId": "A" }, { "expr": "histogram_quantile(0.50, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{job=~\"$job\",component=\"ruler-storage\",operation=\"get_range\"}[$__rate_interval])) by (le)) * 1e3", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "50th Percentile", - "refId": "B", - "step": 10 + "refId": "B" }, { "expr": "sum(rate(thanos_objstore_bucket_operation_duration_seconds_sum{job=~\"$job\",component=\"ruler-storage\",operation=\"get_range\"}[$__rate_interval])) * 1e3 / sum(rate(thanos_objstore_bucket_operation_duration_seconds_count{job=~\"$job\",component=\"ruler-storage\",operation=\"get_range\"}[$__rate_interval]))", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "Average", - "refId": "C", - "step": 10 + "refId": "C" } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, "title": "Latency of Op: GetRange", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, + "type": "timeseries", "yaxes": [ { "format": "ms", @@ -2568,81 +2336,67 @@ ] }, { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", - "fill": 1, + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 1, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "short" + }, + "overrides": [ ] + }, "id": 30, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, "links": [ ], "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 3, - "stack": false, - "steppedLine": false, "targets": [ { "expr": "histogram_quantile(0.99, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{job=~\"$job\",component=\"ruler-storage\",operation=\"upload\"}[$__rate_interval])) by (le)) * 1e3", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "99th Percentile", - "refId": "A", - "step": 10 + "refId": "A" }, { "expr": "histogram_quantile(0.50, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{job=~\"$job\",component=\"ruler-storage\",operation=\"upload\"}[$__rate_interval])) by (le)) * 1e3", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "50th Percentile", - "refId": "B", - "step": 10 + "refId": "B" }, { "expr": "sum(rate(thanos_objstore_bucket_operation_duration_seconds_sum{job=~\"$job\",component=\"ruler-storage\",operation=\"upload\"}[$__rate_interval])) * 1e3 / sum(rate(thanos_objstore_bucket_operation_duration_seconds_count{job=~\"$job\",component=\"ruler-storage\",operation=\"upload\"}[$__rate_interval]))", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "Average", - "refId": "C", - "step": 10 + "refId": "C" } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, "title": "Latency of Op: Upload", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, + "type": "timeseries", "yaxes": [ { "format": "ms", @@ -2663,81 +2417,67 @@ ] }, { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", - "fill": 1, + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 1, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "short" + }, + "overrides": [ ] + }, "id": 31, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, "links": [ ], "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 3, - "stack": false, - "steppedLine": false, "targets": [ { "expr": "histogram_quantile(0.99, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{job=~\"$job\",component=\"ruler-storage\",operation=\"delete\"}[$__rate_interval])) by (le)) * 1e3", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "99th Percentile", - "refId": "A", - "step": 10 + "refId": "A" }, { "expr": "histogram_quantile(0.50, sum(rate(thanos_objstore_bucket_operation_duration_seconds_bucket{job=~\"$job\",component=\"ruler-storage\",operation=\"delete\"}[$__rate_interval])) by (le)) * 1e3", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "50th Percentile", - "refId": "B", - "step": 10 + "refId": "B" }, { "expr": "sum(rate(thanos_objstore_bucket_operation_duration_seconds_sum{job=~\"$job\",component=\"ruler-storage\",operation=\"delete\"}[$__rate_interval])) * 1e3 / sum(rate(thanos_objstore_bucket_operation_duration_seconds_count{job=~\"$job\",component=\"ruler-storage\",operation=\"delete\"}[$__rate_interval]))", "format": "time_series", "interval": "15s", - "intervalFactor": 2, "legendFormat": "Average", - "refId": "C", - "step": 10 + "refId": "C" } ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, "title": "Latency of Op: Delete", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, + "type": "timeseries", "yaxes": [ { "format": "ms", @@ -2779,7 +2519,7 @@ "value": "Cortex" }, "hide": 0, - "label": null, + "label": "Data source", "name": "datasource", "options": [ ], "query": "prometheus", diff --git a/docs/getting-started/rules.yaml b/docs/getting-started/rules.yaml index 6a5646a769..b458ea8688 100644 --- a/docs/getting-started/rules.yaml +++ b/docs/getting-started/rules.yaml @@ -364,7 +364,7 @@ groups: cortex_ingester_memory_series )[24h:] ) - / 1500000 + / 3000000 ) labels: deployment: ingester @@ -373,7 +373,7 @@ groups: - expr: | ceil( sum by (cluster, namespace) (cortex_overrides{limit_name="max_global_series_per_user"}) - * 3 * 0.59999999999999998 / 1500000 + * 3 * 0.59999999999999998 / 3000000 ) labels: deployment: ingester