From 95431d923aa919280e6afec0cb6abfe98c3831eb Mon Sep 17 00:00:00 2001 From: Charles Korn Date: Thu, 4 May 2023 10:48:02 +1000 Subject: [PATCH] Improve out-of-the-box experience with Docker Compose stacks for microservices and read-write mode (#4900) * Make OTel collector, Grafana agent and Prometheus optional in Docker Compose stack and disable all but the Grafana agent by default. * Add `container` labels to data scraped in Docker Compose stacks. * Use Prometheus by default. --- .../config/grafana-agent.yaml | 4 ++++ .../config/otel-collector.yaml | 18 +++++++++++++++++- .../config/prometheus.yaml | 4 ++++ .../docker-compose.jsonnet | 12 +++++++++--- .../docker-compose.yml | 17 ----------------- .../config/grafana-agent.yaml | 3 +++ 6 files changed, 37 insertions(+), 21 deletions(-) diff --git a/development/mimir-microservices-mode/config/grafana-agent.yaml b/development/mimir-microservices-mode/config/grafana-agent.yaml index 60bd7c2a226..55bf4c27058 100644 --- a/development/mimir-microservices-mode/config/grafana-agent.yaml +++ b/development/mimir-microservices-mode/config/grafana-agent.yaml @@ -40,6 +40,10 @@ prometheus: separator: '/' regex: '(.+?)(-\d+)?' replacement: '${1}' + - source_labels: ['pod'] + target_label: 'container' + regex: '(.+?)(-\d+)?' + replacement: '${1}' remote_write: - url: http://distributor-1:8000/api/v1/push diff --git a/development/mimir-microservices-mode/config/otel-collector.yaml b/development/mimir-microservices-mode/config/otel-collector.yaml index 4f30ceb60f5..1c4472c5188 100644 --- a/development/mimir-microservices-mode/config/otel-collector.yaml +++ b/development/mimir-microservices-mode/config/otel-collector.yaml @@ -13,7 +13,7 @@ receivers: global: scrape_interval: 5s external_labels: - scraped_by: otel-collector + scraped_by: otel-collector scrape_configs: - job_name: mimir-microservices-mode/distributor @@ -22,54 +22,70 @@ receivers: labels: cluster: 'docker-compose' namespace: 'mimir-microservices-mode' + container: 'distributor' - job_name: mimir-microservices-mode/ingester static_configs: - targets: ['ingester-1:8002', 'ingester-2:8003'] labels: cluster: 'docker-compose' namespace: 'mimir-microservices-mode' + container: 'ingester' - job_name: mimir-microservices-mode/querier static_configs: - targets: ['querier:8004'] labels: cluster: 'docker-compose' namespace: 'mimir-microservices-mode' + container: 'querier' - job_name: mimir-microservices-mode/ruler static_configs: - targets: ['ruler-1:8021', 'ruler-2:8022'] labels: cluster: 'docker-compose' namespace: 'mimir-microservices-mode' + container: 'ruler' - job_name: mimir-microservices-mode/compactor static_configs: - targets: ['compactor:8006'] labels: cluster: 'docker-compose' namespace: 'mimir-microservices-mode' + container: 'compactor' - job_name: mimir-microservices-mode/query-frontend static_configs: - targets: ['query-frontend:8007'] labels: cluster: 'docker-compose' namespace: 'mimir-microservices-mode' + container: 'query-frontend' - job_name: mimir-microservices-mode/store-gateway static_configs: - targets: ['store-gateway-1:8008', 'store-gateway-2:8009'] labels: cluster: 'docker-compose' namespace: 'mimir-microservices-mode' + container: 'store-gateway' - job_name: mimir-microservices-mode/query-scheduler static_configs: - targets: ['query-scheduler:8011'] labels: cluster: 'docker-compose' namespace: 'mimir-microservices-mode' + container: 'query-scheduler' - job_name: mimir-microservices-mode/memcached-exporter static_configs: - targets: ['memcached-exporter:9150'] labels: cluster: 'docker-compose' namespace: 'mimir-microservices-mode' + container: 'memcached-exporter' + - job_name: mimir-microservices-mode/load-generator + static_configs: + - targets: ['load-generator:9900'] + labels: + cluster: 'docker-compose' + namespace: 'mimir-microservices-mode' + container: 'load-generator' service: pipelines: diff --git a/development/mimir-microservices-mode/config/prometheus.yaml b/development/mimir-microservices-mode/config/prometheus.yaml index 5b9df4ca35a..441c4b613b8 100644 --- a/development/mimir-microservices-mode/config/prometheus.yaml +++ b/development/mimir-microservices-mode/config/prometheus.yaml @@ -33,6 +33,10 @@ scrape_configs: separator: '/' regex: '(.+?)(-\d+)?' replacement: '${1}' + - source_labels: ['pod'] + target_label: 'container' + regex: '(.+?)(-\d+)?' + replacement: '${1}' remote_write: - url: http://distributor-1:8000/api/v1/push diff --git a/development/mimir-microservices-mode/docker-compose.jsonnet b/development/mimir-microservices-mode/docker-compose.jsonnet index 2067630fa05..2f921649a59 100644 --- a/development/mimir-microservices-mode/docker-compose.jsonnet +++ b/development/mimir-microservices-mode/docker-compose.jsonnet @@ -18,6 +18,12 @@ std.manifestYamlDoc({ // - memberlist (consul is not started at all) // - multi (uses consul as primary and memberlist as secondary, but this can be switched in runtime via runtime.yaml) ring: 'memberlist', + + // If true, start and enable scraping by these components. + // Note that if more than one component is enabled, the dashboards shown in Grafana may contain duplicate series or aggregates may be doubled or tripled. + enable_grafana_agent: false, + enable_prometheus: true, // If Prometheus is disabled, recording rules will not be evaluated and so dashboards in Grafana that depend on these recorded series will display no data. + enable_otel_collector: false, }, // We explicitely list all important services here, so that it's easy to disable them by commenting out. @@ -31,10 +37,10 @@ std.manifestYamlDoc({ self.alertmanagers(3) + self.nginx + self.minio + - self.prometheus + + (if $._config.enable_prometheus then self.prometheus else {}) + self.grafana + - self.grafana_agent + - self.otel_collector + + (if $._config.enable_grafana_agent then self.grafana_agent else {}) + + (if $._config.enable_otel_collector then self.otel_collector else {}) + self.jaeger + (if $._config.ring == 'consul' || $._config.ring == 'multi' then self.consul else {}) + (if $._config.cache_backend == 'redis' then self.redis else self.memcached + self.memcached_exporter) + diff --git a/development/mimir-microservices-mode/docker-compose.yml b/development/mimir-microservices-mode/docker-compose.yml index c34f2e0a603..80d471d1743 100644 --- a/development/mimir-microservices-mode/docker-compose.yml +++ b/development/mimir-microservices-mode/docker-compose.yml @@ -152,15 +152,6 @@ - "./config/datasource-mimir.yaml:/etc/grafana/provisioning/datasources/mimir.yaml" - "./config/dashboards-mimir.yaml:/etc/grafana/provisioning/dashboards/mimir.yaml" - "../../operations/mimir-mixin-compiled/dashboards:/var/lib/grafana/dashboards/Mimir" - "grafana-agent": - "command": - - "-config.file=/etc/agent-config/grafana-agent.yaml" - - "-prometheus.wal-directory=/tmp" - "image": "grafana/agent:v0.21.2" - "ports": - - "9091:9091" - "volumes": - - "./config:/etc/agent-config" "ingester-1": "build": "context": "." @@ -252,14 +243,6 @@ - "8080:8080" "volumes": - "../common/config:/etc/nginx/templates" - "otel_collector": - "command": - - "--config=/etc/otel-collector/otel-collector.yaml" - "image": "otel/opentelemetry-collector-contrib:0.54.0" - "ports": - - "8083:8083" - "volumes": - - "./config:/etc/otel-collector" "prometheus": "command": - "--config.file=/etc/prometheus/prometheus.yaml" diff --git a/development/mimir-read-write-mode/config/grafana-agent.yaml b/development/mimir-read-write-mode/config/grafana-agent.yaml index b3602e5f0ac..e6596066fdb 100644 --- a/development/mimir-read-write-mode/config/grafana-agent.yaml +++ b/development/mimir-read-write-mode/config/grafana-agent.yaml @@ -15,18 +15,21 @@ prometheus: labels: cluster: 'docker-compose' namespace: 'mimir-read-write-mode' + container: 'mimir-write' - job_name: mimir-read-write-mode/mimir-read static_configs: - targets: ['mimir-read-1:8080', 'mimir-read-2:8080'] labels: cluster: 'docker-compose' namespace: 'mimir-read-write-mode' + container: 'mimir-read' - job_name: mimir-read-write-mode/mimir-backend static_configs: - targets: ['mimir-backend-1:8080', 'mimir-backend-2:8080'] labels: cluster: 'docker-compose' namespace: 'mimir-read-write-mode' + container: 'mimir-backend' remote_write: - url: http://mimir-write-1:8080/api/v1/push