Improve out-of-the-box experience with Docker Compose stacks for micr…

…oservices and read-write mode (#4900) * Make OTel collector, Grafana agent and Prometheus optional in Docker Compose stack and disable all but the Grafana agent by default. * Add `container` labels to data scraped in Docker Compose stacks. * Use Prometheus by default.
grafana · May 4, 2023 · 95431d9 · 95431d9
1 parent 9768075
commit 95431d9
Show file tree

Hide file tree

Showing 6 changed files with 37 additions and 21 deletions.
diff --git a/development/mimir-microservices-mode/config/grafana-agent.yaml b/development/mimir-microservices-mode/config/grafana-agent.yaml
@@ -40,6 +40,10 @@ prometheus:
               separator: '/'
               regex: '(.+?)(-\d+)?'
               replacement: '${1}'
+            - source_labels: ['pod']
+              target_label: 'container'
+              regex: '(.+?)(-\d+)?'
+              replacement: '${1}'
 
       remote_write:
         - url: http://distributor-1:8000/api/v1/push
diff --git a/development/mimir-microservices-mode/config/otel-collector.yaml b/development/mimir-microservices-mode/config/otel-collector.yaml
@@ -13,7 +13,7 @@ receivers:
       global:
         scrape_interval: 5s
         external_labels:
-          scraped_by: otel-collector 
+          scraped_by: otel-collector
 
       scrape_configs:
         - job_name: mimir-microservices-mode/distributor
@@ -22,54 +22,70 @@ receivers:
               labels:
                 cluster: 'docker-compose'
                 namespace: 'mimir-microservices-mode'
+                container: 'distributor'
         - job_name: mimir-microservices-mode/ingester
           static_configs:
             - targets: ['ingester-1:8002', 'ingester-2:8003']
               labels:
                 cluster: 'docker-compose'
                 namespace: 'mimir-microservices-mode'
+                container: 'ingester'
         - job_name: mimir-microservices-mode/querier
           static_configs:
             - targets: ['querier:8004']
               labels:
                 cluster: 'docker-compose'
                 namespace: 'mimir-microservices-mode'
+                container: 'querier'
         - job_name: mimir-microservices-mode/ruler
           static_configs:
             - targets: ['ruler-1:8021', 'ruler-2:8022']
               labels:
                 cluster: 'docker-compose'
                 namespace: 'mimir-microservices-mode'
+                container: 'ruler'
         - job_name: mimir-microservices-mode/compactor
           static_configs:
             - targets: ['compactor:8006']
               labels:
                 cluster: 'docker-compose'
                 namespace: 'mimir-microservices-mode'
+                container: 'compactor'
         - job_name: mimir-microservices-mode/query-frontend
           static_configs:
             - targets: ['query-frontend:8007']
               labels:
                 cluster: 'docker-compose'
                 namespace: 'mimir-microservices-mode'
+                container: 'query-frontend'
         - job_name: mimir-microservices-mode/store-gateway
           static_configs:
             - targets: ['store-gateway-1:8008', 'store-gateway-2:8009']
               labels:
                 cluster: 'docker-compose'
                 namespace: 'mimir-microservices-mode'
+                container: 'store-gateway'
         - job_name: mimir-microservices-mode/query-scheduler
           static_configs:
             - targets: ['query-scheduler:8011']
               labels:
                 cluster: 'docker-compose'
                 namespace: 'mimir-microservices-mode'
+                container: 'query-scheduler'
         - job_name: mimir-microservices-mode/memcached-exporter
           static_configs:
             - targets: ['memcached-exporter:9150']
               labels:
                 cluster: 'docker-compose'
                 namespace: 'mimir-microservices-mode'
+                container: 'memcached-exporter'
+        - job_name: mimir-microservices-mode/load-generator
+          static_configs:
+            - targets: ['load-generator:9900']
+              labels:
+                cluster: 'docker-compose'
+                namespace: 'mimir-microservices-mode'
+                container: 'load-generator'
 
 service:
   pipelines:

diff --git a/development/mimir-microservices-mode/config/prometheus.yaml b/development/mimir-microservices-mode/config/prometheus.yaml
@@ -33,6 +33,10 @@ scrape_configs:
         separator: '/'
         regex: '(.+?)(-\d+)?'
         replacement: '${1}'
+      - source_labels: ['pod']
+        target_label: 'container'
+        regex: '(.+?)(-\d+)?'
+        replacement: '${1}'
 
 remote_write:
   - url: http://distributor-1:8000/api/v1/push

diff --git a/development/mimir-microservices-mode/docker-compose.jsonnet b/development/mimir-microservices-mode/docker-compose.jsonnet
@@ -18,6 +18,12 @@ std.manifestYamlDoc({
     // - memberlist (consul is not started at all)
     // - multi (uses consul as primary and memberlist as secondary, but this can be switched in runtime via runtime.yaml)
     ring: 'memberlist',
+
+    // If true, start and enable scraping by these components.
+    // Note that if more than one component is enabled, the dashboards shown in Grafana may contain duplicate series or aggregates may be doubled or tripled.
+    enable_grafana_agent: false,
+    enable_prometheus: true, // If Prometheus is disabled, recording rules will not be evaluated and so dashboards in Grafana that depend on these recorded series will display no data.
+    enable_otel_collector: false,
   },
 
   // We explicitely list all important services here, so that it's easy to disable them by commenting out.
@@ -31,10 +37,10 @@ std.manifestYamlDoc({
     self.alertmanagers(3) +
     self.nginx +
     self.minio +
-    self.prometheus +
+    (if $._config.enable_prometheus then self.prometheus else {}) +
     self.grafana +
-    self.grafana_agent +
-    self.otel_collector +
+    (if $._config.enable_grafana_agent then self.grafana_agent else {}) +
+    (if $._config.enable_otel_collector then self.otel_collector else {}) +
     self.jaeger +
     (if $._config.ring == 'consul' || $._config.ring == 'multi' then self.consul else {}) +
     (if $._config.cache_backend == 'redis' then self.redis else self.memcached + self.memcached_exporter) +

diff --git a/development/mimir-microservices-mode/docker-compose.yml b/development/mimir-microservices-mode/docker-compose.yml
@@ -152,15 +152,6 @@
       - "./config/datasource-mimir.yaml:/etc/grafana/provisioning/datasources/mimir.yaml"
       - "./config/dashboards-mimir.yaml:/etc/grafana/provisioning/dashboards/mimir.yaml"
       - "../../operations/mimir-mixin-compiled/dashboards:/var/lib/grafana/dashboards/Mimir"
-  "grafana-agent":
-    "command":
-      - "-config.file=/etc/agent-config/grafana-agent.yaml"
-      - "-prometheus.wal-directory=/tmp"
-    "image": "grafana/agent:v0.21.2"
-    "ports":
-      - "9091:9091"
-    "volumes":
-      - "./config:/etc/agent-config"
   "ingester-1":
     "build":
       "context": "."
@@ -252,14 +243,6 @@
       - "8080:8080"
     "volumes":
       - "../common/config:/etc/nginx/templates"
-  "otel_collector":
-    "command":
-      - "--config=/etc/otel-collector/otel-collector.yaml"
-    "image": "otel/opentelemetry-collector-contrib:0.54.0"
-    "ports":
-      - "8083:8083"
-    "volumes":
-      - "./config:/etc/otel-collector"
   "prometheus":
     "command":
       - "--config.file=/etc/prometheus/prometheus.yaml"

diff --git a/development/mimir-read-write-mode/config/grafana-agent.yaml b/development/mimir-read-write-mode/config/grafana-agent.yaml
@@ -15,18 +15,21 @@ prometheus:
               labels:
                 cluster: 'docker-compose'
                 namespace: 'mimir-read-write-mode'
+                container: 'mimir-write'
         - job_name: mimir-read-write-mode/mimir-read
           static_configs:
             - targets: ['mimir-read-1:8080', 'mimir-read-2:8080']
               labels:
                 cluster: 'docker-compose'
                 namespace: 'mimir-read-write-mode'
+                container: 'mimir-read'
         - job_name: mimir-read-write-mode/mimir-backend
           static_configs:
             - targets: ['mimir-backend-1:8080', 'mimir-backend-2:8080']
               labels:
                 cluster: 'docker-compose'
                 namespace: 'mimir-read-write-mode'
+                container: 'mimir-backend'
 
       remote_write:
         - url: http://mimir-write-1:8080/api/v1/push