[development] Promremote storage local setup (#3791)

docker/grafana/Dockerfile

@@ -10,6 +10,8 @@ COPY ./integrations/grafana/m3db_dashboard.json /tmp/grafana_dashboards/m3db_das
 COPY ./integrations/grafana/temporal_function_comparison.json /tmp/grafana_dashboards/temporal_function_comparison.json
 COPY ./integrations/grafana/m3aggregator_dashboard.json /tmp/grafana_dashboards/m3aggregator_dashboard.json
 COPY ./integrations/grafana/m3aggregator_end_to_end_details.json /tmp/grafana_dashboards/m3aggregator_end_to_end_details.json
+COPY ./scripts/development/m3_prom_remote_stack/prom_remote_demo_dashboard.json /tmp/grafana_dashboards/prom_remote_demo_dashboard.json.json
+
 
 # Need to replace datasource template variable with name of actual data source so auto-import
 # JustWorksTM. Use a temporary directory to host the dashboards since the default

docker/grafana/datasource.yaml

@@ -1,4 +1,12 @@
 datasources:
+  - name: Prometheus Aggregated
+    type: prometheus
+    access: proxy
+    url: http://prometheusagg:9090
+  - name: Prometheus Unaggregated
+    type: prometheus
+    access: proxy
+    url: http://prometheusraw:9090
   - name: Prometheus
     type: prometheus
     access: proxy

scripts/development/m3_prom_remote_stack/README.md

@@ -0,0 +1,34 @@
+# Local Development
+
+This docker-compose file will setup the following environment:
+
+1. 1 ETCD
+2. 1 M3Coordinator node
+3. 1 M3 Aggregator node
+4. 2 Prometheus instances with `remote-write-receiver` feature enabled in order to accept remote writes.   
+5. 1 Grafana node (with a pre-configured Prometheus source)
+6. 1 Prometheus node that scrapes the M3 Components and Prometheus instances. 
+   In addition, it scrapes CAdvisor /metrics endpoints exposed by using `kubectl proxy`.
+
+The environment variables that let's you configure this setup are:
+- `BUILD_M3AGGREGATOR=true`: forces build of local M3 Aggregator.
+- `BUILD_M3COORDINATOR=true`: forces build of local M3 Aggregator.
+- `WIPE_DATA=true`: cleans up docker state when using `stop_m3.sh`.
+
+## Usage
+
+Use the `start_m3.sh` and `stop_m3.sh` scripts.
+
+If you want to scrape CAdvisor metrics from Kubernetes cluster:
+- Run `port_forward_kube.sh`
+- Run `start_m3.sh`
+
+## Grafana
+
+Use Grafana by navigating to `http://localhost:3000` and using `admin` for both the username and password. The M3DB dashboard should already be populated and working.
+
+To pickup latest grafana Docker build remove existing image `docker image rm m3grafana --force`.
+
+## Containers Hanging / Unresponsive
+
+Running the entire stack can be resource intensive. If the containers are unresponsive try increasing the amount of cores and memory that the docker daemon is allowed to use.

scripts/development/m3_prom_remote_stack/docker-compose.yml

@@ -0,0 +1,110 @@
+version: "3.5"
+services:
+  m3aggregator01:
+    expose:
+      - "6001"
+    ports:
+      - "127.0.0.1:6001:6001"
+    networks:
+      - backend
+    environment:
+      - M3AGGREGATOR_HOST_ID=m3aggregator01
+    build:
+      context: ../../../bin
+      dockerfile: ./docker/m3aggregator/development.Dockerfile
+    image: m3aggregator:dev
+    volumes:
+      - "./m3aggregator.yml:/etc/m3aggregator/m3aggregator.yml"
+  m3coordinator01:
+    expose:
+      - "7201"
+    ports:
+      - "0.0.0.0:7201:7201"
+    networks:
+      - backend
+    build:
+      context: ../../../bin
+      dockerfile: ./docker/m3coordinator/development.Dockerfile
+    image: m3coordinator:dev
+    volumes:
+      - "./m3coordinator.yml.tmp:/etc/m3coordinator/m3coordinator.yml"
+  prometheusscraper:
+    networks:
+      - backend
+    image: prom/prometheus:latest
+    volumes:
+      - "./prometheus-scraper.yml.tmp:/etc/prometheus/prometheus.yml"
+    command:
+      - "--config.file=/etc/prometheus/prometheus.yml"
+      - "--storage.tsdb.path=/prometheus"
+      - "--web.console.libraries=/usr/share/prometheus/console_libraries"
+      - "--web.console.templates=/usr/share/prometheus/consoles"
+      - "--log.level=debug"
+  prometheusraw:
+    networks:
+      - backend
+    image: prom/prometheus:latest
+    volumes:
+      - "./prometheus.yml:/etc/prometheus/prometheus.yml"
+    command:
+      - "--config.file=/etc/prometheus/prometheus.yml"
+      - "--storage.tsdb.path=/prometheus"
+      - "--web.console.libraries=/usr/share/prometheus/console_libraries"
+      - "--web.console.templates=/usr/share/prometheus/consoles"
+      - "--enable-feature=remote-write-receiver"
+  prometheusagg:
+    networks:
+      - backend
+    image: prom/prometheus:latest
+    volumes:
+      - "./prometheus.yml:/etc/prometheus/prometheus.yml"
+    command:
+      - "--config.file=/etc/prometheus/prometheus.yml"
+      - "--storage.tsdb.path=/prometheus"
+      - "--web.console.libraries=/usr/share/prometheus/console_libraries"
+      - "--web.console.templates=/usr/share/prometheus/consoles"
+      - "--enable-feature=remote-write-receiver"
+  etcd01:
+    expose:
+      - "2379-2380"
+    ports:
+      - "0.0.0.0:2379-2380:2379-2380"
+    networks:
+      - backend
+    image: quay.io/coreos/etcd:v3.4.3
+    command:
+      - "etcd"
+      - "--name"
+      - "etcd01"
+      - "--listen-peer-urls"
+      - "http://0.0.0.0:2380"
+      - "--listen-client-urls"
+      - "http://0.0.0.0:2379"
+      - "--advertise-client-urls"
+      - "http://etcd01:2379"
+      - "--initial-cluster-token"
+      - "etcd-cluster-1"
+      - "--initial-advertise-peer-urls"
+      - "http://etcd01:2380"
+      - "--initial-cluster"
+      - "etcd01=http://etcd01:2380"
+      - "--initial-cluster-state"
+      - "new"
+      - "--data-dir"
+      - "/var/lib/etcd"
+  grafana:
+    build:
+      context: ../../../
+      dockerfile: ./docker/grafana/Dockerfile
+    expose:
+      - "3000"
+    ports:
+      - "0.0.0.0:3000:3000"
+    networks:
+      - backend
+    image: m3grafana:latest
+networks:
+  backend:
+volumes:
+  prom-raw-data:
+  prom-agg-data:

scripts/development/m3_prom_remote_stack/emit_scrape_configs.sh

@@ -0,0 +1,28 @@
+#!/usr/bin/env bash
+
+set -xe
+
+nodes=()
+while IFS='' read -r line; do nodes+=("$line"); done < <(curl http://localhost:8001/api/v1/nodes | jq '.items[].metadata.name' | tr -d \")
+
+
+cp prometheus-scraper.yml prometheus-scraper.yml.tmp
+
+limit=10
+
+i=0
+for node in "${nodes[@]}" ; do
+  i=$((i+1))
+  if [ "$i" -gt "$limit" ]; then
+    break;
+  fi
+  echo "
+  - job_name: cadvisor_${node}
+    metrics_path: /api/v1/nodes/${node}/proxy/metrics/cadvisor
+    static_configs:
+      - targets:
+        - host.docker.internal:8001
+        labels:
+          instance: ${node}
+  " >> prometheus-scraper.yml.tmp
+done