From 8643fdd76855c6e2d5ae5b768dbf26a65c33d694 Mon Sep 17 00:00:00 2001 From: Zoltan Nagy Date: Sun, 26 Feb 2017 12:07:46 +0100 Subject: [PATCH 01/10] Fix #121: Add example prometheus setup --- README.md | 19 ++++++++++++++++++- docker-compose.yml | 21 +++++++++++++++++++++ prometheus/prometheus.yml | 13 +++++++++++++ 3 files changed, 52 insertions(+), 1 deletion(-) create mode 100644 prometheus/prometheus.yml diff --git a/README.md b/README.md index 0d3417b..5ded153 100644 --- a/README.md +++ b/README.md @@ -180,7 +180,24 @@ To start the NGINX configuration, run: This container doubles as a skeleton for creating proxy configuration around Zipkin like authentication, dealing with CORS with zipkin-js apps, or -terminating SSL. +terminating SSL. + +### Prometheus + +Zipkin comes with a built-in Prometheus metric exporter. The main +`docker-compose.yml` file starts Prometheus configured to scrape Zipkin, exposes +it on port `9090`. You can open `$DOCKER_HOST_IP:9090` and start exploring the +metrics (which are available on the `/promethes` endpoint of Zipkin). + +`docker-compose.yml` also starts a Grafana container with authentication +disabled, exposing it on port 3000. It has built-in support for Prometheus data +sources, so you can point it to `$DOCKER_HOST_IP:9090`, and experiment with +creating dashboards. One immediately useful graph can be showing all the +`response_*` metrics on one graph to show response times per endpoint. + +Note that dashboards will be lost between restarts of the containers - again, +this is not a production environment, it's aimed to help the first steps in +learning Zipkin. If you want to run the zipkin-ui standalone against a remote zipkin server, you need to set `ZIPKIN_BASE_URL` accordingly: diff --git a/docker-compose.yml b/docker-compose.yml index 749ae76..6009ead 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -58,3 +58,24 @@ services: # - JAVA_OPTS=-verbose:gc -Xms1G -Xmx1G depends_on: - storage + + prometheus: + image: prom/prometheus + container_name: prometheus + ports: + - 9090:9090 + depends_on: + - storage + volumes: + - ./prometheus/prometheus.yml:/etc/prometheus/prometheus.yml + + grafana: + image: grafana/grafana + container_name: grafana + ports: + - 3000:3000 + depends_on: + - prometheus + environment: + - GF_AUTH_ANONYMOUS_ENABLED=true + - GF_AUTH_ANONYMOUS_ORG_ROLE=Admin diff --git a/prometheus/prometheus.yml b/prometheus/prometheus.yml new file mode 100644 index 0000000..8a49ae8 --- /dev/null +++ b/prometheus/prometheus.yml @@ -0,0 +1,13 @@ +global: + scrape_interval: 15s # Set the scrape interval to every 15 seconds. Default is every 1 minute. + evaluation_interval: 15s # Evaluate rules every 15 seconds. The default is every 1 minute. + +scrape_configs: + - job_name: 'prometheus' + static_configs: + - targets: ['localhost:9090'] + - job_name: 'zipkin' + scrape_interval: 5s + metrics_path: '/prometheus' + static_configs: + - targets: ['zipkin:9411'] From 7e945d0904fbf735d40a3410a820855869edb6d7 Mon Sep 17 00:00:00 2001 From: Zoltan Nagy Date: Mon, 27 Feb 2017 20:48:09 +0100 Subject: [PATCH 02/10] Mention Zipkin dashboard published at grafana.net --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 5ded153..21693a7 100644 --- a/README.md +++ b/README.md @@ -192,8 +192,8 @@ metrics (which are available on the `/promethes` endpoint of Zipkin). `docker-compose.yml` also starts a Grafana container with authentication disabled, exposing it on port 3000. It has built-in support for Prometheus data sources, so you can point it to `$DOCKER_HOST_IP:9090`, and experiment with -creating dashboards. One immediately useful graph can be showing all the -`response_*` metrics on one graph to show response times per endpoint. +creating dashboards. A good starting point would be importing the dashboard +published at https://grafana.net/dashboards/1598. Note that dashboards will be lost between restarts of the containers - again, this is not a production environment, it's aimed to help the first steps in From 5e316e427b9bf6acaab3536722650b3bd3ed0745 Mon Sep 17 00:00:00 2001 From: Zoltan Nagy Date: Wed, 1 Mar 2017 23:33:40 +0100 Subject: [PATCH 03/10] Set up Prometheus data source and dashboard in docker-compose.yml --- README.md | 14 ++++++-------- docker-compose.yml | 9 +++++++++ prometheus/create-datasource-and-dashboard.sh | 15 +++++++++++++++ 3 files changed, 30 insertions(+), 8 deletions(-) create mode 100755 prometheus/create-datasource-and-dashboard.sh diff --git a/README.md b/README.md index 21693a7..5b0860d 100644 --- a/README.md +++ b/README.md @@ -190,14 +190,12 @@ it on port `9090`. You can open `$DOCKER_HOST_IP:9090` and start exploring the metrics (which are available on the `/promethes` endpoint of Zipkin). `docker-compose.yml` also starts a Grafana container with authentication -disabled, exposing it on port 3000. It has built-in support for Prometheus data -sources, so you can point it to `$DOCKER_HOST_IP:9090`, and experiment with -creating dashboards. A good starting point would be importing the dashboard -published at https://grafana.net/dashboards/1598. - -Note that dashboards will be lost between restarts of the containers - again, -this is not a production environment, it's aimed to help the first steps in -learning Zipkin. +disabled, exposing it on port 3000. On startup it's configured with the +Prometheus instance started by `docker-compose` as a data source, and imports +the dashboard published at https://grafana.net/dashboards/1598. This means that, +after running `docker-compose up`, you can open +`$DOCKER_IP:3000/dashboard/db/zipkin-prometheus` and play around with the +dashboard. If you want to run the zipkin-ui standalone against a remote zipkin server, you need to set `ZIPKIN_BASE_URL` accordingly: diff --git a/docker-compose.yml b/docker-compose.yml index 6009ead..377a465 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -79,3 +79,12 @@ services: environment: - GF_AUTH_ANONYMOUS_ENABLED=true - GF_AUTH_ANONYMOUS_ORG_ROLE=Admin + + setup_grafana_datasource: + image: appropriate/curl + container_name: setup_grafana_datasource + depends_on: + - grafana + volumes: + - ./prometheus/create-datasource-and-dashboard.sh:/create.sh:ro + command: /create.sh diff --git a/prometheus/create-datasource-and-dashboard.sh b/prometheus/create-datasource-and-dashboard.sh new file mode 100755 index 0000000..0716650 --- /dev/null +++ b/prometheus/create-datasource-and-dashboard.sh @@ -0,0 +1,15 @@ +#!/bin/sh + +set -xeuo pipefail + +if ! curl --retry 5 --retry-connrefused --retry-delay 0 -sf http://grafana:3000/api/dashboards/name/prom; then + curl -sf -X POST -H "Content-Type: application/json" \ + --data-binary '{"name":"prom","type":"prometheus","url":"http://prometheus:9090","access":"proxy","isDefault":true}' \ + http://grafana:3000/api/datasources +fi + +curl -s https://grafana.net/api/dashboards/1598/revisions/1/download | \ + xargs -0 -I "{}" curl --retry-connrefused --retry 5 --retry-delay 0 -sf \ + -X POST -H "Content-Type: application/json" \ + --data-binary '{"dashboard": {}, "inputs": [{"name": "DS_PROM", "pluginId": "prometheus", "type": "datasource", "value": "prom"}], "overwrite": false}' \ + http://grafana:3000/api/dashboards/import From b4a480ae397550c98bbfe2d4e72481c987077a7a Mon Sep 17 00:00:00 2001 From: Zoltan Nagy Date: Wed, 1 Mar 2017 23:43:14 +0100 Subject: [PATCH 04/10] Fetch latest revision of dashboard from grafana.net --- prometheus/create-datasource-and-dashboard.sh | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/prometheus/create-datasource-and-dashboard.sh b/prometheus/create-datasource-and-dashboard.sh index 0716650..122396e 100755 --- a/prometheus/create-datasource-and-dashboard.sh +++ b/prometheus/create-datasource-and-dashboard.sh @@ -8,7 +8,10 @@ if ! curl --retry 5 --retry-connrefused --retry-delay 0 -sf http://grafana:3000/ http://grafana:3000/api/datasources fi -curl -s https://grafana.net/api/dashboards/1598/revisions/1/download | \ +dashboard_id=1598 +last_revision=$(curl -sf https://grafana.net/api/dashboards/${dashboard_id}/revisions | grep '"revision":' | sed 's/ *"revision": \([0-9]*\),/\1/' | sort -n | tail -1) + +curl -s https://grafana.net/api/dashboards/${dashboard_id}/revisions/${last_revision}/download | \ xargs -0 -I "{}" curl --retry-connrefused --retry 5 --retry-delay 0 -sf \ -X POST -H "Content-Type: application/json" \ --data-binary '{"dashboard": {}, "inputs": [{"name": "DS_PROM", "pluginId": "prometheus", "type": "datasource", "value": "prom"}], "overwrite": false}' \ From a8952b58a6449655f9f4598d4c578b81040a2787 Mon Sep 17 00:00:00 2001 From: Zoltan Nagy Date: Sun, 19 Mar 2017 19:16:54 +0100 Subject: [PATCH 05/10] Fix typo --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 5b0860d..dbf019c 100644 --- a/README.md +++ b/README.md @@ -187,7 +187,7 @@ terminating SSL. Zipkin comes with a built-in Prometheus metric exporter. The main `docker-compose.yml` file starts Prometheus configured to scrape Zipkin, exposes it on port `9090`. You can open `$DOCKER_HOST_IP:9090` and start exploring the -metrics (which are available on the `/promethes` endpoint of Zipkin). +metrics (which are available on the `/prometheus` endpoint of Zipkin). `docker-compose.yml` also starts a Grafana container with authentication disabled, exposing it on port 3000. On startup it's configured with the From 230718b16338483fdfa354e02c1f02c37bed33f9 Mon Sep 17 00:00:00 2001 From: Zoltan Nagy Date: Sun, 11 Jun 2017 08:14:13 +0200 Subject: [PATCH 06/10] [prometheus] Added relabel rules for nicer response count, timing --- prometheus/prometheus.yml | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/prometheus/prometheus.yml b/prometheus/prometheus.yml index 8a49ae8..032b6f7 100644 --- a/prometheus/prometheus.yml +++ b/prometheus/prometheus.yml @@ -11,3 +11,26 @@ scrape_configs: metrics_path: '/prometheus' static_configs: - targets: ['zipkin:9411'] + metric_relabel_configs: + # Response code count + - source_labels: [__name__] + regex: '^status_(\d+)_(.*)$' + replacement: '${1}' + target_label: status + - source_labels: [__name__] + regex: '^status_(\d+)_(.*)$' + replacement: '${2}' + target_label: path + - source_labels: [__name__] + regex: '^status_(\d+)_(.*)$' + replacement: 'http_requests_count' + target_label: __name__ + # Response time, pending histogram from https://github.com/openzipkin/zipkin/pull/1609 + - source_labels: [__name__] + regex: '^response_(.*)$' + replacement: '${1}' + target_label: path + - source_labels: [__name__] + regex: '^response_(.*)$' + replacement: 'http_request_duration_milliseconds' + target_label: __name__ From a038cdd529a97ab56ed7b39cce1c8d8ea842d38e Mon Sep 17 00:00:00 2001 From: Zoltan Nagy Date: Sun, 11 Jun 2017 08:14:51 +0200 Subject: [PATCH 07/10] grafana.net is now grafana.com --- README.md | 10 +++++----- prometheus/create-datasource-and-dashboard.sh | 4 ++-- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index dbf019c..b5dfa60 100644 --- a/README.md +++ b/README.md @@ -154,14 +154,14 @@ To start the MySQL+Kafka configuration, run: Then configure the [Kafka 0.10 sender](https://github.com/openzipkin/zipkin-reporter-java/blob/master/kafka10/src/main/java/zipkin/reporter/kafka10/KafkaSender.java) or [Kafka 0.8 sender](https://github.com/openzipkin/zipkin-reporter-java/blob/master/kafka08/src/main/java/zipkin/reporter/kafka08/KafkaSender.java) -using a `bootstrapServers` value of `192.168.99.100:9092`. +using a `bootstrapServers` value of `192.168.99.100:9092`. By default, this assumes your Docker host IP is 192.168.99.100. If this is -not the case, adjust `KAFKA_ADVERTISED_HOST_NAME` in `docker-compose-kafka10.yml` -and the `bootstrapServers` configuration of the kafka sender to match your +not the case, adjust `KAFKA_ADVERTISED_HOST_NAME` in `docker-compose-kafka10.yml` +and the `bootstrapServers` configuration of the kafka sender to match your Docker host IP. -If you prefer to activate the +If you prefer to activate the [Kafka 0.8 collector](https://github.com/openzipkin/zipkin/tree/master/zipkin-collector/kafka) use `docker-compose-kafka.yml` instead of `docker-compose-kafka10.yml`: @@ -192,7 +192,7 @@ metrics (which are available on the `/prometheus` endpoint of Zipkin). `docker-compose.yml` also starts a Grafana container with authentication disabled, exposing it on port 3000. On startup it's configured with the Prometheus instance started by `docker-compose` as a data source, and imports -the dashboard published at https://grafana.net/dashboards/1598. This means that, +the dashboard published at https://grafana.com/dashboards/1598. This means that, after running `docker-compose up`, you can open `$DOCKER_IP:3000/dashboard/db/zipkin-prometheus` and play around with the dashboard. diff --git a/prometheus/create-datasource-and-dashboard.sh b/prometheus/create-datasource-and-dashboard.sh index 122396e..e0f6ed2 100755 --- a/prometheus/create-datasource-and-dashboard.sh +++ b/prometheus/create-datasource-and-dashboard.sh @@ -9,9 +9,9 @@ if ! curl --retry 5 --retry-connrefused --retry-delay 0 -sf http://grafana:3000/ fi dashboard_id=1598 -last_revision=$(curl -sf https://grafana.net/api/dashboards/${dashboard_id}/revisions | grep '"revision":' | sed 's/ *"revision": \([0-9]*\),/\1/' | sort -n | tail -1) +last_revision=$(curl -sf https://grafana.com/api/dashboards/${dashboard_id}/revisions | grep '"revision":' | sed 's/ *"revision": \([0-9]*\),/\1/' | sort -n | tail -1) -curl -s https://grafana.net/api/dashboards/${dashboard_id}/revisions/${last_revision}/download | \ +curl -s https://grafana.com/api/dashboards/${dashboard_id}/revisions/${last_revision}/download | \ xargs -0 -I "{}" curl --retry-connrefused --retry 5 --retry-delay 0 -sf \ -X POST -H "Content-Type: application/json" \ --data-binary '{"dashboard": {}, "inputs": [{"name": "DS_PROM", "pluginId": "prometheus", "type": "datasource", "value": "prom"}], "overwrite": false}' \ From 06a21b9a664642872d36d8bf711bffa08138b33b Mon Sep 17 00:00:00 2001 From: Zoltan Nagy Date: Sun, 11 Jun 2017 08:19:29 +0200 Subject: [PATCH 08/10] Oops. The request count is in fact a counter, not a gauge. Meaning that we should name it the same way as the relevant metric from Prometheus itself. --- prometheus/prometheus.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/prometheus/prometheus.yml b/prometheus/prometheus.yml index 032b6f7..88ee911 100644 --- a/prometheus/prometheus.yml +++ b/prometheus/prometheus.yml @@ -23,7 +23,7 @@ scrape_configs: target_label: path - source_labels: [__name__] regex: '^status_(\d+)_(.*)$' - replacement: 'http_requests_count' + replacement: 'http_requests_total' target_label: __name__ # Response time, pending histogram from https://github.com/openzipkin/zipkin/pull/1609 - source_labels: [__name__] From 7bf89c9c99dc2bef1fc045f0adfe1743dbeb3598 Mon Sep 17 00:00:00 2001 From: Zoltan Nagy Date: Mon, 12 Jun 2017 12:14:22 +0200 Subject: [PATCH 09/10] [prometheus] Add zipkin_collector metrics relabeling --- prometheus/create-datasource-and-dashboard.sh | 12 +++++++----- prometheus/prometheus.yml | 9 +++++++++ 2 files changed, 16 insertions(+), 5 deletions(-) diff --git a/prometheus/create-datasource-and-dashboard.sh b/prometheus/create-datasource-and-dashboard.sh index e0f6ed2..ecef423 100755 --- a/prometheus/create-datasource-and-dashboard.sh +++ b/prometheus/create-datasource-and-dashboard.sh @@ -11,8 +11,10 @@ fi dashboard_id=1598 last_revision=$(curl -sf https://grafana.com/api/dashboards/${dashboard_id}/revisions | grep '"revision":' | sed 's/ *"revision": \([0-9]*\),/\1/' | sort -n | tail -1) -curl -s https://grafana.com/api/dashboards/${dashboard_id}/revisions/${last_revision}/download | \ - xargs -0 -I "{}" curl --retry-connrefused --retry 5 --retry-delay 0 -sf \ - -X POST -H "Content-Type: application/json" \ - --data-binary '{"dashboard": {}, "inputs": [{"name": "DS_PROM", "pluginId": "prometheus", "type": "datasource", "value": "prom"}], "overwrite": false}' \ - http://grafana:3000/api/dashboards/import +echo '{"dashboard": ' > data.json +curl -s https://grafana.com/api/dashboards/${dashboard_id}/revisions/${last_revision}/download >> data.json +echo ', "inputs": [{"name": "DS_PROM", "pluginId": "prometheus", "type": "datasource", "value": "prom"}], "overwrite": false}' >> data.json +curl --retry-connrefused --retry 5 --retry-delay 0 -sf \ + -X POST -H "Content-Type: application/json" \ + --data-binary @data.json \ + http://grafana:3000/api/dashboards/import diff --git a/prometheus/prometheus.yml b/prometheus/prometheus.yml index 88ee911..ea159f6 100644 --- a/prometheus/prometheus.yml +++ b/prometheus/prometheus.yml @@ -34,3 +34,12 @@ scrape_configs: regex: '^response_(.*)$' replacement: 'http_request_duration_milliseconds' target_label: __name__ + # Received message count + - source_labels: [__name__] + regex: 'zipkin_collector_(.*)_([^_]*)' + replacement: '${2}' + target_label: transport + - source_labels: [__name__] + regex: 'zipkin_collector_(.*)_([^_]*)' + replacement: 'zipkin_collector_${1}' + target_label: __name__ From 1caa14622b029f717b805aecacc6adbd0a6d2151 Mon Sep 17 00:00:00 2001 From: Zoltan Nagy Date: Wed, 16 Aug 2017 23:39:51 +0200 Subject: [PATCH 10/10] [prometheus] Update rewrite rules for openzipkin/zipkin#1609 --- prometheus/prometheus.yml | 19 +++++-------------- 1 file changed, 5 insertions(+), 14 deletions(-) diff --git a/prometheus/prometheus.yml b/prometheus/prometheus.yml index ea159f6..b97e6e5 100644 --- a/prometheus/prometheus.yml +++ b/prometheus/prometheus.yml @@ -14,32 +14,23 @@ scrape_configs: metric_relabel_configs: # Response code count - source_labels: [__name__] - regex: '^status_(\d+)_(.*)$' + regex: '^counter_status_(\d+)_(.*)$' replacement: '${1}' target_label: status - source_labels: [__name__] - regex: '^status_(\d+)_(.*)$' + regex: '^counter_status_(\d+)_(.*)$' replacement: '${2}' target_label: path - source_labels: [__name__] - regex: '^status_(\d+)_(.*)$' + regex: '^counter_status_(\d+)_(.*)$' replacement: 'http_requests_total' target_label: __name__ - # Response time, pending histogram from https://github.com/openzipkin/zipkin/pull/1609 - - source_labels: [__name__] - regex: '^response_(.*)$' - replacement: '${1}' - target_label: path - - source_labels: [__name__] - regex: '^response_(.*)$' - replacement: 'http_request_duration_milliseconds' - target_label: __name__ # Received message count - source_labels: [__name__] - regex: 'zipkin_collector_(.*)_([^_]*)' + regex: '(?:gauge|counter)_zipkin_collector_(.*)_([^_]*)' replacement: '${2}' target_label: transport - source_labels: [__name__] - regex: 'zipkin_collector_(.*)_([^_]*)' + regex: '(?:gauge|counter)_zipkin_collector_(.*)_([^_]*)' replacement: 'zipkin_collector_${1}' target_label: __name__