diff --git a/contrib/grafana/Oathkeeper-Dashboard.json b/contrib/grafana/Oathkeeper-Dashboard.json new file mode 100644 index 0000000000..e0cae51cb0 --- /dev/null +++ b/contrib/grafana/Oathkeeper-Dashboard.json @@ -0,0 +1,441 @@ +{ + "__inputs": [ + { + "name": "PROMETHEUS", + "label": "prometheus", + "description": "", + "type": "datasource", + "pluginId": "prometheus", + "pluginName": "Prometheus" + } + ], + "__requires": [ + { + "type": "grafana", + "id": "grafana", + "name": "Grafana", + "version": "6.7.1" + }, + { + "type": "panel", + "id": "graph", + "name": "Graph", + "version": "" + }, + { + "type": "datasource", + "id": "prometheus", + "name": "Prometheus", + "version": "1.0.0" + }, + { + "type": "panel", + "id": "stat", + "name": "Stat", + "version": "" + } + ], + "annotations": { + "list": [ + { + "$$hashKey": "object:123", + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "id": null, + "links": [], + "panels": [ + { + "datasource": "${PROMETHEUS}", + "gridPos": { + "h": 5, + "w": 8, + "x": 0, + "y": 0 + }, + "id": 6, + "options": { + "colorMode": "value", + "fieldOptions": { + "calcs": [ + "mean" + ], + "defaults": { + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "rgb(255, 255, 255)", + "value": null + } + ] + } + }, + "overrides": [], + "values": false + }, + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto" + }, + "pluginVersion": "6.7.1", + "targets": [ + { + "expr": "sum(irate(ory_oathkeeper_requests_total[5m])) by (instance)", + "interval": "", + "legendFormat": "", + "refId": "A" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Total RPS", + "type": "stat" + }, + { + "datasource": "${PROMETHEUS}", + "gridPos": { + "h": 5, + "w": 8, + "x": 8, + "y": 0 + }, + "id": 7, + "options": { + "colorMode": "value", + "fieldOptions": { + "calcs": [ + "mean" + ], + "defaults": { + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [], + "values": false + }, + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto" + }, + "pluginVersion": "6.7.1", + "targets": [ + { + "expr": "sum(irate(ory_oathkeeper_requests_total{status_code=\"200\"}[5m])) by (instance)", + "interval": "", + "legendFormat": "", + "refId": "A" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Allowed RPS", + "type": "stat" + }, + { + "cacheTimeout": null, + "datasource": "${PROMETHEUS}", + "gridPos": { + "h": 5, + "w": 8, + "x": 16, + "y": 0 + }, + "id": 8, + "links": [], + "options": { + "colorMode": "value", + "fieldOptions": { + "calcs": [ + "mean" + ], + "defaults": { + "mappings": [ + { + "$$hashKey": "object:311", + "id": 0, + "op": "=", + "text": "0", + "type": 1, + "value": "null" + } + ], + "nullValueMode": "connected", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "dark-red", + "value": null + } + ] + }, + "unit": "none" + }, + "overrides": [], + "values": false + }, + "graphMode": "area", + "justifyMode": "auto", + "orientation": "horizontal" + }, + "pluginVersion": "6.7.1", + "targets": [ + { + "expr": "sum(irate(ory_oathkeeper_requests_total{status_code!=\"200\"}[5m])) by (instance)", + "interval": "", + "legendFormat": "", + "refId": "A" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Denied RPS", + "type": "stat" + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${PROMETHEUS}", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 5 + }, + "hiddenSeries": false, + "id": 2, + "legend": { + "alignAsTable": true, + "avg": true, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(irate(ory_oathkeeper_requests_total[5m])) by (instance, request)", + "interval": "", + "legendFormat": "{{ instance }} - {{ request }}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "RPS", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:373", + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "$$hashKey": "object:374", + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${PROMETHEUS}", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 5 + }, + "hiddenSeries": false, + "id": 4, + "legend": { + "alignAsTable": true, + "avg": true, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "dataLinks": [] + }, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.90, avg(rate(ory_oathkeeper_request_duration_seconds_bucket[5m])) by (le, instance))", + "interval": "", + "legendFormat": "{{ instance }} - p90", + "refId": "A" + }, + { + "expr": "histogram_quantile(0.95, avg(rate(ory_oathkeeper_request_duration_seconds_bucket[5m])) by (le, instance))", + "interval": "", + "legendFormat": "{{ instance }} - p95", + "refId": "B" + }, + { + "expr": "histogram_quantile(0.99, avg(rate(ory_oathkeeper_request_duration_seconds_bucket[5m])) by (le, instance))", + "interval": "", + "legendFormat": "{{ instance }} - p99", + "refId": "C" + }, + { + "expr": "histogram_quantile(1, avg(rate(ory_oathkeeper_request_duration_seconds_bucket[5m])) by (le, instance))", + "interval": "", + "legendFormat": "{{ instance }} - p100", + "refId": "D" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Latencies", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:454", + "decimals": 0, + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "$$hashKey": "object:455", + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + } + ], + "schemaVersion": 22, + "style": "dark", + "tags": [], + "templating": { + "list": [] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": {}, + "timezone": "", + "title": "Oathkeeper", + "uid": "5xCfOseWk", + "variables": { + "list": [] + }, + "version": 3 +} \ No newline at end of file diff --git a/docs/docs/configure-deploy.md b/docs/docs/configure-deploy.md index 897a5f50f4..a1708c78cb 100644 --- a/docs/docs/configure-deploy.md +++ b/docs/docs/configure-deploy.md @@ -369,3 +369,61 @@ $ docker rm -f ory-oathkeeper-demo $ docker rmi -f ory-oathkeeper-demo $ rm -rf oathkeeper-demo ``` + +## Monitoring + +Oathkeeper provides an endpoint for Prometheus to scrape as a target. This endpoint can +be accessed by default at: +[http://localhost:9000/metrics](http://localhost:9000/metrics): + +You can adjust the settings within Oathkeeper's config. +```shell +$ cat << EOF > config.yaml +serve: + prometheus: + port: 9000 + host: localhost + metrics_path: /metrics +EOF +``` + +Prometheus can easily be run as a docker container. More information are available on [https://github.com/prometheus/prometheus](https://github.com/prometheus/prometheus). Start with setting up a prometheus configuration: + + +```shell +$ cat << EOF > prometheus.yml +global: + scrape_interval: 15s # By default, scrape targets every 15 seconds. + +scrape_configs: + - job_name: 'prometheus' + scrape_interval: 15s + static_configs: + - targets: ['localhost:9090'] + - job_name: 'oathkeeper' + scrape_interval: 15s + metrics_path: /metrics + static_configs: + # The target needs to match what you've configured above + - targets: ['localhost:9000'] +``` + +Then start the prometheus server and access it on [http://localhost:9090](http://localhost:9090). + +```shell +$ docker run \ + --config.file=/etc/prometheus/prometheus.yml \ + -v ./prometheus.yml:/etc/prometheus/prometheus.yml \ + --name prometheus \ + -d \ + --net=host + -p 9090:9090 \ + prom/prometheus +``` + +Now where you have a basic monitoring setup running you can extend it by building up nice visualizations eg. using Grafana. More information are available on [https://prometheus.io/docs/visualization/grafana/](https://prometheus.io/docs/visualization/grafana/). + +We have a pre built Dashboard which you can use to get started quickly: +[Oathkeeper-Dashboard.json](https://github.com/ory/oathkeeper/tree/master/contrib/grafana/Oathkeeper-Dashboard.json). + +ORY Oathkeeper with Prometheus and Grafana \ No newline at end of file diff --git a/docs/static/img/docs/grafana.png b/docs/static/img/docs/grafana.png new file mode 100644 index 0000000000..3088781526 Binary files /dev/null and b/docs/static/img/docs/grafana.png differ