diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 5a957c0..7281a75 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -31,5 +31,5 @@ jobs: java-version: '17' - name: Maven build - run: ./mvnw clean package + run: ./mvnw -Drevision=LOCAL-SNAPSHOT clean package diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 99bc2cd..53ceae1 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -24,21 +24,21 @@ jobs: run: echo "sha_short=$(git rev-parse --short HEAD)" >> $GITHUB_OUTPUT - name: Set up JDK - uses: actions/setup-java@v3 + uses: actions/setup-java@v4 with: distribution: 'temurin' java-version: '17' - - name: Maven build - run: ./mvnw clean package - - name: get tag uses: olegtarasov/get-tag@v2.1.2 id: tagName with: tagRegex: "(.*)" + - name: Maven build + run: ./mvnw -Drevision=${{ env.GIT_TAG_NAME }} -clean package + - uses: ncipollo/release-action@v1 with: - artifacts: "target/jfr-exporter-*.jar" + artifacts: "target/jfr-exporter-${{ env.GIT_TAG_NAME }}.jar" body: "Release ${{ env.GIT_TAG_NAME }}" diff --git a/README.md b/README.md index 965947d..7ce1212 100644 --- a/README.md +++ b/README.md @@ -1,25 +1,36 @@ # JfrExporter -Send JFR events to time series databases. +Send JFR events to time series databases. + +Now in "beta", feedback welcome! + +See tutorial in this [blog post](https://perfana.io/continuous-deep-dive-with-jfr-events/). + +[![JfrExporter tutarial video](https://img.youtube.com/vi/rAq2Xo-VoVc/0.jpg)](https://www.youtube.com/watch?v=rAq2Xo-VoVc) Makes use of [JFR Event Streaming](https://openjdk.org/jeps/349) as found in hotspot based JDK 14+. -Now in "beta", feedback welcome! The events and stack traces need accuracy checks. -More events can be added. Only InfluxDB time series database is supported -at the moment. +The InfluxDB time series database is used as storage for the metrics. -The metrics can be used in a Grafana dashboard. +The metrics are displayed in a Grafana dashboard, as shown in the following screenshots. -Shows CPU, Heap, Threads and Memory Allocation Rate: -![dashboard overview 1](https://github.com/perfana/jfr-exporter/blob/main/images/dashboard-1.jpg) +CPU, Heap, Threads and Memory Allocation Rate: +![dashboard overview 1](images/dashboard-6.jpg) -Shows Garbage Collection events: -![dashboard overview 2](https://github.com/perfana/jfr-exporter/blob/main/images/dashboard-2.jpg) +Garbage Collection events: +![dashboard overview 2](images/dashboard-8.jpg) -Shows Safepoints and Big Allocations: -![dashboard overview 3](https://github.com/perfana/jfr-exporter/blob/main/images/dashboard-3.jpg) +Large allocation samples and Big Allocations: +![dashboard overview 3](images/dashboard-7.jpg) -And shows the stacktrace of a big allocations (see screenshot below) +Java Monitor waits and enters: +![dashboard overview 4](images/dashboard-4.jpg) + +Network reads and writes: +![dashboard overview 5](images/dashboard-5.jpg) + +For some events stacktraces are present, such as where in the code a big memory allocation took place. +(see screenshot below) ## Steps @@ -34,7 +45,7 @@ To use JfrExporter: ## Download -Direct [download version 0.2.0](https://github.com/perfana/jfr-exporter/releases/download/0.2.0/jfr-exporter-0.1.0.jar) +Direct [download version 0.3.0](https://github.com/perfana/jfr-exporter/releases/download/0.3.0/jfr-exporter-0.3.0.jar) Download the latest release from the [releases page](https://github.com/perfana/jfr-exporter/releases). @@ -86,30 +97,35 @@ use a saved JFR profile in the JDK used, for example saved as `mySettings`: `-XX ## Events -Currently a subset of JFR events are processed. +These JFR events are processed: * CPU load -* Garbage Collection -* Memory (heap usage, large allocations) +* Thread count +* Classes loaded +* Garbage Collection (GC) events * Safepoints -* Threads -* Classloaders +* Memory (heap usage, large allocations) +* Network read/write +* Java Monitor waits and enters For reference: [list of JFR events](https://bestsolution-at.github.io/jfr-doc/index.html). ## Stacktraces -Stack trace for big allocations are sent to InfuxDB. -Via the dashboard you can see the details by clicking in the big allocations table. +Stack traces for several events are sent to InfuxDB. +Via the dashboard you can see the details by clicking in the stacktrace columns. -Example: -![stacktrace example 1](https://github.com/perfana/jfr-exporter/blob/main/images/stacktrace-1.jpg) +Example of a big memory allocation stacktrace: +![stacktrace example 1](images/stacktrace-2.jpg) ## Dashboard A Grafana dashboard can be imported to view the JFR metrics. + Import the dashboard in the `dashboards` directory into Grafana and connect to an InfluxDB datasource that points to the `jfr` database. +For version 0.3.0 and above use dashboard `jfr-dashboard-export-share-0.3.json`. + ## Troubleshoot Use `-Dio.perfana.jfr.debug=true` to enable debug logging or `--debug` as argument. @@ -117,3 +133,9 @@ Use `-Dio.perfana.jfr.debug=true` to enable debug logging or `--debug` as argume For tracing (more debug logging) use: `-Dio.perfana.jfr.trace=true` Debug and tracing will output a lot of data, so only use for troubleshooting. + +# Releases + +### v0.3.0: January 2024 +* Added new events: Java Monitor waits and enters, Network read/write +* New dashboard with new events diff --git a/dashboards/jfr-dashboard-export-share-0.3.json b/dashboards/jfr-dashboard-export-share-0.3.json new file mode 100644 index 0000000..a9d0ef2 --- /dev/null +++ b/dashboards/jfr-dashboard-export-share-0.3.json @@ -0,0 +1,3221 @@ +{ + "__inputs": [ + { + "name": "DS_INFLUXDB_JFR", + "label": "InfluxDB JFR", + "description": "", + "type": "datasource", + "pluginId": "influxdb", + "pluginName": "InfluxDB" + } + ], + "__elements": {}, + "__requires": [ + { + "type": "panel", + "id": "barchart", + "name": "Bar chart", + "version": "" + }, + { + "type": "grafana", + "id": "grafana", + "name": "Grafana", + "version": "10.1.5" + }, + { + "type": "datasource", + "id": "influxdb", + "name": "InfluxDB", + "version": "1.0.0" + }, + { + "type": "panel", + "id": "stat", + "name": "Stat", + "version": "" + }, + { + "type": "panel", + "id": "table", + "name": "Table", + "version": "" + }, + { + "type": "panel", + "id": "timeseries", + "name": "Time series", + "version": "" + } + ], + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "target": { + "limit": 100, + "matchAny": false, + "tags": [], + "type": "dashboard" + }, + "type": "dashboard" + } + ] + }, + "description": "Show events from JFR", + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "id": null, + "links": [], + "liveNow": false, + "panels": [ + { + "datasource": { + "type": "influxdb", + "uid": "${DS_INFLUXDB_JFR}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineStyle": { + "fill": "solid" + }, + "lineWidth": 1, + "pointSize": 3, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "always", + "spanNulls": 30000, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "percent" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 0 + }, + "id": 2, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "influxdb", + "uid": "${DS_INFLUXDB_JFR}" + }, + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "measurement": "CPU", + "orderByTime": "ASC", + "policy": "default", + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "*" + ], + "type": "field" + }, + { + "params": [], + "type": "max" + } + ] + ], + "tags": [] + } + ], + "title": "CPU", + "type": "timeseries" + }, + { + "datasource": { + "type": "influxdb", + "uid": "${DS_INFLUXDB_JFR}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineStyle": { + "fill": "solid" + }, + "lineWidth": 1, + "pointSize": 3, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "always", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 0 + }, + "id": 7, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "influxdb", + "uid": "${DS_INFLUXDB_JFR}" + }, + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "measurement": "heap", + "orderByTime": "ASC", + "policy": "default", + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "*" + ], + "type": "field" + }, + { + "params": [], + "type": "max" + } + ] + ], + "tags": [] + } + ], + "title": "Heap used", + "type": "timeseries" + }, + { + "datasource": { + "type": "influxdb", + "uid": "${DS_INFLUXDB_JFR}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineStyle": { + "fill": "solid" + }, + "lineWidth": 1, + "pointSize": 3, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "always", + "spanNulls": 30000, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "decimals": 0, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 9 + }, + "id": 6, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "influxdb", + "uid": "${DS_INFLUXDB_JFR}" + }, + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "measurement": "threads", + "orderByTime": "ASC", + "policy": "default", + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "*" + ], + "type": "field" + }, + { + "params": [], + "type": "max" + } + ] + ], + "tags": [] + } + ], + "title": "Threads active", + "type": "timeseries" + }, + { + "datasource": { + "type": "influxdb", + "uid": "${DS_INFLUXDB_JFR}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "smooth", + "lineStyle": { + "fill": "solid" + }, + "lineWidth": 1, + "pointSize": 1, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": 30000, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "Bps" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 9 + }, + "id": 4, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "influxdb", + "uid": "${DS_INFLUXDB_JFR}" + }, + "groupBy": [], + "measurement": "allocation-rate-bytes", + "orderByTime": "ASC", + "policy": "default", + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "bytes" + ], + "type": "field" + } + ] + ], + "tags": [] + } + ], + "title": "Memory allocation rate", + "type": "timeseries" + }, + { + "datasource": { + "type": "influxdb", + "uid": "${DS_INFLUXDB_JFR}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "fillOpacity": 80, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineWidth": 1, + "scaleDistribution": { + "type": "linear" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "ms" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 18 + }, + "id": 3, + "options": { + "barRadius": 0, + "barWidth": 0, + "fullHighlight": false, + "groupWidth": 0.7, + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "orientation": "auto", + "showValue": "auto", + "stacking": "none", + "tooltip": { + "mode": "single", + "sort": "none" + }, + "xTickLabelRotation": 0, + "xTickLabelSpacing": 100 + }, + "targets": [ + { + "datasource": { + "type": "influxdb", + "uid": "${DS_INFLUXDB_JFR}" + }, + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "measurement": "youngGc", + "orderByTime": "ASC", + "policy": "default", + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "duration-ms" + ], + "type": "field" + }, + { + "params": [], + "type": "max" + } + ] + ], + "tags": [] + } + ], + "title": "Garbage collections (young)", + "type": "barchart" + }, + { + "datasource": { + "type": "influxdb", + "uid": "${DS_INFLUXDB_JFR}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "ms" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 6, + "x": 12, + "y": 18 + }, + "id": 10, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "max" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "10.1.5", + "targets": [ + { + "datasource": { + "type": "influxdb", + "uid": "${DS_INFLUXDB_JFR}" + }, + "groupBy": [], + "measurement": "youngGc", + "orderByTime": "ASC", + "policy": "default", + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "duration-ms" + ], + "type": "field" + }, + { + "params": [], + "type": "max" + } + ] + ], + "tags": [] + } + ], + "title": "Garbage collections max (young)", + "type": "stat" + }, + { + "datasource": { + "type": "influxdb", + "uid": "${DS_INFLUXDB_JFR}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "ms" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 6, + "x": 18, + "y": 18 + }, + "id": 24, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "max" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "10.1.5", + "targets": [ + { + "datasource": { + "type": "influxdb", + "uid": "${DS_INFLUXDB_JFR}" + }, + "groupBy": [], + "measurement": "youngGc", + "orderByTime": "ASC", + "policy": "default", + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "duration-ms" + ], + "type": "field" + }, + { + "params": [], + "type": "sum" + } + ] + ], + "tags": [] + } + ], + "title": "Garbage collections total time (young)", + "type": "stat" + }, + { + "datasource": { + "type": "influxdb", + "uid": "${DS_INFLUXDB_JFR}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "fillOpacity": 80, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineWidth": 1, + "scaleDistribution": { + "type": "linear" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "ms" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 27 + }, + "id": 8, + "options": { + "barRadius": 0, + "barWidth": 0, + "fullHighlight": false, + "groupWidth": 0.7, + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "orientation": "auto", + "showValue": "auto", + "stacking": "none", + "tooltip": { + "mode": "single", + "sort": "none" + }, + "xTickLabelRotation": 0, + "xTickLabelSpacing": 100 + }, + "targets": [ + { + "datasource": { + "type": "influxdb", + "uid": "${DS_INFLUXDB_JFR}" + }, + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "measurement": "oldGc", + "orderByTime": "ASC", + "policy": "default", + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "duration-ms" + ], + "type": "field" + }, + { + "params": [], + "type": "max" + } + ] + ], + "tags": [] + } + ], + "title": "Garbage collections (old)", + "type": "barchart" + }, + { + "datasource": { + "type": "influxdb", + "uid": "${DS_INFLUXDB_JFR}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "ms" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 6, + "x": 12, + "y": 27 + }, + "id": 12, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "max" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "10.1.5", + "targets": [ + { + "datasource": { + "type": "influxdb", + "uid": "${DS_INFLUXDB_JFR}" + }, + "groupBy": [], + "measurement": "oldGc", + "orderByTime": "ASC", + "policy": "default", + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "duration-ms" + ], + "type": "field" + }, + { + "params": [], + "type": "max" + } + ] + ], + "tags": [] + } + ], + "title": "Garbage collections max (old)", + "type": "stat" + }, + { + "datasource": { + "type": "influxdb", + "uid": "${DS_INFLUXDB_JFR}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "ms" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 6, + "x": 18, + "y": 27 + }, + "id": 22, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "max" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "10.1.5", + "targets": [ + { + "datasource": { + "type": "influxdb", + "uid": "${DS_INFLUXDB_JFR}" + }, + "groupBy": [], + "measurement": "oldGc", + "orderByTime": "ASC", + "policy": "default", + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "duration-ms" + ], + "type": "field" + }, + { + "params": [], + "type": "sum" + } + ] + ], + "tags": [] + } + ], + "title": "Garbage collections total time (old)", + "type": "stat" + }, + { + "datasource": { + "type": "influxdb", + "uid": "${DS_INFLUXDB_JFR}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "fillOpacity": 80, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineWidth": 1, + "scaleDistribution": { + "type": "linear" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "ms" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 36 + }, + "id": 9, + "options": { + "barRadius": 0, + "barWidth": 0, + "fullHighlight": false, + "groupWidth": 0.7, + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "orientation": "auto", + "showValue": "auto", + "stacking": "none", + "tooltip": { + "mode": "single", + "sort": "none" + }, + "xTickLabelRotation": 0, + "xTickLabelSpacing": 100 + }, + "targets": [ + { + "datasource": { + "type": "influxdb", + "uid": "${DS_INFLUXDB_JFR}" + }, + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "measurement": "safepoint", + "orderByTime": "ASC", + "policy": "default", + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "duration-ms" + ], + "type": "field" + }, + { + "params": [], + "type": "max" + } + ] + ], + "tags": [] + } + ], + "title": "Safe points", + "type": "barchart" + }, + { + "datasource": { + "type": "influxdb", + "uid": "${DS_INFLUXDB_JFR}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "ms" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 6, + "x": 12, + "y": 36 + }, + "id": 11, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "max" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "10.1.5", + "targets": [ + { + "datasource": { + "type": "influxdb", + "uid": "${DS_INFLUXDB_JFR}" + }, + "groupBy": [], + "measurement": "safepoint", + "orderByTime": "ASC", + "policy": "default", + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "duration-ms" + ], + "type": "field" + }, + { + "params": [], + "type": "distinct" + } + ] + ], + "tags": [] + } + ], + "title": "Safe point max", + "type": "stat" + }, + { + "datasource": { + "type": "influxdb", + "uid": "${DS_INFLUXDB_JFR}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "ms" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 6, + "x": 18, + "y": 36 + }, + "id": 23, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "max" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "10.1.5", + "targets": [ + { + "datasource": { + "type": "influxdb", + "uid": "${DS_INFLUXDB_JFR}" + }, + "groupBy": [], + "measurement": "safepoint", + "orderByTime": "ASC", + "policy": "default", + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "duration-ms" + ], + "type": "field" + }, + { + "params": [], + "type": "sum" + } + ] + ], + "tags": [] + } + ], + "title": "Safe point total time", + "type": "stat" + }, + { + "datasource": { + "type": "influxdb", + "uid": "${DS_INFLUXDB_JFR}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "points", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineStyle": { + "fill": "solid" + }, + "lineWidth": 2, + "pointSize": 6, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "decbytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 45 + }, + "id": 13, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "influxdb", + "uid": "${DS_INFLUXDB_JFR}" + }, + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "measurement": "big-allocations", + "orderByTime": "ASC", + "policy": "default", + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "bytes" + ], + "type": "field" + }, + { + "params": [], + "type": "distinct" + } + ] + ], + "tags": [] + } + ], + "title": "Big allocations", + "type": "timeseries" + }, + { + "datasource": { + "type": "influxdb", + "uid": "${DS_INFLUXDB_JFR}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "align": "auto", + "cellOptions": { + "type": "auto" + }, + "inspect": true + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Time" + }, + "properties": [ + { + "id": "custom.width", + "value": 190 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "big-allocations.bytes" + }, + "properties": [ + { + "id": "unit", + "value": "decbytes" + } + ] + } + ] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 45 + }, + "id": 15, + "options": { + "cellHeight": "sm", + "footer": { + "countRows": false, + "fields": "", + "reducer": [ + "sum" + ], + "show": false + }, + "showHeader": true, + "sortBy": [ + { + "desc": true, + "displayName": "big-allocations.bytes" + } + ] + }, + "pluginVersion": "10.1.5", + "targets": [ + { + "datasource": { + "type": "influxdb", + "uid": "${DS_INFLUXDB_JFR}" + }, + "groupBy": [], + "measurement": "big-allocations", + "orderByTime": "ASC", + "policy": "default", + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "bytes" + ], + "type": "field" + } + ] + ], + "tags": [] + }, + { + "datasource": { + "type": "influxdb", + "uid": "${DS_INFLUXDB_JFR}" + }, + "groupBy": [], + "hide": false, + "measurement": "big-allocations", + "orderByTime": "ASC", + "policy": "default", + "query": "SELECT \"stacktrace\",\"objectClass\" FROM \"default\".\"big-allocations\" WHERE $timeFilter", + "rawQuery": false, + "refId": "B", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "stacktrace" + ], + "type": "field" + } + ], + [ + { + "params": [ + "objectClass" + ], + "type": "field" + } + ], + [ + { + "params": [ + "thread" + ], + "type": "field" + } + ] + ], + "tags": [] + } + ], + "title": "Big allocation stacktraces", + "transformations": [ + { + "id": "filterFieldsByName", + "options": { + "include": { + "names": [ + "Time", + "big-allocations.bytes", + "big-allocations.stacktrace", + "big-allocations.thread", + "big-allocations.objectClass" + ] + } + } + }, + { + "id": "seriesToColumns", + "options": { + "byField": "Time" + } + }, + { + "id": "organize", + "options": { + "excludeByName": {}, + "indexByName": {}, + "renameByName": { + "big-allocations.bytes": "size", + "big-allocations.objectClass": "object class", + "big-allocations.stacktrace": "stacktrace", + "big-allocations.thread": "thread" + } + } + } + ], + "type": "table" + }, + { + "datasource": { + "type": "influxdb", + "uid": "${DS_INFLUXDB_JFR}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "points", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineStyle": { + "fill": "solid" + }, + "lineWidth": 2, + "pointSize": 6, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "decbytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 54 + }, + "id": 16, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "influxdb", + "uid": "${DS_INFLUXDB_JFR}" + }, + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "measurement": "object-allocation-sample", + "orderByTime": "ASC", + "policy": "default", + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "bytes" + ], + "type": "field" + }, + { + "params": [], + "type": "distinct" + } + ] + ], + "tags": [] + } + ], + "title": "Large allocation samples", + "type": "timeseries" + }, + { + "datasource": { + "type": "influxdb", + "uid": "${DS_INFLUXDB_JFR}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "align": "auto", + "cellOptions": { + "type": "auto" + }, + "inspect": true + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Time" + }, + "properties": [ + { + "id": "custom.width", + "value": 190 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "object-allocation-sample.bytes" + }, + "properties": [ + { + "id": "unit", + "value": "decbytes" + } + ] + } + ] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 54 + }, + "id": 17, + "options": { + "cellHeight": "sm", + "footer": { + "countRows": false, + "fields": "", + "reducer": [ + "sum" + ], + "show": false + }, + "showHeader": true, + "sortBy": [ + { + "desc": true, + "displayName": "object-allocation-sample.bytes" + } + ] + }, + "pluginVersion": "10.1.5", + "targets": [ + { + "datasource": { + "type": "influxdb", + "uid": "${DS_INFLUXDB_JFR}" + }, + "groupBy": [], + "measurement": "object-allocation-sample", + "orderByTime": "ASC", + "policy": "default", + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "bytes" + ], + "type": "field" + } + ] + ], + "tags": [] + }, + { + "datasource": { + "type": "influxdb", + "uid": "${DS_INFLUXDB_JFR}" + }, + "groupBy": [], + "hide": false, + "measurement": "object-allocation-sample", + "orderByTime": "ASC", + "policy": "default", + "query": "SELECT \"stacktrace\",\"objectClass\" FROM \"default\".\"big-allocations\" WHERE $timeFilter", + "rawQuery": false, + "refId": "B", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "stacktrace" + ], + "type": "field" + } + ], + [ + { + "params": [ + "objectClass" + ], + "type": "field" + } + ], + [ + { + "params": [ + "thread" + ], + "type": "field" + } + ] + ], + "tags": [] + } + ], + "title": "Large allocation samples stacktraces", + "transformations": [ + { + "id": "filterFieldsByName", + "options": { + "include": { + "names": [ + "Time", + "object-allocation-sample.bytes", + "object-allocation-sample.stacktrace", + "object-allocation-sample.objectClass", + "object-allocation-sample.thread" + ] + } + } + }, + { + "id": "seriesToColumns", + "options": { + "byField": "Time" + } + }, + { + "id": "organize", + "options": { + "excludeByName": {}, + "indexByName": {}, + "renameByName": { + "object-allocation-sample.bytes": "size", + "object-allocation-sample.objectClass": "object class", + "object-allocation-sample.stacktrace": "stacktrace", + "object-allocation-sample.thread": "thread" + } + } + } + ], + "type": "table" + }, + { + "datasource": { + "type": "influxdb", + "uid": "${DS_INFLUXDB_JFR}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "bars", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineStyle": { + "fill": "solid" + }, + "lineWidth": 1, + "pointSize": 3, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "always", + "spanNulls": 30000, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "decimals": 0, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "ns" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 63 + }, + "id": 19, + "options": { + "legend": { + "calcs": [ + "max" + ], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "influxdb", + "uid": "${DS_INFLUXDB_JFR}" + }, + "groupBy": [], + "measurement": "java-monitor-enter", + "orderByTime": "ASC", + "policy": "default", + "query": "SELECT sum(\"bytes\") FROM \"SocketRead\" WHERE $timeFilter GROUP BY time($__interval) fill(null)", + "rawQuery": false, + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "duration-ns" + ], + "type": "field" + } + ] + ], + "tags": [] + } + ], + "title": "Java Monitor Enter", + "type": "timeseries" + }, + { + "datasource": { + "type": "influxdb", + "uid": "${DS_INFLUXDB_JFR}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "align": "auto", + "cellOptions": { + "type": "auto" + }, + "inspect": true + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Time" + }, + "properties": [ + { + "id": "custom.width", + "value": 190 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "java-monitor-enter.duration-ns" + }, + "properties": [ + { + "id": "unit", + "value": "ns" + } + ] + } + ] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 63 + }, + "id": 21, + "options": { + "cellHeight": "sm", + "footer": { + "countRows": false, + "fields": "", + "reducer": [ + "sum" + ], + "show": false + }, + "showHeader": true, + "sortBy": [ + { + "desc": true, + "displayName": "java-monitor-enter.duration-ns" + } + ] + }, + "pluginVersion": "10.1.5", + "targets": [ + { + "datasource": { + "type": "influxdb", + "uid": "${DS_INFLUXDB_JFR}" + }, + "groupBy": [], + "measurement": "java-monitor-enter", + "orderByTime": "ASC", + "policy": "default", + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "duration-ns" + ], + "type": "field" + } + ] + ], + "tags": [] + }, + { + "datasource": { + "type": "influxdb", + "uid": "${DS_INFLUXDB_JFR}" + }, + "groupBy": [], + "hide": false, + "measurement": "java-monitor-enter", + "orderByTime": "ASC", + "policy": "default", + "query": "SELECT \"stacktrace\",\"objectClass\" FROM \"default\".\"big-allocations\" WHERE $timeFilter", + "rawQuery": false, + "refId": "B", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "stacktrace" + ], + "type": "field" + } + ], + [ + { + "params": [ + "thread" + ], + "type": "field" + } + ], + [ + { + "params": [ + "address" + ], + "type": "field" + } + ], + [ + { + "params": [ + "monitor-class" + ], + "type": "field" + } + ], + [ + { + "params": [ + "previous-owner" + ], + "type": "field" + } + ] + ], + "tags": [] + } + ], + "title": "Java Monitor Enter stacktraces", + "transformations": [ + { + "id": "filterFieldsByName", + "options": { + "include": { + "names": [ + "java-monitor-enter.duration-ns", + "java-monitor-enter.stacktrace", + "java-monitor-enter.thread", + "java-monitor-enter.address", + "java-monitor-enter.monitor-class", + "java-monitor-enter.previous-owner", + "Time" + ] + } + } + }, + { + "id": "seriesToColumns", + "options": { + "byField": "Time" + } + }, + { + "id": "organize", + "options": { + "excludeByName": {}, + "indexByName": {}, + "renameByName": { + "java-monitor-enter.address": "address", + "java-monitor-enter.duration-ns": "duration", + "java-monitor-enter.monitor-class": "monitor class", + "java-monitor-enter.previous-owner": "previous owner", + "java-monitor-enter.stacktrace": "stacktrace", + "java-monitor-enter.thread": "thread" + } + } + } + ], + "type": "table" + }, + { + "datasource": { + "type": "influxdb", + "uid": "${DS_INFLUXDB_JFR}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "bars", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineStyle": { + "fill": "solid" + }, + "lineWidth": 1, + "pointSize": 3, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "always", + "spanNulls": 30000, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "decimals": 0, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "ns" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 72 + }, + "id": 25, + "options": { + "legend": { + "calcs": [ + "max" + ], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "influxdb", + "uid": "${DS_INFLUXDB_JFR}" + }, + "groupBy": [], + "measurement": "java-monitor-wait", + "orderByTime": "ASC", + "policy": "default", + "query": "SELECT sum(\"bytes\") FROM \"SocketRead\" WHERE $timeFilter GROUP BY time($__interval) fill(null)", + "rawQuery": false, + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "duration-ns" + ], + "type": "field" + } + ] + ], + "tags": [] + } + ], + "title": "Java Monitor Wait", + "type": "timeseries" + }, + { + "datasource": { + "type": "influxdb", + "uid": "${DS_INFLUXDB_JFR}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "align": "auto", + "cellOptions": { + "type": "auto" + }, + "inspect": true + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Time" + }, + "properties": [ + { + "id": "custom.width", + "value": 190 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "java-monitor-wait.duration-ns" + }, + "properties": [ + { + "id": "unit", + "value": "ns" + } + ] + } + ] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 72 + }, + "id": 26, + "options": { + "cellHeight": "sm", + "footer": { + "countRows": false, + "fields": "", + "reducer": [ + "sum" + ], + "show": false + }, + "showHeader": true, + "sortBy": [ + { + "desc": true, + "displayName": "duration" + } + ] + }, + "pluginVersion": "10.1.5", + "targets": [ + { + "datasource": { + "type": "influxdb", + "uid": "${DS_INFLUXDB_JFR}" + }, + "groupBy": [], + "measurement": "java-monitor-wait", + "orderByTime": "ASC", + "policy": "default", + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "duration-ns" + ], + "type": "field" + } + ] + ], + "tags": [] + }, + { + "datasource": { + "type": "influxdb", + "uid": "${DS_INFLUXDB_JFR}" + }, + "groupBy": [], + "hide": false, + "measurement": "java-monitor-wait", + "orderByTime": "ASC", + "policy": "default", + "query": "SELECT \"stacktrace\",\"objectClass\" FROM \"default\".\"big-allocations\" WHERE $timeFilter", + "rawQuery": false, + "refId": "B", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "stacktrace" + ], + "type": "field" + } + ], + [ + { + "params": [ + "thread" + ], + "type": "field" + } + ], + [ + { + "params": [ + "address" + ], + "type": "field" + } + ], + [ + { + "params": [ + "monitor-class" + ], + "type": "field" + } + ], + [ + { + "params": [ + "notifier" + ], + "type": "field" + } + ], + [ + { + "params": [ + "timeout" + ], + "type": "field" + } + ], + [ + { + "params": [ + "timed-out" + ], + "type": "field" + } + ] + ], + "tags": [] + } + ], + "title": "Java Monitor Wait stacktraces", + "transformations": [ + { + "id": "filterFieldsByName", + "options": { + "include": { + "names": [ + "Time", + "java-monitor-wait.stacktrace", + "java-monitor-wait.thread", + "java-monitor-wait.address", + "java-monitor-wait.monitor-class", + "java-monitor-wait.notifier", + "java-monitor-wait.timeout", + "java-monitor-wait.timed-out", + "java-monitor-wait.duration-ns" + ] + } + } + }, + { + "id": "seriesToColumns", + "options": { + "byField": "Time", + "mode": "outer" + } + }, + { + "id": "organize", + "options": { + "excludeByName": {}, + "indexByName": {}, + "renameByName": { + "java-monitor-wait.address": "address", + "java-monitor-wait.duration-ns": "duration", + "java-monitor-wait.monitor-class": "monitor-class", + "java-monitor-wait.notifier": "notifier", + "java-monitor-wait.stacktrace": "stacktrace", + "java-monitor-wait.thread": "thread", + "java-monitor-wait.timed-out": "timed-out", + "java-monitor-wait.timeout": "timeout-ms" + } + } + } + ], + "type": "table" + }, + { + "datasource": { + "type": "influxdb", + "uid": "${DS_INFLUXDB_JFR}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "smooth", + "lineStyle": { + "fill": "solid" + }, + "lineWidth": 1, + "pointSize": 3, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "always", + "spanNulls": 30000, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "decimals": 0, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "Bps" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 81 + }, + "id": 18, + "options": { + "legend": { + "calcs": [ + "max" + ], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "alias": "$tag_host $tag_address $tag_port", + "datasource": { + "type": "influxdb", + "uid": "${DS_INFLUXDB_JFR}" + }, + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "port::tag" + ], + "type": "tag" + }, + { + "params": [ + "host::tag" + ], + "type": "tag" + }, + { + "params": [ + "address::tag" + ], + "type": "tag" + } + ], + "measurement": "socket-read-rate-bytes", + "orderByTime": "ASC", + "policy": "default", + "query": "SELECT sum(\"bytes\") FROM \"socket-read-rate-bytes\" WHERE $timeFilter GROUP BY time($__interval), \"port\"::tag, \"host\"::tag, \"address\"::tag", + "rawQuery": false, + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "bytes" + ], + "type": "field" + }, + { + "params": [], + "type": "sum" + } + ] + ], + "tags": [ + { + "key": "application::tag", + "operator": "=~", + "value": "/^$application$/" + } + ] + } + ], + "title": "Socket read bytes", + "type": "timeseries" + }, + { + "datasource": { + "type": "influxdb", + "uid": "${DS_INFLUXDB_JFR}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "smooth", + "lineStyle": { + "fill": "solid" + }, + "lineWidth": 1, + "pointSize": 3, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "always", + "spanNulls": 30000, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "decimals": 0, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "Bps" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 81 + }, + "id": 20, + "options": { + "legend": { + "calcs": [ + "max" + ], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "alias": "$tag_host $tag_address $tag_port", + "datasource": { + "type": "influxdb", + "uid": "${DS_INFLUXDB_JFR}" + }, + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "port::tag" + ], + "type": "tag" + }, + { + "params": [ + "address::tag" + ], + "type": "tag" + }, + { + "params": [ + "host::tag" + ], + "type": "tag" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "measurement": "socket-write-rate-bytes", + "orderByTime": "ASC", + "policy": "autogen", + "query": "SELECT sum(\"bytes\") FROM \"SocketRead\" WHERE $timeFilter GROUP BY time($__interval) fill(null)", + "rawQuery": false, + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "bytes" + ], + "type": "field" + }, + { + "params": [], + "type": "sum" + } + ] + ], + "tags": [] + } + ], + "title": "Socket write bytes", + "type": "timeseries" + }, + { + "datasource": { + "type": "influxdb", + "uid": "${DS_INFLUXDB_JFR}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineStyle": { + "fill": "solid" + }, + "lineWidth": 1, + "pointSize": 3, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "always", + "spanNulls": 30000, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "decimals": 0, + "displayName": "loaded classes count", + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 90 + }, + "id": 5, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "influxdb", + "uid": "${DS_INFLUXDB_JFR}" + }, + "groupBy": [ + { + "params": [ + "$__interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "measurement": "classes-loaded", + "orderByTime": "ASC", + "policy": "default", + "refId": "A", + "resultFormat": "time_series", + "select": [ + [ + { + "params": [ + "loadedClassCount" + ], + "type": "field" + }, + { + "params": [], + "type": "max" + } + ] + ], + "tags": [] + } + ], + "title": "Classes loaded", + "type": "timeseries" + } + ], + "refresh": false, + "schemaVersion": 38, + "style": "dark", + "tags": [ + "jfr", + "perfana" + ], + "templating": { + "list": [ + { + "current": {}, + "datasource": { + "type": "influxdb", + "uid": "${DS_INFLUXDB_JFR}" + }, + "definition": "SHOW TAG VALUES WITH KEY = \"application\"", + "hide": 0, + "includeAll": false, + "multi": false, + "name": "application", + "options": [], + "query": "SHOW TAG VALUES WITH KEY = \"application\"", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "type": "query" + } + ] + }, + "time": { + "from": "now-15m", + "to": "now" + }, + "timepicker": {}, + "timezone": "", + "title": "Perfana JFR Metrics 0.3", + "uid": "jfr-exporter-0-3", + "version": 14, + "weekStart": "" +} \ No newline at end of file diff --git a/images/dashboard-4.jpg b/images/dashboard-4.jpg new file mode 100644 index 0000000..ae08879 Binary files /dev/null and b/images/dashboard-4.jpg differ diff --git a/images/dashboard-5.jpg b/images/dashboard-5.jpg new file mode 100644 index 0000000..60813e1 Binary files /dev/null and b/images/dashboard-5.jpg differ diff --git a/images/dashboard-6.jpg b/images/dashboard-6.jpg new file mode 100644 index 0000000..81e42fd Binary files /dev/null and b/images/dashboard-6.jpg differ diff --git a/images/dashboard-7.jpg b/images/dashboard-7.jpg new file mode 100644 index 0000000..d1828b6 Binary files /dev/null and b/images/dashboard-7.jpg differ diff --git a/images/dashboard-8.jpg b/images/dashboard-8.jpg new file mode 100644 index 0000000..6ee6a6a Binary files /dev/null and b/images/dashboard-8.jpg differ diff --git a/images/stacktrace-2.jpg b/images/stacktrace-2.jpg new file mode 100644 index 0000000..e9de30b Binary files /dev/null and b/images/stacktrace-2.jpg differ diff --git a/pom.xml b/pom.xml index 64ad479..b37d0ae 100644 --- a/pom.xml +++ b/pom.xml @@ -22,7 +22,7 @@ io.perfana jfr-exporter - 0.2.0 + ${revision} jfr-exporter Export stream of JDK Flight Recorder JVM events to Influx @@ -59,7 +59,7 @@ com.influxdb influxdb-client-java - 6.9.0 + 6.12.0 org.junit.jupiter diff --git a/src/main/java/io/perfana/jfr/Arguments.java b/src/main/java/io/perfana/jfr/Arguments.java index 14f91d0..9cbf6f3 100644 --- a/src/main/java/io/perfana/jfr/Arguments.java +++ b/src/main/java/io/perfana/jfr/Arguments.java @@ -32,8 +32,8 @@ public class Arguments { private String perfanaApiKey = null; private Duration duration = null; private String influxRetentionPolicy = "autogen"; - private long bigObjectThresholdBytes = 32_000L; - private long bigObjectSampleWeightThresholdBytes = 1_000_000L; + private long bigObjectThresholdBytes = 256_000L; + private long bigObjectSampleWeightThresholdBytes = 48_000_000L; public static String usage() { return "Usage: java JfrExporter " + diff --git a/src/main/java/io/perfana/jfr/JfrExporter.java b/src/main/java/io/perfana/jfr/JfrExporter.java index cafb8be..5e20f37 100644 --- a/src/main/java/io/perfana/jfr/JfrExporter.java +++ b/src/main/java/io/perfana/jfr/JfrExporter.java @@ -65,10 +65,12 @@ public void start(Arguments args) { SafepointEvent safepointEvent = new SafepointEvent(eventProcessor); safepointEvent.getEventSettings().forEach(eventHandler::register); - ObjectAllocationSampleEvent objectAllocationSampleEvent = new ObjectAllocationSampleEvent(eventProcessor, args.getBigObjectSampleWeigthThresholdBytes()); + ObjectAllocationSampleEvent objectAllocationSampleEvent = + new ObjectAllocationSampleEvent(eventProcessor, args.getBigObjectSampleWeigthThresholdBytes()); objectAllocationSampleEvent.getEventSettings().forEach(eventHandler::register); - ObjectAllocationEvent objectAllocationEvent = new ObjectAllocationEvent(eventProcessor, args.getBigObjectThresholdBytes()); + ObjectAllocationEvent objectAllocationEvent = + new ObjectAllocationEvent(eventProcessor, args.getBigObjectThresholdBytes()); objectAllocationEvent.getEventSettings().forEach(eventHandler::register); GCHeapEvent gcHeapEvent = new GCHeapEvent(eventProcessor); @@ -77,6 +79,12 @@ public void start(Arguments args) { JavaStatisticsEvent javaStatisticsEvent = new JavaStatisticsEvent(eventProcessor); javaStatisticsEvent.getEventSettings().forEach(eventHandler::register); + MonitorEvent monitorEvent = new MonitorEvent(eventProcessor); + monitorEvent.getEventSettings().forEach(eventHandler::register); + + SocketEvent socketEvent = new SocketEvent(eventProcessor); + socketEvent.getEventSettings().forEach(eventHandler::register); + JfrConnector jfrConnector = new JfrConnector(eventHandler); if (args.getProcessId() == null) { diff --git a/src/main/java/io/perfana/jfr/ProcessedJfrEvent.java b/src/main/java/io/perfana/jfr/ProcessedJfrEvent.java index 32f2521..562f0ba 100644 --- a/src/main/java/io/perfana/jfr/ProcessedJfrEvent.java +++ b/src/main/java/io/perfana/jfr/ProcessedJfrEvent.java @@ -27,13 +27,17 @@ /** * A processed JFR event. * The timestamp can be null if not present. + * Preferably Use one of the static factory methods "of()" to create an instance. */ public record ProcessedJfrEvent(@Nullable Instant timestamp, @Nonnull String measurementName, + @Nonnull Map tags, @Nonnull String field, @Nonnull Number value, - @Nonnull List stacktrace, - @Nonnull Map extraFields) { + @Nonnull Map extraFields, + @Nonnull List stacktrace +) { + public ProcessedJfrEvent { if (measurementName.isBlank()) { throw new IllegalArgumentException("measurementName cannot be blank."); @@ -47,7 +51,7 @@ public static ProcessedJfrEvent of(@Nullable Instant timestamp, @Nonnull String measurementName, @Nonnull String field, @Nonnull Number value) { - return new ProcessedJfrEvent(timestamp, measurementName, field, value, List.of(), Map.of()); + return new ProcessedJfrEvent(timestamp, measurementName, Map.of(), field, value, Map.of(), List.of()); } public static ProcessedJfrEvent of(RecordedEvent event, String measurementName, String metric, MetricCalculation calculation) { @@ -62,10 +66,49 @@ public static ProcessedJfrEvent of(RecordedEvent event, String measurementName, } + public static ProcessedJfrEvent of( + @Nullable Instant timestamp, + @Nonnull String measurementName, + @Nonnull String field, + long value, + @Nonnull Map extraFields) { + return new ProcessedJfrEvent(timestamp, measurementName, Map.of(), field, value, extraFields, List.of()); + } + + public static ProcessedJfrEvent of( + @Nullable Instant timestamp, + @Nonnull String measurementName, + @Nonnull String field, + long value, + @Nonnull Map extraFields, + @Nonnull List stacktrace) { + return new ProcessedJfrEvent(timestamp, measurementName, Map.of(), field, value, extraFields, stacktrace); + } + + public static ProcessedJfrEvent of( + @Nullable Instant timestamp, + @Nonnull String measurementName, + @Nonnull Map tags, + @Nonnull String field, + long value) { + return new ProcessedJfrEvent(timestamp, measurementName, tags, field, value, Map.of(), List.of()); + } + + public static ProcessedJfrEvent of( + @Nullable Instant timestamp, + @Nonnull String measurementName, + @Nonnull Map tags, + @Nonnull String field, + long value, + @Nonnull Map extraFields) { + return new ProcessedJfrEvent(timestamp, measurementName, tags, field, value, extraFields, List.of()); + } + public String toStringShort() { return "ProcessedJfrEvent{" + "timestamp=" + timestamp + ", measurementName='" + measurementName + '\'' + + ", tags=" + tags + ", field='" + field + '\'' + ", value=" + value + ", stacktrace=" + (stacktrace.isEmpty() ? "[]" : stacktrace.get(0) + "...") + diff --git a/src/main/java/io/perfana/jfr/event/GCHeapEvent.java b/src/main/java/io/perfana/jfr/event/GCHeapEvent.java index 7801572..625f746 100644 --- a/src/main/java/io/perfana/jfr/event/GCHeapEvent.java +++ b/src/main/java/io/perfana/jfr/event/GCHeapEvent.java @@ -22,7 +22,6 @@ import jdk.jfr.consumer.RecordedEvent; import java.time.Duration; -import java.util.Collections; import java.util.List; import java.util.Map; @@ -71,7 +70,7 @@ private ProcessedJfrEvent processOldGc(RecordedEvent event) { return ProcessedJfrEvent.of( event.getStartTime(), "oldGc", - "duration", + "duration-ms", durationMs); } @@ -81,7 +80,7 @@ private ProcessedJfrEvent processYoungGc(RecordedEvent event) { return ProcessedJfrEvent.of( event.getStartTime(), "youngGc", - "duration", + "duration-ms", durationMs); } @@ -89,12 +88,11 @@ private ProcessedJfrEvent processGcHeapSummary(RecordedEvent event) { long heapUsed = event.getLong("heapUsed"); long heapCommitted = event.getLong("heapSpace.committedSize"); - return new ProcessedJfrEvent( + return ProcessedJfrEvent.of( event.getStartTime(), "heap", "heapUsed", heapUsed, - Collections.emptyList(), Map.of("heapCommitted", heapCommitted)); } diff --git a/src/main/java/io/perfana/jfr/event/MonitorEvent.java b/src/main/java/io/perfana/jfr/event/MonitorEvent.java new file mode 100644 index 0000000..f01c48f --- /dev/null +++ b/src/main/java/io/perfana/jfr/event/MonitorEvent.java @@ -0,0 +1,138 @@ +/* + * Copyright (C) 2023 Peter Paul Bakker - Perfana + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.perfana.jfr.event; + +import io.perfana.jfr.*; +import jdk.jfr.consumer.RecordedEvent; +import jdk.jfr.consumer.RecordedThread; + +import java.time.Duration; +import java.time.Instant; +import java.util.List; +import java.util.Map; + +public class MonitorEvent implements OnJfrEvent, JfrEventProvider { + + private static final Logger log = Logger.getLogger(MonitorEvent.class); + + public static final String JDK_JAVA_MONITOR_WAIT = "jdk.JavaMonitorWait"; + public static final String JDK_JAVA_MONITOR_ENTER = "jdk.JavaMonitorEnter"; + private final JfrEventProcessor eventProcessor; + + private final long minimumDurationNs = Duration.ofMillis(10).toNanos(); + + public MonitorEvent(JfrEventProcessor eventProcessor) { + if (eventProcessor == null) throw new IllegalArgumentException("eventProcessor must not be null"); + this.eventProcessor = eventProcessor; + } + + @Override + public void onEvent(RecordedEvent event) { + String name = event.getEventType().getName(); + Instant startTime = event.getStartTime(); + long durationNs = event.getLong("duration"); + String monitorClass = event.getClass("monitorClass").getName(); + + long address = event.getValue("address"); + // prefix with 0x to prevent interpretation as number by InfluxDB + String addressAsHex = "0x" + Long.toHexString(address); + + log.trace("%s %s %d %s %s", (startTime == null ? "" : startTime), name, durationNs, monitorClass, addressAsHex); + + if (durationNs > minimumDurationNs) { + + if (event.getStackTrace() == null) { + log.error("No stack trace available for monitor wait of %d nanoseconds of monitorClass '%s'", durationNs, monitorClass); + return; + } + + List stackTrace = JfrUtil.translateStacktrace(event); + + String firstStack = stackTrace.isEmpty() ? "" : stackTrace.get(0); + log.debug("Found monitor wait of %d nanoseconds of '%s' in '%s'", durationNs, monitorClass, firstStack); + + String threadName = nullSafeGetThreadJavaName(event, "eventThread"); + + if (name.equals(JDK_JAVA_MONITOR_WAIT)) { + + if (threadName.startsWith("JFR Event Stream") || threadName.startsWith("Finalizer")) { + log.debug("Ignoring monitor wait of %d nanoseconds of thread '%s'", durationNs, threadName); + return; + } + + String notifier = nullSafeGetThreadJavaName(event,"notifier"); + long timeout = event.getLong("timeout"); + boolean timedOut = event.getBoolean("timedOut"); + + Map extraFields = Map.of( + "monitor-class", monitorClass, + "thread", threadName, + "address", addressAsHex, + "notifier", notifier, + "timeout", timeout, + "timed-out", String.valueOf(timedOut) + ); + + reportMonitor("java-monitor-wait", durationNs, startTime, extraFields, stackTrace); + + } else if (name.equals(JDK_JAVA_MONITOR_ENTER)) { + + String previousOwner = nullSafeGetThreadJavaName(event, "previousOwner"); + + Map extraFields = Map.of( + "monitor-class", monitorClass, + "thread", threadName, + "address", addressAsHex, + "previous-owner", previousOwner + ); + + reportMonitor("java-monitor-enter", durationNs, startTime, extraFields, stackTrace); + + } else { + log.error("Unknown monitor event '%s'", name); + } + } + } + + private static String nullSafeGetThreadJavaName(RecordedEvent event, String attributeName) { + RecordedThread thread = event.getThread(attributeName); + return thread == null ? "" : thread.getJavaName(); + } + + private void reportMonitor(String measurementName, long duration, Instant startTime, Map extraFields, List stackTrace) { + ProcessedJfrEvent processedEvent = ProcessedJfrEvent.of( + startTime, + measurementName, + "duration-ns", + duration, + extraFields, + stackTrace); + + eventProcessor.processEvent(processedEvent); + } + + @Override + public List getEventSettings() { + + JfrEventSettings monitorWait = JfrEventSettings.of(JDK_JAVA_MONITOR_WAIT, this) + .withThreshold(Duration.ofNanos(minimumDurationNs)); + + JfrEventSettings monitorEnter = JfrEventSettings.of(JDK_JAVA_MONITOR_ENTER, this) + .withThreshold(Duration.ofNanos(minimumDurationNs)); + + return List.of(monitorWait, monitorEnter); + } +} diff --git a/src/main/java/io/perfana/jfr/event/ObjectAllocationEvent.java b/src/main/java/io/perfana/jfr/event/ObjectAllocationEvent.java index 723d73b..0b48c03 100644 --- a/src/main/java/io/perfana/jfr/event/ObjectAllocationEvent.java +++ b/src/main/java/io/perfana/jfr/event/ObjectAllocationEvent.java @@ -63,14 +63,18 @@ private void reportBigAllocation(RecordedEvent event, long allocationSize, Strin String firstStack = stackTrace.isEmpty() ? "" : stackTrace.get(0); log.debug("Found big object allocation of %d bytes of %s in '%s'", allocationSize, objectClassTranslation, firstStack); - Map extraFields = Map.of("objectClass", objectClassTranslation, "thread", event.getThread().getJavaName()); + Map extraFields = Map.of( + "objectClass", objectClassTranslation, + "thread", event.getThread().getJavaName() + ); - ProcessedJfrEvent processedEvent = new ProcessedJfrEvent(startTime, + ProcessedJfrEvent processedEvent = ProcessedJfrEvent.of( + startTime, "big-allocations", "bytes", allocationSize, - stackTrace, - extraFields); + extraFields, + stackTrace); eventProcessor.processEvent(processedEvent); } diff --git a/src/main/java/io/perfana/jfr/event/ObjectAllocationSampleEvent.java b/src/main/java/io/perfana/jfr/event/ObjectAllocationSampleEvent.java index 109f792..48a823c 100644 --- a/src/main/java/io/perfana/jfr/event/ObjectAllocationSampleEvent.java +++ b/src/main/java/io/perfana/jfr/event/ObjectAllocationSampleEvent.java @@ -64,13 +64,15 @@ private void reportTotalAllocations(long weight, Instant startTime) { long now = System.currentTimeMillis(); - if (now - lastAllocationRateReport.get() > reportIntervalMs) { + long timePeriodMs = now - lastAllocationRateReport.get(); + + if (timePeriodMs > reportIntervalMs) { lastAllocationRateReport.set(now); long totalAllocations = totalAllocationsBytes.getAndSet(0); - long allocationRate = totalAllocations / (reportIntervalMs / 1000); + long allocationRate = totalAllocations / (timePeriodMs / 1000); log.debug("Total allocations: %d bytes, allocation rate: %d bytes/s", totalAllocations, @@ -102,12 +104,13 @@ private void reportLargeAllocationSample(RecordedEvent event, long weight, Strin Map extraFields = Map.of("objectClass", objectClassTranslation, "thread", event.getThread().getJavaName()); - ProcessedJfrEvent processedEvent = new ProcessedJfrEvent(startTime, + ProcessedJfrEvent processedEvent = ProcessedJfrEvent.of( + startTime, "object-allocation-sample", "bytes", weight, - stackTrace, - extraFields); + extraFields, + stackTrace); eventProcessor.processEvent(processedEvent); } diff --git a/src/main/java/io/perfana/jfr/event/SafepointEvent.java b/src/main/java/io/perfana/jfr/event/SafepointEvent.java index 508b4cf..1b2f9a8 100644 --- a/src/main/java/io/perfana/jfr/event/SafepointEvent.java +++ b/src/main/java/io/perfana/jfr/event/SafepointEvent.java @@ -55,13 +55,15 @@ public void onEvent(RecordedEvent event) { Duration duration = Duration.between(startTime, event.getEndTime()); log.debug("Safepoint duration: %s", duration); - ProcessedJfrEvent processedEvent = ProcessedJfrEvent.of( - event.getStartTime(), - "safepoint", - "duration", - (double) duration.toMillis()); + if (duration.toMillis() > 1) { + ProcessedJfrEvent processedEvent = ProcessedJfrEvent.of( + event.getStartTime(), + "safepoint", + "duration-ms", + (double) duration.toMillis()); - eventProcessor.processEvent(processedEvent); + eventProcessor.processEvent(processedEvent); + } } else { log.debug("Safepoint begin with id %d not found", safepointId); } diff --git a/src/main/java/io/perfana/jfr/event/SocketEvent.java b/src/main/java/io/perfana/jfr/event/SocketEvent.java new file mode 100644 index 0000000..9d7c215 --- /dev/null +++ b/src/main/java/io/perfana/jfr/event/SocketEvent.java @@ -0,0 +1,195 @@ +/* + * Copyright (C) 2023 Peter Paul Bakker - Perfana + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.perfana.jfr.event; + +import io.perfana.jfr.*; +import jdk.jfr.consumer.RecordedEvent; +import org.jetbrains.annotations.NotNull; + +import javax.annotation.Nonnull; +import java.time.Instant; +import java.util.List; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.atomic.AtomicLong; + +public class SocketEvent implements OnJfrEvent, JfrEventProvider { + + private static final Logger log = Logger.getLogger(SocketEvent.class); + + public static final String JDK_SOCKET_READ = "jdk.SocketRead"; + public static final String JDK_SOCKET_WRITE = "jdk.SocketWrite"; + private final JfrEventProcessor eventProcessor; + + private final AtomicLong lastWriteReportTimestamp = new AtomicLong(0); + private final AtomicLong lastReadReportTimestamp = new AtomicLong(0); + + private final Map totalWriteBytesPerHost = new ConcurrentHashMap<>(); + private final Map totalReadBytesPerHost = new ConcurrentHashMap<>(); + + private static final long reportIntervalMs = 2000; + + public SocketEvent(JfrEventProcessor eventProcessor) { + if (eventProcessor == null) throw new IllegalArgumentException("eventProcessor must not be null"); + this.eventProcessor = eventProcessor; + } + + private record TotalBytesHostKey(@Nonnull String host, @Nonnull String address, String port) { + TotalBytesHostKey(String host, String address, int port) { + this(host, address, checkDynamicPorts(port)); + } + + @NotNull + private static String checkDynamicPorts(int port) { + return port > 49151 && port < 65536 ? "dynamic" : String.valueOf(port); + } + } + + @Override + public void onEvent(RecordedEvent event) { + String name = event.getEventType().getName(); + + if (name.equals(JDK_SOCKET_READ)) { + reportSocketRead(event); + } else if (name.equals(JDK_SOCKET_WRITE)) { + reportSocketWrite(event); + } else { + log.debug("Not implemented event: %s", name); + } + + } + + private void reportSocketRead(RecordedEvent event) { + Instant startTime = event.getStartTime(); + + String name = event.getEventType().getName(); + String threadName = event.getThread().getJavaName(); + long durationNs = event.getLong("duration"); + long bytesRead = event.getLong("bytesRead"); + String host = event.getString("host"); + String address = event.getString("address"); + int port = event.getInt("port"); + + Map extraFields = Map.of( + "threadName", threadName, + "durationNs", durationNs, + "host", host, + "address", address, + "port", port + ); + + log.trace("Socket read: %s %s %d %s", + (startTime == null ? "" : startTime), + name, + bytesRead, + extraFields); + + TotalBytesHostKey hostKey = new TotalBytesHostKey(host, address, port); + + processEvent(lastReadReportTimestamp, "read", totalReadBytesPerHost, hostKey, bytesRead); + + } + + private void reportSocketWrite(RecordedEvent event) { + Instant startTime = event.getStartTime(); + + String name = event.getEventType().getName(); + String threadName = event.getThread().getJavaName(); + long durationNs = event.getLong("duration"); + long bytesWritten = event.getLong("bytesWritten"); + String host = event.getString("host"); + String address = event.getString("address"); + int port = event.getInt("port"); + + Map extraFields = Map.of( + "threadName", threadName, + "durationNs", durationNs, + "host", host, + "address", address, + "port", port + ); + + log.trace("Socket write: %s %s %d %s", + (startTime == null ? "" : startTime), + name, + bytesWritten, + extraFields); + + TotalBytesHostKey hostKey = new TotalBytesHostKey(host, address, port); + + processEvent(lastWriteReportTimestamp, "write", totalWriteBytesPerHost, hostKey, bytesWritten); + + } + + private void processEvent(AtomicLong lastReportTimestamp, String readOrWrite, Map totalBytesPerHost, TotalBytesHostKey hostKey, long bytes) { + + if (bytes != 0) { + totalBytesPerHost.computeIfAbsent(hostKey, k -> new AtomicLong(0)).addAndGet(bytes); + } + + long now = System.currentTimeMillis(); + + long timePeriodMs = now - lastReportTimestamp.get(); + + if (timePeriodMs > reportIntervalMs) { + + lastReportTimestamp.set(now); + + log.debug("totalBytesPerHost (%s) before: %s", readOrWrite, totalBytesPerHost); + + Instant timestampNow = Instant.now(); + + totalBytesPerHost.forEach((key, value) -> { + long totalBytesInPeriod = value.getAndSet(0); + + if (totalBytesInPeriod != 0) { + long bytesRate = totalBytesInPeriod / (timePeriodMs / 1000); + + log.debug("Total %s bytes for %s: %d bytes, %s bytes rate: %d bytes/s", + readOrWrite, + key, + totalBytesInPeriod, + readOrWrite, + bytesRate); + + Map tags = Map.of( + "host", key.host(), + "address", key.address(), + "port", key.port() + ); + + ProcessedJfrEvent processedEvent = ProcessedJfrEvent.of( + timestampNow, + "socket-" + readOrWrite + "-rate-bytes", + tags, + "bytes", + bytesRate); + + eventProcessor.processEvent(processedEvent); + } + }); + + log.debug("totalBytesPerHost (%s) after: %s", readOrWrite, totalBytesPerHost); + } + } + + @Override + public List getEventSettings() { + JfrEventSettings readEvent = JfrEventSettings.of(JDK_SOCKET_READ, this); + JfrEventSettings writeEvent = JfrEventSettings.of(JDK_SOCKET_WRITE, this); + return List.of(readEvent, writeEvent); + } +} diff --git a/src/main/java/io/perfana/jfr/influx/InfluxWriterNative.java b/src/main/java/io/perfana/jfr/influx/InfluxWriterNative.java index 7c27e13..2110a27 100644 --- a/src/main/java/io/perfana/jfr/influx/InfluxWriterNative.java +++ b/src/main/java/io/perfana/jfr/influx/InfluxWriterNative.java @@ -28,10 +28,7 @@ import java.nio.charset.StandardCharsets; import java.time.Duration; import java.time.Instant; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; +import java.util.*; import java.util.concurrent.atomic.AtomicLong; import java.util.stream.Collectors; @@ -76,9 +73,15 @@ public void writeMetricPoint(ProcessedJfrEvent event) { Instant timestamp = event.timestamp() == null ? Instant.now() : event.timestamp(); long timestampEpochNano = InfluxWriter.toEpochNs(timestamp); - Map tags = new HashMap<>(); + // tags are sorted alphabetically for better performance in InfluxDB + SortedMap tags = new TreeMap<>(); tags.put("application", config.application()); + for (Map.Entry entry : event.tags().entrySet()) { + String escapedValue = escapeTagForInflux(entry.getValue()); + tags.put(entry.getKey(), escapedValue); + } + String generatedTags = tags.entrySet().stream() .map(e -> e.getKey() + "=" + e.getValue()) .collect(Collectors.joining(",")); @@ -161,15 +164,16 @@ private void sendInfluxData(String data) { try { HttpResponse response = httpClient.send(request, HttpResponse.BodyHandlers.ofString()); - log.trace("InfluxDB response: %d %s", response.statusCode(), response.body()); - if (response.statusCode() != 204) { - log.error("Failed to send request to InfluxDB: %s", response.body()); + int statusCode = response.statusCode(); + log.trace("InfluxDB response: %d %s", statusCode, response.body()); + if (statusCode != 204) { + log.error("Failed to send request to InfluxDB: (%d) %s", statusCode, response.body()); } } catch (IOException e) { - log.error("Failed to send request to InfluxDB: %s", e.getMessage()); + log.error("Failed to send request to InfluxDB: (%s) %s", e.getClass().getSimpleName(), e.getMessage()); } catch (InterruptedException e) { Thread.currentThread().interrupt(); - log.error("Failed to send request to InfluxDB: %s", e.getMessage()); + log.error("Failed to send request to InfluxDB: (%s) %s", e.getClass().getSimpleName(), e.getMessage()); } } @@ -191,6 +195,16 @@ private static String escapeFieldForInflux(Object value) { } return "\"" + value + "\""; } + @NotNull + private static String escapeTagForInflux(String value) { + if (value == null) { + return ""; + } + if (value.isBlank()) { + return ""; + } + return value.replace(" ", "\\ ").replace(",", "\\,"); + } private static String escapeSlashesAndDoubleQuotes(String text) { return text.replace("\"", "\\\"").replace("\\", "\\\\");