From d2c42125aad73fdcb063e0cbc45417cf88989d03 Mon Sep 17 00:00:00 2001 From: Bruno Cadonna Date: Thu, 27 Feb 2020 18:48:53 +0100 Subject: [PATCH] Adapt docs about metrics of Streams according to KIP-444 (#8171) Adapts the docs about metrics of Streams according to https://cwiki.apache.org/confluence/display/KAFKA/KIP-444%253A+Augment+metrics+for+Kafka+Streams Reviewers: Boyang Chen , Guozhang Wang --- docs/ops.html | 520 ++++++++++++++++++++------------------------------ 1 file changed, 203 insertions(+), 317 deletions(-) diff --git a/docs/ops.html b/docs/ops.html index 0fd0ba18f4a17..85699299c7c83 100644 --- a/docs/ops.html +++ b/docs/ops.html @@ -1600,9 +1600,8 @@

Connect Monitoring

Streams Monitoring

- A Kafka Streams instance contains all the producer and consumer metrics as well as additional metrics specific to streams. - By default Kafka Streams has metrics with two recording levels: debug and info. The debug level records all metrics, while - the info level records only the thread-level metrics. + A Kafka Streams instance contains all the producer and consumer metrics as well as additional metrics specific to Streams. + By default Kafka Streams has metrics with two recording levels: debug and info.

Note that the metrics have a 4-layer hierarchy. At the top level there are client-level metrics for each started @@ -1617,7 +1616,7 @@

Streams Mo
metrics.recording.level="info"
Client Metrics
-All the following metrics have a recording level of info: +All of the following metrics have a recording level of info: @@ -1654,7 +1653,7 @@
Thread Metrics
-All the following metrics have a recording level of info: +All of the following metrics have a recording level of info:
@@ -1662,121 +1661,112 @@
Task Metrics
-All the following metrics have a recording level of debug: +All of the following metrics have a recording level of debug, except for metrics +dropped-records-rate and dropped-records-total which have a recording level of info:
@@ -1784,41 +1774,83 @@
+ + + + + + + + + + + + + + + + + + + + - - + + - - + + - - + + - + - - + + - - + + + + + + + + + + + + + + + + + + + + + +
Description Mbean name
process-latency-avgThe average execution time in ns, for processing.kafka.streams:type=stream-task-metrics,thread-id=([-.\w]+),task-id=([-.\w]+)
process-latency-maxThe maximum execution time in ns, for processing.kafka.streams:type=stream-task-metrics,thread-id=([-.\w]+),task-id=([-.\w]+)
process-rateThe average number of processed records per second across all source processor nodes of this task.kafka.streams:type=stream-task-metrics,thread-id=([-.\w]+),task-id=([-.\w]+)
process-totalThe total number of processed records across all source processor nodes of this task.kafka.streams:type=stream-task-metrics,thread-id=([-.\w]+),task-id=([-.\w]+)
commit-latency-avgThe average commit time in ns for this task. kafka.streams:type=stream-task-metrics,client-id=([-.\w]+),task-id=([-.\w]+)The average execution time in ns, for committing.kafka.streams:type=stream-task-metrics,thread-id=([-.\w]+),task-id=([-.\w]+)
commit-latency-maxThe maximum commit time in ns for this task. kafka.streams:type=stream-task-metrics,client-id=([-.\w]+),task-id=([-.\w]+)The maximum execution time in ns, for committing.kafka.streams:type=stream-task-metrics,thread-id=([-.\w]+),task-id=([-.\w]+)
commit-rateThe average number of commit calls per second. kafka.streams:type=stream-task-metrics,client-id=([-.\w]+),task-id=([-.\w]+)The average number of commit calls per second.kafka.streams:type=stream-task-metrics,thread-id=([-.\w]+),task-id=([-.\w]+)
commit-total The total number of commit calls. kafka.streams:type=stream-task-metrics,client-id=([-.\w]+),task-id=([-.\w]+)kafka.streams:type=stream-task-metrics,thread-id=([-.\w]+),task-id=([-.\w]+)
record-lateness-avgThe average observed lateness of records.kafka.streams:type=stream-task-metrics,client-id=([-.\w]+),task-id=([-.\w]+)The average observed lateness of records (stream time - record timestamp).kafka.streams:type=stream-task-metrics,thread-id=([-.\w]+),task-id=([-.\w]+)
record-lateness-maxThe max observed lateness of records.kafka.streams:type=stream-task-metrics,client-id=([-.\w]+),task-id=([-.\w]+)The max observed lateness of records (stream time - record timestamp).kafka.streams:type=stream-task-metrics,thread-id=([-.\w]+),task-id=([-.\w]+)
enforced-processing-rateThe average number of enforced processings per second.kafka.streams:type=stream-task-metrics,thread-id=([-.\w]+),task-id=([-.\w]+)
enforced-processing-totalThe total number enforced processings.kafka.streams:type=stream-task-metrics,thread-id=([-.\w]+),task-id=([-.\w]+)
dropped-records-rateThe average number of records dropped within this task.kafka.streams:type=stream-task-metrics,thread-id=([-.\w]+),task-id=([-.\w]+)
dropped-records-totalThe total number of records dropped within this task.kafka.streams:type=stream-task-metrics,thread-id=([-.\w]+),task-id=([-.\w]+)
Processor Node Metrics
- All the following metrics have a recording level of debug: + The following metrics are only available on certain types of nodes, i.e., process-rate and process-total are + only available for source processor nodes and suppression-emit-rate and suppression-emit-total are only available + for suppression operation nodes. All of the metrics have a recording level of debug: @@ -1826,126 +1858,43 @@
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + - - + + - - + +
Description Mbean name
process-latency-avgThe average process execution time in ns. kafka.streams:type=stream-processor-node-metrics,client-id=([-.\w]+),task-id=([-.\w]+),processor-node-id=([-.\w]+)
process-latency-maxThe maximum process execution time in ns. kafka.streams:type=stream-processor-node-metrics,client-id=([-.\w]+),task-id=([-.\w]+),processor-node-id=([-.\w]+)
punctuate-latency-avgThe average punctuate execution time in ns. kafka.streams:type=stream-processor-node-metrics,client-id=([-.\w]+),task-id=([-.\w]+),processor-node-id=([-.\w]+)
punctuate-latency-maxThe maximum punctuate execution time in ns. kafka.streams:type=stream-processor-node-metrics,client-id=([-.\w]+),task-id=([-.\w]+),processor-node-id=([-.\w]+)
create-latency-avgThe average create execution time in ns. kafka.streams:type=stream-processor-node-metrics,client-id=([-.\w]+),task-id=([-.\w]+),processor-node-id=([-.\w]+)
create-latency-maxThe maximum create execution time in ns. kafka.streams:type=stream-processor-node-metrics,client-id=([-.\w]+),task-id=([-.\w]+),processor-node-id=([-.\w]+)
destroy-latency-avgThe average destroy execution time in ns. kafka.streams:type=stream-processor-node-metrics,client-id=([-.\w]+),task-id=([-.\w]+),processor-node-id=([-.\w]+)
destroy-latency-maxThe maximum destroy execution time in ns. kafka.streams:type=stream-processor-node-metrics,client-id=([-.\w]+),task-id=([-.\w]+),processor-node-id=([-.\w]+)
process-rateThe average number of process operations per second. kafka.streams:type=stream-processor-node-metrics,client-id=([-.\w]+),task-id=([-.\w]+),processor-node-id=([-.\w]+)The average number of records processed by a source processor node per second.kafka.streams:type=stream-processor-node-metrics,thread-id=([-.\w]+),task-id=([-.\w]+),processor-node-id=([-.\w]+)
process-totalThe total number of process operations called. kafka.streams:type=stream-processor-node-metrics,client-id=([-.\w]+),task-id=([-.\w]+),processor-node-id=([-.\w]+)
punctuate-rateThe average number of punctuate operations per second. kafka.streams:type=stream-processor-node-metrics,client-id=([-.\w]+),task-id=([-.\w]+),processor-node-id=([-.\w]+)
punctuate-totalThe total number of punctuate operations called. kafka.streams:type=stream-processor-node-metrics,client-id=([-.\w]+),task-id=([-.\w]+),processor-node-id=([-.\w]+)
create-rateThe average number of create operations per second. kafka.streams:type=stream-processor-node-metrics,client-id=([-.\w]+),task-id=([-.\w]+),processor-node-id=([-.\w]+)
create-totalThe total number of create operations called. kafka.streams:type=stream-processor-node-metrics,client-id=([-.\w]+),task-id=([-.\w]+),processor-node-id=([-.\w]+)
destroy-rateThe average number of destroy operations per second. kafka.streams:type=stream-processor-node-metrics,client-id=([-.\w]+),task-id=([-.\w]+),processor-node-id=([-.\w]+)
destroy-totalThe total number of destroy operations called. kafka.streams:type=stream-processor-node-metrics,client-id=([-.\w]+),task-id=([-.\w]+),processor-node-id=([-.\w]+)
forward-rateThe average rate of records being forwarded downstream, from source nodes only, per second. kafka.streams:type=stream-processor-node-metrics,client-id=([-.\w]+),task-id=([-.\w]+),processor-node-id=([-.\w]+)
forward-totalThe total number of of records being forwarded downstream, from source nodes only. kafka.streams:type=stream-processor-node-metrics,client-id=([-.\w]+),task-id=([-.\w]+),processor-node-id=([-.\w]+)The total number of records processed by a source processor node per second.kafka.streams:type=stream-processor-node-metrics,thread-id=([-.\w]+),task-id=([-.\w]+),processor-node-id=([-.\w]+)
suppression-emit-rate - The rate at which records that have been emitted downstream from suppression operation nodes. - Compare with the process-rate metric to determine how many updates are being suppressed. - kafka.streams:type=stream-processor-node-metrics,client-id=([-.\w]+),task-id=([-.\w]+),processor-node-id=([-.\w]+)The rate at which records that have been emitted downstream from suppression operation nodes.kafka.streams:type=stream-processor-node-metrics,thread-id=([-.\w]+),task-id=([-.\w]+),processor-node-id=([-.\w]+)
suppression-emit-total - The total number of records that have been emitted downstream from suppression operation nodes. - Compare with the process-total metric to determine how many updates are being suppressed. - kafka.streams:type=stream-processor-node-metrics,client-id=([-.\w]+),task-id=([-.\w]+),processor-node-id=([-.\w]+)The total number of records that have been emitted downstream from suppression operation nodes.kafka.streams:type=stream-processor-node-metrics,thread-id=([-.\w]+),task-id=([-.\w]+),processor-node-id=([-.\w]+)
State Store Metrics
- All the following metrics have a recording level of debug. Note that the store-scope value is specified in StoreSupplier#metricsScope() for user's customized + All of the following metrics have a recording level of debug. Note that the store-scope value is specified in StoreSupplier#metricsScope() for user's customized state stores; for built-in state stores, currently we have:
  • in-memory-state
  • in-memory-lru-state
  • in-memory-window-state
  • +
  • in-memory-suppression (for suppression buffers)
  • rocksdb-state (for RocksDB backed key-value store)
  • rocksdb-window-state (for RocksDB backed window store)
  • rocksdb-session-state (for RocksDB backed session store)
+ Metrics suppression-buffer-size-avg, suppression-buffer-size-max, suppression-buffer-count-avg, and suppression-buffer-count-max + are only available for suppression buffers. All other metrics are not available for suppression buffers. @@ -1957,188 +1906,163 @@
RocksDB Metrics
- All the following metrics have a recording level of debug. + All of the following metrics have a recording level of debug. The metrics are collected every minute from the RocksDB state stores. If a state store consists of multiple RocksDB instances as it is the case for aggregations over time and session windows, each metric reports an aggregation over the RocksDB instances of the state store. @@ -2159,88 +2083,88 @@
Record Cache Metrics
- All the following metrics have a recording level of debug: + All of the following metrics have a recording level of debug:
@@ -2250,57 +2174,19 @@
Suppression Buffer Metrics
- All the following metrics have a recording level of debug: - -
- - - - - - - - - - - - - - - - - - - - + + + - - - + + + - - - + + +
suppression-buffer-size-currentThe current total size, in bytes, of the buffered data.kafka.streams:type=stream-buffer-metrics,client-id=([-.\w]+),task-id=([-.\w]+),buffer-id=([-.\w]+)
suppression-buffer-size-avgThe average total size, in bytes, of the buffered data over the sampling window.kafka.streams:type=stream-buffer-metrics,client-id=([-.\w]+),task-id=([-.\w]+),buffer-id=([-.\w]+)
suppression-buffer-size-maxThe maximum total size, in bytes, of the buffered data over the sampling window.kafka.streams:type=stream-buffer-metrics,client-id=([-.\w]+),task-id=([-.\w]+),buffer-id=([-.\w]+)
suppression-buffer-count-currentThe current number of records buffered.kafka.streams:type=stream-buffer-metrics,client-id=([-.\w]+),task-id=([-.\w]+),buffer-id=([-.\w]+)hit-ratio-avgThe average cache hit ratio defined as the ratio of cache read hits over the total cache read requests.kafka.streams:type=stream-record-cache-metrics,thread-id=([-.\w]+),task-id=([-.\w]+),record-cache-id=([-.\w]+)
suppression-buffer-size-avgThe average number of records buffered over the sampling window.kafka.streams:type=stream-buffer-metrics,client-id=([-.\w]+),task-id=([-.\w]+),buffer-id=([-.\w]+)hit-ratio-minThe mininum cache hit ratio.kafka.streams:type=stream-record-cache-metrics,thread-id=([-.\w]+),task-id=([-.\w]+),record-cache-id=([-.\w]+)
suppression-buffer-size-maxThe maximum number of records buffered over the sampling window.kafka.streams:type=stream-buffer-metrics,client-id=([-.\w]+),task-id=([-.\w]+),buffer-id=([-.\w]+)hit-ratio-maxThe maximum cache hit ratio.kafka.streams:type=stream-record-cache-metrics,thread-id=([-.\w]+),task-id=([-.\w]+),record-cache-id=([-.\w]+)