diff --git a/grafana/block_verification.json b/grafana/block_verification.json index 97dfda70244..f6950c23382 100644 --- a/grafana/block_verification.json +++ b/grafana/block_verification.json @@ -3,1072 +3,61 @@ "list": [ { "builtIn": 1, - "datasource": "-- Grafana --", - "enable": true, - "hide": true, - "iconColor": "rgba(0, 211, 255, 1)", - "name": "Annotations & Alerts", - "type": "dashboard" - } - ] - }, - "editable": true, - "gnetId": null, - "graphTooltip": 0, - "id": 1, - "iteration": 1633652785661, - "links": [], - "panels": [ - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "decimals": 0, - "fieldConfig": { - "defaults": {}, - "overrides": [] - }, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 7, - "w": 6, - "x": 0, - "y": 0 - }, - "hiddenSeries": false, - "id": 4, - "legend": { - "alignAsTable": false, - "avg": false, - "current": true, - "max": false, - "min": false, - "rightSide": true, - "show": true, - "sideWidth": null, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "7.5.7", - "pointradius": 2, - "points": false, - "renderer": "flot", - "repeat": "job", - "scopedVars": { - "job": { - "selected": false, - "text": "zebrad-mainnet", - "value": "zebrad-mainnet" - } - }, - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "exemplar": true, - "expr": "zcash_chain_verified_block_height{job=\"$job\"}", - "interval": "", - "legendFormat": "committed", - "refId": "A" - }, - { - "exemplar": true, - "expr": "state_full_verifier_committed_block_height{job=\"$job\"}", - "hide": false, - "interval": "", - "legendFormat": "full verified", - "refId": "B" - }, - { - "exemplar": true, - "expr": "state_checkpoint_finalized_block_height{job=\"$job\"}", - "hide": false, - "interval": "", - "legendFormat": "checkpoint", - "refId": "C" - }, - { - "exemplar": true, - "expr": "state_finalized_block_height{job=\"$job\"}", - "hide": false, - "interval": "", - "legendFormat": "finalized", - "refId": "D" - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "Verified Block Height - $job", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "$$hashKey": "object:84", - "format": "none", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "$$hashKey": "object:85", - "format": "none", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "decimals": 0, - "fieldConfig": { - "defaults": {}, - "overrides": [] - }, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 7, - "w": 6, - "x": 6, - "y": 0 - }, - "hiddenSeries": false, - "id": 6, - "legend": { - "alignAsTable": false, - "avg": false, - "current": true, - "max": false, - "min": false, - "rightSide": true, - "show": true, - "sideWidth": null, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "7.5.7", - "pointradius": 2, - "points": false, - "renderer": "flot", - "repeatIteration": 1633652785661, - "repeatPanelId": 4, - "scopedVars": { - "job": { - "selected": false, - "text": "zebrad-mainnet-tmp", - "value": "zebrad-mainnet-tmp" - } - }, - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "exemplar": true, - "expr": "zcash_chain_verified_block_height{job=\"$job\"}", - "interval": "", - "legendFormat": "committed", - "refId": "A" - }, - { - "exemplar": true, - "expr": "state_full_verifier_committed_block_height{job=\"$job\"}", - "hide": false, - "interval": "", - "legendFormat": "full verified", - "refId": "B" - }, - { - "exemplar": true, - "expr": "state_checkpoint_finalized_block_height{job=\"$job\"}", - "hide": false, - "interval": "", - "legendFormat": "checkpoint", - "refId": "C" - }, - { - "exemplar": true, - "expr": "state_finalized_block_height{job=\"$job\"}", - "hide": false, - "interval": "", - "legendFormat": "finalized", - "refId": "D" - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "Verified Block Height - $job", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "$$hashKey": "object:84", - "format": "none", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "$$hashKey": "object:85", - "format": "none", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "decimals": 0, - "fieldConfig": { - "defaults": {}, - "overrides": [] - }, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 7, - "w": 6, - "x": 12, - "y": 0 - }, - "hiddenSeries": false, - "id": 7, - "legend": { - "alignAsTable": false, - "avg": false, - "current": true, - "max": false, - "min": false, - "rightSide": true, - "show": true, - "sideWidth": null, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "7.5.7", - "pointradius": 2, - "points": false, - "renderer": "flot", - "repeatIteration": 1633652785661, - "repeatPanelId": 4, - "scopedVars": { - "job": { - "selected": false, - "text": "zebrad-testnet", - "value": "zebrad-testnet" - } - }, - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "exemplar": true, - "expr": "zcash_chain_verified_block_height{job=\"$job\"}", - "interval": "", - "legendFormat": "committed", - "refId": "A" - }, - { - "exemplar": true, - "expr": "state_full_verifier_committed_block_height{job=\"$job\"}", - "hide": false, - "interval": "", - "legendFormat": "full verified", - "refId": "B" - }, - { - "exemplar": true, - "expr": "state_checkpoint_finalized_block_height{job=\"$job\"}", - "hide": false, - "interval": "", - "legendFormat": "checkpoint", - "refId": "C" - }, - { - "exemplar": true, - "expr": "state_finalized_block_height{job=\"$job\"}", - "hide": false, - "interval": "", - "legendFormat": "finalized", - "refId": "D" - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "Verified Block Height - $job", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "$$hashKey": "object:84", - "format": "none", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "$$hashKey": "object:85", - "format": "none", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "decimals": 0, - "fieldConfig": { - "defaults": {}, - "overrides": [] - }, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 7, - "w": 6, - "x": 18, - "y": 0 - }, - "hiddenSeries": false, - "id": 8, - "legend": { - "alignAsTable": false, - "avg": false, - "current": true, - "max": false, - "min": false, - "rightSide": true, - "show": true, - "sideWidth": null, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "7.5.7", - "pointradius": 2, - "points": false, - "renderer": "flot", - "repeatIteration": 1633652785661, - "repeatPanelId": 4, - "scopedVars": { - "job": { - "selected": false, - "text": "zebrad-testnet-tmp", - "value": "zebrad-testnet-tmp" - } - }, - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "exemplar": true, - "expr": "zcash_chain_verified_block_height{job=\"$job\"}", - "interval": "", - "legendFormat": "committed", - "refId": "A" - }, - { - "exemplar": true, - "expr": "state_full_verifier_committed_block_height{job=\"$job\"}", - "hide": false, - "interval": "", - "legendFormat": "full verified", - "refId": "B" - }, - { - "exemplar": true, - "expr": "state_checkpoint_finalized_block_height{job=\"$job\"}", - "hide": false, - "interval": "", - "legendFormat": "checkpoint", - "refId": "C" - }, - { - "exemplar": true, - "expr": "state_finalized_block_height{job=\"$job\"}", - "hide": false, - "interval": "", - "legendFormat": "finalized", - "refId": "D" - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "Verified Block Height - $job", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "$$hashKey": "object:84", - "format": "none", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "$$hashKey": "object:85", - "format": "none", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "fieldConfig": { - "defaults": {}, - "overrides": [] - }, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 7, - "w": 6, - "x": 0, - "y": 7 - }, - "hiddenSeries": false, - "id": 5, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "7.5.7", - "pointradius": 2, - "points": false, - "renderer": "flot", - "repeat": "job", - "repeatDirection": "h", - "scopedVars": { - "job": { - "selected": false, - "text": "zebrad-mainnet", - "value": "zebrad-mainnet" - } - }, - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "rate(zcash_chain_verified_block_total{job=\"$job\"}[1s])", - "interval": "", - "legendFormat": "zcash_chain_verified_block_total[1s]", - "refId": "A" - }, - { - "expr": "rate(sync_downloaded_block_count{job=\"$job\"}[1s])", - "interval": "", - "legendFormat": "sync_downloaded_block_count", - "refId": "H" - }, - { - "expr": "sync_downloads_in_flight{job=\"$job\"}", - "interval": "", - "legendFormat": "sync_downloads_in_flight", - "refId": "I" - }, - { - "expr": "rate(sync_verified_block_count{job=\"$job\"}[1s])", - "interval": "", - "legendFormat": "sync_verified_block_count", - "refId": "J" - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "Block Sync Count - $job", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "$$hashKey": "object:167", - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "$$hashKey": "object:168", - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "fieldConfig": { - "defaults": {}, - "overrides": [] - }, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 7, - "w": 6, - "x": 6, - "y": 7 - }, - "hiddenSeries": false, - "id": 9, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "7.5.7", - "pointradius": 2, - "points": false, - "renderer": "flot", - "repeatDirection": "h", - "repeatIteration": 1633652785661, - "repeatPanelId": 5, - "scopedVars": { - "job": { - "selected": false, - "text": "zebrad-mainnet-tmp", - "value": "zebrad-mainnet-tmp" - } - }, - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "rate(zcash_chain_verified_block_total{job=\"$job\"}[1s])", - "interval": "", - "legendFormat": "zcash_chain_verified_block_total[1s]", - "refId": "A" - }, - { - "expr": "rate(sync_downloaded_block_count{job=\"$job\"}[1s])", - "interval": "", - "legendFormat": "sync_downloaded_block_count", - "refId": "H" - }, - { - "expr": "sync_downloads_in_flight{job=\"$job\"}", - "interval": "", - "legendFormat": "sync_downloads_in_flight", - "refId": "I" - }, - { - "expr": "rate(sync_verified_block_count{job=\"$job\"}[1s])", - "interval": "", - "legendFormat": "sync_verified_block_count", - "refId": "J" - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "Block Sync Count - $job", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "$$hashKey": "object:167", - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "$$hashKey": "object:168", - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "fieldConfig": { - "defaults": {}, - "overrides": [] - }, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 7, - "w": 6, - "x": 12, - "y": 7 - }, - "hiddenSeries": false, - "id": 10, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "7.5.7", - "pointradius": 2, - "points": false, - "renderer": "flot", - "repeatDirection": "h", - "repeatIteration": 1633652785661, - "repeatPanelId": 5, - "scopedVars": { - "job": { - "selected": false, - "text": "zebrad-testnet", - "value": "zebrad-testnet" - } - }, - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "rate(zcash_chain_verified_block_total{job=\"$job\"}[1s])", - "interval": "", - "legendFormat": "zcash_chain_verified_block_total[1s]", - "refId": "A" - }, - { - "expr": "rate(sync_downloaded_block_count{job=\"$job\"}[1s])", - "interval": "", - "legendFormat": "sync_downloaded_block_count", - "refId": "H" - }, - { - "expr": "sync_downloads_in_flight{job=\"$job\"}", - "interval": "", - "legendFormat": "sync_downloads_in_flight", - "refId": "I" - }, - { - "expr": "rate(sync_verified_block_count{job=\"$job\"}[1s])", - "interval": "", - "legendFormat": "sync_verified_block_count", - "refId": "J" - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "Block Sync Count - $job", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "$$hashKey": "object:167", - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "$$hashKey": "object:168", - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "fieldConfig": { - "defaults": {}, - "overrides": [] - }, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 7, - "w": 6, - "x": 18, - "y": 7 - }, - "hiddenSeries": false, - "id": 11, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "7.5.7", - "pointradius": 2, - "points": false, - "renderer": "flot", - "repeatDirection": "h", - "repeatIteration": 1633652785661, - "repeatPanelId": 5, - "scopedVars": { - "job": { - "selected": false, - "text": "zebrad-testnet-tmp", - "value": "zebrad-testnet-tmp" - } - }, - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "rate(zcash_chain_verified_block_total{job=\"$job\"}[1s])", - "interval": "", - "legendFormat": "zcash_chain_verified_block_total[1s]", - "refId": "A" - }, - { - "expr": "rate(sync_downloaded_block_count{job=\"$job\"}[1s])", - "interval": "", - "legendFormat": "sync_downloaded_block_count", - "refId": "H" - }, - { - "expr": "sync_downloads_in_flight{job=\"$job\"}", - "interval": "", - "legendFormat": "sync_downloads_in_flight", - "refId": "I" + "datasource": { + "type": "datasource", + "uid": "grafana" }, - { - "expr": "rate(sync_verified_block_count{job=\"$job\"}[1s])", - "interval": "", - "legendFormat": "sync_verified_block_count", - "refId": "J" - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "Block Sync Count - $job", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "$$hashKey": "object:167", - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "target": { + "limit": 100, + "matchAny": false, + "tags": [], + "type": "dashboard" }, - { - "$$hashKey": "object:168", - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null + "type": "dashboard" } - }, + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "id": 1, + "links": [], + "liveNow": false, + "panels": [ { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, - "datasource": null, - "fieldConfig": { - "defaults": {}, - "overrides": [] + "datasource": { + "type": "prometheus", + "uid": "rpR76-qMz" }, + "decimals": 0, "fill": 1, "fillGradient": 0, "gridPos": { "h": 7, "w": 6, "x": 0, - "y": 14 + "y": 0 }, "hiddenSeries": false, - "id": 2, + "id": 4, "legend": { + "alignAsTable": false, "avg": false, - "current": false, + "current": true, "max": false, "min": false, + "rightSide": true, "show": true, "total": false, - "values": false + "values": true }, "lines": true, "linewidth": 1, @@ -1077,177 +66,147 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "7.5.7", + "pluginVersion": "9.1.6", "pointradius": 2, "points": false, "renderer": "flot", "repeat": "job", - "repeatDirection": "h", - "scopedVars": { - "job": { - "selected": false, - "text": "zebrad-mainnet", - "value": "zebrad-mainnet" - } - }, "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { - "expr": "rate(zcash_chain_verified_block_total{job=\"$job\"}[1s])", + "datasource": { + "type": "prometheus", + "uid": "rpR76-qMz" + }, + "editorMode": "code", + "exemplar": true, + "expr": "state_full_verifier_committed_block_height{job=\"$job\"}", + "hide": false, "interval": "", - "legendFormat": "zcash_chain_verified_block_total[1s]", - "refId": "A" + "legendFormat": "full verified", + "range": true, + "refId": "B" }, { - "expr": "rate(gossip_downloaded_block_count{job=\"$job\"}[1s])", + "datasource": { + "type": "prometheus", + "uid": "rpR76-qMz" + }, + "editorMode": "code", + "exemplar": true, + "expr": "state_checkpoint_finalized_block_height{job=\"$job\"}", + "hide": false, "interval": "", - "legendFormat": "gossip_downloaded_block_count[1s]", + "legendFormat": "checkpoint verified", + "range": true, "refId": "C" }, { - "expr": "rate(gossip_verified_block_count{job=\"$job\"}[1s])", - "interval": "", - "legendFormat": "gossip_verified_block_count[1s]", - "refId": "D" + "datasource": { + "type": "prometheus", + "uid": "rpR76-qMz" + }, + "editorMode": "code", + "expr": "state_memory_queued_max_height{job=\"$job\"}", + "hide": false, + "legendFormat": "full queued max", + "range": true, + "refId": "E" }, { - "expr": "gossip_queued_block_count{job=\"$job\"}", - "interval": "", - "legendFormat": "gossip_queued_block_count", - "refId": "E" - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "Block Gossip Count - $job", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ + "datasource": { + "type": "prometheus", + "uid": "rpR76-qMz" + }, + "editorMode": "code", + "expr": "state_memory_queued_min_height{job=\"$job\"}", + "hide": false, + "legendFormat": "full queued min", + "range": true, + "refId": "I" + }, { - "$$hashKey": "object:252", - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true + "datasource": { + "type": "prometheus", + "uid": "rpR76-qMz" + }, + "editorMode": "code", + "expr": "state_checkpoint_queued_max_height{job=\"$job\"}", + "hide": false, + "legendFormat": "checkpoint queued max", + "range": true, + "refId": "F" }, { - "$$hashKey": "object:253", - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": null, - "fieldConfig": { - "defaults": {}, - "overrides": [] - }, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 7, - "w": 6, - "x": 6, - "y": 14 - }, - "hiddenSeries": false, - "id": 12, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "7.5.7", - "pointradius": 2, - "points": false, - "renderer": "flot", - "repeatDirection": "h", - "repeatIteration": 1633652785661, - "repeatPanelId": 2, - "scopedVars": { - "job": { - "selected": false, - "text": "zebrad-mainnet-tmp", - "value": "zebrad-mainnet-tmp" - } - }, - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ + "datasource": { + "type": "prometheus", + "uid": "rpR76-qMz" + }, + "editorMode": "code", + "expr": "state_checkpoint_queued_min_height{job=\"$job\"}", + "hide": false, + "legendFormat": "checkpoint queued min", + "range": true, + "refId": "J" + }, { - "expr": "rate(zcash_chain_verified_block_total{job=\"$job\"}[1s])", - "interval": "", - "legendFormat": "zcash_chain_verified_block_total[1s]", - "refId": "A" + "datasource": { + "type": "prometheus", + "uid": "rpR76-qMz" + }, + "editorMode": "code", + "expr": "state_memory_sent_block_height{job=\"$job\"}", + "hide": false, + "legendFormat": "full sent", + "range": true, + "refId": "H" }, { - "expr": "rate(gossip_downloaded_block_count{job=\"$job\"}[1s])", - "interval": "", - "legendFormat": "gossip_downloaded_block_count[1s]", - "refId": "C" + "datasource": { + "type": "prometheus", + "uid": "rpR76-qMz" + }, + "editorMode": "code", + "expr": "state_checkpoint_sent_block_height{job=\"$job\"}", + "hide": false, + "legendFormat": "checkpoint sent", + "range": true, + "refId": "G" }, { - "expr": "rate(gossip_verified_block_count{job=\"$job\"}[1s])", + "datasource": { + "type": "prometheus", + "uid": "rpR76-qMz" + }, + "editorMode": "code", + "exemplar": true, + "expr": "zcash_chain_verified_block_height{job=\"$job\"}", "interval": "", - "legendFormat": "gossip_verified_block_count[1s]", - "refId": "D" + "legendFormat": "committed", + "range": true, + "refId": "A" }, { - "expr": "gossip_queued_block_count{job=\"$job\"}", + "datasource": { + "type": "prometheus", + "uid": "rpR76-qMz" + }, + "editorMode": "code", + "exemplar": true, + "expr": "state_finalized_block_height{job=\"$job\"}", + "hide": false, "interval": "", - "legendFormat": "gossip_queued_block_count", - "refId": "E" + "legendFormat": "finalized", + "range": true, + "refId": "D" } ], "thresholds": [], - "timeFrom": null, "timeRegions": [], - "timeShift": null, - "title": "Block Gossip Count - $job", + "title": "Verified Block Height - $job", "tooltip": { "shared": true, "sort": 0, @@ -1255,35 +214,26 @@ }, "type": "graph", "xaxis": { - "buckets": null, "mode": "time", - "name": null, "show": true, "values": [] }, "yaxes": [ { - "$$hashKey": "object:252", - "format": "short", - "label": null, + "$$hashKey": "object:84", + "format": "none", "logBase": 1, - "max": null, - "min": null, "show": true }, { - "$$hashKey": "object:253", - "format": "short", - "label": null, + "$$hashKey": "object:85", + "format": "none", "logBase": 1, - "max": null, - "min": null, "show": true } ], "yaxis": { - "align": false, - "alignLevel": null + "align": false } }, { @@ -1291,21 +241,20 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": null, - "fieldConfig": { - "defaults": {}, - "overrides": [] + "datasource": { + "type": "prometheus", + "uid": "rpR76-qMz" }, "fill": 1, "fillGradient": 0, "gridPos": { "h": 7, "w": 6, - "x": 12, - "y": 14 + "x": 0, + "y": 7 }, "hiddenSeries": false, - "id": 13, + "id": 5, "legend": { "avg": false, "current": false, @@ -1322,55 +271,61 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "7.5.7", + "pluginVersion": "9.1.6", "pointradius": 2, "points": false, "renderer": "flot", + "repeat": "job", "repeatDirection": "h", - "repeatIteration": 1633652785661, - "repeatPanelId": 2, - "scopedVars": { - "job": { - "selected": false, - "text": "zebrad-testnet", - "value": "zebrad-testnet" - } - }, "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { + "datasource": { + "type": "prometheus", + "uid": "rpR76-qMz" + }, "expr": "rate(zcash_chain_verified_block_total{job=\"$job\"}[1s])", "interval": "", "legendFormat": "zcash_chain_verified_block_total[1s]", "refId": "A" }, { - "expr": "rate(gossip_downloaded_block_count{job=\"$job\"}[1s])", + "datasource": { + "type": "prometheus", + "uid": "rpR76-qMz" + }, + "expr": "rate(sync_downloaded_block_count{job=\"$job\"}[1s])", "interval": "", - "legendFormat": "gossip_downloaded_block_count[1s]", - "refId": "C" + "legendFormat": "sync_downloaded_block_count", + "refId": "H" }, { - "expr": "rate(gossip_verified_block_count{job=\"$job\"}[1s])", + "datasource": { + "type": "prometheus", + "uid": "rpR76-qMz" + }, + "expr": "sync_downloads_in_flight{job=\"$job\"}", "interval": "", - "legendFormat": "gossip_verified_block_count[1s]", - "refId": "D" + "legendFormat": "sync_downloads_in_flight", + "refId": "I" }, { - "expr": "gossip_queued_block_count{job=\"$job\"}", + "datasource": { + "type": "prometheus", + "uid": "rpR76-qMz" + }, + "expr": "rate(sync_verified_block_count{job=\"$job\"}[1s])", "interval": "", - "legendFormat": "gossip_queued_block_count", - "refId": "E" + "legendFormat": "sync_verified_block_count", + "refId": "J" } ], "thresholds": [], - "timeFrom": null, "timeRegions": [], - "timeShift": null, - "title": "Block Gossip Count - $job", + "title": "Block Sync Count - $job", "tooltip": { "shared": true, "sort": 0, @@ -1378,35 +333,26 @@ }, "type": "graph", "xaxis": { - "buckets": null, "mode": "time", - "name": null, "show": true, "values": [] }, "yaxes": [ { - "$$hashKey": "object:252", + "$$hashKey": "object:167", "format": "short", - "label": null, "logBase": 1, - "max": null, - "min": null, "show": true }, { - "$$hashKey": "object:253", + "$$hashKey": "object:168", "format": "short", - "label": null, "logBase": 1, - "max": null, - "min": null, "show": true } ], "yaxis": { - "align": false, - "alignLevel": null + "align": false } }, { @@ -1414,21 +360,20 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": null, - "fieldConfig": { - "defaults": {}, - "overrides": [] + "datasource": { + "type": "prometheus", + "uid": "rpR76-qMz" }, "fill": 1, "fillGradient": 0, "gridPos": { "h": 7, "w": 6, - "x": 18, + "x": 0, "y": 14 }, "hiddenSeries": false, - "id": 14, + "id": 2, "legend": { "avg": false, "current": false, @@ -1445,44 +390,52 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "7.5.7", + "pluginVersion": "9.1.6", "pointradius": 2, "points": false, "renderer": "flot", + "repeat": "job", "repeatDirection": "h", - "repeatIteration": 1633652785661, - "repeatPanelId": 2, - "scopedVars": { - "job": { - "selected": false, - "text": "zebrad-testnet-tmp", - "value": "zebrad-testnet-tmp" - } - }, "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { + "datasource": { + "type": "prometheus", + "uid": "rpR76-qMz" + }, "expr": "rate(zcash_chain_verified_block_total{job=\"$job\"}[1s])", "interval": "", "legendFormat": "zcash_chain_verified_block_total[1s]", "refId": "A" }, { + "datasource": { + "type": "prometheus", + "uid": "rpR76-qMz" + }, "expr": "rate(gossip_downloaded_block_count{job=\"$job\"}[1s])", "interval": "", "legendFormat": "gossip_downloaded_block_count[1s]", "refId": "C" }, { + "datasource": { + "type": "prometheus", + "uid": "rpR76-qMz" + }, "expr": "rate(gossip_verified_block_count{job=\"$job\"}[1s])", "interval": "", "legendFormat": "gossip_verified_block_count[1s]", "refId": "D" }, { + "datasource": { + "type": "prometheus", + "uid": "rpR76-qMz" + }, "expr": "gossip_queued_block_count{job=\"$job\"}", "interval": "", "legendFormat": "gossip_queued_block_count", @@ -1490,9 +443,7 @@ } ], "thresholds": [], - "timeFrom": null, "timeRegions": [], - "timeShift": null, "title": "Block Gossip Count - $job", "tooltip": { "shared": true, @@ -1501,9 +452,7 @@ }, "type": "graph", "xaxis": { - "buckets": null, "mode": "time", - "name": null, "show": true, "values": [] }, @@ -1511,36 +460,28 @@ { "$$hashKey": "object:252", "format": "short", - "label": null, "logBase": 1, - "max": null, - "min": null, "show": true }, { "$$hashKey": "object:253", "format": "short", - "label": null, "logBase": 1, - "max": null, - "min": null, "show": true } ], "yaxis": { - "align": false, - "alignLevel": null + "align": false } } ], - "refresh": "5s", - "schemaVersion": 27, + "refresh": "1m", + "schemaVersion": 37, "style": "dark", "tags": [], "templating": { "list": [ { - "allValue": null, "current": { "selected": true, "text": [ @@ -1550,13 +491,13 @@ "$__all" ] }, - "datasource": "Prometheus-Zebra", + "datasource": { + "type": "prometheus", + "uid": "rpR76-qMz" + }, "definition": "label_values(zcash_chain_verified_block_height, job)", - "description": null, - "error": null, "hide": 0, "includeAll": true, - "label": null, "multi": true, "name": "job", "options": [], @@ -1569,7 +510,6 @@ "skipUrlSync": false, "sort": 1, "tagValuesQuery": "", - "tags": [], "tagsQuery": "", "type": "query", "useTags": false @@ -1584,5 +524,6 @@ "timezone": "", "title": "block verification", "uid": "rO_Cl5tGz", - "version": 14 + "version": 18, + "weekStart": "" } \ No newline at end of file diff --git a/zebra-state/src/service.rs b/zebra-state/src/service.rs index be1df4d7673..626b819d51f 100644 --- a/zebra-state/src/service.rs +++ b/zebra-state/src/service.rs @@ -183,8 +183,6 @@ pub(crate) struct StateService { /// /// Set to `f64::NAN` if `queued_finalized_blocks` is empty, because grafana shows NaNs /// as a break in the graph. - // - // TODO: add a similar metric for `queued_non_finalized_blocks` max_queued_finalized_height: f64, } @@ -487,7 +485,7 @@ impl StateService { metrics::gauge!( "state.checkpoint.queued.max.height", - self.max_queued_finalized_height + self.max_queued_finalized_height, ); metrics::gauge!( "state.checkpoint.queued.block.count", @@ -527,6 +525,8 @@ impl StateService { .queued_finalized_blocks .remove(&self.last_sent_finalized_block_hash) { + let last_sent_finalized_block_height = queued_block.0.height; + self.last_sent_finalized_block_hash = queued_block.0.hash; // If we've finished sending finalized blocks, ignore any repeated blocks. @@ -545,6 +545,11 @@ impl StateService { self.clear_finalized_block_queue( "block commit task exited. Is Zebra shutting down?", ); + } else { + metrics::gauge!( + "state.checkpoint.sent.block.height", + last_sent_finalized_block_height.0 as f64, + ); }; } } diff --git a/zebra-state/src/service/queued_blocks.rs b/zebra-state/src/service/queued_blocks.rs index 35cd3bbcd04..bacb532dc3f 100644 --- a/zebra-state/src/service/queued_blocks.rs +++ b/zebra-state/src/service/queued_blocks.rs @@ -2,7 +2,7 @@ use std::{ collections::{hash_map::Drain, BTreeMap, HashMap, HashSet, VecDeque}, - mem, + iter, mem, }; use tokio::sync::oneshot; @@ -182,12 +182,19 @@ impl QueuedBlocks { /// Update metrics after the queue is modified fn update_metrics(&self) { + if let Some(min_height) = self.by_height.keys().next() { + metrics::gauge!("state.memory.queued.min.height", min_height.0 as f64); + } else { + // use f64::NAN as a sentinel value for "None", because 0 is a valid height + metrics::gauge!("state.memory.queued.min.height", f64::NAN); + } if let Some(max_height) = self.by_height.keys().next_back() { metrics::gauge!("state.memory.queued.max.height", max_height.0 as f64); } else { // use f64::NAN as a sentinel value for "None", because 0 is a valid height metrics::gauge!("state.memory.queued.max.height", f64::NAN); } + metrics::gauge!("state.memory.queued.block.count", self.blocks.len() as f64); } @@ -198,7 +205,9 @@ impl QueuedBlocks { } /// Clears known_utxos, by_parent, and by_height, then drains blocks. - /// Returns all key-value pairs of blocks as an iterator + /// Returns all key-value pairs of blocks as an iterator. + /// + /// Doesn't update the metrics, because it is only used when the state is being dropped. pub fn drain(&mut self) -> Drain<'_, block::Hash, QueuedNonFinalized> { self.known_utxos.clear(); self.known_utxos.shrink_to_fit(); @@ -248,6 +257,8 @@ impl SentHashes { self.curr_buf.push_back((block.hash, block.height)); self.sent.insert(block.hash, outpoints); + + self.update_metrics_for_block(block.height); } /// Stores the finalized `block`'s hash, height, and UTXOs, so they can be used to check if a @@ -271,6 +282,8 @@ impl SentHashes { self.curr_buf.push_back((block.hash, block.height)); self.sent.insert(block.hash, outpoints); + + self.update_metrics_for_block(block.height); } /// Try to look up this UTXO in any sent block. @@ -315,10 +328,55 @@ impl SentHashes { }); self.sent.shrink_to_fit(); + + self.update_metrics_for_cache(); } /// Returns true if SentHashes contains the `hash` pub fn contains(&self, hash: &block::Hash) -> bool { self.sent.contains_key(hash) } + + /// Update sent block metrics after a block is sent. + fn update_metrics_for_block(&self, height: block::Height) { + metrics::counter!("state.memory.sent.block.count", 1); + metrics::gauge!("state.memory.sent.block.height", height.0 as f64); + + self.update_metrics_for_cache(); + } + + /// Update sent block cache metrics after the sent blocks are modified. + fn update_metrics_for_cache(&self) { + let batch_iter = || self.bufs.iter().chain(iter::once(&self.curr_buf)); + + if let Some(min_height) = batch_iter() + .flat_map(|batch| batch.front().map(|(_hash, height)| height)) + .min() + { + metrics::gauge!("state.memory.sent.cache.min.height", min_height.0 as f64); + } else { + // use f64::NAN as a sentinel value for "None", because 0 is a valid height + metrics::gauge!("state.memory.sent.cache.min.height", f64::NAN); + } + + if let Some(max_height) = batch_iter() + .flat_map(|batch| batch.back().map(|(_hash, height)| height)) + .max() + { + metrics::gauge!("state.memory.sent.cache.max.height", max_height.0 as f64); + } else { + // use f64::NAN as a sentinel value for "None", because 0 is a valid height + metrics::gauge!("state.memory.sent.cache.max.height", f64::NAN); + } + + metrics::gauge!( + "state.memory.sent.cache.block.count", + batch_iter().flatten().count() as f64, + ); + + metrics::gauge!( + "state.memory.sent.cache.batch.count", + batch_iter().count() as f64, + ); + } }