Add teleport_reverse_tunnels_connected Prometheus metric (#9698)

Adds teleport_reverse_tunnels_connected Prometheus metric which tracks reverse tunnels connected to the proxy server by type.

* Update prometheus help

Co-authored-by: Paul Gottschling <[email protected]>

* Update metrics wording

Co-authored-by: Paul Gottschling <[email protected]>

docs/pages/setup/reference/metrics.mdx

@@ -110,6 +110,7 @@ Now you can see the monitoring information by visiting several endpoints:
 | `server_interactive_sessions_total` | gauge | Teleport | Number of active sessions. |
 | `teleport_build_info` | gauge | Teleport | Provides build information of Teleport including gitref (git describe --long --tags), Go version, and Teleport version. The value of this gauge will always be 1. |
 | `teleport_registered_servers` | gauge | Teleport Auth | The number of Teleport servers (a server consists of one or more Teleport services) that have connected to the Teleport cluster, including the Teleport version. After disconnecting, a Teleport server has a TTL of 10 minutes, so this value will include servers that have recently disconnected but have not reached their TTL. |
+| `teleport_reverse_tunnels_connected` | gauge | Teleport Proxy | Number of reverse SSH tunnels connected to the Teleport Proxy Service by Teleport instances. |
 | `trusted_clusters` | gauge | Teleport | Number of tunnels per state. |
 | `tx` | counter | Teleport | Number of bytes transmitted. |
 | `user_login_total` | counter | Teleport Auth | Number of user logins. |

lib/reversetunnel/localsite.go

@@ -420,6 +420,8 @@ func (s *localSite) handleHeartbeat(rconn *remoteConn, ch ssh.Channel, reqC <-ch
 				if len(current) > 0 {
 					rconn.updateProxies(current)
 				}
+				reverseSSHTunnels.WithLabelValues(rconn.tunnelType).Inc()
+				defer reverseSSHTunnels.WithLabelValues(rconn.tunnelType).Dec()
 				firstHeartbeat = false
 			}
 			var timeSent time.Time
@@ -558,6 +560,14 @@ var (
 			Help: "Number of missing SSH tunnels",
 		},
 	)
+	reverseSSHTunnels = prometheus.NewGaugeVec(
+		prometheus.GaugeOpts{
+			Namespace: teleport.MetricNamespace,
+			Name:      teleport.MetricReverseSSHTunnels,
+			Help:      "Number of reverse SSH tunnels connected to the Teleport Proxy Service by Teleport instances",
+		},
+		[]string{teleport.TagType},
+	)
 
-	localClusterCollectors = []prometheus.Collector{missingSSHTunnels}
+	localClusterCollectors = []prometheus.Collector{missingSSHTunnels, reverseSSHTunnels}
 )

metrics.go

@@ -186,6 +186,9 @@ const (
 	// MetricRegisteredServers tracks the number of Teleport servers that have successfully registered with the Teleport cluster and have not reached the end of their ttl
 	MetricRegisteredServers = "registered_servers"
 
+	// MetricReverseSSHTunnels defines the number of connected SSH reverse tunnels to the proxy
+	MetricReverseSSHTunnels = "reverse_tunnels_connected"
+
 	// TagRange is a tag specifying backend requests
 	TagRange = "range"
 
@@ -209,4 +212,7 @@ const (
 
 	// TagGoVersion is a prometheus label for version of Go used to build Teleport
 	TagGoVersion = "goversion"
+
+	// TagType is a prometheus label for type of resource or tunnel connected
+	TagType = "type"
 )