fix the proxy server backend metric error

go.mod

@@ -19,39 +19,6 @@ require (
 	sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.0
 )
 
-require (
-	github.com/beorn7/perks v1.0.1 // indirect
-	github.com/cespare/xxhash/v2 v2.1.1 // indirect
-	github.com/davecgh/go-spew v1.1.1 // indirect
-	github.com/evanphx/json-patch v4.9.0+incompatible // indirect
-	github.com/go-logr/logr v0.2.0 // indirect
-	github.com/gogo/protobuf v1.3.2 // indirect
-	github.com/google/gofuzz v1.1.0 // indirect
-	github.com/googleapis/gnostic v0.4.1 // indirect
-	github.com/imdario/mergo v0.3.5 // indirect
-	github.com/inconshreveable/mousetrap v1.0.0 // indirect
-	github.com/json-iterator/go v1.1.10 // indirect
-	github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369 // indirect
-	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
-	github.com/modern-go/reflect2 v1.0.1 // indirect
-	github.com/pkg/errors v0.9.1 // indirect
-	github.com/prometheus/client_model v0.2.0 // indirect
-	github.com/prometheus/common v0.10.0 // indirect
-	github.com/prometheus/procfs v0.2.0 // indirect
-	golang.org/x/crypto v0.0.0-20201002170205-7f63de1d35b0 // indirect
-	golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d // indirect
-	golang.org/x/sys v0.0.0-20201112073958-5cba982894dd // indirect
-	golang.org/x/text v0.3.4 // indirect
-	golang.org/x/time v0.0.0-20200630173020-3af7569d3a1e // indirect
-	google.golang.org/appengine v1.6.5 // indirect
-	google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013 // indirect
-	google.golang.org/protobuf v1.25.0 // indirect
-	gopkg.in/inf.v0 v0.9.1 // indirect
-	gopkg.in/yaml.v2 v2.2.8 // indirect
-	k8s.io/kube-openapi v0.0.0-20201113171705-d219536bb9fd // indirect
-	k8s.io/utils v0.0.0-20201110183641-67b214c5f920 // indirect
-	sigs.k8s.io/structured-merge-diff/v4 v4.1.2 // indirect
-	sigs.k8s.io/yaml v1.2.0 // indirect
-)
+require github.com/inconshreveable/mousetrap v1.0.0 // indirect
 
 replace sigs.k8s.io/apiserver-network-proxy/konnectivity-client => ./konnectivity-client

go.sum

@@ -78,6 +78,7 @@ github.com/evanphx/json-patch v4.9.0+incompatible h1:kLcOMZeuLAJvL2BPWLMIj5oaZQo
 github.com/evanphx/json-patch v4.9.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk=
 github.com/form3tech-oss/jwt-go v3.2.2+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k=
 github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
+github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4=
 github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ=
 github.com/ghodss/yaml v0.0.0-20150909031657-73d445a93680/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=
 github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=

pkg/server/backend_manager.go

@@ -25,7 +25,7 @@ import (
 	"time"
 
 	"k8s.io/klog/v2"
-	client "sigs.k8s.io/apiserver-network-proxy/konnectivity-client/proto/client"
+	"sigs.k8s.io/apiserver-network-proxy/konnectivity-client/proto/client"
 	pkgagent "sigs.k8s.io/apiserver-network-proxy/pkg/agent"
 	"sigs.k8s.io/apiserver-network-proxy/pkg/server/metrics"
 	"sigs.k8s.io/apiserver-network-proxy/proto/agent"
@@ -133,6 +133,16 @@ func (dbm *DefaultBackendManager) Backend(_ context.Context) (Backend, error) {
 	return dbm.DefaultBackendStorage.GetRandomBackend()
 }
 
+func (dbm *DefaultBackendManager) AddBackend(identifier string, idType pkgagent.IdentifierType, conn agent.AgentService_ConnectServer) Backend {
+	metrics.Metrics.BackendCountInc("DefaultBackendManager", "uid")
+	return dbm.DefaultBackendStorage.AddBackend(identifier, idType, conn)
+}
+
+func (dbm *DefaultBackendManager) RemoveBackend(identifier string, idType pkgagent.IdentifierType, conn agent.AgentService_ConnectServer) {
+	dbm.DefaultBackendStorage.RemoveBackend(identifier, idType, conn)
+	metrics.Metrics.BackendCountDec("DefaultBackendManager", "uid")
+}
+
 // DefaultBackendStorage is the default backend storage.
 type DefaultBackendStorage struct {
 	mu sync.RWMutex //protects the following
@@ -204,7 +214,6 @@ func (s *DefaultBackendStorage) AddBackend(identifier string, idType pkgagent.Id
 		return addedBackend
 	}
 	s.backends[identifier] = []*backend{addedBackend}
-	metrics.Metrics.SetBackendCount(len(s.backends))
 	s.agentIDs = append(s.agentIDs, identifier)
 	if idType == pkgagent.DefaultRoute {
 		s.defaultRouteAgentIDs = append(s.defaultRouteAgentIDs, identifier)
@@ -257,7 +266,6 @@ func (s *DefaultBackendStorage) RemoveBackend(identifier string, idType pkgagent
 	if !found {
 		klog.V(1).InfoS("Could not find connection matching identifier to remove", "connection", conn, "identifier", identifier)
 	}
-	metrics.Metrics.SetBackendCount(len(s.backends))
 }
 
 // NumBackends resturns the number of available backends

pkg/server/default_route_backend_manager.go

@@ -20,7 +20,9 @@ import (
 	"context"
 
 	"k8s.io/klog/v2"
-	"sigs.k8s.io/apiserver-network-proxy/pkg/agent"
+	pkgagent "sigs.k8s.io/apiserver-network-proxy/pkg/agent"
+	"sigs.k8s.io/apiserver-network-proxy/pkg/server/metrics"
+	"sigs.k8s.io/apiserver-network-proxy/proto/agent"
 )
 
 type DefaultRouteBackendManager struct {
@@ -32,20 +34,30 @@ var _ BackendManager = &DefaultRouteBackendManager{}
 func NewDefaultRouteBackendManager() *DefaultRouteBackendManager {
 	return &DefaultRouteBackendManager{
 		DefaultBackendStorage: NewDefaultBackendStorage(
-			[]agent.IdentifierType{agent.DefaultRoute})}
+			[]pkgagent.IdentifierType{pkgagent.DefaultRoute})}
 }
 
 // Backend tries to get a backend associating to the request destination host.
-func (dibm *DefaultRouteBackendManager) Backend(ctx context.Context) (Backend, error) {
-	dibm.mu.RLock()
-	defer dibm.mu.RUnlock()
-	if len(dibm.backends) == 0 {
+func (drbm *DefaultRouteBackendManager) Backend(ctx context.Context) (Backend, error) {
+	drbm.mu.RLock()
+	defer drbm.mu.RUnlock()
+	if len(drbm.backends) == 0 {
 		return nil, &ErrNotFound{}
 	}
-	if len(dibm.defaultRouteAgentIDs) == 0 {
+	if len(drbm.defaultRouteAgentIDs) == 0 {
 		return nil, &ErrNotFound{}
 	}
-	agentID := dibm.defaultRouteAgentIDs[dibm.random.Intn(len(dibm.defaultRouteAgentIDs))]
+	agentID := drbm.defaultRouteAgentIDs[drbm.random.Intn(len(drbm.defaultRouteAgentIDs))]
 	klog.V(4).InfoS("Picked agent as backend", "agentID", agentID)
-	return dibm.backends[agentID][0], nil
+	return drbm.backends[agentID][0], nil
+}
+
+func (drbm *DefaultRouteBackendManager) AddBackend(identifier string, idType pkgagent.IdentifierType, conn agent.AgentService_ConnectServer) Backend {
+	metrics.Metrics.BackendCountInc("DefaultRouteBackendManager", "default-route")
+	return drbm.DefaultBackendStorage.AddBackend(identifier, idType, conn)
+}
+
+func (drbm *DefaultRouteBackendManager) RemoveBackend(identifier string, idType pkgagent.IdentifierType, conn agent.AgentService_ConnectServer) {
+	drbm.DefaultBackendStorage.RemoveBackend(identifier, idType, conn)
+	metrics.Metrics.BackendCountDec("DefaultRouteBackendManager", "default-route")
 }

pkg/server/desthost_backend_manager.go

@@ -20,7 +20,9 @@ import (
 	"context"
 
 	"k8s.io/klog/v2"
-	"sigs.k8s.io/apiserver-network-proxy/pkg/agent"
+	pkgagent "sigs.k8s.io/apiserver-network-proxy/pkg/agent"
+	"sigs.k8s.io/apiserver-network-proxy/pkg/server/metrics"
+	"sigs.k8s.io/apiserver-network-proxy/proto/agent"
 )
 
 type DestHostBackendManager struct {
@@ -32,23 +34,47 @@ var _ BackendManager = &DestHostBackendManager{}
 func NewDestHostBackendManager() *DestHostBackendManager {
 	return &DestHostBackendManager{
 		DefaultBackendStorage: NewDefaultBackendStorage(
-			[]agent.IdentifierType{agent.IPv4, agent.IPv6, agent.Host})}
+			[]pkgagent.IdentifierType{pkgagent.IPv4, pkgagent.IPv6, pkgagent.Host})}
 }
 
 // Backend tries to get a backend associating to the request destination host.
-func (dibm *DestHostBackendManager) Backend(ctx context.Context) (Backend, error) {
-	dibm.mu.RLock()
-	defer dibm.mu.RUnlock()
-	if len(dibm.backends) == 0 {
+func (dhbm *DestHostBackendManager) Backend(ctx context.Context) (Backend, error) {
+	dhbm.mu.RLock()
+	defer dhbm.mu.RUnlock()
+	if len(dhbm.backends) == 0 {
 		return nil, &ErrNotFound{}
 	}
 	destHost := ctx.Value(destHost).(string)
 	if destHost != "" {
-		bes, exist := dibm.backends[destHost]
+		bes, exist := dhbm.backends[destHost]
 		if exist && len(bes) > 0 {
 			klog.V(5).InfoS("Get the backend through the DestHostBackendManager", "destHost", destHost)
-			return dibm.backends[destHost][0], nil
+			return dhbm.backends[destHost][0], nil
 		}
 	}
 	return nil, &ErrNotFound{}
 }
+
+func (dhbm *DestHostBackendManager) AddBackend(identifier string, idType pkgagent.IdentifierType, conn agent.AgentService_ConnectServer) Backend {
+	switch idType {
+	case pkgagent.IPv4:
+		metrics.Metrics.BackendCountInc("DestHostBackendManager", "ipv4")
+	case pkgagent.IPv6:
+		metrics.Metrics.BackendCountInc("DestHostBackendManager", "ipv6")
+	case pkgagent.Host:
+		metrics.Metrics.BackendCountInc("DestHostBackendManager", "host")
+	}
+	return dhbm.DefaultBackendStorage.AddBackend(identifier, idType, conn)
+}
+
+func (dhbm *DestHostBackendManager) RemoveBackend(identifier string, idType pkgagent.IdentifierType, conn agent.AgentService_ConnectServer) {
+	dhbm.DefaultBackendStorage.RemoveBackend(identifier, idType, conn)
+	switch idType {
+	case pkgagent.IPv4:
+		metrics.Metrics.BackendCountDec("DestHostBackendManager", "ipv4")
+	case pkgagent.IPv6:
+		metrics.Metrics.BackendCountDec("DestHostBackendManager", "ipv6")
+	case pkgagent.Host:
+		metrics.Metrics.BackendCountDec("DestHostBackendManager", "host")
+	}
+}

pkg/server/metrics/metrics.go

@@ -99,7 +99,10 @@ func newServerMetrics() *ServerMetrics {
 			Name:      "ready_backend_connections",
 			Help:      "Number of konnectivity agent connected to the proxy server",
 		},
-		[]string{},
+		[]string{
+			"manager",
+			"idType",
+		},
 	)
 	pendingDials := prometheus.NewGaugeVec(
 		prometheus.GaugeOpts{
@@ -164,6 +167,16 @@ func (a *ServerMetrics) SetBackendCount(count int) {
 	a.backend.WithLabelValues().Set(float64(count))
 }
 
+// BackendCountInc increments a new backend connection.
+func (a *ServerMetrics) BackendCountInc(manager string, idType string) {
+	a.backend.WithLabelValues(manager, idType).Inc()
+}
+
+// BackendCountInc decrements a finished backend connection.
+func (a *ServerMetrics) BackendCountDec(manager string, idType string) {
+	a.backend.WithLabelValues(manager, idType).Desc()
+}
+
 // SetPendingDialCount sets the number of pending dials.
 func (a *ServerMetrics) SetPendingDialCount(count int) {
 	a.pendingDials.WithLabelValues().Set(float64(count))