Merge pull request #1856 from panslava/add-l4-ilb-dualstack-metrics

Add L4 ILB Dual-Stack Metrics

pkg/l4lb/l4controller.go

@@ -487,13 +487,21 @@ func (l4c *L4Controller) publishMetrics(result *loadbalancers.L4ILBSyncResult, n
 	case loadbalancers.SyncTypeCreate, loadbalancers.SyncTypeUpdate:
 		klog.V(6).Infof("Internal L4 Loadbalancer for Service %s ensured, updating its state %v in metrics cache", namespacedName, result.MetricsState)
 		l4c.ctx.ControllerMetrics.SetL4ILBService(namespacedName, result.MetricsState)
+		if l4c.enableDualStack {
+			klog.V(6).Infof("Internal L4 DualStack Loadbalancer for Service %s ensured, updating its state %v in metrics cache", namespacedName, result.MetricsState)
+			l4c.ctx.ControllerMetrics.SetL4ILBDualStackService(namespacedName, result.DualStackMetricsState)
+		}
 		l4metrics.PublishILBSyncMetrics(result.Error == nil, result.SyncType, result.GCEResourceInError, utils.GetErrorType(result.Error), result.StartTime)
 
 	case loadbalancers.SyncTypeDelete:
 		// if service is successfully deleted, remove it from cache
 		if result.Error == nil {
 			klog.V(6).Infof("Internal L4 Loadbalancer for Service %s deleted, removing its state from metrics cache", namespacedName)
 			l4c.ctx.ControllerMetrics.DeleteL4ILBService(namespacedName)
+			if l4c.enableDualStack {
+				klog.V(6).Infof("Internal L4 Loadbalancer for Service %s deleted, removing its state from metrics cache", namespacedName)
+				l4c.ctx.ControllerMetrics.DeleteL4ILBDualStackService(namespacedName)
+			}
 		}
 		l4metrics.PublishILBSyncMetrics(result.Error == nil, result.SyncType, result.GCEResourceInError, utils.GetErrorType(result.Error), result.StartTime)
 	default:

pkg/loadbalancers/l4.go

@@ -60,13 +60,14 @@ type L4 struct {
 // L4ILBSyncResult contains information about the outcome of an L4 ILB sync. It stores the list of resource name annotations,
 // sync error, the GCE resource that hit the error along with the error type, metrics and more fields.
 type L4ILBSyncResult struct {
-	Annotations        map[string]string
-	Error              error
-	GCEResourceInError string
-	Status             *corev1.LoadBalancerStatus
-	MetricsState       metrics.L4ILBServiceState
-	SyncType           string
-	StartTime          time.Time
+	Annotations           map[string]string
+	Error                 error
+	GCEResourceInError    string
+	Status                *corev1.LoadBalancerStatus
+	MetricsState          metrics.L4ILBServiceState
+	DualStackMetricsState metrics.L4ILBDualStackServiceState
+	SyncType              string
+	StartTime             time.Time
 }
 
 type L4ILBParams struct {
@@ -278,10 +279,13 @@ func (l4 *L4) getFRNameWithProtocol(protocol string) string {
 // EnsureInternalLoadBalancer ensures that all GCE resources for the given loadbalancer service have
 // been created. It returns a LoadBalancerStatus with the updated ForwardingRule IP address.
 func (l4 *L4) EnsureInternalLoadBalancer(nodeNames []string, svc *corev1.Service) *L4ILBSyncResult {
+	l4.Service = svc
+
 	result := &L4ILBSyncResult{
-		Annotations: make(map[string]string),
-		StartTime:   time.Now(),
-		SyncType:    SyncTypeCreate,
+		Annotations:           make(map[string]string),
+		StartTime:             time.Now(),
+		SyncType:              SyncTypeCreate,
+		DualStackMetricsState: l4.getInitialDualStackMetricsState(),
 	}
 
 	// If service already has an IP assigned, treat it as an update instead of a new Loadbalancer.
@@ -291,8 +295,6 @@ func (l4 *L4) EnsureInternalLoadBalancer(nodeNames []string, svc *corev1.Service
 		result.SyncType = SyncTypeUpdate
 	}
 
-	l4.Service = svc
-
 	hcLink := l4.provideHealthChecks(nodeNames, result)
 	if result.Error != nil {
 		return result
@@ -382,6 +384,9 @@ func (l4 *L4) EnsureInternalLoadBalancer(nodeNames []string, svc *corev1.Service
 	if options.SubnetName != "" {
 		result.MetricsState.EnabledCustomSubnet = true
 	}
+	if l4.enableDualStack {
+		result.DualStackMetricsState.Status = metrics.StatusSuccess
+	}
 	return result
 }
 
@@ -527,6 +532,26 @@ func (l4 *L4) hasAnnotation(annotationKey string) bool {
 	return false
 }
 
+func (l4 *L4) getInitialDualStackMetricsState() metrics.L4ILBDualStackServiceState {
+	// Always init stats with error, and update with Success when service was provisioned
+	state := metrics.L4ILBDualStackServiceState{
+		Status: metrics.StatusError,
+	}
+
+	var ipFamiliesStrings []string
+	for _, ipFamily := range l4.Service.Spec.IPFamilies {
+		ipFamiliesStrings = append(ipFamiliesStrings, string(ipFamily))
+	}
+	state.IPFamilies = strings.Join(ipFamiliesStrings, ",")
+
+	state.IPFamilyPolicy = ""
+	if l4.Service.Spec.IPFamilyPolicy != nil {
+		state.IPFamilyPolicy = string(*l4.Service.Spec.IPFamilyPolicy)
+	}
+
+	return state
+}
+
 func (l4 *L4) getOldForwardingRule() (*composite.ForwardingRule, error) {
 	bsName := l4.namer.L4Backend(l4.Service.Namespace, l4.Service.Name)
 	// Check if protocol has changed for this service. In this case, forwarding rule has different protocol and name

pkg/metrics/l4metrics_test.go

@@ -406,3 +406,105 @@ func checkMetricsComputation(newMetrics *ControllerMetrics, expErrorCount, expSv
 	}
 	return nil
 }
+
+func TestComputeL4ILBDualStackMetrics(t *testing.T) {
+	t.Parallel()
+	for _, tc := range []struct {
+		desc                      string
+		serviceStates             []L4ILBDualStackServiceState
+		expectL4ILBDualStackCount map[L4ILBDualStackServiceState]int
+	}{
+		{
+			desc:                      "empty input",
+			serviceStates:             []L4ILBDualStackServiceState{},
+			expectL4ILBDualStackCount: map[L4ILBDualStackServiceState]int{},
+		},
+		{
+			desc: "one l4 ilb dual-stack service",
+			serviceStates: []L4ILBDualStackServiceState{
+				newL4ILBDualStackServiceState("IPv4", "SingleStack", StatusSuccess),
+			},
+			expectL4ILBDualStackCount: map[L4ILBDualStackServiceState]int{
+				L4ILBDualStackServiceState{
+					"IPv4",
+					"SingleStack",
+					StatusSuccess,
+				}: 1,
+			},
+		},
+		{
+			desc: "l4 ilb dual-stack service in error state",
+			serviceStates: []L4ILBDualStackServiceState{
+				newL4ILBDualStackServiceState("IPv4", "SingleStack", StatusError),
+			},
+			expectL4ILBDualStackCount: map[L4ILBDualStackServiceState]int{
+				L4ILBDualStackServiceState{
+					"IPv4",
+					"SingleStack",
+					StatusError,
+				}: 1,
+			},
+		},
+		{
+			desc: "L4 ILB dual-stack service with IPv4,IPv6 Families",
+			serviceStates: []L4ILBDualStackServiceState{
+				newL4ILBDualStackServiceState("IPv4,IPv6", "RequireDualStack", StatusSuccess),
+			},
+			expectL4ILBDualStackCount: map[L4ILBDualStackServiceState]int{
+				L4ILBDualStackServiceState{
+					"IPv4,IPv6",
+					"RequireDualStack",
+					StatusSuccess,
+				}: 1,
+			},
+		},
+		{
+			desc: "many l4 ilb dual-stack services",
+			serviceStates: []L4ILBDualStackServiceState{
+				newL4ILBDualStackServiceState("IPv4,IPv6", "RequireDualStack", StatusSuccess),
+				newL4ILBDualStackServiceState("IPv4,IPv6", "RequireDualStack", StatusSuccess),
+				newL4ILBDualStackServiceState("IPv4", "SingleStack", StatusError),
+				newL4ILBDualStackServiceState("IPv6", "SingleStack", StatusSuccess),
+				newL4ILBDualStackServiceState("IPv6", "SingleStack", StatusSuccess),
+			},
+			expectL4ILBDualStackCount: map[L4ILBDualStackServiceState]int{
+				L4ILBDualStackServiceState{
+					"IPv4,IPv6",
+					"RequireDualStack",
+					StatusSuccess,
+				}: 2,
+				L4ILBDualStackServiceState{
+					"IPv4",
+					"SingleStack",
+					StatusError,
+				}: 1,
+				L4ILBDualStackServiceState{
+					"IPv6",
+					"SingleStack",
+					StatusSuccess,
+				}: 2,
+			},
+		},
+	} {
+		tc := tc
+		t.Run(tc.desc, func(t *testing.T) {
+			t.Parallel()
+			newMetrics := FakeControllerMetrics()
+			for i, serviceState := range tc.serviceStates {
+				newMetrics.SetL4ILBDualStackService(fmt.Sprint(i), serviceState)
+			}
+			got := newMetrics.computeL4ILBDualStackMetrics()
+			if diff := cmp.Diff(tc.expectL4ILBDualStackCount, got); diff != "" {
+				t.Fatalf("Got diff for L4 ILB Dual-Stack service counts (-want +got):\n%s", diff)
+			}
+		})
+	}
+}
+
+func newL4ILBDualStackServiceState(ipFamilies string, ipFamilyPolicy string, status L4ILBDualStackServiceStateStatus) L4ILBDualStackServiceState {
+	return L4ILBDualStackServiceState{
+		IPFamilies:     ipFamilies,
+		IPFamilyPolicy: ipFamilyPolicy,
+		Status:         status,
+	}
+}

pkg/metrics/metrics.go

@@ -69,6 +69,13 @@ var (
 		},
 		[]string{label},
 	)
+	l4ILBDualStackCount = prometheus.NewGaugeVec(
+		prometheus.GaugeOpts{
+			Name: "number_of_l4_dual_stack_ilbs",
+			Help: "Number of L4 ILBs with DualStack enabled",
+		},
+		[]string{"ipFamilies", "ipFamilyPolicy", "status"},
+	)
 	l4NetLBCount = prometheus.NewGaugeVec(
 		prometheus.GaugeOpts{
 			Name: "number_of_l4_netlbs",
@@ -127,6 +134,9 @@ func init() {
 	klog.V(3).Infof("Registering L4 ILB usage metrics %v", l4ILBCount)
 	prometheus.MustRegister(l4ILBCount)
 
+	klog.V(3).Infof("Registering L4 ILB Dual Stack usage metrics %v", l4ILBDualStackCount)
+	prometheus.MustRegister(l4ILBDualStackCount)
+
 	klog.V(3).Infof("Registering L4 NetLB usage metrics %v", l4NetLBCount)
 	prometheus.MustRegister(l4NetLBCount)
 
@@ -153,6 +163,8 @@ type ControllerMetrics struct {
 	negMap map[string]NegServiceState
 	// l4ILBServiceMap is a map between service key and L4 ILB service state.
 	l4ILBServiceMap map[string]L4ILBServiceState
+	// l4ILBDualStackServiceMap is a map between service key and L4 ILB DualStack service state.
+	l4ILBDualStackServiceMap map[string]L4ILBDualStackServiceState
 	// l4NetLBServiceMap is a map between service key and L4 NetLB service state.
 	l4NetLBServiceMap map[string]L4NetLBServiceState
 	// pscMap is a map between the service attachment key and PSC state
@@ -173,6 +185,7 @@ func NewControllerMetrics(exportInterval, l4NetLBProvisionDeadline time.Duration
 		ingressMap:               make(map[string]IngressState),
 		negMap:                   make(map[string]NegServiceState),
 		l4ILBServiceMap:          make(map[string]L4ILBServiceState),
+		l4ILBDualStackServiceMap: make(map[string]L4ILBDualStackServiceState),
 		l4NetLBServiceMap:        make(map[string]L4NetLBServiceState),
 		pscMap:                   make(map[string]pscmetrics.PSCState),
 		serviceMap:               make(map[string]struct{}),
@@ -272,6 +285,25 @@ func (im *ControllerMetrics) DeleteL4ILBService(svcKey string) {
 	delete(im.l4ILBServiceMap, svcKey)
 }
 
+// SetL4ILBDualStackService implements L4ILBMetricsCollector.
+func (im *ControllerMetrics) SetL4ILBDualStackService(svcKey string, state L4ILBDualStackServiceState) {
+	im.Lock()
+	defer im.Unlock()
+
+	if im.l4ILBDualStackServiceMap == nil {
+		klog.Fatalf("L4 ILB DualStack Metrics failed to initialize correctly.")
+	}
+	im.l4ILBDualStackServiceMap[svcKey] = state
+}
+
+// DeleteL4ILBDualStackService implements L4ILBMetricsCollector.
+func (im *ControllerMetrics) DeleteL4ILBDualStackService(svcKey string) {
+	im.Lock()
+	defer im.Unlock()
+
+	delete(im.l4ILBDualStackServiceMap, svcKey)
+}
+
 // SetL4NetLBService adds metric state for given service to map.
 func (im *ControllerMetrics) SetL4NetLBService(svcKey string, state L4NetLBServiceState) {
 	im.Lock()
@@ -366,6 +398,17 @@ func (im *ControllerMetrics) export() {
 	}
 	klog.V(3).Infof("L4 ILB usage metrics exported.")
 
+	ilbDualStackCount := im.computeL4ILBDualStackMetrics()
+	klog.V(3).Infof("Exporting L4 ILB DualStack usage metrics: %#v", ilbDualStackCount)
+	for state, count := range ilbDualStackCount {
+		l4ILBDualStackCount.With(prometheus.Labels{
+			"ipFamilies":     state.IPFamilies,
+			"ipFamilyPolicy": state.IPFamilyPolicy,
+			"status":         string(state.Status),
+		}).Set(float64(count))
+	}
+	klog.V(3).Infof("L4 ILB DualStack usage metrics exported.")
+
 	netlbCount := im.computeL4NetLBMetrics()
 	klog.V(3).Infof("Exporting L4 NetLB usage metrics: %#v", netlbCount)
 	netlbCount.record()
@@ -519,6 +562,21 @@ func (im *ControllerMetrics) computeL4ILBMetrics() map[feature]int {
 	return counts
 }
 
+// computeL4ILBDualStackMetrics aggregates L4 ILB DualStack metrics in the cache.
+func (im *ControllerMetrics) computeL4ILBDualStackMetrics() map[L4ILBDualStackServiceState]int {
+	im.Lock()
+	defer im.Unlock()
+	klog.V(4).Infof("Computing L4 DualStack ILB usage metrics from service state map: %#v", im.l4ILBDualStackServiceMap)
+	counts := map[L4ILBDualStackServiceState]int{}
+
+	for key, state := range im.l4ILBDualStackServiceMap {
+		klog.V(6).Infof("ILB Service %s has IPFamilies: %v, IPFamilyPolicy: %t, Status: %v", key, state.IPFamilies, state.IPFamilyPolicy, state.Status)
+		counts[state]++
+	}
+	klog.V(4).Info("L4 ILB usage metrics computed.")
+	return counts
+}
+
 // computeL4NetLBMetrics aggregates L4 NetLB metrics in the cache.
 func (im *ControllerMetrics) computeL4NetLBMetrics() netLBFeatureCount {
 	im.Lock()

pkg/metrics/types.go

@@ -71,6 +71,22 @@ type L4ILBServiceState struct {
 	InSuccess bool
 }
 
+type L4ILBDualStackServiceStateStatus string
+
+var StatusSuccess = L4ILBDualStackServiceStateStatus("Success")
+var StatusError = L4ILBDualStackServiceStateStatus("Error")
+
+// L4ILBDualStackServiceState defines ipFamilies, ipFamilyPolicy and status
+// of L4 ILB DualStack service
+type L4ILBDualStackServiceState struct {
+	// IPFamilies stores spec.ipFamilies of Service
+	IPFamilies string
+	// IPFamilyPolicy specifies spec.IPFamilyPolicy of Service
+	IPFamilyPolicy string
+	// Status specifies status of L4 ILB DualStack
+	Status L4ILBDualStackServiceStateStatus
+}
+
 // L4NetLBServiceState defines if network tier is premium and
 // if static ip address is managed bu controller
 // for an L4 NetLB service.