diff --git a/pkg/l4lb/l4controller.go b/pkg/l4lb/l4controller.go index 7e09da607d..9401988b75 100644 --- a/pkg/l4lb/l4controller.go +++ b/pkg/l4lb/l4controller.go @@ -487,6 +487,10 @@ func (l4c *L4Controller) publishMetrics(result *loadbalancers.L4ILBSyncResult, n case loadbalancers.SyncTypeCreate, loadbalancers.SyncTypeUpdate: klog.V(6).Infof("Internal L4 Loadbalancer for Service %s ensured, updating its state %v in metrics cache", namespacedName, result.MetricsState) l4c.ctx.ControllerMetrics.SetL4ILBService(namespacedName, result.MetricsState) + if l4c.enableDualStack { + klog.V(6).Infof("Internal L4 DualStack Loadbalancer for Service %s ensured, updating its state %v in metrics cache", namespacedName, result.MetricsState) + l4c.ctx.ControllerMetrics.SetL4ILBDualStackService(namespacedName, result.DualStackMetricsState) + } l4metrics.PublishILBSyncMetrics(result.Error == nil, result.SyncType, result.GCEResourceInError, utils.GetErrorType(result.Error), result.StartTime) case loadbalancers.SyncTypeDelete: @@ -494,6 +498,10 @@ func (l4c *L4Controller) publishMetrics(result *loadbalancers.L4ILBSyncResult, n if result.Error == nil { klog.V(6).Infof("Internal L4 Loadbalancer for Service %s deleted, removing its state from metrics cache", namespacedName) l4c.ctx.ControllerMetrics.DeleteL4ILBService(namespacedName) + if l4c.enableDualStack { + klog.V(6).Infof("Internal L4 Loadbalancer for Service %s deleted, removing its state from metrics cache", namespacedName) + l4c.ctx.ControllerMetrics.DeleteL4ILBDualStackService(namespacedName) + } } l4metrics.PublishILBSyncMetrics(result.Error == nil, result.SyncType, result.GCEResourceInError, utils.GetErrorType(result.Error), result.StartTime) default: diff --git a/pkg/loadbalancers/l4.go b/pkg/loadbalancers/l4.go index 11a0baabf8..fa97548a11 100644 --- a/pkg/loadbalancers/l4.go +++ b/pkg/loadbalancers/l4.go @@ -60,13 +60,14 @@ type L4 struct { // L4ILBSyncResult contains information about the outcome of an L4 ILB sync. It stores the list of resource name annotations, // sync error, the GCE resource that hit the error along with the error type, metrics and more fields. type L4ILBSyncResult struct { - Annotations map[string]string - Error error - GCEResourceInError string - Status *corev1.LoadBalancerStatus - MetricsState metrics.L4ILBServiceState - SyncType string - StartTime time.Time + Annotations map[string]string + Error error + GCEResourceInError string + Status *corev1.LoadBalancerStatus + MetricsState metrics.L4ILBServiceState + DualStackMetricsState metrics.L4ILBDualStackServiceState + SyncType string + StartTime time.Time } type L4ILBParams struct { @@ -278,10 +279,13 @@ func (l4 *L4) getFRNameWithProtocol(protocol string) string { // EnsureInternalLoadBalancer ensures that all GCE resources for the given loadbalancer service have // been created. It returns a LoadBalancerStatus with the updated ForwardingRule IP address. func (l4 *L4) EnsureInternalLoadBalancer(nodeNames []string, svc *corev1.Service) *L4ILBSyncResult { + l4.Service = svc + result := &L4ILBSyncResult{ - Annotations: make(map[string]string), - StartTime: time.Now(), - SyncType: SyncTypeCreate, + Annotations: make(map[string]string), + StartTime: time.Now(), + SyncType: SyncTypeCreate, + DualStackMetricsState: l4.getInitialDualStackMetricsState(), } // If service already has an IP assigned, treat it as an update instead of a new Loadbalancer. @@ -291,8 +295,6 @@ func (l4 *L4) EnsureInternalLoadBalancer(nodeNames []string, svc *corev1.Service result.SyncType = SyncTypeUpdate } - l4.Service = svc - hcLink := l4.provideHealthChecks(nodeNames, result) if result.Error != nil { return result @@ -382,6 +384,9 @@ func (l4 *L4) EnsureInternalLoadBalancer(nodeNames []string, svc *corev1.Service if options.SubnetName != "" { result.MetricsState.EnabledCustomSubnet = true } + if l4.enableDualStack { + result.DualStackMetricsState.Status = metrics.StatusSuccess + } return result } @@ -527,6 +532,26 @@ func (l4 *L4) hasAnnotation(annotationKey string) bool { return false } +func (l4 *L4) getInitialDualStackMetricsState() metrics.L4ILBDualStackServiceState { + // Always init stats with error, and update with Success when service was provisioned + state := metrics.L4ILBDualStackServiceState{ + Status: metrics.StatusError, + } + + var ipFamiliesStrings []string + for _, ipFamily := range l4.Service.Spec.IPFamilies { + ipFamiliesStrings = append(ipFamiliesStrings, string(ipFamily)) + } + state.IPFamilies = strings.Join(ipFamiliesStrings, ",") + + state.IPFamilyPolicy = "" + if l4.Service.Spec.IPFamilyPolicy != nil { + state.IPFamilyPolicy = string(*l4.Service.Spec.IPFamilyPolicy) + } + + return state +} + func (l4 *L4) getOldForwardingRule() (*composite.ForwardingRule, error) { bsName := l4.namer.L4Backend(l4.Service.Namespace, l4.Service.Name) // Check if protocol has changed for this service. In this case, forwarding rule has different protocol and name diff --git a/pkg/metrics/l4metrics_test.go b/pkg/metrics/l4metrics_test.go index 3efed31608..30d04dec70 100644 --- a/pkg/metrics/l4metrics_test.go +++ b/pkg/metrics/l4metrics_test.go @@ -406,3 +406,105 @@ func checkMetricsComputation(newMetrics *ControllerMetrics, expErrorCount, expSv } return nil } + +func TestComputeL4ILBDualStackMetrics(t *testing.T) { + t.Parallel() + for _, tc := range []struct { + desc string + serviceStates []L4ILBDualStackServiceState + expectL4ILBDualStackCount map[L4ILBDualStackServiceState]int + }{ + { + desc: "empty input", + serviceStates: []L4ILBDualStackServiceState{}, + expectL4ILBDualStackCount: map[L4ILBDualStackServiceState]int{}, + }, + { + desc: "one l4 ilb dual-stack service", + serviceStates: []L4ILBDualStackServiceState{ + newL4ILBDualStackServiceState("IPv4", "SingleStack", StatusSuccess), + }, + expectL4ILBDualStackCount: map[L4ILBDualStackServiceState]int{ + L4ILBDualStackServiceState{ + "IPv4", + "SingleStack", + StatusSuccess, + }: 1, + }, + }, + { + desc: "l4 ilb dual-stack service in error state", + serviceStates: []L4ILBDualStackServiceState{ + newL4ILBDualStackServiceState("IPv4", "SingleStack", StatusError), + }, + expectL4ILBDualStackCount: map[L4ILBDualStackServiceState]int{ + L4ILBDualStackServiceState{ + "IPv4", + "SingleStack", + StatusError, + }: 1, + }, + }, + { + desc: "L4 ILB dual-stack service with IPv4,IPv6 Families", + serviceStates: []L4ILBDualStackServiceState{ + newL4ILBDualStackServiceState("IPv4,IPv6", "RequireDualStack", StatusSuccess), + }, + expectL4ILBDualStackCount: map[L4ILBDualStackServiceState]int{ + L4ILBDualStackServiceState{ + "IPv4,IPv6", + "RequireDualStack", + StatusSuccess, + }: 1, + }, + }, + { + desc: "many l4 ilb dual-stack services", + serviceStates: []L4ILBDualStackServiceState{ + newL4ILBDualStackServiceState("IPv4,IPv6", "RequireDualStack", StatusSuccess), + newL4ILBDualStackServiceState("IPv4,IPv6", "RequireDualStack", StatusSuccess), + newL4ILBDualStackServiceState("IPv4", "SingleStack", StatusError), + newL4ILBDualStackServiceState("IPv6", "SingleStack", StatusSuccess), + newL4ILBDualStackServiceState("IPv6", "SingleStack", StatusSuccess), + }, + expectL4ILBDualStackCount: map[L4ILBDualStackServiceState]int{ + L4ILBDualStackServiceState{ + "IPv4,IPv6", + "RequireDualStack", + StatusSuccess, + }: 2, + L4ILBDualStackServiceState{ + "IPv4", + "SingleStack", + StatusError, + }: 1, + L4ILBDualStackServiceState{ + "IPv6", + "SingleStack", + StatusSuccess, + }: 2, + }, + }, + } { + tc := tc + t.Run(tc.desc, func(t *testing.T) { + t.Parallel() + newMetrics := FakeControllerMetrics() + for i, serviceState := range tc.serviceStates { + newMetrics.SetL4ILBDualStackService(fmt.Sprint(i), serviceState) + } + got := newMetrics.computeL4ILBDualStackMetrics() + if diff := cmp.Diff(tc.expectL4ILBDualStackCount, got); diff != "" { + t.Fatalf("Got diff for L4 ILB Dual-Stack service counts (-want +got):\n%s", diff) + } + }) + } +} + +func newL4ILBDualStackServiceState(ipFamilies string, ipFamilyPolicy string, status L4ILBDualStackServiceStateStatus) L4ILBDualStackServiceState { + return L4ILBDualStackServiceState{ + IPFamilies: ipFamilies, + IPFamilyPolicy: ipFamilyPolicy, + Status: status, + } +} diff --git a/pkg/metrics/metrics.go b/pkg/metrics/metrics.go index a04e8b1078..087fc01c07 100644 --- a/pkg/metrics/metrics.go +++ b/pkg/metrics/metrics.go @@ -69,6 +69,13 @@ var ( }, []string{label}, ) + l4ILBDualStackCount = prometheus.NewGaugeVec( + prometheus.GaugeOpts{ + Name: "number_of_l4_dual_stack_ilbs", + Help: "Number of L4 ILBs with DualStack enabled", + }, + []string{"ipFamilies", "ipFamilyPolicy", "status"}, + ) l4NetLBCount = prometheus.NewGaugeVec( prometheus.GaugeOpts{ Name: "number_of_l4_netlbs", @@ -127,6 +134,9 @@ func init() { klog.V(3).Infof("Registering L4 ILB usage metrics %v", l4ILBCount) prometheus.MustRegister(l4ILBCount) + klog.V(3).Infof("Registering L4 ILB Dual Stack usage metrics %v", l4ILBDualStackCount) + prometheus.MustRegister(l4ILBDualStackCount) + klog.V(3).Infof("Registering L4 NetLB usage metrics %v", l4NetLBCount) prometheus.MustRegister(l4NetLBCount) @@ -153,6 +163,8 @@ type ControllerMetrics struct { negMap map[string]NegServiceState // l4ILBServiceMap is a map between service key and L4 ILB service state. l4ILBServiceMap map[string]L4ILBServiceState + // l4ILBDualStackServiceMap is a map between service key and L4 ILB DualStack service state. + l4ILBDualStackServiceMap map[string]L4ILBDualStackServiceState // l4NetLBServiceMap is a map between service key and L4 NetLB service state. l4NetLBServiceMap map[string]L4NetLBServiceState // pscMap is a map between the service attachment key and PSC state @@ -173,6 +185,7 @@ func NewControllerMetrics(exportInterval, l4NetLBProvisionDeadline time.Duration ingressMap: make(map[string]IngressState), negMap: make(map[string]NegServiceState), l4ILBServiceMap: make(map[string]L4ILBServiceState), + l4ILBDualStackServiceMap: make(map[string]L4ILBDualStackServiceState), l4NetLBServiceMap: make(map[string]L4NetLBServiceState), pscMap: make(map[string]pscmetrics.PSCState), serviceMap: make(map[string]struct{}), @@ -272,6 +285,25 @@ func (im *ControllerMetrics) DeleteL4ILBService(svcKey string) { delete(im.l4ILBServiceMap, svcKey) } +// SetL4ILBDualStackService implements L4ILBMetricsCollector. +func (im *ControllerMetrics) SetL4ILBDualStackService(svcKey string, state L4ILBDualStackServiceState) { + im.Lock() + defer im.Unlock() + + if im.l4ILBDualStackServiceMap == nil { + klog.Fatalf("L4 ILB DualStack Metrics failed to initialize correctly.") + } + im.l4ILBDualStackServiceMap[svcKey] = state +} + +// DeleteL4ILBDualStackService implements L4ILBMetricsCollector. +func (im *ControllerMetrics) DeleteL4ILBDualStackService(svcKey string) { + im.Lock() + defer im.Unlock() + + delete(im.l4ILBDualStackServiceMap, svcKey) +} + // SetL4NetLBService adds metric state for given service to map. func (im *ControllerMetrics) SetL4NetLBService(svcKey string, state L4NetLBServiceState) { im.Lock() @@ -366,6 +398,17 @@ func (im *ControllerMetrics) export() { } klog.V(3).Infof("L4 ILB usage metrics exported.") + ilbDualStackCount := im.computeL4ILBDualStackMetrics() + klog.V(3).Infof("Exporting L4 ILB DualStack usage metrics: %#v", ilbDualStackCount) + for state, count := range ilbDualStackCount { + l4ILBDualStackCount.With(prometheus.Labels{ + "ipFamilies": state.IPFamilies, + "ipFamilyPolicy": state.IPFamilyPolicy, + "status": string(state.Status), + }).Set(float64(count)) + } + klog.V(3).Infof("L4 ILB DualStack usage metrics exported.") + netlbCount := im.computeL4NetLBMetrics() klog.V(3).Infof("Exporting L4 NetLB usage metrics: %#v", netlbCount) netlbCount.record() @@ -519,6 +562,21 @@ func (im *ControllerMetrics) computeL4ILBMetrics() map[feature]int { return counts } +// computeL4ILBDualStackMetrics aggregates L4 ILB DualStack metrics in the cache. +func (im *ControllerMetrics) computeL4ILBDualStackMetrics() map[L4ILBDualStackServiceState]int { + im.Lock() + defer im.Unlock() + klog.V(4).Infof("Computing L4 DualStack ILB usage metrics from service state map: %#v", im.l4ILBDualStackServiceMap) + counts := map[L4ILBDualStackServiceState]int{} + + for key, state := range im.l4ILBDualStackServiceMap { + klog.V(6).Infof("ILB Service %s has IPFamilies: %v, IPFamilyPolicy: %t, Status: %v", key, state.IPFamilies, state.IPFamilyPolicy, state.Status) + counts[state]++ + } + klog.V(4).Info("L4 ILB usage metrics computed.") + return counts +} + // computeL4NetLBMetrics aggregates L4 NetLB metrics in the cache. func (im *ControllerMetrics) computeL4NetLBMetrics() netLBFeatureCount { im.Lock() diff --git a/pkg/metrics/types.go b/pkg/metrics/types.go index f4dc5c8d77..29655bb89b 100644 --- a/pkg/metrics/types.go +++ b/pkg/metrics/types.go @@ -71,6 +71,22 @@ type L4ILBServiceState struct { InSuccess bool } +type L4ILBDualStackServiceStateStatus string + +var StatusSuccess = L4ILBDualStackServiceStateStatus("Success") +var StatusError = L4ILBDualStackServiceStateStatus("Error") + +// L4ILBDualStackServiceState defines ipFamilies, ipFamilyPolicy and status +// of L4 ILB DualStack service +type L4ILBDualStackServiceState struct { + // IPFamilies stores spec.ipFamilies of Service + IPFamilies string + // IPFamilyPolicy specifies spec.IPFamilyPolicy of Service + IPFamilyPolicy string + // Status specifies status of L4 ILB DualStack + Status L4ILBDualStackServiceStateStatus +} + // L4NetLBServiceState defines if network tier is premium and // if static ip address is managed bu controller // for an L4 NetLB service.