Merge pull request #61 from topfreegames/chore/add-custom-metrics

chore: add custom metrics to sgController

.kubernetes/manifests.yaml

@@ -295,7 +295,8 @@ spec:
           [{
             "metrics": [
               "controller*",
-              "workqueue*"
+              "workqueue*",
+              "custom*"
             ],
             "namespace": "kcio",
             "prometheus_url": "http://%%host%%:8080/metrics",
@@ -324,7 +325,7 @@ spec:
               key: account
               name: spotinst-credentials
               optional: true
-        image: tfgco/kubernetes-crossplane-infrastructure-operator:v0.8.0-alpha
+        image: tfgco/kubernetes-crossplane-infrastructure-operator:v0.8.4-alpha
         livenessProbe:
           httpGet:
             path: /healthz

Dockerfile

@@ -16,6 +16,7 @@ COPY cmd/main.go cmd/main.go
 COPY api/ api/
 COPY internal/controller/ internal/controller/
 COPY pkg/ pkg/
+COPY metrics/ metrics/
 
 # Build
 # the GOARCH has not a default value to allow the binary be built according to the host where the command

config/default/manager_datadog_openmetrics_patch.yaml

@@ -13,7 +13,8 @@ spec:
           [{
             "metrics": [
               "controller*",
-              "workqueue*"
+              "workqueue*",
+              "custom*"
             ],
             "namespace": "kcio",
             "prometheus_url": "http://%%host%%:8080/metrics",

config/manager/kustomization.yaml

@@ -13,4 +13,4 @@ kind: Kustomization
 images:
 - name: controller
   newName: tfgco/kubernetes-crossplane-infrastructure-operator
-  newTag: v0.8.3-alpha
+  newTag: v0.8.4-alpha

go.mod

@@ -18,7 +18,7 @@ require (
 	github.com/onsi/ginkgo/v2 v2.11.0
 	github.com/onsi/gomega v1.27.10
 	github.com/pkg/errors v0.9.1
-	github.com/topfreegames/kubernetes-kops-operator v0.11.0-rc
+	github.com/topfreegames/kubernetes-kops-operator v0.13.1-alpha
 	go.uber.org/zap v1.25.0
 	k8s.io/api v0.28.1
 	k8s.io/apimachinery v0.28.1
@@ -95,8 +95,8 @@ require (
 	github.com/modern-go/reflect2 v1.0.2 // indirect
 	github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
 	github.com/pkg/sftp v1.13.6 // indirect
-	github.com/prometheus/client_golang v1.16.0 // indirect
-	github.com/prometheus/client_model v0.4.0 // indirect
+	github.com/prometheus/client_golang v1.16.0
+	github.com/prometheus/client_model v0.4.0
 	github.com/prometheus/common v0.44.0 // indirect
 	github.com/prometheus/procfs v0.10.1 // indirect
 	github.com/spf13/afero v1.9.5 // indirect

go.sum

@@ -360,8 +360,8 @@ github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO
 github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
 github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk=
 github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=
-github.com/topfreegames/kubernetes-kops-operator v0.11.0-rc h1:L4WaAXgb+FM+D+oVJnJgxutOiwWTbCS+1uhpIhLzTqA=
-github.com/topfreegames/kubernetes-kops-operator v0.11.0-rc/go.mod h1:PnaRBjLXOYvuTRDZEBppcE0xLAVvz8NH/5gwi//SUIc=
+github.com/topfreegames/kubernetes-kops-operator v0.13.1-alpha h1:j2a0J3owdH5gB2c0AW42JcjA/Fi5a1L1mJBz0yAknE4=
+github.com/topfreegames/kubernetes-kops-operator v0.13.1-alpha/go.mod h1:+FwqSNyOaJ3bRMI07c2DrVAJD+Ky8rIE1yaVsw5Du9E=
 github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
 github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
 github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=

internal/controller/ec2.aws/securitygroup_controller.go

@@ -43,6 +43,7 @@ import (
 	crossec2v1beta1 "github.com/crossplane-contrib/provider-aws/apis/ec2/v1beta1"
 	"github.com/go-logr/logr"
 	"github.com/hashicorp/go-multierror"
+	custommetrics "github.com/topfreegames/kubernetes-crossplane-infrastructure-operator/metrics"
 	corev1 "k8s.io/api/core/v1"
 	apierrors "k8s.io/apimachinery/pkg/api/errors"
 	"k8s.io/apimachinery/pkg/runtime"
@@ -116,7 +117,7 @@ func DefaultReconciler(mgr manager.Manager) *SecurityGroupReconciler {
 //+kubebuilder:rbac:groups="",resources=events,verbs=get;list;watch;create;patch
 //+kubebuilder:rbac:groups="",resources=secrets,verbs=get;list;watch;create;update
 
-func (c *SecurityGroupReconciler) Reconcile(ctx context.Context, req ctrl.Request) (_ ctrl.Result, rerr error) {
+func (c *SecurityGroupReconciler) Reconcile(ctx context.Context, req ctrl.Request) (_ ctrl.Result, reconciliationErr error) {
 	r := &SecurityGroupReconciliation{
 		SecurityGroupReconciler: *c,
 		start:                   time.Now(),
@@ -141,25 +142,31 @@ func (c *SecurityGroupReconciler) Reconcile(ctx context.Context, req ctrl.Reques
 	r.log.Info(fmt.Sprintf("starting reconcile loop for %s", r.sg.ObjectMeta.GetName()))
 
 	defer func() {
+		environment := r.getEnvironment(ctx)
+
 		securityGroupHelper := r.sg.DeepCopy()
 		if err := r.Update(ctx, r.sg); err != nil {
 			r.Recorder.Eventf(r.sg, corev1.EventTypeWarning, "FailedToUpdate", "failed to update security group %s: %s", r.sg.Name, err)
-			if rerr == nil {
-				rerr = err
+			if reconciliationErr == nil {
+				reconciliationErr = err
 			}
 		}
 
 		if securityGroupHelper.ObjectMeta.DeletionTimestamp.IsZero() {
 			r.sg.Status = securityGroupHelper.Status
 			if err := r.Status().Update(ctx, r.sg); err != nil {
 				r.Recorder.Eventf(r.sg, corev1.EventTypeWarning, "FailedToUpdateStatus", "failed to update security group status %s: %s", r.sg.Name, err)
-				if rerr == nil {
-					rerr = err
+				if reconciliationErr == nil {
+					reconciliationErr = err
 				}
 			}
 		}
-
-		r.log.Info(fmt.Sprintf("finished reconcile loop for %s", r.sg.ObjectMeta.GetName()))
+		if reconciliationErr != nil {
+			custommetrics.ReconciliationConsecutiveErrorsTotal.WithLabelValues("securitygroup", r.sg.Name, environment).Inc()
+		} else {
+			custommetrics.ReconciliationConsecutiveErrorsTotal.WithLabelValues("securitygroup", r.sg.Name, environment).Set(0)
+		}
+		r.log.Info(fmt.Sprintf("finished reconcile loop for %s", r.sg.Name))
 	}()
 
 	err := r.retrieveInfraRefInfo(ctx)
@@ -181,7 +188,7 @@ func (r *SecurityGroupReconciliation) retrieveKCPNetworkInfo(ctx context.Context
 		Name:      name,
 		Namespace: namespace,
 	}
-	if err := r.Client.Get(ctx, key, kcp); err != nil {
+	if err := r.Get(ctx, key, kcp); err != nil {
 		return nil, nil, nil, err
 	}
 
@@ -967,3 +974,45 @@ func (r *SecurityGroupReconciliation) addKCPFinalizer(ctx context.Context, kcp k
 		}
 	}
 }
+
+func (r *SecurityGroupReconciliation) getEnvironment(ctx context.Context) string {
+	var clusterObject clusterv1beta1.Cluster
+	var err error
+
+	for _, ref := range r.sg.Spec.InfrastructureRef {
+		err = nil
+		switch ref.Kind {
+		case "KopsMachinePool":
+			kmp := kinfrastructurev1alpha1.KopsMachinePool{}
+			key := client.ObjectKey{
+				Name:      ref.Name,
+				Namespace: ref.Namespace,
+			}
+			if err = r.Client.Get(ctx, key, &kmp); err != nil {
+				continue
+			}
+			clusterName := &kmp.Spec.ClusterName
+			key = client.ObjectKey{
+				Name:      *clusterName,
+				Namespace: kmp.ObjectMeta.Namespace,
+			}
+			err = r.Client.Get(ctx, key, &clusterObject)
+		case "KopsControlPlane":
+			key := client.ObjectKey{
+				Name:      ref.Name,
+				Namespace: ref.Namespace,
+			}
+			err = r.Client.Get(ctx, key, &clusterObject)
+		}
+		if err != nil {
+			continue
+		} else {
+			break
+		}
+	}
+	if err == nil {
+		return clusterObject.Labels["environment"]
+	} else {
+		return "environmentNotFound"
+	}
+}