feat(kuma-cp) prometheus metrics over mtls

kumahq · Jun 2, 2020 · fc5b7ea · fc5b7ea
1 parent a0aacc9
commit fc5b7ea
Show file tree

Hide file tree

Showing 34 changed files with 544 additions and 469 deletions.
diff --git a/Makefile.kind.mk b/Makefile.kind.mk
@@ -73,6 +73,11 @@ kind/deploy/kuma: build/kumactl kind/load
     	echo "Waiting for the cluster to come up" && sleep 1; \
     done
 
+.PHONY: kind/deploy/metrics
+kind/deploy/metrics: build/kumactl
+	@${BUILD_ARTIFACTS_DIR}/kumactl/kumactl install metrics $(KUMACTL_INSTALL_METRICS_IMAGES) | kubectl apply -f -
+	@KUBECONFIG=$(KIND_KUBECONFIG) kubectl wait --timeout=60s --for=condition=Ready -n kuma-metrics pods -l app=prometheus
+
 .PHONY: kind/deploy/example-app
 kind/deploy/example-app:
 	@KUBECONFIG=$(KIND_KUBECONFIG) kubectl create namespace $(EXAMPLE_NAMESPACE) || true

diff --git a/api/mesh/v1alpha1/dataplane_helpers.go b/api/mesh/v1alpha1/dataplane_helpers.go
@@ -174,11 +174,11 @@ func ParseIP(text string) (string, error) {
 }
 
 func (n *Dataplane_Networking) GetInboundInterface(service string) (*InboundInterface, error) {
-	for i, inbound := range n.Inbound {
+	for _, inbound := range n.Inbound {
 		if inbound.Tags[ServiceTag] != service {
 			continue
 		}
-		iface, err := n.GetInboundInterfaceByIdx(i)
+		iface, err := n.ToInboundInterface(inbound)
 		return &iface, err
 	}
 	return nil, errors.Errorf("Dataplane has no Inbound Interface for service %q", service)
@@ -189,8 +189,8 @@ func (n *Dataplane_Networking) GetInboundInterfaces() ([]InboundInterface, error
 		return nil, nil
 	}
 	ifaces := make([]InboundInterface, len(n.Inbound))
-	for i, _ := range n.Inbound {
-		iface, err := n.GetInboundInterfaceByIdx(i)
+	for i, inbound := range n.Inbound {
+		iface, err := n.ToInboundInterface(inbound)
 		if err != nil {
 			return nil, err
 		}
@@ -199,11 +199,7 @@ func (n *Dataplane_Networking) GetInboundInterfaces() ([]InboundInterface, error
 	return ifaces, nil
 }
 
-func (n *Dataplane_Networking) GetInboundInterfaceByIdx(idx int) (InboundInterface, error) {
-	if idx >= len(n.Inbound) {
-		return InboundInterface{}, errors.Errorf("there is no inbound for index %d. Dataplane has %d inbounds", idx, len(n.Inbound))
-	}
-	inbound := n.Inbound[idx]
+func (n *Dataplane_Networking) ToInboundInterface(inbound *Dataplane_Networking_Inbound) (InboundInterface, error) {
 	if inbound.Interface != "" {
 		return ParseInboundInterface(inbound.Interface)
 	} else {

diff --git a/api/mesh/v1alpha1/metrics.pb.go b/api/mesh/v1alpha1/metrics.pb.go
diff --git a/api/mesh/v1alpha1/metrics.proto b/api/mesh/v1alpha1/metrics.proto
@@ -5,6 +5,7 @@ package kuma.mesh.v1alpha1;
 option go_package = "v1alpha1";
 
 import "google/protobuf/struct.proto";
+import "google/protobuf/wrappers.proto";
 
 // Metrics defines configuration for metrics that should be collected and
 // exposed by dataplanes.
@@ -38,4 +39,16 @@ message PrometheusMetricsBackendConfig {
   // Path on which a dataplane should expose HTTP endpoint with Prometheus
   // metrics.
   string path = 2;
+
+  // Tags associated with an application this dataplane is deployed next to,
+  // e.g. service=web, version=1.0.
+  // `service` tag is mandatory.
+  map<string, string> tags = 3;
+
+  // If true then endpoints for scraping metrics won't require mTLS even if mTLS
+  // is enabled in Mesh. If nil, then it is treated as set to true.
+  //
+  // todo(jakubdyszkiewicz) In next major version of Kuma, change BoolValue to
+  // bool, so the default is false
+  google.protobuf.BoolValue skipMTLS = 4;
 }
diff --git a/app/kumactl/cmd/install/testdata/install-metrics.defaults.golden.yaml b/app/kumactl/cmd/install/testdata/install-metrics.defaults.golden.yaml
@@ -4,6 +4,8 @@ apiVersion: v1
 kind: Namespace
 metadata:
   name: kuma-metrics
+  labels:
+    kuma.io/sidecar-injection: enabled
 ---
 apiVersion: policy/v1beta1
 kind: PodSecurityPolicy
@@ -4847,6 +4849,8 @@ spec:
       labels:
         component: "node-exporter"
         app: prometheus
+      annotations:
+        kuma.io/sidecar-injection: "disabled" # disabled for now, injecting DP crashes K8S cluster
     spec:
       serviceAccountName: prometheus-node-exporter
       containers:
@@ -4996,12 +5000,6 @@ spec:
             - --web.external-url=http://localhost:9093
           ports:
             - containerPort: 9093
-          readinessProbe:
-            httpGet:
-              path: /-/ready
-              port: 9093
-            initialDelaySeconds: 30
-            timeoutSeconds: 30
           resources:
             {}
           volumeMounts:
@@ -5025,7 +5023,6 @@ spec:
       securityContext:
         fsGroup: 65534
         runAsGroup: 65534
-        runAsNonRoot: true
         runAsUser: 65534
       volumes:
         - name: config-volume
@@ -5068,7 +5065,6 @@ spec:
           resources:
             {}
       securityContext:
-        runAsNonRoot: true
         runAsUser: 65534
 ---
 apiVersion: apps/v1
@@ -5099,22 +5095,9 @@ spec:
           args:
           ports:
             - containerPort: 9091
-          livenessProbe:
-            httpGet:
-              path: /-/healthy
-              port: 9091
-            initialDelaySeconds: 10
-            timeoutSeconds: 10
-          readinessProbe:
-            httpGet:
-              path: /-/ready
-              port: 9091
-            initialDelaySeconds: 10
-            timeoutSeconds: 10
           resources:
             {}
       securityContext:
-        runAsNonRoot: true
         runAsUser: 65534
 ---
 apiVersion: apps/v1
@@ -5136,6 +5119,8 @@ spec:
       labels:
         component: "server"
         app: prometheus
+      annotations:
+        kuma.io/direct-access-services: "*"
     spec:
       serviceAccountName: prometheus-server
       containers:
@@ -5163,22 +5148,6 @@ spec:
             - --web.enable-lifecycle
           ports:
             - containerPort: 9090
-          readinessProbe:
-            httpGet:
-              path: /-/ready
-              port: 9090
-            initialDelaySeconds: 30
-            timeoutSeconds: 30
-            failureThreshold: 3
-            successThreshold: 1
-          livenessProbe:
-            httpGet:
-              path: /-/healthy
-              port: 9090
-            initialDelaySeconds: 30
-            timeoutSeconds: 30
-            failureThreshold: 3
-            successThreshold: 1
           resources:
             {}
           volumeMounts:
@@ -5203,7 +5172,6 @@ spec:
       securityContext:
         fsGroup: 65534
         runAsGroup: 65534
-        runAsNonRoot: true
         runAsUser: 65534
       terminationGracePeriodSeconds: 300
       volumes:

diff --git a/app/kumactl/cmd/install/testdata/install-metrics.overrides.golden.yaml b/app/kumactl/cmd/install/testdata/install-metrics.overrides.golden.yaml
@@ -4,6 +4,8 @@ apiVersion: v1
 kind: Namespace
 metadata:
   name: kuma
+  labels:
+    kuma.io/sidecar-injection: enabled
 ---
 apiVersion: policy/v1beta1
 kind: PodSecurityPolicy
@@ -4847,6 +4849,8 @@ spec:
       labels:
         component: "node-exporter"
         app: prometheus
+      annotations:
+        kuma.io/sidecar-injection: "disabled" # disabled for now, injecting DP crashes K8S cluster
     spec:
       serviceAccountName: prometheus-node-exporter
       containers:
@@ -4996,12 +5000,6 @@ spec:
             - --web.external-url=http://localhost:9093
           ports:
             - containerPort: 9093
-          readinessProbe:
-            httpGet:
-              path: /-/ready
-              port: 9093
-            initialDelaySeconds: 30
-            timeoutSeconds: 30
           resources:
             {}
           volumeMounts:
@@ -5025,7 +5023,6 @@ spec:
       securityContext:
         fsGroup: 65534
         runAsGroup: 65534
-        runAsNonRoot: true
         runAsUser: 65534
       volumes:
         - name: config-volume
@@ -5068,7 +5065,6 @@ spec:
           resources:
             {}
       securityContext:
-        runAsNonRoot: true
         runAsUser: 65534
 ---
 apiVersion: apps/v1
@@ -5099,22 +5095,9 @@ spec:
           args:
           ports:
             - containerPort: 9091
-          livenessProbe:
-            httpGet:
-              path: /-/healthy
-              port: 9091
-            initialDelaySeconds: 10
-            timeoutSeconds: 10
-          readinessProbe:
-            httpGet:
-              path: /-/ready
-              port: 9091
-            initialDelaySeconds: 10
-            timeoutSeconds: 10
           resources:
             {}
       securityContext:
-        runAsNonRoot: true
         runAsUser: 65534
 ---
 apiVersion: apps/v1
@@ -5136,6 +5119,8 @@ spec:
       labels:
         component: "server"
         app: prometheus
+      annotations:
+        kuma.io/direct-access-services: "*"
     spec:
       serviceAccountName: prometheus-server
       containers:
@@ -5163,22 +5148,6 @@ spec:
             - --web.enable-lifecycle
           ports:
             - containerPort: 9090
-          readinessProbe:
-            httpGet:
-              path: /-/ready
-              port: 9090
-            initialDelaySeconds: 30
-            timeoutSeconds: 30
-            failureThreshold: 3
-            successThreshold: 1
-          livenessProbe:
-            httpGet:
-              path: /-/healthy
-              port: 9090
-            initialDelaySeconds: 30
-            timeoutSeconds: 30
-            failureThreshold: 3
-            successThreshold: 1
           resources:
             {}
           volumeMounts:
@@ -5203,7 +5172,6 @@ spec:
       securityContext:
         fsGroup: 65534
         runAsGroup: 65534
-        runAsNonRoot: true
         runAsUser: 65534
       terminationGracePeriodSeconds: 300
       volumes: