Expose hubble agent when hubble is enabled

Also enables PKI for the addon
kubernetes · Apr 28, 2021 · 20455af · 20455af
1 parent 06f2f25
commit 20455af
Show file tree

Hide file tree

Showing 5 changed files with 122 additions and 9 deletions.
diff --git a/pkg/apis/kops/validation/validation.go b/pkg/apis/kops/validation/validation.go
@@ -744,8 +744,11 @@ func validateNetworkingCilium(cluster *kops.Cluster, v *kops.CiliumNetworkingSpe
 		}
 
 		if v.Hubble != nil && fi.BoolValue(v.Hubble.Enabled) {
+			if !components.IsCertManagerEnabled(cluster) {
+				allErrs = append(allErrs, field.Forbidden(fldPath.Child("hubble", "enabled"), "Hubble requires that cert manager is enabled"))
+			}
 			if version.Minor < 8 {
-				allErrs = append(allErrs, field.Forbidden(fldPath.Root().Child("hubble", "enabled"), "Hubble requires Cilium 1.8 or newer"))
+				allErrs = append(allErrs, field.Forbidden(fldPath.Child("hubble", "enabled"), "Hubble requires Cilium 1.8 or newer"))
 			}
 		}
 	}

diff --git a/pkg/apis/kops/validation/validation_test.go b/pkg/apis/kops/validation/validation_test.go
@@ -843,6 +843,20 @@ func Test_Validate_Cilium(t *testing.T) {
 					Enabled: fi.Bool(true),
 				},
 			},
+			ExpectedErrors: []string{"Forbidden::cilium.hubble.enabled"},
+		},
+		{
+			Cilium: kops.CiliumNetworkingSpec{
+				Version: "v1.8.0",
+				Hubble: &kops.HubbleSpec{
+					Enabled: fi.Bool(true),
+				},
+			},
+			Spec: kops.ClusterSpec{
+				CertManager: &kops.CertManagerConfig{
+					Enabled: fi.Bool(true),
+				},
+			},
 		},
 	}
 	for _, g := range grid {

diff --git a/upup/models/cloudup/resources/addons/networking.cilium.io/k8s-1.12-v1.9.yaml.template b/upup/models/cloudup/resources/addons/networking.cilium.io/k8s-1.12-v1.9.yaml.template
@@ -193,6 +193,12 @@ data:
   enable-hubble: "true"
   # UNIX domain socket for Hubble server to listen to.
   hubble-socket-path:  "/var/run/cilium/hubble.sock"
+  # An additional address for Hubble server to listen to (e.g. ":4244").
+  hubble-listen-address: ":4244"
+  hubble-disable-tls: "false"
+  hubble-tls-cert-file: /var/lib/cilium/tls/hubble/tls.crt
+  hubble-tls-key-file: /var/lib/cilium/tls/hubble/tls.key
+  hubble-tls-client-ca-files: /var/lib/cilium/tls/hubble/ca.crt
   {{ if .Hubble.Metrics }}
   hubble-metrics-server: ":9091"
   hubble-metrics:
@@ -214,8 +220,13 @@ data:
   config.yaml: |
     peer-service: unix:///var/run/cilium/hubble.sock
     listen-address: :4245
-    disable-client-tls: true
-    disable-server-tls: true
+
+    tls-client-cert-file: /var/lib/hubble-relay/tls/client.crt
+    tls-client-key-file: /var/lib/hubble-relay/tls/client.key
+    tls-hubble-server-ca-files: /var/lib/hubble-relay/tls/hubble-server-ca.crt
+    tls-server-cert-file: /var/lib/hubble-relay/tls/server.crt
+    tls-server-key-file: /var/lib/hubble-relay/tls/server.key
+
 {{ end }}
 ---
 apiVersion: rbac.authorization.k8s.io/v1
@@ -653,6 +664,9 @@ spec:
           readOnly: true
         - mountPath: /run/xtables.lock
           name: xtables-lock
+        - mountPath: /var/lib/cilium/tls/hubble
+          name: hubble-tls
+          readOnly: true
 {{ if CiliumSecret }}
         - mountPath: /etc/ipsec
           name: cilium-ipsec-secrets
@@ -767,6 +781,10 @@ spec:
         secret:
           secretName: cilium-ipsec-keys
 {{ end }}
+      - name: hubble-tls
+        secret:
+          secretName: hubble-server-certs
+          optional: true
 ---
 apiVersion: apps/v1
 kind: Deployment
@@ -902,7 +920,7 @@ spec:
   strategy:
     rollingUpdate:
       maxUnavailable: 1
-      type: RollingUpdate
+    type: RollingUpdate
   template:
     metadata:
       labels:
@@ -928,6 +946,10 @@ spec:
             - "serve"
             - "--peer-service=unix:///var/run/cilium/hubble.sock"
             - "--listen-address=:4245"
+          env:
+            # unfortunately, the addon CAs use only CN
+            - name: GODEBUG
+              value: x509ignoreCN=0
           ports:
             - name: grpc
               containerPort: 4245
@@ -944,6 +966,9 @@ spec:
           - mountPath: /etc/hubble-relay
             name: config
             readOnly: true
+          - mountPath: /var/lib/hubble-relay/tls
+            name: tls
+            readOnly: true
       restartPolicy: Always
       serviceAccount: hubble-relay
       serviceAccountName: hubble-relay
@@ -961,5 +986,72 @@ spec:
           - key: config.yaml
             path: config.yaml
         name: config
+      - projected:
+          sources:
+          - secret:
+              name: hubble-relay-client-certs
+              items:
+                - key: tls.crt
+                  path: client.crt
+                - key: tls.key
+                  path: client.key
+                - key: ca.crt
+                  path: hubble-server-ca.crt
+          - secret:
+              name: hubble-relay-server-certs
+              items:
+                - key: tls.crt
+                  path: server.crt
+                - key: tls.key
+                  path: server.key
+        name: tls
+---
+apiVersion: cert-manager.io/v1
+kind: Certificate
+metadata:
+  labels:
+    k8s-app: cilium
+  name: hubble-server-certs
+  namespace: kube-system
+spec:
+  dnsNames:
+  - "*.default.hubble-grpc.cilium.io"
+  issuerRef:
+    kind: Issuer
+    name: networking.cilium.io
+  secretName: hubble-server-certs
+---
+apiVersion: cert-manager.io/v1
+kind: Certificate
+metadata:
+  labels:
+    k8s-app: cilium
+  name: hubble-relay-server-certs
+  namespace: kube-system
+spec:
+  dnsNames:
+    - hubble-relay.kube-system.svc
+    - hubble-relay.kube-system.svc.cluster.local
+  issuerRef:
+    kind: Issuer
+    name: networking.cilium.io
+  secretName: hubble-relay-server-certs
+---
+apiVersion: cert-manager.io/v1
+kind: Certificate
+metadata:
+  labels:
+    k8s-app: cilium
+  name: hubble-relay-client-certs
+  namespace: kube-system
+spec:
+  dnsNames:
+  - "hubble-relay-client"
+  issuerRef:
+    kind: Issuer
+    name: networking.cilium.io
+  usages:
+  - client auth
+  secretName: hubble-relay-client-certs
 {{ end }}
 {{ end }}
diff --git a/upup/pkg/fi/cloudup/bootstrapchannelbuilder/cilium.go b/upup/pkg/fi/cloudup/bootstrapchannelbuilder/cilium.go
@@ -65,19 +65,23 @@ func addCiliumAddon(b *BootstrapChannelBuilder, addons *api.Addons) error {
 				})
 			}
 		} else if ver.Minor == 9 || ver.Minor == 10 {
-			version := "1.9.0-kops.1"
+			version := "1.9.4-kops.1"
 			{
 				id := "k8s-1.12"
 				location := key + "/" + id + "-v1.9.yaml"
 
-				addons.Spec.Addons = append(addons.Spec.Addons, &api.AddonSpec{
+				addon := &api.AddonSpec{
 					Name:               fi.String(key),
 					Version:            fi.String(version),
 					Selector:           networkingSelector(),
 					Manifest:           fi.String(location),
 					Id:                 id,
 					NeedsRollingUpdate: "all",
-				})
+				}
+				if cilium.Hubble != nil && fi.BoolValue(cilium.Hubble.Enabled) {
+					addon.NeedsPKI = true
+				}
+				addons.Spec.Addons = append(addons.Spec.Addons, addon)
 			}
 		} else {
 			return fmt.Errorf("unknown cilium version: %q", cilium.Version)

diff --git a/upup/pkg/fi/cloudup/tests/bootstrapchannelbuilder/cilium/manifest.yaml b/upup/pkg/fi/cloudup/tests/bootstrapchannelbuilder/cilium/manifest.yaml
@@ -71,9 +71,9 @@ spec:
     version: 1.17.0
   - id: k8s-1.12
     manifest: networking.cilium.io/k8s-1.12-v1.9.yaml
-    manifestHash: 2efe03944de989eca15f035e5d1de72bf1cfbace
+    manifestHash: 1918d794fefdbf0cfa6a5224bcd0b2bf14aee03d
     name: networking.cilium.io
     needsRollingUpdate: all
     selector:
       role.kubernetes.io/networking: "1"
-    version: 1.9.0-kops.1
+    version: 1.9.4-kops.1