giantswarm · QuentinBisson · Sep 19, 2023 · Aug 29, 2023 · Aug 30, 2023 · Sep 14, 2023
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -7,6 +7,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased]
 
+### Changed
+
+- Support flux-managed clusters.
+
 ## [4.47.0] - 2023-09-14
 
 ### Changed

diff --git a/files/templates/scrapeconfigs/_apiserver.yaml b/files/templates/scrapeconfigs/_apiserver.yaml
@@ -1,10 +1,15 @@
 [[- define "_apiserver" -]]
 [[- if ne .ClusterType "management_cluster" ]]
     api_server: https://[[ .APIServerURL ]]
+[[- if eq .AuthenticationType "token" ]]
+    bearer_token_file: /etc/prometheus/secrets/[[ .SecretName ]]/token
+[[- end ]]
     tls_config:
       ca_file: /etc/prometheus/secrets/[[ .SecretName ]]/ca
+[[- if eq .AuthenticationType "certificates" ]]
       cert_file: /etc/prometheus/secrets/[[ .SecretName ]]/crt
       key_file: /etc/prometheus/secrets/[[ .SecretName ]]/key
+[[- end ]]
       insecure_skip_verify: false
 [[- end -]]
 [[- end -]]
diff --git a/files/templates/scrapeconfigs/_tlsconfig.yaml b/files/templates/scrapeconfigs/_tlsconfig.yaml
@@ -1,14 +1,19 @@
 [[- define "_tlsconfig" -]]
 [[- if ne .ClusterType "management_cluster" -]]
-tls_config:
-  ca_file: /etc/prometheus/secrets/[[ .SecretName ]]/ca
-  cert_file: /etc/prometheus/secrets/[[ .SecretName ]]/crt
-  key_file: /etc/prometheus/secrets/[[ .SecretName ]]/key
-  insecure_skip_verify: false
-[[- else -]]
-bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token
-tls_config:
-  ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
-  insecure_skip_verify: true
+  [[- if eq .AuthenticationType "token" ]]
+  bearer_token_file: /etc/prometheus/secrets/[[ .SecretName ]]/token
+  [[- end ]]
+  tls_config:
+    ca_file: /etc/prometheus/secrets/[[ .SecretName ]]/ca
+    [[- if eq .AuthenticationType "certificates" ]]
+    cert_file: /etc/prometheus/secrets/[[ .SecretName ]]/crt
+    key_file: /etc/prometheus/secrets/[[ .SecretName ]]/key
+    [[- end ]]
+    insecure_skip_verify: false
+[[- else ]]
+  bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token
+  tls_config:
+    ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
+    insecure_skip_verify: true
 [[- end -]]
 [[- end -]]
diff --git a/files/templates/scrapeconfigs/_tlsconfig_skip.yaml b/files/templates/scrapeconfigs/_tlsconfig_skip.yaml
@@ -1,14 +1,19 @@
 [[- define "_tlsconfig_skip" -]]
 [[- if ne .ClusterType "management_cluster" -]]
-tls_config:
-  ca_file: /etc/prometheus/secrets/[[ .SecretName ]]/ca
-  cert_file: /etc/prometheus/secrets/[[ .SecretName ]]/crt
-  key_file: /etc/prometheus/secrets/[[ .SecretName ]]/key
-  insecure_skip_verify: true
-[[- else -]]
-bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token
-tls_config:
-  ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
-  insecure_skip_verify: true
+  [[- if eq .AuthenticationType "token" ]]
+  bearer_token_file: /etc/prometheus/secrets/[[ .SecretName ]]/token
+  [[- end ]]
+  tls_config:
+    ca_file: /etc/prometheus/secrets/[[ .SecretName ]]/ca
+    [[- if eq .AuthenticationType "certificates" ]]
+    cert_file: /etc/prometheus/secrets/[[ .SecretName ]]/crt
+    key_file: /etc/prometheus/secrets/[[ .SecretName ]]/key
+    [[- end ]]
+    insecure_skip_verify: true
+[[- else ]]
+  bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token
+  tls_config:
+    ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
+    insecure_skip_verify: true
 [[- end -]]
 [[- end -]]
diff --git a/files/templates/scrapeconfigs/additional-scrape-configs.template.yaml b/files/templates/scrapeconfigs/additional-scrape-configs.template.yaml
@@ -5,7 +5,7 @@
   kubernetes_sd_configs:
   - role: endpoints
 [[- include "_apiserver" . ]]
-[[ include "_tlsconfig_skip" . | indent 2 ]]
+[[- include "_tlsconfig_skip" . ]]
   relabel_configs:
   - source_labels: [__meta_kubernetes_service_label_component]
     regex: apiserver
@@ -26,7 +26,7 @@
   kubernetes_sd_configs:
   - role: node
 [[- include "_apiserver" . ]]
-[[ include "_tlsconfig" . | indent 2 ]]
+[[- include "_tlsconfig" . ]]
   relabel_configs:
   - target_label: __address__
     replacement: [[ .APIServerURL ]]
@@ -56,7 +56,7 @@
       names:
       - giantswarm
 [[- include "_apiserver" . ]]
-[[ include "_tlsconfig_skip" . | indent 2 ]]
+[[- include "_tlsconfig_skip" . ]]
   relabel_configs:
   - source_labels: [__meta_kubernetes_service_label_app_kubernetes_io_name]
     regex: falco-exporter
@@ -74,7 +74,7 @@
   kubernetes_sd_configs:
   - role: node
 [[- include "_apiserver" . ]]
-[[ include "_tlsconfig_skip" . | indent 2 ]]
+[[- include "_tlsconfig_skip" . ]]
   relabel_configs:
   - target_label: app
     replacement: kubelet
@@ -104,7 +104,7 @@
   kubernetes_sd_configs:
   - role: node
 [[- include "_apiserver" . ]]
-[[ include "_tlsconfig" . | indent 2 ]]
+[[- include "_tlsconfig" . ]]
   relabel_configs:
   - source_labels: [__address__]
     target_label: instance
@@ -145,7 +145,7 @@
   kubernetes_sd_configs:
   - role: pod
 [[- include "_apiserver" . ]]
-[[ include "_tlsconfig" . | indent 2 ]]
+[[- include "_tlsconfig" . ]]
   relabel_configs:
   - source_labels: [__address__]
     replacement: ${1}:9091
@@ -177,14 +177,8 @@
   [[- else ]]
   - role: node
   [[- end ]]
-[[- if ne .ClusterType "management_cluster" ]]
-    api_server: https://[[ .APIServerURL ]]
-    tls_config:
-      ca_file: /etc/prometheus/secrets/[[ .SecretName ]]/ca
-      cert_file: /etc/prometheus/secrets/[[ .SecretName ]]/crt
-      key_file: /etc/prometheus/secrets/[[ .SecretName ]]/key
-      insecure_skip_verify: false
-[[- else ]]
+[[- include "_apiserver" . ]]
+[[- if eq .ClusterType "management_cluster" ]]
   bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token
 [[- end ]]
   tls_config:
@@ -215,20 +209,20 @@
     target_label: __address__
     replacement: ${1}:2379
     action: replace
-[[- if eq .ClusterType "management_cluster" ]]
+  [[- if eq .ClusterType "management_cluster" ]]
   # if the 'ip' label is present, use the value
   - source_labels: [__meta_kubernetes_node_label_ip]
     regex: (.+)
     target_label: __address__
     replacement: ${1}:2379
     action: replace
-[[- end ]]
-[[- if and (eq .ClusterType "workload_cluster") (.WorkloadClusterETCDDomain) ]]
+  [[- end ]]
+  [[- if and (eq .ClusterType "workload_cluster") (.WorkloadClusterETCDDomain) ]]
   - source_labels: [__address__]
     target_label: __address__
     replacement: [[ .WorkloadClusterETCDDomain ]]
     action: replace
-[[- end ]]
+  [[- end ]]
 [[- end ]]
   - target_label: app
     replacement: etcd
@@ -251,7 +245,7 @@
       names:
       - kube-system
 [[- include "_apiserver" . ]]
-[[ include "_tlsconfig_skip" . | indent 2 ]]
+[[- include "_tlsconfig_skip" . ]]
   relabel_configs:
   - source_labels: [__address__]
 [[- if or .CAPIManagementCluster (eq .ClusterType "management_cluster") ]]
@@ -303,7 +297,7 @@
       names:
       - kube-system
 [[- include "_apiserver" . ]]
-[[ include "_tlsconfig_skip" . | indent 2 ]]
+[[- include "_tlsconfig_skip" . ]]
   relabel_configs:
   - source_labels: [__address__]
 [[- if or .CAPIManagementCluster (eq .ClusterType "management_cluster") ]]
@@ -354,7 +348,7 @@
       names:
       - kube-system
 [[- include "_apiserver" . ]]
-[[ include "_tlsconfig_skip" . | indent 2 ]]
+[[- include "_tlsconfig_skip" . ]]
   relabel_configs:
   - source_labels: [__address__]
     replacement: $1:10249
@@ -388,7 +382,7 @@
       names:
       - kube-system
 [[- include "_apiserver" . ]]
-[[ include "_tlsconfig_skip" . | indent 2 ]]
+[[- include "_tlsconfig_skip" . ]]
   relabel_configs:
   - source_labels: [__address__]
     target_label: instance
@@ -428,7 +422,7 @@
       - kube-system
 [[- end ]]
 [[- include "_apiserver" . ]]
-[[ include "_tlsconfig_skip" . | indent 2 ]]
+[[- include "_tlsconfig_skip" . ]]
   relabel_configs:
   - source_labels: [__address__]
     target_label: instance
@@ -456,7 +450,7 @@
   kubernetes_sd_configs:
   - role: pod
 [[- include "_apiserver" . ]]
-[[ include "_tlsconfig_skip" . | indent 2 ]]
+[[- include "_tlsconfig_skip" . ]]
   relabel_configs:
   - target_label: __address__
     replacement: [[ .APIServerURL ]]
@@ -487,7 +481,7 @@
   kubernetes_sd_configs:
   - role: endpoints
 [[- include "_apiserver" . ]]
-[[ include "_tlsconfig_skip" . | indent 2 ]]
+[[- include "_tlsconfig_skip" . ]]
   relabel_configs:
   - source_labels: [__meta_kubernetes_service_annotationpresent_giantswarm_io_monitoring, __meta_kubernetes_service_labelpresent_giantswarm_io_monitoring]
     regex: .*(true).*
@@ -825,7 +819,7 @@
       names:
       - kube-system
 [[- include "_apiserver" . ]]
-[[ include "_tlsconfig" . | indent 2 ]]
+[[- include "_tlsconfig" . ]]
   relabel_configs:
   - source_labels: [__meta_kubernetes_service_annotationpresent_giantswarm_io_monitoring, __meta_kubernetes_service_labelpresent_giantswarm_io_monitoring]
     regex: .*(true).*
@@ -876,7 +870,7 @@
       names:
       - kube-system
 [[- include "_apiserver" . ]]
-[[ include "_tlsconfig_skip" . | indent 2 ]]
+[[- include "_tlsconfig_skip" . ]]
   relabel_configs:
   - replacement: http
     target_label: __scheme__

diff --git a/helm/prometheus-meta-operator/templates/alertmanager/alertmanager-psp.yaml b/helm/prometheus-meta-operator/templates/alertmanager/alertmanager-psp.yaml
@@ -5,6 +5,8 @@ metadata:
   labels:
     {{- include "labels.common" . | nindent 4 }}
   name: alertmanager-psp
+  annotations:
+    seccomp.security.alpha.kubernetes.io/allowedProfileNames: runtime/default
 spec:
   allowPrivilegeEscalation: false
   hostNetwork: false

diff --git a/service/controller/managementcluster/resource.go b/service/controller/managementcluster/resource.go
@@ -175,6 +175,7 @@ func newResources(config resourcesConfig) ([]resource.Interface, error) {
 		c := prometheus.Config{
 			Address:            config.PrometheusAddress,
 			PrometheusClient:   config.PrometheusClient,
+			K8sClient:          config.K8sClient,
 			Logger:             config.Logger,
 			Customer:           config.Customer,
 			Installation:       config.Installation,

diff --git a/service/controller/resource/certificates/resource.go b/service/controller/resource/certificates/resource.go
@@ -115,10 +115,14 @@ func (r *Resource) getDesiredObject(ctx context.Context, v interface{}) (*v1.Sec
 				return nil, microerror.Mask(err)
 			}
 			kubeconfigAdminUser := fmt.Sprintf("%s-admin", cluster.GetName())
+			kubeconfigFluxCustomerUser := fmt.Sprintf("%s-capi-admin", cluster.GetName())
+
 			secretData["ca"] = capiKubeconfig.Clusters[cluster.GetName()].CertificateAuthorityData
 			if _, ok := capiKubeconfig.AuthInfos[kubeconfigAdminUser]; ok {
 				secretData["crt"] = capiKubeconfig.AuthInfos[kubeconfigAdminUser].ClientCertificateData
 				secretData["key"] = capiKubeconfig.AuthInfos[kubeconfigAdminUser].ClientKeyData
+			} else if _, ok := capiKubeconfig.AuthInfos[kubeconfigFluxCustomerUser]; ok {
+				secretData["token"] = []byte(capiKubeconfig.AuthInfos[kubeconfigFluxCustomerUser].Token)
 			} else {
 				return nil, errors.New("no supported user found in the CAPI secret")
 			}

diff --git a/service/controller/resource/monitoring/prometheus/resource.go b/service/controller/resource/monitoring/prometheus/resource.go
@@ -4,6 +4,7 @@ import (
 	"fmt"
 	"net/url"
 
+	"github.com/giantswarm/k8sclient/v7/pkg/k8sclient"
 	"github.com/giantswarm/microerror"
 	"github.com/giantswarm/micrologger"
 	"github.com/google/go-cmp/cmp"
@@ -27,6 +28,7 @@ const (
 
 type Config struct {
 	PrometheusClient promclient.Interface
+	K8sClient        k8sclient.Interface
 	Logger           micrologger.Logger
 
 	Address            string
@@ -262,12 +264,21 @@ func toPrometheus(ctx context.Context, v interface{}, config Config) (metav1.Obj
 		prometheus.Spec.APIServerConfig = &promv1.APIServerConfig{
 			Host: fmt.Sprintf("https://%s", key.APIUrl(cluster)),
 			TLSConfig: &promv1.TLSConfig{
-				CAFile:   fmt.Sprintf("/etc/prometheus/secrets/%s/ca", key.APIServerCertificatesSecretName),
-				CertFile: fmt.Sprintf("/etc/prometheus/secrets/%s/crt", key.APIServerCertificatesSecretName),
-				KeyFile:  fmt.Sprintf("/etc/prometheus/secrets/%s/key", key.APIServerCertificatesSecretName),
+				CAFile: fmt.Sprintf("/etc/prometheus/secrets/%s/ca", key.APIServerCertificatesSecretName),
 			},
 		}
 
+		authenticationType, err := key.ApiServerAuthenticationType(ctx, config.K8sClient, key.Namespace(cluster))
+		if err != nil {
+			return nil, microerror.Mask(err)
+		}
+		if authenticationType == "token" {
+			prometheus.Spec.APIServerConfig.BearerTokenFile = fmt.Sprintf("/etc/prometheus/secrets/%s/token", key.APIServerCertificatesSecretName)
+		} else if authenticationType == "certificates" {
+			prometheus.Spec.APIServerConfig.TLSConfig.CertFile = fmt.Sprintf("/etc/prometheus/secrets/%s/crt", key.APIServerCertificatesSecretName)
+			prometheus.Spec.APIServerConfig.TLSConfig.KeyFile = fmt.Sprintf("/etc/prometheus/secrets/%s/key", key.APIServerCertificatesSecretName)
+		}
+
 		prometheus.Spec.Secrets = []string{
 			key.APIServerCertificatesSecretName,
 		}