Add token support

Signed-off-by: QuentinBisson <[email protected]>
giantswarm · Aug 31, 2023 · 78b34b6 · 78b34b6
1 parent b8050cb
commit 78b34b6
Show file tree

Hide file tree

Showing 44 changed files with 771 additions and 789 deletions.
diff --git a/files/templates/scrapeconfigs/_apiserver.yaml b/files/templates/scrapeconfigs/_apiserver.yaml
@@ -1,12 +1,15 @@
 [[- define "_apiserver" -]]
 [[- if ne .ClusterType "management_cluster" ]]
     api_server: https://[[ .APIServerURL ]]
-    authorization:
-      credentials_file: /etc/prometheus/secrets/[[ .SecretName ]]/token
+[[- if eq .AuthenticationType "token" ]]
+    bearer_token_file: /etc/prometheus/secrets/[[ .SecretName ]]/token
+[[- end ]]
     tls_config:
       ca_file: /etc/prometheus/secrets/[[ .SecretName ]]/ca
+[[- if eq .AuthenticationType "certificates" ]]
       cert_file: /etc/prometheus/secrets/[[ .SecretName ]]/crt
       key_file: /etc/prometheus/secrets/[[ .SecretName ]]/key
+[[- end ]]
       insecure_skip_verify: false
 [[- end -]]
 [[- end -]]
diff --git a/files/templates/scrapeconfigs/_tlsconfig.yaml b/files/templates/scrapeconfigs/_tlsconfig.yaml
@@ -1,9 +1,14 @@
 [[- define "_tlsconfig" -]]
 [[- if ne .ClusterType "management_cluster" -]]
+[[- if eq .AuthenticationType "token" ]]
+bearer_token_file: /etc/prometheus/secrets/[[ .SecretName ]]/token
+[[- end ]]
 tls_config:
   ca_file: /etc/prometheus/secrets/[[ .SecretName ]]/ca
+[[- if eq .AuthenticationType "certificates" ]]
   cert_file: /etc/prometheus/secrets/[[ .SecretName ]]/crt
   key_file: /etc/prometheus/secrets/[[ .SecretName ]]/key
+[[- end ]]
   insecure_skip_verify: false
 [[- else -]]
 bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token

diff --git a/files/templates/scrapeconfigs/_tlsconfig_skip.yaml b/files/templates/scrapeconfigs/_tlsconfig_skip.yaml
@@ -1,9 +1,14 @@
 [[- define "_tlsconfig_skip" -]]
 [[- if ne .ClusterType "management_cluster" -]]
+[[- if eq .AuthenticationType "token" ]]
+bearer_token_file: /etc/prometheus/secrets/[[ .SecretName ]]/token
+[[- end ]]
 tls_config:
   ca_file: /etc/prometheus/secrets/[[ .SecretName ]]/ca
+[[- if eq .AuthenticationType "certificates" ]]
   cert_file: /etc/prometheus/secrets/[[ .SecretName ]]/crt
   key_file: /etc/prometheus/secrets/[[ .SecretName ]]/key
+[[- end ]]
   insecure_skip_verify: true
 [[- else -]]
 bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token

diff --git a/files/templates/scrapeconfigs/additional-scrape-configs.template.yaml b/files/templates/scrapeconfigs/additional-scrape-configs.template.yaml
@@ -177,16 +177,10 @@
   [[ else ]]
   - role: node
   [[ end ]]
-[[ if ne .ClusterType "management_cluster" ]]
-    api_server: https://[[ .APIServerURL ]]
-    tls_config:
-      ca_file: /etc/prometheus/secrets/[[ .SecretName ]]/ca
-      cert_file: /etc/prometheus/secrets/[[ .SecretName ]]/crt
-      key_file: /etc/prometheus/secrets/[[ .SecretName ]]/key
-      insecure_skip_verify: false
-[[ else ]]
+[[ include "_apiserver" . ]]
+[[- if eq .ClusterType "management_cluster" ]]
   bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token
-[[ end ]]
+[[- end ]]
   tls_config:
     ca_file: /etc/prometheus/secrets/[[ .EtcdSecretName ]]/ca
     cert_file: /etc/prometheus/secrets/[[ .EtcdSecretName ]]/crt

diff --git a/helm/prometheus-meta-operator/templates/alertmanager/alertmanager-psp.yaml b/helm/prometheus-meta-operator/templates/alertmanager/alertmanager-psp.yaml
@@ -5,6 +5,8 @@ metadata:
   labels:
     {{- include "labels.common" . | nindent 4 }}
   name: alertmanager-psp
+  annotations:
+    seccomp.security.alpha.kubernetes.io/allowedProfileNames: runtime/default
 spec:
   allowPrivilegeEscalation: false
   hostNetwork: false

diff --git a/service/controller/managementcluster/resource.go b/service/controller/managementcluster/resource.go
@@ -174,6 +174,7 @@ func newResources(config resourcesConfig) ([]resource.Interface, error) {
 		c := prometheus.Config{
 			Address:            config.PrometheusAddress,
 			PrometheusClient:   config.PrometheusClient,
+			K8sClient:          config.K8sClient,
 			Logger:             config.Logger,
 			Customer:           config.Customer,
 			Installation:       config.Installation,

diff --git a/service/controller/resource/certificates/resource.go b/service/controller/resource/certificates/resource.go
@@ -115,13 +115,14 @@ func (r *Resource) getDesiredObject(ctx context.Context, v interface{}) (*v1.Sec
 				return nil, microerror.Mask(err)
 			}
 			kubeconfigAdminUser := fmt.Sprintf("%s-admin", cluster.GetName())
-			kubeconfigFluxCustomerUser := fmt.Sprintf("flux-customer@%s", cluster.GetName())
+			kubeconfigFluxCustomerUser := fmt.Sprintf("%s-capi-admin", cluster.GetName())
+
 			secretData["ca"] = capiKubeconfig.Clusters[cluster.GetName()].CertificateAuthorityData
 			if _, ok := capiKubeconfig.AuthInfos[kubeconfigAdminUser]; ok {
 				secretData["crt"] = capiKubeconfig.AuthInfos[kubeconfigAdminUser].ClientCertificateData
 				secretData["key"] = capiKubeconfig.AuthInfos[kubeconfigAdminUser].ClientKeyData
 			} else if _, ok := capiKubeconfig.AuthInfos[kubeconfigFluxCustomerUser]; ok {
-				secretData["token"] = []byte(capiKubeconfig.AuthInfos[kubeconfigAdminUser].Token)
+				secretData["token"] = []byte(capiKubeconfig.AuthInfos[kubeconfigFluxCustomerUser].Token)
 			} else {
 				return nil, errors.New("no supported user found in the CAPI secret")
 			}

diff --git a/service/controller/resource/monitoring/prometheus/resource.go b/service/controller/resource/monitoring/prometheus/resource.go
@@ -4,6 +4,7 @@ import (
 	"fmt"
 	"net/url"
 
+	"github.com/giantswarm/k8sclient/v7/pkg/k8sclient"
 	"github.com/giantswarm/microerror"
 	"github.com/giantswarm/micrologger"
 	"github.com/google/go-cmp/cmp"
@@ -27,6 +28,7 @@ const (
 
 type Config struct {
 	PrometheusClient promclient.Interface
+	K8sClient        k8sclient.Interface
 	Logger           micrologger.Logger
 
 	Address            string
@@ -261,16 +263,22 @@ func toPrometheus(ctx context.Context, v interface{}, config Config) (metav1.Obj
 		// Workload cluster
 		prometheus.Spec.APIServerConfig = &promv1.APIServerConfig{
 			Host: fmt.Sprintf("https://%s", key.APIUrl(cluster)),
-			Authorization: &promv1.Authorization{
-				CredentialsFile: fmt.Sprintf("/etc/prometheus/secrets/%s/token", key.Secret()),
-			},
 			TLSConfig: &promv1.TLSConfig{
-				CAFile:   fmt.Sprintf("/etc/prometheus/secrets/%s/ca", key.APIServerCertificatesSecretName),
-				CertFile: fmt.Sprintf("/etc/prometheus/secrets/%s/crt", key.APIServerCertificatesSecretName),
-				KeyFile:  fmt.Sprintf("/etc/prometheus/secrets/%s/key", key.APIServerCertificatesSecretName),
+				CAFile: fmt.Sprintf("/etc/prometheus/secrets/%s/ca", key.APIServerCertificatesSecretName),
 			},
 		}
 
+		authenticationType, err := key.ApiServerAuthenticationType(ctx, config.K8sClient, key.Namespace(cluster))
+		if err != nil {
+			return nil, microerror.Mask(err)
+		}
+		if authenticationType == "token" {
+			prometheus.Spec.APIServerConfig.BearerTokenFile = fmt.Sprintf("/etc/prometheus/secrets/%s/token", key.APIServerCertificatesSecretName)
+		} else if authenticationType == "certificates" {
+			prometheus.Spec.APIServerConfig.TLSConfig.CertFile = fmt.Sprintf("/etc/prometheus/secrets/%s/crt", key.APIServerCertificatesSecretName)
+			prometheus.Spec.APIServerConfig.TLSConfig.KeyFile = fmt.Sprintf("/etc/prometheus/secrets/%s/key", key.APIServerCertificatesSecretName)
+		}
+
 		prometheus.Spec.Secrets = []string{
 			key.APIServerCertificatesSecretName,
 		}

diff --git a/service/controller/resource/monitoring/prometheus/resource_test.go b/service/controller/resource/monitoring/prometheus/resource_test.go
@@ -6,7 +6,15 @@ import (
 	"path/filepath"
 	"testing"
 
+	"github.com/giantswarm/k8sclient/v7/pkg/k8sclient"
+	"github.com/giantswarm/k8sclient/v7/pkg/k8sclient/fake"
+	"github.com/giantswarm/micrologger"
+	v1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/runtime"
+
 	"github.com/giantswarm/prometheus-meta-operator/v2/pkg/unittest"
+	"github.com/giantswarm/prometheus-meta-operator/v2/service/key"
 )
 
 var update = flag.Bool("update", false, "update the ouput file")
@@ -17,26 +25,66 @@ func TestPrometheus(t *testing.T) {
 		t.Fatal(err)
 	}
 
-	config := Config{
-		Address:            "http://prometheus/cluster",
-		Customer:           "Giant Swarm",
-		EvaluationInterval: "60s",
-		Installation:       "test-installation",
-		Pipeline:           "testing",
-		Provider:           "provider",
-		Region:             "onprem",
-		ImageRepository:    "giantswarm/prometheus",
-		LogLevel:           "debug",
-		Registry:           "quay.io",
-		RetentionDuration:  "2w",
-		ScrapeInterval:     "60s",
-		Version:            "v2.28.1",
+	var logger micrologger.Logger
+	{
+		c := micrologger.Config{}
+
+		logger, err = micrologger.New(c)
+		if err != nil {
+			t.Fatal(err)
+		}
 	}
 
 	c := unittest.Config{
 		OutputDir: outputDir,
 		T:         t,
 		TestFunc: func(v interface{}) (interface{}, error) {
+			cluster, err := key.ToCluster(v)
+			if err != nil {
+				t.Fatal(err)
+			}
+			var secret runtime.Object
+			{
+				secret = &v1.Secret{
+					ObjectMeta: metav1.ObjectMeta{
+						Name:      "cluster-certificates",
+						Namespace: key.Namespace(cluster),
+					},
+					Data: map[string][]byte{
+						"token": []byte("my-token"),
+					},
+				}
+			}
+
+			var k8sClient k8sclient.Interface
+			{
+				c := k8sclient.ClientsConfig{
+					Logger:        logger,
+					SchemeBuilder: k8sclient.SchemeBuilder(v1.SchemeBuilder),
+				}
+				k8sClient, err = fake.NewClients(c, secret)
+				if err != nil {
+					t.Fatal(err)
+				}
+			}
+
+			config := Config{
+				Address:            "http://prometheus/cluster",
+				Customer:           "Giant Swarm",
+				EvaluationInterval: "60s",
+				Installation:       "test-installation",
+				Pipeline:           "testing",
+				K8sClient:          k8sClient,
+				Provider:           "provider",
+				Region:             "onprem",
+				ImageRepository:    "giantswarm/prometheus",
+				LogLevel:           "debug",
+				Registry:           "quay.io",
+				RetentionDuration:  "2w",
+				ScrapeInterval:     "60s",
+				Version:            "v2.28.1",
+			}
+
 			return toPrometheus(context.Background(), v, config)
 		},
 		Update: *update,

diff --git a/service/controller/resource/monitoring/prometheus/test/case-1-awsconfig.golden b/service/controller/resource/monitoring/prometheus/test/case-1-awsconfig.golden
@@ -22,15 +22,12 @@ spec:
           - key: node-role.kubernetes.io/control-plane
             operator: DoesNotExist
   apiserverConfig:
-    authorization:
-      credentialsFile: /etc/prometheus/secrets/cluster-certificates/token
+    bearerTokenFile: /etc/prometheus/secrets/cluster-certificates/token
     host: https://master.alice:443
     tlsConfig:
       ca: {}
       caFile: /etc/prometheus/secrets/cluster-certificates/ca
       cert: {}
-      certFile: /etc/prometheus/secrets/cluster-certificates/crt
-      keyFile: /etc/prometheus/secrets/cluster-certificates/key
   arbitraryFSAccessThroughSMs: {}
   enableFeatures:
   - remote-write-receiver

diff --git a/service/controller/resource/monitoring/prometheus/test/case-2-azureconfig.golden b/service/controller/resource/monitoring/prometheus/test/case-2-azureconfig.golden
@@ -22,15 +22,12 @@ spec:
           - key: node-role.kubernetes.io/control-plane
             operator: DoesNotExist
   apiserverConfig:
-    authorization:
-      credentialsFile: /etc/prometheus/secrets/cluster-certificates/token
+    bearerTokenFile: /etc/prometheus/secrets/cluster-certificates/token
     host: https://master.foo:443
     tlsConfig:
       ca: {}
       caFile: /etc/prometheus/secrets/cluster-certificates/ca
       cert: {}
-      certFile: /etc/prometheus/secrets/cluster-certificates/crt
-      keyFile: /etc/prometheus/secrets/cluster-certificates/key
   arbitraryFSAccessThroughSMs: {}
   enableFeatures:
   - remote-write-receiver

diff --git a/service/controller/resource/monitoring/prometheus/test/case-3-kvmconfig.golden b/service/controller/resource/monitoring/prometheus/test/case-3-kvmconfig.golden
@@ -22,15 +22,12 @@ spec:
           - key: node-role.kubernetes.io/control-plane
             operator: DoesNotExist
   apiserverConfig:
-    authorization:
-      credentialsFile: /etc/prometheus/secrets/cluster-certificates/token
+    bearerTokenFile: /etc/prometheus/secrets/cluster-certificates/token
     host: https://master.bar:443
     tlsConfig:
       ca: {}
       caFile: /etc/prometheus/secrets/cluster-certificates/ca
       cert: {}
-      certFile: /etc/prometheus/secrets/cluster-certificates/crt
-      keyFile: /etc/prometheus/secrets/cluster-certificates/key
   arbitraryFSAccessThroughSMs: {}
   enableFeatures:
   - remote-write-receiver

diff --git a/service/controller/resource/monitoring/prometheus/test/case-5-cluster-api-v1alpha3.golden b/service/controller/resource/monitoring/prometheus/test/case-5-cluster-api-v1alpha3.golden
@@ -22,15 +22,12 @@ spec:
           - key: node-role.kubernetes.io/control-plane
             operator: DoesNotExist
   apiserverConfig:
-    authorization:
-      credentialsFile: /etc/prometheus/secrets/cluster-certificates/token
+    bearerTokenFile: /etc/prometheus/secrets/cluster-certificates/token
     host: https://master.baz:443
     tlsConfig:
       ca: {}
       caFile: /etc/prometheus/secrets/cluster-certificates/ca
       cert: {}
-      certFile: /etc/prometheus/secrets/cluster-certificates/crt
-      keyFile: /etc/prometheus/secrets/cluster-certificates/key
   arbitraryFSAccessThroughSMs: {}
   enableFeatures:
   - remote-write-receiver

diff --git a/service/controller/resource/monitoring/scrapeconfigs/resource.go b/service/controller/resource/monitoring/scrapeconfigs/resource.go
@@ -48,6 +48,7 @@ type Config struct {
 type TemplateData struct {
 	AdditionalScrapeConfigs   string
 	APIServerURL              string
+	AuthenticationType        string
 	Bastions                  []string
 	Provider                  string
 	ClusterID                 string
@@ -176,9 +177,18 @@ func getTemplateData(ctx context.Context, ctrlClient client.Client, cluster meta
 		return nil, microerror.Mask(err)
 	}
 
+	var authenticationType = ""
+	if !key.IsManagementCluster(config.Installation, cluster) {
+		authenticationType, err = key.ApiServerAuthenticationType(ctx, config.K8sClient, key.Namespace(cluster))
+		if err != nil {
+			return nil, microerror.Mask(err)
+		}
+	}
+
 	d := &TemplateData{
 		AdditionalScrapeConfigs:   config.AdditionalScrapeConfigs,
 		APIServerURL:              key.APIUrl(cluster),
+		AuthenticationType:        authenticationType,
 		Bastions:                  config.Bastions,
 		ClusterID:                 key.ClusterID(cluster),
 		ClusterType:               key.ClusterType(config.Installation, cluster),