From ebfdbff70c179f40ce3ee2c5e0623b58ff60dde1 Mon Sep 17 00:00:00 2001 From: David Kirwan Date: Tue, 23 Apr 2019 15:35:42 +0100 Subject: [PATCH] Adding oauth-proxy sidecar to prometheus and grafana --- deploy/roles/prometheus-clusterrole.yaml | 12 +++++++ .../applicationmonitoring_controller.go | 4 +-- .../applicationmonitoring/templateHelper.go | 25 ++++++++++++++ templates/prometheus-proxy-secret.yaml | 11 +++++++ templates/prometheus-route.yaml | 2 +- templates/prometheus-service-account.yaml | 4 ++- templates/prometheus-service.yaml | 6 ++-- templates/prometheus.yaml | 33 ++++++++++++++++++- 8 files changed, 90 insertions(+), 7 deletions(-) create mode 100644 templates/prometheus-proxy-secret.yaml diff --git a/deploy/roles/prometheus-clusterrole.yaml b/deploy/roles/prometheus-clusterrole.yaml index dc29ffae..0496848d 100644 --- a/deploy/roles/prometheus-clusterrole.yaml +++ b/deploy/roles/prometheus-clusterrole.yaml @@ -3,6 +3,18 @@ kind: ClusterRole metadata: name: prometheus-application-monitoring rules: +- apiGroups: + - authentication.k8s.io + resources: + - tokenreviews + verbs: + - create +- apiGroups: + - authorization.k8s.io + resources: + - subjectaccessreviews + verbs: + - create - apiGroups: [""] resources: - nodes diff --git a/pkg/controller/applicationmonitoring/applicationmonitoring_controller.go b/pkg/controller/applicationmonitoring/applicationmonitoring_controller.go index fe311bcd..b463ab20 100644 --- a/pkg/controller/applicationmonitoring/applicationmonitoring_controller.go +++ b/pkg/controller/applicationmonitoring/applicationmonitoring_controller.go @@ -142,7 +142,7 @@ func (r *ReconcileApplicationMonitoring) Reconcile(request reconcile.Request) (r func (r *ReconcileApplicationMonitoring) InstallPrometheusOperator(cr *applicationmonitoringv1alpha1.ApplicationMonitoring) (reconcile.Result, error) { log.Info("Phase: Install PrometheusOperator") - for _, resourceName := range []string{PrometheusOperatorServiceAccountName, PrometheusOperatorName} { + for _, resourceName := range []string{PrometheusOperatorServiceAccountName, PrometheusOperatorName, PrometheusProxySecretsName} { if _, err := r.CreateResource(cr, resourceName); err != nil { log.Info(fmt.Sprintf("Error in InstallPrometheusOperator, resourceName=%s : err=%s", resourceName, err)) // Requeue so it can be attempted again @@ -168,7 +168,7 @@ func (r *ReconcileApplicationMonitoring) CreatePrometheusCRs(cr *applicationmoni return reconcile.Result{Requeue: true}, err } - for _, resourceName := range []string{PrometheusServiceAccountName, PrometheusServiceName, PrometheusRouteName, PrometheusCrName} { + for _, resourceName := range []string{PrometheusServiceAccountName, PrometheusServiceName, PrometheusCrName} { if _, err := r.CreateResource(cr, resourceName); err != nil { log.Info(fmt.Sprintf("Error in CreatePrometheusCRs, resourceName=%s : err=%s", resourceName, err)) // Requeue so it can be attempted again diff --git a/pkg/controller/applicationmonitoring/templateHelper.go b/pkg/controller/applicationmonitoring/templateHelper.go index b780a3ea..17f5cceb 100644 --- a/pkg/controller/applicationmonitoring/templateHelper.go +++ b/pkg/controller/applicationmonitoring/templateHelper.go @@ -2,6 +2,8 @@ package applicationmonitoring import ( "bytes" + "crypto/rand" + "encoding/base64" "fmt" "io/ioutil" "os" @@ -21,6 +23,7 @@ const ( PrometheusOperatorServiceAccountName = "prometheus-operator-service-account" PrometheusCrName = "prometheus" PrometheusRouteName = "prometheus-route" + PrometheusProxySecretsName = "prometheus-proxy-secret" PrometheusServiceAccountName = "prometheus-service-account" PrometheusServiceName = "prometheus-service" AlertManagerServiceAccountName = "alertmanager-service-account" @@ -45,6 +48,7 @@ type Parameters struct { PrometheusCrName string PrometheusRouteName string PrometheusServiceName string + PrometheusSessionSecret string AlertManagerServiceAccountName string AlertManagerCrName string AlertManagerServiceName string @@ -76,6 +80,7 @@ func newTemplateHelper(cr *applicationmonitoring.ApplicationMonitoring, extraPar PrometheusCrName: PrometheusCrName, PrometheusRouteName: PrometheusRouteName, PrometheusServiceName: PrometheusServiceName, + PrometheusSessionSecret: PopulatePrometheusProxySecret(), AlertManagerServiceAccountName: AlertManagerServiceAccountName, AlertManagerCrName: AlertManagerCrName, AlertManagerServiceName: AlertManagerServiceName, @@ -102,6 +107,15 @@ func newTemplateHelper(cr *applicationmonitoring.ApplicationMonitoring, extraPar } } +// Populate the PrometheusServiceName values +func PopulatePrometheusProxySecret() string { + p, err := GeneratePassword(43) + if err != nil { + log.Info("Error creating PopulatePrometheusProxySecret") + } + return p +} + // load a templates from a given resource name. The templates must be located // under ./templates and the filename must be .yaml func (h *TemplateHelper) loadTemplate(name string) ([]byte, error) { @@ -124,3 +138,14 @@ func (h *TemplateHelper) loadTemplate(name string) ([]byte, error) { return buffer.Bytes(), nil } + +// GeneratePassword returns a base64 encoded securely random bytes. +func GeneratePassword(n int) (string, error) { + b := make([]byte, n) + _, err := rand.Read(b) + if err != nil { + return "", err + } + + return base64.StdEncoding.EncodeToString(b), err +} diff --git a/templates/prometheus-proxy-secret.yaml b/templates/prometheus-proxy-secret.yaml new file mode 100644 index 00000000..a4a3c812 --- /dev/null +++ b/templates/prometheus-proxy-secret.yaml @@ -0,0 +1,11 @@ +apiVersion: v1 +data: + session_secret: >- + {{.PrometheusSessionSecret}} +kind: Secret +metadata: + labels: + k8s-app: prometheus-k8s + name: prometheus-k8s-proxy + namespace: {{.Namespace }} +type: Opaque \ No newline at end of file diff --git a/templates/prometheus-route.yaml b/templates/prometheus-route.yaml index ee616b27..3271c82b 100644 --- a/templates/prometheus-route.yaml +++ b/templates/prometheus-route.yaml @@ -7,7 +7,7 @@ spec: port: targetPort: web tls: - termination: edge + termination: Reencrypt to: kind: Service name: {{ .PrometheusServiceName }} diff --git a/templates/prometheus-service-account.yaml b/templates/prometheus-service-account.yaml index 03031d36..843ddb5d 100644 --- a/templates/prometheus-service-account.yaml +++ b/templates/prometheus-service-account.yaml @@ -2,4 +2,6 @@ apiVersion: v1 kind: ServiceAccount metadata: name: prometheus-application-monitoring - namespace: {{ .Namespace }} \ No newline at end of file + namespace: {{ .Namespace }} + annotations: + serviceaccounts.openshift.io/oauth-redirectreference.primary: '{"kind":"OAuthRedirectReference","apiVersion":"v1","reference":{"kind":"Route","name":"{{ .PrometheusRouteName }}"}}' \ No newline at end of file diff --git a/templates/prometheus-service.yaml b/templates/prometheus-service.yaml index 979cf9a8..5b320946 100644 --- a/templates/prometheus-service.yaml +++ b/templates/prometheus-service.yaml @@ -3,15 +3,17 @@ kind: Service metadata: name: {{ .PrometheusServiceName }} namespace: {{ .Namespace }} + annotations: + service.alpha.openshift.io/serving-cert-secret-name: prometheus-k8s-tls labels: application-monitoring: "true" spec: type: ClusterIP ports: - name: web - port: 9090 + port: 9091 protocol: TCP - targetPort: web + targetPort: oproxy selector: app: prometheus sessionAffinity: None \ No newline at end of file diff --git a/templates/prometheus.yaml b/templates/prometheus.yaml index 1cf5f5e5..879a9a90 100644 --- a/templates/prometheus.yaml +++ b/templates/prometheus.yaml @@ -6,6 +6,34 @@ metadata: labels: prometheus: {{ .ApplicationMonitoringName }} spec: + containers: + - args: + - '-provider=openshift' + - '-https-address=:9091' + - '-http-address=' + - '-email-domain=*' + - '-upstream=http://localhost:9090' + - '-openshift-service-account=prometheus-application-monitoring' + - '-openshift-sar={"resource": "namespaces", "verb": "get"}' + - '-openshift-delegate-urls={"/": {"resource": "namespaces", "verb":"get"}}' + - '-tls-cert=/etc/tls/private/tls.crt' + - '-tls-key=/etc/tls/private/tls.key' + - '-client-secret-file=/var/run/secrets/kubernetes.io/serviceaccount/token' + - '-cookie-secret-file=/etc/proxy/secrets/session_secret' + - '-openshift-ca=/etc/pki/tls/cert.pem' + - '-openshift-ca=/var/run/secrets/kubernetes.io/serviceaccount/ca.crt' + - '-skip-auth-regex=^/metrics' + image: 'registry.redhat.io/openshift3/oauth-proxy:v3.11.43' + name: prometheus-proxy + ports: + - containerPort: 9091 + name: oproxy + resources: {} + volumeMounts: + - mountPath: /etc/tls/private + name: secret-prometheus-k8s-tls + - mountPath: /etc/proxy/secrets + name: secret-prometheus-k8s-proxy externalUrl: https://{{ index .ExtraParams "prometheusHost" }} alerting: alertmanagers: @@ -15,6 +43,9 @@ spec: resources: requests: memory: 400Mi + secrets: + - prometheus-k8s-tls + - prometheus-k8s-proxy serviceAccountName: prometheus-application-monitoring serviceMonitorNamespaceSelector: matchLabels: @@ -27,4 +58,4 @@ spec: monitoring-key: {{ .MonitoringKey }} ruleNamespaceSelector: matchLabels: - monitoring-key: {{ .MonitoringKey }} \ No newline at end of file + monitoring-key: {{ .MonitoringKey }}