diff --git a/README.md b/README.md index 194cef7f2..6aee39ed4 100644 --- a/README.md +++ b/README.md @@ -13,6 +13,7 @@ Etcd-backup-restore is collection of components to backup and restore the [etcd] * [Getting started](doc/usage/getting_started.md) * [Manual restoration](doc/usage/manual_restoration.md) * [Monitoring](doc/usage/metrics.md) +* [Generating SSL certificates](doc/usage/generating_ssl_certificates.md) ### Design and Proposals diff --git a/chart/etcd-backup-restore/templates/etcd-bootstrap-configmap.yaml b/chart/etcd-backup-restore/templates/etcd-bootstrap-configmap.yaml index db3d87830..8e3d15fe5 100644 --- a/chart/etcd-backup-restore/templates/etcd-bootstrap-configmap.yaml +++ b/chart/etcd-backup-restore/templates/etcd-bootstrap-configmap.yaml @@ -13,6 +13,13 @@ data: #!/bin/sh VALIDATION_MARKER=/var/etcd/data/validation_marker +{{- if .Values.backupRestoreTLS }} + # install wget from apk in order to pass --ca-certificate flag because + # busybox wget only has bare minimum features, without --ca-certificate option + apk update + apk add wget +{{- end }} + trap_and_propagate() { PID=$1 shift @@ -35,11 +42,11 @@ data: check_and_start_etcd(){ while true; do - wget "http://localhost:{{ .Values.servicePorts.backupRestore }}/initialization/status" -S -O status; + wget {{ if .Values.backupRestoreTLS }}--ca-certificate=/var/etcdbr/ssl/ca/ca.crt "https{{ else }}"http{{ end }}://localhost:{{ .Values.servicePorts.backupRestore }}/initialization/status" -S -O status; STATUS=`cat status`; case $STATUS in "New") - wget "http://localhost:{{ .Values.servicePorts.backupRestore }}/initialization/start?mode=$1{{- if .Values.backup.failBelowRevision }}&failbelowrevision={{ int $.Values.backup.failBelowRevision }}{{- end }}" -S -O - ;; + wget {{ if .Values.backupRestoreTLS }}--ca-certificate=/var/etcdbr/ssl/ca/ca.crt "https{{ else }}"http{{ end }}://localhost:{{ .Values.servicePorts.backupRestore }}/initialization/start?mode=$1{{- if .Values.backup.failBelowRevision }}&failbelowrevision={{ int $.Values.backup.failBelowRevision }}{{- end }}" -S -O - ;; "Progress") sleep 1; continue;; @@ -83,7 +90,7 @@ data: data-dir: /var/etcd/data/new.etcd # metrics configuration - metrics: {{ .Values.metrics }} + metrics: basic # Number of committed transactions to trigger a snapshot to disk. snapshot-count: 75000 @@ -95,11 +102,11 @@ data: {{- end }} # List of comma separated URLs to listen on for client traffic. - listen-client-urls: {{ if .Values.tls }}https{{ else }}http{{ end }}://0.0.0.0:{{ .Values.servicePorts.client }} + listen-client-urls: {{ if .Values.etcdTLS }}https{{ else }}http{{ end }}://0.0.0.0:{{ .Values.servicePorts.client }} # List of this member's client URLs to advertise to the public. # The URLs needed to be a comma-separated list. - advertise-client-urls: {{ if .Values.tls }}https{{ else }}http{{ end }}://0.0.0.0:{{ .Values.servicePorts.client }} + advertise-client-urls: {{ if .Values.etcdTLS }}https{{ else }}http{{ end }}://0.0.0.0:{{ .Values.servicePorts.client }} # Initial cluster token for the etcd cluster during bootstrap. initial-cluster-token: 'new' @@ -107,7 +114,7 @@ data: # Initial cluster state ('new' or 'existing'). initial-cluster-state: 'new' -{{- if .Values.tls }} +{{- if .Values.etcdTLS }} client-transport-security: # Path to the client server TLS cert file. cert-file: /var/etcd/ssl/tls/tls.crt diff --git a/chart/etcd-backup-restore/templates/etcd-ca-secret.yaml b/chart/etcd-backup-restore/templates/etcd-ca-secret.yaml index ff6154d16..7a4abc01c 100644 --- a/chart/etcd-backup-restore/templates/etcd-ca-secret.yaml +++ b/chart/etcd-backup-restore/templates/etcd-ca-secret.yaml @@ -1,5 +1,6 @@ -{{- if .Values.tls }} +{{- if .Values.etcdTLS }} apiVersion: v1 +kind: Secret metadata: name: {{ .Release.Name }}-etcd-ca namespace: {{ .Release.Namespace }} @@ -9,6 +10,5 @@ metadata: app.kubernetes.io/instance: {{ .Release.Name }} type: Opaque data: - ca.crt: {{ .Values.tls.caBundle | b64enc }} -kind: Secret + ca.crt: {{ .Values.etcdTLS.caBundle | b64enc }} {{- end }} \ No newline at end of file diff --git a/chart/etcd-backup-restore/templates/etcd-statefulset.yaml b/chart/etcd-backup-restore/templates/etcd-statefulset.yaml index 760a50286..c11505742 100644 --- a/chart/etcd-backup-restore/templates/etcd-statefulset.yaml +++ b/chart/etcd-backup-restore/templates/etcd-statefulset.yaml @@ -35,6 +35,9 @@ spec: - /var/etcd/bin/bootstrap.sh readinessProbe: httpGet: +{{- if .Values.backupRestoreTLS }} + scheme: HTTPS +{{- end }} path: /healthz port: {{ .Values.servicePorts.backupRestore }} initialDelaySeconds: 5 @@ -46,10 +49,12 @@ spec: - -ec - ETCDCTL_API=3 - etcdctl +{{ if .Values.etcdTLS }} - --cert=/var/etcd/ssl/tls/tls.crt - --key=/var/etcd/ssl/tls/tls.key - --cacert=/var/etcd/ssl/ca/ca.crt - - --endpoints={{ if .Values.tls }}https{{ else }}http{{ end }}://{{ .Release.Name }}-etcd-0:{{ .Values.servicePorts.client }} +{{ end }} + - --endpoints={{ if .Values.etcdTLS }}https{{ else }}http{{ end }}://{{ .Release.Name }}-etcd-0:{{ .Values.servicePorts.client }} {{- if and .Values.etcdAuth.username .Values.etcdAuth.password }} - --user={{ .Values.etcdAuth.username }}:{{ .Values.etcdAuth.password }} {{- end }} @@ -73,11 +78,15 @@ spec: mountPath: /var/etcd/bin/ - name: etcd-config-file mountPath: /var/etcd/config/ -{{- if .Values.tls }} +{{- if .Values.etcdTLS }} - name: ca-etcd mountPath: /var/etcd/ssl/ca - name: etcd-tls mountPath: /var/etcd/ssl/tls +{{- end }} +{{- if .Values.backupRestoreTLS }} + - name: ca-etcdbr + mountPath: /var/etcdbr/ssl/ca {{- end }} - name: backup-restore command: @@ -94,7 +103,7 @@ spec: {{- if .Values.backup.etcdQuotaBytes }} - --embedded-etcd-quota-bytes={{ int $.Values.backup.etcdQuotaBytes }} {{- end }} -{{- if .Values.tls }} +{{- if .Values.etcdTLS }} - --cert=/var/etcd/ssl/tls/tls.crt - --key=/var/etcd/ssl/tls/tls.key - --cacert=/var/etcd/ssl/ca/ca.crt @@ -115,6 +124,10 @@ spec: {{- if and .Values.etcdAuth.username .Values.etcdAuth.password }} - --etcd-username={{ .Values.etcdAuth.username }} - --etcd-password={{ .Values.etcdAuth.password }} +{{- end }} +{{- if .Values.backupRestoreTLS }} + - --server-cert=/var/etcdbr/ssl/tls/tls.crt + - --server-key=/var/etcdbr/ssl/tls/tls.key {{- end }} image: {{ .Values.images.etcdBackupRestore.repository }}:{{ .Values.images.etcdBackupRestore.tag }} imagePullPolicy: {{ .Values.images.etcdBackupRestore.pullPolicy }} @@ -205,12 +218,18 @@ spec: mountPath: /var/etcd/data/ - name: etcd-config-file mountPath: /var/etcd/config/ -{{- if .Values.tls }} +{{- if .Values.etcdTLS }} - name: ca-etcd mountPath: /var/etcd/ssl/ca - name: etcd-tls mountPath: /var/etcd/ssl/tls {{- end }} +{{- if .Values.backupRestoreTLS }} + - name: ca-etcdbr + mountPath: /var/etcdbr/ssl/ca + - name: etcdbr-tls + mountPath: /var/etcdbr/ssl/tls +{{- end }} {{- if eq .Values.backup.storageProvider "GCS" }} - name: etcd-backup mountPath: "/root/.gcp/" @@ -230,13 +249,21 @@ spec: items: - key: etcd.conf.yaml path: etcd.conf.yaml -{{- if .Values.tls }} +{{- if .Values.etcdTLS }} + - name: ca-etcd + secret: + secretName: {{ .Release.Name }}-etcd-ca - name: etcd-tls secret: secretName: {{ .Release.Name }}-etcd-tls - - name: ca-etcd +{{- end }} +{{- if .Values.backupRestoreTLS }} + - name: ca-etcdbr secret: - secretName: {{ .Release.Name }}-etcd-ca + secretName: {{ .Release.Name }}-etcdbr-ca + - name: etcdbr-tls + secret: + secretName: {{ .Release.Name }}-etcdbr-tls {{- end }} {{- if and .Values.backup.storageProvider (not (eq .Values.backup.storageProvider "Local")) }} - name: etcd-backup diff --git a/chart/etcd-backup-restore/templates/etcd-tls-secret.yaml b/chart/etcd-backup-restore/templates/etcd-tls-secret.yaml index c4c6b2c6e..b5bfc3239 100644 --- a/chart/etcd-backup-restore/templates/etcd-tls-secret.yaml +++ b/chart/etcd-backup-restore/templates/etcd-tls-secret.yaml @@ -1,4 +1,4 @@ -{{- if .Values.tls }} +{{- if .Values.etcdTLS }} apiVersion: v1 kind: Secret metadata: @@ -10,6 +10,6 @@ metadata: app.kubernetes.io/instance: {{ .Release.Name }} type: kubernetes.io/tls data: - tls.crt: {{ .Values.tls.crt | b64enc }} - tls.key: {{ .Values.tls.key | b64enc }} + tls.crt: {{ .Values.etcdTLS.crt | b64enc }} + tls.key: {{ .Values.etcdTLS.key | b64enc }} {{- end }} \ No newline at end of file diff --git a/chart/etcd-backup-restore/templates/etcdbr-ca-secret.yaml b/chart/etcd-backup-restore/templates/etcdbr-ca-secret.yaml new file mode 100644 index 000000000..ca456fa05 --- /dev/null +++ b/chart/etcd-backup-restore/templates/etcdbr-ca-secret.yaml @@ -0,0 +1,14 @@ +{{- if .Values.backupRestoreTLS }} +apiVersion: v1 +kind: Secret +metadata: + name: {{ .Release.Name }}-etcdbr-ca + namespace: {{ .Release.Namespace }} + labels: + app.kubernetes.io/name: etcd + app.kubernetes.io/managed-by: {{ .Release.Service }} + app.kubernetes.io/instance: {{ .Release.Name }} +type: Opaque +data: + ca.crt: {{ .Values.backupRestoreTLS.caBundle | b64enc }} +{{- end }} \ No newline at end of file diff --git a/chart/etcd-backup-restore/templates/etcdbr-tls-secret.yaml b/chart/etcd-backup-restore/templates/etcdbr-tls-secret.yaml new file mode 100644 index 000000000..ee8ddf419 --- /dev/null +++ b/chart/etcd-backup-restore/templates/etcdbr-tls-secret.yaml @@ -0,0 +1,15 @@ +{{- if .Values.backupRestoreTLS }} +apiVersion: v1 +kind: Secret +metadata: + name: {{ .Release.Name }}-etcdbr-tls + namespace: {{ .Release.Namespace }} + labels: + app.kubernetes.io/name: etcd + app.kubernetes.io/managed-by: {{ .Release.Service }} + app.kubernetes.io/instance: {{ .Release.Name }} +type: kubernetes.io/tls +data: + tls.crt: {{ .Values.backupRestoreTLS.crt | b64enc }} + tls.key: {{ .Values.backupRestoreTLS.key | b64enc }} +{{- end }} \ No newline at end of file diff --git a/chart/etcd-backup-restore/values.yaml b/chart/etcd-backup-restore/values.yaml index db4461895..3c30edc31 100644 --- a/chart/etcd-backup-restore/values.yaml +++ b/chart/etcd-backup-restore/values.yaml @@ -2,12 +2,12 @@ images: # etcd image to use etcd: repository: quay.io/coreos/etcd - tag: v3.3.12 + tag: v3.3.13 pullPolicy: IfNotPresent # etcd-backup-restore image to use etcdBackupRestore: repository: eu.gcr.io/gardener-project/gardener/etcdbrctl - tag: 0.7.0 + tag: 0.8.0 pullPolicy: IfNotPresent resources: @@ -31,12 +31,7 @@ servicePorts: server: 2380 backupRestore: 8080 -etcdAuth: {} - #username: username - #password: password - backup: - # schedule is cron standard schedule to take full snapshots. schedule: "0 */1 * * *" @@ -90,23 +85,42 @@ backup: # accessKeySecret: secret-access-key-with-object-storage-privileges # accessKeyID: access-key-id-with-object-storage-privileges -metrics: basic - -# tls field contains the pre-created secrets for etcd. Uncomment the -# whole tls section if you dont want to use tls for the etcd. -tls: {} - # caBundle: | - # -----BEGIN CERTIFICATE----- - # ... - # -----END CERTIFICATE----- - # crt: | - # -----BEGIN CERTIFICATE----- - # ... - # -----END CERTIFICATE----- - # key: | - # -----BEGIN RSA PRIVATE KEY----- - # ... - # -----END RSA PRIVATE KEY----- - -# podAnnotations will be placed to the resulting etcd pod +# etcdAuth field contains the pre-created username-password pair +# for etcd. Comment this whole section if you dont want to use +# password-based authentication for the etcd. +etcdAuth: {} + # username: username + # password: password + +etcdTLS: {} +# caBundle: | +# -----BEGIN CERTIFICATE----- +# ... +# -----END CERTIFICATE----- +# crt: | +# -----BEGIN CERTIFICATE----- +# ... +# -----END CERTIFICATE----- +# key: | +# -----BEGIN RSA PRIVATE KEY----- +# ... +# -----END RSA PRIVATE KEY----- + +# backupRestoreTLS field contains the pre-created secrets for backup-restore server. +# Comment this whole section if you dont want to use tls for the backup-restore server. +backupRestoreTLS: {} +# caBundle: | +# -----BEGIN CERTIFICATE----- +# ... +# -----END CERTIFICATE----- +# crt: | +# -----BEGIN CERTIFICATE----- +# ... +# -----END CERTIFICATE----- +# key: | +# -----BEGIN RSA PRIVATE KEY----- +# ... +# -----END RSA PRIVATE KEY----- + +# podAnnotations that will be passed to the resulting etcd pod podAnnotations: {} diff --git a/cmd/server.go b/cmd/server.go index 4e938f26b..b72a12a02 100644 --- a/cmd/server.go +++ b/cmd/server.go @@ -18,6 +18,7 @@ import ( "context" "fmt" "net/http" + "os" "path" "sync/atomic" "time" @@ -102,18 +103,34 @@ func NewServerCommand(ctx context.Context) *cobra.Command { // startHTTPServer creates and starts the HTTP handler // with status 503 (Service Unavailable) -func startHTTPServer(initializer initializer.Initializer, ssr *snapshotter.Snapshotter) *server.HTTPHandler { +func startHTTPServer(initializer initializer.Initializer, ssr *snapshotter.Snapshotter) (*server.HTTPHandler, error) { + enableTLS := serverTLSCertFile != "" && serverTLSKeyFile != "" + if enableTLS { + // Check for existence of server cert and key files before proceeding + if _, err := os.Stat(serverTLSCertFile); err != nil { + logger.Errorf("TLS enabled but server TLS cert file is invalid. Will not start HTTPS server: %v", err) + return nil, err + } + if _, err := os.Stat(serverTLSKeyFile); err != nil { + logger.Errorf("TLS enabled but server TLS key file is invalid. Will not start HTTPS server: %v", err) + return nil, err + } + } + // Start http handler with Error state and wait till snapshotter is up // and running before setting the status to OK. handler := &server.HTTPHandler{ - Port: port, - Initializer: initializer, - Snapshotter: ssr, - Logger: logger, - StopCh: make(chan struct{}), - EnableProfiling: enableProfiling, - ReqCh: make(chan struct{}), - AckCh: make(chan struct{}), + Port: port, + Initializer: initializer, + Snapshotter: ssr, + Logger: logger, + StopCh: make(chan struct{}), + EnableProfiling: enableProfiling, + ReqCh: make(chan struct{}), + AckCh: make(chan struct{}), + EnableTLS: enableTLS, + ServerTLSCertFile: serverTLSCertFile, + ServerTLSKeyFile: serverTLSKeyFile, } handler.SetStatus(http.StatusServiceUnavailable) logger.Info("Registering the http request handlers...") @@ -121,7 +138,7 @@ func startHTTPServer(initializer initializer.Initializer, ssr *snapshotter.Snaps logger.Info("Starting the http server...") go handler.Start() - return handler + return handler, nil } // runServerWithoutSnapshotter runs the etcd-backup-restore @@ -164,7 +181,10 @@ func runServerWithSnapshotter(ctx context.Context, tlsConfig *etcdutil.TLSConfig snapshotterConfig, ) - handler := startHTTPServer(etcdInitializer, ssr) + handler, err := startHTTPServer(etcdInitializer, ssr) + if err != nil { + logger.Fatalf("Failed to start HTTP server: %v", err) + } defer handler.Stop() ssrStopCh := make(chan struct{}) @@ -274,7 +294,10 @@ func runServerWithoutSnapshotter(ctx context.Context, tlsConfig *etcdutil.TLSCon // If no storage provider is given, snapshotter will be nil, in which // case the status is set to OK as soon as etcd probe is successful - handler := startHTTPServer(etcdInitializer, nil) + handler, err := startHTTPServer(etcdInitializer, nil) + if err != nil { + logger.Fatalf("Failed to start HTTP server: %v", err) + } defer handler.Stop() // start defragmentation without trigerring full snapshot @@ -311,12 +334,6 @@ func runEtcdProbeLoopWithoutSnapshotter(ctx context.Context, tlsConfig *etcdutil } } -// initializeServerFlags adds the flags to -func initializeServerFlags(serverCmd *cobra.Command) { - serverCmd.Flags().IntVarP(&port, "server-port", "p", defaultServerPort, "port on which server should listen") - serverCmd.Flags().BoolVar(&enableProfiling, "enable-profiling", false, "enable profiling") -} - // ProbeEtcd will make the snapshotter probe for etcd endpoint to be available // before it starts taking regular snapshots. func ProbeEtcd(tlsConfig *etcdutil.TLSConfig) error { @@ -368,3 +385,11 @@ func handleSsrStopRequest(ctx context.Context, handler *server.HTTPHandler, ssr } } } + +// initializeServerFlags adds the flags to +func initializeServerFlags(serverCmd *cobra.Command) { + serverCmd.Flags().IntVarP(&port, "server-port", "p", defaultServerPort, "port on which server should listen") + serverCmd.Flags().BoolVar(&enableProfiling, "enable-profiling", false, "enable profiling") + serverCmd.Flags().StringVar(&serverTLSCertFile, "server-cert", "", "TLS certificate file for backup-restore server") + serverCmd.Flags().StringVar(&serverTLSKeyFile, "server-key", "", "TLS key file for backup-restore server") +} diff --git a/cmd/types.go b/cmd/types.go index f55461352..b9150a53e 100644 --- a/cmd/types.go +++ b/cmd/types.go @@ -49,8 +49,10 @@ var ( defragmentationSchedule string //server flags - port int - enableProfiling bool + port int + enableProfiling bool + serverTLSCertFile string + serverTLSKeyFile string //restore flags restoreCluster string diff --git a/doc/usage/generating_ssl_certificates.md b/doc/usage/generating_ssl_certificates.md new file mode 100644 index 000000000..cc2f2808c --- /dev/null +++ b/doc/usage/generating_ssl_certificates.md @@ -0,0 +1,160 @@ +# Generating certificates + +If you wish to enable TLS authentication for either etcd or etcdbr server or both, please follow this guide. The SSL certificate configurations given here are meant to facilitate smooth deployment of the etcd setup via the provided [helm chart](../../chart/etcd-backup-restore). + +## Certificates structure + +While deploying the etcd setup via the provided helm chart, TLS can be enabled for the etcd server and/or etcd-backup-restore server by adding the certificate data to the `values.yaml` file as necessary. This data is converted into the respective secrets and mounted onto the pod's containers according to the following directory structure: + +- `etcd` container + +```console +/ +└── var + ├── etcd Contains the CA and server TLS certs for etcd server + | └── ssl + | ├── ca + | | └── ca.crt + | └── tls + | ├── tls.crt + | └── tls.key + └── etcdbr Contains the CA and server TLS certs for etcd backup-restore server + └── ssl + ├── ca + | └── ca.crt + └── tls + ├── tls.crt + └── tls.key +``` + +
+ +- `backup-restore` container + +```console +/ +└── var + ├── etcd Contains the CA and server certs for etcd server + | └── ssl + | ├── ca + | | └── ca.crt + | └── tls + | ├── tls.crt + | └── tls.key + └── etcdbr Contains the CA cert for etcd backup-restore server + └── ssl + └── ca + └── ca.crt +``` + +## Generating the certificates + +### Installing openssl + +```console +# For Mac users +brew install openssl + +# For other flavours of Unix +apk install openssl + +mkdir openssl && cd openssl +``` + +### Generating certs for etcd server authentication + +#### Generating CA cert bundle + +```console +openssl genrsa -out ca.key 2048 +openssl req -new -key ca.key -subj "/CN=etcd" -out ca.csr + +cat > ca.csr.conf < server.csr.conf <