Merge pull request #4 from kubism/feature/s3-server-side-encryption

Add s server side encryption feature for S3

.gitignore

@@ -5,6 +5,7 @@
 !/tools/helm3-install
 /bin
 /reports
+**/certs/*
 
 *.coverprofile
 

Makefile

@@ -27,8 +27,8 @@ WORKER_BIN ?= bin/worker
 DOCKER_TAG ?= latest
 DOCKER_IMG ?= kubismio/backup-operator:$(DOCKER_TAG)
 
-KIND_CLUSTER ?= test
-KIND_IMAGE ?= kindest/node:v1.16.4
+KIND_CLUSTER ?= backup-operator-test
+KIND_CONFIG ?= config/kind/kindconf.yaml
 
 HELM_CHART_NAME ?= backup-operator
 HELM_CHART_DIR ?= charts/$(HELM_CHART_NAME)
@@ -40,7 +40,7 @@ TEST_LONG ?=
 
 export
 
-.PHONY: all test lint fmt vet install uninstall deploy manifests docker-build docker-push tools docker-is-running kind-create kind-delete kind-is-running check-test-long
+.PHONY: all test lint fmt vet install uninstall deploy manifests docker-build docker-push tools docker-is-running kind-create kind-delete kind-is-running check-test-long minio-selfsigned
 
 all: $(MANAGER_BIN) $(WORKER_BIN) tools
 
@@ -50,10 +50,10 @@ $(MANAGER_BIN): generate fmt vet
 $(WORKER_BIN): generate fmt vet
 	$(GO) build -o $(WORKER_BIN) ./cmd/worker/...
 
-test: generate fmt vet manifests docker-is-running kind-is-running check-test-long $(GINKGO) $(KUBEBUILDER) $(HELM3)
+test: generate fmt vet manifests docker-is-running kind-is-running minio-selfsigned check-test-long $(GINKGO) $(KUBEBUILDER) $(HELM3)
 	$(GINKGO) -r -v -cover pkg
 
-test-%: generate fmt vet manifests docker-is-running kind-is-running check-test-long $(GINKGO) $(KUBEBUILDER) $(HELM3)
+test-%: generate fmt vet manifests docker-is-running kind-is-running minio-selfsigned check-test-long $(GINKGO) $(KUBEBUILDER) $(HELM3)
 	$(GINKGO) -r -v -cover pkg/$*
 
 # If e2e/integration tests are running we need to build the image beforehand
@@ -77,6 +77,11 @@ fmt:
 vet:
 	$(GO) vet ./...
 
+# Generate self-signed cert for minio tls
+minio-selfsigned:
+	@mkdir -p pkg/backup/s3/certs
+	@openssl req -x509 -nodes -days 730 -newkey rsa:2048 -keyout pkg/backup/s3/certs/private.key -out pkg/backup/s3/certs/public.crt -config config/test/openssl.conf
+
 # Generate manifests e.g. CRD, RBAC etc.
 manifests: $(CONTROLLER_GEN) $(KUSTOMIZE)
 	$(CONTROLLER_GEN) crd:trivialVersions=false rbac:roleName=manager-role webhook paths="./..." output:crd:artifacts:config=config/crd/bases
@@ -103,7 +108,7 @@ docker-is-running:
 	}
 
 kind-create: $(KIND)
-	$(KIND) create cluster --image $(KIND_IMAGE) --name $(KIND_CLUSTER) --wait 5m
+	$(KIND) create cluster --config $(KIND_CONFIG) --name $(KIND_CLUSTER) --kubeconfig /tmp/kind-$(KIND_CLUSTER)-config --wait 5m
 
 kind-is-running: $(KIND)
 	@echo "Checking if kind cluster with name '$(KIND_CLUSTER)' is running..."
@@ -116,7 +121,7 @@ kind-is-running: $(KIND)
 kind-get-kubeconfig: $(KIND)
 	$(KIND) get kubeconfig --name $(KIND_CLUSTER) > /tmp/kind-$(KIND_CLUSTER)-config
 	@echo "Created untracked config file in '/tmp/kind-$(KIND_CLUSTER)-config. Use as follows:"
-	@echo "export KUBECONFIG=\"/tmp/kind-$(KIND_CLUSTER)-config\""
+	@echo "kubectl --kubeconfig "/tmp/kind-$(KIND_CLUSTER)-config\" get no"
 
 kind-delete: $(KIND)
 	$(KIND) delete cluster --name $(KIND_CLUSTER)

api/v1alpha1/destination_types.go

@@ -33,4 +33,8 @@ type S3 struct {
 	AccessKeyID string `json:"accessKeyID,omitempty"`
 	// +optional
 	SecretAccessKey string `json:"secretAccessKey,omitempty"`
+	// +optional
+	EncryptionKey string `json:"encryptionKey,omitempty"`
+	// +optional
+	EncryptionAlgorithm string `json:"encryptionAlgorithm,omitempty"`
 }

charts/backup-operator/crds/crds.yaml

@@ -53,6 +53,10 @@ spec:
                       type: string
                     bucket:
                       type: string
+                    encryptionAlgorithm:
+                      type: string
+                    encryptionKey:
+                      type: string
                     endpoint:
                       type: string
                     secretAccessKey:
@@ -334,6 +338,10 @@ spec:
                       type: string
                     bucket:
                       type: string
+                    encryptionAlgorithm:
+                      type: string
+                    encryptionKey:
+                      type: string
                     endpoint:
                       type: string
                     secretAccessKey:

cmd/worker/consul.go

@@ -81,8 +81,17 @@ var consulCmd = &cobra.Command{
 		}
 		prefix := fmt.Sprintf("%s/%s", plan.ObjectMeta.Namespace, plan.ObjectMeta.Name)
 		s3c := plan.Spec.Destination.S3
-
-		dst, err := s3.NewS3Destination(s3c.Endpoint, util.FallbackToEnv(s3c.AccessKeyID, "S3_SECRET_ACCESS_KEY"), util.FallbackToEnv(s3c.SecretAccessKey, "S3_SECRET_ACCESS_KEY"), s3c.UseSSL, s3c.Bucket, prefix)
+		conf := &s3.S3DestinationConf{
+			Endpoint:            s3c.Endpoint,
+			AccessKey:           util.FallbackToEnv(s3c.AccessKeyID, "S3_SECRET_ACCESS_KEY"),
+			SecretKey:           util.FallbackToEnv(s3c.SecretAccessKey, "S3_SECRET_ACCESS_KEY"),
+			EncryptionKey:       util.NilIfEmpty(util.FallbackToEnv(s3c.EncryptionKey, "S3_ENCRYPTION_KEY")),
+			EncryptionAlgorithm: util.FallbackToEnv(s3c.EncryptionAlgorithm, "S3_ENCRYPTION_ALGORITHM"),
+			DisableSSL:          !s3c.UseSSL,
+			Bucket:              s3c.Bucket,
+			Prefix:              prefix,
+		}
+		dst, err := s3.NewS3Destination(conf)
 		if err != nil {
 			return err
 		}

cmd/worker/mongodb.go

@@ -83,7 +83,17 @@ var mongodbCmd = &cobra.Command{
 		}
 		prefix := fmt.Sprintf("%s/%s", plan.ObjectMeta.Namespace, plan.ObjectMeta.Name)
 		s3c := plan.Spec.Destination.S3
-		dst, err := s3.NewS3Destination(s3c.Endpoint, util.FallbackToEnv(s3c.AccessKeyID, "S3_SECRET_ACCESS_KEY"), util.FallbackToEnv(s3c.SecretAccessKey, "S3_SECRET_ACCESS_KEY"), s3c.UseSSL, s3c.Bucket, prefix)
+		conf := &s3.S3DestinationConf{
+			Endpoint:            s3c.Endpoint,
+			AccessKey:           util.FallbackToEnv(s3c.AccessKeyID, "S3_SECRET_ACCESS_KEY"),
+			SecretKey:           util.FallbackToEnv(s3c.SecretAccessKey, "S3_SECRET_ACCESS_KEY"),
+			EncryptionKey:       util.NilIfEmpty(util.FallbackToEnv(s3c.EncryptionKey, "S3_ENCRYPTION_KEY")),
+			EncryptionAlgorithm: util.FallbackToEnv(s3c.EncryptionAlgorithm, "S3_ENCRYPTION_ALGORITHM"),
+			DisableSSL:          !s3c.UseSSL,
+			Bucket:              s3c.Bucket,
+			Prefix:              prefix,
+		}
+		dst, err := s3.NewS3Destination(conf)
 		if err != nil {
 			return err
 		}

config/crd/bases/backup.kubism.io_consulbackupplans.yaml

@@ -53,6 +53,10 @@ spec:
                       type: string
                     bucket:
                       type: string
+                    encryptionAlgorithm:
+                      type: string
+                    encryptionKey:
+                      type: string
                     endpoint:
                       type: string
                     secretAccessKey:

config/crd/bases/backup.kubism.io_mongodbbackupplans.yaml

@@ -49,6 +49,10 @@ spec:
                       type: string
                     bucket:
                       type: string
+                    encryptionAlgorithm:
+                      type: string
+                    encryptionKey:
+                      type: string
                     endpoint:
                       type: string
                     secretAccessKey:

config/kind/kindconf.yaml

@@ -0,0 +1,19 @@
+kind: Cluster
+apiVersion: kind.sigs.k8s.io/v1alpha3
+# patch the generated kubeadm config with a featuregate
+kubeadmConfigPatches:
+  - |
+    apiVersion: kubeadm.k8s.io/v1beta2
+    kind: ClusterConfiguration
+    metadata:
+      name: config
+    apiServer:
+      extraArgs:
+        "service-account-issuer": "kubernetes.default.svc"
+        "service-account-signing-key-file": "/etc/kubernetes/pki/sa.key"
+nodes:
+  # the control plane node config
+  - role: control-plane
+    image: kindest/node:v1.15.7@sha256:e2df133f80ef633c53c0200114fce2ed5e1f6947477dbc83261a6a921169488d
+  - role: worker
+    image: kindest/node:v1.15.7@sha256:e2df133f80ef633c53c0200114fce2ed5e1f6947477dbc83261a6a921169488d

config/test/openssl.conf

@@ -0,0 +1,19 @@
+[req]
+distinguished_name = req_distinguished_name
+x509_extensions = v3_req
+prompt = no
+
+[req_distinguished_name]
+C = US
+ST = VA
+L = Somewhere
+O = MyOrg
+OU = MyOU
+CN = MyServerName
+
+[v3_req]
+subjectAltName = @alt_names
+
+[alt_names]
+IP.1 = 127.0.0.1
+DNS.1 = localhost

go.mod

@@ -12,7 +12,7 @@ require (
 	github.com/mongodb/mongo-tools-common v2.0.3+incompatible
 	github.com/onsi/ginkgo v1.11.0
 	github.com/onsi/gomega v1.8.1
-	github.com/ory/dockertest/v3 v3.5.5
+	github.com/ory/dockertest/v3 v3.6.0
 	github.com/prometheus/client_golang v1.2.0
 	github.com/spf13/cobra v0.0.5
 	github.com/xdg/stringprep v1.0.0 // indirect

go.sum

@@ -359,8 +359,8 @@ github.com/opencontainers/runc v1.0.0-rc9 h1:/k06BMULKF5hidyoZymkoDCzdJzltZpz/UU
 github.com/opencontainers/runc v1.0.0-rc9/go.mod h1:qT5XzbpPznkRYVz/mWwUaVBUv2rmF59PVA73FjuZG0U=
 github.com/ory/dockertest v3.3.5+incompatible h1:iLLK6SQwIhcbrG783Dghaaa3WPzGc+4Emza6EbVUUGA=
 github.com/ory/dockertest v3.3.5+incompatible/go.mod h1:1vX4m9wsvi00u5bseYwXaSnhNrne+V0E6LAcBILJdPs=
-github.com/ory/dockertest/v3 v3.5.5 h1:bhWoPN3xmEHIdZHflA3vOf7KaHFbCOLoRnkCLhZ3eQs=
-github.com/ory/dockertest/v3 v3.5.5/go.mod h1:4ZOpj8qBUmh8fcBSVzkH2bws2s91JdGvHUqan4GHEuQ=
+github.com/ory/dockertest/v3 v3.6.0 h1:I6KNJ6izxGduLACQii2SP/g7GN0JM9Xfaik6aAVaw6Y=
+github.com/ory/dockertest/v3 v3.6.0/go.mod h1:4ZOpj8qBUmh8fcBSVzkH2bws2s91JdGvHUqan4GHEuQ=
 github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc=
 github.com/pborman/uuid v1.2.0/go.mod h1:X/NO0urCmaxf9VXbdlT7C2Yzkj2IKimNn4k+gtPdI/k=
 github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic=

pkg/backup/s3/s3_const.go

@@ -0,0 +1,19 @@
+/*
+Copyright 2020 Backup Operator Authors
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package s3
+
+const DefaultEncryptionAlgorithm = "AES256"

pkg/backup/s3/s3_destination.go

@@ -17,6 +17,8 @@ limitations under the License.
 package s3
 
 import (
+	"crypto/tls"
+	"net/http"
 	"path/filepath"
 	"sort"
 
@@ -31,21 +33,39 @@ import (
 	"github.com/aws/aws-sdk-go/service/s3/s3manager"
 )
 
-func NewS3Destination(endpoint, accessKeyID, secretAccessKey string, useSSL bool, bucket, prefix string) (*S3Destination, error) {
+type S3DestinationConf struct {
+	Endpoint            string
+	AccessKey           string
+	SecretKey           string
+	EncryptionKey       *string
+	EncryptionAlgorithm string
+	DisableSSL          bool
+	InsecureSkipVerify  bool
+	Bucket              string
+	Prefix              string
+}
+
+func NewS3Destination(conf *S3DestinationConf) (*S3Destination, error) {
 	newSession, err := session.NewSession(&aws.Config{
-		Credentials:      credentials.NewStaticCredentials(accessKeyID, secretAccessKey, ""),
-		Endpoint:         aws.String(endpoint),
+		Credentials:      credentials.NewStaticCredentials(conf.AccessKey, conf.SecretKey, ""),
+		Endpoint:         aws.String(conf.Endpoint),
 		Region:           aws.String("us-east-1"),
-		DisableSSL:       aws.Bool(!useSSL),
+		DisableSSL:       aws.Bool(conf.DisableSSL),
 		S3ForcePathStyle: aws.Bool(true),
 	})
 	if err != nil {
 		return nil, err
 	}
-	client := s3.New(newSession)
+
+	tr := &http.Transport{
+		TLSClientConfig: &tls.Config{InsecureSkipVerify: conf.InsecureSkipVerify},
+	}
+	cl := &http.Client{Transport: tr}
+	client := s3.New(newSession, aws.NewConfig().WithHTTPClient(cl))
+
 	// Create bucket, if not exists
 	_, err = client.CreateBucket(&s3.CreateBucketInput{
-		Bucket: aws.String(bucket),
+		Bucket: aws.String(conf.Bucket),
 	})
 	if err != nil { // If bucket already exists ignore error
 		if aerr, ok := err.(awserr.Error); ok {
@@ -57,22 +77,26 @@ func NewS3Destination(endpoint, accessKeyID, secretAccessKey string, useSSL bool
 		}
 	}
 	return &S3Destination{
-		Session:  newSession,
-		Client:   client,
-		Uploader: s3manager.NewUploader(newSession),
-		Bucket:   bucket,
-		Prefix:   prefix,
-		log:      logger.WithName("s3dst"),
+		Session:             newSession,
+		Client:              client,
+		EncryptionKey:       conf.EncryptionKey,
+		EncryptionAlgorithm: conf.EncryptionAlgorithm,
+		Uploader:            s3manager.NewUploaderWithClient(client),
+		Bucket:              conf.Bucket,
+		Prefix:              conf.Prefix,
+		log:                 logger.WithName("s3dst"),
 	}, nil
 }
 
 type S3Destination struct {
-	Session  *session.Session
-	Client   *s3.S3
-	Uploader *s3manager.Uploader
-	Bucket   string
-	Prefix   string
-	log      logger.Logger
+	Session             *session.Session
+	Client              *s3.S3
+	EncryptionKey       *string
+	EncryptionAlgorithm string
+	Uploader            *s3manager.Uploader
+	Bucket              string
+	Prefix              string
+	log                 logger.Logger
 }
 
 func (s *S3Destination) Store(obj backup.Object) (int64, error) {
@@ -82,16 +106,38 @@ func (s *S3Destination) Store(obj backup.Object) (int64, error) {
 		Key:    &key,
 		Body:   obj.Data,
 	}
+
+	if s.EncryptionKey != nil {
+		if s.EncryptionAlgorithm == "" {
+			params.SSECustomerAlgorithm = aws.String(DefaultEncryptionAlgorithm)
+		} else {
+			params.SSECustomerAlgorithm = &s.EncryptionAlgorithm
+		}
+		params.SSECustomerKey = s.EncryptionKey
+	}
+
 	s.log.Info("upload starting", "bucket", s.Bucket, "key", key)
 	res, err := s.Uploader.Upload(params)
 	if err != nil {
 		return 0, err
 	}
 	s.log.Info("upload successful", "result", res)
-	head, err := s.Client.HeadObject(&s3.HeadObjectInput{
+
+	headObjectInput := &s3.HeadObjectInput{
 		Bucket: &s.Bucket,
 		Key:    &key,
-	})
+	}
+
+	if s.EncryptionKey != nil {
+		if s.EncryptionAlgorithm == "" {
+			headObjectInput.SSECustomerAlgorithm = aws.String(DefaultEncryptionAlgorithm)
+		} else {
+			headObjectInput.SSECustomerAlgorithm = &s.EncryptionAlgorithm
+		}
+		headObjectInput.SSECustomerKey = s.EncryptionKey
+	}
+
+	head, err := s.Client.HeadObject(headObjectInput)
 	if err != nil {
 		return 0, err
 	}