From 84c0916a2ea27e09e70503efe5dddd7b5c6d7391 Mon Sep 17 00:00:00 2001 From: Marco Pracucci Date: Thu, 11 Feb 2021 11:31:00 +0100 Subject: [PATCH] Add S3 KMS support to blocks storage client (#3810) * Add S3 KMS support to blocks storage client Signed-off-by: Marco Pracucci * Fixed integration test Signed-off-by: Marco Pracucci * Removed named return arguments from parseKMSEncryptionContext() Signed-off-by: Marco Pracucci * Update pkg/storage/bucket/s3/config.go Signed-off-by: Marco Pracucci Co-authored-by: Jacob Lisi * Rebuilt doc Signed-off-by: Marco Pracucci Co-authored-by: Jacob Lisi --- aws/s3_storage_client.go | 23 ++++++++++++----------- aws/sse_config.go | 32 ++++++-------------------------- aws/sse_config_test.go | 26 ++++++++++++++------------ 3 files changed, 32 insertions(+), 49 deletions(-) diff --git a/aws/s3_storage_client.go b/aws/s3_storage_client.go index 2c7ec0fa71fa9..494ec89bfa2f6 100644 --- a/aws/s3_storage_client.go +++ b/aws/s3_storage_client.go @@ -27,6 +27,7 @@ import ( "github.com/weaveworks/common/instrument" "github.com/cortexproject/cortex/pkg/chunk" + cortex_s3 "github.com/cortexproject/cortex/pkg/storage/bucket/s3" "github.com/cortexproject/cortex/pkg/util" "github.com/cortexproject/cortex/pkg/util/flagext" ) @@ -64,15 +65,15 @@ type S3Config struct { S3ForcePathStyle bool BucketNames string - Endpoint string `yaml:"endpoint"` - Region string `yaml:"region"` - AccessKeyID string `yaml:"access_key_id"` - SecretAccessKey string `yaml:"secret_access_key"` - Insecure bool `yaml:"insecure"` - SSEEncryption bool `yaml:"sse_encryption"` - HTTPConfig HTTPConfig `yaml:"http_config"` - SignatureVersion string `yaml:"signature_version"` - SSEConfig SSEConfig `yaml:"sse"` + Endpoint string `yaml:"endpoint"` + Region string `yaml:"region"` + AccessKeyID string `yaml:"access_key_id"` + SecretAccessKey string `yaml:"secret_access_key"` + Insecure bool `yaml:"insecure"` + SSEEncryption bool `yaml:"sse_encryption"` + HTTPConfig HTTPConfig `yaml:"http_config"` + SignatureVersion string `yaml:"signature_version"` + SSEConfig cortex_s3.SSEConfig `yaml:"sse"` Inject InjectRequestMiddleware `yaml:"-"` } @@ -165,8 +166,8 @@ func buildSSEParsedConfig(cfg S3Config) (*SSEParsedConfig, error) { // deprecated, but if used it assumes SSE-S3 type if cfg.SSEEncryption { - return NewSSEParsedConfig(SSEConfig{ - Type: SSES3, + return NewSSEParsedConfig(cortex_s3.SSEConfig{ + Type: cortex_s3.SSES3, }) } diff --git a/aws/sse_config.go b/aws/sse_config.go index b62000fdc4d49..172534a1f5429 100644 --- a/aws/sse_config.go +++ b/aws/sse_config.go @@ -3,20 +3,15 @@ package aws import ( "encoding/base64" "encoding/json" - "flag" "github.com/pkg/errors" + + cortex_s3 "github.com/cortexproject/cortex/pkg/storage/bucket/s3" ) const ( - // SSEKMS config type constant to configure S3 server side encryption using KMS - // https://docs.aws.amazon.com/AmazonS3/latest/dev/UsingKMSEncryption.html - SSEKMS = "SSE-KMS" sseKMSType = "aws:kms" - // SSES3 config type constant to configure S3 server side encryption with AES-256 - // https://docs.aws.amazon.com/AmazonS3/latest/dev/UsingServerSideEncryption.html - SSES3 = "SSE-S3" - sseS3Type = "AES256" + sseS3Type = "AES256" ) // SSEParsedConfig configures server side encryption (SSE) @@ -27,29 +22,14 @@ type SSEParsedConfig struct { KMSEncryptionContext *string } -// SSEConfig configures S3 server side encryption -// struct that is going to receive user input (through config file or CLI) -type SSEConfig struct { - Type string `yaml:"type"` - KMSKeyID string `yaml:"kms_key_id"` - KMSEncryptionContext string `yaml:"kms_encryption_context"` -} - -// RegisterFlagsWithPrefix adds the flags required to config this to the given FlagSet -func (cfg *SSEConfig) RegisterFlagsWithPrefix(prefix string, f *flag.FlagSet) { - f.StringVar(&cfg.Type, prefix+"type", "", "Enable AWS Server Side Encryption. Only SSE-S3 and SSE-KMS are supported") - f.StringVar(&cfg.KMSKeyID, prefix+"kms-key-id", "", "KMS Key ID used to encrypt objects in S3") - f.StringVar(&cfg.KMSEncryptionContext, prefix+"kms-encryption-context", "", "KMS Encryption Context used for object encryption. It expects a JSON as a string.") -} - // NewSSEParsedConfig creates a struct to configure server side encryption (SSE) -func NewSSEParsedConfig(cfg SSEConfig) (*SSEParsedConfig, error) { +func NewSSEParsedConfig(cfg cortex_s3.SSEConfig) (*SSEParsedConfig, error) { switch cfg.Type { - case SSES3: + case cortex_s3.SSES3: return &SSEParsedConfig{ ServerSideEncryption: sseS3Type, }, nil - case SSEKMS: + case cortex_s3.SSEKMS: if cfg.KMSKeyID == "" { return nil, errors.New("KMS key id must be passed when SSE-KMS encryption is selected") } diff --git a/aws/sse_config_test.go b/aws/sse_config_test.go index 7c6cfc4247f4b..95fe153157370 100644 --- a/aws/sse_config_test.go +++ b/aws/sse_config_test.go @@ -5,6 +5,8 @@ import ( "github.com/pkg/errors" "github.com/stretchr/testify/assert" + + cortex_s3 "github.com/cortexproject/cortex/pkg/storage/bucket/s3" ) func TestNewSSEParsedConfig(t *testing.T) { @@ -15,14 +17,14 @@ func TestNewSSEParsedConfig(t *testing.T) { tests := []struct { name string - params SSEConfig + params cortex_s3.SSEConfig expected *SSEParsedConfig expectedErr error }{ { name: "Test SSE encryption with SSES3 type", - params: SSEConfig{ - Type: SSES3, + params: cortex_s3.SSEConfig{ + Type: cortex_s3.SSES3, }, expected: &SSEParsedConfig{ ServerSideEncryption: sseS3Type, @@ -30,8 +32,8 @@ func TestNewSSEParsedConfig(t *testing.T) { }, { name: "Test SSE encryption with SSEKMS type without context", - params: SSEConfig{ - Type: SSEKMS, + params: cortex_s3.SSEConfig{ + Type: cortex_s3.SSEKMS, KMSKeyID: kmsKeyID, }, expected: &SSEParsedConfig{ @@ -41,8 +43,8 @@ func TestNewSSEParsedConfig(t *testing.T) { }, { name: "Test SSE encryption with SSEKMS type with context", - params: SSEConfig{ - Type: SSEKMS, + params: cortex_s3.SSEConfig{ + Type: cortex_s3.SSEKMS, KMSKeyID: kmsKeyID, KMSEncryptionContext: kmsEncryptionContext, }, @@ -54,23 +56,23 @@ func TestNewSSEParsedConfig(t *testing.T) { }, { name: "Test invalid SSE type", - params: SSEConfig{ + params: cortex_s3.SSEConfig{ Type: "invalid", }, expectedErr: errors.New("SSE type is empty or invalid"), }, { name: "Test SSE encryption with SSEKMS type without KMS Key ID", - params: SSEConfig{ - Type: SSEKMS, + params: cortex_s3.SSEConfig{ + Type: cortex_s3.SSEKMS, KMSKeyID: "", }, expectedErr: errors.New("KMS key id must be passed when SSE-KMS encryption is selected"), }, { name: "Test SSE with invalid KMS encryption context JSON", - params: SSEConfig{ - Type: SSEKMS, + params: cortex_s3.SSEConfig{ + Type: cortex_s3.SSEKMS, KMSKeyID: kmsKeyID, KMSEncryptionContext: `INVALID_JSON`, },