Merge pull request #9 from minamijoyo/use-aws-sdk-go-base

Use hashicorp/aws-sdk-go-base to authenticate s3 storage

config/storage_s3_test.go

@@ -15,7 +15,7 @@ func TestParseS3StorageBlock(t *testing.T) {
 		ok     bool
 	}{
 		{
-			desc: "valid",
+			desc: "valid (required)",
 			source: `
 tfmigrate {
   history {
@@ -32,6 +32,41 @@ tfmigrate {
 			},
 			ok: true,
 		},
+		{
+			desc: "valid (with optional)",
+			source: `
+tfmigrate {
+  history {
+    storage "s3" {
+      bucket = "tfmigrate-test"
+      key    = "tfmigrate/history.json"
+
+      region                      = "ap-northeast-1"
+      endpoint                    = "http://localstack:4566"
+      access_key                  = "dummy"
+      secret_key                  = "dummy"
+      profile                     = "dev"
+      skip_credentials_validation = true
+      skip_metadata_api_check     = true
+      force_path_style            = true
+    }
+  }
+}
+`,
+			want: &history.S3StorageConfig{
+				Bucket:                    "tfmigrate-test",
+				Key:                       "tfmigrate/history.json",
+				Region:                    "ap-northeast-1",
+				Endpoint:                  "http://localstack:4566",
+				AccessKey:                 "dummy",
+				SecretKey:                 "dummy",
+				Profile:                   "dev",
+				SkipCredentialsValidation: true,
+				SkipMetadataAPICheck:      true,
+				ForcePathStyle:            true,
+			},
+			ok: true,
+		},
 		{
 			desc: "missing required attribute (bucket)",
 			source: `

go.mod

@@ -6,6 +6,7 @@ require (
 	github.com/aws/aws-sdk-go v1.34.16
 	github.com/davecgh/go-spew v1.1.1
 	github.com/google/go-cmp v0.5.2
+	github.com/hashicorp/aws-sdk-go-base v0.6.0
 	github.com/hashicorp/hcl/v2 v2.6.0
 	github.com/hashicorp/logutils v1.0.0
 	github.com/mattn/go-shellwords v1.0.10

go.sum

@@ -7,6 +7,7 @@ github.com/apparentlymart/go-textseg/v12 v12.0.0 h1:bNEQyAGak9tojivJNkoqWErVCQbj
 github.com/apparentlymart/go-textseg/v12 v12.0.0/go.mod h1:S/4uRK2UtaQttw1GenVJEynmyUenKwP++x/+DdGV/Ec=
 github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310 h1:BUAU3CGlLvorLI26FmByPp2eC2qla6E1Tw+scpcg/to=
 github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8=
+github.com/aws/aws-sdk-go v1.31.9/go.mod h1:5zCpMtNQVjRREroY7sYe8lOMRSxkhG6MZveU8YkpAk0=
 github.com/aws/aws-sdk-go v1.34.16 h1:22jPsMe98UX/van5Ca/5jXnyNsNpJxCJ1rw/wFAlZ+4=
 github.com/aws/aws-sdk-go v1.34.16/go.mod h1:5zCpMtNQVjRREroY7sYe8lOMRSxkhG6MZveU8YkpAk0=
 github.com/bgentry/speakeasy v0.1.0 h1:ByYyxL9InA1OWqxJqqp2A5pYHUrCiAL6K3J+LKSsQkY=
@@ -24,8 +25,12 @@ github.com/google/go-cmp v0.3.1 h1:Xye71clBPdm5HgqGwUkwhbynsUJZhDbS20FvLhQ2izg=
 github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
 github.com/google/go-cmp v0.5.2 h1:X2ev0eStA3AbceY54o37/0PQ/UWqKEiiO2dKL5OPaFM=
 github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/hashicorp/aws-sdk-go-base v0.6.0 h1:qmUbzM36msbBF59YctwuO5w0M2oNXjlilgKpnEhx1uw=
+github.com/hashicorp/aws-sdk-go-base v0.6.0/go.mod h1:2fRjWDv3jJBeN6mVWFHV6hFTNeFBx2gpDLQaZNxUVAY=
 github.com/hashicorp/errwrap v1.0.0 h1:hLrqtEDnRye3+sgx6z4qVLNuviH3MR5aQ0ykNJa/UYA=
 github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4=
+github.com/hashicorp/go-cleanhttp v0.5.0 h1:wvCrVc9TjDls6+YGAF2hAifE1E5U1+b4tH6KdvN3Gig=
+github.com/hashicorp/go-cleanhttp v0.5.0/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80=
 github.com/hashicorp/go-multierror v1.0.0 h1:iVjPR7a6H0tWELX5NxNe7bYopibicUzc7uPribsnS6o=
 github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHhCYQXV3UM06sGGrk=
 github.com/hashicorp/hcl/v2 v2.6.0 h1:3krZOfGY6SziUXa6H9PJU6TyohHn7I+ARYnhbeNBz+o=
@@ -49,6 +54,8 @@ github.com/mattn/go-shellwords v1.0.10 h1:Y7Xqm8piKOO3v10Thp7Z36h4FYFjt5xB//6XvO
 github.com/mattn/go-shellwords v1.0.10/go.mod h1:EZzvwXDESEeg03EKmM+RmDnNOPKG4lLtQsUlTZDWQ8Y=
 github.com/mitchellh/cli v1.1.1 h1:J64v/xD7Clql+JVKSvkYojLOXu1ibnY9ZjGLwSt/89w=
 github.com/mitchellh/cli v1.1.1/go.mod h1:xcISNoH86gajksDmfB23e/pu+B+GeFRMYmoHXxx3xhI=
+github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y=
+github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
 github.com/mitchellh/go-wordwrap v0.0.0-20150314170334-ad45545899c7 h1:DpOJ2HYzCv8LZP15IdmG+YdwD2luVPHITV96TkirNBM=
 github.com/mitchellh/go-wordwrap v0.0.0-20150314170334-ad45545899c7/go.mod h1:ZXFpozHsX6DPmq2I0TCekCxypsnAUbP2oI0UX1GXzOo=
 github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=

history/storage_s3.go

@@ -3,29 +3,54 @@ package history
 import (
 	"bytes"
 	"context"
+	"fmt"
 
 	"github.com/aws/aws-sdk-go/aws"
 	"github.com/aws/aws-sdk-go/aws/awserr"
 	"github.com/aws/aws-sdk-go/aws/request"
-	"github.com/aws/aws-sdk-go/aws/session"
 	"github.com/aws/aws-sdk-go/service/s3"
 	"github.com/aws/aws-sdk-go/service/s3/s3iface"
+
+	awsbase "github.com/hashicorp/aws-sdk-go-base"
 )
 
 // S3StorageConfig is a config for s3 storage.
+// This is expected to have almost the same options as Terraform s3 backend.
+// https://www.terraform.io/docs/backends/types/s3.html
+// However, it has many minor options and it's a pain to test all options from
+// first, so we added only options we need for now. If something missing, feel
+// free to open an issue or submit a pull request.
 type S3StorageConfig struct {
-	// Bucket is a name of s3 bucket.
+	// Name of the bucket.
 	Bucket string `hcl:"bucket"`
-	// Key is a path to a migration history file.
+	// Path to the migration history file.
 	Key string `hcl:"key"`
+
+	// AWS region.
+	Region string `hcl:"region,optional"`
+	// Custom endpoint for the AWS S3 API.
+	Endpoint string `hcl:"endpoint,optional"`
+	// AWS access key.
+	AccessKey string `hcl:"access_key,optional"`
+	// AWS secret key.
+	SecretKey string `hcl:"secret_key,optional"`
+	// Name of AWS profile in AWS shared credentials file.
+	Profile string `hcl:"profile,optional"`
+	// Skip credentials validation via the STS API.
+	SkipCredentialsValidation bool `hcl:"skip_credentials_validation,optional"`
+	// Skip usage of EC2 Metadata API.
+	SkipMetadataAPICheck bool `hcl:"skip_metadata_api_check,optional"`
+	// Enable path-style S3 URLs (https://<HOST>/<BUCKET>
+	// instead of https://<BUCKET>.<HOST>).
+	ForcePathStyle bool `hcl:"force_path_style,optional"`
 }
 
 // S3StorageConfig implements a StorageConfig.
 var _ StorageConfig = (*S3StorageConfig)(nil)
 
 // NewStorage returns a new instance of S3Storage.
 func (c *S3StorageConfig) NewStorage() (Storage, error) {
-	return NewS3Storage(c.Bucket, c.Key, nil)
+	return NewS3Storage(c, nil)
 }
 
 // S3Client is an abstraction layer for AWS S3 API.
@@ -54,66 +79,62 @@ func (c *s3Client) GetObjectWithContext(ctx aws.Context, input *s3.GetObjectInpu
 
 // S3Storage is an implementation of Storage for AWS S3.
 type S3Storage struct {
-	// Client is an instance of S3Client interface to call API.
+	// config is a storage config for s3.
+	config *S3StorageConfig
+	// client is an instance of S3Client interface to call API.
 	// It is intended to be replaced with a mock for testing.
 	client S3Client
-	// Bucket is a name of s3 bucket.
-	bucket string
-	// Key is a path to a migration history file.
-	key string
 }
 
 var _ Storage = (*S3Storage)(nil)
 
-// S3StorageOption customizes a behavior of S3Storage.
-type S3StorageOption struct {
-	// Client is an instance of S3Client interface to call API.
-	// It is intended to be replaced with a mock for testing.
-	// If nil, it means that the real-world client implementation is used.
-	Client S3Client
-}
-
 // NewS3Storage returns a new instance of S3Storage.
-func NewS3Storage(bucket string, key string, option *S3StorageOption) (*S3Storage, error) {
-	var client S3Client
-	if option != nil {
-		if option.Client != nil {
-			client = option.Client
-		}
-	} else {
+func NewS3Storage(config *S3StorageConfig, client S3Client) (*S3Storage, error) {
+	if client == nil {
 		var err error
-		client, err = newS3Client()
+		client, err = newS3Client(config)
 		if err != nil {
 			return nil, err
 		}
 	}
 
 	s := &S3Storage{
+		config: config,
 		client: client,
-		bucket: bucket,
-		key:    key,
 	}
 
 	return s, nil
 }
 
 // newS3Client returns a new instance of S3Client.
-func newS3Client() (S3Client, error) {
-	sess, err := session.NewSession()
-	if err != nil {
-		return nil, err
+func newS3Client(config *S3StorageConfig) (S3Client, error) {
+	cfg := &awsbase.Config{
+		AccessKey:            config.AccessKey,
+		Profile:              config.Profile,
+		Region:               config.Region,
+		SecretKey:            config.SecretKey,
+		SkipCredsValidation:  config.SkipCredentialsValidation,
+		SkipMetadataApiCheck: config.SkipMetadataAPICheck,
 	}
-	client := &s3Client{
-		s3api: s3.New(sess),
+
+	sess, err := awsbase.GetSession(cfg)
+	if err != nil {
+		return nil, fmt.Errorf("failed to new s3 client: %s", err)
 	}
+
+	client := s3.New(sess.Copy(&aws.Config{
+		Endpoint:         aws.String(config.Endpoint),
+		S3ForcePathStyle: aws.Bool(config.ForcePathStyle),
+	}))
+
 	return client, nil
 }
 
 // Write writes migration history data to storage.
 func (s *S3Storage) Write(ctx context.Context, b []byte) error {
 	input := &s3.PutObjectInput{
-		Bucket: aws.String(s.bucket),
-		Key:    aws.String(s.key),
+		Bucket: aws.String(s.config.Bucket),
+		Key:    aws.String(s.config.Key),
 		Body:   bytes.NewReader(b),
 	}
 
@@ -127,8 +148,8 @@ func (s *S3Storage) Write(ctx context.Context, b []byte) error {
 // an empty array instead of an error.
 func (s *S3Storage) Read(ctx context.Context) ([]byte, error) {
 	input := &s3.GetObjectInput{
-		Bucket: aws.String(s.bucket),
-		Key:    aws.String(s.key),
+		Bucket: aws.String(s.config.Bucket),
+		Key:    aws.String(s.config.Key),
 	}
 
 	output, err := s.client.GetObjectWithContext(ctx, input)