diff --git a/config/storage_s3_test.go b/config/storage_s3_test.go index 04bbf32..09489f6 100644 --- a/config/storage_s3_test.go +++ b/config/storage_s3_test.go @@ -15,7 +15,7 @@ func TestParseS3StorageBlock(t *testing.T) { ok bool }{ { - desc: "valid", + desc: "valid (required)", source: ` tfmigrate { history { @@ -32,6 +32,41 @@ tfmigrate { }, ok: true, }, + { + desc: "valid (with optional)", + source: ` +tfmigrate { + history { + storage "s3" { + bucket = "tfmigrate-test" + key = "tfmigrate/history.json" + + region = "ap-northeast-1" + endpoint = "http://localstack:4566" + access_key = "dummy" + secret_key = "dummy" + profile = "dev" + skip_credentials_validation = true + skip_metadata_api_check = true + force_path_style = true + } + } +} +`, + want: &history.S3StorageConfig{ + Bucket: "tfmigrate-test", + Key: "tfmigrate/history.json", + Region: "ap-northeast-1", + Endpoint: "http://localstack:4566", + AccessKey: "dummy", + SecretKey: "dummy", + Profile: "dev", + SkipCredentialsValidation: true, + SkipMetadataAPICheck: true, + ForcePathStyle: true, + }, + ok: true, + }, { desc: "missing required attribute (bucket)", source: ` diff --git a/go.mod b/go.mod index 7985808..c40df78 100644 --- a/go.mod +++ b/go.mod @@ -6,6 +6,7 @@ require ( github.com/aws/aws-sdk-go v1.34.16 github.com/davecgh/go-spew v1.1.1 github.com/google/go-cmp v0.5.2 + github.com/hashicorp/aws-sdk-go-base v0.6.0 github.com/hashicorp/hcl/v2 v2.6.0 github.com/hashicorp/logutils v1.0.0 github.com/mattn/go-shellwords v1.0.10 diff --git a/go.sum b/go.sum index 5ed132b..678edff 100644 --- a/go.sum +++ b/go.sum @@ -7,6 +7,7 @@ github.com/apparentlymart/go-textseg/v12 v12.0.0 h1:bNEQyAGak9tojivJNkoqWErVCQbj github.com/apparentlymart/go-textseg/v12 v12.0.0/go.mod h1:S/4uRK2UtaQttw1GenVJEynmyUenKwP++x/+DdGV/Ec= github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310 h1:BUAU3CGlLvorLI26FmByPp2eC2qla6E1Tw+scpcg/to= github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= +github.com/aws/aws-sdk-go v1.31.9/go.mod h1:5zCpMtNQVjRREroY7sYe8lOMRSxkhG6MZveU8YkpAk0= github.com/aws/aws-sdk-go v1.34.16 h1:22jPsMe98UX/van5Ca/5jXnyNsNpJxCJ1rw/wFAlZ+4= github.com/aws/aws-sdk-go v1.34.16/go.mod h1:5zCpMtNQVjRREroY7sYe8lOMRSxkhG6MZveU8YkpAk0= github.com/bgentry/speakeasy v0.1.0 h1:ByYyxL9InA1OWqxJqqp2A5pYHUrCiAL6K3J+LKSsQkY= @@ -24,8 +25,12 @@ github.com/google/go-cmp v0.3.1 h1:Xye71clBPdm5HgqGwUkwhbynsUJZhDbS20FvLhQ2izg= github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.5.2 h1:X2ev0eStA3AbceY54o37/0PQ/UWqKEiiO2dKL5OPaFM= github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/hashicorp/aws-sdk-go-base v0.6.0 h1:qmUbzM36msbBF59YctwuO5w0M2oNXjlilgKpnEhx1uw= +github.com/hashicorp/aws-sdk-go-base v0.6.0/go.mod h1:2fRjWDv3jJBeN6mVWFHV6hFTNeFBx2gpDLQaZNxUVAY= github.com/hashicorp/errwrap v1.0.0 h1:hLrqtEDnRye3+sgx6z4qVLNuviH3MR5aQ0ykNJa/UYA= github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= +github.com/hashicorp/go-cleanhttp v0.5.0 h1:wvCrVc9TjDls6+YGAF2hAifE1E5U1+b4tH6KdvN3Gig= +github.com/hashicorp/go-cleanhttp v0.5.0/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80= github.com/hashicorp/go-multierror v1.0.0 h1:iVjPR7a6H0tWELX5NxNe7bYopibicUzc7uPribsnS6o= github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHhCYQXV3UM06sGGrk= github.com/hashicorp/hcl/v2 v2.6.0 h1:3krZOfGY6SziUXa6H9PJU6TyohHn7I+ARYnhbeNBz+o= @@ -49,6 +54,8 @@ github.com/mattn/go-shellwords v1.0.10 h1:Y7Xqm8piKOO3v10Thp7Z36h4FYFjt5xB//6XvO github.com/mattn/go-shellwords v1.0.10/go.mod h1:EZzvwXDESEeg03EKmM+RmDnNOPKG4lLtQsUlTZDWQ8Y= github.com/mitchellh/cli v1.1.1 h1:J64v/xD7Clql+JVKSvkYojLOXu1ibnY9ZjGLwSt/89w= github.com/mitchellh/cli v1.1.1/go.mod h1:xcISNoH86gajksDmfB23e/pu+B+GeFRMYmoHXxx3xhI= +github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y= +github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= github.com/mitchellh/go-wordwrap v0.0.0-20150314170334-ad45545899c7 h1:DpOJ2HYzCv8LZP15IdmG+YdwD2luVPHITV96TkirNBM= github.com/mitchellh/go-wordwrap v0.0.0-20150314170334-ad45545899c7/go.mod h1:ZXFpozHsX6DPmq2I0TCekCxypsnAUbP2oI0UX1GXzOo= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= diff --git a/history/storage_s3.go b/history/storage_s3.go index c114108..e4e3f1b 100644 --- a/history/storage_s3.go +++ b/history/storage_s3.go @@ -3,21 +3,46 @@ package history import ( "bytes" "context" + "fmt" "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/aws/awserr" "github.com/aws/aws-sdk-go/aws/request" - "github.com/aws/aws-sdk-go/aws/session" "github.com/aws/aws-sdk-go/service/s3" "github.com/aws/aws-sdk-go/service/s3/s3iface" + + awsbase "github.com/hashicorp/aws-sdk-go-base" ) // S3StorageConfig is a config for s3 storage. +// This is expected to have almost the same options as Terraform s3 backend. +// https://www.terraform.io/docs/backends/types/s3.html +// However, it has many minor options and it's a pain to test all options from +// first, so we added only options we need for now. If something missing, feel +// free to open an issue or submit a pull request. type S3StorageConfig struct { - // Bucket is a name of s3 bucket. + // Name of the bucket. Bucket string `hcl:"bucket"` - // Key is a path to a migration history file. + // Path to the migration history file. Key string `hcl:"key"` + + // AWS region. + Region string `hcl:"region,optional"` + // Custom endpoint for the AWS S3 API. + Endpoint string `hcl:"endpoint,optional"` + // AWS access key. + AccessKey string `hcl:"access_key,optional"` + // AWS secret key. + SecretKey string `hcl:"secret_key,optional"` + // Name of AWS profile in AWS shared credentials file. + Profile string `hcl:"profile,optional"` + // Skip credentials validation via the STS API. + SkipCredentialsValidation bool `hcl:"skip_credentials_validation,optional"` + // Skip usage of EC2 Metadata API. + SkipMetadataAPICheck bool `hcl:"skip_metadata_api_check,optional"` + // Enable path-style S3 URLs (https:/// + // instead of https://.). + ForcePathStyle bool `hcl:"force_path_style,optional"` } // S3StorageConfig implements a StorageConfig. @@ -25,7 +50,7 @@ var _ StorageConfig = (*S3StorageConfig)(nil) // NewStorage returns a new instance of S3Storage. func (c *S3StorageConfig) NewStorage() (Storage, error) { - return NewS3Storage(c.Bucket, c.Key, nil) + return NewS3Storage(c, nil) } // S3Client is an abstraction layer for AWS S3 API. @@ -54,66 +79,62 @@ func (c *s3Client) GetObjectWithContext(ctx aws.Context, input *s3.GetObjectInpu // S3Storage is an implementation of Storage for AWS S3. type S3Storage struct { - // Client is an instance of S3Client interface to call API. + // config is a storage config for s3. + config *S3StorageConfig + // client is an instance of S3Client interface to call API. // It is intended to be replaced with a mock for testing. client S3Client - // Bucket is a name of s3 bucket. - bucket string - // Key is a path to a migration history file. - key string } var _ Storage = (*S3Storage)(nil) -// S3StorageOption customizes a behavior of S3Storage. -type S3StorageOption struct { - // Client is an instance of S3Client interface to call API. - // It is intended to be replaced with a mock for testing. - // If nil, it means that the real-world client implementation is used. - Client S3Client -} - // NewS3Storage returns a new instance of S3Storage. -func NewS3Storage(bucket string, key string, option *S3StorageOption) (*S3Storage, error) { - var client S3Client - if option != nil { - if option.Client != nil { - client = option.Client - } - } else { +func NewS3Storage(config *S3StorageConfig, client S3Client) (*S3Storage, error) { + if client == nil { var err error - client, err = newS3Client() + client, err = newS3Client(config) if err != nil { return nil, err } } s := &S3Storage{ + config: config, client: client, - bucket: bucket, - key: key, } return s, nil } // newS3Client returns a new instance of S3Client. -func newS3Client() (S3Client, error) { - sess, err := session.NewSession() - if err != nil { - return nil, err +func newS3Client(config *S3StorageConfig) (S3Client, error) { + cfg := &awsbase.Config{ + AccessKey: config.AccessKey, + Profile: config.Profile, + Region: config.Region, + SecretKey: config.SecretKey, + SkipCredsValidation: config.SkipCredentialsValidation, + SkipMetadataApiCheck: config.SkipMetadataAPICheck, } - client := &s3Client{ - s3api: s3.New(sess), + + sess, err := awsbase.GetSession(cfg) + if err != nil { + return nil, fmt.Errorf("failed to new s3 client: %s", err) } + + client := s3.New(sess.Copy(&aws.Config{ + Endpoint: aws.String(config.Endpoint), + S3ForcePathStyle: aws.Bool(config.ForcePathStyle), + })) + return client, nil } // Write writes migration history data to storage. func (s *S3Storage) Write(ctx context.Context, b []byte) error { input := &s3.PutObjectInput{ - Bucket: aws.String(s.bucket), - Key: aws.String(s.key), + Bucket: aws.String(s.config.Bucket), + Key: aws.String(s.config.Key), Body: bytes.NewReader(b), } @@ -127,8 +148,8 @@ func (s *S3Storage) Write(ctx context.Context, b []byte) error { // an empty array instead of an error. func (s *S3Storage) Read(ctx context.Context) ([]byte, error) { input := &s3.GetObjectInput{ - Bucket: aws.String(s.bucket), - Key: aws.String(s.key), + Bucket: aws.String(s.config.Bucket), + Key: aws.String(s.config.Key), } output, err := s.client.GetObjectWithContext(ctx, input) diff --git a/history/storage_s3_test.go b/history/storage_s3_test.go index 5f4adf7..96b55fe 100644 --- a/history/storage_s3_test.go +++ b/history/storage_s3_test.go @@ -21,8 +21,16 @@ func TestS3StorageConfigNewStorage(t *testing.T) { { desc: "valid", config: &S3StorageConfig{ - Bucket: "tfmigrate-test", - Key: "tfmigrate/history.json", + Bucket: "tfmigrate-test", + Key: "tfmigrate/history.json", + Region: "ap-northeast-1", + Endpoint: "http://localstack:4566", + AccessKey: "dummy", + SecretKey: "dummy", + Profile: "dev", + SkipCredentialsValidation: true, + SkipMetadataAPICheck: true, + ForcePathStyle: true, }, ok: true, }, @@ -64,35 +72,34 @@ func (c *mockS3Client) GetObjectWithContext(ctx aws.Context, input *s3.GetObject func TestS3StorageWrite(t *testing.T) { cases := []struct { desc string - option *S3StorageOption - bucket string - key string + config *S3StorageConfig + client S3Client contents []byte ok bool }{ { desc: "simple", - option: &S3StorageOption{ - Client: &mockS3Client{ - putOutput: &s3.PutObjectOutput{}, - err: nil, - }, + config: &S3StorageConfig{ + Bucket: "tfmigrate-test", + Key: "tfmigrate/history.json", + }, + client: &mockS3Client{ + putOutput: &s3.PutObjectOutput{}, + err: nil, }, - bucket: "tfmigrate-test", - key: "tfmigrate/history.json", contents: []byte("foo"), ok: true, }, { desc: "bucket does not exist", - option: &S3StorageOption{ - Client: &mockS3Client{ - putOutput: nil, - err: awserr.New("NoSuchBucket", "The specified bucket does not exist.", nil), - }, + config: &S3StorageConfig{ + Bucket: "not-exist-bucket", + Key: "tfmigrate/history.json", + }, + client: &mockS3Client{ + putOutput: nil, + err: awserr.New("NoSuchBucket", "The specified bucket does not exist.", nil), }, - bucket: "not-exist-bucket", - key: "tfmigrate/history.json", contents: []byte("foo"), ok: false, }, @@ -100,7 +107,7 @@ func TestS3StorageWrite(t *testing.T) { for _, tc := range cases { t.Run(tc.desc, func(t *testing.T) { - s, err := NewS3Storage(tc.bucket, tc.key, tc.option) + s, err := NewS3Storage(tc.config, tc.client) if err != nil { t.Fatalf("failed to NewS3Storage: %s", err) } @@ -118,50 +125,49 @@ func TestS3StorageWrite(t *testing.T) { func TestS3StorageRead(t *testing.T) { cases := []struct { desc string - option *S3StorageOption - bucket string - key string + config *S3StorageConfig + client S3Client contents []byte ok bool }{ { desc: "simple", - option: &S3StorageOption{ - Client: &mockS3Client{ - getOutput: &s3.GetObjectOutput{ - Body: ioutil.NopCloser(strings.NewReader("foo")), - }, - err: nil, + config: &S3StorageConfig{ + Bucket: "tfmigrate-test", + Key: "tfmigrate/history.json", + }, + client: &mockS3Client{ + getOutput: &s3.GetObjectOutput{ + Body: ioutil.NopCloser(strings.NewReader("foo")), }, + err: nil, }, - bucket: "tfmigrate-test", - key: "tfmigrate/history.json", contents: []byte("foo"), ok: true, }, { desc: "bucket does not exist", - option: &S3StorageOption{ - Client: &mockS3Client{ - getOutput: nil, - err: awserr.New("NoSuchBucket", "The specified bucket does not exist.", nil), - }, + config: &S3StorageConfig{ + Bucket: "not-exist-bucket", + Key: "tfmigrate/history.json", + }, + client: &mockS3Client{ + getOutput: nil, + err: awserr.New("NoSuchBucket", "The specified bucket does not exist.", nil), }, - bucket: "not-exist-bucket", - key: "tfmigrate/history.json", contents: nil, ok: false, }, { desc: "key does not exist", - option: &S3StorageOption{ - Client: &mockS3Client{ - getOutput: nil, - err: awserr.New("NoSuchKey", "The specified key does not exist.", nil), - }, + config: &S3StorageConfig{ + Bucket: "tfmigrate-test", + Key: "not_exist.json", + }, + client: &mockS3Client{ + getOutput: nil, + err: awserr.New("NoSuchKey", "The specified key does not exist.", nil), }, - bucket: "tfmigrate-test", - key: "not_exist.json", contents: []byte{}, ok: true, }, @@ -169,7 +175,7 @@ func TestS3StorageRead(t *testing.T) { for _, tc := range cases { t.Run(tc.desc, func(t *testing.T) { - s, err := NewS3Storage(tc.bucket, tc.key, tc.option) + s, err := NewS3Storage(tc.config, tc.client) if err != nil { t.Fatalf("failed to NewS3Storage: %s", err) } diff --git a/test-fixtures/storage_s3/.tfmigrate.hcl b/test-fixtures/storage_s3/.tfmigrate.hcl new file mode 100644 index 0000000..69e0a32 --- /dev/null +++ b/test-fixtures/storage_s3/.tfmigrate.hcl @@ -0,0 +1,17 @@ +tfmigrate { + migration_dir = "./tfmigrate" + history { + storage "s3" { + bucket = "tfstate-test" + key = "tfmigrate/history.json" + region = "ap-northeast-1" + + endpoint = "http://localstack:4566" + access_key = "dummy" + secret_key = "dummy" + skip_credentials_validation = true + skip_metadata_api_check = true + force_path_style = true + } + } +} diff --git a/test-fixtures/storage_s3/config.tf b/test-fixtures/storage_s3/config.tf new file mode 100644 index 0000000..d7cfe4e --- /dev/null +++ b/test-fixtures/storage_s3/config.tf @@ -0,0 +1,37 @@ +terraform { + # https://www.terraform.io/docs/backends/types/s3.html + backend "s3" { + region = "ap-northeast-1" + bucket = "tfstate-test" + key = "test/terraform.tfstate" + + // mock s3 endpoint with localstack + endpoint = "http://localstack:4566" + access_key = "dummy" + secret_key = "dummy" + skip_credentials_validation = true + skip_metadata_api_check = true + force_path_style = true + } +} + +# https://www.terraform.io/docs/providers/aws/index.html +# https://www.terraform.io/docs/providers/aws/guides/custom-service-endpoints.html#localstack +provider "aws" { + region = "ap-northeast-1" + + access_key = "dummy" + secret_key = "dummy" + skip_credentials_validation = true + skip_metadata_api_check = true + skip_region_validation = true + skip_requesting_account_id = true + s3_force_path_style = true + + // mock endpoints with localstack + endpoints { + s3 = "http://localstack:4566" + ec2 = "http://localstack:4566" + iam = "http://localstack:4566" + } +} diff --git a/test-fixtures/storage_s3/main.tf b/test-fixtures/storage_s3/main.tf new file mode 100644 index 0000000..f84e085 --- /dev/null +++ b/test-fixtures/storage_s3/main.tf @@ -0,0 +1,4 @@ +resource "null_resource" "foo1" {} +resource "null_resource" "foo2" {} +resource "null_resource" "foo3" {} +resource "null_resource" "foo4" {} diff --git a/test-fixtures/storage_s3/tfmigrate/mv_bar1.hcl b/test-fixtures/storage_s3/tfmigrate/mv_bar1.hcl new file mode 100644 index 0000000..fa7e04a --- /dev/null +++ b/test-fixtures/storage_s3/tfmigrate/mv_bar1.hcl @@ -0,0 +1,5 @@ +migration "state" "bar1" { + actions = [ + "mv null_resource.foo1 null_resource.bar1" + ] +} diff --git a/test-fixtures/storage_s3/tfmigrate/mv_bar2.hcl.bk b/test-fixtures/storage_s3/tfmigrate/mv_bar2.hcl.bk new file mode 100644 index 0000000..eb41488 --- /dev/null +++ b/test-fixtures/storage_s3/tfmigrate/mv_bar2.hcl.bk @@ -0,0 +1,5 @@ +migration "state" "bar1" { + actions = [ + "mv null_resource.foo2 null_resource.bar2" + ] +}