diff --git a/backend/remote-state/s3/backend.go b/backend/remote-state/s3/backend.go index 6e5ecdeba301..1aff51ebb4b7 100644 --- a/backend/remote-state/s3/backend.go +++ b/backend/remote-state/s3/backend.go @@ -226,6 +226,13 @@ func New() backend.Backend { Description: "The maximum number of times an AWS API request is retried on retryable failure.", Default: 5, }, + + "compression": { + Type: schema.TypeBool, + Optional: true, + Description: "Enable gzip compression before sending sate file into bucket", + Default: false, + }, }, } @@ -248,6 +255,7 @@ type Backend struct { kmsKeyID string ddbTable string workspaceKeyPrefix string + compression bool } func (b *Backend) configure(ctx context.Context) error { @@ -264,6 +272,7 @@ func (b *Backend) configure(ctx context.Context) error { b.acl = data.Get("acl").(string) b.kmsKeyID = data.Get("kms_key_id").(string) b.workspaceKeyPrefix = data.Get("workspace_key_prefix").(string) + b.compression = data.Get("compression").(bool) b.ddbTable = data.Get("dynamodb_table").(string) if b.ddbTable == "" { diff --git a/backend/remote-state/s3/backend_state.go b/backend/remote-state/s3/backend_state.go index dc134ecf45c0..0d71ede4e527 100644 --- a/backend/remote-state/s3/backend_state.go +++ b/backend/remote-state/s3/backend_state.go @@ -111,6 +111,7 @@ func (b *Backend) remoteClient(name string) (*RemoteClient, error) { acl: b.acl, kmsKeyID: b.kmsKeyID, ddbTable: b.ddbTable, + compression: b.compression, } return client, nil diff --git a/backend/remote-state/s3/backend_test.go b/backend/remote-state/s3/backend_test.go index be711884df38..4da011a1d5f8 100644 --- a/backend/remote-state/s3/backend_test.go +++ b/backend/remote-state/s3/backend_test.go @@ -40,6 +40,7 @@ func TestBackendConfig(t *testing.T) { "key": "state", "encrypt": true, "dynamodb_table": "dynamoTable", + "compression": true, } b := backend.TestBackendConfig(t, New(), backend.TestWrapConfig(config)).(*Backend) @@ -53,6 +54,9 @@ func TestBackendConfig(t *testing.T) { if b.keyName != "state" { t.Fatalf("Incorrect keyName was populated") } + if !b.compression { + t.Fatalf("Incorrect compression was populated") + } credentials, err := b.s3Client.Config.Credentials.Get() if err != nil { diff --git a/backend/remote-state/s3/client.go b/backend/remote-state/s3/client.go index 12500183b259..016a8143ddd0 100644 --- a/backend/remote-state/s3/client.go +++ b/backend/remote-state/s3/client.go @@ -2,12 +2,14 @@ package s3 import ( "bytes" + "compress/gzip" "crypto/md5" "encoding/hex" "encoding/json" "errors" "fmt" "io" + "io/ioutil" "log" "time" @@ -36,6 +38,7 @@ type RemoteClient struct { acl string kmsKeyID string ddbTable string + compression bool } var ( @@ -122,9 +125,14 @@ func (c *RemoteClient) get() (*remote.Payload, error) { return nil, fmt.Errorf("Failed to read remote state: %s", err) } - sum := md5.Sum(buf.Bytes()) + data, err := c.decompress(buf.Bytes()) + if err != nil { + return nil, err + } + + sum := md5.Sum(data) payload := &remote.Payload{ - Data: buf.Bytes(), + Data: data, MD5: sum[:], } @@ -138,12 +146,17 @@ func (c *RemoteClient) get() (*remote.Payload, error) { func (c *RemoteClient) Put(data []byte) error { contentType := "application/json" - contentLength := int64(len(data)) + objectBody := data + if c.compression { + objectBody = c.compress(data) + contentType = "gzip" + } + contentLength := int64(len(objectBody)) i := &s3.PutObjectInput{ ContentType: &contentType, ContentLength: &contentLength, - Body: bytes.NewReader(data), + Body: bytes.NewReader(objectBody), Bucket: &c.bucketName, Key: &c.path, } @@ -383,6 +396,32 @@ func (c *RemoteClient) lockPath() string { return fmt.Sprintf("%s/%s", c.bucketName, c.path) } +func (c *RemoteClient) decompress(data []byte) ([]byte, error) { + if len(data) == 0 { + return data, nil + } + gz, err := gzip.NewReader(bytes.NewReader(data)) + if err != nil { + if err == gzip.ErrHeader { + // not a gzipped data + return data, nil + } + + return nil, err + } + defer gz.Close() + + return ioutil.ReadAll(gz) +} + +func (c *RemoteClient) compress(data []byte) []byte { + b := &bytes.Buffer{} + gz := gzip.NewWriter(b) + gz.Write(data) + gz.Close() + return b.Bytes() +} + const errBadChecksumFmt = `state data in S3 does not have the expected content. This may be caused by unusually long delays in S3 processing a previous state diff --git a/backend/remote-state/s3/client_test.go b/backend/remote-state/s3/client_test.go index 765f5636b9c2..24facf5eeec6 100644 --- a/backend/remote-state/s3/client_test.go +++ b/backend/remote-state/s3/client_test.go @@ -204,114 +204,130 @@ func TestRemoteClient_clientMD5(t *testing.T) { // verify that a client won't return a state with an incorrect checksum. func TestRemoteClient_stateChecksum(t *testing.T) { - testACC(t) - - bucketName := fmt.Sprintf("terraform-remote-s3-test-%x", time.Now().Unix()) - keyName := "testState" - - b1 := backend.TestBackendConfig(t, New(), backend.TestWrapConfig(map[string]interface{}{ - "bucket": bucketName, - "key": keyName, - "dynamodb_table": bucketName, - })).(*Backend) - - createS3Bucket(t, b1.s3Client, bucketName) - defer deleteS3Bucket(t, b1.s3Client, bucketName) - createDynamoDBTable(t, b1.dynClient, bucketName) - defer deleteDynamoDBTable(t, b1.dynClient, bucketName) - - s1, err := b1.StateMgr(backend.DefaultStateName) - if err != nil { - t.Fatal(err) - } - client1 := s1.(*remote.State).Client - - // create a old and new state version to persist - s := state.TestStateInitial() - sf := &statefile.File{State: s} - var oldState bytes.Buffer - if err := statefile.Write(sf, &oldState); err != nil { - t.Fatal(err) - } - sf.Serial++ - var newState bytes.Buffer - if err := statefile.Write(sf, &newState); err != nil { - t.Fatal(err) - } - - // Use b2 without a dynamodb_table to bypass the lock table to write the state directly. - // client2 will write the "incorrect" state, simulating s3 eventually consistency delays - b2 := backend.TestBackendConfig(t, New(), backend.TestWrapConfig(map[string]interface{}{ - "bucket": bucketName, - "key": keyName, - })).(*Backend) - s2, err := b2.StateMgr(backend.DefaultStateName) - if err != nil { - t.Fatal(err) - } - client2 := s2.(*remote.State).Client - - // write the new state through client2 so that there is no checksum yet - if err := client2.Put(newState.Bytes()); err != nil { - t.Fatal(err) + testCases := map[string]struct { + compression bool + }{ + "without compression": { + compression: false, + }, + "with compression": { + compression: true, + }, } - // verify that we can pull a state without a checksum - if _, err := client1.Get(); err != nil { - t.Fatal(err) - } - - // write the new state back with its checksum - if err := client1.Put(newState.Bytes()); err != nil { - t.Fatal(err) - } - - // put an empty state in place to check for panics during get - if err := client2.Put([]byte{}); err != nil { - t.Fatal(err) - } - - // remove the timeouts so we can fail immediately - origTimeout := consistencyRetryTimeout - origInterval := consistencyRetryPollInterval - defer func() { - consistencyRetryTimeout = origTimeout - consistencyRetryPollInterval = origInterval - }() - consistencyRetryTimeout = 0 - consistencyRetryPollInterval = 0 - - // fetching an empty state through client1 should now error out due to a - // mismatched checksum. - if _, err := client1.Get(); !strings.HasPrefix(err.Error(), errBadChecksumFmt[:80]) { - t.Fatalf("expected state checksum error: got %s", err) - } - - // put the old state in place of the new, without updating the checksum - if err := client2.Put(oldState.Bytes()); err != nil { - t.Fatal(err) - } - - // fetching the wrong state through client1 should now error out due to a - // mismatched checksum. - if _, err := client1.Get(); !strings.HasPrefix(err.Error(), errBadChecksumFmt[:80]) { - t.Fatalf("expected state checksum error: got %s", err) - } - - // update the state with the correct one after we Get again - testChecksumHook = func() { - if err := client2.Put(newState.Bytes()); err != nil { - t.Fatal(err) - } - testChecksumHook = nil - } - - consistencyRetryTimeout = origTimeout - - // this final Get will fail to fail the checksum verification, the above - // callback will update the state with the correct version, and Get should - // retry automatically. - if _, err := client1.Get(); err != nil { - t.Fatal(err) + for name, tc := range testCases { + t.Run(name, func(t *testing.T) { + testACC(t) + + bucketName := fmt.Sprintf("terraform-remote-s3-test-%x", time.Now().Unix()) + keyName := "testState" + + b1 := backend.TestBackendConfig(t, New(), backend.TestWrapConfig(map[string]interface{}{ + "bucket": bucketName, + "key": keyName, + "dynamodb_table": bucketName, + "compression": tc.compression, + })).(*Backend) + + createS3Bucket(t, b1.s3Client, bucketName) + defer deleteS3Bucket(t, b1.s3Client, bucketName) + createDynamoDBTable(t, b1.dynClient, bucketName) + defer deleteDynamoDBTable(t, b1.dynClient, bucketName) + + s1, err := b1.StateMgr(backend.DefaultStateName) + if err != nil { + t.Fatal(err) + } + client1 := s1.(*remote.State).Client + + // create a old and new state version to persist + s := state.TestStateInitial() + sf := &statefile.File{State: s} + var oldState bytes.Buffer + if err := statefile.Write(sf, &oldState); err != nil { + t.Fatal(err) + } + sf.Serial++ + var newState bytes.Buffer + if err := statefile.Write(sf, &newState); err != nil { + t.Fatal(err) + } + + // Use b2 without a dynamodb_table to bypass the lock table to write the state directly. + // client2 will write the "incorrect" state, simulating s3 eventually consistency delays + b2 := backend.TestBackendConfig(t, New(), backend.TestWrapConfig(map[string]interface{}{ + "bucket": bucketName, + "key": keyName, + })).(*Backend) + s2, err := b2.StateMgr(backend.DefaultStateName) + if err != nil { + t.Fatal(err) + } + client2 := s2.(*remote.State).Client + + // write the new state through client2 so that there is no checksum yet + if err := client2.Put(newState.Bytes()); err != nil { + t.Fatal(err) + } + + // verify that we can pull a state without a checksum + if _, err := client1.Get(); err != nil { + t.Fatal(err) + } + + // write the new state back with its checksum + if err := client1.Put(newState.Bytes()); err != nil { + t.Fatal(err) + } + + // put an empty state in place to check for panics during get + if err := client2.Put([]byte{}); err != nil { + t.Fatal(err) + } + + // remove the timeouts so we can fail immediately + origTimeout := consistencyRetryTimeout + origInterval := consistencyRetryPollInterval + defer func() { + consistencyRetryTimeout = origTimeout + consistencyRetryPollInterval = origInterval + }() + consistencyRetryTimeout = 0 + consistencyRetryPollInterval = 0 + + // fetching an empty state through client1 should now error out due to a + // mismatched checksum. + if _, err := client1.Get(); !strings.HasPrefix(err.Error(), errBadChecksumFmt[:80]) { + t.Fatalf("expected state checksum error: got %s", err) + } + + // put the old state in place of the new, without updating the checksum + if err := client2.Put(oldState.Bytes()); err != nil { + t.Fatal(err) + } + + // fetching the wrong state through client1 should now error out due to a + // mismatched checksum. + if _, err := client1.Get(); !strings.HasPrefix(err.Error(), errBadChecksumFmt[:80]) { + t.Fatalf("expected state checksum error: got %s", err) + } + + // update the state with the correct one after we Get again + testChecksumHook = func() { + if err := client2.Put(newState.Bytes()); err != nil { + t.Fatal(err) + } + testChecksumHook = nil + } + + consistencyRetryTimeout = origTimeout + + // this final Get will fail to fail the checksum verification, the above + // callback will update the state with the correct version, and Get should + // retry automatically. + if _, err := client1.Get(); err != nil { + t.Fatal(err) + } + }) } }