GCloud backend

backend/init/init.go

@@ -14,6 +14,7 @@ import (
 	backendAzure "github.com/hashicorp/terraform/backend/remote-state/azure"
 	backendconsul "github.com/hashicorp/terraform/backend/remote-state/consul"
 	backendetcdv3 "github.com/hashicorp/terraform/backend/remote-state/etcdv3"
+	backendGCS "github.com/hashicorp/terraform/backend/remote-state/gcs"
 	backendinmem "github.com/hashicorp/terraform/backend/remote-state/inmem"
 	backendS3 "github.com/hashicorp/terraform/backend/remote-state/s3"
 	backendSwift "github.com/hashicorp/terraform/backend/remote-state/swift"
@@ -47,6 +48,7 @@ func init() {
 			`Warning: "azure" name is deprecated, please use "azurerm"`),
 		"azurerm": func() backend.Backend { return backendAzure.New() },
 		"etcdv3":  func() backend.Backend { return backendetcdv3.New() },
+		"gcs":     func() backend.Backend { return backendGCS.New() },
 	}
 
 	// Add the legacy remote backends that haven't yet been convertd to

backend/remote-state/gcs/backend.go

@@ -0,0 +1,146 @@
+// Package gcs implements remote storage of state on Google Cloud Storage (GCS).
+package gcs
+
+import (
+	"context"
+	"fmt"
+	"os"
+	"strings"
+
+	"cloud.google.com/go/storage"
+	"github.com/hashicorp/terraform/backend"
+	"github.com/hashicorp/terraform/helper/schema"
+	"github.com/hashicorp/terraform/terraform"
+	"google.golang.org/api/option"
+)
+
+// gcsBackend implements "backend".Backend for GCS.
+// Input(), Validate() and Configure() are implemented by embedding *schema.Backend.
+// State(), DeleteState() and States() are implemented explicitly.
+type gcsBackend struct {
+	*schema.Backend
+
+	storageClient  *storage.Client
+	storageContext context.Context
+
+	bucketName       string
+	prefix           string
+	defaultStateFile string
+
+	projectID string
+	region    string
+}
+
+func New() backend.Backend {
+	be := &gcsBackend{}
+	be.Backend = &schema.Backend{
+		ConfigureFunc: be.configure,
+		Schema: map[string]*schema.Schema{
+			"bucket": {
+				Type:        schema.TypeString,
+				Required:    true,
+				Description: "The name of the Google Cloud Storage bucket",
+			},
+
+			"path": {
+				Type:        schema.TypeString,
+				Optional:    true,
+				Description: "Path of the default state file",
+				Deprecated:  "Use the \"prefix\" option instead",
+			},
+
+			"prefix": {
+				Type:        schema.TypeString,
+				Optional:    true,
+				Description: "The directory where state files will be saved inside the bucket",
+			},
+
+			"credentials": {
+				Type:        schema.TypeString,
+				Optional:    true,
+				Description: "Google Cloud JSON Account Key",
+				Default:     "",
+			},
+
+			"project": {
+				Type:        schema.TypeString,
+				Optional:    true,
+				Description: "Google Cloud Project ID",
+				Default:     "",
+			},
+
+			"region": {
+				Type:        schema.TypeString,
+				Optional:    true,
+				Description: "Region / location in which to create the bucket",
+				Default:     "",
+			},
+		},
+	}
+
+	return be
+}
+
+func (b *gcsBackend) configure(ctx context.Context) error {
+	if b.storageClient != nil {
+		return nil
+	}
+
+	// ctx is a background context with the backend config added.
+	// Since no context is passed to remoteClient.Get(), .Lock(), etc. but
+	// one is required for calling the GCP API, we're holding on to this
+	// context here and re-use it later.
+	b.storageContext = ctx
+
+	data := schema.FromContextBackendConfig(b.storageContext)
+
+	b.bucketName = data.Get("bucket").(string)
+	b.prefix = strings.TrimLeft(data.Get("prefix").(string), "/")
+
+	b.defaultStateFile = strings.TrimLeft(data.Get("path").(string), "/")
+
+	b.projectID = data.Get("project").(string)
+	if id := os.Getenv("GOOGLE_PROJECT"); b.projectID == "" && id != "" {
+		b.projectID = id
+	}
+	b.region = data.Get("region").(string)
+	if r := os.Getenv("GOOGLE_REGION"); b.projectID == "" && r != "" {
+		b.region = r
+	}
+
+	opts := []option.ClientOption{
+		option.WithScopes(storage.ScopeReadWrite),
+		option.WithUserAgent(terraform.UserAgentString()),
+	}
+	if credentialsFile := data.Get("credentials").(string); credentialsFile != "" {
+		opts = append(opts, option.WithCredentialsFile(credentialsFile))
+	} else if credentialsFile := os.Getenv("GOOGLE_CREDENTIALS"); credentialsFile != "" {
+		opts = append(opts, option.WithCredentialsFile(credentialsFile))
+	}
+
+	client, err := storage.NewClient(b.storageContext, opts...)
+	if err != nil {
+		return fmt.Errorf("storage.NewClient() failed: %v", err)
+	}
+
+	b.storageClient = client
+
+	return b.ensureBucketExists()
+}
+
+func (b *gcsBackend) ensureBucketExists() error {
+	_, err := b.storageClient.Bucket(b.bucketName).Attrs(b.storageContext)
+	if err != storage.ErrBucketNotExist {
+		return err
+	}
+
+	if b.projectID == "" {
+		return fmt.Errorf("bucket %q does not exist; specify the \"project\" option or create the bucket manually using `gsutil mb gs://%s`", b.bucketName, b.bucketName)
+	}
+
+	attrs := &storage.BucketAttrs{
+		Location: b.region,
+	}
+
+	return b.storageClient.Bucket(b.bucketName).Create(b.storageContext, b.projectID, attrs)
+}

backend/remote-state/gcs/backend_state.go

@@ -0,0 +1,155 @@
+package gcs
+
+import (
+	"fmt"
+	"path"
+	"sort"
+	"strings"
+
+	"cloud.google.com/go/storage"
+	"github.com/hashicorp/terraform/backend"
+	"github.com/hashicorp/terraform/state"
+	"github.com/hashicorp/terraform/state/remote"
+	"github.com/hashicorp/terraform/terraform"
+	"google.golang.org/api/iterator"
+)
+
+const (
+	stateFileSuffix = ".tfstate"
+	lockFileSuffix  = ".tflock"
+)
+
+// States returns a list of names for the states found on GCS. The default
+// state is always returned as the first element in the slice.
+func (b *gcsBackend) States() ([]string, error) {
+	states := []string{backend.DefaultStateName}
+
+	bucket := b.storageClient.Bucket(b.bucketName)
+	objs := bucket.Objects(b.storageContext, &storage.Query{
+		Delimiter: "/",
+		Prefix:    b.prefix,
+	})
+	for {
+		attrs, err := objs.Next()
+		if err == iterator.Done {
+			break
+		}
+		if err != nil {
+			return nil, fmt.Errorf("querying Cloud Storage failed: %v", err)
+		}
+
+		name := path.Base(attrs.Name)
+		if !strings.HasSuffix(name, stateFileSuffix) {
+			continue
+		}
+		st := strings.TrimSuffix(name, stateFileSuffix)
+
+		if st != backend.DefaultStateName {
+			states = append(states, st)
+		}
+	}
+
+	sort.Strings(states[1:])
+	return states, nil
+}
+
+// DeleteState deletes the named state. The "default" state cannot be deleted.
+func (b *gcsBackend) DeleteState(name string) error {
+	if name == backend.DefaultStateName {
+		return fmt.Errorf("cowardly refusing to delete the %q state", name)
+	}
+
+	c, err := b.client(name)
+	if err != nil {
+		return err
+	}
+
+	return c.Delete()
+}
+
+// client returns a remoteClient for the named state.
+func (b *gcsBackend) client(name string) (*remoteClient, error) {
+	if name == "" {
+		return nil, fmt.Errorf("%q is not a valid state name", name)
+	}
+
+	return &remoteClient{
+		storageContext: b.storageContext,
+		storageClient:  b.storageClient,
+		bucketName:     b.bucketName,
+		stateFilePath:  b.stateFile(name),
+		lockFilePath:   b.lockFile(name),
+	}, nil
+}
+
+// State reads and returns the named state from GCS. If the named state does
+// not yet exist, a new state file is created.
+func (b *gcsBackend) State(name string) (state.State, error) {
+	c, err := b.client(name)
+	if err != nil {
+		return nil, err
+	}
+
+	st := &remote.State{Client: c}
+	lockInfo := state.NewLockInfo()
+	lockInfo.Operation = "init"
+	lockID, err := st.Lock(lockInfo)
+	if err != nil {
+		return nil, err
+	}
+
+	// Local helper function so we can call it multiple places
+	unlock := func(baseErr error) error {
+		if err := st.Unlock(lockID); err != nil {
+			const unlockErrMsg = `%v
+Additionally, unlocking the state file on Google Cloud Storage failed:
+
+  Error message: %q
+  Lock ID (gen): %v
+  Lock file URL: %v
+
+You may have to force-unlock this state in order to use it again.
+The GCloud backend acquires a lock during initialization to ensure
+the initial state file is created.`
+			return fmt.Errorf(unlockErrMsg, baseErr, err.Error(), lockID, c.lockFileURL())
+		}
+
+		return baseErr
+	}
+
+	// Grab the value
+	if err := st.RefreshState(); err != nil {
+		return nil, unlock(err)
+	}
+
+	// If we have no state, we have to create an empty state
+	if v := st.State(); v == nil {
+		if err := st.WriteState(terraform.NewState()); err != nil {
+			return nil, unlock(err)
+		}
+		if err := st.PersistState(); err != nil {
+			return nil, unlock(err)
+		}
+	}
+
+	// Unlock, the state should now be initialized
+	if err := unlock(nil); err != nil {
+		return nil, err
+	}
+
+	return st, nil
+}
+
+func (b *gcsBackend) stateFile(name string) string {
+	if name == backend.DefaultStateName && b.defaultStateFile != "" {
+		return b.defaultStateFile
+	}
+	return path.Join(b.prefix, name+stateFileSuffix)
+}
+
+func (b *gcsBackend) lockFile(name string) string {
+	if name == backend.DefaultStateName && b.defaultStateFile != "" {
+		return strings.TrimSuffix(b.defaultStateFile, stateFileSuffix) + lockFileSuffix
+	}
+	return path.Join(b.prefix, name+lockFileSuffix)
+}