backend/cos: Add TencentCloud backend cos with lock (#22540)

* add TencentCloud COS backend for remote state

* add vendor of dependence

* fixed error not handle and remove default value for prefix argument

* get appid from TF_COS_APPID environment variables

CODEOWNERS

@@ -5,3 +5,4 @@
 /backend/remote-state/azure     @hashicorp/terraform-azure
 /backend/remote-state/gcs       @hashicorp/terraform-google
 /backend/remote-state/s3        @hashicorp/terraform-aws
+/backend/remote-state/s3        @likexian

backend/init/init.go

@@ -16,6 +16,7 @@ import (
 	backendArtifactory "github.com/hashicorp/terraform/backend/remote-state/artifactory"
 	backendAzure "github.com/hashicorp/terraform/backend/remote-state/azure"
 	backendConsul "github.com/hashicorp/terraform/backend/remote-state/consul"
+	backendCos "github.com/hashicorp/terraform/backend/remote-state/cos"
 	backendEtcdv2 "github.com/hashicorp/terraform/backend/remote-state/etcdv2"
 	backendEtcdv3 "github.com/hashicorp/terraform/backend/remote-state/etcdv3"
 	backendGCS "github.com/hashicorp/terraform/backend/remote-state/gcs"
@@ -57,6 +58,7 @@ func Init(services *disco.Disco) {
 		"atlas":       func() backend.Backend { return backendAtlas.New() },
 		"azurerm":     func() backend.Backend { return backendAzure.New() },
 		"consul":      func() backend.Backend { return backendConsul.New() },
+		"cos":         func() backend.Backend { return backendCos.New() },
 		"etcd":        func() backend.Backend { return backendEtcdv2.New() },
 		"etcdv3":      func() backend.Backend { return backendEtcdv3.New() },
 		"gcs":         func() backend.Backend { return backendGCS.New() },

backend/init/init_test.go

@@ -18,6 +18,7 @@ func TestInit_backend(t *testing.T) {
 		{"atlas", "*atlas.Backend"},
 		{"azurerm", "*azure.Backend"},
 		{"consul", "*consul.Backend"},
+		{"cos", "*cos.Backend"},
 		{"etcdv3", "*etcd.Backend"},
 		{"gcs", "*gcs.Backend"},
 		{"inmem", "*inmem.Backend"},

backend/remote-state/cos/backend.go

@@ -0,0 +1,169 @@
+package cos
+
+import (
+	"context"
+	"fmt"
+	"net/http"
+	"net/url"
+	"strings"
+	"time"
+
+	"github.com/hashicorp/terraform/backend"
+	"github.com/hashicorp/terraform/helper/schema"
+	"github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/common"
+	"github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/common/profile"
+	tag "github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/tag/v20180813"
+	"github.com/tencentyun/cos-go-sdk-v5"
+)
+
+// Default value from environment variable
+const (
+	PROVIDER_SECRET_ID  = "TENCENTCLOUD_SECRET_ID"
+	PROVIDER_SECRET_KEY = "TENCENTCLOUD_SECRET_KEY"
+	PROVIDER_REGION     = "TENCENTCLOUD_REGION"
+)
+
+// Backend implements "backend".Backend for tencentCloud cos
+type Backend struct {
+	*schema.Backend
+
+	cosContext context.Context
+	cosClient  *cos.Client
+	tagClient  *tag.Client
+
+	region  string
+	bucket  string
+	prefix  string
+	key     string
+	encrypt bool
+	acl     string
+}
+
+// New creates a new backend for TencentCloud cos remote state.
+func New() backend.Backend {
+	s := &schema.Backend{
+		Schema: map[string]*schema.Schema{
+			"secret_id": {
+				Type:        schema.TypeString,
+				Required:    true,
+				DefaultFunc: schema.EnvDefaultFunc(PROVIDER_SECRET_ID, nil),
+				Description: "Secret id of Tencent Cloud",
+			},
+			"secret_key": {
+				Type:        schema.TypeString,
+				Required:    true,
+				DefaultFunc: schema.EnvDefaultFunc(PROVIDER_SECRET_KEY, nil),
+				Description: "Secret key of Tencent Cloud",
+				Sensitive:   true,
+			},
+			"region": {
+				Type:         schema.TypeString,
+				Required:     true,
+				DefaultFunc:  schema.EnvDefaultFunc(PROVIDER_REGION, nil),
+				Description:  "The region of the COS bucket",
+				InputDefault: "ap-guangzhou",
+			},
+			"bucket": {
+				Type:        schema.TypeString,
+				Required:    true,
+				Description: "The name of the COS bucket",
+			},
+			"prefix": {
+				Type:        schema.TypeString,
+				Optional:    true,
+				Description: "The directory for saving the state file in bucket",
+				ValidateFunc: func(v interface{}, s string) ([]string, []error) {
+					prefix := v.(string)
+					if strings.HasPrefix(prefix, "/") || strings.HasPrefix(prefix, "./") {
+						return nil, []error{fmt.Errorf("prefix must not start with '/' or './'")}
+					}
+					return nil, nil
+				},
+			},
+			"key": {
+				Type:        schema.TypeString,
+				Optional:    true,
+				Description: "The path for saving the state file in bucket",
+				Default:     "terraform.tfstate",
+				ValidateFunc: func(v interface{}, s string) ([]string, []error) {
+					if strings.HasPrefix(v.(string), "/") || strings.HasSuffix(v.(string), "/") {
+						return nil, []error{fmt.Errorf("key can not start and end with '/'")}
+					}
+					return nil, nil
+				},
+			},
+			"encrypt": {
+				Type:        schema.TypeBool,
+				Optional:    true,
+				Description: "Whether to enable server side encryption of the state file",
+				Default:     true,
+			},
+			"acl": {
+				Type:        schema.TypeString,
+				Optional:    true,
+				Description: "Object ACL to be applied to the state file",
+				Default:     "private",
+				ValidateFunc: func(v interface{}, s string) ([]string, []error) {
+					value := v.(string)
+					if value != "private" && value != "public-read" {
+						return nil, []error{fmt.Errorf(
+							"acl value invalid, expected %s or %s, got %s",
+							"private", "public-read", value)}
+					}
+					return nil, nil
+				},
+			},
+		},
+	}
+
+	result := &Backend{Backend: s}
+	result.Backend.ConfigureFunc = result.configure
+
+	return result
+}
+
+// configure init cos client
+func (b *Backend) configure(ctx context.Context) error {
+	if b.cosClient != nil {
+		return nil
+	}
+
+	b.cosContext = ctx
+	data := schema.FromContextBackendConfig(b.cosContext)
+
+	b.region = data.Get("region").(string)
+	b.bucket = data.Get("bucket").(string)
+	b.prefix = data.Get("prefix").(string)
+	b.key = data.Get("key").(string)
+	b.encrypt = data.Get("encrypt").(bool)
+	b.acl = data.Get("acl").(string)
+
+	u, err := url.Parse(fmt.Sprintf("https://%s.cos.%s.myqcloud.com", b.bucket, b.region))
+	if err != nil {
+		return err
+	}
+
+	b.cosClient = cos.NewClient(
+		&cos.BaseURL{BucketURL: u},
+		&http.Client{
+			Timeout: 60 * time.Second,
+			Transport: &cos.AuthorizationTransport{
+				SecretID:  data.Get("secret_id").(string),
+				SecretKey: data.Get("secret_key").(string),
+			},
+		},
+	)
+
+	credential := common.NewCredential(
+		data.Get("secret_id").(string),
+		data.Get("secret_key").(string),
+	)
+
+	cpf := profile.NewClientProfile()
+	cpf.HttpProfile.ReqMethod = "POST"
+	cpf.HttpProfile.ReqTimeout = 300
+	cpf.Language = "en-US"
+	b.tagClient, err = tag.NewClient(credential, b.region, cpf)
+
+	return err
+}

backend/remote-state/cos/backend_state.go

@@ -0,0 +1,178 @@
+package cos
+
+import (
+	"fmt"
+	"log"
+	"path"
+	"sort"
+	"strings"
+
+	"github.com/hashicorp/terraform/backend"
+	"github.com/hashicorp/terraform/state"
+	"github.com/hashicorp/terraform/state/remote"
+	"github.com/hashicorp/terraform/states"
+	"github.com/likexian/gokit/assert"
+)
+
+// Define file suffix
+const (
+	stateFileSuffix = ".tfstate"
+	lockFileSuffix  = ".tflock"
+)
+
+// Workspaces returns a list of names for the workspaces
+func (b *Backend) Workspaces() ([]string, error) {
+	c, err := b.client("tencentcloud")
+	if err != nil {
+		return nil, err
+	}
+
+	obs, err := c.getBucket(b.prefix)
+	log.Printf("[DEBUG] list all workspaces, objects: %v, error: %v", obs, err)
+	if err != nil {
+		return nil, err
+	}
+
+	ws := []string{backend.DefaultStateName}
+	for _, vv := range obs {
+		// <name>.tfstate
+		if !strings.HasSuffix(vv.Key, stateFileSuffix) {
+			continue
+		}
+		// default worksapce
+		if path.Join(b.prefix, b.key) == vv.Key {
+			continue
+		}
+		// <prefix>/<worksapce>/<key>
+		prefix := strings.TrimRight(b.prefix, "/") + "/"
+		parts := strings.Split(strings.TrimPrefix(vv.Key, prefix), "/")
+		if len(parts) > 0 && parts[0] != "" {
+			ws = append(ws, parts[0])
+		}
+	}
+
+	sort.Strings(ws[1:])
+	log.Printf("[DEBUG] list all workspaces, workspaces: %v", ws)
+
+	return ws, nil
+}
+
+// DeleteWorkspace deletes the named workspaces. The "default" state cannot be deleted.
+func (b *Backend) DeleteWorkspace(name string) error {
+	log.Printf("[DEBUG] delete workspace, workspace: %v", name)
+
+	if name == backend.DefaultStateName || name == "" {
+		return fmt.Errorf("default state is not allow to delete")
+	}
+
+	c, err := b.client(name)
+	if err != nil {
+		return err
+	}
+
+	return c.Delete()
+}
+
+// StateMgr manage the state, if the named state not exists, a new file will created
+func (b *Backend) StateMgr(name string) (state.State, error) {
+	log.Printf("[DEBUG] state manager, current workspace: %v", name)
+
+	c, err := b.client(name)
+	if err != nil {
+		return nil, err
+	}
+	stateMgr := &remote.State{Client: c}
+
+	ws, err := b.Workspaces()
+	if err != nil {
+		return nil, err
+	}
+
+	if !assert.IsContains(ws, name) {
+		log.Printf("[DEBUG] workspace %v not exists", name)
+
+		// take a lock on this state while we write it
+		lockInfo := state.NewLockInfo()
+		lockInfo.Operation = "init"
+		lockId, err := c.Lock(lockInfo)
+		if err != nil {
+			return nil, fmt.Errorf("Failed to lock cos state: %s", err)
+		}
+
+		// Local helper function so we can call it multiple places
+		lockUnlock := func(e error) error {
+			if err := stateMgr.Unlock(lockId); err != nil {
+				return fmt.Errorf(unlockErrMsg, err, lockId)
+			}
+			return e
+		}
+
+		// Grab the value
+		if err := stateMgr.RefreshState(); err != nil {
+			err = lockUnlock(err)
+			return nil, err
+		}
+
+		// If we have no state, we have to create an empty state
+		if v := stateMgr.State(); v == nil {
+			if err := stateMgr.WriteState(states.NewState()); err != nil {
+				err = lockUnlock(err)
+				return nil, err
+			}
+			if err := stateMgr.PersistState(); err != nil {
+				err = lockUnlock(err)
+				return nil, err
+			}
+		}
+
+		// Unlock, the state should now be initialized
+		if err := lockUnlock(nil); err != nil {
+			return nil, err
+		}
+	}
+
+	return stateMgr, nil
+}
+
+// client returns a remoteClient for the named state.
+func (b *Backend) client(name string) (*remoteClient, error) {
+	if strings.TrimSpace(name) == "" {
+		return nil, fmt.Errorf("state name not allow to be empty")
+	}
+
+	return &remoteClient{
+		cosContext: b.cosContext,
+		cosClient:  b.cosClient,
+		tagClient:  b.tagClient,
+		bucket:     b.bucket,
+		stateFile:  b.stateFile(name),
+		lockFile:   b.lockFile(name),
+		encrypt:    b.encrypt,
+		acl:        b.acl,
+	}, nil
+}
+
+// stateFile returns state file path by name
+func (b *Backend) stateFile(name string) string {
+	if name == backend.DefaultStateName {
+		return path.Join(b.prefix, b.key)
+	}
+	return path.Join(b.prefix, name, b.key)
+}
+
+// lockFile returns lock file path by name
+func (b *Backend) lockFile(name string) string {
+	return b.stateFile(name) + lockFileSuffix
+}
+
+// unlockErrMsg is error msg for unlock failed
+const unlockErrMsg = `
+Unlocking the state file on TencentCloud cos backend failed:
+
+Error message: %v
+Lock ID (gen): %s
+
+You may have to force-unlock this state in order to use it again.
+The TencentCloud backend acquires a lock during initialization
+to ensure the initial state file is created.
+`