Add a new field emulatorEnabled in gcp secret & configure Gcloud clie…

…nt for GCS Emulator

chart/etcd-backup-restore/templates/etcd-backup-secret.yaml

@@ -28,6 +28,9 @@ data:
   {{- if .Values.backup.gcs.storageAPIEndpoint }}
   storageAPIEndpoint: {{ .Values.backup.gcs.storageAPIEndpoint | b64enc}}
   {{- end }}
+  {{- if .Values.backup.gcs.emulatorEnabled }}
+  emulatorEnabled: {{ .Values.backup.gcs.emulatorEnabled | b64enc}}
+  {{- end }}
 {{- else if eq .Values.backup.storageProvider "Swift" }}
   authURL: {{ .Values.backup.swift.authURL | b64enc }}
   domainName: {{ .Values.backup.swift.domainName | b64enc }}

chart/etcd-backup-restore/templates/etcd-statefulset.yaml

@@ -218,6 +218,13 @@ spec:
               key: "storageAPIEndpoint"
               optional: true
   {{- end }}
+  {{- if .Values.backup.gcs.emulatorEnabled }}
+        - name: "GOOGLE_EMULATOR_ENABLED"
+          valueFrom:
+            secretKeyRef:
+              name: {{ .Release.Name }}-etcd-backup
+              key: "emulatorEnabled"
+  {{- end }}
 {{- else if eq .Values.backup.storageProvider "Swift" }}
         - name: "OS_AUTH_URL"
           valueFrom:
@@ -356,4 +363,4 @@ spec:
       - "ReadWriteOnce"
       resources:
         requests:
-          storage: {{ .Values.storageCapacity }}
+          storage: {{ .Values.storageCapacity }}

chart/etcd-backup-restore/values.yaml

@@ -91,6 +91,7 @@ backup:
   # gcs:
   #   serviceAccountJson: service-account-json-with-object-storage-privileges
   #   storageAPIEndpoint: endpoint-override-for-storage-api # optional
+  #   emulatorEnabled: boolean-flag-to-configure-etcdbr-to-use-gcs-emulator # optional
   # abs:
   #   storageAccount: storage-account-with-object-storage-privileges
   #   storageKey: storage-key-with-object-storage-privileges

example/storage-provider-secrets/04-google-cloud-storage-secret.yaml

@@ -6,4 +6,5 @@ metadata:
 type: Opaque
 data:
   serviceaccount.json: ...
-#  storageAPIEndpoint: # http[s]://host[:port]/storage/v1/
+#  storageAPIEndpoint: aHR0cFtzXTovL2hvc3RbOnBvcnRdL3N0b3JhZ2UvdjEv # http[s]://host[:port]/storage/v1/
+#  emulatorEnabled: dHJ1ZQ== # true (optional)

pkg/snapstore/gcs_snapstore.go

@@ -12,6 +12,7 @@ import (
 	"os"
 	"path"
 	"sort"
+	"strconv"
 	"strings"
 	"sync"
 	"time"
@@ -29,6 +30,7 @@ const (
 	envStoreCredentials       = "GOOGLE_APPLICATION_CREDENTIALS"
 	envStorageAPIEndpoint     = "GOOGLE_STORAGE_API_ENDPOINT"
 	envSourceStoreCredentials = "SOURCE_GOOGLE_APPLICATION_CREDENTIALS"
+	envEmulatorEnabled        = "GOOGLE_EMULATOR_ENABLED"
 )
 
 // GCSSnapStore is snapstore with GCS object store as backend.
@@ -40,6 +42,13 @@ type GCSSnapStore struct {
 	maxParallelChunkUploads uint
 	minChunkSize            int64
 	tempDir                 string
+	chunkDirSuffix          string
+}
+
+// gcsEmulatorConfig holds the configuration for the fake GCS emulator
+type gcsEmulatorConfig struct {
+	enabled  bool   // whether the fake GCS emulator is enabled
+	endpoint string // the endpoint of the fake GCS emulator
 }
 
 const (
@@ -50,16 +59,28 @@ const (
 // NewGCSSnapStore create new GCSSnapStore from shared configuration with specified bucket.
 func NewGCSSnapStore(config *brtypes.SnapstoreConfig) (*GCSSnapStore, error) {
 	ctx := context.TODO()
+	var emulatorConfig gcsEmulatorConfig
+	emulatorConfig.enabled = isEmulatorEnabled()
 	var opts []option.ClientOption // no need to explicitly set store credentials here since the Google SDK picks it up from the standard environment variable
 
-	if _, ok := os.LookupEnv(envSourceStoreCredentials); !ok { // do not set endpoint override when copying backups between buckets, since the buckets may reside on different regions
+	if _, ok := os.LookupEnv(envSourceStoreCredentials); !ok || emulatorConfig.enabled { // do not set endpoint override when copying backups between buckets, since the buckets may reside on different regions
 		endpoint := strings.TrimSpace(os.Getenv(envStorageAPIEndpoint))
 		if endpoint != "" {
 			opts = append(opts, option.WithEndpoint(endpoint))
+			if emulatorConfig.enabled {
+				emulatorConfig.endpoint = endpoint
+			}
 		}
 	}
-
-	if config.IsSource {
+	var chunkDirSuffix string
+	if emulatorConfig.enabled {
+		err := emulatorConfig.configureClient(opts)
+		if err != nil {
+			return nil, err
+		}
+		chunkDirSuffix = brtypes.ChunkDirSuffix
+	}
+	if config.IsSource && !emulatorConfig.enabled {
 		filename := os.Getenv(envSourceStoreCredentials)
 		if filename == "" {
 			return nil, fmt.Errorf("environment variable %s is not set", envSourceStoreCredentials)
@@ -73,19 +94,43 @@ func NewGCSSnapStore(config *brtypes.SnapstoreConfig) (*GCSSnapStore, error) {
 	}
 	gcsClient := stiface.AdaptClient(cli)
 
-	return NewGCSSnapStoreFromClient(config.Container, config.Prefix, config.TempDir, config.MaxParallelChunkUploads, config.MinChunkSize, gcsClient), nil
+	return NewGCSSnapStoreFromClient(config.Container, config.Prefix, config.TempDir, config.MaxParallelChunkUploads, config.MinChunkSize, chunkDirSuffix, gcsClient), nil
 }
 
 // NewGCSSnapStoreFromClient create new GCSSnapStore from shared configuration with specified bucket.
-func NewGCSSnapStoreFromClient(bucket, prefix, tempDir string, maxParallelChunkUploads uint, minChunkSize int64, cli stiface.Client) *GCSSnapStore {
+func NewGCSSnapStoreFromClient(bucket, prefix, tempDir string, maxParallelChunkUploads uint, minChunkSize int64, chunkDirSuffix string, cli stiface.Client) *GCSSnapStore {
 	return &GCSSnapStore{
 		prefix:                  prefix,
 		client:                  cli,
 		bucket:                  bucket,
 		maxParallelChunkUploads: maxParallelChunkUploads,
 		minChunkSize:            minChunkSize,
 		tempDir:                 tempDir,
+		chunkDirSuffix:          chunkDirSuffix,
+	}
+}
+
+// isEmulatorEnabled checks if the fake GCS emulator is enabled
+func isEmulatorEnabled() bool {
+	isFakeGCSEnabled, ok := os.LookupEnv(envEmulatorEnabled)
+	if !ok {
+		return false
 	}
+	emulatorEnabled, err := strconv.ParseBool(isFakeGCSEnabled)
+	if err != nil {
+		return false
+	}
+	return emulatorEnabled
+}
+
+// configureClient configures the fake gcs emulator
+func (e *gcsEmulatorConfig) configureClient(opts []option.ClientOption) error {
+	err := os.Setenv("STORAGE_EMULATOR_HOST", strings.TrimPrefix(e.endpoint, "http://"))
+	if err != nil {
+		return fmt.Errorf("failed to set the environment variable for the fake GCS emulator: %v", err)
+	}
+	opts = append(opts, option.WithoutAuthentication())
+	return nil
 }
 
 // Fetch should open reader for the snapshot file from store.
@@ -155,7 +200,7 @@ func (s *GCSSnapStore) Save(snap brtypes.Snapshot, rc io.ReadCloser) error {
 	var subObjects []stiface.ObjectHandle
 	prefix := adaptPrefix(&snap, s.prefix)
 	for partNumber := int64(1); partNumber <= noOfChunks; partNumber++ {
-		name := path.Join(prefix, snap.SnapDir, snap.SnapName, fmt.Sprintf("%010d", partNumber))
+		name := path.Join(prefix, snap.SnapDir, fmt.Sprintf("%s%s", snap.SnapName, s.chunkDirSuffix), fmt.Sprintf("%010d", partNumber))
 		obj := bh.Object(name)
 		subObjects = append(subObjects, obj)
 	}
@@ -184,7 +229,7 @@ func (s *GCSSnapStore) uploadComponent(snap *brtypes.Snapshot, file *os.File, of
 	sr := io.NewSectionReader(file, offset, size)
 	bh := s.client.Bucket(s.bucket)
 	partNumber := ((offset / chunkSize) + 1)
-	name := path.Join(adaptPrefix(snap, s.prefix), snap.SnapDir, snap.SnapName, fmt.Sprintf("%010d", partNumber))
+	name := path.Join(adaptPrefix(snap, s.prefix), snap.SnapDir, fmt.Sprintf("%s%s", snap.SnapName, s.chunkDirSuffix), fmt.Sprintf("%010d", partNumber))
 	obj := bh.Object(name)
 	ctx, cancel := context.WithTimeout(context.TODO(), chunkUploadTimeout)
 	defer cancel()

pkg/snapstore/snapshot.go

@@ -116,6 +116,10 @@ func ParseSnapshot(snapPath string) (*brtypes.Snapshot, error) {
 	//parse creation time as well as parse the Snapshot compression suffix
 	lastNameToken := strings.Split(tokens[3], "/")
 	timeWithSnapSuffix := strings.Split(lastNameToken[0], ".")
+	// Check & remove if the last token is a chunk directory. The chunkDirSuffix is set by only GCS snapstore when using a emulator for testing.
+	if fmt.Sprintf(".%s", timeWithSnapSuffix[len(timeWithSnapSuffix)-1]) == brtypes.ChunkDirSuffix {
+		timeWithSnapSuffix = timeWithSnapSuffix[:len(timeWithSnapSuffix)-1]
+	}
 	if len(timeWithSnapSuffix) >= 2 {
 		if "."+timeWithSnapSuffix[1] != brtypes.FinalSuffix {
 			s.CompressionSuffix = "." + timeWithSnapSuffix[1]

pkg/types/snapstore.go

@@ -48,6 +48,10 @@ const (
 	// FinalSuffix is the suffix appended to the names of final snapshots.
 	FinalSuffix = ".final"
 
+	// ChunkDirSuffix is the suffix appended to the name of chunk snapshot folder when using fakegcs emulator for testing.
+	// Refer to this github issue for more details: https://github.com/fsouza/fake-gcs-server/issues/1434
+	ChunkDirSuffix = ".chunk"
+
 	backupFormatVersion = "v2"
 
 	// MinChunkSize is set to 5Mib since it is lower chunk size limit for AWS.