Incorporating feedback from pull request.

Giving Env paramaters for secrets, not just CLI.
Rename prefix to be storage_root i.a.w what we do for s3 and gcs

Signed-off-by: Dan Kozlowski <[email protected]>

go.mod

@@ -5,6 +5,7 @@ go 1.12
 require (
 	cloud.google.com/go v0.45.1
 	github.com/Azure/azure-storage-blob-go v0.8.0
+	github.com/Azure/go-autorest/autorest/adal v0.8.1 // indirect
 	github.com/Bowery/prompt v0.0.0-20190419144237-972d0ceb96f5 // indirect
 	github.com/GeertJohan/go.rice v1.0.0
 	github.com/armon/go-metrics v0.0.0-20190430140413-ec5e00d3c878 // indirect
@@ -79,7 +80,7 @@ require (
 	github.com/ugorji/go v1.1.7 // indirect
 	github.com/uudashr/gocognit v1.0.1 // indirect
 	github.com/z-division/go-zookeeper v0.0.0-20190128072838-6d7457066b9b
-	golang.org/x/crypto v0.0.0-20191205180655-e7c4368fe9dd
+	golang.org/x/crypto v0.0.0-20191206172530-e9b2fee46413
 	golang.org/x/lint v0.0.0-20190409202823-959b441ac422
 	golang.org/x/net v0.0.0-20190926025831-c00fd9afed17
 	golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45

go.sum

@@ -16,6 +16,20 @@ github.com/Azure/azure-pipeline-go v0.2.1 h1:OLBdZJ3yvOn2MezlWvbrBMTEUQC72zAftRZ
 github.com/Azure/azure-pipeline-go v0.2.1/go.mod h1:UGSo8XybXnIGZ3epmeBw7Jdz+HiUVpqIlpz/HKHylF4=
 github.com/Azure/azure-storage-blob-go v0.8.0 h1:53qhf0Oxa0nOjgbDeeYPUeyiNmafAFEY95rZLK0Tj6o=
 github.com/Azure/azure-storage-blob-go v0.8.0/go.mod h1:lPI3aLPpuLTeUwh1sViKXFxwl2B6teiRqI0deQUvsw0=
+github.com/Azure/go-autorest/autorest v0.9.0 h1:MRvx8gncNaXJqOoLmhNjUAKh33JJF8LyxPhomEtOsjs=
+github.com/Azure/go-autorest/autorest v0.9.0/go.mod h1:xyHB1BMZT0cuDHU7I0+g046+BFDTQ8rEZB0s4Yfa6bI=
+github.com/Azure/go-autorest/autorest/adal v0.5.0/go.mod h1:8Z9fGy2MpX0PvDjB1pEgQTmVqjGhiHBW7RJJEciWzS0=
+github.com/Azure/go-autorest/autorest/adal v0.8.1 h1:pZdL8o72rK+avFWl+p9nE8RWi1JInZrWJYlnpfXJwHk=
+github.com/Azure/go-autorest/autorest/adal v0.8.1/go.mod h1:ZjhuQClTqx435SRJ2iMlOxPYt3d2C/T/7TiQCVZSn3Q=
+github.com/Azure/go-autorest/autorest/date v0.1.0/go.mod h1:plvfp3oPSKwf2DNjlBjWF/7vwR+cUD/ELuzDCXwHUVA=
+github.com/Azure/go-autorest/autorest/date v0.2.0 h1:yW+Zlqf26583pE43KhfnhFcdmSWlm5Ew6bxipnr/tbM=
+github.com/Azure/go-autorest/autorest/date v0.2.0/go.mod h1:vcORJHLJEh643/Ioh9+vPmf1Ij9AEBM5FuBIXLmIy0g=
+github.com/Azure/go-autorest/autorest/mocks v0.1.0/go.mod h1:OTyCOPRA2IgIlWxVYxBee2F5Gr4kF2zd2J5cFRaIDN0=
+github.com/Azure/go-autorest/autorest/mocks v0.2.0/go.mod h1:OTyCOPRA2IgIlWxVYxBee2F5Gr4kF2zd2J5cFRaIDN0=
+github.com/Azure/go-autorest/autorest/mocks v0.3.0/go.mod h1:a8FDP3DYzQ4RYfVAxAN3SVSiiO77gL2j2ronKKP0syM=
+github.com/Azure/go-autorest/logger v0.1.0/go.mod h1:oExouG+K6PryycPJfVSxi/koC6LSNgds39diKLz7Vrc=
+github.com/Azure/go-autorest/tracing v0.5.0 h1:TRn4WjSnkcSy5AEG3pnbtFSwNtwzjr4VYyQflFE619k=
+github.com/Azure/go-autorest/tracing v0.5.0/go.mod h1:r/s2XiOKccPW3HrqB+W0TQzfbtp2fGCgRFtBroKn4Dk=
 github.com/Bowery/prompt v0.0.0-20190419144237-972d0ceb96f5 h1:7tNlRGC3pUEPKS3DwgX5L0s+cBloaq/JBoi9ceN1MCM=
 github.com/Bowery/prompt v0.0.0-20190419144237-972d0ceb96f5/go.mod h1:4/6eNcqZ09BZ9wLK3tZOjBA1nDj+B0728nlX5YRlSmQ=
 github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ=
@@ -697,8 +711,8 @@ golang.org/x/crypto v0.0.0-20190829043050-9756ffdc2472/go.mod h1:yigFU9vqHzYiE8U
 golang.org/x/crypto v0.0.0-20190923035154-9ee001bba392/go.mod h1:/lpIB1dKB+9EgE3H3cr1v9wB50oz8l4C4h62xy7jSTY=
 golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550 h1:ObdrDkeb4kJdCP557AjRjq69pTHfNouLtWZG7j9rPN8=
 golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
-golang.org/x/crypto v0.0.0-20191205180655-e7c4368fe9dd h1:GGJVjV8waZKRHrgwvtH66z9ZGVurTD1MT0n1Bb+q4aM=
-golang.org/x/crypto v0.0.0-20191205180655-e7c4368fe9dd/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
+golang.org/x/crypto v0.0.0-20191206172530-e9b2fee46413 h1:ULYEB3JvPRE/IfO+9uO7vKV/xzVTO7XPAwm8xbf4w2g=
+golang.org/x/crypto v0.0.0-20191206172530-e9b2fee46413/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
 golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8=
 golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js=

go/vt/mysqlctl/azblobbackupstorage/azblob.go

@@ -24,6 +24,7 @@ import (
 	"fmt"
 	"io"
 	"net/url"
+	"os"
 	"strings"
 	"sync"
 	"time"
@@ -36,26 +37,44 @@ import (
 
 var (
 	// This is the account name
-	accountName = flag.String("azblob_backup_account_name", "", "Azure Account Name for Backups")
+	accountName = flag.String("azblob_backup_account_name", "", "Azure Account Name for Backups, alternative environment paramater is VITESS_AZBLOB_ACCOUNT_NAME")
 
 	// This is the private access key
-	accountKey = flag.String("azblob_backup_account_key", "", "Azure Account Key")
+	accountKey = flag.String("azblob_backup_account_key", "", "Azure Account Key, alternative environment paramater is VITESS_AZBLOB_ACCOUNT_KEY")
 
 	// This is the name of the container that will store the backups
-	containerName = flag.String("azblob_backup_contain_name", "", "Azure Blob Contain Name")
+	containerName = flag.String("azblob_backup_container_name", "", "Azure Blob Container Name")
 
 	// This is an optional previx to prepend to all files
-	prefix = flag.String("azblob_backup_prefix", "", "Azure Blob prefix")
+	storageRoot = flag.String("azblob_backup_storage_root", "", "Azure Blob storage root")
 
-	azBlobParallelism = flag.Int("azblob_parallelism", 1, "Azure blob operation parallelism (requires extra memory when increased)")
+	azBlobParallelism = flag.Int("azblob_backup_parallelism", 1, "Azure blob operation parallelism (requires extra memory when increased)")
 )
 
 const (
 	defaultRetryCount = 5
 	delimiter         = "/"
 )
 
+// Return a Shared credential from the available credential sources.
+// We will use credentials in the following order
+// 1. Direct Command Line Flag (azblob_backup_account_name, azblob_backup_account_key)
+// 2. Environment Paramaters
 func azCredentials() (*azblob.SharedKeyCredential, error) {
+	actName := *accountName
+	if len(actName) == 0 {
+		// Check the Environmental Value
+		actName = os.Getenv("VITESS_AZBLOB_ACCOUNT_NAME")
+	}
+
+	actKey := *accountKey
+	if len(actKey) == 0 {
+		actKey = os.Getenv("VITESS_AZBLOB_ACCOUNT_NAME")
+	}
+
+	if len(actName) == 0 || len(actKey) == 0 {
+		return nil, fmt.Errorf("can not get Account Credentials from CLI or Environment paramaters")
+	}
 	return azblob.NewSharedKeyCredential(*accountName, *accountKey)
 }
 
@@ -152,7 +171,7 @@ func (bh *AZBlobBackupHandle) ReadFile(ctx context.Context, filename string) (io
 		return nil, fmt.Errorf("ReadFile cannot be called on read-write backup")
 	}
 
-	obj := objName(bh.dir, bh.name, filename)
+	obj := objName(bh.dir, filename)
 	containerURL, err := bh.bs.containerURL()
 	if err != nil {
 		return nil, err
@@ -166,7 +185,10 @@ func (bh *AZBlobBackupHandle) ReadFile(ctx context.Context, filename string) (io
 	return resp.Body(azblob.RetryReaderOptions{
 		MaxRetryRequests: defaultRetryCount,
 		NotifyFailedRead: func(failureCount int, lastError error, offset int64, count int64, willRetry bool) {
-			bh.errors.RecordError(lastError)
+			log.Infof("ReadFile: [azblob] container: %s, directory: %s, filename: %s, error: %v", *containerName, objName(bh.dir, ""), filename, lastError)
+			if !willRetry {
+				bh.errors.RecordError(lastError)
+			}
 		},
 		TreatEarlyCloseAsError: true,
 	}), nil
@@ -187,7 +209,7 @@ func (bs *AZBlobBackupStorage) containerURL() (*azblob.ContainerURL, error) {
 
 // ListBackups implements BackupStorage.
 func (bs *AZBlobBackupStorage) ListBackups(ctx context.Context, dir string) ([]backupstorage.BackupHandle, error) {
-	log.Infof("ListBackups: [azblob] container: %v, prefix: %v", *containerName, dir)
+	log.Infof("ListBackups: [azblob] container: %s, directory: %v", *containerName, objName(dir, ""))
 
 	containerURL, err := bs.containerURL()
 	if err != nil {
@@ -200,6 +222,7 @@ func (bs *AZBlobBackupStorage) ListBackups(ctx context.Context, dir string) ([]b
 	var subdirs []string
 
 	for marker := (azblob.Marker{}); marker.NotDone(); {
+		// This returns Blobs in sorted order so we don't need to sort them a second time
 		resp, err := containerURL.ListBlobsHierarchySegment(ctx, marker, delimiter, azblob.ListBlobsSegmentOptions{
 			Prefix:     searchPrefix,
 			MaxResults: 0,
@@ -242,7 +265,7 @@ func (bs *AZBlobBackupStorage) StartBackup(ctx context.Context, dir, name string
 
 // RemoveBackup implements BackupStorage.
 func (bs *AZBlobBackupStorage) RemoveBackup(ctx context.Context, dir, name string) error {
-	log.Infof("ListBackups: [azblob] container: %v, prefix: %v", *containerName, dir)
+	log.Infof("ListBackups: [azblob] container: %s, directory: %s", *containerName, objName(dir, ""))
 
 	containerURL, err := bs.containerURL()
 	if err != nil {
@@ -284,10 +307,10 @@ func (bs *AZBlobBackupStorage) Close() error {
 
 // objName joins path parts into an object name.
 // Unlike path.Join, it doesn't collapse ".." or strip trailing slashes.
-// It also adds the value of the -gcs_backup_storage_root flag if set.
+// It also adds the value of the -azblob_backup_storage_root flag if set.
 func objName(parts ...string) string {
-	if *prefix != "" {
-		return *prefix + "/" + strings.Join(parts, "/")
+	if *storageRoot != "" {
+		return *storageRoot + "/" + strings.Join(parts, "/")
 	}
 	return strings.Join(parts, "/")
 }