diff --git a/go.mod b/go.mod index 787033c55cf..a17bd6ddd53 100644 --- a/go.mod +++ b/go.mod @@ -5,6 +5,7 @@ go 1.12 require ( cloud.google.com/go v0.45.1 github.com/Azure/azure-storage-blob-go v0.8.0 + github.com/Azure/go-autorest/autorest/adal v0.8.1 // indirect github.com/Bowery/prompt v0.0.0-20190419144237-972d0ceb96f5 // indirect github.com/GeertJohan/go.rice v1.0.0 github.com/armon/go-metrics v0.0.0-20190430140413-ec5e00d3c878 // indirect @@ -78,7 +79,7 @@ require ( github.com/ugorji/go v1.1.7 // indirect github.com/uudashr/gocognit v1.0.1 // indirect github.com/z-division/go-zookeeper v0.0.0-20190128072838-6d7457066b9b - golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550 + golang.org/x/crypto v0.0.0-20191206172530-e9b2fee46413 golang.org/x/lint v0.0.0-20190409202823-959b441ac422 golang.org/x/net v0.0.0-20190926025831-c00fd9afed17 golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45 diff --git a/go.sum b/go.sum index f4f408d2716..9009e83b66b 100644 --- a/go.sum +++ b/go.sum @@ -11,6 +11,20 @@ github.com/Azure/azure-pipeline-go v0.2.1 h1:OLBdZJ3yvOn2MezlWvbrBMTEUQC72zAftRZ github.com/Azure/azure-pipeline-go v0.2.1/go.mod h1:UGSo8XybXnIGZ3epmeBw7Jdz+HiUVpqIlpz/HKHylF4= github.com/Azure/azure-storage-blob-go v0.8.0 h1:53qhf0Oxa0nOjgbDeeYPUeyiNmafAFEY95rZLK0Tj6o= github.com/Azure/azure-storage-blob-go v0.8.0/go.mod h1:lPI3aLPpuLTeUwh1sViKXFxwl2B6teiRqI0deQUvsw0= +github.com/Azure/go-autorest/autorest v0.9.0 h1:MRvx8gncNaXJqOoLmhNjUAKh33JJF8LyxPhomEtOsjs= +github.com/Azure/go-autorest/autorest v0.9.0/go.mod h1:xyHB1BMZT0cuDHU7I0+g046+BFDTQ8rEZB0s4Yfa6bI= +github.com/Azure/go-autorest/autorest/adal v0.5.0/go.mod h1:8Z9fGy2MpX0PvDjB1pEgQTmVqjGhiHBW7RJJEciWzS0= +github.com/Azure/go-autorest/autorest/adal v0.8.1 h1:pZdL8o72rK+avFWl+p9nE8RWi1JInZrWJYlnpfXJwHk= +github.com/Azure/go-autorest/autorest/adal v0.8.1/go.mod h1:ZjhuQClTqx435SRJ2iMlOxPYt3d2C/T/7TiQCVZSn3Q= +github.com/Azure/go-autorest/autorest/date v0.1.0/go.mod h1:plvfp3oPSKwf2DNjlBjWF/7vwR+cUD/ELuzDCXwHUVA= +github.com/Azure/go-autorest/autorest/date v0.2.0 h1:yW+Zlqf26583pE43KhfnhFcdmSWlm5Ew6bxipnr/tbM= +github.com/Azure/go-autorest/autorest/date v0.2.0/go.mod h1:vcORJHLJEh643/Ioh9+vPmf1Ij9AEBM5FuBIXLmIy0g= +github.com/Azure/go-autorest/autorest/mocks v0.1.0/go.mod h1:OTyCOPRA2IgIlWxVYxBee2F5Gr4kF2zd2J5cFRaIDN0= +github.com/Azure/go-autorest/autorest/mocks v0.2.0/go.mod h1:OTyCOPRA2IgIlWxVYxBee2F5Gr4kF2zd2J5cFRaIDN0= +github.com/Azure/go-autorest/autorest/mocks v0.3.0/go.mod h1:a8FDP3DYzQ4RYfVAxAN3SVSiiO77gL2j2ronKKP0syM= +github.com/Azure/go-autorest/logger v0.1.0/go.mod h1:oExouG+K6PryycPJfVSxi/koC6LSNgds39diKLz7Vrc= +github.com/Azure/go-autorest/tracing v0.5.0 h1:TRn4WjSnkcSy5AEG3pnbtFSwNtwzjr4VYyQflFE619k= +github.com/Azure/go-autorest/tracing v0.5.0/go.mod h1:r/s2XiOKccPW3HrqB+W0TQzfbtp2fGCgRFtBroKn4Dk= github.com/Bowery/prompt v0.0.0-20190419144237-972d0ceb96f5 h1:7tNlRGC3pUEPKS3DwgX5L0s+cBloaq/JBoi9ceN1MCM= github.com/Bowery/prompt v0.0.0-20190419144237-972d0ceb96f5/go.mod h1:4/6eNcqZ09BZ9wLK3tZOjBA1nDj+B0728nlX5YRlSmQ= github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ= @@ -524,6 +538,8 @@ golang.org/x/crypto v0.0.0-20190829043050-9756ffdc2472/go.mod h1:yigFU9vqHzYiE8U golang.org/x/crypto v0.0.0-20190923035154-9ee001bba392/go.mod h1:/lpIB1dKB+9EgE3H3cr1v9wB50oz8l4C4h62xy7jSTY= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550 h1:ObdrDkeb4kJdCP557AjRjq69pTHfNouLtWZG7j9rPN8= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20191206172530-e9b2fee46413 h1:ULYEB3JvPRE/IfO+9uO7vKV/xzVTO7XPAwm8xbf4w2g= +golang.org/x/crypto v0.0.0-20191206172530-e9b2fee46413/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= diff --git a/go/vt/mysqlctl/azblobbackupstorage/azblob.go b/go/vt/mysqlctl/azblobbackupstorage/azblob.go index b829b8605f5..4567fe823c1 100644 --- a/go/vt/mysqlctl/azblobbackupstorage/azblob.go +++ b/go/vt/mysqlctl/azblobbackupstorage/azblob.go @@ -24,6 +24,7 @@ import ( "fmt" "io" "net/url" + "os" "strings" "sync" "time" @@ -36,18 +37,18 @@ import ( var ( // This is the account name - accountName = flag.String("azblob_backup_account_name", "", "Azure Account Name for Backups") + accountName = flag.String("azblob_backup_account_name", "", "Azure Account Name for Backups, alternative environment paramater is VITESS_AZBLOB_ACCOUNT_NAME") // This is the private access key - accountKey = flag.String("azblob_backup_account_key", "", "Azure Account Key") + accountKey = flag.String("azblob_backup_account_key", "", "Azure Account Key, alternative environment paramater is VITESS_AZBLOB_ACCOUNT_KEY") // This is the name of the container that will store the backups - containerName = flag.String("azblob_backup_contain_name", "", "Azure Blob Contain Name") + containerName = flag.String("azblob_backup_container_name", "", "Azure Blob Container Name") // This is an optional previx to prepend to all files - prefix = flag.String("azblob_backup_prefix", "", "Azure Blob prefix") + storageRoot = flag.String("azblob_backup_storage_root", "", "Azure Blob storage root") - azBlobParallelism = flag.Int("azblob_parallelism", 1, "Azure blob operation parallelism (requires extra memory when increased)") + azBlobParallelism = flag.Int("azblob_backup_parallelism", 1, "Azure blob operation parallelism (requires extra memory when increased)") ) const ( @@ -55,7 +56,25 @@ const ( delimiter = "/" ) +// Return a Shared credential from the available credential sources. +// We will use credentials in the following order +// 1. Direct Command Line Flag (azblob_backup_account_name, azblob_backup_account_key) +// 2. Environment Paramaters func azCredentials() (*azblob.SharedKeyCredential, error) { + actName := *accountName + if len(actName) == 0 { + // Check the Environmental Value + actName = os.Getenv("VITESS_AZBLOB_ACCOUNT_NAME") + } + + actKey := *accountKey + if len(actKey) == 0 { + actKey = os.Getenv("VITESS_AZBLOB_ACCOUNT_NAME") + } + + if len(actName) == 0 || len(actKey) == 0 { + return nil, fmt.Errorf("can not get Account Credentials from CLI or Environment paramaters") + } return azblob.NewSharedKeyCredential(*accountName, *accountKey) } @@ -152,7 +171,7 @@ func (bh *AZBlobBackupHandle) ReadFile(ctx context.Context, filename string) (io return nil, fmt.Errorf("ReadFile cannot be called on read-write backup") } - obj := objName(bh.dir, bh.name, filename) + obj := objName(bh.dir, filename) containerURL, err := bh.bs.containerURL() if err != nil { return nil, err @@ -166,7 +185,10 @@ func (bh *AZBlobBackupHandle) ReadFile(ctx context.Context, filename string) (io return resp.Body(azblob.RetryReaderOptions{ MaxRetryRequests: defaultRetryCount, NotifyFailedRead: func(failureCount int, lastError error, offset int64, count int64, willRetry bool) { - bh.errors.RecordError(lastError) + log.Infof("ReadFile: [azblob] container: %s, directory: %s, filename: %s, error: %v", *containerName, objName(bh.dir, ""), filename, lastError) + if !willRetry { + bh.errors.RecordError(lastError) + } }, TreatEarlyCloseAsError: true, }), nil @@ -187,7 +209,7 @@ func (bs *AZBlobBackupStorage) containerURL() (*azblob.ContainerURL, error) { // ListBackups implements BackupStorage. func (bs *AZBlobBackupStorage) ListBackups(ctx context.Context, dir string) ([]backupstorage.BackupHandle, error) { - log.Infof("ListBackups: [azblob] container: %v, prefix: %v", *containerName, dir) + log.Infof("ListBackups: [azblob] container: %s, directory: %v", *containerName, objName(dir, "")) containerURL, err := bs.containerURL() if err != nil { @@ -200,6 +222,7 @@ func (bs *AZBlobBackupStorage) ListBackups(ctx context.Context, dir string) ([]b var subdirs []string for marker := (azblob.Marker{}); marker.NotDone(); { + // This returns Blobs in sorted order so we don't need to sort them a second time resp, err := containerURL.ListBlobsHierarchySegment(ctx, marker, delimiter, azblob.ListBlobsSegmentOptions{ Prefix: searchPrefix, MaxResults: 0, @@ -242,7 +265,7 @@ func (bs *AZBlobBackupStorage) StartBackup(ctx context.Context, dir, name string // RemoveBackup implements BackupStorage. func (bs *AZBlobBackupStorage) RemoveBackup(ctx context.Context, dir, name string) error { - log.Infof("ListBackups: [azblob] container: %v, prefix: %v", *containerName, dir) + log.Infof("ListBackups: [azblob] container: %s, directory: %s", *containerName, objName(dir, "")) containerURL, err := bs.containerURL() if err != nil { @@ -284,10 +307,10 @@ func (bs *AZBlobBackupStorage) Close() error { // objName joins path parts into an object name. // Unlike path.Join, it doesn't collapse ".." or strip trailing slashes. -// It also adds the value of the -gcs_backup_storage_root flag if set. +// It also adds the value of the -azblob_backup_storage_root flag if set. func objName(parts ...string) string { - if *prefix != "" { - return *prefix + "/" + strings.Join(parts, "/") + if *storageRoot != "" { + return *storageRoot + "/" + strings.Join(parts, "/") } return strings.Join(parts, "/") }