diff --git a/go.mod b/go.mod index 7f7f0100e04..787033c55cf 100644 --- a/go.mod +++ b/go.mod @@ -4,6 +4,7 @@ go 1.12 require ( cloud.google.com/go v0.45.1 + github.com/Azure/azure-storage-blob-go v0.8.0 github.com/Bowery/prompt v0.0.0-20190419144237-972d0ceb96f5 // indirect github.com/GeertJohan/go.rice v1.0.0 github.com/armon/go-metrics v0.0.0-20190430140413-ec5e00d3c878 // indirect @@ -50,6 +51,8 @@ require ( github.com/mattn/go-runewidth v0.0.1 // indirect github.com/minio/minio-go v0.0.0-20190131015406-c8a261de75c1 github.com/mitchellh/go-testing-interface v1.0.0 // indirect + github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect + github.com/modern-go/reflect2 v1.0.1 // indirect github.com/olekukonko/tablewriter v0.0.0-20160115111002-cca8bbc07984 github.com/opentracing-contrib/go-grpc v0.0.0-20180928155321-4b5a12d3ff02 github.com/opentracing/opentracing-go v1.1.0 diff --git a/go.sum b/go.sum index cb9c1f13a95..f4f408d2716 100644 --- a/go.sum +++ b/go.sum @@ -7,6 +7,10 @@ cloud.google.com/go v0.45.1 h1:lRi0CHyU+ytlvylOlFKKq0af6JncuyoRh1J+QJBqQx0= cloud.google.com/go v0.45.1/go.mod h1:RpBamKRgapWJb87xiFSdk4g1CME7QZg3uwTez+TSTjc= cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o= cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= +github.com/Azure/azure-pipeline-go v0.2.1 h1:OLBdZJ3yvOn2MezlWvbrBMTEUQC72zAftRZOMdj5HYo= +github.com/Azure/azure-pipeline-go v0.2.1/go.mod h1:UGSo8XybXnIGZ3epmeBw7Jdz+HiUVpqIlpz/HKHylF4= +github.com/Azure/azure-storage-blob-go v0.8.0 h1:53qhf0Oxa0nOjgbDeeYPUeyiNmafAFEY95rZLK0Tj6o= +github.com/Azure/azure-storage-blob-go v0.8.0/go.mod h1:lPI3aLPpuLTeUwh1sViKXFxwl2B6teiRqI0deQUvsw0= github.com/Bowery/prompt v0.0.0-20190419144237-972d0ceb96f5 h1:7tNlRGC3pUEPKS3DwgX5L0s+cBloaq/JBoi9ceN1MCM= github.com/Bowery/prompt v0.0.0-20190419144237-972d0ceb96f5/go.mod h1:4/6eNcqZ09BZ9wLK3tZOjBA1nDj+B0728nlX5YRlSmQ= github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ= @@ -308,6 +312,8 @@ github.com/matoous/godox v0.0.0-20190911065817-5d6d842e92eb h1:RHba4YImhrUVQDHUC github.com/matoous/godox v0.0.0-20190911065817-5d6d842e92eb/go.mod h1:1BELzlh859Sh1c6+90blK8lbYy0kwQf1bYlBhBysy1s= github.com/mattn/go-colorable v0.1.4 h1:snbPLB8fVfU9iwbbo30TPtbLRzwWu6aJS6Xh4eaaviA= github.com/mattn/go-colorable v0.1.4/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= +github.com/mattn/go-ieproxy v0.0.0-20190610004146-91bb50d98149 h1:HfxbT6/JcvIljmERptWhwa8XzP7H3T+Z2N26gTsaDaA= +github.com/mattn/go-ieproxy v0.0.0-20190610004146-91bb50d98149/go.mod h1:31jz6HNzdxOmlERGGEc4v/dMssOfmp2p5bT/okiKFFc= github.com/mattn/go-isatty v0.0.8 h1:HLtExJ+uU2HOZ+wI0Tt5DtUDrx8yhUqDcp7fYERX4CE= github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= github.com/mattn/go-isatty v0.0.11 h1:FxPOTFNqGkuDUGi3H/qkUbQO4ZiBa2brKq5r0l8TGeM= @@ -617,13 +623,13 @@ golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgw golang.org/x/tools v0.0.0-20190719005602-e377ae9d6386/go.mod h1:jcCCGcm9btYwXyDqrUWc6MKQKKGJCWEQ3AfLSRIbEuI= golang.org/x/tools v0.0.0-20190830154057-c17b040389b9 h1:5/jaG/gKlo3xxvUn85ReNyTlN7BvlPPsxC6sHZKjGEE= golang.org/x/tools v0.0.0-20190830154057-c17b040389b9/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191209225234-22774f7dae43 h1:NfPq5mgc5ArFgVLCpeS4z07IoxSAqVfV/gQ5vxdgaxI= -golang.org/x/tools v0.0.0-20191209225234-22774f7dae43/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20190910044552-dd2b5c81c578/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20190911151314-feee8acb394c/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20190930201159-7c411dea38b0/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191010075000-0337d82405ff h1:XdBG6es/oFDr1HwaxkxgVve7NB281QhxgK/i4voubFs= golang.org/x/tools v0.0.0-20191010075000-0337d82405ff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191209225234-22774f7dae43 h1:NfPq5mgc5ArFgVLCpeS4z07IoxSAqVfV/gQ5vxdgaxI= +golang.org/x/tools v0.0.0-20191209225234-22774f7dae43/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191217033636-bbbf87ae2631/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20191219041853-979b82bfef62 h1:vDaiisQl0rGVXqk3wT2yc43gSnwlj4haEG5J78IGZP4= golang.org/x/tools v0.0.0-20191219041853-979b82bfef62/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= diff --git a/go/cmd/vtbackup/plugin_azblobbackupstorage.go b/go/cmd/vtbackup/plugin_azblobbackupstorage.go new file mode 100644 index 00000000000..a4ca64096a9 --- /dev/null +++ b/go/cmd/vtbackup/plugin_azblobbackupstorage.go @@ -0,0 +1,21 @@ +/* +Copyright 2020 The Vitess Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package main + +import ( + _ "vitess.io/vitess/go/vt/mysqlctl/azblobbackupstorage" +) diff --git a/go/cmd/vtctl/plugin_azblobbackupstorage.go b/go/cmd/vtctl/plugin_azblobbackupstorage.go new file mode 100644 index 00000000000..a4ca64096a9 --- /dev/null +++ b/go/cmd/vtctl/plugin_azblobbackupstorage.go @@ -0,0 +1,21 @@ +/* +Copyright 2020 The Vitess Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package main + +import ( + _ "vitess.io/vitess/go/vt/mysqlctl/azblobbackupstorage" +) diff --git a/go/cmd/vtctld/plugin_azblobbackupstorage.go b/go/cmd/vtctld/plugin_azblobbackupstorage.go new file mode 100644 index 00000000000..a4ca64096a9 --- /dev/null +++ b/go/cmd/vtctld/plugin_azblobbackupstorage.go @@ -0,0 +1,21 @@ +/* +Copyright 2020 The Vitess Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package main + +import ( + _ "vitess.io/vitess/go/vt/mysqlctl/azblobbackupstorage" +) diff --git a/go/cmd/vttablet/plugin_azblobbackupstorage.go b/go/cmd/vttablet/plugin_azblobbackupstorage.go new file mode 100644 index 00000000000..a4ca64096a9 --- /dev/null +++ b/go/cmd/vttablet/plugin_azblobbackupstorage.go @@ -0,0 +1,21 @@ +/* +Copyright 2020 The Vitess Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package main + +import ( + _ "vitess.io/vitess/go/vt/mysqlctl/azblobbackupstorage" +) diff --git a/go/vt/mysqlctl/azblobbackupstorage/azblob.go b/go/vt/mysqlctl/azblobbackupstorage/azblob.go new file mode 100644 index 00000000000..b829b8605f5 --- /dev/null +++ b/go/vt/mysqlctl/azblobbackupstorage/azblob.go @@ -0,0 +1,297 @@ +/* +Copyright 2020 The Vitess Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Package azblobbackupstorage implements the BackupStorage interface +// for Azure Blob Storage +package azblobbackupstorage + +import ( + "context" + "flag" + "fmt" + "io" + "net/url" + "strings" + "sync" + "time" + + "github.com/Azure/azure-storage-blob-go/azblob" + "vitess.io/vitess/go/vt/concurrency" + "vitess.io/vitess/go/vt/log" + "vitess.io/vitess/go/vt/mysqlctl/backupstorage" +) + +var ( + // This is the account name + accountName = flag.String("azblob_backup_account_name", "", "Azure Account Name for Backups") + + // This is the private access key + accountKey = flag.String("azblob_backup_account_key", "", "Azure Account Key") + + // This is the name of the container that will store the backups + containerName = flag.String("azblob_backup_contain_name", "", "Azure Blob Contain Name") + + // This is an optional previx to prepend to all files + prefix = flag.String("azblob_backup_prefix", "", "Azure Blob prefix") + + azBlobParallelism = flag.Int("azblob_parallelism", 1, "Azure blob operation parallelism (requires extra memory when increased)") +) + +const ( + defaultRetryCount = 5 + delimiter = "/" +) + +func azCredentials() (*azblob.SharedKeyCredential, error) { + return azblob.NewSharedKeyCredential(*accountName, *accountKey) +} + +func azServiceURL(credentials azblob.Credential) azblob.ServiceURL { + pipeline := azblob.NewPipeline(credentials, azblob.PipelineOptions{ + Retry: azblob.RetryOptions{ + Policy: azblob.RetryPolicyFixed, + MaxTries: defaultRetryCount, + // Per https://godoc.org/github.com/Azure/azure-storage-blob-go/azblob#RetryOptions + // This shuld be set to a very nigh number ( they claim 60s per MB ). That could end up being days so we are limiting this to four hours + TryTimeout: 4 * time.Hour, + }, + }) + u, _ := url.Parse(fmt.Sprintf("https://%s.blob.core.windows.net/", *accountName)) + return azblob.NewServiceURL(*u, pipeline) +} + +// AZBlobBackupHandle implements BackupHandle for Azure Blob service. +type AZBlobBackupHandle struct { + bs *AZBlobBackupStorage + dir string + name string + readOnly bool + waitGroup sync.WaitGroup + errors concurrency.AllErrorRecorder +} + +// Directory implements BackupHandle. +func (bh *AZBlobBackupHandle) Directory() string { + return bh.dir +} + +// Name implements BackupHandle. +func (bh *AZBlobBackupHandle) Name() string { + return bh.name +} + +// AddFile implements BackupHandle. +func (bh *AZBlobBackupHandle) AddFile(ctx context.Context, filename string, filesize int64) (io.WriteCloser, error) { + if bh.readOnly { + return nil, fmt.Errorf("AddFile cannot be called on read-only backup") + } + // Error out if the file size it too large ( ~4.75 TB) + if filesize > azblob.BlockBlobMaxStageBlockBytes*azblob.BlockBlobMaxBlocks { + return nil, fmt.Errorf("filesize is too large to upload to az blob (max size %v)", azblob.BlockBlobMaxStageBlockBytes*azblob.BlockBlobMaxBlocks) + } + + obj := objName(bh.dir, bh.name, filename) + containerURL, err := bh.bs.containerURL() + if err != nil { + return nil, err + } + + blockBlobURL := containerURL.NewBlockBlobURL(obj) + + reader, writer := io.Pipe() + bh.waitGroup.Add(1) + + go func() { + defer bh.waitGroup.Done() + _, err := azblob.UploadStreamToBlockBlob(ctx, reader, blockBlobURL, azblob.UploadStreamToBlockBlobOptions{ + BufferSize: azblob.BlockBlobMaxStageBlockBytes, + MaxBuffers: *azBlobParallelism, + }) + if err != nil { + reader.CloseWithError(err) + bh.errors.RecordError(err) + } + }() + + return writer, nil +} + +// EndBackup implements BackupHandle. +func (bh *AZBlobBackupHandle) EndBackup(ctx context.Context) error { + if bh.readOnly { + return fmt.Errorf("EndBackup cannot be called on read-only backup") + } + bh.waitGroup.Wait() + return bh.errors.Error() +} + +// AbortBackup implements BackupHandle. +func (bh *AZBlobBackupHandle) AbortBackup(ctx context.Context) error { + if bh.readOnly { + return fmt.Errorf("AbortBackup cannot be called on read-only backup") + } + return bh.bs.RemoveBackup(ctx, bh.dir, bh.name) +} + +// ReadFile implements BackupHandle. +func (bh *AZBlobBackupHandle) ReadFile(ctx context.Context, filename string) (io.ReadCloser, error) { + if !bh.readOnly { + return nil, fmt.Errorf("ReadFile cannot be called on read-write backup") + } + + obj := objName(bh.dir, bh.name, filename) + containerURL, err := bh.bs.containerURL() + if err != nil { + return nil, err + } + blobURL := containerURL.NewBlobURL(obj) + + resp, err := blobURL.Download(ctx, 0, azblob.CountToEnd, azblob.BlobAccessConditions{}, false) + if err != nil { + return nil, err + } + return resp.Body(azblob.RetryReaderOptions{ + MaxRetryRequests: defaultRetryCount, + NotifyFailedRead: func(failureCount int, lastError error, offset int64, count int64, willRetry bool) { + bh.errors.RecordError(lastError) + }, + TreatEarlyCloseAsError: true, + }), nil +} + +// AZBlobBackupStorage structs implements the BackupStorage interface for AZBlob +type AZBlobBackupStorage struct { +} + +func (bs *AZBlobBackupStorage) containerURL() (*azblob.ContainerURL, error) { + credentials, err := azCredentials() + if err != nil { + return nil, err + } + u := azServiceURL(credentials).NewContainerURL(*containerName) + return &u, nil +} + +// ListBackups implements BackupStorage. +func (bs *AZBlobBackupStorage) ListBackups(ctx context.Context, dir string) ([]backupstorage.BackupHandle, error) { + log.Infof("ListBackups: [azblob] container: %v, prefix: %v", *containerName, dir) + + containerURL, err := bs.containerURL() + if err != nil { + return nil, err + } + + searchPrefix := objName(dir, "") + + result := make([]backupstorage.BackupHandle, 0) + var subdirs []string + + for marker := (azblob.Marker{}); marker.NotDone(); { + resp, err := containerURL.ListBlobsHierarchySegment(ctx, marker, delimiter, azblob.ListBlobsSegmentOptions{ + Prefix: searchPrefix, + MaxResults: 0, + }) + + if err != nil { + return nil, err + } + + for _, item := range resp.Segment.BlobPrefixes { + subdir := strings.TrimPrefix(item.Name, searchPrefix) + subdir = strings.TrimSuffix(subdir, delimiter) + subdirs = append(subdirs, subdir) + } + + marker = resp.NextMarker + } + + for _, subdir := range subdirs { + result = append(result, &AZBlobBackupHandle{ + bs: bs, + dir: dir, + name: subdir, + readOnly: true, + }) + } + + return result, nil +} + +// StartBackup implements BackupStorage. +func (bs *AZBlobBackupStorage) StartBackup(ctx context.Context, dir, name string) (backupstorage.BackupHandle, error) { + return &AZBlobBackupHandle{ + bs: bs, + dir: dir, + name: name, + readOnly: false, + }, nil +} + +// RemoveBackup implements BackupStorage. +func (bs *AZBlobBackupStorage) RemoveBackup(ctx context.Context, dir, name string) error { + log.Infof("ListBackups: [azblob] container: %v, prefix: %v", *containerName, dir) + + containerURL, err := bs.containerURL() + if err != nil { + return err + } + + searchPrefix := objName(dir, name, "") + + for marker := (azblob.Marker{}); marker.NotDone(); { + resp, err := containerURL.ListBlobsHierarchySegment(ctx, marker, delimiter, azblob.ListBlobsSegmentOptions{ + Prefix: searchPrefix, + MaxResults: 0, + }) + + if err != nil { + return err + } + + // Right now there is no batch delete so we must iterate over all the blobs to delete them one by one + // One day we will be able to use this https://docs.microsoft.com/en-us/rest/api/storageservices/blob-batch + // but currently it is listed as a preview and its not in the go API + for _, item := range resp.Segment.BlobItems { + _, err = containerURL.NewBlobURL(item.Name).Delete(ctx, azblob.DeleteSnapshotsOptionNone, azblob.BlobAccessConditions{}) + if err != nil { + return err + } + } + marker = resp.NextMarker + } + + return nil +} + +// Close implements BackupStorage. +func (bs *AZBlobBackupStorage) Close() error { + // This function is a No-op + return nil +} + +// objName joins path parts into an object name. +// Unlike path.Join, it doesn't collapse ".." or strip trailing slashes. +// It also adds the value of the -gcs_backup_storage_root flag if set. +func objName(parts ...string) string { + if *prefix != "" { + return *prefix + "/" + strings.Join(parts, "/") + } + return strings.Join(parts, "/") +} + +func init() { + backupstorage.BackupStorageMap["azblob"] = &AZBlobBackupStorage{} +}