Skip to content

Commit

Permalink
Merge pull request #2603 from yuwaMSFT2/newazuresdk
Browse files Browse the repository at this point in the history 
closes #2496 and #2552
  • Loading branch information
@dmcgowan
dmcgowan authored Jun 11, 2018
2 parents f0cc927 + 6279723 commit 749f6af
Show file tree
Hide file tree
Showing 76 changed files with 9,896 additions and 2,761 deletions.
93 changes: 59 additions & 34 deletions registry/storage/driver/azure/azure.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -87,7 +87,7 @@ func New(accountName, accountKey, container, realm string) (*Driver, error) {

// Create registry container
containerRef := blobClient.GetContainerReference(container)
if _, err = containerRef.CreateIfNotExists(); err != nil {
if _, err = containerRef.CreateIfNotExists(nil); err != nil {
return nil, err
}

Expand All @@ -104,7 +104,8 @@ func (d *driver) Name() string {

// GetContent retrieves the content stored at "path" as a []byte.
func (d *driver) GetContent(ctx context.Context, path string) ([]byte, error) {
blob, err := d.client.GetBlob(d.container, path)
blobRef := d.client.GetContainerReference(d.container).GetBlobReference(path)
blob, err := blobRef.Get(nil)
if err != nil {
if is404(err) {
return nil, storagedriver.PathNotFoundError{Path: path}
Expand All @@ -118,7 +119,10 @@ func (d *driver) GetContent(ctx context.Context, path string) ([]byte, error) {

// PutContent stores the []byte content at a location designated by "path".
func (d *driver) PutContent(ctx context.Context, path string, contents []byte) error {
if limit := 64 * 1024 * 1024; len(contents) > limit { // max size for block blobs uploaded via single "Put Blob"
// max size for block blobs uploaded via single "Put Blob" for version after "2016-05-31"
// https://docs.microsoft.com/en-us/rest/api/storageservices/put-blob#remarks
const limit = 256 * 1024 * 1024
if len(contents) > limit {
return fmt.Errorf("uploading %d bytes with PutContent is not supported; limit: %d bytes", len(contents), limit)
}

Expand All @@ -133,41 +137,49 @@ func (d *driver) PutContent(ctx context.Context, path string, contents []byte) e
// losing the existing data while migrating it to BlockBlob type. However,
// expectation is the clients pushing will be retrying when they get an error
// response.
props, err := d.client.GetBlobProperties(d.container, path)
blobRef := d.client.GetContainerReference(d.container).GetBlobReference(path)
err := blobRef.GetProperties(nil)
if err != nil && !is404(err) {
return fmt.Errorf("failed to get blob properties: %v", err)
}
if err == nil && props.BlobType != azure.BlobTypeBlock {
if err := d.client.DeleteBlob(d.container, path, nil); err != nil {
return fmt.Errorf("failed to delete legacy blob (%s): %v", props.BlobType, err)
if err == nil && blobRef.Properties.BlobType != azure.BlobTypeBlock {
if err := blobRef.Delete(nil); err != nil {
return fmt.Errorf("failed to delete legacy blob (%s): %v", blobRef.Properties.BlobType, err)
}
}

r := bytes.NewReader(contents)
return d.client.CreateBlockBlobFromReader(d.container, path, uint64(len(contents)), r, nil)
// reset properties to empty before doing overwrite
blobRef.Properties = azure.BlobProperties{}
return blobRef.CreateBlockBlobFromReader(r, nil)
}

// Reader retrieves an io.ReadCloser for the content stored at "path" with a
// given byte offset.
func (d *driver) Reader(ctx context.Context, path string, offset int64) (io.ReadCloser, error) {
if ok, err := d.client.BlobExists(d.container, path); err != nil {
blobRef := d.client.GetContainerReference(d.container).GetBlobReference(path)
if ok, err := blobRef.Exists(); err != nil {
return nil, err
} else if !ok {
return nil, storagedriver.PathNotFoundError{Path: path}
}

info, err := d.client.GetBlobProperties(d.container, path)
err := blobRef.GetProperties(nil)
if err != nil {
return nil, err
}

info := blobRef.Properties
size := int64(info.ContentLength)
if offset >= size {
return ioutil.NopCloser(bytes.NewReader(nil)), nil
}

bytesRange := fmt.Sprintf("%v-", offset)
resp, err := d.client.GetBlobRange(d.container, path, bytesRange, nil)
resp, err := blobRef.GetRange(&azure.GetBlobRangeOptions{
Range: &azure.BlobRange{
Start: uint64(offset),
End: 0,
},
})
if err != nil {
return nil, err
}
Expand All @@ -177,20 +189,22 @@ func (d *driver) Reader(ctx context.Context, path string, offset int64) (io.Read
// Writer returns a FileWriter which will store the content written to it
// at the location designated by "path" after the call to Commit.
func (d *driver) Writer(ctx context.Context, path string, append bool) (storagedriver.FileWriter, error) {
blobExists, err := d.client.BlobExists(d.container, path)
blobRef := d.client.GetContainerReference(d.container).GetBlobReference(path)
blobExists, err := blobRef.Exists()
if err != nil {
return nil, err
}
var size int64
if blobExists {
if append {
blobProperties, err := d.client.GetBlobProperties(d.container, path)
err = blobRef.GetProperties(nil)
if err != nil {
return nil, err
}
blobProperties := blobRef.Properties
size = blobProperties.ContentLength
} else {
err := d.client.DeleteBlob(d.container, path, nil)
err = blobRef.Delete(nil)
if err != nil {
return nil, err
}
Expand All @@ -199,7 +213,7 @@ func (d *driver) Writer(ctx context.Context, path string, append bool) (storaged
if append {
return nil, storagedriver.PathNotFoundError{Path: path}
}
err := d.client.PutAppendBlob(d.container, path, nil)
err = blobRef.PutAppendBlob(nil)
if err != nil {
return nil, err
}
Expand All @@ -211,24 +225,21 @@ func (d *driver) Writer(ctx context.Context, path string, append bool) (storaged
// Stat retrieves the FileInfo for the given path, including the current size
// in bytes and the creation time.
func (d *driver) Stat(ctx context.Context, path string) (storagedriver.FileInfo, error) {
blobRef := d.client.GetContainerReference(d.container).GetBlobReference(path)
// Check if the path is a blob
if ok, err := d.client.BlobExists(d.container, path); err != nil {
if ok, err := blobRef.Exists(); err != nil {
return nil, err
} else if ok {
blob, err := d.client.GetBlobProperties(d.container, path)
if err != nil {
return nil, err
}

mtim, err := time.Parse(http.TimeFormat, blob.LastModified)
err = blobRef.GetProperties(nil)
if err != nil {
return nil, err
}
blobProperties := blobRef.Properties

return storagedriver.FileInfoInternal{FileInfoFields: storagedriver.FileInfoFields{
Path: path,
Size: int64(blob.ContentLength),
ModTime: mtim,
Size: int64(blobProperties.ContentLength),
ModTime: time.Time(blobProperties.LastModified),
IsDir: false,
}}, nil
}
Expand Down Expand Up @@ -281,21 +292,24 @@ func (d *driver) List(ctx context.Context, path string) ([]string, error) {
// Move moves an object stored at sourcePath to destPath, removing the original
// object.
func (d *driver) Move(ctx context.Context, sourcePath string, destPath string) error {
sourceBlobURL := d.client.GetBlobURL(d.container, sourcePath)
err := d.client.CopyBlob(d.container, destPath, sourceBlobURL)
srcBlobRef := d.client.GetContainerReference(d.container).GetBlobReference(sourcePath)
sourceBlobURL := srcBlobRef.GetURL()
destBlobRef := d.client.GetContainerReference(d.container).GetBlobReference(destPath)
err := destBlobRef.Copy(sourceBlobURL, nil)
if err != nil {
if is404(err) {
return storagedriver.PathNotFoundError{Path: sourcePath}
}
return err
}

return d.client.DeleteBlob(d.container, sourcePath, nil)
return srcBlobRef.Delete(nil)
}

// Delete recursively deletes all objects stored at "path" and its subpaths.
func (d *driver) Delete(ctx context.Context, path string) error {
ok, err := d.client.DeleteBlobIfExists(d.container, path, nil)
blobRef := d.client.GetContainerReference(d.container).GetBlobReference(path)
ok, err := blobRef.DeleteIfExists(nil)
if err != nil {
return err
}
Expand All @@ -310,7 +324,8 @@ func (d *driver) Delete(ctx context.Context, path string) error {
}

for _, b := range blobs {
if err = d.client.DeleteBlob(d.container, b, nil); err != nil {
blobRef = d.client.GetContainerReference(d.container).GetBlobReference(b)
if err = blobRef.Delete(nil); err != nil {
return err
}
}
Expand All @@ -333,7 +348,15 @@ func (d *driver) URLFor(ctx context.Context, path string, options map[string]int
expiresTime = t
}
}
return d.client.GetBlobSASURI(d.container, path, expiresTime, "r")
blobRef := d.client.GetContainerReference(d.container).GetBlobReference(path)
return blobRef.GetSASURI(azure.BlobSASOptions{
BlobServiceSASPermissions: azure.BlobServiceSASPermissions{
Read: true,
},
SASOptions: azure.SASOptions{
Expiry: expiresTime,
},
})
}

// Walk traverses a filesystem defined within driver, starting
Expand Down Expand Up @@ -467,7 +490,8 @@ func (w *writer) Cancel() error {
return fmt.Errorf("already committed")
}
w.cancelled = true
return w.driver.client.DeleteBlob(w.driver.container, w.path, nil)
blobRef := w.driver.client.GetContainerReference(w.driver.container).GetBlobReference(w.path)
return blobRef.Delete(nil)
}

func (w *writer) Commit() error {
Expand All @@ -490,12 +514,13 @@ type blockWriter struct {

func (bw *blockWriter) Write(p []byte) (int, error) {
n := 0
blobRef := bw.client.GetContainerReference(bw.container).GetBlobReference(bw.path)
for offset := 0; offset < len(p); offset += maxChunkSize {
chunkSize := maxChunkSize
if offset+chunkSize > len(p) {
chunkSize = len(p) - offset
}
err := bw.client.AppendBlock(bw.container, bw.path, p[offset:offset+chunkSize], nil)
err := blobRef.AppendBlock(p[offset:offset+chunkSize], nil)
if err != nil {
return n, err
}
Expand Down
6 changes: 4 additions & 2 deletions vendor.conf
View file
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
github.com/Azure/azure-sdk-for-go 088007b3b08cc02b27f2eadfdcd870958460ce7e
github.com/Azure/go-autorest ec5f4903f77ed9927ac95b19ab8e44ada64c1356
github.com/Azure/azure-sdk-for-go 4650843026a7fdec254a8d9cf893693a254edd0b
github.com/Azure/go-autorest eaa7994b2278094c904d31993d26f56324db3052
github.com/sirupsen/logrus 3d4380f53a34dcdc95f0c1db702615992b38d9a4
github.com/aws/aws-sdk-go 5bcc0a238d880469f949fc7cd24e35f32ab80cbd
github.com/bshuster-repo/logrus-logstash-hook d2c0ecc1836d91814e15e23bb5dc309c3ef51f4a
Expand All @@ -19,6 +19,8 @@ github.com/gorilla/handlers 60c7bfde3e33c201519a200a4507a158cc03a17b
github.com/gorilla/mux 599cba5e7b6137d46ddf58fb1765f5d928e69604
github.com/inconshreveable/mousetrap 76626ae9c91c4f2a10f34cad8ce83ea42c93bb75
github.com/jmespath/go-jmespath bd40a432e4c76585ef6b72d3fd96fb9b6dc7b68d
github.com/marstr/guid 8bd9a64bf37eb297b492a4101fb28e80ac0b290f
github.com/satori/go.uuid f58768cc1a7a7e77a3bd49e98cdd21419399b6a3
github.com/matttproud/golang_protobuf_extensions c12348ce28de40eed0136aa2b644d0ee0650e56c
github.com/miekg/dns 271c58e0c14f552178ea321a545ff9af38930f39
github.com/mitchellh/mapstructure 482a9fd5fa83e8c4e7817413b80f3eb8feec03ef
Expand Down
5 changes: 5 additions & 0 deletions vendor/github.com/Azure/azure-sdk-for-go/NOTICE
View file

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

Loading

0 comments on commit 749f6af

Please sign in to comment.