Skip to content

Commit

Permalink
chore: bump azidentity and storage/azblob to latest version (argoproj…
Browse files Browse the repository at this point in the history 
…#11254) (argoproj#11255)

Signed-off-by: Brian Loss <[email protected]>
  • Loading branch information
@brianloss
brianloss authored Jun 22, 2023
1 parent b89c98b commit 873a58d
Show file tree
Hide file tree
Showing 3 changed files with 55 additions and 71 deletions.
10 changes: 5 additions & 5 deletions go.mod
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,8 +4,8 @@ go 1.20

require (
cloud.google.com/go/storage v1.30.1
github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.2.2
github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v0.4.1
github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.3.0
github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.0.0
github.com/Knetic/govaluate v3.0.1-0.20171022003610-9aa49832a739+incompatible
github.com/Masterminds/sprig/v3 v3.2.3
github.com/TwiN/go-color v1.4.0
Expand Down Expand Up @@ -86,8 +86,8 @@ require (
cloud.google.com/go/compute/metadata v0.2.3 // indirect
cloud.google.com/go/iam v0.13.0 // indirect
github.com/Azure/azure-sdk-for-go v62.0.0+incompatible // indirect
github.com/Azure/azure-sdk-for-go/sdk/azcore v1.4.0 // indirect
github.com/Azure/azure-sdk-for-go/sdk/internal v1.2.0 // indirect
github.com/Azure/azure-sdk-for-go/sdk/azcore v1.6.0 // indirect
github.com/Azure/azure-sdk-for-go/sdk/internal v1.3.0 // indirect
github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 // indirect
github.com/Azure/go-autorest v14.2.0+incompatible // indirect
github.com/Azure/go-autorest/autorest v0.11.24 // indirect
Expand All @@ -97,7 +97,7 @@ require (
github.com/Azure/go-autorest/autorest/date v0.3.0 // indirect
github.com/Azure/go-autorest/logger v0.2.1 // indirect
github.com/Azure/go-autorest/tracing v0.6.0 // indirect
github.com/AzureAD/microsoft-authentication-library-for-go v0.9.0 // indirect
github.com/AzureAD/microsoft-authentication-library-for-go v1.0.0 // indirect
github.com/MakeNowJust/heredoc v0.0.0-20170808103936-bb23615498cd // indirect
github.com/Masterminds/goutils v1.1.1 // indirect
github.com/Masterminds/semver/v3 v3.2.0 // indirect
Expand Down
22 changes: 11 additions & 11 deletions go.sum
View file
Original file line number Diff line number Diff line change
Expand Up @@ -68,14 +68,14 @@ github.com/Antonboom/nilnil v0.1.0/go.mod h1:PhHLvRPSghY5Y7mX4TW+BHZQYo1A8flE5H2
github.com/Azure/azure-sdk-for-go v46.4.0+incompatible/go.mod h1:9XXNKU+eRnpl9moKnB4QOLf1HestfXbmab5FXxiDBjc=
github.com/Azure/azure-sdk-for-go v62.0.0+incompatible h1:8N2k27SYtc12qj5nTsuFMFJPZn5CGmgMWqTy4y9I7Jw=
github.com/Azure/azure-sdk-for-go v62.0.0+incompatible/go.mod h1:9XXNKU+eRnpl9moKnB4QOLf1HestfXbmab5FXxiDBjc=
github.com/Azure/azure-sdk-for-go/sdk/azcore v1.4.0 h1:rTnT/Jrcm+figWlYz4Ixzt0SJVR2cMC8lvZcimipiEY=
github.com/Azure/azure-sdk-for-go/sdk/azcore v1.4.0/go.mod h1:ON4tFdPTwRcgWEaVDrN3584Ef+b7GgSJaXxe5fW9t4M=
github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.2.2 h1:uqM+VoHjVH6zdlkLF2b6O0ZANcHoj3rO0PoQ3jglUJA=
github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.2.2/go.mod h1:twTKAa1E6hLmSDjLhaCkbTMQKc7p/rNLU40rLxGEOCI=
github.com/Azure/azure-sdk-for-go/sdk/internal v1.2.0 h1:leh5DwKv6Ihwi+h60uHtn6UWAxBbZ0q8DwQVMzf61zw=
github.com/Azure/azure-sdk-for-go/sdk/internal v1.2.0/go.mod h1:eWRD7oawr1Mu1sLCawqVc0CUiF43ia3qQMxLscsKQ9w=
github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v0.4.1 h1:QSdcrd/UFJv6Bp/CfoVf2SrENpFn9P6Yh8yb+xNhYMM=
github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v0.4.1/go.mod h1:eZ4g6GUvXiGulfIbbhh1Xr4XwUYaYaWMqzGD/284wCA=
github.com/Azure/azure-sdk-for-go/sdk/azcore v1.6.0 h1:8kDqDngH+DmVBiCtIjCFTGa7MBnsIOkF9IccInFEbjk=
github.com/Azure/azure-sdk-for-go/sdk/azcore v1.6.0/go.mod h1:bjGvMhVMb+EEm3VRNQawDMUyMMjo+S5ewNjflkep/0Q=
github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.3.0 h1:vcYCAze6p19qBW7MhZybIsqD8sMV8js0NyQM8JDnVtg=
github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.3.0/go.mod h1:OQeznEEkTZ9OrhHJoDD8ZDq51FHgXjqtP9z6bEwBq9U=
github.com/Azure/azure-sdk-for-go/sdk/internal v1.3.0 h1:sXr+ck84g/ZlZUOZiNELInmMgOsuGwdjjVkEIde0OtY=
github.com/Azure/azure-sdk-for-go/sdk/internal v1.3.0/go.mod h1:okt5dMMTOFjX/aovMlrjvvXoPMBVSPzk9185BT0+eZM=
github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.0.0 h1:u/LLAOFgsMv7HmNL4Qufg58y+qElGOt5qv0z1mURkRY=
github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.0.0/go.mod h1:2e8rMJtl2+2j+HXbTBwnyGpm5Nou7KhvSfxOq8JpTag=
github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 h1:UQHMgLO+TxOElx5B5HZ4hJQsoJ/PvUvKRhJHDQXO8P8=
github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1/go.mod h1:xomTg63KZ2rFqZQzSB4Vz2SUXa1BpHTVz9L5PTmPC4E=
github.com/Azure/go-autorest v14.2.0+incompatible h1:V5VMDjClD3GiElqLWO7mz2MxNAK/vTfRHdAubSIPRgs=
Expand Down Expand Up @@ -111,8 +111,8 @@ github.com/Azure/go-autorest/logger v0.2.1/go.mod h1:T9E3cAhj2VqvPOtCYAvby9aBXkZ
github.com/Azure/go-autorest/tracing v0.5.0/go.mod h1:r/s2XiOKccPW3HrqB+W0TQzfbtp2fGCgRFtBroKn4Dk=
github.com/Azure/go-autorest/tracing v0.6.0 h1:TYi4+3m5t6K48TGI9AUdb+IzbnSxvnvUMfuitfgcfuo=
github.com/Azure/go-autorest/tracing v0.6.0/go.mod h1:+vhtPC754Xsa23ID7GlGsrdKBpUA79WCAKPPZVC2DeU=
github.com/AzureAD/microsoft-authentication-library-for-go v0.9.0 h1:UE9n9rkJF62ArLb1F3DEjRt8O3jLwMWdSoypKV4f3MU=
github.com/AzureAD/microsoft-authentication-library-for-go v0.9.0/go.mod h1:kgDmCTgBzIEPFElEF+FK0SdjAor06dRq2Go927dnQ6o=
github.com/AzureAD/microsoft-authentication-library-for-go v1.0.0 h1:OBhqkivkhkMqLPymWEppkm7vgPQY2XsHoEkaMQ0AdZY=
github.com/AzureAD/microsoft-authentication-library-for-go v1.0.0/go.mod h1:kgDmCTgBzIEPFElEF+FK0SdjAor06dRq2Go927dnQ6o=
github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
github.com/BurntSushi/toml v0.4.1/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ=
github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo=
Expand Down Expand Up @@ -324,7 +324,7 @@ github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8
github.com/dimchansky/utfbom v1.1.0/go.mod h1:rO41eb7gLfo8SF1jd9F8HplJm1Fewwi4mQvIirEdv+8=
github.com/dimchansky/utfbom v1.1.1 h1:vV6w1AhK4VMnhBno/TPVCoK9U/LP0PkLCS9tbxHdi/U=
github.com/dimchansky/utfbom v1.1.1/go.mod h1:SxdoEBH5qIqFocHMyGOXVAybYJdr71b1Q/j0mACtrfE=
github.com/dnaeon/go-vcr v1.1.0 h1:ReYa/UBrRyQdant9B4fNHGoCNKw6qh6P0fsdGmZpR7c=
github.com/dnaeon/go-vcr v1.2.0 h1:zHCHvJYTMh1N7xnV7zf1m1GPBF9Ad0Jk/whtQ1663qI=
github.com/docker/cli v20.10.17+incompatible h1:eO2KS7ZFeov5UJeaDmIs1NFEDRf32PaqRpvoEkKBy5M=
github.com/docker/cli v20.10.17+incompatible/go.mod h1:JLrzqnKDaYBop7H2jaqPtU4hHvMKP+vjCwu2uszcLI8=
github.com/docker/distribution v2.8.1+incompatible h1:Q50tZOPR6T/hjNsyc9g8/syEs6bk8XXApsHjKukMl68=
Expand Down
94 changes: 39 additions & 55 deletions workflow/artifacts/azure/azure.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,8 @@ import (

"github.com/Azure/azure-sdk-for-go/sdk/azidentity"
"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob"
"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/bloberror"
"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/container"
log "github.com/sirupsen/logrus"

argoerrors "github.com/argoproj/argo-workflows/v3/errors"
Expand All @@ -33,10 +35,10 @@ type ArtifactDriver struct {

var _ artifactscommon.ArtifactDriver = &ArtifactDriver{}

// newAzureContainerClient creates a new azblob.ContainerClient for interacting with the specified Azure Blob Storage container
// newAzureContainerClient creates a new container.Client for interacting with the specified Azure Blob Storage container
// The container client is created with the default azblob.ClientOptions which does include retry behavior
// for failed requests.
func (azblobDriver *ArtifactDriver) newAzureContainerClient() (*azblob.ContainerClient, error) {
func (azblobDriver *ArtifactDriver) newAzureContainerClient() (*container.Client, error) {

containerUrl, err := url.Parse(azblobDriver.Endpoint)
if err != nil {
Expand All @@ -53,7 +55,7 @@ func (azblobDriver *ArtifactDriver) newAzureContainerClient() (*azblob.Container
if err != nil {
return nil, fmt.Errorf("unable to create default Azure credential: %s", err)
}
containerClient, err := azblob.NewContainerClient(containerUrl.String(), credential, nil)
containerClient, err := container.NewClient(containerUrl.String(), credential, nil)
return containerClient, err
} else {
if azblobDriver.AccountKey == "" {
Expand All @@ -67,7 +69,7 @@ func (azblobDriver *ArtifactDriver) newAzureContainerClient() (*azblob.Container
if err != nil {
return nil, fmt.Errorf("unable to create Azure shared key credential: %s", err)
}
containerClient, err := azblob.NewContainerClientWithSharedKey(containerUrl.String(), credential, nil)
containerClient, err := container.NewClientWithSharedKeyCredential(containerUrl.String(), credential, nil)
return containerClient, err
}
}
Expand Down Expand Up @@ -118,7 +120,7 @@ func (azblobDriver *ArtifactDriver) Load(artifact *wfv1.Artifact, path string) e
return nil
}
isEmptyFile = true
} else if !IsAzureError(origErr, azblob.StorageErrorCodeBlobNotFound) {
} else if !bloberror.HasCode(origErr, bloberror.BlobNotFound) {
return fmt.Errorf("unable to download blob %s: %s", artifact.Azure.Blob, origErr)
}

Expand Down Expand Up @@ -149,11 +151,8 @@ func (azblobDriver *ArtifactDriver) Load(artifact *wfv1.Artifact, path string) e
}

// DownloadFile downloads a single file from Azure Blob Storage
func DownloadFile(containerClient *azblob.ContainerClient, blobName, path string) error {
blobClient, err := containerClient.NewBlobClient(blobName)
if err != nil {
return fmt.Errorf("unable to create Azure Blob client for %s: %s", blobName, err)
}
func DownloadFile(containerClient *container.Client, blobName, path string) error {
blobClient := containerClient.NewBlobClient(blobName)

outFile, err := os.Create(path)
if err != nil {
Expand All @@ -165,11 +164,12 @@ func DownloadFile(containerClient *azblob.ContainerClient, blobName, path string
}
}()

return blobClient.DownloadToFile(context.TODO(), 0, 0, outFile, azblob.DownloadOptions{})
_, err = blobClient.DownloadFile(context.TODO(), outFile, nil)
return err
}

// DownloadDirectory downloads all of the files starting with the named blob prefix into a local directory.
func (azblobDriver *ArtifactDriver) DownloadDirectory(containerClient *azblob.ContainerClient, artifact *wfv1.Artifact, path string) error {
func (azblobDriver *ArtifactDriver) DownloadDirectory(containerClient *container.Client, artifact *wfv1.Artifact, path string) error {
log.WithFields(log.Fields{"endpoint": artifact.Azure.Endpoint, "container": artifact.Azure.Container,
"blob": artifact.Azure.Blob}).Info("Downloading directory from Azure Blob Storage")

Expand Down Expand Up @@ -209,24 +209,21 @@ func (azblobDriver *ArtifactDriver) OpenStream(artifact *wfv1.Artifact) (io.Read
return nil, fmt.Errorf("unable to create Azure Blob Container client: %s", err)
}

blobClient, err := containerClient.NewBlockBlobClient(artifact.Azure.Blob)
if err != nil {
return nil, fmt.Errorf("unable to create Azure Blob client for %s: %s", artifact.Azure.Blob, err)
}
blobClient := containerClient.NewBlockBlobClient(artifact.Azure.Blob)

// Attempt the download. If it fails with a BlobNotFound error, or succeeds but with
// a content length of 0, then it could be that we're attempting to stream a directory.
// Check if the blob represents a directory and return an error if so. If not, then
// return either the original BlobNotFound error or the empty file stream.
emptyFile := false
response, origErr := blobClient.Download(context.TODO(), nil)
response, origErr := blobClient.DownloadStream(context.TODO(), nil)
if origErr == nil {
emptyFile = *response.ContentLength == 0
// We have a normal file blob, so just return the response body stream
if !emptyFile {
return response.Body(nil), nil
return response.Body, nil
}
} else if !IsAzureError(origErr, azblob.StorageErrorCodeBlobNotFound) {
} else if !bloberror.HasCode(origErr, bloberror.BlobNotFound) {
return nil, fmt.Errorf("unable to open stream for blob %s: %s", artifact.Azure.Blob, origErr)
}

Expand All @@ -242,7 +239,7 @@ func (azblobDriver *ArtifactDriver) OpenStream(artifact *wfv1.Artifact) (io.Read
return nil, fmt.Errorf("unable to open blob stream for %s: %s", artifact.Azure.Blob, origErr)
}

return response.Body(nil), nil
return response.Body, nil
}

// Save saves an artifact to Azure Blob Storage
Expand Down Expand Up @@ -276,11 +273,8 @@ func (azblobDriver *ArtifactDriver) Save(path string, outputArtifact *wfv1.Artif
}

// PutFile uploads a file to Azure Blob Storage
func PutFile(containerClient *azblob.ContainerClient, blobName, path string) error {
blobClient, err := containerClient.NewBlockBlobClient(blobName)
if err != nil {
return fmt.Errorf("unable to create Azure Blob client: %s", err)
}
func PutFile(containerClient *container.Client, blobName, path string) error {
blobClient := containerClient.NewBlockBlobClient(blobName)

file, err := os.Open(path)
if err != nil {
Expand All @@ -292,12 +286,12 @@ func PutFile(containerClient *azblob.ContainerClient, blobName, path string) err
}
}()

_, err = blobClient.UploadFile(context.TODO(), file, azblob.UploadOption{})
_, err = blobClient.UploadFile(context.TODO(), file, nil)
return err
}

// PutDirectory uploads all files in a directory to Azure Blob Storage
func PutDirectory(containerClient *azblob.ContainerClient, blobName, path string) error {
func PutDirectory(containerClient *container.Client, blobName, path string) error {
for putTask := range generatePutTasks(blobName, path) {
err := PutFile(containerClient, putTask.blobName, putTask.path)
if err != nil {
Expand Down Expand Up @@ -346,15 +340,12 @@ func (azblobDriver *ArtifactDriver) Delete(artifact *wfv1.Artifact) error {
return nil
}

func DeleteBlob(containerClient *azblob.ContainerClient, blobName string, allowNonExistent bool) error {
blobClient, err := containerClient.NewBlobClient(blobName)
if err != nil {
return fmt.Errorf("unable to create Azure Blob client for %s: %s", blobName, err)
}
func DeleteBlob(containerClient *container.Client, blobName string, allowNonExistent bool) error {
blobClient := containerClient.NewBlobClient(blobName)

_, err = blobClient.Delete(context.TODO(), nil)
_, err := blobClient.Delete(context.TODO(), nil)
if err != nil {
if allowNonExistent && IsAzureError(err, azblob.StorageErrorCodeBlobNotFound) {
if allowNonExistent && bloberror.HasCode(err, bloberror.BlobNotFound) {
log.Debugf("blob to delete '%s' does not exist: %s", blobName, err)
return nil
} else {
Expand All @@ -376,21 +367,21 @@ func (azblobDriver *ArtifactDriver) ListObjects(artifact *wfv1.Artifact) ([]stri
return nil, fmt.Errorf("unable to create Azure Blob Container client: %s", err)
}

listOpts := azblob.ContainerListBlobsFlatOptions{
listOpts := azblob.ListBlobsFlatOptions{
Prefix: &artifact.Azure.Blob,
Marker: nil,
}
ctx := context.TODO()
pager := containerClient.ListBlobsFlat(&listOpts)
for pager.NextPage(ctx) {
resp := pager.PageResponse()
pager := containerClient.NewListBlobsFlatPager(&listOpts)
for pager.More() {
resp, err := pager.NextPage(ctx)
if err != nil {
return nil, fmt.Errorf("error listing blobs %s in Azure Blob Storage container: %s", artifact.Azure.Blob, err)
}
for _, v := range resp.Segment.BlobItems {
files = append(files, *v.Name)
}
}
if pager.Err() != nil {
return nil, fmt.Errorf("error listing blobs %s in Azure Blob Storage container: %s", artifact.Azure.Blob, pager.Err())
}
return files, nil
}

Expand All @@ -410,33 +401,26 @@ func (azblobDriver *ArtifactDriver) IsDirectory(artifact *wfv1.Artifact) (bool,
return false, fmt.Errorf("unable to create Azure Blob Container client: %s", err)
}

listOpts := azblob.ContainerListBlobsFlatOptions{
listOpts := azblob.ListBlobsFlatOptions{
Prefix: &blobPrefix,
Marker: nil,
}
pager := containerClient.ListBlobsFlat(&listOpts)
if pager.NextPage(context.TODO()) {
resp := pager.PageResponse()
pager := containerClient.NewListBlobsFlatPager(&listOpts)
if pager.More() {
resp, err := pager.NextPage(context.TODO())
if err != nil {
return false, fmt.Errorf("error listing blobs %s in Azure Blob Storage container: %s", artifact.Azure.Blob, err)
}
if len(resp.Segment.BlobItems) == 1 {
return strings.HasPrefix(*resp.Segment.BlobItems[0].Name, blobPrefix), nil
} else {
return len(resp.Segment.BlobItems) > 0, nil
}
} else if pager.Err() != nil {
return false, fmt.Errorf("error listing blobs %s in Azure Blob Storage container: %s", artifact.Azure.Blob, pager.Err())
}

return false, nil
}

func IsAzureError(err error, code azblob.StorageErrorCode) bool {
var storageErr *azblob.StorageError
if errors.As(err, &storageErr) {
return storageErr.ErrorCode == code
}
return false
}

type uploadTask struct {
blobName string
path string
Expand Down

0 comments on commit 873a58d

Please sign in to comment.