From 873a58de7dd9dad76d5577b8c4294a58b52849b8 Mon Sep 17 00:00:00 2001 From: Brian Loss Date: Thu, 22 Jun 2023 13:09:25 -0400 Subject: [PATCH] chore: bump azidentity and storage/azblob to latest version (#11254) (#11255) Signed-off-by: Brian Loss --- go.mod | 10 ++-- go.sum | 22 ++++---- workflow/artifacts/azure/azure.go | 94 +++++++++++++------------------ 3 files changed, 55 insertions(+), 71 deletions(-) diff --git a/go.mod b/go.mod index 7d8b2d5353e2..27e8a443fe19 100644 --- a/go.mod +++ b/go.mod @@ -4,8 +4,8 @@ go 1.20 require ( cloud.google.com/go/storage v1.30.1 - github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.2.2 - github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v0.4.1 + github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.3.0 + github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.0.0 github.com/Knetic/govaluate v3.0.1-0.20171022003610-9aa49832a739+incompatible github.com/Masterminds/sprig/v3 v3.2.3 github.com/TwiN/go-color v1.4.0 @@ -86,8 +86,8 @@ require ( cloud.google.com/go/compute/metadata v0.2.3 // indirect cloud.google.com/go/iam v0.13.0 // indirect github.com/Azure/azure-sdk-for-go v62.0.0+incompatible // indirect - github.com/Azure/azure-sdk-for-go/sdk/azcore v1.4.0 // indirect - github.com/Azure/azure-sdk-for-go/sdk/internal v1.2.0 // indirect + github.com/Azure/azure-sdk-for-go/sdk/azcore v1.6.0 // indirect + github.com/Azure/azure-sdk-for-go/sdk/internal v1.3.0 // indirect github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 // indirect github.com/Azure/go-autorest v14.2.0+incompatible // indirect github.com/Azure/go-autorest/autorest v0.11.24 // indirect @@ -97,7 +97,7 @@ require ( github.com/Azure/go-autorest/autorest/date v0.3.0 // indirect github.com/Azure/go-autorest/logger v0.2.1 // indirect github.com/Azure/go-autorest/tracing v0.6.0 // indirect - github.com/AzureAD/microsoft-authentication-library-for-go v0.9.0 // indirect + github.com/AzureAD/microsoft-authentication-library-for-go v1.0.0 // indirect github.com/MakeNowJust/heredoc v0.0.0-20170808103936-bb23615498cd // indirect github.com/Masterminds/goutils v1.1.1 // indirect github.com/Masterminds/semver/v3 v3.2.0 // indirect diff --git a/go.sum b/go.sum index ed91bf435e91..63b4b6be7ebb 100644 --- a/go.sum +++ b/go.sum @@ -68,14 +68,14 @@ github.com/Antonboom/nilnil v0.1.0/go.mod h1:PhHLvRPSghY5Y7mX4TW+BHZQYo1A8flE5H2 github.com/Azure/azure-sdk-for-go v46.4.0+incompatible/go.mod h1:9XXNKU+eRnpl9moKnB4QOLf1HestfXbmab5FXxiDBjc= github.com/Azure/azure-sdk-for-go v62.0.0+incompatible h1:8N2k27SYtc12qj5nTsuFMFJPZn5CGmgMWqTy4y9I7Jw= github.com/Azure/azure-sdk-for-go v62.0.0+incompatible/go.mod h1:9XXNKU+eRnpl9moKnB4QOLf1HestfXbmab5FXxiDBjc= -github.com/Azure/azure-sdk-for-go/sdk/azcore v1.4.0 h1:rTnT/Jrcm+figWlYz4Ixzt0SJVR2cMC8lvZcimipiEY= -github.com/Azure/azure-sdk-for-go/sdk/azcore v1.4.0/go.mod h1:ON4tFdPTwRcgWEaVDrN3584Ef+b7GgSJaXxe5fW9t4M= -github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.2.2 h1:uqM+VoHjVH6zdlkLF2b6O0ZANcHoj3rO0PoQ3jglUJA= -github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.2.2/go.mod h1:twTKAa1E6hLmSDjLhaCkbTMQKc7p/rNLU40rLxGEOCI= -github.com/Azure/azure-sdk-for-go/sdk/internal v1.2.0 h1:leh5DwKv6Ihwi+h60uHtn6UWAxBbZ0q8DwQVMzf61zw= -github.com/Azure/azure-sdk-for-go/sdk/internal v1.2.0/go.mod h1:eWRD7oawr1Mu1sLCawqVc0CUiF43ia3qQMxLscsKQ9w= -github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v0.4.1 h1:QSdcrd/UFJv6Bp/CfoVf2SrENpFn9P6Yh8yb+xNhYMM= -github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v0.4.1/go.mod h1:eZ4g6GUvXiGulfIbbhh1Xr4XwUYaYaWMqzGD/284wCA= +github.com/Azure/azure-sdk-for-go/sdk/azcore v1.6.0 h1:8kDqDngH+DmVBiCtIjCFTGa7MBnsIOkF9IccInFEbjk= +github.com/Azure/azure-sdk-for-go/sdk/azcore v1.6.0/go.mod h1:bjGvMhVMb+EEm3VRNQawDMUyMMjo+S5ewNjflkep/0Q= +github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.3.0 h1:vcYCAze6p19qBW7MhZybIsqD8sMV8js0NyQM8JDnVtg= +github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.3.0/go.mod h1:OQeznEEkTZ9OrhHJoDD8ZDq51FHgXjqtP9z6bEwBq9U= +github.com/Azure/azure-sdk-for-go/sdk/internal v1.3.0 h1:sXr+ck84g/ZlZUOZiNELInmMgOsuGwdjjVkEIde0OtY= +github.com/Azure/azure-sdk-for-go/sdk/internal v1.3.0/go.mod h1:okt5dMMTOFjX/aovMlrjvvXoPMBVSPzk9185BT0+eZM= +github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.0.0 h1:u/LLAOFgsMv7HmNL4Qufg58y+qElGOt5qv0z1mURkRY= +github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.0.0/go.mod h1:2e8rMJtl2+2j+HXbTBwnyGpm5Nou7KhvSfxOq8JpTag= github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 h1:UQHMgLO+TxOElx5B5HZ4hJQsoJ/PvUvKRhJHDQXO8P8= github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1/go.mod h1:xomTg63KZ2rFqZQzSB4Vz2SUXa1BpHTVz9L5PTmPC4E= github.com/Azure/go-autorest v14.2.0+incompatible h1:V5VMDjClD3GiElqLWO7mz2MxNAK/vTfRHdAubSIPRgs= @@ -111,8 +111,8 @@ github.com/Azure/go-autorest/logger v0.2.1/go.mod h1:T9E3cAhj2VqvPOtCYAvby9aBXkZ github.com/Azure/go-autorest/tracing v0.5.0/go.mod h1:r/s2XiOKccPW3HrqB+W0TQzfbtp2fGCgRFtBroKn4Dk= github.com/Azure/go-autorest/tracing v0.6.0 h1:TYi4+3m5t6K48TGI9AUdb+IzbnSxvnvUMfuitfgcfuo= github.com/Azure/go-autorest/tracing v0.6.0/go.mod h1:+vhtPC754Xsa23ID7GlGsrdKBpUA79WCAKPPZVC2DeU= -github.com/AzureAD/microsoft-authentication-library-for-go v0.9.0 h1:UE9n9rkJF62ArLb1F3DEjRt8O3jLwMWdSoypKV4f3MU= -github.com/AzureAD/microsoft-authentication-library-for-go v0.9.0/go.mod h1:kgDmCTgBzIEPFElEF+FK0SdjAor06dRq2Go927dnQ6o= +github.com/AzureAD/microsoft-authentication-library-for-go v1.0.0 h1:OBhqkivkhkMqLPymWEppkm7vgPQY2XsHoEkaMQ0AdZY= +github.com/AzureAD/microsoft-authentication-library-for-go v1.0.0/go.mod h1:kgDmCTgBzIEPFElEF+FK0SdjAor06dRq2Go927dnQ6o= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/toml v0.4.1/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= @@ -324,7 +324,7 @@ github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8 github.com/dimchansky/utfbom v1.1.0/go.mod h1:rO41eb7gLfo8SF1jd9F8HplJm1Fewwi4mQvIirEdv+8= github.com/dimchansky/utfbom v1.1.1 h1:vV6w1AhK4VMnhBno/TPVCoK9U/LP0PkLCS9tbxHdi/U= github.com/dimchansky/utfbom v1.1.1/go.mod h1:SxdoEBH5qIqFocHMyGOXVAybYJdr71b1Q/j0mACtrfE= -github.com/dnaeon/go-vcr v1.1.0 h1:ReYa/UBrRyQdant9B4fNHGoCNKw6qh6P0fsdGmZpR7c= +github.com/dnaeon/go-vcr v1.2.0 h1:zHCHvJYTMh1N7xnV7zf1m1GPBF9Ad0Jk/whtQ1663qI= github.com/docker/cli v20.10.17+incompatible h1:eO2KS7ZFeov5UJeaDmIs1NFEDRf32PaqRpvoEkKBy5M= github.com/docker/cli v20.10.17+incompatible/go.mod h1:JLrzqnKDaYBop7H2jaqPtU4hHvMKP+vjCwu2uszcLI8= github.com/docker/distribution v2.8.1+incompatible h1:Q50tZOPR6T/hjNsyc9g8/syEs6bk8XXApsHjKukMl68= diff --git a/workflow/artifacts/azure/azure.go b/workflow/artifacts/azure/azure.go index 645badd32d1c..389445f1fbfe 100644 --- a/workflow/artifacts/azure/azure.go +++ b/workflow/artifacts/azure/azure.go @@ -12,6 +12,8 @@ import ( "github.com/Azure/azure-sdk-for-go/sdk/azidentity" "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob" + "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/bloberror" + "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/container" log "github.com/sirupsen/logrus" argoerrors "github.com/argoproj/argo-workflows/v3/errors" @@ -33,10 +35,10 @@ type ArtifactDriver struct { var _ artifactscommon.ArtifactDriver = &ArtifactDriver{} -// newAzureContainerClient creates a new azblob.ContainerClient for interacting with the specified Azure Blob Storage container +// newAzureContainerClient creates a new container.Client for interacting with the specified Azure Blob Storage container // The container client is created with the default azblob.ClientOptions which does include retry behavior // for failed requests. -func (azblobDriver *ArtifactDriver) newAzureContainerClient() (*azblob.ContainerClient, error) { +func (azblobDriver *ArtifactDriver) newAzureContainerClient() (*container.Client, error) { containerUrl, err := url.Parse(azblobDriver.Endpoint) if err != nil { @@ -53,7 +55,7 @@ func (azblobDriver *ArtifactDriver) newAzureContainerClient() (*azblob.Container if err != nil { return nil, fmt.Errorf("unable to create default Azure credential: %s", err) } - containerClient, err := azblob.NewContainerClient(containerUrl.String(), credential, nil) + containerClient, err := container.NewClient(containerUrl.String(), credential, nil) return containerClient, err } else { if azblobDriver.AccountKey == "" { @@ -67,7 +69,7 @@ func (azblobDriver *ArtifactDriver) newAzureContainerClient() (*azblob.Container if err != nil { return nil, fmt.Errorf("unable to create Azure shared key credential: %s", err) } - containerClient, err := azblob.NewContainerClientWithSharedKey(containerUrl.String(), credential, nil) + containerClient, err := container.NewClientWithSharedKeyCredential(containerUrl.String(), credential, nil) return containerClient, err } } @@ -118,7 +120,7 @@ func (azblobDriver *ArtifactDriver) Load(artifact *wfv1.Artifact, path string) e return nil } isEmptyFile = true - } else if !IsAzureError(origErr, azblob.StorageErrorCodeBlobNotFound) { + } else if !bloberror.HasCode(origErr, bloberror.BlobNotFound) { return fmt.Errorf("unable to download blob %s: %s", artifact.Azure.Blob, origErr) } @@ -149,11 +151,8 @@ func (azblobDriver *ArtifactDriver) Load(artifact *wfv1.Artifact, path string) e } // DownloadFile downloads a single file from Azure Blob Storage -func DownloadFile(containerClient *azblob.ContainerClient, blobName, path string) error { - blobClient, err := containerClient.NewBlobClient(blobName) - if err != nil { - return fmt.Errorf("unable to create Azure Blob client for %s: %s", blobName, err) - } +func DownloadFile(containerClient *container.Client, blobName, path string) error { + blobClient := containerClient.NewBlobClient(blobName) outFile, err := os.Create(path) if err != nil { @@ -165,11 +164,12 @@ func DownloadFile(containerClient *azblob.ContainerClient, blobName, path string } }() - return blobClient.DownloadToFile(context.TODO(), 0, 0, outFile, azblob.DownloadOptions{}) + _, err = blobClient.DownloadFile(context.TODO(), outFile, nil) + return err } // DownloadDirectory downloads all of the files starting with the named blob prefix into a local directory. -func (azblobDriver *ArtifactDriver) DownloadDirectory(containerClient *azblob.ContainerClient, artifact *wfv1.Artifact, path string) error { +func (azblobDriver *ArtifactDriver) DownloadDirectory(containerClient *container.Client, artifact *wfv1.Artifact, path string) error { log.WithFields(log.Fields{"endpoint": artifact.Azure.Endpoint, "container": artifact.Azure.Container, "blob": artifact.Azure.Blob}).Info("Downloading directory from Azure Blob Storage") @@ -209,24 +209,21 @@ func (azblobDriver *ArtifactDriver) OpenStream(artifact *wfv1.Artifact) (io.Read return nil, fmt.Errorf("unable to create Azure Blob Container client: %s", err) } - blobClient, err := containerClient.NewBlockBlobClient(artifact.Azure.Blob) - if err != nil { - return nil, fmt.Errorf("unable to create Azure Blob client for %s: %s", artifact.Azure.Blob, err) - } + blobClient := containerClient.NewBlockBlobClient(artifact.Azure.Blob) // Attempt the download. If it fails with a BlobNotFound error, or succeeds but with // a content length of 0, then it could be that we're attempting to stream a directory. // Check if the blob represents a directory and return an error if so. If not, then // return either the original BlobNotFound error or the empty file stream. emptyFile := false - response, origErr := blobClient.Download(context.TODO(), nil) + response, origErr := blobClient.DownloadStream(context.TODO(), nil) if origErr == nil { emptyFile = *response.ContentLength == 0 // We have a normal file blob, so just return the response body stream if !emptyFile { - return response.Body(nil), nil + return response.Body, nil } - } else if !IsAzureError(origErr, azblob.StorageErrorCodeBlobNotFound) { + } else if !bloberror.HasCode(origErr, bloberror.BlobNotFound) { return nil, fmt.Errorf("unable to open stream for blob %s: %s", artifact.Azure.Blob, origErr) } @@ -242,7 +239,7 @@ func (azblobDriver *ArtifactDriver) OpenStream(artifact *wfv1.Artifact) (io.Read return nil, fmt.Errorf("unable to open blob stream for %s: %s", artifact.Azure.Blob, origErr) } - return response.Body(nil), nil + return response.Body, nil } // Save saves an artifact to Azure Blob Storage @@ -276,11 +273,8 @@ func (azblobDriver *ArtifactDriver) Save(path string, outputArtifact *wfv1.Artif } // PutFile uploads a file to Azure Blob Storage -func PutFile(containerClient *azblob.ContainerClient, blobName, path string) error { - blobClient, err := containerClient.NewBlockBlobClient(blobName) - if err != nil { - return fmt.Errorf("unable to create Azure Blob client: %s", err) - } +func PutFile(containerClient *container.Client, blobName, path string) error { + blobClient := containerClient.NewBlockBlobClient(blobName) file, err := os.Open(path) if err != nil { @@ -292,12 +286,12 @@ func PutFile(containerClient *azblob.ContainerClient, blobName, path string) err } }() - _, err = blobClient.UploadFile(context.TODO(), file, azblob.UploadOption{}) + _, err = blobClient.UploadFile(context.TODO(), file, nil) return err } // PutDirectory uploads all files in a directory to Azure Blob Storage -func PutDirectory(containerClient *azblob.ContainerClient, blobName, path string) error { +func PutDirectory(containerClient *container.Client, blobName, path string) error { for putTask := range generatePutTasks(blobName, path) { err := PutFile(containerClient, putTask.blobName, putTask.path) if err != nil { @@ -346,15 +340,12 @@ func (azblobDriver *ArtifactDriver) Delete(artifact *wfv1.Artifact) error { return nil } -func DeleteBlob(containerClient *azblob.ContainerClient, blobName string, allowNonExistent bool) error { - blobClient, err := containerClient.NewBlobClient(blobName) - if err != nil { - return fmt.Errorf("unable to create Azure Blob client for %s: %s", blobName, err) - } +func DeleteBlob(containerClient *container.Client, blobName string, allowNonExistent bool) error { + blobClient := containerClient.NewBlobClient(blobName) - _, err = blobClient.Delete(context.TODO(), nil) + _, err := blobClient.Delete(context.TODO(), nil) if err != nil { - if allowNonExistent && IsAzureError(err, azblob.StorageErrorCodeBlobNotFound) { + if allowNonExistent && bloberror.HasCode(err, bloberror.BlobNotFound) { log.Debugf("blob to delete '%s' does not exist: %s", blobName, err) return nil } else { @@ -376,21 +367,21 @@ func (azblobDriver *ArtifactDriver) ListObjects(artifact *wfv1.Artifact) ([]stri return nil, fmt.Errorf("unable to create Azure Blob Container client: %s", err) } - listOpts := azblob.ContainerListBlobsFlatOptions{ + listOpts := azblob.ListBlobsFlatOptions{ Prefix: &artifact.Azure.Blob, Marker: nil, } ctx := context.TODO() - pager := containerClient.ListBlobsFlat(&listOpts) - for pager.NextPage(ctx) { - resp := pager.PageResponse() + pager := containerClient.NewListBlobsFlatPager(&listOpts) + for pager.More() { + resp, err := pager.NextPage(ctx) + if err != nil { + return nil, fmt.Errorf("error listing blobs %s in Azure Blob Storage container: %s", artifact.Azure.Blob, err) + } for _, v := range resp.Segment.BlobItems { files = append(files, *v.Name) } } - if pager.Err() != nil { - return nil, fmt.Errorf("error listing blobs %s in Azure Blob Storage container: %s", artifact.Azure.Blob, pager.Err()) - } return files, nil } @@ -410,33 +401,26 @@ func (azblobDriver *ArtifactDriver) IsDirectory(artifact *wfv1.Artifact) (bool, return false, fmt.Errorf("unable to create Azure Blob Container client: %s", err) } - listOpts := azblob.ContainerListBlobsFlatOptions{ + listOpts := azblob.ListBlobsFlatOptions{ Prefix: &blobPrefix, Marker: nil, } - pager := containerClient.ListBlobsFlat(&listOpts) - if pager.NextPage(context.TODO()) { - resp := pager.PageResponse() + pager := containerClient.NewListBlobsFlatPager(&listOpts) + if pager.More() { + resp, err := pager.NextPage(context.TODO()) + if err != nil { + return false, fmt.Errorf("error listing blobs %s in Azure Blob Storage container: %s", artifact.Azure.Blob, err) + } if len(resp.Segment.BlobItems) == 1 { return strings.HasPrefix(*resp.Segment.BlobItems[0].Name, blobPrefix), nil } else { return len(resp.Segment.BlobItems) > 0, nil } - } else if pager.Err() != nil { - return false, fmt.Errorf("error listing blobs %s in Azure Blob Storage container: %s", artifact.Azure.Blob, pager.Err()) } return false, nil } -func IsAzureError(err error, code azblob.StorageErrorCode) bool { - var storageErr *azblob.StorageError - if errors.As(err, &storageErr) { - return storageErr.ErrorCode == code - } - return false -} - type uploadTask struct { blobName string path string