diff --git a/test/e2e/azure_edgezone.go b/test/e2e/azure_edgezone.go index 3b8d1dba61e..f13bdb4d7b0 100644 --- a/test/e2e/azure_edgezone.go +++ b/test/e2e/azure_edgezone.go @@ -22,7 +22,8 @@ package e2e import ( "context" - "github.com/Azure/azure-sdk-for-go/services/compute/mgmt/2021-11-01/compute" + "github.com/Azure/azure-sdk-for-go/sdk/azidentity" + "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/compute/armcompute/v5" "github.com/Azure/go-autorest/autorest/azure/auth" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" @@ -75,26 +76,31 @@ func AzureEdgeZoneClusterSpec(ctx context.Context, inputGetter func() AzureEdgeZ settings, err := auth.GetSettingsFromEnvironment() Expect(err).NotTo(HaveOccurred()) subscriptionID := settings.GetSubscriptionID() - auth, err := settings.GetAuthorizer() + cred, err := azidentity.NewDefaultAzureCredential(nil) Expect(err).NotTo(HaveOccurred()) if len(machineList.Items) > 0 { By("Creating a VM client") // create a VM client - vmClient := compute.NewVirtualMachinesClient(subscriptionID) - vmClient.Authorizer = auth + vmClient, err := armcompute.NewVirtualMachinesClient(subscriptionID, cred, nil) + Expect(err).NotTo(HaveOccurred()) // get the resource group name resource, err := azureutil.ParseResourceID(*machineList.Items[0].Spec.ProviderID) Expect(err).NotTo(HaveOccurred()) - vmListResults, err := vmClient.List(ctx, resource.ResourceGroupName, "") - Expect(err).NotTo(HaveOccurred()) + var vms []*armcompute.VirtualMachine + pager := vmClient.NewListPager(resource.ResourceGroupName, nil) + for pager.More() { + nextResult, err := pager.NextPage(ctx) + Expect(err).NotTo(HaveOccurred()) + vms = append(vms, nextResult.Value...) + } By("Verifying VMs' extendedLocation property is correct") - for _, machine := range vmListResults.Values() { + for _, machine := range vms { Expect(*machine.ExtendedLocation.Name).To(Equal(extendedLocationName)) - Expect(string(machine.ExtendedLocation.Type)).To(Equal(extendedLocationType)) + Expect(string(*machine.ExtendedLocation.Type)).To(Equal(extendedLocationType)) } } } diff --git a/test/e2e/azure_logcollector.go b/test/e2e/azure_logcollector.go index 2e399bf7a62..77302261327 100644 --- a/test/e2e/azure_logcollector.go +++ b/test/e2e/azure_logcollector.go @@ -28,7 +28,8 @@ import ( "strings" "time" - "github.com/Azure/azure-sdk-for-go/services/compute/mgmt/2021-11-01/compute" + "github.com/Azure/azure-sdk-for-go/sdk/azidentity" + "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/compute/armcompute/v5" "github.com/Azure/go-autorest/autorest/azure/auth" "github.com/pkg/errors" apierrors "k8s.io/apimachinery/pkg/api/errors" @@ -420,10 +421,13 @@ func collectVMBootLog(ctx context.Context, am *infrav1.AzureMachine, outputPath return errors.Wrap(err, "failed to get settings from environment") } - vmClient := compute.NewVirtualMachinesClient(settings.GetSubscriptionID()) - vmClient.Authorizer, err = azureutil.GetAuthorizer(settings) + cred, err := azidentity.NewDefaultAzureCredential(nil) if err != nil { - return errors.Wrap(err, "failed to get authorizer") + return errors.Wrap(err, "failed to get default azure credential") + } + vmClient, err := armcompute.NewVirtualMachinesClient(settings.GetSubscriptionID(), cred, nil) + if err != nil { + return errors.Wrap(err, "failed to create virtual machines client") } bootDiagnostics, err := vmClient.RetrieveBootDiagnosticsData(ctx, resource.ResourceGroupName, resource.Name, nil) @@ -431,7 +435,7 @@ func collectVMBootLog(ctx context.Context, am *infrav1.AzureMachine, outputPath return errors.Wrap(err, "failed to get boot diagnostics data") } - return writeBootLog(bootDiagnostics, outputPath) + return writeBootLog(bootDiagnostics.RetrieveBootDiagnosticsDataResult, outputPath) } // collectVMSSBootLog collects boot logs of the scale set by using azure boot diagnostics. @@ -452,10 +456,13 @@ func collectVMSSBootLog(ctx context.Context, providerID string, outputPath strin return errors.Wrap(err, "failed to get settings from environment") } - vmssClient := compute.NewVirtualMachineScaleSetVMsClient(settings.GetSubscriptionID()) - vmssClient.Authorizer, err = azureutil.GetAuthorizer(settings) + cred, err := azidentity.NewDefaultAzureCredential(nil) + if err != nil { + return errors.Wrap(err, "failed to get default azure credential") + } + vmssClient, err := armcompute.NewVirtualMachineScaleSetVMsClient(settings.GetSubscriptionID(), cred, nil) if err != nil { - return errors.Wrap(err, "failed to get authorizer") + return errors.Wrap(err, "failed to create virtual machine scale set VMs client") } bootDiagnostics, err := vmssClient.RetrieveBootDiagnosticsData(ctx, resource.ResourceGroupName, resource.Name, instanceID, nil) @@ -463,10 +470,10 @@ func collectVMSSBootLog(ctx context.Context, providerID string, outputPath strin return errors.Wrap(err, "failed to get boot diagnostics data") } - return writeBootLog(bootDiagnostics, outputPath) + return writeBootLog(bootDiagnostics.RetrieveBootDiagnosticsDataResult, outputPath) } -func writeBootLog(bootDiagnostics compute.RetrieveBootDiagnosticsDataResult, outputPath string) error { +func writeBootLog(bootDiagnostics armcompute.RetrieveBootDiagnosticsDataResult, outputPath string) error { var err error req, err := http.NewRequestWithContext(context.TODO(), http.MethodGet, *bootDiagnostics.SerialConsoleLogBlobURI, http.NoBody) if err != nil { diff --git a/test/e2e/azure_securitygroups.go b/test/e2e/azure_securitygroups.go index 6ada8cce1e3..272edef37ac 100644 --- a/test/e2e/azure_securitygroups.go +++ b/test/e2e/azure_securitygroups.go @@ -22,14 +22,14 @@ package e2e import ( "context" - "github.com/Azure/azure-sdk-for-go/services/network/mgmt/2021-08-01/network" + "github.com/Azure/azure-sdk-for-go/sdk/azidentity" + "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/network/armnetwork/v4" "github.com/Azure/go-autorest/autorest/azure/auth" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" corev1 "k8s.io/api/core/v1" "k8s.io/utils/ptr" infrav1 "sigs.k8s.io/cluster-api-provider-azure/api/v1beta1" - azureutil "sigs.k8s.io/cluster-api-provider-azure/util/azure" clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1" "sigs.k8s.io/cluster-api/test/framework" "sigs.k8s.io/controller-runtime/pkg/client" @@ -91,18 +91,17 @@ func AzureSecurityGroupsSpec(ctx context.Context, inputGetter func() AzureSecuri settings, err := auth.GetSettingsFromEnvironment() Expect(err).NotTo(HaveOccurred()) subscriptionID := settings.GetSubscriptionID() - auth, err := azureutil.GetAuthorizer(settings) Expect(err).NotTo(HaveOccurred()) - By("creating a subnets client") - subnetsClient := network.NewSubnetsClient(subscriptionID) - subnetsClient.Authorizer = auth + By("creating securitygroups and securityrules clients") + cred, err := azidentity.NewDefaultAzureCredential(nil) + Expect(err).NotTo(HaveOccurred()) - securityGroupsClient := network.NewSecurityGroupsClient(subscriptionID) - securityGroupsClient.Authorizer = auth + securityGroupsClient, err := armnetwork.NewSecurityGroupsClient(subscriptionID, cred, nil) + Expect(err).NotTo(HaveOccurred()) - securityRulesClient := network.NewSecurityRulesClient(subscriptionID) - securityRulesClient.Authorizer = auth + securityRulesClient, err := armnetwork.NewSecurityRulesClient(subscriptionID, cred, nil) + Expect(err).NotTo(HaveOccurred()) azureCluster := &infrav1.AzureCluster{} err = mgmtClient.Get(ctx, client.ObjectKey{ @@ -114,18 +113,23 @@ func AzureSecurityGroupsSpec(ctx context.Context, inputGetter func() AzureSecuri var expectedSubnets infrav1.Subnets checkSubnets := func(g Gomega) { for _, expectedSubnet := range expectedSubnets { - securityGroup, err := securityGroupsClient.Get(ctx, azureCluster.Spec.ResourceGroup, expectedSubnet.SecurityGroup.Name, "") + securityGroup, err := securityGroupsClient.Get(ctx, azureCluster.Spec.ResourceGroup, expectedSubnet.SecurityGroup.Name, nil) g.Expect(err).NotTo(HaveOccurred()) - securityRules, err := securityRulesClient.List(ctx, azureCluster.Spec.ResourceGroup, *securityGroup.Name) - g.Expect(err).NotTo(HaveOccurred()) + var securityRules []*armnetwork.SecurityRule + pager := securityRulesClient.NewListPager(azureCluster.Spec.ResourceGroup, *securityGroup.Name, nil) + for pager.More() { + nextResult, err := pager.NextPage(ctx) + Expect(err).NotTo(HaveOccurred()) + securityRules = append(securityRules, nextResult.Value...) + } var expectedSecurityRuleNames []string for _, expectedSecurityRule := range expectedSubnet.SecurityGroup.SecurityRules { expectedSecurityRuleNames = append(expectedSecurityRuleNames, expectedSecurityRule.Name) } - for _, securityRule := range securityRules.Values() { + for _, securityRule := range securityRules { g.Expect(expectedSecurityRuleNames).To(ContainElement(*securityRule.Name)) } } diff --git a/test/e2e/azure_vmextensions.go b/test/e2e/azure_vmextensions.go index 2b677ddf258..86ad920b600 100644 --- a/test/e2e/azure_vmextensions.go +++ b/test/e2e/azure_vmextensions.go @@ -22,7 +22,8 @@ package e2e import ( "context" - "github.com/Azure/azure-sdk-for-go/services/compute/mgmt/2021-11-01/compute" + "github.com/Azure/azure-sdk-for-go/sdk/azidentity" + "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/compute/armcompute/v5" "github.com/Azure/go-autorest/autorest/azure/auth" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" @@ -73,8 +74,6 @@ func AzureVMExtensionsSpec(ctx context.Context, inputGetter func() AzureVMExtens settings, err := auth.GetSettingsFromEnvironment() Expect(err).NotTo(HaveOccurred()) subscriptionID := settings.GetSubscriptionID() - auth, err := azureutil.GetAuthorizer(settings) - Expect(err).NotTo(HaveOccurred()) if len(machineList.Items) > 0 { By("Creating a mapping of machine IDs to array of expected VM extensions") @@ -86,31 +85,39 @@ func AzureVMExtensionsSpec(ctx context.Context, inputGetter func() AzureVMExtens } By("Creating a VM and VM extension client") + cred, err := azidentity.NewDefaultAzureCredential(nil) + Expect(err).NotTo(HaveOccurred()) + // create a VM client - vmClient := compute.NewVirtualMachinesClient(subscriptionID) - vmClient.Authorizer = auth + vmClient, err := armcompute.NewVirtualMachinesClient(subscriptionID, cred, nil) + Expect(err).NotTo(HaveOccurred()) // create a VM extension client - vmExtensionsClient := compute.NewVirtualMachineExtensionsClient(subscriptionID) - vmExtensionsClient.Authorizer = auth + vmExtensionsClient, err := armcompute.NewVirtualMachineExtensionsClient(subscriptionID, cred, nil) + Expect(err).NotTo(HaveOccurred()) // get the resource group name resource, err := azureutil.ParseResourceID(*machineList.Items[0].Spec.ProviderID) Expect(err).NotTo(HaveOccurred()) - vmListResults, err := vmClient.List(ctx, resource.ResourceGroupName, "") - Expect(err).NotTo(HaveOccurred()) + var vms []*armcompute.VirtualMachine + pager := vmClient.NewListPager(resource.ResourceGroupName, nil) + for pager.More() { + nextResult, err := pager.NextPage(ctx) + Expect(err).NotTo(HaveOccurred()) + vms = append(vms, nextResult.Value...) + } By("Verifying specified VM extensions are created on Azure") - for _, machine := range vmListResults.Values() { - vmExtensionListResult, err := vmExtensionsClient.List(ctx, resource.ResourceGroupName, *machine.Name, "") + for _, machine := range vms { + vmExtensionListResult, err := vmExtensionsClient.List(ctx, resource.ResourceGroupName, *machine.Name, nil) Expect(err).NotTo(HaveOccurred()) - vmExtensionList := *vmExtensionListResult.Value + vmExtensionList := vmExtensionListResult.Value var vmExtensionNames []string for _, vmExtension := range vmExtensionList { vmExtensionNames = append(vmExtensionNames, *vmExtension.Name) } - osName := string(machine.VirtualMachineProperties.StorageProfile.OsDisk.OsType) + osName := string(*machine.Properties.StorageProfile.OSDisk.OSType) Expect(vmExtensionNames).To(ContainElements("CAPZ." + osName + ".Bootstrapping")) Expect(vmExtensionNames).To(ContainElements(expectedVMExtensionMap[*machine.ID])) } @@ -133,31 +140,43 @@ func AzureVMExtensionsSpec(ctx context.Context, inputGetter func() AzureVMExtens } By("Creating a VMSS and VMSS extension client") + cred, err := azidentity.NewDefaultAzureCredential(nil) + Expect(err).NotTo(HaveOccurred()) + // create a VMSS client - vmssClient := compute.NewVirtualMachineScaleSetsClient(subscriptionID) - vmssClient.Authorizer = auth + vmssClient, err := armcompute.NewVirtualMachineScaleSetsClient(subscriptionID, cred, nil) + Expect(err).NotTo(HaveOccurred()) // create a VMSS extension client - vmssExtensionsClient := compute.NewVirtualMachineScaleSetExtensionsClient(subscriptionID) - vmssExtensionsClient.Authorizer = auth + vmssExtensionsClient, err := armcompute.NewVirtualMachineScaleSetExtensionsClient(subscriptionID, cred, nil) + Expect(err).NotTo(HaveOccurred()) // get the resource group name resource, err := azureutil.ParseResourceID(machinePoolList.Items[0].Spec.ProviderID) Expect(err).NotTo(HaveOccurred()) - vmssListResults, err := vmssClient.List(ctx, resource.ResourceGroupName) - Expect(err).NotTo(HaveOccurred()) + var vmsses []*armcompute.VirtualMachineScaleSet + pager := vmssClient.NewListPager(resource.ResourceGroupName, nil) + for pager.More() { + nextResult, err := pager.NextPage(ctx) + Expect(err).NotTo(HaveOccurred()) + vmsses = append(vmsses, nextResult.Value...) + } By("Verifying VMSS extensions are created on Azure") - for _, machinePool := range vmssListResults.Values() { - vmssExtensionListResult, err := vmssExtensionsClient.List(ctx, resource.ResourceGroupName, *machinePool.Name) - Expect(err).NotTo(HaveOccurred()) - vmssExtensionList := vmssExtensionListResult.Values() + for _, machinePool := range vmsses { + var vmssExts []*armcompute.VirtualMachineScaleSetExtension + pager := vmssExtensionsClient.NewListPager(resource.ResourceGroupName, *machinePool.Name, nil) + for pager.More() { + nextResult, err := pager.NextPage(ctx) + Expect(err).NotTo(HaveOccurred()) + vmssExts = append(vmssExts, nextResult.Value...) + } var vmssExtensionNames []string - for _, vmssExtension := range vmssExtensionList { + for _, vmssExtension := range vmssExts { vmssExtensionNames = append(vmssExtensionNames, *vmssExtension.Name) } - osName := string(machinePool.VirtualMachineScaleSetProperties.VirtualMachineProfile.StorageProfile.OsDisk.OsType) + osName := string(*machinePool.Properties.VirtualMachineProfile.StorageProfile.OSDisk.OSType) Expect(vmssExtensionNames).To(ContainElements("CAPZ." + osName + ".Bootstrapping")) Expect(vmssExtensionNames).To(ContainElements(expectedVMSSExtensionMap[*machinePool.ID])) } diff --git a/test/e2e/common.go b/test/e2e/common.go index 30674d3e79b..eea819490d0 100644 --- a/test/e2e/common.go +++ b/test/e2e/common.go @@ -28,7 +28,8 @@ import ( "path/filepath" "strings" - "github.com/Azure/azure-sdk-for-go/services/resources/mgmt/2020-10-01/resources" + "github.com/Azure/azure-sdk-for-go/sdk/azidentity" + "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/resources/armresources" "github.com/Azure/go-autorest/autorest/azure/auth" . "github.com/onsi/ginkgo/v2" "github.com/onsi/ginkgo/v2/types" @@ -40,7 +41,6 @@ import ( "k8s.io/utils/ptr" "sigs.k8s.io/cluster-api-provider-azure/azure" e2e_namespace "sigs.k8s.io/cluster-api-provider-azure/test/e2e/kubernetes/namespace" - azureutil "sigs.k8s.io/cluster-api-provider-azure/util/azure" clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1" kubeadmv1 "sigs.k8s.io/cluster-api/controlplane/kubeadm/api/v1beta1" capi_e2e "sigs.k8s.io/cluster-api/test/e2e" @@ -209,11 +209,11 @@ func ExpectResourceGroupToBe404(ctx context.Context) { settings, err := auth.GetSettingsFromEnvironment() Expect(err).NotTo(HaveOccurred()) subscriptionID := settings.GetSubscriptionID() - authorizer, err := azureutil.GetAuthorizer(settings) + cred, err := azidentity.NewDefaultAzureCredential(nil) Expect(err).NotTo(HaveOccurred()) - groupsClient := resources.NewGroupsClient(subscriptionID) - groupsClient.Authorizer = authorizer - _, err = groupsClient.Get(ctx, os.Getenv(AzureResourceGroup)) + groupsClient, err := armresources.NewResourceGroupsClient(subscriptionID, cred, nil) + Expect(err).NotTo(HaveOccurred()) + _, err = groupsClient.Get(ctx, os.Getenv(AzureResourceGroup), nil) Expect(azure.ResourceNotFound(err)).To(BeTrue(), "The resource group in Azure still exists. After deleting the cluster all of the Azure resources should also be deleted.") } diff --git a/test/e2e/helpers.go b/test/e2e/helpers.go index b15c96e4c7a..6544598e87a 100644 --- a/test/e2e/helpers.go +++ b/test/e2e/helpers.go @@ -35,7 +35,8 @@ import ( "text/tabwriter" "time" - "github.com/Azure/azure-sdk-for-go/services/compute/mgmt/2022-03-02/compute" + "github.com/Azure/azure-sdk-for-go/sdk/azidentity" + "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/compute/armcompute/v5" "github.com/Azure/go-autorest/autorest/azure/auth" "github.com/blang/semver" . "github.com/onsi/ginkgo/v2" @@ -55,7 +56,6 @@ import ( infrav1 "sigs.k8s.io/cluster-api-provider-azure/api/v1beta1" "sigs.k8s.io/cluster-api-provider-azure/azure" infrav1exp "sigs.k8s.io/cluster-api-provider-azure/exp/api/v1beta1" - azureutil "sigs.k8s.io/cluster-api-provider-azure/util/azure" capi_e2e "sigs.k8s.io/cluster-api/test/e2e" "sigs.k8s.io/cluster-api/test/framework" "sigs.k8s.io/cluster-api/test/framework/clusterctl" @@ -759,38 +759,38 @@ func resolveFlatcarVersion(config *clusterctl.E2EConfig, versions semver.Version } // newImagesClient returns a new VM images client using environmental settings for auth. -func newImagesClient() compute.VirtualMachineImagesClient { +func newImagesClient() *armcompute.VirtualMachineImagesClient { settings, err := auth.GetSettingsFromEnvironment() Expect(err).NotTo(HaveOccurred()) subscriptionID := settings.GetSubscriptionID() - authorizer, err := azureutil.GetAuthorizer(settings) + cred, err := azidentity.NewDefaultAzureCredential(nil) + Expect(err).NotTo(HaveOccurred()) + imagesClient, err := armcompute.NewVirtualMachineImagesClient(subscriptionID, cred, nil) Expect(err).NotTo(HaveOccurred()) - imagesClient := compute.NewVirtualMachineImagesClient(subscriptionID) - imagesClient.Authorizer = authorizer return imagesClient } -func newCommunityGalleryImagesClient() compute.CommunityGalleryImagesClient { +func newCommunityGalleryImagesClient() *armcompute.CommunityGalleryImagesClient { settings, err := auth.GetSettingsFromEnvironment() Expect(err).NotTo(HaveOccurred()) subscriptionID := settings.GetSubscriptionID() - authorizer, err := azureutil.GetAuthorizer(settings) + cred, err := azidentity.NewDefaultAzureCredential(nil) + Expect(err).NotTo(HaveOccurred()) + communityGalleryImagesClient, err := armcompute.NewCommunityGalleryImagesClient(subscriptionID, cred, nil) Expect(err).NotTo(HaveOccurred()) - communityGalleryImagesClient := compute.NewCommunityGalleryImagesClient(subscriptionID) - communityGalleryImagesClient.Authorizer = authorizer return communityGalleryImagesClient } -func newCommunityGalleryImageVersionsClient() compute.CommunityGalleryImageVersionsClient { +func newCommunityGalleryImageVersionsClient() *armcompute.CommunityGalleryImageVersionsClient { settings, err := auth.GetSettingsFromEnvironment() Expect(err).NotTo(HaveOccurred()) subscriptionID := settings.GetSubscriptionID() - authorizer, err := azureutil.GetAuthorizer(settings) + cred, err := azidentity.NewDefaultAzureCredential(nil) + Expect(err).NotTo(HaveOccurred()) + communityGalleryImageVersionsClient, err := armcompute.NewCommunityGalleryImageVersionsClient(subscriptionID, cred, nil) Expect(err).NotTo(HaveOccurred()) - communityGalleryImageVersionsClient := compute.NewCommunityGalleryImageVersionsClient(subscriptionID) - communityGalleryImageVersionsClient.Authorizer = authorizer return communityGalleryImageVersionsClient } @@ -803,31 +803,31 @@ func getVersionsInOffer(ctx context.Context, location, publisher, offer string) capiVersion := regexp.MustCompile(`^(\d)(\d{1,2})\.(\d{1,2})\.\d{8}$`) oldCapiSku := regexp.MustCompile(`^k8s-(0|[1-9][0-9]*)dot(0|[1-9][0-9]*)dot(0|[1-9][0-9]*)-[a-z]*.*$`) imagesClient := newImagesClient() - skus, err := imagesClient.ListSkus(ctx, location, publisher, offer) + resp, err := imagesClient.ListSKUs(ctx, location, publisher, offer, nil) Expect(err).NotTo(HaveOccurred()) - if skus.Value != nil { - versions = make(map[string]semver.Version, len(*skus.Value)) - for _, sku := range *skus.Value { - res, err := imagesClient.List(ctx, location, publisher, offer, *sku.Name, "", nil, "") - Expect(err).NotTo(HaveOccurred()) - // Don't use SKUs without existing images. See https://github.com/Azure/azure-cli/issues/20115. - if res.Value != nil && len(*res.Value) > 0 { - // New SKUs don't contain the Kubernetes version and are named like "ubuntu-2004-gen1". - if match := capiSku.FindStringSubmatch(*sku.Name); len(match) > 0 { - for _, vmImage := range *res.Value { - // Versions are named like "121.13.20220601", for Kubernetes v1.21.13 published on June 1, 2022. - match = capiVersion.FindStringSubmatch(*vmImage.Name) - stringVer := fmt.Sprintf("%s.%s.%s", match[1], match[2], match[3]) - versions[stringVer] = semver.MustParse(stringVer) - } - continue - } - // Old SKUs before 1.21.12, 1.22.9, or 1.23.6 are named like "k8s-1dot21dot2-ubuntu-2004". - if match := oldCapiSku.FindStringSubmatch(*sku.Name); len(match) > 0 { + skus := resp.VirtualMachineImageResourceArray + + versions = make(map[string]semver.Version, len(skus)) + for _, sku := range skus { + res, err := imagesClient.List(ctx, location, publisher, offer, *sku.Name, nil) + Expect(err).NotTo(HaveOccurred()) + // Don't use SKUs without existing images. See https://github.com/Azure/azure-cli/issues/20115. + if len(res.VirtualMachineImageResourceArray) > 0 { + // New SKUs don't contain the Kubernetes version and are named like "ubuntu-2004-gen1". + if match := capiSku.FindStringSubmatch(*sku.Name); len(match) > 0 { + for _, vmImage := range res.VirtualMachineImageResourceArray { + // Versions are named like "121.13.20220601", for Kubernetes v1.21.13 published on June 1, 2022. + match = capiVersion.FindStringSubmatch(*vmImage.Name) stringVer := fmt.Sprintf("%s.%s.%s", match[1], match[2], match[3]) versions[stringVer] = semver.MustParse(stringVer) } + continue + } + // Old SKUs before 1.21.12, 1.22.9, or 1.23.6 are named like "k8s-1dot21dot2-ubuntu-2004". + if match := oldCapiSku.FindStringSubmatch(*sku.Name); len(match) > 0 { + stringVer := fmt.Sprintf("%s.%s.%s", match[1], match[2], match[3]) + versions[stringVer] = semver.MustParse(stringVer) } } } @@ -867,15 +867,15 @@ func getFlatcarVersions(ctx context.Context, location, galleryName, k8sVersion s Logf("Finding Flatcar versions in community gallery %q in location %q for image %q", galleryName, location, image) var versions semver.Versions communityGalleryImageVersionsClient := newCommunityGalleryImageVersionsClient() - imageVersionsIterator, err := communityGalleryImageVersionsClient.List(ctx, location, galleryName, image) - Expect(err).NotTo(HaveOccurred()) - imageVersions := imageVersionsIterator.Response() - - if imageVersions.Value == nil { - return versions + var imageVersions []*armcompute.CommunityGalleryImageVersion + pager := communityGalleryImageVersionsClient.NewListPager(location, galleryName, image, nil) + for pager.More() { + nextResult, err := pager.NextPage(ctx) + Expect(err).NotTo(HaveOccurred()) + imageVersions = append(imageVersions, nextResult.Value...) } - for _, imageVersion := range *imageVersions.Value { + for _, imageVersion := range imageVersions { versions = append(versions, semver.MustParse(*imageVersion.Name)) } @@ -888,20 +888,24 @@ func getFlatcarK8sVersions(ctx context.Context, location, communityGalleryName s k8sVersion := regexp.MustCompile(`flatcar-stable-amd64-capi-v(\d+)\.(\d+).(\d+)`) communityGalleryImagesClient := newCommunityGalleryImagesClient() communityGalleryImageVersionsClient := newCommunityGalleryImageVersionsClient() - imagesIterator, err := communityGalleryImagesClient.ListComplete(ctx, location, communityGalleryName) - Expect(err).NotTo(HaveOccurred()) - images := imagesIterator.Response() - - if images.Value == nil { - return versions + var images []*armcompute.CommunityGalleryImage + pager := communityGalleryImagesClient.NewListPager(location, communityGalleryName, nil) + for pager.More() { + nextResult, err := pager.NextPage(ctx) + Expect(err).NotTo(HaveOccurred()) + images = append(images, nextResult.Value...) } - for _, image := range *images.Value { - resIterator, err := communityGalleryImageVersionsClient.List(ctx, location, communityGalleryName, *image.Name) - Expect(err).NotTo(HaveOccurred()) - res := resIterator.Response() + for _, image := range images { + var imageVersions []*armcompute.CommunityGalleryImageVersion + pager := communityGalleryImageVersionsClient.NewListPager(location, communityGalleryName, *image.Name, nil) + for pager.More() { + nextResult, err := pager.NextPage(ctx) + Expect(err).NotTo(HaveOccurred()) + imageVersions = append(imageVersions, nextResult.Value...) + } - if res.Value == nil || len(*res.Value) == 0 { + if len(imageVersions) == 0 { continue }