Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Expose AKS preview features #4617

Merged
merged 1 commit into from
Mar 7, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions api/v1beta1/azuremanagedcontrolplane_webhook.go
Original file line number Diff line number Diff line change
Expand Up @@ -81,6 +81,7 @@ func (mw *azureManagedControlPlaneWebhook) Default(ctx context.Context, obj runt
setDefault[*Identity](&m.Spec.Identity, &Identity{
Type: ManagedControlPlaneIdentityTypeSystemAssigned,
})
setDefault[*bool](&m.Spec.EnablePreviewFeatures, ptr.To(false))
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

For next time, the compiler can usually figure out what the type parameters are so they don't always need to be included.

Suggested change
setDefault[*bool](&m.Spec.EnablePreviewFeatures, ptr.To(false))
setDefault(&m.Spec.EnablePreviewFeatures, ptr.To(false))

m.Spec.Version = setDefaultVersion(m.Spec.Version)
m.Spec.SKU = setDefaultSku(m.Spec.SKU)
m.Spec.AutoScalerProfile = setDefaultAutoScalerProfile(m.Spec.AutoScalerProfile)
Expand Down
5 changes: 5 additions & 0 deletions api/v1beta1/azuremanagedcontrolplane_webhook_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -77,6 +77,8 @@ func TestDefaultingWebhook(t *testing.T) {
g.Expect(amcp.Spec.DNSPrefix).NotTo(BeNil())
g.Expect(*amcp.Spec.DNSPrefix).To(Equal(amcp.Name))
g.Expect(amcp.Spec.Extensions[0].Plan.Name).To(Equal("fooName-test-product"))
g.Expect(amcp.Spec.EnablePreviewFeatures).NotTo(BeNil())
g.Expect(*amcp.Spec.EnablePreviewFeatures).To(BeFalse())

t.Logf("Testing amcp defaulting webhook with baseline")
netPlug := "kubenet"
Expand Down Expand Up @@ -106,6 +108,7 @@ func TestDefaultingWebhook(t *testing.T) {
IntervalHours: ptr.To(48),
},
}
amcp.Spec.EnablePreviewFeatures = ptr.To(true)

err = mcpw.Default(context.Background(), amcp)
g.Expect(err).NotTo(HaveOccurred())
Expand All @@ -129,6 +132,8 @@ func TestDefaultingWebhook(t *testing.T) {
g.Expect(amcp.Spec.SecurityProfile.ImageCleaner).NotTo(BeNil())
g.Expect(amcp.Spec.SecurityProfile.ImageCleaner.IntervalHours).NotTo(BeNil())
g.Expect(*amcp.Spec.SecurityProfile.ImageCleaner.IntervalHours).To(Equal(48))
g.Expect(amcp.Spec.EnablePreviewFeatures).NotTo(BeNil())
g.Expect(*amcp.Spec.EnablePreviewFeatures).To(BeTrue())

t.Logf("Testing amcp defaulting webhook with overlay")
amcp = &AzureManagedControlPlane{
Expand Down
1 change: 1 addition & 0 deletions api/v1beta1/azuremanagedcontrolplanetemplate_default.go
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@ import (
func (mcp *AzureManagedControlPlaneTemplate) setDefaults() {
setDefault[*string](&mcp.Spec.Template.Spec.NetworkPlugin, ptr.To(AzureNetworkPluginName))
setDefault[*string](&mcp.Spec.Template.Spec.LoadBalancerSKU, ptr.To("Standard"))
setDefault[*bool](&mcp.Spec.Template.Spec.EnablePreviewFeatures, ptr.To(false))

if mcp.Spec.Template.Spec.Version != "" && !strings.HasPrefix(mcp.Spec.Template.Spec.Version, "v") {
mcp.Spec.Template.Spec.Version = setDefaultVersion(mcp.Spec.Template.Spec.Version)
Expand Down
3 changes: 3 additions & 0 deletions api/v1beta1/azuremanagedcontrolplanetemplate_webhook_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -41,6 +41,7 @@ func TestControlPlaneTemplateDefaultingWebhook(t *testing.T) {
g.Expect(amcpt.Spec.Template.Spec.VirtualNetwork.Subnet.Name).To(Equal("fooName"))
g.Expect(amcpt.Spec.Template.Spec.VirtualNetwork.Subnet.CIDRBlock).To(Equal(defaultAKSNodeSubnetCIDR))
g.Expect(amcpt.Spec.Template.Spec.SKU.Tier).To(Equal(FreeManagedControlPlaneTier))
g.Expect(*amcpt.Spec.Template.Spec.EnablePreviewFeatures).To(BeFalse())

t.Logf("Testing amcp defaulting webhook with baseline")
netPlug := "kubenet"
Expand All @@ -53,6 +54,7 @@ func TestControlPlaneTemplateDefaultingWebhook(t *testing.T) {
amcpt.Spec.Template.Spec.VirtualNetwork.Name = "fooVnetName"
amcpt.Spec.Template.Spec.VirtualNetwork.Subnet.Name = "fooSubnetName"
amcpt.Spec.Template.Spec.SKU.Tier = PaidManagedControlPlaneTier
amcpt.Spec.Template.Spec.EnablePreviewFeatures = ptr.To(true)

err = mcptw.Default(context.Background(), amcpt)
g.Expect(err).NotTo(HaveOccurred())
Expand All @@ -63,6 +65,7 @@ func TestControlPlaneTemplateDefaultingWebhook(t *testing.T) {
g.Expect(amcpt.Spec.Template.Spec.VirtualNetwork.Name).To(Equal("fooVnetName"))
g.Expect(amcpt.Spec.Template.Spec.VirtualNetwork.Subnet.Name).To(Equal("fooSubnetName"))
g.Expect(amcpt.Spec.Template.Spec.SKU.Tier).To(Equal(StandardManagedControlPlaneTier))
g.Expect(*amcpt.Spec.Template.Spec.EnablePreviewFeatures).To(BeTrue())
}

func TestControlPlaneTemplateUpdateWebhook(t *testing.T) {
Expand Down
4 changes: 4 additions & 0 deletions api/v1beta1/types_class.go
Original file line number Diff line number Diff line change
Expand Up @@ -242,6 +242,10 @@ type AzureManagedControlPlaneClassSpec struct {
// operation is possible.
// +optional
ASOManagedClusterPatches []string `json:"asoManagedClusterPatches,omitempty"`

// EnablePreviewFeatures enables preview features for the cluster.
// +optional
EnablePreviewFeatures *bool `json:"enablePreviewFeatures,omitempty"`
}

// ManagedClusterAutoUpgradeProfile defines the auto upgrade profile for a managed cluster.
Expand Down
5 changes: 5 additions & 0 deletions api/v1beta1/zz_generated.deepcopy.go

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

42 changes: 42 additions & 0 deletions azure/converters/managedagentpool.go
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@
package converters

import (
asocontainerservicev1preview "github.com/Azure/azure-service-operator/v2/api/containerservice/v1api20230202preview"
mboersma marked this conversation as resolved.
Show resolved Hide resolved
asocontainerservicev1 "github.com/Azure/azure-service-operator/v2/api/containerservice/v1api20231001"
"k8s.io/utils/ptr"
)
Expand Down Expand Up @@ -59,3 +60,44 @@
}
return agentPool
}

// AgentPoolToManagedClusterAgentPoolPreviewProfile converts an AgentPoolSpec to an Azure SDK ManagedClusterAgentPoolPreviewProfile used in managedcluster reconcile.
func AgentPoolToManagedClusterAgentPoolPreviewProfile(pool *asocontainerservicev1preview.ManagedClustersAgentPool) asocontainerservicev1preview.ManagedClusterAgentPoolProfile {
properties := pool.Spec

// Populate the same properties as the stable version since the patcher will handle the preview-only fields.
agentPool := asocontainerservicev1preview.ManagedClusterAgentPoolProfile{
Name: ptr.To(pool.AzureName()),
VmSize: properties.VmSize,
OsType: properties.OsType,
OsDiskSizeGB: properties.OsDiskSizeGB,
Count: properties.Count,
Type: properties.Type,
OrchestratorVersion: properties.OrchestratorVersion,
VnetSubnetReference: properties.VnetSubnetReference,
Mode: properties.Mode,
EnableAutoScaling: properties.EnableAutoScaling,
MaxCount: properties.MaxCount,
MinCount: properties.MinCount,
NodeTaints: properties.NodeTaints,
AvailabilityZones: properties.AvailabilityZones,
MaxPods: properties.MaxPods,
OsDiskType: properties.OsDiskType,
NodeLabels: properties.NodeLabels,
EnableUltraSSD: properties.EnableUltraSSD,
EnableNodePublicIP: properties.EnableNodePublicIP,
NodePublicIPPrefixReference: properties.NodePublicIPPrefixReference,
ScaleSetPriority: properties.ScaleSetPriority,
ScaleDownMode: properties.ScaleDownMode,
SpotMaxPrice: properties.SpotMaxPrice,
Tags: properties.Tags,
KubeletDiskType: properties.KubeletDiskType,
LinuxOSConfig: properties.LinuxOSConfig,
EnableFIPS: properties.EnableFIPS,
EnableEncryptionAtHost: properties.EnableEncryptionAtHost,
}
if properties.KubeletConfig != nil {
agentPool.KubeletConfig = properties.KubeletConfig

Check warning on line 100 in azure/converters/managedagentpool.go

View check run for this annotation

Codecov / codecov/patch

azure/converters/managedagentpool.go#L100

Added line #L100 was not covered by tests
}
return agentPool
}
84 changes: 84 additions & 0 deletions azure/converters/managedagentpool_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@ package converters
import (
"testing"

asocontainerservicev1preview "github.com/Azure/azure-service-operator/v2/api/containerservice/v1api20230202preview"
asocontainerservicev1 "github.com/Azure/azure-service-operator/v2/api/containerservice/v1api20231001"
"github.com/Azure/azure-service-operator/v2/pkg/genruntime"
. "github.com/onsi/gomega"
Expand Down Expand Up @@ -107,3 +108,86 @@ func Test_AgentPoolToManagedClusterAgentPoolProfile(t *testing.T) {
})
}
}

func Test_AgentPoolToManagedClusterAgentPoolPreviewProfile(t *testing.T) {
cases := []struct {
name string
pool *asocontainerservicev1preview.ManagedClustersAgentPool
expect func(*GomegaWithT, asocontainerservicev1preview.ManagedClusterAgentPoolProfile)
}{
{
name: "Should set all values correctly",
pool: &asocontainerservicev1preview.ManagedClustersAgentPool{
Spec: asocontainerservicev1preview.ManagedClusters_AgentPool_Spec{
AzureName: "agentpool1",
VmSize: ptr.To("Standard_D2s_v3"),
OsType: ptr.To(asocontainerservicev1preview.OSType_Linux),
OsDiskSizeGB: ptr.To[asocontainerservicev1preview.ContainerServiceOSDisk](100),
Count: ptr.To(2),
Type: ptr.To(asocontainerservicev1preview.AgentPoolType_VirtualMachineScaleSets),
OrchestratorVersion: ptr.To("1.22.6"),
VnetSubnetReference: &genruntime.ResourceReference{
ARMID: "/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/rg-123/providers/Microsoft.Network/virtualNetworks/vnet-123/subnets/subnet-123",
},
Mode: ptr.To(asocontainerservicev1preview.AgentPoolMode_User),
EnableAutoScaling: ptr.To(true),
MaxCount: ptr.To(5),
MinCount: ptr.To(2),
NodeTaints: []string{"key1=value1:NoSchedule"},
AvailabilityZones: []string{"zone1"},
MaxPods: ptr.To(60),
OsDiskType: ptr.To(asocontainerservicev1preview.OSDiskType_Managed),
NodeLabels: map[string]string{
"custom": "default",
},
Tags: map[string]string{
"custom": "default",
},
EnableFIPS: ptr.To(true),
EnableEncryptionAtHost: ptr.To(true),
},
},

expect: func(g *GomegaWithT, result asocontainerservicev1preview.ManagedClusterAgentPoolProfile) {
g.Expect(result).To(Equal(asocontainerservicev1preview.ManagedClusterAgentPoolProfile{
Name: ptr.To("agentpool1"),
VmSize: ptr.To("Standard_D2s_v3"),
OsType: ptr.To(asocontainerservicev1preview.OSType_Linux),
OsDiskSizeGB: ptr.To[asocontainerservicev1preview.ContainerServiceOSDisk](100),
Count: ptr.To(2),
Type: ptr.To(asocontainerservicev1preview.AgentPoolType_VirtualMachineScaleSets),
OrchestratorVersion: ptr.To("1.22.6"),
VnetSubnetReference: &genruntime.ResourceReference{
ARMID: "/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/rg-123/providers/Microsoft.Network/virtualNetworks/vnet-123/subnets/subnet-123",
},
Mode: ptr.To(asocontainerservicev1preview.AgentPoolMode_User),
EnableAutoScaling: ptr.To(true),
MaxCount: ptr.To(5),
MinCount: ptr.To(2),
NodeTaints: []string{"key1=value1:NoSchedule"},
AvailabilityZones: []string{"zone1"},
MaxPods: ptr.To(60),
OsDiskType: ptr.To(asocontainerservicev1preview.OSDiskType_Managed),
NodeLabels: map[string]string{
"custom": "default",
},
Tags: map[string]string{
"custom": "default",
},
EnableFIPS: ptr.To(true),
EnableEncryptionAtHost: ptr.To(true),
}))
},
},
}

for _, c := range cases {
c := c
t.Run(c.name, func(t *testing.T) {
t.Parallel()
g := NewGomegaWithT(t)
result := AgentPoolToManagedClusterAgentPoolPreviewProfile(c.pool)
c.expect(g, result)
})
}
}
14 changes: 10 additions & 4 deletions azure/scope/managedcontrolplane.go
Original file line number Diff line number Diff line change
Expand Up @@ -24,11 +24,11 @@
"time"

asocontainerservicev1preview "github.com/Azure/azure-service-operator/v2/api/containerservice/v1api20230315preview"
asocontainerservicev1 "github.com/Azure/azure-service-operator/v2/api/containerservice/v1api20231001"
asokubernetesconfigurationv1 "github.com/Azure/azure-service-operator/v2/api/kubernetesconfiguration/v1api20230501"
asonetworkv1api20201101 "github.com/Azure/azure-service-operator/v2/api/network/v1api20201101"
asonetworkv1api20220701 "github.com/Azure/azure-service-operator/v2/api/network/v1api20220701"
asoresourcesv1 "github.com/Azure/azure-service-operator/v2/api/resources/v1api20200601"
"github.com/Azure/azure-service-operator/v2/pkg/genruntime"
"github.com/pkg/errors"
"golang.org/x/mod/semver"
"gopkg.in/yaml.v3"
Expand Down Expand Up @@ -528,6 +528,11 @@
return isManagedVersionUpgrade(s.ControlPlane)
}

// IsPreviewEnabled checks if the preview feature is enabled.
func (s *ManagedControlPlaneScope) IsPreviewEnabled() bool {
return ptr.Deref(s.ControlPlane.Spec.EnablePreviewFeatures, false)

Check warning on line 533 in azure/scope/managedcontrolplane.go

View check run for this annotation

Codecov / codecov/patch

azure/scope/managedcontrolplane.go#L532-L533

Added lines #L532 - L533 were not covered by tests
}

func isManagedVersionUpgrade(managedControlPlane *infrav1.AzureManagedControlPlane) bool {
return managedControlPlane.Spec.AutoUpgradeProfile != nil &&
managedControlPlane.Spec.AutoUpgradeProfile.UpgradeChannel != nil &&
Expand All @@ -536,7 +541,7 @@
}

// ManagedClusterSpec returns the managed cluster spec.
func (s *ManagedControlPlaneScope) ManagedClusterSpec() azure.ASOResourceSpecGetter[*asocontainerservicev1.ManagedCluster] {
func (s *ManagedControlPlaneScope) ManagedClusterSpec() azure.ASOResourceSpecGetter[genruntime.MetaObject] {
managedClusterSpec := managedclusters.ManagedClusterSpec{
Name: s.ControlPlane.Name,
ResourceGroup: s.ControlPlane.Spec.ResourceGroupName,
Expand All @@ -559,6 +564,7 @@
NetworkPluginMode: s.ControlPlane.Spec.NetworkPluginMode,
DNSPrefix: s.ControlPlane.Spec.DNSPrefix,
Patches: s.ControlPlane.Spec.ASOManagedClusterPatches,
Preview: ptr.Deref(s.ControlPlane.Spec.EnablePreviewFeatures, false),
}

if s.ControlPlane.Spec.SSHPublicKey != nil {
Expand Down Expand Up @@ -726,9 +732,9 @@
}

// GetAllAgentPoolSpecs gets a slice of azure.AgentPoolSpec for the list of agent pools.
func (s *ManagedControlPlaneScope) GetAllAgentPoolSpecs() ([]azure.ASOResourceSpecGetter[*asocontainerservicev1.ManagedClustersAgentPool], error) {
func (s *ManagedControlPlaneScope) GetAllAgentPoolSpecs() ([]azure.ASOResourceSpecGetter[genruntime.MetaObject], error) {
var (
ammps = make([]azure.ASOResourceSpecGetter[*asocontainerservicev1.ManagedClustersAgentPool], 0, len(s.ManagedMachinePools))
ammps = make([]azure.ASOResourceSpecGetter[genruntime.MetaObject], 0, len(s.ManagedMachinePools))
foundSystemPool = false
)
for _, pool := range s.ManagedMachinePools {
Expand Down
12 changes: 6 additions & 6 deletions azure/scope/managedcontrolplane_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -21,10 +21,10 @@ import (
"reflect"
"testing"

asocontainerservicev1 "github.com/Azure/azure-service-operator/v2/api/containerservice/v1api20231001"
asokubernetesconfigurationv1 "github.com/Azure/azure-service-operator/v2/api/kubernetesconfiguration/v1api20230501"
asonetworkv1 "github.com/Azure/azure-service-operator/v2/api/network/v1api20220701"
asoresourcesv1 "github.com/Azure/azure-service-operator/v2/api/resources/v1api20200601"
"github.com/Azure/azure-service-operator/v2/pkg/genruntime"
. "github.com/onsi/gomega"
corev1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
Expand Down Expand Up @@ -141,7 +141,7 @@ func TestManagedControlPlaneScope_PoolVersion(t *testing.T) {
cases := []struct {
Name string
Input ManagedControlPlaneScopeParams
Expected []azure.ASOResourceSpecGetter[*asocontainerservicev1.ManagedClustersAgentPool]
Expected []azure.ASOResourceSpecGetter[genruntime.MetaObject]
Err string
}{
{
Expand Down Expand Up @@ -176,7 +176,7 @@ func TestManagedControlPlaneScope_PoolVersion(t *testing.T) {
},
},
},
Expected: []azure.ASOResourceSpecGetter[*asocontainerservicev1.ManagedClustersAgentPool]{
Expected: []azure.ASOResourceSpecGetter[genruntime.MetaObject]{
&agentpools.AgentPoolSpec{
Name: "pool0",
AzureName: "pool0",
Expand Down Expand Up @@ -221,7 +221,7 @@ func TestManagedControlPlaneScope_PoolVersion(t *testing.T) {
},
},
},
Expected: []azure.ASOResourceSpecGetter[*asocontainerservicev1.ManagedClustersAgentPool]{
Expected: []azure.ASOResourceSpecGetter[genruntime.MetaObject]{
&agentpools.AgentPoolSpec{
Name: "pool0",
AzureName: "pool0",
Expand Down Expand Up @@ -428,7 +428,7 @@ func TestManagedControlPlaneScope_OSType(t *testing.T) {
cases := []struct {
Name string
Input ManagedControlPlaneScopeParams
Expected []azure.ASOResourceSpecGetter[*asocontainerservicev1.ManagedClustersAgentPool]
Expected []azure.ASOResourceSpecGetter[genruntime.MetaObject]
Err string
}{
{
Expand Down Expand Up @@ -472,7 +472,7 @@ func TestManagedControlPlaneScope_OSType(t *testing.T) {
},
},
},
Expected: []azure.ASOResourceSpecGetter[*asocontainerservicev1.ManagedClustersAgentPool]{
Expected: []azure.ASOResourceSpecGetter[genruntime.MetaObject]{
&agentpools.AgentPoolSpec{
Name: "pool0",
AzureName: "pool0",
Expand Down
12 changes: 9 additions & 3 deletions azure/scope/managedmachinepool.go
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,7 @@ import (
"fmt"
"strings"

asocontainerservicev1 "github.com/Azure/azure-service-operator/v2/api/containerservice/v1api20231001"
"github.com/Azure/azure-service-operator/v2/pkg/genruntime"
"github.com/pkg/errors"
"k8s.io/utils/ptr"
infrav1 "sigs.k8s.io/cluster-api-provider-azure/api/v1beta1"
Expand Down Expand Up @@ -146,7 +146,7 @@ func (s *ManagedMachinePoolScope) SetSubnetName() {
}

// AgentPoolSpec returns an azure.ResourceSpecGetter for currently reconciled AzureManagedMachinePool.
func (s *ManagedMachinePoolScope) AgentPoolSpec() azure.ASOResourceSpecGetter[*asocontainerservicev1.ManagedClustersAgentPool] {
func (s *ManagedMachinePoolScope) AgentPoolSpec() azure.ASOResourceSpecGetter[genruntime.MetaObject] {
return buildAgentPoolSpec(s.ControlPlane, s.MachinePool, s.InfraMachinePool)
}

Expand All @@ -159,7 +159,7 @@ func getAgentPoolSubnet(controlPlane *infrav1.AzureManagedControlPlane, infraMac

func buildAgentPoolSpec(managedControlPlane *infrav1.AzureManagedControlPlane,
machinePool *expv1.MachinePool,
managedMachinePool *infrav1.AzureManagedMachinePool) azure.ASOResourceSpecGetter[*asocontainerservicev1.ManagedClustersAgentPool] {
managedMachinePool *infrav1.AzureManagedMachinePool) azure.ASOResourceSpecGetter[genruntime.MetaObject] {
normalizedVersion := getManagedMachinePoolVersion(managedControlPlane, machinePool)

replicas := int32(1)
Expand Down Expand Up @@ -198,6 +198,7 @@ func buildAgentPoolSpec(managedControlPlane *infrav1.AzureManagedControlPlane,
EnableFIPS: managedMachinePool.Spec.EnableFIPS,
EnableEncryptionAtHost: managedMachinePool.Spec.EnableEncryptionAtHost,
Patches: managedMachinePool.Spec.ASOManagedClustersAgentPoolPatches,
Preview: ptr.Deref(managedControlPlane.Spec.EnablePreviewFeatures, false),
}

if managedMachinePool.Spec.OSDiskSizeGB != nil {
Expand Down Expand Up @@ -243,6 +244,11 @@ func buildAgentPoolSpec(managedControlPlane *infrav1.AzureManagedControlPlane,
return agentPoolSpec
}

// IsPreviewEnabled returns the value of the EnablePreviewFeatures field from the AzureManagedControlPlane.
func (s *ManagedMachinePoolScope) IsPreviewEnabled() bool {
return ptr.Deref(s.ControlPlane.Spec.EnablePreviewFeatures, false)
}

// SetAgentPoolProviderIDList sets a list of agent pool's Azure VM IDs.
func (s *ManagedMachinePoolScope) SetAgentPoolProviderIDList(providerIDs []string) {
s.InfraMachinePool.Spec.ProviderIDList = providerIDs
Expand Down
Loading