storage - allow azurerm_storage_account to be used in Data Plane …

…restrictive environments (#27818)

* add feature flag for toggling storage data plane availability

* add dataplane tolerance to storage account.
matches features being back-ported to 3.x to allow users to migrate more easily.

* add fivepointoh feature flagging

* customise diff major release feature flagging

* make generate

* use correct val for set in create for deprecated blocks

* changes from review

* restore return read

.github/labeler-issue-triage.yml

@@ -312,7 +312,7 @@ service/spring:
   - '### (|New or )Affected Resource\(s\)\/Data Source\(s\)((.|\n)*)azurerm_(spring_cloud_accelerator\W+|spring_cloud_active_deployment\W+|spring_cloud_api_portal\W+|spring_cloud_api_portal_custom_domain\W+|spring_cloud_app\W+|spring_cloud_app_cosmosdb_association\W+|spring_cloud_app_dynamics_application_performance_monitoring\W+|spring_cloud_app_mysql_association\W+|spring_cloud_app_redis_association\W+|spring_cloud_application_insights_application_performance_monitoring\W+|spring_cloud_application_live_view\W+|spring_cloud_build_deployment\W+|spring_cloud_build_pack_binding\W+|spring_cloud_builder\W+|spring_cloud_certificate\W+|spring_cloud_configuration_service\W+|spring_cloud_container_deployment\W+|spring_cloud_custom_domain\W+|spring_cloud_customized_accelerator\W+|spring_cloud_dev_tool_portal\W+|spring_cloud_dynatrace_application_performance_monitoring\W+|spring_cloud_elastic_application_performance_monitoring\W+|spring_cloud_gateway\W+|spring_cloud_gateway_custom_domain\W+|spring_cloud_gateway_route_config\W+|spring_cloud_java_deployment\W+|spring_cloud_new_relic_application_performance_monitoring\W+|spring_cloud_service\W+|spring_cloud_storage\W+)((.|\n)*)###'
 
 service/storage:
-  - '### (|New or )Affected Resource\(s\)\/Data Source\(s\)((.|\n)*)azurerm_(storage_account\W+|storage_account_blob_container_sas\W+|storage_account_customer_managed_key\W+|storage_account_local_user\W+|storage_account_network_rules\W+|storage_account_sas\W+|storage_blob\W+|storage_blob_inventory_policy\W+|storage_container\W+|storage_container_immutability_policy\W+|storage_containers\W+|storage_data_lake_gen2_filesystem\W+|storage_data_lake_gen2_path\W+|storage_encryption_scope\W+|storage_management_policy\W+|storage_object_replication\W+|storage_queue\W+|storage_share\W+|storage_share_directory\W+|storage_share_file\W+|storage_sync\W+|storage_sync_cloud_endpoint\W+|storage_sync_group\W+|storage_sync_server_endpoint\W+|storage_table\W+|storage_table\W+|storage_table_entities\W+|storage_table_entity\W+)((.|\n)*)###'
+  - '### (|New or )Affected Resource\(s\)\/Data Source\(s\)((.|\n)*)azurerm_(storage_account\W+|storage_account_blob_container_sas\W+|storage_account_customer_managed_key\W+|storage_account_local_user\W+|storage_account_network_rules\W+|storage_account_queue_properties\W+|storage_account_sas\W+|storage_account_static_website\W+|storage_blob\W+|storage_blob_inventory_policy\W+|storage_container\W+|storage_container_immutability_policy\W+|storage_containers\W+|storage_data_lake_gen2_filesystem\W+|storage_data_lake_gen2_path\W+|storage_encryption_scope\W+|storage_management_policy\W+|storage_object_replication\W+|storage_queue\W+|storage_share\W+|storage_share_directory\W+|storage_share_file\W+|storage_sync\W+|storage_sync_cloud_endpoint\W+|storage_sync_group\W+|storage_sync_server_endpoint\W+|storage_table\W+|storage_table\W+|storage_table_entities\W+|storage_table_entity\W+)((.|\n)*)###'
 
 service/storagemover:
   - '### (|New or )Affected Resource\(s\)\/Data Source\(s\)((.|\n)*)azurerm_storage_mover((.|\n)*)###'

internal/features/defaults.go

@@ -62,6 +62,9 @@ func Default() UserFeatures {
 			RollInstancesWhenRequired: true,
 			ScaleToZeroOnDelete:       true,
 		},
+		Storage: StorageFeatures{
+			DataPlaneAvailable: true,
+		},
 		Subscription: SubscriptionFeatures{
 			PreventCancellationOnDestroy: false,
 		},

internal/features/user_flags.go

@@ -16,6 +16,7 @@ type UserFeatures struct {
 	ResourceGroup            ResourceGroupFeatures
 	RecoveryServicesVault    RecoveryServicesVault
 	ManagedDisk              ManagedDiskFeatures
+	Storage                  StorageFeatures
 	Subscription             SubscriptionFeatures
 	PostgresqlFlexibleServer PostgresqlFlexibleServerFeatures
 	MachineLearning          MachineLearningFeatures
@@ -84,6 +85,10 @@ type AppConfigurationFeatures struct {
 	RecoverSoftDeleted       bool
 }
 
+type StorageFeatures struct {
+	DataPlaneAvailable bool
+}
+
 type SubscriptionFeatures struct {
 	PreventCancellationOnDestroy bool
 }

internal/provider/features.go

@@ -309,6 +309,21 @@ func schemaFeatures(supportLegacyTestSuite bool) *pluginsdk.Schema {
 			},
 		},
 
+		"storage": {
+			Type:     pluginsdk.TypeList,
+			Optional: true,
+			MaxItems: 1,
+			Elem: &pluginsdk.Resource{
+				Schema: map[string]*schema.Schema{
+					"data_plane_available": {
+						Type:     pluginsdk.TypeBool,
+						Optional: true,
+						Default:  true,
+					},
+				},
+			},
+		},
+
 		"subscription": {
 			Type:     pluginsdk.TypeList,
 			Optional: true,
@@ -580,6 +595,15 @@ func expandFeatures(input []interface{}) features.UserFeatures {
 			}
 		}
 	}
+	if raw, ok := val["storage"]; ok {
+		items := raw.([]interface{})
+		if len(items) > 0 {
+			storageRaw := items[0].(map[string]interface{})
+			if v, ok := storageRaw["data_plane_available"]; ok {
+				featuresMap.Storage.DataPlaneAvailable = v.(bool)
+			}
+		}
+	}
 
 	if raw, ok := val["subscription"]; ok {
 		items := raw.([]interface{})

internal/provider/features_test.go

@@ -75,6 +75,9 @@ func TestExpandFeatures(t *testing.T) {
 				RecoveryServicesVault: features.RecoveryServicesVault{
 					RecoverSoftDeletedBackupProtectedVM: true,
 				},
+				Storage: features.StorageFeatures{
+					DataPlaneAvailable: true,
+				},
 				Subscription: features.SubscriptionFeatures{
 					PreventCancellationOnDestroy: false,
 				},
@@ -156,6 +159,11 @@ func TestExpandFeatures(t *testing.T) {
 							"recover_soft_deleted_backup_protected_vm": true,
 						},
 					},
+					"storage": []interface{}{
+						map[string]interface{}{
+							"data_plane_available": true,
+						},
+					},
 					"subscription": []interface{}{
 						map[string]interface{}{
 							"prevent_cancellation_on_destroy": true,
@@ -235,6 +243,9 @@ func TestExpandFeatures(t *testing.T) {
 				RecoveryServicesVault: features.RecoveryServicesVault{
 					RecoverSoftDeletedBackupProtectedVM: true,
 				},
+				Storage: features.StorageFeatures{
+					DataPlaneAvailable: true,
+				},
 				Subscription: features.SubscriptionFeatures{
 					PreventCancellationOnDestroy: true,
 				},
@@ -331,6 +342,11 @@ func TestExpandFeatures(t *testing.T) {
 							"recover_soft_deleted_backup_protected_vm": false,
 						},
 					},
+					"storage": []interface{}{
+						map[string]interface{}{
+							"data_plane_available": false,
+						},
+					},
 					"subscription": []interface{}{
 						map[string]interface{}{
 							"prevent_cancellation_on_destroy": false,
@@ -410,6 +426,9 @@ func TestExpandFeatures(t *testing.T) {
 				RecoveryServicesVault: features.RecoveryServicesVault{
 					RecoverSoftDeletedBackupProtectedVM: false,
 				},
+				Storage: features.StorageFeatures{
+					DataPlaneAvailable: false,
+				},
 				Subscription: features.SubscriptionFeatures{
 					PreventCancellationOnDestroy: false,
 				},
@@ -1431,6 +1450,54 @@ func TestExpandFeaturesManagedDisk(t *testing.T) {
 	}
 }
 
+func TestExpandFeaturesStorage(t *testing.T) {
+	testData := []struct {
+		Name     string
+		Input    []interface{}
+		EnvVars  map[string]interface{}
+		Expected features.UserFeatures
+	}{
+		{
+			Name: "Empty Block",
+			Input: []interface{}{
+				map[string]interface{}{
+					"storage": []interface{}{},
+				},
+			},
+			Expected: features.UserFeatures{
+				Storage: features.StorageFeatures{
+					DataPlaneAvailable: true,
+				},
+			},
+		},
+		{
+			Name: "Storage Data Plane on Create is Disabled",
+			Input: []interface{}{
+				map[string]interface{}{
+					"storage": []interface{}{
+						map[string]interface{}{
+							"data_plane_available": false,
+						},
+					},
+				},
+			},
+			Expected: features.UserFeatures{
+				Storage: features.StorageFeatures{
+					DataPlaneAvailable: false,
+				},
+			},
+		},
+	}
+
+	for _, testCase := range testData {
+		t.Logf("[DEBUG] Test Case: %q", testCase.Name)
+		result := expandFeatures(testCase.Input)
+		if !reflect.DeepEqual(result.Storage, testCase.Expected.Storage) {
+			t.Fatalf("Expected %+v but got %+v", result.Storage, testCase.Expected.Storage)
+		}
+	}
+}
+
 func TestExpandFeaturesSubscription(t *testing.T) {
 	testData := []struct {
 		Name     string

internal/provider/framework/config.go

@@ -415,6 +415,19 @@ func (p *ProviderConfig) Load(ctx context.Context, data *ProviderModel, tfVersio
 			f.ManagedDisk.ExpandWithoutDowntime = true
 		}
 
+		if !features.Storage.IsNull() && !features.Storage.IsUnknown() {
+			var feature []Storage
+			d := features.Storage.ElementsAs(ctx, &feature, true)
+			diags.Append(d...)
+			if diags.HasError() {
+				return
+			}
+			f.Storage.DataPlaneAvailable = true
+			if !feature[0].DataPlaneAvailable.IsNull() && !feature[0].DataPlaneAvailable.IsUnknown() {
+				f.Storage.DataPlaneAvailable = feature[0].DataPlaneAvailable.ValueBool()
+			}
+		}
+
 		if !features.Subscription.IsNull() && !features.Subscription.IsUnknown() {
 			var feature []Subscription
 			d := features.Subscription.ElementsAs(ctx, &feature, true)

internal/provider/framework/config_test.go

@@ -275,6 +275,11 @@ func defaultFeaturesList() types.List {
 	})
 	managedDiskList, _ := basetypes.NewListValue(types.ObjectType{}.WithAttributeTypes(ManagedDiskAttributes), []attr.Value{managedDisk})
 
+	storage, _ := basetypes.NewObjectValueFrom(context.Background(), StorageAttributes, map[string]attr.Value{
+		"data_plane_available": basetypes.NewBoolNull(),
+	})
+	storageList, _ := basetypes.NewListValue(types.ObjectType{}.WithAttributeTypes(StorageAttributes), []attr.Value{storage})
+
 	subscription, _ := basetypes.NewObjectValueFrom(context.Background(), SubscriptionAttributes, map[string]attr.Value{
 		"prevent_cancellation_on_destroy": basetypes.NewBoolNull(),
 	})
@@ -314,6 +319,7 @@ func defaultFeaturesList() types.List {
 		"virtual_machine_scale_set":  virtualMachineScaleSetList,
 		"resource_group":             resourceGroupList,
 		"managed_disk":               managedDiskList,
+		"storage":                    storageList,
 		"subscription":               subscriptionList,
 		"postgresql_flexible_server": postgresqlFlexibleServerList,
 		"machine_learning":           machineLearningList,

internal/provider/framework/model.go

@@ -52,6 +52,7 @@ type Features struct {
 	VirtualMachineScaleSet   types.List `tfsdk:"virtual_machine_scale_set"`
 	ResourceGroup            types.List `tfsdk:"resource_group"`
 	ManagedDisk              types.List `tfsdk:"managed_disk"`
+	Storage                  types.List `tfsdk:"storage"`
 	Subscription             types.List `tfsdk:"subscription"`
 	PostgresqlFlexibleServer types.List `tfsdk:"postgresql_flexible_server"`
 	MachineLearning          types.List `tfsdk:"machine_learning"`
@@ -73,6 +74,7 @@ var FeaturesAttributes = map[string]attr.Type{
 	"virtual_machine_scale_set":  types.ListType{}.WithElementType(types.ObjectType{}.WithAttributeTypes(VirtualMachineScaleSetAttributes)),
 	"resource_group":             types.ListType{}.WithElementType(types.ObjectType{}.WithAttributeTypes(ResourceGroupAttributes)),
 	"managed_disk":               types.ListType{}.WithElementType(types.ObjectType{}.WithAttributeTypes(ManagedDiskAttributes)),
+	"storage":                    types.ListType{}.WithElementType(types.ObjectType{}.WithAttributeTypes(StorageAttributes)),
 	"subscription":               types.ListType{}.WithElementType(types.ObjectType{}.WithAttributeTypes(SubscriptionAttributes)),
 	"postgresql_flexible_server": types.ListType{}.WithElementType(types.ObjectType{}.WithAttributeTypes(PostgresqlFlexibleServerAttributes)),
 	"machine_learning":           types.ListType{}.WithElementType(types.ObjectType{}.WithAttributeTypes(MachineLearningAttributes)),
@@ -204,6 +206,14 @@ var ManagedDiskAttributes = map[string]attr.Type{
 	"expand_without_downtime": types.BoolType,
 }
 
+type Storage struct {
+	DataPlaneAvailable types.Bool `tfsdk:"data_plane_available"`
+}
+
+var StorageAttributes = map[string]attr.Type{
+	"data_plane_available": types.BoolType,
+}
+
 type Subscription struct {
 	PreventCancellationOnDestroy types.Bool `tfsdk:"prevent_cancellation_on_destroy"`
 }

internal/provider/framework/provider.go

@@ -406,6 +406,15 @@ func (p *azureRmFrameworkProvider) Schema(_ context.Context, _ provider.SchemaRe
 								},
 							},
 						},
+						"storage": schema.ListNestedBlock{
+							NestedObject: schema.NestedBlockObject{
+								Attributes: map[string]schema.Attribute{
+									"data_plane_available": schema.BoolAttribute{
+										Optional: true,
+									},
+								},
+							},
+						},
 						"subscription": schema.ListNestedBlock{
 							NestedObject: schema.NestedBlockObject{
 								Attributes: map[string]schema.Attribute{

internal/services/storage/custompollers/data_plane_blob_containers_availability_poller.go

@@ -35,6 +35,11 @@ func NewDataPlaneBlobContainersAvailabilityPoller(ctx context.Context, client *s
 func (d *DataPlaneBlobContainersAvailabilityPoller) Poll(ctx context.Context) (*pollers.PollResult, error) {
 	resp, err := d.client.GetServiceProperties(ctx, d.accountName)
 	if err != nil {
+		if resp.HttpResponse == nil {
+			return nil, pollers.PollingDroppedConnectionError{
+				Message: err.Error(),
+			}
+		}
 		if !response.WasNotFound(resp.HttpResponse) {
 			return nil, pollers.PollingFailedError{
 				Message: err.Error(),

internal/services/storage/custompollers/data_plane_queues_availability_poller.go

@@ -5,6 +5,7 @@ package custompollers
 
 import (
 	"context"
+	"errors"
 	"fmt"
 	"time"
 
@@ -35,6 +36,10 @@ func NewDataPlaneQueuesAvailabilityPoller(ctx context.Context, client *storageCl
 
 func (d *DataPlaneQueuesAvailabilityPoller) Poll(ctx context.Context) (*pollers.PollResult, error) {
 	resp, err := d.client.GetServiceProperties(ctx)
+	var e pollers.PollingDroppedConnectionError
+	if errors.As(err, &e) {
+		return nil, err
+	}
 	if err != nil {
 		return nil, pollers.PollingFailedError{
 			Message:      err.Error(),

internal/services/storage/custompollers/data_plane_static_website_availability_poller.go

@@ -39,6 +39,11 @@ func (d *DataPlaneStaticWebsiteAvailabilityPoller) Poll(ctx context.Context) (*p
 	resp, err := d.client.GetServiceProperties(ctx, d.storageAccountId.StorageAccountName)
 	if err != nil {
 		if !response.WasNotFound(resp.HttpResponse) {
+			if resp.HttpResponse == nil {
+				return nil, pollers.PollingDroppedConnectionError{
+					Message: err.Error(),
+				}
+			}
 			return nil, pollers.PollingFailedError{
 				Message: err.Error(),
 				HttpResponse: &client.Response{

internal/services/storage/registration.go

@@ -82,6 +82,8 @@ func (r Registration) DataSources() []sdk.DataSource {
 
 func (r Registration) Resources() []sdk.Resource {
 	return []sdk.Resource{
+		AccountQueuePropertiesResource{},
+		AccountStaticWebsiteResource{},
 		LocalUserResource{},
 		StorageContainerImmutabilityPolicyResource{},
 		SyncServerEndpointResource{},

internal/services/storage/shim/queues_data_plane.go

@@ -8,6 +8,7 @@ import (
 
 	"github.com/hashicorp/go-azure-helpers/lang/pointer"
 	"github.com/hashicorp/go-azure-helpers/lang/response"
+	"github.com/hashicorp/go-azure-sdk/sdk/client/pollers"
 	"github.com/tombuildsstuff/giovanni/storage/2023-11-03/queue/queues"
 )
 
@@ -62,6 +63,11 @@ func (w DataPlaneStorageQueueWrapper) Get(ctx context.Context, queueName string)
 func (w DataPlaneStorageQueueWrapper) GetServiceProperties(ctx context.Context) (*queues.StorageServiceProperties, error) {
 	serviceProps, err := w.client.GetServiceProperties(ctx)
 	if err != nil {
+		if serviceProps.HttpResponse == nil {
+			return nil, pollers.PollingDroppedConnectionError{
+				Message: err.Error(),
+			}
+		}
 		if response.WasNotFound(serviceProps.HttpResponse) {
 			return nil, nil
 		}

internal/services/storage/shim/shares_data_plane.go

@@ -9,6 +9,7 @@ import (
 
 	"github.com/hashicorp/go-azure-helpers/lang/pointer"
 	"github.com/hashicorp/go-azure-helpers/lang/response"
+	"github.com/hashicorp/go-azure-sdk/sdk/client/pollers"
 	"github.com/tombuildsstuff/giovanni/storage/2023-11-03/file/shares"
 )
 
@@ -51,6 +52,11 @@ func (w DataPlaneStorageShareWrapper) Exists(ctx context.Context, shareName stri
 func (w DataPlaneStorageShareWrapper) Get(ctx context.Context, shareName string) (*StorageShareProperties, error) {
 	props, err := w.client.GetProperties(ctx, shareName)
 	if err != nil {
+		if props.HttpResponse == nil {
+			return nil, pollers.PollingDroppedConnectionError{
+				Message: err.Error(),
+			}
+		}
 		if response.WasNotFound(props.HttpResponse) {
 			return nil, nil
 		}