From e7a8f8059c4883bab35c6828a78d14a505ab49e6 Mon Sep 17 00:00:00 2001 From: njucz Date: Mon, 28 Jun 2021 23:35:32 -0500 Subject: [PATCH] new resource "azurerm_data_factory_linked_custom_service" (#12224) fix #9860 fix #9431 a generic resource for data factory linked service. Users could use a json string and construct a specific type linked service. the same with azure cli implementation docs.microsoft.com/en-us/cli/azure/datafactory/linked-service?view=azure-cli-latest#az_datafactory_linked_service_create there are some sensitive properties in property_json not returned in the response, so not set it in read function and no supressDiff func. --- ..._factory_linked_custom_service_resource.go | 335 ++++++++++++++++++ ...ory_linked_custom_service_resource_test.go | 320 +++++++++++++++++ .../services/datafactory/registration.go | 1 + ...actory_linked_custom_service.html.markdown | 112 ++++++ 4 files changed, 768 insertions(+) create mode 100644 azurerm/internal/services/datafactory/data_factory_linked_custom_service_resource.go create mode 100644 azurerm/internal/services/datafactory/data_factory_linked_custom_service_resource_test.go create mode 100644 website/docs/r/data_factory_linked_custom_service.html.markdown diff --git a/azurerm/internal/services/datafactory/data_factory_linked_custom_service_resource.go b/azurerm/internal/services/datafactory/data_factory_linked_custom_service_resource.go new file mode 100644 index 000000000000..1b6817b122fa --- /dev/null +++ b/azurerm/internal/services/datafactory/data_factory_linked_custom_service_resource.go @@ -0,0 +1,335 @@ +package datafactory + +import ( + "encoding/json" + "fmt" + "time" + + "github.com/Azure/azure-sdk-for-go/services/datafactory/mgmt/2018-06-01/datafactory" + "github.com/terraform-providers/terraform-provider-azurerm/azurerm/helpers/tf" + "github.com/terraform-providers/terraform-provider-azurerm/azurerm/internal/clients" + "github.com/terraform-providers/terraform-provider-azurerm/azurerm/internal/services/datafactory/parse" + "github.com/terraform-providers/terraform-provider-azurerm/azurerm/internal/services/datafactory/validate" + "github.com/terraform-providers/terraform-provider-azurerm/azurerm/internal/tf/pluginsdk" + "github.com/terraform-providers/terraform-provider-azurerm/azurerm/internal/tf/validation" + "github.com/terraform-providers/terraform-provider-azurerm/azurerm/internal/timeouts" + "github.com/terraform-providers/terraform-provider-azurerm/azurerm/utils" +) + +func resourceDataFactoryLinkedCustomService() *pluginsdk.Resource { + return &pluginsdk.Resource{ + Create: resourceDataFactoryLinkedCustomServiceCreateUpdate, + Read: resourceDataFactoryLinkedCustomServiceRead, + Update: resourceDataFactoryLinkedCustomServiceCreateUpdate, + Delete: resourceDataFactoryLinkedCustomServiceDelete, + + Importer: pluginsdk.ImporterValidatingResourceId(func(id string) error { + _, err := parse.LinkedServiceID(id) + return err + }), + + Timeouts: &pluginsdk.ResourceTimeout{ + Create: pluginsdk.DefaultTimeout(30 * time.Minute), + Read: pluginsdk.DefaultTimeout(5 * time.Minute), + Update: pluginsdk.DefaultTimeout(30 * time.Minute), + Delete: pluginsdk.DefaultTimeout(30 * time.Minute), + }, + + Schema: map[string]*pluginsdk.Schema{ + "name": { + Type: pluginsdk.TypeString, + Required: true, + ForceNew: true, + ValidateFunc: validate.LinkedServiceDatasetName, + }, + + "data_factory_id": { + Type: pluginsdk.TypeString, + Required: true, + ForceNew: true, + ValidateFunc: validate.DataFactoryID, + }, + + "type": { + Type: pluginsdk.TypeString, + Required: true, + ForceNew: true, + }, + + "type_properties_json": { + Type: pluginsdk.TypeString, + Required: true, + StateFunc: utils.NormalizeJson, + DiffSuppressFunc: suppressJsonOrderingDifference, + }, + + "description": { + Type: pluginsdk.TypeString, + Optional: true, + ValidateFunc: validation.StringIsNotEmpty, + }, + + "integration_runtime": { + Type: pluginsdk.TypeList, + Optional: true, + MaxItems: 1, + Elem: &pluginsdk.Resource{ + Schema: map[string]*pluginsdk.Schema{ + "name": { + Type: pluginsdk.TypeString, + Required: true, + ValidateFunc: validation.StringIsNotEmpty, + }, + + "parameters": { + Type: pluginsdk.TypeMap, + Optional: true, + Elem: &pluginsdk.Schema{ + Type: pluginsdk.TypeString, + }, + }, + }, + }, + }, + + "parameters": { + Type: pluginsdk.TypeMap, + Optional: true, + Elem: &pluginsdk.Schema{ + Type: pluginsdk.TypeString, + }, + }, + + "annotations": { + Type: pluginsdk.TypeList, + Optional: true, + Elem: &pluginsdk.Schema{ + Type: pluginsdk.TypeString, + }, + }, + + "additional_properties": { + Type: pluginsdk.TypeMap, + Optional: true, + Elem: &pluginsdk.Schema{ + Type: pluginsdk.TypeString, + }, + }, + }, + } +} + +func resourceDataFactoryLinkedCustomServiceCreateUpdate(d *pluginsdk.ResourceData, meta interface{}) error { + client := meta.(*clients.Client).DataFactory.LinkedServiceClient + subscriptionId := meta.(*clients.Client).Account.SubscriptionId + ctx, cancel := timeouts.ForCreateUpdate(meta.(*clients.Client).StopContext, d) + defer cancel() + + dataFactoryId, err := parse.DataFactoryID(d.Get("data_factory_id").(string)) + if err != nil { + return err + } + + id := parse.NewLinkedServiceID(subscriptionId, dataFactoryId.ResourceGroup, dataFactoryId.FactoryName, d.Get("name").(string)) + if d.IsNewResource() { + existing, err := client.Get(ctx, id.ResourceGroup, id.FactoryName, id.Name, "") + if err != nil { + if !utils.ResponseWasNotFound(existing.Response) { + return fmt.Errorf("checking for presence of existing %s: %+v", id, err) + } + } + if !utils.ResponseWasNotFound(existing.Response) { + return tf.ImportAsExistsError("azurerm_data_factory_linked_custom_service", id.ID()) + } + } + + props := map[string]interface{}{ + "type": d.Get("type").(string), + "connectVia": expandDataFactoryLinkedServiceIntegrationRuntimeV2(d.Get("integration_runtime").([]interface{})), + } + + jsonDataStr := fmt.Sprintf(`{ "typeProperties": %s }`, d.Get("type_properties_json").(string)) + if err = json.Unmarshal([]byte(jsonDataStr), &props); err != nil { + return err + } + + if v, ok := d.GetOk("description"); ok { + props["description"] = v.(string) + } + + if v, ok := d.GetOk("parameters"); ok { + props["parameters"] = expandDataFactoryParameters(v.(map[string]interface{})) + } + + if v, ok := d.GetOk("annotations"); ok { + props["annotations"] = v.([]interface{}) + } + + additionalProperties := d.Get("additional_properties").(map[string]interface{}) + for k, v := range additionalProperties { + props[k] = v + } + + jsonData, err := json.Marshal(map[string]interface{}{ + "properties": props, + }) + if err != nil { + return err + } + + linkedService := &datafactory.LinkedServiceResource{} + if err := linkedService.UnmarshalJSON(jsonData); err != nil { + return err + } + + if _, err := client.CreateOrUpdate(ctx, id.ResourceGroup, id.FactoryName, id.Name, *linkedService, ""); err != nil { + return fmt.Errorf("creating/updating %s: %+v", id, err) + } + + d.SetId(id.ID()) + + return resourceDataFactoryLinkedCustomServiceRead(d, meta) +} + +func resourceDataFactoryLinkedCustomServiceRead(d *pluginsdk.ResourceData, meta interface{}) error { + client := meta.(*clients.Client).DataFactory.LinkedServiceClient + subscriptionId := meta.(*clients.Client).Account.SubscriptionId + ctx, cancel := timeouts.ForRead(meta.(*clients.Client).StopContext, d) + defer cancel() + + id, err := parse.LinkedServiceID(d.Id()) + if err != nil { + return err + } + + resp, err := client.Get(ctx, id.ResourceGroup, id.FactoryName, id.Name, "") + if err != nil { + if utils.ResponseWasNotFound(resp.Response) { + d.SetId("") + return nil + } + + return fmt.Errorf("retrieving %s: %+v", id, err) + } + + d.Set("name", id.Name) + d.Set("data_factory_id", parse.NewDataFactoryID(subscriptionId, id.ResourceGroup, id.FactoryName).ID()) + + byteArr, err := json.Marshal(resp.Properties) + if err != nil { + return err + } + + var m map[string]*json.RawMessage + if err = json.Unmarshal(byteArr, &m); err != nil { + return err + } + + description := "" + if v, ok := m["description"]; ok && v != nil { + if err := json.Unmarshal(*v, &description); err != nil { + return err + } + delete(m, "description") + } + d.Set("description", description) + + t := "" + if v, ok := m["type"]; ok && v != nil { + if err := json.Unmarshal(*v, &t); err != nil { + return err + } + delete(m, "type") + } + d.Set("type", t) + + annotations := make([]interface{}, 0) + if v, ok := m["annotations"]; ok && v != nil { + if err := json.Unmarshal(*v, &annotations); err != nil { + return err + } + delete(m, "annotations") + } + d.Set("annotations", annotations) + + parameters := make(map[string]*datafactory.ParameterSpecification) + if v, ok := m["parameters"]; ok && v != nil { + if err := json.Unmarshal(*v, ¶meters); err != nil { + return err + } + delete(m, "parameters") + } + if err := d.Set("parameters", flattenDataFactoryParameters(parameters)); err != nil { + return fmt.Errorf("setting `parameters`: %+v", err) + } + + var integrationRuntime *datafactory.IntegrationRuntimeReference + if v, ok := m["connectVia"]; ok && v != nil { + integrationRuntime = &datafactory.IntegrationRuntimeReference{} + if err := json.Unmarshal(*v, &integrationRuntime); err != nil { + return err + } + delete(m, "connectVia") + } + if err := d.Set("integration_runtime", flattenDataFactoryLinkedServiceIntegrationRuntimeV2(integrationRuntime)); err != nil { + return fmt.Errorf("setting `integration_runtime`: %+v", err) + } + + delete(m, "typeProperties") + + additionalProperties := make(map[string]interface{}) + for k, v := range m { + additionalProperties[k] = v + } + d.Set("additional_properties", additionalProperties) + + return nil +} + +func resourceDataFactoryLinkedCustomServiceDelete(d *pluginsdk.ResourceData, meta interface{}) error { + client := meta.(*clients.Client).DataFactory.LinkedServiceClient + ctx, cancel := timeouts.ForDelete(meta.(*clients.Client).StopContext, d) + defer cancel() + + id, err := parse.LinkedServiceID(d.Id()) + if err != nil { + return err + } + + if _, err := client.Delete(ctx, id.ResourceGroup, id.FactoryName, id.Name); err != nil { + return fmt.Errorf("deleting %s: %+v", id, err) + } + + return nil +} + +func expandDataFactoryLinkedServiceIntegrationRuntimeV2(input []interface{}) *datafactory.IntegrationRuntimeReference { + if len(input) == 0 || input[0] == nil { + return nil + } + + v := input[0].(map[string]interface{}) + return &datafactory.IntegrationRuntimeReference{ + ReferenceName: utils.String(v["name"].(string)), + Type: utils.String("IntegrationRuntimeReference"), + Parameters: v["parameters"].(map[string]interface{}), + } +} + +func flattenDataFactoryLinkedServiceIntegrationRuntimeV2(input *datafactory.IntegrationRuntimeReference) []interface{} { + if input == nil { + return []interface{}{} + } + + name := "" + if input.ReferenceName != nil { + name = *input.ReferenceName + } + + return []interface{}{ + map[string]interface{}{ + "name": name, + "parameters": input.Parameters, + }, + } +} diff --git a/azurerm/internal/services/datafactory/data_factory_linked_custom_service_resource_test.go b/azurerm/internal/services/datafactory/data_factory_linked_custom_service_resource_test.go new file mode 100644 index 000000000000..1369fbbb564d --- /dev/null +++ b/azurerm/internal/services/datafactory/data_factory_linked_custom_service_resource_test.go @@ -0,0 +1,320 @@ +package datafactory_test + +import ( + "context" + "fmt" + "testing" + + "github.com/terraform-providers/terraform-provider-azurerm/azurerm/internal/acceptance" + "github.com/terraform-providers/terraform-provider-azurerm/azurerm/internal/acceptance/check" + "github.com/terraform-providers/terraform-provider-azurerm/azurerm/internal/clients" + "github.com/terraform-providers/terraform-provider-azurerm/azurerm/internal/services/datafactory/parse" + "github.com/terraform-providers/terraform-provider-azurerm/azurerm/internal/tf/pluginsdk" + "github.com/terraform-providers/terraform-provider-azurerm/azurerm/utils" +) + +type LinkedCustomServiceResource struct { +} + +func TestAccDataFactoryLinkedCustomService_basic(t *testing.T) { + data := acceptance.BuildTestData(t, "azurerm_data_factory_linked_custom_service", "test") + r := LinkedCustomServiceResource{} + + data.ResourceTest(t, r, []acceptance.TestStep{ + { + Config: r.basic(data), + Check: acceptance.ComposeTestCheckFunc( + check.That(data.ResourceName).ExistsInAzure(r), + ), + }, + data.ImportStep("type_properties_json"), + }) +} + +func TestAccDataFactoryLinkedCustomService_requiresImport(t *testing.T) { + data := acceptance.BuildTestData(t, "azurerm_data_factory_linked_custom_service", "test") + r := LinkedCustomServiceResource{} + + data.ResourceTest(t, r, []acceptance.TestStep{ + { + Config: r.basic(data), + Check: acceptance.ComposeTestCheckFunc( + check.That(data.ResourceName).ExistsInAzure(r), + ), + }, + data.RequiresImportErrorStep(r.requiresImport), + }) +} + +func TestAccDataFactoryLinkedCustomService_complete(t *testing.T) { + data := acceptance.BuildTestData(t, "azurerm_data_factory_linked_custom_service", "test") + r := LinkedCustomServiceResource{} + + data.ResourceTest(t, r, []acceptance.TestStep{ + { + Config: r.complete(data), + Check: acceptance.ComposeTestCheckFunc( + check.That(data.ResourceName).ExistsInAzure(r), + ), + }, + data.ImportStep("type_properties_json"), + }) +} + +func TestAccDataFactoryLinkedCustomService_update(t *testing.T) { + data := acceptance.BuildTestData(t, "azurerm_data_factory_linked_custom_service", "test") + r := LinkedCustomServiceResource{} + + data.ResourceTest(t, r, []acceptance.TestStep{ + { + Config: r.basic(data), + Check: acceptance.ComposeTestCheckFunc( + check.That(data.ResourceName).ExistsInAzure(r), + ), + }, + data.ImportStep("type_properties_json"), + { + Config: r.complete(data), + Check: acceptance.ComposeTestCheckFunc( + check.That(data.ResourceName).ExistsInAzure(r), + ), + }, + data.ImportStep("type_properties_json"), + { + Config: r.basic(data), + Check: acceptance.ComposeTestCheckFunc( + check.That(data.ResourceName).ExistsInAzure(r), + ), + }, + data.ImportStep("type_properties_json"), + }) +} + +func TestAccDataFactoryLinkedCustomService_web(t *testing.T) { + data := acceptance.BuildTestData(t, "azurerm_data_factory_linked_custom_service", "test") + r := LinkedCustomServiceResource{} + + data.ResourceTest(t, r, []acceptance.TestStep{ + { + Config: r.web(data), + Check: acceptance.ComposeTestCheckFunc( + check.That(data.ResourceName).ExistsInAzure(r), + ), + }, + data.ImportStep("type_properties_json"), + }) +} + +func TestAccDataFactoryLinkedCustomService_search(t *testing.T) { + data := acceptance.BuildTestData(t, "azurerm_data_factory_linked_custom_service", "test") + r := LinkedCustomServiceResource{} + + data.ResourceTest(t, r, []acceptance.TestStep{ + { + Config: r.search(data), + Check: acceptance.ComposeTestCheckFunc( + check.That(data.ResourceName).ExistsInAzure(r), + ), + }, + data.ImportStep("type_properties_json"), + }) +} + +func (t LinkedCustomServiceResource) Exists(ctx context.Context, clients *clients.Client, state *pluginsdk.InstanceState) (*bool, error) { + id, err := parse.LinkedServiceID(state.ID) + if err != nil { + return nil, err + } + + resp, err := clients.DataFactory.LinkedServiceClient.Get(ctx, id.ResourceGroup, id.FactoryName, id.Name, "") + if err != nil { + return nil, fmt.Errorf("reading %s: %+v", id, err) + } + + return utils.Bool(resp.ID != nil), nil +} + +func (r LinkedCustomServiceResource) basic(data acceptance.TestData) string { + return fmt.Sprintf(` +%s + +resource "azurerm_data_factory_linked_custom_service" "test" { + name = "acctestls%d" + data_factory_id = azurerm_data_factory.test.id + type = "AzureBlobStorage" + type_properties_json = <