From 4f770a0c72ee88acb87ff287b1e0daa6e301d60f Mon Sep 17 00:00:00 2001 From: Arnab <45350738+c0d3r-arnab@users.noreply.github.com> Date: Wed, 29 Sep 2021 17:02:35 +0530 Subject: [PATCH] Add developer_portal_url, disable_gateway, enable_client_certificate, api_version_constraint, certificates, custom_properties, identity_user_assigned_identities, virtual_network_type, restore, scm_url, zones and diagnostic_settings columns to the table azure_api_management. Closes #292 (#336) --- .../test-get-expected.json | 8 +- .../azure_api_management/test-get-query.sql | 2 +- .../test-hydrate-expected.json | 4 +- .../test-hydrate-query.sql | 2 +- .../test-list-expected.json | 4 +- .../azure_api_management/test-list-query.sql | 2 +- .../test-not-found-expected.json | 2 +- .../test-not-found-query.sql | 2 +- .../test-turbot-expected.json | 9 +- .../test-turbot-query.sql | 4 +- .../tests/azure_api_management/variables.tf | 78 +++---- azure/table_azure_api_management.go | 207 ++++++++++++++---- 12 files changed, 212 insertions(+), 112 deletions(-) diff --git a/azure-test/tests/azure_api_management/test-get-expected.json b/azure-test/tests/azure_api_management/test-get-expected.json index faed0fc4..9898051d 100644 --- a/azure-test/tests/azure_api_management/test-get-expected.json +++ b/azure-test/tests/azure_api_management/test-get-expected.json @@ -1,12 +1,12 @@ [ { "id": "{{ output.resource_id.value }}", - "name": "{{resourceName}}", + "name": "{{ resourceName }}", "publisher_email": "test@turbot.com", "publisher_name": "TurbotHQ", - "region": "{{ output.location.value }}", + "region": "{{ output.region.value }}", "sku_capacity": 1, - "sku_name": "Developer_1", + "sku_name": "Developer", "type": "Microsoft.ApiManagement/service" } -] \ No newline at end of file +] diff --git a/azure-test/tests/azure_api_management/test-get-query.sql b/azure-test/tests/azure_api_management/test-get-query.sql index 0ad4dcf4..cc6b4f66 100644 --- a/azure-test/tests/azure_api_management/test-get-query.sql +++ b/azure-test/tests/azure_api_management/test-get-query.sql @@ -1,3 +1,3 @@ select name, id, region, type, publisher_email, publisher_name, sku_capacity, sku_name from azure.azure_api_management -where name = '{{resourceName}}' and resource_group = '{{ output.resource_group_name.value }}' +where name = '{{ resourceName }}' and resource_group = '{{ resourceName }}'; diff --git a/azure-test/tests/azure_api_management/test-hydrate-expected.json b/azure-test/tests/azure_api_management/test-hydrate-expected.json index d2d90152..72568a88 100644 --- a/azure-test/tests/azure_api_management/test-hydrate-expected.json +++ b/azure-test/tests/azure_api_management/test-hydrate-expected.json @@ -1,6 +1,6 @@ [ { - "name": "{{resourceName}}", + "name": "{{ resourceName }}", "sku_name": "Developer" } -] \ No newline at end of file +] diff --git a/azure-test/tests/azure_api_management/test-hydrate-query.sql b/azure-test/tests/azure_api_management/test-hydrate-query.sql index 27c49d8d..5fa6197b 100644 --- a/azure-test/tests/azure_api_management/test-hydrate-query.sql +++ b/azure-test/tests/azure_api_management/test-hydrate-query.sql @@ -1,3 +1,3 @@ select name, sku_name from azure.azure_api_management -where name = '{{resourceName}}' and resource_group = '{{ output.resource_group_name.value }}' +where name = '{{ resourceName }}' and resource_group = '{{ resourceName }}'; diff --git a/azure-test/tests/azure_api_management/test-list-expected.json b/azure-test/tests/azure_api_management/test-list-expected.json index db8cea80..e854f350 100644 --- a/azure-test/tests/azure_api_management/test-list-expected.json +++ b/azure-test/tests/azure_api_management/test-list-expected.json @@ -1,6 +1,6 @@ [ { "id": "{{ output.resource_id.value }}", - "name": "{{resourceName}}" + "name": "{{ resourceName }}" } -] \ No newline at end of file +] diff --git a/azure-test/tests/azure_api_management/test-list-query.sql b/azure-test/tests/azure_api_management/test-list-query.sql index f0dd511e..6495cd79 100644 --- a/azure-test/tests/azure_api_management/test-list-query.sql +++ b/azure-test/tests/azure_api_management/test-list-query.sql @@ -1,3 +1,3 @@ select name, id from azure.azure_api_management -where name = '{{resourceName}}' +where id = '{{ output.resource_id.value }}'; diff --git a/azure-test/tests/azure_api_management/test-not-found-expected.json b/azure-test/tests/azure_api_management/test-not-found-expected.json index ec747fa4..19765bd5 100644 --- a/azure-test/tests/azure_api_management/test-not-found-expected.json +++ b/azure-test/tests/azure_api_management/test-not-found-expected.json @@ -1 +1 @@ -null \ No newline at end of file +null diff --git a/azure-test/tests/azure_api_management/test-not-found-query.sql b/azure-test/tests/azure_api_management/test-not-found-query.sql index eb347ee5..ef266424 100644 --- a/azure-test/tests/azure_api_management/test-not-found-query.sql +++ b/azure-test/tests/azure_api_management/test-not-found-query.sql @@ -1,3 +1,3 @@ select name, id, region, type, publisher_email, publisher_name, sku_capacity from azure.azure_api_management -where name = '{{resourceName}}' and resource_group = 'dummy-{{resourceName}}' +where name = '{{ resourceName }}' and resource_group = 'dummy-{{ resourceName }}'; diff --git a/azure-test/tests/azure_api_management/test-turbot-expected.json b/azure-test/tests/azure_api_management/test-turbot-expected.json index 1f74446d..b6272600 100644 --- a/azure-test/tests/azure_api_management/test-turbot-expected.json +++ b/azure-test/tests/azure_api_management/test-turbot-expected.json @@ -1,13 +1,10 @@ [ { "akas": [ - "{{output.resource_aka.value }}", + "{{ output.resource_aka.value }}", "{{ output.resource_aka_lower.value }}" ], - "name": "{{resourceName}}", - "tags": { - "name": "{{resourceName}}" - }, - "title": "{{resourceName}}" + "name": "{{ resourceName }}", + "title": "{{ resourceName }}" } ] \ No newline at end of file diff --git a/azure-test/tests/azure_api_management/test-turbot-query.sql b/azure-test/tests/azure_api_management/test-turbot-query.sql index e4906b7c..1c6b7e94 100644 --- a/azure-test/tests/azure_api_management/test-turbot-query.sql +++ b/azure-test/tests/azure_api_management/test-turbot-query.sql @@ -1,3 +1,3 @@ -select name, akas, tags, title +select name, akas, title from azure.azure_api_management -where name = '{{resourceName}}' and resource_group = '{{ output.resource_group_name.value }}' +where name = '{{ resourceName }}' and resource_group = '{{ resourceName }}'; diff --git a/azure-test/tests/azure_api_management/variables.tf b/azure-test/tests/azure_api_management/variables.tf index 95cc0608..37a1d04b 100644 --- a/azure-test/tests/azure_api_management/variables.tf +++ b/azure-test/tests/azure_api_management/variables.tf @@ -1,4 +1,3 @@ - variable "resource_name" { type = string default = "turbot-test-20200125-create-update" @@ -17,62 +16,50 @@ variable "azure_subscription" { description = "Azure subscription used for the test." } -variable "azure_resource_group" { - type = string - default = "integration_test_rg" - description = "Name of the resource group used throughout the test." -} - -resource "azurerm_resource_group" "named_test_resource" { - name = var.resource_name - location = "West US" +terraform { + required_providers { + azurerm = { + source = "hashicorp/azurerm" + version = "=2.77.0" + } + } } provider "azurerm" { # Cannot be passed as a variable - version = "=1.36.0" + features {} environment = var.azure_environment subscription_id = var.azure_subscription } -data "azurerm_client_config" "current" {} - -data "null_data_source" "resource" { - inputs = { - scope = "azure:///subscriptions/${data.azurerm_client_config.current.subscription_id}" - } +resource "azurerm_resource_group" "named_test_resource" { + name = var.resource_name + location = "West Europe" } - + resource "azurerm_api_management" "named_test_resource" { - name = var.resource_name - location = azurerm_resource_group.named_test_resource.location - resource_group_name = azurerm_resource_group.named_test_resource.name - publisher_name = "TurbotHQ" - publisher_email = "test@turbot.com" - - sku_name = "Developer_1" + name = var.resource_name + location = azurerm_resource_group.named_test_resource.location + resource_group_name = azurerm_resource_group.named_test_resource.name + publisher_name = "TurbotHQ" + publisher_email = "test@turbot.com" + + sku_name = "Developer_1" +} - policy { - xml_content = < - - - - - -XML - } - tags = { - name = var.resource_name - } +output "region" { + depends_on = [azurerm_api_management.named_test_resource] + value = azurerm_resource_group.named_test_resource.location } output "resource_aka" { - value = "azure://${azurerm_api_management.named_test_resource.id}" + depends_on = [azurerm_api_management.named_test_resource] + value = "azure://${azurerm_api_management.named_test_resource.id}" } output "resource_aka_lower" { - value = "azure://${lower(azurerm_api_management.named_test_resource.id)}" + depends_on = [azurerm_api_management.named_test_resource] + value = "azure://${lower(azurerm_api_management.named_test_resource.id)}" } output "resource_name" { @@ -80,17 +67,10 @@ output "resource_name" { } output "resource_id" { - value = azurerm_api_management.named_test_resource.id -} - -output "location" { - value = azurerm_resource_group.named_test_resource.location + depends_on = [azurerm_api_management.named_test_resource] + value = azurerm_api_management.named_test_resource.id } output "subscription_id" { value = var.azure_subscription } - -output "resource_group_name" { - value = var.azure_resource_group -} diff --git a/azure/table_azure_api_management.go b/azure/table_azure_api_management.go index d3f0a681..d8fb9038 100644 --- a/azure/table_azure_api_management.go +++ b/azure/table_azure_api_management.go @@ -3,7 +3,8 @@ package azure import ( "context" - "github.com/Azure/azure-sdk-for-go/services/apimanagement/mgmt/2019-12-01/apimanagement" + "github.com/Azure/azure-sdk-for-go/profiles/2020-09-01/monitor/mgmt/insights" + "github.com/Azure/azure-sdk-for-go/services/apimanagement/mgmt/2020-12-01/apimanagement" "github.com/turbot/steampipe-plugin-sdk/grpc/proto" "github.com/turbot/steampipe-plugin-sdk/plugin/transform" @@ -28,101 +29,128 @@ func tableAzureAPIManagement(_ context.Context) *plugin.Table { { Name: "name", Type: proto.ColumnType_STRING, - Description: "A friendly name that identifies an api management", + Description: "A friendly name that identifies an API management service.", }, { Name: "id", - Description: "Contains ID to identify an api management uniquely", + Description: "Contains ID to identify an API management service uniquely.", Type: proto.ColumnType_STRING, Transform: transform.FromGo(), }, { - Name: "etag", - Description: "An unique read-only string that changes whenever the resource is updated", + Name: "provisioning_state", + Description: "The current provisioning state of the API management service. Possible values include: 'Created', 'Activating', 'Succeeded', 'Updating', 'Failed', 'Stopped', 'Terminating', 'TerminationFailed', 'Deleted'.", Type: proto.ColumnType_STRING, + Transform: transform.FromField("ServiceProperties.ProvisioningState"), }, { Name: "type", - Description: "Type of the resource", - Type: proto.ColumnType_STRING, - }, - { - Name: "provisioning_state", - Description: "The current provisioning state of the API Management service", + Description: "Type of the resource.", Type: proto.ColumnType_STRING, - Transform: transform.FromField("ServiceProperties.ProvisioningState"), }, { Name: "created_at_utc", - Description: "Creation UTC date of the API Management service", + Description: "Creation UTC date of the API management service.", Type: proto.ColumnType_TIMESTAMP, Transform: transform.FromField("ServiceProperties.CreatedAtUtc").Transform(convertDateToTime), }, + { + Name: "developer_portal_url", + Description: "Developer Portal endpoint URL of the API management service.", + Type: proto.ColumnType_STRING, + Transform: transform.FromField("ServiceProperties.DeveloperPortalURL"), + }, + { + Name: "disable_gateway", + Description: "Property only valid for an API management service deployed in multiple locations. This can be used to disable the gateway in master region.", + Type: proto.ColumnType_BOOL, + Transform: transform.FromField("ServiceProperties.DisableGateway"), + Default: false, + }, + { + Name: "enable_client_certificate", + Description: "Property only meant to be used for Consumption SKU Service. This enforces a client certificate to be presented on each request to the gateway. This also enables the ability to authenticate the certificate in the policy on the gateway.", + Type: proto.ColumnType_BOOL, + Transform: transform.FromField("ServiceProperties.EnableClientCertificate"), + Default: false, + }, + { + Name: "etag", + Description: "An unique read-only string that changes whenever the resource is updated.", + Type: proto.ColumnType_STRING, + }, { Name: "gateway_regional_url", - Description: "Gateway URL of the API Management service in the Default Region", + Description: "Gateway URL of the API management service in the default region.", Type: proto.ColumnType_STRING, Transform: transform.FromField("ServiceProperties.GatewayRegionalURL"), }, { Name: "gateway_url", - Description: "Gateway URL of the API Management service", + Description: "Gateway URL of the API management service.", Type: proto.ColumnType_STRING, Transform: transform.FromField("ServiceProperties.GatewayURL"), }, { Name: "identity_principal_id", - Description: "The principal id of the identity", + Description: "The principal id of the identity.", Type: proto.ColumnType_STRING, Transform: transform.FromField("Identity.PrincipalID").Transform(transform.ToString), }, { Name: "identity_tenant_id", - Description: "The client tenant id of the identity", + Description: "The client tenant id of the identity.", Type: proto.ColumnType_STRING, Transform: transform.FromField("Identity.TenantID").Transform(transform.ToString), }, { Name: "identity_type", - Description: "The type of identity used for the resource", + Description: "The type of identity used for the resource.", Type: proto.ColumnType_STRING, Transform: transform.FromField("Identity.Type").Transform(transform.ToString), }, { Name: "management_api_url", - Description: "Management API endpoint URL of the API Management service", + Description: "Management API endpoint URL of the API management service.", Type: proto.ColumnType_STRING, Transform: transform.FromField("ServiceProperties.ManagementAPIURL"), }, { Name: "notification_sender_email", - Description: "Email address from which the notification will be sent", + Description: "Email address from which the notification will be sent.", Type: proto.ColumnType_STRING, Transform: transform.FromField("ServiceProperties.NotificationSenderEmail"), }, { Name: "portal_url", - Description: "Publisher portal endpoint Url of the API Management service", + Description: "Publisher portal endpoint URL of the API management service.", Type: proto.ColumnType_STRING, Transform: transform.FromField("ServiceProperties.PortalURL"), }, { Name: "publisher_email", - Description: "Email address of the publisher of the API Management service", + Description: "Publisher email of the API management service.", Type: proto.ColumnType_STRING, Transform: transform.FromField("ServiceProperties.PublisherEmail"), }, { Name: "publisher_name", - Description: "Name of the publisher of the API Management service", + Description: "Publisher name of the API management service.", Type: proto.ColumnType_STRING, Transform: transform.FromField("ServiceProperties.PublisherName"), }, { - Name: "sku_name", - Description: "Name of the Sku", + Name: "restore", + Description: "Undelete API management service if it was previously soft-deleted.", + Type: proto.ColumnType_BOOL, + Transform: transform.FromField("ServiceProperties.Restore"), + Default: false, + }, + { + Name: "scm_url", + Description: "SCM endpoint URL of the API management service.", Type: proto.ColumnType_STRING, - Transform: transform.FromField("Sku.Name").Transform(transform.ToString), + Transform: transform.FromField("ServiceProperties.ScmURL"), }, { Name: "sku_capacity", @@ -131,55 +159,103 @@ func tableAzureAPIManagement(_ context.Context) *plugin.Table { Transform: transform.FromField("Sku.Capacity"), }, { - Name: "target_provisioning_state", - Description: "The provisioning state of the API Management service, which is targeted by the long running operation started on the service", + Name: "sku_name", + Description: "Name of the Sku", Type: proto.ColumnType_STRING, - Transform: transform.FromField("ServiceProperties.TargetProvisioningState"), + Transform: transform.FromField("Sku.Name").Transform(transform.ToString), }, { - Name: "virtual_network_configuration_id", - Description: "Contains the virtual network ID", + Name: "target_provisioning_state", + Description: "The provisioning state of the API management service, which is targeted by the long running operation started on the service.", Type: proto.ColumnType_STRING, - Transform: transform.FromField("ServiceProperties.VirtualNetworkConfiguration.Vnetid"), + Transform: transform.FromField("ServiceProperties.TargetProvisioningState"), }, { Name: "virtual_network_configuration_subnet_name", - Description: "Contains the name of the subnet", + Description: "The name of the subnet.", Type: proto.ColumnType_STRING, Transform: transform.FromField("ServiceProperties.VirtualNetworkConfiguration.Subnetname"), }, { Name: "virtual_network_configuration_subnet_resource_id", - Description: "The full resource ID of a subnet in a virtual network to deploy the API Management service in", + Description: "The full resource ID of a subnet in a virtual network to deploy the API Management service in.", Type: proto.ColumnType_STRING, Transform: transform.FromField("ServiceProperties.VirtualNetworkConfiguration.SubnetResourceID"), }, + { + Name: "virtual_network_configuration_id", + Description: "The virtual network ID.", + Type: proto.ColumnType_STRING, + Transform: transform.FromField("ServiceProperties.VirtualNetworkConfiguration.Vnetid"), + }, + { + Name: "virtual_network_type", + Description: "The type of VPN in which API management service needs to be configured in. None (Default Value) means the API management service is not part of any Virtual Network, External means the API management deployment is set up inside a Virtual Network having an Internet Facing Endpoint, and Internal means that API management deployment is setup inside a Virtual Network having an Intranet Facing Endpoint only. Possible values include: 'None', 'External', 'Internal'", + Type: proto.ColumnType_STRING, + Transform: transform.FromField("ServiceProperties.VirtualNetworkType"), + }, { Name: "additional_locations", - Description: "Additional datacenter locations of the API Management service", + Description: "Additional datacenter locations of the API management service.", Type: proto.ColumnType_JSON, Transform: transform.FromField("ServiceProperties.AdditionalLocations"), }, + { + Name: "api_version_constraint", + Description: "Control plane APIs version constraint for the API management service.", + Type: proto.ColumnType_JSON, + Transform: transform.FromField("ServiceProperties.APIVersionConstraint"), + }, + { + Name: "certificates", + Description: "List of certificates that need to be installed in the API management service.", + Type: proto.ColumnType_JSON, + Transform: transform.FromField("ServiceProperties.Certificates"), + }, + { + Name: "custom_properties", + Description: "Custom properties of the API management service.", + Type: proto.ColumnType_JSON, + Transform: transform.FromField("ServiceProperties.CustomProperties"), + }, + { + Name: "diagnostic_settings", + Description: "A list of active diagnostic settings for the API management service.", + Type: proto.ColumnType_JSON, + Hydrate: listAPIManagementDiagnosticSettings, + Transform: transform.FromValue(), + }, { Name: "host_name_configurations", - Description: "A list of custom hostname configuration of the API Management service", + Description: "Custom hostname configuration of the API management service.", Type: proto.ColumnType_JSON, Transform: transform.FromField("ServiceProperties.HostnameConfigurations"), }, + { + Name: "identity_user_assigned_identities", + Description: "The list of user identities associated with the resource.", + Type: proto.ColumnType_JSON, + Transform: transform.FromField("Identity.UserAssignedIdentities"), + }, { Name: "private_ip_addresses", - Description: "A list of private Static Load Balanced IP addresses of the API Management service in Primary region which is deployed in an Internal Virtual Network", + Description: "Private static load balanced IP addresses of the API management service in primary region which is deployed in an internal virtual network. Available only for 'Basic', 'Standard', 'Premium' and 'Isolated' SKU.", Type: proto.ColumnType_JSON, Transform: transform.FromField("ServiceProperties.PrivateIPAddresses"), }, { Name: "public_ip_addresses", - Description: "A list of public Static Load Balanced IP addresses of the API Management service in Primary region", + Description: "Public static load balanced IP addresses of the API management service in primary region. Available only for 'Basic', 'Standard', 'Premium' and 'Isolated' SKU.", Type: proto.ColumnType_JSON, Transform: transform.FromField("ServiceProperties.PublicIPAddresses"), }, + { + Name: "zones", + Description: "A list of availability zones denoting where the resource needs to come from.", + Type: proto.ColumnType_JSON, + }, - // Standard columns + // Steampipe standard columns { Name: "title", Description: ColumnDescriptionTitle, @@ -197,11 +273,13 @@ func tableAzureAPIManagement(_ context.Context) *plugin.Table { Type: proto.ColumnType_JSON, Transform: transform.FromField("ID").Transform(idToAkas), }, + + // Azure standard columns { Name: "region", Description: ColumnDescriptionRegion, Type: proto.ColumnType_STRING, - Transform: transform.FromField("Location").Transform(toLower), + Transform: transform.FromField("Location").Transform(formatRegion).Transform(toLower), }, { Name: "resource_group", @@ -219,7 +297,7 @@ func tableAzureAPIManagement(_ context.Context) *plugin.Table { } } -//// FETCH FUNCTIONS //// +//// LIST FUNCTION func listAPIManagements(ctx context.Context, d *plugin.QueryData, _ *plugin.HydrateData) (interface{}, error) { session, err := GetNewSession(ctx, d, "MANAGEMENT") @@ -233,6 +311,7 @@ func listAPIManagements(ctx context.Context, d *plugin.QueryData, _ *plugin.Hydr result, err := apiManagementClient.List(ctx) if err != nil { + plugin.Logger(ctx).Error("listAPIManagements", "list", err) return nil, err } for _, apiManagement := range result.Values() { @@ -242,6 +321,7 @@ func listAPIManagements(ctx context.Context, d *plugin.QueryData, _ *plugin.Hydr for result.NotDone() { err = result.NextWithContext(ctx) if err != nil { + plugin.Logger(ctx).Error("listAPIManagements", "list_paging", err) return nil, err } @@ -253,7 +333,7 @@ func listAPIManagements(ctx context.Context, d *plugin.QueryData, _ *plugin.Hydr return nil, err } -//// HYDRATE FUNCTIONS //// +//// HYDRATE FUNCTIONS func getAPIManagement(ctx context.Context, d *plugin.QueryData, h *plugin.HydrateData) (interface{}, error) { plugin.Logger(ctx).Trace("getAPIManagement") @@ -279,8 +359,51 @@ func getAPIManagement(ctx context.Context, d *plugin.QueryData, h *plugin.Hydrat op, err := apiManagementClient.Get(ctx, resourceGroup, name) if err != nil { + plugin.Logger(ctx).Error("getAPIManagement", "get", err) return nil, err } return op, nil } + +func listAPIManagementDiagnosticSettings(ctx context.Context, d *plugin.QueryData, h *plugin.HydrateData) (interface{}, error) { + plugin.Logger(ctx).Trace("listAPIManagementDiagnosticSettings") + id := *h.Item.(apimanagement.ServiceResource).ID + + // Create session + session, err := GetNewSession(ctx, d, "MANAGEMENT") + if err != nil { + return nil, err + } + subscriptionID := session.SubscriptionID + + client := insights.NewDiagnosticSettingsClient(subscriptionID) + client.Authorizer = session.Authorizer + + op, err := client.List(ctx, id) + if err != nil { + plugin.Logger(ctx).Error("listAPIManagementDiagnosticSettings", "list", err) + return nil, err + } + + // If we return the API response directly, the output does not provide + // all the contents of DiagnosticSettings + var diagnosticSettings []map[string]interface{} + for _, i := range *op.Value { + objectMap := make(map[string]interface{}) + if i.ID != nil { + objectMap["id"] = i.ID + } + if i.Name != nil { + objectMap["name"] = i.Name + } + if i.Type != nil { + objectMap["type"] = i.Type + } + if i.DiagnosticSettings != nil { + objectMap["properties"] = i.DiagnosticSettings + } + diagnosticSettings = append(diagnosticSettings, objectMap) + } + return diagnosticSettings, nil +}