From 1dfbbaebefed479c701ad1bf1413aed9d457d264 Mon Sep 17 00:00:00 2001 From: Arnab Ghosh Date: Thu, 7 Oct 2021 18:25:51 +0530 Subject: [PATCH] Add table azure_hdinsight_cluster --- .../azure_hdinsight_cluster/dependencies.txt | 0 .../test-get-expected.json | 10 + .../test-get-query.sql | 3 + .../test-list-expected.json | 10 + .../test-list-query.sql | 3 + .../test-not-found-expected.json | 1 + .../test-not-found-query.sql | 3 + .../test-turbot-expected.json | 10 + .../test-turbot-query.sql | 3 + .../azure_hdinsight_cluster/variables.json | 1 + .../azure_hdinsight_cluster/variables.tf | 126 +++++++ azure/plugin.go | 1 + azure/table_azure_hdinsight_cluster.go | 350 ++++++++++++++++++ docs/tables/azure_hdinsight_cluster.md | 67 ++++ 14 files changed, 588 insertions(+) create mode 100644 azure-test/tests/azure_hdinsight_cluster/dependencies.txt create mode 100644 azure-test/tests/azure_hdinsight_cluster/test-get-expected.json create mode 100644 azure-test/tests/azure_hdinsight_cluster/test-get-query.sql create mode 100644 azure-test/tests/azure_hdinsight_cluster/test-list-expected.json create mode 100644 azure-test/tests/azure_hdinsight_cluster/test-list-query.sql create mode 100644 azure-test/tests/azure_hdinsight_cluster/test-not-found-expected.json create mode 100644 azure-test/tests/azure_hdinsight_cluster/test-not-found-query.sql create mode 100644 azure-test/tests/azure_hdinsight_cluster/test-turbot-expected.json create mode 100644 azure-test/tests/azure_hdinsight_cluster/test-turbot-query.sql create mode 100644 azure-test/tests/azure_hdinsight_cluster/variables.json create mode 100644 azure-test/tests/azure_hdinsight_cluster/variables.tf create mode 100644 azure/table_azure_hdinsight_cluster.go create mode 100644 docs/tables/azure_hdinsight_cluster.md diff --git a/azure-test/tests/azure_hdinsight_cluster/dependencies.txt b/azure-test/tests/azure_hdinsight_cluster/dependencies.txt new file mode 100644 index 00000000..e69de29b diff --git a/azure-test/tests/azure_hdinsight_cluster/test-get-expected.json b/azure-test/tests/azure_hdinsight_cluster/test-get-expected.json new file mode 100644 index 00000000..e1c7598b --- /dev/null +++ b/azure-test/tests/azure_hdinsight_cluster/test-get-expected.json @@ -0,0 +1,10 @@ +[ + { + "id": "{{ output.resource_id.value }}", + "name": "{{ resourceName }}", + "region": "{{ output.region.value }}", + "resource_group": "{{ resourceName }}", + "subscription_id": "{{ output.subscription_id.value }}", + "type": "Microsoft.HDInsight/clusters" + } +] diff --git a/azure-test/tests/azure_hdinsight_cluster/test-get-query.sql b/azure-test/tests/azure_hdinsight_cluster/test-get-query.sql new file mode 100644 index 00000000..441270ef --- /dev/null +++ b/azure-test/tests/azure_hdinsight_cluster/test-get-query.sql @@ -0,0 +1,3 @@ +select name, id, type, region, resource_group, subscription_id +from azure.azure_hdinsight_cluster +where name = '{{ resourceName }}' and resource_group = '{{ resourceName }}'; diff --git a/azure-test/tests/azure_hdinsight_cluster/test-list-expected.json b/azure-test/tests/azure_hdinsight_cluster/test-list-expected.json new file mode 100644 index 00000000..e1c7598b --- /dev/null +++ b/azure-test/tests/azure_hdinsight_cluster/test-list-expected.json @@ -0,0 +1,10 @@ +[ + { + "id": "{{ output.resource_id.value }}", + "name": "{{ resourceName }}", + "region": "{{ output.region.value }}", + "resource_group": "{{ resourceName }}", + "subscription_id": "{{ output.subscription_id.value }}", + "type": "Microsoft.HDInsight/clusters" + } +] diff --git a/azure-test/tests/azure_hdinsight_cluster/test-list-query.sql b/azure-test/tests/azure_hdinsight_cluster/test-list-query.sql new file mode 100644 index 00000000..9e123114 --- /dev/null +++ b/azure-test/tests/azure_hdinsight_cluster/test-list-query.sql @@ -0,0 +1,3 @@ +select name, id, type, region, resource_group, subscription_id +from azure.azure_hdinsight_cluster +where id = '{{ output.resource_id.value }}'; diff --git a/azure-test/tests/azure_hdinsight_cluster/test-not-found-expected.json b/azure-test/tests/azure_hdinsight_cluster/test-not-found-expected.json new file mode 100644 index 00000000..19765bd5 --- /dev/null +++ b/azure-test/tests/azure_hdinsight_cluster/test-not-found-expected.json @@ -0,0 +1 @@ +null diff --git a/azure-test/tests/azure_hdinsight_cluster/test-not-found-query.sql b/azure-test/tests/azure_hdinsight_cluster/test-not-found-query.sql new file mode 100644 index 00000000..01f568d6 --- /dev/null +++ b/azure-test/tests/azure_hdinsight_cluster/test-not-found-query.sql @@ -0,0 +1,3 @@ +select name, id, type, region +from azure.azure_hdinsight_cluster +where name = 'dummy-test{{ resourceName }}' and resource_group = '{{ resourceName }}'; diff --git a/azure-test/tests/azure_hdinsight_cluster/test-turbot-expected.json b/azure-test/tests/azure_hdinsight_cluster/test-turbot-expected.json new file mode 100644 index 00000000..02cd3c76 --- /dev/null +++ b/azure-test/tests/azure_hdinsight_cluster/test-turbot-expected.json @@ -0,0 +1,10 @@ +[ + { + "akas": [ + "{{ output.resource_aka.value }}", + "{{ output.resource_aka_lower.value }}" + ], + "name": "{{ resourceName }}", + "title": "{{ resourceName }}" + } +] diff --git a/azure-test/tests/azure_hdinsight_cluster/test-turbot-query.sql b/azure-test/tests/azure_hdinsight_cluster/test-turbot-query.sql new file mode 100644 index 00000000..311f671c --- /dev/null +++ b/azure-test/tests/azure_hdinsight_cluster/test-turbot-query.sql @@ -0,0 +1,3 @@ +select name, akas, title +from azure.azure_hdinsight_cluster +where name = '{{ resourceName }}' and resource_group = '{{ resourceName }}'; diff --git a/azure-test/tests/azure_hdinsight_cluster/variables.json b/azure-test/tests/azure_hdinsight_cluster/variables.json new file mode 100644 index 00000000..0967ef42 --- /dev/null +++ b/azure-test/tests/azure_hdinsight_cluster/variables.json @@ -0,0 +1 @@ +{} diff --git a/azure-test/tests/azure_hdinsight_cluster/variables.tf b/azure-test/tests/azure_hdinsight_cluster/variables.tf new file mode 100644 index 00000000..7c50a372 --- /dev/null +++ b/azure-test/tests/azure_hdinsight_cluster/variables.tf @@ -0,0 +1,126 @@ +variable "resource_name" { + type = string + default = "turbot-test-20211007-create-update" + description = "Name of the resource used throughout the test." +} + +variable "azure_environment" { + type = string + default = "public" + description = "Azure environment used for the test." +} + +variable "azure_subscription" { + type = string + default = "3510ae4d-530b-497d-8f30-53b9616fc6c1" + description = "Azure environment used for the test." +} + +terraform { + required_providers { + azurerm = { + source = "hashicorp/azurerm" + version = "=2.79.1" + } + } +} + +provider "azurerm" { + # Cannot be passed as a variable + features {} + environment = var.azure_environment + subscription_id = var.azure_subscription +} + +resource "azurerm_resource_group" "named_test_resource" { + name = var.resource_name + location = "West US" +} + +resource "azurerm_storage_account" "named_test_resource" { + name = var.resource_name + resource_group_name = azurerm_resource_group.named_test_resource.name + location = azurerm_resource_group.named_test_resource.location + account_tier = "Standard" + account_replication_type = "LRS" +} + +resource "azurerm_storage_container" "named_test_resource" { + name = var.resource_name + storage_account_name = azurerm_storage_account.named_test_resource.name + container_access_type = "private" +} + +resource "azurerm_hdinsight_hadoop_cluster" "named_test_resource" { + name = var.resource_name + resource_group_name = azurerm_resource_group.named_test_resource.name + location = azurerm_resource_group.named_test_resource.location + cluster_version = "3.6" + tier = "Standard" + + component_version { + hadoop = "2.7" + } + + gateway { + enabled = true + username = "acctestusrgw" + password = "TerrAform123!" + } + + storage_account { + storage_container_id = azurerm_storage_container.named_test_resource.id + storage_account_key = azurerm_storage_account.named_test_resource.primary_access_key + is_default = true + } + + roles { + head_node { + vm_size = "Standard_D3_V2" + username = "acctestusrvm" + password = "AccTestvdSC4daf986!" + } + + worker_node { + vm_size = "Standard_D4_V2" + username = "acctestusrvm" + password = "AccTestvdSC4daf986!" + target_instance_count = 3 + } + + zookeeper_node { + vm_size = "Standard_D3_V2" + username = "acctestusrvm" + password = "AccTestvdSC4daf986!" + } + } +} + +output "region" { + value = azurerm_resource_group.named_test_resource.location +} + +output "resource_aka" { + depends_on = [azurerm_hdinsight_hadoop_cluster.named_test_resource] + value = "azure://${azurerm_hdinsight_hadoop_cluster.named_test_resource.id}" +} + +output "resource_aka_lower" { + value = "azure://${lower(azurerm_hdinsight_hadoop_cluster.named_test_resource.id)}" +} + +output "resource_name" { + value = var.resource_name +} + +output "resource_id" { + value = azurerm_hdinsight_hadoop_cluster.named_test_resource.id +} + +output "subscription_id" { + value = var.azure_subscription +} + +output "cluster_version" { + value = azurerm_hdinsight_hadoop_cluster.named_test_resource.cluster_version +} diff --git a/azure/plugin.go b/azure/plugin.go index f2a440e4..0197094c 100644 --- a/azure/plugin.go +++ b/azure/plugin.go @@ -70,6 +70,7 @@ func Plugin(ctx context.Context) *plugin.Plugin { "azure_express_route_circuit": tableAzureExpressRouteCircuit(ctx), "azure_firewall": tableAzureFirewall(ctx), "azure_frontdoor": tableAzureFrontDoor(ctx), + "azure_hdinsight_cluster": tableAzureHDInsightCluster(ctx), "azure_healthcare_service": tableAzureHealthcareService(ctx), "azure_hpc_cache": tableAzureHPCCache(ctx), "azure_hybrid_compute_machine": tableAzureHybridComputeMachine(ctx), diff --git a/azure/table_azure_hdinsight_cluster.go b/azure/table_azure_hdinsight_cluster.go new file mode 100644 index 00000000..ba445036 --- /dev/null +++ b/azure/table_azure_hdinsight_cluster.go @@ -0,0 +1,350 @@ +package azure + +import ( + "context" + + "github.com/Azure/azure-sdk-for-go/profiles/2020-09-01/monitor/mgmt/insights" + "github.com/Azure/azure-sdk-for-go/services/hdinsight/mgmt/2018-06-01/hdinsight" + "github.com/turbot/steampipe-plugin-sdk/grpc/proto" + "github.com/turbot/steampipe-plugin-sdk/plugin/transform" + + "github.com/turbot/steampipe-plugin-sdk/plugin" +) + +//// TABLE DEFINITION + +func tableAzureHDInsightCluster(_ context.Context) *plugin.Table { + return &plugin.Table{ + Name: "azure_hdinsight_cluster", + Description: "Azure HDInsight Cluster", + Get: &plugin.GetConfig{ + KeyColumns: plugin.AllColumns([]string{"name", "resource_group"}), + Hydrate: getHDInsightCluster, + ShouldIgnoreError: isNotFoundError([]string{"ResourceNotFound", "ResourceGroupNotFound", "404"}), + }, + List: &plugin.ListConfig{ + Hydrate: listHDInsightClusters, + }, + Columns: []*plugin.Column{ + { + Name: "name", + Description: "The name of the resource.", + Type: proto.ColumnType_STRING, + }, + { + Name: "id", + Description: "Fully qualified resource Id for the resource.", + Type: proto.ColumnType_STRING, + Transform: transform.FromGo(), + }, + { + Name: "provisioning_state", + Description: "The provisioning state, which only appears in the response. Possible values include: 'InProgress', 'Failed', 'Succeeded', 'Canceled', 'Deleting'.", + Type: proto.ColumnType_STRING, + Transform: transform.FromField("Properties.ProvisioningState").Transform(transform.ToString), + }, + { + Name: "type", + Description: "The type of the resource.", + Type: proto.ColumnType_STRING, + }, + { + Name: "cluster_hdp_version", + Description: "The hdp version of the cluster.", + Type: proto.ColumnType_STRING, + Transform: transform.FromField("Properties.ClusterHdpVersion"), + }, + { + Name: "cluster_id", + Description: "The cluster id.", + Type: proto.ColumnType_STRING, + Transform: transform.FromField("Properties.ClusterID"), + }, + { + Name: "cluster_state", + Description: "The state of the cluster.", + Type: proto.ColumnType_STRING, + Transform: transform.FromField("Properties.ClusterState"), + }, + { + Name: "cluster_version", + Description: "The version of the cluster.", + Type: proto.ColumnType_STRING, + Transform: transform.FromField("Properties.ClusterVersion"), + }, + { + Name: "created_date", + Description: "The date on which the cluster was created.", + Type: proto.ColumnType_STRING, + Transform: transform.FromField("Properties.CreatedDate"), + }, + { + Name: "etag", + Description: "The ETag for the resource.", + Type: proto.ColumnType_STRING, + }, + { + Name: "min_supported_tls_version", + Description: "The minimal supported tls version of the cluster.", + Type: proto.ColumnType_STRING, + Transform: transform.FromField("Properties.MinSupportedTLSVersion"), + }, + { + Name: "os_type", + Description: "The type of operating system. Possible values include: 'Windows', 'Linux'.", + Type: proto.ColumnType_STRING, + Transform: transform.FromField("Properties.OsType").Transform(transform.ToString), + }, + { + Name: "tier", + Description: "The cluster tier. Possible values include: 'Standard', 'Premium'.", + Type: proto.ColumnType_STRING, + Transform: transform.FromField("Properties.Tier").Transform(transform.ToString), + }, + { + Name: "cluster_definition", + Description: "The cluster definition.", + Type: proto.ColumnType_JSON, + Transform: transform.FromField("Properties.ClusterDefinition"), + }, + { + Name: "compute_isolation_properties", + Description: "The compute isolation properties of the cluster.", + Type: proto.ColumnType_JSON, + Transform: transform.FromField("Properties.ComputeIsolationProperties"), + }, + { + Name: "compute_profile", + Description: "The complete profile of the cluster.", + Type: proto.ColumnType_JSON, + Transform: transform.FromField("Properties.ComputeProfile"), + }, + { + Name: "connectivity_endpoints", + Description: "The list of connectivity endpoints.", + Type: proto.ColumnType_JSON, + Transform: transform.FromField("Properties.ConnectivityEndpoints"), + }, + { + Name: "diagnostic_settings", + Description: "A list of active diagnostic settings for the cluster.", + Type: proto.ColumnType_JSON, + Hydrate: listHDInsightClusterDiagnosticSettings, + Transform: transform.FromValue(), + }, + { + Name: "disk_encryption_properties", + Description: "The disk encryption properties of the cluster.", + Type: proto.ColumnType_JSON, + Transform: transform.FromField("Properties.DiskEncryptionProperties"), + }, + { + Name: "encryption_in_transit_properties", + Description: "The encryption-in-transit properties of the cluster.", + Type: proto.ColumnType_JSON, + Transform: transform.FromField("Properties.EncryptionInTransitProperties"), + }, + { + Name: "errors", + Description: "The list of errors.", + Type: proto.ColumnType_JSON, + Transform: transform.FromField("Properties.Errors"), + }, + { + Name: "excluded_services_config", + Description: "The excluded services config of the cluster.", + Type: proto.ColumnType_JSON, + Transform: transform.FromField("Properties.ExcludedServicesConfig"), + }, + { + Name: "identity", + Description: "The identity of the cluster, if configured.", + Type: proto.ColumnType_JSON, + }, + { + Name: "kafka_rest_properties", + Description: "The cluster kafka rest proxy configuration.", + Type: proto.ColumnType_JSON, + Transform: transform.FromField("Properties.KafkaRestProperties"), + }, + { + Name: "network_properties", + Description: "The network properties of the cluster.", + Type: proto.ColumnType_JSON, + Transform: transform.FromField("Properties.NetworkProperties"), + }, + { + Name: "quota_info", + Description: "The quota information of the cluster.", + Type: proto.ColumnType_JSON, + Transform: transform.FromField("Properties.QuotaInfo"), + }, + { + Name: "security_profile", + Description: "The security profile of the cluster.", + Type: proto.ColumnType_JSON, + Transform: transform.FromField("Properties.SecurityProfile"), + }, + { + Name: "storage_profile", + Description: "The storage profile of the cluster.", + Type: proto.ColumnType_JSON, + Transform: transform.FromField("Properties.StorageProfile"), + }, + + // Steampipe standard columns + { + Name: "title", + Description: ColumnDescriptionTitle, + Type: proto.ColumnType_STRING, + Transform: transform.FromField("Name"), + }, + { + Name: "tags", + Description: ColumnDescriptionTags, + Type: proto.ColumnType_JSON, + }, + { + Name: "akas", + Description: ColumnDescriptionAkas, + Type: proto.ColumnType_JSON, + Transform: transform.FromField("ID").Transform(idToAkas), + }, + + // Azure standard columns + { + Name: "region", + Description: ColumnDescriptionRegion, + Type: proto.ColumnType_STRING, + Transform: transform.FromField("Location").Transform(formatRegion).Transform(toLower), + }, + { + Name: "resource_group", + Description: ColumnDescriptionResourceGroup, + Type: proto.ColumnType_STRING, + Transform: transform.FromField("ID").Transform(extractResourceGroupFromID), + }, + { + Name: "subscription_id", + Description: ColumnDescriptionSubscription, + Type: proto.ColumnType_STRING, + Transform: transform.FromField("ID").Transform(idToSubscriptionID), + }, + }, + } +} + +//// LIST FUNCTION + +func listHDInsightClusters(ctx context.Context, d *plugin.QueryData, _ *plugin.HydrateData) (interface{}, error) { + session, err := GetNewSession(ctx, d, "MANAGEMENT") + if err != nil { + return nil, err + } + subscriptionID := session.SubscriptionID + + client := hdinsight.NewClustersClient(subscriptionID) + client.Authorizer = session.Authorizer + + result, err := client.List(ctx) + if err != nil { + plugin.Logger(ctx).Error("listHDInsightClusters", "list", err) + return nil, err + } + + for _, cluster := range result.Values() { + d.StreamListItem(ctx, cluster) + } + + for result.NotDone() { + err = result.NextWithContext(ctx) + if err != nil { + plugin.Logger(ctx).Error("listHDInsightClusters", "list_paging", err) + return nil, err + } + for _, cluster := range result.Values() { + d.StreamListItem(ctx, cluster) + } + } + + return nil, err +} + +//// HYDRATE FUNCTIONS + +func getHDInsightCluster(ctx context.Context, d *plugin.QueryData, h *plugin.HydrateData) (interface{}, error) { + plugin.Logger(ctx).Trace("getHDInsightCluster") + + name := d.KeyColumnQuals["name"].GetStringValue() + resourceGroup := d.KeyColumnQuals["resource_group"].GetStringValue() + + // Handle empty name or resourceGroup + if name == "" || resourceGroup == "" { + return nil, nil + } + + session, err := GetNewSession(ctx, d, "MANAGEMENT") + if err != nil { + return nil, err + } + subscriptionID := session.SubscriptionID + + client := hdinsight.NewClustersClient(subscriptionID) + client.Authorizer = session.Authorizer + + op, err := client.Get(ctx, resourceGroup, name) + if err != nil { + plugin.Logger(ctx).Error("getHDInsightCluster", "get", err) + return nil, err + } + + // In some cases resource does not give any notFound error + // instead of notFound error, it returns empty data + if op.ID != nil { + return op, nil + } + + return nil, nil +} + +func listHDInsightClusterDiagnosticSettings(ctx context.Context, d *plugin.QueryData, h *plugin.HydrateData) (interface{}, error) { + plugin.Logger(ctx).Trace("listHDInsightClusterDiagnosticSettings") + id := *h.Item.(hdinsight.Cluster).ID + + // Create session + session, err := GetNewSession(ctx, d, "MANAGEMENT") + if err != nil { + return nil, err + } + subscriptionID := session.SubscriptionID + + client := insights.NewDiagnosticSettingsClient(subscriptionID) + client.Authorizer = session.Authorizer + + op, err := client.List(ctx, id) + if err != nil { + plugin.Logger(ctx).Error("listHDInsightClusterDiagnosticSettings", "list", err) + return nil, err + } + + // If we return the API response directly, the output does not provide all + // the contents of DiagnosticSettings + var diagnosticSettings []map[string]interface{} + for _, i := range *op.Value { + objectMap := make(map[string]interface{}) + if i.ID != nil { + objectMap["id"] = i.ID + } + if i.Name != nil { + objectMap["name"] = i.Name + } + if i.Type != nil { + objectMap["type"] = i.Type + } + if i.DiagnosticSettings != nil { + objectMap["properties"] = i.DiagnosticSettings + } + diagnosticSettings = append(diagnosticSettings, objectMap) + } + return diagnosticSettings, nil +} diff --git a/docs/tables/azure_hdinsight_cluster.md b/docs/tables/azure_hdinsight_cluster.md new file mode 100644 index 00000000..a71a46c1 --- /dev/null +++ b/docs/tables/azure_hdinsight_cluster.md @@ -0,0 +1,67 @@ +# Table: azure_hdinsight_cluster + +Azure HDInsight is a managed, full-spectrum, open-source analytics service in the cloud for enterprises. You can use open-source frameworks such as Hadoop, Apache Spark, Apache Hive, LLAP, Apache Kafka, Apache Storm, R, and more. + +## Examples + +### Basic info + +```sql +select + name, + id, + provisioning_state, + type, + cluster_hdp_version, + cluster_id, + cluster_state, + cluster_version, + created_date +from + azure_hdinsight_cluster; +``` + +### List clusters with encryption in transit enabled + +```sql +select + name, + id, + encryption_in_transit_properties -> 'isEncryptionInTransitEnabled' as is_encryption_in_transit_enabled +from + azure_hdinsight_cluster +where + (encryption_in_transit_properties ->> 'isEncryptionInTransitEnabled')::boolean; +``` + +### List disk encryption details + +```sql +select + name, + id, + disk_encryption_properties ->> 'encryptionAlgorithm' as encryption_algorithm, + disk_encryption_properties -> 'encryptionAtHost' as encryption_at_host, + disk_encryption_properties ->> 'keyName' as key_name, + disk_encryption_properties ->> 'keyVersion' as key_version, + disk_encryption_properties ->> 'msiResourceId' as msi_resource_id, + disk_encryption_properties ->> 'vaultUri' as vault_uri +from + azure_hdinsight_cluster; +``` + +### List connectivity endpoint details + +```sql +select + name, + id, + endpoint ->> 'location' as endpoint_location, + endpoint ->> 'name' as endpoint_name, + endpoint -> 'port' as endpoint_port, + endpoint ->> 'protocol' as endpoint_protocol, + endpoint ->> 'privateIpAddress' as endpoint_private_ip_address +from + azure_hdinsight_cluster, + jsonb_array_elements(connectivity_endpoints) as endpoint; +```