diff --git a/azure-test/tests/azure_data_lake_store/dependencies.txt b/azure-test/tests/azure_data_lake_store/dependencies.txt new file mode 100644 index 00000000..e69de29b diff --git a/azure-test/tests/azure_data_lake_store/test-get-expected.json b/azure-test/tests/azure_data_lake_store/test-get-expected.json new file mode 100644 index 00000000..ab0435e6 --- /dev/null +++ b/azure-test/tests/azure_data_lake_store/test-get-expected.json @@ -0,0 +1,9 @@ +[ + { + "id": "{{ output.resource_id.value }}", + "name": "{{ resourceName }}", + "resource_group": "{{ resourceName }}", + "subscription_id": "{{ output.subscription_id.value }}", + "type": "Microsoft.DataLakeStore/accounts" + } +] diff --git a/azure-test/tests/azure_data_lake_store/test-get-query.sql b/azure-test/tests/azure_data_lake_store/test-get-query.sql new file mode 100644 index 00000000..c40610a6 --- /dev/null +++ b/azure-test/tests/azure_data_lake_store/test-get-query.sql @@ -0,0 +1,3 @@ +select name, id, type, resource_group, subscription_id +from azure.azure_data_lake_store +where name = '{{ resourceName }}' and resource_group = '{{ resourceName }}'; \ No newline at end of file diff --git a/azure-test/tests/azure_data_lake_store/test-list-expected.json b/azure-test/tests/azure_data_lake_store/test-list-expected.json new file mode 100644 index 00000000..6241c66d --- /dev/null +++ b/azure-test/tests/azure_data_lake_store/test-list-expected.json @@ -0,0 +1,7 @@ +[ + { + "id": "{{ output.resource_id.value }}", + "name": "{{ resourceName }}", + "type": "Microsoft.DataLakeStore/accounts" + } +] diff --git a/azure-test/tests/azure_data_lake_store/test-list-query.sql b/azure-test/tests/azure_data_lake_store/test-list-query.sql new file mode 100644 index 00000000..602a5517 --- /dev/null +++ b/azure-test/tests/azure_data_lake_store/test-list-query.sql @@ -0,0 +1,3 @@ +select name, id, type +from azure.azure_data_lake_store +where name = '{{ resourceName }}'; \ No newline at end of file diff --git a/azure-test/tests/azure_data_lake_store/test-not-found-expected.json b/azure-test/tests/azure_data_lake_store/test-not-found-expected.json new file mode 100644 index 00000000..19765bd5 --- /dev/null +++ b/azure-test/tests/azure_data_lake_store/test-not-found-expected.json @@ -0,0 +1 @@ +null diff --git a/azure-test/tests/azure_data_lake_store/test-not-found-query.sql b/azure-test/tests/azure_data_lake_store/test-not-found-query.sql new file mode 100644 index 00000000..d49e37dc --- /dev/null +++ b/azure-test/tests/azure_data_lake_store/test-not-found-query.sql @@ -0,0 +1,3 @@ +select name +from azure.azure_data_lake_store +where name = 'dummy-{{ resourceName }}' and resource_group = '{{ resourceName }}'; \ No newline at end of file diff --git a/azure-test/tests/azure_data_lake_store/test-turbot-expected.json b/azure-test/tests/azure_data_lake_store/test-turbot-expected.json new file mode 100644 index 00000000..0fad76be --- /dev/null +++ b/azure-test/tests/azure_data_lake_store/test-turbot-expected.json @@ -0,0 +1,9 @@ +[ + { + "akas": [ + "{{ output.resource_aka.value }}", + "{{ output.resource_aka_lower.value }}" + ], + "title": "{{ resourceName }}" + } +] diff --git a/azure-test/tests/azure_data_lake_store/test-turbot-query.sql b/azure-test/tests/azure_data_lake_store/test-turbot-query.sql new file mode 100644 index 00000000..c61d418e --- /dev/null +++ b/azure-test/tests/azure_data_lake_store/test-turbot-query.sql @@ -0,0 +1,3 @@ +select title, akas +from azure.azure_data_lake_store +where name = '{{ resourceName }}' and resource_group = '{{ resourceName }}'; \ No newline at end of file diff --git a/azure-test/tests/azure_data_lake_store/variables.json b/azure-test/tests/azure_data_lake_store/variables.json new file mode 100644 index 00000000..0967ef42 --- /dev/null +++ b/azure-test/tests/azure_data_lake_store/variables.json @@ -0,0 +1 @@ +{} diff --git a/azure-test/tests/azure_data_lake_store/variables.tf b/azure-test/tests/azure_data_lake_store/variables.tf new file mode 100644 index 00000000..1ec7373b --- /dev/null +++ b/azure-test/tests/azure_data_lake_store/variables.tf @@ -0,0 +1,65 @@ +variable "resource_name" { + type = string + default = "steampipe-test" + description = "Name of the resource used throughout the test." +} + +variable "azure_environment" { + type = string + default = "public" + description = "Azure environment used for the test." +} + +variable "azure_subscription" { + type = string + default = "3510ae4d-530b-497d-8f30-53b9616fc6c1" + description = "Azure subscription used for the test." +} + +provider "azurerm" { + # Cannot be passed as a variable + version = "=1.36.0" + environment = var.azure_environment + subscription_id = var.azure_subscription +} + +data "azurerm_client_config" "current" {} + +data "null_data_source" "resource" { + inputs = { + scope = "azure:///subscriptions/${data.azurerm_client_config.current.subscription_id}" + } +} + +resource "azurerm_resource_group" "named_test_resource" { + name = var.resource_name + location = "East US2" +} + +resource "azurerm_data_lake_store" "named_test_resource" { + name = var.resource_name + resource_group_name = azurerm_resource_group.named_test_resource.name + location = azurerm_resource_group.named_test_resource.location + encryption_state = "Enabled" + encryption_type = "ServiceManaged" +} + +output "resource_aka" { + value = "azure://${azurerm_data_lake_store.named_test_resource.id}" +} + +output "resource_aka_lower" { + value = "azure://${lower(azurerm_data_lake_store.named_test_resource.id)}" +} + +output "resource_name" { + value = var.resource_name +} + +output "resource_id" { + value = azurerm_data_lake_store.named_test_resource.id +} + +output "subscription_id" { + value = var.azure_subscription +} diff --git a/azure/plugin.go b/azure/plugin.go index 00799d95..1b121a46 100644 --- a/azure/plugin.go +++ b/azure/plugin.go @@ -54,6 +54,7 @@ func Plugin(ctx context.Context) *plugin.Plugin { "azure_data_factory": tableAzureDataFactory(ctx), "azure_data_factory_dataset": tableAzureDataFactoryDataset(ctx), "azure_data_factory_pipeline": tableAzureDataFactoryPipeline(ctx), + "azure_data_lake_store": tableAzureDataLakeStore(ctx), "azure_diagnostic_setting": tableAzureDiagnosticSetting(ctx), "azure_eventhub_namespace": tableAzureEventHubNamespace(ctx), "azure_express_route_circuit": tableAzureExpressRouteCircuit(ctx), diff --git a/azure/table_azure_data_lake_store.go b/azure/table_azure_data_lake_store.go new file mode 100644 index 00000000..7dc72609 --- /dev/null +++ b/azure/table_azure_data_lake_store.go @@ -0,0 +1,337 @@ +package azure + +import ( + "context" + "strings" + + "github.com/Azure/azure-sdk-for-go/profiles/2020-09-01/monitor/mgmt/insights" + "github.com/Azure/azure-sdk-for-go/services/datalake/store/mgmt/2016-11-01/account" + "github.com/turbot/steampipe-plugin-sdk/grpc/proto" + "github.com/turbot/steampipe-plugin-sdk/plugin/transform" + + "github.com/turbot/steampipe-plugin-sdk/plugin" +) + +//// TABLE DEFINITION + +func tableAzureDataLakeStore(_ context.Context) *plugin.Table { + return &plugin.Table{ + Name: "azure_data_lake_store", + Description: "Azure Data Lake Store", + Get: &plugin.GetConfig{ + KeyColumns: plugin.AllColumns([]string{"name", "resource_group"}), + Hydrate: getDataLakeStore, + ShouldIgnoreError: isNotFoundError([]string{"ResourceNotFound", "ResourceGroupNotFound", "400"}), + }, + List: &plugin.ListConfig{ + Hydrate: listDataLakeStores, + }, + Columns: []*plugin.Column{ + { + Name: "name", + Description: "The resource name.", + Type: proto.ColumnType_STRING, + }, + { + Name: "id", + Description: "The resource identifier.", + Type: proto.ColumnType_STRING, + Transform: transform.FromGo(), + }, + { + Name: "type", + Description: "The resource type.", + Type: proto.ColumnType_STRING, + }, + { + Name: "account_id", + Description: "The unique identifier associated with this data lake store account.", + Type: proto.ColumnType_STRING, + Transform: transform.FromField("DataLakeStoreAccountPropertiesBasic.AccountID", "DataLakeStoreAccountProperties.AccountID"), + }, + { + Name: "creation_time", + Description: "The account creation time.", + Type: proto.ColumnType_TIMESTAMP, + Transform: transform.FromField("DataLakeStoreAccountPropertiesBasic.CreationTime", "DataLakeStoreAccountProperties.CreationTime").Transform(convertDateToTime), + }, + { + Name: "current_tier", + Description: "The commitment tier in use for current month.", + Type: proto.ColumnType_STRING, + Hydrate: getDataLakeStore, + Transform: transform.FromField("DataLakeStoreAccountProperties.CurrentTier"), + }, + { + Name: "default_group", + Description: "The default owner group for all new folders and files created in the data lake store account.", + Type: proto.ColumnType_STRING, + Hydrate: getDataLakeStore, + Transform: transform.FromField("DataLakeStoreAccountProperties.DefaultGroup"), + }, + { + Name: "encryption_state", + Description: "The current state of encryption for this data lake store account.", + Type: proto.ColumnType_STRING, + Hydrate: getDataLakeStore, + Transform: transform.FromField("DataLakeStoreAccountProperties.EncryptionState"), + }, + { + Name: "encryption_provisioning_state", + Description: "The current state of encryption provisioning for this data lake store account.", + Type: proto.ColumnType_STRING, + Hydrate: getDataLakeStore, + Transform: transform.FromField("DataLakeStoreAccountProperties.EncryptionProvisioningState"), + }, + { + Name: "endpoint", + Description: "The full cname endpoint for this account.", + Type: proto.ColumnType_STRING, + Transform: transform.FromField("DataLakeStoreAccountPropertiesBasic.Endpoint", "DataLakeStoreAccountProperties.Endpoint"), + }, + { + Name: "firewall_state", + Description: "The current state of the IP address firewall for this data lake store account.", + Type: proto.ColumnType_STRING, + Hydrate: getDataLakeStore, + Transform: transform.FromField("DataLakeStoreAccountProperties.FirewallState"), + }, + { + Name: "last_modified_time", + Description: "The account last modified time.", + Type: proto.ColumnType_TIMESTAMP, + Transform: transform.FromField("DataLakeStoreAccountPropertiesBasic.LastModifiedTime", "DataLakeStoreAccountProperties.LastModifiedTime").Transform(convertDateToTime), + }, + { + Name: "new_tier", + Description: "The commitment tier to use for next month.", + Type: proto.ColumnType_STRING, + Hydrate: getDataLakeStore, + Transform: transform.FromField("DataLakeStoreAccountProperties.NewTier"), + }, + { + Name: "provisioning_state", + Description: "The provisioning status of the data lake store account.", + Type: proto.ColumnType_STRING, + Transform: transform.FromField("DataLakeStoreAccountPropertiesBasic.ProvisioningState", "DataLakeStoreAccountProperties.ProvisioningState"), + }, + { + Name: "state", + Description: "The state of the data lake store account.", + Type: proto.ColumnType_STRING, + Transform: transform.FromField("DataLakeStoreAccountPropertiesBasic.State", "DataLakeStoreAccountProperties.State"), + }, + { + Name: "trusted_id_provider_state", + Description: "The current state of the trusted identity provider feature for this data lake store account.", + Type: proto.ColumnType_STRING, + Hydrate: getDataLakeStore, + Transform: transform.FromField("DataLakeStoreAccountProperties.TrustedIDProviderState"), + }, + { + Name: "diagnostic_settings", + Description: "A list of active diagnostic settings for the data lake store.", + Type: proto.ColumnType_JSON, + Hydrate: listDataLakeStoreDiagnosticSettings, + Transform: transform.FromValue(), + }, + { + Name: "encryption_config", + Description: "The key vault encryption configuration.", + Type: proto.ColumnType_JSON, + Hydrate: getDataLakeStore, + Transform: transform.FromField("DataLakeStoreAccountProperties.EncryptionConfig"), + }, + { + Name: "firewall_allow_azure_ips", + Description: "The current state of allowing or disallowing IPs originating within azure through the firewall. If the firewall is disabled, this is not enforced.", + Type: proto.ColumnType_JSON, + Hydrate: getDataLakeStore, + Transform: transform.FromField("DataLakeStoreAccountProperties.FirewallAllowAzureIps"), + }, + { + Name: "firewall_rules", + Description: "The list of firewall rules associated with this data lake store account.", + Type: proto.ColumnType_JSON, + Hydrate: getDataLakeStore, + Transform: transform.FromField("DataLakeStoreAccountProperties.FirewallRules"), + }, + { + Name: "identity", + Description: "The key vault encryption identity, if any.", + Type: proto.ColumnType_JSON, + Hydrate: getDataLakeStore, + }, + { + Name: "trusted_id_providers", + Description: "The list of trusted identity providers associated with this data lake store account.", + Type: proto.ColumnType_JSON, + Hydrate: getDataLakeStore, + Transform: transform.FromField("DataLakeStoreAccountProperties.TrustedIDProviders"), + }, + { + Name: "virtual_network_rules", + Description: "The list of virtual network rules associated with this data lake store account.", + Type: proto.ColumnType_JSON, + Hydrate: getDataLakeStore, + Transform: transform.FromField("DataLakeStoreAccountProperties.VirtualNetworkRules"), + }, + + // Steampipe standard columns + { + Name: "title", + Description: ColumnDescriptionTitle, + Type: proto.ColumnType_STRING, + Transform: transform.FromField("Name"), + }, + { + Name: "tags", + Description: ColumnDescriptionTags, + Type: proto.ColumnType_JSON, + }, + { + Name: "akas", + Description: ColumnDescriptionAkas, + Type: proto.ColumnType_JSON, + Transform: transform.FromField("ID").Transform(idToAkas), + }, + + // Azure standard columns + { + Name: "region", + Description: ColumnDescriptionRegion, + Type: proto.ColumnType_STRING, + Transform: transform.FromField("Location").Transform(toLower), + }, + { + Name: "resource_group", + Description: ColumnDescriptionResourceGroup, + Type: proto.ColumnType_STRING, + Transform: transform.FromField("ID").Transform(extractResourceGroupFromID), + }, + { + Name: "subscription_id", + Description: ColumnDescriptionSubscription, + Type: proto.ColumnType_STRING, + Transform: transform.FromField("ID").Transform(idToSubscriptionID), + }, + }, + } +} + +//// LIST FUNCTION + +func listDataLakeStores(ctx context.Context, d *plugin.QueryData, _ *plugin.HydrateData) (interface{}, error) { + session, err := GetNewSession(ctx, d, "MANAGEMENT") + if err != nil { + return nil, err + } + subscriptionID := session.SubscriptionID + + accountClient := account.NewAccountsClient(subscriptionID) + accountClient.Authorizer = session.Authorizer + + pagesLeft := true + for pagesLeft { + result, err := accountClient.List(context.Background(), "", nil, nil, "", "", nil) + if err != nil { + return nil, err + } + for _, account := range result.Values() { + d.StreamListItem(ctx, account) + } + result.NextWithContext(context.Background()) + pagesLeft = result.NotDone() + } + return nil, err +} + +//// HYDRATE FUNCTIONS + +func getDataLakeStore(ctx context.Context, d *plugin.QueryData, h *plugin.HydrateData) (interface{}, error) { + plugin.Logger(ctx).Trace("getDataLakeStore") + + // Create session + session, err := GetNewSession(ctx, d, "MANAGEMENT") + if err != nil { + return nil, err + } + subscriptionID := session.SubscriptionID + + accountClient := account.NewAccountsClient(subscriptionID) + accountClient.Authorizer = session.Authorizer + + var name, resourceGroup string + if h.Item != nil { + data := h.Item.(account.DataLakeStoreAccountBasic) + splitID := strings.Split(*data.ID, "/") + name = *data.Name + resourceGroup = splitID[4] + } else { + name = d.KeyColumnQuals["name"].GetStringValue() + resourceGroup = d.KeyColumnQuals["resource_group"].GetStringValue() + } + + // Return nil, if no input provide + if name == "" || resourceGroup == "" { + return nil, nil + } + + op, err := accountClient.Get(ctx, resourceGroup, name) + if err != nil { + return nil, err + } + + return op, nil +} + +func listDataLakeStoreDiagnosticSettings(ctx context.Context, d *plugin.QueryData, h *plugin.HydrateData) (interface{}, error) { + plugin.Logger(ctx).Trace("listDataLakeStoreDiagnosticSettings") + id := getLakeStoreID(h.Item) + + // Create session + session, err := GetNewSession(ctx, d, "MANAGEMENT") + if err != nil { + return nil, err + } + subscriptionID := session.SubscriptionID + + client := insights.NewDiagnosticSettingsClient(subscriptionID) + client.Authorizer = session.Authorizer + + op, err := client.List(ctx, id) + if err != nil { + return nil, err + } + + // If we return the API response directly, the output only gives + // the contents of DiagnosticSettings + var diagnosticSettings []map[string]interface{} + for _, i := range *op.Value { + objectMap := make(map[string]interface{}) + if i.ID != nil { + objectMap["id"] = i.ID + } + if i.Name != nil { + objectMap["name"] = i.Name + } + if i.Type != nil { + objectMap["type"] = i.Type + } + if i.DiagnosticSettings != nil { + objectMap["properties"] = i.DiagnosticSettings + } + diagnosticSettings = append(diagnosticSettings, objectMap) + } + return diagnosticSettings, nil +} + +func getLakeStoreID(item interface{}) string { + switch item := item.(type) { + case account.DataLakeStoreAccountBasic: + return *item.ID + case account.DataLakeStoreAccount: + return *item.ID + } + return "" +} diff --git a/docs/tables/azure_data_lake_store.md b/docs/tables/azure_data_lake_store.md new file mode 100644 index 00000000..095bbb0e --- /dev/null +++ b/docs/tables/azure_data_lake_store.md @@ -0,0 +1,45 @@ +# Table: azure_data_lake_store + +Azure Data Lake Storage is an enterprise-wide hyper-scale repository for big data analytic workloads. Azure Data Lake enables you to capture data of any size, type, and ingestion speed in one single place for operational and exploratory analytics. + +## Examples + +### Basic info + +```sql +select + name, + id, + type, + provisioning_state +from + azure_data_lake_store; +``` + +### List data lake stores with encryption disabled + +```sql +select + name, + id, + type, + provisioning_state +from + azure_data_lake_store +where + encryption_state = 'Disabled'; +``` + +### List data lake stores with firewall disabled + +```sql +select + name, + id, + type, + provisioning_state +from + azure_data_lake_store +where + firewall_state = 'Disabled'; +```