From d7440ba5d4841d3a27ecea50c280a72dabede249 Mon Sep 17 00:00:00 2001 From: bigdatasourav Date: Fri, 30 Jul 2021 19:09:48 +0530 Subject: [PATCH 1/6] New table --- .../azure_data_lake_store/dependencies.txt | 0 .../test-get-expected.json | 9 + .../azure_data_lake_store/test-get-query.sql | 3 + .../test-list-expected.json | 7 + .../azure_data_lake_store/test-list-query.sql | 3 + .../test-not-found-expected.json | 1 + .../test-not-found-query.sql | 3 + .../test-turbot-expected.json | 9 + .../test-turbot-query.sql | 3 + .../azure_data_lake_store/variables.json | 1 + .../tests/azure_data_lake_store/variables.tf | 65 +++++++ azure/plugin.go | 1 + azure/table_azure_data_lake_store.go | 178 ++++++++++++++++++ docs/tables/azure_data_lake_store.md | 17 ++ 14 files changed, 300 insertions(+) create mode 100644 azure-test/tests/azure_data_lake_store/dependencies.txt create mode 100644 azure-test/tests/azure_data_lake_store/test-get-expected.json create mode 100644 azure-test/tests/azure_data_lake_store/test-get-query.sql create mode 100644 azure-test/tests/azure_data_lake_store/test-list-expected.json create mode 100644 azure-test/tests/azure_data_lake_store/test-list-query.sql create mode 100644 azure-test/tests/azure_data_lake_store/test-not-found-expected.json create mode 100644 azure-test/tests/azure_data_lake_store/test-not-found-query.sql create mode 100644 azure-test/tests/azure_data_lake_store/test-turbot-expected.json create mode 100644 azure-test/tests/azure_data_lake_store/test-turbot-query.sql create mode 100644 azure-test/tests/azure_data_lake_store/variables.json create mode 100644 azure-test/tests/azure_data_lake_store/variables.tf create mode 100644 azure/table_azure_data_lake_store.go create mode 100644 docs/tables/azure_data_lake_store.md diff --git a/azure-test/tests/azure_data_lake_store/dependencies.txt b/azure-test/tests/azure_data_lake_store/dependencies.txt new file mode 100644 index 00000000..e69de29b diff --git a/azure-test/tests/azure_data_lake_store/test-get-expected.json b/azure-test/tests/azure_data_lake_store/test-get-expected.json new file mode 100644 index 00000000..ab0435e6 --- /dev/null +++ b/azure-test/tests/azure_data_lake_store/test-get-expected.json @@ -0,0 +1,9 @@ +[ + { + "id": "{{ output.resource_id.value }}", + "name": "{{ resourceName }}", + "resource_group": "{{ resourceName }}", + "subscription_id": "{{ output.subscription_id.value }}", + "type": "Microsoft.DataLakeStore/accounts" + } +] diff --git a/azure-test/tests/azure_data_lake_store/test-get-query.sql b/azure-test/tests/azure_data_lake_store/test-get-query.sql new file mode 100644 index 00000000..c40610a6 --- /dev/null +++ b/azure-test/tests/azure_data_lake_store/test-get-query.sql @@ -0,0 +1,3 @@ +select name, id, type, resource_group, subscription_id +from azure.azure_data_lake_store +where name = '{{ resourceName }}' and resource_group = '{{ resourceName }}'; \ No newline at end of file diff --git a/azure-test/tests/azure_data_lake_store/test-list-expected.json b/azure-test/tests/azure_data_lake_store/test-list-expected.json new file mode 100644 index 00000000..6241c66d --- /dev/null +++ b/azure-test/tests/azure_data_lake_store/test-list-expected.json @@ -0,0 +1,7 @@ +[ + { + "id": "{{ output.resource_id.value }}", + "name": "{{ resourceName }}", + "type": "Microsoft.DataLakeStore/accounts" + } +] diff --git a/azure-test/tests/azure_data_lake_store/test-list-query.sql b/azure-test/tests/azure_data_lake_store/test-list-query.sql new file mode 100644 index 00000000..602a5517 --- /dev/null +++ b/azure-test/tests/azure_data_lake_store/test-list-query.sql @@ -0,0 +1,3 @@ +select name, id, type +from azure.azure_data_lake_store +where name = '{{ resourceName }}'; \ No newline at end of file diff --git a/azure-test/tests/azure_data_lake_store/test-not-found-expected.json b/azure-test/tests/azure_data_lake_store/test-not-found-expected.json new file mode 100644 index 00000000..19765bd5 --- /dev/null +++ b/azure-test/tests/azure_data_lake_store/test-not-found-expected.json @@ -0,0 +1 @@ +null diff --git a/azure-test/tests/azure_data_lake_store/test-not-found-query.sql b/azure-test/tests/azure_data_lake_store/test-not-found-query.sql new file mode 100644 index 00000000..d49e37dc --- /dev/null +++ b/azure-test/tests/azure_data_lake_store/test-not-found-query.sql @@ -0,0 +1,3 @@ +select name +from azure.azure_data_lake_store +where name = 'dummy-{{ resourceName }}' and resource_group = '{{ resourceName }}'; \ No newline at end of file diff --git a/azure-test/tests/azure_data_lake_store/test-turbot-expected.json b/azure-test/tests/azure_data_lake_store/test-turbot-expected.json new file mode 100644 index 00000000..0fad76be --- /dev/null +++ b/azure-test/tests/azure_data_lake_store/test-turbot-expected.json @@ -0,0 +1,9 @@ +[ + { + "akas": [ + "{{ output.resource_aka.value }}", + "{{ output.resource_aka_lower.value }}" + ], + "title": "{{ resourceName }}" + } +] diff --git a/azure-test/tests/azure_data_lake_store/test-turbot-query.sql b/azure-test/tests/azure_data_lake_store/test-turbot-query.sql new file mode 100644 index 00000000..c61d418e --- /dev/null +++ b/azure-test/tests/azure_data_lake_store/test-turbot-query.sql @@ -0,0 +1,3 @@ +select title, akas +from azure.azure_data_lake_store +where name = '{{ resourceName }}' and resource_group = '{{ resourceName }}'; \ No newline at end of file diff --git a/azure-test/tests/azure_data_lake_store/variables.json b/azure-test/tests/azure_data_lake_store/variables.json new file mode 100644 index 00000000..0967ef42 --- /dev/null +++ b/azure-test/tests/azure_data_lake_store/variables.json @@ -0,0 +1 @@ +{} diff --git a/azure-test/tests/azure_data_lake_store/variables.tf b/azure-test/tests/azure_data_lake_store/variables.tf new file mode 100644 index 00000000..26d27cc9 --- /dev/null +++ b/azure-test/tests/azure_data_lake_store/variables.tf @@ -0,0 +1,65 @@ +variable "resource_name" { + type = string + default = "steampipe-test" + description = "Name of the resource used throughout the test." +} + +variable "azure_environment" { + type = string + default = "public" + description = "Azure environment used for the test." +} + +variable "azure_subscription" { + type = string + default = "d7245080-b4ae-4fe5-b6fa-2e71b3dae6c8" + description = "Azure subscription used for the test." +} + +provider "azurerm" { + # Cannot be passed as a variable + version = "=1.36.0" + environment = var.azure_environment + subscription_id = var.azure_subscription +} + +data "azurerm_client_config" "current" {} + +data "null_data_source" "resource" { + inputs = { + scope = "azure:///subscriptions/${data.azurerm_client_config.current.subscription_id}" + } +} + +resource "azurerm_resource_group" "named_test_resource" { + name = var.resource_name + location = "East US2" +} + +resource "azurerm_data_lake_store" "named_test_resource" { + name = var.resource_name + resource_group_name = azurerm_resource_group.named_test_resource.name + location = azurerm_resource_group.named_test_resource.location + encryption_state = "Enabled" + encryption_type = "ServiceManaged" +} + +output "resource_aka" { + value = "azure://${azurerm_data_lake_store.named_test_resource.id}" +} + +output "resource_aka_lower" { + value = "azure://${lower(azurerm_data_lake_store.named_test_resource.id)}" +} + +output "resource_name" { + value = var.resource_name +} + +output "resource_id" { + value = azurerm_data_lake_store.named_test_resource.id +} + +output "subscription_id" { + value = var.azure_subscription +} diff --git a/azure/plugin.go b/azure/plugin.go index 00799d95..1b121a46 100644 --- a/azure/plugin.go +++ b/azure/plugin.go @@ -54,6 +54,7 @@ func Plugin(ctx context.Context) *plugin.Plugin { "azure_data_factory": tableAzureDataFactory(ctx), "azure_data_factory_dataset": tableAzureDataFactoryDataset(ctx), "azure_data_factory_pipeline": tableAzureDataFactoryPipeline(ctx), + "azure_data_lake_store": tableAzureDataLakeStore(ctx), "azure_diagnostic_setting": tableAzureDiagnosticSetting(ctx), "azure_eventhub_namespace": tableAzureEventHubNamespace(ctx), "azure_express_route_circuit": tableAzureExpressRouteCircuit(ctx), diff --git a/azure/table_azure_data_lake_store.go b/azure/table_azure_data_lake_store.go new file mode 100644 index 00000000..485eb82e --- /dev/null +++ b/azure/table_azure_data_lake_store.go @@ -0,0 +1,178 @@ +package azure + +import ( + "context" + + "github.com/Azure/azure-sdk-for-go/services/datalake/store/mgmt/2016-11-01/account" + "github.com/turbot/steampipe-plugin-sdk/grpc/proto" + "github.com/turbot/steampipe-plugin-sdk/plugin/transform" + + "github.com/turbot/steampipe-plugin-sdk/plugin" +) + +//// TABLE DEFINITION + +func tableAzureDataLakeStore(_ context.Context) *plugin.Table { + return &plugin.Table{ + Name: "azure_data_lake_store", + Description: "Azure Data Lake Store", + Get: &plugin.GetConfig{ + KeyColumns: plugin.AllColumns([]string{"name", "resource_group"}), + Hydrate: getDataLakeStore, + ShouldIgnoreError: isNotFoundError([]string{"ResourceNotFound", "ResourceGroupNotFound", "Invalid input"}), + }, + List: &plugin.ListConfig{ + Hydrate: listDataLakeStores, + }, + Columns: []*plugin.Column{ + { + Name: "name", + Description: "The resource name.", + Type: proto.ColumnType_STRING, + }, + { + Name: "id", + Description: "The resource identifier.", + Type: proto.ColumnType_STRING, + Transform: transform.FromGo(), + }, + { + Name: "type", + Description: "The resource type.", + Type: proto.ColumnType_STRING, + }, + { + Name: "account_id", + Description: "The unique identifier associated with this Data Lake Store account.", + Type: proto.ColumnType_STRING, + Transform: transform.FromField("DataLakeStoreAccountPropertiesBasic.AccountID", "DataLakeStoreAccountProperties.AccountID"), + }, + { + Name: "creation_time", + Description: "The account creation time.", + Type: proto.ColumnType_TIMESTAMP, + Transform: transform.FromField("DataLakeStoreAccountPropertiesBasic.CreationTime").Transform(convertDateToTime), + }, + { + Name: "endpoint", + Description: "The full CName endpoint for this account.", + Type: proto.ColumnType_STRING, + }, + { + Name: "provisioning_state", + Description: "The provisioning status of the Data Lake Store account.", + Type: proto.ColumnType_STRING, + Transform: transform.FromField("DataLakeStoreAccountPropertiesBasic.ProvisioningState"), + }, + { + Name: "state", + Description: "The state of the Data Lake Store account.", + Type: proto.ColumnType_STRING, + Transform: transform.FromField("DataLakeStoreAccountPropertiesBasic.State"), + }, + { + Name: "last_modified_time", + Description: "Managed service identity of the factory.", + Type: proto.ColumnType_TIMESTAMP, + Transform: transform.FromField("DataLakeStoreAccountPropertiesBasic.LastModifiedTime").Transform(convertDateToTime), + }, + + // Steampipe standard columns + { + Name: "title", + Description: ColumnDescriptionTitle, + Type: proto.ColumnType_STRING, + Transform: transform.FromField("Name"), + }, + { + Name: "tags", + Description: ColumnDescriptionTags, + Type: proto.ColumnType_JSON, + }, + { + Name: "akas", + Description: ColumnDescriptionAkas, + Type: proto.ColumnType_JSON, + Transform: transform.FromField("ID").Transform(idToAkas), + }, + + // Azure standard column + { + Name: "region", + Description: ColumnDescriptionRegion, + Type: proto.ColumnType_STRING, + Transform: transform.FromField("Location").Transform(toLower), + }, + { + Name: "resource_group", + Description: ColumnDescriptionResourceGroup, + Type: proto.ColumnType_STRING, + Transform: transform.FromField("ID").Transform(extractResourceGroupFromID), + }, + { + Name: "subscription_id", + Description: ColumnDescriptionSubscription, + Type: proto.ColumnType_STRING, + Transform: transform.FromField("ID").Transform(idToSubscriptionID), + }, + }, + } +} + +//// LIST FUNCTION + +func listDataLakeStores(ctx context.Context, d *plugin.QueryData, _ *plugin.HydrateData) (interface{}, error) { + session, err := GetNewSession(ctx, d, "MANAGEMENT") + if err != nil { + return nil, err + } + subscriptionID := session.SubscriptionID + + accountClient := account.NewAccountsClient(subscriptionID) + accountClient.Authorizer = session.Authorizer + + pagesLeft := true + for pagesLeft { + result, err := accountClient.List(context.Background(), "", nil, nil, "", "", nil) + if err != nil { + return nil, err + } + for _, account := range result.Values() { + d.StreamListItem(ctx, account) + } + result.NextWithContext(context.Background()) + pagesLeft = result.NotDone() + } + return nil, err +} + +//// HYDRATE FUNCTIONS + +func getDataLakeStore(ctx context.Context, d *plugin.QueryData, h *plugin.HydrateData) (interface{}, error) { + plugin.Logger(ctx).Trace("getDataLakeStore") + + // Create session + session, err := GetNewSession(ctx, d, "MANAGEMENT") + if err != nil { + return nil, err + } + subscriptionID := session.SubscriptionID + + accountClient := account.NewAccountsClient(subscriptionID) + accountClient.Authorizer = session.Authorizer + + name := d.KeyColumnQuals["name"].GetStringValue() + resourceGroup := d.KeyColumnQuals["resource_group"].GetStringValue() + + // Return nil, if no input provide + if name == "" || resourceGroup == "" { + return nil, nil + } + + op, err := accountClient.Get(ctx, resourceGroup, name) + if err != nil { + return nil, err + } + + return op, nil +} diff --git a/docs/tables/azure_data_lake_store.md b/docs/tables/azure_data_lake_store.md new file mode 100644 index 00000000..5839ce66 --- /dev/null +++ b/docs/tables/azure_data_lake_store.md @@ -0,0 +1,17 @@ +# Table: azure_data_lake_store + +Azure Data Lake Storage Gen1 is an enterprise-wide hyper-scale repository for big data analytic workloads. Azure Data Lake enables you to capture data of any size, type, and ingestion speed in one single place for operational and exploratory analytics. + +## Examples + +### Basic info + +```sql +select + name, + id, + type, + provisioning_state +from + azure_data_lake_store; +``` From 677399222aa054e07fd6c074d72d86afcb530297 Mon Sep 17 00:00:00 2001 From: bigdatasourav Date: Sat, 31 Jul 2021 15:43:00 +0530 Subject: [PATCH 2/6] doc update --- .../tests/azure_data_lake_store/variables.tf | 2 +- azure/table_azure_data_lake_store.go | 179 +++++++++++++++++- docs/tables/azure_data_lake_store.md | 30 ++- 3 files changed, 199 insertions(+), 12 deletions(-) diff --git a/azure-test/tests/azure_data_lake_store/variables.tf b/azure-test/tests/azure_data_lake_store/variables.tf index 26d27cc9..1ec7373b 100644 --- a/azure-test/tests/azure_data_lake_store/variables.tf +++ b/azure-test/tests/azure_data_lake_store/variables.tf @@ -12,7 +12,7 @@ variable "azure_environment" { variable "azure_subscription" { type = string - default = "d7245080-b4ae-4fe5-b6fa-2e71b3dae6c8" + default = "3510ae4d-530b-497d-8f30-53b9616fc6c1" description = "Azure subscription used for the test." } diff --git a/azure/table_azure_data_lake_store.go b/azure/table_azure_data_lake_store.go index 485eb82e..b4ea59fe 100644 --- a/azure/table_azure_data_lake_store.go +++ b/azure/table_azure_data_lake_store.go @@ -2,7 +2,9 @@ package azure import ( "context" + "strings" + "github.com/Azure/azure-sdk-for-go/profiles/2020-09-01/monitor/mgmt/insights" "github.com/Azure/azure-sdk-for-go/services/datalake/store/mgmt/2016-11-01/account" "github.com/turbot/steampipe-plugin-sdk/grpc/proto" "github.com/turbot/steampipe-plugin-sdk/plugin/transform" @@ -19,7 +21,7 @@ func tableAzureDataLakeStore(_ context.Context) *plugin.Table { Get: &plugin.GetConfig{ KeyColumns: plugin.AllColumns([]string{"name", "resource_group"}), Hydrate: getDataLakeStore, - ShouldIgnoreError: isNotFoundError([]string{"ResourceNotFound", "ResourceGroupNotFound", "Invalid input"}), + ShouldIgnoreError: isNotFoundError([]string{"ResourceNotFound", "ResourceGroupNotFound"}), }, List: &plugin.ListConfig{ Hydrate: listDataLakeStores, @@ -51,30 +53,128 @@ func tableAzureDataLakeStore(_ context.Context) *plugin.Table { Name: "creation_time", Description: "The account creation time.", Type: proto.ColumnType_TIMESTAMP, - Transform: transform.FromField("DataLakeStoreAccountPropertiesBasic.CreationTime").Transform(convertDateToTime), + Transform: transform.FromField("DataLakeStoreAccountPropertiesBasic.CreationTime", "DataLakeStoreAccountProperties.CreationTime").Transform(convertDateToTime), + }, + { + Name: "current_tier", + Description: "The commitment tier in use for current month.", + Type: proto.ColumnType_STRING, + Hydrate: getDataLakeStore, + Transform: transform.FromField("DataLakeStoreAccountProperties.CurrentTier"), + }, + { + Name: "default_group", + Description: "The default owner group for all new folders and files created in the Data Lake Store account.", + Type: proto.ColumnType_STRING, + Hydrate: getDataLakeStore, + Transform: transform.FromField("DataLakeStoreAccountProperties.DefaultGroup"), + }, + { + Name: "encryption_state", + Description: "The current state of encryption for this Data Lake Store account.", + Type: proto.ColumnType_STRING, + Hydrate: getDataLakeStore, + Transform: transform.FromField("DataLakeStoreAccountProperties.EncryptionState"), + }, + { + Name: "encryption_provisioning_state", + Description: "The current state of encryption provisioning for this Data Lake Store account.", + Type: proto.ColumnType_STRING, + Hydrate: getDataLakeStore, + Transform: transform.FromField("DataLakeStoreAccountProperties.EncryptionProvisioningState"), }, { Name: "endpoint", Description: "The full CName endpoint for this account.", Type: proto.ColumnType_STRING, + Transform: transform.FromField("DataLakeStoreAccountPropertiesBasic.Endpoint", "DataLakeStoreAccountProperties.Endpoint"), + }, + { + Name: "firewall_state", + Description: "The current state of the IP address firewall for this Data Lake Store account.", + Type: proto.ColumnType_STRING, + Hydrate: getDataLakeStore, + Transform: transform.FromField("DataLakeStoreAccountProperties.FirewallState"), + }, + { + Name: "last_modified_time", + Description: "The account last modified time.", + Type: proto.ColumnType_TIMESTAMP, + Transform: transform.FromField("DataLakeStoreAccountPropertiesBasic.LastModifiedTime", "DataLakeStoreAccountProperties.LastModifiedTime").Transform(convertDateToTime), + }, + { + Name: "new_tier", + Description: "The commitment tier to use for next month.", + Type: proto.ColumnType_STRING, + Hydrate: getDataLakeStore, + Transform: transform.FromField("DataLakeStoreAccountProperties.NewTier"), }, { Name: "provisioning_state", Description: "The provisioning status of the Data Lake Store account.", Type: proto.ColumnType_STRING, - Transform: transform.FromField("DataLakeStoreAccountPropertiesBasic.ProvisioningState"), + Transform: transform.FromField("DataLakeStoreAccountPropertiesBasic.ProvisioningState", "DataLakeStoreAccountProperties.ProvisioningState"), }, { Name: "state", Description: "The state of the Data Lake Store account.", Type: proto.ColumnType_STRING, - Transform: transform.FromField("DataLakeStoreAccountPropertiesBasic.State"), + Transform: transform.FromField("DataLakeStoreAccountPropertiesBasic.State", "DataLakeStoreAccountProperties.State"), }, { - Name: "last_modified_time", - Description: "Managed service identity of the factory.", - Type: proto.ColumnType_TIMESTAMP, - Transform: transform.FromField("DataLakeStoreAccountPropertiesBasic.LastModifiedTime").Transform(convertDateToTime), + Name: "trusted_id_provider_state", + Description: "The current state of the trusted identity provider feature for this Data Lake Store account.", + Type: proto.ColumnType_STRING, + Hydrate: getDataLakeStore, + Transform: transform.FromField("DataLakeStoreAccountProperties.TrustedIDProviderState"), + }, + { + Name: "diagnostic_settings", + Description: "A list of active diagnostic settings for the Lake Store.", + Type: proto.ColumnType_JSON, + Hydrate: listDataLakeStoreDiagnosticSettings, + Transform: transform.FromValue(), + }, + { + Name: "encryption_config", + Description: "The Key Vault encryption configuration.", + Type: proto.ColumnType_JSON, + Hydrate: getDataLakeStore, + Transform: transform.FromField("DataLakeStoreAccountProperties.EncryptionConfig"), + }, + { + Name: "firewall_allow_azure_ips", + Description: "The current state of allowing or disallowing IPs originating within Azure through the firewall. If the firewall is disabled, this is not enforced.", + Type: proto.ColumnType_JSON, + Hydrate: getDataLakeStore, + Transform: transform.FromField("DataLakeStoreAccountProperties.FirewallAllowAzureIps"), + }, + { + Name: "firewall_rules", + Description: "The list of firewall rules associated with this Data Lake Store account.", + Type: proto.ColumnType_JSON, + Hydrate: getDataLakeStore, + Transform: transform.FromField("DataLakeStoreAccountProperties.FirewallRules"), + }, + { + Name: "identity", + Description: "The Key Vault encryption identity, if any.", + Type: proto.ColumnType_JSON, + Hydrate: getDataLakeStore, + }, + { + Name: "trusted_id_providers", + Description: "The list of trusted identity providers associated with this Data Lake Store account.", + Type: proto.ColumnType_JSON, + Hydrate: getDataLakeStore, + Transform: transform.FromField("DataLakeStoreAccountProperties.TrustedIDProviders"), + }, + { + Name: "virtual_network_rules", + Description: "The list of virtual network rules associated with this Data Lake Store account.", + Type: proto.ColumnType_JSON, + Hydrate: getDataLakeStore, + Transform: transform.FromField("DataLakeStoreAccountProperties.VirtualNetworkRules"), }, // Steampipe standard columns @@ -161,8 +261,16 @@ func getDataLakeStore(ctx context.Context, d *plugin.QueryData, h *plugin.Hydrat accountClient := account.NewAccountsClient(subscriptionID) accountClient.Authorizer = session.Authorizer - name := d.KeyColumnQuals["name"].GetStringValue() - resourceGroup := d.KeyColumnQuals["resource_group"].GetStringValue() + var name, resourceGroup string + if h.Item != nil { + data := h.Item.(account.DataLakeStoreAccountBasic) + splitID := strings.Split(*data.ID, "/") + name = *data.Name + resourceGroup = splitID[4] + } else { + name = d.KeyColumnQuals["name"].GetStringValue() + resourceGroup = d.KeyColumnQuals["resource_group"].GetStringValue() + } // Return nil, if no input provide if name == "" || resourceGroup == "" { @@ -176,3 +284,54 @@ func getDataLakeStore(ctx context.Context, d *plugin.QueryData, h *plugin.Hydrat return op, nil } + +func listDataLakeStoreDiagnosticSettings(ctx context.Context, d *plugin.QueryData, h *plugin.HydrateData) (interface{}, error) { + plugin.Logger(ctx).Trace("listDataLakeStoreDiagnosticSettings") + id := getLakeStoreID(h.Item) + + // Create session + session, err := GetNewSession(ctx, d, "MANAGEMENT") + if err != nil { + return nil, err + } + subscriptionID := session.SubscriptionID + + client := insights.NewDiagnosticSettingsClient(subscriptionID) + client.Authorizer = session.Authorizer + + op, err := client.List(ctx, id) + if err != nil { + return nil, err + } + + // If we return the API response directly, the output only gives + // the contents of DiagnosticSettings + var diagnosticSettings []map[string]interface{} + for _, i := range *op.Value { + objectMap := make(map[string]interface{}) + if i.ID != nil { + objectMap["id"] = i.ID + } + if i.Name != nil { + objectMap["name"] = i.Name + } + if i.Type != nil { + objectMap["type"] = i.Type + } + if i.DiagnosticSettings != nil { + objectMap["properties"] = i.DiagnosticSettings + } + diagnosticSettings = append(diagnosticSettings, objectMap) + } + return diagnosticSettings, nil +} + +func getLakeStoreID(item interface{}) string { + switch item := item.(type) { + case account.DataLakeStoreAccountBasic: + return *item.ID + case account.DataLakeStoreAccount: + return *item.ID + } + return "" +} diff --git a/docs/tables/azure_data_lake_store.md b/docs/tables/azure_data_lake_store.md index 5839ce66..095bbb0e 100644 --- a/docs/tables/azure_data_lake_store.md +++ b/docs/tables/azure_data_lake_store.md @@ -1,6 +1,6 @@ # Table: azure_data_lake_store -Azure Data Lake Storage Gen1 is an enterprise-wide hyper-scale repository for big data analytic workloads. Azure Data Lake enables you to capture data of any size, type, and ingestion speed in one single place for operational and exploratory analytics. +Azure Data Lake Storage is an enterprise-wide hyper-scale repository for big data analytic workloads. Azure Data Lake enables you to capture data of any size, type, and ingestion speed in one single place for operational and exploratory analytics. ## Examples @@ -15,3 +15,31 @@ select from azure_data_lake_store; ``` + +### List data lake stores with encryption disabled + +```sql +select + name, + id, + type, + provisioning_state +from + azure_data_lake_store +where + encryption_state = 'Disabled'; +``` + +### List data lake stores with firewall disabled + +```sql +select + name, + id, + type, + provisioning_state +from + azure_data_lake_store +where + firewall_state = 'Disabled'; +``` From 10fd266aff428296e03ff8b356865e5f954992f8 Mon Sep 17 00:00:00 2001 From: bigdatasourav Date: Sat, 31 Jul 2021 15:55:24 +0530 Subject: [PATCH 3/6] update --- azure/table_azure_data_lake_store.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/azure/table_azure_data_lake_store.go b/azure/table_azure_data_lake_store.go index b4ea59fe..0f7a3586 100644 --- a/azure/table_azure_data_lake_store.go +++ b/azure/table_azure_data_lake_store.go @@ -130,7 +130,7 @@ func tableAzureDataLakeStore(_ context.Context) *plugin.Table { }, { Name: "diagnostic_settings", - Description: "A list of active diagnostic settings for the Lake Store.", + Description: "A list of active diagnostic settings for the Data Lake Store.", Type: proto.ColumnType_JSON, Hydrate: listDataLakeStoreDiagnosticSettings, Transform: transform.FromValue(), From fb95ca28dab22548f87fbc63634c5d816aec71b7 Mon Sep 17 00:00:00 2001 From: bigdatasourav Date: Mon, 2 Aug 2021 10:34:41 +0530 Subject: [PATCH 4/6] update --- azure/table_azure_data_lake_store.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/azure/table_azure_data_lake_store.go b/azure/table_azure_data_lake_store.go index 0f7a3586..f06f5526 100644 --- a/azure/table_azure_data_lake_store.go +++ b/azure/table_azure_data_lake_store.go @@ -196,7 +196,7 @@ func tableAzureDataLakeStore(_ context.Context) *plugin.Table { Transform: transform.FromField("ID").Transform(idToAkas), }, - // Azure standard column + // Azure standard columns { Name: "region", Description: ColumnDescriptionRegion, From 9a8c0fbcc8ecb695f700fa838b0c7e048f8937c6 Mon Sep 17 00:00:00 2001 From: bigdatasourav Date: Thu, 5 Aug 2021 13:58:46 +0530 Subject: [PATCH 5/6] update --- azure/table_azure_data_lake_store.go | 32 ++++++++++++++-------------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/azure/table_azure_data_lake_store.go b/azure/table_azure_data_lake_store.go index f06f5526..2f6ab5bd 100644 --- a/azure/table_azure_data_lake_store.go +++ b/azure/table_azure_data_lake_store.go @@ -45,7 +45,7 @@ func tableAzureDataLakeStore(_ context.Context) *plugin.Table { }, { Name: "account_id", - Description: "The unique identifier associated with this Data Lake Store account.", + Description: "The unique identifier associated with this data lake store account.", Type: proto.ColumnType_STRING, Transform: transform.FromField("DataLakeStoreAccountPropertiesBasic.AccountID", "DataLakeStoreAccountProperties.AccountID"), }, @@ -64,34 +64,34 @@ func tableAzureDataLakeStore(_ context.Context) *plugin.Table { }, { Name: "default_group", - Description: "The default owner group for all new folders and files created in the Data Lake Store account.", + Description: "The default owner group for all new folders and files created in the data lake store account.", Type: proto.ColumnType_STRING, Hydrate: getDataLakeStore, Transform: transform.FromField("DataLakeStoreAccountProperties.DefaultGroup"), }, { Name: "encryption_state", - Description: "The current state of encryption for this Data Lake Store account.", + Description: "The current state of encryption for this data lake store account.", Type: proto.ColumnType_STRING, Hydrate: getDataLakeStore, Transform: transform.FromField("DataLakeStoreAccountProperties.EncryptionState"), }, { Name: "encryption_provisioning_state", - Description: "The current state of encryption provisioning for this Data Lake Store account.", + Description: "The current state of encryption provisioning for this data lake store account.", Type: proto.ColumnType_STRING, Hydrate: getDataLakeStore, Transform: transform.FromField("DataLakeStoreAccountProperties.EncryptionProvisioningState"), }, { Name: "endpoint", - Description: "The full CName endpoint for this account.", + Description: "The full cname endpoint for this account.", Type: proto.ColumnType_STRING, Transform: transform.FromField("DataLakeStoreAccountPropertiesBasic.Endpoint", "DataLakeStoreAccountProperties.Endpoint"), }, { Name: "firewall_state", - Description: "The current state of the IP address firewall for this Data Lake Store account.", + Description: "The current state of the IP address firewall for this data lake store account.", Type: proto.ColumnType_STRING, Hydrate: getDataLakeStore, Transform: transform.FromField("DataLakeStoreAccountProperties.FirewallState"), @@ -111,67 +111,67 @@ func tableAzureDataLakeStore(_ context.Context) *plugin.Table { }, { Name: "provisioning_state", - Description: "The provisioning status of the Data Lake Store account.", + Description: "The provisioning status of the data lake store account.", Type: proto.ColumnType_STRING, Transform: transform.FromField("DataLakeStoreAccountPropertiesBasic.ProvisioningState", "DataLakeStoreAccountProperties.ProvisioningState"), }, { Name: "state", - Description: "The state of the Data Lake Store account.", + Description: "The state of the data lake store account.", Type: proto.ColumnType_STRING, Transform: transform.FromField("DataLakeStoreAccountPropertiesBasic.State", "DataLakeStoreAccountProperties.State"), }, { Name: "trusted_id_provider_state", - Description: "The current state of the trusted identity provider feature for this Data Lake Store account.", + Description: "The current state of the trusted identity provider feature for this data lake store account.", Type: proto.ColumnType_STRING, Hydrate: getDataLakeStore, Transform: transform.FromField("DataLakeStoreAccountProperties.TrustedIDProviderState"), }, { Name: "diagnostic_settings", - Description: "A list of active diagnostic settings for the Data Lake Store.", + Description: "A list of active diagnostic settings for the data lake store.", Type: proto.ColumnType_JSON, Hydrate: listDataLakeStoreDiagnosticSettings, Transform: transform.FromValue(), }, { Name: "encryption_config", - Description: "The Key Vault encryption configuration.", + Description: "The key vault encryption configuration.", Type: proto.ColumnType_JSON, Hydrate: getDataLakeStore, Transform: transform.FromField("DataLakeStoreAccountProperties.EncryptionConfig"), }, { Name: "firewall_allow_azure_ips", - Description: "The current state of allowing or disallowing IPs originating within Azure through the firewall. If the firewall is disabled, this is not enforced.", + Description: "The current state of allowing or disallowing IPs originating within azure through the firewall. If the firewall is disabled, this is not enforced.", Type: proto.ColumnType_JSON, Hydrate: getDataLakeStore, Transform: transform.FromField("DataLakeStoreAccountProperties.FirewallAllowAzureIps"), }, { Name: "firewall_rules", - Description: "The list of firewall rules associated with this Data Lake Store account.", + Description: "The list of firewall rules associated with this data lake store account.", Type: proto.ColumnType_JSON, Hydrate: getDataLakeStore, Transform: transform.FromField("DataLakeStoreAccountProperties.FirewallRules"), }, { Name: "identity", - Description: "The Key Vault encryption identity, if any.", + Description: "The key vault encryption identity, if any.", Type: proto.ColumnType_JSON, Hydrate: getDataLakeStore, }, { Name: "trusted_id_providers", - Description: "The list of trusted identity providers associated with this Data Lake Store account.", + Description: "The list of trusted identity providers associated with this data lake store account.", Type: proto.ColumnType_JSON, Hydrate: getDataLakeStore, Transform: transform.FromField("DataLakeStoreAccountProperties.TrustedIDProviders"), }, { Name: "virtual_network_rules", - Description: "The list of virtual network rules associated with this Data Lake Store account.", + Description: "The list of virtual network rules associated with this data lake store account.", Type: proto.ColumnType_JSON, Hydrate: getDataLakeStore, Transform: transform.FromField("DataLakeStoreAccountProperties.VirtualNetworkRules"), From efb21362f3e8bdabfe628e4e2213dc0ac8f8e763 Mon Sep 17 00:00:00 2001 From: bigdatasourav Date: Thu, 5 Aug 2021 18:13:22 +0530 Subject: [PATCH 6/6] update --- azure/table_azure_data_lake_store.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/azure/table_azure_data_lake_store.go b/azure/table_azure_data_lake_store.go index 2f6ab5bd..7dc72609 100644 --- a/azure/table_azure_data_lake_store.go +++ b/azure/table_azure_data_lake_store.go @@ -21,7 +21,7 @@ func tableAzureDataLakeStore(_ context.Context) *plugin.Table { Get: &plugin.GetConfig{ KeyColumns: plugin.AllColumns([]string{"name", "resource_group"}), Hydrate: getDataLakeStore, - ShouldIgnoreError: isNotFoundError([]string{"ResourceNotFound", "ResourceGroupNotFound"}), + ShouldIgnoreError: isNotFoundError([]string{"ResourceNotFound", "ResourceGroupNotFound", "400"}), }, List: &plugin.ListConfig{ Hydrate: listDataLakeStores,