From 48e2d848ecfbd2514516a26f78dc6c50c0539343 Mon Sep 17 00:00:00 2001 From: rajeshbal65 Date: Thu, 1 Jul 2021 14:14:46 +0530 Subject: [PATCH 1/6] Add table azure_data_factory. Closes #156 --- .../tests/azure_data_factory/dependencies.txt | 0 .../azure_data_factory/test-get-expected.json | 13 ++ .../azure_data_factory/test-get-query.sql | 3 + .../test-list-expected.json | 7 + .../azure_data_factory/test-list-query.sql | 3 + .../test-not-found-expected.json | 1 + .../test-not-found-query.sql | 3 + .../test-turbot-expected.json | 13 ++ .../azure_data_factory/test-turbot-query.sql | 3 + .../tests/azure_data_factory/variables.json | 1 + .../tests/azure_data_factory/variables.tf | 67 ++++++ azure/plugin.go | 1 + azure/table_azure_data_factory.go | 195 ++++++++++++++++++ docs/tables/azure_data_factory.md | 48 +++++ 14 files changed, 358 insertions(+) create mode 100644 azure-test/tests/azure_data_factory/dependencies.txt create mode 100644 azure-test/tests/azure_data_factory/test-get-expected.json create mode 100644 azure-test/tests/azure_data_factory/test-get-query.sql create mode 100644 azure-test/tests/azure_data_factory/test-list-expected.json create mode 100644 azure-test/tests/azure_data_factory/test-list-query.sql create mode 100644 azure-test/tests/azure_data_factory/test-not-found-expected.json create mode 100644 azure-test/tests/azure_data_factory/test-not-found-query.sql create mode 100644 azure-test/tests/azure_data_factory/test-turbot-expected.json create mode 100644 azure-test/tests/azure_data_factory/test-turbot-query.sql create mode 100644 azure-test/tests/azure_data_factory/variables.json create mode 100644 azure-test/tests/azure_data_factory/variables.tf create mode 100644 azure/table_azure_data_factory.go create mode 100644 docs/tables/azure_data_factory.md diff --git a/azure-test/tests/azure_data_factory/dependencies.txt b/azure-test/tests/azure_data_factory/dependencies.txt new file mode 100644 index 00000000..e69de29b diff --git a/azure-test/tests/azure_data_factory/test-get-expected.json b/azure-test/tests/azure_data_factory/test-get-expected.json new file mode 100644 index 00000000..619e2748 --- /dev/null +++ b/azure-test/tests/azure_data_factory/test-get-expected.json @@ -0,0 +1,13 @@ +[ + { + "id": "{{ output.resource_id.value }}", + "name": "{{resourceName}}", + "region": "eastus", + "resource_group": "{{resourceName}}", + "subscription_id": "{{ output.subscription_id.value }}", + "tags": { + "name": "{{resourceName}}" + }, + "type": "Microsoft.DataFactory/factories" + } +] \ No newline at end of file diff --git a/azure-test/tests/azure_data_factory/test-get-query.sql b/azure-test/tests/azure_data_factory/test-get-query.sql new file mode 100644 index 00000000..83808052 --- /dev/null +++ b/azure-test/tests/azure_data_factory/test-get-query.sql @@ -0,0 +1,3 @@ +select name, id, type, tags, region, resource_group, subscription_id +from azure.azure_data_factory +where name = '{{resourceName}}' and resource_group = '{{resourceName}}' \ No newline at end of file diff --git a/azure-test/tests/azure_data_factory/test-list-expected.json b/azure-test/tests/azure_data_factory/test-list-expected.json new file mode 100644 index 00000000..f15afcd8 --- /dev/null +++ b/azure-test/tests/azure_data_factory/test-list-expected.json @@ -0,0 +1,7 @@ +[ + { + "id": "{{ output.resource_id.value }}", + "name": "{{resourceName}}", + "type": "Microsoft.DataFactory/factories" + } +] \ No newline at end of file diff --git a/azure-test/tests/azure_data_factory/test-list-query.sql b/azure-test/tests/azure_data_factory/test-list-query.sql new file mode 100644 index 00000000..f900f679 --- /dev/null +++ b/azure-test/tests/azure_data_factory/test-list-query.sql @@ -0,0 +1,3 @@ +select name, id, type +from azure.azure_data_factory +where name = '{{resourceName}}' \ No newline at end of file diff --git a/azure-test/tests/azure_data_factory/test-not-found-expected.json b/azure-test/tests/azure_data_factory/test-not-found-expected.json new file mode 100644 index 00000000..ec747fa4 --- /dev/null +++ b/azure-test/tests/azure_data_factory/test-not-found-expected.json @@ -0,0 +1 @@ +null \ No newline at end of file diff --git a/azure-test/tests/azure_data_factory/test-not-found-query.sql b/azure-test/tests/azure_data_factory/test-not-found-query.sql new file mode 100644 index 00000000..04ecb573 --- /dev/null +++ b/azure-test/tests/azure_data_factory/test-not-found-query.sql @@ -0,0 +1,3 @@ +select name, tags, title, akas +from azure.azure_data_factory +where name = 'dummy-{{resourceName}}' and resource_group = '{{resourceName}}' \ No newline at end of file diff --git a/azure-test/tests/azure_data_factory/test-turbot-expected.json b/azure-test/tests/azure_data_factory/test-turbot-expected.json new file mode 100644 index 00000000..3ceae2f7 --- /dev/null +++ b/azure-test/tests/azure_data_factory/test-turbot-expected.json @@ -0,0 +1,13 @@ +[ + { + "akas": [ + "{{ output.resource_aka.value }}", + "{{ output.resource_aka_lower.value }}" + ], + "name": "{{resourceName}}", + "tags": { + "name": "{{resourceName}}" + }, + "title": "{{resourceName}}" + } +] \ No newline at end of file diff --git a/azure-test/tests/azure_data_factory/test-turbot-query.sql b/azure-test/tests/azure_data_factory/test-turbot-query.sql new file mode 100644 index 00000000..434fc5ae --- /dev/null +++ b/azure-test/tests/azure_data_factory/test-turbot-query.sql @@ -0,0 +1,3 @@ +select name, tags, title, akas +from azure.azure_data_factory +where name = '{{resourceName}}' and resource_group = '{{resourceName}}' \ No newline at end of file diff --git a/azure-test/tests/azure_data_factory/variables.json b/azure-test/tests/azure_data_factory/variables.json new file mode 100644 index 00000000..0967ef42 --- /dev/null +++ b/azure-test/tests/azure_data_factory/variables.json @@ -0,0 +1 @@ +{} diff --git a/azure-test/tests/azure_data_factory/variables.tf b/azure-test/tests/azure_data_factory/variables.tf new file mode 100644 index 00000000..20dd0733 --- /dev/null +++ b/azure-test/tests/azure_data_factory/variables.tf @@ -0,0 +1,67 @@ + +variable "resource_name" { + type = string + default = "steampipe-test" + description = "Name of the resource used throughout the test." +} + +variable "azure_environment" { + type = string + default = "public" + description = "Azure environment used for the test." +} + +variable "azure_subscription" { + type = string + default = "3510ae4d-530b-497d-8f30-53b9616fc6c1" + description = "Azure subscription used for the test." +} + +provider "azurerm" { + # Cannot be passed as a variable + version = "=1.36.0" + environment = var.azure_environment + subscription_id = var.azure_subscription +} + +data "azurerm_client_config" "current" {} + +data "null_data_source" "resource" { + inputs = { + scope = "azure:///subscriptions/${data.azurerm_client_config.current.subscription_id}" + } +} + +resource "azurerm_resource_group" "named_test_resource" { + name = var.resource_name + location = "East US" +} + +resource "azurerm_data_factory" "named_test_resource" { + name = var.resource_name + location = "East US" + resource_group_name = azurerm_resource_group.named_test_resource.name + tags = { + name = var.resource_name + } +} + +output "resource_aka" { + value = "azure://${azurerm_data_factory.named_test_resource.id}" +} + +output "resource_aka_lower" { + value = "azure://${lower(azurerm_data_factory.named_test_resource.id)}" +} + +output "resource_name" { + value = var.resource_name +} + +output "resource_id" { + value = azurerm_data_factory.named_test_resource.id +} + +output "subscription_id" { + value = var.azure_subscription +} diff --git a/azure/plugin.go b/azure/plugin.go index 63feab55..c165e4c8 100644 --- a/azure/plugin.go +++ b/azure/plugin.go @@ -41,6 +41,7 @@ func Plugin(ctx context.Context) *plugin.Plugin { "azure_cosmosdb_account": tableAzureCosmosDBAccount(ctx), "azure_cosmosdb_mongo_database": tableAzureCosmosDBMongoDatabase(ctx), "azure_cosmosdb_sql_database": tableAzureCosmosDBSQLDatabase(ctx), + "azure_data_factory": tableAzureDataFactory(ctx), "azure_diagnostic_setting": tableAzureDiagnosticSetting(ctx), "azure_firewall": tableAzureFirewall(ctx), "azure_key_vault": tableAzureKeyVault(ctx), diff --git a/azure/table_azure_data_factory.go b/azure/table_azure_data_factory.go new file mode 100644 index 00000000..ae62b6f2 --- /dev/null +++ b/azure/table_azure_data_factory.go @@ -0,0 +1,195 @@ +package azure + +import ( + "context" + + "github.com/Azure/azure-sdk-for-go/services/datafactory/mgmt/2018-06-01/datafactory" + "github.com/turbot/steampipe-plugin-sdk/grpc/proto" + "github.com/turbot/steampipe-plugin-sdk/plugin/transform" + + "github.com/turbot/steampipe-plugin-sdk/plugin" +) + +//// TABLE DEFINITION //// + +func tableAzureDataFactory(_ context.Context) *plugin.Table { + return &plugin.Table{ + Name: "azure_data_factory", + Description: "Azure Data Factory", + Get: &plugin.GetConfig{ + KeyColumns: plugin.AllColumns([]string{"name", "resource_group"}), + Hydrate: getFactory, + ShouldIgnoreError: isNotFoundError([]string{"ResourceNotFound", "ResourceGroupNotFound", "404"}), + }, + List: &plugin.ListConfig{ + Hydrate: listFactories, + }, + Columns: []*plugin.Column{ + { + Name: "name", + Description: "The resource name.", + Type: proto.ColumnType_STRING, + }, + { + Name: "id", + Description: "The resource identifier.", + Type: proto.ColumnType_STRING, + Transform: transform.FromGo(), + }, + { + Name: "etag", + Description: "Etag identifies change in the resource.", + Type: proto.ColumnType_STRING, + Transform: transform.FromField("ETag"), + }, + { + Name: "type", + Description: "The resource type.", + Type: proto.ColumnType_STRING, + }, + { + Name: "provisioning_state", + Description: "Factory provisioning state, example Succeeded.", + Type: proto.ColumnType_STRING, + Transform: transform.FromField("FactoryProperties.ProvisioningState"), + }, + { + Name: "create_time", + Description: "Time the factory was created in ISO8601 format.", + Type: proto.ColumnType_TIMESTAMP, + Transform: transform.FromField("FactoryProperties.CreateTime").Transform(convertDateToTime), + }, + { + Name: "version", + Description: "Version of the factory.", + Type: proto.ColumnType_STRING, + Transform: transform.FromField("FactoryProperties.Version"), + }, + { + Name: "public_network_access", + Description: "Whether or not public network access is allowed for the data factory.", + Type: proto.ColumnType_STRING, + Transform: transform.FromField("FactoryProperties.PublicNetworkAccess"), + }, + { + Name: "identity", + Description: "Managed service identity of the factory.", + Type: proto.ColumnType_JSON, + }, + { + Name: "encryption", + Description: "Properties to enable Customer Managed Key for the factory.", + Type: proto.ColumnType_JSON, + Transform: transform.FromField("FactoryProperties.EncryptionConfiguration"), + }, + { + Name: "repo_configuration", + Description: "Git repo information of the factory.", + Type: proto.ColumnType_JSON, + Transform: transform.FromField("FactoryProperties.RepoConfiguration"), + }, + { + Name: "global_parameters", + Description: "List of parameters for factory.", + Type: proto.ColumnType_JSON, + Transform: transform.FromField("FactoryProperties.GlobalParameters"), + }, + + // Standard columns + { + Name: "title", + Description: ColumnDescriptionTitle, + Type: proto.ColumnType_STRING, + Transform: transform.FromField("Name"), + }, + { + Name: "tags", + Description: ColumnDescriptionTags, + Type: proto.ColumnType_JSON, + }, + { + Name: "akas", + Description: ColumnDescriptionAkas, + Type: proto.ColumnType_JSON, + Transform: transform.FromField("ID").Transform(idToAkas), + }, + { + Name: "region", + Description: ColumnDescriptionRegion, + Type: proto.ColumnType_STRING, + Transform: transform.FromField("Location").Transform(toLower), + }, + { + Name: "resource_group", + Description: ColumnDescriptionResourceGroup, + Type: proto.ColumnType_STRING, + Transform: transform.FromField("ID").Transform(extractResourceGroupFromID), + }, + { + Name: "subscription_id", + Description: ColumnDescriptionSubscription, + Type: proto.ColumnType_STRING, + Transform: transform.FromField("ID").Transform(idToSubscriptionID), + }, + }, + } +} + +//// LIST FUNCTIONS //// + +func listFactories(ctx context.Context, d *plugin.QueryData, _ *plugin.HydrateData) (interface{}, error) { + session, err := GetNewSession(ctx, d, "MANAGEMENT") + if err != nil { + return nil, err + } + subscriptionID := session.SubscriptionID + + factoryClient := datafactory.NewFactoriesClient(subscriptionID) + factoryClient.Authorizer = session.Authorizer + pagesLeft := true + + for pagesLeft { + result, err := factoryClient.List(context.Background()) + if err != nil { + return nil, err + } + + for _, factory := range result.Values() { + d.StreamListItem(ctx, factory) + } + result.NextWithContext(context.Background()) + pagesLeft = result.NotDone() + } + return nil, err +} + +//// HYDRATE FUNCTIONS //// + +func getFactory(ctx context.Context, d *plugin.QueryData, h *plugin.HydrateData) (interface{}, error) { + plugin.Logger(ctx).Trace("getFactory") + + name := d.KeyColumnQuals["name"].GetStringValue() + resourceGroup := d.KeyColumnQuals["resource_group"].GetStringValue() + + session, err := GetNewSession(ctx, d, "MANAGEMENT") + if err != nil { + return nil, err + } + subscriptionID := session.SubscriptionID + + factoryClient := datafactory.NewFactoriesClient(subscriptionID) + factoryClient.Authorizer = session.Authorizer + + op, err := factoryClient.Get(ctx, resourceGroup, name, "*") + if err != nil { + return nil, err + } + + // In some cases resource does not give any notFound error + // instead of notFound error, it returns empty data + if op.ID != nil { + return op, nil + } + + return nil, nil +} diff --git a/docs/tables/azure_data_factory.md b/docs/tables/azure_data_factory.md new file mode 100644 index 00000000..9d2aef7e --- /dev/null +++ b/docs/tables/azure_data_factory.md @@ -0,0 +1,48 @@ +# Table: azure_data_factory + +Azure Data Factory is the platform that solves such data scenarios. It is the cloud-based ETL and data integration service that allows to create data-driven workflows for orchestrating data movement and transforming data at scale. + +## Examples + +### Basic info + +```sql +select + name, + id, + type, + provisioning_state, + etag +from + azure_data_factory; +``` + + +### List system assigned identity type Factories + +```sql +select + name, + id, + type, + identity ->> 'type' as identity_type +from + azure_data_factory +where + identity ->> 'type' = 'SystemAssigned'; +``` + + +### List factories with plubic network access is allowed + +```sql +select + name, + id, + type, + public_network_access +from + azure_data_factory +where + public_network_access = 'Enabled'; +``` \ No newline at end of file From 74a0b022a4f57033506361c8aac3a9b42e12720d Mon Sep 17 00:00:00 2001 From: rajeshbal65 Date: Mon, 5 Jul 2021 14:20:46 +0530 Subject: [PATCH 2/6] wip --- .../dependencies.txt | 0 .../test-get-expected.json | 9 + .../test-get-query.sql | 3 + .../test-list-expected.json | 7 + .../test-list-query.sql | 3 + .../test-not-found-expected.json | 1 + .../test-not-found-query.sql | 3 + .../test-turbot-expected.json | 10 ++ .../test-turbot-query.sql | 3 + .../azure_data_factory_dataset/variables.json | 1 + .../azure_data_factory_dataset/variables.tf | 81 +++++++++ azure/plugin.go | 1 + azure/table_azure_data_factory_dataset.go | 159 ++++++++++++++++++ 13 files changed, 281 insertions(+) create mode 100644 azure-test/tests/azure_data_factory_dataset/dependencies.txt create mode 100644 azure-test/tests/azure_data_factory_dataset/test-get-expected.json create mode 100644 azure-test/tests/azure_data_factory_dataset/test-get-query.sql create mode 100644 azure-test/tests/azure_data_factory_dataset/test-list-expected.json create mode 100644 azure-test/tests/azure_data_factory_dataset/test-list-query.sql create mode 100644 azure-test/tests/azure_data_factory_dataset/test-not-found-expected.json create mode 100644 azure-test/tests/azure_data_factory_dataset/test-not-found-query.sql create mode 100644 azure-test/tests/azure_data_factory_dataset/test-turbot-expected.json create mode 100644 azure-test/tests/azure_data_factory_dataset/test-turbot-query.sql create mode 100644 azure-test/tests/azure_data_factory_dataset/variables.json create mode 100644 azure-test/tests/azure_data_factory_dataset/variables.tf create mode 100644 azure/table_azure_data_factory_dataset.go diff --git a/azure-test/tests/azure_data_factory_dataset/dependencies.txt b/azure-test/tests/azure_data_factory_dataset/dependencies.txt new file mode 100644 index 00000000..e69de29b diff --git a/azure-test/tests/azure_data_factory_dataset/test-get-expected.json b/azure-test/tests/azure_data_factory_dataset/test-get-expected.json new file mode 100644 index 00000000..f7248842 --- /dev/null +++ b/azure-test/tests/azure_data_factory_dataset/test-get-expected.json @@ -0,0 +1,9 @@ +[ + { + "id": "{{ output.resource_id.value }}", + "name": "{{resourceName}}", + "resource_group": "{{resourceName}}", + "subscription_id": "{{ output.subscription_id.value }}", + "type": "Microsoft.DataFactory/factories" + } +] \ No newline at end of file diff --git a/azure-test/tests/azure_data_factory_dataset/test-get-query.sql b/azure-test/tests/azure_data_factory_dataset/test-get-query.sql new file mode 100644 index 00000000..f3ad768a --- /dev/null +++ b/azure-test/tests/azure_data_factory_dataset/test-get-query.sql @@ -0,0 +1,3 @@ +select name, id, type, resource_group, subscription_id +from azure.azure_data_factory_dataset +where name = '{{resourceName}}' and resource_group = '{{resourceName}}' \ No newline at end of file diff --git a/azure-test/tests/azure_data_factory_dataset/test-list-expected.json b/azure-test/tests/azure_data_factory_dataset/test-list-expected.json new file mode 100644 index 00000000..f15afcd8 --- /dev/null +++ b/azure-test/tests/azure_data_factory_dataset/test-list-expected.json @@ -0,0 +1,7 @@ +[ + { + "id": "{{ output.resource_id.value }}", + "name": "{{resourceName}}", + "type": "Microsoft.DataFactory/factories" + } +] \ No newline at end of file diff --git a/azure-test/tests/azure_data_factory_dataset/test-list-query.sql b/azure-test/tests/azure_data_factory_dataset/test-list-query.sql new file mode 100644 index 00000000..2f6a45eb --- /dev/null +++ b/azure-test/tests/azure_data_factory_dataset/test-list-query.sql @@ -0,0 +1,3 @@ +select name, id, type +from azure.azure_data_factory_dataset +where name = '{{resourceName}}' \ No newline at end of file diff --git a/azure-test/tests/azure_data_factory_dataset/test-not-found-expected.json b/azure-test/tests/azure_data_factory_dataset/test-not-found-expected.json new file mode 100644 index 00000000..ec747fa4 --- /dev/null +++ b/azure-test/tests/azure_data_factory_dataset/test-not-found-expected.json @@ -0,0 +1 @@ +null \ No newline at end of file diff --git a/azure-test/tests/azure_data_factory_dataset/test-not-found-query.sql b/azure-test/tests/azure_data_factory_dataset/test-not-found-query.sql new file mode 100644 index 00000000..61771cb4 --- /dev/null +++ b/azure-test/tests/azure_data_factory_dataset/test-not-found-query.sql @@ -0,0 +1,3 @@ +select name, tags, title, akas +from azure.azure_data_factory_dataset +where name = 'dummy-{{resourceName}}' and resource_group = '{{resourceName}}' \ No newline at end of file diff --git a/azure-test/tests/azure_data_factory_dataset/test-turbot-expected.json b/azure-test/tests/azure_data_factory_dataset/test-turbot-expected.json new file mode 100644 index 00000000..5d3863e6 --- /dev/null +++ b/azure-test/tests/azure_data_factory_dataset/test-turbot-expected.json @@ -0,0 +1,10 @@ +[ + { + "akas": [ + "{{ output.resource_aka.value }}", + "{{ output.resource_aka_lower.value }}" + ], + "name": "{{resourceName}}", + "title": "{{resourceName}}" + } +] \ No newline at end of file diff --git a/azure-test/tests/azure_data_factory_dataset/test-turbot-query.sql b/azure-test/tests/azure_data_factory_dataset/test-turbot-query.sql new file mode 100644 index 00000000..d7bb537b --- /dev/null +++ b/azure-test/tests/azure_data_factory_dataset/test-turbot-query.sql @@ -0,0 +1,3 @@ +select name, title, akas +from azure.azure_data_factory_dataset +where name = '{{resourceName}}' and resource_group = '{{resourceName}}' \ No newline at end of file diff --git a/azure-test/tests/azure_data_factory_dataset/variables.json b/azure-test/tests/azure_data_factory_dataset/variables.json new file mode 100644 index 00000000..0967ef42 --- /dev/null +++ b/azure-test/tests/azure_data_factory_dataset/variables.json @@ -0,0 +1 @@ +{} diff --git a/azure-test/tests/azure_data_factory_dataset/variables.tf b/azure-test/tests/azure_data_factory_dataset/variables.tf new file mode 100644 index 00000000..eed86cf6 --- /dev/null +++ b/azure-test/tests/azure_data_factory_dataset/variables.tf @@ -0,0 +1,81 @@ + +variable "resource_name" { + type = string + default = "steampipe-test" + description = "Name of the resource used throughout the test." +} + +variable "azure_environment" { + type = string + default = "public" + description = "Azure environment used for the test." +} + +variable "azure_subscription" { + type = string + default = "3510ae4d-530b-497d-8f30-53b9616fc6c1" + description = "Azure subscription used for the test." +} + +provider "azurerm" { + # Cannot be passed as a variable + version = "=1.36.0" + environment = var.azure_environment + subscription_id = var.azure_subscription +} + +data "azurerm_client_config" "current" {} + +data "null_data_source" "resource" { + inputs = { + scope = "azure:///subscriptions/${data.azurerm_client_config.current.subscription_id}" + } +} + +resource "azurerm_resource_group" "named_test_resource" { + name = var.resource_name + location = "East US" +} + +resource "azurerm_data_factory" "named_test_resource" { + name = var.resource_name + location = "East US" + resource_group_name = azurerm_resource_group.named_test_resource.name + tags = { + name = var.resource_name + } +} + +resource "azurerm_data_factory_linked_service_mysql" "named_test_resource" { + name = var.resource_name + resource_group_name = azurerm_resource_group.named_test_resource.name + data_factory_name = azurerm_data_factory.named_test_resource.name + connection_string = "Server=test;Port=3306;Database=test;User=test;SSLMode=1;UseSystemTrustStore=0;Password=test" +} + +resource "azurerm_data_factory_dataset_mysql" "named_test_resource" { + name = var.resource_name + resource_group_name = azurerm_resource_group.named_test_resource.name + data_factory_name = azurerm_data_factory.named_test_resource.name + linked_service_name = azurerm_data_factory_linked_service_mysql.named_test_resource.name +} + +output "resource_aka" { + value = "azure://${azurerm_data_factory_dataset_mysql.named_test_resource.id}" +} + +output "resource_aka_lower" { + value = "azure://${lower(azurerm_data_factory_dataset_mysql.named_test_resource.id)}" +} + +output "resource_name" { + value = var.resource_name +} + +output "resource_id" { + value = azurerm_data_factory_dataset_mysql.named_test_resource.id +} + +output "subscription_id" { + value = var.azure_subscription +} diff --git a/azure/plugin.go b/azure/plugin.go index c165e4c8..af103ce9 100644 --- a/azure/plugin.go +++ b/azure/plugin.go @@ -42,6 +42,7 @@ func Plugin(ctx context.Context) *plugin.Plugin { "azure_cosmosdb_mongo_database": tableAzureCosmosDBMongoDatabase(ctx), "azure_cosmosdb_sql_database": tableAzureCosmosDBSQLDatabase(ctx), "azure_data_factory": tableAzureDataFactory(ctx), + "azure_data_factory_dataset": tableAzureDataFactoryDataset(ctx), "azure_diagnostic_setting": tableAzureDiagnosticSetting(ctx), "azure_firewall": tableAzureFirewall(ctx), "azure_key_vault": tableAzureKeyVault(ctx), diff --git a/azure/table_azure_data_factory_dataset.go b/azure/table_azure_data_factory_dataset.go new file mode 100644 index 00000000..4f079569 --- /dev/null +++ b/azure/table_azure_data_factory_dataset.go @@ -0,0 +1,159 @@ +package azure + +import ( + "context" + "strings" + + "github.com/Azure/azure-sdk-for-go/services/datafactory/mgmt/2018-06-01/datafactory" + "github.com/turbot/steampipe-plugin-sdk/grpc/proto" + "github.com/turbot/steampipe-plugin-sdk/plugin/transform" + + "github.com/turbot/steampipe-plugin-sdk/plugin" +) + +//// TABLE DEFINITION //// + +func tableAzureDataFactoryDataset(_ context.Context) *plugin.Table { + return &plugin.Table{ + Name: "azure_data_factory_dataset", + Description: "Azure Data Factory Dataset", + Get: &plugin.GetConfig{ + KeyColumns: plugin.AllColumns([]string{"name", "resource_group", "factory_name"}), + Hydrate: getDataset, + ShouldIgnoreError: isNotFoundError([]string{"ResourceNotFound", "ResourceGroupNotFound", "404"}), + }, + List: &plugin.ListConfig{ + Hydrate: listDatasets, + ParentHydrate: listFactories, + }, + Columns: []*plugin.Column{ + { + Name: "name", + Description: "The resource name.", + Type: proto.ColumnType_STRING, + }, + { + Name: "id", + Description: "The resource identifier.", + Type: proto.ColumnType_STRING, + Transform: transform.FromGo(), + }, + { + Name: "etag", + Description: "Etag identifies change in the resource.", + Type: proto.ColumnType_STRING, + Transform: transform.FromField("Etag"), + }, + { + Name: "type", + Description: "The resource type.", + Type: proto.ColumnType_STRING, + }, + { + Name: "factory_name", + Description: "Time the factory was created in ISO8601 format.", + Type: proto.ColumnType_STRING, + }, + { + Name: "properties", + Description: "Dataset ElapsedTime Metric Policy.", + Type: proto.ColumnType_JSON, + Transform: transform.FromField("Properties"), + }, + // Standard columns + { + Name: "title", + Description: ColumnDescriptionTitle, + Type: proto.ColumnType_STRING, + Transform: transform.FromField("Name"), + }, + { + Name: "akas", + Description: ColumnDescriptionAkas, + Type: proto.ColumnType_JSON, + Transform: transform.FromField("ID").Transform(idToAkas), + }, + { + Name: "resource_group", + Description: ColumnDescriptionResourceGroup, + Type: proto.ColumnType_STRING, + Transform: transform.FromField("ID").Transform(extractResourceGroupFromID), + }, + { + Name: "subscription_id", + Description: ColumnDescriptionSubscription, + Type: proto.ColumnType_STRING, + Transform: transform.FromField("ID").Transform(idToSubscriptionID), + }, + }, + } +} + +type DatasetInfo = struct { + datafactory.DatasetResource + FactoryName string +} + +//// LIST FUNCTIONS //// + +func listDatasets(ctx context.Context, d *plugin.QueryData, h *plugin.HydrateData) (interface{}, error) { + session, err := GetNewSession(ctx, d, "MANAGEMENT") + if err != nil { + return nil, err + } + + factoryInfo := h.Item.(datafactory.Factory) + resourceGroup := strings.Split(*factoryInfo.ID, "/")[4] + + subscriptionID := session.SubscriptionID + + datasetClient := datafactory.NewDatasetsClient(subscriptionID) + datasetClient.Authorizer = session.Authorizer + pagesLeft := true + + for pagesLeft { + result, err := datasetClient.ListByFactory(ctx, resourceGroup, *factoryInfo.Name) + if err != nil { + return nil, err + } + + for _, dataset := range result.Values() { + d.StreamListItem(ctx, DatasetInfo{dataset, *factoryInfo.Name}) + } + result.NextWithContext(context.Background()) + pagesLeft = result.NotDone() + } + return nil, err +} + +//// HYDRATE FUNCTIONS //// + +func getDataset(ctx context.Context, d *plugin.QueryData, h *plugin.HydrateData) (interface{}, error) { + plugin.Logger(ctx).Trace("getDataset") + + DatasetName := d.KeyColumnQuals["name"].GetStringValue() + resourceGroup := d.KeyColumnQuals["resource_group"].GetStringValue() + factoryName := d.KeyColumnQuals["factory_name"].GetStringValue() + + session, err := GetNewSession(ctx, d, "MANAGEMENT") + if err != nil { + return nil, err + } + subscriptionID := session.SubscriptionID + + datasetClient := datafactory.NewDatasetsClient(subscriptionID) + datasetClient.Authorizer = session.Authorizer + + op, err := datasetClient.Get(ctx, resourceGroup, factoryName, DatasetName, "") + if err != nil { + return nil, err + } + + // In some cases resource does not give any notFound error + // instead of notFound error, it returns empty data + if op.ID != nil { + return op, nil + } + + return op, nil +} From 69e2602977ad5e331604025fc120f57a126dc71b Mon Sep 17 00:00:00 2001 From: rajeshbal65 Date: Mon, 5 Jul 2021 14:57:36 +0530 Subject: [PATCH 3/6] Add table azure_datafactory_dataset. Closes #150 --- .../test-get-expected.json | 2 +- .../test-get-query.sql | 2 +- .../test-list-expected.json | 2 +- .../test-not-found-query.sql | 2 +- .../azure_data_factory_dataset/variables.tf | 2 +- azure/table_azure_data_factory_dataset.go | 11 +++++-- docs/tables/azure_data_factory_dataset.md | 33 +++++++++++++++++++ 7 files changed, 46 insertions(+), 8 deletions(-) create mode 100644 docs/tables/azure_data_factory_dataset.md diff --git a/azure-test/tests/azure_data_factory_dataset/test-get-expected.json b/azure-test/tests/azure_data_factory_dataset/test-get-expected.json index f7248842..8e7fd942 100644 --- a/azure-test/tests/azure_data_factory_dataset/test-get-expected.json +++ b/azure-test/tests/azure_data_factory_dataset/test-get-expected.json @@ -4,6 +4,6 @@ "name": "{{resourceName}}", "resource_group": "{{resourceName}}", "subscription_id": "{{ output.subscription_id.value }}", - "type": "Microsoft.DataFactory/factories" + "type": "Microsoft.DataFactory/factories/datasets" } ] \ No newline at end of file diff --git a/azure-test/tests/azure_data_factory_dataset/test-get-query.sql b/azure-test/tests/azure_data_factory_dataset/test-get-query.sql index f3ad768a..94b15b66 100644 --- a/azure-test/tests/azure_data_factory_dataset/test-get-query.sql +++ b/azure-test/tests/azure_data_factory_dataset/test-get-query.sql @@ -1,3 +1,3 @@ select name, id, type, resource_group, subscription_id from azure.azure_data_factory_dataset -where name = '{{resourceName}}' and resource_group = '{{resourceName}}' \ No newline at end of file +where name = '{{resourceName}}' and resource_group = '{{resourceName}}' and factory_name = '{{resourceName}}' \ No newline at end of file diff --git a/azure-test/tests/azure_data_factory_dataset/test-list-expected.json b/azure-test/tests/azure_data_factory_dataset/test-list-expected.json index f15afcd8..11083c14 100644 --- a/azure-test/tests/azure_data_factory_dataset/test-list-expected.json +++ b/azure-test/tests/azure_data_factory_dataset/test-list-expected.json @@ -2,6 +2,6 @@ { "id": "{{ output.resource_id.value }}", "name": "{{resourceName}}", - "type": "Microsoft.DataFactory/factories" + "type": "Microsoft.DataFactory/factories/datasets" } ] \ No newline at end of file diff --git a/azure-test/tests/azure_data_factory_dataset/test-not-found-query.sql b/azure-test/tests/azure_data_factory_dataset/test-not-found-query.sql index 61771cb4..76611712 100644 --- a/azure-test/tests/azure_data_factory_dataset/test-not-found-query.sql +++ b/azure-test/tests/azure_data_factory_dataset/test-not-found-query.sql @@ -1,3 +1,3 @@ -select name, tags, title, akas +select name, title, akas from azure.azure_data_factory_dataset where name = 'dummy-{{resourceName}}' and resource_group = '{{resourceName}}' \ No newline at end of file diff --git a/azure-test/tests/azure_data_factory_dataset/variables.tf b/azure-test/tests/azure_data_factory_dataset/variables.tf index eed86cf6..53b1d09b 100644 --- a/azure-test/tests/azure_data_factory_dataset/variables.tf +++ b/azure-test/tests/azure_data_factory_dataset/variables.tf @@ -13,7 +13,7 @@ variable "azure_environment" { variable "azure_subscription" { type = string - default = "3510ae4d-530b-497d-8f30-53b9616fc6c1" + default = "d7245080-b4ae-4fe5-b6fa-2e71b3dae6c8" description = "Azure subscription used for the test." } diff --git a/azure/table_azure_data_factory_dataset.go b/azure/table_azure_data_factory_dataset.go index 4f079569..184a7834 100644 --- a/azure/table_azure_data_factory_dataset.go +++ b/azure/table_azure_data_factory_dataset.go @@ -38,6 +38,12 @@ func tableAzureDataFactoryDataset(_ context.Context) *plugin.Table { Type: proto.ColumnType_STRING, Transform: transform.FromGo(), }, + { + Name: "description", + Description: "The description of the Dataset.", + Type: proto.ColumnType_STRING, + Transform: transform.FromField("Dataset.Description"), + }, { Name: "etag", Description: "Etag identifies change in the resource.", @@ -58,7 +64,6 @@ func tableAzureDataFactoryDataset(_ context.Context) *plugin.Table { Name: "properties", Description: "Dataset ElapsedTime Metric Policy.", Type: proto.ColumnType_JSON, - Transform: transform.FromField("Properties"), }, // Standard columns { @@ -152,8 +157,8 @@ func getDataset(ctx context.Context, d *plugin.QueryData, h *plugin.HydrateData) // In some cases resource does not give any notFound error // instead of notFound error, it returns empty data if op.ID != nil { - return op, nil + return DatasetInfo{op, factoryName}, nil } - return op, nil + return nil, nil } diff --git a/docs/tables/azure_data_factory_dataset.md b/docs/tables/azure_data_factory_dataset.md new file mode 100644 index 00000000..509e8555 --- /dev/null +++ b/docs/tables/azure_data_factory_dataset.md @@ -0,0 +1,33 @@ +# Table: azure_data_factory_dataset + + Datasets identify data within different data stores, such as tables, files, folders, and documents. + +## Examples + +### Basic info + +```sql +select + name, + id, + description, + etag, + type +from + azure_data_factory_dataset; +``` + + +### List relational table type Datasets + +```sql +select + name, + id, + type, + properties ->> 'type' as dataset_type +from + azure_data_factory_dataset +where + properties ->> 'type' = 'RelationalTable'; +``` \ No newline at end of file From 5b205d6d43a8cee1d95f8c7061d95d355d2dff44 Mon Sep 17 00:00:00 2001 From: rajeshbal65 Date: Mon, 5 Jul 2021 15:01:05 +0530 Subject: [PATCH 4/6] update --- .../tests/azure_data_factory/dependencies.txt | 0 .../azure_data_factory/test-get-expected.json | 13 -- .../azure_data_factory/test-get-query.sql | 3 - .../test-list-expected.json | 7 - .../azure_data_factory/test-list-query.sql | 3 - .../test-not-found-expected.json | 1 - .../test-not-found-query.sql | 3 - .../test-turbot-expected.json | 13 -- .../azure_data_factory/test-turbot-query.sql | 3 - .../tests/azure_data_factory/variables.json | 1 - .../tests/azure_data_factory/variables.tf | 67 ------ .../azure_data_factory_dataset/variables.tf | 2 +- azure/plugin.go | 1 - azure/table_azure_data_factory.go | 195 ------------------ docs/tables/azure_data_factory.md | 48 ----- 15 files changed, 1 insertion(+), 359 deletions(-) delete mode 100644 azure-test/tests/azure_data_factory/dependencies.txt delete mode 100644 azure-test/tests/azure_data_factory/test-get-expected.json delete mode 100644 azure-test/tests/azure_data_factory/test-get-query.sql delete mode 100644 azure-test/tests/azure_data_factory/test-list-expected.json delete mode 100644 azure-test/tests/azure_data_factory/test-list-query.sql delete mode 100644 azure-test/tests/azure_data_factory/test-not-found-expected.json delete mode 100644 azure-test/tests/azure_data_factory/test-not-found-query.sql delete mode 100644 azure-test/tests/azure_data_factory/test-turbot-expected.json delete mode 100644 azure-test/tests/azure_data_factory/test-turbot-query.sql delete mode 100644 azure-test/tests/azure_data_factory/variables.json delete mode 100644 azure-test/tests/azure_data_factory/variables.tf delete mode 100644 azure/table_azure_data_factory.go delete mode 100644 docs/tables/azure_data_factory.md diff --git a/azure-test/tests/azure_data_factory/dependencies.txt b/azure-test/tests/azure_data_factory/dependencies.txt deleted file mode 100644 index e69de29b..00000000 diff --git a/azure-test/tests/azure_data_factory/test-get-expected.json b/azure-test/tests/azure_data_factory/test-get-expected.json deleted file mode 100644 index 619e2748..00000000 --- a/azure-test/tests/azure_data_factory/test-get-expected.json +++ /dev/null @@ -1,13 +0,0 @@ -[ - { - "id": "{{ output.resource_id.value }}", - "name": "{{resourceName}}", - "region": "eastus", - "resource_group": "{{resourceName}}", - "subscription_id": "{{ output.subscription_id.value }}", - "tags": { - "name": "{{resourceName}}" - }, - "type": "Microsoft.DataFactory/factories" - } -] \ No newline at end of file diff --git a/azure-test/tests/azure_data_factory/test-get-query.sql b/azure-test/tests/azure_data_factory/test-get-query.sql deleted file mode 100644 index 83808052..00000000 --- a/azure-test/tests/azure_data_factory/test-get-query.sql +++ /dev/null @@ -1,3 +0,0 @@ -select name, id, type, tags, region, resource_group, subscription_id -from azure.azure_data_factory -where name = '{{resourceName}}' and resource_group = '{{resourceName}}' \ No newline at end of file diff --git a/azure-test/tests/azure_data_factory/test-list-expected.json b/azure-test/tests/azure_data_factory/test-list-expected.json deleted file mode 100644 index f15afcd8..00000000 --- a/azure-test/tests/azure_data_factory/test-list-expected.json +++ /dev/null @@ -1,7 +0,0 @@ -[ - { - "id": "{{ output.resource_id.value }}", - "name": "{{resourceName}}", - "type": "Microsoft.DataFactory/factories" - } -] \ No newline at end of file diff --git a/azure-test/tests/azure_data_factory/test-list-query.sql b/azure-test/tests/azure_data_factory/test-list-query.sql deleted file mode 100644 index f900f679..00000000 --- a/azure-test/tests/azure_data_factory/test-list-query.sql +++ /dev/null @@ -1,3 +0,0 @@ -select name, id, type -from azure.azure_data_factory -where name = '{{resourceName}}' \ No newline at end of file diff --git a/azure-test/tests/azure_data_factory/test-not-found-expected.json b/azure-test/tests/azure_data_factory/test-not-found-expected.json deleted file mode 100644 index ec747fa4..00000000 --- a/azure-test/tests/azure_data_factory/test-not-found-expected.json +++ /dev/null @@ -1 +0,0 @@ -null \ No newline at end of file diff --git a/azure-test/tests/azure_data_factory/test-not-found-query.sql b/azure-test/tests/azure_data_factory/test-not-found-query.sql deleted file mode 100644 index 04ecb573..00000000 --- a/azure-test/tests/azure_data_factory/test-not-found-query.sql +++ /dev/null @@ -1,3 +0,0 @@ -select name, tags, title, akas -from azure.azure_data_factory -where name = 'dummy-{{resourceName}}' and resource_group = '{{resourceName}}' \ No newline at end of file diff --git a/azure-test/tests/azure_data_factory/test-turbot-expected.json b/azure-test/tests/azure_data_factory/test-turbot-expected.json deleted file mode 100644 index 3ceae2f7..00000000 --- a/azure-test/tests/azure_data_factory/test-turbot-expected.json +++ /dev/null @@ -1,13 +0,0 @@ -[ - { - "akas": [ - "{{ output.resource_aka.value }}", - "{{ output.resource_aka_lower.value }}" - ], - "name": "{{resourceName}}", - "tags": { - "name": "{{resourceName}}" - }, - "title": "{{resourceName}}" - } -] \ No newline at end of file diff --git a/azure-test/tests/azure_data_factory/test-turbot-query.sql b/azure-test/tests/azure_data_factory/test-turbot-query.sql deleted file mode 100644 index 434fc5ae..00000000 --- a/azure-test/tests/azure_data_factory/test-turbot-query.sql +++ /dev/null @@ -1,3 +0,0 @@ -select name, tags, title, akas -from azure.azure_data_factory -where name = '{{resourceName}}' and resource_group = '{{resourceName}}' \ No newline at end of file diff --git a/azure-test/tests/azure_data_factory/variables.json b/azure-test/tests/azure_data_factory/variables.json deleted file mode 100644 index 0967ef42..00000000 --- a/azure-test/tests/azure_data_factory/variables.json +++ /dev/null @@ -1 +0,0 @@ -{} diff --git a/azure-test/tests/azure_data_factory/variables.tf b/azure-test/tests/azure_data_factory/variables.tf deleted file mode 100644 index 20dd0733..00000000 --- a/azure-test/tests/azure_data_factory/variables.tf +++ /dev/null @@ -1,67 +0,0 @@ - -variable "resource_name" { - type = string - default = "steampipe-test" - description = "Name of the resource used throughout the test." -} - -variable "azure_environment" { - type = string - default = "public" - description = "Azure environment used for the test." -} - -variable "azure_subscription" { - type = string - default = "3510ae4d-530b-497d-8f30-53b9616fc6c1" - description = "Azure subscription used for the test." -} - -provider "azurerm" { - # Cannot be passed as a variable - version = "=1.36.0" - environment = var.azure_environment - subscription_id = var.azure_subscription -} - -data "azurerm_client_config" "current" {} - -data "null_data_source" "resource" { - inputs = { - scope = "azure:///subscriptions/${data.azurerm_client_config.current.subscription_id}" - } -} - -resource "azurerm_resource_group" "named_test_resource" { - name = var.resource_name - location = "East US" -} - -resource "azurerm_data_factory" "named_test_resource" { - name = var.resource_name - location = "East US" - resource_group_name = azurerm_resource_group.named_test_resource.name - tags = { - name = var.resource_name - } -} - -output "resource_aka" { - value = "azure://${azurerm_data_factory.named_test_resource.id}" -} - -output "resource_aka_lower" { - value = "azure://${lower(azurerm_data_factory.named_test_resource.id)}" -} - -output "resource_name" { - value = var.resource_name -} - -output "resource_id" { - value = azurerm_data_factory.named_test_resource.id -} - -output "subscription_id" { - value = var.azure_subscription -} diff --git a/azure-test/tests/azure_data_factory_dataset/variables.tf b/azure-test/tests/azure_data_factory_dataset/variables.tf index 53b1d09b..eed86cf6 100644 --- a/azure-test/tests/azure_data_factory_dataset/variables.tf +++ b/azure-test/tests/azure_data_factory_dataset/variables.tf @@ -13,7 +13,7 @@ variable "azure_environment" { variable "azure_subscription" { type = string - default = "d7245080-b4ae-4fe5-b6fa-2e71b3dae6c8" + default = "3510ae4d-530b-497d-8f30-53b9616fc6c1" description = "Azure subscription used for the test." } diff --git a/azure/plugin.go b/azure/plugin.go index af103ce9..683ea2ba 100644 --- a/azure/plugin.go +++ b/azure/plugin.go @@ -41,7 +41,6 @@ func Plugin(ctx context.Context) *plugin.Plugin { "azure_cosmosdb_account": tableAzureCosmosDBAccount(ctx), "azure_cosmosdb_mongo_database": tableAzureCosmosDBMongoDatabase(ctx), "azure_cosmosdb_sql_database": tableAzureCosmosDBSQLDatabase(ctx), - "azure_data_factory": tableAzureDataFactory(ctx), "azure_data_factory_dataset": tableAzureDataFactoryDataset(ctx), "azure_diagnostic_setting": tableAzureDiagnosticSetting(ctx), "azure_firewall": tableAzureFirewall(ctx), diff --git a/azure/table_azure_data_factory.go b/azure/table_azure_data_factory.go deleted file mode 100644 index ae62b6f2..00000000 --- a/azure/table_azure_data_factory.go +++ /dev/null @@ -1,195 +0,0 @@ -package azure - -import ( - "context" - - "github.com/Azure/azure-sdk-for-go/services/datafactory/mgmt/2018-06-01/datafactory" - "github.com/turbot/steampipe-plugin-sdk/grpc/proto" - "github.com/turbot/steampipe-plugin-sdk/plugin/transform" - - "github.com/turbot/steampipe-plugin-sdk/plugin" -) - -//// TABLE DEFINITION //// - -func tableAzureDataFactory(_ context.Context) *plugin.Table { - return &plugin.Table{ - Name: "azure_data_factory", - Description: "Azure Data Factory", - Get: &plugin.GetConfig{ - KeyColumns: plugin.AllColumns([]string{"name", "resource_group"}), - Hydrate: getFactory, - ShouldIgnoreError: isNotFoundError([]string{"ResourceNotFound", "ResourceGroupNotFound", "404"}), - }, - List: &plugin.ListConfig{ - Hydrate: listFactories, - }, - Columns: []*plugin.Column{ - { - Name: "name", - Description: "The resource name.", - Type: proto.ColumnType_STRING, - }, - { - Name: "id", - Description: "The resource identifier.", - Type: proto.ColumnType_STRING, - Transform: transform.FromGo(), - }, - { - Name: "etag", - Description: "Etag identifies change in the resource.", - Type: proto.ColumnType_STRING, - Transform: transform.FromField("ETag"), - }, - { - Name: "type", - Description: "The resource type.", - Type: proto.ColumnType_STRING, - }, - { - Name: "provisioning_state", - Description: "Factory provisioning state, example Succeeded.", - Type: proto.ColumnType_STRING, - Transform: transform.FromField("FactoryProperties.ProvisioningState"), - }, - { - Name: "create_time", - Description: "Time the factory was created in ISO8601 format.", - Type: proto.ColumnType_TIMESTAMP, - Transform: transform.FromField("FactoryProperties.CreateTime").Transform(convertDateToTime), - }, - { - Name: "version", - Description: "Version of the factory.", - Type: proto.ColumnType_STRING, - Transform: transform.FromField("FactoryProperties.Version"), - }, - { - Name: "public_network_access", - Description: "Whether or not public network access is allowed for the data factory.", - Type: proto.ColumnType_STRING, - Transform: transform.FromField("FactoryProperties.PublicNetworkAccess"), - }, - { - Name: "identity", - Description: "Managed service identity of the factory.", - Type: proto.ColumnType_JSON, - }, - { - Name: "encryption", - Description: "Properties to enable Customer Managed Key for the factory.", - Type: proto.ColumnType_JSON, - Transform: transform.FromField("FactoryProperties.EncryptionConfiguration"), - }, - { - Name: "repo_configuration", - Description: "Git repo information of the factory.", - Type: proto.ColumnType_JSON, - Transform: transform.FromField("FactoryProperties.RepoConfiguration"), - }, - { - Name: "global_parameters", - Description: "List of parameters for factory.", - Type: proto.ColumnType_JSON, - Transform: transform.FromField("FactoryProperties.GlobalParameters"), - }, - - // Standard columns - { - Name: "title", - Description: ColumnDescriptionTitle, - Type: proto.ColumnType_STRING, - Transform: transform.FromField("Name"), - }, - { - Name: "tags", - Description: ColumnDescriptionTags, - Type: proto.ColumnType_JSON, - }, - { - Name: "akas", - Description: ColumnDescriptionAkas, - Type: proto.ColumnType_JSON, - Transform: transform.FromField("ID").Transform(idToAkas), - }, - { - Name: "region", - Description: ColumnDescriptionRegion, - Type: proto.ColumnType_STRING, - Transform: transform.FromField("Location").Transform(toLower), - }, - { - Name: "resource_group", - Description: ColumnDescriptionResourceGroup, - Type: proto.ColumnType_STRING, - Transform: transform.FromField("ID").Transform(extractResourceGroupFromID), - }, - { - Name: "subscription_id", - Description: ColumnDescriptionSubscription, - Type: proto.ColumnType_STRING, - Transform: transform.FromField("ID").Transform(idToSubscriptionID), - }, - }, - } -} - -//// LIST FUNCTIONS //// - -func listFactories(ctx context.Context, d *plugin.QueryData, _ *plugin.HydrateData) (interface{}, error) { - session, err := GetNewSession(ctx, d, "MANAGEMENT") - if err != nil { - return nil, err - } - subscriptionID := session.SubscriptionID - - factoryClient := datafactory.NewFactoriesClient(subscriptionID) - factoryClient.Authorizer = session.Authorizer - pagesLeft := true - - for pagesLeft { - result, err := factoryClient.List(context.Background()) - if err != nil { - return nil, err - } - - for _, factory := range result.Values() { - d.StreamListItem(ctx, factory) - } - result.NextWithContext(context.Background()) - pagesLeft = result.NotDone() - } - return nil, err -} - -//// HYDRATE FUNCTIONS //// - -func getFactory(ctx context.Context, d *plugin.QueryData, h *plugin.HydrateData) (interface{}, error) { - plugin.Logger(ctx).Trace("getFactory") - - name := d.KeyColumnQuals["name"].GetStringValue() - resourceGroup := d.KeyColumnQuals["resource_group"].GetStringValue() - - session, err := GetNewSession(ctx, d, "MANAGEMENT") - if err != nil { - return nil, err - } - subscriptionID := session.SubscriptionID - - factoryClient := datafactory.NewFactoriesClient(subscriptionID) - factoryClient.Authorizer = session.Authorizer - - op, err := factoryClient.Get(ctx, resourceGroup, name, "*") - if err != nil { - return nil, err - } - - // In some cases resource does not give any notFound error - // instead of notFound error, it returns empty data - if op.ID != nil { - return op, nil - } - - return nil, nil -} diff --git a/docs/tables/azure_data_factory.md b/docs/tables/azure_data_factory.md deleted file mode 100644 index 9d2aef7e..00000000 --- a/docs/tables/azure_data_factory.md +++ /dev/null @@ -1,48 +0,0 @@ -# Table: azure_data_factory - -Azure Data Factory is the platform that solves such data scenarios. It is the cloud-based ETL and data integration service that allows to create data-driven workflows for orchestrating data movement and transforming data at scale. - -## Examples - -### Basic info - -```sql -select - name, - id, - type, - provisioning_state, - etag -from - azure_data_factory; -``` - - -### List system assigned identity type Factories - -```sql -select - name, - id, - type, - identity ->> 'type' as identity_type -from - azure_data_factory -where - identity ->> 'type' = 'SystemAssigned'; -``` - - -### List factories with plubic network access is allowed - -```sql -select - name, - id, - type, - public_network_access -from - azure_data_factory -where - public_network_access = 'Enabled'; -``` \ No newline at end of file From 9e71ae07ec90133c8522bbd3fba6282a6f220594 Mon Sep 17 00:00:00 2001 From: rajeshbal65 Date: Wed, 7 Jul 2021 10:34:27 +0530 Subject: [PATCH 5/6] update --- azure/table_azure_data_factory_dataset.go | 39 +++++++++++++---------- docs/tables/azure_data_factory_dataset.md | 2 +- 2 files changed, 24 insertions(+), 17 deletions(-) diff --git a/azure/table_azure_data_factory_dataset.go b/azure/table_azure_data_factory_dataset.go index 184a7834..776c2428 100644 --- a/azure/table_azure_data_factory_dataset.go +++ b/azure/table_azure_data_factory_dataset.go @@ -11,7 +11,7 @@ import ( "github.com/turbot/steampipe-plugin-sdk/plugin" ) -//// TABLE DEFINITION //// +//// TABLE DEFINITION func tableAzureDataFactoryDataset(_ context.Context) *plugin.Table { return &plugin.Table{ @@ -19,11 +19,11 @@ func tableAzureDataFactoryDataset(_ context.Context) *plugin.Table { Description: "Azure Data Factory Dataset", Get: &plugin.GetConfig{ KeyColumns: plugin.AllColumns([]string{"name", "resource_group", "factory_name"}), - Hydrate: getDataset, + Hydrate: getDataFactoryDataset, ShouldIgnoreError: isNotFoundError([]string{"ResourceNotFound", "ResourceGroupNotFound", "404"}), }, List: &plugin.ListConfig{ - Hydrate: listDatasets, + Hydrate: listDataFactoryDatasets, ParentHydrate: listFactories, }, Columns: []*plugin.Column{ @@ -38,6 +38,11 @@ func tableAzureDataFactoryDataset(_ context.Context) *plugin.Table { Type: proto.ColumnType_STRING, Transform: transform.FromGo(), }, + { + Name: "factory_name", + Description: "Name of the factory the dataset belongs", + Type: proto.ColumnType_STRING, + }, { Name: "description", Description: "The description of the Dataset.", @@ -55,17 +60,13 @@ func tableAzureDataFactoryDataset(_ context.Context) *plugin.Table { Description: "The resource type.", Type: proto.ColumnType_STRING, }, - { - Name: "factory_name", - Description: "Time the factory was created in ISO8601 format.", - Type: proto.ColumnType_STRING, - }, { Name: "properties", Description: "Dataset ElapsedTime Metric Policy.", Type: proto.ColumnType_JSON, }, - // Standard columns + + // Steampipe standard columns { Name: "title", Description: ColumnDescriptionTitle, @@ -78,6 +79,8 @@ func tableAzureDataFactoryDataset(_ context.Context) *plugin.Table { Type: proto.ColumnType_JSON, Transform: transform.FromField("ID").Transform(idToAkas), }, + + // Azure standard column { Name: "resource_group", Description: ColumnDescriptionResourceGroup, @@ -99,9 +102,9 @@ type DatasetInfo = struct { FactoryName string } -//// LIST FUNCTIONS //// +//// LIST FUNCTION -func listDatasets(ctx context.Context, d *plugin.QueryData, h *plugin.HydrateData) (interface{}, error) { +func listDataFactoryDatasets(ctx context.Context, d *plugin.QueryData, h *plugin.HydrateData) (interface{}, error) { session, err := GetNewSession(ctx, d, "MANAGEMENT") if err != nil { return nil, err @@ -131,15 +134,19 @@ func listDatasets(ctx context.Context, d *plugin.QueryData, h *plugin.HydrateDat return nil, err } -//// HYDRATE FUNCTIONS //// +//// HYDRATE FUNCTIONS -func getDataset(ctx context.Context, d *plugin.QueryData, h *plugin.HydrateData) (interface{}, error) { - plugin.Logger(ctx).Trace("getDataset") +func getDataFactoryDataset(ctx context.Context, d *plugin.QueryData, h *plugin.HydrateData) (interface{}, error) { + plugin.Logger(ctx).Trace("getDataFactoryDataset") - DatasetName := d.KeyColumnQuals["name"].GetStringValue() + datasetName := d.KeyColumnQuals["name"].GetStringValue() resourceGroup := d.KeyColumnQuals["resource_group"].GetStringValue() factoryName := d.KeyColumnQuals["factory_name"].GetStringValue() + if datasetName == "" || resourceGroup == "" || factoryName == "" { + return nil, nil + } + session, err := GetNewSession(ctx, d, "MANAGEMENT") if err != nil { return nil, err @@ -149,7 +156,7 @@ func getDataset(ctx context.Context, d *plugin.QueryData, h *plugin.HydrateData) datasetClient := datafactory.NewDatasetsClient(subscriptionID) datasetClient.Authorizer = session.Authorizer - op, err := datasetClient.Get(ctx, resourceGroup, factoryName, DatasetName, "") + op, err := datasetClient.Get(ctx, resourceGroup, factoryName, datasetName, "") if err != nil { return nil, err } diff --git a/docs/tables/azure_data_factory_dataset.md b/docs/tables/azure_data_factory_dataset.md index 509e8555..aa81f49f 100644 --- a/docs/tables/azure_data_factory_dataset.md +++ b/docs/tables/azure_data_factory_dataset.md @@ -18,7 +18,7 @@ from ``` -### List relational table type Datasets +### List relational table type datasets ```sql select From 6c9dedfa0c7d702066af86103491e31b693f0396 Mon Sep 17 00:00:00 2001 From: Subhajit Kumar Mondal Date: Fri, 9 Jul 2021 14:39:24 +0530 Subject: [PATCH 6/6] update --- .../test-get-expected.json | 6 +-- .../test-get-query.sql | 2 +- .../test-list-expected.json | 4 +- .../test-list-query.sql | 2 +- .../test-not-found-expected.json | 2 +- .../test-not-found-query.sql | 2 +- .../test-turbot-expected.json | 6 +-- .../test-turbot-query.sql | 2 +- azure/table_azure_data_factory_dataset.go | 44 ++++++++----------- docs/tables/azure_data_factory_dataset.md | 2 +- 10 files changed, 33 insertions(+), 39 deletions(-) diff --git a/azure-test/tests/azure_data_factory_dataset/test-get-expected.json b/azure-test/tests/azure_data_factory_dataset/test-get-expected.json index 8e7fd942..7f99d11e 100644 --- a/azure-test/tests/azure_data_factory_dataset/test-get-expected.json +++ b/azure-test/tests/azure_data_factory_dataset/test-get-expected.json @@ -1,9 +1,9 @@ [ { "id": "{{ output.resource_id.value }}", - "name": "{{resourceName}}", - "resource_group": "{{resourceName}}", + "name": "{{ resourceName }}", + "resource_group": "{{ resourceName }}", "subscription_id": "{{ output.subscription_id.value }}", "type": "Microsoft.DataFactory/factories/datasets" } -] \ No newline at end of file +] diff --git a/azure-test/tests/azure_data_factory_dataset/test-get-query.sql b/azure-test/tests/azure_data_factory_dataset/test-get-query.sql index 94b15b66..526b015a 100644 --- a/azure-test/tests/azure_data_factory_dataset/test-get-query.sql +++ b/azure-test/tests/azure_data_factory_dataset/test-get-query.sql @@ -1,3 +1,3 @@ select name, id, type, resource_group, subscription_id from azure.azure_data_factory_dataset -where name = '{{resourceName}}' and resource_group = '{{resourceName}}' and factory_name = '{{resourceName}}' \ No newline at end of file +where name = '{{ resourceName }}' and resource_group = '{{ resourceName }}' and factory_name = '{{ resourceName }}'; \ No newline at end of file diff --git a/azure-test/tests/azure_data_factory_dataset/test-list-expected.json b/azure-test/tests/azure_data_factory_dataset/test-list-expected.json index 11083c14..4ce51358 100644 --- a/azure-test/tests/azure_data_factory_dataset/test-list-expected.json +++ b/azure-test/tests/azure_data_factory_dataset/test-list-expected.json @@ -1,7 +1,7 @@ [ { "id": "{{ output.resource_id.value }}", - "name": "{{resourceName}}", + "name": "{{ resourceName }}", "type": "Microsoft.DataFactory/factories/datasets" } -] \ No newline at end of file +] diff --git a/azure-test/tests/azure_data_factory_dataset/test-list-query.sql b/azure-test/tests/azure_data_factory_dataset/test-list-query.sql index 2f6a45eb..bfe31fc2 100644 --- a/azure-test/tests/azure_data_factory_dataset/test-list-query.sql +++ b/azure-test/tests/azure_data_factory_dataset/test-list-query.sql @@ -1,3 +1,3 @@ select name, id, type from azure.azure_data_factory_dataset -where name = '{{resourceName}}' \ No newline at end of file +where name = '{{ resourceName }}'; \ No newline at end of file diff --git a/azure-test/tests/azure_data_factory_dataset/test-not-found-expected.json b/azure-test/tests/azure_data_factory_dataset/test-not-found-expected.json index ec747fa4..19765bd5 100644 --- a/azure-test/tests/azure_data_factory_dataset/test-not-found-expected.json +++ b/azure-test/tests/azure_data_factory_dataset/test-not-found-expected.json @@ -1 +1 @@ -null \ No newline at end of file +null diff --git a/azure-test/tests/azure_data_factory_dataset/test-not-found-query.sql b/azure-test/tests/azure_data_factory_dataset/test-not-found-query.sql index 76611712..0ecd79c5 100644 --- a/azure-test/tests/azure_data_factory_dataset/test-not-found-query.sql +++ b/azure-test/tests/azure_data_factory_dataset/test-not-found-query.sql @@ -1,3 +1,3 @@ select name, title, akas from azure.azure_data_factory_dataset -where name = 'dummy-{{resourceName}}' and resource_group = '{{resourceName}}' \ No newline at end of file +where name = 'dummy-{{ resourceName }}' and resource_group = '{{ resourceName }}'; \ No newline at end of file diff --git a/azure-test/tests/azure_data_factory_dataset/test-turbot-expected.json b/azure-test/tests/azure_data_factory_dataset/test-turbot-expected.json index 5d3863e6..02cd3c76 100644 --- a/azure-test/tests/azure_data_factory_dataset/test-turbot-expected.json +++ b/azure-test/tests/azure_data_factory_dataset/test-turbot-expected.json @@ -4,7 +4,7 @@ "{{ output.resource_aka.value }}", "{{ output.resource_aka_lower.value }}" ], - "name": "{{resourceName}}", - "title": "{{resourceName}}" + "name": "{{ resourceName }}", + "title": "{{ resourceName }}" } -] \ No newline at end of file +] diff --git a/azure-test/tests/azure_data_factory_dataset/test-turbot-query.sql b/azure-test/tests/azure_data_factory_dataset/test-turbot-query.sql index d7bb537b..9449c61e 100644 --- a/azure-test/tests/azure_data_factory_dataset/test-turbot-query.sql +++ b/azure-test/tests/azure_data_factory_dataset/test-turbot-query.sql @@ -1,3 +1,3 @@ select name, title, akas from azure.azure_data_factory_dataset -where name = '{{resourceName}}' and resource_group = '{{resourceName}}' \ No newline at end of file +where name = '{{ resourceName }}' and resource_group = '{{ resourceName }}'; \ No newline at end of file diff --git a/azure/table_azure_data_factory_dataset.go b/azure/table_azure_data_factory_dataset.go index 776c2428..674fac0d 100644 --- a/azure/table_azure_data_factory_dataset.go +++ b/azure/table_azure_data_factory_dataset.go @@ -24,7 +24,7 @@ func tableAzureDataFactoryDataset(_ context.Context) *plugin.Table { }, List: &plugin.ListConfig{ Hydrate: listDataFactoryDatasets, - ParentHydrate: listFactories, + ParentHydrate: listDataFactories, }, Columns: []*plugin.Column{ { @@ -40,20 +40,13 @@ func tableAzureDataFactoryDataset(_ context.Context) *plugin.Table { }, { Name: "factory_name", - Description: "Name of the factory the dataset belongs", + Description: "Name of the factory the dataset belongs.", Type: proto.ColumnType_STRING, }, - { - Name: "description", - Description: "The description of the Dataset.", - Type: proto.ColumnType_STRING, - Transform: transform.FromField("Dataset.Description"), - }, { Name: "etag", - Description: "Etag identifies change in the resource.", + Description: "An unique read-only string that changes whenever the resource is updated.", Type: proto.ColumnType_STRING, - Transform: transform.FromField("Etag"), }, { Name: "type", @@ -62,7 +55,7 @@ func tableAzureDataFactoryDataset(_ context.Context) *plugin.Table { }, { Name: "properties", - Description: "Dataset ElapsedTime Metric Policy.", + Description: "Dataset properties.", Type: proto.ColumnType_JSON, }, @@ -109,22 +102,21 @@ func listDataFactoryDatasets(ctx context.Context, d *plugin.QueryData, h *plugin if err != nil { return nil, err } + subscriptionID := session.SubscriptionID + // Get factory details factoryInfo := h.Item.(datafactory.Factory) resourceGroup := strings.Split(*factoryInfo.ID, "/")[4] - subscriptionID := session.SubscriptionID - datasetClient := datafactory.NewDatasetsClient(subscriptionID) datasetClient.Authorizer = session.Authorizer - pagesLeft := true + pagesLeft := true for pagesLeft { result, err := datasetClient.ListByFactory(ctx, resourceGroup, *factoryInfo.Name) if err != nil { return nil, err } - for _, dataset := range result.Values() { d.StreamListItem(ctx, DatasetInfo{dataset, *factoryInfo.Name}) } @@ -136,17 +128,10 @@ func listDataFactoryDatasets(ctx context.Context, d *plugin.QueryData, h *plugin //// HYDRATE FUNCTIONS -func getDataFactoryDataset(ctx context.Context, d *plugin.QueryData, h *plugin.HydrateData) (interface{}, error) { +func getDataFactoryDataset(ctx context.Context, d *plugin.QueryData, _ *plugin.HydrateData) (interface{}, error) { plugin.Logger(ctx).Trace("getDataFactoryDataset") - datasetName := d.KeyColumnQuals["name"].GetStringValue() - resourceGroup := d.KeyColumnQuals["resource_group"].GetStringValue() - factoryName := d.KeyColumnQuals["factory_name"].GetStringValue() - - if datasetName == "" || resourceGroup == "" || factoryName == "" { - return nil, nil - } - + // Create session session, err := GetNewSession(ctx, d, "MANAGEMENT") if err != nil { return nil, err @@ -156,7 +141,16 @@ func getDataFactoryDataset(ctx context.Context, d *plugin.QueryData, h *plugin.H datasetClient := datafactory.NewDatasetsClient(subscriptionID) datasetClient.Authorizer = session.Authorizer - op, err := datasetClient.Get(ctx, resourceGroup, factoryName, datasetName, "") + datasetName := d.KeyColumnQuals["name"].GetStringValue() + resourceGroup := d.KeyColumnQuals["resource_group"].GetStringValue() + factoryName := d.KeyColumnQuals["factory_name"].GetStringValue() + + // Return nil, of no input provided + if datasetName == "" || resourceGroup == "" || factoryName == "" { + return nil, nil + } + + op, err := datasetClient.Get(ctx, resourceGroup, factoryName, datasetName, "*") if err != nil { return nil, err } diff --git a/docs/tables/azure_data_factory_dataset.md b/docs/tables/azure_data_factory_dataset.md index aa81f49f..b04c5d78 100644 --- a/docs/tables/azure_data_factory_dataset.md +++ b/docs/tables/azure_data_factory_dataset.md @@ -1,6 +1,6 @@ # Table: azure_data_factory_dataset - Datasets identify data within different data stores, such as tables, files, folders, and documents. +Azure Data Factory datasets identify data within different data stores, such as tables, files, folders, and documents. ## Examples