diff --git a/azure-test/tests/azure_data_factory_dataset/dependencies.txt b/azure-test/tests/azure_data_factory_dataset/dependencies.txt new file mode 100644 index 00000000..e69de29b diff --git a/azure-test/tests/azure_data_factory_dataset/test-get-expected.json b/azure-test/tests/azure_data_factory_dataset/test-get-expected.json new file mode 100644 index 00000000..7f99d11e --- /dev/null +++ b/azure-test/tests/azure_data_factory_dataset/test-get-expected.json @@ -0,0 +1,9 @@ +[ + { + "id": "{{ output.resource_id.value }}", + "name": "{{ resourceName }}", + "resource_group": "{{ resourceName }}", + "subscription_id": "{{ output.subscription_id.value }}", + "type": "Microsoft.DataFactory/factories/datasets" + } +] diff --git a/azure-test/tests/azure_data_factory_dataset/test-get-query.sql b/azure-test/tests/azure_data_factory_dataset/test-get-query.sql new file mode 100644 index 00000000..526b015a --- /dev/null +++ b/azure-test/tests/azure_data_factory_dataset/test-get-query.sql @@ -0,0 +1,3 @@ +select name, id, type, resource_group, subscription_id +from azure.azure_data_factory_dataset +where name = '{{ resourceName }}' and resource_group = '{{ resourceName }}' and factory_name = '{{ resourceName }}'; \ No newline at end of file diff --git a/azure-test/tests/azure_data_factory_dataset/test-list-expected.json b/azure-test/tests/azure_data_factory_dataset/test-list-expected.json new file mode 100644 index 00000000..4ce51358 --- /dev/null +++ b/azure-test/tests/azure_data_factory_dataset/test-list-expected.json @@ -0,0 +1,7 @@ +[ + { + "id": "{{ output.resource_id.value }}", + "name": "{{ resourceName }}", + "type": "Microsoft.DataFactory/factories/datasets" + } +] diff --git a/azure-test/tests/azure_data_factory_dataset/test-list-query.sql b/azure-test/tests/azure_data_factory_dataset/test-list-query.sql new file mode 100644 index 00000000..bfe31fc2 --- /dev/null +++ b/azure-test/tests/azure_data_factory_dataset/test-list-query.sql @@ -0,0 +1,3 @@ +select name, id, type +from azure.azure_data_factory_dataset +where name = '{{ resourceName }}'; \ No newline at end of file diff --git a/azure-test/tests/azure_data_factory_dataset/test-not-found-expected.json b/azure-test/tests/azure_data_factory_dataset/test-not-found-expected.json new file mode 100644 index 00000000..19765bd5 --- /dev/null +++ b/azure-test/tests/azure_data_factory_dataset/test-not-found-expected.json @@ -0,0 +1 @@ +null diff --git a/azure-test/tests/azure_data_factory_dataset/test-not-found-query.sql b/azure-test/tests/azure_data_factory_dataset/test-not-found-query.sql new file mode 100644 index 00000000..0ecd79c5 --- /dev/null +++ b/azure-test/tests/azure_data_factory_dataset/test-not-found-query.sql @@ -0,0 +1,3 @@ +select name, title, akas +from azure.azure_data_factory_dataset +where name = 'dummy-{{ resourceName }}' and resource_group = '{{ resourceName }}'; \ No newline at end of file diff --git a/azure-test/tests/azure_data_factory_dataset/test-turbot-expected.json b/azure-test/tests/azure_data_factory_dataset/test-turbot-expected.json new file mode 100644 index 00000000..02cd3c76 --- /dev/null +++ b/azure-test/tests/azure_data_factory_dataset/test-turbot-expected.json @@ -0,0 +1,10 @@ +[ + { + "akas": [ + "{{ output.resource_aka.value }}", + "{{ output.resource_aka_lower.value }}" + ], + "name": "{{ resourceName }}", + "title": "{{ resourceName }}" + } +] diff --git a/azure-test/tests/azure_data_factory_dataset/test-turbot-query.sql b/azure-test/tests/azure_data_factory_dataset/test-turbot-query.sql new file mode 100644 index 00000000..9449c61e --- /dev/null +++ b/azure-test/tests/azure_data_factory_dataset/test-turbot-query.sql @@ -0,0 +1,3 @@ +select name, title, akas +from azure.azure_data_factory_dataset +where name = '{{ resourceName }}' and resource_group = '{{ resourceName }}'; \ No newline at end of file diff --git a/azure-test/tests/azure_data_factory_dataset/variables.json b/azure-test/tests/azure_data_factory_dataset/variables.json new file mode 100644 index 00000000..0967ef42 --- /dev/null +++ b/azure-test/tests/azure_data_factory_dataset/variables.json @@ -0,0 +1 @@ +{} diff --git a/azure-test/tests/azure_data_factory_dataset/variables.tf b/azure-test/tests/azure_data_factory_dataset/variables.tf new file mode 100644 index 00000000..eed86cf6 --- /dev/null +++ b/azure-test/tests/azure_data_factory_dataset/variables.tf @@ -0,0 +1,81 @@ + +variable "resource_name" { + type = string + default = "steampipe-test" + description = "Name of the resource used throughout the test." +} + +variable "azure_environment" { + type = string + default = "public" + description = "Azure environment used for the test." +} + +variable "azure_subscription" { + type = string + default = "3510ae4d-530b-497d-8f30-53b9616fc6c1" + description = "Azure subscription used for the test." +} + +provider "azurerm" { + # Cannot be passed as a variable + version = "=1.36.0" + environment = var.azure_environment + subscription_id = var.azure_subscription +} + +data "azurerm_client_config" "current" {} + +data "null_data_source" "resource" { + inputs = { + scope = "azure:///subscriptions/${data.azurerm_client_config.current.subscription_id}" + } +} + +resource "azurerm_resource_group" "named_test_resource" { + name = var.resource_name + location = "East US" +} + +resource "azurerm_data_factory" "named_test_resource" { + name = var.resource_name + location = "East US" + resource_group_name = azurerm_resource_group.named_test_resource.name + tags = { + name = var.resource_name + } +} + +resource "azurerm_data_factory_linked_service_mysql" "named_test_resource" { + name = var.resource_name + resource_group_name = azurerm_resource_group.named_test_resource.name + data_factory_name = azurerm_data_factory.named_test_resource.name + connection_string = "Server=test;Port=3306;Database=test;User=test;SSLMode=1;UseSystemTrustStore=0;Password=test" +} + +resource "azurerm_data_factory_dataset_mysql" "named_test_resource" { + name = var.resource_name + resource_group_name = azurerm_resource_group.named_test_resource.name + data_factory_name = azurerm_data_factory.named_test_resource.name + linked_service_name = azurerm_data_factory_linked_service_mysql.named_test_resource.name +} + +output "resource_aka" { + value = "azure://${azurerm_data_factory_dataset_mysql.named_test_resource.id}" +} + +output "resource_aka_lower" { + value = "azure://${lower(azurerm_data_factory_dataset_mysql.named_test_resource.id)}" +} + +output "resource_name" { + value = var.resource_name +} + +output "resource_id" { + value = azurerm_data_factory_dataset_mysql.named_test_resource.id +} + +output "subscription_id" { + value = var.azure_subscription +} diff --git a/azure/plugin.go b/azure/plugin.go index e5467c02..0823f23a 100644 --- a/azure/plugin.go +++ b/azure/plugin.go @@ -51,6 +51,7 @@ func Plugin(ctx context.Context) *plugin.Plugin { "azure_cosmosdb_mongo_database": tableAzureCosmosDBMongoDatabase(ctx), "azure_cosmosdb_sql_database": tableAzureCosmosDBSQLDatabase(ctx), "azure_data_factory": tableAzureDataFactory(ctx), + "azure_data_factory_dataset": tableAzureDataFactoryDataset(ctx), "azure_diagnostic_setting": tableAzureDiagnosticSetting(ctx), "azure_express_route_circuit": tableAzureExpressRouteCircuit(ctx), "azure_firewall": tableAzureFirewall(ctx), diff --git a/azure/table_azure_data_factory_dataset.go b/azure/table_azure_data_factory_dataset.go new file mode 100644 index 00000000..674fac0d --- /dev/null +++ b/azure/table_azure_data_factory_dataset.go @@ -0,0 +1,165 @@ +package azure + +import ( + "context" + "strings" + + "github.com/Azure/azure-sdk-for-go/services/datafactory/mgmt/2018-06-01/datafactory" + "github.com/turbot/steampipe-plugin-sdk/grpc/proto" + "github.com/turbot/steampipe-plugin-sdk/plugin/transform" + + "github.com/turbot/steampipe-plugin-sdk/plugin" +) + +//// TABLE DEFINITION + +func tableAzureDataFactoryDataset(_ context.Context) *plugin.Table { + return &plugin.Table{ + Name: "azure_data_factory_dataset", + Description: "Azure Data Factory Dataset", + Get: &plugin.GetConfig{ + KeyColumns: plugin.AllColumns([]string{"name", "resource_group", "factory_name"}), + Hydrate: getDataFactoryDataset, + ShouldIgnoreError: isNotFoundError([]string{"ResourceNotFound", "ResourceGroupNotFound", "404"}), + }, + List: &plugin.ListConfig{ + Hydrate: listDataFactoryDatasets, + ParentHydrate: listDataFactories, + }, + Columns: []*plugin.Column{ + { + Name: "name", + Description: "The resource name.", + Type: proto.ColumnType_STRING, + }, + { + Name: "id", + Description: "The resource identifier.", + Type: proto.ColumnType_STRING, + Transform: transform.FromGo(), + }, + { + Name: "factory_name", + Description: "Name of the factory the dataset belongs.", + Type: proto.ColumnType_STRING, + }, + { + Name: "etag", + Description: "An unique read-only string that changes whenever the resource is updated.", + Type: proto.ColumnType_STRING, + }, + { + Name: "type", + Description: "The resource type.", + Type: proto.ColumnType_STRING, + }, + { + Name: "properties", + Description: "Dataset properties.", + Type: proto.ColumnType_JSON, + }, + + // Steampipe standard columns + { + Name: "title", + Description: ColumnDescriptionTitle, + Type: proto.ColumnType_STRING, + Transform: transform.FromField("Name"), + }, + { + Name: "akas", + Description: ColumnDescriptionAkas, + Type: proto.ColumnType_JSON, + Transform: transform.FromField("ID").Transform(idToAkas), + }, + + // Azure standard column + { + Name: "resource_group", + Description: ColumnDescriptionResourceGroup, + Type: proto.ColumnType_STRING, + Transform: transform.FromField("ID").Transform(extractResourceGroupFromID), + }, + { + Name: "subscription_id", + Description: ColumnDescriptionSubscription, + Type: proto.ColumnType_STRING, + Transform: transform.FromField("ID").Transform(idToSubscriptionID), + }, + }, + } +} + +type DatasetInfo = struct { + datafactory.DatasetResource + FactoryName string +} + +//// LIST FUNCTION + +func listDataFactoryDatasets(ctx context.Context, d *plugin.QueryData, h *plugin.HydrateData) (interface{}, error) { + session, err := GetNewSession(ctx, d, "MANAGEMENT") + if err != nil { + return nil, err + } + subscriptionID := session.SubscriptionID + + // Get factory details + factoryInfo := h.Item.(datafactory.Factory) + resourceGroup := strings.Split(*factoryInfo.ID, "/")[4] + + datasetClient := datafactory.NewDatasetsClient(subscriptionID) + datasetClient.Authorizer = session.Authorizer + + pagesLeft := true + for pagesLeft { + result, err := datasetClient.ListByFactory(ctx, resourceGroup, *factoryInfo.Name) + if err != nil { + return nil, err + } + for _, dataset := range result.Values() { + d.StreamListItem(ctx, DatasetInfo{dataset, *factoryInfo.Name}) + } + result.NextWithContext(context.Background()) + pagesLeft = result.NotDone() + } + return nil, err +} + +//// HYDRATE FUNCTIONS + +func getDataFactoryDataset(ctx context.Context, d *plugin.QueryData, _ *plugin.HydrateData) (interface{}, error) { + plugin.Logger(ctx).Trace("getDataFactoryDataset") + + // Create session + session, err := GetNewSession(ctx, d, "MANAGEMENT") + if err != nil { + return nil, err + } + subscriptionID := session.SubscriptionID + + datasetClient := datafactory.NewDatasetsClient(subscriptionID) + datasetClient.Authorizer = session.Authorizer + + datasetName := d.KeyColumnQuals["name"].GetStringValue() + resourceGroup := d.KeyColumnQuals["resource_group"].GetStringValue() + factoryName := d.KeyColumnQuals["factory_name"].GetStringValue() + + // Return nil, of no input provided + if datasetName == "" || resourceGroup == "" || factoryName == "" { + return nil, nil + } + + op, err := datasetClient.Get(ctx, resourceGroup, factoryName, datasetName, "*") + if err != nil { + return nil, err + } + + // In some cases resource does not give any notFound error + // instead of notFound error, it returns empty data + if op.ID != nil { + return DatasetInfo{op, factoryName}, nil + } + + return nil, nil +} diff --git a/docs/tables/azure_data_factory_dataset.md b/docs/tables/azure_data_factory_dataset.md new file mode 100644 index 00000000..b04c5d78 --- /dev/null +++ b/docs/tables/azure_data_factory_dataset.md @@ -0,0 +1,33 @@ +# Table: azure_data_factory_dataset + +Azure Data Factory datasets identify data within different data stores, such as tables, files, folders, and documents. + +## Examples + +### Basic info + +```sql +select + name, + id, + description, + etag, + type +from + azure_data_factory_dataset; +``` + + +### List relational table type datasets + +```sql +select + name, + id, + type, + properties ->> 'type' as dataset_type +from + azure_data_factory_dataset +where + properties ->> 'type' = 'RelationalTable'; +``` \ No newline at end of file