Add table azure_data_factory_dataset. Closes #150

azure-test/tests/azure_data_factory_dataset/test-get-expected.json

@@ -0,0 +1,9 @@
+[
+  {
+    "id": "{{ output.resource_id.value }}",
+    "name": "{{resourceName}}",
+    "resource_group": "{{resourceName}}",
+    "subscription_id": "{{ output.subscription_id.value }}",
+    "type": "Microsoft.DataFactory/factories/datasets"
+  }
+]

azure-test/tests/azure_data_factory_dataset/test-get-query.sql

@@ -0,0 +1,3 @@
+select name, id, type, resource_group, subscription_id
+from azure.azure_data_factory_dataset
+where name = '{{resourceName}}' and resource_group = '{{resourceName}}' and factory_name = '{{resourceName}}'

azure-test/tests/azure_data_factory_dataset/test-list-expected.json

@@ -0,0 +1,7 @@
+[
+  {
+    "id": "{{ output.resource_id.value }}",
+    "name": "{{resourceName}}",
+    "type": "Microsoft.DataFactory/factories/datasets"
+  }
+]

azure-test/tests/azure_data_factory_dataset/test-list-query.sql

@@ -0,0 +1,3 @@
+select name, id, type
+from azure.azure_data_factory_dataset
+where name = '{{resourceName}}'

azure-test/tests/azure_data_factory_dataset/test-not-found-expected.json

@@ -0,0 +1 @@
+null

azure-test/tests/azure_data_factory_dataset/test-not-found-query.sql

@@ -0,0 +1,3 @@
+select name, title, akas
+from azure.azure_data_factory_dataset
+where name = 'dummy-{{resourceName}}' and resource_group = '{{resourceName}}'

azure-test/tests/azure_data_factory_dataset/test-turbot-expected.json

@@ -0,0 +1,10 @@
+[
+  {
+    "akas": [
+      "{{ output.resource_aka.value }}",
+      "{{ output.resource_aka_lower.value }}"
+    ],
+    "name": "{{resourceName}}",
+    "title": "{{resourceName}}"
+  }
+]

azure-test/tests/azure_data_factory_dataset/test-turbot-query.sql

@@ -0,0 +1,3 @@
+select name, title, akas
+from azure.azure_data_factory_dataset
+where name = '{{resourceName}}' and resource_group = '{{resourceName}}'

azure-test/tests/azure_data_factory_dataset/variables.json

@@ -0,0 +1 @@
+{}

azure-test/tests/azure_data_factory_dataset/variables.tf

@@ -0,0 +1,81 @@
+
+variable "resource_name" {
+  type        = string
+  default     = "steampipe-test"
+  description = "Name of the resource used throughout the test."
+}
+
+variable "azure_environment" {
+  type        = string
+  default     = "public"
+  description = "Azure environment used for the test."
+}
+
+variable "azure_subscription" {
+  type        = string
+  default     = "3510ae4d-530b-497d-8f30-53b9616fc6c1"
+  description = "Azure subscription used for the test."
+}
+
+provider "azurerm" {
+  # Cannot be passed as a variable
+  version         = "=1.36.0"
+  environment     = var.azure_environment
+  subscription_id = var.azure_subscription
+}
+
+data "azurerm_client_config" "current" {}
+
+data "null_data_source" "resource" {
+  inputs = {
+    scope = "azure:///subscriptions/${data.azurerm_client_config.current.subscription_id}"
+  }
+}
+
+resource "azurerm_resource_group" "named_test_resource" {
+  name     = var.resource_name
+  location = "East US"
+}
+
+resource "azurerm_data_factory" "named_test_resource" {
+  name                = var.resource_name
+  location            = "East US"
+  resource_group_name = azurerm_resource_group.named_test_resource.name
+  tags = {
+    name = var.resource_name
+  }
+}
+
+resource "azurerm_data_factory_linked_service_mysql" "named_test_resource" {
+  name                = var.resource_name
+  resource_group_name = azurerm_resource_group.named_test_resource.name
+  data_factory_name   = azurerm_data_factory.named_test_resource.name
+  connection_string   = "Server=test;Port=3306;Database=test;User=test;SSLMode=1;UseSystemTrustStore=0;Password=test"
+}
+
+resource "azurerm_data_factory_dataset_mysql" "named_test_resource" {
+  name                = var.resource_name
+  resource_group_name = azurerm_resource_group.named_test_resource.name
+  data_factory_name   = azurerm_data_factory.named_test_resource.name
+  linked_service_name = azurerm_data_factory_linked_service_mysql.named_test_resource.name
+}
+
+output "resource_aka" {
+  value = "azure://${azurerm_data_factory_dataset_mysql.named_test_resource.id}"
+}
+
+output "resource_aka_lower" {
+  value = "azure://${lower(azurerm_data_factory_dataset_mysql.named_test_resource.id)}"
+}
+
+output "resource_name" {
+  value = var.resource_name
+}
+
+output "resource_id" {
+  value = azurerm_data_factory_dataset_mysql.named_test_resource.id
+}
+
+output "subscription_id" {
+  value = var.azure_subscription
+}

azure/plugin.go

@@ -51,6 +51,7 @@ func Plugin(ctx context.Context) *plugin.Plugin {
 			"azure_cosmosdb_mongo_database":                               tableAzureCosmosDBMongoDatabase(ctx),
 			"azure_cosmosdb_sql_database":                                 tableAzureCosmosDBSQLDatabase(ctx),
 			"azure_data_factory":                                          tableAzureDataFactory(ctx),
+			"azure_data_factory_dataset":                                  tableAzureDataFactoryDataset(ctx),
 			"azure_diagnostic_setting":                                    tableAzureDiagnosticSetting(ctx),
 			"azure_express_route_circuit":                                 tableAzureExpressRouteCircuit(ctx),
 			"azure_firewall":                                              tableAzureFirewall(ctx),

azure/table_azure_data_factory_dataset.go

@@ -0,0 +1,171 @@
+package azure
+
+import (
+	"context"
+	"strings"
+
+	"github.com/Azure/azure-sdk-for-go/services/datafactory/mgmt/2018-06-01/datafactory"
+	"github.com/turbot/steampipe-plugin-sdk/grpc/proto"
+	"github.com/turbot/steampipe-plugin-sdk/plugin/transform"
+
+	"github.com/turbot/steampipe-plugin-sdk/plugin"
+)
+
+//// TABLE DEFINITION
+
+func tableAzureDataFactoryDataset(_ context.Context) *plugin.Table {
+	return &plugin.Table{
+		Name:        "azure_data_factory_dataset",
+		Description: "Azure Data Factory Dataset",
+		Get: &plugin.GetConfig{
+			KeyColumns:        plugin.AllColumns([]string{"name", "resource_group", "factory_name"}),
+			Hydrate:           getDataFactoryDataset,
+			ShouldIgnoreError: isNotFoundError([]string{"ResourceNotFound", "ResourceGroupNotFound", "404"}),
+		},
+		List: &plugin.ListConfig{
+			Hydrate:       listDataFactoryDatasets,
+			ParentHydrate: listFactories,
+		},
+		Columns: []*plugin.Column{
+			{
+				Name:        "name",
+				Description: "The resource name.",
+				Type:        proto.ColumnType_STRING,
+			},
+			{
+				Name:        "id",
+				Description: "The resource identifier.",
+				Type:        proto.ColumnType_STRING,
+				Transform:   transform.FromGo(),
+			},
+			{
+				Name:        "factory_name",
+				Description: "Name of the factory the dataset belongs",
+				Type:        proto.ColumnType_STRING,
+			},
+			{
+				Name:        "description",
+				Description: "The description of the Dataset.",
+				Type:        proto.ColumnType_STRING,
+				Transform:   transform.FromField("Dataset.Description"),
+			},
+			{
+				Name:        "etag",
+				Description: "Etag identifies change in the resource.",
+				Type:        proto.ColumnType_STRING,
+				Transform:   transform.FromField("Etag"),
+			},
+			{
+				Name:        "type",
+				Description: "The resource type.",
+				Type:        proto.ColumnType_STRING,
+			},
+			{
+				Name:        "properties",
+				Description: "Dataset ElapsedTime Metric Policy.",
+				Type:        proto.ColumnType_JSON,
+			},
+
+			// Steampipe standard columns
+			{
+				Name:        "title",
+				Description: ColumnDescriptionTitle,
+				Type:        proto.ColumnType_STRING,
+				Transform:   transform.FromField("Name"),
+			},
+			{
+				Name:        "akas",
+				Description: ColumnDescriptionAkas,
+				Type:        proto.ColumnType_JSON,
+				Transform:   transform.FromField("ID").Transform(idToAkas),
+			},
+
+			// Azure standard column
+			{
+				Name:        "resource_group",
+				Description: ColumnDescriptionResourceGroup,
+				Type:        proto.ColumnType_STRING,
+				Transform:   transform.FromField("ID").Transform(extractResourceGroupFromID),
+			},
+			{
+				Name:        "subscription_id",
+				Description: ColumnDescriptionSubscription,
+				Type:        proto.ColumnType_STRING,
+				Transform:   transform.FromField("ID").Transform(idToSubscriptionID),
+			},
+		},
+	}
+}
+
+type DatasetInfo = struct {
+	datafactory.DatasetResource
+	FactoryName string
+}
+
+//// LIST FUNCTION
+
+func listDataFactoryDatasets(ctx context.Context, d *plugin.QueryData, h *plugin.HydrateData) (interface{}, error) {
+	session, err := GetNewSession(ctx, d, "MANAGEMENT")
+	if err != nil {
+		return nil, err
+	}
+
+	factoryInfo := h.Item.(datafactory.Factory)
+	resourceGroup := strings.Split(*factoryInfo.ID, "/")[4]
+
+	subscriptionID := session.SubscriptionID
+
+	datasetClient := datafactory.NewDatasetsClient(subscriptionID)
+	datasetClient.Authorizer = session.Authorizer
+	pagesLeft := true
+
+	for pagesLeft {
+		result, err := datasetClient.ListByFactory(ctx, resourceGroup, *factoryInfo.Name)
+		if err != nil {
+			return nil, err
+		}
+
+		for _, dataset := range result.Values() {
+			d.StreamListItem(ctx, DatasetInfo{dataset, *factoryInfo.Name})
+		}
+		result.NextWithContext(context.Background())
+		pagesLeft = result.NotDone()
+	}
+	return nil, err
+}
+
+//// HYDRATE FUNCTIONS
+
+func getDataFactoryDataset(ctx context.Context, d *plugin.QueryData, h *plugin.HydrateData) (interface{}, error) {
+	plugin.Logger(ctx).Trace("getDataFactoryDataset")
+
+	datasetName := d.KeyColumnQuals["name"].GetStringValue()
+	resourceGroup := d.KeyColumnQuals["resource_group"].GetStringValue()
+	factoryName := d.KeyColumnQuals["factory_name"].GetStringValue()
+
+	if datasetName == "" || resourceGroup == "" || factoryName == "" {
+		return nil, nil
+	}
+
+	session, err := GetNewSession(ctx, d, "MANAGEMENT")
+	if err != nil {
+		return nil, err
+	}
+	subscriptionID := session.SubscriptionID
+
+	datasetClient := datafactory.NewDatasetsClient(subscriptionID)
+	datasetClient.Authorizer = session.Authorizer
+
+	op, err := datasetClient.Get(ctx, resourceGroup, factoryName, datasetName, "")
+	if err != nil {
+		return nil, err
+	}
+
+	// In some cases resource does not give any notFound error
+	// instead of notFound error, it returns empty data
+	if op.ID != nil {
+		return DatasetInfo{op, factoryName}, nil
+	}
+
+	return nil, nil
+}

docs/tables/azure_data_factory_dataset.md

@@ -0,0 +1,33 @@
+# Table: azure_data_factory_dataset
+
+ Datasets identify data within different data stores, such as tables, files, folders, and documents.
+
+## Examples
+
+### Basic info
+
+```sql
+select
+  name,
+  id,
+  description,
+  etag,
+  type
+from
+  azure_data_factory_dataset;
+```
+
+
+### List relational table type datasets
+
+```sql
+select
+  name,
+  id,
+  type,
+  properties ->> 'type' as dataset_type
+from
+  azure_data_factory_dataset
+where
+  properties ->> 'type' = 'RelationalTable';
+```