Skip to content

Commit

Permalink
Add table azure_data_factory_dataset. Closes #150 (#168)
Browse files Browse the repository at this point in the history
  • Loading branch information
rajeshbal65 authored Jul 13, 2021
1 parent 048bce9 commit 85be243
Show file tree
Hide file tree
Showing 14 changed files with 320 additions and 0 deletions.
Empty file.
Original file line number Diff line number Diff line change
@@ -0,0 +1,9 @@
[
{
"id": "{{ output.resource_id.value }}",
"name": "{{ resourceName }}",
"resource_group": "{{ resourceName }}",
"subscription_id": "{{ output.subscription_id.value }}",
"type": "Microsoft.DataFactory/factories/datasets"
}
]
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
select name, id, type, resource_group, subscription_id
from azure.azure_data_factory_dataset
where name = '{{ resourceName }}' and resource_group = '{{ resourceName }}' and factory_name = '{{ resourceName }}';
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
[
{
"id": "{{ output.resource_id.value }}",
"name": "{{ resourceName }}",
"type": "Microsoft.DataFactory/factories/datasets"
}
]
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
select name, id, type
from azure.azure_data_factory_dataset
where name = '{{ resourceName }}';
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
null
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
select name, title, akas
from azure.azure_data_factory_dataset
where name = 'dummy-{{ resourceName }}' and resource_group = '{{ resourceName }}';
Original file line number Diff line number Diff line change
@@ -0,0 +1,10 @@
[
{
"akas": [
"{{ output.resource_aka.value }}",
"{{ output.resource_aka_lower.value }}"
],
"name": "{{ resourceName }}",
"title": "{{ resourceName }}"
}
]
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
select name, title, akas
from azure.azure_data_factory_dataset
where name = '{{ resourceName }}' and resource_group = '{{ resourceName }}';
1 change: 1 addition & 0 deletions azure-test/tests/azure_data_factory_dataset/variables.json
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
{}
81 changes: 81 additions & 0 deletions azure-test/tests/azure_data_factory_dataset/variables.tf
Original file line number Diff line number Diff line change
@@ -0,0 +1,81 @@

variable "resource_name" {
type = string
default = "steampipe-test"
description = "Name of the resource used throughout the test."
}

variable "azure_environment" {
type = string
default = "public"
description = "Azure environment used for the test."
}

variable "azure_subscription" {
type = string
default = "3510ae4d-530b-497d-8f30-53b9616fc6c1"
description = "Azure subscription used for the test."
}

provider "azurerm" {
# Cannot be passed as a variable
version = "=1.36.0"
environment = var.azure_environment
subscription_id = var.azure_subscription
}

data "azurerm_client_config" "current" {}

data "null_data_source" "resource" {
inputs = {
scope = "azure:///subscriptions/${data.azurerm_client_config.current.subscription_id}"
}
}

resource "azurerm_resource_group" "named_test_resource" {
name = var.resource_name
location = "East US"
}

resource "azurerm_data_factory" "named_test_resource" {
name = var.resource_name
location = "East US"
resource_group_name = azurerm_resource_group.named_test_resource.name
tags = {
name = var.resource_name
}
}

resource "azurerm_data_factory_linked_service_mysql" "named_test_resource" {
name = var.resource_name
resource_group_name = azurerm_resource_group.named_test_resource.name
data_factory_name = azurerm_data_factory.named_test_resource.name
connection_string = "Server=test;Port=3306;Database=test;User=test;SSLMode=1;UseSystemTrustStore=0;Password=test"
}

resource "azurerm_data_factory_dataset_mysql" "named_test_resource" {
name = var.resource_name
resource_group_name = azurerm_resource_group.named_test_resource.name
data_factory_name = azurerm_data_factory.named_test_resource.name
linked_service_name = azurerm_data_factory_linked_service_mysql.named_test_resource.name
}

output "resource_aka" {
value = "azure://${azurerm_data_factory_dataset_mysql.named_test_resource.id}"
}

output "resource_aka_lower" {
value = "azure://${lower(azurerm_data_factory_dataset_mysql.named_test_resource.id)}"
}

output "resource_name" {
value = var.resource_name
}

output "resource_id" {
value = azurerm_data_factory_dataset_mysql.named_test_resource.id
}

output "subscription_id" {
value = var.azure_subscription
}
1 change: 1 addition & 0 deletions azure/plugin.go
Original file line number Diff line number Diff line change
Expand Up @@ -51,6 +51,7 @@ func Plugin(ctx context.Context) *plugin.Plugin {
"azure_cosmosdb_mongo_database": tableAzureCosmosDBMongoDatabase(ctx),
"azure_cosmosdb_sql_database": tableAzureCosmosDBSQLDatabase(ctx),
"azure_data_factory": tableAzureDataFactory(ctx),
"azure_data_factory_dataset": tableAzureDataFactoryDataset(ctx),
"azure_diagnostic_setting": tableAzureDiagnosticSetting(ctx),
"azure_express_route_circuit": tableAzureExpressRouteCircuit(ctx),
"azure_firewall": tableAzureFirewall(ctx),
Expand Down
165 changes: 165 additions & 0 deletions azure/table_azure_data_factory_dataset.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,165 @@
package azure

import (
"context"
"strings"

"github.com/Azure/azure-sdk-for-go/services/datafactory/mgmt/2018-06-01/datafactory"
"github.com/turbot/steampipe-plugin-sdk/grpc/proto"
"github.com/turbot/steampipe-plugin-sdk/plugin/transform"

"github.com/turbot/steampipe-plugin-sdk/plugin"
)

//// TABLE DEFINITION

func tableAzureDataFactoryDataset(_ context.Context) *plugin.Table {
return &plugin.Table{
Name: "azure_data_factory_dataset",
Description: "Azure Data Factory Dataset",
Get: &plugin.GetConfig{
KeyColumns: plugin.AllColumns([]string{"name", "resource_group", "factory_name"}),
Hydrate: getDataFactoryDataset,
ShouldIgnoreError: isNotFoundError([]string{"ResourceNotFound", "ResourceGroupNotFound", "404"}),
},
List: &plugin.ListConfig{
Hydrate: listDataFactoryDatasets,
ParentHydrate: listDataFactories,
},
Columns: []*plugin.Column{
{
Name: "name",
Description: "The resource name.",
Type: proto.ColumnType_STRING,
},
{
Name: "id",
Description: "The resource identifier.",
Type: proto.ColumnType_STRING,
Transform: transform.FromGo(),
},
{
Name: "factory_name",
Description: "Name of the factory the dataset belongs.",
Type: proto.ColumnType_STRING,
},
{
Name: "etag",
Description: "An unique read-only string that changes whenever the resource is updated.",
Type: proto.ColumnType_STRING,
},
{
Name: "type",
Description: "The resource type.",
Type: proto.ColumnType_STRING,
},
{
Name: "properties",
Description: "Dataset properties.",
Type: proto.ColumnType_JSON,
},

// Steampipe standard columns
{
Name: "title",
Description: ColumnDescriptionTitle,
Type: proto.ColumnType_STRING,
Transform: transform.FromField("Name"),
},
{
Name: "akas",
Description: ColumnDescriptionAkas,
Type: proto.ColumnType_JSON,
Transform: transform.FromField("ID").Transform(idToAkas),
},

// Azure standard column
{
Name: "resource_group",
Description: ColumnDescriptionResourceGroup,
Type: proto.ColumnType_STRING,
Transform: transform.FromField("ID").Transform(extractResourceGroupFromID),
},
{
Name: "subscription_id",
Description: ColumnDescriptionSubscription,
Type: proto.ColumnType_STRING,
Transform: transform.FromField("ID").Transform(idToSubscriptionID),
},
},
}
}

type DatasetInfo = struct {
datafactory.DatasetResource
FactoryName string
}

//// LIST FUNCTION

func listDataFactoryDatasets(ctx context.Context, d *plugin.QueryData, h *plugin.HydrateData) (interface{}, error) {
session, err := GetNewSession(ctx, d, "MANAGEMENT")
if err != nil {
return nil, err
}
subscriptionID := session.SubscriptionID

// Get factory details
factoryInfo := h.Item.(datafactory.Factory)
resourceGroup := strings.Split(*factoryInfo.ID, "/")[4]

datasetClient := datafactory.NewDatasetsClient(subscriptionID)
datasetClient.Authorizer = session.Authorizer

pagesLeft := true
for pagesLeft {
result, err := datasetClient.ListByFactory(ctx, resourceGroup, *factoryInfo.Name)
if err != nil {
return nil, err
}
for _, dataset := range result.Values() {
d.StreamListItem(ctx, DatasetInfo{dataset, *factoryInfo.Name})
}
result.NextWithContext(context.Background())
pagesLeft = result.NotDone()
}
return nil, err
}

//// HYDRATE FUNCTIONS

func getDataFactoryDataset(ctx context.Context, d *plugin.QueryData, _ *plugin.HydrateData) (interface{}, error) {
plugin.Logger(ctx).Trace("getDataFactoryDataset")

// Create session
session, err := GetNewSession(ctx, d, "MANAGEMENT")
if err != nil {
return nil, err
}
subscriptionID := session.SubscriptionID

datasetClient := datafactory.NewDatasetsClient(subscriptionID)
datasetClient.Authorizer = session.Authorizer

datasetName := d.KeyColumnQuals["name"].GetStringValue()
resourceGroup := d.KeyColumnQuals["resource_group"].GetStringValue()
factoryName := d.KeyColumnQuals["factory_name"].GetStringValue()

// Return nil, of no input provided
if datasetName == "" || resourceGroup == "" || factoryName == "" {
return nil, nil
}

op, err := datasetClient.Get(ctx, resourceGroup, factoryName, datasetName, "*")
if err != nil {
return nil, err
}

// In some cases resource does not give any notFound error
// instead of notFound error, it returns empty data
if op.ID != nil {
return DatasetInfo{op, factoryName}, nil
}

return nil, nil
}
33 changes: 33 additions & 0 deletions docs/tables/azure_data_factory_dataset.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,33 @@
# Table: azure_data_factory_dataset

Azure Data Factory datasets identify data within different data stores, such as tables, files, folders, and documents.

## Examples

### Basic info

```sql
select
name,
id,
description,
etag,
type
from
azure_data_factory_dataset;
```


### List relational table type datasets

```sql
select
name,
id,
type,
properties ->> 'type' as dataset_type
from
azure_data_factory_dataset
where
properties ->> 'type' = 'RelationalTable';
```

0 comments on commit 85be243

Please sign in to comment.