Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add table azure_data_factory_dataset. Closes #150 #168

Merged
merged 7 commits into from
Jul 13, 2021
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Empty file.
Original file line number Diff line number Diff line change
@@ -0,0 +1,9 @@
[
{
"id": "{{ output.resource_id.value }}",
"name": "{{ resourceName }}",
"resource_group": "{{ resourceName }}",
"subscription_id": "{{ output.subscription_id.value }}",
"type": "Microsoft.DataFactory/factories/datasets"
}
]
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
select name, id, type, resource_group, subscription_id
from azure.azure_data_factory_dataset
where name = '{{ resourceName }}' and resource_group = '{{ resourceName }}' and factory_name = '{{ resourceName }}';
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
[
{
"id": "{{ output.resource_id.value }}",
"name": "{{ resourceName }}",
"type": "Microsoft.DataFactory/factories/datasets"
}
]
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
select name, id, type
from azure.azure_data_factory_dataset
where name = '{{ resourceName }}';
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
null
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
select name, title, akas
from azure.azure_data_factory_dataset
where name = 'dummy-{{ resourceName }}' and resource_group = '{{ resourceName }}';
Original file line number Diff line number Diff line change
@@ -0,0 +1,10 @@
[
{
"akas": [
"{{ output.resource_aka.value }}",
"{{ output.resource_aka_lower.value }}"
],
"name": "{{ resourceName }}",
"title": "{{ resourceName }}"
}
]
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
select name, title, akas
from azure.azure_data_factory_dataset
where name = '{{ resourceName }}' and resource_group = '{{ resourceName }}';
1 change: 1 addition & 0 deletions azure-test/tests/azure_data_factory_dataset/variables.json
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
{}
81 changes: 81 additions & 0 deletions azure-test/tests/azure_data_factory_dataset/variables.tf
Original file line number Diff line number Diff line change
@@ -0,0 +1,81 @@

variable "resource_name" {
type = string
default = "steampipe-test"
description = "Name of the resource used throughout the test."
}

variable "azure_environment" {
type = string
default = "public"
description = "Azure environment used for the test."
}

variable "azure_subscription" {
type = string
default = "3510ae4d-530b-497d-8f30-53b9616fc6c1"
description = "Azure subscription used for the test."
}

provider "azurerm" {
# Cannot be passed as a variable
version = "=1.36.0"
environment = var.azure_environment
subscription_id = var.azure_subscription
}

data "azurerm_client_config" "current" {}

data "null_data_source" "resource" {
inputs = {
scope = "azure:///subscriptions/${data.azurerm_client_config.current.subscription_id}"
}
}

resource "azurerm_resource_group" "named_test_resource" {
name = var.resource_name
location = "East US"
}

resource "azurerm_data_factory" "named_test_resource" {
name = var.resource_name
location = "East US"
resource_group_name = azurerm_resource_group.named_test_resource.name
tags = {
name = var.resource_name
}
}

resource "azurerm_data_factory_linked_service_mysql" "named_test_resource" {
name = var.resource_name
resource_group_name = azurerm_resource_group.named_test_resource.name
data_factory_name = azurerm_data_factory.named_test_resource.name
connection_string = "Server=test;Port=3306;Database=test;User=test;SSLMode=1;UseSystemTrustStore=0;Password=test"
}

resource "azurerm_data_factory_dataset_mysql" "named_test_resource" {
name = var.resource_name
resource_group_name = azurerm_resource_group.named_test_resource.name
data_factory_name = azurerm_data_factory.named_test_resource.name
linked_service_name = azurerm_data_factory_linked_service_mysql.named_test_resource.name
}

output "resource_aka" {
value = "azure://${azurerm_data_factory_dataset_mysql.named_test_resource.id}"
}

output "resource_aka_lower" {
value = "azure://${lower(azurerm_data_factory_dataset_mysql.named_test_resource.id)}"
}

output "resource_name" {
value = var.resource_name
}

output "resource_id" {
value = azurerm_data_factory_dataset_mysql.named_test_resource.id
}

output "subscription_id" {
value = var.azure_subscription
}
1 change: 1 addition & 0 deletions azure/plugin.go
Original file line number Diff line number Diff line change
Expand Up @@ -51,6 +51,7 @@ func Plugin(ctx context.Context) *plugin.Plugin {
"azure_cosmosdb_mongo_database": tableAzureCosmosDBMongoDatabase(ctx),
"azure_cosmosdb_sql_database": tableAzureCosmosDBSQLDatabase(ctx),
"azure_data_factory": tableAzureDataFactory(ctx),
"azure_data_factory_dataset": tableAzureDataFactoryDataset(ctx),
"azure_diagnostic_setting": tableAzureDiagnosticSetting(ctx),
"azure_express_route_circuit": tableAzureExpressRouteCircuit(ctx),
"azure_firewall": tableAzureFirewall(ctx),
Expand Down
165 changes: 165 additions & 0 deletions azure/table_azure_data_factory_dataset.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,165 @@
package azure

import (
"context"
"strings"

"github.com/Azure/azure-sdk-for-go/services/datafactory/mgmt/2018-06-01/datafactory"
"github.com/turbot/steampipe-plugin-sdk/grpc/proto"
"github.com/turbot/steampipe-plugin-sdk/plugin/transform"

"github.com/turbot/steampipe-plugin-sdk/plugin"
)

//// TABLE DEFINITION

func tableAzureDataFactoryDataset(_ context.Context) *plugin.Table {
return &plugin.Table{
Name: "azure_data_factory_dataset",
Description: "Azure Data Factory Dataset",
Get: &plugin.GetConfig{
KeyColumns: plugin.AllColumns([]string{"name", "resource_group", "factory_name"}),
Hydrate: getDataFactoryDataset,
ShouldIgnoreError: isNotFoundError([]string{"ResourceNotFound", "ResourceGroupNotFound", "404"}),
},
List: &plugin.ListConfig{
Hydrate: listDataFactoryDatasets,
ParentHydrate: listDataFactories,
},
Columns: []*plugin.Column{
{
Name: "name",
Description: "The resource name.",
Type: proto.ColumnType_STRING,
},
{
Name: "id",
Description: "The resource identifier.",
Type: proto.ColumnType_STRING,
Transform: transform.FromGo(),
},
{
Name: "factory_name",
Description: "Name of the factory the dataset belongs.",
Type: proto.ColumnType_STRING,
},
{
Name: "etag",
Description: "An unique read-only string that changes whenever the resource is updated.",
Type: proto.ColumnType_STRING,
},
{
Name: "type",
Description: "The resource type.",
Type: proto.ColumnType_STRING,
},
{
Name: "properties",
Description: "Dataset properties.",
Type: proto.ColumnType_JSON,
},

// Steampipe standard columns
{
Name: "title",
Description: ColumnDescriptionTitle,
Type: proto.ColumnType_STRING,
Transform: transform.FromField("Name"),
},
{
Name: "akas",
Description: ColumnDescriptionAkas,
Type: proto.ColumnType_JSON,
Transform: transform.FromField("ID").Transform(idToAkas),
},

// Azure standard column
{
Name: "resource_group",
Description: ColumnDescriptionResourceGroup,
Type: proto.ColumnType_STRING,
Transform: transform.FromField("ID").Transform(extractResourceGroupFromID),
},
{
Name: "subscription_id",
Description: ColumnDescriptionSubscription,
Type: proto.ColumnType_STRING,
Transform: transform.FromField("ID").Transform(idToSubscriptionID),
},
},
}
}

type DatasetInfo = struct {
datafactory.DatasetResource
FactoryName string
}

//// LIST FUNCTION

func listDataFactoryDatasets(ctx context.Context, d *plugin.QueryData, h *plugin.HydrateData) (interface{}, error) {
session, err := GetNewSession(ctx, d, "MANAGEMENT")
if err != nil {
return nil, err
}
subscriptionID := session.SubscriptionID

// Get factory details
factoryInfo := h.Item.(datafactory.Factory)
resourceGroup := strings.Split(*factoryInfo.ID, "/")[4]

datasetClient := datafactory.NewDatasetsClient(subscriptionID)
datasetClient.Authorizer = session.Authorizer

pagesLeft := true
for pagesLeft {
result, err := datasetClient.ListByFactory(ctx, resourceGroup, *factoryInfo.Name)
if err != nil {
return nil, err
}
for _, dataset := range result.Values() {
d.StreamListItem(ctx, DatasetInfo{dataset, *factoryInfo.Name})
}
result.NextWithContext(context.Background())
pagesLeft = result.NotDone()
}
return nil, err
}

//// HYDRATE FUNCTIONS

func getDataFactoryDataset(ctx context.Context, d *plugin.QueryData, _ *plugin.HydrateData) (interface{}, error) {
plugin.Logger(ctx).Trace("getDataFactoryDataset")

Subhajit97 marked this conversation as resolved.
Show resolved Hide resolved
// Create session
session, err := GetNewSession(ctx, d, "MANAGEMENT")
if err != nil {
return nil, err
}
subscriptionID := session.SubscriptionID

datasetClient := datafactory.NewDatasetsClient(subscriptionID)
datasetClient.Authorizer = session.Authorizer

datasetName := d.KeyColumnQuals["name"].GetStringValue()
resourceGroup := d.KeyColumnQuals["resource_group"].GetStringValue()
factoryName := d.KeyColumnQuals["factory_name"].GetStringValue()

// Return nil, of no input provided
if datasetName == "" || resourceGroup == "" || factoryName == "" {
return nil, nil
}

op, err := datasetClient.Get(ctx, resourceGroup, factoryName, datasetName, "*")
if err != nil {
return nil, err
}

// In some cases resource does not give any notFound error
// instead of notFound error, it returns empty data
if op.ID != nil {
return DatasetInfo{op, factoryName}, nil
}

return nil, nil
}
33 changes: 33 additions & 0 deletions docs/tables/azure_data_factory_dataset.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,33 @@
# Table: azure_data_factory_dataset

Azure Data Factory datasets identify data within different data stores, such as tables, files, folders, and documents.

## Examples

### Basic info

```sql
select
name,
id,
description,
etag,
type
from
azure_data_factory_dataset;
```


### List relational table type datasets

```sql
select
name,
id,
type,
properties ->> 'type' as dataset_type
from
azure_data_factory_dataset
where
properties ->> 'type' = 'RelationalTable';
```