diff --git a/azure-test/tests/azure_machine_learning_workspace/dependencies.txt b/azure-test/tests/azure_machine_learning_workspace/dependencies.txt new file mode 100644 index 00000000..e69de29b diff --git a/azure-test/tests/azure_machine_learning_workspace/test-get-expected.json b/azure-test/tests/azure_machine_learning_workspace/test-get-expected.json new file mode 100644 index 00000000..ee683041 --- /dev/null +++ b/azure-test/tests/azure_machine_learning_workspace/test-get-expected.json @@ -0,0 +1,13 @@ +[ + { + "id": "{{ output.resource_id.value }}", + "name": "{{ resourceName }}", + "region": "{{ output.location.value }}", + "resource_group": "{{ resourceName }}", + "subscription_id": "{{ output.subscription_id.value }}", + "tags": { + "name": "{{ resourceName }}" + }, + "type": "Microsoft.MachineLearningServices/workspaces" + } +] diff --git a/azure-test/tests/azure_machine_learning_workspace/test-get-query.sql b/azure-test/tests/azure_machine_learning_workspace/test-get-query.sql new file mode 100644 index 00000000..e84fca51 --- /dev/null +++ b/azure-test/tests/azure_machine_learning_workspace/test-get-query.sql @@ -0,0 +1,3 @@ +select name, id, type, tags, region, resource_group, subscription_id +from azure.azure_machine_learning_workspace +where name = '{{ resourceName }}' and resource_group = '{{ resourceName }}'; \ No newline at end of file diff --git a/azure-test/tests/azure_machine_learning_workspace/test-list-expected.json b/azure-test/tests/azure_machine_learning_workspace/test-list-expected.json new file mode 100644 index 00000000..2317d14e --- /dev/null +++ b/azure-test/tests/azure_machine_learning_workspace/test-list-expected.json @@ -0,0 +1,7 @@ +[ + { + "id": "{{ output.resource_id.value }}", + "name": "{{ resourceName }}", + "type": "Microsoft.MachineLearningServices/workspaces" + } +] diff --git a/azure-test/tests/azure_machine_learning_workspace/test-list-query.sql b/azure-test/tests/azure_machine_learning_workspace/test-list-query.sql new file mode 100644 index 00000000..e5f8445d --- /dev/null +++ b/azure-test/tests/azure_machine_learning_workspace/test-list-query.sql @@ -0,0 +1,3 @@ +select name, id, type +from azure.azure_machine_learning_workspace +where name = '{{ resourceName }}'; \ No newline at end of file diff --git a/azure-test/tests/azure_machine_learning_workspace/test-not-found-expected.json b/azure-test/tests/azure_machine_learning_workspace/test-not-found-expected.json new file mode 100644 index 00000000..19765bd5 --- /dev/null +++ b/azure-test/tests/azure_machine_learning_workspace/test-not-found-expected.json @@ -0,0 +1 @@ +null diff --git a/azure-test/tests/azure_machine_learning_workspace/test-not-found-query.sql b/azure-test/tests/azure_machine_learning_workspace/test-not-found-query.sql new file mode 100644 index 00000000..ad59e610 --- /dev/null +++ b/azure-test/tests/azure_machine_learning_workspace/test-not-found-query.sql @@ -0,0 +1,3 @@ +select name, tags, title, akas +from azure.azure_machine_learning_workspace +where name = 'dummy-{{ resourceName }}' and resource_group = '{{ resourceName }}'; \ No newline at end of file diff --git a/azure-test/tests/azure_machine_learning_workspace/test-turbot-expected.json b/azure-test/tests/azure_machine_learning_workspace/test-turbot-expected.json new file mode 100644 index 00000000..43ffbc2c --- /dev/null +++ b/azure-test/tests/azure_machine_learning_workspace/test-turbot-expected.json @@ -0,0 +1,13 @@ +[ + { + "akas": [ + "{{ output.resource_aka.value }}", + "{{ output.resource_aka_lower.value }}" + ], + "name": "{{ resourceName }}", + "tags": { + "name": "{{ resourceName }}" + }, + "title": "{{ resourceName }}" + } +] diff --git a/azure-test/tests/azure_machine_learning_workspace/test-turbot-query.sql b/azure-test/tests/azure_machine_learning_workspace/test-turbot-query.sql new file mode 100644 index 00000000..6b999b21 --- /dev/null +++ b/azure-test/tests/azure_machine_learning_workspace/test-turbot-query.sql @@ -0,0 +1,3 @@ +select name, tags, title, akas +from azure.azure_machine_learning_workspace +where name = '{{ resourceName }}' and resource_group = '{{ resourceName }}'; \ No newline at end of file diff --git a/azure-test/tests/azure_machine_learning_workspace/variables.json b/azure-test/tests/azure_machine_learning_workspace/variables.json new file mode 100644 index 00000000..0967ef42 --- /dev/null +++ b/azure-test/tests/azure_machine_learning_workspace/variables.json @@ -0,0 +1 @@ +{} diff --git a/azure-test/tests/azure_machine_learning_workspace/variables.tf b/azure-test/tests/azure_machine_learning_workspace/variables.tf new file mode 100644 index 00000000..f639a327 --- /dev/null +++ b/azure-test/tests/azure_machine_learning_workspace/variables.tf @@ -0,0 +1,103 @@ + +variable "resource_name" { + type = string + default = "steampipe-test" + description = "Name of the resource used throughout the test." +} + +variable "azure_environment" { + type = string + default = "public" + description = "Azure environment used for the test." +} + +variable "azure_subscription" { + type = string + default = "3510ae4d-530b-497d-8f30-53b9616fc6c1" + description = "Azure subscription used for the test." +} + +provider "azurerm" { + # Cannot be passed as a variable + version = "=2.41.0" + environment = var.azure_environment + subscription_id = var.azure_subscription + features {} +} + +data "azurerm_client_config" "current" {} + +data "null_data_source" "resource" { + inputs = { + scope = "azure:///subscriptions/${data.azurerm_client_config.current.subscription_id}" + } +} + +resource "azurerm_resource_group" "named_test_resource" { + name = var.resource_name + location = "East US" +} + +resource "azurerm_application_insights" "named_test_resource" { + name = var.resource_name + location = azurerm_resource_group.named_test_resource.location + resource_group_name = azurerm_resource_group.named_test_resource.name + application_type = "web" +} + +resource "azurerm_key_vault" "named_test_resource" { + name = var.resource_name + location = azurerm_resource_group.named_test_resource.location + resource_group_name = azurerm_resource_group.named_test_resource.name + tenant_id = data.azurerm_client_config.current.tenant_id + sku_name = "premium" +} + +resource "azurerm_storage_account" "named_test_resource" { + name = var.resource_name + location = azurerm_resource_group.named_test_resource.location + resource_group_name = azurerm_resource_group.named_test_resource.name + account_tier = "Standard" + account_replication_type = "GRS" +} + +resource "azurerm_machine_learning_workspace" "named_test_resource" { + name = var.resource_name + location = azurerm_resource_group.named_test_resource.location + resource_group_name = azurerm_resource_group.named_test_resource.name + application_insights_id = azurerm_application_insights.named_test_resource.id + key_vault_id = azurerm_key_vault.named_test_resource.id + storage_account_id = azurerm_storage_account.named_test_resource.id + + identity { + type = "SystemAssigned" + } + + tags = { + "name" = var.resource_name + } +} + +output "resource_aka" { + value = "azure://${azurerm_machine_learning_workspace.named_test_resource.id}" +} + +output "resource_aka_lower" { + value = "azure://${lower(azurerm_machine_learning_workspace.named_test_resource.id)}" +} + +output "resource_name" { + value = var.resource_name +} + +output "resource_id" { + value = azurerm_machine_learning_workspace.named_test_resource.id +} + +output "location" { + value = azurerm_resource_group.named_test_resource.location +} + +output "subscription_id" { + value = var.azure_subscription +} diff --git a/azure/plugin.go b/azure/plugin.go index 0ad2edcf..0ecaa0de 100644 --- a/azure/plugin.go +++ b/azure/plugin.go @@ -79,6 +79,7 @@ func Plugin(ctx context.Context) *plugin.Plugin { "azure_log_alert": tableAzureLogAlert(ctx), "azure_log_profile": tableAzureLogProfile(ctx), "azure_logic_app_workflow": tableAzureLogicAppWorkflow(ctx), + "azure_machine_learning_workspace": tableAzureMachineLearningWorkspace(ctx), "azure_management_lock": tableAzureManagementLock(ctx), "azure_mariadb_server": tableAzureMariaDBServer(ctx), "azure_mssql_elasticpool": tableAzureMSSQLElasticPool(ctx), diff --git a/azure/table_azure_machine_learning_workspace.go b/azure/table_azure_machine_learning_workspace.go new file mode 100644 index 00000000..e53749bd --- /dev/null +++ b/azure/table_azure_machine_learning_workspace.go @@ -0,0 +1,315 @@ +package azure + +import ( + "context" + + "github.com/Azure/azure-sdk-for-go/services/preview/machinelearningservices/mgmt/2020-02-18-preview/machinelearningservices" + "github.com/Azure/go-autorest/autorest/date" + "github.com/turbot/steampipe-plugin-sdk/grpc/proto" + "github.com/turbot/steampipe-plugin-sdk/plugin/transform" + + "github.com/turbot/steampipe-plugin-sdk/plugin" +) + +//// TABLE DEFINITION + +func tableAzureMachineLearningWorkspace(_ context.Context) *plugin.Table { + return &plugin.Table{ + Name: "azure_machine_learning_workspace", + Description: "Azure Machine Learning Worspace", + Get: &plugin.GetConfig{ + KeyColumns: plugin.AllColumns([]string{"name", "resource_group"}), + Hydrate: getMachineLearningWorkspace, + ShouldIgnoreError: isNotFoundError([]string{"ResourceNotFound", "ResourceGroupNotFound", "Invalid input"}), + }, + List: &plugin.ListConfig{ + Hydrate: listMachineLearningWorkspaces, + }, + Columns: []*plugin.Column{ + { + Name: "name", + Description: "The resource name.", + Type: proto.ColumnType_STRING, + }, + { + Name: "friendly_name", + Description: "The friendly name for this workspace. This name in mutable.", + Type: proto.ColumnType_STRING, + }, + { + Name: "id", + Description: "The resource identifier.", + Type: proto.ColumnType_STRING, + Transform: transform.FromGo(), + }, + { + Name: "provisioning_state", + Description: "The current deployment state of workspace resource, The provisioningState is to indicate states for resource provisioning. Possible values include: 'Unknown', 'Updating', 'Creating', 'Deleting', 'Succeeded', 'Failed', 'Canceled'.", + Type: proto.ColumnType_STRING, + }, + { + Name: "creation_time", + Description: "The creation time for this workspace resource.", + Type: proto.ColumnType_TIMESTAMP, + Transform: transform.FromField("CreationTime").Transform(convertDateToTime), + }, + { + Name: "workspace_id", + Description: "The immutable id associated with this workspace.", + Type: proto.ColumnType_STRING, + Transform: transform.FromField("WorkspaceID"), + }, + { + Name: "application_insights", + Description: "ARM id of the application insights associated with this workspace. This cannot be changed once the workspace has been created.", + Type: proto.ColumnType_STRING, + }, + { + Name: "container_registry", + Description: "ARM id of the container registry associated with this workspace. This cannot be changed once the workspace has been created.", + Type: proto.ColumnType_STRING, + }, + { + Name: "description", + Description: "The description of this workspace.", + Type: proto.ColumnType_STRING, + }, + { + Name: "discovery_url", + Description: "ARM id of the container registry associated with this workspace. This cannot be changed once the workspace has been created.", + Type: proto.ColumnType_STRING, + Transform: transform.FromField("DiscoveryURL"), + }, + { + Name: "hbi_workspace", + Description: "The flag to signal HBI data in the workspace and reduce diagnostic data collected by the service.", + Type: proto.ColumnType_BOOL, + }, + { + Name: "key_vault", + Description: "ARM id of the key vault associated with this workspace, This cannot be changed once the workspace has been created.", + Type: proto.ColumnType_STRING, + }, + { + Name: "location", + Description: "The location of the resource. This cannot be changed after the resource is created.", + Type: proto.ColumnType_STRING, + }, + { + Name: "service_provisioned_resource_group", + Description: "The name of the managed resource group created by workspace RP in customer subscription if the workspace is CMK workspace.", + Type: proto.ColumnType_STRING, + }, + { + Name: "storage_account", + Description: "ARM id of the storage account associated with this workspace. This cannot be changed once the workspace has been created.", + Type: proto.ColumnType_STRING, + }, + { + Name: "studio_endpoint", + Description: "The regional endpoint for the machine learning studio service which hosts this workspace.", + Type: proto.ColumnType_STRING, + }, + { + Name: "type", + Description: "The resource type.", + Type: proto.ColumnType_STRING, + }, + { + Name: "encryption", + Description: "The encryption settings of Azure ML workspace.", + Type: proto.ColumnType_JSON, + }, + { + Name: "identity", + Description: "The identity of the resource.", + Type: proto.ColumnType_JSON, + }, + + // Steampipe standard columns + { + Name: "title", + Description: ColumnDescriptionTitle, + Type: proto.ColumnType_STRING, + Transform: transform.FromField("Name"), + }, + { + Name: "tags", + Description: ColumnDescriptionTags, + Type: proto.ColumnType_JSON, + }, + { + Name: "akas", + Description: ColumnDescriptionAkas, + Type: proto.ColumnType_JSON, + Transform: transform.FromField("ID").Transform(idToAkas), + }, + + // Azure standard column + { + Name: "region", + Description: ColumnDescriptionRegion, + Type: proto.ColumnType_STRING, + Transform: transform.FromField("Location").Transform(toLower), + }, + { + Name: "resource_group", + Description: ColumnDescriptionResourceGroup, + Type: proto.ColumnType_STRING, + Transform: transform.FromField("ID").Transform(extractResourceGroupFromID), + }, + { + Name: "subscription_id", + Description: ColumnDescriptionSubscription, + Type: proto.ColumnType_STRING, + Transform: transform.FromField("ID").Transform(idToSubscriptionID), + }, + }, + } +} + +type WorkspaceInfo struct { + ID *string + Name *string + WorkspaceID *string + Description *string + FriendlyName *string + CreationTime *date.Time + Encryption interface{} + HbiWorkspace *bool + KeyVault *string + ApplicationInsights *string + ContainerRegistry *string + StorageAccount *string + ServiceProvisionedResourceGroup *string + DiscoveryURL *string + ProvisioningState string + Identity interface{} + Location *string + Type *string + Tags map[string]*string +} + +//// LIST FUNCTION + +func listMachineLearningWorkspaces(ctx context.Context, d *plugin.QueryData, _ *plugin.HydrateData) (interface{}, error) { + session, err := GetNewSession(ctx, d, "MANAGEMENT") + if err != nil { + return nil, err + } + subscriptionID := session.SubscriptionID + + worspaceClient := machinelearningservices.NewWorkspacesClient(subscriptionID) + worspaceClient.Authorizer = session.Authorizer + + result, err := worspaceClient.ListBySubscription(ctx, "") + if err != nil { + return nil, err + } + for _, workspace := range result.Values() { + d.StreamListItem(ctx, WorkspaceInfo{ + ID: workspace.ID, + Name: workspace.Name, + WorkspaceID: workspace.WorkspaceID, + FriendlyName: workspace.FriendlyName, + Description: workspace.Description, + CreationTime: workspace.CreationTime, + Encryption: workspace.Encryption, + HbiWorkspace: workspace.HbiWorkspace, + ServiceProvisionedResourceGroup: workspace.ServiceProvisionedResourceGroup, + KeyVault: workspace.KeyVault, + ApplicationInsights: workspace.ApplicationInsights, + ContainerRegistry: workspace.ContainerRegistry, + StorageAccount: workspace.StorageAccount, + DiscoveryURL: workspace.DiscoveryURL, + ProvisioningState: string(workspace.ProvisioningState), + Identity: workspace.Identity, + Location: workspace.Location, + Type: workspace.Type, + Tags: workspace.Tags, + }) + } + + for result.NotDone() { + err = result.NextWithContext(ctx) + if err != nil { + return nil, err + } + for _, workspace := range result.Values() { + d.StreamListItem(ctx, WorkspaceInfo{ + ID: workspace.ID, + Name: workspace.Name, + WorkspaceID: workspace.WorkspaceID, + FriendlyName: workspace.FriendlyName, + Description: workspace.Description, + CreationTime: workspace.CreationTime, + Encryption: workspace.Encryption, + HbiWorkspace: workspace.HbiWorkspace, + ServiceProvisionedResourceGroup: workspace.ServiceProvisionedResourceGroup, + KeyVault: workspace.KeyVault, + ApplicationInsights: workspace.ApplicationInsights, + ContainerRegistry: workspace.ContainerRegistry, + StorageAccount: workspace.StorageAccount, + DiscoveryURL: workspace.DiscoveryURL, + ProvisioningState: string(workspace.ProvisioningState), + Identity: workspace.Identity, + Location: workspace.Location, + Type: workspace.Type, + Tags: workspace.Tags, + }) + } + + } + return nil, err +} + +//// HYDRATE FUNCTIONS + +func getMachineLearningWorkspace(ctx context.Context, d *plugin.QueryData, h *plugin.HydrateData) (interface{}, error) { + plugin.Logger(ctx).Trace("getDataFactory") + + // Create session + session, err := GetNewSession(ctx, d, "MANAGEMENT") + if err != nil { + return nil, err + } + subscriptionID := session.SubscriptionID + + worspaceClient := machinelearningservices.NewWorkspacesClient(subscriptionID) + worspaceClient.Authorizer = session.Authorizer + + name := d.KeyColumnQuals["name"].GetStringValue() + resourceGroup := d.KeyColumnQuals["resource_group"].GetStringValue() + + // Return nil, if no input provide + if name == "" || resourceGroup == "" { + return nil, nil + } + + workspace, err := worspaceClient.Get(ctx, resourceGroup, name) + if err != nil { + return nil, err + } + + return WorkspaceInfo{ + ID: workspace.ID, + Name: workspace.Name, + WorkspaceID: workspace.WorkspaceID, + FriendlyName: workspace.FriendlyName, + Description: workspace.Description, + CreationTime: workspace.CreationTime, + Encryption: workspace.Encryption, + HbiWorkspace: workspace.HbiWorkspace, + ServiceProvisionedResourceGroup: workspace.ServiceProvisionedResourceGroup, + KeyVault: workspace.KeyVault, + ApplicationInsights: workspace.ApplicationInsights, + ContainerRegistry: workspace.ContainerRegistry, + StorageAccount: workspace.StorageAccount, + DiscoveryURL: workspace.DiscoveryURL, + ProvisioningState: string(workspace.ProvisioningState), + Identity: workspace.Identity, + Location: workspace.Location, + Type: workspace.Type, + Tags: workspace.Tags, + }, nil +} diff --git a/docs/tables/azure_machine_learning_workspace.md b/docs/tables/azure_machine_learning_workspace.md new file mode 100644 index 00000000..dc23b0b7 --- /dev/null +++ b/docs/tables/azure_machine_learning_workspace.md @@ -0,0 +1,46 @@ +# Table: azure_machine_learning_workspace + +The workspace is the top-level resource for Azure Machine Learning, providing a centralized place to work with all the artifacts you create when you use Azure Machine Learning. The workspace keeps a history of all training runs, including logs, metrics, output, and a snapshot of your scripts. You use this information to determine which training run produces the best model. + +## Examples + +### Basic info + +```sql +select + name, + id, + type, + provisioning_state, + etag +from + azure_machine_learning_workspace; +``` + +### List system assigned identity type workspace + +```sql +select + name, + id, + type, + identity ->> 'type' as identity_type +from + azure_machine_learning_workspace +where + identity ->> 'type' = 'SystemAssigned'; +``` + +### List key vault used by workspaces with soft deleteion disabled + +```sql +select + m.name as workspace_name, + m.id as workspace_id, + v.soft_delete_enabled +from + azure_machine_learning_workspace as m, + azure_key_vault as v +where + lower(m.key_vault) = lower(v.id) and not v.soft_delete_enabled; +``` \ No newline at end of file