From 1cabbac6c6f7be7d6c7a3c81ec3118ad994b6cfe Mon Sep 17 00:00:00 2001 From: Arnab <45350738+c0d3r-arnab@users.noreply.github.com> Date: Tue, 5 Oct 2021 13:17:24 +0530 Subject: [PATCH] Add table azure_hpc_cache. Closes #320 (#374) --- .../tests/azure_hpc_cache/dependencies.txt | 0 .../azure_hpc_cache/test-get-expected.json | 11 + .../tests/azure_hpc_cache/test-get-query.sql | 3 + .../azure_hpc_cache/test-list-expected.json | 11 + .../tests/azure_hpc_cache/test-list-query.sql | 3 + .../test-not-found-expected.json | 1 + .../azure_hpc_cache/test-not-found-query.sql | 3 + .../azure_hpc_cache/test-turbot-expected.json | 10 + .../azure_hpc_cache/test-turbot-query.sql | 3 + .../tests/azure_hpc_cache/variables.json | 1 + azure-test/tests/azure_hpc_cache/variables.tf | 94 ++++++ azure/plugin.go | 1 + azure/table_azure_hpc_cache.go | 307 ++++++++++++++++++ docs/tables/azure_hpc_cache.md | 46 +++ go.sum | 2 - 15 files changed, 494 insertions(+), 2 deletions(-) create mode 100644 azure-test/tests/azure_hpc_cache/dependencies.txt create mode 100644 azure-test/tests/azure_hpc_cache/test-get-expected.json create mode 100644 azure-test/tests/azure_hpc_cache/test-get-query.sql create mode 100644 azure-test/tests/azure_hpc_cache/test-list-expected.json create mode 100644 azure-test/tests/azure_hpc_cache/test-list-query.sql create mode 100644 azure-test/tests/azure_hpc_cache/test-not-found-expected.json create mode 100644 azure-test/tests/azure_hpc_cache/test-not-found-query.sql create mode 100644 azure-test/tests/azure_hpc_cache/test-turbot-expected.json create mode 100644 azure-test/tests/azure_hpc_cache/test-turbot-query.sql create mode 100644 azure-test/tests/azure_hpc_cache/variables.json create mode 100644 azure-test/tests/azure_hpc_cache/variables.tf create mode 100644 azure/table_azure_hpc_cache.go create mode 100644 docs/tables/azure_hpc_cache.md diff --git a/azure-test/tests/azure_hpc_cache/dependencies.txt b/azure-test/tests/azure_hpc_cache/dependencies.txt new file mode 100644 index 00000000..e69de29b diff --git a/azure-test/tests/azure_hpc_cache/test-get-expected.json b/azure-test/tests/azure_hpc_cache/test-get-expected.json new file mode 100644 index 00000000..f90adff7 --- /dev/null +++ b/azure-test/tests/azure_hpc_cache/test-get-expected.json @@ -0,0 +1,11 @@ +[ + { + "id": "{{ output.resource_id.value }}", + "name": "{{ resourceName }}", + "region": "{{ output.region.value }}", + "resource_group": "{{ resourceName }}", + "sku_name": "{{ output.sku_name.value }}", + "subscription_id": "{{ output.subscription_id.value }}", + "type": "Microsoft.StorageCache/caches" + } +] diff --git a/azure-test/tests/azure_hpc_cache/test-get-query.sql b/azure-test/tests/azure_hpc_cache/test-get-query.sql new file mode 100644 index 00000000..b6122fd3 --- /dev/null +++ b/azure-test/tests/azure_hpc_cache/test-get-query.sql @@ -0,0 +1,3 @@ +select name, id, type, region, resource_group, subscription_id, sku_name +from azure.azure_hpc_cache +where name = '{{ resourceName }}' and resource_group = '{{ resourceName }}'; diff --git a/azure-test/tests/azure_hpc_cache/test-list-expected.json b/azure-test/tests/azure_hpc_cache/test-list-expected.json new file mode 100644 index 00000000..22bf5b96 --- /dev/null +++ b/azure-test/tests/azure_hpc_cache/test-list-expected.json @@ -0,0 +1,11 @@ +[ + { + "id": "{{ output.resource_id_upper.value }}", + "name": "{{ resourceName }}", + "region": "{{ output.region.value }}", + "resource_group": "{{ resourceName }}", + "sku_name": "{{ output.sku_name.value }}", + "subscription_id": "{{ output.subscription_id.value }}", + "type": "Microsoft.StorageCache/caches" + } +] diff --git a/azure-test/tests/azure_hpc_cache/test-list-query.sql b/azure-test/tests/azure_hpc_cache/test-list-query.sql new file mode 100644 index 00000000..23408300 --- /dev/null +++ b/azure-test/tests/azure_hpc_cache/test-list-query.sql @@ -0,0 +1,3 @@ +select name, id, type, region, resource_group, subscription_id, sku_name +from azure.azure_hpc_cache +where name = '{{ resourceName }}'; \ No newline at end of file diff --git a/azure-test/tests/azure_hpc_cache/test-not-found-expected.json b/azure-test/tests/azure_hpc_cache/test-not-found-expected.json new file mode 100644 index 00000000..19765bd5 --- /dev/null +++ b/azure-test/tests/azure_hpc_cache/test-not-found-expected.json @@ -0,0 +1 @@ +null diff --git a/azure-test/tests/azure_hpc_cache/test-not-found-query.sql b/azure-test/tests/azure_hpc_cache/test-not-found-query.sql new file mode 100644 index 00000000..69028744 --- /dev/null +++ b/azure-test/tests/azure_hpc_cache/test-not-found-query.sql @@ -0,0 +1,3 @@ +select name, id, type, region +from azure.azure_hpc_cache +where name = 'dummy-test{{ resourceName }}' and resource_group = '{{ resourceName }}'; diff --git a/azure-test/tests/azure_hpc_cache/test-turbot-expected.json b/azure-test/tests/azure_hpc_cache/test-turbot-expected.json new file mode 100644 index 00000000..02cd3c76 --- /dev/null +++ b/azure-test/tests/azure_hpc_cache/test-turbot-expected.json @@ -0,0 +1,10 @@ +[ + { + "akas": [ + "{{ output.resource_aka.value }}", + "{{ output.resource_aka_lower.value }}" + ], + "name": "{{ resourceName }}", + "title": "{{ resourceName }}" + } +] diff --git a/azure-test/tests/azure_hpc_cache/test-turbot-query.sql b/azure-test/tests/azure_hpc_cache/test-turbot-query.sql new file mode 100644 index 00000000..e86dae2f --- /dev/null +++ b/azure-test/tests/azure_hpc_cache/test-turbot-query.sql @@ -0,0 +1,3 @@ +select name, akas, title +from azure.azure_hpc_cache +where name = '{{ resourceName }}' and resource_group = '{{ resourceName }}'; diff --git a/azure-test/tests/azure_hpc_cache/variables.json b/azure-test/tests/azure_hpc_cache/variables.json new file mode 100644 index 00000000..0967ef42 --- /dev/null +++ b/azure-test/tests/azure_hpc_cache/variables.json @@ -0,0 +1 @@ +{} diff --git a/azure-test/tests/azure_hpc_cache/variables.tf b/azure-test/tests/azure_hpc_cache/variables.tf new file mode 100644 index 00000000..341f4209 --- /dev/null +++ b/azure-test/tests/azure_hpc_cache/variables.tf @@ -0,0 +1,94 @@ +variable "resource_name" { + type = string + default = "turbot-test-20201001-create-update" + description = "Name of the resource used throughout the test." +} + +variable "azure_environment" { + type = string + default = "public" + description = "Azure environment used for the test." +} + +variable "azure_subscription" { + type = string + default = "3510ae4d-530b-497d-8f30-53b9616fc6c1" + description = "Azure environment used for the test." +} + +terraform { + required_providers { + azurerm = { + source = "hashicorp/azurerm" + version = "=2.78.0" + } + } +} + +provider "azurerm" { + # Cannot be passed as a variable + features {} + environment = var.azure_environment + subscription_id = var.azure_subscription +} + +resource "azurerm_resource_group" "named_test_resource" { + name = var.resource_name + location = "West Europe" +} + +resource "azurerm_virtual_network" "named_test_resource" { + name = var.resource_name + address_space = ["10.0.0.0/16"] + location = azurerm_resource_group.named_test_resource.location + resource_group_name = azurerm_resource_group.named_test_resource.name +} + +resource "azurerm_subnet" "named_test_resource" { + name = var.resource_name + resource_group_name = azurerm_resource_group.named_test_resource.name + virtual_network_name = azurerm_virtual_network.named_test_resource.name + address_prefixes = ["10.0.1.0/24"] +} + +resource "azurerm_hpc_cache" "named_test_resource" { + name = var.resource_name + resource_group_name = azurerm_resource_group.named_test_resource.name + location = azurerm_resource_group.named_test_resource.location + cache_size_in_gb = 3072 + subnet_id = azurerm_subnet.named_test_resource.id + sku_name = "Standard_2G" +} + +output "region" { + value = azurerm_resource_group.named_test_resource.location +} + +output "resource_aka" { + depends_on = [azurerm_hpc_cache.named_test_resource] + value = "azure://${azurerm_hpc_cache.named_test_resource.id}" +} + +output "resource_aka_lower" { + value = "azure://${lower(azurerm_hpc_cache.named_test_resource.id)}" +} + +output "resource_name" { + value = var.resource_name +} + +output "resource_id" { + value = azurerm_hpc_cache.named_test_resource.id +} + +output "resource_id_upper" { + value = "/subscriptions/${var.azure_subscription}/resourceGroups/${upper(var.resource_name)}/providers/Microsoft.StorageCache/caches/${var.resource_name}" +} + +output "subscription_id" { + value = var.azure_subscription +} + +output "sku_name" { + value = azurerm_hpc_cache.named_test_resource.sku_name +} diff --git a/azure/plugin.go b/azure/plugin.go index 08f84f7b..d7126c2a 100644 --- a/azure/plugin.go +++ b/azure/plugin.go @@ -70,6 +70,7 @@ func Plugin(ctx context.Context) *plugin.Plugin { "azure_firewall": tableAzureFirewall(ctx), "azure_frontdoor": tableAzureFrontDoor(ctx), "azure_healthcare_service": tableAzureHealthcareService(ctx), + "azure_hpc_cache": tableAzureHPCCache(ctx), "azure_iothub": tableAzureIotHub(ctx), "azure_iothub_dps": tableAzureIotHubDps(ctx), "azure_key_vault": tableAzureKeyVault(ctx), diff --git a/azure/table_azure_hpc_cache.go b/azure/table_azure_hpc_cache.go new file mode 100644 index 00000000..8159baa6 --- /dev/null +++ b/azure/table_azure_hpc_cache.go @@ -0,0 +1,307 @@ +package azure + +import ( + "context" + + "github.com/Azure/go-autorest/autorest/date" + "github.com/Azure/azure-sdk-for-go/services/storagecache/mgmt/2021-05-01/storagecache" + "github.com/turbot/steampipe-plugin-sdk/grpc/proto" + "github.com/turbot/steampipe-plugin-sdk/plugin/transform" + + "github.com/turbot/steampipe-plugin-sdk/plugin" +) + +//// TABLE DEFINITION + +func tableAzureHPCCache(_ context.Context) *plugin.Table { + return &plugin.Table{ + Name: "azure_hpc_cache", + Description: "Azure HPC Cache", + Get: &plugin.GetConfig{ + KeyColumns: plugin.AllColumns([]string{"name", "resource_group"}), + Hydrate: getHPCCache, + ShouldIgnoreError: isNotFoundError([]string{"ResourceNotFound", "ResourceGroupNotFound", "404"}), + }, + List: &plugin.ListConfig{ + Hydrate: listHPCCaches, + }, + Columns: []*plugin.Column{ + { + Name: "name", + Description: "The name of the cache.", + Type: proto.ColumnType_STRING, + }, + { + Name: "id", + Description: "The resource ID of the cache.", + Type: proto.ColumnType_STRING, + Transform: transform.FromGo(), + }, + { + Name: "provisioning_state", + Description: "ARM provisioning state. Possible values include: 'Succeeded', 'Failed', 'Cancelled', 'Creating', 'Deleting', 'Updating'.", + Type: proto.ColumnType_STRING, + Transform: transform.FromField("CacheProperties.ProvisioningState"), + }, + { + Name: "type", + Description: "The type of the cache.", + Type: proto.ColumnType_STRING, + }, + { + Name: "cache_size_gb", + Description: "The size of the cache, in GB.", + Type: proto.ColumnType_INT, + Transform: transform.FromField("CacheProperties.CacheSizeGB"), + }, + { + Name: "sku_name", + Description: "The SKU for the cache.", + Type: proto.ColumnType_STRING, + Transform: transform.FromField("Sku.Name"), + }, + { + Name: "subnet", + Description: "Subnet used for the cache.", + Type: proto.ColumnType_STRING, + Transform: transform.FromField("CacheProperties.Subnet"), + }, + { + Name: "directory_services_settings", + Description: "Specifies directory services settings of the cache.", + Type: proto.ColumnType_JSON, + Transform: transform.FromField("CacheProperties.DirectoryServicesSettings"), + }, + { + Name: "encryption_settings", + Description: "Specifies encryption settings of the cache.", + Type: proto.ColumnType_JSON, + Transform: transform.FromField("CacheProperties.EncryptionSettings"), + }, + { + Name: "health", + Description: "The health of the cache.", + Type: proto.ColumnType_JSON, + Transform: transform.FromField("CacheProperties.Health"), + }, + { + Name: "identity", + Description: "The identity of the cache, if configured.", + Type: proto.ColumnType_JSON, + }, + { + Name: "mount_addresses", + Description: "Array of IP addresses that can be used by clients mounting the cache.", + Type: proto.ColumnType_JSON, + Transform: transform.FromField("CacheProperties.MountAddresses"), + }, + { + Name: "network_settings", + Description: "Specifies network settings of the cache.", + Type: proto.ColumnType_JSON, + Transform: transform.From(extractHPCCacheNetworkSettings), + }, + { + Name: "security_settings", + Description: "Specifies security settings of the cache.", + Type: proto.ColumnType_JSON, + Transform: transform.FromField("CacheProperties.SecuritySettings"), + }, + { + Name: "system_data", + Description: "The system meta data relating to the resource.", + Type: proto.ColumnType_JSON, + }, + { + Name: "upgrade_status", + Description: "Upgrade status of the cache.", + Type: proto.ColumnType_JSON, + Transform: transform.From(extractHPCCacheUpgradeStatus), + }, + + // Steampipe standard columns + { + Name: "title", + Description: ColumnDescriptionTitle, + Type: proto.ColumnType_STRING, + Transform: transform.FromField("Name"), + }, + { + Name: "tags", + Description: ColumnDescriptionTags, + Type: proto.ColumnType_JSON, + }, + { + Name: "akas", + Description: ColumnDescriptionAkas, + Type: proto.ColumnType_JSON, + Transform: transform.FromField("ID").Transform(idToAkas), + }, + + // Azure standard columns + { + Name: "region", + Description: ColumnDescriptionRegion, + Type: proto.ColumnType_STRING, + Transform: transform.FromField("Location").Transform(toLower), + }, + { + Name: "resource_group", + Description: ColumnDescriptionResourceGroup, + Type: proto.ColumnType_STRING, + Transform: transform.FromField("ID").Transform(extractResourceGroupFromID), + }, + { + Name: "subscription_id", + Description: ColumnDescriptionSubscription, + Type: proto.ColumnType_STRING, + Transform: transform.FromField("ID").Transform(idToSubscriptionID), + }, + }, + } +} + +type CacheNetworkSettingsInfo struct { + Mtu *int32 + UtilityAddresses *[]string + DNSServers *[]string + DNSSearchDomain *string + NtpServer *string +} + +type CacheUpgradeStatusInfo struct { + CurrentFirmwareVersion *string + FirmwareUpdateStatus interface{} + FirmwareUpdateDeadline *date.Time + LastFirmwareUpdate *date.Time + PendingFirmwareVersion *string +} + +//// LIST FUNCTION + +func listHPCCaches(ctx context.Context, d *plugin.QueryData, _ *plugin.HydrateData) (interface{}, error) { + session, err := GetNewSession(ctx, d, "MANAGEMENT") + if err != nil { + return nil, err + } + subscriptionID := session.SubscriptionID + + client := storagecache.NewCachesClient(subscriptionID) + client.Authorizer = session.Authorizer + + result, err := client.List(ctx) + if err != nil { + plugin.Logger(ctx).Error("listHPCCaches", "list", err) + return nil, err + } + + for _, cache := range result.Values() { + d.StreamListItem(ctx, cache) + } + + for result.NotDone() { + err = result.NextWithContext(ctx) + if err != nil { + plugin.Logger(ctx).Error("listHPCCaches", "list_paging", err) + return nil, err + } + for _, cache := range result.Values() { + d.StreamListItem(ctx, cache) + } + } + + return nil, err +} + +//// HYDRATE FUNCTION + +func getHPCCache(ctx context.Context, d *plugin.QueryData, h *plugin.HydrateData) (interface{}, error) { + plugin.Logger(ctx).Trace("getHPCCache") + + name := d.KeyColumnQuals["name"].GetStringValue() + resourceGroup := d.KeyColumnQuals["resource_group"].GetStringValue() + + // Handle empty name or resourceGroup + if name == "" || resourceGroup == "" { + return nil, nil + } + + session, err := GetNewSession(ctx, d, "MANAGEMENT") + if err != nil { + return nil, err + } + subscriptionID := session.SubscriptionID + + client := storagecache.NewCachesClient(subscriptionID) + client.Authorizer = session.Authorizer + + op, err := client.Get(ctx, resourceGroup, name) + if err != nil { + plugin.Logger(ctx).Error("getHPCCache", "get", err) + return nil, err + } + + // In some cases resource does not give any notFound error + // instead of notFound error, it returns empty data + if op.ID != nil { + return op, nil + } + + return nil, nil +} + +//// TRANSFORM FUNCTIONS + +// If we return the API response directly, the output does not provide +// all the properties of NetworkSettings +func extractHPCCacheNetworkSettings(ctx context.Context, d *transform.TransformData) (interface{}, error) { + cache := d.HydrateItem.(storagecache.Cache) + var properties CacheNetworkSettingsInfo + + if cache.CacheProperties.NetworkSettings != nil { + if cache.CacheProperties.NetworkSettings.Mtu != nil { + properties.Mtu = cache.CacheProperties.NetworkSettings.Mtu + } + if cache.CacheProperties.NetworkSettings.UtilityAddresses != nil { + properties.UtilityAddresses = cache.CacheProperties.NetworkSettings.UtilityAddresses + } + if cache.CacheProperties.NetworkSettings.DNSServers != nil { + properties.DNSServers = cache.CacheProperties.NetworkSettings.DNSServers + } + if cache.CacheProperties.NetworkSettings.DNSSearchDomain != nil { + properties.DNSSearchDomain = cache.CacheProperties.NetworkSettings.DNSSearchDomain + } + if cache.CacheProperties.NetworkSettings.NtpServer != nil { + properties.NtpServer = cache.CacheProperties.NetworkSettings.NtpServer + } + } + + return properties, nil +} + +// If we return the API response directly, the output does not provide +// all the properties of UpgradeStatus +func extractHPCCacheUpgradeStatus(ctx context.Context, d *transform.TransformData) (interface{}, error) { + cache := d.HydrateItem.(storagecache.Cache) + var properties CacheUpgradeStatusInfo + + if cache.CacheProperties.UpgradeStatus != nil { + if cache.CacheProperties.UpgradeStatus.CurrentFirmwareVersion != nil { + properties.CurrentFirmwareVersion = cache.CacheProperties.UpgradeStatus.CurrentFirmwareVersion + } + if len(cache.CacheProperties.UpgradeStatus.FirmwareUpdateStatus) > 0 { + properties.FirmwareUpdateStatus = cache.CacheProperties.UpgradeStatus.FirmwareUpdateStatus + } + if cache.CacheProperties.UpgradeStatus.FirmwareUpdateDeadline != nil { + properties.FirmwareUpdateDeadline = cache.CacheProperties.UpgradeStatus.FirmwareUpdateDeadline + } + if cache.CacheProperties.UpgradeStatus.LastFirmwareUpdate != nil { + properties.LastFirmwareUpdate = cache.CacheProperties.UpgradeStatus.LastFirmwareUpdate + } + if cache.CacheProperties.UpgradeStatus.PendingFirmwareVersion != nil { + properties.PendingFirmwareVersion = cache.CacheProperties.UpgradeStatus.PendingFirmwareVersion + } + } + + return properties, nil +} diff --git a/docs/tables/azure_hpc_cache.md b/docs/tables/azure_hpc_cache.md new file mode 100644 index 00000000..3773e3af --- /dev/null +++ b/docs/tables/azure_hpc_cache.md @@ -0,0 +1,46 @@ +# Table: azure_hpc_cache + +Azure HPC Cache speeds access to your data for high-performance computing (HPC) tasks. By caching files in Azure, Azure HPC Cache brings the scalability of cloud computing to your existing workflow. This service can be used even for workflows where your data is stored across WAN links, such as in your local datacenter network-attached storage (NAS) environment. + +## Examples + +### Basic info + +```sql +select + id, + name, + type, + provisioning_state, + sku_name +from + azure_hpc_cache; +``` + +### List network settings details + +```sql +select + id, + name, + network_settings ->> 'DNSSearchDomain' as dns_search_domain, + network_settings -> 'Mtu' as mtu, + network_settings ->> 'NtpServer' as ntp_server, + jsonb_pretty(network_settings -> 'DNSServers') as dns_servers, + jsonb_pretty(network_settings -> 'UtilityAddresses') as utility_addresses +from + azure_hpc_cache; +``` + +### List encryption settings details + +```sql +select + id, + name, + encryption_settings -> 'keyEncryptionKey' ->> 'keyUrl' as key_url, + encryption_settings -> 'keyEncryptionKey' -> 'sourceVault' ->> 'id' as source_vault_id, + network_settings -> 'rotationToLatestKeyVersionEnabled' as rotation_to_latest_key_version_enabled +from + azure_hpc_cache; +``` diff --git a/go.sum b/go.sum index bd2600f1..4df7cc76 100644 --- a/go.sum +++ b/go.sum @@ -3,8 +3,6 @@ github.com/Azure/azure-pipeline-go v0.2.3 h1:7U9HBg1JFK3jHl5qmo4CTZKFTVgMwdFHMVt github.com/Azure/azure-pipeline-go v0.2.3/go.mod h1:x841ezTBIMG6O3lAcl8ATHnsOPVl2bqk7S3ta6S6u4k= github.com/Azure/azure-sdk-for-go v45.0.0+incompatible/go.mod h1:9XXNKU+eRnpl9moKnB4QOLf1HestfXbmab5FXxiDBjc= github.com/Azure/azure-sdk-for-go v47.1.0+incompatible/go.mod h1:9XXNKU+eRnpl9moKnB4QOLf1HestfXbmab5FXxiDBjc= -github.com/Azure/azure-sdk-for-go v55.4.0+incompatible h1:zn3+QwgCjRgIRgE0iWpF2LrFy5er/T0e5/EHrp1Zgak= -github.com/Azure/azure-sdk-for-go v55.4.0+incompatible/go.mod h1:9XXNKU+eRnpl9moKnB4QOLf1HestfXbmab5FXxiDBjc= github.com/Azure/azure-sdk-for-go v58.0.0+incompatible h1:Cw16jiP4dI+CK761aq44ol4RV5dUiIIXky1+EKpoiVM= github.com/Azure/azure-sdk-for-go v58.0.0+incompatible/go.mod h1:9XXNKU+eRnpl9moKnB4QOLf1HestfXbmab5FXxiDBjc= github.com/Azure/azure-storage-blob-go v0.12.0 h1:7bFXA1QB+lOK2/ASWHhp6/vnxjaeeZq6t8w1Jyp0Iaw=