Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add tables for monitoring statistics of Virtual Machine cpu utilization and disk read/write ops #166

Merged
merged 16 commits into from
Jul 8, 2021
Merged

Add tables for monitoring statistics of Virtual Machine cpu utilization and disk read/write ops #166

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
16 commits
Select commit Hold shift + click to select a range
8d5f005
Add table azure_compute_virtual_machine_metric_cpu_utilization_ close…
ParthaI Jul 1, 2021
6de1191
Add table azure_compute_virtual_machine_metric_cpu_utilization_daily …
ParthaI Jul 1, 2021
3925ca9
Add table azure_compute_virtual_machine_metric_cpu_utilization_hourly…
ParthaI Jul 1, 2021
292526e
Add table azure_compute_disk_metric_read_ops closes #161
ParthaI Jul 1, 2021
be86b2e
Add table azure_compute_disk_metric_read_ops_hourly closes #162
ParthaI Jul 1, 2021
1b11b21
Add table azure_compute_disk_metric_read_ops_daily closes #148
ParthaI Jul 1, 2021
6c12bbb
Add table azure_compute_disk_metric_write_ops closes #164
ParthaI Jul 2, 2021
d2800fa
Update the doc
ParthaI Jul 2, 2021
311537b
Add table azure_compute_disk_metric_write_ops_hourly closes #165
ParthaI Jul 2, 2021
b8ede4b
Add table azure_compute_disk_metric_write_ops_daily closes #149
ParthaI Jul 2, 2021
6580a28
Made changes as per review comments
ParthaI Jul 5, 2021
23c852c
Merge branch 'main' of github.com:turbot/steampipe-plugin-azure into …
Subhajit97 Jul 5, 2021
852b5ff
Fixed table descriptions
Subhajit97 Jul 5, 2021
5dce283
Updated docs
Subhajit97 Jul 6, 2021
449c808
Merge branch 'main' into issue-147
cbruno10 Jul 8, 2021
24a4e6a
Update monitoring_metric.go
cbruno10 Jul 8, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
164 changes: 164 additions & 0 deletions azure/monitoring_metric.go
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,164 @@
package azure

import (
"context"
"strings"
"time"

"github.com/Azure/azure-sdk-for-go/services/preview/monitor/mgmt/2021-04-01-preview/insights"
"github.com/turbot/steampipe-plugin-sdk/grpc/proto"
"github.com/turbot/steampipe-plugin-sdk/plugin"
"github.com/turbot/steampipe-plugin-sdk/plugin/transform"
)

type monitoringMetric struct {
// Resource Name
DimensionValue string
// MetadataValue represents a metric metadata value.
MetaData *insights.MetadataValue
// Metric the result data of a query.
Metric *insights.Metric
// The maximum metric value for the data point.
Maximum *float64
// The minimum metric value for the data point.
Minimum *float64
// The average of the metric values that correspond to the data point.
Average *float64
// The number of metric values that contributed to the aggregate value of this data point.
SampleCount *float64
// The sum of the metric values for the data point.
Sum *float64
// The time stamp used for the data point.
TimeStamp string
// The units in which the metric value is reported.
Unit string
}

//// TABLE DEFINITION

func monitoringMetricColumns(columns []*plugin.Column) []*plugin.Column {
return append(columns, commonMonitoringMetricColumns()...)
}

func commonMonitoringMetricColumns() []*plugin.Column {
return []*plugin.Column{
{
Name: "maximum",
Description: "The maximum metric value for the data point.",
Type: proto.ColumnType_DOUBLE,
},
{
Name: "minimum",
Description: "The minimum metric value for the data point.",
Type: proto.ColumnType_DOUBLE,
},
{
Name: "average",
Description: "The average of the metric values that correspond to the data point.",
Type: proto.ColumnType_DOUBLE,
},
{
Name: "sample_count",
Description: "The number of metric values that contributed to the aggregate value of this data point.",
Type: proto.ColumnType_DOUBLE,
},
{
Name: "sum",
Description: "The sum of the metric values for the data point.",
Type: proto.ColumnType_DOUBLE,
},
{
Name: "timestamp",
Description: "The time stamp used for the data point.",
Type: proto.ColumnType_TIMESTAMP,
Transform: transform.FromField("TimeStamp"),
},
{
Name: "unit",
Description: "The units in which the metric value is reported.",
Type: proto.ColumnType_STRING,
},
{
Name: "resource_group",
Description: ColumnDescriptionResourceGroup,
Type: proto.ColumnType_STRING,
Transform: transform.FromField("DimensionValue").Transform(extractResourceGroupFromID),
},
{
Name: "subscription_id",
Description: ColumnDescriptionSubscription,
Type: proto.ColumnType_STRING,
Transform: transform.FromField("DimensionValue").Transform(idToSubscriptionID),
},
}
}

func getMonitoringIntervalForGranularity(granularity string) string {
switch strings.ToUpper(granularity) {
case "DAILY":
// 24 hours
return "PT24H"
case "HOURLY":
// 1 hour
return "PT1H"
}
// else 5 minutes
return "PT5M"
}

func getMonitoringStartDateForGranularity(granularity string) string {
switch strings.ToUpper(granularity) {
case "DAILY":
// Last 1 year
return time.Now().UTC().AddDate(-1, 0, 0).Format(time.RFC3339)
case "HOURLY":
// Last 60 days
return time.Now().UTC().AddDate(0, 0, -60).Format(time.RFC3339)
}
// Last 5 days
return time.Now().UTC().AddDate(0, 0, -5).Format(time.RFC3339)
}

func listAzureMonitorMetricStatistics(ctx context.Context, d *plugin.QueryData, granularity string, metricNameSpace string, metricNames string, dimensionValue string) (interface{}, error) {
session, err := GetNewSession(ctx, d, "MANAGEMENT")
if err != nil {
return nil, err
}
subscriptionID := session.SubscriptionID

monitoringClient := insights.NewMetricsClient(subscriptionID)
monitoringClient.Authorizer = session.Authorizer

// Define param values
interval := getMonitoringIntervalForGranularity(granularity)
aggregation := "average,count,maximum,minimum,total"
timeSpan := getMonitoringStartDateForGranularity(granularity) + "/" + time.Now().UTC().AddDate(0, 0, 1).Format(time.RFC3339) // Retrieve data within a year
orderBy := "timestamp"
top := int32(1000) // Maximum number of record fetch with given interval
filter := ""

result, err := monitoringClient.List(ctx, dimensionValue, timeSpan, &interval, metricNames, aggregation, &top, orderBy, filter, insights.ResultTypeData, metricNameSpace)
if err != nil {
return nil, err
}
for _, metric := range *result.Value {
for _, timeseries := range *metric.Timeseries {
for _, data := range *timeseries.Data {
if data.Average != nil {
d.StreamListItem(ctx, &monitoringMetric{
DimensionValue: dimensionValue,
TimeStamp: data.TimeStamp.Format(time.RFC3339),
Maximum: data.Maximum,
Minimum: data.Minimum,
Average: data.Average,
Sum: data.Total,
SampleCount: data.Count,
Unit: string(metric.Unit),
})
}
}
}
}

return nil, nil
}
131 changes: 70 additions & 61 deletions azure/plugin.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,67 +22,76 @@ func Plugin(ctx context.Context) *plugin.Plugin {
Schema: ConfigSchema,
},
TableMap: map[string]*plugin.Table{
"azure_ad_group": tableAzureAdGroup(ctx),
"azure_ad_service_principal": tableAzureAdServicePrincipal(ctx),
"azure_ad_user": tableAzureAdUser(ctx),
"azure_api_management": tableAzureAPIManagement(ctx),
"azure_app_service_environment": tableAzureAppServiceEnvironment(ctx),
"azure_app_service_function_app": tableAzureAppServiceFunctionApp(ctx),
"azure_app_service_plan": tableAzureAppServicePlan(ctx),
"azure_app_service_web_app": tableAzureAppServiceWebApp(ctx),
"azure_application_security_group": tableAzureApplicationSecurityGroup(ctx),
"azure_compute_availability_set": tableAzureComputeAvailabilitySet(ctx),
"azure_compute_disk": tableAzureComputeDisk(ctx),
"azure_compute_disk_encryption_set": tableAzureComputeDiskEncryptionSet(ctx),
"azure_compute_image": tableAzureComputeImage(ctx),
"azure_compute_resource_sku": tableAzureResourceSku(ctx),
"azure_compute_snapshot": tableAzureComputeSnapshot(ctx),
"azure_compute_virtual_machine": tableAzureComputeVirtualMachine(ctx),
"azure_cosmosdb_account": tableAzureCosmosDBAccount(ctx),
"azure_cosmosdb_mongo_database": tableAzureCosmosDBMongoDatabase(ctx),
"azure_cosmosdb_sql_database": tableAzureCosmosDBSQLDatabase(ctx),
"azure_diagnostic_setting": tableAzureDiagnosticSetting(ctx),
"azure_express_route_circuit": tableAzureExpressRouteCircuit(ctx),
"azure_firewall": tableAzureFirewall(ctx),
"azure_key_vault": tableAzureKeyVault(ctx),
"azure_key_vault_key": tableAzureKeyVaultKey(ctx),
"azure_key_vault_secret": tableAzureKeyVaultSecret(ctx),
"azure_kubernetes_cluster": tableAzureKubernetesCluster(ctx),
"azure_location": tableAzureLocation(ctx),
"azure_log_alert": tableAzureLogAlert(ctx),
"azure_log_profile": tableAzureLogProfile(ctx),
"azure_management_lock": tableAzureManagementLock(ctx),
"azure_mysql_server": tableAzureMySQLServer(ctx),
"azure_network_interface": tableAzureNetworkInterface(ctx),
"azure_network_security_group": tableAzureNetworkSecurityGroup(ctx),
"azure_network_watcher": tableAzureNetworkWatcher(ctx),
"azure_network_watcher_flow_log": tableAzureNetworkWatcherFlowLog(ctx),
"azure_policy_assignment": tableAzurePolicyAssignment(ctx),
"azure_policy_definition": tableAzurePolicyDefinition(ctx),
"azure_postgresql_server": tableAzurePostgreSqlServer(ctx),
"azure_provider": tableAzureProvider(ctx),
"azure_public_ip": tableAzurePublicIP(ctx),
"azure_resource_group": tableAzureResourceGroup(ctx),
"azure_role_assignment": tableAzureIamRoleAssignment(ctx),
"azure_role_definition": tableAzureIamRoleDefinition(ctx),
"azure_route_table": tableAzureRouteTable(ctx),
"azure_security_center_auto_provisioning": tableAzureSecurityCenterAutoProvisioning(ctx),
"azure_security_center_contact": tableAzureSecurityCenterContact(ctx),
"azure_security_center_setting": tableAzureSecurityCenterSetting(ctx),
"azure_security_center_subscription_pricing": tableAzureSecurityCenterPricing(ctx),
"azure_sql_database": tableAzureSqlDatabase(ctx),
"azure_sql_server": tableAzureSQLServer(ctx),
"azure_storage_account": tableAzureStorageAccount(ctx),
"azure_storage_blob": tableAzureStorageBlob(ctx),
"azure_storage_blob_service": tableAzureStorageBlobService(ctx),
"azure_storage_container": tableAzureStorageContainer(ctx),
"azure_storage_queue": tableAzureStorageQueue(ctx),
"azure_storage_table": tableAzureStorageTable(ctx),
"azure_storage_table_service": tableAzureStorageTableService(ctx),
"azure_subnet": tableAzureSubnet(ctx),
"azure_subscription": tableAzureSubscription(ctx),
"azure_tenant": tableAzureTenant(ctx),
"azure_virtual_network": tableAzureVirtualNetwork(ctx),
"azure_ad_group": tableAzureAdGroup(ctx),
"azure_ad_service_principal": tableAzureAdServicePrincipal(ctx),
"azure_ad_user": tableAzureAdUser(ctx),
"azure_api_management": tableAzureAPIManagement(ctx),
"azure_app_service_environment": tableAzureAppServiceEnvironment(ctx),
"azure_app_service_function_app": tableAzureAppServiceFunctionApp(ctx),
"azure_app_service_plan": tableAzureAppServicePlan(ctx),
"azure_app_service_web_app": tableAzureAppServiceWebApp(ctx),
"azure_application_security_group": tableAzureApplicationSecurityGroup(ctx),
"azure_compute_availability_set": tableAzureComputeAvailabilitySet(ctx),
"azure_compute_disk": tableAzureComputeDisk(ctx),
"azure_compute_disk_encryption_set": tableAzureComputeDiskEncryptionSet(ctx),
"azure_compute_disk_metric_read_ops": tableComputeDisksReadOpsMetric(ctx),
"azure_compute_disk_metric_read_ops_hourly": tableComputeDisksReadOpsMetricHourly(ctx),
"azure_compute_disk_metric_read_ops_daily": tableComputeDisksReadOpsMetricDaily(ctx),
"azure_compute_disk_metric_write_ops": tableComputeDisksWriteOpsMetric(ctx),
"azure_compute_disk_metric_write_ops_daily": tableComputeDisksWriteOpsMetricDaily(ctx),
"azure_compute_disk_metric_write_ops_hourly": tableComputeDisksWriteOpsMetricHourly(ctx),
"azure_compute_image": tableAzureComputeImage(ctx),
"azure_compute_resource_sku": tableAzureResourceSku(ctx),
"azure_compute_snapshot": tableAzureComputeSnapshot(ctx),
"azure_compute_virtual_machine": tableAzureComputeVirtualMachine(ctx),
"azure_compute_virtual_machine_metric_cpu_utilization": tableComputeInstanceCpuUtilizationMetric(ctx),
"azure_compute_virtual_machine_metric_cpu_utilization_daily": tableComputeInstanceCpuUtilizationMetricDaily(ctx),
"azure_compute_virtual_machine_metric_cpu_utilization_hourly": tableComputeInstanceCpuUtilizationMetricHourly(ctx),
"azure_cosmosdb_account": tableAzureCosmosDBAccount(ctx),
"azure_cosmosdb_mongo_database": tableAzureCosmosDBMongoDatabase(ctx),
"azure_cosmosdb_sql_database": tableAzureCosmosDBSQLDatabase(ctx),
"azure_diagnostic_setting": tableAzureDiagnosticSetting(ctx),
"azure_express_route_circuit": tableAzureExpressRouteCircuit(ctx),
"azure_firewall": tableAzureFirewall(ctx),
"azure_key_vault": tableAzureKeyVault(ctx),
"azure_key_vault_key": tableAzureKeyVaultKey(ctx),
"azure_key_vault_secret": tableAzureKeyVaultSecret(ctx),
"azure_kubernetes_cluster": tableAzureKubernetesCluster(ctx),
"azure_location": tableAzureLocation(ctx),
"azure_log_alert": tableAzureLogAlert(ctx),
"azure_log_profile": tableAzureLogProfile(ctx),
"azure_management_lock": tableAzureManagementLock(ctx),
"azure_mysql_server": tableAzureMySQLServer(ctx),
"azure_network_interface": tableAzureNetworkInterface(ctx),
"azure_network_security_group": tableAzureNetworkSecurityGroup(ctx),
"azure_network_watcher": tableAzureNetworkWatcher(ctx),
"azure_network_watcher_flow_log": tableAzureNetworkWatcherFlowLog(ctx),
"azure_policy_assignment": tableAzurePolicyAssignment(ctx),
"azure_policy_definition": tableAzurePolicyDefinition(ctx),
"azure_postgresql_server": tableAzurePostgreSqlServer(ctx),
"azure_provider": tableAzureProvider(ctx),
"azure_public_ip": tableAzurePublicIP(ctx),
"azure_resource_group": tableAzureResourceGroup(ctx),
"azure_role_assignment": tableAzureIamRoleAssignment(ctx),
"azure_role_definition": tableAzureIamRoleDefinition(ctx),
"azure_route_table": tableAzureRouteTable(ctx),
"azure_security_center_auto_provisioning": tableAzureSecurityCenterAutoProvisioning(ctx),
"azure_security_center_contact": tableAzureSecurityCenterContact(ctx),
"azure_security_center_setting": tableAzureSecurityCenterSetting(ctx),
"azure_security_center_subscription_pricing": tableAzureSecurityCenterPricing(ctx),
"azure_sql_database": tableAzureSqlDatabase(ctx),
"azure_sql_server": tableAzureSQLServer(ctx),
"azure_storage_account": tableAzureStorageAccount(ctx),
"azure_storage_blob": tableAzureStorageBlob(ctx),
"azure_storage_blob_service": tableAzureStorageBlobService(ctx),
"azure_storage_container": tableAzureStorageContainer(ctx),
"azure_storage_queue": tableAzureStorageQueue(ctx),
"azure_storage_table": tableAzureStorageTable(ctx),
"azure_storage_table_service": tableAzureStorageTableService(ctx),
"azure_subnet": tableAzureSubnet(ctx),
"azure_subscription": tableAzureSubscription(ctx),
"azure_tenant": tableAzureTenant(ctx),
"azure_virtual_network": tableAzureVirtualNetwork(ctx),
// "azure_storage_table": tableAzureStorageTable(ctx),
},
}
Expand Down
39 changes: 39 additions & 0 deletions azure/table_azure_compute_disk_metric_read_ops.go
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,39 @@
package azure

import (
"context"

"github.com/Azure/azure-sdk-for-go/services/compute/mgmt/2020-06-01/compute"
"github.com/turbot/steampipe-plugin-sdk/grpc/proto"
"github.com/turbot/steampipe-plugin-sdk/plugin"
"github.com/turbot/steampipe-plugin-sdk/plugin/transform"
)

//// TABLE DEFINITION

func tableComputeDisksReadOpsMetric(_ context.Context) *plugin.Table {
return &plugin.Table{
Name: "azure_compute_disk_metric_read_ops",
Description: "Azure Compute Disk Metrics - Read Ops",
List: &plugin.ListConfig{
ParentHydrate: listAzureComputeDisks,
Hydrate: listComputeDiskMetricReadOps,
},
Columns: monitoringMetricColumns([]*plugin.Column{
{
Name: "name",
Description: "The name of the disk.",
Type: proto.ColumnType_STRING,
Transform: transform.FromField("DimensionValue").Transform(lastPathElement),
},
}),
}
}

//// LIST FUNCTION

func listComputeDiskMetricReadOps(ctx context.Context, d *plugin.QueryData, h *plugin.HydrateData) (interface{}, error) {
diskInfo := h.Item.(compute.Disk)

return listAzureMonitorMetricStatistics(ctx, d, "FIVE_MINUTES", "Microsoft.Compute/disks", "Composite Disk Read Operations/sec", *diskInfo.ID)
}
39 changes: 39 additions & 0 deletions azure/table_azure_compute_disk_metric_read_ops_daily.go
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,39 @@
package azure

import (
"context"

"github.com/Azure/azure-sdk-for-go/services/compute/mgmt/2020-06-01/compute"
"github.com/turbot/steampipe-plugin-sdk/grpc/proto"
"github.com/turbot/steampipe-plugin-sdk/plugin"
"github.com/turbot/steampipe-plugin-sdk/plugin/transform"
)

//// TABLE DEFINITION

func tableComputeDisksReadOpsMetricDaily(_ context.Context) *plugin.Table {
return &plugin.Table{
Name: "azure_compute_disk_metric_read_ops_daily",
Description: "Azure Compute Disk Metrics - Read Ops (Daily)",
List: &plugin.ListConfig{
ParentHydrate: listAzureComputeDisks,
Hydrate: listComputeDiskMetricReadOpsDaily,
},
Columns: monitoringMetricColumns([]*plugin.Column{
{
Name: "name",
Description: "The name of the disk.",
Type: proto.ColumnType_STRING,
Transform: transform.FromField("DimensionValue").Transform(lastPathElement),
},
}),
}
}

//// LIST FUNCTION

func listComputeDiskMetricReadOpsDaily(ctx context.Context, d *plugin.QueryData, h *plugin.HydrateData) (interface{}, error) {
diskInfo := h.Item.(compute.Disk)

return listAzureMonitorMetricStatistics(ctx, d, "DAILY", "Microsoft.Compute/disks", "Composite Disk Read Operations/sec", *diskInfo.ID)
}
Loading