Skip to content

Commit

Permalink
Merge pull request #64 from bergmannf/add-aws-account-id-to-metrics
Browse files Browse the repository at this point in the history 
Add const labels to metrics
  • Loading branch information
@janboll
janboll authored Sep 22, 2022
2 parents 084f655 + caf2ef9 commit 0309210
Show file tree
Hide file tree
Showing 7 changed files with 117 additions and 32 deletions.
26 changes: 22 additions & 4 deletions main.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@ import (
"github.com/app-sre/aws-resource-exporter/pkg"
"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/aws/session"
"github.com/aws/aws-sdk-go/service/sts"
"github.com/go-kit/kit/log/level"
"github.com/prometheus/common/promlog"
"github.com/prometheus/common/promlog/flag"
Expand All @@ -37,6 +38,16 @@ func main() {
os.Exit(run())
}

func getAwsAccountNumber(logger log.Logger, sess *session.Session) (string, error) {
stsClient := sts.New(sess)
identityOutput, err := stsClient.GetCallerIdentity(&sts.GetCallerIdentityInput{})
if err != nil {
level.Error(logger).Log("msg", "Could not retrieve caller identity of the aws account", "err", err)
return "", err
}
return *identityOutput.Account, nil
}

func setupCollectors(logger log.Logger, configFile string) ([]prometheus.Collector, error) {
var collectors []prometheus.Collector
config, err := pkg.LoadExporterConfiguration(logger, configFile)
Expand All @@ -47,15 +58,22 @@ func setupCollectors(logger log.Logger, configFile string) ([]prometheus.Collect
level.Info(logger).Log("msg", "Configuring rds with regions", "regions", strings.Join(config.RdsConfig.Regions, ","))
level.Info(logger).Log("msg", "Configuring ec2 with regions", "regions", strings.Join(config.EC2Config.Regions, ","))
level.Info(logger).Log("msg", "Configuring route53 with region", "region", config.Route53Config.Region)
var vpcSessions []*session.Session
level.Info(logger).Log("msg", "Will VPC metrics be gathered?", "vpc-enabled", config.VpcConfig.Enabled)
// Create a single session here, because we need the accountid, before we create the other configs
awsConfig := aws.NewConfig().WithRegion("us-east-1")
sess := session.Must(session.NewSession(awsConfig))
awsAccountId, err := getAwsAccountNumber(logger, sess)
if err != nil {
return collectors, err
}
var vpcSessions []*session.Session
if config.VpcConfig.Enabled {
for _, region := range config.VpcConfig.Regions {
config := aws.NewConfig().WithRegion(region)
sess := session.Must(session.NewSession(config))
vpcSessions = append(vpcSessions, sess)
}
vpcExporter := pkg.NewVPCExporter(vpcSessions, logger, config.VpcConfig)
vpcExporter := pkg.NewVPCExporter(vpcSessions, logger, config.VpcConfig, awsAccountId)
collectors = append(collectors, vpcExporter)
go vpcExporter.CollectLoop()
}
Expand All @@ -79,15 +97,15 @@ func setupCollectors(logger log.Logger, configFile string) ([]prometheus.Collect
sess := session.Must(session.NewSession(config))
ec2Sessions = append(ec2Sessions, sess)
}
ec2Exporter := pkg.NewEC2Exporter(ec2Sessions, logger, config.EC2Config)
ec2Exporter := pkg.NewEC2Exporter(ec2Sessions, logger, config.EC2Config, awsAccountId)
collectors = append(collectors, ec2Exporter)
go ec2Exporter.CollectLoop()
}
level.Info(logger).Log("msg", "Will Route53 metrics be gathered?", "route53-enabled", config.Route53Config.Enabled)
if config.Route53Config.Enabled {
awsConfig := aws.NewConfig().WithRegion(config.Route53Config.Region)
sess := session.Must(session.NewSession(awsConfig))
r53Exporter := pkg.NewRoute53Exporter(sess, logger, config.Route53Config)
r53Exporter := pkg.NewRoute53Exporter(sess, logger, config.Route53Config, awsAccountId)
collectors = append(collectors, r53Exporter)
go r53Exporter.CollectLoop()
}
Expand Down
4 changes: 3 additions & 1 deletion pkg/constats.go
View file
Original file line number Diff line number Diff line change
@@ -1,5 +1,7 @@
package pkg

const (
namespace = "aws_resources_exporter"
namespace = "aws_resources_exporter"
SERVICE_CODE_KEY = "service_code"
QUOTA_CODE_KEY = "quota_code"
)
11 changes: 8 additions & 3 deletions pkg/ec2.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,8 +20,8 @@ const (
ec2ServiceCode string = "ec2"
)

var TransitGatewaysQuota *prometheus.Desc = prometheus.NewDesc(prometheus.BuildFQName(namespace, "", "ec2_transitgatewaysperregion_quota"), "Quota for maximum number of Transitgateways in this account", []string{"aws_region"}, nil)
var TransitGatewaysUsage *prometheus.Desc = prometheus.NewDesc(prometheus.BuildFQName(namespace, "", "ec2_transitgatewaysperregion_usage"), "Number of Tranitgatewyas in the AWS Account", []string{"aws_region"}, nil)
var TransitGatewaysQuota *prometheus.Desc
var TransitGatewaysUsage *prometheus.Desc

type EC2Exporter struct {
sessions []*session.Session
Expand All @@ -32,9 +32,14 @@ type EC2Exporter struct {
interval time.Duration
}

func NewEC2Exporter(sessions []*session.Session, logger log.Logger, config EC2Config) *EC2Exporter {
func NewEC2Exporter(sessions []*session.Session, logger log.Logger, config EC2Config, awsAccountId string) *EC2Exporter {

level.Info(logger).Log("msg", "Initializing EC2 exporter")
constLabels := map[string]string{"aws_account_id": awsAccountId, QUOTA_CODE_KEY: transitGatewayPerAccountQuotaCode, SERVICE_CODE_KEY: ec2ServiceCode}

TransitGatewaysQuota = prometheus.NewDesc(prometheus.BuildFQName(namespace, "", "ec2_transitgatewaysperregion_quota"), "Quota for maximum number of Transitgateways in this account", []string{"aws_region"}, constLabels)
TransitGatewaysUsage = prometheus.NewDesc(prometheus.BuildFQName(namespace, "", "ec2_transitgatewaysperregion_usage"), "Number of Tranitgatewyas in the AWS Account", []string{"aws_region"}, constLabels)

return &EC2Exporter{
sessions: sessions,
cache: *NewMetricsCache(*config.CacheTTL),
Expand Down
24 changes: 13 additions & 11 deletions pkg/route53.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,11 +18,12 @@ import (
)

const (
maxRetries = 10
route53MaxConcurrency = 5
route53ServiceCode = "route53"
hostedZonesQuotaCode = "L-4EA4796A"
errorCodeThrottling = "Throttling"
maxRetries = 10
route53MaxConcurrency = 5
route53ServiceCode = "route53"
hostedZonesQuotaCode = "L-4EA4796A"
recordsPerHostedZoneQuotaCode = "L-E209CC9F"
errorCodeThrottling = "Throttling"
)

type Route53Exporter struct {
Expand All @@ -40,17 +41,18 @@ type Route53Exporter struct {
timeout time.Duration
}

func NewRoute53Exporter(sess *session.Session, logger log.Logger, config Route53Config) *Route53Exporter {
func NewRoute53Exporter(sess *session.Session, logger log.Logger, config Route53Config, awsAccountId string) *Route53Exporter {

level.Info(logger).Log("msg", "Initializing Route53 exporter")
constLabels := map[string]string{"aws_account_id": awsAccountId, SERVICE_CODE_KEY: route53ServiceCode}

exporter := &Route53Exporter{
sess: sess,
RecordsPerHostedZoneQuota: prometheus.NewDesc(prometheus.BuildFQName(namespace, "", "route53_recordsperhostedzone_quota"), "Quota for maximum number of records in a Route53 hosted zone", []string{"hostedzoneid", "hostedzonename"}, nil),
RecordsPerHostedZoneUsage: prometheus.NewDesc(prometheus.BuildFQName(namespace, "", "route53_recordsperhostedzone_total"), "Number of Resource records", []string{"hostedzoneid", "hostedzonename"}, nil),
HostedZonesPerAccountQuota: prometheus.NewDesc(prometheus.BuildFQName(namespace, "", "route53_hostedzonesperaccount_quota"), "Quota for maximum number of Route53 hosted zones in an account", []string{}, nil),
HostedZonesPerAccountUsage: prometheus.NewDesc(prometheus.BuildFQName(namespace, "", "route53_hostedzonesperaccount_total"), "Number of Resource records", []string{}, nil),
LastUpdateTime: prometheus.NewDesc(prometheus.BuildFQName(namespace, "", "route53_last_updated_timestamp_seconds"), "Last time, the route53 metrics were sucessfully updated", []string{}, nil),
RecordsPerHostedZoneQuota: prometheus.NewDesc(prometheus.BuildFQName(namespace, "", "route53_recordsperhostedzone_quota"), "Quota for maximum number of records in a Route53 hosted zone", []string{"hostedzoneid", "hostedzonename"}, WithKeyValue(constLabels, QUOTA_CODE_KEY, recordsPerHostedZoneQuotaCode)),
RecordsPerHostedZoneUsage: prometheus.NewDesc(prometheus.BuildFQName(namespace, "", "route53_recordsperhostedzone_total"), "Number of Resource records", []string{"hostedzoneid", "hostedzonename"}, WithKeyValue(constLabels, QUOTA_CODE_KEY, recordsPerHostedZoneQuotaCode)),
HostedZonesPerAccountQuota: prometheus.NewDesc(prometheus.BuildFQName(namespace, "", "route53_hostedzonesperaccount_quota"), "Quota for maximum number of Route53 hosted zones in an account", []string{}, WithKeyValue(constLabels, QUOTA_CODE_KEY, hostedZonesQuotaCode)),
HostedZonesPerAccountUsage: prometheus.NewDesc(prometheus.BuildFQName(namespace, "", "route53_hostedzonesperaccount_total"), "Number of Resource records", []string{}, WithKeyValue(constLabels, QUOTA_CODE_KEY, hostedZonesQuotaCode)),
LastUpdateTime: prometheus.NewDesc(prometheus.BuildFQName(namespace, "", "route53_last_updated_timestamp_seconds"), "Last time, the route53 metrics were sucessfully updated", []string{}, constLabels),
cache: *NewMetricsCache(*config.CacheTTL),
logger: logger,
interval: *config.Interval,
Expand Down
10 changes: 10 additions & 0 deletions pkg/util.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,3 +23,13 @@ func GetEnvIntValue(envname string) (*int, error) {
func durationPtr(duration time.Duration) *time.Duration {
return &duration
}

// Add a new key to the map and return the new map
func WithKeyValue(m map[string]string, key string, value string) map[string]string {
newMap := make(map[string]string)
for k, v := range m {
newMap[k] = v
}
newMap[key] = value
return newMap
}
45 changes: 45 additions & 0 deletions pkg/util_test.go
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,45 @@
package pkg

import (
"reflect"
"testing"
)

func TestWithKeyValue(t *testing.T) {
type args struct {
m map[string]string
key string
value string
}
tests := []struct {
name string
args args
want map[string]string
}{
{
name: "Adding a key-value-pair to empty map returns a map with one key-value-pair",
args: args{
m: map[string]string{},
key: "new",
value: "new",
},
want: map[string]string{"new": "new"},
},
{
name: "Adding a key-value-pair to existing map returns a new map with an additional key-value-pair",
args: args{
m: map[string]string{"old": "old"},
key: "new",
value: "new",
},
want: map[string]string{"old": "old", "new": "new"},
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
if got := WithKeyValue(tt.args.m, tt.args.key, tt.args.value); !reflect.DeepEqual(got, tt.want) {
t.Errorf("WithKeyValue() = %v, want %v", got, tt.want)
}
})
}
}
29 changes: 16 additions & 13 deletions pkg/vpc.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@ const (
)

type VPCExporter struct {
awsAccountId string
sessions []*session.Session
VpcsPerRegionQuota *prometheus.Desc
VpcsPerRegionUsage *prometheus.Desc
Expand Down Expand Up @@ -53,22 +54,24 @@ type VPCCollector struct {
wg *sync.WaitGroup
}

func NewVPCExporter(sess []*session.Session, logger log.Logger, config VPCConfig) *VPCExporter {
func NewVPCExporter(sess []*session.Session, logger log.Logger, config VPCConfig, awsAccountId string) *VPCExporter {
level.Info(logger).Log("msg", "Initializing VPC exporter")
constLabels := map[string]string{"aws_account_id": awsAccountId, SERVICE_CODE_KEY: SERVICE_CODE_VPC}
return &VPCExporter{
awsAccountId: awsAccountId,
sessions: sess,
VpcsPerRegionQuota: prometheus.NewDesc(prometheus.BuildFQName(namespace, "", "vpc_vpcsperregion_quota"), "The quota of VPCs per region", []string{"aws_region"}, nil),
VpcsPerRegionUsage: prometheus.NewDesc(prometheus.BuildFQName(namespace, "", "vpc_vpcsperregion_usage"), "The usage of VPCs per region", []string{"aws_region"}, nil),
SubnetsPerVpcQuota: prometheus.NewDesc(prometheus.BuildFQName(namespace, "", "vpc_subnetspervpc_quota"), "The quota of subnets per VPC", []string{"aws_region"}, nil),
SubnetsPerVpcUsage: prometheus.NewDesc(prometheus.BuildFQName(namespace, "", "vpc_subnetspervpc_usage"), "The usage of subnets per VPC", []string{"aws_region", "vpcid"}, nil),
RoutesPerRouteTableQuota: prometheus.NewDesc(prometheus.BuildFQName(namespace, "", "vpc_routesperroutetable_quota"), "The quota of routes per routetable", []string{"aws_region"}, nil),
RoutesPerRouteTableUsage: prometheus.NewDesc(prometheus.BuildFQName(namespace, "", "vpc_routesperroutetable_usage"), "The usage of routes per routetable", []string{"aws_region", "vpcid", "routetableid"}, nil),
InterfaceVpcEndpointsPerVpcQuota: prometheus.NewDesc(prometheus.BuildFQName(namespace, "", "vpc_interfacevpcendpointspervpc_quota"), "The quota of interface vpc endpoints per vpc", []string{"aws_region"}, nil),
InterfaceVpcEndpointsPerVpcUsage: prometheus.NewDesc(prometheus.BuildFQName(namespace, "", "vpc_interfacevpcendpointspervpc_usage"), "The usage of interface vpc endpoints per vpc", []string{"aws_region", "vpcid"}, nil),
RouteTablesPerVpcQuota: prometheus.NewDesc(prometheus.BuildFQName(namespace, "", "vpc_routetablespervpc_quota"), "The quota of route tables per vpc", []string{"aws_region"}, nil),
RouteTablesPerVpcUsage: prometheus.NewDesc(prometheus.BuildFQName(namespace, "", "vpc_routetablespervpc_usage"), "The usage of route tables per vpc", []string{"aws_region", "vpcid"}, nil),
IPv4BlocksPerVpcQuota: prometheus.NewDesc(prometheus.BuildFQName(namespace, "", "vpc_ipv4blockspervpc_quota"), "The quota of ipv4 blocks per vpc", []string{"aws_region"}, nil),
IPv4BlocksPerVpcUsage: prometheus.NewDesc(prometheus.BuildFQName(namespace, "", "vpc_ipv4blockspervpc_usage"), "The usage of ipv4 blocks per vpc", []string{"aws_region", "vpcid"}, nil),
VpcsPerRegionQuota: prometheus.NewDesc(prometheus.BuildFQName(namespace, "", "vpc_vpcsperregion_quota"), "The quota of VPCs per region", []string{"aws_region"}, WithKeyValue(constLabels, QUOTA_CODE_KEY, QUOTA_VPCS_PER_REGION)),
VpcsPerRegionUsage: prometheus.NewDesc(prometheus.BuildFQName(namespace, "", "vpc_vpcsperregion_usage"), "The usage of VPCs per region", []string{"aws_region"}, WithKeyValue(constLabels, QUOTA_CODE_KEY, QUOTA_VPCS_PER_REGION)),
SubnetsPerVpcQuota: prometheus.NewDesc(prometheus.BuildFQName(namespace, "", "vpc_subnetspervpc_quota"), "The quota of subnets per VPC", []string{"aws_region"}, WithKeyValue(constLabels, QUOTA_CODE_KEY, QUOTA_SUBNETS_PER_VPC)),
SubnetsPerVpcUsage: prometheus.NewDesc(prometheus.BuildFQName(namespace, "", "vpc_subnetspervpc_usage"), "The usage of subnets per VPC", []string{"aws_region", "vpcid"}, WithKeyValue(constLabels, QUOTA_CODE_KEY, QUOTA_SUBNETS_PER_VPC)),
RoutesPerRouteTableQuota: prometheus.NewDesc(prometheus.BuildFQName(namespace, "", "vpc_routesperroutetable_quota"), "The quota of routes per routetable", []string{"aws_region"}, WithKeyValue(constLabels, QUOTA_CODE_KEY, QUOTA_ROUTES_PER_ROUTE_TABLE)),
RoutesPerRouteTableUsage: prometheus.NewDesc(prometheus.BuildFQName(namespace, "", "vpc_routesperroutetable_usage"), "The usage of routes per routetable", []string{"aws_region", "vpcid", "routetableid"}, WithKeyValue(constLabels, QUOTA_CODE_KEY, QUOTA_ROUTES_PER_ROUTE_TABLE)),
InterfaceVpcEndpointsPerVpcQuota: prometheus.NewDesc(prometheus.BuildFQName(namespace, "", "vpc_interfacevpcendpointspervpc_quota"), "The quota of interface vpc endpoints per vpc", []string{"aws_region"}, WithKeyValue(constLabels, QUOTA_CODE_KEY, QUOTA_INTERFACE_VPC_ENDPOINTS_PER_VPC)),
InterfaceVpcEndpointsPerVpcUsage: prometheus.NewDesc(prometheus.BuildFQName(namespace, "", "vpc_interfacevpcendpointspervpc_usage"), "The usage of interface vpc endpoints per vpc", []string{"aws_region", "vpcid"}, WithKeyValue(constLabels, QUOTA_CODE_KEY, QUOTA_INTERFACE_VPC_ENDPOINTS_PER_VPC)),
RouteTablesPerVpcQuota: prometheus.NewDesc(prometheus.BuildFQName(namespace, "", "vpc_routetablespervpc_quota"), "The quota of route tables per vpc", []string{"aws_region"}, WithKeyValue(constLabels, QUOTA_CODE_KEY, QUOTA_ROUTE_TABLES_PER_VPC)),
RouteTablesPerVpcUsage: prometheus.NewDesc(prometheus.BuildFQName(namespace, "", "vpc_routetablespervpc_usage"), "The usage of route tables per vpc", []string{"aws_region", "vpcid"}, WithKeyValue(constLabels, QUOTA_CODE_KEY, QUOTA_ROUTE_TABLES_PER_VPC)),
IPv4BlocksPerVpcQuota: prometheus.NewDesc(prometheus.BuildFQName(namespace, "", "vpc_ipv4blockspervpc_quota"), "The quota of ipv4 blocks per vpc", []string{"aws_region"}, WithKeyValue(constLabels, QUOTA_CODE_KEY, QUOTA_IPV4_BLOCKS_PER_VPC)),
IPv4BlocksPerVpcUsage: prometheus.NewDesc(prometheus.BuildFQName(namespace, "", "vpc_ipv4blockspervpc_usage"), "The usage of ipv4 blocks per vpc", []string{"aws_region", "vpcid"}, WithKeyValue(constLabels, QUOTA_CODE_KEY, QUOTA_IPV4_BLOCKS_PER_VPC)),
logger: logger,
timeout: *config.Timeout,
cache: *NewMetricsCache(*config.CacheTTL),
Expand Down

0 comments on commit 0309210

Please sign in to comment.