Merging AgentMetrics branch into dev

THIRD-PARTY

@@ -14,6 +14,7 @@
 ** github.com/opencontainers/image-spec; version v1.0.1 -- https://github.com/opencontainers/image-spec
 ** github.com/opencontainers/runc; version v0.1.1 -- https://github.com/opencontainers/runc
 ** github.com/opencontainers/runtime-spec; version d349388c43b01b2ea695965ae561b5bddb81318f -- https://github.com/opencontainers/runtime-spec
+** github.com/prometheus/client_golang; version 0.9.0 -- https://github.com/prometheus/client_golang
 ** github.com/vishvananda/netlink; version fe3b5664d23a11b52ba59bece4ff29c52772a56b -- https://github.com/vishvananda/netlink
 ** github.com/vishvananda/netns; version be1fbeda19366dea804f00efff2dd73a1642fdcc -- https://github.com/vishvananda/netns
 
@@ -237,6 +238,32 @@ Copyright 2016 The Linux Foundation.
 Copyright 2014 Docker, Inc.
 * For github.com/opencontainers/runtime-spec see also this required NOTICE:
 Copyright 2015 The Linux Foundation.
+* For github.com/prometheus/client_golang see also this required NOTICE:
+Prometheus instrumentation library for Go applications
+Copyright 2012-2015 The Prometheus Authors
+
+This product includes software developed at
+SoundCloud Ltd. (http://soundcloud.com/).
+
+
+The following components are included in this product:
+
+perks - a fork of https://github.com/bmizerany/perks
+https://github.com/beorn7/perks
+Copyright 2013-2015 Blake Mizerany, Björn Rabenstein
+See https://github.com/beorn7/perks/blob/master/README.md for license
+details.
+
+Go support for Protocol Buffers - Google's data interchange format
+http://github.com/golang/protobuf/
+Copyright 2010 The Go Authors
+See source code for license details.
+
+Support for streaming Protocol Buffer messages for the Go language
+(golang).
+https://github.com/matttproud/golang_protobuf_extensions
+Copyright 2013 Matt T. Proud
+Licensed under the Apache License, Version 2.0
 * For github.com/vishvananda/netlink see also this required NOTICE:
 Copyright 2014 Vishvananda Ishaya.
 Copyright 2014 Docker, Inc.

agent/Gopkg.toml

@@ -88,3 +88,7 @@ required = ["github.com/golang/mock/mockgen/model"]
 [[constraint]]
   name = "github.com/gorilla/mux"
   version = "1.6.2"
+
+[[constraint]]
+  name = "github.com/prometheus/client_golang"
+  version = "0.9.0"

agent/app/agent.go

@@ -18,6 +18,8 @@ import (
 	"errors"
 	"fmt"
 
+	"github.com/aws/amazon-ecs-agent/agent/metrics"
+
 	acshandler "github.com/aws/amazon-ecs-agent/agent/acs/handler"
 	"github.com/aws/amazon-ecs-agent/agent/api"
 	"github.com/aws/amazon-ecs-agent/agent/api/ecsclient"
@@ -239,6 +241,8 @@ func (agent *ecsAgent) doStart(containerChangeEventStream *eventstream.EventStre
 		return exitcodes.ExitTerminal
 	}
 
+	agent.initMetricsEngine()
+
 	// Initialize the state manager
 	stateManager, err := agent.newStateManager(taskEngine,
 		&agent.cfg.Cluster, &agent.containerInstanceARN, &currentEC2InstanceID, &agent.availabilityZone)
@@ -374,6 +378,20 @@ func (agent *ecsAgent) newTaskEngine(containerChangeEventStream *eventstream.Eve
 	return previousTaskEngine, currentEC2InstanceID, nil
 }
 
+func (agent *ecsAgent) initMetricsEngine() {
+	// In case of a panic during set-up, we will recover quietly and resume
+	// normal Agent execution.
+	defer func() {
+		if r := recover(); r != nil {
+			seelog.Errorf("MetricsEngine Set-up panicked. Recovering quietly: %s", r)
+		}
+	}()
+
+	// We init the global MetricsEngine before we publish metrics
+	metrics.MustInit(agent.cfg)
+	metrics.PublishMetrics()
+}
+
 // setClusterInConfig sets the cluster name in the config object based on
 // previous state. It returns an error if there's a mismatch between the
 // the current cluster name with what's restored from the cluster state

agent/config/config.go

@@ -47,6 +47,9 @@ const (
 	// AgentCredentialsPort is used to serve the credentials for tasks.
 	AgentCredentialsPort = 51679
 
+	// AgentPrometheusExpositionPort is used to expose Prometheus metrics that can be scraped by a Prometheus server
+	AgentPrometheusExpositionPort = 51680
+
 	// defaultConfigFileName is the default (json-formatted) config file
 	defaultConfigFileName = "/etc/ecs_container_agent/config.json"
 

agent/config/config_unix.go

@@ -16,9 +16,11 @@ package config
 
 import (
 	"fmt"
+	"os"
 	"time"
 
 	"github.com/aws/amazon-ecs-agent/agent/dockerclient"
+	"github.com/aws/amazon-ecs-agent/agent/utils"
 )
 
 const (
@@ -73,10 +75,16 @@ func DefaultConfig() Config {
 		SharedVolumeMatchFullConfig:        false, // only requiring shared volumes to match on name, which is default docker behavior
 		ImagePullInactivityTimeout:         defaultImagePullInactivityTimeout,
 		ContainerInstancePropagateTagsFrom: ContainerInstancePropagateTagsFromNoneType,
+		PrometheusMetricsEnabled:           false,
 	}
 }
 
-func (cfg *Config) platformOverrides() {}
+func (cfg *Config) platformOverrides() {
+	cfg.PrometheusMetricsEnabled = utils.ParseBool(os.Getenv("ECS_ENABLE_PROMETHEUS_METRICS"), false)
+	if cfg.PrometheusMetricsEnabled {
+		cfg.ReservedPorts = append(cfg.ReservedPorts, AgentPrometheusExpositionPort)
+	}
+}
 
 // platformString returns platform-specific config data that can be serialized
 // to string for debugging

agent/config/config_unix_test.go

@@ -180,6 +180,17 @@ func TestBadFileContent(t *testing.T) {
 	assert.Error(t, err, "create configuration should fail")
 }
 
+func TestPrometheusMetricsPlatformOverrides(t *testing.T) {
+	defer setTestRegion()()
+	cfg, err := NewConfig(ec2.NewBlackholeEC2MetadataClient())
+	require.NoError(t, err)
+
+	defer setTestEnv("ECS_ENABLE_PROMETHEUS_METRICS", "true")()
+	cfg.platformOverrides()
+	assert.True(t, cfg.PrometheusMetricsEnabled, "Prometheus metrics should be enabled")
+	assert.Equal(t, 6, len(cfg.ReservedPorts), "Reserved ports should have added Prometheus endpoint")
+}
+
 // setupFileConfiguration create a temp file store the configuration
 func setupFileConfiguration(t *testing.T, configContent string) string {
 	file, err := ioutil.TempFile("", "ecs-test")

agent/config/types.go

@@ -189,6 +189,11 @@ type Config struct {
 	// the image from the tarball; the referenced image must already be loaded.
 	PauseContainerTag string
 
+	// PrometheusMetricsEnabled configures whether Agent metrics should be
+	// collected and published to the specified endpoint. This is disabled by
+	// default.
+	PrometheusMetricsEnabled bool
+
 	// AWSVPCBlockInstanceMetdata specifies if InstanceMetadata endpoint should be blocked
 	// for tasks that are launched with network mode "awsvpc" when ECS_AWSVPC_BLOCK_IMDS=true
 	AWSVPCBlockInstanceMetdata bool