Metrics initialization (#2767)

* Add builder metrics initialization

* Add pull_closed metrics initialization

* Add builder metrics initialization

* use InitCounter from metrics package to initialize prometheus counters

server/events/instrumented_project_command_builder.go

@@ -4,21 +4,21 @@ import (
 	"github.com/runatlantis/atlantis/server/events/command"
 	"github.com/runatlantis/atlantis/server/logging"
 	"github.com/runatlantis/atlantis/server/metrics"
+	"github.com/uber-go/tally"
 )
 
 type InstrumentedProjectCommandBuilder struct {
 	ProjectCommandBuilder
 	Logger logging.SimpleLogging
+	scope  tally.Scope
 }
 
 func (b *InstrumentedProjectCommandBuilder) BuildApplyCommands(ctx *command.Context, comment *CommentCommand) ([]command.ProjectContext, error) {
-	scope := ctx.Scope.SubScope("builder")
-
-	timer := scope.Timer(metrics.ExecutionTimeMetric).Start()
+	timer := b.scope.Timer(metrics.ExecutionTimeMetric).Start()
 	defer timer.Stop()
 
-	executionSuccess := scope.Counter(metrics.ExecutionSuccessMetric)
-	executionError := scope.Counter(metrics.ExecutionErrorMetric)
+	executionSuccess := b.scope.Counter(metrics.ExecutionSuccessMetric)
+	executionError := b.scope.Counter(metrics.ExecutionErrorMetric)
 
 	projectCmds, err := b.ProjectCommandBuilder.BuildApplyCommands(ctx, comment)
 
@@ -33,13 +33,11 @@ func (b *InstrumentedProjectCommandBuilder) BuildApplyCommands(ctx *command.Cont
 
 }
 func (b *InstrumentedProjectCommandBuilder) BuildAutoplanCommands(ctx *command.Context) ([]command.ProjectContext, error) {
-	scope := ctx.Scope.SubScope("builder")
-
-	timer := scope.Timer(metrics.ExecutionTimeMetric).Start()
+	timer := b.scope.Timer(metrics.ExecutionTimeMetric).Start()
 	defer timer.Stop()
 
-	executionSuccess := scope.Counter(metrics.ExecutionSuccessMetric)
-	executionError := scope.Counter(metrics.ExecutionErrorMetric)
+	executionSuccess := b.scope.Counter(metrics.ExecutionSuccessMetric)
+	executionError := b.scope.Counter(metrics.ExecutionErrorMetric)
 
 	projectCmds, err := b.ProjectCommandBuilder.BuildAutoplanCommands(ctx)
 
@@ -54,13 +52,11 @@ func (b *InstrumentedProjectCommandBuilder) BuildAutoplanCommands(ctx *command.C
 
 }
 func (b *InstrumentedProjectCommandBuilder) BuildPlanCommands(ctx *command.Context, comment *CommentCommand) ([]command.ProjectContext, error) {
-	scope := ctx.Scope.SubScope("builder")
-
-	timer := scope.Timer(metrics.ExecutionTimeMetric).Start()
+	timer := b.scope.Timer(metrics.ExecutionTimeMetric).Start()
 	defer timer.Stop()
 
-	executionSuccess := scope.Counter(metrics.ExecutionSuccessMetric)
-	executionError := scope.Counter(metrics.ExecutionErrorMetric)
+	executionSuccess := b.scope.Counter(metrics.ExecutionSuccessMetric)
+	executionError := b.scope.Counter(metrics.ExecutionErrorMetric)
 
 	projectCmds, err := b.ProjectCommandBuilder.BuildPlanCommands(ctx, comment)
 

server/events/instrumented_project_command_runner.go

@@ -3,28 +3,52 @@ package events
 import (
 	"github.com/runatlantis/atlantis/server/events/command"
 	"github.com/runatlantis/atlantis/server/metrics"
+	"github.com/uber-go/tally"
 )
 
+type IntrumentedCommandRunner interface {
+	Plan(ctx command.ProjectContext) command.ProjectResult
+	PolicyCheck(ctx command.ProjectContext) command.ProjectResult
+	Apply(ctx command.ProjectContext) command.ProjectResult
+	ApprovePolicies(ctx command.ProjectContext) command.ProjectResult
+}
+
 type InstrumentedProjectCommandRunner struct {
-	ProjectCommandRunner
+	projectCommandRunner ProjectCommandRunner
+	scope                tally.Scope
+}
+
+func NewInstrumentedProjectCommandRunner(scope tally.Scope, projectCommandRunner ProjectCommandRunner) *InstrumentedProjectCommandRunner {
+	scope = scope.SubScope("project")
+
+	for _, m := range []string{metrics.ExecutionSuccessMetric, metrics.ExecutionErrorMetric, metrics.ExecutionFailureMetric} {
+		metrics.InitCounter(scope, m)
+	}
+
+	return &InstrumentedProjectCommandRunner{
+		projectCommandRunner: projectCommandRunner,
+		scope:                scope,
+	}
 }
 
 func (p *InstrumentedProjectCommandRunner) Plan(ctx command.ProjectContext) command.ProjectResult {
-	return RunAndEmitStats("plan", ctx, p.ProjectCommandRunner.Plan)
+	return RunAndEmitStats("plan", ctx, p.projectCommandRunner.Plan, p.scope)
 }
 
 func (p *InstrumentedProjectCommandRunner) PolicyCheck(ctx command.ProjectContext) command.ProjectResult {
-	return RunAndEmitStats("policy check", ctx, p.ProjectCommandRunner.PolicyCheck)
+	return RunAndEmitStats("policy check", ctx, p.projectCommandRunner.PolicyCheck, p.scope)
 }
 
 func (p *InstrumentedProjectCommandRunner) Apply(ctx command.ProjectContext) command.ProjectResult {
-	return RunAndEmitStats("apply", ctx, p.ProjectCommandRunner.Apply)
+	return RunAndEmitStats("apply", ctx, p.projectCommandRunner.Apply, p.scope)
 }
 
-func RunAndEmitStats(commandName string, ctx command.ProjectContext, execute func(ctx command.ProjectContext) command.ProjectResult) command.ProjectResult {
+func (p *InstrumentedProjectCommandRunner) ApprovePolicies(ctx command.ProjectContext) command.ProjectResult {
+	return RunAndEmitStats("approve policies", ctx, p.projectCommandRunner.Apply, p.scope)
+}
 
+func RunAndEmitStats(commandName string, ctx command.ProjectContext, execute func(ctx command.ProjectContext) command.ProjectResult, scope tally.Scope) command.ProjectResult {
 	// ensures we are differentiating between project level command and overall command
-	scope := ctx.Scope.SubScope("project")
 	scope = ctx.SetScopeTags(scope)
 	logger := ctx.Log
 

server/events/instrumented_pull_closed_executor.go

@@ -18,9 +18,14 @@ type InstrumentedPullClosedExecutor struct {
 func NewInstrumentedPullClosedExecutor(
 	scope tally.Scope, log logging.SimpleLogging, cleaner PullCleaner,
 ) PullCleaner {
+	scope = scope.SubScope("pullclosed_cleanup")
+
+	for _, m := range []string{metrics.ExecutionSuccessMetric, metrics.ExecutionErrorMetric} {
+		metrics.InitCounter(scope, m)
+	}
 
 	return &InstrumentedPullClosedExecutor{
-		scope:   scope.SubScope("pullclosed_cleanup"),
+		scope:   scope,
 		log:     log,
 		cleaner: cleaner,
 	}

server/events/project_command_builder.go

@@ -11,6 +11,7 @@ import (
 
 	"github.com/runatlantis/atlantis/server/core/config/valid"
 	"github.com/runatlantis/atlantis/server/logging"
+	"github.com/runatlantis/atlantis/server/metrics"
 
 	"github.com/pkg/errors"
 
@@ -54,6 +55,12 @@ func NewInstrumentedProjectCommandBuilder(
 	scope tally.Scope,
 	logger logging.SimpleLogging,
 ) *InstrumentedProjectCommandBuilder {
+	scope = scope.SubScope("builder")
+
+	for _, m := range []string{metrics.ExecutionSuccessMetric, metrics.ExecutionErrorMetric} {
+		metrics.InitCounter(scope, m)
+	}
+
 	return &InstrumentedProjectCommandBuilder{
 		ProjectCommandBuilder: NewProjectCommandBuilder(
 			policyChecksSupported,
@@ -74,6 +81,7 @@ func NewInstrumentedProjectCommandBuilder(
 			logger,
 		),
 		Logger: logger,
+		scope:  scope,
 	}
 }
 

server/metrics/counter.go

@@ -0,0 +1,8 @@
+package metrics
+
+import "github.com/uber-go/tally"
+
+func InitCounter(scope tally.Scope, name string) {
+	s := scope.Counter(name)
+	s.Inc(0)
+}

server/server.go

@@ -628,9 +628,10 @@ func NewServer(userConfig UserConfig, config Config) (*Server, error) {
 		ProjectCommandRunner: projectCommandRunner,
 		JobURLSetter:         jobs.NewJobURLSetter(router, commitStatusUpdater),
 	}
-	instrumentedProjectCmdRunner := &events.InstrumentedProjectCommandRunner{
-		ProjectCommandRunner: projectOutputWrapper,
-	}
+	instrumentedProjectCmdRunner := events.NewInstrumentedProjectCommandRunner(
+		statsScope,
+		projectOutputWrapper,
+	)
 
 	policyCheckCommandRunner := events.NewPolicyCheckCommandRunner(
 		dbUpdater,