Stream Terraform output through web interface

go.mod

@@ -22,6 +22,7 @@ require (
 	github.com/google/go-cmp v0.5.2 // indirect
 	github.com/google/go-github/v31 v31.0.0
 	github.com/gorilla/mux v1.8.0
+	github.com/gorilla/websocket v1.4.2
 	github.com/hashicorp/go-getter v1.5.1
 	github.com/hashicorp/go-version v1.2.0
 	github.com/hashicorp/hcl/v2 v2.6.0 // indirect

go.sum

@@ -138,6 +138,7 @@ github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1 h1:EGx4pi6eqNxGa
 github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY=
 github.com/gorilla/mux v1.8.0 h1:i40aqfkR1h2SlN9hojwV5ZA91wcXFOvkdNIeFDP5koI=
 github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So=
+github.com/gorilla/websocket v1.4.2 h1:+/TMaTYc4QFitKJxsQ7Yye35DkWvkdLcvGKqM+x0Ufc=
 github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
 github.com/graph-gophers/graphql-go v0.0.0-20191115155744-f33e81362277/go.mod h1:9CQHMSxwO4MprSdzoIEobiHpoLtHm77vfxsvsIN5Vuc=
 github.com/grpc-ecosystem/go-grpc-middleware v1.0.0/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs=

server/events/command_runner.go

@@ -105,12 +105,14 @@ type DefaultCommandRunner struct {
 	ProjectCommandRunner    ProjectCommandRunner
 	// GlobalAutomerge is true if we should automatically merge pull requests if all
 	// plans have been successfully applied. This is set via a CLI flag.
-	GlobalAutomerge   bool
-	PendingPlanFinder PendingPlanFinder
-	WorkingDir        WorkingDir
-	DB                *db.BoltDB
-	Drainer           *Drainer
-	DeleteLockCommand DeleteLockCommand
+	GlobalAutomerge     bool
+	PendingPlanFinder   PendingPlanFinder
+	WorkingDir          WorkingDir
+	DB                  *db.BoltDB
+	Drainer             *Drainer
+	DeleteLockCommand   DeleteLockCommand
+	TerraformOutputChan chan<- *models.TerraformOutputLine
+	JobURLGenerator     JobURLGenerator
 }
 
 // RunAutoplanCommand runs plan when a pull request is opened or updated.
@@ -140,7 +142,7 @@ func (c *DefaultCommandRunner) RunAutoplanCommand(baseRepo models.Repo, headRepo
 
 	projectCmds, err := c.ProjectCommandBuilder.BuildAutoplanCommands(ctx)
 	if err != nil {
-		if statusErr := c.CommitStatusUpdater.UpdateCombined(ctx.Pull.BaseRepo, ctx.Pull, models.FailedCommitStatus, models.PlanCommand); statusErr != nil {
+		if statusErr := c.CommitStatusUpdater.UpdateCombined(ctx.Pull.BaseRepo, ctx.Pull, models.FailedCommitStatus, models.PlanCommand, ""); statusErr != nil {
 			ctx.Log.Warn("unable to update commit status: %s", statusErr)
 		}
 
@@ -154,15 +156,15 @@ func (c *DefaultCommandRunner) RunAutoplanCommand(baseRepo models.Repo, headRepo
 			// with 0/0 projects planned successfully because some users require
 			// the Atlantis status to be passing for all pull requests.
 			ctx.Log.Debug("setting VCS status to success with no projects found")
-			if err := c.CommitStatusUpdater.UpdateCombinedCount(baseRepo, pull, models.SuccessCommitStatus, models.PlanCommand, 0, 0); err != nil {
+			if err := c.CommitStatusUpdater.UpdateCombinedCount(baseRepo, pull, models.SuccessCommitStatus, models.PlanCommand, 0, 0, ""); err != nil {
 				ctx.Log.Warn("unable to update commit status: %s", err)
 			}
 		}
 		return
 	}
 
 	// At this point we are sure Atlantis has work to do, so set commit status to pending
-	if err := c.CommitStatusUpdater.UpdateCombined(ctx.Pull.BaseRepo, ctx.Pull, models.PendingCommitStatus, models.PlanCommand); err != nil {
+	if err := c.CommitStatusUpdater.UpdateCombined(ctx.Pull.BaseRepo, ctx.Pull, models.PendingCommitStatus, models.PlanCommand, c.JobURLGenerator.GenerateJobURL(ctx.Pull)); err != nil {
 		ctx.Log.Warn("unable to update commit status: %s", err)
 	}
 
@@ -291,7 +293,7 @@ func (c *DefaultCommandRunner) RunCommentCommand(baseRepo models.Repo, maybeHead
 		ctx.Log.Info("pull request mergeable status: %t", ctx.PullMergeable)
 	}
 
-	if err = c.CommitStatusUpdater.UpdateCombined(baseRepo, pull, models.PendingCommitStatus, cmd.CommandName()); err != nil {
+	if err = c.CommitStatusUpdater.UpdateCombined(baseRepo, pull, models.PendingCommitStatus, cmd.CommandName(), c.JobURLGenerator.GenerateJobURL(ctx.Pull)); err != nil {
 		ctx.Log.Warn("unable to update commit status: %s", err)
 	}
 
@@ -306,7 +308,7 @@ func (c *DefaultCommandRunner) RunCommentCommand(baseRepo models.Repo, maybeHead
 		return
 	}
 	if err != nil {
-		if statusErr := c.CommitStatusUpdater.UpdateCombined(ctx.Pull.BaseRepo, ctx.Pull, models.FailedCommitStatus, cmd.CommandName()); statusErr != nil {
+		if statusErr := c.CommitStatusUpdater.UpdateCombined(ctx.Pull.BaseRepo, ctx.Pull, models.FailedCommitStatus, cmd.CommandName(), c.JobURLGenerator.GenerateJobURL(ctx.Pull)); statusErr != nil {
 			ctx.Log.Warn("unable to update commit status: %s", statusErr)
 		}
 		c.updatePull(ctx, cmd, CommandResult{Error: err})
@@ -376,7 +378,11 @@ func (c *DefaultCommandRunner) updateCommitStatus(ctx *CommandContext, cmd model
 		}
 	}
 
-	if err := c.CommitStatusUpdater.UpdateCombinedCount(ctx.Pull.BaseRepo, ctx.Pull, status, cmd, numSuccess, len(pullStatus.Projects)); err != nil {
+	url := ""
+	if status == models.PendingCommitStatus {
+		url = c.JobURLGenerator.GenerateJobURL(ctx.Pull)
+	}
+	if err := c.CommitStatusUpdater.UpdateCombinedCount(ctx.Pull.BaseRepo, ctx.Pull, status, cmd, numSuccess, len(pullStatus.Projects), url); err != nil {
 		ctx.Log.Warn("unable to update commit status: %s", err)
 	}
 }
@@ -447,7 +453,21 @@ func (c *DefaultCommandRunner) runProjectCmdsParallel(cmds []models.ProjectComma
 
 func (c *DefaultCommandRunner) runProjectCmds(cmds []models.ProjectCommandContext, cmdName models.CommandName) CommandResult {
 	var results []models.ProjectResult
-	for _, pCmd := range cmds {
+	for idx, pCmd := range cmds {
+		if idx == 0 {
+			c.TerraformOutputChan <- &models.TerraformOutputLine{
+				PullSlug:    pCmd.PullSlug(),
+				ClearBefore: true,
+				Line:        fmt.Sprintf(":: Start processing pull request %s", pCmd.PullSlug()),
+			}
+		}
+		c.TerraformOutputChan <- &models.TerraformOutputLine{
+			PullSlug: pCmd.PullSlug(),
+			Line: fmt.Sprintf(
+				":::: [%d/%d] Start processing project %s",
+				idx+1, len(cmds), pCmd.ProjectDesc(),
+			),
+		}
 		var res models.ProjectResult
 		switch cmdName {
 		case models.PlanCommand:
@@ -456,6 +476,20 @@ func (c *DefaultCommandRunner) runProjectCmds(cmds []models.ProjectCommandContex
 			res = c.ProjectCommandRunner.Apply(pCmd)
 		}
 		results = append(results, res)
+		c.TerraformOutputChan <- &models.TerraformOutputLine{
+			PullSlug: pCmd.PullSlug(),
+			Line: fmt.Sprintf(
+				":::: [%d/%d] Finish processing project %s",
+				idx+1, len(cmds), pCmd.ProjectDesc(),
+			),
+		}
+		if idx == len(cmds)-1 {
+			c.TerraformOutputChan <- &models.TerraformOutputLine{
+				PullSlug:   pCmd.PullSlug(),
+				Line:       fmt.Sprintf(":: Finish processing pull request %s", pCmd.PullSlug()),
+				ClearAfter: true,
+			}
+		}
 	}
 	return CommandResult{ProjectResults: results}
 }

server/events/commit_status_updater.go

@@ -28,10 +28,10 @@ import (
 type CommitStatusUpdater interface {
 	// UpdateCombined updates the combined status of the head commit of pull.
 	// A combined status represents all the projects modified in the pull.
-	UpdateCombined(repo models.Repo, pull models.PullRequest, status models.CommitStatus, command models.CommandName) error
+	UpdateCombined(repo models.Repo, pull models.PullRequest, status models.CommitStatus, command models.CommandName, url string) error
 	// UpdateCombinedCount updates the combined status to reflect the
 	// numSuccess out of numTotal.
-	UpdateCombinedCount(repo models.Repo, pull models.PullRequest, status models.CommitStatus, command models.CommandName, numSuccess int, numTotal int) error
+	UpdateCombinedCount(repo models.Repo, pull models.PullRequest, status models.CommitStatus, command models.CommandName, numSuccess int, numTotal int, url string) error
 	// UpdateProject sets the commit status for the project represented by
 	// ctx.
 	UpdateProject(ctx models.ProjectCommandContext, cmdName models.CommandName, status models.CommitStatus, url string) error
@@ -44,7 +44,7 @@ type DefaultCommitStatusUpdater struct {
 	StatusName string
 }
 
-func (d *DefaultCommitStatusUpdater) UpdateCombined(repo models.Repo, pull models.PullRequest, status models.CommitStatus, command models.CommandName) error {
+func (d *DefaultCommitStatusUpdater) UpdateCombined(repo models.Repo, pull models.PullRequest, status models.CommitStatus, command models.CommandName, url string) error {
 	src := fmt.Sprintf("%s/%s", d.StatusName, command.String())
 	var descripWords string
 	switch status {
@@ -56,16 +56,16 @@ func (d *DefaultCommitStatusUpdater) UpdateCombined(repo models.Repo, pull model
 		descripWords = "succeeded."
 	}
 	descrip := fmt.Sprintf("%s %s", strings.Title(command.String()), descripWords)
-	return d.Client.UpdateStatus(repo, pull, status, src, descrip, "")
+	return d.Client.UpdateStatus(repo, pull, status, src, descrip, url)
 }
 
-func (d *DefaultCommitStatusUpdater) UpdateCombinedCount(repo models.Repo, pull models.PullRequest, status models.CommitStatus, command models.CommandName, numSuccess int, numTotal int) error {
+func (d *DefaultCommitStatusUpdater) UpdateCombinedCount(repo models.Repo, pull models.PullRequest, status models.CommitStatus, command models.CommandName, numSuccess int, numTotal int, url string) error {
 	src := fmt.Sprintf("%s/%s", d.StatusName, command.String())
 	cmdVerb := "planned"
 	if command == models.ApplyCommand {
 		cmdVerb = "applied"
 	}
-	return d.Client.UpdateStatus(repo, pull, status, src, fmt.Sprintf("%d/%d projects %s successfully.", numSuccess, numTotal, cmdVerb), "")
+	return d.Client.UpdateStatus(repo, pull, status, src, fmt.Sprintf("%d/%d projects %s successfully.", numSuccess, numTotal, cmdVerb), url)
 }
 
 func (d *DefaultCommitStatusUpdater) UpdateProject(ctx models.ProjectCommandContext, cmdName models.CommandName, status models.CommitStatus, url string) error {

server/events/delete_lock_command.go

@@ -15,6 +15,10 @@ type DeleteLockCommand interface {
 	DeleteLocksByPull(repoFullName string, pullNum int) error
 }
 
+type JobURLGenerator interface {
+	GenerateJobURL(pull models.PullRequest) string
+}
+
 // DefaultDeleteLockCommand deletes a specific lock after a request from the LocksController.
 type DefaultDeleteLockCommand struct {
 	Locker           locking.Locker

server/events/models/models.go

@@ -359,6 +359,40 @@ type ProjectCommandContext struct {
 	Workspace string
 }
 
+// The next two functions are kind of hacks. In a few different places
+// we need to get a unique identifier for the current pull request as
+// a single string (and it has to be constructed the same way each
+// time). Same for getting a string that describes the current
+// Atlantis workspace (or "project", which I think is the technically
+// correct term). The easy way is to put this shared code as methods
+// on the ProjectCommandContext struct, so that it's easy to get the
+// relevant strings from anywhere. There might be a more appropriate
+// way to do it; I haven't looked very hard.
+
+func (c *ProjectCommandContext) PullSlug() string {
+	return fmt.Sprintf("%s/%d", c.BaseRepo.FullName, c.Pull.Num)
+}
+
+// ProjectDesc returns a string that describes the current project in
+// human-readable terms, as concisely as possible.
+func (c *ProjectCommandContext) ProjectDesc() string {
+	// If project name is set, it's supposed to be a unique
+	// string, we'll just use that.
+	if c.ProjectName != "" {
+		return c.ProjectName
+	}
+	// Otherwise we want to report the Terraform working directory
+	// which is the primary disambiguator.
+	desc := c.RepoRelDir
+	// Only if the configuration has specified custom workspaces
+	// do we want to tack those on (because multiple projects
+	// might then use the same working directory).
+	if c.Workspace != "default" {
+		desc += " (" + c.Workspace + ")"
+	}
+	return desc
+}
+
 // SplitRepoFullName splits a repo full name up into its owner and repo
 // name segments. If the repoFullName is malformed, may return empty
 // strings for owner or repo.
@@ -535,3 +569,17 @@ func (c CommandName) String() string {
 	}
 	return ""
 }
+
+type TerraformOutputLine struct {
+	// pull request that generated this output line
+	PullSlug string
+
+	// thing to print in console on web interface, no trailing newline
+	Line string
+
+	// true means we should clear the buffer of accumulated log
+	// lines before (resp. after) logging the given line, so that
+	// new clients connecting to the web interface don't see them
+	ClearBefore bool
+	ClearAfter  bool
+}

server/events/runtime/apply_step_runner.go

@@ -48,7 +48,7 @@ func (a *ApplyStepRunner) Run(ctx models.ProjectCommandContext, extraArgs []stri
 		// NOTE: we need to quote the plan path because Bitbucket Server can
 		// have spaces in its repo owner names which is part of the path.
 		args := append(append(append([]string{"apply", "-input=false", "-no-color"}, extraArgs...), ctx.EscapedCommentArgs...), fmt.Sprintf("%q", planPath))
-		out, err = a.TerraformExecutor.RunCommandWithVersion(ctx.Log, path, args, envs, ctx.TerraformVersion, ctx.Workspace)
+		out, err = a.TerraformExecutor.RunCommandWithVersion(ctx, path, args, envs, ctx.TerraformVersion)
 	}
 
 	// If the apply was successful, delete the plan.
@@ -140,7 +140,7 @@ func (a *ApplyStepRunner) runRemoteApply(
 
 	// Start the async command execution.
 	ctx.Log.Debug("starting async tf remote operation")
-	inCh, outCh := a.AsyncTFExec.RunCommandAsync(ctx.Log, filepath.Clean(path), applyArgs, envs, tfVersion, ctx.Workspace)
+	inCh, outCh := a.AsyncTFExec.RunCommandAsync(ctx, filepath.Clean(path), applyArgs, envs, tfVersion)
 	var lines []string
 	nextLineIsRunURL := false
 	var runURL string

server/events/runtime/init_step_runner.go

@@ -24,7 +24,7 @@ func (i *InitStepRunner) Run(ctx models.ProjectCommandContext, extraArgs []strin
 		terraformInitCmd = append([]string{"get", "-no-color", "-upgrade"}, extraArgs...)
 	}
 
-	out, err := i.TerraformExecutor.RunCommandWithVersion(ctx.Log, path, terraformInitCmd, envs, tfVersion, ctx.Workspace)
+	out, err := i.TerraformExecutor.RunCommandWithVersion(ctx, path, terraformInitCmd, envs, tfVersion)
 	// Only include the init output if there was an error. Otherwise it's
 	// unnecessary and lengthens the comment.
 	if err != nil {

server/events/runtime/plan_step_runner.go

@@ -47,7 +47,7 @@ func (p *PlanStepRunner) Run(ctx models.ProjectCommandContext, extraArgs []strin
 
 	planFile := filepath.Join(path, GetPlanFilename(ctx.Workspace, ctx.ProjectName))
 	planCmd := p.buildPlanCmd(ctx, extraArgs, path, tfVersion, planFile)
-	output, err := p.TerraformExecutor.RunCommandWithVersion(ctx.Log, filepath.Clean(path), planCmd, envs, tfVersion, ctx.Workspace)
+	output, err := p.TerraformExecutor.RunCommandWithVersion(ctx, filepath.Clean(path), planCmd, envs, tfVersion)
 	if p.isRemoteOpsErr(output, err) {
 		ctx.Log.Debug("detected that this project is using TFE remote ops")
 		return p.remotePlan(ctx, extraArgs, path, tfVersion, planFile, envs)
@@ -130,7 +130,7 @@ func (p *PlanStepRunner) switchWorkspace(ctx models.ProjectCommandContext, path
 	// already in the right workspace then no need to switch. This will save us
 	// about ten seconds. This command is only available in > 0.10.
 	if !runningZeroPointNine {
-		workspaceShowOutput, err := p.TerraformExecutor.RunCommandWithVersion(ctx.Log, path, []string{workspaceCmd, "show"}, envs, tfVersion, ctx.Workspace)
+		workspaceShowOutput, err := p.TerraformExecutor.RunCommandWithVersion(ctx, path, []string{workspaceCmd, "show"}, envs, tfVersion)
 		if err != nil {
 			return err
 		}
@@ -145,11 +145,11 @@ func (p *PlanStepRunner) switchWorkspace(ctx models.ProjectCommandContext, path
 	// To do this we can either select and catch the error or use list and then
 	// look for the workspace. Both commands take the same amount of time so
 	// that's why we're running select here.
-	_, err := p.TerraformExecutor.RunCommandWithVersion(ctx.Log, path, []string{workspaceCmd, "select", "-no-color", ctx.Workspace}, envs, tfVersion, ctx.Workspace)
+	_, err := p.TerraformExecutor.RunCommandWithVersion(ctx, path, []string{workspaceCmd, "select", "-no-color", ctx.Workspace}, envs, tfVersion)
 	if err != nil {
 		// If terraform workspace select fails we run terraform workspace
 		// new to create a new workspace automatically.
-		out, err := p.TerraformExecutor.RunCommandWithVersion(ctx.Log, path, []string{workspaceCmd, "new", "-no-color", ctx.Workspace}, envs, tfVersion, ctx.Workspace)
+		out, err := p.TerraformExecutor.RunCommandWithVersion(ctx, path, []string{workspaceCmd, "new", "-no-color", ctx.Workspace}, envs, tfVersion)
 		if err != nil {
 			return fmt.Errorf("%s: %s", err, out)
 		}
@@ -264,7 +264,7 @@ func (p *PlanStepRunner) runRemotePlan(
 
 	// Start the async command execution.
 	ctx.Log.Debug("starting async tf remote operation")
-	_, outCh := p.AsyncTFExec.RunCommandAsync(ctx.Log, filepath.Clean(path), cmdArgs, envs, tfVersion, ctx.Workspace)
+	_, outCh := p.AsyncTFExec.RunCommandAsync(ctx, filepath.Clean(path), cmdArgs, envs, tfVersion)
 	var lines []string
 	nextLineIsRunURL := false
 	var runURL string

server/events/runtime/runtime.go

@@ -24,7 +24,7 @@ const (
 // TerraformExec brings the interface from TerraformClient into this package
 // without causing circular imports.
 type TerraformExec interface {
-	RunCommandWithVersion(log *logging.SimpleLogger, path string, args []string, envs map[string]string, v *version.Version, workspace string) (string, error)
+	RunCommandWithVersion(ctx models.ProjectCommandContext, path string, args []string, envs map[string]string, v *version.Version) (string, error)
 	EnsureVersion(log *logging.SimpleLogger, v *version.Version) error
 }
 
@@ -39,7 +39,7 @@ type AsyncTFExec interface {
 	// Callers can use the input channel to pass stdin input to the command.
 	// If any error is passed on the out channel, there will be no
 	// further output (so callers are free to exit).
-	RunCommandAsync(log *logging.SimpleLogger, path string, args []string, envs map[string]string, v *version.Version, workspace string) (chan<- string, <-chan terraform.Line)
+	RunCommandAsync(ctx models.ProjectCommandContext, path string, args []string, envs map[string]string, v *version.Version) (chan<- string, <-chan terraform.Line)
 }
 
 // StatusUpdater brings the interface from CommitStatusUpdater into this package