Skip to content

Commit

Permalink
feat: Jobs now shown in a table in the main page (runatlantis#3784)
Browse files Browse the repository at this point in the history 
* feat: better logging for UpdateStatus

* feat: jobs now shown in a table in the main page

---------

Co-authored-by: PePe Amengual <[email protected]>
  • Loading branch information
@marcosdiez @jamengual
marcosdiez and jamengual authored Dec 12, 2023
1 parent 070fd3e commit ce95f8e
Show file tree
Hide file tree
Showing 8 changed files with 221 additions and 33 deletions.
38 changes: 36 additions & 2 deletions server/controllers/templates/web_templates.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,8 @@ import (
"html/template"
"io"
"time"

"github.com/runatlantis/atlantis/server/jobs"
)

//go:generate pegomock generate --package mocks -o mocks/mock_template_writer.go TemplateWriter
Expand Down Expand Up @@ -50,7 +52,9 @@ type ApplyLockData struct {

// IndexData holds the data for rendering the index page
type IndexData struct {
Locks []LockIndexData
Locks []LockIndexData
PullToJobMapping []jobs.PullInfoWithJobIDs

ApplyLock ApplyLockData
AtlantisVersion string
// CleanedBasePath is the path Atlantis is accessible at externally. If
Expand Down Expand Up @@ -113,8 +117,8 @@ var IndexTemplate = template.Must(template.New("index.html.tmpl").Parse(`
<br>
<section>
<p class="title-heading small"><strong>Locks</strong></p>
{{ if .Locks }}
{{ $basePath := .CleanedBasePath }}
{{ if .Locks }}
<div class="lock-grid">
<div class="lock-header">
<span>Repository</span>
Expand Down Expand Up @@ -151,6 +155,36 @@ var IndexTemplate = template.Must(template.New("index.html.tmpl").Parse(`
<p class="placeholder">No locks found.</p>
{{ end }}
</section>
<br>
<br>
<br>
<section>
<p class="title-heading small"><strong>Jobs</strong></p>
{{ if .PullToJobMapping }}
<div class="pulls-grid">
<div class="lock-header">
<span>Repository</span>
<span>Project</span>
<span>Workspace</span>
<span>Jobs</span>
</div>
{{ range .PullToJobMapping }}
<div class="pulls-row">
<span class="pulls-element">{{.Pull.RepoFullName}} #{{.Pull.PullNum}}</span>
<span class="pulls-element"><code>{{.Pull.Path}}</code></span>
<span class="pulls-element"><code>{{.Pull.Workspace}}</code></span>
<span class="pulls-element">
{{ range .JobIDInfos }}
<div><a href="{{ $basePath }}{{ .JobIDUrl }}" target="_blank">{{ .TimeFormatted }}</a></div>
{{ end }}
</span>
</div>
{{ end }}
</div>
{{ else }}
<p class="placeholder">No jobs found.</p>
{{ end }}
</section>
<div id="applyLockMessageModal" class="modal">
<!-- Modal content -->
<div class="modal-content">
Expand Down
10 changes: 9 additions & 1 deletion server/events/vcs/instrumented_client.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -218,14 +218,22 @@ func (c *InstrumentedClient) PullIsMergeable(repo models.Repo, pull models.PullR
func (c *InstrumentedClient) UpdateStatus(repo models.Repo, pull models.PullRequest, state models.CommitStatus, src string, description string, url string) error {
scope := c.StatsScope.SubScope("update_status")
scope = SetGitScopeTags(scope, repo.FullName, pull.Num)
logger := c.Logger.WithHistory(fmtLogSrc(repo, pull.Num)...)
logger := c.Logger.WithHistory([]interface{}{
"repository", fmt.Sprintf("%s/%s", repo.Owner, repo.Name),
"pull-num", strconv.Itoa(pull.Num),
"src", src,
"description", description,
"state", state,
"url", url,
}...)

executionTime := scope.Timer(metrics.ExecutionTimeMetric).Start()
defer executionTime.Stop()

executionSuccess := scope.Counter(metrics.ExecutionSuccessMetric)
executionError := scope.Counter(metrics.ExecutionErrorMetric)

logger.Info("updating vcs status")
if err := c.Client.UpdateStatus(repo, pull, state, src, description, url); err != nil {
executionError.Inc(1)
logger.Err("Unable to update status at url: %s, error: %s", url, err.Error())
Expand Down
32 changes: 32 additions & 0 deletions server/jobs/mocks/mock_project_command_output_handler.go
View file

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

82 changes: 68 additions & 14 deletions server/jobs/project_command_output_handler.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@ package jobs

import (
"sync"
"time"

"github.com/runatlantis/atlantis/server/events/command"
"github.com/runatlantis/atlantis/server/events/models"
Expand All @@ -14,10 +15,24 @@ type OutputBuffer struct {
}

type PullInfo struct {
PullNum int
Repo string
ProjectName string
Workspace string
PullNum int
Repo string
RepoFullName string
ProjectName string
Path string
Workspace string
}

type JobIDInfo struct {
JobID string
JobIDUrl string
Time time.Time
TimeFormatted string
}

type PullInfoWithJobIDs struct {
Pull PullInfo
JobIDInfos []JobIDInfo
}

type JobInfo struct {
Expand Down Expand Up @@ -71,6 +86,9 @@ type ProjectCommandOutputHandler interface {

// Cleans up resources for a pull
CleanUp(pullInfo PullInfo)

// Returns a map from Pull Requests to Jobs
GetPullToJobMapping() []PullInfoWithJobIDs
}

func NewAsyncProjectCommandOutputHandler(
Expand All @@ -86,6 +104,36 @@ func NewAsyncProjectCommandOutputHandler(
}
}

func (p *AsyncProjectCommandOutputHandler) GetPullToJobMapping() []PullInfoWithJobIDs {

pullToJobMappings := []PullInfoWithJobIDs{}
i := 0

p.pullToJobMapping.Range(func(key, value interface{}) bool {
pullInfo := key.(PullInfo)
jobIDMap := value.(map[string]time.Time)

p := PullInfoWithJobIDs{
Pull: pullInfo,
JobIDInfos: make([]JobIDInfo, 0, len(jobIDMap)),
}

for jobID, theTime := range jobIDMap {
jobIDInfo := JobIDInfo{
JobID: jobID,
Time: theTime,
}
p.JobIDInfos = append(p.JobIDInfos, jobIDInfo)
}

pullToJobMappings = append(pullToJobMappings, p)
i++
return true
})

return pullToJobMappings
}

func (p *AsyncProjectCommandOutputHandler) IsKeyExists(key string) bool {
p.projectOutputBuffersLock.RLock()
defer p.projectOutputBuffersLock.RUnlock()
Expand All @@ -99,10 +147,12 @@ func (p *AsyncProjectCommandOutputHandler) Send(ctx command.ProjectContext, msg
JobInfo: JobInfo{
HeadCommit: ctx.Pull.HeadCommit,
PullInfo: PullInfo{
PullNum: ctx.Pull.Num,
Repo: ctx.BaseRepo.Name,
ProjectName: ctx.ProjectName,
Workspace: ctx.Workspace,
PullNum: ctx.Pull.Num,
Repo: ctx.BaseRepo.Name,
RepoFullName: ctx.BaseRepo.FullName,
ProjectName: ctx.ProjectName,
Path: ctx.RepoRelDir,
Workspace: ctx.Workspace,
},
},
Line: msg,
Expand Down Expand Up @@ -138,11 +188,11 @@ func (p *AsyncProjectCommandOutputHandler) Handle() {

// Add job to pullToJob mapping
if _, ok := p.pullToJobMapping.Load(msg.JobInfo.PullInfo); !ok {
p.pullToJobMapping.Store(msg.JobInfo.PullInfo, map[string]bool{})
p.pullToJobMapping.Store(msg.JobInfo.PullInfo, map[string]time.Time{})
}
value, _ := p.pullToJobMapping.Load(msg.JobInfo.PullInfo)
jobMapping := value.(map[string]bool)
jobMapping[msg.JobID] = true
jobMapping := value.(map[string]time.Time)
jobMapping[msg.JobID] = time.Now()

// Forward new message to all receiver channels and output buffer
p.writeLogLine(msg.JobID, msg.Line)
Expand Down Expand Up @@ -239,16 +289,16 @@ func (p *AsyncProjectCommandOutputHandler) GetProjectOutputBuffer(jobID string)
return p.projectOutputBuffers[jobID]
}

func (p *AsyncProjectCommandOutputHandler) GetJobIDMapForPull(pullInfo PullInfo) map[string]bool {
func (p *AsyncProjectCommandOutputHandler) GetJobIDMapForPull(pullInfo PullInfo) map[string]time.Time {
if value, ok := p.pullToJobMapping.Load(pullInfo); ok {
return value.(map[string]bool)
return value.(map[string]time.Time)
}
return nil
}

func (p *AsyncProjectCommandOutputHandler) CleanUp(pullInfo PullInfo) {
if value, ok := p.pullToJobMapping.Load(pullInfo); ok {
jobMapping := value.(map[string]bool)
jobMapping := value.(map[string]time.Time)
for jobID := range jobMapping {
p.projectOutputBuffersLock.Lock()
delete(p.projectOutputBuffers, jobID)
Expand Down Expand Up @@ -286,3 +336,7 @@ func (p *NoopProjectOutputHandler) CleanUp(_ PullInfo) {
func (p *NoopProjectOutputHandler) IsKeyExists(_ string) bool {
return false
}

func (p *NoopProjectOutputHandler) GetPullToJobMapping() []PullInfoWithJobIDs {
return []PullInfoWithJobIDs{}
}
10 changes: 6 additions & 4 deletions server/jobs/project_command_output_handler_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -161,10 +161,12 @@ func TestProjectCommandOutputHandler(t *testing.T) {
projectOutputHandler.Send(ctx, "Complete", false)

pullContext := jobs.PullInfo{
PullNum: ctx.Pull.Num,
Repo: ctx.BaseRepo.Name,
ProjectName: ctx.ProjectName,
Workspace: ctx.Workspace,
PullNum: ctx.Pull.Num,
Repo: ctx.BaseRepo.Name,
RepoFullName: ctx.BaseRepo.FullName,
ProjectName: ctx.ProjectName,
Path: ctx.RepoRelDir,
Workspace: ctx.Workspace,
}
wg.Wait() // Must finish reading messages before cleaning up
projectOutputHandler.CleanUp(pullContext)
Expand Down
44 changes: 40 additions & 4 deletions server/server.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -521,6 +521,7 @@ func NewServer(userConfig UserConfig, config Config) (*Server, error) {
VCSClient: vcsClient,
},
)

eventParser := &events.EventParser{
GithubUser: userConfig.GithubUser,
GithubToken: userConfig.GithubToken,
Expand Down Expand Up @@ -1075,16 +1076,51 @@ func (s *Server) Index(w http.ResponseWriter, _ *http.Request) {
sort.SliceStable(lockResults, func(i, j int) bool { return lockResults[i].Time.After(lockResults[j].Time) })

err = s.IndexTemplate.Execute(w, templates.IndexData{
Locks: lockResults,
ApplyLock: applyLockData,
AtlantisVersion: s.AtlantisVersion,
CleanedBasePath: s.AtlantisURL.Path,
Locks: lockResults,
PullToJobMapping: preparePullToJobMappings(s),
ApplyLock: applyLockData,
AtlantisVersion: s.AtlantisVersion,
CleanedBasePath: s.AtlantisURL.Path,
})
if err != nil {
s.Logger.Err(err.Error())
}
}

func preparePullToJobMappings(s *Server) []jobs.PullInfoWithJobIDs {

pullToJobMappings := s.ProjectCmdOutputHandler.GetPullToJobMapping()

for i := range pullToJobMappings {
for j := range pullToJobMappings[i].JobIDInfos {
jobUrl, _ := s.Router.Get(ProjectJobsViewRouteName).URL("job-id", pullToJobMappings[i].JobIDInfos[j].JobID)
pullToJobMappings[i].JobIDInfos[j].JobIDUrl = jobUrl.String()
pullToJobMappings[i].JobIDInfos[j].TimeFormatted = pullToJobMappings[i].JobIDInfos[j].Time.Format("02-01-2006 15:04:05")
}

//Sort by date - newest to oldest.
sort.SliceStable(pullToJobMappings[i].JobIDInfos, func(x, y int) bool {
return pullToJobMappings[i].JobIDInfos[x].Time.After(pullToJobMappings[i].JobIDInfos[y].Time)
})
}

//Sort by repository, project, path, workspace then date.
sort.SliceStable(pullToJobMappings, func(x, y int) bool {
if pullToJobMappings[x].Pull.RepoFullName != pullToJobMappings[y].Pull.RepoFullName {
return pullToJobMappings[x].Pull.RepoFullName < pullToJobMappings[y].Pull.RepoFullName
}
if pullToJobMappings[x].Pull.ProjectName != pullToJobMappings[y].Pull.ProjectName {
return pullToJobMappings[x].Pull.ProjectName < pullToJobMappings[y].Pull.ProjectName
}
if pullToJobMappings[x].Pull.Path != pullToJobMappings[y].Pull.Path {
return pullToJobMappings[x].Pull.Path < pullToJobMappings[y].Pull.Path
}
return pullToJobMappings[x].Pull.Workspace < pullToJobMappings[y].Pull.Workspace
})

return pullToJobMappings
}

func mkSubDir(parentDir string, subDir string) (string, error) {
fullDir := filepath.Join(parentDir, subDir)
if err := os.MkdirAll(fullDir, 0700); err != nil {
Expand Down
Loading

0 comments on commit ce95f8e

Please sign in to comment.