cmd/coordinator,internal/coordinator: add queue UI

This adds a UI for viewing what builds are in each queue. For golang/go#48857 Change-Id: I025d01510833cc9c0d23556d7f90a62119eb39fb Reviewed-on: https://go-review.googlesource.com/c/build/+/419429 TryBot-Result: Gopher Robot <[email protected]> Reviewed-by: Heschi Kreinick <[email protected]> Run-TryBot: Jenny Rakoczy <[email protected]> Auto-Submit: Jenny Rakoczy <[email protected]>
golang · Jul 28, 2022 · b32452c · b32452c
1 parent 669914d
commit b32452c
Show file tree

Hide file tree

Showing 9 changed files with 224 additions and 2 deletions.
diff --git a/cmd/coordinator/coordinator.go b/cmd/coordinator/coordinator.go
@@ -287,6 +287,10 @@ func main() {
 			Name:     "linux-amd64",
 			HostType: "host-linux-amd64-localdev",
 		}
+		dashboard.Builders["linux-amd64-localdev"] = &dashboard.BuildConfig{
+			Name:     "linux-amd64",
+			HostType: "host-linux-amd64-localdev",
+		}
 	}
 
 	go pool.CoordinatorProcess().UpdateInstanceRecord()
@@ -375,6 +379,7 @@ func main() {
 	mux.HandleFunc("/status/reverse.json", pool.ReversePool().ServeReverseStatusJSON)
 	mux.HandleFunc("/status/post-submit-active.json", handlePostSubmitActiveJSON)
 	mux.Handle("/dashboard", dashV2)
+	mux.HandleFunc("/queues", handleQueues)
 	mux.Handle("/buildlet/create", requireBuildletProxyAuth(http.HandlerFunc(handleBuildletCreate)))
 	mux.Handle("/buildlet/list", requireBuildletProxyAuth(http.HandlerFunc(handleBuildletList)))
 	if *mode == "dev" {

diff --git a/cmd/coordinator/queues.go b/cmd/coordinator/queues.go
@@ -0,0 +1,43 @@
+// Copyright 2022 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+//go:build go1.16 && (linux || darwin)
+// +build go1.16
+// +build linux darwin
+
+package main
+
+import (
+	_ "embed"
+	"html/template"
+	"log"
+	"net/http"
+
+	"golang.org/x/build/internal/coordinator/pool"
+	"golang.org/x/build/internal/coordinator/pool/queue"
+)
+
+//go:embed queues.html
+var queuesTemplateStr string
+
+var queuesTemplate = template.Must(baseTmpl.New("queues.html").Parse(queuesTemplateStr))
+
+type QueuesResponse struct {
+	Queues map[string]*queue.QuotaStats
+}
+
+func handleQueues(w http.ResponseWriter, _ *http.Request) {
+	resp := QueuesResponse{Queues: map[string]*queue.QuotaStats{}}
+	mergeStats := func(qs map[string]*queue.QuotaStats) {
+		for name, stats := range qs {
+			resp.Queues[name] = stats
+		}
+	}
+	mergeStats(pool.ReversePool().QuotaStats())
+	mergeStats(pool.EC2BuildetPool().QuotaStats())
+	mergeStats(pool.NewGCEConfiguration().BuildletPool().QuotaStats())
+	if err := queuesTemplate.Execute(w, resp); err != nil {
+		log.Printf("handleQueues: %v", err)
+	}
+}
diff --git a/cmd/coordinator/queues.html b/cmd/coordinator/queues.html
@@ -0,0 +1,73 @@
+<!DOCTYPE html>
+<!--
+ Copyright 2022 The Go Authors All rights reserved.
+ Use of this source code is governed by a BSD-style
+ license that can be found in the LICENSE file.
+-->
+
+<html lang="en">
+  <head>
+    <link rel="stylesheet" href="/style.css" />
+    <title>Go Farmer Queues</title>
+  </head>
+  <body>
+    {{template "build-header"}}
+    <h2>Queues</h2>
+    <div class="QueueStats">
+      {{range $name, $stats := .Queues}}
+        <div class="QueueStats-queue">
+          <table class="QueueStats-queueTable">
+            <caption>
+              <a href="#{{$name}}" id="{{$name}}">
+                {{$name}}
+              </a>
+            </caption>
+            <thead>
+              <tr>
+                <th class="QueueStats-queueTableHeader">Name</th>
+                <th class="QueueStats-queueTableHeader">Cost</th>
+                <th class="QueueStats-queueTableHeader">Type/Priority</th>
+                <th class="QueueStats-queueTableHeader">User</th>
+              </tr>
+            </thead>
+            <tbody>
+              {{range $item := $stats.Items}}
+                <tr class="QueueStats-queueTableRow">
+                  {{$build := $item.Build}}
+                  <td class="QueueStats-queueTableColumn">
+                    {{$item.Build.HostType}}
+                  </td>
+                  <td class="QueueStats-queueTableColumn">
+                    {{$item.Cost}}
+                  </td>
+                  <td class="QueueStats-queueTableColumn">
+                    {{if $build.IsRelease}}
+                      Release
+                    {{else if $build.IsGomote}}
+                      Gomote
+                    {{else if $build.IsTry}}
+                      Trybot
+                    {{else}}
+                      Post-submit
+                    {{end}}
+                    /
+                    {{$build.Priority}}
+                  </td>
+                  <td class="QueueStats-queueTableColumn">
+                    {{$build.User}}
+                  </td>
+                </tr>
+              {{else}}
+                <tr class="QueueStats-queueTableRow">
+                  <td class="QueueStats-queueTableColumn" colspan="4">
+                    Queue empty.
+                  </td>
+                </tr>
+              {{end}}
+            </tbody>
+          </table>
+        </div>
+      {{end}}
+    </div>
+  </body>
+</html>
diff --git a/cmd/coordinator/style.css b/cmd/coordinator/style.css
@@ -185,3 +185,15 @@ table.Build tbody tr.commit:hover {
   color: #000;
   text-decoration: none;
 }
+.QueueStats {
+  display: flex;
+  flex-wrap: wrap;
+  justify-content: space-around;
+}
+.QueueStats-queueTable {
+  table-layout: fixed;
+}
+.QueueStats-queue {
+  margin: 1rem;
+  min-width: 35rem;
+}
diff --git a/internal/coordinator/pool/ec2.go b/internal/coordinator/pool/ec2.go
@@ -222,6 +222,12 @@ func (eb *EC2Buildlet) GetBuildlet(ctx context.Context, hostType string, lg Logg
 	return bc, nil
 }
 
+func (eb *EC2Buildlet) QuotaStats() map[string]*queue.QuotaStats {
+	return map[string]*queue.QuotaStats{
+		"ec2-cpu": eb.ledger.cpuQueue.ToExported(),
+	}
+}
+
 // String gives a report of capacity usage for the EC2 buildlet pool.
 func (eb *EC2Buildlet) String() string {
 	return fmt.Sprintf("EC2 pool capacity: %s", eb.capacityString())

diff --git a/internal/coordinator/pool/gce.go b/internal/coordinator/pool/gce.go
@@ -371,6 +371,16 @@ func (p *GCEBuildlet) pollQuota() {
 	}
 }
 
+func (p *GCEBuildlet) QuotaStats() map[string]*queue.QuotaStats {
+	return map[string]*queue.QuotaStats{
+		"gce-cpu":       p.cpuQueue.ToExported(),
+		"gce-c2-cpu":    p.c2cpuQueue.ToExported(),
+		"gce-n2-cpu":    p.n2cpuQueue.ToExported(),
+		"gce-n2d-cpu":   p.n2dcpuQueue.ToExported(),
+		"gce-instances": p.instQueue.ToExported(),
+	}
+}
+
 // SetEnabled marks the buildlet pool as enabled.
 func (p *GCEBuildlet) SetEnabled(enabled bool) {
 	p.mu.Lock()
@@ -481,13 +491,14 @@ func (p *GCEBuildlet) String() string {
 func (p *GCEBuildlet) capacityString() string {
 	p.mu.Lock()
 	defer p.mu.Unlock()
+	qLen := p.cpuQueue.Len()
 	cpuUsage, cpuLimit := p.cpuQueue.Quotas()
 	c2Usage, c2Limit := p.c2cpuQueue.Quotas()
 	instUsage, instLimit := p.instQueue.Quotas()
 	n2Usage, n2Limit := p.n2cpuQueue.Quotas()
 	n2dUsage, n2dLimit := p.n2dcpuQueue.Quotas()
-	return fmt.Sprintf("%d/%d instances; %d/%d CPUs, %d/%d C2_CPUS, %d/%d N2_CPUS, %d/%d N2D_CPUS",
-		instUsage, instLimit,
+	return fmt.Sprintf("%d/%d instances; %d/%d CPUs (%d), %d/%d C2_CPUS, %d/%d N2_CPUS, %d/%d N2D_CPUS",
+		instUsage, instLimit, qLen,
 		cpuUsage, cpuLimit,
 		c2Limit, c2Usage,
 		n2Limit, n2Usage,

diff --git a/internal/coordinator/pool/queue/quota.go b/internal/coordinator/pool/queue/quota.go
@@ -7,6 +7,7 @@ package queue
 import (
 	"container/heap"
 	"context"
+	"sort"
 	"sync"
 )
 
@@ -133,6 +134,36 @@ func (q *Quota) AwaitQueue(ctx context.Context, cost int, si *SchedItem) error {
 	return q.Enqueue(cost, si).Await(ctx)
 }
 
+type QuotaStats struct {
+	Used  int
+	Limit int
+	Items []ItemStats
+}
+
+type ItemStats struct {
+	Build *SchedItem
+	Cost  int
+}
+
+func (q *Quota) ToExported() *QuotaStats {
+	q.mu.Lock()
+	qs := &QuotaStats{
+		Used:  q.used,
+		Limit: q.limit,
+		Items: make([]ItemStats, q.queue.Len()),
+	}
+	for i, item := range *q.queue {
+		qs.Items[i].Build = item.SchedItem()
+		qs.Items[i].Cost = item.cost
+	}
+	q.mu.Unlock()
+
+	sort.Slice(qs.Items, func(i, j int) bool {
+		return qs.Items[i].Build.Less(qs.Items[j].Build)
+	})
+	return qs
+}
+
 // An Item is something we manage in a priority buildletQueue.
 type Item struct {
 	build   *SchedItem

diff --git a/internal/coordinator/pool/queue/quota_test.go b/internal/coordinator/pool/queue/quota_test.go
@@ -9,6 +9,8 @@ import (
 	"sync"
 	"testing"
 	"time"
+
+	"github.com/google/go-cmp/cmp"
 )
 
 func TestQueueEmpty(t *testing.T) {
@@ -164,3 +166,32 @@ func TestQueueCancel(t *testing.T) {
 		t.Errorf("q.Quotas() = %d, %d, wanted %d, %d", used, limit, 0, 15)
 	}
 }
+
+func TestQueueToExported(t *testing.T) {
+	q := NewQuota()
+	q.UpdateLimit(10)
+	q.Enqueue(100, &SchedItem{IsTry: true})
+	q.Enqueue(100, &SchedItem{IsTry: true})
+	q.Enqueue(100, &SchedItem{IsTry: true})
+	q.Enqueue(100, &SchedItem{IsGomote: true})
+	q.Enqueue(100, &SchedItem{IsGomote: true})
+	q.Enqueue(100, &SchedItem{IsGomote: true})
+	q.Enqueue(100, &SchedItem{IsRelease: true})
+	want := &QuotaStats{
+		Used:  0,
+		Limit: 10,
+		Items: []ItemStats{
+			{Build: &SchedItem{IsRelease: true}, Cost: 100},
+			{Build: &SchedItem{IsGomote: true}, Cost: 100},
+			{Build: &SchedItem{IsGomote: true}, Cost: 100},
+			{Build: &SchedItem{IsGomote: true}, Cost: 100},
+			{Build: &SchedItem{IsTry: true}, Cost: 100},
+			{Build: &SchedItem{IsTry: true}, Cost: 100},
+			{Build: &SchedItem{IsTry: true}, Cost: 100},
+		},
+	}
+	got := q.ToExported()
+	if diff := cmp.Diff(want, got); diff != "" {
+		t.Errorf("q.ToExported() mismatch (-want +got):\n%s", diff)
+	}
+}
diff --git a/internal/coordinator/pool/reverse.go b/internal/coordinator/pool/reverse.go
@@ -401,6 +401,16 @@ func (p *ReverseBuildletPool) WriteHTMLStatus(w io.Writer) {
 	fmt.Fprintf(w, "<b>Reverse pool machine detail</b><ul>%s</ul>", buf.Bytes())
 }
 
+func (p *ReverseBuildletPool) QuotaStats() map[string]*queue.QuotaStats {
+	p.mu.Lock()
+	defer p.mu.Unlock()
+	ret := make(map[string]*queue.QuotaStats)
+	for typ, queue := range p.hostQueue {
+		ret[fmt.Sprintf("reverse-%s", typ)] = queue.ToExported()
+	}
+	return ret
+}
+
 // HostTypeCount iterates through the running reverse buildlets, and
 // constructs a count of running buildlets per hostType.
 func (p *ReverseBuildletPool) HostTypeCount() map[string]int {