diff --git a/cmd/coordinator/coordinator.go b/cmd/coordinator/coordinator.go index e6d667e384..13f18a5b1e 100644 --- a/cmd/coordinator/coordinator.go +++ b/cmd/coordinator/coordinator.go @@ -287,6 +287,10 @@ func main() { Name: "linux-amd64", HostType: "host-linux-amd64-localdev", } + dashboard.Builders["linux-amd64-localdev"] = &dashboard.BuildConfig{ + Name: "linux-amd64", + HostType: "host-linux-amd64-localdev", + } } go pool.CoordinatorProcess().UpdateInstanceRecord() @@ -375,6 +379,7 @@ func main() { mux.HandleFunc("/status/reverse.json", pool.ReversePool().ServeReverseStatusJSON) mux.HandleFunc("/status/post-submit-active.json", handlePostSubmitActiveJSON) mux.Handle("/dashboard", dashV2) + mux.HandleFunc("/queues", handleQueues) mux.Handle("/buildlet/create", requireBuildletProxyAuth(http.HandlerFunc(handleBuildletCreate))) mux.Handle("/buildlet/list", requireBuildletProxyAuth(http.HandlerFunc(handleBuildletList))) if *mode == "dev" { diff --git a/cmd/coordinator/queues.go b/cmd/coordinator/queues.go new file mode 100644 index 0000000000..a9e5a429a1 --- /dev/null +++ b/cmd/coordinator/queues.go @@ -0,0 +1,43 @@ +// Copyright 2022 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build go1.16 && (linux || darwin) +// +build go1.16 +// +build linux darwin + +package main + +import ( + _ "embed" + "html/template" + "log" + "net/http" + + "golang.org/x/build/internal/coordinator/pool" + "golang.org/x/build/internal/coordinator/pool/queue" +) + +//go:embed queues.html +var queuesTemplateStr string + +var queuesTemplate = template.Must(baseTmpl.New("queues.html").Parse(queuesTemplateStr)) + +type QueuesResponse struct { + Queues map[string]*queue.QuotaStats +} + +func handleQueues(w http.ResponseWriter, _ *http.Request) { + resp := QueuesResponse{Queues: map[string]*queue.QuotaStats{}} + mergeStats := func(qs map[string]*queue.QuotaStats) { + for name, stats := range qs { + resp.Queues[name] = stats + } + } + mergeStats(pool.ReversePool().QuotaStats()) + mergeStats(pool.EC2BuildetPool().QuotaStats()) + mergeStats(pool.NewGCEConfiguration().BuildletPool().QuotaStats()) + if err := queuesTemplate.Execute(w, resp); err != nil { + log.Printf("handleQueues: %v", err) + } +} diff --git a/cmd/coordinator/queues.html b/cmd/coordinator/queues.html new file mode 100644 index 0000000000..65715e9b1a --- /dev/null +++ b/cmd/coordinator/queues.html @@ -0,0 +1,73 @@ + + + + + + + Go Farmer Queues + + + {{template "build-header"}} +

Queues

+
+ {{range $name, $stats := .Queues}} +
+ + + + + + + + + + + + {{range $item := $stats.Items}} + + {{$build := $item.Build}} + + + + + + {{else}} + + + + {{end}} + +
+ + {{$name}} + +
NameCostType/PriorityUser
+ {{$item.Build.HostType}} + + {{$item.Cost}} + + {{if $build.IsRelease}} + Release + {{else if $build.IsGomote}} + Gomote + {{else if $build.IsTry}} + Trybot + {{else}} + Post-submit + {{end}} + / + {{$build.Priority}} + + {{$build.User}} +
+ Queue empty. +
+
+ {{end}} +
+ + diff --git a/cmd/coordinator/style.css b/cmd/coordinator/style.css index 89914f6f6f..03f873937e 100644 --- a/cmd/coordinator/style.css +++ b/cmd/coordinator/style.css @@ -185,3 +185,15 @@ table.Build tbody tr.commit:hover { color: #000; text-decoration: none; } +.QueueStats { + display: flex; + flex-wrap: wrap; + justify-content: space-around; +} +.QueueStats-queueTable { + table-layout: fixed; +} +.QueueStats-queue { + margin: 1rem; + min-width: 35rem; +} diff --git a/internal/coordinator/pool/ec2.go b/internal/coordinator/pool/ec2.go index af0a2bf40f..7c1a533d74 100644 --- a/internal/coordinator/pool/ec2.go +++ b/internal/coordinator/pool/ec2.go @@ -222,6 +222,12 @@ func (eb *EC2Buildlet) GetBuildlet(ctx context.Context, hostType string, lg Logg return bc, nil } +func (eb *EC2Buildlet) QuotaStats() map[string]*queue.QuotaStats { + return map[string]*queue.QuotaStats{ + "ec2-cpu": eb.ledger.cpuQueue.ToExported(), + } +} + // String gives a report of capacity usage for the EC2 buildlet pool. func (eb *EC2Buildlet) String() string { return fmt.Sprintf("EC2 pool capacity: %s", eb.capacityString()) diff --git a/internal/coordinator/pool/gce.go b/internal/coordinator/pool/gce.go index 5835ecf018..aa05f59d89 100644 --- a/internal/coordinator/pool/gce.go +++ b/internal/coordinator/pool/gce.go @@ -371,6 +371,16 @@ func (p *GCEBuildlet) pollQuota() { } } +func (p *GCEBuildlet) QuotaStats() map[string]*queue.QuotaStats { + return map[string]*queue.QuotaStats{ + "gce-cpu": p.cpuQueue.ToExported(), + "gce-c2-cpu": p.c2cpuQueue.ToExported(), + "gce-n2-cpu": p.n2cpuQueue.ToExported(), + "gce-n2d-cpu": p.n2dcpuQueue.ToExported(), + "gce-instances": p.instQueue.ToExported(), + } +} + // SetEnabled marks the buildlet pool as enabled. func (p *GCEBuildlet) SetEnabled(enabled bool) { p.mu.Lock() @@ -481,13 +491,14 @@ func (p *GCEBuildlet) String() string { func (p *GCEBuildlet) capacityString() string { p.mu.Lock() defer p.mu.Unlock() + qLen := p.cpuQueue.Len() cpuUsage, cpuLimit := p.cpuQueue.Quotas() c2Usage, c2Limit := p.c2cpuQueue.Quotas() instUsage, instLimit := p.instQueue.Quotas() n2Usage, n2Limit := p.n2cpuQueue.Quotas() n2dUsage, n2dLimit := p.n2dcpuQueue.Quotas() - return fmt.Sprintf("%d/%d instances; %d/%d CPUs, %d/%d C2_CPUS, %d/%d N2_CPUS, %d/%d N2D_CPUS", - instUsage, instLimit, + return fmt.Sprintf("%d/%d instances; %d/%d CPUs (%d), %d/%d C2_CPUS, %d/%d N2_CPUS, %d/%d N2D_CPUS", + instUsage, instLimit, qLen, cpuUsage, cpuLimit, c2Limit, c2Usage, n2Limit, n2Usage, diff --git a/internal/coordinator/pool/queue/quota.go b/internal/coordinator/pool/queue/quota.go index 8a647d86b2..7cbeb6aea2 100644 --- a/internal/coordinator/pool/queue/quota.go +++ b/internal/coordinator/pool/queue/quota.go @@ -7,6 +7,7 @@ package queue import ( "container/heap" "context" + "sort" "sync" ) @@ -133,6 +134,36 @@ func (q *Quota) AwaitQueue(ctx context.Context, cost int, si *SchedItem) error { return q.Enqueue(cost, si).Await(ctx) } +type QuotaStats struct { + Used int + Limit int + Items []ItemStats +} + +type ItemStats struct { + Build *SchedItem + Cost int +} + +func (q *Quota) ToExported() *QuotaStats { + q.mu.Lock() + qs := &QuotaStats{ + Used: q.used, + Limit: q.limit, + Items: make([]ItemStats, q.queue.Len()), + } + for i, item := range *q.queue { + qs.Items[i].Build = item.SchedItem() + qs.Items[i].Cost = item.cost + } + q.mu.Unlock() + + sort.Slice(qs.Items, func(i, j int) bool { + return qs.Items[i].Build.Less(qs.Items[j].Build) + }) + return qs +} + // An Item is something we manage in a priority buildletQueue. type Item struct { build *SchedItem diff --git a/internal/coordinator/pool/queue/quota_test.go b/internal/coordinator/pool/queue/quota_test.go index 1398d31970..a2ae8009f5 100644 --- a/internal/coordinator/pool/queue/quota_test.go +++ b/internal/coordinator/pool/queue/quota_test.go @@ -9,6 +9,8 @@ import ( "sync" "testing" "time" + + "github.com/google/go-cmp/cmp" ) func TestQueueEmpty(t *testing.T) { @@ -164,3 +166,32 @@ func TestQueueCancel(t *testing.T) { t.Errorf("q.Quotas() = %d, %d, wanted %d, %d", used, limit, 0, 15) } } + +func TestQueueToExported(t *testing.T) { + q := NewQuota() + q.UpdateLimit(10) + q.Enqueue(100, &SchedItem{IsTry: true}) + q.Enqueue(100, &SchedItem{IsTry: true}) + q.Enqueue(100, &SchedItem{IsTry: true}) + q.Enqueue(100, &SchedItem{IsGomote: true}) + q.Enqueue(100, &SchedItem{IsGomote: true}) + q.Enqueue(100, &SchedItem{IsGomote: true}) + q.Enqueue(100, &SchedItem{IsRelease: true}) + want := &QuotaStats{ + Used: 0, + Limit: 10, + Items: []ItemStats{ + {Build: &SchedItem{IsRelease: true}, Cost: 100}, + {Build: &SchedItem{IsGomote: true}, Cost: 100}, + {Build: &SchedItem{IsGomote: true}, Cost: 100}, + {Build: &SchedItem{IsGomote: true}, Cost: 100}, + {Build: &SchedItem{IsTry: true}, Cost: 100}, + {Build: &SchedItem{IsTry: true}, Cost: 100}, + {Build: &SchedItem{IsTry: true}, Cost: 100}, + }, + } + got := q.ToExported() + if diff := cmp.Diff(want, got); diff != "" { + t.Errorf("q.ToExported() mismatch (-want +got):\n%s", diff) + } +} diff --git a/internal/coordinator/pool/reverse.go b/internal/coordinator/pool/reverse.go index 35b77c3432..a58fb43f2a 100644 --- a/internal/coordinator/pool/reverse.go +++ b/internal/coordinator/pool/reverse.go @@ -401,6 +401,16 @@ func (p *ReverseBuildletPool) WriteHTMLStatus(w io.Writer) { fmt.Fprintf(w, "Reverse pool machine detail", buf.Bytes()) } +func (p *ReverseBuildletPool) QuotaStats() map[string]*queue.QuotaStats { + p.mu.Lock() + defer p.mu.Unlock() + ret := make(map[string]*queue.QuotaStats) + for typ, queue := range p.hostQueue { + ret[fmt.Sprintf("reverse-%s", typ)] = queue.ToExported() + } + return ret +} + // HostTypeCount iterates through the running reverse buildlets, and // constructs a count of running buildlets per hostType. func (p *ReverseBuildletPool) HostTypeCount() map[string]int {