feat(trigger.buildkite): emit event for every buildkite queue

examples/cron-buildkite-aws-ec2-asg.toml

@@ -0,0 +1,37 @@
+#
+# cron -> buildkite
+#
+# For every 1 minute, waymond will check if there is new jobs waiting to be scheduled in buildkite
+# If yes, it will try to schedule it in the respective autoscaling group
+
+[[trigger]]
+type = "cron"
+id = "global_cron"
+expression = "*/1 * * * *"
+
+[[trigger]]
+type = "buildkite"
+id = "my_buildkite_org"
+# set BUILDKITE_TOKEN environment variable
+
+[[connect]]
+type = "direct"
+id = "check_my_buildkite_org_queues_periodically"
+from = "trigger.global_cron"
+to = "trigger.my_buildkite_org"
+
+[[scaler]]
+type = "aws_ec2_asg"
+id = "my_aws_org_asg_scaler"
+
+[[connect]]
+type = "go_template"
+id = "connect_buildkite_metrics_to_aws_asg"
+from = "trigger.my_buildkite_org"
+to = "scaler.my_aws_org_asg_scaler"
+output = """
+{
+  "asg_name": "{{ .Queue }}",
+  "desired_count": {{ .SchduledJobCount }}
+}
+"""

internal/trigger/buildkite/buildkite.go

@@ -71,6 +71,14 @@ func (t *Trigger) Do(_ []byte) error {
 	var metrics struct {
 		Jobs struct {
 			Queues map[string]struct {
+				// Scheduled will give us the number of jobs that are waiting for an agent to pick up
+				Scheduled int
+
+				// Waiting will give the jobs which are waiting to be scheduled.
+				// example: job B will be waiting for job A if those are connected by the `depends_on` and job A is not finished
+				// Besides jobs to go to "Waiting" state infinitely if there is a missing step dependency.
+				// example: job B depends on job C and job C is not present in the build.
+				// Hence, it is better to neglect it during auto-scaling.
 				Waiting int
 			}
 		} `json:"jobs"`
@@ -80,8 +88,21 @@ func (t *Trigger) Do(_ []byte) error {
 		return fmt.Errorf("unable to decode response body: %s", err)
 	}
 
+	type outputData struct {
+		Queue              string `json:"queue"`
+		ScheduledJobsCount int    `json:"scheduled_jobs_count"`
+	}
 	for qName, q := range metrics.Jobs.Queues {
-		t.log.Debugf("qName: %s, waitingSize: %d \n", qName, q.Waiting)
+		t.log.Debugf("qName: %s, waitingSize: %d \n", qName, q.Scheduled)
+		data := outputData{
+			Queue:              qName,
+			ScheduledJobsCount: q.Scheduled,
+		}
+		rawData, err := json.Marshal(data)
+		if err != nil {
+			event.B.Publish(fmt.Sprintf("%s.error", t.namespacedID), []byte(err.Error()))
+		}
+		event.B.Publish(fmt.Sprintf("%s.output", t.namespacedID), rawData)
 	}
 
 	return nil