Ignore cluster if it is empty

\#227 added support for clusters, but as was somewhat anticipated, it
broke compatibility with the some existing backends because it added a
new dimension/label/tag or some equivalent for the cluster. In this PR,
we refrain from doing so if the cluster is the empty string.

One thing to note is that the cluster is not specified when the process
starts, it is encoded in the cluster token, and the backend include the name
of the cluster in its response. If the token is for unclustered agents,
that part of the response will be the empty string.

backend/cloudwatch.go

@@ -69,10 +69,14 @@ func (cb *CloudWatchBackend) Collect(r *collector.Result) error {
 			Name:  aws.String("Org"),
 			Value: aws.String(r.Org),
 		},
-		{
+	}
+
+	// Add cluster dimension if a cluster token was used
+	if r.Cluster != "" {
+		dimensions = append(dimensions, &cloudwatch.Dimension{
 			Name:  aws.String("Cluster"),
 			Value: aws.String(r.Cluster),
-		},
+		})
 	}
 
 	// Add custom dimension if provided

backend/newrelic.go

@@ -56,8 +56,11 @@ func (nr *NewRelicBackend) Collect(r *collector.Result) error {
 // toCustomEvent converts a map of metrics to a valid New Relic event body
 func toCustomEvent(clusterName, queueName string, queueMetrics map[string]int) map[string]any {
 	eventData := map[string]any{
-		"Cluster": clusterName,
-		"Queue":   queueName,
+		"Queue": queueName,
+	}
+
+	if clusterName != "" {
+		eventData["Cluster"] = clusterName
 	}
 
 	for k, v := range queueMetrics {

backend/prometheus.go

@@ -39,37 +39,53 @@ func (p *Prometheus) Serve(path, addr string) {
 }
 
 func (p *Prometheus) Collect(r *collector.Result) error {
-
 	// Clear the gauges to prevent stale values from persisting forever.
 	for _, gauge := range p.queues {
 		gauge.Reset()
 	}
 
 	for name, value := range r.Totals {
+		labelNames := []string{}
+		if r.Cluster != "" {
+			labelNames = append(labelNames, "cluster")
+		}
 		gauge, ok := p.totals[name]
 		if !ok {
 			gauge = prometheus.NewGaugeVec(prometheus.GaugeOpts{
 				Name: fmt.Sprintf("buildkite_total_%s", camelToUnderscore(name)),
 				Help: fmt.Sprintf("Buildkite Total: %s", name),
-			}, []string{"cluster"})
+			}, labelNames)
 			prometheus.MustRegister(gauge)
 			p.totals[name] = gauge
 		}
-		gauge.WithLabelValues(r.Cluster).Set(float64(value))
+
+		if r.Cluster != "" {
+			gauge.WithLabelValues(r.Cluster).Set(float64(value))
+		} else {
+			gauge.WithLabelValues().Set(float64(value))
+		}
 	}
 
 	for queue, counts := range r.Queues {
 		for name, value := range counts {
 			gauge, ok := p.queues[name]
 			if !ok {
+				labelNames := []string{"queue"}
+				if r.Cluster != "" {
+					labelNames = append(labelNames, "cluster")
+				}
 				gauge = prometheus.NewGaugeVec(prometheus.GaugeOpts{
 					Name: fmt.Sprintf("buildkite_queues_%s", camelToUnderscore(name)),
 					Help: fmt.Sprintf("Buildkite Queues: %s", name),
-				}, []string{"cluster", "queue"})
+				}, labelNames)
 				prometheus.MustRegister(gauge)
 				p.queues[name] = gauge
 			}
-			gauge.WithLabelValues(r.Cluster, queue).Set(float64(value))
+			if r.Cluster != "" {
+				gauge.WithLabelValues(queue, r.Cluster).Set(float64(value))
+			} else {
+				gauge.WithLabelValues(queue).Set(float64(value))
+			}
 		}
 	}