Terminate Beyla if we can't open a configured Prometheus port (#1111)

pkg/export/prom/prom_test.go

@@ -5,13 +5,17 @@ import (
 	"fmt"
 	"io"
 	"net/http"
+	"os"
+	"os/signal"
 	"regexp"
 	"sync"
+	"syscall"
 	"testing"
 	"time"
 
 	"github.com/mariomac/guara/pkg/test"
 	"github.com/mariomac/pipes/pipe"
+	"github.com/prometheus/client_golang/prometheus"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 
@@ -341,6 +345,56 @@ func TestSpanMetricsDiscarded(t *testing.T) {
 	}
 }
 
+func TestTerminatesOnBadPromPort(t *testing.T) {
+	now := syncedClock{now: time.Now()}
+	timeNow = now.Now
+
+	ctx, cancelCtx := context.WithCancel(context.Background())
+	defer cancelCtx()
+	openPort, err := test.FreeTCPPort()
+	require.NoError(t, err)
+
+	// Grab the port we just allocated for something else
+	handler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		fmt.Fprintf(w, "Hello, %v, http: %v\n", r.URL.Path, r.TLS == nil)
+	})
+	server := http.Server{Addr: fmt.Sprintf(":%d", openPort), Handler: handler}
+	serverUp := make(chan bool, 1)
+
+	go func() {
+		go func() {
+			time.Sleep(5 * time.Second)
+			serverUp <- true
+		}()
+		err := server.ListenAndServe()
+		fmt.Printf("Terminating server %v\n", err)
+	}()
+
+	sigChan := make(chan os.Signal, 1)
+	signal.Notify(sigChan, syscall.SIGINT)
+
+	pm := connector.PrometheusManager{}
+
+	c := prometheus.NewGaugeVec(prometheus.GaugeOpts{
+		Name: TracesTargetInfo,
+		Help: "target service information in trace span metric format",
+	}, []string{"a"}).MetricVec
+
+	pm.Register(openPort, "/metrics", c)
+	go pm.StartHTTP(ctx)
+
+	ok := false
+	select {
+	case sig := <-sigChan:
+		assert.Equal(t, sig, syscall.SIGINT)
+		ok = true
+	case <-time.After(5 * time.Second):
+		ok = false
+	}
+
+	assert.True(t, ok)
+}
+
 var mmux = sync.Mutex{}
 
 func getMetrics(t require.TestingT, promURL string) string {

pkg/internal/connector/prommgr.go

@@ -8,8 +8,10 @@ import (
 	"fmt"
 	"log/slog"
 	"net/http"
+	"os"
 	"strconv"
 	"sync/atomic"
+	"syscall"
 
 	"github.com/prometheus/client_golang/prometheus"
 	"github.com/prometheus/client_golang/prometheus/promhttp"
@@ -109,9 +111,13 @@ func (pm *PrometheusManager) listenAndServe(ctx context.Context, port int, handl
 	go func() {
 		err := server.ListenAndServe()
 		if errors.Is(err, http.ErrServerClosed) {
-			log.Debug("HTTP server was closed", "error", err)
+			log.Debug("Prometheus endpoint server was closed", "error", err)
 		} else {
-			log.Error("HTTP service ended unexpectedly", "error", err)
+			log.Error("Prometheus endpoint service ended unexpectedly", "error", err)
+			err = syscall.Kill(os.Getpid(), syscall.SIGINT) // interrupt for graceful shutdown, instead of os.Exit
+			if err != nil {
+				log.Error("unable to terminate Beyla", "error", err)
+			}
 		}
 	}()
 	go func() {