Add health check

build/service/Dockerfile

@@ -10,4 +10,13 @@ WORKDIR /app
 
 COPY --from=build /build/service .
 
+RUN apk --no-cache add curl
+HEALTHCHECK \
+    --start-period=10s \
+    --start-interval=1s \
+    --interval=1m \
+    --timeout=20s \
+    --retries=3 \
+    CMD curl -f http://localhost:8080/health || exit 1
+
 CMD ./service

cmd/service/main.go

@@ -16,6 +16,7 @@ import (
 	"github.com/nijeti/cinema-keeper/internal/handlers/unlock"
 	cfgPkg "github.com/nijeti/cinema-keeper/internal/pkg/config"
 	"github.com/nijeti/cinema-keeper/internal/pkg/dbUtils"
+	"github.com/nijeti/cinema-keeper/internal/pkg/healthcheck"
 )
 
 type config struct {
@@ -25,27 +26,30 @@ type config struct {
 
 func main() {
 	log.Println("starting")
+	defer log.Println("shutdown complete")
 
 	cfg := cfgPkg.ReadConfig[config]()
 
 	ctx, cancel := context.WithCancel(context.Background())
-	defer cancel()
 	stop := make(chan os.Signal, 1)
 	signal.Notify(stop, os.Interrupt)
 
 	// logging
 	logger := slog.New(slog.NewJSONHandler(os.Stdout, nil))
 	dbLogger := logger.WithGroup("db")
 	cmdLogger := logger.WithGroup("command")
+	hcLogger := logger.WithGroup("healthcheck")
 
 	// db
 	dbConn := db.Connect(cfg.DB)
 	defer dbConn.Close()
+	dbProbe := db.NewProbe(dbConn)
 	txWrapper := dbUtils.NewTxWrapper(dbLogger, dbConn)
 
 	// discord
 	discordConn := discord.Connect(cfg.Discord.Token)
 	defer discordConn.Close()
+	discordProbe := discord.NewProbe(discordConn)
 
 	// repos
 	quotesRepo := db.NewQuotesRepo(dbLogger, dbConn, txWrapper)
@@ -78,8 +82,20 @@ func main() {
 	discord.RegisterCommands(discordConn, cmds, cfg.Discord.Guild)
 	defer discord.UnregisterCommands(discordConn, cmds, cfg.Discord.Guild)
 
+	// healthcheck
+	hc := healthcheck.New(
+		hcLogger,
+		healthcheck.WithProbe("db", dbProbe),
+		healthcheck.WithProbe("discord", discordProbe),
+	)
+	hcServer := hc.Serve(":8080")
+	defer hcServer.Shutdown()
+
 	// run
-	log.Println("started")
+	log.Println("startup complete")
 	<-stop
+
+	// shutdown
 	log.Println("shutting down")
+	cancel()
 }

go.mod

@@ -11,16 +11,19 @@ require (
 	github.com/knadh/koanf/v2 v2.1.1
 	github.com/lib/pq v1.10.9
 	github.com/stretchr/testify v1.9.0
+	github.com/valyala/fasthttp v1.55.0
 )
 
 require (
+	github.com/andybalholm/brotli v1.1.0 // indirect
 	github.com/bytedance/sonic/loader v0.1.1 // indirect
 	github.com/cloudwego/base64x v0.1.4 // indirect
 	github.com/cloudwego/iasm v0.2.0 // indirect
 	github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect
 	github.com/fsnotify/fsnotify v1.7.0 // indirect
 	github.com/go-viper/mapstructure/v2 v2.0.0 // indirect
 	github.com/gorilla/websocket v1.5.3 // indirect
+	github.com/klauspost/compress v1.17.9 // indirect
 	github.com/klauspost/cpuid/v2 v2.2.8 // indirect
 	github.com/knadh/koanf/maps v0.1.1 // indirect
 	github.com/kr/pretty v0.3.1 // indirect
@@ -29,6 +32,7 @@ require (
 	github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect
 	github.com/stretchr/objx v0.5.2 // indirect
 	github.com/twitchyliquid64/golang-asm v0.15.1 // indirect
+	github.com/valyala/bytebufferpool v1.0.0 // indirect
 	golang.org/x/arch v0.8.0 // indirect
 	golang.org/x/crypto v0.25.0 // indirect
 	golang.org/x/sys v0.22.0 // indirect

go.sum

@@ -1,3 +1,5 @@
+github.com/andybalholm/brotli v1.1.0 h1:eLKJA0d02Lf0mVpIDgYnqXcUn0GqVmEFny3VuID1U3M=
+github.com/andybalholm/brotli v1.1.0/go.mod h1:sms7XGricyQI9K10gOSf56VKKWS4oLer58Q+mhRPtnY=
 github.com/bwmarrin/discordgo v0.28.1 h1:gXsuo2GBO7NbR6uqmrrBDplPUx2T3nzu775q/Rd1aG4=
 github.com/bwmarrin/discordgo v0.28.1/go.mod h1:NJZpH+1AfhIcyQsPeuBKsUtYrRnjkyu0kIVMCHkZtRY=
 github.com/bytedance/sonic v1.11.9 h1:LFHENlIY/SLzDWverzdOvgMztTxcfcF+cqNsz9pK5zg=
@@ -20,6 +22,8 @@ github.com/go-viper/mapstructure/v2 v2.0.0/go.mod h1:oJDH3BJKyqBA2TXFhDsKDGDTlnd
 github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
 github.com/gorilla/websocket v1.5.3 h1:saDtZ6Pbx/0u+bgYQ3q96pZgCzfhKXGPqt7kZ72aNNg=
 github.com/gorilla/websocket v1.5.3/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
+github.com/klauspost/compress v1.17.9 h1:6KIumPrER1LHsvBVuDa0r5xaG0Es51mhhB9BQB2qeMA=
+github.com/klauspost/compress v1.17.9/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw=
 github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg=
 github.com/klauspost/cpuid/v2 v2.2.8 h1:+StwCXwm9PdpiEkPyzBXIy+M9KUb4ODm0Zarf1kS5BM=
 github.com/klauspost/cpuid/v2 v2.2.8/go.mod h1:Lcz8mBdAVJIBVzewtcLocK12l3Y+JytZYpaMropDUws=
@@ -63,6 +67,10 @@ github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsT
 github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
 github.com/twitchyliquid64/golang-asm v0.15.1 h1:SU5vSMR7hnwNxj24w34ZyCi/FmDZTkS4MhqMhdFk5YI=
 github.com/twitchyliquid64/golang-asm v0.15.1/go.mod h1:a1lVb/DtPvCB8fslRZhAngC2+aY1QWCk3Cedj/Gdt08=
+github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw=
+github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc=
+github.com/valyala/fasthttp v1.55.0 h1:Zkefzgt6a7+bVKHnu/YaYSOPfNYNisSVBo/unVCf8k8=
+github.com/valyala/fasthttp v1.55.0/go.mod h1:NkY9JtkrpPKmgwV3HTaS2HWaJss9RSIsRVfcxxoHiOM=
 golang.org/x/arch v0.0.0-20210923205945-b76863e36670/go.mod h1:5om86z9Hs0C8fWVUuoMHwpExlXzs5Tkyp9hOrfG7pp8=
 golang.org/x/arch v0.8.0 h1:3wRIsP3pM4yUptoR96otTUOXI367OS0+c9eeRi9doIc=
 golang.org/x/arch v0.8.0/go.mod h1:FEVrYAQjsQXMVJ1nsMoVVXPZg6p2JE2mx8psSWTDQys=

internal/db/migrator.go

@@ -22,7 +22,7 @@ func NewMigrator(
 	txWrapper dbUtils.TxWrapper,
 ) Migrator {
 	return Migrator{
-		log:       log,
+		log:       log.With("type", "migrator"),
 		db:        db,
 		txWrapper: txWrapper,
 	}

internal/db/probe.go

@@ -0,0 +1,20 @@
+package db
+
+import (
+	"context"
+	"database/sql"
+)
+
+type Probe struct {
+	db *sql.DB
+}
+
+func NewProbe(db *sql.DB) Probe {
+	return Probe{
+		db: db,
+	}
+}
+
+func (p Probe) Check(ctx context.Context) error {
+	return p.db.PingContext(ctx)
+}

internal/discord/probe.go

@@ -0,0 +1,24 @@
+package discord
+
+import (
+	"context"
+
+	"github.com/bwmarrin/discordgo"
+)
+
+type Probe struct {
+	discord *discordgo.Session
+}
+
+func NewProbe(
+	discord *discordgo.Session,
+) Probe {
+	return Probe{
+		discord: discord,
+	}
+}
+
+func (p Probe) Check(_ context.Context) error {
+	_, err := p.discord.User("@me")
+	return err
+}

internal/pkg/healthcheck/healthcheck.go

@@ -0,0 +1,151 @@
+package healthcheck
+
+import (
+	"context"
+	"log"
+	"log/slog"
+	"sync"
+	"time"
+
+	"github.com/valyala/fasthttp"
+)
+
+type Healthcheck struct {
+	log              *slog.Logger
+	probes           map[string]Probe
+	timeoutDegraded  time.Duration
+	timeoutUnhealthy time.Duration
+}
+
+func New(
+	log *slog.Logger,
+	opts ...Option,
+) *Healthcheck {
+	hc := &Healthcheck{
+		log:              log,
+		probes:           map[string]Probe{},
+		timeoutDegraded:  1 * time.Second,
+		timeoutUnhealthy: 10 * time.Second,
+	}
+
+	for _, opt := range opts {
+		opt(hc)
+	}
+
+	return hc
+}
+
+func (hc *Healthcheck) Serve(address string) *fasthttp.Server {
+	s := &fasthttp.Server{
+		Handler: func(ctx *fasthttp.RequestCtx) {
+			if string(ctx.Path()) != "/health" || !ctx.IsGet() {
+				ctx.NotFound()
+				return
+			}
+
+			status := hc.handle(ctx)
+
+			if status == StatusHealthy {
+				ctx.SetStatusCode(fasthttp.StatusOK)
+			} else {
+				ctx.SetStatusCode(fasthttp.StatusServiceUnavailable)
+			}
+			ctx.SetBodyString(status.String())
+		},
+	}
+
+	go func() {
+		err := s.ListenAndServe(address)
+		if err != nil {
+			log.Fatalln("healthcheck server error", "error", err)
+		}
+	}()
+
+	return s
+}
+
+func (hc *Healthcheck) handle(ctx context.Context) Status {
+	nProbes := len(hc.probes)
+
+	wg := &sync.WaitGroup{}
+	wg.Add(nProbes)
+
+	statuses := make(chan Status, nProbes)
+
+	for name, probe := range hc.probes {
+		hc.probeCheck(
+			hc.log.With("probe", name), ctx, wg, probe, statuses,
+		)
+	}
+
+	wg.Wait()
+	close(statuses)
+
+	status := StatusHealthy
+	for s := range statuses {
+		if s <= status {
+			continue
+		}
+
+		status = s
+		if status == StatusUnhealthy {
+			break
+		}
+	}
+
+	return status
+}
+
+func (hc *Healthcheck) probeCheck(
+	log *slog.Logger,
+	ctx context.Context,
+	wg *sync.WaitGroup,
+	probe Probe,
+	status chan Status,
+) {
+	timedCtx, cancel := context.WithTimeout(ctx, hc.timeoutUnhealthy)
+
+	defer func() {
+		if err := recover(); err != nil {
+			log.ErrorContext(
+				ctx,
+				"probe panicked",
+				"panic", err,
+			)
+
+			status <- StatusUnhealthy
+		}
+
+		cancel()
+		wg.Done()
+	}()
+
+	probeTime := time.Now()
+	err := probe.Check(timedCtx)
+	probeDuration := time.Since(probeTime)
+
+	if err != nil || probeDuration > hc.timeoutUnhealthy {
+		log.ErrorContext(
+			ctx,
+			"failed to probe",
+			"error", err,
+			"duration", probeDuration.String(),
+		)
+
+		status <- StatusUnhealthy
+		return
+	}
+
+	if probeDuration > hc.timeoutDegraded {
+		log.WarnContext(
+			ctx,
+			"probe is degraded",
+			"duration", probeDuration.String(),
+		)
+
+		status <- StatusDegraded
+		return
+	}
+
+	status <- StatusHealthy
+}

internal/pkg/healthcheck/options.go

@@ -0,0 +1,39 @@
+package healthcheck
+
+import (
+	"fmt"
+	"time"
+)
+
+type Option func(hc *Healthcheck)
+
+func WithProbe(name string, probe Probe) Option {
+	return func(hc *Healthcheck) {
+		if _, ok := hc.probes[name]; ok {
+			p := fmt.Sprintf("healthcheck probe '%s' already registered", name)
+			panic(p)
+		}
+
+		hc.probes[name] = probe
+	}
+}
+
+func WithTimeoutDegraded(timeout time.Duration) Option {
+	return func(hc *Healthcheck) {
+		if timeout <= 0 {
+			panic("healthcheck timeout must be greater than zero")
+		}
+
+		hc.timeoutDegraded = timeout
+	}
+}
+
+func WithTimeoutUnhealthy(timeout time.Duration) Option {
+	return func(hc *Healthcheck) {
+		if timeout <= 0 {
+			panic("healthcheck timeout must be greater than zero")
+		}
+
+		hc.timeoutUnhealthy = timeout
+	}
+}

internal/pkg/healthcheck/probe.go

@@ -0,0 +1,9 @@
+package healthcheck
+
+import (
+	"context"
+)
+
+type Probe interface {
+	Check(ctx context.Context) error
+}