From 1625dd798690110f130bd68c4f0521be5c951d29 Mon Sep 17 00:00:00 2001 From: Arkdady Skvoretsky Date: Tue, 23 Jul 2024 23:09:11 +0300 Subject: [PATCH] Add health check --- build/service/Dockerfile | 9 ++ cmd/service/main.go | 20 +++- go.mod | 4 + go.sum | 8 ++ internal/db/migrator.go | 2 +- internal/db/probe.go | 20 ++++ internal/discord/probe.go | 24 ++++ internal/pkg/healthcheck/healthcheck.go | 151 ++++++++++++++++++++++++ internal/pkg/healthcheck/options.go | 39 ++++++ internal/pkg/healthcheck/probe.go | 9 ++ internal/pkg/healthcheck/status.go | 26 ++++ 11 files changed, 309 insertions(+), 3 deletions(-) create mode 100644 internal/db/probe.go create mode 100644 internal/discord/probe.go create mode 100644 internal/pkg/healthcheck/healthcheck.go create mode 100644 internal/pkg/healthcheck/options.go create mode 100644 internal/pkg/healthcheck/probe.go create mode 100644 internal/pkg/healthcheck/status.go diff --git a/build/service/Dockerfile b/build/service/Dockerfile index c49b5e6..a962a02 100644 --- a/build/service/Dockerfile +++ b/build/service/Dockerfile @@ -10,4 +10,13 @@ WORKDIR /app COPY --from=build /build/service . +RUN apk --no-cache add curl +HEALTHCHECK \ + --start-period=10s \ + --start-interval=1s \ + --interval=1m \ + --timeout=20s \ + --retries=3 \ + CMD curl -f http://localhost:8080/health || exit 1 + CMD ./service diff --git a/cmd/service/main.go b/cmd/service/main.go index 220d97a..d7baddc 100644 --- a/cmd/service/main.go +++ b/cmd/service/main.go @@ -16,6 +16,7 @@ import ( "github.com/nijeti/cinema-keeper/internal/handlers/unlock" cfgPkg "github.com/nijeti/cinema-keeper/internal/pkg/config" "github.com/nijeti/cinema-keeper/internal/pkg/dbUtils" + "github.com/nijeti/cinema-keeper/internal/pkg/healthcheck" ) type config struct { @@ -25,11 +26,11 @@ type config struct { func main() { log.Println("starting") + defer log.Println("shutdown complete") cfg := cfgPkg.ReadConfig[config]() ctx, cancel := context.WithCancel(context.Background()) - defer cancel() stop := make(chan os.Signal, 1) signal.Notify(stop, os.Interrupt) @@ -37,15 +38,18 @@ func main() { logger := slog.New(slog.NewJSONHandler(os.Stdout, nil)) dbLogger := logger.WithGroup("db") cmdLogger := logger.WithGroup("command") + hcLogger := logger.WithGroup("healthcheck") // db dbConn := db.Connect(cfg.DB) defer dbConn.Close() + dbProbe := db.NewProbe(dbConn) txWrapper := dbUtils.NewTxWrapper(dbLogger, dbConn) // discord discordConn := discord.Connect(cfg.Discord.Token) defer discordConn.Close() + discordProbe := discord.NewProbe(discordConn) // repos quotesRepo := db.NewQuotesRepo(dbLogger, dbConn, txWrapper) @@ -78,8 +82,20 @@ func main() { discord.RegisterCommands(discordConn, cmds, cfg.Discord.Guild) defer discord.UnregisterCommands(discordConn, cmds, cfg.Discord.Guild) + // healthcheck + hc := healthcheck.New( + hcLogger, + healthcheck.WithProbe("db", dbProbe), + healthcheck.WithProbe("discord", discordProbe), + ) + hcServer := hc.Serve(":8080") + defer hcServer.Shutdown() + // run - log.Println("started") + log.Println("startup complete") <-stop + + // shutdown log.Println("shutting down") + cancel() } diff --git a/go.mod b/go.mod index 46768ce..897697b 100644 --- a/go.mod +++ b/go.mod @@ -11,9 +11,11 @@ require ( github.com/knadh/koanf/v2 v2.1.1 github.com/lib/pq v1.10.9 github.com/stretchr/testify v1.9.0 + github.com/valyala/fasthttp v1.55.0 ) require ( + github.com/andybalholm/brotli v1.1.0 // indirect github.com/bytedance/sonic/loader v0.1.1 // indirect github.com/cloudwego/base64x v0.1.4 // indirect github.com/cloudwego/iasm v0.2.0 // indirect @@ -21,6 +23,7 @@ require ( github.com/fsnotify/fsnotify v1.7.0 // indirect github.com/go-viper/mapstructure/v2 v2.0.0 // indirect github.com/gorilla/websocket v1.5.3 // indirect + github.com/klauspost/compress v1.17.9 // indirect github.com/klauspost/cpuid/v2 v2.2.8 // indirect github.com/knadh/koanf/maps v0.1.1 // indirect github.com/kr/pretty v0.3.1 // indirect @@ -29,6 +32,7 @@ require ( github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect github.com/stretchr/objx v0.5.2 // indirect github.com/twitchyliquid64/golang-asm v0.15.1 // indirect + github.com/valyala/bytebufferpool v1.0.0 // indirect golang.org/x/arch v0.8.0 // indirect golang.org/x/crypto v0.25.0 // indirect golang.org/x/sys v0.22.0 // indirect diff --git a/go.sum b/go.sum index 3e7586f..53db082 100644 --- a/go.sum +++ b/go.sum @@ -1,3 +1,5 @@ +github.com/andybalholm/brotli v1.1.0 h1:eLKJA0d02Lf0mVpIDgYnqXcUn0GqVmEFny3VuID1U3M= +github.com/andybalholm/brotli v1.1.0/go.mod h1:sms7XGricyQI9K10gOSf56VKKWS4oLer58Q+mhRPtnY= github.com/bwmarrin/discordgo v0.28.1 h1:gXsuo2GBO7NbR6uqmrrBDplPUx2T3nzu775q/Rd1aG4= github.com/bwmarrin/discordgo v0.28.1/go.mod h1:NJZpH+1AfhIcyQsPeuBKsUtYrRnjkyu0kIVMCHkZtRY= github.com/bytedance/sonic v1.11.9 h1:LFHENlIY/SLzDWverzdOvgMztTxcfcF+cqNsz9pK5zg= @@ -20,6 +22,8 @@ github.com/go-viper/mapstructure/v2 v2.0.0/go.mod h1:oJDH3BJKyqBA2TXFhDsKDGDTlnd github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= github.com/gorilla/websocket v1.5.3 h1:saDtZ6Pbx/0u+bgYQ3q96pZgCzfhKXGPqt7kZ72aNNg= github.com/gorilla/websocket v1.5.3/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= +github.com/klauspost/compress v1.17.9 h1:6KIumPrER1LHsvBVuDa0r5xaG0Es51mhhB9BQB2qeMA= +github.com/klauspost/compress v1.17.9/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw= github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= github.com/klauspost/cpuid/v2 v2.2.8 h1:+StwCXwm9PdpiEkPyzBXIy+M9KUb4ODm0Zarf1kS5BM= github.com/klauspost/cpuid/v2 v2.2.8/go.mod h1:Lcz8mBdAVJIBVzewtcLocK12l3Y+JytZYpaMropDUws= @@ -63,6 +67,10 @@ github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsT github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= github.com/twitchyliquid64/golang-asm v0.15.1 h1:SU5vSMR7hnwNxj24w34ZyCi/FmDZTkS4MhqMhdFk5YI= github.com/twitchyliquid64/golang-asm v0.15.1/go.mod h1:a1lVb/DtPvCB8fslRZhAngC2+aY1QWCk3Cedj/Gdt08= +github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw= +github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc= +github.com/valyala/fasthttp v1.55.0 h1:Zkefzgt6a7+bVKHnu/YaYSOPfNYNisSVBo/unVCf8k8= +github.com/valyala/fasthttp v1.55.0/go.mod h1:NkY9JtkrpPKmgwV3HTaS2HWaJss9RSIsRVfcxxoHiOM= golang.org/x/arch v0.0.0-20210923205945-b76863e36670/go.mod h1:5om86z9Hs0C8fWVUuoMHwpExlXzs5Tkyp9hOrfG7pp8= golang.org/x/arch v0.8.0 h1:3wRIsP3pM4yUptoR96otTUOXI367OS0+c9eeRi9doIc= golang.org/x/arch v0.8.0/go.mod h1:FEVrYAQjsQXMVJ1nsMoVVXPZg6p2JE2mx8psSWTDQys= diff --git a/internal/db/migrator.go b/internal/db/migrator.go index 40463c5..1cf953d 100644 --- a/internal/db/migrator.go +++ b/internal/db/migrator.go @@ -22,7 +22,7 @@ func NewMigrator( txWrapper dbUtils.TxWrapper, ) Migrator { return Migrator{ - log: log, + log: log.With("type", "migrator"), db: db, txWrapper: txWrapper, } diff --git a/internal/db/probe.go b/internal/db/probe.go new file mode 100644 index 0000000..cedca22 --- /dev/null +++ b/internal/db/probe.go @@ -0,0 +1,20 @@ +package db + +import ( + "context" + "database/sql" +) + +type Probe struct { + db *sql.DB +} + +func NewProbe(db *sql.DB) Probe { + return Probe{ + db: db, + } +} + +func (p Probe) Check(ctx context.Context) error { + return p.db.PingContext(ctx) +} diff --git a/internal/discord/probe.go b/internal/discord/probe.go new file mode 100644 index 0000000..e550e6d --- /dev/null +++ b/internal/discord/probe.go @@ -0,0 +1,24 @@ +package discord + +import ( + "context" + + "github.com/bwmarrin/discordgo" +) + +type Probe struct { + discord *discordgo.Session +} + +func NewProbe( + discord *discordgo.Session, +) Probe { + return Probe{ + discord: discord, + } +} + +func (p Probe) Check(_ context.Context) error { + _, err := p.discord.User("@me") + return err +} diff --git a/internal/pkg/healthcheck/healthcheck.go b/internal/pkg/healthcheck/healthcheck.go new file mode 100644 index 0000000..334df9a --- /dev/null +++ b/internal/pkg/healthcheck/healthcheck.go @@ -0,0 +1,151 @@ +package healthcheck + +import ( + "context" + "log" + "log/slog" + "sync" + "time" + + "github.com/valyala/fasthttp" +) + +type Healthcheck struct { + log *slog.Logger + probes map[string]Probe + timeoutDegraded time.Duration + timeoutUnhealthy time.Duration +} + +func New( + log *slog.Logger, + opts ...Option, +) *Healthcheck { + hc := &Healthcheck{ + log: log, + probes: map[string]Probe{}, + timeoutDegraded: 1 * time.Second, + timeoutUnhealthy: 10 * time.Second, + } + + for _, opt := range opts { + opt(hc) + } + + return hc +} + +func (hc *Healthcheck) Serve(address string) *fasthttp.Server { + s := &fasthttp.Server{ + Handler: func(ctx *fasthttp.RequestCtx) { + if string(ctx.Path()) != "/health" || !ctx.IsGet() { + ctx.NotFound() + return + } + + status := hc.handle(ctx) + + if status == StatusHealthy { + ctx.SetStatusCode(fasthttp.StatusOK) + } else { + ctx.SetStatusCode(fasthttp.StatusServiceUnavailable) + } + ctx.SetBodyString(status.String()) + }, + } + + go func() { + err := s.ListenAndServe(address) + if err != nil { + log.Fatalln("healthcheck server error", "error", err) + } + }() + + return s +} + +func (hc *Healthcheck) handle(ctx context.Context) Status { + nProbes := len(hc.probes) + + wg := &sync.WaitGroup{} + wg.Add(nProbes) + + statuses := make(chan Status, nProbes) + + for name, probe := range hc.probes { + hc.probeCheck( + hc.log.With("probe", name), ctx, wg, probe, statuses, + ) + } + + wg.Wait() + close(statuses) + + status := StatusHealthy + for s := range statuses { + if s <= status { + continue + } + + status = s + if status == StatusUnhealthy { + break + } + } + + return status +} + +func (hc *Healthcheck) probeCheck( + log *slog.Logger, + ctx context.Context, + wg *sync.WaitGroup, + probe Probe, + status chan Status, +) { + timedCtx, cancel := context.WithTimeout(ctx, hc.timeoutUnhealthy) + + defer func() { + if err := recover(); err != nil { + log.ErrorContext( + ctx, + "probe panicked", + "panic", err, + ) + + status <- StatusUnhealthy + } + + cancel() + wg.Done() + }() + + probeTime := time.Now() + err := probe.Check(timedCtx) + probeDuration := time.Since(probeTime) + + if err != nil || probeDuration > hc.timeoutUnhealthy { + log.ErrorContext( + ctx, + "failed to probe", + "error", err, + "duration", probeDuration.String(), + ) + + status <- StatusUnhealthy + return + } + + if probeDuration > hc.timeoutDegraded { + log.WarnContext( + ctx, + "probe is degraded", + "duration", probeDuration.String(), + ) + + status <- StatusDegraded + return + } + + status <- StatusHealthy +} diff --git a/internal/pkg/healthcheck/options.go b/internal/pkg/healthcheck/options.go new file mode 100644 index 0000000..532209e --- /dev/null +++ b/internal/pkg/healthcheck/options.go @@ -0,0 +1,39 @@ +package healthcheck + +import ( + "fmt" + "time" +) + +type Option func(hc *Healthcheck) + +func WithProbe(name string, probe Probe) Option { + return func(hc *Healthcheck) { + if _, ok := hc.probes[name]; ok { + p := fmt.Sprintf("healthcheck probe '%s' already registered", name) + panic(p) + } + + hc.probes[name] = probe + } +} + +func WithTimeoutDegraded(timeout time.Duration) Option { + return func(hc *Healthcheck) { + if timeout <= 0 { + panic("healthcheck timeout must be greater than zero") + } + + hc.timeoutDegraded = timeout + } +} + +func WithTimeoutUnhealthy(timeout time.Duration) Option { + return func(hc *Healthcheck) { + if timeout <= 0 { + panic("healthcheck timeout must be greater than zero") + } + + hc.timeoutUnhealthy = timeout + } +} diff --git a/internal/pkg/healthcheck/probe.go b/internal/pkg/healthcheck/probe.go new file mode 100644 index 0000000..12e0be6 --- /dev/null +++ b/internal/pkg/healthcheck/probe.go @@ -0,0 +1,9 @@ +package healthcheck + +import ( + "context" +) + +type Probe interface { + Check(ctx context.Context) error +} diff --git a/internal/pkg/healthcheck/status.go b/internal/pkg/healthcheck/status.go new file mode 100644 index 0000000..2732f26 --- /dev/null +++ b/internal/pkg/healthcheck/status.go @@ -0,0 +1,26 @@ +package healthcheck + +type Status int + +const ( + StatusHealthy = Status(iota) + StatusDegraded + StatusUnhealthy +) + +func (s Status) Int() int { + return int(s) +} + +func (s Status) String() string { + switch s { + case 0: + return "healthy" + case 1: + return "degraded" + case 2: + return "unhealthy" + default: + panic("invalid status") + } +}