Skip to content

Commit

Permalink
Health status updating is now on by default
Browse files Browse the repository at this point in the history
  • Loading branch information
mtharp committed Jul 3, 2018
1 parent 74a5139 commit c8a81ed
Show file tree
Hide file tree
Showing 6 changed files with 31 additions and 37 deletions.
6 changes: 3 additions & 3 deletions cmdline/servecmd/servecmd.go
Original file line number Diff line number Diff line change
Expand Up @@ -36,11 +36,11 @@ var ServeCmd = &cobra.Command{
RunE: serveCmd,
}

var argForce, argTest bool
var argTest bool

func init() {
shared.RootCmd.AddCommand(ServeCmd)
ServeCmd.Flags().BoolVarP(&argForce, "force", "f", false, "Start even if the initial health check fails")
ServeCmd.Flags().BoolP("force", "f", false, "(ignored)")
ServeCmd.Flags().BoolVarP(&argTest, "test", "t", false, "Test configuration and exit")
}

Expand All @@ -63,7 +63,7 @@ func MakeServer() (*daemon.Daemon, error) {
if shared.CurrentConfig.Server.Listen == "" {
shared.CurrentConfig.Server.Listen = ":6300"
}
return daemon.New(shared.CurrentConfig, argForce, argTest)
return daemon.New(shared.CurrentConfig, argTest)
}

func listenDebug() error {
Expand Down
17 changes: 2 additions & 15 deletions cmdline/workercmd/handler.go
Original file line number Diff line number Diff line change
Expand Up @@ -38,11 +38,6 @@ import (
"github.com/sassoftware/relic/token"
)

const (
defaultInterval = 60 * time.Second
defaultTimeout = 30 * time.Second
)

// an arbitarily-chosen set of error codes that indicate that the token session
// is busted and that the worker should exit and start over
var fatalErrors = map[pkcs11Error]bool{
Expand All @@ -59,16 +54,8 @@ var fatalErrors = map[pkcs11Error]bool{
}

func (h *handler) healthCheck() {
interval := defaultInterval
timeout := defaultTimeout
if shared.CurrentConfig.Server != nil {
if shared.CurrentConfig.Server.TokenCheckInterval != 0 {
interval = time.Duration(shared.CurrentConfig.Server.TokenCheckInterval) * time.Second
}
if shared.CurrentConfig.Server.TokenCheckTimeout != 0 {
timeout = time.Duration(shared.CurrentConfig.Server.TokenCheckTimeout) * time.Second
}
}
interval := time.Duration(shared.CurrentConfig.Server.TokenCheckInterval) * time.Second
timeout := time.Duration(shared.CurrentConfig.Server.TokenCheckTimeout) * time.Second
ppid := os.Getppid()
tick := time.NewTicker(interval)
tmt := time.NewTimer(timeout)
Expand Down
11 changes: 11 additions & 0 deletions config/config.go
Original file line number Diff line number Diff line change
Expand Up @@ -200,6 +200,17 @@ func (config *Config) Normalize(path string) error {
keyConf.token = config.Tokens[keyConf.Token]
}
}
if s := config.Server; s != nil {
if s.TokenCheckInterval == 0 {
s.TokenCheckInterval = 60
}
if s.TokenCheckTimeout == 0 {
s.TokenCheckTimeout = 60
}
if s.TokenCheckFailures == 0 {
s.TokenCheckFailures = 3
}
}
return nil
}

Expand Down
4 changes: 2 additions & 2 deletions server/daemon/daemon.go
Original file line number Diff line number Diff line change
Expand Up @@ -79,8 +79,8 @@ func getListener(laddr string, tconf *tls.Config) (net.Listener, error) {
return listener, err
}

func New(config *config.Config, force, test bool) (*Daemon, error) {
srv, err := server.New(config, force)
func New(config *config.Config, test bool) (*Daemon, error) {
srv, err := server.New(config)
if err != nil {
return nil, err
}
Expand Down
4 changes: 2 additions & 2 deletions server/server.go
Original file line number Diff line number Diff line change
Expand Up @@ -214,7 +214,7 @@ func (s *Server) ReopenLogger() error {
return nil
}

func New(config *config.Config, force bool) (*Server, error) {
func New(config *config.Config) (*Server, error) {
closed := make(chan bool)
s := &Server{
Config: config,
Expand All @@ -236,7 +236,7 @@ func New(config *config.Config, force bool) (*Server, error) {
}
s.tokens[name] = tok
}
if err := s.startHealthCheck(force); err != nil {
if err := s.startHealthCheck(); err != nil {
return nil, err
}
return s, nil
Expand Down
26 changes: 11 additions & 15 deletions server/view_health.go
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,6 @@ package server

import (
"context"
"errors"
"net/http"
"sync"
"time"
Expand All @@ -32,25 +31,25 @@ var (
healthMu sync.Mutex
)

func (s *Server) startHealthCheck(force bool) error {
if s.Config.Server.TokenCheckInterval > 0 {
healthStatus = s.Config.Server.TokenCheckFailures
if !s.healthCheck() && !force {
return errors.New("health check failed")
}
go s.healthCheckLoop()
}
func (s *Server) healthCheckInterval() time.Duration {
return time.Second * time.Duration(s.Config.Server.TokenCheckInterval)
}

func (s *Server) startHealthCheck() error {
healthStatus = s.Config.Server.TokenCheckFailures
go s.healthCheckLoop()
return nil
}

func (s *Server) healthCheckLoop() {
t := time.NewTimer(time.Second * time.Duration(s.Config.Server.TokenCheckInterval))
interval := s.healthCheckInterval()
t := time.NewTimer(0)
defer t.Stop()
for {
select {
case <-t.C:
s.healthCheck()
t.Reset(time.Second * time.Duration(s.Config.Server.TokenCheckInterval))
t.Reset(time.Second * time.Duration(interval))
case <-s.Closed:
break
}
Expand Down Expand Up @@ -106,12 +105,9 @@ func (s *Server) Healthy(request *http.Request) bool {
if s.Config.Server.Disabled {
return false
}
if s.Config.Server.TokenCheckInterval <= 0 {
return true
}
healthMu.Lock()
defer healthMu.Unlock()
if time.Since(healthLastPing) > 3*time.Second*time.Duration(s.Config.Server.TokenCheckInterval) {
if time.Since(healthLastPing) > 3*s.healthCheckInterval() {
if request != nil {
s.Logr(request, "error: health check AWOL for %d seconds", time.Since(healthLastPing)/time.Second)
}
Expand Down

0 comments on commit c8a81ed

Please sign in to comment.