Skip to content

Commit

Permalink
Add health check API (#41)
Browse files Browse the repository at this point in the history
* Uptick go version

* Add test stage

* Add health check for metric provider servers

* Address review comments

Signed-off-by: Abdul Qadeer <[email protected]>
  • Loading branch information
Abdul Qadeer committed Nov 2, 2021
1 parent 9589865 commit 32392ed
Show file tree
Hide file tree
Showing 9 changed files with 84 additions and 2 deletions.
3 changes: 3 additions & 0 deletions .github/workflows/ci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -20,3 +20,6 @@ jobs:

- name: Build
run: go build -o load-watcher main.go

- name: Test
run: go test ./...
2 changes: 1 addition & 1 deletion go.mod
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
module github.com/paypal/load-watcher

go 1.15
go 1.16

require (
github.com/francoispqt/gojay v1.2.13
Expand Down
11 changes: 11 additions & 0 deletions pkg/watcher/internal/metricsprovider/k8s.go
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,8 @@ package metricsprovider

import (
"context"
"fmt"
"net/http"
"os"

"github.com/paypal/load-watcher/pkg/watcher"
Expand Down Expand Up @@ -153,3 +155,12 @@ func (m metricsServerClient) FetchAllHostsMetrics(window *watcher.Window) (map[s

return metrics, nil
}

func (m metricsServerClient) Health() (int, error) {
var status int
m.metricsClientSet.RESTClient().Verb("HEAD").Do(context.Background()).StatusCode(&status)
if status != http.StatusOK {
return -1, fmt.Errorf("received response status code: %v", status)
}
return 0, nil
}
16 changes: 16 additions & 0 deletions pkg/watcher/internal/metricsprovider/prometheus.go
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@ package metricsprovider
import (
"context"
"fmt"
"net/http"
"time"

"github.com/paypal/load-watcher/pkg/watcher"
Expand Down Expand Up @@ -133,6 +134,21 @@ func (s promClient) FetchAllHostsMetrics(window *watcher.Window) (map[string][]w
return hostMetrics, anyerr
}

func (s promClient) Health() (int, error) {
req, err := http.NewRequest("HEAD", DefaultPromAddress, nil)
if err != nil {
return -1, err
}
resp, _, err := s.client.Do(context.Background(), req)
if err != nil {
return -1, err
}
if resp.StatusCode != http.StatusOK {
return -1, fmt.Errorf("received response status code: %v", resp.StatusCode)
}
return 0, nil
}

func (s promClient) buildPromQuery(host string, metric string, method string, rollup string) string {
var promQuery string

Expand Down
21 changes: 21 additions & 0 deletions pkg/watcher/internal/metricsprovider/signalfx.go
Original file line number Diff line number Diff line change
Expand Up @@ -191,6 +191,27 @@ func (s signalFxClient) FetchAllHostsMetrics(window *watcher.Window) (map[string
return metrics, nil
}

func (s signalFxClient) Health() (int, error) {
return Ping(s.client, s.signalFxAddress)
}

// Simple ping utility to a given URL
// Returns -1 if unhealthy, 0 if healthy along with error if any
func Ping(client http.Client, url string) (int, error) {
req, err := http.NewRequest("HEAD", url, nil)
if err != nil {
return -1, err
}
resp, err := client.Do(req)
if err != nil {
return -1, err
}
if resp.StatusCode != http.StatusOK {
return -1, fmt.Errorf("received response code: %v", resp.StatusCode)
}
return 0, nil
}

func addMetadata(metric *watcher.Metric, metricType string) {
metric.Operator = watcher.Average
if metricType == cpuUtilizationMetric {
Expand Down
5 changes: 4 additions & 1 deletion pkg/watcher/metricsprovider.go
Original file line number Diff line number Diff line change
Expand Up @@ -50,11 +50,14 @@ type MetricsProviderClient interface {
FetchHostMetrics(host string, window *Window) ([]Metric, error)
// Fetch metrics for all hosts
FetchAllHostsMetrics(window *Window) (map[string][]Metric, error)
// Get metric provider server health status
// Returns 0 if healthy, -1 if unhealthy along with error if any
Health() (int, error)
}

// Generic metrics provider options
type MetricsProviderOpts struct {
Name string
Address string
AuthToken string
}
}
4 changes: 4 additions & 0 deletions pkg/watcher/testserver.go
Original file line number Diff line number Diff line change
Expand Up @@ -115,3 +115,7 @@ func (t testServerClient) FetchAllHostsMetrics(window *Window) (map[string][]Met

return FifteenMinutesMetricsMap, nil
}

func (t testServerClient) Health() (int, error) {
return 0, nil
}
13 changes: 13 additions & 0 deletions pkg/watcher/watcher.go
Original file line number Diff line number Diff line change
Expand Up @@ -37,6 +37,7 @@ import (

const (
BaseUrl = "/watcher"
HealthCheckUrl = "/watcher/health"
FifteenMinutes = "15m"
TenMinutes = "10m"
FiveMinutes = "5m"
Expand Down Expand Up @@ -153,6 +154,7 @@ func (w *Watcher) StartWatching() {
}

http.HandleFunc(BaseUrl, w.handler)
http.HandleFunc(HealthCheckUrl, w.healthCheckHandler)
server := &http.Server{
Addr: ":2020",
Handler: http.DefaultServeMux,
Expand All @@ -176,6 +178,7 @@ func (w *Watcher) StartWatching() {
w.mutex.Lock()
w.isStarted = true
w.mutex.Unlock()
log.Info("Started watching metrics")
}

// StartWatching() should be called before calling this.
Expand Down Expand Up @@ -297,6 +300,16 @@ func (w *Watcher) handler(resp http.ResponseWriter, r *http.Request) {
}
}

// Simple server status handler
func (w *Watcher) healthCheckHandler(resp http.ResponseWriter, r *http.Request) {
if status, err := w.client.Health(); status != 0 {
log.Warnf("health check failed with: %v", err)
resp.WriteHeader(http.StatusServiceUnavailable)
return
}
resp.WriteHeader(http.StatusOK)
}

// Utility functions

func metricMapToWatcherMetrics(metricMap map[string][]Metric, clientName string, window Window) WatcherMetrics {
Expand Down
11 changes: 11 additions & 0 deletions pkg/watcher/watcher_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -120,6 +120,17 @@ func TestWatcherInternalServerError(t *testing.T) {
assert.Equal(t, http.StatusInternalServerError, rr.Code)
}

func TestWatcherHealthCheck(t *testing.T) {
req, err := http.NewRequest("GET", HealthCheckUrl, nil)
require.Nil(t, err)

rr := httptest.NewRecorder()
handler := http.HandlerFunc(w.handler)

handler.ServeHTTP(rr, req)
require.Equal(t, http.StatusOK, rr.Code)
}

func TestMain(m *testing.M) {
client := NewTestMetricsServerClient()
w = NewWatcher(client)
Expand Down

0 comments on commit 32392ed

Please sign in to comment.