Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Filterable Health Tags #1304

Merged
merged 13 commits into from
Apr 10, 2023
Merged

Filterable Health Tags #1304

Show file tree
Hide file tree
Changes from 6 commits
Commits
Show all changes
13 commits
Select commit Hold shift + click to select a range
823b69d
add filterable tags to health checks
ceyonur Apr 6, 2023
219d02e
add unit tests
ceyonur Apr 6, 2023
931cef3
Merge branch 'dev' into subnet-healths
ceyonur Apr 6, 2023
008dc14
review fixes
ceyonur Apr 7, 2023
4f118e3
Merge branch 'subnet-healths' of github.com:ava-labs/avalanchego into…
ceyonur Apr 7, 2023
86afdcc
Merge branch 'dev' into subnet-healths
ceyonur Apr 7, 2023
4ad7ab4
Update chains/manager.go
ceyonur Apr 10, 2023
3921183
use require eventually
ceyonur Apr 10, 2023
679acd2
Merge branch 'dev' into subnet-healths
ceyonur Apr 10, 2023
238a786
increase timeout
ceyonur Apr 10, 2023
02f592d
Merge branch 'subnet-healths' of github.com:ava-labs/avalanchego into…
ceyonur Apr 10, 2023
15ee77e
move init requirement
ceyonur Apr 10, 2023
63c3dcb
use empty slice
ceyonur Apr 10, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
35 changes: 18 additions & 17 deletions api/health/client.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,11 +17,11 @@ var _ Client = (*client)(nil)
// AwaitHealthy, and AwaitAlive.
type Client interface {
// Readiness returns if the node has finished initialization
Readiness(context.Context, ...rpc.Option) (*APIReply, error)
Readiness(ctx context.Context, tags []string, options ...rpc.Option) (*APIReply, error)
// Health returns a summation of the health of the node
Health(context.Context, ...rpc.Option) (*APIReply, error)
Health(ctx context.Context, tags []string, options ...rpc.Option) (*APIReply, error)
// Liveness returns if the node is in need of a restart
Liveness(context.Context, ...rpc.Option) (*APIReply, error)
Liveness(ctx context.Context, tags []string, options ...rpc.Option) (*APIReply, error)
StephenButtolph marked this conversation as resolved.
Show resolved Hide resolved
}

// Client implementation for Avalanche Health API Endpoint
Expand All @@ -36,53 +36,54 @@ func NewClient(uri string) Client {
)}
}

func (c *client) Readiness(ctx context.Context, options ...rpc.Option) (*APIReply, error) {
func (c *client) Readiness(ctx context.Context, tags []string, options ...rpc.Option) (*APIReply, error) {
res := &APIReply{}
err := c.requester.SendRequest(ctx, "health.readiness", struct{}{}, res, options...)
err := c.requester.SendRequest(ctx, "health.readiness", &APIArgs{Tags: tags}, res, options...)
return res, err
}

func (c *client) Health(ctx context.Context, options ...rpc.Option) (*APIReply, error) {
func (c *client) Health(ctx context.Context, tags []string, options ...rpc.Option) (*APIReply, error) {
res := &APIReply{}
err := c.requester.SendRequest(ctx, "health.health", struct{}{}, res, options...)
err := c.requester.SendRequest(ctx, "health.health", &APIArgs{Tags: tags}, res, options...)
return res, err
}

func (c *client) Liveness(ctx context.Context, options ...rpc.Option) (*APIReply, error) {
func (c *client) Liveness(ctx context.Context, tags []string, options ...rpc.Option) (*APIReply, error) {
res := &APIReply{}
err := c.requester.SendRequest(ctx, "health.liveness", struct{}{}, res, options...)
err := c.requester.SendRequest(ctx, "health.liveness", &APIArgs{Tags: tags}, res, options...)
return res, err
}

// AwaitReady polls the node every [freq] until the node reports ready.
// Only returns an error if [ctx] returns an error.
func AwaitReady(ctx context.Context, c Client, freq time.Duration, options ...rpc.Option) (bool, error) {
return await(ctx, freq, c.Readiness, options...)
func AwaitReady(ctx context.Context, c Client, freq time.Duration, tags []string, options ...rpc.Option) (bool, error) {
return await(ctx, freq, c.Readiness, tags, options...)
}

// AwaitHealthy polls the node every [freq] until the node reports healthy.
// Only returns an error if [ctx] returns an error.
func AwaitHealthy(ctx context.Context, c Client, freq time.Duration, options ...rpc.Option) (bool, error) {
return await(ctx, freq, c.Health, options...)
func AwaitHealthy(ctx context.Context, c Client, freq time.Duration, tags []string, options ...rpc.Option) (bool, error) {
return await(ctx, freq, c.Health, tags, options...)
}

// AwaitAlive polls the node every [freq] until the node reports liveness.
// Only returns an error if [ctx] returns an error.
func AwaitAlive(ctx context.Context, c Client, freq time.Duration, options ...rpc.Option) (bool, error) {
return await(ctx, freq, c.Liveness, options...)
func AwaitAlive(ctx context.Context, c Client, freq time.Duration, tags []string, options ...rpc.Option) (bool, error) {
return await(ctx, freq, c.Liveness, tags, options...)
}

func await(
ctx context.Context,
freq time.Duration,
check func(context.Context, ...rpc.Option) (*APIReply, error),
check func(ctx context.Context, tags []string, options ...rpc.Option) (*APIReply, error),
tags []string,
options ...rpc.Option,
) (bool, error) {
ticker := time.NewTicker(freq)
defer ticker.Stop()

for {
res, err := check(ctx, options...)
res, err := check(ctx, tags, options...)
if err == nil && res.Healthy {
return true, nil
}
Expand Down
18 changes: 9 additions & 9 deletions api/health/client_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -48,34 +48,34 @@ func TestClient(t *testing.T) {
}

{
readiness, err := c.Readiness(context.Background())
readiness, err := c.Readiness(context.Background(), nil)
require.NoError(err)
require.True(readiness.Healthy)
}

{
health, err := c.Health(context.Background())
health, err := c.Health(context.Background(), nil)
require.NoError(err)
require.True(health.Healthy)
}

{
liveness, err := c.Liveness(context.Background())
liveness, err := c.Liveness(context.Background(), nil)
require.NoError(err)
require.True(liveness.Healthy)
}

{
ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second)
healthy, err := AwaitHealthy(ctx, c, time.Second)
healthy, err := AwaitHealthy(ctx, c, time.Second, nil)
cancel()
require.NoError(err)
require.True(healthy)
}

{
ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second)
healthy, err := AwaitReady(ctx, c, time.Second)
healthy, err := AwaitReady(ctx, c, time.Second, nil)
cancel()
require.NoError(err)
require.True(healthy)
Expand All @@ -85,15 +85,15 @@ func TestClient(t *testing.T) {

{
ctx, cancel := context.WithTimeout(context.Background(), 20*time.Microsecond)
healthy, err := AwaitHealthy(ctx, c, time.Microsecond)
healthy, err := AwaitHealthy(ctx, c, time.Microsecond, nil)
cancel()
require.Error(err)
require.False(healthy)
}

{
ctx, cancel := context.WithTimeout(context.Background(), 20*time.Microsecond)
healthy, err := AwaitReady(ctx, c, time.Microsecond)
healthy, err := AwaitReady(ctx, c, time.Microsecond, nil)
cancel()
require.Error(err)
require.False(healthy)
Expand All @@ -104,14 +104,14 @@ func TestClient(t *testing.T) {
}

{
healthy, err := AwaitHealthy(context.Background(), c, time.Microsecond)
healthy, err := AwaitHealthy(context.Background(), c, time.Microsecond, nil)
require.NoError(err)
require.True(healthy)
}

mc.reply.Healthy = false
{
healthy, err := AwaitReady(context.Background(), c, time.Microsecond)
healthy, err := AwaitReady(context.Background(), c, time.Microsecond, nil)
require.NoError(err)
require.True(healthy)
}
Expand Down
5 changes: 3 additions & 2 deletions api/health/handler.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -47,12 +47,13 @@ func NewGetAndPostHandler(log logging.Logger, reporter Reporter) (http.Handler,

// NewGetHandler return a health handler that supports GET requests reporting
// the result of the provided [reporter].
func NewGetHandler(reporter func() (map[string]Result, bool)) http.Handler {
func NewGetHandler(reporter func(tags ...string) (map[string]Result, bool)) http.Handler {
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
// Make sure the content type is set before writing the header.
w.Header().Set("Content-Type", "application/json")

checks, healthy := reporter()
tags := r.URL.Query()["tag"]
checks, healthy := reporter(tags...)
if !healthy {
// If a health check has failed, we should return a 503.
w.WriteHeader(http.StatusServiceUnavailable)
Expand Down
42 changes: 24 additions & 18 deletions api/health/health.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,12 @@ import (
"github.com/ava-labs/avalanchego/utils/logging"
)

// GlobalTag is the tag that is returned for all health check results,
// regardless of the tags passed to the Reporter.
// Registering a health check with this tag will ensure that it is always
// included in the results.
const GlobalTag = "global"

var _ Health = (*health)(nil)

// Health defines the full health service interface for registering, reporting
Expand All @@ -28,16 +34,16 @@ type Health interface {

// Registerer defines how to register new components to check the health of.
type Registerer interface {
RegisterReadinessCheck(name string, checker Checker) error
RegisterHealthCheck(name string, checker Checker) error
RegisterLivenessCheck(name string, checker Checker) error
RegisterReadinessCheck(name string, checker Checker, tags ...string) error
RegisterHealthCheck(name string, checker Checker, tags ...string) error
RegisterLivenessCheck(name string, checker Checker, tags ...string) error
}

// Reporter returns the current health status.
type Reporter interface {
Readiness() (map[string]Result, bool)
Health() (map[string]Result, bool)
Liveness() (map[string]Result, bool)
Readiness(tags ...string) (map[string]Result, bool)
Health(tags ...string) (map[string]Result, bool)
Liveness(tags ...string) (map[string]Result, bool)
}

type health struct {
Expand Down Expand Up @@ -67,20 +73,20 @@ func New(log logging.Logger, registerer prometheus.Registerer) (Health, error) {
}, err
}

func (h *health) RegisterReadinessCheck(name string, checker Checker) error {
return h.readiness.RegisterMonotonicCheck(name, checker)
func (h *health) RegisterReadinessCheck(name string, checker Checker, tags ...string) error {
return h.readiness.RegisterMonotonicCheck(name, checker, tags...)
}

func (h *health) RegisterHealthCheck(name string, checker Checker) error {
return h.health.RegisterCheck(name, checker)
func (h *health) RegisterHealthCheck(name string, checker Checker, tags ...string) error {
return h.health.RegisterCheck(name, checker, tags...)
}

func (h *health) RegisterLivenessCheck(name string, checker Checker) error {
return h.liveness.RegisterCheck(name, checker)
func (h *health) RegisterLivenessCheck(name string, checker Checker, tags ...string) error {
return h.liveness.RegisterCheck(name, checker, tags...)
}

func (h *health) Readiness() (map[string]Result, bool) {
results, healthy := h.readiness.Results()
func (h *health) Readiness(tags ...string) (map[string]Result, bool) {
results, healthy := h.readiness.Results(tags...)
if !healthy {
h.log.Warn("failing readiness check",
zap.Reflect("reason", results),
Expand All @@ -89,8 +95,8 @@ func (h *health) Readiness() (map[string]Result, bool) {
return results, healthy
}

func (h *health) Health() (map[string]Result, bool) {
results, healthy := h.health.Results()
func (h *health) Health(tags ...string) (map[string]Result, bool) {
results, healthy := h.health.Results(tags...)
if !healthy {
h.log.Warn("failing health check",
zap.Reflect("reason", results),
Expand All @@ -99,8 +105,8 @@ func (h *health) Health() (map[string]Result, bool) {
return results, healthy
}

func (h *health) Liveness() (map[string]Result, bool) {
results, healthy := h.liveness.Results()
func (h *health) Liveness(tags ...string) (map[string]Result, bool) {
results, healthy := h.liveness.Results(tags...)
if !healthy {
h.log.Warn("failing liveness check",
zap.Reflect("reason", results),
Expand Down
Loading