Skip to content

Commit

Permalink
Allow min-ready to be set to zero
Browse files Browse the repository at this point in the history
Also, return 503 is min ready exceeds total number of instances.
  • Loading branch information
enocom committed Oct 26, 2022
1 parent d5d4279 commit d15373a
Show file tree
Hide file tree
Showing 2 changed files with 54 additions and 33 deletions.
35 changes: 22 additions & 13 deletions internal/healthcheck/healthcheck.go
Original file line number Diff line number Diff line change
Expand Up @@ -89,47 +89,56 @@ func (c *Check) HandleReadiness(w http.ResponseWriter, req *http.Request) {
return
}

var minReady int
var minReady *int
q := req.URL.Query()
if v := q.Get("min-ready"); v != "" {
n, err := strconv.Atoi(v)
if err != nil {
c.logger.Errorf("[Health Check] min-ready %q was not a valid integer", v)
w.WriteHeader(http.StatusServiceUnavailable)
w.WriteHeader(http.StatusBadRequest)
fmt.Fprintf(w, "min-query must be a valid integer, got = %q", v)
return
}
minReady = n
minReady = &n
}

n, err := c.proxy.CheckConnections(ctx)
if !ready(err, minReady, n) {
c.logger.Errorf("[Health Check] Readiness failed: %v", err)
if rErr := ready(err, minReady, n); rErr != nil {
c.logger.Errorf("[Health Check] Readiness failed: %v", rErr)
w.WriteHeader(http.StatusServiceUnavailable)
w.Write([]byte(err.Error()))
w.Write([]byte(rErr.Error()))
return
}

w.WriteHeader(http.StatusOK)
w.Write([]byte("ok"))
}

func ready(err error, minReady, total int) bool {
func ready(err error, minReady *int, total int) error {
// If err is nil, then the proxy is ready.
if err == nil {
return true
if minReady != nil && *minReady > total {
return fmt.Errorf(
"min-ready (%v) is greater than registered instances (%v)",
*minReady, total,
)
}
return nil
}
// When minReady is not configured, any error means the proxy is not ready.
if minReady == 0 {
return false
if minReady == nil {
return err
}
mErr, ok := err.(proxy.MultiErr)
if !ok {
return false
return err
}
notReady := len(mErr)
areReady := total - notReady
return areReady >= minReady

if areReady < *minReady {
return err
}
return nil
}

// HandleLiveness indicates the process is up and responding to HTTP requests.
Expand Down
52 changes: 32 additions & 20 deletions internal/healthcheck/healthcheck_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -263,50 +263,62 @@ func TestReadinessWithMinReady(t *testing.T) {
desc string
minReady string
wantStatus int
dialer cloudsql.Dialer
}{
{
desc: "when min ready is not configured",
minReady: "0",
wantStatus: http.StatusServiceUnavailable,
desc: "when min ready is zero",
minReady: "0",
// Required 0 to be ready, so status OK
// even if all checks fail.
wantStatus: http.StatusOK,
dialer: &errorDialer{},
},
{
desc: "when only one instance must be ready",
minReady: "1",
wantStatus: http.StatusOK,
dialer: &flakeyDialer{}, // fails on first call, succeeds on second
},
{
desc: "when all instances must be ready",
minReady: "2",
wantStatus: http.StatusServiceUnavailable,
dialer: &errorDialer{},
},
{
desc: "when min ready is greater than the number of instances",
minReady: "3",
wantStatus: http.StatusServiceUnavailable,
dialer: &fakeDialer{},
},
{
desc: "when min ready is bogus",
minReady: "bogus",
wantStatus: http.StatusServiceUnavailable,
wantStatus: http.StatusBadRequest,
dialer: &fakeDialer{},
},
{
desc: "when min ready is not set",
minReady: "",
wantStatus: http.StatusServiceUnavailable,
wantStatus: http.StatusOK,
dialer: &fakeDialer{},
},
}
p := newProxyWithParams(t, 0,
// for every two calls, flaky dialer fails for the first, succeeds for
// the second
&flakeyDialer{},
[]proxy.InstanceConnConfig{
{Name: "p:r:instance-1"},
{Name: "p:r:instance-2"},
},
)
defer func() {
if err := p.Close(); err != nil {
t.Logf("failed to close proxy client: %v", err)
}
}()

for _, tc := range tcs {
t.Run(tc.desc, func(t *testing.T) {
p := newProxyWithParams(t, 0,
tc.dialer,
[]proxy.InstanceConnConfig{
{Name: "p:r:instance-1"},
{Name: "p:r:instance-2"},
},
)
defer func() {
if err := p.Close(); err != nil {
t.Logf("failed to close proxy client: %v", err)
}
}()

check := healthcheck.NewCheck(p, logger)
check.NotifyStarted()
u, err := url.Parse(fmt.Sprintf("/readiness?min-ready=%s", tc.minReady))
Expand Down

0 comments on commit d15373a

Please sign in to comment.