Skip to content

Commit

Permalink
Add a way to mark http requests as failed
Browse files Browse the repository at this point in the history
This is done through running a callback on every request before emitting
the metrics. Currently only a built-in metric looking at good statuses
is possible, but a possibility for future JS based callbacks is left
open.

The implementation specifically makes it hard to figure out anything
about the returned callback from JS and tries not to change any other
code so it makes it easier for future implementation, but instead tries
to do the bare minimum without imposing any limitations on the future
work.

Additionally because it turned out to be easy, setting the callback to
null will make the http library to neither tag requests with `expected_response`
nor emit the new `http_req_failed` metric, essentially giving users a way to
go back to the previous behaviour.

part of #1828
  • Loading branch information
mstoykov committed Mar 2, 2021
1 parent 7e81f66 commit d86e24a
Show file tree
Hide file tree
Showing 21 changed files with 1,200 additions and 95 deletions.
14 changes: 14 additions & 0 deletions core/engine.go
Original file line number Diff line number Diff line change
Expand Up @@ -108,6 +108,20 @@ func NewEngine(
e.submetrics[parent] = append(e.submetrics[parent], sm)
}

// TODO: refactor this out of here when https://github.com/loadimpact/k6/issues/1832 lands and
// there is a better way to enable a metric with tag
if opts.SystemTags.Has(stats.TagExpectedResponse) {
for _, name := range []string{
"http_req_duration{expected_response:true}",
} {
if _, ok := e.thresholds[name]; ok {
continue
}
parent, sm := stats.NewSubmetric(name)
e.submetrics[parent] = append(e.submetrics[parent], sm)
}
}

return e, nil
}

Expand Down
13 changes: 7 additions & 6 deletions core/local/local_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -329,12 +329,13 @@ func TestExecutionSchedulerSystemTags(t *testing.T) {
}()

expCommonTrailTags := stats.IntoSampleTags(&map[string]string{
"group": "",
"method": "GET",
"name": sr("HTTPBIN_IP_URL/"),
"url": sr("HTTPBIN_IP_URL/"),
"proto": "HTTP/1.1",
"status": "200",
"group": "",
"method": "GET",
"name": sr("HTTPBIN_IP_URL/"),
"url": sr("HTTPBIN_IP_URL/"),
"proto": "HTTP/1.1",
"status": "200",
"expected_response": "true",
})
expTrailPVUTagsRaw := expCommonTrailTags.CloneTags()
expTrailPVUTagsRaw["scenario"] = "per_vu_test"
Expand Down
4 changes: 4 additions & 0 deletions js/modules/k6/http/http.go
Original file line number Diff line number Diff line change
Expand Up @@ -73,6 +73,8 @@ func (g *GlobalHTTP) NewModuleInstancePerVU() interface{} { // this here needs t
OCSP_REASON_REMOVE_FROM_CRL: netext.OCSP_REASON_REMOVE_FROM_CRL,
OCSP_REASON_PRIVILEGE_WITHDRAWN: netext.OCSP_REASON_PRIVILEGE_WITHDRAWN,
OCSP_REASON_AA_COMPROMISE: netext.OCSP_REASON_AA_COMPROMISE,

responseCallback: defaultExpectedStatuses.match,
}
}

Expand All @@ -97,6 +99,8 @@ type HTTP struct {
OCSP_REASON_REMOVE_FROM_CRL string `js:"OCSP_REASON_REMOVE_FROM_CRL"`
OCSP_REASON_PRIVILEGE_WITHDRAWN string `js:"OCSP_REASON_PRIVILEGE_WITHDRAWN"`
OCSP_REASON_AA_COMPROMISE string `js:"OCSP_REASON_AA_COMPROMISE"`

responseCallback func(int) bool
}

func (*HTTP) XCookieJar(ctx *context.Context) *HTTPCookieJar {
Expand Down
21 changes: 16 additions & 5 deletions js/modules/k6/http/request.go
Original file line number Diff line number Diff line change
Expand Up @@ -134,12 +134,14 @@ func (h *HTTP) parseRequest(
URL: reqURL.GetURL(),
Header: make(http.Header),
},
Timeout: 60 * time.Second,
Throw: state.Options.Throw.Bool,
Redirects: state.Options.MaxRedirects,
Cookies: make(map[string]*httpext.HTTPRequestCookie),
Tags: make(map[string]string),
Timeout: 60 * time.Second,
Throw: state.Options.Throw.Bool,
Redirects: state.Options.MaxRedirects,
Cookies: make(map[string]*httpext.HTTPRequestCookie),
Tags: make(map[string]string),
ResponseCallback: h.responseCallback,
}

if state.Options.DiscardResponseBodies.Bool {
result.ResponseType = httpext.ResponseTypeNone
} else {
Expand Down Expand Up @@ -349,6 +351,15 @@ func (h *HTTP) parseRequest(
return nil, err
}
result.ResponseType = responseType
case "responseCallback":
v := params.Get(k).Export()
if v == nil {
result.ResponseCallback = nil
} else if c, ok := v.(*expectedStatuses); ok {
result.ResponseCallback = c.match
} else {
return nil, fmt.Errorf("unsupported responseCallback")
}
}
}
}
Expand Down
52 changes: 39 additions & 13 deletions js/modules/k6/http/request_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -81,6 +81,7 @@ func TestRunES6String(t *testing.T) {
})
}

// TODO replace this with the Single version
func assertRequestMetricsEmitted(t *testing.T, sampleContainers []stats.SampleContainer, method, url, name string, status int, group string) {
if name == "" {
name = url
Expand Down Expand Up @@ -130,6 +131,29 @@ func assertRequestMetricsEmitted(t *testing.T, sampleContainers []stats.SampleCo
assert.True(t, seenReceiving, "url %s didn't emit Receiving", url)
}

func assertRequestMetricsEmittedSingle(t *testing.T, sampleContainer stats.SampleContainer, expectedTags map[string]string, metrics []*stats.Metric, callback func(sample stats.Sample)) {
t.Helper()

metricMap := make(map[string]bool, len(metrics))
for _, m := range metrics {
metricMap[m.Name] = false
}
for _, sample := range sampleContainer.GetSamples() {
tags := sample.Tags.CloneTags()
v, ok := metricMap[sample.Metric.Name]
assert.True(t, ok, "unexpected metric %s", sample.Metric.Name)
assert.False(t, v, "second metric %s", sample.Metric.Name)
metricMap[sample.Metric.Name] = true
assert.EqualValues(t, expectedTags, tags, "%s", tags)
if callback != nil {
callback(sample)
}
}
for k, v := range metricMap {
assert.True(t, v, "didn't emit %s", k)
}
}

func newRuntime(
t testing.TB,
) (*httpmultibin.HTTPMultiBin, *lib.State, chan stats.SampleContainer, *goja.Runtime, *context.Context) {
Expand Down Expand Up @@ -1945,26 +1969,28 @@ func TestRedirectMetricTags(t *testing.T) {

checkTags := func(sc stats.SampleContainer, expTags map[string]string) {
allSamples := sc.GetSamples()
assert.Len(t, allSamples, 8)
assert.Len(t, allSamples, 9)
for _, s := range allSamples {
assert.Equal(t, expTags, s.Tags.CloneTags())
}
}
expPOSTtags := map[string]string{
"group": "",
"method": "POST",
"url": sr("HTTPBIN_URL/redirect/post"),
"name": sr("HTTPBIN_URL/redirect/post"),
"status": "301",
"proto": "HTTP/1.1",
"group": "",
"method": "POST",
"url": sr("HTTPBIN_URL/redirect/post"),
"name": sr("HTTPBIN_URL/redirect/post"),
"status": "301",
"proto": "HTTP/1.1",
"expected_response": "true",
}
expGETtags := map[string]string{
"group": "",
"method": "GET",
"url": sr("HTTPBIN_URL/get"),
"name": sr("HTTPBIN_URL/get"),
"status": "200",
"proto": "HTTP/1.1",
"group": "",
"method": "GET",
"url": sr("HTTPBIN_URL/get"),
"name": sr("HTTPBIN_URL/get"),
"status": "200",
"proto": "HTTP/1.1",
"expected_response": "true",
}
checkTags(<-samples, expPOSTtags)
checkTags(<-samples, expGETtags)
Expand Down
117 changes: 117 additions & 0 deletions js/modules/k6/http/response_callback.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,117 @@
/*
*
* k6 - a next-generation load testing tool
* Copyright (C) 2021 Load Impact
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*
*/

package http

import (
"context"
"errors"
"fmt"

"github.com/dop251/goja"
"github.com/loadimpact/k6/js/common"
)

//nolint:gochecknoglobals
var defaultExpectedStatuses = expectedStatuses{
minmax: [][2]int{{200, 399}},
}

// expectedStatuses is specifically totally unexported so it can't be used for anything else but
// SetResponseCallback and nothing can be done from the js side to modify it or make an instance of
// it except using ExpectedStatuses
type expectedStatuses struct {
minmax [][2]int
exact []int
}

func (e expectedStatuses) match(status int) bool {
for _, v := range e.exact {
if v == status {
return true
}
}

for _, v := range e.minmax {
if v[0] <= status && status <= v[1] {
return true
}
}
return false
}

// ExpectedStatuses returns expectedStatuses object based on the provided arguments.
// The arguments must be either integers or object of `{min: <integer>, max: <integer>}`
// kind. The "integer"ness is checked by the Number.isInteger.
func (*HTTP) ExpectedStatuses(ctx context.Context, args ...goja.Value) *expectedStatuses { //nolint: golint
rt := common.GetRuntime(ctx)

if len(args) == 0 {
common.Throw(rt, errors.New("no arguments"))
}
var result expectedStatuses

jsIsInt, _ := goja.AssertFunction(rt.GlobalObject().Get("Number").ToObject(rt).Get("isInteger"))
isInt := func(a goja.Value) bool {
v, err := jsIsInt(goja.Undefined(), a)
return err == nil && v.ToBoolean()
}

errMsg := "argument number %d to expectedStatuses was neither an integer nor an object like {min:100, max:329}"
for i, arg := range args {
o := arg.ToObject(rt)
if o == nil {
common.Throw(rt, fmt.Errorf(errMsg, i+1))
}

if isInt(arg) {
result.exact = append(result.exact, int(o.ToInteger()))
} else {
min := o.Get("min")
max := o.Get("max")
if min == nil || max == nil {
common.Throw(rt, fmt.Errorf(errMsg, i+1))
}
if !(isInt(min) && isInt(max)) {
common.Throw(rt, fmt.Errorf("both min and max need to be integers for argument number %d", i+1))
}

result.minmax = append(result.minmax, [2]int{int(min.ToInteger()), int(max.ToInteger())})
}
}
return &result
}

// SetResponseCallback sets the responseCallback to the value provided. Supported values are
// expectedStatuses object or a `null` which means that metrics shouldn't be tagged as failed and
// `http_req_failed` should not be emitted - the behaviour previous to this
func (h *HTTP) SetResponseCallback(ctx context.Context, val goja.Value) {
if val != nil && !goja.IsNull(val) {
// This is done this way as ExportTo exports functions to empty structs without an error
if es, ok := val.Export().(*expectedStatuses); ok {
h.responseCallback = es.match
} else {
//nolint:golint
common.Throw(common.GetRuntime(ctx), fmt.Errorf("unsupported argument, expected http.expectedStatuses"))
}
} else {
h.responseCallback = nil
}
}
Loading

0 comments on commit d86e24a

Please sign in to comment.