From 8b05abd231c16eefef1bb8b649c22c8aea35623b Mon Sep 17 00:00:00 2001 From: Mihail Stoykov Date: Fri, 26 Feb 2021 17:17:47 +0200 Subject: [PATCH] Add a way to mark http requests as failed This is done through running a callback on every request before emitting the metrics. Currently only a built-in metric looking at good statuses is possible, but a possibility for future JS based callbacks is left open. The implementation specifically makes it hard to figure out anything about the returned callback from JS and tries not to change any other code so it makes it easier for future implementation, but instead tries to do the bare minimum without imposing any limitations on the future work. Additionally because it turned out to be easy, setting the callback to null will make the http library to neither tag requests with `expected_response` nor emit the new `http_req_failed` metric, essentially giving users a way to go back to the previous behaviour. part of #1828 --- core/engine.go | 14 ++++ core/local/local_test.go | 13 ++-- js/modules/k6/http/http.go | 4 ++ js/modules/k6/http/request.go | 21 ++++-- js/modules/k6/http/request_test.go | 52 ++++++++++---- js/modules/k6/http/response_test.go | 1 + js/runner.go | 3 +- lib/metrics/metrics.go | 3 +- lib/netext/httpext/request.go | 48 +++++++++---- lib/netext/httpext/tracer.go | 2 + lib/netext/httpext/transport.go | 49 ++++++++++--- stats/cloud/cloud_easyjson.go | 66 ++++++++++++++++-- stats/cloud/data.go | 46 +++++++++++++ stats/cloud/data_test.go | 102 +++++++++++++++++++++++++++- stats/system_tag.go | 3 +- stats/system_tag_set_gen.go | 48 ++++++------- stats/units.go | 8 +++ 17 files changed, 404 insertions(+), 79 deletions(-) diff --git a/core/engine.go b/core/engine.go index daf5bed35052..1c5273572819 100644 --- a/core/engine.go +++ b/core/engine.go @@ -108,6 +108,20 @@ func NewEngine( e.submetrics[parent] = append(e.submetrics[parent], sm) } + // TODO: refactor this out of here when https://github.com/loadimpact/k6/issues/1832 lands and + // there is a better way to enable a metric with tag + if opts.SystemTags.Has(stats.TagExpectedResponse) { + for _, name := range []string{ + "http_req_duration{expected_response:true}", + } { + if _, ok := e.thresholds[name]; ok { + continue + } + parent, sm := stats.NewSubmetric(name) + e.submetrics[parent] = append(e.submetrics[parent], sm) + } + } + return e, nil } diff --git a/core/local/local_test.go b/core/local/local_test.go index 382dd1a0028e..c86f687b8cd5 100644 --- a/core/local/local_test.go +++ b/core/local/local_test.go @@ -329,12 +329,13 @@ func TestExecutionSchedulerSystemTags(t *testing.T) { }() expCommonTrailTags := stats.IntoSampleTags(&map[string]string{ - "group": "", - "method": "GET", - "name": sr("HTTPBIN_IP_URL/"), - "url": sr("HTTPBIN_IP_URL/"), - "proto": "HTTP/1.1", - "status": "200", + "group": "", + "method": "GET", + "name": sr("HTTPBIN_IP_URL/"), + "url": sr("HTTPBIN_IP_URL/"), + "proto": "HTTP/1.1", + "status": "200", + "expected_response": "true", }) expTrailPVUTagsRaw := expCommonTrailTags.CloneTags() expTrailPVUTagsRaw["scenario"] = "per_vu_test" diff --git a/js/modules/k6/http/http.go b/js/modules/k6/http/http.go index ea6e4b0dbdb3..3118e26cbd72 100644 --- a/js/modules/k6/http/http.go +++ b/js/modules/k6/http/http.go @@ -73,6 +73,8 @@ func (g *GlobalHTTP) NewVUModule() interface{} { // this here needs to return in OCSP_REASON_REMOVE_FROM_CRL: netext.OCSP_REASON_REMOVE_FROM_CRL, OCSP_REASON_PRIVILEGE_WITHDRAWN: netext.OCSP_REASON_PRIVILEGE_WITHDRAWN, OCSP_REASON_AA_COMPROMISE: netext.OCSP_REASON_AA_COMPROMISE, + + responseCallback: defaultExpectedStatuses.match, } } @@ -97,6 +99,8 @@ type HTTP struct { OCSP_REASON_REMOVE_FROM_CRL string `js:"OCSP_REASON_REMOVE_FROM_CRL"` OCSP_REASON_PRIVILEGE_WITHDRAWN string `js:"OCSP_REASON_PRIVILEGE_WITHDRAWN"` OCSP_REASON_AA_COMPROMISE string `js:"OCSP_REASON_AA_COMPROMISE"` + + responseCallback func(int) bool } func (*HTTP) XCookieJar(ctx *context.Context) *HTTPCookieJar { diff --git a/js/modules/k6/http/request.go b/js/modules/k6/http/request.go index 91fc9a73c883..22f18ab9dace 100644 --- a/js/modules/k6/http/request.go +++ b/js/modules/k6/http/request.go @@ -134,12 +134,14 @@ func (h *HTTP) parseRequest( URL: reqURL.GetURL(), Header: make(http.Header), }, - Timeout: 60 * time.Second, - Throw: state.Options.Throw.Bool, - Redirects: state.Options.MaxRedirects, - Cookies: make(map[string]*httpext.HTTPRequestCookie), - Tags: make(map[string]string), + Timeout: 60 * time.Second, + Throw: state.Options.Throw.Bool, + Redirects: state.Options.MaxRedirects, + Cookies: make(map[string]*httpext.HTTPRequestCookie), + Tags: make(map[string]string), + ResponseCallback: h.responseCallback, } + if state.Options.DiscardResponseBodies.Bool { result.ResponseType = httpext.ResponseTypeNone } else { @@ -349,6 +351,15 @@ func (h *HTTP) parseRequest( return nil, err } result.ResponseType = responseType + case "responseCallback": + v := params.Get(k).Export() + if v == nil { + result.ResponseCallback = nil + } else if c, ok := v.(*expectedStatuses); ok { + result.ResponseCallback = c.match + } else { + return nil, fmt.Errorf("unsupported responseCallback") + } } } } diff --git a/js/modules/k6/http/request_test.go b/js/modules/k6/http/request_test.go index 1e927b13d518..12ae3d201ac0 100644 --- a/js/modules/k6/http/request_test.go +++ b/js/modules/k6/http/request_test.go @@ -81,6 +81,7 @@ func TestRunES6String(t *testing.T) { }) } +// TODO replace this with the Single version func assertRequestMetricsEmitted(t *testing.T, sampleContainers []stats.SampleContainer, method, url, name string, status int, group string) { if name == "" { name = url @@ -130,6 +131,29 @@ func assertRequestMetricsEmitted(t *testing.T, sampleContainers []stats.SampleCo assert.True(t, seenReceiving, "url %s didn't emit Receiving", url) } +func assertRequestMetricsEmittedSingle(t *testing.T, sampleContainer stats.SampleContainer, expectedTags map[string]string, metrics []*stats.Metric, callback func(sample stats.Sample)) { + t.Helper() + + metricMap := make(map[string]bool, len(metrics)) + for _, m := range metrics { + metricMap[m.Name] = false + } + for _, sample := range sampleContainer.GetSamples() { + tags := sample.Tags.CloneTags() + v, ok := metricMap[sample.Metric.Name] + assert.True(t, ok, "unexpected metric %s", sample.Metric.Name) + assert.False(t, v, "second metric %s", sample.Metric.Name) + metricMap[sample.Metric.Name] = true + assert.EqualValues(t, expectedTags, tags, "%s", tags) + if callback != nil { + callback(sample) + } + } + for k, v := range metricMap { + assert.True(t, v, "didn't emit %s", k) + } +} + func newRuntime( t testing.TB, ) (*httpmultibin.HTTPMultiBin, *lib.State, chan stats.SampleContainer, *goja.Runtime, *context.Context) { @@ -1945,26 +1969,28 @@ func TestRedirectMetricTags(t *testing.T) { checkTags := func(sc stats.SampleContainer, expTags map[string]string) { allSamples := sc.GetSamples() - assert.Len(t, allSamples, 8) + assert.Len(t, allSamples, 9) for _, s := range allSamples { assert.Equal(t, expTags, s.Tags.CloneTags()) } } expPOSTtags := map[string]string{ - "group": "", - "method": "POST", - "url": sr("HTTPBIN_URL/redirect/post"), - "name": sr("HTTPBIN_URL/redirect/post"), - "status": "301", - "proto": "HTTP/1.1", + "group": "", + "method": "POST", + "url": sr("HTTPBIN_URL/redirect/post"), + "name": sr("HTTPBIN_URL/redirect/post"), + "status": "301", + "proto": "HTTP/1.1", + "expected_response": "true", } expGETtags := map[string]string{ - "group": "", - "method": "GET", - "url": sr("HTTPBIN_URL/get"), - "name": sr("HTTPBIN_URL/get"), - "status": "200", - "proto": "HTTP/1.1", + "group": "", + "method": "GET", + "url": sr("HTTPBIN_URL/get"), + "name": sr("HTTPBIN_URL/get"), + "status": "200", + "proto": "HTTP/1.1", + "expected_response": "true", } checkTags(<-samples, expPOSTtags) checkTags(<-samples, expGETtags) diff --git a/js/modules/k6/http/response_test.go b/js/modules/k6/http/response_test.go index dbce89e0a5bd..6b346e117544 100644 --- a/js/modules/k6/http/response_test.go +++ b/js/modules/k6/http/response_test.go @@ -55,6 +55,7 @@ const testGetFormHTML = ` ` + const jsonData = `{"glossary": { "friends": [ {"first": "Dale", "last": "Murphy", "age": 44}, diff --git a/js/runner.go b/js/runner.go index 1d1afbe3a7d1..9651c3e32b9a 100644 --- a/js/runner.go +++ b/js/runner.go @@ -179,7 +179,7 @@ func (r *Runner) newVU(id int64, samplesOut chan<- stats.SampleContainer) (*VU, } tlsConfig := &tls.Config{ - InsecureSkipVerify: r.Bundle.Options.InsecureSkipTLSVerify.Bool, + InsecureSkipVerify: r.Bundle.Options.InsecureSkipTLSVerify.Bool, //nolint:gosec CipherSuites: cipherSuites, MinVersion: uint16(tlsVersions.Min), MaxVersion: uint16(tlsVersions.Max), @@ -244,6 +244,7 @@ func (r *Runner) newVU(id int64, samplesOut chan<- stats.SampleContainer) (*VU, return vu, nil } +// Setup runs the setup function if there is one and sets the setupData to the returned value func (r *Runner) Setup(ctx context.Context, out chan<- stats.SampleContainer) error { setupCtx, setupCancel := context.WithTimeout(ctx, r.getTimeoutFor(consts.SetupFn)) defer setupCancel() diff --git a/lib/metrics/metrics.go b/lib/metrics/metrics.go index cb7eda3b0b2b..a47aa156ebaf 100644 --- a/lib/metrics/metrics.go +++ b/lib/metrics/metrics.go @@ -24,7 +24,7 @@ import ( "github.com/loadimpact/k6/stats" ) -//TODO: refactor this, using non thread-safe global variables seems like a bad idea for various reasons... +// TODO: refactor this, using non thread-safe global variables seems like a bad idea for various reasons... //nolint:gochecknoglobals var ( @@ -42,6 +42,7 @@ var ( // HTTP-related. HTTPReqs = stats.New("http_reqs", stats.Counter) + HTTPReqFailed = stats.New("http_req_failed", stats.Rate) HTTPReqDuration = stats.New("http_req_duration", stats.Trend, stats.Time) HTTPReqBlocked = stats.New("http_req_blocked", stats.Trend, stats.Time) HTTPReqConnecting = stats.New("http_req_connecting", stats.Trend, stats.Time) diff --git a/lib/netext/httpext/request.go b/lib/netext/httpext/request.go index 5642f0235d57..9205ec6a49dc 100644 --- a/lib/netext/httpext/request.go +++ b/lib/netext/httpext/request.go @@ -103,18 +103,19 @@ type Request struct { // ParsedHTTPRequest a represantion of a request after it has been parsed from a user script type ParsedHTTPRequest struct { - URL *URL - Body *bytes.Buffer - Req *http.Request - Timeout time.Duration - Auth string - Throw bool - ResponseType ResponseType - Compressions []CompressionType - Redirects null.Int - ActiveJar *cookiejar.Jar - Cookies map[string]*HTTPRequestCookie - Tags map[string]string + URL *URL + Body *bytes.Buffer + Req *http.Request + Timeout time.Duration + Auth string + Throw bool + ResponseType ResponseType + ResponseCallback func(int) bool + Compressions []CompressionType + Redirects null.Int + ActiveJar *cookiejar.Jar + Cookies map[string]*HTTPRequestCookie + Tags map[string]string } // Matches non-compliant io.Closer implementations (e.g. zstd.Decoder) @@ -139,7 +140,7 @@ func (r readCloser) Close() error { } func stdCookiesToHTTPRequestCookies(cookies []*http.Cookie) map[string][]*HTTPRequestCookie { - var result = make(map[string][]*HTTPRequestCookie, len(cookies)) + result := make(map[string][]*HTTPRequestCookie, len(cookies)) for _, cookie := range cookies { result[cookie.Name] = append(result[cookie.Name], &HTTPRequestCookie{Name: cookie.Name, Value: cookie.Value}) @@ -249,7 +250,7 @@ func MakeRequest(ctx context.Context, preq *ParsedHTTPRequest) (*Response, error } } - tracerTransport := newTransport(ctx, state, tags) + tracerTransport := newTransport(ctx, state, tags, preq.ResponseCallback) var transport http.RoundTripper = tracerTransport // Combine tags with common log fields @@ -269,8 +270,25 @@ func MakeRequest(ctx context.Context, preq *ParsedHTTPRequest) (*Response, error } if preq.Auth == "digest" { + // In both digest and NTLM it is expected that the first response will be 401 + // this does mean that a non 401 response will be marked as failed/unexpected + if tracerTransport.responseCallback != nil { + originalResponseCallback := tracerTransport.responseCallback + tracerTransport.responseCallback = func(status int) bool { + tracerTransport.responseCallback = originalResponseCallback + return status == 401 + } + } transport = digestTransport{originalTransport: transport} } else if preq.Auth == "ntlm" { + if tracerTransport.responseCallback != nil { + originalResponseCallback := tracerTransport.responseCallback + tracerTransport.responseCallback = func(status int) bool { + tracerTransport.responseCallback = originalResponseCallback + // ntlm is connection-level based so we could've already authorized the connection and to now reuse it + return status == 401 || originalResponseCallback(status) + } + } transport = ntlmssp.Negotiator{RoundTripper: transport} } @@ -381,7 +399,7 @@ func MakeRequest(ctx context.Context, preq *ParsedHTTPRequest) (*Response, error // SetRequestCookies sets the cookies of the requests getting those cookies both from the jar and // from the reqCookies map. The Replace field of the HTTPRequestCookie will be taken into account func SetRequestCookies(req *http.Request, jar *cookiejar.Jar, reqCookies map[string]*HTTPRequestCookie) { - var replacedCookies = make(map[string]struct{}) + replacedCookies := make(map[string]struct{}) for key, reqCookie := range reqCookies { req.AddCookie(&http.Cookie{Name: key, Value: reqCookie.Value}) if reqCookie.Replace { diff --git a/lib/netext/httpext/tracer.go b/lib/netext/httpext/tracer.go index 60c02b548c7d..d4896d732d16 100644 --- a/lib/netext/httpext/tracer.go +++ b/lib/netext/httpext/tracer.go @@ -29,6 +29,7 @@ import ( "github.com/loadimpact/k6/lib/metrics" "github.com/loadimpact/k6/stats" + "gopkg.in/guregu/null.v3" ) // A Trail represents detailed information about an HTTP request. @@ -53,6 +54,7 @@ type Trail struct { ConnReused bool ConnRemoteAddr net.Addr + Failed null.Bool // Populated by SaveSamples() Tags *stats.SampleTags Samples []stats.Sample diff --git a/lib/netext/httpext/transport.go b/lib/netext/httpext/transport.go index bc754125f8bc..6c77648b0566 100644 --- a/lib/netext/httpext/transport.go +++ b/lib/netext/httpext/transport.go @@ -29,6 +29,7 @@ import ( "sync" "github.com/loadimpact/k6/lib" + "github.com/loadimpact/k6/lib/metrics" "github.com/loadimpact/k6/lib/netext" "github.com/loadimpact/k6/stats" ) @@ -36,9 +37,10 @@ import ( // transport is an implementation of http.RoundTripper that will measure and emit // different metrics for each roundtrip type transport struct { - ctx context.Context - state *lib.State - tags map[string]string + ctx context.Context + state *lib.State + tags map[string]string + responseCallback func(int) bool lastRequest *unfinishedRequest lastRequestLock *sync.Mutex @@ -76,17 +78,20 @@ func newTransport( ctx context.Context, state *lib.State, tags map[string]string, + responseCallback func(int) bool, ) *transport { return &transport{ - ctx: ctx, - state: state, - tags: tags, - lastRequestLock: new(sync.Mutex), + ctx: ctx, + state: state, + tags: tags, + responseCallback: responseCallback, + lastRequestLock: new(sync.Mutex), } } // Helper method to finish the tracer trail, assemble the tag values and emits // the metric samples for the supplied unfinished request. +//nolint:nestif,funlen func (t *transport) measureAndEmitMetrics(unfReq *unfinishedRequest) *finishedRequest { trail := unfReq.tracer.Done() @@ -101,7 +106,6 @@ func (t *transport) measureAndEmitMetrics(unfReq *unfinishedRequest) *finishedRe } enabledTags := t.state.Options.SystemTags - urlEnabled := enabledTags.Has(stats.TagURL) var setName bool if _, ok := tags["name"]; !ok && enabledTags.Has(stats.TagName) { @@ -164,8 +168,35 @@ func (t *transport) measureAndEmitMetrics(unfReq *unfinishedRequest) *finishedRe tags["ip"] = ip } } + var failed float64 + if t.responseCallback != nil { + var statusCode int + if unfReq.response != nil { + statusCode = unfReq.response.StatusCode + } + expected := t.responseCallback(statusCode) + if !expected { + failed = 1 + } + + if enabledTags.Has(stats.TagExpectedResponse) { + tags[stats.TagExpectedResponse.String()] = strconv.FormatBool(expected) + } + } - trail.SaveSamples(stats.IntoSampleTags(&tags)) + finalTags := stats.IntoSampleTags(&tags) + trail.SaveSamples(finalTags) + if t.responseCallback != nil { + trail.Failed.SetValid(true) + if failed == 1 { + trail.Failed.Bool = true + } + trail.Samples = append(trail.Samples, + stats.Sample{ + Metric: metrics.HTTPReqFailed, Time: trail.EndTime, Tags: finalTags, Value: failed, + }, + ) + } stats.PushIfNotDone(t.ctx, t.state.Samples, trail) return result diff --git a/stats/cloud/cloud_easyjson.go b/stats/cloud/cloud_easyjson.go index 7f412efc1fac..1a06e49c4629 100644 --- a/stats/cloud/cloud_easyjson.go +++ b/stats/cloud/cloud_easyjson.go @@ -382,6 +382,7 @@ func easyjson9def2ecdDecode(in *jlexer.Lexer, out *struct { Sending AggregatedMetric `json:"http_req_sending"` Waiting AggregatedMetric `json:"http_req_waiting"` Receiving AggregatedMetric `json:"http_req_receiving"` + Failed AggregatedRate `json:"http_req_failed,omitempty"` }) { isTopLevel := in.IsStart() if in.IsNull() { @@ -415,6 +416,8 @@ func easyjson9def2ecdDecode(in *jlexer.Lexer, out *struct { easyjson9def2ecdDecodeGithubComLoadimpactK6StatsCloud4(in, &out.Waiting) case "http_req_receiving": easyjson9def2ecdDecodeGithubComLoadimpactK6StatsCloud4(in, &out.Receiving) + case "http_req_failed": + easyjson9def2ecdDecodeGithubComLoadimpactK6StatsCloud5(in, &out.Failed) default: in.SkipRecursive() } @@ -433,6 +436,7 @@ func easyjson9def2ecdEncode(out *jwriter.Writer, in struct { Sending AggregatedMetric `json:"http_req_sending"` Waiting AggregatedMetric `json:"http_req_waiting"` Receiving AggregatedMetric `json:"http_req_receiving"` + Failed AggregatedRate `json:"http_req_failed,omitempty"` }) { out.RawByte('{') first := true @@ -472,6 +476,60 @@ func easyjson9def2ecdEncode(out *jwriter.Writer, in struct { out.RawString(prefix) easyjson9def2ecdEncodeGithubComLoadimpactK6StatsCloud4(out, in.Receiving) } + if (in.Failed).IsDefined() { + const prefix string = ",\"http_req_failed\":" + out.RawString(prefix) + easyjson9def2ecdEncodeGithubComLoadimpactK6StatsCloud5(out, in.Failed) + } + out.RawByte('}') +} +func easyjson9def2ecdDecodeGithubComLoadimpactK6StatsCloud5(in *jlexer.Lexer, out *AggregatedRate) { + isTopLevel := in.IsStart() + if in.IsNull() { + if isTopLevel { + in.Consumed() + } + in.Skip() + return + } + in.Delim('{') + for !in.IsDelim('}') { + key := in.UnsafeFieldName(false) + in.WantColon() + if in.IsNull() { + in.Skip() + in.WantComma() + continue + } + switch key { + case "count": + out.Count = float64(in.Float64()) + case "nz_count": + out.NzCount = float64(in.Float64()) + default: + in.SkipRecursive() + } + in.WantComma() + } + in.Delim('}') + if isTopLevel { + in.Consumed() + } +} +func easyjson9def2ecdEncodeGithubComLoadimpactK6StatsCloud5(out *jwriter.Writer, in AggregatedRate) { + out.RawByte('{') + first := true + _ = first + { + const prefix string = ",\"count\":" + out.RawString(prefix[1:]) + out.Float64(float64(in.Count)) + } + { + const prefix string = ",\"nz_count\":" + out.RawString(prefix) + out.Float64(float64(in.NzCount)) + } out.RawByte('}') } func easyjson9def2ecdDecodeGithubComLoadimpactK6StatsCloud4(in *jlexer.Lexer, out *AggregatedMetric) { @@ -530,7 +588,7 @@ func easyjson9def2ecdEncodeGithubComLoadimpactK6StatsCloud4(out *jwriter.Writer, } out.RawByte('}') } -func easyjson9def2ecdDecodeGithubComLoadimpactK6StatsCloud5(in *jlexer.Lexer, out *Sample) { +func easyjson9def2ecdDecodeGithubComLoadimpactK6StatsCloud6(in *jlexer.Lexer, out *Sample) { isTopLevel := in.IsStart() if in.IsNull() { if isTopLevel { @@ -571,7 +629,7 @@ func easyjson9def2ecdDecodeGithubComLoadimpactK6StatsCloud5(in *jlexer.Lexer, ou in.Consumed() } } -func easyjson9def2ecdEncodeGithubComLoadimpactK6StatsCloud5(out *jwriter.Writer, in Sample) { +func easyjson9def2ecdEncodeGithubComLoadimpactK6StatsCloud6(out *jwriter.Writer, in Sample) { out.RawByte('{') first := true _ = first @@ -601,10 +659,10 @@ func easyjson9def2ecdEncodeGithubComLoadimpactK6StatsCloud5(out *jwriter.Writer, // MarshalEasyJSON supports easyjson.Marshaler interface func (v Sample) MarshalEasyJSON(w *jwriter.Writer) { - easyjson9def2ecdEncodeGithubComLoadimpactK6StatsCloud5(w, v) + easyjson9def2ecdEncodeGithubComLoadimpactK6StatsCloud6(w, v) } // UnmarshalEasyJSON supports easyjson.Unmarshaler interface func (v *Sample) UnmarshalEasyJSON(l *jlexer.Lexer) { - easyjson9def2ecdDecodeGithubComLoadimpactK6StatsCloud5(l, v) + easyjson9def2ecdDecodeGithubComLoadimpactK6StatsCloud6(l, v) } diff --git a/stats/cloud/data.go b/stats/cloud/data.go index 0a63372128b2..b072abd5e9c3 100644 --- a/stats/cloud/data.go +++ b/stats/cloud/data.go @@ -110,6 +110,29 @@ type SampleDataMap struct { // NewSampleFromTrail just creates a ready-to-send Sample instance // directly from a httpext.Trail. func NewSampleFromTrail(trail *httpext.Trail) *Sample { + if trail.Failed.Valid { + return &Sample{ + Type: DataTypeMap, + Metric: "http_req_li_all", + Data: &SampleDataMap{ + Time: toMicroSecond(trail.GetTime()), + Tags: trail.GetTags(), + Values: map[string]float64{ + metrics.HTTPReqs.Name: 1, + metrics.HTTPReqDuration.Name: stats.D(trail.Duration), + + metrics.HTTPReqBlocked.Name: stats.D(trail.Blocked), + metrics.HTTPReqConnecting.Name: stats.D(trail.Connecting), + metrics.HTTPReqTLSHandshaking.Name: stats.D(trail.TLSHandshaking), + metrics.HTTPReqSending.Name: stats.D(trail.Sending), + metrics.HTTPReqWaiting.Name: stats.D(trail.Waiting), + metrics.HTTPReqReceiving.Name: stats.D(trail.Receiving), + metrics.HTTPReqFailed.Name: stats.B(trail.Failed.Bool), + }, + }, + } + } + return &Sample{ Type: DataTypeMap, Metric: "http_req_li_all", @@ -146,6 +169,7 @@ type SampleDataAggregatedHTTPReqs struct { Sending AggregatedMetric `json:"http_req_sending"` Waiting AggregatedMetric `json:"http_req_waiting"` Receiving AggregatedMetric `json:"http_req_receiving"` + Failed AggregatedRate `json:"http_req_failed,omitempty"` } `json:"values"` } @@ -159,6 +183,9 @@ func (sdagg *SampleDataAggregatedHTTPReqs) Add(trail *httpext.Trail) { sdagg.Values.Sending.Add(trail.Sending) sdagg.Values.Waiting.Add(trail.Waiting) sdagg.Values.Receiving.Add(trail.Receiving) + if trail.Failed.Valid { + sdagg.Values.Failed.Add(trail.Failed.Bool) + } } // CalcAverages calculates and sets all `Avg` properties in the `Values` struct @@ -173,6 +200,25 @@ func (sdagg *SampleDataAggregatedHTTPReqs) CalcAverages() { sdagg.Values.Receiving.Calc(count) } +// AggregatedRate is an aggregation of a Rate metric +type AggregatedRate struct { + Count float64 `json:"count"` + NzCount float64 `json:"nz_count"` +} + +// Add a boolean to the aggregated rate +func (ar *AggregatedRate) Add(b bool) { + ar.Count++ + if b { + ar.NzCount++ + } +} + +// IsDefined implements easyjson.Optional +func (ar AggregatedRate) IsDefined() bool { + return ar.Count != 0 +} + // AggregatedMetric is used to store aggregated information for a // particular metric in an SampleDataAggregatedMap. type AggregatedMetric struct { diff --git a/stats/cloud/data_test.go b/stats/cloud/data_test.go index d9fd5a58fe73..02c1c63535f1 100644 --- a/stats/cloud/data_test.go +++ b/stats/cloud/data_test.go @@ -29,6 +29,7 @@ import ( "github.com/mailru/easyjson" "github.com/stretchr/testify/assert" + "gopkg.in/guregu/null.v3" "github.com/loadimpact/k6/lib/metrics" "github.com/loadimpact/k6/lib/netext/httpext" @@ -87,6 +88,105 @@ func TestSampleMarshaling(t *testing.T) { }), fmt.Sprintf(`{"type":"Points","metric":"http_req_li_all","data":{"time":"%d","type":"counter","values":{"http_req_blocked":0.001,"http_req_connecting":0.002,"http_req_duration":0.123,"http_req_receiving":0.006,"http_req_sending":0.004,"http_req_tls_handshaking":0.003,"http_req_waiting":0.005,"http_reqs":1}}}`, exptoMicroSecond), }, + { + NewSampleFromTrail(&httpext.Trail{ + EndTime: now, + Duration: 123000, + Blocked: 1000, + Connecting: 2000, + TLSHandshaking: 3000, + Sending: 4000, + Waiting: 5000, + Receiving: 6000, + Failed: null.NewBool(false, true), + }), + fmt.Sprintf(`{"type":"Points","metric":"http_req_li_all","data":{"time":"%d","type":"counter","values":{"http_req_blocked":0.001,"http_req_connecting":0.002,"http_req_duration":0.123,"http_req_failed":0,"http_req_receiving":0.006,"http_req_sending":0.004,"http_req_tls_handshaking":0.003,"http_req_waiting":0.005,"http_reqs":1}}}`, exptoMicroSecond), + }, + { + func() *Sample { + aggrData := &SampleDataAggregatedHTTPReqs{ + Time: exptoMicroSecond, + Type: "aggregated_trend", + Tags: stats.IntoSampleTags(&map[string]string{"test": "mest"}), + } + aggrData.Add( + &httpext.Trail{ + EndTime: now, + Duration: 123000, + Blocked: 1000, + Connecting: 2000, + TLSHandshaking: 3000, + Sending: 4000, + Waiting: 5000, + Receiving: 6000, + }, + ) + + aggrData.Add( + &httpext.Trail{ + EndTime: now, + Duration: 13000, + Blocked: 3000, + Connecting: 1000, + TLSHandshaking: 4000, + Sending: 5000, + Waiting: 8000, + Receiving: 8000, + }, + ) + aggrData.CalcAverages() + + return &Sample{ + Type: DataTypeAggregatedHTTPReqs, + Metric: "http_req_li_all", + Data: aggrData, + } + }(), + fmt.Sprintf(`{"type":"AggregatedPoints","metric":"http_req_li_all","data":{"time":"%d","type":"aggregated_trend","count":2,"tags":{"test":"mest"},"values":{"http_req_duration":{"min":0.013,"max":0.123,"avg":0.068},"http_req_blocked":{"min":0.001,"max":0.003,"avg":0.002},"http_req_connecting":{"min":0.001,"max":0.002,"avg":0.0015},"http_req_tls_handshaking":{"min":0.003,"max":0.004,"avg":0.0035},"http_req_sending":{"min":0.004,"max":0.005,"avg":0.0045},"http_req_waiting":{"min":0.005,"max":0.008,"avg":0.0065},"http_req_receiving":{"min":0.006,"max":0.008,"avg":0.007}}}}`, exptoMicroSecond), + }, + { + func() *Sample { + aggrData := &SampleDataAggregatedHTTPReqs{ + Time: exptoMicroSecond, + Type: "aggregated_trend", + Tags: stats.IntoSampleTags(&map[string]string{"test": "mest"}), + } + aggrData.Add( + &httpext.Trail{ + EndTime: now, + Duration: 123000, + Blocked: 1000, + Connecting: 2000, + TLSHandshaking: 3000, + Sending: 4000, + Waiting: 5000, + Receiving: 6000, + Failed: null.BoolFrom(false), + }, + ) + + aggrData.Add( + &httpext.Trail{ + EndTime: now, + Duration: 13000, + Blocked: 3000, + Connecting: 1000, + TLSHandshaking: 4000, + Sending: 5000, + Waiting: 8000, + Receiving: 8000, + }, + ) + aggrData.CalcAverages() + + return &Sample{ + Type: DataTypeAggregatedHTTPReqs, + Metric: "http_req_li_all", + Data: aggrData, + } + }(), + fmt.Sprintf(`{"type":"AggregatedPoints","metric":"http_req_li_all","data":{"time":"%d","type":"aggregated_trend","count":2,"tags":{"test":"mest"},"values":{"http_req_duration":{"min":0.013,"max":0.123,"avg":0.068},"http_req_blocked":{"min":0.001,"max":0.003,"avg":0.002},"http_req_connecting":{"min":0.001,"max":0.002,"avg":0.0015},"http_req_tls_handshaking":{"min":0.003,"max":0.004,"avg":0.0035},"http_req_sending":{"min":0.004,"max":0.005,"avg":0.0045},"http_req_waiting":{"min":0.005,"max":0.008,"avg":0.0065},"http_req_receiving":{"min":0.006,"max":0.008,"avg":0.007},"http_req_failed":{"count":1,"nz_count":0}}}}`, exptoMicroSecond), + }, } for _, tc := range testCases { @@ -140,6 +240,7 @@ func getDurations(count int, min, multiplier float64) durations { } return data } + func BenchmarkDurationBounds(b *testing.B) { iqrRadius := 0.25 // If it's something different, the Q in IQR won't make much sense... iqrLowerCoef := 1.5 @@ -223,7 +324,6 @@ func TestQuickSelectAndBounds(t *testing.T) { assert.Equal(t, dataForSort[k], data.quickSelect(k)) }) } - }) } } diff --git a/stats/system_tag.go b/stats/system_tag.go index c549e0745096..1cf8ee0a96fd 100644 --- a/stats/system_tag.go +++ b/stats/system_tag.go @@ -97,6 +97,7 @@ const ( TagTLSVersion TagScenario TagService + TagExpectedResponse // System tags not enabled by default. TagIter @@ -109,7 +110,7 @@ const ( // Other tags that are not enabled by default include: iter, vu, ocsp_status, ip //nolint:gochecknoglobals var DefaultSystemTagSet = TagProto | TagSubproto | TagStatus | TagMethod | TagURL | TagName | TagGroup | - TagCheck | TagCheck | TagError | TagErrorCode | TagTLSVersion | TagScenario | TagService + TagCheck | TagCheck | TagError | TagErrorCode | TagTLSVersion | TagScenario | TagService | TagExpectedResponse // Add adds a tag to tag set. func (i *SystemTagSet) Add(tag SystemTagSet) { diff --git a/stats/system_tag_set_gen.go b/stats/system_tag_set_gen.go index a8ccc3d223da..b65604d01ecc 100644 --- a/stats/system_tag_set_gen.go +++ b/stats/system_tag_set_gen.go @@ -7,26 +7,27 @@ import ( "fmt" ) -const _SystemTagSetName = "protosubprotostatusmethodurlnamegroupcheckerrorerror_codetls_versionscenarioserviceitervuocsp_statusip" +const _SystemTagSetName = "protosubprotostatusmethodurlnamegroupcheckerrorerror_codetls_versionscenarioserviceexpected_responseitervuocsp_statusip" var _SystemTagSetMap = map[SystemTagSet]string{ - 1: _SystemTagSetName[0:5], - 2: _SystemTagSetName[5:13], - 4: _SystemTagSetName[13:19], - 8: _SystemTagSetName[19:25], - 16: _SystemTagSetName[25:28], - 32: _SystemTagSetName[28:32], - 64: _SystemTagSetName[32:37], - 128: _SystemTagSetName[37:42], - 256: _SystemTagSetName[42:47], - 512: _SystemTagSetName[47:57], - 1024: _SystemTagSetName[57:68], - 2048: _SystemTagSetName[68:76], - 4096: _SystemTagSetName[76:83], - 8192: _SystemTagSetName[83:87], - 16384: _SystemTagSetName[87:89], - 32768: _SystemTagSetName[89:100], - 65536: _SystemTagSetName[100:102], + 1: _SystemTagSetName[0:5], + 2: _SystemTagSetName[5:13], + 4: _SystemTagSetName[13:19], + 8: _SystemTagSetName[19:25], + 16: _SystemTagSetName[25:28], + 32: _SystemTagSetName[28:32], + 64: _SystemTagSetName[32:37], + 128: _SystemTagSetName[37:42], + 256: _SystemTagSetName[42:47], + 512: _SystemTagSetName[47:57], + 1024: _SystemTagSetName[57:68], + 2048: _SystemTagSetName[68:76], + 4096: _SystemTagSetName[76:83], + 8192: _SystemTagSetName[83:100], + 16384: _SystemTagSetName[100:104], + 32768: _SystemTagSetName[104:106], + 65536: _SystemTagSetName[106:117], + 131072: _SystemTagSetName[117:119], } func (i SystemTagSet) String() string { @@ -36,7 +37,7 @@ func (i SystemTagSet) String() string { return fmt.Sprintf("SystemTagSet(%d)", i) } -var _SystemTagSetValues = []SystemTagSet{1, 2, 4, 8, 16, 32, 64, 128, 256, 512, 1024, 2048, 4096, 8192, 16384, 32768, 65536} +var _SystemTagSetValues = []SystemTagSet{1, 2, 4, 8, 16, 32, 64, 128, 256, 512, 1024, 2048, 4096, 8192, 16384, 32768, 65536, 131072} var _SystemTagSetNameToValueMap = map[string]SystemTagSet{ _SystemTagSetName[0:5]: 1, @@ -52,10 +53,11 @@ var _SystemTagSetNameToValueMap = map[string]SystemTagSet{ _SystemTagSetName[57:68]: 1024, _SystemTagSetName[68:76]: 2048, _SystemTagSetName[76:83]: 4096, - _SystemTagSetName[83:87]: 8192, - _SystemTagSetName[87:89]: 16384, - _SystemTagSetName[89:100]: 32768, - _SystemTagSetName[100:102]: 65536, + _SystemTagSetName[83:100]: 8192, + _SystemTagSetName[100:104]: 16384, + _SystemTagSetName[104:106]: 32768, + _SystemTagSetName[106:117]: 65536, + _SystemTagSetName[117:119]: 131072, } // SystemTagSetString retrieves an enum value from the enum constants string name. diff --git a/stats/units.go b/stats/units.go index 4380f2bc6e9f..9e5997f2bd75 100644 --- a/stats/units.go +++ b/stats/units.go @@ -37,3 +37,11 @@ func D(d time.Duration) float64 { func ToD(d float64) time.Duration { return time.Duration(d * float64(timeUnit)) } + +// B formats a boolena for emission +func B(b bool) float64 { + if b { + return 1 + } + return 0 +}