Implement dynamic body property for HTTP responses

Kind of resolves #1841 (comment)

js/modules/k6/http/request.go

@@ -48,47 +48,47 @@ var ErrHTTPForbiddenInInitContext = common.NewInitContextError("Making http requ
 var ErrBatchForbiddenInInitContext = common.NewInitContextError("Using batch in the init context is not supported")
 
 // Get makes an HTTP GET request and returns a corresponding response by taking goja.Values as arguments
-func (h *HTTP) Get(ctx context.Context, url goja.Value, args ...goja.Value) (*Response, error) {
+func (h *HTTP) Get(ctx context.Context, url goja.Value, args ...goja.Value) (*goja.Object, error) {
 	// The body argument is always undefined for GETs and HEADs.
 	args = append([]goja.Value{goja.Undefined()}, args...)
 	return h.Request(ctx, HTTP_METHOD_GET, url, args...)
 }
 
 // Head makes an HTTP HEAD request and returns a corresponding response by taking goja.Values as arguments
-func (h *HTTP) Head(ctx context.Context, url goja.Value, args ...goja.Value) (*Response, error) {
+func (h *HTTP) Head(ctx context.Context, url goja.Value, args ...goja.Value) (*goja.Object, error) {
 	// The body argument is always undefined for GETs and HEADs.
 	args = append([]goja.Value{goja.Undefined()}, args...)
 	return h.Request(ctx, HTTP_METHOD_HEAD, url, args...)
 }
 
 // Post makes an HTTP POST request and returns a corresponding response by taking goja.Values as arguments
-func (h *HTTP) Post(ctx context.Context, url goja.Value, args ...goja.Value) (*Response, error) {
+func (h *HTTP) Post(ctx context.Context, url goja.Value, args ...goja.Value) (*goja.Object, error) {
 	return h.Request(ctx, HTTP_METHOD_POST, url, args...)
 }
 
 // Put makes an HTTP PUT request and returns a corresponding response by taking goja.Values as arguments
-func (h *HTTP) Put(ctx context.Context, url goja.Value, args ...goja.Value) (*Response, error) {
+func (h *HTTP) Put(ctx context.Context, url goja.Value, args ...goja.Value) (*goja.Object, error) {
 	return h.Request(ctx, HTTP_METHOD_PUT, url, args...)
 }
 
 // Patch makes a patch request and returns a corresponding response by taking goja.Values as arguments
-func (h *HTTP) Patch(ctx context.Context, url goja.Value, args ...goja.Value) (*Response, error) {
+func (h *HTTP) Patch(ctx context.Context, url goja.Value, args ...goja.Value) (*goja.Object, error) {
 	return h.Request(ctx, HTTP_METHOD_PATCH, url, args...)
 }
 
 // Del makes an HTTP DELETE and returns a corresponding response by taking goja.Values as arguments
-func (h *HTTP) Del(ctx context.Context, url goja.Value, args ...goja.Value) (*Response, error) {
+func (h *HTTP) Del(ctx context.Context, url goja.Value, args ...goja.Value) (*goja.Object, error) {
 	return h.Request(ctx, HTTP_METHOD_DELETE, url, args...)
 }
 
 // Options makes an HTTP OPTIONS request and returns a corresponding response by taking goja.Values as arguments
-func (h *HTTP) Options(ctx context.Context, url goja.Value, args ...goja.Value) (*Response, error) {
+func (h *HTTP) Options(ctx context.Context, url goja.Value, args ...goja.Value) (*goja.Object, error) {
 	return h.Request(ctx, HTTP_METHOD_OPTIONS, url, args...)
 }
 
 // Request makes an http request of the provided `method` and returns a corresponding response by
 // taking goja.Values as arguments
-func (h *HTTP) Request(ctx context.Context, method string, url goja.Value, args ...goja.Value) (*Response, error) {
+func (h *HTTP) Request(ctx context.Context, method string, url goja.Value, args ...goja.Value) (*goja.Object, error) {
 	u, err := ToURL(url)
 	if err != nil {
 		return nil, err
@@ -113,8 +113,7 @@ func (h *HTTP) Request(ctx context.Context, method string, url goja.Value, args
 	if err != nil {
 		return nil, err
 	}
-	processResponse(ctx, resp, req.ResponseType)
-	return responseFromHttpext(resp), nil
+	return newResponse(ctx, resp, req.ResponseType), nil
 }
 
 //TODO break this function up
@@ -363,10 +362,10 @@ func (h *HTTP) parseRequest(
 
 func (h *HTTP) prepareBatchArray(
 	ctx context.Context, requests []interface{},
-) ([]httpext.BatchParsedHTTPRequest, []*Response, error) {
+) ([]httpext.BatchParsedHTTPRequest, []*goja.Object, error) {
 	reqCount := len(requests)
 	batchReqs := make([]httpext.BatchParsedHTTPRequest, reqCount)
-	results := make([]*Response, reqCount)
+	results := make([]*goja.Object, reqCount)
 
 	for i, req := range requests {
 		parsedReq, err := h.parseBatchRequest(ctx, i, req)
@@ -378,18 +377,18 @@ func (h *HTTP) prepareBatchArray(
 			ParsedHTTPRequest: parsedReq,
 			Response:          response,
 		}
-		results[i] = &Response{response}
+		results[i] = newResponse(ctx, response, parsedReq.ResponseType)
 	}
 
 	return batchReqs, results, nil
 }
 
 func (h *HTTP) prepareBatchObject(
 	ctx context.Context, requests map[string]interface{},
-) ([]httpext.BatchParsedHTTPRequest, map[string]*Response, error) {
+) ([]httpext.BatchParsedHTTPRequest, map[string]*goja.Object, error) {
 	reqCount := len(requests)
 	batchReqs := make([]httpext.BatchParsedHTTPRequest, reqCount)
-	results := make(map[string]*Response, reqCount)
+	results := make(map[string]*goja.Object, reqCount)
 
 	i := 0
 	for key, req := range requests {
@@ -402,7 +401,7 @@ func (h *HTTP) prepareBatchObject(
 			ParsedHTTPRequest: parsedReq,
 			Response:          response,
 		}
-		results[key] = &Response{response}
+		results[key] = newResponse(ctx, response, parsedReq.ResponseType)
 		i++
 	}
 
@@ -420,7 +419,7 @@ func (h *HTTP) Batch(ctx context.Context, reqsV goja.Value) (goja.Value, error)
 	var (
 		err       error
 		batchReqs []httpext.BatchParsedHTTPRequest
-		results   interface{} // either []*Response or map[string]*Response
+		results   interface{} // either []*goja.Object or map[string]*goja.Object
 	)
 
 	switch v := reqsV.Export().(type) {
@@ -440,7 +439,6 @@ func (h *HTTP) Batch(ctx context.Context, reqsV goja.Value) (goja.Value, error)
 	errs := httpext.MakeBatchRequests(
 		ctx, batchReqs, reqCount,
 		int(state.Options.Batch.Int64), int(state.Options.BatchPerHost.Int64),
-		processResponse,
 	)
 
 	for i := 0; i < reqCount; i++ {

js/modules/k6/http/response.go

@@ -39,19 +39,34 @@ type Response struct {
 	*httpext.Response `js:"-"`
 }
 
-// processResponse stores the body as an ArrayBuffer if indicated by
-// respType. This is done here instead of in httpext.readResponseBody to avoid
-// a reverse dependency on js/common or goja.
-func processResponse(ctx context.Context, resp *httpext.Response, respType httpext.ResponseType) {
-	if respType == httpext.ResponseTypeBinary {
-		rt := common.GetRuntime(ctx)
-		resp.Body = rt.NewArrayBuffer(resp.Body.([]byte))
+func newResponse(ctx context.Context, resp *httpext.Response, respType httpext.ResponseType) *goja.Object {
+	rt := common.GetRuntime(ctx)
+
+	getBody := rt.ToValue(func() interface{} {
+		switch respType {
+		case httpext.ResponseTypeBinary:
+			ab := rt.NewArrayBuffer(resp.Body)
+			return &ab
+		case httpext.ResponseTypeText:
+			return string(resp.Body)
+		case httpext.ResponseTypeNone:
+		default:
+			common.Throw(rt, errors.New("invalid response type"))
+		}
+		return nil
+	})
+
+	res := &Response{resp}
+	resObj := rt.ToValue(res).ToObject(rt)
+	obj := rt.NewObject()
+	if err := obj.DefineAccessorProperty("body", getBody, nil, goja.FLAG_FALSE, goja.FLAG_TRUE); err != nil {
+		common.Throw(rt, err)
+	}
+	if err := obj.SetPrototype(resObj); err != nil {
+		common.Throw(rt, err)
 	}
-}
 
-func responseFromHttpext(resp *httpext.Response) *Response {
-	res := Response{resp}
-	return &res
+	return obj
 }
 
 // JSON parses the body of a response as json and returns it to the goja VM
@@ -68,17 +83,7 @@ func (res *Response) JSON(selector ...string) goja.Value {
 
 // HTML returns the body as an html.Selection
 func (res *Response) HTML(selector ...string) html.Selection {
-	var body string
-	switch b := res.Body.(type) {
-	case []byte:
-		body = string(b)
-	case string:
-		body = b
-	default:
-		common.Throw(common.GetRuntime(res.GetCtx()), errors.New("invalid response type"))
-	}
-
-	sel, err := html.HTML{}.ParseHTML(res.GetCtx(), body)
+	sel, err := html.HTML{}.ParseHTML(res.GetCtx(), string(res.Body))
 	if err != nil {
 		common.Throw(common.GetRuntime(res.GetCtx()), err)
 	}
@@ -91,7 +96,8 @@ func (res *Response) HTML(selector ...string) html.Selection {
 
 // SubmitForm parses the body as an html looking for a from and then submitting it
 // TODO: document the actual arguments that can be provided
-func (res *Response) SubmitForm(args ...goja.Value) (*Response, error) {
+//nolint: funlen
+func (res *Response) SubmitForm(args ...goja.Value) (*goja.Object, error) {
 	rt := common.GetRuntime(res.GetCtx())
 
 	formSelector := "form"
@@ -177,7 +183,7 @@ func (res *Response) SubmitForm(args ...goja.Value) (*Response, error) {
 
 // ClickLink parses the body as an html, looks for a link and than makes a request as if the link was
 // clicked
-func (res *Response) ClickLink(args ...goja.Value) (*Response, error) {
+func (res *Response) ClickLink(args ...goja.Value) (*goja.Object, error) {
 	rt := common.GetRuntime(res.GetCtx())
 
 	selector := "a[href]"

js/modules/k6/http/response_test.go

@@ -413,15 +413,15 @@ func BenchmarkResponseJson(b *testing.B) {
 		tc := tc
 		b.Run(fmt.Sprintf("Selector %s ", tc.selector), func(b *testing.B) {
 			for n := 0; n < b.N; n++ {
-				resp := responseFromHttpext(&httpext.Response{Body: jsonData})
+				resp := Response{&httpext.Response{Body: []byte(jsonData)}}
 				resp.JSON(tc.selector)
 			}
 		})
 	}
 
 	b.Run("Without selector", func(b *testing.B) {
 		for n := 0; n < b.N; n++ {
-			resp := responseFromHttpext(&httpext.Response{Body: jsonData})
+			resp := Response{&httpext.Response{Body: []byte(jsonData)}}
 			resp.JSON()
 		}
 	})

lib/netext/httpext/batch.go

@@ -42,13 +42,10 @@ type BatchParsedHTTPRequest struct {
 // pre-initialized. In addition, each processed request would emit either a nil
 // value, or an error, via the returned errors channel. The goroutines exit when
 // the requests channel is closed.
-// The processResponse callback can be used to modify the response, e.g.
-// to replace the body.
 func MakeBatchRequests(
 	ctx context.Context,
 	requests []BatchParsedHTTPRequest,
 	reqCount, globalLimit, perHostLimit int,
-	processResponse func(context.Context, *Response, ResponseType),
 ) <-chan error {
 	workers := globalLimit
 	if reqCount < workers {
@@ -65,7 +62,6 @@ func MakeBatchRequests(
 
 		resp, err := MakeRequest(ctx, req.ParsedHTTPRequest)
 		if resp != nil {
-			processResponse(ctx, resp, req.ParsedHTTPRequest.ResponseType)
 			*req.Response = *resp
 		}
 		result <- err

lib/netext/httpext/compression.go

@@ -140,7 +140,7 @@ func readResponseBody(
 	respType ResponseType,
 	resp *http.Response,
 	respErr error,
-) (interface{}, error) {
+) ([]byte, error) {
 	if resp == nil || respErr != nil {
 		return nil, respErr
 	}
@@ -204,22 +204,10 @@ func readResponseBody(
 		respErr = wrapDecompressionError(err)
 	}
 
-	var result interface{}
-	// Binary or string
-	switch respType {
-	case ResponseTypeText:
-		result = buf.String()
-	case ResponseTypeBinary:
-		// Copy the data to a new slice before we return the buffer to the pool,
-		// because buf.Bytes() points to the underlying buffer byte slice.
-		// The ArrayBuffer wrapping will be done in the js/modules/k6/http
-		// package to avoid a reverse dependency, since it depends on goja.
-		binData := make([]byte, buf.Len())
-		copy(binData, buf.Bytes())
-		result = binData
-	default:
-		respErr = fmt.Errorf("unknown responseType %s", respType)
-	}
+	// Copy the data to a new slice before we return the buffer to the pool,
+	// because buf.Bytes() points to the underlying buffer byte slice.
+	binData := make([]byte, buf.Len())
+	copy(binData, buf.Bytes())
 
-	return result, respErr
+	return binData, respErr
 }

lib/netext/httpext/response.go

@@ -26,8 +26,6 @@ import (
 	"encoding/json"
 	"fmt"
 
-	"github.com/dop251/goja"
-	"github.com/pkg/errors"
 	"github.com/tidwall/gjson"
 
 	"github.com/loadimpact/k6/lib/netext"
@@ -101,7 +99,7 @@ type Response struct {
 	Proto          string                   `json:"proto"`
 	Headers        map[string]string        `json:"headers"`
 	Cookies        map[string][]*HTTPCookie `json:"cookies"`
-	Body           interface{}              `json:"body"`
+	Body           []byte                   `json:"body"`
 	Timings        ResponseTimings          `json:"timings"`
 	TLSVersion     string                   `json:"tls_version"`
 	TLSCipherSuite string                   `json:"tls_cipher_suite"`
@@ -131,37 +129,25 @@ func (res *Response) JSON(selector ...string) (interface{}, error) {
 	hasSelector := len(selector) > 0
 	if res.cachedJSON == nil || hasSelector {
 		var v interface{}
-		var body []byte
-		switch b := res.Body.(type) {
-		case []byte:
-			body = b
-		case string:
-			body = []byte(b)
-		case goja.ArrayBuffer:
-			body = b.Bytes()
-		default:
-			return nil, errors.New("invalid response type")
-		}
-
 		if hasSelector {
 			if !res.validatedJSON {
-				if !gjson.ValidBytes(body) {
+				if !gjson.ValidBytes(res.Body) {
 					return nil, nil
 				}
 				res.validatedJSON = true
 			}
 
-			result := gjson.GetBytes(body, selector[0])
+			result := gjson.GetBytes(res.Body, selector[0])
 
 			if !result.Exists() {
 				return nil, nil
 			}
 			return result.Value(), nil
 		}
 
-		if err := json.Unmarshal(body, &v); err != nil {
+		if err := json.Unmarshal(res.Body, &v); err != nil {
 			if syntaxError, ok := err.(*json.SyntaxError); ok {
-				err = checkErrorInJSON(body, int(syntaxError.Offset), err)
+				err = checkErrorInJSON(res.Body, int(syntaxError.Offset), err)
 			}
 			return nil, err
 		}