Fix response error handling and slightly improve the digest auth code

lib/netext/httpext/httpdebug_transport.go

@@ -0,0 +1,67 @@
+/*
+ *
+ * k6 - a next-generation load testing tool
+ * Copyright (C) 2019 Load Impact
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ *
+ */
+
+package httpext
+
+import (
+	"fmt"
+	"net/http"
+	"net/http/httputil"
+
+	log "github.com/sirupsen/logrus"
+)
+
+type httpDebugTransport struct {
+	//TODO: get the state and log to its Logger
+	originalTransport http.RoundTripper
+	httpDebugOption   string
+}
+
+// RoundTrip prints passing HTTP requests and received responses
+//
+// TODO: massively improve this, because the printed information can be wrong:
+//  - https://github.com/loadimpact/k6/issues/986
+//  - https://github.com/loadimpact/k6/issues/1042
+//  - https://github.com/loadimpact/k6/issues/774
+func (t httpDebugTransport) RoundTrip(req *http.Request) (*http.Response, error) {
+	// Make an initial reques
+	t.debugRequest(req)
+	resp, err := t.originalTransport.RoundTrip(req)
+	t.debugResponse(resp)
+	return resp, err
+}
+
+func (t httpDebugTransport) debugRequest(req *http.Request) {
+	dump, err := httputil.DumpRequestOut(req, t.httpDebugOption == "full")
+	if err != nil {
+		log.Fatal(err) //TODO: fix...
+	}
+	fmt.Printf("Request:\n%s\n", dump) //TODO: fix...
+}
+
+func (t httpDebugTransport) debugResponse(res *http.Response) {
+	if res != nil {
+		dump, err := httputil.DumpResponse(res, t.httpDebugOption == "full")
+		if err != nil {
+			log.Fatal(err) //TODO: fix...
+		}
+		fmt.Printf("Response:\n%s\n", dump) //TODO: fix...
+	}
+}

lib/netext/httpext/request.go

@@ -31,7 +31,6 @@ import (
 	"net"
 	"net/http"
 	"net/http/cookiejar"
-	"net/http/httputil"
 	"net/url"
 	"strconv"
 	"strings"
@@ -433,11 +432,12 @@ func MakeRequest(ctx context.Context, preq *ParsedHTTPRequest) (*Response, error
 	tracerTransport := newTransport(state, tags)
 	var transport http.RoundTripper = tracerTransport
 
-	// TODO: if HttpDebug is enabled, inject the debug transport here? or use
-	// something like a virtual proxy for more accurate results, so we can catch
-	// things like HTTP/2 and exact headers? Connected issues:
-	// https://github.com/loadimpact/k6/issues/986,
-	// https://github.com/loadimpact/k6/issues/1042
+	if state.Options.HttpDebug.String != "" {
+		transport = httpDebugTransport{
+			originalTransport: transport,
+			httpDebugOption:   state.Options.HttpDebug.String,
+		}
+	}
 
 	if preq.Auth == "digest" {
 		transport = digestTransport{originalTransport: transport}
@@ -451,7 +451,6 @@ func MakeRequest(ctx context.Context, preq *ParsedHTTPRequest) (*Response, error
 		Timeout:   preq.Timeout,
 		CheckRedirect: func(req *http.Request, via []*http.Request) error {
 			resp.URL = req.URL.String()
-			debugResponse(state, req.Response, "RedirectResponse")
 
 			// Update active jar with cookies found in "Set-Cookie" header(s) of redirect response
 			if preq.ActiveJar != nil {
@@ -474,15 +473,12 @@ func MakeRequest(ctx context.Context, preq *ParsedHTTPRequest) (*Response, error
 				}
 				return http.ErrUseLastResponse
 			}
-			debugRequest(state, req, "RedirectRequest")
 			return nil
 		},
 	}
 
-	debugRequest(state, preq.Req, "Request")
 	mreq := preq.Req.WithContext(ctx)
 	res, resErr := client.Do(mreq)
-	debugResponse(state, res, "Response")
 
 	resp.Body, resErr = readResponseBody(state, preq.ResponseType, res, resErr)
 	finishedReq := tracerTransport.processLastSavedRequest(wrapDecompressionError(resErr))
@@ -558,16 +554,3 @@ func SetRequestCookies(req *http.Request, jar *cookiejar.Jar, reqCookies map[str
 		}
 	}
 }
-
-func debugRequest(state *lib.State, req *http.Request, description string) {
-	if state.Options.HttpDebug.String != "" {
-		dump, err := httputil.DumpRequestOut(req, state.Options.HttpDebug.String == "full")
-		if err != nil {
-			log.Fatal(err)
-		}
-		logDump(description, dump)
-	}
-}
-func logDump(description string, dump []byte) {
-	fmt.Printf("%s:\n%s\n", description, dump)
-}

lib/netext/httpext/response.go

@@ -24,13 +24,9 @@ import (
 	"context"
 	"crypto/tls"
 	"encoding/json"
-	"net/http"
-	"net/http/httputil"
 
-	"github.com/loadimpact/k6/lib"
 	"github.com/loadimpact/k6/lib/netext"
 	"github.com/pkg/errors"
-	log "github.com/sirupsen/logrus"
 	"github.com/tidwall/gjson"
 )
 
@@ -115,16 +111,6 @@ func (res *Response) GetCtx() context.Context {
 	return res.ctx
 }
 
-func debugResponse(state *lib.State, res *http.Response, description string) {
-	if state.Options.HttpDebug.String != "" && res != nil {
-		dump, err := httputil.DumpResponse(res, state.Options.HttpDebug.String == "full")
-		if err != nil {
-			log.Fatal(err)
-		}
-		logDump(description, dump)
-	}
-}
-
 // JSON parses the body of a response as json and returns it to the goja VM
 func (res *Response) JSON(selector ...string) (interface{}, error) {
 	hasSelector := len(selector) > 0