Refactor the http-debug code in its own low-level transport

This makes the code slighly saner and fixes the bug where the first NTLM and digest authentication requests weren't displayed. But by no means does this fix all of the bugs in the http-debug code... For those, see:
 - #986
 - #1042
 - #774

 A discussion how some or all of them can be fixed can be found here: #1102 (comment)

lib/netext/httpext/httpdebug_transport.go

@@ -0,0 +1,67 @@
+/*
+ *
+ * k6 - a next-generation load testing tool
+ * Copyright (C) 2019 Load Impact
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ *
+ */
+
+package httpext
+
+import (
+	"fmt"
+	"net/http"
+	"net/http/httputil"
+
+	log "github.com/sirupsen/logrus"
+)
+
+type httpDebugTransport struct {
+	//TODO: get the state and log to its Logger
+	originalTransport http.RoundTripper
+	httpDebugOption   string
+}
+
+// RoundTrip prints passing HTTP requests and received responses
+//
+// TODO: massively improve this, because the printed information can be wrong:
+//  - https://github.com/loadimpact/k6/issues/986
+//  - https://github.com/loadimpact/k6/issues/1042
+//  - https://github.com/loadimpact/k6/issues/774
+func (t httpDebugTransport) RoundTrip(req *http.Request) (*http.Response, error) {
+	// Make an initial reques
+	t.debugRequest(req)
+	resp, err := t.originalTransport.RoundTrip(req)
+	t.debugResponse(resp)
+	return resp, err
+}
+
+func (t httpDebugTransport) debugRequest(req *http.Request) {
+	dump, err := httputil.DumpRequestOut(req, t.httpDebugOption == "full")
+	if err != nil {
+		log.Fatal(err) //TODO: fix...
+	}
+	fmt.Printf("Request:\n%s\n", dump) //TODO: fix...
+}
+
+func (t httpDebugTransport) debugResponse(res *http.Response) {
+	if res != nil {
+		dump, err := httputil.DumpResponse(res, t.httpDebugOption == "full")
+		if err != nil {
+			log.Fatal(err) //TODO: fix...
+		}
+		fmt.Printf("Response:\n%s\n", dump) //TODO: fix...
+	}
+}

lib/netext/httpext/request.go

@@ -31,7 +31,6 @@ import (
 	"net"
 	"net/http"
 	"net/http/cookiejar"
-	"net/http/httputil"
 	"net/url"
 	"strconv"
 	"strings"
@@ -433,11 +432,12 @@ func MakeRequest(ctx context.Context, preq *ParsedHTTPRequest) (*Response, error
 	tracerTransport := newTransport(state, tags)
 	var transport http.RoundTripper = tracerTransport
 
-	// TODO: if HttpDebug is enabled, inject the debug transport here? or use
-	// something like a virtual proxy for more accurate results, so we can catch
-	// things like HTTP/2 and exact headers? Connected issues:
-	// https://github.com/loadimpact/k6/issues/986,
-	// https://github.com/loadimpact/k6/issues/1042
+	if state.Options.HttpDebug.String != "" {
+		transport = httpDebugTransport{
+			originalTransport: transport,
+			httpDebugOption:   state.Options.HttpDebug.String,
+		}
+	}
 
 	if preq.Auth == "digest" {
 		transport = digestTransport{originalTransport: transport}
@@ -451,7 +451,6 @@ func MakeRequest(ctx context.Context, preq *ParsedHTTPRequest) (*Response, error
 		Timeout:   preq.Timeout,
 		CheckRedirect: func(req *http.Request, via []*http.Request) error {
 			resp.URL = req.URL.String()
-			debugResponse(state, req.Response, "RedirectResponse")
 
 			// Update active jar with cookies found in "Set-Cookie" header(s) of redirect response
 			if preq.ActiveJar != nil {
@@ -474,15 +473,12 @@ func MakeRequest(ctx context.Context, preq *ParsedHTTPRequest) (*Response, error
 				}
 				return http.ErrUseLastResponse
 			}
-			debugRequest(state, req, "RedirectRequest")
 			return nil
 		},
 	}
 
-	debugRequest(state, preq.Req, "Request")
 	mreq := preq.Req.WithContext(ctx)
 	res, resErr := client.Do(mreq)
-	debugResponse(state, res, "Response")
 
 	resp.Body, resErr = readResponseBody(state, preq.ResponseType, res, resErr)
 	finishedReq := tracerTransport.processLastSavedRequest(wrapDecompressionError(resErr))
@@ -558,16 +554,3 @@ func SetRequestCookies(req *http.Request, jar *cookiejar.Jar, reqCookies map[str
 		}
 	}
 }
-
-func debugRequest(state *lib.State, req *http.Request, description string) {
-	if state.Options.HttpDebug.String != "" {
-		dump, err := httputil.DumpRequestOut(req, state.Options.HttpDebug.String == "full")
-		if err != nil {
-			log.Fatal(err)
-		}
-		logDump(description, dump)
-	}
-}
-func logDump(description string, dump []byte) {
-	fmt.Printf("%s:\n%s\n", description, dump)
-}

lib/netext/httpext/response.go

@@ -24,13 +24,9 @@ import (
 	"context"
 	"crypto/tls"
 	"encoding/json"
-	"net/http"
-	"net/http/httputil"
 
-	"github.com/loadimpact/k6/lib"
 	"github.com/loadimpact/k6/lib/netext"
 	"github.com/pkg/errors"
-	log "github.com/sirupsen/logrus"
 	"github.com/tidwall/gjson"
 )
 
@@ -115,16 +111,6 @@ func (res *Response) GetCtx() context.Context {
 	return res.ctx
 }
 
-func debugResponse(state *lib.State, res *http.Response, description string) {
-	if state.Options.HttpDebug.String != "" && res != nil {
-		dump, err := httputil.DumpResponse(res, state.Options.HttpDebug.String == "full")
-		if err != nil {
-			log.Fatal(err)
-		}
-		logDump(description, dump)
-	}
-}
-
 // JSON parses the body of a response as json and returns it to the goja VM
 func (res *Response) JSON(selector ...string) (interface{}, error) {
 	hasSelector := len(selector) > 0