Fix response error handling and slightly improve the digest auth code

js/modules/k6/http/request_test.go

@@ -45,6 +45,7 @@ import (
 	"github.com/loadimpact/k6/lib/metrics"
 	"github.com/loadimpact/k6/lib/testutils"
 	"github.com/loadimpact/k6/stats"
+	"github.com/mccutchen/go-httpbin/httpbin"
 	"github.com/oxtoacart/bpool"
 	"github.com/sirupsen/logrus"
 	logtest "github.com/sirupsen/logrus/hooks/test"
@@ -1592,20 +1593,16 @@ func TestErrorCodes(t *testing.T) {
 			script:            `let res = http.request("GET", "HTTPBIN_URL/redirect-to?url=http://dafsgdhfjg/");`,
 		},
 		{
-			name:                "Non location redirect",
-			expectedErrorCode:   0,
-			expectedErrorMsg:    "",
-			script:              `let res = http.request("GET", "HTTPBIN_URL/no-location-redirect");`,
-			expectedScriptError: sr(`GoError: Get HTTPBIN_URL/no-location-redirect: 302 response missing Location header`),
+			name:              "Non location redirect",
+			expectedErrorCode: 1000,
+			expectedErrorMsg:  "302 response missing Location header",
+			script:            `let res = http.request("GET", "HTTPBIN_URL/no-location-redirect");`,
 		},
 		{
 			name:              "Bad location redirect",
-			expectedErrorCode: 0,
-			expectedErrorMsg:  "",
+			expectedErrorCode: 1000,
+			expectedErrorMsg:  "failed to parse Location header \"h\\t:/\": parse h\t:/: net/url: invalid control character in URL", //nolint: lll
 			script:            `let res = http.request("GET", "HTTPBIN_URL/bad-location-redirect");`,
-			expectedScriptError: sr(
-				"GoError: Get HTTPBIN_URL/bad-location-redirect: failed to parse Location header" +
-					" \"h\\t:/\": parse h\t:/: net/url: invalid control character in URL"),
 		},
 		{
 			name:              "Missing protocol",
@@ -1641,7 +1638,7 @@ func TestErrorCodes(t *testing.T) {
 			_, err := common.RunString(rt,
 				sr(testCase.script+"\n"+fmt.Sprintf(`
 			if (res.status != %d) { throw new Error("wrong status: "+ res.status);}
-			if (res.error != '%s') { throw new Error("wrong error: "+ res.error);}
+			if (res.error != %q) { throw new Error("wrong error: '" + res.error + "'");}
 			if (res.error_code != %d) { throw new Error("wrong error_code: "+ res.error_code);}
 			`, testCase.status, testCase.expectedErrorMsg, testCase.expectedErrorCode)))
 			if testCase.expectedScriptError == "" {
@@ -1824,3 +1821,62 @@ func BenchmarkHandlingOfResponseBodies(b *testing.B) {
 	b.Run("binary", testResponseType("binary"))
 	b.Run("none", testResponseType("none"))
 }
+
+func TestErrorsWithDecompression(t *testing.T) {
+	t.Parallel()
+	tb, state, _, rt, _ := newRuntime(t)
+	defer tb.Cleanup()
+
+	state.Options.Throw = null.BoolFrom(false)
+
+	tb.Mux.HandleFunc("/broken-archive", http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		enc := r.URL.Query()["encoding"][0]
+		w.Header().Set("Content-Encoding", enc)
+		_, _ = fmt.Fprintf(w, "Definitely not %s, but it's all cool...", enc)
+	}))
+
+	_, err := common.RunString(rt, tb.Replacer.Replace(`
+		function handleResponseEncodingError (encoding) {
+			let resp = http.get("HTTPBIN_URL/broken-archive?encoding=" + encoding);
+			if (resp.error_code != 1701) {
+				throw new Error("Expected error_code 1701 for '" + encoding +"', but got " + resp.error_code);
+			}
+		}
+
+		["gzip", "deflate", "br", "zstd"].forEach(handleResponseEncodingError);
+	`))
+	assert.NoError(t, err)
+}
+
+func TestDigestAuthWithBody(t *testing.T) {
+	t.Parallel()
+	tb, state, samples, rt, _ := newRuntime(t)
+	defer tb.Cleanup()
+
+	state.Options.Throw = null.BoolFrom(true)
+
+	tb.Mux.HandleFunc("/digest-auth-with-post/", http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		require.Equal(t, "POST", r.Method)
+		body, err := ioutil.ReadAll(r.Body)
+		require.NoError(t, err)
+		require.Equal(t, "super secret body", string(body))
+		httpbin.New().DigestAuth(w, r) // this doesn't read the body
+	}))
+
+	// TODO: fix, the metric tags shouldn't have credentials (https://github.com/loadimpact/k6/issues/1103)
+	urlWithCreds := tb.Replacer.Replace(
+		"http://testuser:testpwd@HTTPBIN_IP:HTTPBIN_PORT/digest-auth-with-post/auth/testuser/testpwd",
+	)
+
+	_, err := common.RunString(rt, fmt.Sprintf(`
+		let res = http.post(%q, "super secret body", { auth: "digest" });
+		if (res.status !== 200) { throw new Error("wrong status: " + res.status); }
+		if (res.error_code !== 0) { throw new Error("wrong error code: " + res.error_code); }
+	`, urlWithCreds))
+	require.NoError(t, err)
+
+	expectedURL := tb.Replacer.Replace("HTTPBIN_IP_URL/digest-auth-with-post/auth/testuser/testpwd")
+	sampleContainers := stats.GetBufferedSamples(samples)
+	assertRequestMetricsEmitted(t, sampleContainers[0:1], "POST", expectedURL, urlWithCreds, 401, "")
+	assertRequestMetricsEmitted(t, sampleContainers[1:2], "POST", expectedURL, urlWithCreds, 200, "")
+}

lib/netext/httpext/digest_transport.go

@@ -0,0 +1,84 @@
+/*
+ *
+ * k6 - a next-generation load testing tool
+ * Copyright (C) 2019 Load Impact
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ *
+ */
+
+package httpext
+
+import (
+	"io/ioutil"
+	"net/http"
+
+	digest "github.com/Soontao/goHttpDigestClient"
+)
+
+type digestTransport struct {
+	originalTransport http.RoundTripper
+}
+
+// RoundTrip handles digest auth by behaving like an http.RoundTripper
+//
+// TODO: fix - this is a preliminary solution and is somewhat broken! we're
+// always making 2 HTTP requests when digest authentication is enabled... we
+// should cache the nonces and behave more like a browser... or we should
+// ditch the hacky http.RoundTripper approach and write our own client...
+//
+// Github issue: https://github.com/loadimpact/k6/issues/800
+func (t digestTransport) RoundTrip(req *http.Request) (*http.Response, error) {
+	// Make the initial request authentication params to compute the
+	// authorization header
+	username := req.URL.User.Username()
+	password, _ := req.URL.User.Password()
+
+	// Remove the user data from the URL to avoid sending the authorization
+	// header for basic auth
+	req.URL.User = nil
+
+	noAuthResponse, err := t.originalTransport.RoundTrip(req)
+	if err != nil || noAuthResponse.StatusCode != http.StatusUnauthorized {
+		// If there was an error, or if the remote server didn't respond with
+		// status 401, we simply return, so the upstream code can deal with it.
+		return noAuthResponse, err
+	}
+
+	respBody, err := ioutil.ReadAll(noAuthResponse.Body)
+	if err != nil {
+		return nil, err
+	}
+	_ = noAuthResponse.Body.Close()
+
+	// Calculate the Authorization header
+	// TODO: determine if we actually need the body, since I'm not sure that's
+	// what the `entity` means... maybe a moot point if we change the used
+	// digest auth library...
+	challenge := digest.GetChallengeFromHeader(&noAuthResponse.Header)
+	challenge.ComputeResponse(req.Method, req.URL.RequestURI(), string(respBody), username, password)
+	authorization := challenge.ToAuthorizationStr()
+	req.Header.Set(digest.KEY_AUTHORIZATION, authorization)
+
+	if req.GetBody != nil {
+		// Reset the request body if we need to
+		req.Body, err = req.GetBody()
+		if err != nil {
+			return nil, err
+		}
+	}
+
+	// Actually make the HTTP request with the proper Authorization
+	return t.originalTransport.RoundTrip(req)
+}

lib/netext/httpext/error_codes.go

@@ -35,6 +35,10 @@ import (
 	"golang.org/x/net/http2"
 )
 
+// TODO: maybe rename the type errorCode, so we can have errCode variables? and
+// also the constants would probably be better of if `ErrorCode` was a prefix,
+// not a suffix - they would be much easier for auto-autocompletion at least...
+
 type errCode uint32
 
 const (
@@ -72,6 +76,10 @@ const (
 	// HTTP2 Connection errors
 	unknownHTTP2ConnectionErrorCode errCode = 1650
 	// errors till 1651 + 13 are other HTTP2 Connection errors with a specific errCode
+
+	// Custom k6 content errors, i.e. when the magic fails
+	//defaultContentError errCode = 1700 // reserved for future use
+	responseDecompressionErrorCode errCode = 1701
 )
 
 const (
@@ -99,6 +107,8 @@ func http2ErrCodeOffset(code http2.ErrCode) errCode {
 // errorCodeForError returns the errorCode and a specific error message for given error.
 func errorCodeForError(err error) (errCode, string) {
 	switch e := errors.Cause(err).(type) {
+	case K6Error:
+		return e.Code, e.Message
 	case *net.DNSError:
 		switch e.Err {
 		case "no such host": // defined as private in the go stdlib
@@ -170,3 +180,27 @@ func errorCodeForError(err error) (errCode, string) {
 		return defaultErrorCode, err.Error()
 	}
 }
+
+// K6Error is a helper struct that enhances Go errors with custom k6-specific
+// error-codes and more user-readable error messages.
+type K6Error struct {
+	Code          errCode
+	Message       string
+	OriginalError error
+}
+
+// NewK6Error is the constructor for K6Error
+func NewK6Error(code errCode, msg string, originalErr error) K6Error {
+	return K6Error{code, msg, originalErr}
+}
+
+// Error implements the `error` interface, so K6Errors are normal Go errors.
+func (k6Err K6Error) Error() string {
+	return k6Err.Message
+}
+
+// Unwrap implements the `xerrors.Wrapper` interface, so K6Errors are a bit
+// future-proof Go 2 errors.
+func (k6Err K6Error) Unwrap() error {
+	return k6Err.OriginalError
+}

lib/netext/httpext/httpdebug_transport.go

@@ -0,0 +1,66 @@
+/*
+ *
+ * k6 - a next-generation load testing tool
+ * Copyright (C) 2019 Load Impact
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ *
+ */
+
+package httpext
+
+import (
+	"fmt"
+	"net/http"
+	"net/http/httputil"
+
+	log "github.com/sirupsen/logrus"
+)
+
+type httpDebugTransport struct {
+	//TODO: get the state and log to its Logger
+	originalTransport http.RoundTripper
+	httpDebugOption   string
+}
+
+// RoundTrip prints passing HTTP requests and received responses
+//
+// TODO: massively improve this, because the printed information can be wrong:
+//  - https://github.com/loadimpact/k6/issues/986
+//  - https://github.com/loadimpact/k6/issues/1042
+//  - https://github.com/loadimpact/k6/issues/774
+func (t httpDebugTransport) RoundTrip(req *http.Request) (*http.Response, error) {
+	t.debugRequest(req)
+	resp, err := t.originalTransport.RoundTrip(req)
+	t.debugResponse(resp)
+	return resp, err
+}
+
+func (t httpDebugTransport) debugRequest(req *http.Request) {
+	dump, err := httputil.DumpRequestOut(req, t.httpDebugOption == "full")
+	if err != nil {
+		log.Fatal(err) //TODO: fix...
+	}
+	fmt.Printf("Request:\n%s\n", dump) //TODO: fix...
+}
+
+func (t httpDebugTransport) debugResponse(res *http.Response) {
+	if res != nil {
+		dump, err := httputil.DumpResponse(res, t.httpDebugOption == "full")
+		if err != nil {
+			log.Fatal(err) //TODO: fix...
+		}
+		fmt.Printf("Response:\n%s\n", dump) //TODO: fix...
+	}
+}