use simpler io.Writer for stdout, add --hash-body option

.gitignore

@@ -7,3 +7,4 @@ out
 .idea
 TODO.md
 dist
+ganda-amd64

Makefile

@@ -0,0 +1,22 @@
+all: lint test build
+
+lint:
+	go fmt
+	golint
+
+build: 
+	go build -o ganda -v
+
+test:
+	go test -v ./...
+
+install: lint test build
+	go install
+
+clean: 
+	go clean
+	rm -f ganda
+	rm -f ganda-amd64
+
+build-linux:
+	CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -o ganda-amd64 -v

config/config.go

@@ -11,6 +11,7 @@ type Config struct {
 	Insecure              bool
 	NoColor               bool
 	JsonEnvelope          bool
+	HashBody              bool
 	BaseDirectory         string
 	DataTemplate          string
 	RequestWorkers        int
@@ -31,6 +32,7 @@ func New() *Config {
 		Silent:                false,
 		NoColor:               false,
 		JsonEnvelope:          false,
+		HashBody:              false,
 		DataTemplate:          "",
 		RequestWorkers:        30,
 		SubdirLength:          0,

execcontext/execcontext.go

@@ -9,12 +9,14 @@ import (
 	"math"
 	"os"
 	"time"
+	"io"
 )
 
 type Context struct {
 	RequestMethod          string
 	WriteFiles             bool
 	JsonEnvelope           bool
+	HashBody               bool
 	Insecure               bool
 	BaseDirectory          string
 	DataTemplate           string
@@ -25,7 +27,7 @@ type Context struct {
 	ThrottlePerSecond      int
 	Retries                int
 	Logger                 *logger.LeveledLogger
-	Out                    *log.Logger
+	Out                    io.Writer
 	RequestHeaders         []config.RequestHeader
 	RequestScanner         *bufio.Scanner
 }
@@ -37,6 +39,7 @@ func New(conf *config.Config) (*Context, error) {
 		ConnectTimeoutDuration: time.Duration(conf.ConnectTimeoutSeconds) * time.Second,
 		Insecure:               conf.Insecure,
 		JsonEnvelope:           conf.JsonEnvelope,
+		HashBody:               conf.HashBody,
 		RequestMethod:          conf.RequestMethod,
 		BaseDirectory:          conf.BaseDirectory,
 		DataTemplate:           conf.DataTemplate,
@@ -45,7 +48,7 @@ func New(conf *config.Config) (*Context, error) {
 		ResponseWorkers:        conf.ResponseWorkers,
 		RequestHeaders:         conf.RequestHeaders,
 		ThrottlePerSecond:      math.MaxInt32,
-		Out:                    log.New(os.Stdout, "", 0),
+		Out:                    os.Stdout,
 		Logger:                 createLeveledLogger(conf),
 	}
 

main.go

@@ -98,9 +98,14 @@ func createApp() *cli.App {
 		},
 		cli.BoolFlag{
 			Name:        "json-envelope",
-			Usage:       "EXPERIMENTAL: if flag is present, emit result with JSON envelope with url, status, length, and body fields, assumes result is valid json",
+			Usage:       "EXPERIMENTAL: emit result with JSON envelope with url, status, length, and body fields, assumes result is valid json",
 			Destination: &conf.JsonEnvelope,
 		},
+		cli.BoolFlag{
+			Name:        "hash-body",
+			Usage:       "EXPERIMENTAL: instead of emitting full body in JSON, emit the SHA256 of the bytes of the body, useful for checksums, only has meaning with --json-envelope flag",
+			Destination: &conf.HashBody,
+		},
 		cli.IntFlag{
 			Name:        "retry",
 			Usage:       "max number of retries on transient errors (5XX status codes/timeouts) to attempt",
@@ -136,9 +141,8 @@ func createApp() *cli.App {
 }
 
 func run(context *execcontext.Context) {
-	bufferSize := 256
-	requestsChannel := make(chan *http.Request, bufferSize)
-	responsesChannel := make(chan *http.Response, bufferSize)
+	requestsChannel := make(chan *http.Request)
+	responsesChannel := make(chan *http.Response)
 
 	requestWaitGroup := requests.StartRequestWorkers(requestsChannel, responsesChannel, context)
 	responseWaitGroup := responses.StartResponseWorkers(responsesChannel, context)

main_test.go

@@ -8,6 +8,7 @@ import (
 	"github.com/tednaleid/ganda/config"
 	"github.com/tednaleid/ganda/execcontext"
 	"github.com/tednaleid/ganda/logger"
+	"io"
 	"log"
 	"math"
 	"net/http"
@@ -22,7 +23,7 @@ type Scaffold struct {
 	BaseURL           string
 	StandardOutBuffer *bytes.Buffer
 	LogBuffer         *bytes.Buffer
-	StandardOutMock   *log.Logger
+	StandardOutMock   io.Writer
 	LoggerMock        *log.Logger
 }
 
@@ -36,7 +37,7 @@ func NewScaffold(handler http.Handler) *Scaffold {
 		LogBuffer:         new(bytes.Buffer),
 	}
 
-	scaffold.StandardOutMock = log.New(scaffold.StandardOutBuffer, "", 0)
+	scaffold.StandardOutMock = scaffold.StandardOutBuffer
 	scaffold.LoggerMock = log.New(scaffold.LogBuffer, "", 0)
 
 	return &scaffold
@@ -49,7 +50,7 @@ func TestRequestHappyPathHeadersAndResults(t *testing.T) {
 		assert.Equal(t, r.Header["User-Agent"][0], "Go-http-client/1.1", "User-Agent header")
 		assert.Equal(t, r.Header["Connection"][0], "keep-alive", "Connection header")
 		assert.Equal(t, r.Header["Accept-Encoding"][0], "gzip", "Accept-Encoding header")
-		fmt.Fprintln(w, "Hello", r.URL.Path)
+		fmt.Fprint(w, "Hello ", r.URL.Path)
 	}))
 	defer scaffold.Server.Close()
 
@@ -65,7 +66,7 @@ func TestRequestHappyPathHeadersAndResults(t *testing.T) {
 func TestResponseHasJsonEnvelopeWhenRequested(t *testing.T) {
 	t.Parallel()
 	scaffold := NewScaffold(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-		fmt.Fprintln(w, "{ \"foo\": true }")
+		fmt.Fprint(w, "{ \"foo\": true }")
 	}))
 	defer scaffold.Server.Close()
 
@@ -75,7 +76,7 @@ func TestResponseHasJsonEnvelopeWhenRequested(t *testing.T) {
 	run(context)
 
 	assertOutput(t, scaffold,
-		"{ \"url\": \""+scaffold.BaseURL+"/bar\", \"code\": 200, \"length\": 16, \"body\": { \"foo\": true }\n }\n",
+		"{ \"url\": \""+scaffold.BaseURL+"/bar\", \"code\": 200, \"length\": 15, \"body\": { \"foo\": true } }\n",
 		"Response: 200 "+scaffold.BaseURL+"/bar\n")
 }
 
@@ -99,7 +100,7 @@ func TestTimeout(t *testing.T) {
 	t.Parallel()
 	scaffold := NewScaffold(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
 		time.Sleep(10 * time.Millisecond)
-		fmt.Fprintln(w, "Should not get this, should time out first")
+		fmt.Fprint(w, "Should not get this, should time out first")
 	}))
 	defer scaffold.Server.Close()
 
@@ -122,7 +123,7 @@ func TestRetryEnabledShouldRetry5XX(t *testing.T) {
 		if requests == 1 {
 			w.WriteHeader(500)
 		} else {
-			fmt.Fprintln(w, "Retried request")
+			fmt.Fprint(w, "Retried request")
 		}
 	}))
 	defer scaffold.Server.Close()
@@ -189,7 +190,7 @@ func TestRetryEnabledShouldRetryTimeout(t *testing.T) {
 			time.Sleep(10 * time.Millisecond)
 		}
 		requestCount++
-		fmt.Fprintln(w, "Request", requestCount)
+		fmt.Fprint(w, "Request ", requestCount)
 	}))
 	defer scaffold.Server.Close()
 
@@ -212,7 +213,7 @@ func TestAddHeadersToRequestCreatesCanonicalKeys(t *testing.T) {
 		// turns to uppercase versions for header key when transmitted
 		assert.Equal(t, r.Header["Foo"][0], "bar", "foo header")
 		assert.Equal(t, r.Header["X-Baz"][0], "qux", "baz header")
-		fmt.Fprintln(w, "Hello", r.URL.Path)
+		fmt.Fprint(w, "Hello ", r.URL.Path)
 	}))
 	defer scaffold.Server.Close()
 
@@ -242,8 +243,10 @@ func newTestContext(scaffold *Scaffold, expectedURLPaths []string) *execcontext.
 }
 
 func assertOutput(t *testing.T, scaffold *Scaffold, expectedStandardOut string, expectedLog string) {
-	assert.Equal(t, expectedStandardOut, scaffold.StandardOutBuffer.String(), "expected stdout")
-	assert.Equal(t, expectedLog, scaffold.LogBuffer.String(), "expected logger stderr")
+	actualOut := scaffold.StandardOutBuffer.String()
+	assert.Equal(t, expectedStandardOut, actualOut, "expected stdout")
+	actualLog := scaffold.LogBuffer.String()
+	assert.Equal(t, expectedLog, actualLog, "expected logger stderr")
 }
 
 func urlsScanner(urls []string) *bufio.Scanner {

responses/responses.go

@@ -5,12 +5,12 @@ import (
 	"crypto/md5"
 	"fmt"
 	"github.com/tednaleid/ganda/execcontext"
-	"github.com/tednaleid/ganda/logger"
 	"io"
 	"net/http"
 	"os"
 	"regexp"
 	"sync"
+	"crypto/sha256"
 )
 
 func StartResponseWorkers(responses <-chan *http.Response, context *execcontext.Context) *sync.WaitGroup {
@@ -34,46 +34,73 @@ func StartResponseWorkers(responses <-chan *http.Response, context *execcontext.
 func responseSavingWorker(responses <-chan *http.Response, context *execcontext.Context) {
 	specialCharactersRegexp := regexp.MustCompile("[^A-Za-z0-9]+")
 
-	responseWorker(responses, context.Logger, func(response *http.Response) {
+	responseWorker(responses, func(response *http.Response) {
 		filename := specialCharactersRegexp.ReplaceAllString(response.Request.URL.String(), "-")
 		fullPath := saveBodyToFile(context.BaseDirectory, context.SubdirLength, filename, response.Body)
 		context.Logger.LogResponse(response.StatusCode, response.Request.URL.String()+" -> "+fullPath)
 	})
 }
 
 func responsePrintingWorker(responses <-chan *http.Response, context *execcontext.Context) {
-
-	responseWorker(responses, context.Logger, func(response *http.Response) {
-		printResponse(response, context)
+	emitResponseFn := determineEmitResponseFn(context)
+	out := context.Out
+	responseWorker(responses, func(response *http.Response) {
+		context.Logger.LogResponse(response.StatusCode, response.Request.URL.String())
+		emitResponseFn(response, out)
 	})
 }
 
-func printResponse(response *http.Response, context *execcontext.Context) {
+type emitResponseFn func(response *http.Response, out io.Writer)
+
+func determineEmitResponseFn(context *execcontext.Context) emitResponseFn {
+	if context.JsonEnvelope {
+		if context.HashBody {
+			return emitJsonMessageSha256
+		}
+		return emitJsonMessages
+	}
+	return emitRawMessages
+}
+
+func emitRawMessages(response *http.Response, out io.Writer) {
 	defer response.Body.Close()
-	context.Logger.LogResponse(response.StatusCode, response.Request.URL.String())
 	buf := new(bytes.Buffer)
 	buf.ReadFrom(response.Body)
 
-	if context.JsonEnvelope {
-		if buf.Len() > 0 {
-			context.Out.Printf("{ \"url\": \"%s\", \"code\": %d, \"length\": %d, \"body\": %s }", response.Request.URL.String(), response.StatusCode, buf.Len(), buf)
+	if buf.Len() > 0 {
+		buf.WriteByte('\n')
+		out.Write(buf.Bytes())
+	}
+}
 
-		} else {
-			context.Out.Printf("{ \"url\": \"%s\", \"code\": %d, \"length\": %d, \"body\": null }", response.Request.URL.String(), response.StatusCode, 0)
-		}
+func emitJsonMessages(response *http.Response, out io.Writer) {
+	defer response.Body.Close()
+	buf := new(bytes.Buffer)
+	buf.ReadFrom(response.Body)
+
+	if buf.Len() > 0 {
+		fmt.Fprintf(out, "{ \"url\": \"%s\", \"code\": %d, \"length\": %d, \"body\": %s }\n", response.Request.URL.String(), response.StatusCode, buf.Len(), buf)
 	} else {
-		if buf.Len() > 0 {
-			context.Out.Printf("%s", buf)
-		}
+		fmt.Fprintf(out, "{ \"url\": \"%s\", \"code\": %d, \"length\": %d, \"body\": null }\n", response.Request.URL.String(), response.StatusCode, 0)
 	}
+}
+
+func emitJsonMessageSha256(response *http.Response, out io.Writer) {
+	defer response.Body.Close()
+	buf := new(bytes.Buffer)
+	buf.ReadFrom(response.Body)
 
+	if buf.Len() > 0 {
+		fmt.Fprintf(out, "{ \"url\": \"%s\", \"code\": %d, \"length\": %d, \"body\": \"%x\" }\n", response.Request.URL.String(), response.StatusCode, buf.Len(), sha256.Sum256(buf.Bytes()))
+	} else {
+		fmt.Fprintf(out, "{ \"url\": \"%s\", \"code\": %d, \"length\": %d, \"body\": null }\n", response.Request.URL.String(), response.StatusCode, 0)
+	}
 }
 
-func responseWorker(responses <-chan *http.Response, logger *logger.LeveledLogger, responseHandler func(*http.Response)) {
+func responseWorker(responses <-chan *http.Response, responseHandler func(*http.Response)) {
 	for response := range responses {
 		responseHandler(response)
 	}
-
 }
 
 func saveBodyToFile(baseDirectory string, subdirLength int, filename string, body io.ReadCloser) string {