pass through context accepts a TSV and supports RFC 4180

parser/parser.go

@@ -46,6 +46,8 @@ func SendUrlsRequests(
 	staticHeaders []config.RequestHeader,
 ) {
 	csvReader := csv.NewReader(reader)
+	csvReader.Comma = '\t'
+	csvReader.FieldsPerRecord = -1
 
 	for {
 		record, err := csvReader.Read()

parser/parser_test.go

@@ -13,20 +13,19 @@ func TestSendGetRequestUrlsHaveDefaultHeaders(t *testing.T) {
 	t.Parallel()
 
 	requestsWithContext := make(chan parser.RequestWithContext)
-	firstUrl := "https://example.com/bar"
-	secondUrl := "https://example.com/qux"
+	defer close(requestsWithContext)
 
-	var in = inputReader([]string{firstUrl, secondUrl})
-	var requestMethod = "GET"
-	var requestHeaders []config.RequestHeader
+	firstLine := "https://example.com/bar"
+	secondLine := "https://example.com/qux"
+	var in = inputReader([]string{firstLine, secondLine})
 
-	go parser.SendRequests(requestsWithContext, in, requestMethod, requestHeaders)
+	go parser.SendRequests(requestsWithContext, in, "GET", []config.RequestHeader{})
 
 	requestWithContext := <-requestsWithContext
 	request := requestWithContext.Request
 	requestContext := requestWithContext.RequestContext
 
-	assert.Equal(t, firstUrl, request.URL.String(), "expected url")
+	assert.Equal(t, firstLine, request.URL.String(), "expected url")
 	assert.Equal(t, "GET", request.Method, "expected method")
 	assert.Equal(t, request.Header["Connection"][0], "keep-alive", "Connection header")
 	assert.Equal(t, []string{}, requestContext, "expected nil context")
@@ -35,15 +34,60 @@ func TestSendGetRequestUrlsHaveDefaultHeaders(t *testing.T) {
 	secondRequest := secondRequestWithContext.Request
 	secondRequestContext := secondRequestWithContext.RequestContext
 
-	assert.Equal(t, secondUrl, secondRequest.URL.String(), "expected url")
+	assert.Equal(t, secondLine, secondRequest.URL.String(), "expected url")
 	assert.Equal(t, []string{}, secondRequestContext, "expected nil context")
 
-	close(requestsWithContext)
 }
 
-// TODO start here: test sending context through
+func TestSendRequestsHasRaggedRequestContext(t *testing.T) {
+	t.Parallel()
+
+	requestsWithContext := make(chan parser.RequestWithContext)
+	defer close(requestsWithContext)
+
+	// we allow ragged numbers of fields in the TSV input
+	// we also follow the quoting rules in RFC 4180, so:
+	//    a double quote in a quoted field is escaped with another double quote
+	//    whitespace inside a quoted field is preserved
+	inputLines := `
+        https://ex.com/bar	foo	"quoted content"	
+		https://ex.com/qux	quux	"  ""quoted with whitespace""  "	456
+		https://ex.com/123	"baz"
+    `
+
+	var in = trimmedInputReader(inputLines)
+
+	go parser.SendRequests(requestsWithContext, in, "GET", []config.RequestHeader{})
+
+	expectedResults := []struct {
+		url     string
+		context []string
+	}{
+		{"https://ex.com/bar", []string{"foo", "quoted content"}},
+		{"https://ex.com/qux", []string{"quux", "  \"quoted with whitespace\"  ", "456"}},
+		{"https://ex.com/123", []string{"baz"}},
+	}
+
+	for _, expectedResult := range expectedResults {
+		requestWithContext := <-requestsWithContext
+		request := requestWithContext.Request
+		requestContext := requestWithContext.RequestContext
+
+		assert.Equal(t, expectedResult.url, request.URL.String(), "expected url")
+		assert.Equal(t, expectedResult.context, requestContext, "expected context")
+	}
+
+}
 
 func inputReader(urls []string) io.Reader {
 	stringUrls := strings.Join(urls, "\n")
 	return strings.NewReader(stringUrls)
 }
+
+func trimmedInputReader(s string) io.Reader {
+	lines := strings.Split(s, "\n")
+	for i, line := range lines {
+		lines[i] = strings.TrimSpace(line)
+	}
+	return strings.NewReader(strings.Join(lines, "\n"))
+}