Add ability to add cpu and memory profile with loadgenerator (knative…

…#701)

* Add ability to add cpu and memory profile with loadgenerator

* Store profiles in artifacts dir

* Pass the filename to results

* Fail test when err happens

shared/loadgenerator/loadgenerator.go

@@ -23,6 +23,7 @@ import (
 	"fmt"
 	"log"
 	"os"
+	"path"
 	"time"
 
 	"fortio.org/fortio/fhttp"
@@ -64,11 +65,19 @@ type GeneratorOptions struct {
 	// LoadFactors defines the multiplier for baseQPS.
 	// Len(loadfactors) defines the number of QPS changes.
 	LoadFactors []float64
+	// FileNamePrefix is the prefix used to identify the stored files in artifacts.
+	// If not empty, this can be used to store cpu/mem profile from loadgenerator.
+	// Typically, we can use t.Name() to differentiate between the tests.
+	FileNamePrefix string
 }
 
 // GeneratorResults contains the results of running the per test
 type GeneratorResults struct {
 	Result []*fhttp.HTTPRunnerResults
+	// FileNamePrefix is the prefix used to identify the stored files in artifacts.
+	// This will be used to store the JSON output from loadgenerator.
+	// Typically, we can use t.Name() to differentiate between the tests.
+	FileNamePrefix string
 }
 
 // addDefaults adds default values to non mandatory params
@@ -87,7 +96,7 @@ func (g *GeneratorOptions) addDefaults() {
 }
 
 // CreateRunnerOptions sets up the fortio client with the knobs needed to run the load test
-func (g *GeneratorOptions) CreateRunnerOptions(resolvableDomain bool) *fhttp.HTTPRunnerOptions {
+func (g *GeneratorOptions) CreateRunnerOptions(resolvableDomain bool) (*fhttp.HTTPRunnerOptions, error) {
 	o := fhttp.NewHTTPOptions(g.URL)
 	o.NumConnections = g.NumConnections
 	o.HTTPReqTimeOut = g.RequestTimeout
@@ -97,7 +106,7 @@ func (g *GeneratorOptions) CreateRunnerOptions(resolvableDomain bool) *fhttp.HTT
 		o.AddAndValidateExtraHeader(fmt.Sprintf("Host: %s", g.Domain))
 	}
 
-	return &fhttp.HTTPRunnerOptions{
+	ro := fhttp.HTTPRunnerOptions{
 		RunnerOptions: periodic.RunnerOptions{
 			Duration:    g.Duration,
 			NumThreads:  g.NumThreads,
@@ -107,6 +116,17 @@ func (g *GeneratorOptions) CreateRunnerOptions(resolvableDomain bool) *fhttp.HTT
 		HTTPOptions:        *o,
 		AllowInitialErrors: g.AllowInitialErrors,
 	}
+
+	if len(g.FileNamePrefix) != 0 {
+		dir := prow.GetLocalArtifactsDir()
+		if err := common.CreateDir(dir); err != nil {
+			return nil, err
+		}
+
+		ro.Profiler = path.Join(dir, g.FileNamePrefix)
+	}
+
+	return &ro, nil
 }
 
 /*
@@ -115,8 +135,8 @@ By default, LoadFactors = [1] => test full load directly with no intermediate st
 
 For LoadFactors=[1,2,4], baseQPS=q, duration=d
 	QPS
-	|		|---d---|
-	|		|   |	|
+	|               |---d---|
+	|               |   |   |
 	|       |---d---|   |4q	|
 	|---d---|   |2q	    |	|
 	|___|q______|_______|___|____duration(time)
@@ -128,24 +148,28 @@ func (g *GeneratorOptions) RunLoadTest(resolvableDomain bool) (*GeneratorResults
 
 	for i, f := range g.LoadFactors {
 		g.BaseQPS = g.BaseQPS * f
-		r, err := fhttp.RunHTTPTest(g.CreateRunnerOptions(resolvableDomain))
+		ro, err := g.CreateRunnerOptions(resolvableDomain)
+		if err != nil {
+			return &GeneratorResults{Result: res}, err
+		}
+		r, err := fhttp.RunHTTPTest(ro)
 		if err != nil {
 			return &GeneratorResults{Result: res}, err
 		}
 		res[i] = r
 	}
 
-	return &GeneratorResults{Result: res}, nil
+	return &GeneratorResults{Result: res, FileNamePrefix: g.FileNamePrefix}, nil
 }
 
 // SaveJSON saves the results as Json in the artifacts directory
-func (gr *GeneratorResults) SaveJSON(testName string) error {
+func (gr *GeneratorResults) SaveJSON() error {
 	dir := prow.GetLocalArtifactsDir()
 	if err := common.CreateDir(dir); err != nil {
 		return err
 	}
 
-	outputFile := dir + "/" + testName + jsonExt
+	outputFile := path.Join(dir, gr.FileNamePrefix+jsonExt)
 	log.Printf("Storing json output in %s", outputFile)
 	f, err := os.OpenFile(outputFile, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0644)
 	if err != nil {

shared/loadgenerator/loadgenerator_test.go

@@ -25,8 +25,8 @@ import (
 )
 
 func TestSaveJSON(t *testing.T) {
-	res := &loadgenerator.GeneratorResults{}
-	err := res.SaveJSON("TestSaveJSON")
+	res := &loadgenerator.GeneratorResults{FileNamePrefix: t.Name()}
+	err := res.SaveJSON()
 	if err != nil {
 		t.Fatalf("Cannot save JSON: %v", err)
 	}

test/e2e/load_generator_test.go

@@ -25,30 +25,41 @@ import (
 	"github.com/knative/test-infra/shared/loadgenerator"
 )
 
-func loadTest(t *testing.T, factors []float64) {
+func loadTest(t *testing.T, factors bool, profiler bool) {
 	opts := loadgenerator.GeneratorOptions{
 		URL:            "http://www.google.com",
 		Duration:       10 * time.Second,
 		BaseQPS:        10,
 		NumThreads:     1,
 		NumConnections: 1,
 		RequestTimeout: 10 * time.Second,
-		LoadFactors:    factors,
+	}
+
+	if factors {
+		opts.LoadFactors = []float64{1, 2, 4}
+	}
+
+	if profiler {
+		opts.FileNamePrefix = t.Name()
 	}
 
 	res, err := opts.RunLoadTest(true)
 	if err != nil {
 		t.Fatalf("Error performing load test: %v", err)
 	}
 
-	if len(res.Result) != len(factors) {
-		t.Logf("got:%d, want: %d", len(res.Result), len(factors))
+	if factors && len(res.Result) != 3 {
+		t.Fatalf("got:%d, want: 3", len(res.Result))
 	}
 }
-func TestLoadGeneratorFullLoad(t *testing.T) {
-	loadTest(t, []float64{1})
+func TestFullLoad(t *testing.T) {
+	loadTest(t, false, false)
+}
+
+func TestStepLoad(t *testing.T) {
+	loadTest(t, true, false)
 }
 
-func TestLoadGeneratorStepLoad(t *testing.T) {
-	loadTest(t, []float64{1, 2, 4})
+func TestProfiler(t *testing.T) {
+	loadTest(t, false, true)
 }