From d07fdcb2e41fa5df2b1032c741f53ca852bc8092 Mon Sep 17 00:00:00 2001 From: Gregory Shimansky Date: Tue, 13 Nov 2018 13:07:37 -0600 Subject: [PATCH] Added support for Wrk benchmark tool --- test/framework/dockerlauncher.go | 59 ++++++++++++++++++++------------ test/framework/main/perf.json | 2 +- test/framework/report.go | 11 +++++- test/framework/report_test.go | 28 ++++++++++----- test/framework/testsuite.go | 8 +++++ test/framework/types.go | 29 +++++++++++----- 6 files changed, 97 insertions(+), 40 deletions(-) diff --git a/test/framework/dockerlauncher.go b/test/framework/dockerlauncher.go index 96757812..a83514f8 100644 --- a/test/framework/dockerlauncher.go +++ b/test/framework/dockerlauncher.go @@ -50,6 +50,17 @@ var ( regexp.MustCompile(`average= *(\d+\.?\d*) μs$`), regexp.MustCompile(`stddev= *(\d+\.?\d*) μs$`), } + WrkStatsRegexps = [2]*regexp.Regexp{ + regexp.MustCompile(`Requests/sec: *(\d+\.?\d*)$`), + regexp.MustCompile(`Transfer/sec: *(\d+\.?\d*)([K|M|G|T|P])B$`), + } + unitScale = map[string]float64{ + "K": 1.0, + "M": 1024.0, + "G": 1024.0 * 1024.0, + "T": 1024.0 * 1024.0 * 1024.0, + "P": 1024.0 * 1024.0 * 1024.0 * 1024.0, + } NoDeleteContainersOnExit = false username string @@ -77,6 +88,7 @@ type RunningApp struct { CoresStats []CoresInfo abs *ApacheBenchmarkStats lats *LatencyStats + wrks *WrkBenchmarkStats Logger *Logger benchStartTime time.Time benchEndTime time.Time @@ -138,9 +150,10 @@ func (app *RunningApp) testRoutine(report chan<- TestReport, done <-chan struct{ app.Status = TestRunning if app.config.Type == TestAppApacheBenchmark { app.abs = &ApacheBenchmarkStats{} - } - if app.config.Type == TestAppLatency { + } else if app.config.Type == TestAppLatency { app.lats = &LatencyStats{} + } else if app.config.Type == TestAppWrkBenchmark { + app.wrks = &WrkBenchmarkStats{} } scanner := bufio.NewScanner(logs) @@ -156,6 +169,23 @@ func (app *RunningApp) testRoutine(report chan<- TestReport, done <-chan struct{ } else if TestFailedRegexp.FindStringIndex(str) != nil { status = TestReportedFailed } else { + fillstats := func(stats []float64, rexps []*regexp.Regexp) { + for iii := range rexps { + matches := rexps[iii].FindStringSubmatch(str) + if len(matches) >= 2 { + var value float64 + n, err := fmt.Sscanf(matches[1], "%f", &value) + if err == nil && n == 1 { + stats[iii] = value + if len(matches) == 3 { + // Next match is unit letter + stats[iii] *= unitScale[matches[2]] + } + } + } + } + } + // Scan for strings specific to test application type if app.config.Type == TestAppGo { // Get cores number information for NFF-Go application @@ -175,28 +205,13 @@ func (app *RunningApp) testRoutine(report chan<- TestReport, done <-chan struct{ } } else if app.config.Type == TestAppApacheBenchmark { // Get Apache Benchmark output - for iii := range ABStatsRegexps { - matches := ABStatsRegexps[iii].FindStringSubmatch(str) - if len(matches) == 2 { - var value float32 - n, err := fmt.Sscanf(matches[1], "%f", &value) - if err == nil && n == 1 { - app.abs.Stats[iii] = value - } - } - } + fillstats(app.abs.Stats[:], ABStatsRegexps[:]) } else if app.config.Type == TestAppLatency { // Get Latency perf test output - for iii := range LatStatsRegexps { - matches := LatStatsRegexps[iii].FindStringSubmatch(str) - if len(matches) == 2 { - var value float32 - n, err := fmt.Sscanf(matches[1], "%f", &value) - if err == nil && n == 1 { - app.lats.Stats[iii] = value - } - } - } + fillstats(app.lats.Stats[:], LatStatsRegexps[:]) + } else if app.config.Type == TestAppWrkBenchmark { + // Get Wrk Benchmark output + fillstats(app.wrks.Stats[:], WrkStatsRegexps[:]) } } diff --git a/test/framework/main/perf.json b/test/framework/main/perf.json index a460a3be..31be48d5 100644 --- a/test/framework/main/perf.json +++ b/test/framework/main/perf.json @@ -17,7 +17,7 @@ "OUTPORT1_2": "1", "PKTGENCOREMASK": "0x1ff", "PKTGENPORT": "[1:2-3].0, [4-5:6].1", - "CORES": "0-43" + "CORES": "0-43" }, "tests": [ { diff --git a/test/framework/report.go b/test/framework/report.go index bed5d888..a4079cec 100644 --- a/test/framework/report.go +++ b/test/framework/report.go @@ -30,6 +30,8 @@ type TestcaseReportInfo struct { ABStats *ApacheBenchmarkStats `json:",omitempty"` // Latency type tests LatStats *LatencyStats `json:",omitempty"` + // Latency type tests + WStats *WrkBenchmarkStats `json:",omitempty"` // Per application statistics Apps []RunningApp `json:"-"` } @@ -151,7 +153,14 @@ const ( Requested speed [Pkts/sec]{{index .Stats 1}} - {{end}}{{/* end with .LatStats */}}{{end}}{{/* end if .LatStats */}} + {{end}}{{/* end with .LatStats */}}{{end}}{{/* end if .LatStats */}}{{if .WStats}}{{with .WStats}} + + + + + + +
Requests per second [#/sec]{{index .Stats 0}}
Transfer rate [Kbytes/sec] received{{index .Stats 1}}
{{end}}{{/* end with .WStats */}}{{end}}{{/* end if .WStats */}} {{range $appindex, $appelement := .Apps}} diff --git a/test/framework/report_test.go b/test/framework/report_test.go index c6c9f3c2..31770669 100644 --- a/test/framework/report_test.go +++ b/test/framework/report_test.go @@ -54,12 +54,7 @@ func testScenarios(t *testing.T, logdir string, tests []TestcaseReportInfo) { } for iii := range tests { - select { - case report.Pipe <- tests[iii]: - t.Log("Reported test", iii) - case err := <-report.Done: - t.Fatal(err) - } + report.AddTestResult(&tests[iii]) } report.FinishReport() @@ -88,6 +83,8 @@ func testManyApps(t *testing.T, testtype TestType) { appConfig[iii].Type = TestAppApacheBenchmark } else if iii == 0 && testtype == TestTypeLatency { appConfig[iii].Type = TestAppLatency + } else if iii == 0 && testtype == TestTypeWrkBenchmark { + appConfig[iii].Type = TestAppWrkBenchmark } else { appConfig[iii].Type = TestAppGo } @@ -132,11 +129,15 @@ func testManyApps(t *testing.T, testtype TestType) { } } else if appConfig[iii].Type == TestAppApacheBenchmark { apps[iii].abs = &ApacheBenchmarkStats{ - Stats: [4]float32{111.111, 222.222, 333.333, 444.444}, + Stats: [4]float64{111.111, 222.222, 333.333, 444.444}, } } else if appConfig[iii].Type == TestAppLatency { apps[iii].lats = &LatencyStats{ - Stats: [5]float32{111.111, 1000000, 222.222, 333.333, 444.444}, + Stats: [5]float64{111.111, 1000000, 222.222, 333.333, 444.444}, + } + } else if appConfig[iii].Type == TestAppWrkBenchmark { + apps[iii].wrks = &WrkBenchmarkStats{ + Stats: [2]float64{111.111, 222.222}, } } else { // appConfig[iii].Type == TestAppGo apps[iii].CoresStats = make([]CoresInfo, NUM_MEASUREMENTS) @@ -183,6 +184,13 @@ func testManyApps(t *testing.T, testtype TestType) { break } } + } else if testtype == TestTypeWrkBenchmark { + for iii := range apps { + if apps[iii].config.Type == TestAppWrkBenchmark { + tests[jjj].WStats = apps[iii].wrks + break + } + } } } @@ -204,3 +212,7 @@ func TestApacheBenchmarkManyApps(t *testing.T) { func TestLatencyManyApps(t *testing.T) { testManyApps(t, TestTypeLatency) } + +func TestWrkBenchmarkManyApps(t *testing.T) { + testManyApps(t, TestTypeWrkBenchmark) +} diff --git a/test/framework/testsuite.go b/test/framework/testsuite.go index 40c1df9f..35e40da5 100644 --- a/test/framework/testsuite.go +++ b/test/framework/testsuite.go @@ -188,6 +188,14 @@ func (config *TestsuiteConfig) executeOneTest(test *TestConfig, logdir string, break } } + } else if test.Type == TestTypeWrkBenchmark { + // Find which app has Wrk Benchmark statistics report + for iii := range apps { + if apps[iii].config.Type == TestAppWrkBenchmark { + tri.WStats = apps[iii].wrks + break + } + } } return &tri diff --git a/test/framework/types.go b/test/framework/types.go index 3b1bfbb7..914806b8 100644 --- a/test/framework/types.go +++ b/test/framework/types.go @@ -61,6 +61,7 @@ const ( TestAppPktgen TestAppApacheBenchmark TestAppLatency + TestAppWrkBenchmark ) // UnmarshalJSON unmarshals data and checks app type validity. @@ -77,6 +78,7 @@ func (at *AppType) UnmarshalJSON(data []byte) error { "TestAppPktgen": TestAppPktgen, "TestAppApacheBenchmark": TestAppApacheBenchmark, "TestAppLatency": TestAppLatency, + "TestAppWrkBenchmark": TestAppWrkBenchmark, }[s] if !ok { return fmt.Errorf("invalid AppType %q", s) @@ -91,9 +93,10 @@ type TestType int // Constants for different test types. const ( TestTypeBenchmark TestType = iota + TestTypeScenario TestTypeApacheBenchmark TestTypeLatency - TestTypeScenario + TestTypeWrkBenchmark ) // UnmarshalJSON unmarshals data and checks test type validity. @@ -107,9 +110,10 @@ func (at *TestType) UnmarshalJSON(data []byte) error { // Use map to get int keys for string values got, ok := map[string]TestType{ "TestTypeBenchmark": TestTypeBenchmark, + "TestTypeScenario": TestTypeScenario, "TestTypeApacheBenchmark": TestTypeApacheBenchmark, "TestTypeLatency": TestTypeLatency, - "TestTypeScenario": TestTypeScenario, + "TestTypeWrkBenchmark": TestTypeWrkBenchmark, }[s] if !ok { return fmt.Errorf("invalid TestType %q", s) @@ -137,16 +141,16 @@ type ReportCoresInfo struct { // Indexes in array of Apache Benchmark stats ApacheBenchmarkStats const ( - RequestsPerSecond = iota - TimePerRequest - TimePerRequestConcurrent - TransferRate + AbRequestsPerSecond = iota + AbTimePerRequest + AbTimePerRequestConcurrent + AbTransferRate ) // ApacheBenchmarkStats has info about running Apache Benchmark web // client. type ApacheBenchmarkStats struct { - Stats [4]float32 + Stats [4]float64 } // Indexes in array of latency stats LatencyStats @@ -160,7 +164,16 @@ const ( // LatencyStats has info about finished latency perf test type LatencyStats struct { - Stats [5]float32 + Stats [5]float64 +} + +const ( + WrkRequestsPerSecond = iota + WrkTransferRate +) + +type WrkBenchmarkStats struct { + Stats [2]float64 } // TestReport has info about test status and application.