Merge pull request #496 from intel-go/develop

Release 0.6.1 with testing infrastructure improvements

low/low.go

@@ -643,7 +643,7 @@ func ReportMempoolsState() {
 func CreateKni(portId uint16, core uint, name string) error {
 	mempool := (*C.struct_rte_mempool)(CreateMempool("KNI"))
 	if C.create_kni(C.uint16_t(portId), C.uint32_t(core), C.CString(name), mempool) != 0 {
-		common.WrapWithNFError(nil, "Error with KNI allocation\n", common.FailToCreateKNI)
+		return common.WrapWithNFError(nil, "Error with KNI allocation\n", common.FailToCreateKNI)
 	}
 	return nil
 }

test/framework/dockerlauncher.go

@@ -50,6 +50,17 @@ var (
 		regexp.MustCompile(`average= *(\d+\.?\d*) μs$`),
 		regexp.MustCompile(`stddev= *(\d+\.?\d*) μs$`),
 	}
+	WrkStatsRegexps = [2]*regexp.Regexp{
+		regexp.MustCompile(`Requests/sec: *(\d+\.?\d*)$`),
+		regexp.MustCompile(`Transfer/sec: *(\d+\.?\d*)([K|M|G|T|P])B$`),
+	}
+	unitScale = map[string]float64{
+		"K": 1.0,
+		"M": 1024.0,
+		"G": 1024.0 * 1024.0,
+		"T": 1024.0 * 1024.0 * 1024.0,
+		"P": 1024.0 * 1024.0 * 1024.0 * 1024.0,
+	}
 
 	NoDeleteContainersOnExit = false
 	username                 string
@@ -77,6 +88,7 @@ type RunningApp struct {
 	CoresStats      []CoresInfo
 	abs             *ApacheBenchmarkStats
 	lats            *LatencyStats
+	wrks            *WrkBenchmarkStats
 	Logger          *Logger
 	benchStartTime  time.Time
 	benchEndTime    time.Time
@@ -138,9 +150,10 @@ func (app *RunningApp) testRoutine(report chan<- TestReport, done <-chan struct{
 	app.Status = TestRunning
 	if app.config.Type == TestAppApacheBenchmark {
 		app.abs = &ApacheBenchmarkStats{}
-	}
-	if app.config.Type == TestAppLatency {
+	} else if app.config.Type == TestAppLatency {
 		app.lats = &LatencyStats{}
+	} else if app.config.Type == TestAppWrkBenchmark {
+		app.wrks = &WrkBenchmarkStats{}
 	}
 
 	scanner := bufio.NewScanner(logs)
@@ -156,6 +169,23 @@ func (app *RunningApp) testRoutine(report chan<- TestReport, done <-chan struct{
 		} else if TestFailedRegexp.FindStringIndex(str) != nil {
 			status = TestReportedFailed
 		} else {
+			fillstats := func(stats []float64, rexps []*regexp.Regexp) {
+				for iii := range rexps {
+					matches := rexps[iii].FindStringSubmatch(str)
+					if len(matches) >= 2 {
+						var value float64
+						n, err := fmt.Sscanf(matches[1], "%f", &value)
+						if err == nil && n == 1 {
+							stats[iii] = value
+							if len(matches) == 3 {
+								// Next match is unit letter
+								stats[iii] *= unitScale[matches[2]]
+							}
+						}
+					}
+				}
+			}
+
 			// Scan for strings specific to test application type
 			if app.config.Type == TestAppGo {
 				// Get cores number information for NFF-Go application
@@ -175,28 +205,13 @@ func (app *RunningApp) testRoutine(report chan<- TestReport, done <-chan struct{
 				}
 			} else if app.config.Type == TestAppApacheBenchmark {
 				// Get Apache Benchmark output
-				for iii := range ABStatsRegexps {
-					matches := ABStatsRegexps[iii].FindStringSubmatch(str)
-					if len(matches) == 2 {
-						var value float32
-						n, err := fmt.Sscanf(matches[1], "%f", &value)
-						if err == nil && n == 1 {
-							app.abs.Stats[iii] = value
-						}
-					}
-				}
+				fillstats(app.abs.Stats[:], ABStatsRegexps[:])
 			} else if app.config.Type == TestAppLatency {
 				// Get Latency perf test output
-				for iii := range LatStatsRegexps {
-					matches := LatStatsRegexps[iii].FindStringSubmatch(str)
-					if len(matches) == 2 {
-						var value float32
-						n, err := fmt.Sscanf(matches[1], "%f", &value)
-						if err == nil && n == 1 {
-							app.lats.Stats[iii] = value
-						}
-					}
-				}
+				fillstats(app.lats.Stats[:], LatStatsRegexps[:])
+			} else if app.config.Type == TestAppWrkBenchmark {
+				// Get Wrk Benchmark output
+				fillstats(app.wrks.Stats[:], WrkStatsRegexps[:])
 			}
 		}
 

test/framework/main/perf.json

@@ -17,7 +17,7 @@
         "OUTPORT1_2": "1",
         "PKTGENCOREMASK": "0x1ff",
         "PKTGENPORT": "[1:2-3].0, [4-5:6].1",
-	"CORES": "0-43"
+	    "CORES": "0-43"
     },
     "tests": [
         {

test/framework/report.go

@@ -30,6 +30,8 @@ type TestcaseReportInfo struct {
 	ABStats *ApacheBenchmarkStats `json:",omitempty"`
 	// Latency type tests
 	LatStats *LatencyStats `json:",omitempty"`
+	// Latency type tests
+	WStats *WrkBenchmarkStats `json:",omitempty"`
 	// Per application statistics
 	Apps []RunningApp `json:"-"`
 }
@@ -151,7 +153,14 @@ const (
             </tr><tr>
                 <td class="thinrborderstatnames">Requested speed [Pkts/sec]</td><td>{{index .Stats 1}}</td>
             </tr>
-        </table>{{end}}{{/* end with .LatStats */}}{{end}}{{/* end if .LatStats */}}
+        </table>{{end}}{{/* end with .LatStats */}}{{end}}{{/* end if .LatStats */}}{{if .WStats}}{{with .WStats}}
+        <table class="bench">
+            <tr>
+                <td class="thinrborderstatnames">Requests per second [#/sec]</td><td>{{index .Stats 0}}</td>
+            </tr><tr>
+                <td class="thinrborderstatnames">Transfer rate [Kbytes/sec] received</td><td>{{index .Stats 1}}</td>
+            </tr>
+        </table>{{end}}{{/* end with .WStats */}}{{end}}{{/* end if .WStats */}}
     </td>
     </tr>{{range $appindex, $appelement := .Apps}}<tr style='display:none' id='{{genappid $testindex $appindex}}'>
     <td>

test/framework/report_test.go

@@ -54,12 +54,7 @@ func testScenarios(t *testing.T, logdir string, tests []TestcaseReportInfo) {
 	}
 
 	for iii := range tests {
-		select {
-		case report.Pipe <- tests[iii]:
-			t.Log("Reported test", iii)
-		case err := <-report.Done:
-			t.Fatal(err)
-		}
+		report.AddTestResult(&tests[iii])
 	}
 
 	report.FinishReport()
@@ -88,6 +83,8 @@ func testManyApps(t *testing.T, testtype TestType) {
 				appConfig[iii].Type = TestAppApacheBenchmark
 			} else if iii == 0 && testtype == TestTypeLatency {
 				appConfig[iii].Type = TestAppLatency
+			} else if iii == 0 && testtype == TestTypeWrkBenchmark {
+				appConfig[iii].Type = TestAppWrkBenchmark
 			} else {
 				appConfig[iii].Type = TestAppGo
 			}
@@ -132,11 +129,15 @@ func testManyApps(t *testing.T, testtype TestType) {
 				}
 			} else if appConfig[iii].Type == TestAppApacheBenchmark {
 				apps[iii].abs = &ApacheBenchmarkStats{
-					Stats: [4]float32{111.111, 222.222, 333.333, 444.444},
+					Stats: [4]float64{111.111, 222.222, 333.333, 444.444},
 				}
 			} else if appConfig[iii].Type == TestAppLatency {
 				apps[iii].lats = &LatencyStats{
-					Stats: [5]float32{111.111, 1000000, 222.222, 333.333, 444.444},
+					Stats: [5]float64{111.111, 1000000, 222.222, 333.333, 444.444},
+				}
+			} else if appConfig[iii].Type == TestAppWrkBenchmark {
+				apps[iii].wrks = &WrkBenchmarkStats{
+					Stats: [2]float64{111.111, 222.222},
 				}
 			} else { // appConfig[iii].Type == TestAppGo
 				apps[iii].CoresStats = make([]CoresInfo, NUM_MEASUREMENTS)
@@ -183,6 +184,13 @@ func testManyApps(t *testing.T, testtype TestType) {
 					break
 				}
 			}
+		} else if testtype == TestTypeWrkBenchmark {
+			for iii := range apps {
+				if apps[iii].config.Type == TestAppWrkBenchmark {
+					tests[jjj].WStats = apps[iii].wrks
+					break
+				}
+			}
 		}
 	}
 
@@ -204,3 +212,7 @@ func TestApacheBenchmarkManyApps(t *testing.T) {
 func TestLatencyManyApps(t *testing.T) {
 	testManyApps(t, TestTypeLatency)
 }
+
+func TestWrkBenchmarkManyApps(t *testing.T) {
+	testManyApps(t, TestTypeWrkBenchmark)
+}

test/framework/testsuite.go

@@ -188,6 +188,14 @@ func (config *TestsuiteConfig) executeOneTest(test *TestConfig, logdir string,
 				break
 			}
 		}
+	} else if test.Type == TestTypeWrkBenchmark {
+		// Find which app has Wrk Benchmark statistics report
+		for iii := range apps {
+			if apps[iii].config.Type == TestAppWrkBenchmark {
+				tri.WStats = apps[iii].wrks
+				break
+			}
+		}
 	}
 
 	return &tri

test/framework/types.go

@@ -61,6 +61,7 @@ const (
 	TestAppPktgen
 	TestAppApacheBenchmark
 	TestAppLatency
+	TestAppWrkBenchmark
 )
 
 // UnmarshalJSON unmarshals data and checks app type validity.
@@ -77,6 +78,7 @@ func (at *AppType) UnmarshalJSON(data []byte) error {
 		"TestAppPktgen":          TestAppPktgen,
 		"TestAppApacheBenchmark": TestAppApacheBenchmark,
 		"TestAppLatency":         TestAppLatency,
+		"TestAppWrkBenchmark":    TestAppWrkBenchmark,
 	}[s]
 	if !ok {
 		return fmt.Errorf("invalid AppType %q", s)
@@ -91,9 +93,10 @@ type TestType int
 // Constants for different test types.
 const (
 	TestTypeBenchmark TestType = iota
+	TestTypeScenario
 	TestTypeApacheBenchmark
 	TestTypeLatency
-	TestTypeScenario
+	TestTypeWrkBenchmark
 )
 
 // UnmarshalJSON unmarshals data and checks test type validity.
@@ -107,9 +110,10 @@ func (at *TestType) UnmarshalJSON(data []byte) error {
 	// Use map to get int keys for string values
 	got, ok := map[string]TestType{
 		"TestTypeBenchmark":       TestTypeBenchmark,
+		"TestTypeScenario":        TestTypeScenario,
 		"TestTypeApacheBenchmark": TestTypeApacheBenchmark,
 		"TestTypeLatency":         TestTypeLatency,
-		"TestTypeScenario":        TestTypeScenario,
+		"TestTypeWrkBenchmark":    TestTypeWrkBenchmark,
 	}[s]
 	if !ok {
 		return fmt.Errorf("invalid TestType %q", s)
@@ -137,16 +141,16 @@ type ReportCoresInfo struct {
 
 // Indexes in array of Apache Benchmark stats ApacheBenchmarkStats
 const (
-	RequestsPerSecond = iota
-	TimePerRequest
-	TimePerRequestConcurrent
-	TransferRate
+	AbRequestsPerSecond = iota
+	AbTimePerRequest
+	AbTimePerRequestConcurrent
+	AbTransferRate
 )
 
 // ApacheBenchmarkStats has info about running Apache Benchmark web
 // client.
 type ApacheBenchmarkStats struct {
-	Stats [4]float32
+	Stats [4]float64
 }
 
 // Indexes in array of latency stats LatencyStats
@@ -160,7 +164,16 @@ const (
 
 // LatencyStats has info about finished latency perf test
 type LatencyStats struct {
-	Stats [5]float32
+	Stats [5]float64
+}
+
+const (
+	WrkRequestsPerSecond = iota
+	WrkTransferRate
+)
+
+type WrkBenchmarkStats struct {
+	Stats [2]float64
 }
 
 // TestReport has info about test status and application.

vagrant/Vagrantfile

@@ -33,7 +33,7 @@ Vagrant.configure(2) do |config|
   config.vm.provider "libvirt" do |lv|
     lv.driver = "kvm"
     lv.memory = "4096"
-    lv.cpus = 8
+    lv.cpus = 16
     lv.storage_pool_name = "images"
   end
 
@@ -52,7 +52,10 @@ sudo sed -i -e 's,biosdevname=0,biosdevname=1,' /etc/default/grub
 sudo sed -i -e 's,net.ifnames=0,net.ifnames=1,' /etc/default/grub
 sudo grub2-mkconfig -o /boot/grub2/grub.cfg
 
-if [ $(readlink /etc/udev/rules.d/80-net-setup-link.rules) == /dev/null ]
+echo Disabling SELinux
+sudo sed -i -e 's,SELINUX=enforcing,SELINUX=disabled,' /etc/selinux/config
+
+if [ "$(readlink /etc/udev/rules.d/80-net-setup-link.rules)" == /dev/null ]
 then
     echo Fixing udev to use consistent interface names
     sudo rm /etc/udev/rules.d/80-net-setup-link.rules
@@ -80,16 +83,21 @@ echo Reassigning "${syscon}" interface to system name
 sudo nmcli c mod "${syscon}" connection.id 'System connection'
 
 echo Unpacking Go language into /opt
-(cd /opt; sudo sh -c 'curl -L -s https://dl.google.com/go/go1.11.1.linux-amd64.tar.gz | tar zx')
+(cd /opt; sudo sh -c 'curl -L -s https://dl.google.com/go/go1.11.2.linux-amd64.tar.gz | tar zx')
 mkdir go
 chmod +x ~/scripts.sh
 . ~/scripts.sh
 echo . ~/scripts.sh >> .bashrc
-setupdocker
+setuptesthost
 
-echo Downloading and building NFF-GO
+echo Downloading and building NFF-GO framework
 go get -d -v github.com/intel-go/nff-go
-(cd \"$GOPATH\"/src/github.com/intel-go/nff-go; git checkout develop; ./scripts/get-depends.sh; make)
+(cd "${GOPATH}"/src/github.com/intel-go/nff-go; git checkout develop; ./scripts/get-depends.sh; make)
+echo Downloading and building NFF-GO NAT example and its dependencies
+go get github.com/golang/protobuf/protoc-gen-go
+git clone -b develop --recurse-submodules http://github.com/intel-go/nff-go-nat
+(cd nff-go-nat; . env.sh; make)
+./nff-go-nat/test/httpperfserv/install-systemd-service.sh "-port 8008"
 
 echo Setting up 1024 huge pages
 sudo sh -c 'echo 1024 > /sys/devices/system/node/node0/hugepages/hugepages-2048kB/nr_hugepages'