cmd: test command flags update (#3390)

After QA testing and review by Oisin, some changes were made.

- Rename `--cluster-lock-file-path` to `--lock-file` and `--cluster-definition-file-path` to `--definition-file` in test peers command. This is to be aligned with the rest of the CLI.
- Allow passing of cluster definition by URL to `--definition-file` flag.
- Remove [REQUIRED] from the MEV test `--beacon-node-endpoint` flag description, as this flag is required only if `--load-test` is enabled.
- Do single MEV block creation test, to which users can specify number of blocks to smoothen the result. Previously we had 2 test, one with single block and second with multiple (by default 3). Now it is only the second test with a default of 1.
- Rename the `--load-test-blocks` flag in MEV test to `--number-of-payloads`.
- Round the measurements - > 1s to 2 digits after the decimal, < 1s to whole number.
- Delete the fio command created test file.
- Add a missed return statement.
- Bring back the test command to alpha state 😔.

category: feature
ticket: none

cmd/cmd.go

@@ -49,6 +49,14 @@ func New() *cobra.Command {
 		newAlphaCmd(
 			newAddValidatorsCmd(runAddValidatorsSolo),
 			newViewClusterManifestCmd(runViewClusterManifest),
+			newTestCmd(
+				newTestAllCmd(runTestAll),
+				newTestPeersCmd(runTestPeers),
+				newTestBeaconCmd(runTestBeacon),
+				newTestValidatorCmd(runTestValidator),
+				newTestMEVCmd(runTestMEV),
+				newTestInfraCmd(runTestInfra),
+			),
 		),
 		newExitCmd(
 			newListActiveValidatorsCmd(runListActiveValidatorsCmd),
@@ -57,14 +65,6 @@ func New() *cobra.Command {
 			newFetchExitCmd(runFetchExit),
 		),
 		newUnsafeCmd(newRunCmd(app.Run, true)),
-		newTestCmd(
-			newTestAllCmd(runTestAll),
-			newTestPeersCmd(runTestPeers),
-			newTestBeaconCmd(runTestBeacon),
-			newTestValidatorCmd(runTestValidator),
-			newTestMEVCmd(runTestMEV),
-			newTestInfraCmd(runTestInfra),
-		),
 	)
 }
 

cmd/duration.go

@@ -65,3 +65,16 @@ func (d *Duration) UnmarshalText(b []byte) error {
 
 	return nil
 }
+
+func RoundDuration(d Duration) Duration {
+	switch {
+	case d.Duration > time.Second:
+		return Duration{d.Round(10 * time.Millisecond)}
+	case d.Duration > time.Millisecond:
+		return Duration{d.Round(time.Millisecond)}
+	case d.Duration > time.Microsecond:
+		return Duration{d.Round(time.Microsecond)}
+	default:
+		return d
+	}
+}

cmd/duration_test.go

@@ -294,3 +294,43 @@ func TestDurationUnmarshalText(t *testing.T) {
 		})
 	}
 }
+
+func TestRoundDuration(t *testing.T) {
+	tests := []struct {
+		name     string
+		in       cmd.Duration
+		expected cmd.Duration
+	}{
+		{
+			name:     "15.151 milliseconds",
+			in:       cmd.Duration{15151 * time.Microsecond},
+			expected: cmd.Duration{15 * time.Millisecond},
+		},
+		{
+			name:     "15.151515 milliseconds",
+			in:       cmd.Duration{15151515 * time.Nanosecond},
+			expected: cmd.Duration{15 * time.Millisecond},
+		},
+		{
+			name:     "2.344444 seconds",
+			in:       cmd.Duration{2344444 * time.Microsecond},
+			expected: cmd.Duration{2340 * time.Millisecond},
+		},
+		{
+			name:     "2.345555 seconds",
+			in:       cmd.Duration{2345555 * time.Microsecond},
+			expected: cmd.Duration{2350 * time.Millisecond},
+		},
+		{
+			name:     "15.151 microsecond",
+			in:       cmd.Duration{15151 * time.Nanosecond},
+			expected: cmd.Duration{15 * time.Microsecond},
+		},
+	}
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			res := cmd.RoundDuration(test.in)
+			require.Equal(t, test.expected, res)
+		})
+	}
+}

cmd/test.go

@@ -380,7 +380,7 @@ func evaluateRTT(rtt time.Duration, testRes testResult, avg time.Duration, poor
 	} else {
 		testRes.Verdict = testVerdictGood
 	}
-	testRes.Measurement = Duration{rtt}.String()
+	testRes.Measurement = RoundDuration(Duration{rtt}).String()
 
 	return testRes
 }

cmd/testbeacon.go

@@ -830,14 +830,7 @@ func beaconSimulationTest(ctx context.Context, conf *testBeaconConfig, target st
 			highestRTT = sim.Max
 		}
 	}
-	if highestRTT.Duration > thresholdBeaconSimulationPoor {
-		testRes.Verdict = testVerdictPoor
-	} else if highestRTT.Duration > thresholdBeaconSimulationAvg {
-		testRes.Verdict = testVerdictAvg
-	} else {
-		testRes.Verdict = testVerdictGood
-	}
-	testRes.Measurement = highestRTT.String()
+	testRes = evaluateRTT(highestRTT.Duration, testRes, thresholdBeaconSimulationAvg, thresholdBeaconSimulationPoor)
 
 	log.Info(ctx, "Validators simulation finished",
 		z.Any("validators_count", params.TotalValidatorsCount),

cmd/testinfra.go

@@ -267,7 +267,7 @@ func infraDiskWriteSpeedTest(ctx context.Context, conf *testInfraConfig) testRes
 	} else {
 		testRes.Verdict = testVerdictGood
 	}
-	testRes.Measurement = strconv.FormatFloat(diskWriteMBs, 'f', 4, 64) + "MB/s"
+	testRes.Measurement = strconv.FormatFloat(diskWriteMBs, 'f', 2, 64) + "MB/s"
 
 	return testRes
 }
@@ -370,7 +370,7 @@ func infraDiskReadSpeedTest(ctx context.Context, conf *testInfraConfig) testResu
 	} else {
 		testRes.Verdict = testVerdictGood
 	}
-	testRes.Measurement = strconv.FormatFloat(diskReadMBs, 'f', 4, 64) + "MB/s"
+	testRes.Measurement = strconv.FormatFloat(diskReadMBs, 'f', 2, 64) + "MB/s"
 
 	return testRes
 }
@@ -514,16 +514,7 @@ func infraInternetLatencyTest(ctx context.Context, conf *testInfraConfig) testRe
 	if err != nil {
 		return failedTestResult(testRes, err)
 	}
-	latency := server.Latency
-
-	if latency > internetLatencyPoor {
-		testRes.Verdict = testVerdictPoor
-	} else if latency > internetLatencyAvg {
-		testRes.Verdict = testVerdictAvg
-	} else {
-		testRes.Verdict = testVerdictGood
-	}
-	testRes.Measurement = latency.Round(time.Microsecond).String()
+	testRes = evaluateRTT(server.Latency, testRes, internetLatencyAvg, internetLatencyPoor)
 
 	return testRes
 }
@@ -612,6 +603,11 @@ func fioCommand(ctx context.Context, filename string, blocksize int, operation s
 		return nil, errors.Wrap(err, "exec fio command")
 	}
 
+	err = os.Remove(fmt.Sprintf("%v/fiotest", filename))
+	if err != nil {
+		return nil, errors.Wrap(err, "delete fio test file")
+	}
+
 	return cmd, nil
 }
 

cmd/testmev.go

@@ -33,7 +33,7 @@ type testMEVConfig struct {
 	Endpoints          []string
 	BeaconNodeEndpoint string
 	LoadTest           bool
-	LoadTestBlocks     uint
+	NumberOfPayloads   uint
 }
 
 type testCaseMEV func(context.Context, *testMEVConfig, string) testResult
@@ -75,26 +75,29 @@ func newTestMEVCmd(runFunc func(context.Context, io.Writer, testMEVConfig) (test
 			return errors.New("beacon-node-endpoint should be specified when load-test is")
 		}
 
+		if loadTest == "false" && beaconNodeEndpoint != "" {
+			return errors.New("beacon-node-endpoint should be used only when load-test is")
+		}
+
 		return nil
 	})
 
 	return cmd
 }
 
 func bindTestMEVFlags(cmd *cobra.Command, config *testMEVConfig, flagsPrefix string) {
-	cmd.Flags().StringSliceVar(&config.Endpoints, flagsPrefix+"endpoints", nil, "[REQUIRED] Comma separated list of one or more MEV relay endpoint URLs.")
+	cmd.Flags().StringSliceVar(&config.Endpoints, flagsPrefix+"endpoints", nil, "Comma separated list of one or more MEV relay endpoint URLs.")
 	cmd.Flags().StringVar(&config.BeaconNodeEndpoint, flagsPrefix+"beacon-node-endpoint", "", "[REQUIRED] Beacon node endpoint URL used for block creation test.")
 	cmd.Flags().BoolVar(&config.LoadTest, flagsPrefix+"load-test", false, "Enable load test.")
-	cmd.Flags().UintVar(&config.LoadTestBlocks, flagsPrefix+"load-test-blocks", 3, "Amount of blocks the 'createMultipleBlocks' test will create.")
+	cmd.Flags().UintVar(&config.NumberOfPayloads, flagsPrefix+"number-of-payloads", 1, "Increases the accuracy of the load test by asking for multiple payloads. Increases test duration.")
 	mustMarkFlagRequired(cmd, flagsPrefix+"endpoints")
 }
 
 func supportedMEVTestCases() map[testCaseName]testCaseMEV {
 	return map[testCaseName]testCaseMEV{
-		{name: "Ping", order: 1}:                 mevPingTest,
-		{name: "PingMeasure", order: 2}:          mevPingMeasureTest,
-		{name: "CreateBlock", order: 3}:          mevCreateBlockTest,
-		{name: "CreateMultipleBlocks", order: 4}: mevCreateMultipleBlocksTest,
+		{name: "Ping", order: 1}:        mevPingTest,
+		{name: "PingMeasure", order: 2}: mevPingMeasureTest,
+		{name: "CreateBlock", order: 3}: mevCreateBlockTest,
 	}
 }
 
@@ -305,61 +308,16 @@ func mevCreateBlockTest(ctx context.Context, conf *testMEVConfig, target string)
 		return failedTestResult(testRes, err)
 	}
 
-	log.Info(ctx, "Starting attempts for block creation", z.Any("mev_relay", target))
-	rtt, err := createMEVBlock(ctx, conf, target, nextSlot, latestBlock, proposerDuties)
-	if err != nil {
-		return failedTestResult(testRes, err)
-	}
-	testRes = evaluateRTT(rtt, testRes, thresholdMEVBlockAvg, thresholdMEVBlockPoor)
-
-	return testRes
-}
-
-func mevCreateMultipleBlocksTest(ctx context.Context, conf *testMEVConfig, target string) testResult {
-	testRes := testResult{Name: "CreateMultipleBlocks"}
-
-	if !conf.LoadTest {
-		testRes.Verdict = testVerdictSkipped
-		return testRes
-	}
-
-	latestBlock, err := latestBeaconBlock(ctx, conf.BeaconNodeEndpoint)
-	if err != nil {
-		return failedTestResult(testRes, err)
-	}
-
-	// wait for beginning of next slot, as the block for current one might have already been proposed
-	latestBlockTSUnix, err := strconv.ParseInt(latestBlock.Body.ExecutionPayload.Timestamp, 10, 64)
-	if err != nil {
-		failedTestResult(testRes, err)
-	}
-	latestBlockTS := time.Unix(latestBlockTSUnix, 0)
-	nextBlockTS := latestBlockTS.Add(slotTime)
-	for time.Now().Before(nextBlockTS) && ctx.Err() == nil {
-		sleepWithContext(ctx, time.Millisecond)
-	}
-
-	latestSlot, err := strconv.ParseInt(latestBlock.Slot, 10, 64)
-	if err != nil {
-		return failedTestResult(testRes, err)
-	}
-	nextSlot := latestSlot + 1
-	epoch := nextSlot / slotsInEpoch
-	proposerDuties, err := fetchProposersForEpoch(ctx, conf, epoch)
-	if err != nil {
-		return failedTestResult(testRes, err)
-	}
-
 	allBlocksRTT := []time.Duration{}
-	log.Info(ctx, "Starting attempts for multiple block creation", z.Any("mev_relay", target), z.Any("blocks", conf.LoadTestBlocks))
+	log.Info(ctx, "Starting attempts for block creation", z.Any("mev_relay", target), z.Any("blocks", conf.NumberOfPayloads))
 	for ctx.Err() == nil {
 		startIteration := time.Now()
 		rtt, err := createMEVBlock(ctx, conf, target, nextSlot, latestBlock, proposerDuties)
 		if err != nil {
 			return failedTestResult(testRes, err)
 		}
 		allBlocksRTT = append(allBlocksRTT, rtt)
-		if len(allBlocksRTT) == int(conf.LoadTestBlocks) {
+		if len(allBlocksRTT) == int(conf.NumberOfPayloads) {
 			break
 		}
 		// wait for the next slot - time it took createMEVBlock - 1 sec

cmd/testmev_internal_test.go

@@ -57,7 +57,6 @@ func TestMEVTest(t *testing.T) {
 						{Name: "Ping", Verdict: testVerdictOk, Measurement: "", Suggestion: "", Error: testResultError{}},
 						{Name: "PingMeasure", Verdict: testVerdictGood, Measurement: "", Suggestion: "", Error: testResultError{}},
 						{Name: "CreateBlock", Verdict: testVerdictSkipped, Measurement: "", Suggestion: "", Error: testResultError{}},
-						{Name: "CreateMultipleBlocks", Verdict: testVerdictSkipped, Measurement: "", Suggestion: "", Error: testResultError{}},
 					},
 				},
 			},
@@ -82,7 +81,6 @@ func TestMEVTest(t *testing.T) {
 						{Name: "Ping", Verdict: testVerdictOk, Measurement: "", Suggestion: "", Error: testResultError{}},
 						{Name: "PingMeasure", Verdict: testVerdictGood, Measurement: "", Suggestion: "", Error: testResultError{}},
 						{Name: "CreateBlock", Verdict: testVerdictFail, Measurement: "", Suggestion: "", Error: testResultError{}},
-						{Name: "CreateMultipleBlocks", Verdict: testVerdictFail, Measurement: "", Suggestion: "", Error: testResultError{}},
 					},
 				},
 			},
@@ -105,13 +103,11 @@ func TestMEVTest(t *testing.T) {
 						{Name: "Ping", Verdict: testVerdictFail, Measurement: "", Suggestion: "", Error: testResultError{errors.New(fmt.Sprintf(`%v: connect: connection refused`, port1))}},
 						{Name: "PingMeasure", Verdict: testVerdictFail, Measurement: "", Suggestion: "", Error: testResultError{errors.New(fmt.Sprintf(`%v: connect: connection refused`, port1))}},
 						{Name: "CreateBlock", Verdict: testVerdictSkipped, Measurement: "", Suggestion: "", Error: testResultError{}},
-						{Name: "CreateMultipleBlocks", Verdict: testVerdictSkipped, Measurement: "", Suggestion: "", Error: testResultError{}},
 					},
 					endpoint2: {
 						{Name: "Ping", Verdict: testVerdictFail, Measurement: "", Suggestion: "", Error: testResultError{errors.New(fmt.Sprintf(`%v: connect: connection refused`, port2))}},
 						{Name: "PingMeasure", Verdict: testVerdictFail, Measurement: "", Suggestion: "", Error: testResultError{errors.New(fmt.Sprintf(`%v: connect: connection refused`, port2))}},
 						{Name: "CreateBlock", Verdict: testVerdictSkipped, Measurement: "", Suggestion: "", Error: testResultError{}},
-						{Name: "CreateMultipleBlocks", Verdict: testVerdictSkipped, Measurement: "", Suggestion: "", Error: testResultError{}},
 					},
 				},
 			},
@@ -157,13 +153,11 @@ func TestMEVTest(t *testing.T) {
 						{Name: "Ping", Verdict: testVerdictFail, Measurement: "", Suggestion: "", Error: testResultError{errors.New(fmt.Sprintf(`%v: connect: connection refused`, port1))}},
 						{Name: "PingMeasure", Verdict: testVerdictFail, Measurement: "", Suggestion: "", Error: testResultError{errors.New(fmt.Sprintf(`%v: connect: connection refused`, port1))}},
 						{Name: "CreateBlock", Verdict: testVerdictSkipped, Measurement: "", Suggestion: "", Error: testResultError{}},
-						{Name: "CreateMultipleBlocks", Verdict: testVerdictSkipped, Measurement: "", Suggestion: "", Error: testResultError{}},
 					},
 					endpoint2: {
 						{Name: "Ping", Verdict: testVerdictFail, Measurement: "", Suggestion: "", Error: testResultError{errors.New(fmt.Sprintf(`%v: connect: connection refused`, port2))}},
 						{Name: "PingMeasure", Verdict: testVerdictFail, Measurement: "", Suggestion: "", Error: testResultError{errors.New(fmt.Sprintf(`%v: connect: connection refused`, port2))}},
 						{Name: "CreateBlock", Verdict: testVerdictSkipped, Measurement: "", Suggestion: "", Error: testResultError{}},
-						{Name: "CreateMultipleBlocks", Verdict: testVerdictSkipped, Measurement: "", Suggestion: "", Error: testResultError{}},
 					},
 				},
 			},
@@ -223,13 +217,11 @@ func TestMEVTest(t *testing.T) {
 						{Name: "Ping", Verdict: testVerdictFail, Measurement: "", Suggestion: "", Error: testResultError{errors.New(fmt.Sprintf(`%v: connect: connection refused`, port1))}},
 						{Name: "PingMeasure", Verdict: testVerdictFail, Measurement: "", Suggestion: "", Error: testResultError{errors.New(fmt.Sprintf(`%v: connect: connection refused`, port1))}},
 						{Name: "CreateBlock", Verdict: testVerdictSkipped, Measurement: "", Suggestion: "", Error: testResultError{}},
-						{Name: "CreateMultipleBlocks", Verdict: testVerdictSkipped, Measurement: "", Suggestion: "", Error: testResultError{}},
 					},
 					endpoint2: {
 						{Name: "Ping", Verdict: testVerdictFail, Measurement: "", Suggestion: "", Error: testResultError{errors.New(fmt.Sprintf(`%v: connect: connection refused`, port2))}},
 						{Name: "PingMeasure", Verdict: testVerdictFail, Measurement: "", Suggestion: "", Error: testResultError{errors.New(fmt.Sprintf(`%v: connect: connection refused`, port2))}},
 						{Name: "CreateBlock", Verdict: testVerdictSkipped, Measurement: "", Suggestion: "", Error: testResultError{}},
-						{Name: "CreateMultipleBlocks", Verdict: testVerdictSkipped, Measurement: "", Suggestion: "", Error: testResultError{}},
 					},
 				},
 				Score:        categoryScoreC,