Skip to content

Commit

Permalink
oasis-test-runner: custom CLI parameters
Browse files Browse the repository at this point in the history
  • Loading branch information
matevz committed Feb 25, 2020
1 parent 71b155d commit 2ade430
Show file tree
Hide file tree
Showing 11 changed files with 367 additions and 89 deletions.
2 changes: 1 addition & 1 deletion go/oasis-node/cmd/common/metrics/metrics.go
Original file line number Diff line number Diff line change
Expand Up @@ -224,7 +224,7 @@ func New(ctx context.Context) (service.BackgroundService, error) {
}

func init() {
Flags.String(cfgMetricsMode, metricsModeNone, "metrics (prometheus) mode")
Flags.String(cfgMetricsMode, metricsModeNone, "metrics (prometheus) mode: none, pull, push")
Flags.String(cfgMetricsAddr, "127.0.0.1:3000", "metrics pull/push address")
Flags.String(cfgMetricsPushJobName, "", "metrics push job name")
Flags.String(cfgMetricsPushInstanceLabel, "", "metrics push instance label")
Expand Down
2 changes: 1 addition & 1 deletion go/oasis-node/cmd/common/pprof/pprof.go
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,6 @@ package pprof

import (
"context"
"github.com/pkg/errors"
"net"
"net/http"
"net/http/pprof"
Expand All @@ -12,6 +11,7 @@ import (
runtimePprof "runtime/pprof"
"strconv"

"github.com/pkg/errors"
flag "github.com/spf13/pflag"
"github.com/spf13/viper"

Expand Down
273 changes: 218 additions & 55 deletions go/oasis-test-runner/cmd/root.go
Original file line number Diff line number Diff line change
Expand Up @@ -2,11 +2,14 @@
package cmd

import (
"encoding/json"
"fmt"
"io"
"io/ioutil"
"os"
"path/filepath"
"sort"
"strconv"
"strings"

"github.com/pkg/errors"
Expand All @@ -27,6 +30,7 @@ const (
cfgLogFmt = "log.format"
cfgLogLevel = "log.level"
cfgLogNoStdout = "log.no_stdout"
cfgNumRuns = "num_runs"
cfgTest = "test"
cfgParallelJobCount = "parallel.job_count"
cfgParallelJobIndex = "parallel.job_index"
Expand All @@ -49,6 +53,7 @@ var (
rootFlags = flag.NewFlagSet("", flag.ContinueOnError)

cfgFile string
numRuns int

scenarioMap = make(map[string]scenario.Scenario)
defaultScenarios []scenario.Scenario
Expand All @@ -72,14 +77,141 @@ func Execute() {
}

// RegisterNondefault adds a scenario to the runner.
func RegisterNondefault(scenario scenario.Scenario) error {
n := strings.ToLower(scenario.Name())
func RegisterNondefault(s scenario.Scenario) error {
n := strings.ToLower(s.Name())
if _, ok := scenarioMap[n]; ok {
return errors.New("root: scenario already registered: " + n)
}

scenarioMap[n] = scenario
scenarios = append(scenarios, scenario)
scenarioMap[n] = s
scenarios = append(scenarios, s)

params := s.Parameters()
if len(params) > 0 {
for k, v := range scenario.ParametersToStringMap(params) {
// Re-register rootFlags for test parameters.
rootFlags.StringSlice("params."+n+"."+k, []string{v}, "")
rootCmd.PersistentFlags().AddFlagSet(rootFlags)
_ = viper.BindPFlag("params."+n+"."+k, rootFlags.Lookup("params."+n+"."+k))
}
}

return nil
}

// parseTestParams parses --params.<test_name>.<key1>=<val1>,<val2>... flags combinations, clones provided proto-
// scenarios, and populates them so that each scenario instance has unique paramater set. Returns mapping test name ->
// list of scenario instances.
func parseTestParams(toRun []scenario.Scenario) (map[string][]scenario.Scenario, error) {
r := make(map[string][]scenario.Scenario)
for _, s := range toRun {
zippedParams := make(map[string][]string)
for k := range s.Parameters() {
userVal := viper.GetStringSlice("params." + s.Name() + "." + k)
if userVal == nil {
continue
}
zippedParams[k] = userVal
}

parameterSets := computeParamSets(zippedParams, map[string]string{})

// For each parameter set combination, clone a scenario and apply user-provided parameter value.
for _, ps := range parameterSets {
sCloned := s.Clone()
for k, userVal := range ps {
v := sCloned.Parameters()[k]
switch v := v.(type) {
case *int:
val, err := strconv.ParseInt(userVal, 10, 32)
if err != nil {
return nil, err
}
*v = int(val)
case *int64:
val, err := strconv.ParseInt(userVal, 10, 64)
if err != nil {
return nil, err
}
*v = val
case *float64:
val, err := strconv.ParseFloat(userVal, 64)
if err != nil {
return nil, err
}
*v = val
case *bool:
val, err := strconv.ParseBool(userVal)
if err != nil {
return nil, err
}
*v = val
case *string:
*v = userVal
default:
return nil, errors.New(fmt.Sprintf("cannot parse parameter. Unknown type %v", v))
}
}
r[s.Name()] = append(r[s.Name()], sCloned)
}

// No parameters provided over CLI, keep a single copy.
if len(parameterSets) == 0 {
r[s.Name()] = []scenario.Scenario{s}
}
}

return r, nil
}

// computeParamSets recursively combines a map of string slices into all possible key=>value parameter sets.
func computeParamSets(zp map[string][]string, ps map[string]string) []map[string]string {
// Recursion stops when zp is empty.
if len(zp) == 0 {
// XXX: How do I clone a map in golang?
psCloned := map[string]string{}
for k, v := range ps {
psCloned[k] = v
}
return []map[string]string{psCloned}
}

rps := []map[string]string{}

// XXX: How do I clone a map in golang?
zpCloned := map[string][]string{}
for k, v := range zp {
zpCloned[k] = v
}
// Take first element from cloned zp and do recursion.
for k, vals := range zpCloned {
delete(zpCloned, k)
for _, v := range vals {
ps[k] = v
rps = append(rps, computeParamSets(zpCloned, ps)...)
}
break
}

return rps
}

// writeParamSetToFile dumps test instance parameter set to test_instance_info.json file for debugging afterwards.
func writeTestInstanceInfo(dirName string, s scenario.Scenario, run int) error {
testInfo := scenario.TestInstanceInfo{
Name: s.Name(),
ParameterSet: scenario.ParametersToStringMap(s.Parameters()),
Run: run,
}

b, err := json.Marshal(testInfo)
if err != nil {
return err
}
if err = ioutil.WriteFile(dirName+"/test_instance_info.json", b, 0644); err != nil {
return err
}

return nil
}

Expand Down Expand Up @@ -176,61 +308,82 @@ func runRoot(cmd *cobra.Command, args []string) error {
parallelJobCount := viper.GetInt(cfgParallelJobCount)
parallelJobIndex := viper.GetInt(cfgParallelJobIndex)

for index, v := range toRun {
n := v.Name()

if index%parallelJobCount != parallelJobIndex {
logger.Info("skipping test case (assigned to different parallel job)",
"test", n,
)
continue
}

if excludeMap[strings.ToLower(n)] {
logger.Info("skipping test case (excluded by environment)",
"test", n,
)
continue
}
// Parse test parameters passed by CLI.
var toRunExploded map[string][]scenario.Scenario
toRunExploded, err = parseTestParams(toRun)
if err != nil {
return errors.Wrap(err, "root: failed to parse test params")
}

logger.Info("running test case",
"test", n,
)

childEnv, err := rootEnv.NewChild(n)
if err != nil {
logger.Error("failed to setup child environment",
"err", err,
"test", n,
)
return errors.Wrap(err, "root: failed to setup child environment")
}
// Run all test instances.
index := 0
for run := 0; run < numRuns; run++ {
for name, sc := range toRunExploded {
for i, v := range sc {
// Maintain unique scenario datadir.
n := fmt.Sprintf("%s-%d", name, run*len(sc)+i)

if index%parallelJobCount != parallelJobIndex {
logger.Info("skipping test case (assigned to different parallel job)",
"test", n,
)
continue
}

if excludeMap[strings.ToLower(n)] {
logger.Info("skipping test case (excluded by environment)",
"test", n,
)
continue
}

logger.Info("running test case",
"test", n,
)

if err = doScenario(childEnv, v); err != nil {
logger.Error("failed to run test case",
"err", err,
"test", n,
)
err = errors.Wrap(err, "root: failed to run test case")
}
childEnv, err := rootEnv.NewChild(n)
if err != nil {
logger.Error("failed to setup child environment",
"err", err,
"test", n,
)
return errors.Wrap(err, "root: failed to setup child environment")
}

// Dump current parameter set to file.
if err = writeTestInstanceInfo(childEnv.Dir(), v, run); err != nil {
return err
}

if err = doScenario(childEnv, v); err != nil {
logger.Error("failed to run test case",
"err", err,
"test", n,
)
err = errors.Wrap(err, "root: failed to run test case")
}

if cleanErr := doCleanup(childEnv); cleanErr != nil {
logger.Error("failed to clean up child envionment",
"err", cleanErr,
"test", n,
)
if err == nil {
err = errors.Wrap(cleanErr, "root: failed to clean up child enviroment")
}
}

if err != nil {
return err
}

logger.Info("passed test case",
"test", n,
)

if cleanErr := doCleanup(childEnv); cleanErr != nil {
logger.Error("failed to clean up child envionment",
"err", cleanErr,
"test", n,
)
if err == nil {
err = errors.Wrap(cleanErr, "root: failed to clean up child enviroment")
index++
}
}

if err != nil {
return err
}

logger.Info("passed test case",
"test", n,
)
}

return nil
Expand Down Expand Up @@ -297,7 +450,16 @@ func runList(cmd *cobra.Command, args []string) {
})

for _, v := range scenarios {
fmt.Printf(" * %v\n", v.Name())
fmt.Printf(" * %v", v.Name())
params := v.Parameters()
if len(params) > 0 {
fmt.Printf(" (parameters:")
for p := range params {
fmt.Printf(" %v", p)
}
fmt.Printf(")")
}
fmt.Printf("\n")
}
}
}
Expand All @@ -312,6 +474,7 @@ func init() {
rootFlags.Var(&logLevel, cfgLogLevel, "log level")
rootFlags.Bool(cfgLogNoStdout, false, "do not mutiplex logs to stdout")
rootFlags.StringSliceP(cfgTest, "t", nil, "test(s) to run")
rootFlags.IntVarP(&numRuns, cfgNumRuns, "n", 1, "number of runs for given test(s)")
rootFlags.Int(cfgParallelJobCount, 1, "(for CI) number of overall parallel jobs")
rootFlags.Int(cfgParallelJobIndex, 0, "(for CI) index of this parallel job")
_ = viper.BindPFlags(rootFlags)
Expand Down
25 changes: 25 additions & 0 deletions go/oasis-test-runner/cmd/root_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,25 @@
package cmd

import (
"testing"

"github.com/stretchr/testify/require"
)

func TestComputeParamSets(t *testing.T) {
zippedParams := map[string][]string{
"testParam1": []string{"1", "2", "3"},
"testParam2": []string{"a", "b"},
}

expectedParamSets := []map[string]string{
{"testParam1": "1", "testParam2": "a"},
{"testParam1": "1", "testParam2": "b"},
{"testParam1": "2", "testParam2": "a"},
{"testParam1": "2", "testParam2": "b"},
{"testParam1": "3", "testParam2": "a"},
{"testParam1": "3", "testParam2": "b"},
}

require.Equal(t, computeParamSets(zippedParams, map[string]string{}), expectedParamSets)
}
1 change: 1 addition & 0 deletions go/oasis-test-runner/oasis/oasis.go
Original file line number Diff line number Diff line change
Expand Up @@ -605,6 +605,7 @@ func (net *Network) startOasisNode(
"--log.format", "json",
"--log.file", nodeLogPath(node.dir),
"--genesis.file", net.GenesisPath(),
// TODO Matevz: Forward oasis-test-runner's metrics parameters here.
}
if len(subCmd) == 0 {
extraArgs = extraArgs.
Expand Down
Loading

0 comments on commit 2ade430

Please sign in to comment.