Merge pull request #2627 from grafana/test-pre-init-and-run-state

Create distinct test state objects for the pre-init and run phases and thread them everywhere

api/server_test.go

@@ -81,10 +81,19 @@ func TestWithEngine(t *testing.T) {
 	logger := logrus.New()
 	logger.SetOutput(testutils.NewTestOutput(t))
 	registry := metrics.NewRegistry()
-	builtinMetrics := metrics.RegisterBuiltinMetrics(registry)
-	execScheduler, err := local.NewExecutionScheduler(&minirunner.MiniRunner{}, builtinMetrics, logger)
+	testState := &lib.TestRunState{
+		TestPreInitState: &lib.TestPreInitState{
+			Logger:         logger,
+			Registry:       registry,
+			BuiltinMetrics: metrics.RegisterBuiltinMetrics(registry),
+		},
+		Options: lib.Options{},
+		Runner:  &minirunner.MiniRunner{},
+	}
+
+	execScheduler, err := local.NewExecutionScheduler(testState)
 	require.NoError(t, err)
-	engine, err := core.NewEngine(execScheduler, lib.Options{}, lib.RuntimeOptions{}, nil, logger, registry)
+	engine, err := core.NewEngine(testState, execScheduler, nil)
 	require.NoError(t, err)
 
 	rw := httptest.NewRecorder()

api/v1/group_routes_test.go

@@ -26,7 +26,6 @@ import (
 	"net/http/httptest"
 	"testing"
 
-	"github.com/sirupsen/logrus"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 
@@ -38,6 +37,25 @@ import (
 	"go.k6.io/k6/metrics"
 )
 
+func getTestPreInitState(tb testing.TB) *lib.TestPreInitState {
+	reg := metrics.NewRegistry()
+	return &lib.TestPreInitState{
+		Logger:         testutils.NewLogger(tb),
+		RuntimeOptions: lib.RuntimeOptions{},
+		Registry:       reg,
+		BuiltinMetrics: metrics.RegisterBuiltinMetrics(reg),
+	}
+}
+
+func getTestRunState(tb testing.TB, options lib.Options, runner lib.Runner) *lib.TestRunState {
+	require.NoError(tb, runner.SetOptions(runner.GetOptions().Apply(options)))
+	return &lib.TestRunState{
+		TestPreInitState: getTestPreInitState(tb),
+		Options:          options,
+		Runner:           runner,
+	}
+}
+
 func TestGetGroups(t *testing.T) {
 	g0, err := lib.NewGroup("", nil)
 	assert.NoError(t, err)
@@ -46,14 +64,10 @@ func TestGetGroups(t *testing.T) {
 	g2, err := g1.Group("group 2")
 	assert.NoError(t, err)
 
-	logger := logrus.New()
-	logger.SetOutput(testutils.NewTestOutput(t))
-
-	registry := metrics.NewRegistry()
-	builtinMetrics := metrics.RegisterBuiltinMetrics(registry)
-	execScheduler, err := local.NewExecutionScheduler(&minirunner.MiniRunner{Group: g0}, builtinMetrics, logger)
+	testState := getTestRunState(t, lib.Options{}, &minirunner.MiniRunner{Group: g0})
+	execScheduler, err := local.NewExecutionScheduler(testState)
 	require.NoError(t, err)
-	engine, err := core.NewEngine(execScheduler, lib.Options{}, lib.RuntimeOptions{}, nil, logger, registry)
+	engine, err := core.NewEngine(testState, execScheduler, nil)
 	require.NoError(t, err)
 
 	t.Run("list", func(t *testing.T) {

api/v1/metric_routes_test.go

@@ -26,31 +26,26 @@ import (
 	"net/http/httptest"
 	"testing"
 
-	"github.com/sirupsen/logrus"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 	"gopkg.in/guregu/null.v3"
 
 	"go.k6.io/k6/core"
 	"go.k6.io/k6/core/local"
 	"go.k6.io/k6/lib"
-	"go.k6.io/k6/lib/testutils"
 	"go.k6.io/k6/lib/testutils/minirunner"
 	"go.k6.io/k6/metrics"
 )
 
 func TestGetMetrics(t *testing.T) {
 	t.Parallel()
 
-	logger := logrus.New()
-	logger.SetOutput(testutils.NewTestOutput(t))
-	registry := metrics.NewRegistry()
-	builtinMetrics := metrics.RegisterBuiltinMetrics(registry)
-	testMetric, err := registry.NewMetric("my_metric", metrics.Trend, metrics.Time)
+	testState := getTestRunState(t, lib.Options{}, &minirunner.MiniRunner{})
+	testMetric, err := testState.Registry.NewMetric("my_metric", metrics.Trend, metrics.Time)
 	require.NoError(t, err)
-	execScheduler, err := local.NewExecutionScheduler(&minirunner.MiniRunner{}, builtinMetrics, logger)
+	execScheduler, err := local.NewExecutionScheduler(testState)
 	require.NoError(t, err)
-	engine, err := core.NewEngine(execScheduler, lib.Options{}, lib.RuntimeOptions{}, nil, logger, registry)
+	engine, err := core.NewEngine(testState, execScheduler, nil)
 	require.NoError(t, err)
 
 	engine.MetricsEngine.ObservedMetrics = map[string]*metrics.Metric{
@@ -104,15 +99,12 @@ func TestGetMetrics(t *testing.T) {
 func TestGetMetric(t *testing.T) {
 	t.Parallel()
 
-	logger := logrus.New()
-	logger.SetOutput(testutils.NewTestOutput(t))
-	registry := metrics.NewRegistry()
-	testMetric, err := registry.NewMetric("my_metric", metrics.Trend, metrics.Time)
+	testState := getTestRunState(t, lib.Options{}, &minirunner.MiniRunner{})
+	testMetric, err := testState.Registry.NewMetric("my_metric", metrics.Trend, metrics.Time)
 	require.NoError(t, err)
-	builtinMetrics := metrics.RegisterBuiltinMetrics(registry)
-	execScheduler, err := local.NewExecutionScheduler(&minirunner.MiniRunner{}, builtinMetrics, logger)
+	execScheduler, err := local.NewExecutionScheduler(testState)
 	require.NoError(t, err)
-	engine, err := core.NewEngine(execScheduler, lib.Options{}, lib.RuntimeOptions{}, nil, logger, registry)
+	engine, err := core.NewEngine(testState, execScheduler, nil)
 	require.NoError(t, err)
 
 	engine.MetricsEngine.ObservedMetrics = map[string]*metrics.Metric{

api/v1/setup_teardown_routes_test.go

@@ -31,7 +31,6 @@ import (
 	"testing"
 	"time"
 
-	"github.com/sirupsen/logrus"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 	"gopkg.in/guregu/null.v3"
@@ -40,10 +39,8 @@ import (
 	"go.k6.io/k6/core/local"
 	"go.k6.io/k6/js"
 	"go.k6.io/k6/lib"
-	"go.k6.io/k6/lib/testutils"
 	"go.k6.io/k6/lib/types"
 	"go.k6.io/k6/loader"
-	"go.k6.io/k6/metrics"
 )
 
 func TestSetupData(t *testing.T) {
@@ -140,32 +137,28 @@ func TestSetupData(t *testing.T) {
 		t.Run(testCase.name, func(t *testing.T) {
 			t.Parallel()
 
-			logger := logrus.New()
-			logger.SetOutput(testutils.NewTestOutput(t))
-			registry := metrics.NewRegistry()
-			builtinMetrics := metrics.RegisterBuiltinMetrics(registry)
-
+			piState := getTestPreInitState(t)
 			runner, err := js.New(
-				&lib.RuntimeState{
-					Logger:         logger,
-					BuiltinMetrics: builtinMetrics,
-					Registry:       registry,
-				},
-				&loader.SourceData{URL: &url.URL{Path: "/script.js"}, Data: testCase.script},
-				nil,
+				piState, &loader.SourceData{URL: &url.URL{Path: "/script.js"}, Data: testCase.script}, nil,
 			)
 			require.NoError(t, err)
-			runner.SetOptions(lib.Options{
+			require.NoError(t, runner.SetOptions(lib.Options{
 				Paused:          null.BoolFrom(true),
 				VUs:             null.IntFrom(2),
 				Iterations:      null.IntFrom(3),
 				NoSetup:         null.BoolFrom(true),
 				SetupTimeout:    types.NullDurationFrom(5 * time.Second),
 				TeardownTimeout: types.NullDurationFrom(5 * time.Second),
-			})
-			execScheduler, err := local.NewExecutionScheduler(runner, builtinMetrics, logger)
+			}))
+			testState := &lib.TestRunState{
+				TestPreInitState: piState,
+				Options:          runner.GetOptions(),
+				Runner:           runner,
+			}
+
+			execScheduler, err := local.NewExecutionScheduler(testState)
 			require.NoError(t, err)
-			engine, err := core.NewEngine(execScheduler, runner.GetOptions(), lib.RuntimeOptions{}, nil, logger, registry)
+			engine, err := core.NewEngine(testState, execScheduler, nil)
 			require.NoError(t, err)
 
 			require.NoError(t, engine.OutputManager.StartOutputs())

api/v1/status_routes_test.go

@@ -29,29 +29,23 @@ import (
 	"testing"
 	"time"
 
-	"github.com/sirupsen/logrus"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 	"gopkg.in/guregu/null.v3"
 
 	"go.k6.io/k6/core"
 	"go.k6.io/k6/core/local"
 	"go.k6.io/k6/lib"
-	"go.k6.io/k6/lib/testutils"
 	"go.k6.io/k6/lib/testutils/minirunner"
-	"go.k6.io/k6/metrics"
 )
 
 func TestGetStatus(t *testing.T) {
 	t.Parallel()
 
-	logger := logrus.New()
-	logger.SetOutput(testutils.NewTestOutput(t))
-	registry := metrics.NewRegistry()
-	builtinMetrics := metrics.RegisterBuiltinMetrics(registry)
-	execScheduler, err := local.NewExecutionScheduler(&minirunner.MiniRunner{}, builtinMetrics, logger)
+	testState := getTestRunState(t, lib.Options{}, &minirunner.MiniRunner{})
+	execScheduler, err := local.NewExecutionScheduler(testState)
 	require.NoError(t, err)
-	engine, err := core.NewEngine(execScheduler, lib.Options{}, lib.RuntimeOptions{}, nil, logger, registry)
+	engine, err := core.NewEngine(testState, execScheduler, nil)
 	require.NoError(t, err)
 
 	rw := httptest.NewRecorder()
@@ -128,21 +122,17 @@ func TestPatchStatus(t *testing.T) {
 	for name, testCase := range testData {
 		t.Run(name, func(t *testing.T) {
 			t.Parallel()
-			logger := logrus.New()
-			logger.SetOutput(testutils.NewTestOutput(t))
 
 			scenarios := lib.ScenarioConfigs{}
 			err := json.Unmarshal([]byte(`
 			{"external": {"executor": "externally-controlled",
 			"vus": 0, "maxVUs": 10, "duration": "0"}}`), &scenarios)
 			require.NoError(t, err)
-			options := lib.Options{Scenarios: scenarios}
 
-			registry := metrics.NewRegistry()
-			builtinMetrics := metrics.RegisterBuiltinMetrics(registry)
-			execScheduler, err := local.NewExecutionScheduler(&minirunner.MiniRunner{Options: options}, builtinMetrics, logger)
+			testState := getTestRunState(t, lib.Options{Scenarios: scenarios}, &minirunner.MiniRunner{})
+			execScheduler, err := local.NewExecutionScheduler(testState)
 			require.NoError(t, err)
-			engine, err := core.NewEngine(execScheduler, options, lib.RuntimeOptions{}, nil, logger, registry)
+			engine, err := core.NewEngine(testState, execScheduler, nil)
 			require.NoError(t, err)
 
 			require.NoError(t, engine.OutputManager.StartOutputs())

cmd/archive.go

@@ -43,13 +43,13 @@ func (c *cmdArchive) run(cmd *cobra.Command, args []string) error {
 	// an execution shortcut option (e.g. `iterations` or `duration`),
 	// we will have multiple conflicting execution options since the
 	// derivation will set `scenarios` as well.
-	err = test.initRunner.SetOptions(test.consolidatedConfig.Options)
+	testRunState, err := test.buildTestRunState(test.consolidatedConfig.Options)
 	if err != nil {
 		return err
 	}
 
 	// Archive.
-	arc := test.initRunner.MakeArchive()
+	arc := testRunState.Runner.MakeArchive()
 	f, err := c.gs.fs.Create(c.archiveOut)
 	if err != nil {
 		return err

cmd/cloud.go

@@ -102,7 +102,7 @@ func (c *cmdCloud) run(cmd *cobra.Command, args []string) error {
 	// an execution shortcut option (e.g. `iterations` or `duration`),
 	// we will have multiple conflicting execution options since the
 	// derivation will set `scenarios` as well.
-	err = test.initRunner.SetOptions(test.consolidatedConfig.Options)
+	testRunState, err := test.buildTestRunState(test.consolidatedConfig.Options)
 	if err != nil {
 		return err
 	}
@@ -112,7 +112,7 @@ func (c *cmdCloud) run(cmd *cobra.Command, args []string) error {
 	// TODO: move those validations to a separate function and reuse validateConfig()?
 
 	modifyAndPrintBar(c.gs, progressBar, pb.WithConstProgress(0, "Building the archive..."))
-	arc := test.initRunner.MakeArchive()
+	arc := testRunState.Runner.MakeArchive()
 
 	// TODO: Fix this
 	// We reuse cloud.Config for parsing options.ext.loadimpact, but this probably shouldn't be

cmd/inspect.go

@@ -40,7 +40,7 @@ func getCmdInspect(gs *globalState) *cobra.Command {
 		Long:  `Inspect a script or archive.`,
 		Args:  cobra.ExactArgs(1),
 		RunE: func(cmd *cobra.Command, args []string) error {
-			test, err := loadAndConfigureTest(gs, cmd, args, nil)
+			test, err := loadTest(gs, cmd, args)
 			if err != nil {
 				return err
 			}
@@ -82,24 +82,28 @@ func getCmdInspect(gs *globalState) *cobra.Command {
 // derive the value of `scenarios` and calculate the max test duration and VUs.
 func inspectOutputWithExecRequirements(gs *globalState, cmd *cobra.Command, test *loadedTest) (interface{}, error) {
 	// we don't actually support CLI flags here, so we pass nil as the getter
-	if err := test.consolidateDeriveAndValidateConfig(gs, cmd, nil); err != nil {
+	configuredTest, err := test.consolidateDeriveAndValidateConfig(gs, cmd, nil)
+	if err != nil {
 		return nil, err
 	}
 
-	et, err := lib.NewExecutionTuple(test.derivedConfig.ExecutionSegment, test.derivedConfig.ExecutionSegmentSequence)
+	et, err := lib.NewExecutionTuple(
+		configuredTest.derivedConfig.ExecutionSegment,
+		configuredTest.derivedConfig.ExecutionSegmentSequence,
+	)
 	if err != nil {
 		return nil, err
 	}
 
-	executionPlan := test.derivedConfig.Scenarios.GetFullExecutionRequirements(et)
+	executionPlan := configuredTest.derivedConfig.Scenarios.GetFullExecutionRequirements(et)
 	duration, _ := lib.GetEndOffset(executionPlan)
 
 	return struct {
 		lib.Options
 		TotalDuration types.NullDuration `json:"totalDuration"`
 		MaxVUs        uint64             `json:"maxVUs"`
 	}{
-		test.derivedConfig.Options,
+		configuredTest.derivedConfig.Options,
 		types.NewNullDuration(duration, true),
 		lib.GetMaxPossibleVUs(executionPlan),
 	}, nil