diff --git a/cmd/run.go b/cmd/run.go index 206480f4069..0da375d883f 100644 --- a/cmd/run.go +++ b/cmd/run.go @@ -47,6 +47,7 @@ import ( "go.k6.io/k6/errext" "go.k6.io/k6/errext/exitcodes" "go.k6.io/k6/js" + "go.k6.io/k6/js/common" "go.k6.io/k6/lib" "go.k6.io/k6/lib/consts" "go.k6.io/k6/lib/metrics" @@ -120,7 +121,7 @@ a commandline interface for interacting with it.`, builtinMetrics := metrics.RegisterBuiltinMetrics(registry) initRunner, err := newRunner(logger, src, runType, filesystems, runtimeOptions, builtinMetrics, registry) if err != nil { - return err + return common.UnwrapGojaInterruptedError(err) } logger.Debug("Getting the script options...") @@ -250,6 +251,7 @@ a commandline interface for interacting with it.`, initBar.Modify(pb.WithConstProgress(0, "Init VUs...")) engineRun, engineWait, err := engine.Init(globalCtx, runCtx) if err != nil { + err = common.UnwrapGojaInterruptedError(err) // Add a generic engine exit code if we don't have a more specific one return errext.WithExitCodeIfNone(err, exitcodes.GenericEngine) } @@ -273,8 +275,18 @@ a commandline interface for interacting with it.`, // Start the test run initBar.Modify(pb.WithConstProgress(0, "Starting test...")) - if err := engineRun(); err != nil { - return errext.WithExitCodeIfNone(err, exitcodes.GenericEngine) + var interrupt error + err = engineRun() + if err != nil { + err = common.UnwrapGojaInterruptedError(err) + if common.IsInterruptError(err) { + // Don't return here since we need to work with --linger, + // show the end-of-test summary and exit cleanly. + interrupt = err + } + if !conf.Linger.Bool && interrupt == nil { + return errext.WithExitCodeIfNone(err, exitcodes.GenericEngine) + } } runCancel() logger.Debug("Engine run terminated cleanly") @@ -323,6 +335,9 @@ a commandline interface for interacting with it.`, logger.Debug("Waiting for engine processes to finish...") engineWait() logger.Debug("Everything has finished, exiting k6!") + if interrupt != nil { + return interrupt + } if engine.IsTainted() { return errext.WithExitCodeIfNone(errors.New("some thresholds have failed"), exitcodes.ThresholdsHaveFailed) } diff --git a/cmd/run_test.go b/cmd/run_test.go index 59b9b8a7bd1..c2a77e50f93 100644 --- a/cmd/run_test.go +++ b/cmd/run_test.go @@ -22,19 +22,28 @@ package cmd import ( "bytes" + "context" "errors" "io" "io/ioutil" "os" + "path" "path/filepath" "runtime" + "strings" "testing" + "github.com/sirupsen/logrus" "github.com/spf13/afero" + "github.com/spf13/pflag" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + "go.k6.io/k6/errext" + "go.k6.io/k6/errext/exitcodes" + "go.k6.io/k6/js/common" "go.k6.io/k6/lib/fsext" + "go.k6.io/k6/lib/testutils" ) type mockWriter struct { @@ -125,3 +134,85 @@ func TestHandleSummaryResultError(t *testing.T) { assertEqual(t, "file summary 1", files[filePath1]) assertEqual(t, "file summary 2", files[filePath2]) } + +func TestAbortTest(t *testing.T) { //nolint: tparallel + t.Parallel() + + testCases := []struct { + testFilename, expLogOutput string + }{ + { + testFilename: "abort.js", + }, + { + testFilename: "abort_initerr.js", + }, + { + testFilename: "abort_initvu.js", + }, + { + testFilename: "abort_teardown.js", + expLogOutput: "Calling teardown function after test.abort()", + }, + } + + for _, tc := range testCases { //nolint: paralleltest + tc := tc + t.Run(tc.testFilename, func(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + logger := logrus.New() + logger.SetLevel(logrus.InfoLevel) + logger.Out = ioutil.Discard + hook := testutils.SimpleLogrusHook{ + HookedLevels: []logrus.Level{logrus.InfoLevel}, + } + logger.AddHook(&hook) + + cmd := getRunCmd(ctx, logger) + // Redefine the flag to avoid a nil pointer panic on lookup. + cmd.Flags().AddFlag(&pflag.Flag{ + Name: "address", + Hidden: true, + }) + a, err := filepath.Abs(path.Join("testdata", tc.testFilename)) + require.NoError(t, err) + cmd.SetArgs([]string{a}) + err = cmd.Execute() + var e errext.HasExitCode + require.ErrorAs(t, err, &e) + assert.Equalf(t, exitcodes.ScriptAborted, e.ExitCode(), + "Status code must be %d", exitcodes.ScriptAborted) + assert.Contains(t, e.Error(), common.AbortTest) + + if tc.expLogOutput != "" { + var gotMsg bool + for _, entry := range hook.Drain() { + if strings.Contains(entry.Message, tc.expLogOutput) { + gotMsg = true + break + } + } + assert.True(t, gotMsg) + } + }) + } +} + +func TestInitErrExitCode(t *testing.T) { //nolint: paralleltest + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + logger := testutils.NewLogger(t) + + cmd := getRunCmd(ctx, logger) + a, err := filepath.Abs("testdata/initerr.js") + require.NoError(t, err) + cmd.SetArgs([]string{a}) + err = cmd.Execute() + var e errext.HasExitCode + require.ErrorAs(t, err, &e) + assert.Equalf(t, exitcodes.ScriptException, e.ExitCode(), + "Status code must be %d", exitcodes.ScriptException) + assert.Contains(t, err.Error(), "ReferenceError: someUndefinedVar is not defined") +} diff --git a/cmd/testdata/abort.js b/cmd/testdata/abort.js new file mode 100644 index 00000000000..b4b4a793ae1 --- /dev/null +++ b/cmd/testdata/abort.js @@ -0,0 +1,5 @@ +import exec from 'k6/execution'; + +export default function () { + exec.test.abort(); +} diff --git a/cmd/testdata/abort_initerr.js b/cmd/testdata/abort_initerr.js new file mode 100644 index 00000000000..d9dba86df30 --- /dev/null +++ b/cmd/testdata/abort_initerr.js @@ -0,0 +1,2 @@ +import exec from 'k6/execution'; +exec.test.abort(); diff --git a/cmd/testdata/abort_initvu.js b/cmd/testdata/abort_initvu.js new file mode 100644 index 00000000000..252b0e58a7d --- /dev/null +++ b/cmd/testdata/abort_initvu.js @@ -0,0 +1,8 @@ +import exec from 'k6/execution'; + +// This won't fail on initial parsing of the script, but on VU initialization. +if (__VU == 1) { + exec.test.abort(); +} + +export default function() {} diff --git a/cmd/testdata/abort_teardown.js b/cmd/testdata/abort_teardown.js new file mode 100644 index 00000000000..3403c16ce38 --- /dev/null +++ b/cmd/testdata/abort_teardown.js @@ -0,0 +1,9 @@ +import exec from 'k6/execution'; + +export default function () { + exec.test.abort(); +} + +export function teardown() { + console.log('Calling teardown function after test.abort()'); +} diff --git a/cmd/testdata/initerr.js b/cmd/testdata/initerr.js new file mode 100644 index 00000000000..9dbabe3eac2 --- /dev/null +++ b/cmd/testdata/initerr.js @@ -0,0 +1 @@ +someUndefinedVar diff --git a/core/engine.go b/core/engine.go index df61f5a2380..599421e187c 100644 --- a/core/engine.go +++ b/core/engine.go @@ -31,6 +31,7 @@ import ( "gopkg.in/guregu/null.v3" "go.k6.io/k6/errext" + "go.k6.io/k6/js/common" "go.k6.io/k6/lib" "go.k6.io/k6/lib/metrics" "go.k6.io/k6/output" @@ -263,9 +264,12 @@ func (e *Engine) startBackgroundProcesses( if err != nil { e.logger.WithError(err).Debug("run: execution scheduler returned an error") var serr errext.Exception - if errors.As(err, &serr) { + switch { + case errors.As(err, &serr): e.setRunStatus(lib.RunStatusAbortedScriptError) - } else { + case common.IsInterruptError(err): + e.setRunStatus(lib.RunStatusAbortedUser) + default: e.setRunStatus(lib.RunStatusAbortedSystem) } } else { diff --git a/core/local/local.go b/core/local/local.go index b27988ffbdd..e4bd50a77f6 100644 --- a/core/local/local.go +++ b/core/local/local.go @@ -30,7 +30,9 @@ import ( "github.com/sirupsen/logrus" "go.k6.io/k6/errext" + "go.k6.io/k6/js/common" "go.k6.io/k6/lib" + "go.k6.io/k6/lib/executor" "go.k6.io/k6/lib/metrics" "go.k6.io/k6/stats" "go.k6.io/k6/ui/pb" @@ -346,7 +348,13 @@ func (e *ExecutionScheduler) Run( executorsCount := len(e.executors) logger := e.logger.WithField("phase", "local-execution-scheduler-run") e.initProgress.Modify(pb.WithConstLeft("Run")) - defer e.state.MarkEnded() + var interrupted bool + defer func() { + e.state.MarkEnded() + if interrupted { + e.state.SetExecutionStatus(lib.ExecutionStatusInterrupted) + } + }() if e.state.IsPaused() { logger.Debug("Execution is paused, waiting for resume or interrupt...") @@ -386,8 +394,14 @@ func (e *ExecutionScheduler) Run( // Start all executors at their particular startTime in a separate goroutine... logger.Debug("Start all executors...") e.state.SetExecutionStatus(lib.ExecutionStatusRunning) + + // We are using this context to allow lib.Executor implementations to cancel + // this context effectively stopping all executions. + // + // This is for addressing test.abort(). + execCtx := executor.Context(runSubCtx) for _, exec := range e.executors { - go e.runExecutor(runSubCtx, runResults, engineOut, exec, builtinMetrics) + go e.runExecutor(execCtx, runResults, engineOut, exec, builtinMetrics) } // Wait for all executors to finish @@ -414,7 +428,10 @@ func (e *ExecutionScheduler) Run( return err } } - + if err := executor.CancelReason(execCtx); err != nil && common.IsInterruptError(err) { + interrupted = true + return err + } return firstErr } diff --git a/errext/exitcodes/codes.go b/errext/exitcodes/codes.go index d98aaf4ab72..c0e0b48c5ee 100644 --- a/errext/exitcodes/codes.go +++ b/errext/exitcodes/codes.go @@ -36,4 +36,5 @@ const ( ExternalAbort errext.ExitCode = 105 CannotStartRESTAPI errext.ExitCode = 106 ScriptException errext.ExitCode = 107 + ScriptAborted errext.ExitCode = 108 ) diff --git a/js/common/interrupt_error.go b/js/common/interrupt_error.go new file mode 100644 index 00000000000..eac3273afd8 --- /dev/null +++ b/js/common/interrupt_error.go @@ -0,0 +1,69 @@ +/* + * + * k6 - a next-generation load testing tool + * Copyright (C) 2021 Load Impact + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + */ + +package common + +import ( + "errors" + + "github.com/dop251/goja" + "go.k6.io/k6/errext" + "go.k6.io/k6/errext/exitcodes" +) + +// InterruptError is an error that halts engine execution +type InterruptError struct { + Reason string +} + +var _ errext.HasExitCode = &InterruptError{} + +// Error returns the reason of the interruption. +func (i *InterruptError) Error() string { + return i.Reason +} + +// ExitCode returns the status code used when the k6 process exits. +func (i *InterruptError) ExitCode() errext.ExitCode { + return exitcodes.ScriptAborted +} + +// AbortTest is the reason emitted when a test script calls test.abort() +const AbortTest = "test aborted" + +// IsInterruptError returns true if err is *InterruptError. +func IsInterruptError(err error) bool { + if err == nil { + return false + } + var intErr *InterruptError + return errors.As(err, &intErr) +} + +// UnwrapGojaInterruptedError returns the internal error handled by goja. +func UnwrapGojaInterruptedError(err error) error { + var gojaErr *goja.InterruptedError + if errors.As(err, &gojaErr) { + if e, ok := gojaErr.Value().(error); ok { + return e + } + } + return err +} diff --git a/js/modules/k6/execution/execution.go b/js/modules/k6/execution/execution.go index 668a7074728..2094bcf3959 100644 --- a/js/modules/k6/execution/execution.go +++ b/js/modules/k6/execution/execution.go @@ -73,8 +73,9 @@ func (*RootModule) NewModuleInstance(vu modules.VU) modules.Instance { common.Throw(rt, err) } } - defProp("scenario", mi.newScenarioInfo) defProp("instance", mi.newInstanceInfo) + defProp("scenario", mi.newScenarioInfo) + defProp("test", mi.newTestInfo) defProp("vu", mi.newVUInfo) mi.obj = o @@ -90,15 +91,11 @@ func (mi *ModuleInstance) Exports() modules.Exports { // newScenarioInfo returns a goja.Object with property accessors to retrieve // information about the scenario the current VU is running in. func (mi *ModuleInstance) newScenarioInfo() (*goja.Object, error) { - ctx := mi.vu.Context() - rt := common.GetRuntime(ctx) + rt := mi.vu.Runtime() vuState := mi.vu.State() if vuState == nil { return nil, errors.New("getting scenario information in the init context is not supported") } - if rt == nil { - return nil, errors.New("goja runtime is nil in context") - } getScenarioState := func() *lib.ScenarioState { ss := lib.GetScenarioState(mi.vu.Context()) if ss == nil { @@ -140,16 +137,11 @@ func (mi *ModuleInstance) newScenarioInfo() (*goja.Object, error) { // newInstanceInfo returns a goja.Object with property accessors to retrieve // information about the local instance stats. func (mi *ModuleInstance) newInstanceInfo() (*goja.Object, error) { - ctx := mi.vu.Context() - es := lib.GetExecutionState(ctx) + es := lib.GetExecutionState(mi.vu.Context()) if es == nil { return nil, errors.New("getting instance information in the init context is not supported") } - - rt := common.GetRuntime(ctx) - if rt == nil { - return nil, errors.New("goja runtime is nil in context") - } + rt := mi.vu.Runtime() ti := map[string]func() interface{}{ "currentTestRunDuration": func() interface{} { @@ -172,19 +164,34 @@ func (mi *ModuleInstance) newInstanceInfo() (*goja.Object, error) { return newInfoObj(rt, ti) } +// newTestInfo returns a goja.Object with property accessors to retrieve +// information and control execution of the overall test run. +func (mi *ModuleInstance) newTestInfo() (*goja.Object, error) { + rt := mi.vu.Runtime() + ti := map[string]func() interface{}{ + // stop the test run + "abort": func() interface{} { + return func(msg goja.Value) { + reason := common.AbortTest + if msg != nil && !goja.IsUndefined(msg) { + reason = fmt.Sprintf("%s: %s", reason, msg.String()) + } + rt.Interrupt(&common.InterruptError{Reason: reason}) + } + }, + } + + return newInfoObj(rt, ti) +} + // newVUInfo returns a goja.Object with property accessors to retrieve // information about the currently executing VU. func (mi *ModuleInstance) newVUInfo() (*goja.Object, error) { - ctx := mi.vu.Context() - vuState := lib.GetState(ctx) + vuState := mi.vu.State() if vuState == nil { return nil, errors.New("getting VU information in the init context is not supported") } - - rt := common.GetRuntime(ctx) - if rt == nil { - return nil, errors.New("goja runtime is nil in context") - } + rt := mi.vu.Runtime() vi := map[string]func() interface{}{ "idInInstance": func() interface{} { return vuState.VUID }, diff --git a/js/modules/k6/execution/execution_test.go b/js/modules/k6/execution/execution_test.go index 6827f774482..f6d7ca1dbf2 100644 --- a/js/modules/k6/execution/execution_test.go +++ b/js/modules/k6/execution/execution_test.go @@ -183,3 +183,40 @@ func TestVUTags(t *testing.T) { }) }) } + +func TestAbortTest(t *testing.T) { //nolint: tparallel + t.Parallel() + + rt := goja.New() + ctx := common.WithRuntime(context.Background(), rt) + state := &lib.State{} + ctx = lib.WithState(ctx, state) + + m, ok := New().NewModuleInstance( + &modulestest.VU{ + RuntimeField: rt, + InitEnvField: &common.InitEnvironment{}, + CtxField: ctx, + StateField: state, + }, + ).(*ModuleInstance) + require.True(t, ok) + require.NoError(t, rt.Set("exec", m.Exports().Default)) + + prove := func(t *testing.T, script, reason string) { + _, err := rt.RunString(script) + require.NotNil(t, err) + var x *goja.InterruptedError + assert.ErrorAs(t, err, &x) + v, ok := x.Value().(*common.InterruptError) + require.True(t, ok) + require.Equal(t, v.Reason, reason) + } + + t.Run("default reason", func(t *testing.T) { //nolint: paralleltest + prove(t, "exec.test.abort()", common.AbortTest) + }) + t.Run("custom reason", func(t *testing.T) { //nolint: paralleltest + prove(t, `exec.test.abort("mayday")`, fmt.Sprintf("%s: mayday", common.AbortTest)) + }) +} diff --git a/js/runner.go b/js/runner.go index a25ef086364..6e27c8accaf 100644 --- a/js/runner.go +++ b/js/runner.go @@ -726,6 +726,15 @@ func (u *ActiveVU) RunOnce() error { // Call the exported function. _, isFullIteration, totalTime, err := u.runFn(u.RunContext, true, fn, u.setupData) + if err != nil { + var x *goja.InterruptedError + if errors.As(err, &x) { + if v, ok := x.Value().(*common.InterruptError); ok { + v.Reason = x.Error() + err = v + } + } + } // If MinIterationDuration is specified and the iteration wasn't canceled // and was less than it, sleep for the remainder diff --git a/js/runner_test.go b/js/runner_test.go index dbc8370fa5c..34abe6a8b60 100644 --- a/js/runner_test.go +++ b/js/runner_test.go @@ -1505,6 +1505,13 @@ func TestInitContextForbidden(t *testing.T) { exports.default = function() { console.log("p"); }`, k6.ErrCheckInInitContext.Error(), }, + { + "abortTest", + `var test = require("k6/execution").test; + test.abort(); + exports.default = function() { console.log("p"); }`, + common.AbortTest, + }, { "group", `var group = require("k6").group; diff --git a/lib/execution.go b/lib/execution.go index 1511558f1ee..14c090414ed 100644 --- a/lib/execution.go +++ b/lib/execution.go @@ -123,6 +123,7 @@ const ( ExecutionStatusRunning ExecutionStatusTeardown ExecutionStatusEnded + ExecutionStatusInterrupted ) // ExecutionState contains a few different things: diff --git a/lib/execution_status_gen.go b/lib/execution_status_gen.go index aafbcc6b303..3c8e670e4f5 100644 --- a/lib/execution_status_gen.go +++ b/lib/execution_status_gen.go @@ -7,9 +7,9 @@ import ( "fmt" ) -const _ExecutionStatusName = "CreatedInitVUsInitExecutorsInitDonePausedBeforeRunStartedSetupRunningTeardownEnded" +const _ExecutionStatusName = "CreatedInitVUsInitExecutorsInitDonePausedBeforeRunStartedSetupRunningTeardownEndedInterrupted" -var _ExecutionStatusIndex = [...]uint8{0, 7, 14, 27, 35, 50, 57, 62, 69, 77, 82} +var _ExecutionStatusIndex = [...]uint8{0, 7, 14, 27, 35, 50, 57, 62, 69, 77, 82, 93} func (i ExecutionStatus) String() string { if i >= ExecutionStatus(len(_ExecutionStatusIndex)-1) { @@ -18,7 +18,7 @@ func (i ExecutionStatus) String() string { return _ExecutionStatusName[_ExecutionStatusIndex[i]:_ExecutionStatusIndex[i+1]] } -var _ExecutionStatusValues = []ExecutionStatus{0, 1, 2, 3, 4, 5, 6, 7, 8, 9} +var _ExecutionStatusValues = []ExecutionStatus{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10} var _ExecutionStatusNameToValueMap = map[string]ExecutionStatus{ _ExecutionStatusName[0:7]: 0, @@ -31,6 +31,7 @@ var _ExecutionStatusNameToValueMap = map[string]ExecutionStatus{ _ExecutionStatusName[62:69]: 7, _ExecutionStatusName[69:77]: 8, _ExecutionStatusName[77:82]: 9, + _ExecutionStatusName[82:93]: 10, } // ExecutionStatusString retrieves an enum value from the enum constants string name. diff --git a/lib/executor/helpers.go b/lib/executor/helpers.go index 86c514325ad..97ff18337e3 100644 --- a/lib/executor/helpers.go +++ b/lib/executor/helpers.go @@ -30,6 +30,7 @@ import ( "github.com/sirupsen/logrus" "go.k6.io/k6/errext" + "go.k6.io/k6/js/common" "go.k6.io/k6/lib" "go.k6.io/k6/lib/types" "go.k6.io/k6/ui/pb" @@ -76,6 +77,62 @@ func validateStages(stages []Stage) []error { return errors } +// cancelKey is the key used to store the cancel function for the context of an +// executor. This is a work around to avoid excessive changes for the ability of +// nested functions to cancel the passed context. +type cancelKey struct{} + +type cancelExec struct { + cancel context.CancelFunc + reason error +} + +// Context returns context.Context that can be cancelled by calling +// CancelExecutorContext. Use this to initialize context that will be passed to +// executors. +// +// This allows executors to globally halt any executions that uses this context. +// Example use case is when a script calls test.abort(). +func Context(ctx context.Context) context.Context { + ctx, cancel := context.WithCancel(ctx) + return context.WithValue(ctx, cancelKey{}, &cancelExec{cancel: cancel}) +} + +// cancelExecutorContext cancels executor context found in ctx, ctx can be a +// child of a context that was created with Context function. +func cancelExecutorContext(ctx context.Context, err error) { + if x := ctx.Value(cancelKey{}); x != nil { + if v, ok := x.(*cancelExec); ok { + v.reason = err + v.cancel() + } + } +} + +// CancelReason returns a reason the executor context was cancelled. This will +// return nil if ctx is not an executor context(ctx or any of its parents was +// never created by Context function). +func CancelReason(ctx context.Context) error { + if x := ctx.Value(cancelKey{}); x != nil { + if v, ok := x.(*cancelExec); ok { + return v.reason + } + } + return nil +} + +// handleInterrupt returns true if err is InterruptError and if so it +// cancels the executor context passed with ctx. +func handleInterrupt(ctx context.Context, err error) bool { + if err != nil { + if common.IsInterruptError(err) { + cancelExecutorContext(ctx, err) + return true + } + } + return false +} + // getIterationRunner is a helper function that returns an iteration executor // closure. It takes care of updating the execution state statistics and // warning messages. And returns whether a full iteration was finished or not @@ -98,6 +155,11 @@ func getIterationRunner( return false default: if err != nil { + if handleInterrupt(ctx, err) { + executionState.AddInterruptedIterations(1) + return false + } + var exception errext.Exception if errors.As(err, &exception) { // TODO don't count this as a full iteration?