From e7314bbadc49cb7c8caf8c5e03199061a2505801 Mon Sep 17 00:00:00 2001 From: gernest Date: Mon, 22 Mar 2021 10:52:49 +0300 Subject: [PATCH] Add abortTest() helper function MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Ivan Mirić Closes #1001 This adds abortTest() helper function to the k6 module. This function when called inside a script it will - stop the whole test run and k6 will exit with 107 status code - stops immediately the VU that called it and no more iterations are started - make sure the teardown() is called - the engine run status is 7 (RunStatusAbortedScriptError) `(*goja.Runtime).Interrupt` is used for halting script execution and capturing stack traces for better error message of what is happening with the script. We introduce InterruptError which is used with `(*goja.Runtime).Interrupt` to identify interrupts emitted by abortTest(). This way we use special handling of this type. Example script is ```js import { abortTest, sleep } from 'k6'; export default function () { // We abort the test on second iteration if (__ITER == 1) { abortTest(); } sleep(1); } export function teardown() { console.log('This function will be called even when we abort script'); } ``` abortTest() can be called in both default and setup functions, however you can't use it in the init context The following script will fail with the error ``` ERRO[0000] Using abortTest() in the init context is not supported at (...path to the script )init.js:13:43(34) ``` ```js import { abortTest } from 'k6'; abortTest(); export function setup() { } export default function () { // ... some test logic ... console.log('mayday, mayday'); } ``` You can customize the reason for abortTest() by passing values to the function ```js abortTest("Exceeded expectations"); ``` Will emit `"Exceeded expectations"` on the logs --- cmd/run.go | 21 +++++++- cmd/run_test.go | 52 +++++++++++++++++++ cmd/testdata/abort.js | 5 ++ cmd/testdata/teardown.js | 9 ++++ core/local/local.go | 15 +++++- js/common/interrupt_error.go | 44 ++++++++++++++++ js/modules/k6/execution/execution.go | 47 +++++++++-------- js/modules/k6/execution/execution_test.go | 37 ++++++++++++++ js/modules/k6/k6.go | 12 +++++ js/modules/k6/k6_test.go | 32 ++++++++++++ js/runner.go | 9 ++++ js/runner_test.go | 7 +++ lib/executor/helpers.go | 62 +++++++++++++++++++++++ 13 files changed, 328 insertions(+), 24 deletions(-) create mode 100644 cmd/testdata/abort.js create mode 100644 cmd/testdata/teardown.js create mode 100644 js/common/interrupt_error.go diff --git a/cmd/run.go b/cmd/run.go index 206480f40695..f921fca483f5 100644 --- a/cmd/run.go +++ b/cmd/run.go @@ -47,6 +47,7 @@ import ( "go.k6.io/k6/errext" "go.k6.io/k6/errext/exitcodes" "go.k6.io/k6/js" + "go.k6.io/k6/js/common" "go.k6.io/k6/lib" "go.k6.io/k6/lib/consts" "go.k6.io/k6/lib/metrics" @@ -250,6 +251,9 @@ a commandline interface for interacting with it.`, initBar.Modify(pb.WithConstProgress(0, "Init VUs...")) engineRun, engineWait, err := engine.Init(globalCtx, runCtx) if err != nil { + if common.IsInterruptError(err) { + return errext.WithExitCodeIfNone(err, exitcodes.ScriptException) + } // Add a generic engine exit code if we don't have a more specific one return errext.WithExitCodeIfNone(err, exitcodes.GenericEngine) } @@ -273,8 +277,18 @@ a commandline interface for interacting with it.`, // Start the test run initBar.Modify(pb.WithConstProgress(0, "Starting test...")) - if err := engineRun(); err != nil { - return errext.WithExitCodeIfNone(err, exitcodes.GenericEngine) + var interrupt error + err = engineRun() + if err != nil { + if common.IsInterruptError(err) { + interrupt = err + } + if !conf.Linger.Bool { + if interrupt == nil { + return errext.WithExitCodeIfNone(err, exitcodes.GenericEngine) + } + return errext.WithExitCodeIfNone(interrupt, exitcodes.ScriptException) + } } runCancel() logger.Debug("Engine run terminated cleanly") @@ -323,6 +337,9 @@ a commandline interface for interacting with it.`, logger.Debug("Waiting for engine processes to finish...") engineWait() logger.Debug("Everything has finished, exiting k6!") + if interrupt != nil { + return errext.WithExitCodeIfNone(interrupt, exitcodes.ScriptException) + } if engine.IsTainted() { return errext.WithExitCodeIfNone(errors.New("some thresholds have failed"), exitcodes.ThresholdsHaveFailed) } diff --git a/cmd/run_test.go b/cmd/run_test.go index 59b9b8a7bd1f..e0ea78c72d08 100644 --- a/cmd/run_test.go +++ b/cmd/run_test.go @@ -22,6 +22,7 @@ package cmd import ( "bytes" + "context" "errors" "io" "io/ioutil" @@ -30,11 +31,17 @@ import ( "runtime" "testing" + "github.com/sirupsen/logrus" "github.com/spf13/afero" + "github.com/spf13/pflag" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + "go.k6.io/k6/errext" + "go.k6.io/k6/errext/exitcodes" + "go.k6.io/k6/js/common" "go.k6.io/k6/lib/fsext" + "go.k6.io/k6/lib/testutils" ) type mockWriter struct { @@ -125,3 +132,48 @@ func TestHandleSummaryResultError(t *testing.T) { assertEqual(t, "file summary 1", files[filePath1]) assertEqual(t, "file summary 2", files[filePath2]) } + +func TestAbortTest(t *testing.T) { //nolint: tparallel + t.Parallel() + + t.Run("Check status code is 107", func(t *testing.T) { //nolint: paralleltest + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + logger := testutils.NewLogger(t) + + cmd := getRunCmd(ctx, logger) + a, err := filepath.Abs("testdata/abort.js") + require.NoError(t, err) + cmd.SetArgs([]string{a}) + err = cmd.Execute() + var e errext.HasExitCode + require.ErrorAs(t, err, &e) + require.Equal(t, exitcodes.ScriptException, e.ExitCode(), "Status code must be 107") + require.Contains(t, e.Error(), common.AbortTest) + }) + + t.Run("Check that teardown is called", func(t *testing.T) { //nolint: paralleltest + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + msg := "Calling teardown function after abortTest()" + var buf bytes.Buffer + logger := logrus.New() + logger.SetOutput(&buf) + + cmd := getRunCmd(ctx, logger) + // Redefine the flag to avoid a nil pointer panic on lookup. + cmd.Flags().AddFlag(&pflag.Flag{ + Name: "address", + Hidden: true, + }) + a, err := filepath.Abs("testdata/teardown.js") + require.NoError(t, err) + cmd.SetArgs([]string{a}) + err = cmd.Execute() + var e errext.HasExitCode + require.ErrorAs(t, err, &e) + assert.Equal(t, exitcodes.ScriptException, e.ExitCode(), "Status code must be 107") + assert.Contains(t, e.Error(), common.AbortTest) + assert.Contains(t, buf.String(), msg) + }) +} diff --git a/cmd/testdata/abort.js b/cmd/testdata/abort.js new file mode 100644 index 000000000000..a451ed512b5a --- /dev/null +++ b/cmd/testdata/abort.js @@ -0,0 +1,5 @@ +import { abortTest } from 'k6'; + +export default function () { + abortTest(); +} diff --git a/cmd/testdata/teardown.js b/cmd/testdata/teardown.js new file mode 100644 index 000000000000..aa870819b0d0 --- /dev/null +++ b/cmd/testdata/teardown.js @@ -0,0 +1,9 @@ +import { abortTest } from 'k6'; + +export default function () { + abortTest(); +} + +export function teardown() { + console.log('Calling teardown function after abortTest()'); +} \ No newline at end of file diff --git a/core/local/local.go b/core/local/local.go index b27988ffbddf..7f45e458b8ff 100644 --- a/core/local/local.go +++ b/core/local/local.go @@ -30,7 +30,9 @@ import ( "github.com/sirupsen/logrus" "go.k6.io/k6/errext" + "go.k6.io/k6/js/common" "go.k6.io/k6/lib" + "go.k6.io/k6/lib/executor" "go.k6.io/k6/lib/metrics" "go.k6.io/k6/stats" "go.k6.io/k6/ui/pb" @@ -386,8 +388,14 @@ func (e *ExecutionScheduler) Run( // Start all executors at their particular startTime in a separate goroutine... logger.Debug("Start all executors...") e.state.SetExecutionStatus(lib.ExecutionStatusRunning) + + // We are using this context to allow lib.Executor implementations to cancel + // this context effectively stopping all executions. + // + // This is for addressing abortTest() + execCtx := executor.Context(runSubCtx) for _, exec := range e.executors { - go e.runExecutor(runSubCtx, runResults, engineOut, exec, builtinMetrics) + go e.runExecutor(execCtx, runResults, engineOut, exec, builtinMetrics) } // Wait for all executors to finish @@ -414,7 +422,10 @@ func (e *ExecutionScheduler) Run( return err } } - + if err := executor.CancelReason(execCtx); err != nil && common.IsInterruptError(err) { + // The execution was interupted + return err + } return firstErr } diff --git a/js/common/interrupt_error.go b/js/common/interrupt_error.go new file mode 100644 index 000000000000..ce2476a560fe --- /dev/null +++ b/js/common/interrupt_error.go @@ -0,0 +1,44 @@ +/* + * + * k6 - a next-generation load testing tool + * Copyright (C) 2021 Load Impact + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + */ + +package common + +import "errors" + +// InterruptError is an error that halts engine execution +type InterruptError struct { + Reason string +} + +func (i *InterruptError) Error() string { + return i.Reason +} + +// AbortTest is a reason emitted when a test script calls abortTest() without arguments +const AbortTest = "abortTest() was called in a script" + +// IsInterruptError returns true if err is *InterruptError. +func IsInterruptError(err error) bool { + if err == nil { + return false + } + var intErr *InterruptError + return errors.As(err, &intErr) +} diff --git a/js/modules/k6/execution/execution.go b/js/modules/k6/execution/execution.go index 668a70747284..2094bcf3959e 100644 --- a/js/modules/k6/execution/execution.go +++ b/js/modules/k6/execution/execution.go @@ -73,8 +73,9 @@ func (*RootModule) NewModuleInstance(vu modules.VU) modules.Instance { common.Throw(rt, err) } } - defProp("scenario", mi.newScenarioInfo) defProp("instance", mi.newInstanceInfo) + defProp("scenario", mi.newScenarioInfo) + defProp("test", mi.newTestInfo) defProp("vu", mi.newVUInfo) mi.obj = o @@ -90,15 +91,11 @@ func (mi *ModuleInstance) Exports() modules.Exports { // newScenarioInfo returns a goja.Object with property accessors to retrieve // information about the scenario the current VU is running in. func (mi *ModuleInstance) newScenarioInfo() (*goja.Object, error) { - ctx := mi.vu.Context() - rt := common.GetRuntime(ctx) + rt := mi.vu.Runtime() vuState := mi.vu.State() if vuState == nil { return nil, errors.New("getting scenario information in the init context is not supported") } - if rt == nil { - return nil, errors.New("goja runtime is nil in context") - } getScenarioState := func() *lib.ScenarioState { ss := lib.GetScenarioState(mi.vu.Context()) if ss == nil { @@ -140,16 +137,11 @@ func (mi *ModuleInstance) newScenarioInfo() (*goja.Object, error) { // newInstanceInfo returns a goja.Object with property accessors to retrieve // information about the local instance stats. func (mi *ModuleInstance) newInstanceInfo() (*goja.Object, error) { - ctx := mi.vu.Context() - es := lib.GetExecutionState(ctx) + es := lib.GetExecutionState(mi.vu.Context()) if es == nil { return nil, errors.New("getting instance information in the init context is not supported") } - - rt := common.GetRuntime(ctx) - if rt == nil { - return nil, errors.New("goja runtime is nil in context") - } + rt := mi.vu.Runtime() ti := map[string]func() interface{}{ "currentTestRunDuration": func() interface{} { @@ -172,19 +164,34 @@ func (mi *ModuleInstance) newInstanceInfo() (*goja.Object, error) { return newInfoObj(rt, ti) } +// newTestInfo returns a goja.Object with property accessors to retrieve +// information and control execution of the overall test run. +func (mi *ModuleInstance) newTestInfo() (*goja.Object, error) { + rt := mi.vu.Runtime() + ti := map[string]func() interface{}{ + // stop the test run + "abort": func() interface{} { + return func(msg goja.Value) { + reason := common.AbortTest + if msg != nil && !goja.IsUndefined(msg) { + reason = fmt.Sprintf("%s: %s", reason, msg.String()) + } + rt.Interrupt(&common.InterruptError{Reason: reason}) + } + }, + } + + return newInfoObj(rt, ti) +} + // newVUInfo returns a goja.Object with property accessors to retrieve // information about the currently executing VU. func (mi *ModuleInstance) newVUInfo() (*goja.Object, error) { - ctx := mi.vu.Context() - vuState := lib.GetState(ctx) + vuState := mi.vu.State() if vuState == nil { return nil, errors.New("getting VU information in the init context is not supported") } - - rt := common.GetRuntime(ctx) - if rt == nil { - return nil, errors.New("goja runtime is nil in context") - } + rt := mi.vu.Runtime() vi := map[string]func() interface{}{ "idInInstance": func() interface{} { return vuState.VUID }, diff --git a/js/modules/k6/execution/execution_test.go b/js/modules/k6/execution/execution_test.go index 6827f774482e..f6d7ca1dbf25 100644 --- a/js/modules/k6/execution/execution_test.go +++ b/js/modules/k6/execution/execution_test.go @@ -183,3 +183,40 @@ func TestVUTags(t *testing.T) { }) }) } + +func TestAbortTest(t *testing.T) { //nolint: tparallel + t.Parallel() + + rt := goja.New() + ctx := common.WithRuntime(context.Background(), rt) + state := &lib.State{} + ctx = lib.WithState(ctx, state) + + m, ok := New().NewModuleInstance( + &modulestest.VU{ + RuntimeField: rt, + InitEnvField: &common.InitEnvironment{}, + CtxField: ctx, + StateField: state, + }, + ).(*ModuleInstance) + require.True(t, ok) + require.NoError(t, rt.Set("exec", m.Exports().Default)) + + prove := func(t *testing.T, script, reason string) { + _, err := rt.RunString(script) + require.NotNil(t, err) + var x *goja.InterruptedError + assert.ErrorAs(t, err, &x) + v, ok := x.Value().(*common.InterruptError) + require.True(t, ok) + require.Equal(t, v.Reason, reason) + } + + t.Run("default reason", func(t *testing.T) { //nolint: paralleltest + prove(t, "exec.test.abort()", common.AbortTest) + }) + t.Run("custom reason", func(t *testing.T) { //nolint: paralleltest + prove(t, `exec.test.abort("mayday")`, fmt.Sprintf("%s: mayday", common.AbortTest)) + }) +} diff --git a/js/modules/k6/k6.go b/js/modules/k6/k6.go index 3e94cdb54f9a..094315a58fde 100644 --- a/js/modules/k6/k6.go +++ b/js/modules/k6/k6.go @@ -22,6 +22,7 @@ package k6 import ( + "context" "errors" "math/rand" "sync/atomic" @@ -151,6 +152,17 @@ func (mi *K6) Group(name string, fn goja.Callable) (goja.Value, error) { return ret, err } +// AbortTest exposes abortTest function in the k6 module. When called it will +// interrupt the active goja runtime passed with ctx. +func (*K6) AbortTest(ctx context.Context, msg goja.Value) { + rt := common.GetRuntime(ctx) + reason := common.AbortTest + if !goja.IsUndefined(msg) { + reason = msg.String() + } + rt.Interrupt(&common.InterruptError{Reason: reason}) +} + // Check will emit check metrics for the provided checks. //nolint:cyclop func (mi *K6) Check(arg0, checks goja.Value, extras ...goja.Value) (bool, error) { diff --git a/js/modules/k6/k6_test.go b/js/modules/k6/k6_test.go index 6e541d9033f8..f1a2d285c57a 100644 --- a/js/modules/k6/k6_test.go +++ b/js/modules/k6/k6_test.go @@ -474,3 +474,35 @@ func TestCheckTags(t *testing.T) { }, sample.Tags.CloneTags()) } } + +func TestAbortTest(t *testing.T) { //nolint: tparallel + t.Parallel() + + rt := goja.New() + baseCtx := common.WithRuntime(context.Background(), rt) + + ctx := new(context.Context) + *ctx = baseCtx + err := rt.Set("k6", common.Bind(rt, New(), ctx)) + require.Nil(t, err) + prove := func(t *testing.T, script, reason string) { + _, err := rt.RunString(script) + require.NotNil(t, err) + var x *goja.InterruptedError + assert.ErrorAs(t, err, &x) + v, ok := x.Value().(*common.InterruptError) + require.True(t, ok) + require.Equal(t, v.Reason, reason) + } + t.Run("Without state", func(t *testing.T) { //nolint: paralleltest + prove(t, "k6.abortTest()", common.AbortTest) + }) + t.Run("With state and default reason", func(t *testing.T) { //nolint: paralleltest + *ctx = lib.WithState(baseCtx, &lib.State{}) + prove(t, "k6.abortTest()", common.AbortTest) + }) + t.Run("With state and custom reason", func(t *testing.T) { //nolint: paralleltest + *ctx = lib.WithState(baseCtx, &lib.State{}) + prove(t, `k6.abortTest("mayday")`, "mayday") + }) +} diff --git a/js/runner.go b/js/runner.go index a25ef086364a..6e27c8accaf8 100644 --- a/js/runner.go +++ b/js/runner.go @@ -726,6 +726,15 @@ func (u *ActiveVU) RunOnce() error { // Call the exported function. _, isFullIteration, totalTime, err := u.runFn(u.RunContext, true, fn, u.setupData) + if err != nil { + var x *goja.InterruptedError + if errors.As(err, &x) { + if v, ok := x.Value().(*common.InterruptError); ok { + v.Reason = x.Error() + err = v + } + } + } // If MinIterationDuration is specified and the iteration wasn't canceled // and was less than it, sleep for the remainder diff --git a/js/runner_test.go b/js/runner_test.go index dbc8370fa5ca..d9e76f6494d5 100644 --- a/js/runner_test.go +++ b/js/runner_test.go @@ -1505,6 +1505,13 @@ func TestInitContextForbidden(t *testing.T) { exports.default = function() { console.log("p"); }`, k6.ErrCheckInInitContext.Error(), }, + { + "abortTest", + `var abortTest = require("k6").abortTest; + abortTest(); + exports.default = function() { console.log("p"); }`, + common.AbortTest, + }, { "group", `var group = require("k6").group; diff --git a/lib/executor/helpers.go b/lib/executor/helpers.go index 86c514325ad0..4e16c7a2f920 100644 --- a/lib/executor/helpers.go +++ b/lib/executor/helpers.go @@ -30,6 +30,7 @@ import ( "github.com/sirupsen/logrus" "go.k6.io/k6/errext" + "go.k6.io/k6/js/common" "go.k6.io/k6/lib" "go.k6.io/k6/lib/types" "go.k6.io/k6/ui/pb" @@ -76,6 +77,62 @@ func validateStages(stages []Stage) []error { return errors } +// cancelKey is the key used to store the cancel function for the context of an +// executor. This is a work around to avoid excessive changes for the ability of +// nested functions to cancel the passed context. +type cancelKey struct{} + +type cancelExec struct { + cancel context.CancelFunc + reason error +} + +// Context returns context.Context that can be cancelled by calling +// CancelExecutorContext. Use this to initialize context that will be passed to +// executors. +// +// This allows executors to globally halt any executions that uses this context. +// Example use case is when a script calls abortTest() +func Context(ctx context.Context) context.Context { + ctx, cancel := context.WithCancel(ctx) + return context.WithValue(ctx, cancelKey{}, &cancelExec{cancel: cancel}) +} + +// cancelExecutorContext cancels executor context found in ctx, ctx can be a +// child of a context that was created with Context function. +func cancelExecutorContext(ctx context.Context, err error) { + if x := ctx.Value(cancelKey{}); x != nil { + if v, ok := x.(*cancelExec); ok { + v.reason = err + v.cancel() + } + } +} + +// CancelReason returns a reason the executor context was cancelled. This will +// return nil if ctx is not an executor context(ctx or any of its parents was +// never created by Context function). +func CancelReason(ctx context.Context) error { + if x := ctx.Value(cancelKey{}); x != nil { + if v, ok := x.(*cancelExec); ok { + return v.reason + } + } + return nil +} + +// handleInterrupt returns true if err is InterruptError and if so it +// cancels the executor context passed with ctx. +func handleInterrupt(ctx context.Context, err error) bool { + if err != nil { + if common.IsInterruptError(err) { + cancelExecutorContext(ctx, err) + return true + } + } + return false +} + // getIterationRunner is a helper function that returns an iteration executor // closure. It takes care of updating the execution state statistics and // warning messages. And returns whether a full iteration was finished or not @@ -98,6 +155,11 @@ func getIterationRunner( return false default: if err != nil { + if handleInterrupt(ctx, err) { + executionState.AddInterruptedIterations(1) + return false + } + var exception errext.Exception if errors.As(err, &exception) { // TODO don't count this as a full iteration?