Report ThresholdsHaveFailed on Cloud runs (#3876)

cmd/cloud.go

@@ -14,6 +14,7 @@ import (
 
 	"github.com/fatih/color"
 	"go.k6.io/k6/cloudapi"
+	"go.k6.io/k6/cmd/state"
 	"go.k6.io/k6/errext"
 	"go.k6.io/k6/errext/exitcodes"
 	"go.k6.io/k6/lib"
@@ -22,8 +23,6 @@ import (
 
 	"github.com/spf13/cobra"
 	"github.com/spf13/pflag"
-
-	"go.k6.io/k6/cmd/state"
 )
 
 // cmdCloud handles the `k6 cloud` sub-command
@@ -313,9 +312,16 @@ func (c *cmdCloud) run(cmd *cobra.Command, args []string) error {
 		logger.WithField("run_status", testProgress.RunStatusText).Debug("Test finished")
 	}
 
+	//nolint:stylecheck,golint
 	if testProgress.ResultStatus == cloudapi.ResultStatusFailed {
+		// Although by looking at [ResultStatus] and [RunStatus] isn't self-explanatory,
+		// the scenario when the test run has finished, but it failed is an exceptional case for those situations
+		// when thresholds have been crossed (failed). So, we report this situation as such.
+		if testProgress.RunStatus == cloudapi.RunStatusFinished {
+			return errext.WithExitCodeIfNone(errors.New("Thresholds have been crossed"), exitcodes.ThresholdsHaveFailed)
+		}
+
 		// TODO: use different exit codes for failed thresholds vs failed test (e.g. aborted by system/limit)
-		//nolint:stylecheck,golint
 		return errext.WithExitCodeIfNone(errors.New("The test has failed"), exitcodes.CloudTestRunFailed)
 	}
 

cmd/tests/cmd_cloud_test.go

@@ -10,14 +10,14 @@ import (
 	"path/filepath"
 	"testing"
 
-	"go.k6.io/k6/lib/testutils"
-
+	"go.k6.io/k6/cloudapi"
 	"go.k6.io/k6/cmd"
+	"go.k6.io/k6/errext/exitcodes"
+	"go.k6.io/k6/lib/fsext"
+	"go.k6.io/k6/lib/testutils"
 
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
-	"go.k6.io/k6/cloudapi"
-	"go.k6.io/k6/lib/fsext"
 )
 
 func TestK6Cloud(t *testing.T) {
@@ -236,6 +236,27 @@ func runCloudTests(t *testing.T, setupCmd setupCommandFunc) {
 		assert.Contains(t, stdout, `output: https://app.k6.io/runs/123`)
 		assert.Contains(t, stdout, `test status: Finished`)
 	})
+
+	t.Run("TestCloudThresholdsHaveFailed", func(t *testing.T) {
+		t.Parallel()
+
+		progressCallback := func() cloudapi.TestProgressResponse {
+			return cloudapi.TestProgressResponse{
+				RunStatusText: "Finished",
+				RunStatus:     cloudapi.RunStatusFinished,
+				ResultStatus:  cloudapi.ResultStatusFailed,
+				Progress:      1.0,
+			}
+		}
+		ts := getSimpleCloudTestState(t, nil, setupCmd, nil, nil, progressCallback)
+		ts.ExpectedExitCode = int(exitcodes.ThresholdsHaveFailed)
+
+		cmd.ExecuteWithGlobalState(ts.GlobalState)
+
+		stdout := ts.Stdout.String()
+		t.Log(stdout)
+		assert.Contains(t, stdout, `Thresholds have been crossed`)
+	})
 }
 
 func cloudTestStartSimple(tb testing.TB, testRunID int) http.Handler {