From 2b0bc1f7b17712a77e0951dc41094d001dcf51af Mon Sep 17 00:00:00 2001 From: oleiade Date: Tue, 23 Jul 2024 14:52:48 +0200 Subject: [PATCH] Refactor cloud run command and tests to leverage existing code --- cmd/cloud.go | 29 ++- cmd/cloud_run.go | 343 +------------------------- cmd/tests/cmd_cloud_run_test.go | 246 +----------------- cmd/tests/cmd_cloud_test.go | 424 ++++++++++++++++---------------- 4 files changed, 244 insertions(+), 798 deletions(-) diff --git a/cmd/cloud.go b/cmd/cloud.go index b04a36865bf..640b0a7e9ed 100644 --- a/cmd/cloud.go +++ b/cmd/cloud.go @@ -13,16 +13,17 @@ import ( "time" "github.com/fatih/color" - "github.com/spf13/cobra" - "github.com/spf13/pflag" - "go.k6.io/k6/cloudapi" - "go.k6.io/k6/cmd/state" "go.k6.io/k6/errext" "go.k6.io/k6/errext/exitcodes" "go.k6.io/k6/lib" "go.k6.io/k6/lib/consts" "go.k6.io/k6/ui/pb" + + "github.com/spf13/cobra" + "github.com/spf13/pflag" + + "go.k6.io/k6/cmd/state" ) // cmdCloud handles the `k6 cloud` sub-command @@ -34,7 +35,6 @@ type cmdCloud struct { uploadOnly bool } -//nolint:dupl // function duplicated from the deprecated `k6 cloud` command, stmt can go when the command is remove func (c *cmdCloud) preRun(cmd *cobra.Command, _ []string) error { // TODO: refactor (https://github.com/loadimpact/k6/issues/883) // @@ -119,7 +119,10 @@ func (c *cmdCloud) run(cmd *cobra.Command, args []string) error { return err } if !cloudConfig.Token.Valid { - return errors.New("Not logged in, please use `k6 cloud login`") //nolint:golint,stylecheck + return errors.New( //nolint:golint + "not logged in, please login to the Grafana Cloud k6 " + + "using the \"k6 cloud login\" command", + ) } // Display config warning if needed @@ -345,6 +348,12 @@ func getCmdCloud(gs *state.GlobalState) *cobra.Command { } exampleText := getExampleText(gs, ` + # [deprecated] Run a k6 script in the Grafana Cloud k6 + $ {{.}} cloud script.js + + # [deprecated] Run a k6 archive in the Grafana Cloud k6 + $ {{.}} cloud archive.tar + # Authenticate with Grafana Cloud k6 $ {{.}} cloud login @@ -352,13 +361,7 @@ func getCmdCloud(gs *state.GlobalState) *cobra.Command { $ {{.}} cloud run script.js # Run a k6 archive in the Grafana Cloud k6 - $ {{.}} cloud run archive.tar - - # [deprecated] Run a k6 script in the Grafana Cloud k6 - $ {{.}} cloud script.js - - # [deprecated] Run a k6 archive in the Grafana Cloud k6 - $ {{.}} cloud archive.tar`[1:]) + $ {{.}} cloud run archive.tar`[1:]) cloudCmd := &cobra.Command{ Use: "cloud", diff --git a/cmd/cloud_run.go b/cmd/cloud_run.go index 06adb4ef7d4..82bfab570a9 100644 --- a/cmd/cloud_run.go +++ b/cmd/cloud_run.go @@ -1,40 +1,14 @@ package cmd import ( - "context" - "encoding/json" - "errors" - "fmt" - "os" - "path/filepath" - "strconv" - "sync" - "time" - - "github.com/fatih/color" "github.com/spf13/cobra" - "github.com/spf13/pflag" - - "go.k6.io/k6/cloudapi" "go.k6.io/k6/cmd/state" - "go.k6.io/k6/errext" - "go.k6.io/k6/errext/exitcodes" - "go.k6.io/k6/lib" - "go.k6.io/k6/lib/consts" - "go.k6.io/k6/ui/pb" ) -// cmdCloudRun handles the `k6 cloud run` sub-command -type cmdCloudRun struct { - gs *state.GlobalState - - showCloudLogs bool - exitOnRunning bool - uploadOnly bool -} +const cloudRunCommandName string = "run" func getCmdCloudRun(gs *state.GlobalState) *cobra.Command { - c := &cmdCloudRun{ + deprecatedCloudCmd := &cmdCloud{ gs: gs, showCloudLogs: true, exitOnRunning: false, @@ -63,319 +37,12 @@ against Grafana Cloud k6. Use the "k6 cloud login" command to authenticate.`, "the k6 cloud run command expects a single argument consisting in either a path to a script or "+ "archive file, or the \"-\" symbol indicating the script or archive should be read from stdin", ), - PreRunE: c.preRun, - RunE: c.run, + PreRunE: deprecatedCloudCmd.preRun, + RunE: deprecatedCloudCmd.run, } cloudRunCmd.Flags().SortFlags = false - cloudRunCmd.Flags().AddFlagSet(c.flagSet()) + cloudRunCmd.Flags().AddFlagSet(deprecatedCloudCmd.flagSet()) return cloudRunCmd } - -//nolint:dupl // function duplicated from the deprecated `k6 cloud` command, stmt can go when the command is remove -func (c *cmdCloudRun) preRun(cmd *cobra.Command, _ []string) error { - // TODO: refactor (https://github.com/loadimpact/k6/issues/883) - // - // We deliberately parse the env variables, to validate for wrong - // values, even if we don't subsequently use them (if the respective - // CLI flag was specified, since it has a higher priority). - if showCloudLogsEnv, ok := c.gs.Env["K6_SHOW_CLOUD_LOGS"]; ok { - showCloudLogsValue, err := strconv.ParseBool(showCloudLogsEnv) - if err != nil { - return fmt.Errorf("parsing K6_SHOW_CLOUD_LOGS returned an error: %w", err) - } - if !cmd.Flags().Changed("show-logs") { - c.showCloudLogs = showCloudLogsValue - } - } - - if exitOnRunningEnv, ok := c.gs.Env["K6_EXIT_ON_RUNNING"]; ok { - exitOnRunningValue, err := strconv.ParseBool(exitOnRunningEnv) - if err != nil { - return fmt.Errorf("parsing K6_EXIT_ON_RUNNING returned an error: %w", err) - } - if !cmd.Flags().Changed("exit-on-running") { - c.exitOnRunning = exitOnRunningValue - } - } - if uploadOnlyEnv, ok := c.gs.Env["K6_CLOUD_UPLOAD_ONLY"]; ok { - uploadOnlyValue, err := strconv.ParseBool(uploadOnlyEnv) - if err != nil { - return fmt.Errorf("parsing K6_CLOUD_UPLOAD_ONLY returned an error: %w", err) - } - if !cmd.Flags().Changed("upload-only") { - c.uploadOnly = uploadOnlyValue - } - } - - return nil -} - -// TODO: split apart some more -// -//nolint:funlen,gocognit,cyclop -func (c *cmdCloudRun) run(cmd *cobra.Command, args []string) error { - printBanner(c.gs) - - progressBar := pb.New( - pb.WithConstLeft("Init"), - pb.WithConstProgress(0, "Loading test script..."), - ) - printBar(c.gs, progressBar) - - test, err := loadAndConfigureLocalTest(c.gs, cmd, args, getPartialConfig) - if err != nil { - return err - } - - // It's important to NOT set the derived options back to the runner - // here, only the consolidated ones. Otherwise, if the script used - // an execution shortcut option (e.g. `iterations` or `duration`), - // we will have multiple conflicting execution options since the - // derivation will set `scenarios` as well. - testRunState, err := test.buildTestRunState(test.consolidatedConfig.Options) - if err != nil { - return err - } - - // TODO: validate for usage of execution segment - // TODO: validate for externally controlled executor (i.e. executors that aren't distributable) - // TODO: move those validations to a separate function and reuse validateConfig()? - - modifyAndPrintBar(c.gs, progressBar, pb.WithConstProgress(0, "Building the archive...")) - arc := testRunState.Runner.MakeArchive() - - tmpCloudConfig, err := cloudapi.GetTemporaryCloudConfig(arc.Options.Cloud, arc.Options.External) - if err != nil { - return err - } - - // Cloud config - cloudConfig, warn, err := cloudapi.GetConsolidatedConfig( - test.derivedConfig.Collectors["cloud"], c.gs.Env, "", arc.Options.Cloud, arc.Options.External) - if err != nil { - return err - } - if !cloudConfig.Token.Valid { - return errors.New( //nolint:golint - "not logged in, please login to the Grafana Cloud k6 " + - "using the `k6 cloud login` command", - ) - } - - // Display config warning if needed - if warn != "" { - modifyAndPrintBar(c.gs, progressBar, pb.WithConstProgress(0, "Warning: "+warn)) - } - - if cloudConfig.Token.Valid { - tmpCloudConfig["token"] = cloudConfig.Token - } - if cloudConfig.Name.Valid { - tmpCloudConfig["name"] = cloudConfig.Name - } - if cloudConfig.ProjectID.Valid { - tmpCloudConfig["projectID"] = cloudConfig.ProjectID - } - - if arc.Options.External == nil { - arc.Options.External = make(map[string]json.RawMessage) - } - - b, err := json.Marshal(tmpCloudConfig) - if err != nil { - return err - } - - arc.Options.Cloud = b - arc.Options.External[cloudapi.LegacyCloudConfigKey] = b - - name := cloudConfig.Name.String - if !cloudConfig.Name.Valid || cloudConfig.Name.String == "" { - name = filepath.Base(test.sourceRootPath) - } - - globalCtx, globalCancel := context.WithCancel(c.gs.Ctx) - defer globalCancel() - - logger := c.gs.Logger - - // Start cloud test run - modifyAndPrintBar(c.gs, progressBar, pb.WithConstProgress(0, "Validating script options")) - client := cloudapi.NewClient( - logger, cloudConfig.Token.String, cloudConfig.Host.String, consts.Version, cloudConfig.Timeout.TimeDuration()) - if err = client.ValidateOptions(arc.Options); err != nil { - return err - } - - modifyAndPrintBar(c.gs, progressBar, pb.WithConstProgress(0, "Uploading archive")) - - var cloudTestRun *cloudapi.CreateTestRunResponse - if c.uploadOnly { - cloudTestRun, err = client.UploadTestOnly(name, cloudConfig.ProjectID.Int64, arc) - } else { - cloudTestRun, err = client.StartCloudTestRun(name, cloudConfig.ProjectID.Int64, arc) - } - - if err != nil { - return err - } - - refID := cloudTestRun.ReferenceID - if cloudTestRun.ConfigOverride != nil { - cloudConfig = cloudConfig.Apply(*cloudTestRun.ConfigOverride) - } - - // Trap Interrupts, SIGINTs and SIGTERMs. - gracefulStop := func(sig os.Signal) { - logger.WithField("sig", sig).Print("Stopping cloud test run in response to signal...") - // Do this in a separate goroutine so that if it blocks, the - // second signal can still abort the process execution. - go func() { - stopErr := client.StopCloudTestRun(refID) - if stopErr != nil { - logger.WithError(stopErr).Error("Stop cloud test error") - } else { - logger.Info("Successfully sent signal to stop the cloud test, now waiting for it to actually stop...") - } - globalCancel() - }() - } - onHardStop := func(sig os.Signal) { - logger.WithField("sig", sig).Error("Aborting k6 in response to signal, we won't wait for the test to end.") - } - stopSignalHandling := handleTestAbortSignals(c.gs, gracefulStop, onHardStop) - defer stopSignalHandling() - - et, err := lib.NewExecutionTuple(test.derivedConfig.ExecutionSegment, test.derivedConfig.ExecutionSegmentSequence) - if err != nil { - return err - } - testURL := cloudapi.URLForResults(refID, cloudConfig) - executionPlan := test.derivedConfig.Scenarios.GetFullExecutionRequirements(et) - printExecutionDescription( - c.gs, "cloud", test.sourceRootPath, testURL, test.derivedConfig, et, executionPlan, nil, - ) - - modifyAndPrintBar( - c.gs, progressBar, - pb.WithConstLeft("Run "), pb.WithConstProgress(0, "Initializing the cloud test"), - ) - - progressCtx, progressCancel := context.WithCancel(globalCtx) - progressBarWG := &sync.WaitGroup{} - progressBarWG.Add(1) - defer progressBarWG.Wait() - defer progressCancel() - go func() { - showProgress(progressCtx, c.gs, []*pb.ProgressBar{progressBar}, logger) - progressBarWG.Done() - }() - - var ( - startTime time.Time - maxDuration time.Duration - ) - maxDuration, _ = lib.GetEndOffset(executionPlan) - - testProgressLock := &sync.Mutex{} - var testProgress *cloudapi.TestProgressResponse - progressBar.Modify( - pb.WithProgress(func() (float64, []string) { - testProgressLock.Lock() - defer testProgressLock.Unlock() - - if testProgress == nil { - return 0, []string{"Waiting..."} - } - - statusText := testProgress.RunStatusText - - if testProgress.RunStatus == cloudapi.RunStatusFinished { - testProgress.Progress = 1 - } else if testProgress.RunStatus == cloudapi.RunStatusRunning { - if startTime.IsZero() { - startTime = time.Now() - } - spent := time.Since(startTime) - if spent > maxDuration { - statusText = maxDuration.String() - } else { - statusText = fmt.Sprintf("%s/%s", pb.GetFixedLengthDuration(spent, maxDuration), maxDuration) - } - } - - return testProgress.Progress, []string{statusText} - }), - ) - - ticker := time.NewTicker(time.Millisecond * 2000) - if c.showCloudLogs { - go func() { - logger.Debug("Connecting to cloud logs server...") - if err := cloudConfig.StreamLogsToLogger(globalCtx, logger, refID, 0); err != nil { - logger.WithError(err).Error("error while tailing cloud logs") - } - }() - } - - for range ticker.C { - newTestProgress, progressErr := client.GetTestProgress(refID) - if progressErr != nil { - logger.WithError(progressErr).Error("Test progress error") - continue - } - - testProgressLock.Lock() - testProgress = newTestProgress - testProgressLock.Unlock() - - if (newTestProgress.RunStatus > cloudapi.RunStatusRunning) || - (c.exitOnRunning && newTestProgress.RunStatus == cloudapi.RunStatusRunning) { - globalCancel() - break - } - } - - if testProgress == nil { - //nolint:stylecheck,golint - return errext.WithExitCodeIfNone(errors.New("Test progress error"), exitcodes.CloudFailedToGetProgress) - } - - if !c.gs.Flags.Quiet { - valueColor := getColor(c.gs.Flags.NoColor || !c.gs.Stdout.IsTTY, color.FgCyan) - printToStdout(c.gs, fmt.Sprintf( - " test status: %s\n", valueColor.Sprint(testProgress.RunStatusText), - )) - } else { - logger.WithField("run_status", testProgress.RunStatusText).Debug("Test finished") - } - - if testProgress.ResultStatus == cloudapi.ResultStatusFailed { - // TODO: use different exit codes for failed thresholds vs failed test (e.g. aborted by system/limit) - //nolint:stylecheck,golint - return errext.WithExitCodeIfNone(errors.New("The test has failed"), exitcodes.CloudTestRunFailed) - } - - return nil -} - -func (c *cmdCloudRun) flagSet() *pflag.FlagSet { - flags := pflag.NewFlagSet("", pflag.ContinueOnError) - flags.SortFlags = false - flags.AddFlagSet(optionFlagSet()) - flags.AddFlagSet(runtimeOptionFlagSet(false)) - - // TODO: Figure out a better way to handle the CLI flags - flags.BoolVar(&c.exitOnRunning, "exit-on-running", c.exitOnRunning, - "exits when test reaches the running status") - flags.BoolVar(&c.showCloudLogs, "show-logs", c.showCloudLogs, - "enable showing of logs when a test is executed in the cloud") - flags.BoolVar(&c.uploadOnly, "upload-only", c.uploadOnly, - "only upload the test to the cloud without actually starting a test run") - - return flags -} - -const cloudRunCommandName string = "run" diff --git a/cmd/tests/cmd_cloud_run_test.go b/cmd/tests/cmd_cloud_run_test.go index f41378c2884..b16ec5b1a1c 100644 --- a/cmd/tests/cmd_cloud_run_test.go +++ b/cmd/tests/cmd_cloud_run_test.go @@ -1,248 +1,12 @@ package tests -import ( - "encoding/json" - "fmt" - "io" - "net/http" - "os" - "path/filepath" - "testing" +import "testing" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - "go.k6.io/k6/cloudapi" - "go.k6.io/k6/cmd" - "go.k6.io/k6/lib/fsext" - "go.k6.io/k6/lib/testutils" -) - -func TestCloudRunNotLoggedIn(t *testing.T) { +func TestK6CloudRun(t *testing.T) { t.Parallel() - - ts := getSimpleCloudRunTestState(t, nil, nil, nil, nil) - delete(ts.Env, "K6_CLOUD_TOKEN") - ts.ExpectedExitCode = -1 // TODO: use a more specific exit code? - cmd.ExecuteWithGlobalState(ts.GlobalState) - - stdout := ts.Stdout.String() - t.Log(stdout) - assert.Contains(t, stdout, `not logged in`) + runCloudTests(t, setupK6CloudRunCmd) } -func TestCloudRunLoggedInWithScriptToken(t *testing.T) { - t.Parallel() - - script := ` - export let options = { - ext: { - loadimpact: { - token: "asdf", - name: "my load test", - projectID: 124, - note: 124, - }, - } - }; - - export default function() {}; - ` - - ts := getSimpleCloudRunTestState(t, []byte(script), nil, nil, nil) - delete(ts.Env, "K6_CLOUD_TOKEN") - cmd.ExecuteWithGlobalState(ts.GlobalState) - - stdout := ts.Stdout.String() - t.Log(stdout) - assert.NotContains(t, stdout, `Not logged in`) - assert.Contains(t, stdout, `execution: cloud`) - assert.Contains(t, stdout, `output: https://app.k6.io/runs/123`) - assert.Contains(t, stdout, `test status: Finished`) -} - -func TestCloudRunExitOnRunning(t *testing.T) { - t.Parallel() - - cs := func() cloudapi.TestProgressResponse { - return cloudapi.TestProgressResponse{ - RunStatusText: "Running", - RunStatus: cloudapi.RunStatusRunning, - } - } - - ts := getSimpleCloudRunTestState(t, nil, []string{"--exit-on-running", "--log-output=stdout"}, nil, cs) - cmd.ExecuteWithGlobalState(ts.GlobalState) - - stdout := ts.Stdout.String() - t.Log(stdout) - assert.Contains(t, stdout, `execution: cloud`) - assert.Contains(t, stdout, `output: https://app.k6.io/runs/123`) - assert.Contains(t, stdout, `test status: Running`) -} - -func TestCloudRunUploadOnly(t *testing.T) { - t.Parallel() - - cs := func() cloudapi.TestProgressResponse { - return cloudapi.TestProgressResponse{ - RunStatusText: "Archived", - RunStatus: cloudapi.RunStatusArchived, - } - } - - ts := getSimpleCloudRunTestState(t, nil, []string{"--upload-only", "--log-output=stdout"}, nil, cs) - cmd.ExecuteWithGlobalState(ts.GlobalState) - - stdout := ts.Stdout.String() - t.Log(stdout) - assert.Contains(t, stdout, `execution: cloud`) - assert.Contains(t, stdout, `output: https://app.k6.io/runs/123`) - assert.Contains(t, stdout, `test status: Archived`) -} - -func TestCloudRunWithConfigOverride(t *testing.T) { - t.Parallel() - - configOverride := http.HandlerFunc(func(resp http.ResponseWriter, _ *http.Request) { - resp.WriteHeader(http.StatusOK) - _, err := fmt.Fprint(resp, `{ - "reference_id": "123", - "config": { - "webAppURL": "https://bogus.url", - "testRunDetails": "something from the cloud" - }, - "logs": [ - {"level": "invalid", "message": "test debug message"}, - {"level": "warning", "message": "test warning"}, - {"level": "error", "message": "test error"} - ] - }`) - assert.NoError(t, err) - }) - ts := getSimpleCloudRunTestState(t, nil, nil, configOverride, nil) - cmd.ExecuteWithGlobalState(ts.GlobalState) - - stdout := ts.Stdout.String() - t.Log(stdout) - assert.Contains(t, stdout, "execution: cloud") - assert.Contains(t, stdout, "output: something from the cloud") - assert.Contains(t, stdout, `level=debug msg="invalid message level 'invalid' for message 'test debug message'`) - assert.Contains(t, stdout, `level=error msg="test debug message" source=grafana-k6-cloud`) - assert.Contains(t, stdout, `level=warning msg="test warning" source=grafana-k6-cloud`) - assert.Contains(t, stdout, `level=error msg="test error" source=grafana-k6-cloud`) -} - -// TestCloudRunWithArchive tests that if k6 uses a static archive with the script inside that has cloud options like: -// -// export let options = { -// ext: { -// loadimpact: { -// name: "my load test", -// projectID: 124, -// note: "lorem ipsum", -// }, -// } -// }; -// -// actually sends to the cloud the archive with the correct metadata (metadata.json), like: -// -// "ext": { -// "loadimpact": { -// "name": "my load test", -// "note": "lorem ipsum", -// "projectID": 124 -// } -// } -func TestCloudRunWithArchive(t *testing.T) { - t.Parallel() - - testRunID := 123 - ts := NewGlobalTestState(t) - - archiveUpload := http.HandlerFunc(func(resp http.ResponseWriter, req *http.Request) { - // check the archive - file, _, err := req.FormFile("file") - assert.NoError(t, err) - assert.NotNil(t, file) - - // temporary write the archive for file system - data, err := io.ReadAll(file) - assert.NoError(t, err) - - tmpPath := filepath.Join(ts.Cwd, "archive_to_cloud.tar") - require.NoError(t, fsext.WriteFile(ts.FS, tmpPath, data, 0o644)) - - // check what inside - require.NoError(t, testutils.Untar(t, ts.FS, tmpPath, "tmp/")) - - metadataRaw, err := fsext.ReadFile(ts.FS, "tmp/metadata.json") - require.NoError(t, err) - - metadata := struct { - Options struct { - Cloud struct { - Name string `json:"name"` - Note string `json:"note"` - ProjectID int `json:"projectID"` - } `json:"cloud"` - } `json:"options"` - }{} - - // then unpacked metadata should not contain any environment variables passed at the moment of archive creation - require.NoError(t, json.Unmarshal(metadataRaw, &metadata)) - require.Equal(t, "my load test", metadata.Options.Cloud.Name) - require.Equal(t, "lorem ipsum", metadata.Options.Cloud.Note) - require.Equal(t, 124, metadata.Options.Cloud.ProjectID) - - // respond with the test run ID - resp.WriteHeader(http.StatusOK) - _, err = fmt.Fprintf(resp, `{"reference_id": "%d"}`, testRunID) - assert.NoError(t, err) - }) - - srv := getMockCloud(t, testRunID, archiveUpload, nil) - - data, err := os.ReadFile(filepath.Join("testdata/archives", "archive_v0.46.0_with_loadimpact_option.tar")) //nolint:forbidigo // it's a test - require.NoError(t, err) - - require.NoError(t, fsext.WriteFile(ts.FS, filepath.Join(ts.Cwd, "archive.tar"), data, 0o644)) - - ts.CmdArgs = []string{"k6", "cloud", "--verbose", "--log-output=stdout", "archive.tar"} - ts.Env["K6_SHOW_CLOUD_LOGS"] = "false" // no mock for the logs yet - ts.Env["K6_CLOUD_HOST"] = srv.URL - ts.Env["K6_CLOUD_TOKEN"] = "foo" // doesn't matter, we mock the cloud - - cmd.ExecuteWithGlobalState(ts.GlobalState) - - stdout := ts.Stdout.String() - t.Log(stdout) - assert.NotContains(t, stdout, `Not logged in`) - assert.Contains(t, stdout, `execution: cloud`) - assert.Contains(t, stdout, `hello world from archive`) - assert.Contains(t, stdout, `output: https://app.k6.io/runs/123`) - assert.Contains(t, stdout, `test status: Finished`) -} - -func getSimpleCloudRunTestState( - t *testing.T, script []byte, cliFlags []string, - archiveUpload http.Handler, progressCallback func() cloudapi.TestProgressResponse, -) *GlobalTestState { - if script == nil { - script = []byte(`export default function() {}`) - } - - if cliFlags == nil { - cliFlags = []string{"--verbose", "--log-output=stdout"} - } - - srv := getMockCloud(t, 123, archiveUpload, progressCallback) - - ts := NewGlobalTestState(t) - require.NoError(t, fsext.WriteFile(ts.FS, filepath.Join(ts.Cwd, "test.js"), script, 0o644)) - ts.CmdArgs = append(append([]string{"k6", "cloud", "run"}, cliFlags...), "test.js") - ts.Env["K6_SHOW_CLOUD_LOGS"] = "false" // no mock for the logs yet - ts.Env["K6_CLOUD_HOST"] = srv.URL - ts.Env["K6_CLOUD_TOKEN"] = "foo" // doesn't matter, we mock the cloud - - return ts +func setupK6CloudRunCmd(cliFlags []string) []string { + return append([]string{"k6", "cloud", "run"}, append(cliFlags, "test.js")...) } diff --git a/cmd/tests/cmd_cloud_test.go b/cmd/tests/cmd_cloud_test.go index a8e76e38006..838cb60a461 100644 --- a/cmd/tests/cmd_cloud_test.go +++ b/cmd/tests/cmd_cloud_test.go @@ -10,97 +10,45 @@ import ( "path/filepath" "testing" + "go.k6.io/k6/lib/testutils" + + "go.k6.io/k6/cmd" + "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "go.k6.io/k6/cloudapi" - "go.k6.io/k6/cmd" "go.k6.io/k6/lib/fsext" - "go.k6.io/k6/lib/testutils" ) -func cloudTestStartSimple(tb testing.TB, testRunID int) http.Handler { - return http.HandlerFunc(func(resp http.ResponseWriter, _ *http.Request) { - resp.WriteHeader(http.StatusOK) - _, err := fmt.Fprintf(resp, `{"reference_id": "%d"}`, testRunID) - assert.NoError(tb, err) - }) +func TestK6Cloud(t *testing.T) { + t.Parallel() + runCloudTests(t, setupK6CloudCmd) } -func getMockCloud( - t *testing.T, testRunID int, - archiveUpload http.Handler, progressCallback func() cloudapi.TestProgressResponse, -) *httptest.Server { - if archiveUpload == nil { - archiveUpload = cloudTestStartSimple(t, testRunID) - } - testProgressURL := fmt.Sprintf("GET ^/v1/test-progress/%d$", testRunID) - defaultProgress := cloudapi.TestProgressResponse{ - RunStatusText: "Finished", - RunStatus: cloudapi.RunStatusFinished, - ResultStatus: cloudapi.ResultStatusPassed, - Progress: 1, - } - - srv := getTestServer(t, map[string]http.Handler{ - "POST ^/v1/archive-upload$": archiveUpload, - testProgressURL: http.HandlerFunc(func(resp http.ResponseWriter, _ *http.Request) { - testProgress := defaultProgress - if progressCallback != nil { - testProgress = progressCallback() - } - respBody, err := json.Marshal(testProgress) - assert.NoError(t, err) - _, err = fmt.Fprint(resp, string(respBody)) - assert.NoError(t, err) - }), - }) - - t.Cleanup(srv.Close) - - return srv +func setupK6CloudCmd(cliFlags []string) []string { + return append([]string{"k6", "cloud"}, append(cliFlags, "test.js")...) } -func getSimpleCloudTestState( - t *testing.T, script []byte, cliFlags []string, - archiveUpload http.Handler, progressCallback func() cloudapi.TestProgressResponse, -) *GlobalTestState { - if script == nil { - script = []byte(`export default function() {}`) - } - - if cliFlags == nil { - cliFlags = []string{"--verbose", "--log-output=stdout"} - } - - srv := getMockCloud(t, 123, archiveUpload, progressCallback) - - ts := NewGlobalTestState(t) - require.NoError(t, fsext.WriteFile(ts.FS, filepath.Join(ts.Cwd, "test.js"), script, 0o644)) - ts.CmdArgs = append(append([]string{"k6", "cloud"}, cliFlags...), "test.js") - ts.Env["K6_SHOW_CLOUD_LOGS"] = "false" // no mock for the logs yet - ts.Env["K6_CLOUD_HOST"] = srv.URL - ts.Env["K6_CLOUD_TOKEN"] = "foo" // doesn't matter, we mock the cloud - - return ts -} +type setupCommandFunc func(cliFlags []string) []string -func TestCloudNotLoggedIn(t *testing.T) { - t.Parallel() +func runCloudTests(t *testing.T, setupCmd setupCommandFunc) { + t.Run("TestCloudNotLoggedIn", func(t *testing.T) { + t.Parallel() - ts := getSimpleCloudTestState(t, nil, nil, nil, nil) - delete(ts.Env, "K6_CLOUD_TOKEN") - ts.ExpectedExitCode = -1 // TODO: use a more specific exit code? - cmd.ExecuteWithGlobalState(ts.GlobalState) + ts := getSimpleCloudTestState(t, nil, setupCmd, nil, nil, nil) + delete(ts.Env, "K6_CLOUD_TOKEN") + ts.ExpectedExitCode = -1 // TODO: use a more specific exit code? + cmd.ExecuteWithGlobalState(ts.GlobalState) - stdout := ts.Stdout.String() - t.Log(stdout) - assert.Contains(t, stdout, `Not logged in`) -} + stdout := ts.Stdout.String() + t.Log(stdout) + assert.Contains(t, stdout, `not logged in`) + }) -func TestCloudLoggedInWithScriptToken(t *testing.T) { - t.Parallel() + t.Run("TestCloudLoggedInWithScriptToken", func(t *testing.T) { + t.Parallel() - script := ` + script := ` export let options = { ext: { loadimpact: { @@ -114,64 +62,64 @@ func TestCloudLoggedInWithScriptToken(t *testing.T) { export default function() {}; ` - ts := getSimpleCloudTestState(t, []byte(script), nil, nil, nil) - delete(ts.Env, "K6_CLOUD_TOKEN") - cmd.ExecuteWithGlobalState(ts.GlobalState) + ts := getSimpleCloudTestState(t, []byte(script), setupCmd, nil, nil, nil) + delete(ts.Env, "K6_CLOUD_TOKEN") + cmd.ExecuteWithGlobalState(ts.GlobalState) - stdout := ts.Stdout.String() - t.Log(stdout) - assert.NotContains(t, stdout, `Not logged in`) - assert.Contains(t, stdout, `execution: cloud`) - assert.Contains(t, stdout, `output: https://app.k6.io/runs/123`) - assert.Contains(t, stdout, `test status: Finished`) -} + stdout := ts.Stdout.String() + t.Log(stdout) + assert.NotContains(t, stdout, `not logged in`) + assert.Contains(t, stdout, `execution: cloud`) + assert.Contains(t, stdout, `output: https://app.k6.io/runs/123`) + assert.Contains(t, stdout, `test status: Finished`) + }) -func TestCloudExitOnRunning(t *testing.T) { - t.Parallel() + t.Run("TestCloudExitOnRunning", func(t *testing.T) { + t.Parallel() - cs := func() cloudapi.TestProgressResponse { - return cloudapi.TestProgressResponse{ - RunStatusText: "Running", - RunStatus: cloudapi.RunStatusRunning, + cs := func() cloudapi.TestProgressResponse { + return cloudapi.TestProgressResponse{ + RunStatusText: "Running", + RunStatus: cloudapi.RunStatusRunning, + } } - } - ts := getSimpleCloudTestState(t, nil, []string{"--exit-on-running", "--log-output=stdout"}, nil, cs) - cmd.ExecuteWithGlobalState(ts.GlobalState) + ts := getSimpleCloudTestState(t, nil, setupCmd, []string{"--exit-on-running", "--log-output=stdout"}, nil, cs) + cmd.ExecuteWithGlobalState(ts.GlobalState) - stdout := ts.Stdout.String() - t.Log(stdout) - assert.Contains(t, stdout, `execution: cloud`) - assert.Contains(t, stdout, `output: https://app.k6.io/runs/123`) - assert.Contains(t, stdout, `test status: Running`) -} + stdout := ts.Stdout.String() + t.Log(stdout) + assert.Contains(t, stdout, `execution: cloud`) + assert.Contains(t, stdout, `output: https://app.k6.io/runs/123`) + assert.Contains(t, stdout, `test status: Running`) + }) -func TestCloudUploadOnly(t *testing.T) { - t.Parallel() + t.Run("TestCloudUploadOnly", func(t *testing.T) { + t.Parallel() - cs := func() cloudapi.TestProgressResponse { - return cloudapi.TestProgressResponse{ - RunStatusText: "Archived", - RunStatus: cloudapi.RunStatusArchived, + cs := func() cloudapi.TestProgressResponse { + return cloudapi.TestProgressResponse{ + RunStatusText: "Archived", + RunStatus: cloudapi.RunStatusArchived, + } } - } - ts := getSimpleCloudTestState(t, nil, []string{"--upload-only", "--log-output=stdout"}, nil, cs) - cmd.ExecuteWithGlobalState(ts.GlobalState) + ts := getSimpleCloudTestState(t, nil, setupCmd, []string{"--upload-only", "--log-output=stdout"}, nil, cs) + cmd.ExecuteWithGlobalState(ts.GlobalState) - stdout := ts.Stdout.String() - t.Log(stdout) - assert.Contains(t, stdout, `execution: cloud`) - assert.Contains(t, stdout, `output: https://app.k6.io/runs/123`) - assert.Contains(t, stdout, `test status: Archived`) -} + stdout := ts.Stdout.String() + t.Log(stdout) + assert.Contains(t, stdout, `execution: cloud`) + assert.Contains(t, stdout, `output: https://app.k6.io/runs/123`) + assert.Contains(t, stdout, `test status: Archived`) + }) -func TestCloudWithConfigOverride(t *testing.T) { - t.Parallel() + t.Run("TestCloudWithConfigOverride", func(t *testing.T) { + t.Parallel() - configOverride := http.HandlerFunc(func(resp http.ResponseWriter, _ *http.Request) { - resp.WriteHeader(http.StatusOK) - _, err := fmt.Fprint(resp, `{ + configOverride := http.HandlerFunc(func(resp http.ResponseWriter, _ *http.Request) { + resp.WriteHeader(http.StatusOK) + _, err := fmt.Fprint(resp, `{ "reference_id": "123", "config": { "webAppURL": "https://bogus.url", @@ -183,108 +131,172 @@ func TestCloudWithConfigOverride(t *testing.T) { {"level": "error", "message": "test error"} ] }`) - assert.NoError(t, err) + assert.NoError(t, err) + }) + ts := getSimpleCloudTestState(t, nil, setupCmd, nil, configOverride, nil) + cmd.ExecuteWithGlobalState(ts.GlobalState) + + stdout := ts.Stdout.String() + t.Log(stdout) + assert.Contains(t, stdout, "execution: cloud") + assert.Contains(t, stdout, "output: something from the cloud") + assert.Contains(t, stdout, `level=debug msg="invalid message level 'invalid' for message 'test debug message'`) + assert.Contains(t, stdout, `level=error msg="test debug message" source=grafana-k6-cloud`) + assert.Contains(t, stdout, `level=warning msg="test warning" source=grafana-k6-cloud`) + assert.Contains(t, stdout, `level=error msg="test error" source=grafana-k6-cloud`) }) - ts := getSimpleCloudTestState(t, nil, nil, configOverride, nil) - cmd.ExecuteWithGlobalState(ts.GlobalState) - - stdout := ts.Stdout.String() - t.Log(stdout) - assert.Contains(t, stdout, "execution: cloud") - assert.Contains(t, stdout, "output: something from the cloud") - assert.Contains(t, stdout, `level=debug msg="invalid message level 'invalid' for message 'test debug message'`) - assert.Contains(t, stdout, `level=error msg="test debug message" source=grafana-k6-cloud`) - assert.Contains(t, stdout, `level=warning msg="test warning" source=grafana-k6-cloud`) - assert.Contains(t, stdout, `level=error msg="test error" source=grafana-k6-cloud`) -} -// TestCloudWithArchive tests that if k6 uses a static archive with the script inside that has cloud options like: -// -// export let options = { -// ext: { -// loadimpact: { -// name: "my load test", -// projectID: 124, -// note: "lorem ipsum", -// }, -// } -// }; -// -// actually sends to the cloud the archive with the correct metadata (metadata.json), like: -// -// "ext": { -// "loadimpact": { -// "name": "my load test", -// "note": "lorem ipsum", -// "projectID": 124 -// } -// } -func TestCloudWithArchive(t *testing.T) { - t.Parallel() + // TestCloudWithArchive tests that if k6 uses a static archive with the script inside that has cloud options like: + // + // export let options = { + // ext: { + // loadimpact: { + // name: "my load test", + // projectID: 124, + // note: "lorem ipsum", + // }, + // } + // }; + // + // actually sends to the cloud the archive with the correct metadata (metadata.json), like: + // + // "ext": { + // "loadimpact": { + // "name": "my load test", + // "note": "lorem ipsum", + // "projectID": 124 + // } + // } + t.Run("TestCloudWithArchive", func(t *testing.T) { + t.Parallel() + + testRunID := 123 + ts := NewGlobalTestState(t) + + archiveUpload := http.HandlerFunc(func(resp http.ResponseWriter, req *http.Request) { + // check the archive + file, _, err := req.FormFile("file") + assert.NoError(t, err) + assert.NotNil(t, file) - testRunID := 123 - ts := NewGlobalTestState(t) + // temporary write the archive for file system + data, err := io.ReadAll(file) + assert.NoError(t, err) - archiveUpload := http.HandlerFunc(func(resp http.ResponseWriter, req *http.Request) { - // check the archive - file, _, err := req.FormFile("file") - assert.NoError(t, err) - assert.NotNil(t, file) + tmpPath := filepath.Join(ts.Cwd, "archive_to_cloud.tar") + require.NoError(t, fsext.WriteFile(ts.FS, tmpPath, data, 0o644)) + + // check what inside + require.NoError(t, testutils.Untar(t, ts.FS, tmpPath, "tmp/")) + + metadataRaw, err := fsext.ReadFile(ts.FS, "tmp/metadata.json") + require.NoError(t, err) + + metadata := struct { + Options struct { + Cloud struct { + Name string `json:"name"` + Note string `json:"note"` + ProjectID int `json:"projectID"` + } `json:"cloud"` + } `json:"options"` + }{} + + // then unpacked metadata should not contain any environment variables passed at the moment of archive creation + require.NoError(t, json.Unmarshal(metadataRaw, &metadata)) + require.Equal(t, "my load test", metadata.Options.Cloud.Name) + require.Equal(t, "lorem ipsum", metadata.Options.Cloud.Note) + require.Equal(t, 124, metadata.Options.Cloud.ProjectID) + + // respond with the test run ID + resp.WriteHeader(http.StatusOK) + _, err = fmt.Fprintf(resp, `{"reference_id": "%d"}`, testRunID) + assert.NoError(t, err) + }) - // temporary write the archive for file system - data, err := io.ReadAll(file) - assert.NoError(t, err) + srv := getMockCloud(t, testRunID, archiveUpload, nil) - tmpPath := filepath.Join(ts.Cwd, "archive_to_cloud.tar") - require.NoError(t, fsext.WriteFile(ts.FS, tmpPath, data, 0o644)) + data, err := os.ReadFile(filepath.Join("testdata/archives", "archive_v0.46.0_with_loadimpact_option.tar")) //nolint:forbidigo // it's a test + require.NoError(t, err) - // check what inside - require.NoError(t, testutils.Untar(t, ts.FS, tmpPath, "tmp/")) + require.NoError(t, fsext.WriteFile(ts.FS, filepath.Join(ts.Cwd, "archive.tar"), data, 0o644)) - metadataRaw, err := fsext.ReadFile(ts.FS, "tmp/metadata.json") - require.NoError(t, err) + ts.CmdArgs = []string{"k6", "cloud", "--verbose", "--log-output=stdout", "archive.tar"} + ts.Env["K6_SHOW_CLOUD_LOGS"] = "false" // no mock for the logs yet + ts.Env["K6_CLOUD_HOST"] = srv.URL + ts.Env["K6_CLOUD_TOKEN"] = "foo" // doesn't matter, we mock the cloud - metadata := struct { - Options struct { - Cloud struct { - Name string `json:"name"` - Note string `json:"note"` - ProjectID int `json:"projectID"` - } `json:"cloud"` - } `json:"options"` - }{} - - // then unpacked metadata should not contain any environment variables passed at the moment of archive creation - require.NoError(t, json.Unmarshal(metadataRaw, &metadata)) - require.Equal(t, "my load test", metadata.Options.Cloud.Name) - require.Equal(t, "lorem ipsum", metadata.Options.Cloud.Note) - require.Equal(t, 124, metadata.Options.Cloud.ProjectID) - - // respond with the test run ID + cmd.ExecuteWithGlobalState(ts.GlobalState) + + stdout := ts.Stdout.String() + t.Log(stdout) + assert.NotContains(t, stdout, `not logged in`) + assert.Contains(t, stdout, `execution: cloud`) + assert.Contains(t, stdout, `hello world from archive`) + assert.Contains(t, stdout, `output: https://app.k6.io/runs/123`) + assert.Contains(t, stdout, `test status: Finished`) + }) +} + +func cloudTestStartSimple(tb testing.TB, testRunID int) http.Handler { + return http.HandlerFunc(func(resp http.ResponseWriter, _ *http.Request) { resp.WriteHeader(http.StatusOK) - _, err = fmt.Fprintf(resp, `{"reference_id": "%d"}`, testRunID) - assert.NoError(t, err) + _, err := fmt.Fprintf(resp, `{"reference_id": "%d"}`, testRunID) + assert.NoError(tb, err) + }) +} + +func getMockCloud( + t *testing.T, testRunID int, + archiveUpload http.Handler, progressCallback func() cloudapi.TestProgressResponse, +) *httptest.Server { + if archiveUpload == nil { + archiveUpload = cloudTestStartSimple(t, testRunID) + } + testProgressURL := fmt.Sprintf("GET ^/v1/test-progress/%d$", testRunID) + defaultProgress := cloudapi.TestProgressResponse{ + RunStatusText: "Finished", + RunStatus: cloudapi.RunStatusFinished, + ResultStatus: cloudapi.ResultStatusPassed, + Progress: 1, + } + + srv := getTestServer(t, map[string]http.Handler{ + "POST ^/v1/archive-upload$": archiveUpload, + testProgressURL: http.HandlerFunc(func(resp http.ResponseWriter, _ *http.Request) { + testProgress := defaultProgress + if progressCallback != nil { + testProgress = progressCallback() + } + respBody, err := json.Marshal(testProgress) + assert.NoError(t, err) + _, err = fmt.Fprint(resp, string(respBody)) + assert.NoError(t, err) + }), }) - srv := getMockCloud(t, testRunID, archiveUpload, nil) + t.Cleanup(srv.Close) + + return srv +} + +func getSimpleCloudTestState(t *testing.T, script []byte, setupCmd setupCommandFunc, cliFlags []string, archiveUpload http.Handler, progressCallback func() cloudapi.TestProgressResponse) *GlobalTestState { + if script == nil { + script = []byte(`export default function() {}`) + } - data, err := os.ReadFile(filepath.Join("testdata/archives", "archive_v0.46.0_with_loadimpact_option.tar")) //nolint:forbidigo // it's a test - require.NoError(t, err) + if cliFlags == nil { + cliFlags = []string{"--verbose", "--log-output=stdout"} + } - require.NoError(t, fsext.WriteFile(ts.FS, filepath.Join(ts.Cwd, "archive.tar"), data, 0o644)) + srv := getMockCloud(t, 123, archiveUpload, progressCallback) - ts.CmdArgs = []string{"k6", "cloud", "--verbose", "--log-output=stdout", "archive.tar"} + ts := NewGlobalTestState(t) + require.NoError(t, fsext.WriteFile(ts.FS, filepath.Join(ts.Cwd, "test.js"), script, 0o644)) + ts.CmdArgs = setupCmd(cliFlags) ts.Env["K6_SHOW_CLOUD_LOGS"] = "false" // no mock for the logs yet ts.Env["K6_CLOUD_HOST"] = srv.URL ts.Env["K6_CLOUD_TOKEN"] = "foo" // doesn't matter, we mock the cloud - cmd.ExecuteWithGlobalState(ts.GlobalState) - - stdout := ts.Stdout.String() - t.Log(stdout) - assert.NotContains(t, stdout, `Not logged in`) - assert.Contains(t, stdout, `execution: cloud`) - assert.Contains(t, stdout, `hello world from archive`) - assert.Contains(t, stdout, `output: https://app.k6.io/runs/123`) - assert.Contains(t, stdout, `test status: Finished`) + return ts }