diff --git a/CHANGELOG.md b/CHANGELOG.md index ca6e31a2..43246ef9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,6 @@ # v2.3.0 + - Preserve test execution order alphabetical order - Add property `skip`, adds the ability to skip test cases # v2.2.0 diff --git a/README.md b/README.md index c5400b87..20c4b1de 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ [![Build Status](https://travis-ci.org/commander-cli/commander.svg?branch=master)](https://travis-ci.org/commander-cli/commander) [![GoDoc](https://godoc.org/github.com/commander-cli/commander?status.svg)](https://godoc.org/github.com/commander-cli/commander) -[![Go Report Card](https://goreportcard.com/badge/github.com/commander-cli/commander)](https://goreportcard.com/report/github.com/SimonBaeumer/commander) +[![Go Report Card](https://goreportcard.com/badge/github.com/commander-cli/commander)](https://goreportcard.com/report/github.com/commander-cli/commander) [![Maintainability](https://api.codeclimate.com/v1/badges/cc848165784e0f809a51/maintainability)](https://codeclimate.com/github/commander-cli/commander/maintainability) [![Test Coverage](https://api.codeclimate.com/v1/badges/cc848165784e0f809a51/test_coverage)](https://codeclimate.com/github/commander-cli/commander/test_coverage) [![Github All Releases](https://img.shields.io/github/downloads/commander-cli/commander/total.svg)](https://github.com/commander-cli/commander/releases) diff --git a/cmd/commander/commander.go b/cmd/commander/commander.go index df668241..785a029a 100644 --- a/cmd/commander/commander.go +++ b/cmd/commander/commander.go @@ -46,7 +46,22 @@ func createCliApp() *cli.App { func createTestCommand() cli.Command { return cli.Command{ Name: "test", - Usage: "Execute the test suite, by default it will use the commander.yaml from your current directory", + Usage: `Execute cli app tests + +By default it will use the commander.yaml from your current directory. +Tests are always executed in alphabetical order. + +Examples: + +Filtering tests: +test commander.yaml --filter="my test" + +Multiple filters: +test commander.yaml --filter=filter1 --filter=filter2 + +Regex filters: +test commander.yaml --filter="^filter1$" +`, ArgsUsage: "[file] [--filter]", Flags: []cli.Flag{ cli.BoolFlag{ @@ -65,14 +80,7 @@ func createTestCommand() cli.Command { }, cli.StringSliceFlag{ Name: "filter", - Usage: `Filter tests by a given regex pattern. Tests are filtered by its title. - -Example: -test commander.yaml --filter="my test" - -Apply multiple filters: -test commander.yaml --filter=filter1 --filter=filter2 -`, + Usage: `Filter tests by a given regex pattern. Tests are filtered by its title.`, }, }, Action: func(c *cli.Context) error { diff --git a/commander_unix.yaml b/commander_unix.yaml index facbeae1..248dfed7 100644 --- a/commander_unix.yaml +++ b/commander_unix.yaml @@ -111,4 +111,15 @@ tests: - ✓ [local] should be ignored not-contains: - executed at the beginning is ignored - exit-code: 0 \ No newline at end of file + exit-code: 0 + + it should be executed in alphabetical order: + command: ./commander test integration/unix/alphabetically_order.yaml + stdout: + contains: + - |- + ✓ [local] --- + ✓ [local] 123 + ✓ [local] a + ✓ [local] b + exit-code: 0 diff --git a/examples/ordering.yaml b/examples/ordering.yaml new file mode 100644 index 00000000..02919ed8 --- /dev/null +++ b/examples/ordering.yaml @@ -0,0 +1,18 @@ +# Tests are always executed in alphabetical order + +tests: + 001 - test: + command: exit 0 + exit-code: 0 + + 002 - test: + command: exit 0 + exit-code: 0 + + 003 - test: + command: exit 0 + exit-code: 0 + + 004 - test: + command: exit 0 + exit-code: 0 \ No newline at end of file diff --git a/integration/unix/alphabetically_order.yaml b/integration/unix/alphabetically_order.yaml new file mode 100644 index 00000000..6a8b1a6d --- /dev/null +++ b/integration/unix/alphabetically_order.yaml @@ -0,0 +1,13 @@ +tests: + a: + command: echo test + exit-code: 0 + b: + command: echo test + exit-code: 0 + 123: + command: echo test + exit-code: 0 + "---": + command: echo test + exit-code: 0 \ No newline at end of file diff --git a/pkg/runtime/runtime.go b/pkg/runtime/runtime.go index 0f0cce10..405088df 100644 --- a/pkg/runtime/runtime.go +++ b/pkg/runtime/runtime.go @@ -2,6 +2,7 @@ package runtime import ( "log" + "sort" "time" ) @@ -131,6 +132,11 @@ type Result struct { // Start starts the given test suite and executes all tests func (r *Runtime) Start(tests []TestCase) Result { + // Sort tests alphabetically to preserve a reproducible execution order + sort.SliceStable(tests, func(i, j int) bool { + return tests[i].Title < tests[j].Title + }) + result := Result{} testCh := r.Runner.Run(tests) start := time.Now() diff --git a/pkg/runtime/runtime_test.go b/pkg/runtime/runtime_test.go index ef25bb67..b2a0a940 100644 --- a/pkg/runtime/runtime_test.go +++ b/pkg/runtime/runtime_test.go @@ -47,6 +47,27 @@ func TestRuntime_WithRetries(t *testing.T) { assert.Equal(t, 1, counter) } +func Test_AlphabeticalOrder(t *testing.T) { + tests := []TestCase{ + {Title: "bbb", Command: CommandUnderTest{Cmd: "exit 0;"}}, + {Title: "aaa"}, + {Title: "111"}, + {Title: "_"}, + } + + got := []string{} + runtime := NewRuntime(&EventHandler{TestFinished: func(r TestResult) { + got = append(got, r.TestCase.Title) + }}) + + runtime.Start(tests) + + assert.Equal(t, "111", got[0]) + assert.Equal(t, "_", got[1]) + assert.Equal(t, "aaa", got[2]) + assert.Equal(t, "bbb", got[3]) +} + func Test_RuntimeWithRetriesAndInterval(t *testing.T) { s := getExampleTestCases() s[0].Command.Retries = 3