From 4ce699e3b69efe87afae448484aceac985fe9965 Mon Sep 17 00:00:00 2001 From: Simon Date: Wed, 22 Jul 2020 13:43:54 +0200 Subject: [PATCH 1/5] Preserve execution order alphabetically by test title --- commander_unix.yaml | 13 ++++++++++++- integration/unix/alphabetically_order.yaml | 13 +++++++++++++ pkg/runtime/runtime.go | 7 +++++++ 3 files changed, 32 insertions(+), 1 deletion(-) create mode 100644 integration/unix/alphabetically_order.yaml diff --git a/commander_unix.yaml b/commander_unix.yaml index facbeae1..248dfed7 100644 --- a/commander_unix.yaml +++ b/commander_unix.yaml @@ -111,4 +111,15 @@ tests: - ✓ [local] should be ignored not-contains: - executed at the beginning is ignored - exit-code: 0 \ No newline at end of file + exit-code: 0 + + it should be executed in alphabetical order: + command: ./commander test integration/unix/alphabetically_order.yaml + stdout: + contains: + - |- + ✓ [local] --- + ✓ [local] 123 + ✓ [local] a + ✓ [local] b + exit-code: 0 diff --git a/integration/unix/alphabetically_order.yaml b/integration/unix/alphabetically_order.yaml new file mode 100644 index 00000000..6a8b1a6d --- /dev/null +++ b/integration/unix/alphabetically_order.yaml @@ -0,0 +1,13 @@ +tests: + a: + command: echo test + exit-code: 0 + b: + command: echo test + exit-code: 0 + 123: + command: echo test + exit-code: 0 + "---": + command: echo test + exit-code: 0 \ No newline at end of file diff --git a/pkg/runtime/runtime.go b/pkg/runtime/runtime.go index 0f0cce10..842ac888 100644 --- a/pkg/runtime/runtime.go +++ b/pkg/runtime/runtime.go @@ -2,6 +2,7 @@ package runtime import ( "log" + "sort" "time" ) @@ -131,10 +132,16 @@ type Result struct { // Start starts the given test suite and executes all tests func (r *Runtime) Start(tests []TestCase) Result { + // Sort tests alphabetically to preserve a reproducible execution order + sort.SliceStable(tests, func(i, j int) bool { + return tests[i].Title < tests[j].Title + }) + result := Result{} testCh := r.Runner.Run(tests) start := time.Now() for tr := range testCh { + r.EventHandler.TestFinished(tr) if tr.Skipped { result.Skipped++ From b52ef39e387ad3479805064c8a6a07d9aedd1fec Mon Sep 17 00:00:00 2001 From: Simon Date: Sat, 1 Aug 2020 14:02:34 +0200 Subject: [PATCH 2/5] Add CHANGELOG and tests --- CHANGELOG.md | 1 + README.md | 2 +- pkg/runtime/runtime_test.go | 21 +++++++++++++++++++++ 3 files changed, 23 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ca6e31a2..43246ef9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,6 @@ # v2.3.0 + - Preserve test execution order alphabetical order - Add property `skip`, adds the ability to skip test cases # v2.2.0 diff --git a/README.md b/README.md index c5400b87..20c4b1de 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ [![Build Status](https://travis-ci.org/commander-cli/commander.svg?branch=master)](https://travis-ci.org/commander-cli/commander) [![GoDoc](https://godoc.org/github.com/commander-cli/commander?status.svg)](https://godoc.org/github.com/commander-cli/commander) -[![Go Report Card](https://goreportcard.com/badge/github.com/commander-cli/commander)](https://goreportcard.com/report/github.com/SimonBaeumer/commander) +[![Go Report Card](https://goreportcard.com/badge/github.com/commander-cli/commander)](https://goreportcard.com/report/github.com/commander-cli/commander) [![Maintainability](https://api.codeclimate.com/v1/badges/cc848165784e0f809a51/maintainability)](https://codeclimate.com/github/commander-cli/commander/maintainability) [![Test Coverage](https://api.codeclimate.com/v1/badges/cc848165784e0f809a51/test_coverage)](https://codeclimate.com/github/commander-cli/commander/test_coverage) [![Github All Releases](https://img.shields.io/github/downloads/commander-cli/commander/total.svg)](https://github.com/commander-cli/commander/releases) diff --git a/pkg/runtime/runtime_test.go b/pkg/runtime/runtime_test.go index ef25bb67..b2a0a940 100644 --- a/pkg/runtime/runtime_test.go +++ b/pkg/runtime/runtime_test.go @@ -47,6 +47,27 @@ func TestRuntime_WithRetries(t *testing.T) { assert.Equal(t, 1, counter) } +func Test_AlphabeticalOrder(t *testing.T) { + tests := []TestCase{ + {Title: "bbb", Command: CommandUnderTest{Cmd: "exit 0;"}}, + {Title: "aaa"}, + {Title: "111"}, + {Title: "_"}, + } + + got := []string{} + runtime := NewRuntime(&EventHandler{TestFinished: func(r TestResult) { + got = append(got, r.TestCase.Title) + }}) + + runtime.Start(tests) + + assert.Equal(t, "111", got[0]) + assert.Equal(t, "_", got[1]) + assert.Equal(t, "aaa", got[2]) + assert.Equal(t, "bbb", got[3]) +} + func Test_RuntimeWithRetriesAndInterval(t *testing.T) { s := getExampleTestCases() s[0].Command.Retries = 3 From a553dbed7760586d9755f122558c3875797d814d Mon Sep 17 00:00:00 2001 From: Simon Date: Thu, 6 Aug 2020 10:52:27 +0200 Subject: [PATCH 3/5] Add usage test command usage documentation --- cmd/commander/commander.go | 26 +++++++++++++++++--------- 1 file changed, 17 insertions(+), 9 deletions(-) diff --git a/cmd/commander/commander.go b/cmd/commander/commander.go index df668241..785a029a 100644 --- a/cmd/commander/commander.go +++ b/cmd/commander/commander.go @@ -46,7 +46,22 @@ func createCliApp() *cli.App { func createTestCommand() cli.Command { return cli.Command{ Name: "test", - Usage: "Execute the test suite, by default it will use the commander.yaml from your current directory", + Usage: `Execute cli app tests + +By default it will use the commander.yaml from your current directory. +Tests are always executed in alphabetical order. + +Examples: + +Filtering tests: +test commander.yaml --filter="my test" + +Multiple filters: +test commander.yaml --filter=filter1 --filter=filter2 + +Regex filters: +test commander.yaml --filter="^filter1$" +`, ArgsUsage: "[file] [--filter]", Flags: []cli.Flag{ cli.BoolFlag{ @@ -65,14 +80,7 @@ func createTestCommand() cli.Command { }, cli.StringSliceFlag{ Name: "filter", - Usage: `Filter tests by a given regex pattern. Tests are filtered by its title. - -Example: -test commander.yaml --filter="my test" - -Apply multiple filters: -test commander.yaml --filter=filter1 --filter=filter2 -`, + Usage: `Filter tests by a given regex pattern. Tests are filtered by its title.`, }, }, Action: func(c *cli.Context) error { From 2699727f65b17bd95eabcc21bb67a67789f12141 Mon Sep 17 00:00:00 2001 From: Simon Date: Thu, 6 Aug 2020 11:05:44 +0200 Subject: [PATCH 4/5] Add ordering example --- examples/ordering.yaml | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) create mode 100644 examples/ordering.yaml diff --git a/examples/ordering.yaml b/examples/ordering.yaml new file mode 100644 index 00000000..02919ed8 --- /dev/null +++ b/examples/ordering.yaml @@ -0,0 +1,18 @@ +# Tests are always executed in alphabetical order + +tests: + 001 - test: + command: exit 0 + exit-code: 0 + + 002 - test: + command: exit 0 + exit-code: 0 + + 003 - test: + command: exit 0 + exit-code: 0 + + 004 - test: + command: exit 0 + exit-code: 0 \ No newline at end of file From ba142f2c3ec72ca26fc2a709a68ef1e01cbcfc97 Mon Sep 17 00:00:00 2001 From: Simon Date: Fri, 7 Aug 2020 13:31:01 +0200 Subject: [PATCH 5/5] Remove unused TestFinsihed event --- pkg/runtime/runtime.go | 1 - 1 file changed, 1 deletion(-) diff --git a/pkg/runtime/runtime.go b/pkg/runtime/runtime.go index 842ac888..405088df 100644 --- a/pkg/runtime/runtime.go +++ b/pkg/runtime/runtime.go @@ -141,7 +141,6 @@ func (r *Runtime) Start(tests []TestCase) Result { testCh := r.Runner.Run(tests) start := time.Now() for tr := range testCh { - r.EventHandler.TestFinished(tr) if tr.Skipped { result.Skipped++