Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Preserve execution order alphabetically by test title #142

Merged
merged 5 commits into from
Aug 7, 2020
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
# v2.3.0

- Preserve test execution order alphabetical order
- Add property `skip`, adds the ability to skip test cases

# v2.2.0
Expand Down
2 changes: 1 addition & 1 deletion README.md
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
[![Build Status](https://travis-ci.org/commander-cli/commander.svg?branch=master)](https://travis-ci.org/commander-cli/commander)
[![GoDoc](https://godoc.org/github.com/commander-cli/commander?status.svg)](https://godoc.org/github.com/commander-cli/commander)
[![Go Report Card](https://goreportcard.com/badge/github.com/commander-cli/commander)](https://goreportcard.com/report/github.com/SimonBaeumer/commander)
[![Go Report Card](https://goreportcard.com/badge/github.com/commander-cli/commander)](https://goreportcard.com/report/github.com/commander-cli/commander)
[![Maintainability](https://api.codeclimate.com/v1/badges/cc848165784e0f809a51/maintainability)](https://codeclimate.com/github/commander-cli/commander/maintainability)
[![Test Coverage](https://api.codeclimate.com/v1/badges/cc848165784e0f809a51/test_coverage)](https://codeclimate.com/github/commander-cli/commander/test_coverage)
[![Github All Releases](https://img.shields.io/github/downloads/commander-cli/commander/total.svg)](https://github.com/commander-cli/commander/releases)
Expand Down
26 changes: 17 additions & 9 deletions cmd/commander/commander.go
Original file line number Diff line number Diff line change
Expand Up @@ -46,7 +46,22 @@ func createCliApp() *cli.App {
func createTestCommand() cli.Command {
return cli.Command{
Name: "test",
Usage: "Execute the test suite, by default it will use the commander.yaml from your current directory",
Usage: `Execute cli app tests

By default it will use the commander.yaml from your current directory.
Tests are always executed in alphabetical order.

Examples:

Filtering tests:
test commander.yaml --filter="my test"

Multiple filters:
test commander.yaml --filter=filter1 --filter=filter2

Regex filters:
test commander.yaml --filter="^filter1$"
`,
ArgsUsage: "[file] [--filter]",
Flags: []cli.Flag{
cli.BoolFlag{
Expand All @@ -65,14 +80,7 @@ func createTestCommand() cli.Command {
},
cli.StringSliceFlag{
Name: "filter",
Usage: `Filter tests by a given regex pattern. Tests are filtered by its title.

Example:
test commander.yaml --filter="my test"

Apply multiple filters:
test commander.yaml --filter=filter1 --filter=filter2
`,
Usage: `Filter tests by a given regex pattern. Tests are filtered by its title.`,
},
},
Action: func(c *cli.Context) error {
Expand Down
13 changes: 12 additions & 1 deletion commander_unix.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -111,4 +111,15 @@ tests:
- ✓ [local] should be ignored
not-contains:
- executed at the beginning is ignored
exit-code: 0
exit-code: 0

it should be executed in alphabetical order:
command: ./commander test integration/unix/alphabetically_order.yaml
stdout:
contains:
- |-
✓ [local] ---
✓ [local] 123
✓ [local] a
✓ [local] b
exit-code: 0
18 changes: 18 additions & 0 deletions examples/ordering.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,18 @@
# Tests are always executed in alphabetical order

tests:
001 - test:
command: exit 0
exit-code: 0

002 - test:
command: exit 0
exit-code: 0

003 - test:
command: exit 0
exit-code: 0

004 - test:
command: exit 0
exit-code: 0
13 changes: 13 additions & 0 deletions integration/unix/alphabetically_order.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,13 @@
tests:
a:
command: echo test
exit-code: 0
b:
command: echo test
exit-code: 0
123:
command: echo test
exit-code: 0
"---":
command: echo test
exit-code: 0
6 changes: 6 additions & 0 deletions pkg/runtime/runtime.go
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@ package runtime

import (
"log"
"sort"
"time"
)

Expand Down Expand Up @@ -131,6 +132,11 @@ type Result struct {

// Start starts the given test suite and executes all tests
func (r *Runtime) Start(tests []TestCase) Result {
// Sort tests alphabetically to preserve a reproducible execution order
sort.SliceStable(tests, func(i, j int) bool {
return tests[i].Title < tests[j].Title
})

result := Result{}
testCh := r.Runner.Run(tests)
start := time.Now()
Expand Down
21 changes: 21 additions & 0 deletions pkg/runtime/runtime_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -47,6 +47,27 @@ func TestRuntime_WithRetries(t *testing.T) {
assert.Equal(t, 1, counter)
}

func Test_AlphabeticalOrder(t *testing.T) {
tests := []TestCase{
{Title: "bbb", Command: CommandUnderTest{Cmd: "exit 0;"}},
{Title: "aaa"},
{Title: "111"},
{Title: "_"},
}

got := []string{}
runtime := NewRuntime(&EventHandler{TestFinished: func(r TestResult) {
got = append(got, r.TestCase.Title)
}})

runtime.Start(tests)

assert.Equal(t, "111", got[0])
assert.Equal(t, "_", got[1])
assert.Equal(t, "aaa", got[2])
assert.Equal(t, "bbb", got[3])
}

func Test_RuntimeWithRetriesAndInterval(t *testing.T) {
s := getExampleTestCases()
s[0].Command.Retries = 3
Expand Down