diff --git a/CHANGELOG.md b/CHANGELOG.md index 934577ef..ca6e31a2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,7 @@ +# v2.3.0 + + - Add property `skip`, adds the ability to skip test cases + # v2.2.0 - Move from `github.com/SimonBaeumer` to `github.com/commander-cli` diff --git a/README.md b/README.md index 5bb6c57f..c5400b87 100644 --- a/README.md +++ b/README.md @@ -41,6 +41,7 @@ For more information take a look at the [quick start](#quick-start), the [exampl * [not-contains](#not-contains) * [xml](#xml) - [stderr](#stderr) + - [skip](#skip) + [Config](#user-content-config-config) - [dir](#dir) - [env](#env) @@ -146,6 +147,11 @@ tests: stdout: hello # Default is to check if it contains the given characters exit-code: 0 # Assert exit-code + it should skip: + command: echo "I should be skipped" + stdout: I should be skipped + skip: true + it should fail: command: invalid stderr: @@ -162,7 +168,8 @@ tests: xml: "//book//auhtor": Steven King # Make assertions on xml documents exit-code: 127 - + skip: false + it has configs: command: echo hello stdout: @@ -536,6 +543,20 @@ See [stdout](#stdout) for more information. line-count: 1 ``` +#### skip + +`skip` is a `boolean` type, setting this field to `true` will skip the test case. + + - name: `skip` + - type: `bool` + - default: `false` + +```yaml +echo test: + stdout: test + skip: true +``` + ### Config You can add configs which will be applied globally to all tests or just for a specific test case, i.e.: diff --git a/commander_linux.yaml b/commander_linux.yaml index 6151ba22..49990ed6 100644 --- a/commander_linux.yaml +++ b/commander_linux.yaml @@ -6,6 +6,9 @@ tests: - ✓ [ssh-host] it should test ssh host - ✓ [ssh-host] it should set env variable - ✓ [ssh-host-default] it should be executed on ssh-host-default + - "- [ssh-host-default] it should skip, was skipped" + - "- [ssh-host] it should skip, was skipped" + - "- [local] it should skip, was skipped" - ✓ [ssh-host] it should test multiple hosts - ✓ [ssh-host-default] it should test multiple hosts - ✓ [local] it should test multiple hosts diff --git a/commander_unix.yaml b/commander_unix.yaml index 593e57b3..facbeae1 100644 --- a/commander_unix.yaml +++ b/commander_unix.yaml @@ -15,7 +15,8 @@ tests: stdout: contains: - ✓ [local] it should exit with error code - line-count: 16 + - "- [local] it should skip, was skipped" + line-count: 17 exit-code: 0 it should assert that commander will fail: diff --git a/commander_windows.yaml b/commander_windows.yaml index 7d274b2e..8a0f7571 100644 --- a/commander_windows.yaml +++ b/commander_windows.yaml @@ -14,6 +14,7 @@ tests: command: commander.exe test ./integration/windows/commander_test.yaml stdout: contains: + - "- [local] it should skip, was skipped" - test exit-code: 0 diff --git a/examples/commander.yaml b/examples/commander.yaml index e687ed0b..859c0aa5 100644 --- a/examples/commander.yaml +++ b/examples/commander.yaml @@ -20,3 +20,8 @@ tests: command: invalid stderr: "/bin/sh: 1: command invalid: not found" exit-code: 127 + + it should skip: + command: echo "I should be skipped" + stdout: I should be skipped + skip: true \ No newline at end of file diff --git a/integration/linux/nodes.yaml b/integration/linux/nodes.yaml index eac02d3e..e0373cb2 100644 --- a/integration/linux/nodes.yaml +++ b/integration/linux/nodes.yaml @@ -40,6 +40,16 @@ tests: it should be executed on ssh-host-default: command: whoami stdout: root + + it should skip: + command: whoami + stdout: root + config: + nodes: + - ssh-host-default + - ssh-host + - local + skip: true it should test multiple hosts: command: echo hello diff --git a/integration/unix/commander_test.yaml b/integration/unix/commander_test.yaml index e47931da..3f91b489 100644 --- a/integration/unix/commander_test.yaml +++ b/integration/unix/commander_test.yaml @@ -65,3 +65,8 @@ tests: command: echo $USER stdout: from_parent exit-code: 0 + + it should skip: + command: whoami + stdout: root + skip: true diff --git a/integration/windows/commander_test.yaml b/integration/windows/commander_test.yaml index c6617db5..b5f26302 100644 --- a/integration/windows/commander_test.yaml +++ b/integration/windows/commander_test.yaml @@ -38,4 +38,9 @@ tests: command: echo lorem ipsum stdout: not-contains: - - not contains \ No newline at end of file + - not contains + + it should skip: + command: whoami + stdout: root + skip: true \ No newline at end of file diff --git a/pkg/output/cli.go b/pkg/output/cli.go index df1b3dc4..36add127 100644 --- a/pkg/output/cli.go +++ b/pkg/output/cli.go @@ -44,6 +44,7 @@ type TestResult struct { FailedProperty string Diff string Error error + Skipped bool } // GetEventHandler create a new runtime.EventHandler @@ -53,6 +54,12 @@ func (w *OutputWriter) GetEventHandler() *runtime.EventHandler { tr := convertTestResult(testResult) w.printResult(tr) } + + handler.TestSkipped = func(testResult runtime.TestResult) { + tr := convertTestResult(testResult) + w.printSkip(tr) + } + return &handler } @@ -74,7 +81,7 @@ func (w *OutputWriter) PrintSummary(result runtime.Result) bool { return result.Failed == 0 } -// PrintResult prints the simple output form of a TestReault +// printResult prints the simple output form of a TestReault func (w *OutputWriter) printResult(r TestResult) { if !r.Success { w.fprintf(w.au.Red(w.template.testResult(r))) @@ -83,6 +90,10 @@ func (w *OutputWriter) printResult(r TestResult) { w.fprintf(w.template.testResult(r)) } +func (w *OutputWriter) printSkip(r TestResult) { + w.fprintf(fmt.Sprintf("- [%s] %s, was skipped", r.Node, r.Title)) +} + func (w *OutputWriter) printFailures(results []runtime.TestResult) { w.fprintf("") w.fprintf(w.au.Bold("Results")) @@ -90,11 +101,16 @@ func (w *OutputWriter) printFailures(results []runtime.TestResult) { for _, tr := range results { r := convertTestResult(tr) + if r.Skipped { + continue + } + if r.Error != nil { w.fprintf(w.au.Bold(w.au.Red(w.template.errors(r)))) w.fprintf(r.Error.Error()) continue } + if !r.Success { w.fprintf(w.au.Bold(w.au.Red(w.template.failures(r)))) w.fprintf(r.Diff) @@ -119,6 +135,7 @@ func convertTestResult(tr runtime.TestResult) TestResult { FailedProperty: tr.FailedProperty, Diff: tr.ValidationResult.Diff, Error: tr.TestCase.Result.Error, + Skipped: tr.Skipped, } return testResult diff --git a/pkg/output/cli_template.go b/pkg/output/cli_template.go index 4127fdc0..ffdf66bf 100644 --- a/pkg/output/cli_template.go +++ b/pkg/output/cli_template.go @@ -40,7 +40,7 @@ var commanderTmpl = ` // Summary {{define "summary" -}} - Count: {{len .TestResults}}, Failed: {{ .Failed }} + Count: {{len .TestResults}}, Failed: {{ .Failed }}, Skipped: {{ .Skipped }} {{- end -}} // Result diff --git a/pkg/output/cli_test.go b/pkg/output/cli_test.go index 74f84ef4..239b2a97 100644 --- a/pkg/output/cli_test.go +++ b/pkg/output/cli_test.go @@ -30,14 +30,28 @@ func Test_EventHandlerTestFinished(t *testing.T) { for _, tr := range testResults { eh.TestFinished(tr) - } output := buf.String() - assert.Contains(t, output, "✗ [192.168.0.1] Failed test") assert.Contains(t, output, "✓ [docker-host] Successful test") +} + +func Test_EventHandlerTestSkipped(t *testing.T) { + var buf bytes.Buffer + writer := NewCliOutput(true) + writer.out = &buf + eh := writer.GetEventHandler() + + testResults := createFakeTestResults() + for _, tr := range testResults { + if tr.Skipped { + eh.TestSkipped(tr) + } + } + output := buf.String() + assert.Contains(t, output, "- [192.168.0.1] Skipped test, was skipped") } func Test_PrintSummary(t *testing.T) { @@ -98,5 +112,15 @@ func createFakeTestResults() []runtime.TestResult { Tries: 2, } - return []runtime.TestResult{tr, tr2, tr3} + tr4 := runtime.TestResult{ + TestCase: runtime.TestCase{ + Title: "Skipped test", + Command: runtime.CommandUnderTest{}, + }, + FailedProperty: "Stdout", + Node: "192.168.0.1", + Skipped: true, + } + + return []runtime.TestResult{tr, tr2, tr3, tr4} } diff --git a/pkg/runtime/runner.go b/pkg/runtime/runner.go index 37a91cde..c1edd464 100644 --- a/pkg/runtime/runner.go +++ b/pkg/runtime/runner.go @@ -12,8 +12,8 @@ type Runner struct { Nodes []Node } -// Execute the runner -func (r *Runner) Execute(tests []TestCase) <-chan TestResult { +// Run the runner +func (r *Runner) Run(tests []TestCase) <-chan TestResult { in := make(chan TestCase) out := make(chan TestResult) @@ -40,6 +40,11 @@ func (r *Runner) Execute(tests []TestCase) <-chan TestResult { result := TestResult{} for i := 1; i <= t.Command.GetRetries(); i++ { + if t.Skip { + result = TestResult{TestCase: t, Skipped: true, Node: n} + break + } + e := r.getExecutor(n) result = e.Execute(t) result.Node = n @@ -53,7 +58,6 @@ func (r *Runner) Execute(tests []TestCase) <-chan TestResult { } out <- result } - } }(in) diff --git a/pkg/runtime/runner_test.go b/pkg/runtime/runner_test.go index 9a095b72..b7666960 100644 --- a/pkg/runtime/runner_test.go +++ b/pkg/runtime/runner_test.go @@ -12,7 +12,7 @@ func Test_RunnerExexcute(t *testing.T) { Nodes: getExampleNodes(), } - got := r.Execute(s) + got := r.Run(s) assert.IsType(t, make(<-chan TestResult), got) diff --git a/pkg/runtime/runtime.go b/pkg/runtime/runtime.go index e2f4327d..0f0cce10 100644 --- a/pkg/runtime/runtime.go +++ b/pkg/runtime/runtime.go @@ -1,6 +1,7 @@ package runtime import ( + "log" "time" ) @@ -12,16 +13,6 @@ const ( LineCount = "LineCount" ) -// Result status codes -const ( - //Success status - Success ResultStatus = iota - // Failed status - Failed - // Skipped status - Skipped -) - type Filters []string // NewRuntime creates a new runtime and inits default nodes @@ -52,6 +43,7 @@ type Runtime struct { // EventHandler is a configurable event system that handles events such as test completion type EventHandler struct { TestFinished func(TestResult) + TestSkipped func(TestResult) } // TestCase represents a test case which will be executed by the runtime @@ -62,6 +54,7 @@ type TestCase struct { Result CommandResult Nodes []string FileName string + Skip bool } //GlobalTestConfig represents the configuration for a test @@ -125,7 +118,7 @@ type TestResult struct { FailedProperty string Tries int Node string - FileName string + Skipped bool } // Result respresents the aggregation of all TestResults/summary of a runtime @@ -133,21 +126,33 @@ type Result struct { TestResults []TestResult Duration time.Duration Failed int + Skipped int } // Start starts the given test suite and executes all tests func (r *Runtime) Start(tests []TestCase) Result { result := Result{} - testCh := r.Runner.Execute(tests) + testCh := r.Runner.Run(tests) start := time.Now() for tr := range testCh { + if tr.Skipped { + result.Skipped++ - r.EventHandler.TestFinished(tr) + log.Println("title: '"+tr.TestCase.Title+"'", " was skipped") + log.Println("title: '"+tr.TestCase.Title+"'", " Command: ", tr.TestCase.Command.Cmd) + log.Println("title: '"+tr.TestCase.Title+"'", " Directory: ", tr.TestCase.Command.Dir) + log.Println("title: '"+tr.TestCase.Title+"'", " Env: ", tr.TestCase.Command.Env) + + r.EventHandler.TestSkipped(tr) + result.TestResults = append(result.TestResults, tr) + continue + } if !tr.ValidationResult.Success { result.Failed++ } + r.EventHandler.TestFinished(tr) result.TestResults = append(result.TestResults, tr) } result.Duration = time.Since(start) diff --git a/pkg/runtime/runtime_test.go b/pkg/runtime/runtime_test.go index 1af1b7e5..ef25bb67 100644 --- a/pkg/runtime/runtime_test.go +++ b/pkg/runtime/runtime_test.go @@ -69,11 +69,24 @@ func Test_RuntimeWithRetriesAndInterval(t *testing.T) { assert.True(t, duration.Seconds() > 0.15, "Retry interval did not work") } +func Test_RuntimeWithSkip(t *testing.T) { + s := getExampleTestCases() + s[0].Skip = true + + r := getRuntime() + got := r.Start(s) + + assert.Equal(t, 1, got.Skipped) +} + func getRuntime() Runtime { eh := EventHandler{ TestFinished: func(tr TestResult) { fmt.Println("I do nothing") }, + TestSkipped: func(tr TestResult) { + fmt.Printf("%s was skipped", tr.TestCase.Title) + }, } runtime := NewRuntime(&eh, Node{Name: "test"}, Node{Name: "test2"}) diff --git a/pkg/suite/yaml_suite.go b/pkg/suite/yaml_suite.go index 848d2ca4..720f4bee 100644 --- a/pkg/suite/yaml_suite.go +++ b/pkg/suite/yaml_suite.go @@ -47,6 +47,7 @@ type YAMLTest struct { Stdout interface{} `yaml:"stdout,omitempty"` Stderr interface{} `yaml:"stderr,omitempty"` Config YAMLTestConfigConf `yaml:"config,omitempty"` + Skip bool `yaml:"skip,omitempty"` } // ParseYAML parses the Suite from a yaml byte slice @@ -118,6 +119,7 @@ func convertYAMLSuiteConfToTestCases(conf YAMLSuiteConf, fileName string) []runt }, Nodes: t.Config.Nodes, FileName: fileName, + Skip: t.Skip, }) } @@ -152,6 +154,7 @@ func (y *YAMLSuiteConf) UnmarshalYAML(unmarshal func(interface{}) error) error { Stdout: y.convertToExpectedOut(v.Stdout), Stderr: y.convertToExpectedOut(v.Stderr), Config: y.mergeConfigs(v.Config, params.Config), + Skip: v.Skip, } // Set key as command, if command property was empty diff --git a/pkg/suite/yaml_suite_test.go b/pkg/suite/yaml_suite_test.go index 4dd2f7e1..10bf0faf 100644 --- a/pkg/suite/yaml_suite_test.go +++ b/pkg/suite/yaml_suite_test.go @@ -74,6 +74,28 @@ tests: assert.Equal(t, "line4", tests[0].Expected.Stdout.Lines[3]) } +func TestYAMLConfig_UnmarshalYAML_ShouldDisable(t *testing.T) { + yaml := []byte(` +tests: + it should print hello: + command: echo hello + exit-code: 0 + stdout: hello + stderr: anything + skip: true +`) + got := ParseYAML(yaml, "") + tests := got.GetTests() + + assert.Len(t, tests, 1) + assert.Equal(t, "echo hello", tests[0].Command.Cmd) + assert.Equal(t, 0, tests[0].Expected.ExitCode) + assert.Equal(t, "it should print hello", tests[0].Title) + assert.Equal(t, "hello", tests[0].Expected.Stdout.Contains[0]) + assert.Equal(t, "anything", tests[0].Expected.Stderr.Contains[0]) + assert.True(t, tests[0].Skip) +} + func TestYAMLConfig_UnmarshalYAML_ShouldConvertStdoutToExpectedOut(t *testing.T) { yaml := []byte(` tests: