Merge pull request #144 from commander-cli/skip-test-case

Skip test case

CHANGELOG.md

@@ -1,3 +1,7 @@
+# v2.3.0
+
+ - Add property `skip`, adds the ability to skip test cases 
+
 # v2.2.0
 
  - Move from `github.com/SimonBaeumer` to `github.com/commander-cli`

README.md

@@ -41,6 +41,7 @@ For more information take a look at the [quick start](#quick-start), the [exampl
       * [not-contains](#not-contains)
       * [xml](#xml)
     - [stderr](#stderr)
+    - [skip](#skip)
   + [Config](#user-content-config-config)
     - [dir](#dir)
     - [env](#env)
@@ -146,6 +147,11 @@ tests:
         stdout: hello # Default is to check if it contains the given characters
         exit-code: 0 # Assert exit-code
 
+    it should skip:
+        command: echo "I should be skipped"
+        stdout: I should be skipped
+        skip: true
+
     it should fail:
         command: invalid
         stderr:
@@ -162,7 +168,8 @@ tests:
             xml:
                 "//book//auhtor": Steven King # Make assertions on xml documents
         exit-code: 127
-
+        skip: false
+
     it has configs:
         command: echo hello
         stdout:
@@ -536,6 +543,20 @@ See [stdout](#stdout) for more information.
     line-count: 1
 ```
 
+#### skip
+
+`skip` is a `boolean` type, setting this field to `true` will skip the test case.
+
+ - name: `skip`
+ - type: `bool`
+ - default: `false`
+
+```yaml
+echo test:
+  stdout: test
+  skip: true
+```
+
 ### <a name="config-config"></a>Config
 
 You can add configs which will be applied globally to all tests or just for a specific test case, i.e.:

commander_linux.yaml

@@ -6,6 +6,9 @@ tests:
         - ✓ [ssh-host] it should test ssh host
         - ✓ [ssh-host] it should set env variable
         - ✓ [ssh-host-default] it should be executed on ssh-host-default
+        - "- [ssh-host-default] it should skip, was skipped" 
+        - "- [ssh-host] it should skip, was skipped" 
+        - "- [local] it should skip, was skipped" 
         - ✓ [ssh-host] it should test multiple hosts
         - ✓ [ssh-host-default] it should test multiple hosts
         - ✓ [local] it should test multiple hosts

commander_unix.yaml

@@ -15,7 +15,8 @@ tests:
     stdout:
       contains:
         - ✓ [local] it should exit with error code
-      line-count: 16
+        - "- [local] it should skip, was skipped"
+      line-count: 17
     exit-code: 0
 
   it should assert that commander will fail:

commander_windows.yaml

@@ -14,6 +14,7 @@ tests:
     command: commander.exe test ./integration/windows/commander_test.yaml
     stdout:
       contains:
+        - "- [local] it should skip, was skipped"
         - test
     exit-code: 0
 

examples/commander.yaml

@@ -20,3 +20,8 @@ tests:
     command: invalid
     stderr: "/bin/sh: 1: command invalid: not found"
     exit-code: 127
+
+  it should skip:
+    command: echo "I should be skipped"
+    stdout: I should be skipped
+    skip: true

integration/linux/nodes.yaml

@@ -40,6 +40,16 @@ tests:
   it should be executed on ssh-host-default:
     command: whoami
     stdout: root
+
+  it should skip:
+    command: whoami
+    stdout: root
+    config:
+      nodes:
+      - ssh-host-default
+      - ssh-host
+      - local
+    skip: true
 
   it should test multiple hosts:
     command: echo hello

integration/unix/commander_test.yaml

@@ -65,3 +65,8 @@ tests:
     command: echo $USER
     stdout: from_parent
     exit-code: 0
+
+  it should skip:
+    command: whoami
+    stdout: root
+    skip: true

integration/windows/commander_test.yaml

@@ -38,4 +38,9 @@ tests:
     command: echo lorem ipsum
     stdout:
       not-contains:
-      - not contains
+      - not contains
+
+  it should skip:
+    command: whoami
+    stdout: root
+    skip: true

pkg/output/cli.go

@@ -44,6 +44,7 @@ type TestResult struct {
 	FailedProperty string
 	Diff           string
 	Error          error
+	Skipped        bool
 }
 
 // GetEventHandler create a new runtime.EventHandler
@@ -53,6 +54,12 @@ func (w *OutputWriter) GetEventHandler() *runtime.EventHandler {
 		tr := convertTestResult(testResult)
 		w.printResult(tr)
 	}
+
+	handler.TestSkipped = func(testResult runtime.TestResult) {
+		tr := convertTestResult(testResult)
+		w.printSkip(tr)
+	}
+
 	return &handler
 }
 
@@ -74,7 +81,7 @@ func (w *OutputWriter) PrintSummary(result runtime.Result) bool {
 	return result.Failed == 0
 }
 
-// PrintResult prints the simple output form of a TestReault
+// printResult prints the simple output form of a TestReault
 func (w *OutputWriter) printResult(r TestResult) {
 	if !r.Success {
 		w.fprintf(w.au.Red(w.template.testResult(r)))
@@ -83,18 +90,27 @@ func (w *OutputWriter) printResult(r TestResult) {
 	w.fprintf(w.template.testResult(r))
 }
 
+func (w *OutputWriter) printSkip(r TestResult) {
+	w.fprintf(fmt.Sprintf("- [%s] %s, was skipped", r.Node, r.Title))
+}
+
 func (w *OutputWriter) printFailures(results []runtime.TestResult) {
 	w.fprintf("")
 	w.fprintf(w.au.Bold("Results"))
 	w.fprintf(w.au.Bold(""))
 
 	for _, tr := range results {
 		r := convertTestResult(tr)
+		if r.Skipped {
+			continue
+		}
+
 		if r.Error != nil {
 			w.fprintf(w.au.Bold(w.au.Red(w.template.errors(r))))
 			w.fprintf(r.Error.Error())
 			continue
 		}
+
 		if !r.Success {
 			w.fprintf(w.au.Bold(w.au.Red(w.template.failures(r))))
 			w.fprintf(r.Diff)
@@ -119,6 +135,7 @@ func convertTestResult(tr runtime.TestResult) TestResult {
 		FailedProperty: tr.FailedProperty,
 		Diff:           tr.ValidationResult.Diff,
 		Error:          tr.TestCase.Result.Error,
+		Skipped:        tr.Skipped,
 	}
 
 	return testResult

pkg/output/cli_template.go

@@ -40,7 +40,7 @@ var commanderTmpl = `
 
 // Summary
 {{define "summary" -}}
-	Count: {{len .TestResults}}, Failed: {{ .Failed }}
+	Count: {{len .TestResults}}, Failed: {{ .Failed }}, Skipped: {{ .Skipped }}
 {{- end -}}
 
 // Result

pkg/output/cli_test.go

@@ -30,14 +30,28 @@ func Test_EventHandlerTestFinished(t *testing.T) {
 
 	for _, tr := range testResults {
 		eh.TestFinished(tr)
-
 	}
 
 	output := buf.String()
-
 	assert.Contains(t, output, "✗ [192.168.0.1] Failed test")
 	assert.Contains(t, output, "✓ [docker-host] Successful test")
+}
+
+func Test_EventHandlerTestSkipped(t *testing.T) {
+	var buf bytes.Buffer
+	writer := NewCliOutput(true)
+	writer.out = &buf
+	eh := writer.GetEventHandler()
+
+	testResults := createFakeTestResults()
 
+	for _, tr := range testResults {
+		if tr.Skipped {
+			eh.TestSkipped(tr)
+		}
+	}
+	output := buf.String()
+	assert.Contains(t, output, "- [192.168.0.1] Skipped test, was skipped")
 }
 
 func Test_PrintSummary(t *testing.T) {
@@ -98,5 +112,15 @@ func createFakeTestResults() []runtime.TestResult {
 		Tries: 2,
 	}
 
-	return []runtime.TestResult{tr, tr2, tr3}
+	tr4 := runtime.TestResult{
+		TestCase: runtime.TestCase{
+			Title:   "Skipped test",
+			Command: runtime.CommandUnderTest{},
+		},
+		FailedProperty: "Stdout",
+		Node:           "192.168.0.1",
+		Skipped:        true,
+	}
+
+	return []runtime.TestResult{tr, tr2, tr3, tr4}
 }

pkg/runtime/runner.go

@@ -12,8 +12,8 @@ type Runner struct {
 	Nodes []Node
 }
 
-// Execute the runner
-func (r *Runner) Execute(tests []TestCase) <-chan TestResult {
+// Run the runner
+func (r *Runner) Run(tests []TestCase) <-chan TestResult {
 	in := make(chan TestCase)
 	out := make(chan TestResult)
 
@@ -40,6 +40,11 @@ func (r *Runner) Execute(tests []TestCase) <-chan TestResult {
 				result := TestResult{}
 				for i := 1; i <= t.Command.GetRetries(); i++ {
 
+					if t.Skip {
+						result = TestResult{TestCase: t, Skipped: true, Node: n}
+						break
+					}
+
 					e := r.getExecutor(n)
 					result = e.Execute(t)
 					result.Node = n
@@ -53,7 +58,6 @@ func (r *Runner) Execute(tests []TestCase) <-chan TestResult {
 				}
 				out <- result
 			}
-
 		}
 	}(in)
 

pkg/runtime/runner_test.go

@@ -12,7 +12,7 @@ func Test_RunnerExexcute(t *testing.T) {
 		Nodes: getExampleNodes(),
 	}
 
-	got := r.Execute(s)
+	got := r.Run(s)
 
 	assert.IsType(t, make(<-chan TestResult), got)
 

pkg/runtime/runtime.go

@@ -1,6 +1,7 @@
 package runtime
 
 import (
+	"log"
 	"time"
 )
 
@@ -12,16 +13,6 @@ const (
 	LineCount = "LineCount"
 )
 
-// Result status codes
-const (
-	//Success status
-	Success ResultStatus = iota
-	// Failed status
-	Failed
-	// Skipped status
-	Skipped
-)
-
 type Filters []string
 
 // NewRuntime creates a new runtime and inits default nodes
@@ -52,6 +43,7 @@ type Runtime struct {
 // EventHandler is a configurable event system that handles events such as test completion
 type EventHandler struct {
 	TestFinished func(TestResult)
+	TestSkipped  func(TestResult)
 }
 
 // TestCase represents a test case which will be executed by the runtime
@@ -62,6 +54,7 @@ type TestCase struct {
 	Result   CommandResult
 	Nodes    []string
 	FileName string
+	Skip     bool
 }
 
 //GlobalTestConfig represents the configuration for a test
@@ -125,29 +118,41 @@ type TestResult struct {
 	FailedProperty   string
 	Tries            int
 	Node             string
-	FileName         string
+	Skipped          bool
 }
 
 // Result respresents the aggregation of all TestResults/summary of a runtime
 type Result struct {
 	TestResults []TestResult
 	Duration    time.Duration
 	Failed      int
+	Skipped     int
 }
 
 // Start starts the given test suite and executes all tests
 func (r *Runtime) Start(tests []TestCase) Result {
 	result := Result{}
-	testCh := r.Runner.Execute(tests)
+	testCh := r.Runner.Run(tests)
 	start := time.Now()
 	for tr := range testCh {
+		if tr.Skipped {
+			result.Skipped++
 
-		r.EventHandler.TestFinished(tr)
+			log.Println("title: '"+tr.TestCase.Title+"'", " was skipped")
+			log.Println("title: '"+tr.TestCase.Title+"'", " Command: ", tr.TestCase.Command.Cmd)
+			log.Println("title: '"+tr.TestCase.Title+"'", " Directory: ", tr.TestCase.Command.Dir)
+			log.Println("title: '"+tr.TestCase.Title+"'", " Env: ", tr.TestCase.Command.Env)
+
+			r.EventHandler.TestSkipped(tr)
+			result.TestResults = append(result.TestResults, tr)
+			continue
+		}
 
 		if !tr.ValidationResult.Success {
 			result.Failed++
 		}
 
+		r.EventHandler.TestFinished(tr)
 		result.TestResults = append(result.TestResults, tr)
 	}
 	result.Duration = time.Since(start)