diff --git a/CHANGELOG.md b/CHANGELOG.md index 41ba16dc..c394eda1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,7 @@ - Add ability to test suite from a url - Add ability to test suite from stdin + - Add `file` assertion to `stdout` and `stderr` # v2.3.0 diff --git a/README.md b/README.md index 906fbad7..fdbbd2ce 100644 --- a/README.md +++ b/README.md @@ -40,6 +40,7 @@ For more information take a look at the [quick start](#quick-start), the [exampl * [line-count](#line-count) * [not-contains](#not-contains) * [xml](#xml) + * [file](#file) - [stderr](#stderr) - [skip](#skip) + [Config](#user-content-config-config) @@ -167,6 +168,7 @@ tests: object.attr: hello # Make assertions on json objects xml: "//book//auhtor": Steven King # Make assertions on xml documents + file: correct-output.txt exit-code: 127 skip: false @@ -529,6 +531,21 @@ cat some.xml: ``` +##### file + +`file` is a file path, relative to the working directory that will have +its entire contents matched against the command output. Other than +reading from a file this works the same as [exactly](#exactly). + +The example below will always pass. + +```yaml +output should match file: + command: cat output.txt + stdout: + file: output.txt +``` + #### stderr See [stdout](#stdout) for more information. diff --git a/commander_unix.yaml b/commander_unix.yaml index 73458fb8..3ca0362c 100644 --- a/commander_unix.yaml +++ b/commander_unix.yaml @@ -16,17 +16,18 @@ tests: contains: - ✓ [local] it should exit with error code - "- [local] it should skip, was skipped" - line-count: 17 + line-count: 19 exit-code: 0 it should assert that commander will fail: command: ./commander test ./integration/unix/failing_suite.yaml stdout: contains: + - ✗ [local] 'file matcher should fail when file and output are different', on property 'Stdout' - ✗ [local] 'it will fail', on property 'ExitCode' - ✗ [local] 'test timeout' could not be executed with error message - Command timed out after 10ms - - "Count: 2, Failed: 2" + - "Count: 3, Failed: 3" exit-code: 1 it should validate a big output: diff --git a/examples/_fixtures/output.txt b/examples/_fixtures/output.txt new file mode 100644 index 00000000..4c00b399 --- /dev/null +++ b/examples/_fixtures/output.txt @@ -0,0 +1,2 @@ +line one +line two \ No newline at end of file diff --git a/examples/commander.yaml b/examples/commander.yaml index 859c0aa5..7eafe51f 100644 --- a/examples/commander.yaml +++ b/examples/commander.yaml @@ -24,4 +24,9 @@ tests: it should skip: command: echo "I should be skipped" stdout: I should be skipped - skip: true \ No newline at end of file + skip: true + + it should match file output: + command: printf "line one\nline two" + stdout: + file: ../../examples/_fixtures/output.txt diff --git a/integration/unix/_fixtures/file_output_0.txt b/integration/unix/_fixtures/file_output_0.txt new file mode 100644 index 00000000..9c40573a --- /dev/null +++ b/integration/unix/_fixtures/file_output_0.txt @@ -0,0 +1,2 @@ +first line +second line \ No newline at end of file diff --git a/integration/unix/_fixtures/file_output_1.txt b/integration/unix/_fixtures/file_output_1.txt new file mode 100644 index 00000000..272b2e47 --- /dev/null +++ b/integration/unix/_fixtures/file_output_1.txt @@ -0,0 +1,3 @@ +first line +second line +third line \ No newline at end of file diff --git a/integration/unix/commander_test.yaml b/integration/unix/commander_test.yaml index 3f91b489..534e117f 100644 --- a/integration/unix/commander_test.yaml +++ b/integration/unix/commander_test.yaml @@ -59,6 +59,16 @@ tests: /books/0/author: J. R. R. Tokien /books/1/author: Joanne K. Rowling + it should assert file contents on stdout: + command: cat ./integration/unix/_fixtures/big_out.txt + stdout: + file: ./integration/unix/_fixtures/big_out.txt + + it should assert file contents on stderr: + command: cat ./integration/unix/_fixtures/big_out.txt >&2 + stderr: + file: ./integration/unix/_fixtures/big_out.txt + it should inherit from parent env: config: inherit-env: true diff --git a/integration/unix/failing_suite.yaml b/integration/unix/failing_suite.yaml index 36eab02f..8ffd2cb3 100644 --- a/integration/unix/failing_suite.yaml +++ b/integration/unix/failing_suite.yaml @@ -7,4 +7,9 @@ tests: command: sleep 1 config: timeout: 10ms - exit-code: 0 \ No newline at end of file + exit-code: 0 + + file matcher should fail when file and output are different: + command: cat ./integration/unix/_fixtures/file_output_1.txt + stdout: + file: ./integration/unix/_fixtures/file_output_0.txt diff --git a/pkg/matcher/matcher.go b/pkg/matcher/matcher.go index d01c4147..15d9f3d0 100644 --- a/pkg/matcher/matcher.go +++ b/pkg/matcher/matcher.go @@ -5,6 +5,7 @@ import ( "github.com/antchfx/xmlquery" "github.com/pmezard/go-difflib/difflib" "github.com/tidwall/gjson" + "io/ioutil" "log" "strings" ) @@ -20,8 +21,13 @@ const ( NotContains = "notcontains" JSON = "json" XML = "xml" + File = "file" ) +// The function used to open files when necessary for matching +// Allows the file IO to be overridden during tests +var ReadFile = ioutil.ReadFile + // NewMatcher creates a new matcher by type func NewMatcher(matcher string) Matcher { switch matcher { @@ -37,6 +43,8 @@ func NewMatcher(matcher string) Matcher { return JSONMatcher{} case XML: return XMLMatcher{} + case File: + return FileMatcher{} default: panic(fmt.Sprintf("Validator '%s' does not exist!", matcher)) } @@ -240,3 +248,32 @@ to be equal to return result } + +// FileMatcher matches output captured from stdout or stderr +// against the contents of a file +type FileMatcher struct { +} + +func (m FileMatcher) Match(got interface{}, expected interface{}) MatcherResult { + expectedText, err := ReadFile(expected.(string)) + if err != nil { + panic(err.Error()) + } + expectedString := string(expectedText) + + result := got == expectedString + + diff := difflib.UnifiedDiff{ + A: difflib.SplitLines(got.(string)), + B: difflib.SplitLines(expectedString), + FromFile: "Got", + ToFile: "Expected", + Context: 3, + } + diffText, _ := difflib.GetUnifiedDiffString(diff) + + return MatcherResult{ + Diff: diffText, + Success: result, + } +} diff --git a/pkg/matcher/matcher_test.go b/pkg/matcher/matcher_test.go index 7e8a2cca..d085c0ec 100644 --- a/pkg/matcher/matcher_test.go +++ b/pkg/matcher/matcher_test.go @@ -48,16 +48,16 @@ func TestTextMatcher_ValidateFails(t *testing.T) { func TestEqualMatcher_Validate(t *testing.T) { m := EqualMatcher{} - got := m.Match(1, 1) + got := m.Match(2, 2) assert.True(t, got.Success) } func TestEqualMatcher_ValidateFails(t *testing.T) { m := EqualMatcher{} - got := m.Match(1, 0) + got := m.Match(2, 3) assert.False(t, got.Success) - assert.Contains(t, got.Diff, "+0") - assert.Contains(t, got.Diff, "-1") + assert.Contains(t, got.Diff, "+3") + assert.Contains(t, got.Diff, "-2") } func TestContainsMatcher_Match(t *testing.T) { @@ -183,3 +183,24 @@ another` assert.False(t, r.Success) assert.Equal(t, diff, r.Diff) } + +func TestFileMatcher_Match(t *testing.T) { + ReadFile = func(filename string) ([]byte, error) { + return []byte("line one\nline two"), nil + } + m := FileMatcher{} + got := m.Match("line one\nline two", "fake.txt") + assert.True(t, got.Success) + assert.Equal(t, "", got.Diff) +} + +func TestFileMatcher_ValidateFails(t *testing.T) { + ReadFile = func(filename string) ([]byte, error) { + return []byte("line one\nline two"), nil + } + m := FileMatcher{} + got := m.Match("line one\nline three", "fake.txt") + assert.False(t, got.Success) + assert.Contains(t, got.Diff, "+line two") + assert.Contains(t, got.Diff, "-line three") +} diff --git a/pkg/runtime/runtime.go b/pkg/runtime/runtime.go index 405088df..5e933211 100644 --- a/pkg/runtime/runtime.go +++ b/pkg/runtime/runtime.go @@ -99,6 +99,7 @@ type ExpectedOut struct { NotContains []string `yaml:"not-contains,omitempty"` JSON map[string]string `yaml:"json,omitempty"` XML map[string]string `yaml:"xml,omitempty"` + File string `yaml:"file,omitempty"` } // CommandUnderTest represents the command under test diff --git a/pkg/runtime/validator.go b/pkg/runtime/validator.go index 3ef66254..0c712ded 100644 --- a/pkg/runtime/validator.go +++ b/pkg/runtime/validator.go @@ -118,6 +118,13 @@ func validateExpectedOut(got string, expected ExpectedOut) matcher.MatcherResult } } + if expected.File != "" { + m = matcher.NewMatcher(matcher.File) + if result = m.Match(got, expected.File); !result.Success { + return result + } + } + return result } diff --git a/pkg/runtime/validator_test.go b/pkg/runtime/validator_test.go index d09bbc7a..fc632812 100644 --- a/pkg/runtime/validator_test.go +++ b/pkg/runtime/validator_test.go @@ -3,6 +3,7 @@ package runtime import ( "github.com/commander-cli/commander/pkg/matcher" "github.com/stretchr/testify/assert" + "io/ioutil" "testing" ) @@ -192,6 +193,29 @@ test` assert.Equal(t, diff, r.Diff) } +func Test_ValidateExpectedOut_ValidateFile(t *testing.T) { + content := "line one" + matcher.ReadFile = func(filename string) ([]byte, error) { + return []byte(content), nil + } + r := validateExpectedOut(content, ExpectedOut{File: "fake.txt"}) + assert.True(t, r.Success) + assert.Equal(t, "", r.Diff) + + diff := `--- Got ++++ Expected +@@ -1,2 +1 @@ + line one +-line two +` + + r = validateExpectedOut(content+"\nline two", ExpectedOut{File: "fake.txt"}) + assert.False(t, r.Success) + assert.Equal(t, diff, r.Diff) + + matcher.ReadFile = ioutil.ReadFile +} + func Test_ValidateExpectedOut_ValidateXML(t *testing.T) { xml := ` J. R. R. Tolkien diff --git a/pkg/suite/yaml_suite.go b/pkg/suite/yaml_suite.go index 720f4bee..618e4076 100644 --- a/pkg/suite/yaml_suite.go +++ b/pkg/suite/yaml_suite.go @@ -222,6 +222,7 @@ func (y *YAMLSuiteConf) convertToExpectedOut(value interface{}) runtime.Expected "lines", "json", "xml", + "file", "not-contains": break default: @@ -242,6 +243,11 @@ func (y *YAMLSuiteConf) convertToExpectedOut(value interface{}) runtime.Expected exp.Exactly = toString(exactly) } + // Parse file key + if file := v["file"]; file != nil { + exp.File = toString(file) + } + //Parse line-count key if lc := v["line-count"]; lc != nil { exp.LineCount = lc.(int)