From f7ba99f780b951957b40f718a64d81d9fe439581 Mon Sep 17 00:00:00 2001 From: Matt Fellows Date: Sat, 11 Nov 2017 16:46:00 +1100 Subject: [PATCH] feat(verification): add testing.T to VerifyProvider for better test reporting BREAKING CHANGE: implements feature request #58 --- README.md | 33 +++++++++---------- daemon/daemon_test.go | 4 +-- daemon/verification_service.go | 8 ++--- daemon/verification_service_test.go | 7 ++-- doc.go | 18 ++++------ dsl/client_test.go | 2 +- dsl/pact.go | 30 +++++++++++++++-- dsl/pact_test.go | 8 ++--- examples/gin/provider/user_service_test.go | 28 +++------------- examples/go-kit/provider/user_service_test.go | 28 +++------------- examples/mux/provider/user_service_test.go | 28 +++------------- scripts/pact.ps1 | 2 +- types/verify_request.go | 4 +-- 13 files changed, 79 insertions(+), 121 deletions(-) diff --git a/README.md b/README.md index 763e4b5ac..483b1d2f6 100644 --- a/README.md +++ b/README.md @@ -255,24 +255,22 @@ Read more about [flexible matching](https://github.com/realestate-com-au/pact/wi satisfy the requirements of each of your known consumers: ```go - response := pact.VerifyProvider(types.VerifyRequest{ - ProviderBaseURL: "http://localhost:8000", - PactURLs: []string{"./pacts/my_consumer-my_provider.json"}, - ProviderStatesSetupURL: "http://localhost:8000/setup", - }) - - if err != nil { - t.Fatal("Error:", err) - } + pact.VerifyProvider(t, types.VerifyRequest{ + ProviderBaseURL: "http://localhost:8000", + PactURLs: []string{"./pacts/my_consumer-my_provider.json"}, + ProviderStatesSetupURL: "http://localhost:8000/setup", + }) ``` - Note that `PactURLs` is a list of local pact files or remote based - urls (e.g. from a - [Pact Broker](http://docs.pact.io/documentation/sharings_pacts.html)). + The `VerifyProvider` will handle all verifications, treating them as subtests + and giving you granular test reporting. If you don't like this behaviour, you may call `VerifyProviderRaw` directly and handle the errors manually. + Note that `PactURLs` may a list of local pact files or remote based + urls (e.g. from a + [Pact Broker](http://docs.pact.io/documentation/sharings_pacts.html)). - See the `Skip()'ed` [integration tests](https://github.com/pact-foundation/pact-go/blob/master/dsl/pact_test.go) - for a more complete E2E example. + See the `Skip()'ed` [integration tests](https://github.com/pact-foundation/pact-go/blob/master/dsl/pact_test.go) + for a more complete E2E example. #### Provider Verification @@ -281,7 +279,7 @@ When validating a Provider, you have 3 options to provide the Pact files: 1. Use `PactURLs` to specify the exact set of pacts to be replayed: ```go - response = pact.VerifyProvider(types.VerifyRequest{ + pact.VerifyProvider(t, types.VerifyRequest{ ProviderBaseURL: "http://myproviderhost", PactURLs: []string{"http://broker/pacts/provider/them/consumer/me/latest/dev"}, ProviderStatesSetupURL: "http://myproviderhost/setup", @@ -292,7 +290,7 @@ When validating a Provider, you have 3 options to provide the Pact files: 1. Use `PactBroker` to automatically find all of the latest consumers: ```go - response = pact.VerifyProvider(types.VerifyRequest{ + pact.VerifyProvider(t, types.VerifyRequest{ ProviderBaseURL: "http://myproviderhost", BrokerURL: "http://brokerHost", ProviderStatesSetupURL: "http://myproviderhost/setup", @@ -303,7 +301,7 @@ When validating a Provider, you have 3 options to provide the Pact files: 1. Use `PactBroker` and `Tags` to automatically find all of the latest consumers: ```go - response = pact.VerifyProvider(types.VerifyRequest{ + pact.VerifyProvider(t, types.VerifyRequest{ ProviderBaseURL: "http://myproviderhost", BrokerURL: "http://brokerHost", Tags: []string{"latest", "sit4"}, @@ -319,7 +317,6 @@ in development. See this [article](http://rea.tech/enter-the-pact-matrix-or-how-to-decouple-the-release-cycles-of-your-microservices/) for more on this strategy. -e See the examples or read more at http://docs.pact.io/documentation/provider_states.html. diff --git a/daemon/daemon_test.go b/daemon/daemon_test.go index 3ff6ec2da..1c67f7bff 100644 --- a/daemon/daemon_test.go +++ b/daemon/daemon_test.go @@ -255,7 +255,7 @@ func TestVerifyProvider_MissingProviderBaseURL(t *testing.T) { t.Fatal("Expected an error") } - if !strings.Contains(err.Error(), "ProviderBaseURL is mandatory") { + if !strings.Contains(err.Error(), "Provider base URL is mandatory") { t.Fatalf("Expected error message but got '%s'", err.Error()) } } @@ -273,7 +273,7 @@ func TestVerifyProvider_MissingPactURLs(t *testing.T) { t.Fatal("Expected an error") } - if !strings.Contains(err.Error(), "PactURLs is mandatory") { + if !strings.Contains(err.Error(), "Pact URLs is mandatory") { t.Fatalf("Expected error message but got '%s'", err.Error()) } } diff --git a/daemon/verification_service.go b/daemon/verification_service.go index 32cff2e27..fed294f9b 100644 --- a/daemon/verification_service.go +++ b/daemon/verification_service.go @@ -24,12 +24,12 @@ type VerificationService struct { // --broker-password // --publish_verification_results // --provider_app_version -func (m *VerificationService) NewService(args []string) Service { +func (v *VerificationService) NewService(args []string) Service { log.Printf("[DEBUG] starting verification service with args: %v\n", args) - m.Args = args - m.Cmd = getVerifierCommandPath() - return m + v.Args = args + v.Cmd = getVerifierCommandPath() + return v } func getVerifierCommandPath() string { diff --git a/daemon/verification_service_test.go b/daemon/verification_service_test.go index 064c338c5..abd736b95 100644 --- a/daemon/verification_service_test.go +++ b/daemon/verification_service_test.go @@ -1,6 +1,9 @@ package daemon -import "testing" +import ( + "fmt" + "testing" +) func TestVerificationService_NewService(t *testing.T) { s := &VerificationService{} @@ -11,6 +14,6 @@ func TestVerificationService_NewService(t *testing.T) { } if s.Args[0] != "--foo bar" { - t.Fatalf("Expected '--foo bar' argument to be passed") + t.Fatalf(fmt.Sprintf(`Expected "--foo bar" argument to be passed, got "%s"`, s.Args[0])) } } diff --git a/doc.go b/doc.go index a7b4c4a83..b23601bcc 100644 --- a/doc.go +++ b/doc.go @@ -130,24 +130,18 @@ A typical Provider side test would like something like: } go startMyAPI("http://localhost:8000") - response, err := pact.VerifyProvider(types.VerifyRequest{ + pact.VerifyProvider(types.VerifyRequest{ ProviderBaseURL: "http://localhost:8000", PactURLs: []string{"./pacts/my_consumer-my_provider.json"}, ProviderStatesSetupURL: "http://localhost:8000/setup", }) - - if err != nil { - t.Fatal("Error:", err) - } - - for _, example := range response.Examples { - if example.Status != "passed" { - t.Errorf("%s\n%s\n", example.FullDescription, example.Exception.Message) - } - } } -Note that `PactURLs` can be a list of local pact files or remote based +The `VerifyProvider` will handle all verifications, treating them as subtests +and giving you granular test reporting. If you don't like this behaviour, you may +call `VerifyProviderRaw` directly and handle the errors manually. + +Note that `PactURLs` may be a list of local pact files or remote based urls (possibly from a Pact Broker - http://docs.pact.io/documentation/sharings_pacts.html). diff --git a/dsl/client_test.go b/dsl/client_test.go index 204170a4c..36e96d06c 100644 --- a/dsl/client_test.go +++ b/dsl/client_test.go @@ -262,7 +262,7 @@ func TestClient_VerifyProviderFailValidation(t *testing.T) { t.Fatal("Expected a error but got none") } - if !strings.Contains(err.Error(), "PactURLs is mandatory") { + if !strings.Contains(err.Error(), "Pact URLs is mandatory") { t.Fatalf("Expected a proper error message but got '%s'", err.Error()) } } diff --git a/dsl/pact.go b/dsl/pact.go index 3bbd43afd..93ff287fa 100644 --- a/dsl/pact.go +++ b/dsl/pact.go @@ -9,6 +9,7 @@ import ( "log" "os" "path/filepath" + "testing" "github.com/hashicorp/logutils" "github.com/pact-foundation/pact-go/types" @@ -234,9 +235,9 @@ func (p *Pact) WritePact() error { return nil } -// VerifyProvider reads the provided pact files and runs verification against -// a running Provider API. -func (p *Pact) VerifyProvider(request types.VerifyRequest) (types.ProviderVerifierResponse, error) { +// VerifyProviderRaw reads the provided pact files and runs verification against +// a running Provider API, providing raw response from the Verification process. +func (p *Pact) VerifyProviderRaw(request types.VerifyRequest) (types.ProviderVerifierResponse, error) { p.Setup(false) // If we provide a Broker, we go to it to find consumers @@ -252,3 +253,26 @@ func (p *Pact) VerifyProvider(request types.VerifyRequest) (types.ProviderVerifi return p.pactClient.VerifyProvider(request) } + +// VerifyProvider accepts an instance of `*testing.T` +// running the provider verification with granular test reporting and +// automatic failure reporting for nice, simple tests. +func (p *Pact) VerifyProvider(t *testing.T, request types.VerifyRequest) (types.ProviderVerifierResponse, error) { + res, err := p.VerifyProviderRaw(request) + + if err != nil { + t.Fatal("Error:", err) + return res, err + } + + for _, example := range res.Examples { + t.Run(example.Description, func(st *testing.T) { + st.Log(example.FullDescription) + if example.Status != "passed" { + st.Errorf("%s\n", example.Exception.Message) + } + }) + } + + return res, err +} diff --git a/dsl/pact_test.go b/dsl/pact_test.go index fa277afea..1e953f1ea 100644 --- a/dsl/pact_test.go +++ b/dsl/pact_test.go @@ -271,7 +271,7 @@ func TestPact_VerifyProvider(t *testing.T) { waitForPortInTest(port, t) pact := &Pact{Port: port, LogLevel: "DEBUG", pactClient: &PactClient{Port: port}} - _, err := pact.VerifyProvider(types.VerifyRequest{ + _, err := pact.VerifyProviderRaw(types.VerifyRequest{ ProviderBaseURL: "http://www.foo.com", PactURLs: []string{"foo.json", "bar.json"}, }) @@ -294,7 +294,7 @@ func TestPact_VerifyProviderBroker(t *testing.T) { waitForPortInTest(port, t) pact := &Pact{Port: port, LogLevel: "DEBUG", pactClient: &PactClient{Port: port}, Provider: "bobby"} - _, err := pact.VerifyProvider(types.VerifyRequest{ + _, err := pact.VerifyProviderRaw(types.VerifyRequest{ ProviderBaseURL: "http://www.foo.com", BrokerURL: s.URL, PublishVerificationResults: true, @@ -319,7 +319,7 @@ func TestPact_VerifyProviderBrokerNoConsumers(t *testing.T) { waitForPortInTest(port, t) pact := &Pact{Port: port, LogLevel: "DEBUG", pactClient: &PactClient{Port: port}, Provider: "providernotexist"} - _, err := pact.VerifyProvider(types.VerifyRequest{ + _, err := pact.VerifyProviderRaw(types.VerifyRequest{ ProviderBaseURL: "http://www.foo.com", BrokerURL: s.URL, }) @@ -340,7 +340,7 @@ func TestPact_VerifyProviderFail(t *testing.T) { waitForPortInTest(port, t) pact := &Pact{Port: port, LogLevel: "DEBUG", pactClient: &PactClient{Port: port}} - _, err := pact.VerifyProvider(types.VerifyRequest{ + _, err := pact.VerifyProviderRaw(types.VerifyRequest{ ProviderBaseURL: "http://www.foo.com", PactURLs: []string{"foo.json", "bar.json"}, }) diff --git a/examples/gin/provider/user_service_test.go b/examples/gin/provider/user_service_test.go index 28bedfdfb..70d412d74 100644 --- a/examples/gin/provider/user_service_test.go +++ b/examples/gin/provider/user_service_test.go @@ -20,23 +20,18 @@ func TestPact_Provider(t *testing.T) { pact := createPact() // Verify the Provider with local Pact Files - res, err := pact.VerifyProvider(types.VerifyRequest{ + pact.VerifyProvider(t, types.VerifyRequest{ ProviderBaseURL: fmt.Sprintf("http://localhost:%d", port), PactURLs: []string{filepath.ToSlash(fmt.Sprintf("%s/billy-bobby.json", pactDir))}, ProviderStatesSetupURL: fmt.Sprintf("http://localhost:%d/setup", port), }) - if err != nil { - t.Fatal("Error:", err) - } - assertExamples(t, res) - // Pull from pact broker, used in e2e/integrated tests for pact-go release if os.Getenv("PACT_INTEGRATED_TESTS") != "" { var brokerHost = os.Getenv("PACT_BROKER_HOST") // Verify the Provider - Specific Published Pacts - res, err = pact.VerifyProvider(types.VerifyRequest{ + pact.VerifyProvider(t, types.VerifyRequest{ ProviderBaseURL: fmt.Sprintf("http://127.0.0.1:%d", port), PactURLs: []string{fmt.Sprintf("%s/pacts/provider/bobby/consumer/billy/latest/sit4", brokerHost)}, ProviderStatesSetupURL: fmt.Sprintf("http://127.0.0.1:%d/setup", port), @@ -46,13 +41,8 @@ func TestPact_Provider(t *testing.T) { ProviderVersion: "1.0.0", }) - if err != nil { - t.Fatal("Error:", err) - } - assertExamples(t, res) - // Verify the Provider - Latest Published Pacts for any known consumers - res, err = pact.VerifyProvider(types.VerifyRequest{ + pact.VerifyProvider(t, types.VerifyRequest{ ProviderBaseURL: fmt.Sprintf("http://127.0.0.1:%d", port), BrokerURL: brokerHost, ProviderStatesSetupURL: fmt.Sprintf("http://127.0.0.1:%d/setup", port), @@ -62,13 +52,8 @@ func TestPact_Provider(t *testing.T) { ProviderVersion: "1.0.0", }) - if err != nil { - t.Fatal("Error:", err) - } - assertExamples(t, res) - // Verify the Provider - Tag-based Published Pacts for any known consumers - res, err = pact.VerifyProvider(types.VerifyRequest{ + pact.VerifyProvider(t, types.VerifyRequest{ ProviderBaseURL: fmt.Sprintf("http://127.0.0.1:%d", port), ProviderStatesSetupURL: fmt.Sprintf("http://127.0.0.1:%d/setup", port), BrokerURL: brokerHost, @@ -79,11 +64,6 @@ func TestPact_Provider(t *testing.T) { ProviderVersion: "1.0.0", }) - if err != nil { - t.Fatal("Error:", err) - } - assertExamples(t, res) - } else { t.Log("Skipping pulling from broker as PACT_INTEGRATED_TESTS is not set") } diff --git a/examples/go-kit/provider/user_service_test.go b/examples/go-kit/provider/user_service_test.go index d9dcb3cd0..b48bdcdaf 100644 --- a/examples/go-kit/provider/user_service_test.go +++ b/examples/go-kit/provider/user_service_test.go @@ -58,23 +58,18 @@ func TestPact_Provider(t *testing.T) { pact := createPact() // Verify the Provider with local Pact Files - res, err := pact.VerifyProvider(types.VerifyRequest{ + pact.VerifyProvider(t, types.VerifyRequest{ ProviderBaseURL: fmt.Sprintf("http://localhost:%d", port), PactURLs: []string{filepath.ToSlash(fmt.Sprintf("%s/billy-bobby.json", pactDir))}, ProviderStatesSetupURL: fmt.Sprintf("http://localhost:%d/setup", port), }) - if err != nil { - t.Fatal("Error:", err) - } - assertExamples(t, res) - // Pull from pact broker, used in e2e/integrated tests for pact-go release if os.Getenv("PACT_INTEGRATED_TESTS") != "" { var brokerHost = os.Getenv("PACT_BROKER_HOST") // Verify the Provider - Specific Published Pacts - res, err = pact.VerifyProvider(types.VerifyRequest{ + pact.VerifyProvider(t, types.VerifyRequest{ ProviderBaseURL: fmt.Sprintf("http://127.0.0.1:%d", port), PactURLs: []string{fmt.Sprintf("%s/pacts/provider/bobby/consumer/billy/latest/sit4", brokerHost)}, ProviderStatesSetupURL: fmt.Sprintf("http://127.0.0.1:%d/setup", port), @@ -84,13 +79,8 @@ func TestPact_Provider(t *testing.T) { ProviderVersion: "1.0.0", }) - if err != nil { - t.Fatal("Error:", err) - } - assertExamples(t, res) - // Verify the Provider - Latest Published Pacts for any known consumers - res, err = pact.VerifyProvider(types.VerifyRequest{ + pact.VerifyProvider(t, types.VerifyRequest{ ProviderBaseURL: fmt.Sprintf("http://127.0.0.1:%d", port), BrokerURL: brokerHost, ProviderStatesSetupURL: fmt.Sprintf("http://127.0.0.1:%d/setup", port), @@ -100,13 +90,8 @@ func TestPact_Provider(t *testing.T) { ProviderVersion: "1.0.0", }) - if err != nil { - t.Fatal("Error:", err) - } - assertExamples(t, res) - // Verify the Provider - Tag-based Published Pacts for any known consumers - res, err = pact.VerifyProvider(types.VerifyRequest{ + pact.VerifyProvider(t, types.VerifyRequest{ ProviderBaseURL: fmt.Sprintf("http://127.0.0.1:%d", port), ProviderStatesSetupURL: fmt.Sprintf("http://127.0.0.1:%d/setup", port), BrokerURL: brokerHost, @@ -117,10 +102,6 @@ func TestPact_Provider(t *testing.T) { ProviderVersion: "1.0.0", }) - if err != nil { - t.Fatal("Error:", err) - } - assertExamples(t, res) } else { t.Log("Skipping pulling from broker as PACT_INTEGRATED_TESTS is not set") } @@ -179,7 +160,6 @@ func startInstrumentedProvider() { if err != nil { return } - json.Unmarshal(body, &state) svc := s.(*loggingMiddleware).next.(*userService) diff --git a/examples/mux/provider/user_service_test.go b/examples/mux/provider/user_service_test.go index c0316b11b..dd6459c4f 100644 --- a/examples/mux/provider/user_service_test.go +++ b/examples/mux/provider/user_service_test.go @@ -24,23 +24,18 @@ func TestPact_Provider(t *testing.T) { pact := createPact() // Verify the Provider with local Pact Files - res, err := pact.VerifyProvider(types.VerifyRequest{ + pact.VerifyProvider(t, types.VerifyRequest{ ProviderBaseURL: fmt.Sprintf("http://localhost:%d", port), PactURLs: []string{filepath.ToSlash(fmt.Sprintf("%s/billy-bobby.json", pactDir))}, ProviderStatesSetupURL: fmt.Sprintf("http://localhost:%d/setup", port), }) - if err != nil { - t.Fatal("Error:", err) - } - assertExamples(t, res) - // Pull from pact broker, used in e2e/integrated tests for pact-go release if os.Getenv("PACT_INTEGRATED_TESTS") != "" { var brokerHost = os.Getenv("PACT_BROKER_HOST") // Verify the Provider - Specific Published Pacts - res, err = pact.VerifyProvider(types.VerifyRequest{ + pact.VerifyProvider(t, types.VerifyRequest{ ProviderBaseURL: fmt.Sprintf("http://127.0.0.1:%d", port), PactURLs: []string{fmt.Sprintf("%s/pacts/provider/bobby/consumer/billy/latest/sit4", brokerHost)}, ProviderStatesSetupURL: fmt.Sprintf("http://127.0.0.1:%d/setup", port), @@ -50,13 +45,8 @@ func TestPact_Provider(t *testing.T) { ProviderVersion: "1.0.0", }) - if err != nil { - t.Fatal("Error:", err) - } - assertExamples(t, res) - // Verify the Provider - Latest Published Pacts for any known consumers - res, err = pact.VerifyProvider(types.VerifyRequest{ + pact.VerifyProvider(t, types.VerifyRequest{ ProviderBaseURL: fmt.Sprintf("http://127.0.0.1:%d", port), BrokerURL: brokerHost, ProviderStatesSetupURL: fmt.Sprintf("http://127.0.0.1:%d/setup", port), @@ -66,13 +56,8 @@ func TestPact_Provider(t *testing.T) { ProviderVersion: "1.0.0", }) - if err != nil { - t.Fatal("Error:", err) - } - assertExamples(t, res) - // Verify the Provider - Tag-based Published Pacts for any known consumers - res, err = pact.VerifyProvider(types.VerifyRequest{ + pact.VerifyProvider(t, types.VerifyRequest{ ProviderBaseURL: fmt.Sprintf("http://127.0.0.1:%d", port), ProviderStatesSetupURL: fmt.Sprintf("http://127.0.0.1:%d/setup", port), BrokerURL: brokerHost, @@ -83,11 +68,6 @@ func TestPact_Provider(t *testing.T) { ProviderVersion: "1.0.0", }) - if err != nil { - t.Fatal("Error:", err) - } - assertExamples(t, res) - } else { t.Log("Skipping pulling from broker as PACT_INTEGRATED_TESTS is not set") } diff --git a/scripts/pact.ps1 b/scripts/pact.ps1 index 732db1776..57e681f47 100644 --- a/scripts/pact.ps1 +++ b/scripts/pact.ps1 @@ -27,7 +27,7 @@ go build -o "$pactDir\pact-go.exe" "github.com\pact-foundation\pact-go" Write-Verbose "--> Creating pact daemon (downloading Ruby binaries)" $downloadDir = $env:TEMP -$url = "https://github.com/pact-foundation/pact-ruby-standalone/releases/download/v1.3.1/pact-1.3.1-win32.zip" +$url = "https://github.com/pact-foundation/pact-ruby-standalone/releases/download/v1.20.0/pact-1.20.0-win32.zip" Write-Verbose " Downloading $url" $zip = "$downloadDir\pact.zip" diff --git a/types/verify_request.go b/types/verify_request.go index 171e224f2..2ea989d96 100644 --- a/types/verify_request.go +++ b/types/verify_request.go @@ -55,7 +55,7 @@ func (v *VerifyRequest) Validate() error { if len(v.PactURLs) != 0 { v.Args = append(v.Args, strings.Join(v.PactURLs, " ")) } else { - return fmt.Errorf("PactURLs is mandatory.") + return fmt.Errorf("Pact URLs is mandatory") } v.Args = append(v.Args, "--format", "json") @@ -64,7 +64,7 @@ func (v *VerifyRequest) Validate() error { v.Args = append(v.Args, "--provider-base-url") v.Args = append(v.Args, v.ProviderBaseURL) } else { - return fmt.Errorf("ProviderBaseURL is mandatory.") + return fmt.Errorf("Provider base URL is mandatory") } if v.ProviderStatesSetupURL != "" {