From bb98a078961d9452cd4c8f2d310ed95aee4151c2 Mon Sep 17 00:00:00 2001 From: nouseforaname <34882943+nouseforaname@users.noreply.github.com> Date: Thu, 12 Sep 2024 10:27:29 +0200 Subject: [PATCH] update tests to cover shutdown and updated recover in progress --- .../fake-uuid-provision.tf | 11 +- .../termination-recovery/manifest.yml | 2 + integrationtest/integrationtest_suite_test.go | 16 +- integrationtest/termination_recovery_test.go | 314 +++++++++++------- internal/storage/storage_suite_test.go | 2 + pkg/providers/tf/deployment_manager_test.go | 7 +- 6 files changed, 232 insertions(+), 120 deletions(-) diff --git a/integrationtest/fixtures/termination-recovery/fake-uuid-provision.tf b/integrationtest/fixtures/termination-recovery/fake-uuid-provision.tf index db12d4a33..ba683a089 100644 --- a/integrationtest/fixtures/termination-recovery/fake-uuid-provision.tf +++ b/integrationtest/fixtures/termination-recovery/fake-uuid-provision.tf @@ -3,9 +3,16 @@ terraform { random = { source = "registry.terraform.io/hashicorp/random" } + null = { + source = "registry.terraform.io/hashicorp/null" + } + } +} +resource "null_resource" "sleeper" { + provisioner "local-exec" { + command = "sleep 10" } } - resource "random_uuid" "random" {} -output provision_output { value = random_uuid.random.result } \ No newline at end of file +output provision_output { value = random_uuid.random.result } diff --git a/integrationtest/fixtures/termination-recovery/manifest.yml b/integrationtest/fixtures/termination-recovery/manifest.yml index 306bb39ab..b5758ffe9 100644 --- a/integrationtest/fixtures/termination-recovery/manifest.yml +++ b/integrationtest/fixtures/termination-recovery/manifest.yml @@ -14,5 +14,7 @@ terraform_binaries: source: https://github.com/opentofu/opentofu/archive/refs/tags/v1.6.0.zip - name: terraform-provider-random version: 3.1.0 +- name: terraform-provider-null + version: 3.2.2 service_definitions: - fake-uuid-service.yml diff --git a/integrationtest/integrationtest_suite_test.go b/integrationtest/integrationtest_suite_test.go index cae7ec363..280c9d235 100644 --- a/integrationtest/integrationtest_suite_test.go +++ b/integrationtest/integrationtest_suite_test.go @@ -30,7 +30,7 @@ var ( var _ = SynchronizedBeforeSuite( func() []byte { // -gcflags enabled "gops", but had to be removed as this doesn't compile with Go 1.19 - //path, err := Build("github.com/cloudfoundry/cloud-service-broker", `-gcflags="all=-N -l"`) + // path, err := Build("github.com/cloudfoundry/cloud-service-broker", `-gcflags="all=-N -l"`) path := must(Build("github.com/cloudfoundry/cloud-service-broker/v2")) return []byte(path) }, @@ -45,8 +45,18 @@ var _ = SynchronizedBeforeSuite( ) var _ = SynchronizedAfterSuite( - func() {}, - func() { CleanupBuildArtifacts() }, + func() { + }, + func() { + CleanupBuildArtifacts() + files, err := filepath.Glob("/tmp/brokerpak*") + Expect(err).ToNot(HaveOccurred()) + for _, f := range files { + if err := os.RemoveAll(f); err != nil { + Expect(err).ToNot(HaveOccurred()) + } + } + }, ) var _ = BeforeEach(func() { diff --git a/integrationtest/termination_recovery_test.go b/integrationtest/termination_recovery_test.go index 79c547c70..fa356fc17 100644 --- a/integrationtest/termination_recovery_test.go +++ b/integrationtest/termination_recovery_test.go @@ -3,6 +3,8 @@ package integrationtest_test import ( "fmt" "net/http" + "os" + "time" "github.com/cloudfoundry/cloud-service-broker/v2/integrationtest/packer" "github.com/cloudfoundry/cloud-service-broker/v2/internal/testdrive" @@ -28,145 +30,233 @@ var _ = Describe("Recovery From Broker Termination", func() { BeforeEach(func() { brokerpak = must(packer.BuildBrokerpak(csb, fixtures("termination-recovery"))) - + }) + BeforeEach(func() { stdout = NewBuffer() stderr = NewBuffer() - broker = must(testdrive.StartBroker(csb, brokerpak, database, testdrive.WithOutputs(stdout, stderr))) - - DeferCleanup(func() { - Expect(broker.Stop()).To(Succeed()) - cleanup(brokerpak) + }) + Describe("running csb on a VM", func() { + Describe("when a vm broker properly drains", func() { + BeforeEach(func() { + broker = must(testdrive.StartBroker(csb, brokerpak, database, testdrive.WithOutputs(stdout, stderr))) + + DeferCleanup(func() { + Expect(broker.Terminate()).To(Succeed()) + }) + }) + + It("can finish the in flight operation", func() { + By("starting to provision") + instanceGUID := uuid.NewString() + response := broker.Client.Provision(instanceGUID, serviceOfferingGUID, servicePlanGUID, uuid.NewString(), nil) + + Expect(response.Error).NotTo(HaveOccurred()) + Expect(response.StatusCode).To(Equal(http.StatusAccepted)) + Eventually(stdout, time.Second*5).Should(Say(`tofu","apply","-auto-approve"`)) + By("gracefully stopping the broker") + // Stop seems to be blocking, so run it in a routine so we can check that the broker actually rejects requests until it's fully stopped. + go func() { + Expect(broker.Stop()).To(Succeed()) + }() + + By("logging a message") + Eventually(stdout).Should(Say("received SIGTERM")) + Eventually(stdout).Should(Say("draining csb in progress")) + + By("ensuring that the broker rejects requests") + Expect(broker.Client.LastOperation(instanceGUID, uuid.NewString()).Error).To(HaveOccurred()) + + // Fun stuff, do not optimize this with a SatisfyAll().. The relevant part of the docs is: + // When Say succeeds, it fast forwards the gbytes.Buffer's read cursor to just after the successful match. + // meaning if below lines will be partially matched at first attempt, no further attempt can succeed because we + // forwarded past the location of the initial first match. + + Eventually(stdout, time.Second*20).Should(Say(fmt.Sprintf("successfully stored state for tf:%s:", instanceGUID))) + Eventually(stdout, time.Second*20).Should(Say("draining complete")) + Consistently(stderr, time.Second*20).ShouldNot(Say("shutdown error")) + + broker = must(testdrive.StartBroker(csb, brokerpak, database, testdrive.WithOutputs(stdout, stderr))) + + By("checking that the resource finished successfully") + response = broker.Client.LastOperation(instanceGUID, uuid.NewString()) + Expect(string(response.ResponseBody)).To(ContainSubstring(`{"state":"succeeded","description":"provision succeeded"}`)) + Expect(response.Error).NotTo(HaveOccurred()) + Expect(response.StatusCode).To(Equal(http.StatusOK)) + + By("ensuring SI can be successfully deleted") + si := testdrive.ServiceInstance{GUID: instanceGUID, ServiceOfferingGUID: serviceOfferingGUID, ServicePlanGUID: servicePlanGUID} + Expect(broker.Deprovision(si)).To(Succeed()) + }) + }) + Describe("when a vm broker did not properly drain", func() { + var dirDefault string + BeforeEach(func() { + By("ensuring that the expected lockdir exists") + + dirDefault, _ = os.MkdirTemp("/tmp/", "lockfiles") + broker = must(testdrive.StartBroker(csb, brokerpak, database, testdrive.WithOutputs(stdout, stderr), testdrive.WithEnv(fmt.Sprintf("CSB_LOCKFILE_DIR=%s", dirDefault)))) + }) + + It("fails service instances that have a lockfile on start", func() { + instanceGUID := uuid.NewString() + response := broker.Client.Provision(instanceGUID, serviceOfferingGUID, servicePlanGUID, uuid.NewString(), nil) + Expect(response.Error).NotTo(HaveOccurred()) + Expect(response.StatusCode).To(Equal(http.StatusAccepted)) + + Eventually(stdout, time.Second*5).Should(Say(`tofu","apply","-auto-approve"`)) + By("forcefully stopping the broker") + // Stop seems to be blocking, so run it in a routine so we can check that the broker actually rejects requests until it's fully stopped. + go func() { + Expect(broker.Terminate()).To(Succeed()) + }() + + broker = must(testdrive.StartBroker(csb, brokerpak, database, testdrive.WithOutputs(stdout, stderr), testdrive.WithEnv(fmt.Sprintf("CSB_LOCKFILE_DIR=%s", dirDefault)))) + lastOperation, err := broker.LastOperation(instanceGUID) + Expect(err).NotTo(HaveOccurred()) + Expect(lastOperation.Description).To(Equal("the broker restarted while the operation was in progress")) + Expect(lastOperation.State).To(BeEquivalentTo("failed")) + }) }) }) - It("can recover from a terminated create", func() { - By("starting to provision") - instanceGUID := uuid.NewString() - response := broker.Client.Provision(instanceGUID, serviceOfferingGUID, servicePlanGUID, uuid.NewString(), nil) - Expect(response.Error).NotTo(HaveOccurred()) - Expect(response.StatusCode).To(Equal(http.StatusAccepted)) - - By("terminating and restarting the broker") - Expect(broker.Stop()).To(Succeed()) - broker = must(testdrive.StartBroker(csb, brokerpak, database, testdrive.WithOutputs(stdout, stderr))) - - By("reporting that an operation failed") - lastOperation, err := broker.LastOperation(instanceGUID) - Expect(err).NotTo(HaveOccurred()) - Expect(lastOperation.Description).To(Equal("the broker restarted while the operation was in progress")) - Expect(lastOperation.State).To(BeEquivalentTo("failed")) - - By("logging a message") - ws := fmt.Sprintf(`"workspace_id":"tf:%s:"`, instanceGUID) - Expect(string(stdout.Contents())).To(SatisfyAll(ContainSubstring("recover-in-progress-operations.mark-as-failed"), ContainSubstring(ws))) - - // OSBAPI requires that HTTP 409 (Conflict) is returned - By("refusing to allow a duplicate instance") - response = broker.Client.Provision(instanceGUID, serviceOfferingGUID, servicePlanGUID, uuid.NewString(), nil) - Expect(response.Error).NotTo(HaveOccurred()) - Expect(response.StatusCode).To(Equal(http.StatusConflict)) - - By("allowing the instance to be cleaned up") - response = broker.Client.Deprovision(instanceGUID, serviceOfferingGUID, servicePlanGUID, uuid.NewString()) - Expect(response.Error).NotTo(HaveOccurred()) - Expect(response.StatusCode).To(Equal(http.StatusOK)) - }) + Describe("running csb as a CF app", func() { + BeforeEach(func() { + broker = must(testdrive.StartBroker(csb, brokerpak, database, testdrive.WithOutputs(stdout, stderr), testdrive.WithEnv("CF_INSTANCE_GUID=dcfa061e-c0e3-4237-a805-734578347393"))) + + DeferCleanup(func() { + Expect(broker.Terminate()).To(Succeed()) + }) + }) - It("can recover from a terminated update", func() { - By("successfully provisioning a service instance") - instance, err := broker.Provision(serviceOfferingGUID, servicePlanGUID) - Expect(err).NotTo(HaveOccurred()) + It("can recover from a terminated create", func() { + By("starting to provision") + instanceGUID := uuid.NewString() + response := broker.Client.Provision(instanceGUID, serviceOfferingGUID, servicePlanGUID, uuid.NewString(), nil) + Expect(response.Error).NotTo(HaveOccurred()) + Expect(response.StatusCode).To(Equal(http.StatusAccepted)) + + By("terminating and restarting the broker") + Expect(broker.Terminate()).To(Succeed()) + broker = must(testdrive.StartBroker(csb, brokerpak, database, testdrive.WithOutputs(stdout, stderr), testdrive.WithEnv("CF_INSTANCE_GUID=dcfa061e-c0e3-4237-a805-734578347393"))) + + By("reporting that an operation failed") + lastOperation, err := broker.LastOperation(instanceGUID) + Expect(err).NotTo(HaveOccurred()) + Expect(lastOperation.Description).To(Equal("the broker restarted while the operation was in progress")) + Expect(lastOperation.State).To(BeEquivalentTo("failed")) + + By("logging a message") + ws := fmt.Sprintf(`"workspace_id":"tf:%s:"`, instanceGUID) + Expect(string(stdout.Contents())).To(SatisfyAll(ContainSubstring("recover-in-progress-operations.mark-as-failed"), ContainSubstring(ws))) + + // OSBAPI requires that HTTP 409 (Conflict) is returned + By("refusing to allow a duplicate instance") + response = broker.Client.Provision(instanceGUID, serviceOfferingGUID, servicePlanGUID, uuid.NewString(), nil) + Expect(response.Error).NotTo(HaveOccurred()) + Expect(response.StatusCode).To(Equal(http.StatusConflict)) + + By("allowing the instance to be cleaned up") + response = broker.Client.Deprovision(instanceGUID, serviceOfferingGUID, servicePlanGUID, uuid.NewString()) + Expect(response.Error).NotTo(HaveOccurred()) + Expect(response.StatusCode).To(Equal(http.StatusOK)) + }) - By("starting to update") - response := broker.Client.Update(instance.GUID, serviceOfferingGUID, servicePlanGUID, uuid.NewString(), nil, domain.PreviousValues{}, nil) - Expect(response.Error).NotTo(HaveOccurred()) - Expect(response.StatusCode).To(Equal(http.StatusAccepted)) + It("can recover from a terminated update", func() { + By("successfully provisioning a service instance") + instance, err := broker.Provision(serviceOfferingGUID, servicePlanGUID) + Expect(err).NotTo(HaveOccurred()) - By("terminating and restarting the broker") - Expect(broker.Stop()).To(Succeed()) - broker = must(testdrive.StartBroker(csb, brokerpak, database, testdrive.WithOutputs(stdout, stderr))) + By("starting to update") + response := broker.Client.Update(instance.GUID, serviceOfferingGUID, servicePlanGUID, uuid.NewString(), nil, domain.PreviousValues{}, nil) + Expect(response.Error).NotTo(HaveOccurred()) + Expect(response.StatusCode).To(Equal(http.StatusAccepted)) - By("reporting that an operation failed") - lastOperation, err := broker.LastOperation(instance.GUID) - Expect(err).NotTo(HaveOccurred()) - Expect(lastOperation.Description).To(Equal("the broker restarted while the operation was in progress")) - Expect(lastOperation.State).To(BeEquivalentTo("failed")) + By("terminating and restarting the broker") + Expect(broker.Terminate()).To(Succeed()) + broker = must(testdrive.StartBroker(csb, brokerpak, database, testdrive.WithOutputs(stdout, stderr), testdrive.WithEnv("CF_INSTANCE_GUID=dcfa061e-c0e3-4237-a805-734578347393"))) - By("logging a message") - ws := fmt.Sprintf(`"workspace_id":"tf:%s:"`, instance.GUID) - Expect(string(stdout.Contents())).To(SatisfyAll(ContainSubstring("recover-in-progress-operations.mark-as-failed"), ContainSubstring(ws))) + By("reporting that an operation failed") + lastOperation, err := broker.LastOperation(instance.GUID) + Expect(err).NotTo(HaveOccurred()) + Expect(lastOperation.Description).To(Equal("the broker restarted while the operation was in progress")) + Expect(lastOperation.State).To(BeEquivalentTo("failed")) - By("allowing the operation to be restarted") - Expect(broker.UpdateService(instance)).To(Succeed()) - }) + By("logging a message") + ws := fmt.Sprintf(`"workspace_id":"tf:%s:"`, instance.GUID) + Expect(string(stdout.Contents())).To(SatisfyAll(ContainSubstring("recover-in-progress-operations.mark-as-failed"), ContainSubstring(ws))) - It("can recover from a terminated delete", func() { - By("successfully provisioning a service instance") - instance, err := broker.Provision(serviceOfferingGUID, servicePlanGUID) - Expect(err).NotTo(HaveOccurred()) + By("allowing the operation to be restarted") + Expect(broker.UpdateService(instance)).To(Succeed()) + }) - By("starting to delete") - response := broker.Client.Deprovision(instance.GUID, serviceOfferingGUID, servicePlanGUID, uuid.NewString()) - Expect(response.Error).NotTo(HaveOccurred()) - Expect(response.StatusCode).To(Equal(http.StatusAccepted)) + It("can recover from a terminated delete", func() { + By("successfully provisioning a service instance") + instance, err := broker.Provision(serviceOfferingGUID, servicePlanGUID) + Expect(err).NotTo(HaveOccurred()) - By("terminating and restarting the broker") - Expect(broker.Stop()).To(Succeed()) - broker = must(testdrive.StartBroker(csb, brokerpak, database, testdrive.WithOutputs(stdout, stderr))) + By("starting to delete") + response := broker.Client.Deprovision(instance.GUID, serviceOfferingGUID, servicePlanGUID, uuid.NewString()) + Expect(response.Error).NotTo(HaveOccurred()) + Expect(response.StatusCode).To(Equal(http.StatusAccepted)) - By("reporting that an operation failed") - lastOperation, err := broker.LastOperation(instance.GUID) - Expect(err).NotTo(HaveOccurred()) - Expect(lastOperation.Description).To(Equal("the broker restarted while the operation was in progress")) - Expect(lastOperation.State).To(BeEquivalentTo("failed")) + By("terminating and restarting the broker") + Expect(broker.Terminate()).To(Succeed()) + broker = must(testdrive.StartBroker(csb, brokerpak, database, testdrive.WithOutputs(stdout, stderr), testdrive.WithEnv("CF_INSTANCE_GUID=dcfa061e-c0e3-4237-a805-734578347393"))) - By("logging a message") - ws := fmt.Sprintf(`"workspace_id":"tf:%s:"`, instance.GUID) - Expect(string(stdout.Contents())).To(SatisfyAll(ContainSubstring("recover-in-progress-operations.mark-as-failed"), ContainSubstring(ws))) + By("reporting that an operation failed") + lastOperation, err := broker.LastOperation(instance.GUID) + Expect(err).NotTo(HaveOccurred()) + Expect(lastOperation.Description).To(Equal("the broker restarted while the operation was in progress")) + Expect(lastOperation.State).To(BeEquivalentTo("failed")) - By("allowing the operation to be restarted") - Expect(broker.Deprovision(instance)).To(Succeed()) - }) + By("logging a message") + ws := fmt.Sprintf(`"workspace_id":"tf:%s:"`, instance.GUID) + Expect(string(stdout.Contents())).To(SatisfyAll(ContainSubstring("recover-in-progress-operations.mark-as-failed"), ContainSubstring(ws))) - It("can recover from a terminated bind", func() { - By("successfully provisioning a service instance") - instance, err := broker.Provision(serviceOfferingGUID, servicePlanGUID) - Expect(err).NotTo(HaveOccurred()) + By("allowing the operation to be restarted") + Expect(broker.Deprovision(instance)).To(Succeed()) + }) - By("starting to bind") - bindingGUID := uuid.NewString() - go broker.CreateBinding(instance, testdrive.WithBindingGUID(bindingGUID)) + It("can recover from a terminated bind", func() { + By("successfully provisioning a service instance") + instance, err := broker.Provision(serviceOfferingGUID, servicePlanGUID) + Expect(err).NotTo(HaveOccurred()) - Eventually(stdout).Should(Say(fmt.Sprintf(`"cloud-service-broker.Binding".*"binding_id":"%s"`, bindingGUID))) + By("starting to bind") + bindingGUID := uuid.NewString() + go broker.CreateBinding(instance, testdrive.WithBindingGUID(bindingGUID)) - By("terminating and restarting the broker") - Expect(broker.Stop()).To(Succeed()) - broker = must(testdrive.StartBroker(csb, brokerpak, database, testdrive.WithOutputs(stdout, stderr))) + Eventually(stdout).Should(Say(fmt.Sprintf(`"cloud-service-broker.Binding".*"binding_id":"%s"`, bindingGUID))) - By("allowing the operation to be restarted") - _, err = broker.CreateBinding(instance, testdrive.WithBindingGUID(bindingGUID)) - Expect(err).NotTo(HaveOccurred()) - }) + By("terminating and restarting the broker") + Expect(broker.Terminate()).To(Succeed()) + broker = must(testdrive.StartBroker(csb, brokerpak, database, testdrive.WithOutputs(stdout, stderr), testdrive.WithEnv("CF_INSTANCE_GUID=dcfa061e-c0e3-4237-a805-734578347393"))) + + By("allowing the operation to be restarted") + _, err = broker.CreateBinding(instance, testdrive.WithBindingGUID(bindingGUID)) + Expect(err).NotTo(HaveOccurred()) + }) - It("can recover from a terminated unbind", func() { - By("successfully provisioning a service instance and binding") - instance, err := broker.Provision(serviceOfferingGUID, servicePlanGUID) - Expect(err).NotTo(HaveOccurred()) + It("can recover from a terminated unbind", func() { + By("successfully provisioning a service instance and binding") + instance, err := broker.Provision(serviceOfferingGUID, servicePlanGUID) + Expect(err).NotTo(HaveOccurred()) - bindingGUID := uuid.NewString() - _, err = broker.CreateBinding(instance, testdrive.WithBindingGUID(bindingGUID)) - Expect(err).NotTo(HaveOccurred()) + bindingGUID := uuid.NewString() + _, err = broker.CreateBinding(instance, testdrive.WithBindingGUID(bindingGUID)) + Expect(err).NotTo(HaveOccurred()) - By("starting to unbind") - go broker.DeleteBinding(instance, bindingGUID) + By("starting to unbind") + go broker.DeleteBinding(instance, bindingGUID) - Eventually(stdout).Should(Say(fmt.Sprintf(`"cloud-service-broker.Unbinding".*"binding_id":"%s"`, bindingGUID))) + Eventually(stdout).Should(Say(fmt.Sprintf(`"cloud-service-broker.Unbinding".*"binding_id":"%s"`, bindingGUID))) - By("terminating and restarting the broker") - Expect(broker.Stop()).To(Succeed()) - broker = must(testdrive.StartBroker(csb, brokerpak, database, testdrive.WithOutputs(stdout, stderr))) + By("terminating and restarting the broker") + Expect(broker.Terminate()).To(Succeed()) + broker = must(testdrive.StartBroker(csb, brokerpak, database, testdrive.WithOutputs(stdout, stderr), testdrive.WithEnv("CF_INSTANCE_GUID=dcfa061e-c0e3-4237-a805-734578347393"))) - By("allowing the operation to be restarted") - Expect(broker.DeleteBinding(instance, bindingGUID)).To(Succeed()) + By("allowing the operation to be restarted") + Expect(broker.DeleteBinding(instance, bindingGUID)).To(Succeed()) + }) }) }) diff --git a/internal/storage/storage_suite_test.go b/internal/storage/storage_suite_test.go index c02209966..31728a8c0 100644 --- a/internal/storage/storage_suite_test.go +++ b/internal/storage/storage_suite_test.go @@ -5,6 +5,7 @@ import ( "strings" "testing" + "code.cloudfoundry.org/lager/v3/lagertest" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" "gorm.io/driver/sqlite" @@ -19,6 +20,7 @@ var ( db *gorm.DB encryptor *storagefakes.FakeEncryptor store *storage.Storage + logger *lagertest.TestLogger ) func TestStorage(t *testing.T) { diff --git a/pkg/providers/tf/deployment_manager_test.go b/pkg/providers/tf/deployment_manager_test.go index 0e04c3b20..30c8d5eb5 100644 --- a/pkg/providers/tf/deployment_manager_test.go +++ b/pkg/providers/tf/deployment_manager_test.go @@ -205,7 +205,8 @@ var _ = Describe("DeploymentManager", func() { storedDeployment := fakeStore.StoreTerraformDeploymentArgsForCall(0) Expect(storedDeployment.LastOperationState).To(Equal("succeeded")) Expect(storedDeployment.LastOperationMessage).To(Equal("provision succeeded: apply completed successfully")) - Expect(fakeLogger.Logs()).To(BeEmpty()) + Expect(fakeLogger.Logs()).To(HaveLen(1)) + Expect(fakeLogger.Logs()[0].Message).To(Equal("broker.successfully stored state for deploymentID")) }) }) @@ -222,12 +223,12 @@ var _ = Describe("DeploymentManager", func() { Expect(storedDeployment.LastOperationType).To(Equal(existingDeployment.LastOperationType)) Expect(storedDeployment.LastOperationState).To(Equal("failed")) Expect(storedDeployment.LastOperationMessage).To(Equal("provision failed: operation failed dramatically")) - Expect(fakeLogger.Logs()).To(HaveLen(1)) + Expect(fakeLogger.Logs()).To(HaveLen(2)) Expect(fakeLogger.Logs()[0].Message).To(ContainSubstring("operation-failed")) Expect(fakeLogger.Logs()[0].Data).To(HaveKeyWithValue("error", Equal("operation failed dramatically"))) Expect(fakeLogger.Logs()[0].Data).To(HaveKeyWithValue("message", Equal("provision failed: operation failed dramatically"))) Expect(fakeLogger.Logs()[0].Data).To(HaveKeyWithValue("deploymentID", Equal(existingDeployment.ID))) - + Expect(fakeLogger.Logs()[1].Message).To(Equal("broker.successfully stored state for deploymentID")) }) }) })