From 5f722b26e5b0cf39982fdc4aeafa9b1ecb8255fa Mon Sep 17 00:00:00 2001 From: shreyas-s-rao Date: Wed, 30 Jan 2019 16:37:44 +0530 Subject: [PATCH] Negative tests for network interruption --- pkg/snapshot/snapshotter/snapshotter.go | 2 +- pkg/snapstore/abs_snapstore_test.go | 24 +++++++++++++ pkg/snapstore/gcs_snapstore.go | 6 ++++ pkg/snapstore/gcs_snapstore_test.go | 15 ++++++++ pkg/snapstore/s3_snapstore_test.go | 28 +++++++++++++++ pkg/snapstore/snapstore_suite_test.go | 5 +-- pkg/snapstore/snapstore_test.go | 47 ++++++++++++++++++++++--- pkg/snapstore/swift_snapstore_test.go | 20 +++++++++++ 8 files changed, 140 insertions(+), 7 deletions(-) diff --git a/pkg/snapshot/snapshotter/snapshotter.go b/pkg/snapshot/snapshotter/snapshotter.go index 54ab2b334..1faa91da8 100644 --- a/pkg/snapshot/snapshotter/snapshotter.go +++ b/pkg/snapshot/snapshotter/snapshotter.go @@ -171,7 +171,7 @@ func (ssr *Snapshotter) TakeFullSnapshotAndResetTimer() error { } else { ssr.logger.Infof("Stopping full snapshot...") ssr.fullSnapshotTimer.Stop() - ssr.logger.Infof("Reseting full snapshot to run after %d secs.", effective.Sub(now)) + ssr.logger.Infof("Reseting full snapshot to run after %f secs.", effective.Sub(now).Seconds()) ssr.fullSnapshotTimer.Reset(effective.Sub(now)) } ssr.logger.Infof("Will take next full snapshot at time: %s", effective) diff --git a/pkg/snapstore/abs_snapstore_test.go b/pkg/snapstore/abs_snapstore_test.go index 717302044..d191aabb3 100644 --- a/pkg/snapstore/abs_snapstore_test.go +++ b/pkg/snapstore/abs_snapstore_test.go @@ -136,6 +136,12 @@ func (p *fakePolicy) handleContainerGetOperation(w *http.Response) error { // handleListObjectNames responds with a blob `List` response. func (p *fakePolicy) handleListObjects(w *http.Response) error { + if networkTimeoutFlag { + w.StatusCode = http.StatusRequestTimeout + w.Body = http.NoBody + return nil + } + var ( keys []string blobs []blobItem @@ -195,6 +201,12 @@ func (p *fakePolicy) handleListObjects(w *http.Response) error { // handleBlobCreateOperation responds with a blob `Put` response. func (p *fakePolicy) handleBlobPutOperation(w *http.Response) { + if networkTimeoutFlag { + w.StatusCode = http.StatusRequestTimeout + w.Body = http.NoBody + return + } + var ( query = w.Request.URL.Query() comp = query.Get("comp") @@ -248,6 +260,12 @@ func (p *fakePolicy) handleBlobPutOperation(w *http.Response) { // handleBlobGetOperation on GET request `/testContainer/testObject` responds with a `Get` response. func (p *fakePolicy) handleBlobGetOperation(w *http.Response) { + if networkTimeoutFlag { + w.StatusCode = http.StatusRequestTimeout + w.Body = http.NoBody + return + } + key := parseObjectNamefromURL(w.Request.URL) if _, ok := p.objectMap[key]; ok { w.StatusCode = http.StatusOK @@ -260,6 +278,12 @@ func (p *fakePolicy) handleBlobGetOperation(w *http.Response) { // handleDeleteObject on delete request `/testContainer/testObject` responds with a `Delete` response. func (p *fakePolicy) handleDeleteObject(w *http.Response) { + if networkTimeoutFlag { + w.StatusCode = http.StatusRequestTimeout + w.Body = http.NoBody + return + } + key := parseObjectNamefromURL(w.Request.URL) if _, ok := p.objectMap[key]; ok { delete(p.objectMap, key) diff --git a/pkg/snapstore/gcs_snapstore.go b/pkg/snapstore/gcs_snapstore.go index c692531c1..c4c6ea6ed 100644 --- a/pkg/snapstore/gcs_snapstore.go +++ b/pkg/snapstore/gcs_snapstore.go @@ -167,6 +167,9 @@ func (s *GCSSnapStore) uploadComponent(snap *Snapshot, file *os.File, offset, ch ctx, cancel := context.WithTimeout(context.TODO(), chunkUploadTimeout) defer cancel() w := obj.NewWriter(ctx) + if w == nil { + return fmt.Errorf("failed to create writer for snapshot") + } if _, err := io.Copy(w, sr); err != nil { w.Close() return err @@ -197,6 +200,9 @@ func (s *GCSSnapStore) componentUploader(wg *sync.WaitGroup, stopCh <-chan struc // List will list the snapshots from store. func (s *GCSSnapStore) List() (SnapList, error) { it := s.client.Bucket(s.bucket).Objects(context.TODO(), &storage.Query{Prefix: s.prefix}) + if it == nil { + return nil, fmt.Errorf("failed listing objects") + } var attrs []*storage.ObjectAttrs for { diff --git a/pkg/snapstore/gcs_snapstore_test.go b/pkg/snapstore/gcs_snapstore_test.go index fa34a7bfc..00c93dd7f 100644 --- a/pkg/snapstore/gcs_snapstore_test.go +++ b/pkg/snapstore/gcs_snapstore_test.go @@ -51,6 +51,10 @@ func (m *mockBucketHandle) Object(name string) stiface.ObjectHandle { } func (m *mockBucketHandle) Objects(context.Context, *storage.Query) stiface.ObjectIterator { + if networkTimeoutFlag { + return nil + } + var keys []string for key, _ := range m.client.objects { keys = append(keys, key) @@ -66,6 +70,10 @@ type mockObjectHandle struct { } func (m *mockObjectHandle) NewReader(ctx context.Context) (stiface.Reader, error) { + if networkTimeoutFlag { + return nil, fmt.Errorf("network timeout for NewReader()") + } + if value, ok := m.client.objects[m.object]; ok { return &mockObjectReader{reader: ioutil.NopCloser(bytes.NewReader(*value))}, nil } @@ -73,6 +81,9 @@ func (m *mockObjectHandle) NewReader(ctx context.Context) (stiface.Reader, error } func (m *mockObjectHandle) NewWriter(context.Context) stiface.Writer { + if networkTimeoutFlag { + return nil + } return &mockObjectWriter{object: m.object, client: m.client} } @@ -85,6 +96,10 @@ func (m *mockObjectHandle) ComposerFrom(objects ...stiface.ObjectHandle) stiface } func (m *mockObjectHandle) Delete(context.Context) error { + if networkTimeoutFlag { + return fmt.Errorf("network timeout for Delete()") + } + if _, ok := m.client.objects[m.object]; ok { delete(m.client.objects, m.object) return nil diff --git a/pkg/snapstore/s3_snapstore_test.go b/pkg/snapstore/s3_snapstore_test.go index dfd4bac12..f52c28671 100644 --- a/pkg/snapstore/s3_snapstore_test.go +++ b/pkg/snapstore/s3_snapstore_test.go @@ -41,6 +41,10 @@ type mockS3Client struct { // GetObject returns the object from map for mock test func (m *mockS3Client) GetObject(in *s3.GetObjectInput) (*s3.GetObjectOutput, error) { + if networkTimeoutFlag { + return nil, fmt.Errorf("network timeout for GetObject()") + } + if m.objects[*in.Key] == nil { return nil, fmt.Errorf("object not found") } @@ -51,8 +55,13 @@ func (m *mockS3Client) GetObject(in *s3.GetObjectInput) (*s3.GetObjectOutput, er return &out, nil } +/* // PutObject adds the object to the map for mock test func (m *mockS3Client) PutObject(in *s3.PutObjectInput) (*s3.PutObjectOutput, error) { + if networkTimeoutFlag { + return nil, fmt.Errorf("network timeout for PutObject()") + } + size, err := in.Body.Seek(0, io.SeekEnd) if err != nil { return nil, fmt.Errorf("failed to seek at the end of body %v", err) @@ -68,6 +77,7 @@ func (m *mockS3Client) PutObject(in *s3.PutObjectInput) (*s3.PutObjectOutput, er out := s3.PutObjectOutput{} return &out, nil } +*/ func (m *mockS3Client) CreateMultipartUploadWithContext(ctx aws.Context, in *s3.CreateMultipartUploadInput, opts ...request.Option) (*s3.CreateMultipartUploadOutput, error) { uploadID := time.Now().String() @@ -81,6 +91,10 @@ func (m *mockS3Client) CreateMultipartUploadWithContext(ctx aws.Context, in *s3. } func (m *mockS3Client) UploadPartWithContext(ctx aws.Context, in *s3.UploadPartInput, opts ...request.Option) (*s3.UploadPartOutput, error) { + if networkTimeoutFlag { + return nil, fmt.Errorf("network timeout for UploadPartWithContext()") + } + if *in.PartNumber < 0 { return nil, fmt.Errorf("part number should be positive integer") } @@ -146,8 +160,13 @@ func (m *mockS3Client) AbortMultipartUploadWithContext(ctx aws.Context, in *s3.A return out, nil } +/* // ListObject returns the objects from map for mock test func (m *mockS3Client) ListObjects(in *s3.ListObjectsInput) (*s3.ListObjectsOutput, error) { + if networkTimeoutFlag { + return nil, fmt.Errorf("network timeout for ListObjects()") + } + var contents []*s3.Object for key := range m.objects { if strings.HasPrefix(key, *in.Prefix) { @@ -165,9 +184,14 @@ func (m *mockS3Client) ListObjects(in *s3.ListObjectsInput) (*s3.ListObjectsOutp } return out, nil } +*/ // ListObject returns the objects from map for mock test func (m *mockS3Client) ListObjectsPages(in *s3.ListObjectsInput, callback func(*s3.ListObjectsOutput, bool) bool) error { + if networkTimeoutFlag { + return fmt.Errorf("network timeout for ListObjectsPages()") + } + var ( count int64 = 0 limit int64 = 1 // aws default is 1000. @@ -216,6 +240,10 @@ func (m *mockS3Client) ListObjectsPages(in *s3.ListObjectsInput, callback func(* // DeleteObject deletes the object from map for mock test func (m *mockS3Client) DeleteObject(in *s3.DeleteObjectInput) (*s3.DeleteObjectOutput, error) { + if networkTimeoutFlag { + return nil, fmt.Errorf("network timeout for DeleteObject()") + } + delete(m.objects, *in.Key) return &s3.DeleteObjectOutput{}, nil } diff --git a/pkg/snapstore/snapstore_suite_test.go b/pkg/snapstore/snapstore_suite_test.go index f3df50fad..04ea474a9 100644 --- a/pkg/snapstore/snapstore_suite_test.go +++ b/pkg/snapstore/snapstore_suite_test.go @@ -27,8 +27,9 @@ import ( ) var ( - testObj *testing.T - swiftStore snapstore.SnapStore + testObj *testing.T + swiftStore snapstore.SnapStore + networkTimeoutFlag bool = false ) func TestSnapstore(t *testing.T) { diff --git a/pkg/snapstore/snapstore_test.go b/pkg/snapstore/snapstore_test.go index 53f430352..6d455fc3b 100644 --- a/pkg/snapstore/snapstore_test.go +++ b/pkg/snapstore/snapstore_test.go @@ -90,10 +90,14 @@ var _ = Describe("Snapstore", func() { } }) + AfterEach(func() { + networkTimeoutFlag = false + }) + Describe("Fetch operation", func() { It("fetches snapshot", func() { for key, snapStore := range snapstores { - logrus.Infof("Running tests for %s", key) + logrus.Infof("Running test for Fetch() for %s", key) resetObjectMap() objectMap[path.Join(prefix, snap1.SnapDir, snap1.SnapName)] = &expectedVal1 objectMap[path.Join(prefix, snap2.SnapDir, snap2.SnapName)] = &expectedVal2 @@ -105,26 +109,43 @@ var _ = Describe("Snapstore", func() { Expect(err).ShouldNot(HaveOccurred()) Expect(buf.Bytes()).To(Equal(expectedVal1)) } + networkTimeoutFlag = true + for key, snapStore := range snapstores { + logrus.Infof("Running negative test for Fetch() for %s", key) + resetObjectMap() + objectMap[path.Join(prefix, snap1.SnapDir, snap1.SnapName)] = &expectedVal1 + objectMap[path.Join(prefix, snap2.SnapDir, snap2.SnapName)] = &expectedVal2 + _, err := snapStore.Fetch(snap1) + Expect(err).Should(HaveOccurred()) + } }) }) Describe("Save snapshot", func() { It("saves snapshot", func() { for key, snapStore := range snapstores { - logrus.Infof("Running tests for %s", key) + logrus.Infof("Running test for Save() for %s", key) resetObjectMap() dummyData := make([]byte, 6*1024*1024) err := snapStore.Save(snap3, bytes.NewReader(dummyData)) Expect(err).ShouldNot(HaveOccurred()) Expect(len(objectMap)).Should(BeNumerically(">", 0)) } + networkTimeoutFlag = true + for key, snapStore := range snapstores { + logrus.Infof("Running negative test for Save() for %s", key) + resetObjectMap() + dummyData := make([]byte, 6*1024*1024) + err := snapStore.Save(snap3, bytes.NewReader(dummyData)) + Expect(err).Should(HaveOccurred()) + } }) }) Describe("List snapshot", func() { It("gives sorted list of snapshot", func() { for key, snapStore := range snapstores { - logrus.Infof("Running tests for %s", key) + logrus.Infof("Running test for List() for %s", key) resetObjectMap() objectMap[path.Join(prefix, snap1.SnapDir, snap1.SnapName)] = &expectedVal1 objectMap[path.Join(prefix, snap2.SnapDir, snap2.SnapName)] = &expectedVal2 @@ -133,13 +154,22 @@ var _ = Describe("Snapstore", func() { Expect(snapList.Len()).To(Equal(2)) Expect(snapList[0].SnapName).To(Equal(snap1.SnapName)) } + networkTimeoutFlag = true + for key, snapStore := range snapstores { + logrus.Infof("Running negative test for List() for %s", key) + resetObjectMap() + objectMap[path.Join(prefix, snap1.SnapDir, snap1.SnapName)] = &expectedVal1 + objectMap[path.Join(prefix, snap2.SnapDir, snap2.SnapName)] = &expectedVal2 + _, err := snapStore.List() + Expect(err).Should(HaveOccurred()) + } }) }) Describe("Delete snapshot", func() { It("deletes snapshot", func() { for key, snapStore := range snapstores { - logrus.Infof("Running tests for %s", key) + logrus.Infof("Running test for Delete() for %s", key) resetObjectMap() objectMap[path.Join(prefix, snap1.SnapDir, snap1.SnapName)] = &expectedVal1 objectMap[path.Join(prefix, snap2.SnapDir, snap2.SnapName)] = &expectedVal2 @@ -148,6 +178,15 @@ var _ = Describe("Snapstore", func() { Expect(err).ShouldNot(HaveOccurred()) Expect(len(objectMap)).To(Equal(prevLen - 1)) } + networkTimeoutFlag = true + for key, snapStore := range snapstores { + logrus.Infof("Running negative test for Delete() for %s", key) + resetObjectMap() + objectMap[path.Join(prefix, snap1.SnapDir, snap1.SnapName)] = &expectedVal1 + objectMap[path.Join(prefix, snap2.SnapDir, snap2.SnapName)] = &expectedVal2 + err := snapStore.Delete(snap2) + Expect(err).Should(HaveOccurred()) + } }) }) }) diff --git a/pkg/snapstore/swift_snapstore_test.go b/pkg/snapstore/swift_snapstore_test.go index fa21fe07e..3b0566469 100644 --- a/pkg/snapstore/swift_snapstore_test.go +++ b/pkg/snapstore/swift_snapstore_test.go @@ -61,6 +61,11 @@ func initializeMockSwiftServer(t *testing.T) { // handleCreateTextObject creates an HTTP handler at `/testContainer/testObject` on the test handler mux // that responds with a `Create` response. func handleCreateTextObject(w http.ResponseWriter, r *http.Request) { + if networkTimeoutFlag { + w.WriteHeader(http.StatusRequestTimeout) + return + } + var ( content []byte err error @@ -94,6 +99,11 @@ func handleCreateTextObject(w http.ResponseWriter, r *http.Request) { // handleDownloadObject creates an HTTP handler at `/testContainer/testObject` on the test handler mux that // responds with a `Download` response. func handleDownloadObject(w http.ResponseWriter, r *http.Request) { + if networkTimeoutFlag { + w.WriteHeader(http.StatusRequestTimeout) + return + } + prefix := parseObjectNamefromURL(r.URL) var contents []byte for key, val := range objectMap { @@ -109,6 +119,11 @@ func handleDownloadObject(w http.ResponseWriter, r *http.Request) { // handleListObjectNames creates an HTTP handler at `/testContainer` on the test handler mux that // responds with a `List` response when only object names are requested. func handleListObjectNames(w http.ResponseWriter, r *http.Request) { + if networkTimeoutFlag { + w.WriteHeader(http.StatusRequestTimeout) + return + } + marker := r.URL.Query().Get("marker") // To store the keys in slice in sorted order @@ -132,6 +147,11 @@ func handleListObjectNames(w http.ResponseWriter, r *http.Request) { // handleDeleteObject creates an HTTP handler at `/testContainer/testObject` on the test handler mux that // responds with a `Delete` response. func handleDeleteObject(w http.ResponseWriter, r *http.Request) { + if networkTimeoutFlag { + w.WriteHeader(http.StatusRequestTimeout) + return + } + key := parseObjectNamefromURL(r.URL) if _, ok := objectMap[key]; ok { delete(objectMap, key)