[WIP] Negative tests for network interruption

pkg/snapshot/snapshotter/snapshotter.go

@@ -171,7 +171,7 @@ func (ssr *Snapshotter) TakeFullSnapshotAndResetTimer() error {
 	} else {
 		ssr.logger.Infof("Stopping full snapshot...")
 		ssr.fullSnapshotTimer.Stop()
-		ssr.logger.Infof("Reseting full snapshot to run after %d secs.", effective.Sub(now))
+		ssr.logger.Infof("Reseting full snapshot to run after %f secs.", effective.Sub(now).Seconds())
 		ssr.fullSnapshotTimer.Reset(effective.Sub(now))
 	}
 	ssr.logger.Infof("Will take next full snapshot at time: %s", effective)

pkg/snapstore/abs_snapstore_test.go

@@ -136,6 +136,12 @@ func (p *fakePolicy) handleContainerGetOperation(w *http.Response) error {
 
 // handleListObjectNames responds with a blob `List` response.
 func (p *fakePolicy) handleListObjects(w *http.Response) error {
+	if networkTimeoutFlag {
+		w.StatusCode = http.StatusRequestTimeout
+		w.Body = http.NoBody
+		return nil
+	}
+
 	var (
 		keys          []string
 		blobs         []blobItem
@@ -195,6 +201,12 @@ func (p *fakePolicy) handleListObjects(w *http.Response) error {
 
 // handleBlobCreateOperation responds with a blob `Put` response.
 func (p *fakePolicy) handleBlobPutOperation(w *http.Response) {
+	if networkTimeoutFlag {
+		w.StatusCode = http.StatusRequestTimeout
+		w.Body = http.NoBody
+		return
+	}
+
 	var (
 		query   = w.Request.URL.Query()
 		comp    = query.Get("comp")
@@ -248,6 +260,12 @@ func (p *fakePolicy) handleBlobPutOperation(w *http.Response) {
 
 // handleBlobGetOperation on GET request `/testContainer/testObject` responds with a `Get` response.
 func (p *fakePolicy) handleBlobGetOperation(w *http.Response) {
+	if networkTimeoutFlag {
+		w.StatusCode = http.StatusRequestTimeout
+		w.Body = http.NoBody
+		return
+	}
+
 	key := parseObjectNamefromURL(w.Request.URL)
 	if _, ok := p.objectMap[key]; ok {
 		w.StatusCode = http.StatusOK
@@ -260,6 +278,12 @@ func (p *fakePolicy) handleBlobGetOperation(w *http.Response) {
 
 // handleDeleteObject on delete request `/testContainer/testObject` responds with a `Delete` response.
 func (p *fakePolicy) handleDeleteObject(w *http.Response) {
+	if networkTimeoutFlag {
+		w.StatusCode = http.StatusRequestTimeout
+		w.Body = http.NoBody
+		return
+	}
+
 	key := parseObjectNamefromURL(w.Request.URL)
 	if _, ok := p.objectMap[key]; ok {
 		delete(p.objectMap, key)

pkg/snapstore/gcs_snapstore.go

@@ -167,6 +167,9 @@ func (s *GCSSnapStore) uploadComponent(snap *Snapshot, file *os.File, offset, ch
 	ctx, cancel := context.WithTimeout(context.TODO(), chunkUploadTimeout)
 	defer cancel()
 	w := obj.NewWriter(ctx)
+	if w == nil {
+		return fmt.Errorf("failed to create writer for snapshot")
+	}
 	if _, err := io.Copy(w, sr); err != nil {
 		w.Close()
 		return err
@@ -197,6 +200,9 @@ func (s *GCSSnapStore) componentUploader(wg *sync.WaitGroup, stopCh <-chan struc
 // List will list the snapshots from store.
 func (s *GCSSnapStore) List() (SnapList, error) {
 	it := s.client.Bucket(s.bucket).Objects(context.TODO(), &storage.Query{Prefix: s.prefix})
+	if it == nil {
+		return nil, fmt.Errorf("failed listing objects")
+	}
 
 	var attrs []*storage.ObjectAttrs
 	for {

pkg/snapstore/gcs_snapstore_test.go

@@ -51,6 +51,10 @@ func (m *mockBucketHandle) Object(name string) stiface.ObjectHandle {
 }
 
 func (m *mockBucketHandle) Objects(context.Context, *storage.Query) stiface.ObjectIterator {
+	if networkTimeoutFlag {
+		return nil
+	}
+
 	var keys []string
 	for key, _ := range m.client.objects {
 		keys = append(keys, key)
@@ -66,13 +70,20 @@ type mockObjectHandle struct {
 }
 
 func (m *mockObjectHandle) NewReader(ctx context.Context) (stiface.Reader, error) {
+	if networkTimeoutFlag {
+		return nil, fmt.Errorf("network timeout for NewReader()")
+	}
+
 	if value, ok := m.client.objects[m.object]; ok {
 		return &mockObjectReader{reader: ioutil.NopCloser(bytes.NewReader(*value))}, nil
 	}
 	return nil, fmt.Errorf("object %s not found", m.object)
 }
 
 func (m *mockObjectHandle) NewWriter(context.Context) stiface.Writer {
+	if networkTimeoutFlag {
+		return nil
+	}
 	return &mockObjectWriter{object: m.object, client: m.client}
 }
 
@@ -85,6 +96,10 @@ func (m *mockObjectHandle) ComposerFrom(objects ...stiface.ObjectHandle) stiface
 }
 
 func (m *mockObjectHandle) Delete(context.Context) error {
+	if networkTimeoutFlag {
+		return fmt.Errorf("network timeout for Delete()")
+	}
+
 	if _, ok := m.client.objects[m.object]; ok {
 		delete(m.client.objects, m.object)
 		return nil

pkg/snapstore/s3_snapstore_test.go

@@ -41,6 +41,10 @@ type mockS3Client struct {
 
 // GetObject returns the object from map for mock test
 func (m *mockS3Client) GetObject(in *s3.GetObjectInput) (*s3.GetObjectOutput, error) {
+	if networkTimeoutFlag {
+		return nil, fmt.Errorf("network timeout for GetObject()")
+	}
+
 	if m.objects[*in.Key] == nil {
 		return nil, fmt.Errorf("object not found")
 	}
@@ -51,8 +55,13 @@ func (m *mockS3Client) GetObject(in *s3.GetObjectInput) (*s3.GetObjectOutput, er
 	return &out, nil
 }
 
+/*
 // PutObject adds the object to the map for mock test
 func (m *mockS3Client) PutObject(in *s3.PutObjectInput) (*s3.PutObjectOutput, error) {
+	if networkTimeoutFlag {
+		return nil, fmt.Errorf("network timeout for PutObject()")
+	}
+
 	size, err := in.Body.Seek(0, io.SeekEnd)
 	if err != nil {
 		return nil, fmt.Errorf("failed to seek at the end of body %v", err)
@@ -68,6 +77,7 @@ func (m *mockS3Client) PutObject(in *s3.PutObjectInput) (*s3.PutObjectOutput, er
 	out := s3.PutObjectOutput{}
 	return &out, nil
 }
+*/
 
 func (m *mockS3Client) CreateMultipartUploadWithContext(ctx aws.Context, in *s3.CreateMultipartUploadInput, opts ...request.Option) (*s3.CreateMultipartUploadOutput, error) {
 	uploadID := time.Now().String()
@@ -81,6 +91,10 @@ func (m *mockS3Client) CreateMultipartUploadWithContext(ctx aws.Context, in *s3.
 }
 
 func (m *mockS3Client) UploadPartWithContext(ctx aws.Context, in *s3.UploadPartInput, opts ...request.Option) (*s3.UploadPartOutput, error) {
+	if networkTimeoutFlag {
+		return nil, fmt.Errorf("network timeout for UploadPartWithContext()")
+	}
+
 	if *in.PartNumber < 0 {
 		return nil, fmt.Errorf("part number should be positive integer")
 	}
@@ -146,8 +160,13 @@ func (m *mockS3Client) AbortMultipartUploadWithContext(ctx aws.Context, in *s3.A
 	return out, nil
 }
 
+/*
 // ListObject returns the objects from map for mock test
 func (m *mockS3Client) ListObjects(in *s3.ListObjectsInput) (*s3.ListObjectsOutput, error) {
+	if networkTimeoutFlag {
+		return nil, fmt.Errorf("network timeout for ListObjects()")
+	}
+
 	var contents []*s3.Object
 	for key := range m.objects {
 		if strings.HasPrefix(key, *in.Prefix) {
@@ -165,9 +184,14 @@ func (m *mockS3Client) ListObjects(in *s3.ListObjectsInput) (*s3.ListObjectsOutp
 	}
 	return out, nil
 }
+*/
 
 // ListObject returns the objects from map for mock test
 func (m *mockS3Client) ListObjectsPages(in *s3.ListObjectsInput, callback func(*s3.ListObjectsOutput, bool) bool) error {
+	if networkTimeoutFlag {
+		return fmt.Errorf("network timeout for ListObjectsPages()")
+	}
+
 	var (
 		count    int64 = 0
 		limit    int64 = 1 // aws default is 1000.
@@ -216,6 +240,10 @@ func (m *mockS3Client) ListObjectsPages(in *s3.ListObjectsInput, callback func(*
 
 // DeleteObject deletes the object from map for mock test
 func (m *mockS3Client) DeleteObject(in *s3.DeleteObjectInput) (*s3.DeleteObjectOutput, error) {
+	if networkTimeoutFlag {
+		return nil, fmt.Errorf("network timeout for DeleteObject()")
+	}
+
 	delete(m.objects, *in.Key)
 	return &s3.DeleteObjectOutput{}, nil
 }

pkg/snapstore/snapstore_suite_test.go

@@ -27,8 +27,9 @@ import (
 )
 
 var (
-	testObj    *testing.T
-	swiftStore snapstore.SnapStore
+	testObj            *testing.T
+	swiftStore         snapstore.SnapStore
+	networkTimeoutFlag bool = false
 )
 
 func TestSnapstore(t *testing.T) {

pkg/snapstore/snapstore_test.go

@@ -90,10 +90,14 @@ var _ = Describe("Snapstore", func() {
 		}
 	})
 
+	AfterEach(func() {
+		networkTimeoutFlag = false
+	})
+
 	Describe("Fetch operation", func() {
 		It("fetches snapshot", func() {
 			for key, snapStore := range snapstores {
-				logrus.Infof("Running tests for %s", key)
+				logrus.Infof("Running test for Fetch() for %s", key)
 				resetObjectMap()
 				objectMap[path.Join(prefix, snap1.SnapDir, snap1.SnapName)] = &expectedVal1
 				objectMap[path.Join(prefix, snap2.SnapDir, snap2.SnapName)] = &expectedVal2
@@ -105,26 +109,43 @@ var _ = Describe("Snapstore", func() {
 				Expect(err).ShouldNot(HaveOccurred())
 				Expect(buf.Bytes()).To(Equal(expectedVal1))
 			}
+			networkTimeoutFlag = true
+			for key, snapStore := range snapstores {
+				logrus.Infof("Running negative test for Fetch() for %s", key)
+				resetObjectMap()
+				objectMap[path.Join(prefix, snap1.SnapDir, snap1.SnapName)] = &expectedVal1
+				objectMap[path.Join(prefix, snap2.SnapDir, snap2.SnapName)] = &expectedVal2
+				_, err := snapStore.Fetch(snap1)
+				Expect(err).Should(HaveOccurred())
+			}
 		})
 	})
 
 	Describe("Save snapshot", func() {
 		It("saves snapshot", func() {
 			for key, snapStore := range snapstores {
-				logrus.Infof("Running tests for %s", key)
+				logrus.Infof("Running test for Save() for %s", key)
 				resetObjectMap()
 				dummyData := make([]byte, 6*1024*1024)
 				err := snapStore.Save(snap3, bytes.NewReader(dummyData))
 				Expect(err).ShouldNot(HaveOccurred())
 				Expect(len(objectMap)).Should(BeNumerically(">", 0))
 			}
+			networkTimeoutFlag = true
+			for key, snapStore := range snapstores {
+				logrus.Infof("Running negative test for Save() for %s", key)
+				resetObjectMap()
+				dummyData := make([]byte, 6*1024*1024)
+				err := snapStore.Save(snap3, bytes.NewReader(dummyData))
+				Expect(err).Should(HaveOccurred())
+			}
 		})
 	})
 
 	Describe("List snapshot", func() {
 		It("gives sorted list of snapshot", func() {
 			for key, snapStore := range snapstores {
-				logrus.Infof("Running tests for %s", key)
+				logrus.Infof("Running test for List() for %s", key)
 				resetObjectMap()
 				objectMap[path.Join(prefix, snap1.SnapDir, snap1.SnapName)] = &expectedVal1
 				objectMap[path.Join(prefix, snap2.SnapDir, snap2.SnapName)] = &expectedVal2
@@ -133,13 +154,22 @@ var _ = Describe("Snapstore", func() {
 				Expect(snapList.Len()).To(Equal(2))
 				Expect(snapList[0].SnapName).To(Equal(snap1.SnapName))
 			}
+			networkTimeoutFlag = true
+			for key, snapStore := range snapstores {
+				logrus.Infof("Running negative test for List() for %s", key)
+				resetObjectMap()
+				objectMap[path.Join(prefix, snap1.SnapDir, snap1.SnapName)] = &expectedVal1
+				objectMap[path.Join(prefix, snap2.SnapDir, snap2.SnapName)] = &expectedVal2
+				_, err := snapStore.List()
+				Expect(err).Should(HaveOccurred())
+			}
 		})
 	})
 
 	Describe("Delete snapshot", func() {
 		It("deletes snapshot", func() {
 			for key, snapStore := range snapstores {
-				logrus.Infof("Running tests for %s", key)
+				logrus.Infof("Running test for Delete() for %s", key)
 				resetObjectMap()
 				objectMap[path.Join(prefix, snap1.SnapDir, snap1.SnapName)] = &expectedVal1
 				objectMap[path.Join(prefix, snap2.SnapDir, snap2.SnapName)] = &expectedVal2
@@ -148,6 +178,15 @@ var _ = Describe("Snapstore", func() {
 				Expect(err).ShouldNot(HaveOccurred())
 				Expect(len(objectMap)).To(Equal(prevLen - 1))
 			}
+			networkTimeoutFlag = true
+			for key, snapStore := range snapstores {
+				logrus.Infof("Running negative test for Delete() for %s", key)
+				resetObjectMap()
+				objectMap[path.Join(prefix, snap1.SnapDir, snap1.SnapName)] = &expectedVal1
+				objectMap[path.Join(prefix, snap2.SnapDir, snap2.SnapName)] = &expectedVal2
+				err := snapStore.Delete(snap2)
+				Expect(err).Should(HaveOccurred())
+			}
 		})
 	})
 })

pkg/snapstore/swift_snapstore_test.go

@@ -61,6 +61,11 @@ func initializeMockSwiftServer(t *testing.T) {
 // handleCreateTextObject creates an HTTP handler at `/testContainer/testObject` on the test handler mux
 // that responds with a `Create` response.
 func handleCreateTextObject(w http.ResponseWriter, r *http.Request) {
+	if networkTimeoutFlag {
+		w.WriteHeader(http.StatusRequestTimeout)
+		return
+	}
+
 	var (
 		content []byte
 		err     error
@@ -94,6 +99,11 @@ func handleCreateTextObject(w http.ResponseWriter, r *http.Request) {
 // handleDownloadObject creates an HTTP handler at `/testContainer/testObject` on the test handler mux that
 // responds with a `Download` response.
 func handleDownloadObject(w http.ResponseWriter, r *http.Request) {
+	if networkTimeoutFlag {
+		w.WriteHeader(http.StatusRequestTimeout)
+		return
+	}
+
 	prefix := parseObjectNamefromURL(r.URL)
 	var contents []byte
 	for key, val := range objectMap {
@@ -109,6 +119,11 @@ func handleDownloadObject(w http.ResponseWriter, r *http.Request) {
 // handleListObjectNames creates an HTTP handler at `/testContainer` on the test handler mux that
 // responds with a `List` response when only object names are requested.
 func handleListObjectNames(w http.ResponseWriter, r *http.Request) {
+	if networkTimeoutFlag {
+		w.WriteHeader(http.StatusRequestTimeout)
+		return
+	}
+
 	marker := r.URL.Query().Get("marker")
 
 	// To store the keys in slice in sorted order
@@ -132,6 +147,11 @@ func handleListObjectNames(w http.ResponseWriter, r *http.Request) {
 // handleDeleteObject creates an HTTP handler at `/testContainer/testObject` on the test handler mux that
 // responds with a `Delete` response.
 func handleDeleteObject(w http.ResponseWriter, r *http.Request) {
+	if networkTimeoutFlag {
+		w.WriteHeader(http.StatusRequestTimeout)
+		return
+	}
+
 	key := parseObjectNamefromURL(r.URL)
 	if _, ok := objectMap[key]; ok {
 		delete(objectMap, key)