Added negative tests for restore of snapshots

.ci/unit_test

@@ -39,8 +39,8 @@ function test_with_coverage() {
   local output_dir=test/output
   local coverprofile_file=coverprofile.out
   mkdir -p test/output
-  ginkgo $GINKGO_COMMON_FLAGS --coverprofile ${coverprofile_file} -covermode=set -outputdir ${output_dir} ${TEST_PACKAGES}
-  sed -i '/mode: set/d' ${output_dir}/${coverprofile_file}
+  ginkgo $GINKGO_COMMON_FLAGS  --coverprofile ${coverprofile_file} -covermode=set -outputdir ${output_dir} ${TEST_PACKAGES}
+  sed -i='' '/mode: set/d' ${output_dir}/${coverprofile_file}
   {( echo "mode: set"; cat ${output_dir}/${coverprofile_file} )} > ${output_dir}/${coverprofile_file}.temp
   mv ${output_dir}/${coverprofile_file}.temp ${output_dir}/${coverprofile_file}
   go tool cover -func ${output_dir}/${coverprofile_file}
@@ -49,10 +49,18 @@ function test_with_coverage() {
 ################################################################################
 
 TEST_PACKAGES="cmd pkg"
-GINKGO_COMMON_FLAGS="-r -timeout=1h0m0s --randomizeAllSpecs --randomizeSuites --failOnPending --progress"
+GINKGO_COMMON_FLAGS="-r -timeout=1h0m0s --progress"
+
 if [ -z $COVER ] || [ "$COVER" = false ] ; then
   echo "[INFO] Test coverage is disabled."
-  ginkgo -race -trace $GINKGO_COMMON_FLAGS ${TEST_PACKAGES}
+
+  # run everything which is not part of negative scenarios with randomizeAllSpecs parameters.
+  ginkgo -race -trace $GINKGO_COMMON_FLAGS --randomizeAllSpecs --randomizeSuites --failOnPending --skip="NEGATIVE\:.*" ${TEST_PACKAGES} 
+
+
+  #run negative scenario in a sequenced manner (removed failOnPending as one spec in restore test is marked as 'X' for excluding) 
+  ginkgo -race -trace $GINKGO_COMMON_FLAGS --focus="NEGATIVE\:.*" ${TEST_PACKAGES} 
+
 else
   test_with_coverage
 fi

pkg/miscellaneous/miscellaneous.go

@@ -38,5 +38,6 @@ func GetLatestFullSnapshotAndDeltaSnapList(store snapstore.SnapStore) (*snapstor
 		}
 		deltaSnapList = append(deltaSnapList, snapList[index-1])
 	}
+	sort.Sort(deltaSnapList) //added to ensure the list is well formed for only deltasnapshots scenarios as well
 	return nil, deltaSnapList, nil
 }

pkg/snapshot/restorer/restorer.go

@@ -67,6 +67,7 @@ func (r *Restorer) Restore(ro RestoreOptions) error {
 		r.logger.Infof("Quota size for etcd must be greater than 0. Input EmbeddedEtcdQuotaBytes: %d. Defaulting to 8GB.", ro.EmbeddedEtcdQuotaBytes)
 		ro.EmbeddedEtcdQuotaBytes = int64(8 * 1024 * 1024 * 1024)
 	}
+	ro.RestoreDataDir = path.Clean(ro.RestoreDataDir)
 	if err := r.restoreFromBaseSnapshot(ro); err != nil {
 		return fmt.Errorf("failed to restore from the base snapshot :%v", err)
 	}

pkg/snapshot/restorer/restorer_suite_test.go

@@ -17,7 +17,9 @@ package restorer_test
 import (
 	"context"
 	"fmt"
+	"math"
 	"os"
+	"path"
 	"strconv"
 	"sync"
 	"testing"
@@ -38,7 +40,7 @@ const (
 	etcdDir                    = outputDir + "/default.etcd"
 	snapstoreDir               = outputDir + "/snapshotter.bkp"
 	etcdEndpoint               = "http://localhost:2379"
-	snapshotterDurationSeconds = 100
+	snapshotterDurationSeconds = 20
 	keyPrefix                  = "key-"
 	valuePrefix                = "val-"
 	keyFrom                    = 1
@@ -72,7 +74,7 @@ var _ = SynchronizedBeforeSuite(func() []byte {
 	etcd, err = startEmbeddedEtcd(etcdDir, logger)
 	Expect(err).ShouldNot(HaveOccurred())
 	wg := &sync.WaitGroup{}
-	deltaSnapshotPeriod := 5
+	deltaSnapshotPeriod := 1
 	wg.Add(1)
 	go populateEtcd(wg, logger, endpoints, errCh, populatorStopCh)
 	go func() {
@@ -83,21 +85,39 @@ var _ = SynchronizedBeforeSuite(func() []byte {
 		close(ssrStopCh)
 	}()
 
-	err = runSnapshotter(logger, deltaSnapshotPeriod, endpoints, ssrStopCh)
+	err = runSnapshotter(logger, deltaSnapshotPeriod, endpoints, ssrStopCh, true)
 	Expect(err).ShouldNot(HaveOccurred())
 
 	etcd.Server.Stop()
 	etcd.Close()
 	return data
+
 }, func(data []byte) {})
 
-var _ = SynchronizedAfterSuite(func() {}, func() {
+var _ = SynchronizedAfterSuite(func() {}, cleanUp)
+
+func cleanUp() {
 	err = os.RemoveAll(etcdDir)
 	Expect(err).ShouldNot(HaveOccurred())
 
 	err = os.RemoveAll(snapstoreDir)
 	Expect(err).ShouldNot(HaveOccurred())
-})
+
+	//for the negative scenario for invalid restoredir set to "" we need to cleanup the member folder in the working directory
+	restoreDir := path.Clean("")
+	info, err := os.Stat(fmt.Sprintf("%s/%s", restoreDir, "member"))
+	if err != nil {
+		if os.IsNotExist(err) {
+			return
+		} else {
+			Expect(err).ShouldNot(HaveOccurred())
+		}
+	}
+	if info.IsDir() {
+		err = os.RemoveAll(info.Name())
+		Expect(err).ShouldNot(HaveOccurred())
+	}
+}
 
 // startEmbeddedEtcd starts an embedded etcd server
 func startEmbeddedEtcd(dir string, logger *logrus.Logger) (*embed.Etcd, error) {
@@ -124,7 +144,7 @@ func startEmbeddedEtcd(dir string, logger *logrus.Logger) (*embed.Etcd, error) {
 }
 
 // runSnapshotter creates a snapshotter object and runs it for a duration specified by 'snapshotterDurationSeconds'
-func runSnapshotter(logger *logrus.Logger, deltaSnapshotPeriod int, endpoints []string, stopCh chan struct{}) error {
+func runSnapshotter(logger *logrus.Logger, deltaSnapshotPeriod int, endpoints []string, stopCh chan struct{}, startWithFullSnapshot bool) error {
 	var (
 		store                          snapstore.SnapStore
 		certFile                       string
@@ -177,7 +197,7 @@ func runSnapshotter(logger *logrus.Logger, deltaSnapshotPeriod int, endpoints []
 		snapshotterConfig,
 	)
 
-	return ssr.Run(stopCh, true)
+	return ssr.Run(stopCh, startWithFullSnapshot)
 }
 
 // populateEtcd sequentially puts key-value pairs into the embedded etcd, until stopped
@@ -218,6 +238,19 @@ func populateEtcd(wg *sync.WaitGroup, logger *logrus.Logger, endpoints []string,
 				return
 			}
 			time.Sleep(time.Second * 1)
+			//call a delete for every 10th Key after putting it in the store to check deletes in consistency check
+			// handles deleted keys as every 10th key is deleted during populate etcd call
+			// this handling is also done in the checkDataConsistency() in restorer_test.go file
+			// also it assumes that the deltaSnapshotDuration is more than 10 --
+			// if you change the constant please change the factor accordingly to have coverage of delete scenarios.
+			if math.Mod(float64(currKey), 10) == 0 {
+				_, err = cli.Delete(context.TODO(), key)
+				if err != nil {
+					errCh <- fmt.Errorf("unable to delete key  (%s) from embedded etcd: %v", key, err)
+					return
+				}
+			}
+
 		}
 	}
 }