BDD for restore of remote backup in different namespace

Signed-off-by: mayank <[email protected]>

.travis.yml

@@ -42,7 +42,7 @@ script:
       kubectl cluster-info;
       export VELERO_RELEASE=v1.0.0;
       export OPENEBS_RELEASE=master;
-      ./script/install-openebs.sh && ./script/install-velero.sh && travis_wait make test || travis_terminate 1;
+      ./script/install-openebs.sh && ./script/install-velero.sh && travis_wait 30 make test || travis_terminate 1;
     fi
 
 after_success:

pkg/clouduploader/operation.go

@@ -48,7 +48,7 @@ func (c *Conn) Upload(file string, fileSize int64) bool {
 	if err != nil {
 		c.Log.Errorf("Failed to upload snapshot to bucket: %s", err.Error())
 		if c.bucket.Delete(c.ctx, file) != nil {
-			c.Log.Errorf("Failed to remove snapshot{%s} from cloud", file)
+			c.Log.Errorf("Failed to delete uncompleted snapshot{%s} from cloud", file)
 		}
 		return false
 	}

pkg/clouduploader/server.go

@@ -134,19 +134,23 @@ func (s *Server) acceptClient(fd, epfd int) (int, error) {
 		return (-1), err
 	}
 
+	readerWriter := s.cl.Create(s.OpType)
+	if readerWriter == nil {
+		s.Log.Errorf("Failed to create file interface")
+		if err = syscall.Close(connFd); err != nil {
+			s.Log.Warnf("Failed to close cline {%v} : %s", connFd, err.Error())
+		}
+		return (-1), errors.New("failed to create file interface")
+	}
+
 	c = new(Client)
 	c.fd = connFd
-	c.file = s.cl.Create(s.OpType)
+	c.file = readerWriter
 	c.bufferLen = ReadBufferLen
 	c.buffer = make([]byte, c.bufferLen)
 	c.status = TransferStatusInit
 	c.next = nil
 
-	if c.file == nil {
-		s.Log.Errorf("Failed to create file interface")
-		panic(errors.New("failed to create file interface"))
-	}
-
 	event = new(syscall.EpollEvent)
 	if s.OpType == OpBackup {
 		event.Events = syscall.EPOLLIN | syscall.EPOLLRDHUP | syscall.EPOLLHUP | syscall.EPOLLERR | EPOLLET
@@ -276,6 +280,11 @@ func (s *Server) Run(opType ServerOperation) error {
 	for {
 		nevents, err := syscall.EpollWait(epfd, events[:], EPOLLTIMEOUT)
 		if err != nil {
+			if isEINTR(err) {
+				s.Log.Warningf("Epoll wait failed : %s", err.Error())
+				continue
+			}
+
 			s.Log.Errorf("Epoll wait failed : %s", err.Error())
 			return err
 		}

pkg/clouduploader/server_utils.go

@@ -40,16 +40,18 @@ func (s *Server) removeFromClientList(c *Client) {
 	var prevClient *Client
 
 	if s.FirstClient == nil || s.state.runningCount == 0 {
-		s.Log.Errorf("ClientList is empty")
-		panic(errors.New("clientList list is empty"))
+		// epoll may have returned multiple event for the same fd
+		s.Log.Warningf("ClientList is empty")
+		return
 	} else if s.FirstClient == c {
 		s.FirstClient = c.next
 	} else {
 		curClient := s.FirstClient
 		for curClient != c {
 			if curClient.next == nil {
-				s.Log.Errorf("entry{%v} not found in ClientList", c.fd)
-				panic(errors.Errorf("entry{%v} not found in ClientList", c.fd))
+				// epoll may have returned multiple event for the same fd
+				s.Log.Warningf("entry{%v} not found in ClientList", c.fd)
+				return
 			}
 
 			prevClient = curClient
@@ -193,3 +195,16 @@ func (s *Server) disconnectAllClient(efd int) {
 		curClient = nextClient
 	}
 }
+
+// isEINTR check if given error is generated because of EINTR
+func isEINTR(err error) bool {
+	if err == nil {
+		return false
+	}
+
+	errno, ok := err.(syscall.Errno)
+	if ok && errno == syscall.EINTR {
+		return true
+	}
+	return false
+}

pkg/cstor/cvr_operation.go

@@ -58,7 +58,8 @@ func (p *Plugin) waitForAllCVR(vol *Volume) error {
 		for _, cvr := range cvrList.Items {
 			if cvr.Status.Phase == v1alpha1.CVRStatusOnline ||
 				cvr.Status.Phase == v1alpha1.CVRStatusError ||
-				cvr.Status.Phase == v1alpha1.CVRStatusDegraded {
+				cvr.Status.Phase == v1alpha1.CVRStatusDegraded ||
+				cvr.Status.Phase == v1alpha1.CVRStatusOffline {
 				cvrCount++
 			}
 		}

pkg/cstor/pv_operation.go

@@ -112,6 +112,7 @@ func (p *Plugin) getVolumeForLocalRestore(volumeID, snapName string) (*Volume, e
 func (p *Plugin) getVolumeForRemoteRestore(volumeID, snapName string) (*Volume, error) {
 	vol, err := p.createPVC(volumeID, snapName)
 	if err != nil {
+		p.Log.Errorf("CreatePVC returned error=%s", err)
 		return nil, err
 	}
 

pkg/cstor/pvc_operation.go

@@ -114,7 +114,7 @@ func (p *Plugin) createPVC(volumeID, snapName string) (*Volume, error) {
 		return newVol, nil
 	}
 
-	p.Log.Infof("Creating PVC for volumeID:%s snapshot:%s", volumeID, snapName)
+	p.Log.Infof("Creating PVC for volumeID:%s snapshot:%s in namespace=%s", volumeID, snapName, targetedNs)
 
 	pvc.Annotations = make(map[string]string)
 	pvc.Annotations["openebs.io/created-through"] = "restore"

pkg/cstor/status.go

@@ -21,7 +21,6 @@ import (
 	"time"
 
 	"github.com/openebs/maya/pkg/apis/openebs.io/v1alpha1"
-	"github.com/pkg/errors"
 )
 
 // checkBackupStatus queries MayaAPI server for given backup status
@@ -34,13 +33,17 @@ func (p *Plugin) checkBackupStatus(bkp *v1alpha1.CStorBackup) {
 	bkpvolume, exists := p.volumes[bkp.Spec.VolumeName]
 	if !exists {
 		p.Log.Errorf("Failed to fetch volume info for {%s}", bkp.Spec.VolumeName)
-		panic(errors.Errorf("Failed to fetch volume info for {%s}", bkp.Spec.VolumeName))
+		p.cl.ExitServer = true
+		bkpvolume.backupStatus = v1alpha1.BKPCStorStatusInvalid
+		return
 	}
 
 	bkpData, err := json.Marshal(bkp)
 	if err != nil {
 		p.Log.Errorf("JSON marshal failed : %s", err.Error())
-		panic(errors.Errorf("JSON marshal failed : %s", err.Error()))
+		p.cl.ExitServer = true
+		bkpvolume.backupStatus = v1alpha1.BKPCStorStatusInvalid
+		return
 	}
 
 	for !bkpDone {
@@ -82,7 +85,8 @@ func (p *Plugin) checkRestoreStatus(rst *v1alpha1.CStorRestore, vol *Volume) {
 	rstData, err := json.Marshal(rst)
 	if err != nil {
 		p.Log.Errorf("JSON marshal failed : %s", err.Error())
-		panic(errors.Errorf("JSON marshal failed : %s", err.Error()))
+		vol.restoreStatus = v1alpha1.RSTCStorStatusInvalid
+		p.cl.ExitServer = true
 	}
 
 	for !rstDone {

tests/openebs/logs.go

@@ -17,6 +17,8 @@ limitations under the License.
 package openebs
 
 import (
+	"fmt"
+
 	k8s "github.com/openebs/velero-plugin/tests/k8s"
 	corev1 "k8s.io/api/core/v1"
 )
@@ -28,22 +30,26 @@ const (
 )
 
 // DumpLogs will dump openebs logs
-func (c *ClientSet) DumpLogs() error {
+func (c *ClientSet) DumpLogs() {
 	mayaPod := c.getMayaAPIServerPodName()
 	spcPod := c.getSPCPodName()
 	pvcPod := c.getPVCPodName()
 
 	for _, v := range mayaPod {
-		_ = k8s.Client.DumpLogs(OpenEBSNs, v[0], v[1])
+		if err := k8s.Client.DumpLogs(OpenEBSNs, v[0], v[1]); err != nil {
+			fmt.Printf("Failed to dump maya-apiserver logs err=%s\n", err)
+		}
 	}
 	for _, v := range spcPod {
-		_ = k8s.Client.DumpLogs(OpenEBSNs, v[0], v[1])
+		if err := k8s.Client.DumpLogs(OpenEBSNs, v[0], v[1]); err != nil {
+			fmt.Printf("Failed to dump cstor pod logs err=%s\n", err)
+		}
 	}
 	for _, v := range pvcPod {
-		_ = k8s.Client.DumpLogs(OpenEBSNs, v[0], v[1])
+		if err := k8s.Client.DumpLogs(OpenEBSNs, v[0], v[1]); err != nil {
+			fmt.Printf("Failed to dump target pod logs err=%s\n", err)
+		}
 	}
-
-	return nil
 }
 
 // getMayaAPIServerPodName return Maya-API server pod name and container

tests/openebs/storage_install.go

@@ -51,8 +51,8 @@ const (
 	// OpenEBSNs openebs Namespace
 	OpenEBSNs = "openebs"
 
-	// PVCDeploymentLabel for target pod Deployment
-	PVCDeploymentLabel = "openebs.io/persistent-volume-claim"
+	// PVDeploymentLabel for target pod Deployment
+	PVDeploymentLabel = "openebs.io/persistent-volume"
 )
 
 func init() {
@@ -123,12 +123,17 @@ func (c *ClientSet) DeleteVolume(pvcYAML, pvcNs string) error {
 		return err
 	}
 
+	pv, err := c.getPVCVolumeName(pvc.Name, pvcNs)
+	if err != nil {
+		return err
+	}
+
 	pvc.Namespace = pvcNs
 	if err := k8s.Client.DeletePVC(pvc); err != nil {
 		return err
 	}
 
 	return k8s.Client.WaitForDeploymentCleanup(
-		PVCDeploymentLabel+"="+pvc.Name,
+		PVDeploymentLabel+"="+pv,
 		OpenEBSNs)
 }