feat(remote-restore): enabling restore in different namespace

.travis.yml

@@ -42,7 +42,7 @@ script:
       kubectl cluster-info;
       export VELERO_RELEASE=v1.0.0;
       export OPENEBS_RELEASE=master;
-      ./script/install-openebs.sh && ./script/install-velero.sh && travis_wait make test || travis_terminate 1;
+      ./script/install-openebs.sh && ./script/install-velero.sh && travis_wait 30 make test || travis_terminate 1;
     fi
 
 after_success:

README.md

@@ -246,6 +246,13 @@ defaultbackup-20190513113453   defaultbackup   Completed   0          0
 
 Once the restore is completed you should see the restore marked as `Completed`.
 
+
+To restore in different namespace, run the following command:
+
+```
+velero restore create --from-backup backup_name --restore-volumes=true --namespace-mappings source_ns:destination_ns
+```
+
 *Note: After restore for remote backup is completed, you need to set target-ip for the volume in pool pod. If restore is from local snapshot then you don't need to update target-ip*
 *Steps to get target-ip*
 1. kubectl get svc -n openebs <PV_NAME> -ojsonpath='{.spec.clusterIP}'
@@ -255,9 +262,6 @@ Once the restore is completed you should see the restore marked as `Completed`.
 2. zfs set io.openebs:targetip=<TARGET_IP> <POOL_NAME/VOLUME_NAME>
 ```
 
-*Limitation:*
-- _Restore of remote/cloud-backup in different namespace(--namespace-remapping) is not supported_
-
 ### Creating a scheduled remote backup
 OpenEBS velero-plugin provides incremental remote backup support for CStor persistent volumes.
 
@@ -307,6 +311,9 @@ velero restore create --from-backup sched-20190513103034 --restore-volumes=true
 velero restore create --from-backup sched-20190513103534 --restore-volumes=true
 velero restore create --from-backup sched-20190513104034 --restore-volumes=true
 ```
+
+You can restore scheduled remote backup to different namespace using `--namespace-mappings` argument [while creating a restore](#creating-a-restore-for-remote-backup).
+
 *Note: Velero clean-up the backups according to retain policy. By default retain policy is 30days. So you need to set retain policy for scheduled remote/cloud-backup accordingly.*
 
 ## License

changelogs/unreleased/72-mynktl

@@ -0,0 +1 @@
+Adding support to restore remote backup in different namespace

pkg/clouduploader/operation.go

@@ -48,7 +48,7 @@ func (c *Conn) Upload(file string, fileSize int64) bool {
 	if err != nil {
 		c.Log.Errorf("Failed to upload snapshot to bucket: %s", err.Error())
 		if c.bucket.Delete(c.ctx, file) != nil {
-			c.Log.Errorf("Failed to remove snapshot{%s} from cloud", file)
+			c.Log.Errorf("Failed to delete uncompleted snapshot{%s} from cloud", file)
 		}
 		return false
 	}

pkg/clouduploader/server.go

@@ -134,19 +134,23 @@ func (s *Server) acceptClient(fd, epfd int) (int, error) {
 		return (-1), err
 	}
 
+	readerWriter := s.cl.Create(s.OpType)
+	if readerWriter == nil {
+		s.Log.Errorf("Failed to create file interface")
+		if err = syscall.Close(connFd); err != nil {
+			s.Log.Warnf("Failed to close cline {%v} : %s", connFd, err.Error())
+		}
+		return (-1), errors.New("failed to create file interface")
+	}
+
 	c = new(Client)
 	c.fd = connFd
-	c.file = s.cl.Create(s.OpType)
+	c.file = readerWriter
 	c.bufferLen = ReadBufferLen
 	c.buffer = make([]byte, c.bufferLen)
 	c.status = TransferStatusInit
 	c.next = nil
 
-	if c.file == nil {
-		s.Log.Errorf("Failed to create file interface")
-		panic(errors.New("failed to create file interface"))
-	}
-
 	event = new(syscall.EpollEvent)
 	if s.OpType == OpBackup {
 		event.Events = syscall.EPOLLIN | syscall.EPOLLRDHUP | syscall.EPOLLHUP | syscall.EPOLLERR | EPOLLET
@@ -276,6 +280,11 @@ func (s *Server) Run(opType ServerOperation) error {
 	for {
 		nevents, err := syscall.EpollWait(epfd, events[:], EPOLLTIMEOUT)
 		if err != nil {
+			if isEINTR(err) {
+				s.Log.Warningf("Epoll wait failed : %s", err.Error())
+				continue
+			}
+
 			s.Log.Errorf("Epoll wait failed : %s", err.Error())
 			return err
 		}

pkg/clouduploader/server_utils.go

@@ -40,16 +40,18 @@ func (s *Server) removeFromClientList(c *Client) {
 	var prevClient *Client
 
 	if s.FirstClient == nil || s.state.runningCount == 0 {
-		s.Log.Errorf("ClientList is empty")
-		panic(errors.New("clientList list is empty"))
+		// epoll may have returned multiple event for the same fd
+		s.Log.Warningf("ClientList is empty")
+		return
 	} else if s.FirstClient == c {
 		s.FirstClient = c.next
 	} else {
 		curClient := s.FirstClient
 		for curClient != c {
 			if curClient.next == nil {
-				s.Log.Errorf("entry{%v} not found in ClientList", c.fd)
-				panic(errors.Errorf("entry{%v} not found in ClientList", c.fd))
+				// epoll may have returned multiple event for the same fd
+				s.Log.Warningf("entry{%v} not found in ClientList", c.fd)
+				return
 			}
 
 			prevClient = curClient
@@ -193,3 +195,16 @@ func (s *Server) disconnectAllClient(efd int) {
 		curClient = nextClient
 	}
 }
+
+// isEINTR check if given error is generated because of EINTR
+func isEINTR(err error) bool {
+	if err == nil {
+		return false
+	}
+
+	errno, ok := err.(syscall.Errno)
+	if ok && errno == syscall.EINTR {
+		return true
+	}
+	return false
+}

pkg/cstor/cstor.go

@@ -30,6 +30,7 @@ import (
 	 */
 	v1alpha1 "github.com/openebs/maya/pkg/apis/openebs.io/v1alpha1"
 	openebs "github.com/openebs/maya/pkg/client/generated/clientset/versioned"
+	velero "github.com/openebs/velero-plugin/pkg/velero"
 	"github.com/sirupsen/logrus"
 	v1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/api/resource"
@@ -57,6 +58,9 @@ const (
 
 	// LocalSnapshot config key for local snapshot
 	LocalSnapshot = "local"
+
+	// SnapshotIDIdentifier is a word to generate snapshotID from volume name and backup name
+	SnapshotIDIdentifier = "-velero-bkp-"
 )
 
 // Plugin defines snapshot plugin for CStor volume
@@ -209,6 +213,10 @@ func (p *Plugin) Init(config map[string]string) error {
 		return nil
 	}
 
+	if err := velero.InitializeClientSet(conf); err != nil {
+		return errors.Wrapf(err, "failed to initialize velero clientSet")
+	}
+
 	p.cl = &cloud.Conn{Log: p.Log}
 	return p.cl.Init(config)
 }
@@ -349,7 +357,7 @@ func (p *Plugin) CreateSnapshot(volumeID, volumeAZ string, tags map[string]strin
 
 	if p.local {
 		// local snapshot
-		return volumeID + "-velero-bkp-" + bkpname, nil
+		return generateSnapshotID(volumeID, bkpname), nil
 	}
 
 	filename := p.cl.GenerateRemoteFilename(vol.snapshotTag, vol.backupName)
@@ -365,15 +373,14 @@ func (p *Plugin) CreateSnapshot(volumeID, volumeAZ string, tags map[string]strin
 	}
 
 	if vol.backupStatus == v1alpha1.BKPCStorStatusDone {
-		return volumeID + "-velero-bkp-" + bkpname, nil
+		return generateSnapshotID(volumeID, bkpname), nil
 	}
+
 	return "", errors.Errorf("Failed to upload snapshot, status:{%v}", vol.backupStatus)
 }
 
 func (p *Plugin) getSnapInfo(snapshotID string) (*Snapshot, error) {
-	s := strings.Split(snapshotID, "-velero-bkp-")
-	volumeID := s[0]
-	bkpName := s[1]
+	volumeID, bkpName := getInfoFromSnapshotID(snapshotID)
 
 	pv, err := p.K8sClient.
 		CoreV1().
@@ -397,32 +404,39 @@ func (p *Plugin) getSnapInfo(snapshotID string) (*Snapshot, error) {
 // CreateVolumeFromSnapshot create CStor volume for given
 // snapshotID and perform restore operation on it
 func (p *Plugin) CreateVolumeFromSnapshot(snapshotID, volumeType, volumeAZ string, iops *int64) (string, error) {
+	var (
+		newVol *Volume
+		err    error
+	)
+
 	if volumeType != "cstor-snapshot" {
 		return "", errors.Errorf("Invalid volume type{%s}", volumeType)
 	}
 
-	s := strings.Split(snapshotID, "-velero-bkp-")
-	volumeID := s[0]
-	snapName := s[1]
+	volumeID, snapName := getInfoFromSnapshotID(snapshotID)
 
-	snapType := "cloud"
+	snapType := "remote"
 	if p.local {
 		snapType = "local"
 	}
 
 	p.Log.Infof("Restoring %s snapshot{%s} for volume:%s", snapType, snapName, volumeID)
 
-	newVol, err := p.getVolInfo(volumeID, snapName)
-	if err != nil {
-		return "", errors.Wrapf(err, "Failed to read PVC for volumeID=%s snap=%s", volumeID, snapName)
-	}
-
-	fn := p.restoreVolumeFromCloud
 	if p.local {
-		fn = p.restoreVolumeFromLocal
-	}
+		newVol, err = p.getVolumeForLocalRestore(volumeID, snapName)
+		if err != nil {
+			return "", errors.Wrapf(err, "Failed to read PVC for volumeID=%s snap=%s", volumeID, snapName)
+		}
 
-	err = fn(newVol)
+		err = p.restoreVolumeFromLocal(newVol)
+	} else {
+		newVol, err = p.getVolumeForRemoteRestore(volumeID, snapName)
+		if err != nil {
+			return "", errors.Wrapf(err, "Failed to read PVC for volumeID=%s snap=%s", volumeID, snapName)
+		}
+
+		err = p.restoreVolumeFromCloud(newVol)
+	}
 
 	if err != nil {
 		p.Log.Errorf("Failed to restore volume : %s", err)
@@ -473,39 +487,6 @@ func (p *Plugin) SetVolumeID(unstructuredPV runtime.Unstructured, volumeID strin
 	return &unstructured.Unstructured{Object: res}, nil
 }
 
-func (p *Plugin) getVolInfo(volumeID, snapName string) (*Volume, error) {
-	// To support namespace re-mapping for cloud snapshot,
-	// change createPVC to getPVCInfo
-	fn := p.createPVC
-	if p.local {
-		fn = p.getPVInfo
-	}
-
-	vol, err := fn(volumeID, snapName)
-	if err != nil {
-		return nil, err
-	}
-
-	// To support namespace re-mapping for cloud-snapshot remove below check
-	if p.local {
-		// Let's rename PV if already created
-		// -- PV may exist in-case of namespace-remapping or stale PV
-		newVolName, err := p.generateRestorePVName(vol.volname)
-		if err != nil {
-			return nil, errors.Wrapf(err, "Failed to generate PV name")
-		}
-
-		delete(p.volumes, vol.volname)
-		vol.volname = newVolName
-	}
-
-	p.volumes[vol.volname] = vol
-
-	p.Log.Infof("Generated PV name is %s", vol.volname)
-
-	return vol, nil
-}
-
 // getScheduleName return the schedule name for the given backup
 // It will check if backup name have 'bkp-20060102150405' format
 func (p *Plugin) getScheduleName(backupName string) string {
@@ -524,3 +505,15 @@ func (p *Plugin) getScheduleName(backupName string) string {
 	}
 	return scheduleOrBackupName
 }
+
+// getInfoFromSnapshotID return backup name and volume id from the given snapshotID
+func getInfoFromSnapshotID(snapshotID string) (volumeID, backupName string) {
+	s := strings.Split(snapshotID, SnapshotIDIdentifier)
+	volumeID = s[0]
+	backupName = s[1]
+	return
+}
+
+func generateSnapshotID(volumeID, backupName string) string {
+	return volumeID + SnapshotIDIdentifier + backupName
+}