diff --git a/.travis.yml b/.travis.yml index c2bae1d4..c58a6092 100644 --- a/.travis.yml +++ b/.travis.yml @@ -42,7 +42,7 @@ script: kubectl cluster-info; export VELERO_RELEASE=v1.0.0; export OPENEBS_RELEASE=master; - ./script/install-openebs.sh && ./script/install-velero.sh && travis_wait make test || travis_terminate 1; + ./script/install-openebs.sh && ./script/install-velero.sh && travis_wait 30 make test || travis_terminate 1; fi after_success: diff --git a/README.md b/README.md index ba2ebca0..64f5944d 100644 --- a/README.md +++ b/README.md @@ -246,6 +246,13 @@ defaultbackup-20190513113453 defaultbackup Completed 0 0 Once the restore is completed you should see the restore marked as `Completed`. + +To restore in different namespace, run the following command: + +``` +velero restore create --from-backup backup_name --restore-volumes=true --namespace-mappings source_ns:destination_ns +``` + *Note: After restore for remote backup is completed, you need to set target-ip for the volume in pool pod. If restore is from local snapshot then you don't need to update target-ip* *Steps to get target-ip* 1. kubectl get svc -n openebs -ojsonpath='{.spec.clusterIP}' @@ -255,9 +262,6 @@ Once the restore is completed you should see the restore marked as `Completed`. 2. zfs set io.openebs:targetip= ``` -*Limitation:* -- _Restore of remote/cloud-backup in different namespace(--namespace-remapping) is not supported_ - ### Creating a scheduled remote backup OpenEBS velero-plugin provides incremental remote backup support for CStor persistent volumes. @@ -307,6 +311,9 @@ velero restore create --from-backup sched-20190513103034 --restore-volumes=true velero restore create --from-backup sched-20190513103534 --restore-volumes=true velero restore create --from-backup sched-20190513104034 --restore-volumes=true ``` + +You can restore scheduled remote backup to different namespace using `--namespace-mappings` argument [while creating a restore](#creating-a-restore-for-remote-backup). + *Note: Velero clean-up the backups according to retain policy. By default retain policy is 30days. So you need to set retain policy for scheduled remote/cloud-backup accordingly.* ## License diff --git a/changelogs/unreleased/72-mynktl b/changelogs/unreleased/72-mynktl new file mode 100644 index 00000000..c4e623da --- /dev/null +++ b/changelogs/unreleased/72-mynktl @@ -0,0 +1 @@ +Adding support to restore remote backup in different namespace diff --git a/pkg/clouduploader/operation.go b/pkg/clouduploader/operation.go index cab451f8..1faea99e 100644 --- a/pkg/clouduploader/operation.go +++ b/pkg/clouduploader/operation.go @@ -48,7 +48,7 @@ func (c *Conn) Upload(file string, fileSize int64) bool { if err != nil { c.Log.Errorf("Failed to upload snapshot to bucket: %s", err.Error()) if c.bucket.Delete(c.ctx, file) != nil { - c.Log.Errorf("Failed to remove snapshot{%s} from cloud", file) + c.Log.Errorf("Failed to delete uncompleted snapshot{%s} from cloud", file) } return false } diff --git a/pkg/clouduploader/server.go b/pkg/clouduploader/server.go index 93306ade..eab6d6a3 100644 --- a/pkg/clouduploader/server.go +++ b/pkg/clouduploader/server.go @@ -134,19 +134,23 @@ func (s *Server) acceptClient(fd, epfd int) (int, error) { return (-1), err } + readerWriter := s.cl.Create(s.OpType) + if readerWriter == nil { + s.Log.Errorf("Failed to create file interface") + if err = syscall.Close(connFd); err != nil { + s.Log.Warnf("Failed to close cline {%v} : %s", connFd, err.Error()) + } + return (-1), errors.New("failed to create file interface") + } + c = new(Client) c.fd = connFd - c.file = s.cl.Create(s.OpType) + c.file = readerWriter c.bufferLen = ReadBufferLen c.buffer = make([]byte, c.bufferLen) c.status = TransferStatusInit c.next = nil - if c.file == nil { - s.Log.Errorf("Failed to create file interface") - panic(errors.New("failed to create file interface")) - } - event = new(syscall.EpollEvent) if s.OpType == OpBackup { event.Events = syscall.EPOLLIN | syscall.EPOLLRDHUP | syscall.EPOLLHUP | syscall.EPOLLERR | EPOLLET @@ -276,6 +280,11 @@ func (s *Server) Run(opType ServerOperation) error { for { nevents, err := syscall.EpollWait(epfd, events[:], EPOLLTIMEOUT) if err != nil { + if isEINTR(err) { + s.Log.Warningf("Epoll wait failed : %s", err.Error()) + continue + } + s.Log.Errorf("Epoll wait failed : %s", err.Error()) return err } diff --git a/pkg/clouduploader/server_utils.go b/pkg/clouduploader/server_utils.go index 2b78b4fd..062ec438 100644 --- a/pkg/clouduploader/server_utils.go +++ b/pkg/clouduploader/server_utils.go @@ -40,16 +40,18 @@ func (s *Server) removeFromClientList(c *Client) { var prevClient *Client if s.FirstClient == nil || s.state.runningCount == 0 { - s.Log.Errorf("ClientList is empty") - panic(errors.New("clientList list is empty")) + // epoll may have returned multiple event for the same fd + s.Log.Warningf("ClientList is empty") + return } else if s.FirstClient == c { s.FirstClient = c.next } else { curClient := s.FirstClient for curClient != c { if curClient.next == nil { - s.Log.Errorf("entry{%v} not found in ClientList", c.fd) - panic(errors.Errorf("entry{%v} not found in ClientList", c.fd)) + // epoll may have returned multiple event for the same fd + s.Log.Warningf("entry{%v} not found in ClientList", c.fd) + return } prevClient = curClient @@ -193,3 +195,16 @@ func (s *Server) disconnectAllClient(efd int) { curClient = nextClient } } + +// isEINTR check if given error is generated because of EINTR +func isEINTR(err error) bool { + if err == nil { + return false + } + + errno, ok := err.(syscall.Errno) + if ok && errno == syscall.EINTR { + return true + } + return false +} diff --git a/pkg/cstor/cstor.go b/pkg/cstor/cstor.go index 09401e91..51c409f8 100644 --- a/pkg/cstor/cstor.go +++ b/pkg/cstor/cstor.go @@ -30,6 +30,7 @@ import ( */ v1alpha1 "github.com/openebs/maya/pkg/apis/openebs.io/v1alpha1" openebs "github.com/openebs/maya/pkg/client/generated/clientset/versioned" + velero "github.com/openebs/velero-plugin/pkg/velero" "github.com/sirupsen/logrus" v1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/resource" @@ -57,6 +58,9 @@ const ( // LocalSnapshot config key for local snapshot LocalSnapshot = "local" + + // SnapshotIDIdentifier is a word to generate snapshotID from volume name and backup name + SnapshotIDIdentifier = "-velero-bkp-" ) // Plugin defines snapshot plugin for CStor volume @@ -209,6 +213,10 @@ func (p *Plugin) Init(config map[string]string) error { return nil } + if err := velero.InitializeClientSet(conf); err != nil { + return errors.Wrapf(err, "failed to initialize velero clientSet") + } + p.cl = &cloud.Conn{Log: p.Log} return p.cl.Init(config) } @@ -349,7 +357,7 @@ func (p *Plugin) CreateSnapshot(volumeID, volumeAZ string, tags map[string]strin if p.local { // local snapshot - return volumeID + "-velero-bkp-" + bkpname, nil + return generateSnapshotID(volumeID, bkpname), nil } filename := p.cl.GenerateRemoteFilename(vol.snapshotTag, vol.backupName) @@ -365,15 +373,14 @@ func (p *Plugin) CreateSnapshot(volumeID, volumeAZ string, tags map[string]strin } if vol.backupStatus == v1alpha1.BKPCStorStatusDone { - return volumeID + "-velero-bkp-" + bkpname, nil + return generateSnapshotID(volumeID, bkpname), nil } + return "", errors.Errorf("Failed to upload snapshot, status:{%v}", vol.backupStatus) } func (p *Plugin) getSnapInfo(snapshotID string) (*Snapshot, error) { - s := strings.Split(snapshotID, "-velero-bkp-") - volumeID := s[0] - bkpName := s[1] + volumeID, bkpName := getInfoFromSnapshotID(snapshotID) pv, err := p.K8sClient. CoreV1(). @@ -397,32 +404,39 @@ func (p *Plugin) getSnapInfo(snapshotID string) (*Snapshot, error) { // CreateVolumeFromSnapshot create CStor volume for given // snapshotID and perform restore operation on it func (p *Plugin) CreateVolumeFromSnapshot(snapshotID, volumeType, volumeAZ string, iops *int64) (string, error) { + var ( + newVol *Volume + err error + ) + if volumeType != "cstor-snapshot" { return "", errors.Errorf("Invalid volume type{%s}", volumeType) } - s := strings.Split(snapshotID, "-velero-bkp-") - volumeID := s[0] - snapName := s[1] + volumeID, snapName := getInfoFromSnapshotID(snapshotID) - snapType := "cloud" + snapType := "remote" if p.local { snapType = "local" } p.Log.Infof("Restoring %s snapshot{%s} for volume:%s", snapType, snapName, volumeID) - newVol, err := p.getVolInfo(volumeID, snapName) - if err != nil { - return "", errors.Wrapf(err, "Failed to read PVC for volumeID=%s snap=%s", volumeID, snapName) - } - - fn := p.restoreVolumeFromCloud if p.local { - fn = p.restoreVolumeFromLocal - } + newVol, err = p.getVolumeForLocalRestore(volumeID, snapName) + if err != nil { + return "", errors.Wrapf(err, "Failed to read PVC for volumeID=%s snap=%s", volumeID, snapName) + } - err = fn(newVol) + err = p.restoreVolumeFromLocal(newVol) + } else { + newVol, err = p.getVolumeForRemoteRestore(volumeID, snapName) + if err != nil { + return "", errors.Wrapf(err, "Failed to read PVC for volumeID=%s snap=%s", volumeID, snapName) + } + + err = p.restoreVolumeFromCloud(newVol) + } if err != nil { p.Log.Errorf("Failed to restore volume : %s", err) @@ -473,39 +487,6 @@ func (p *Plugin) SetVolumeID(unstructuredPV runtime.Unstructured, volumeID strin return &unstructured.Unstructured{Object: res}, nil } -func (p *Plugin) getVolInfo(volumeID, snapName string) (*Volume, error) { - // To support namespace re-mapping for cloud snapshot, - // change createPVC to getPVCInfo - fn := p.createPVC - if p.local { - fn = p.getPVInfo - } - - vol, err := fn(volumeID, snapName) - if err != nil { - return nil, err - } - - // To support namespace re-mapping for cloud-snapshot remove below check - if p.local { - // Let's rename PV if already created - // -- PV may exist in-case of namespace-remapping or stale PV - newVolName, err := p.generateRestorePVName(vol.volname) - if err != nil { - return nil, errors.Wrapf(err, "Failed to generate PV name") - } - - delete(p.volumes, vol.volname) - vol.volname = newVolName - } - - p.volumes[vol.volname] = vol - - p.Log.Infof("Generated PV name is %s", vol.volname) - - return vol, nil -} - // getScheduleName return the schedule name for the given backup // It will check if backup name have 'bkp-20060102150405' format func (p *Plugin) getScheduleName(backupName string) string { @@ -524,3 +505,15 @@ func (p *Plugin) getScheduleName(backupName string) string { } return scheduleOrBackupName } + +// getInfoFromSnapshotID return backup name and volume id from the given snapshotID +func getInfoFromSnapshotID(snapshotID string) (volumeID, backupName string) { + s := strings.Split(snapshotID, SnapshotIDIdentifier) + volumeID = s[0] + backupName = s[1] + return +} + +func generateSnapshotID(volumeID, backupName string) string { + return volumeID + SnapshotIDIdentifier + backupName +} diff --git a/pkg/cstor/pv_operation.go b/pkg/cstor/pv_operation.go index 2bfd8379..2cf78d24 100644 --- a/pkg/cstor/pv_operation.go +++ b/pkg/cstor/pv_operation.go @@ -7,10 +7,14 @@ import ( v1alpha1 "github.com/openebs/maya/pkg/apis/openebs.io/v1alpha1" "github.com/pkg/errors" v1 "k8s.io/api/core/v1" - apierror "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) +const ( + // PvClonePrefix prefix for clone volume in case restore from local backup + PvClonePrefix = "cstor-clone-" +) + func (p *Plugin) updateVolCASInfo(data []byte, volumeID string) error { var cas v1alpha1.CASVolume @@ -60,40 +64,40 @@ func (p *Plugin) restoreVolumeFromCloud(vol *Volume) error { return nil } -func (p *Plugin) generateRestorePVName(volumeID string) (string, error) { - _, err := p.K8sClient. +func (p *Plugin) getPV(volumeID string) (*v1.PersistentVolume, error) { + return p.K8sClient. CoreV1(). PersistentVolumes(). Get(volumeID, metav1.GetOptions{}) +} + +func (p *Plugin) restoreVolumeFromLocal(vol *Volume) error { + _, err := p.sendRestoreRequest(vol) if err != nil { - if apierror.IsNotFound(err) { - return volumeID, nil - } - return "", errors.Wrapf(err, "Error checking if PV with same name exist") + return errors.Wrapf(err, "Restore request to apiServer failed") } + vol.restoreStatus = v1alpha1.RSTCStorStatusDone + return nil +} - nuuid, err := uuid.NewV4() +// getVolumeForLocalRestore return volume information to restore locally for the given volumeID and snapName +// volumeID : pv name from backup +// snapName : snapshot name from where new volume will be created +func (p *Plugin) getVolumeForLocalRestore(volumeID, snapName string) (*Volume, error) { + pv, err := p.getPV(volumeID) if err != nil { - return "", errors.Wrapf(err, "Error generating uuid for PV rename") + return nil, errors.Wrapf(err, "error fetching PV=%s", volumeID) } - oldVolumeID, volumeID := volumeID, "cstor-clone-"+nuuid.String() - p.Log.Infof("Renaming PV %s to %s", oldVolumeID, volumeID) - return volumeID, nil -} - -func (p *Plugin) getPVInfo(volumeID, snapName string) (*Volume, error) { - pv, err := p.K8sClient. - CoreV1(). - PersistentVolumes(). - Get(volumeID, metav1.GetOptions{}) + clonePvName, err := generateClonePVName() if err != nil { - return nil, errors.Errorf("Error fetching volume{%s} : %s", volumeID, err.Error()) + return nil, err } + p.Log.Infof("Renaming PV %s to %s", pv.Name, clonePvName) vol := &Volume{ - volname: volumeID, - srcVolname: volumeID, + volname: clonePvName, + srcVolname: pv.Name, backupName: snapName, storageClass: pv.Spec.StorageClassName, size: pv.Spec.Capacity[v1.ResourceStorage], @@ -102,11 +106,29 @@ func (p *Plugin) getPVInfo(volumeID, snapName string) (*Volume, error) { return vol, nil } -func (p *Plugin) restoreVolumeFromLocal(vol *Volume) error { - _, err := p.sendRestoreRequest(vol) +// getVolumeForRemoteRestore return volume information to restore from remote backup for the given volumeID and snapName +// volumeID : pv name from backup +// snapName : snapshot name from where new volume will be created +func (p *Plugin) getVolumeForRemoteRestore(volumeID, snapName string) (*Volume, error) { + vol, err := p.createPVC(volumeID, snapName) if err != nil { - return errors.Wrapf(err, "Restore request to apiServer failed") + p.Log.Errorf("CreatePVC returned error=%s", err) + return nil, err } - vol.restoreStatus = v1alpha1.RSTCStorStatusDone - return nil + + p.volumes[vol.volname] = vol + + p.Log.Infof("Generated PV name is %s", vol.volname) + + return vol, nil +} + +// generateClonePVName return new name for clone pv for the given pv +func generateClonePVName() (string, error) { + nuuid, err := uuid.NewV4() + if err != nil { + return "", errors.Wrapf(err, "Error generating uuid for PV rename") + } + + return PvClonePrefix + nuuid.String(), nil } diff --git a/pkg/cstor/pvc_operation.go b/pkg/cstor/pvc_operation.go index b22aaddc..2fd22f2b 100644 --- a/pkg/cstor/pvc_operation.go +++ b/pkg/cstor/pvc_operation.go @@ -20,6 +20,7 @@ import ( "encoding/json" "time" + velero "github.com/openebs/velero-plugin/pkg/velero" "github.com/pkg/errors" v1 "k8s.io/api/core/v1" k8serrors "k8s.io/apimachinery/pkg/api/errors" @@ -89,36 +90,31 @@ func (p *Plugin) backupPVC(volumeID string) error { // createPVC create PVC for given volume name func (p *Plugin) createPVC(volumeID, snapName string) (*Volume, error) { - pvc := &v1.PersistentVolumeClaim{} var vol *Volume - var data []byte - var ok bool - filename := p.cl.GenerateRemoteFilename(volumeID, snapName) - if filename == "" { - return nil, errors.New("error creating remote file name for pvc backup") + pvc, err := p.downloadPVC(volumeID, snapName) + if err != nil { + return nil, errors.Wrapf(err, "failed to download pvc") } - if data, ok = p.cl.Read(filename + ".pvc"); !ok { - return nil, errors.Errorf("Failed to download PVC file=%s", filename+".pvc") + targetedNs, err := velero.GetRestoreNamespace(pvc.Namespace, snapName, p.Log) + if err != nil { + return nil, err } + pvc.Namespace = targetedNs - if err := json.Unmarshal(data, pvc); err != nil { - return nil, errors.Errorf("Failed to decode pvc file=%s", filename+".pvc") + newVol, err := p.getVolumeFromPVC(*pvc) + if err != nil { + return nil, err } - newVol, err := p.getVolumeFromPVC(*pvc) if newVol != nil { newVol.backupName = snapName newVol.snapshotTag = volumeID return newVol, nil } - if err != nil { - return nil, err - } - - p.Log.Infof("Creating PVC for volumeID:%s snapshot:%s", volumeID, snapName) + p.Log.Infof("Creating PVC for volumeID:%s snapshot:%s in namespace=%s", volumeID, snapName, targetedNs) pvc.Annotations = make(map[string]string) pvc.Annotations["openebs.io/created-through"] = "restore" @@ -200,7 +196,6 @@ func (p *Plugin) getPVCInfo(volumeID, snapName string) (*Volume, error) { return vol, nil } -// nolint: unused // getVolumeFromPVC returns volume info for given PVC if PVC is in bound state func (p *Plugin) getVolumeFromPVC(pvc v1.PersistentVolumeClaim) (*Volume, error) { rpvc, err := p.K8sClient. @@ -231,3 +226,20 @@ func (p *Plugin) getVolumeFromPVC(pvc v1.PersistentVolumeClaim) (*Volume, error) } return vol, nil } + +func (p *Plugin) downloadPVC(volumeID, snapName string) (*v1.PersistentVolumeClaim, error) { + pvc := &v1.PersistentVolumeClaim{} + + filename := p.cl.GenerateRemoteFilename(volumeID, snapName) + + data, ok := p.cl.Read(filename + ".pvc") + if !ok { + return nil, errors.Errorf("failed to download PVC file=%s", filename+".pvc") + } + + if err := json.Unmarshal(data, pvc); err != nil { + return nil, errors.Errorf("failed to decode pvc file=%s", filename+".pvc") + } + + return pvc, nil +} diff --git a/pkg/cstor/status.go b/pkg/cstor/status.go index 915e3f23..f92cb61f 100644 --- a/pkg/cstor/status.go +++ b/pkg/cstor/status.go @@ -21,7 +21,6 @@ import ( "time" "github.com/openebs/maya/pkg/apis/openebs.io/v1alpha1" - "github.com/pkg/errors" ) // checkBackupStatus queries MayaAPI server for given backup status @@ -34,13 +33,17 @@ func (p *Plugin) checkBackupStatus(bkp *v1alpha1.CStorBackup) { bkpvolume, exists := p.volumes[bkp.Spec.VolumeName] if !exists { p.Log.Errorf("Failed to fetch volume info for {%s}", bkp.Spec.VolumeName) - panic(errors.Errorf("Failed to fetch volume info for {%s}", bkp.Spec.VolumeName)) + p.cl.ExitServer = true + bkpvolume.backupStatus = v1alpha1.BKPCStorStatusInvalid + return } bkpData, err := json.Marshal(bkp) if err != nil { p.Log.Errorf("JSON marshal failed : %s", err.Error()) - panic(errors.Errorf("JSON marshal failed : %s", err.Error())) + p.cl.ExitServer = true + bkpvolume.backupStatus = v1alpha1.BKPCStorStatusInvalid + return } for !bkpDone { @@ -82,7 +85,8 @@ func (p *Plugin) checkRestoreStatus(rst *v1alpha1.CStorRestore, vol *Volume) { rstData, err := json.Marshal(rst) if err != nil { p.Log.Errorf("JSON marshal failed : %s", err.Error()) - panic(errors.Errorf("JSON marshal failed : %s", err.Error())) + vol.restoreStatus = v1alpha1.RSTCStorStatusInvalid + p.cl.ExitServer = true } for !rstDone { diff --git a/pkg/velero/restore.go b/pkg/velero/restore.go new file mode 100644 index 00000000..668c9e24 --- /dev/null +++ b/pkg/velero/restore.go @@ -0,0 +1,43 @@ +package velero + +import ( + "sort" + + "github.com/pkg/errors" + "github.com/sirupsen/logrus" + velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +// GetRestoreNamespace return the namespace mapping for the given namespace +// if namespace mapping not found then it will return the same namespace in which backup was created +// if namespace mapping found then it will return the mapping/target namespace +// +// velero doesn't pass the restore name to plugin, so we are following the below +// approach to fetch the namespace mapping: +// +// plugin find the relevant restore from the sorted list(creationTimestamp in decreasing order) of +// restore resource using following criteria: +// - retore is in in-progress state AND +// backup for that restore matches with the backup name from snapshotID +// Above approach works because velero support sequential restore +func GetRestoreNamespace(ns, bkpName string, log logrus.FieldLogger) (string, error) { + listOpts := metav1.ListOptions{} + list, err := clientSet.VeleroV1().Restores(veleroNs).List(listOpts) + if err != nil { + return "", errors.Wrapf(err, "failed to get list of restore") + } + + sort.Sort(sort.Reverse(RestoreByCreationTimestamp(list.Items))) + + for _, r := range list.Items { + if r.Status.Phase == velerov1api.RestorePhaseInProgress && r.Spec.BackupName == bkpName { + targetedNs, ok := r.Spec.NamespaceMapping[ns] + if ok { + return targetedNs, nil + } + return ns, nil + } + } + return "", errors.Errorf("restore not found for backup %s", bkpName) +} diff --git a/pkg/velero/sort.go b/pkg/velero/sort.go new file mode 100644 index 00000000..492493ab --- /dev/null +++ b/pkg/velero/sort.go @@ -0,0 +1,15 @@ +package velero + +import velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" + +// RestoreByCreationTimestamp sorts a list of Restore by creation timestamp, using their names as a tie breaker. +type RestoreByCreationTimestamp []velerov1api.Restore + +func (o RestoreByCreationTimestamp) Len() int { return len(o) } +func (o RestoreByCreationTimestamp) Swap(i, j int) { o[i], o[j] = o[j], o[i] } +func (o RestoreByCreationTimestamp) Less(i, j int) bool { + if o[i].CreationTimestamp.Equal(&o[j].CreationTimestamp) { + return o[i].Name < o[j].Name + } + return o[i].CreationTimestamp.Before(&o[j].CreationTimestamp) +} diff --git a/pkg/velero/velero.go b/pkg/velero/velero.go new file mode 100644 index 00000000..be30d92b --- /dev/null +++ b/pkg/velero/velero.go @@ -0,0 +1,32 @@ +package velero + +import ( + "os" + + veleroclient "github.com/vmware-tanzu/velero/pkg/generated/clientset/versioned" + "k8s.io/client-go/rest" +) + +var ( + // clientSet will be used to fetch velero customo resources + clientSet veleroclient.Interface + + // veleroNs velero installation namespace + veleroNs string +) + +func init() { + veleroNs = os.Getenv("VELERO_NAMESPACE") +} + +// InitializeClientSet initialize velero clientset +func InitializeClientSet(config *rest.Config) error { + var err error + + clientSet, err = veleroclient.NewForConfig(config) + if err != nil { + return err + } + + return nil +} diff --git a/tests/openebs/logs.go b/tests/openebs/logs.go index 78162b6d..1e6f0245 100644 --- a/tests/openebs/logs.go +++ b/tests/openebs/logs.go @@ -17,6 +17,8 @@ limitations under the License. package openebs import ( + "fmt" + k8s "github.com/openebs/velero-plugin/tests/k8s" corev1 "k8s.io/api/core/v1" ) @@ -28,22 +30,26 @@ const ( ) // DumpLogs will dump openebs logs -func (c *ClientSet) DumpLogs() error { +func (c *ClientSet) DumpLogs() { mayaPod := c.getMayaAPIServerPodName() spcPod := c.getSPCPodName() pvcPod := c.getPVCPodName() for _, v := range mayaPod { - _ = k8s.Client.DumpLogs(OpenEBSNs, v[0], v[1]) + if err := k8s.Client.DumpLogs(OpenEBSNs, v[0], v[1]); err != nil { + fmt.Printf("Failed to dump maya-apiserver logs err=%s\n", err) + } } for _, v := range spcPod { - _ = k8s.Client.DumpLogs(OpenEBSNs, v[0], v[1]) + if err := k8s.Client.DumpLogs(OpenEBSNs, v[0], v[1]); err != nil { + fmt.Printf("Failed to dump cstor pod logs err=%s\n", err) + } } for _, v := range pvcPod { - _ = k8s.Client.DumpLogs(OpenEBSNs, v[0], v[1]) + if err := k8s.Client.DumpLogs(OpenEBSNs, v[0], v[1]); err != nil { + fmt.Printf("Failed to dump target pod logs err=%s\n", err) + } } - - return nil } // getMayaAPIServerPodName return Maya-API server pod name and container diff --git a/tests/openebs/storage_install.go b/tests/openebs/storage_install.go index a71f3a95..1b4f4c32 100644 --- a/tests/openebs/storage_install.go +++ b/tests/openebs/storage_install.go @@ -51,8 +51,8 @@ const ( // OpenEBSNs openebs Namespace OpenEBSNs = "openebs" - // PVCDeploymentLabel for target pod Deployment - PVCDeploymentLabel = "openebs.io/persistent-volume-claim" + // PVDeploymentLabel for target pod Deployment + PVDeploymentLabel = "openebs.io/persistent-volume" ) func init() { @@ -123,12 +123,17 @@ func (c *ClientSet) DeleteVolume(pvcYAML, pvcNs string) error { return err } + pv, err := c.getPVCVolumeName(pvc.Name, pvcNs) + if err != nil { + return err + } + pvc.Namespace = pvcNs if err := k8s.Client.DeletePVC(pvc); err != nil { return err } return k8s.Client.WaitForDeploymentCleanup( - PVCDeploymentLabel+"="+pvc.Name, + PVDeploymentLabel+"="+pv, OpenEBSNs) } diff --git a/tests/sanity/backup_test.go b/tests/sanity/backup_test.go index 8bde8760..02a103d6 100644 --- a/tests/sanity/backup_test.go +++ b/tests/sanity/backup_test.go @@ -32,7 +32,12 @@ import ( ) const ( - AppNs = "test" + // AppNs application namespace + AppNs = "test" + + // TargetedNs namespace used for restore in different namespace + TargetedNs = "ns1" + BackupLocation = "default" SnapshotLocation = "default" ) @@ -52,6 +57,9 @@ var _ = BeforeSuite(func() { err = app.CreateNamespace(AppNs) Expect(err).NotTo(HaveOccurred()) + err = app.CreateNamespace(TargetedNs) + Expect(err).NotTo(HaveOccurred()) + err = k8s.Client.CreateStorageClass(openebs.SCYaml) Expect(err).NotTo(HaveOccurred()) @@ -77,21 +85,21 @@ var _ = Describe("Backup/Restore Test", func() { By("Creating a backup") err = openebs.Client.WaitForHealthyCVR(openebs.AppPVC) - Expect(err).NotTo(HaveOccurred()) + Expect(err).NotTo(HaveOccurred(), "No healthy CVR for %s", openebs.AppPVC) // There are chances that istgt is not updated, but replica is healthy time.Sleep(30 * time.Second) backupName, status, err = velero.Client.CreateBackup(AppNs) - if ((err != nil) || status != v1.BackupPhaseCompleted) && - backupName != "" { + if (err != nil) || status != v1.BackupPhaseCompleted { _ = velero.Client.DumpBackupLogs(backupName) - _ = openebs.Client.DumpLogs() + openebs.Client.DumpLogs() } - Expect(err).NotTo(HaveOccurred()) - Expect(status).To(Equal(v1.BackupPhaseCompleted)) + Expect(err).NotTo(HaveOccurred(), "Failed to create backup=%s for namespace=%s", backupName, AppNs) + Expect(status).To(Equal(v1.BackupPhaseCompleted), "Backup=%s for namespace=%s failed", backupName, AppNs) + isExist, err = openebs.Client.IsBackupResourcesExist(backupName, app.PVCName, AppNs) - Expect(err).NotTo(HaveOccurred()) - Expect(isExist).To(BeFalse()) + Expect(err).NotTo(HaveOccurred(), "Failed to verify snapshot cleanup for backup=%s", backupName) + Expect(isExist).To(BeFalse(), "Snapshot for backup=%s still exist", backupName) }) }) @@ -102,18 +110,20 @@ var _ = Describe("Backup/Restore Test", func() { By("Creating a scheduled backup") scheduleName, status, err = velero.Client.CreateSchedule(AppNs, "*/2 * * * *", 3) - Expect(err).NotTo(HaveOccurred()) - Expect(status).To(Equal(v1.BackupPhaseCompleted)) + Expect(err).NotTo(HaveOccurred(), "Failed create schedule:%s status=%s", scheduleName, status) + Expect(status).To(Equal(v1.BackupPhaseCompleted), "Schedule=%s failed", scheduleName) + err = velero.Client.DeleteSchedule(scheduleName) - Expect(err).NotTo(HaveOccurred()) + Expect(err).NotTo(HaveOccurred(), "Failed to delete schedule=%s", scheduleName) bkplist, serr := velero.Client.GetScheduledBackups(scheduleName) - Expect(serr).NotTo(HaveOccurred()) + Expect(serr).NotTo(HaveOccurred(), "Failed to get backup list for schedule=%s", scheduleName) - for _, bkp := range bkplist { + for i, bkp := range bkplist { isExist, err = openebs.Client.IsBackupResourcesExist(bkp, app.PVCName, AppNs) - Expect(err).NotTo(HaveOccurred()) - Expect(isExist).To(BeFalse()) + Expect(err).NotTo(HaveOccurred(), + "Failed to verify snapshot cleanup for backup=%s, with incremental count=%d", bkp, i) + Expect(isExist).To(BeFalse(), "Snapshot for backup=%s, with incremental count=%d, still exist", bkp, i) } }) }) @@ -122,64 +132,154 @@ var _ = Describe("Backup/Restore Test", func() { BeforeEach(func() { By("Destroying Application and Volume") err = app.DestroyApplication(app.BusyboxYaml, AppNs) - Expect(err).NotTo(HaveOccurred()) + Expect(err).NotTo(HaveOccurred(), "Failed to destroy application in namespace=%s", AppNs) err = openebs.Client.DeleteVolume(openebs.PVCYaml, AppNs) - Expect(err).NotTo(HaveOccurred()) + Expect(err).NotTo(HaveOccurred(), "Failed to delete volume for namespace=%s", AppNs) }) - It("Restore from non-scheduled backup Test 1", func() { - var status v1.RestorePhase + It("Restore from non-scheduled backup", func() { + var ( + status v1.RestorePhase + phase corev1.PersistentVolumeClaimPhase + ) By("Restoring from a non-scheduled backup") - status, err = velero.Client.CreateRestore(AppNs, backupName) - Expect(err).NotTo(HaveOccurred()) - Expect(status).To(Equal(v1.RestorePhaseCompleted)) + status, err = velero.Client.CreateRestore(AppNs, AppNs, backupName) + if err != nil || status != v1.RestorePhaseCompleted { + dumpLogs() + } + + Expect(err).NotTo(HaveOccurred(), "Failed to create a restore from backup=%s", backupName) + Expect(status).To(Equal(v1.RestorePhaseCompleted), "Restore from backup=%s failed", backupName) By("Checking if restored PVC is bound or not") phase, perr := k8s.Client.GetPVCPhase(app.PVCName, AppNs) - Expect(perr).NotTo(HaveOccurred()) - Expect(phase).To(Equal(corev1.ClaimBound)) + Expect(perr).NotTo(HaveOccurred(), "Failed to verify PVC=%s bound status for namespace=%s", app.PVCName, AppNs) + Expect(phase).To(Equal(corev1.ClaimBound), "PVC=%s not bound", app.PVCName) By("Checking if restored CVR are in error state") ok := openebs.Client.CheckCVRStatus(app.PVCName, AppNs, v1alpha1.CVRStatusError) - Expect(ok).To(BeTrue()) + Expect(ok).To(BeTrue(), "CVR for PVC=%s are not in errored state", app.PVCName) }) - It("Restore from scheduled backup Test 1", func() { + It("Restore from scheduled backup without base-backup", func() { var status v1.RestorePhase By("Restoring from a scheduled backup") - status, err = velero.Client.CreateRestoreFromSchedule(AppNs, scheduleName, 1) - Expect(err).NotTo(HaveOccurred()) - Expect(status).To(Equal(v1.RestorePhasePartiallyFailed)) + status, err = velero.Client.CreateRestoreFromSchedule(AppNs, AppNs, scheduleName, 1) + if err != nil || status != v1.RestorePhasePartiallyFailed { + dumpLogs() + } + + Expect(err).NotTo(HaveOccurred(), "Failed to create a restore from schedule=%s", scheduleName) + Expect(status).To(Equal(v1.RestorePhasePartiallyFailed), "Restore for schedule=%s should have failed", scheduleName) }) - It("Restore from scheduled backup Test 2", func() { - var status v1.RestorePhase + It("Restore from scheduled backup using base-backup", func() { + var ( + status v1.RestorePhase + phase corev1.PersistentVolumeClaimPhase + ) By("Restoring from a scheduled backup") - status, err = velero.Client.CreateRestoreFromSchedule(AppNs, scheduleName, 0) - Expect(err).NotTo(HaveOccurred()) - Expect(status).To(Equal(v1.RestorePhaseCompleted)) + status, err = velero.Client.CreateRestoreFromSchedule(AppNs, AppNs, scheduleName, 0) + if err != nil || status != v1.RestorePhaseCompleted { + dumpLogs() + } + Expect(err).NotTo(HaveOccurred(), "Failed to create a restore from schedule=%s", scheduleName) + Expect(status).To(Equal(v1.RestorePhaseCompleted), "Restore from schedule=%s failed", scheduleName) By("Checking if restored PVC is bound or not") - phase, err := k8s.Client.GetPVCPhase(app.PVCName, AppNs) - Expect(err).NotTo(HaveOccurred()) - Expect(phase).To(Equal(corev1.ClaimBound)) + phase, err = k8s.Client.GetPVCPhase(app.PVCName, AppNs) + Expect(err).NotTo(HaveOccurred(), "Failed to verify PVC=%s bound status for namespace=%s", app.PVCName, AppNs) + Expect(phase).To(Equal(corev1.ClaimBound), "PVC=%s not bound", app.PVCName) By("Checking if restored CVR are in error state") ok := openebs.Client.CheckCVRStatus(app.PVCName, AppNs, v1alpha1.CVRStatusError) - Expect(ok).To(BeTrue()) + Expect(ok).To(BeTrue(), "CVR for PVC=%s are not in errored state", app.PVCName) + + By("Checking if restore has created snapshot or not") + snapshotList, serr := velero.Client.GetRestoredSnapshotFromSchedule(scheduleName) + Expect(serr).NotTo(HaveOccurred()) + for snapshot := range snapshotList { + ok, err = openebs.Client.CheckSnapshot(app.PVCName, AppNs, snapshot) + if err != nil { + dumpLogs() + } + Expect(err).NotTo(HaveOccurred(), "Failed to verify restored snapshot from schedule=%s", scheduleName) + Expect(ok).Should(BeTrue(), "Snapshots are not restored from schedule=%s", scheduleName) + } + }) + }) + + Context("Restore Test in different namespace", func() { + AfterEach(func() { + By("Destroying Application and Volume") + err = app.DestroyApplication(app.BusyboxYaml, TargetedNs) + Expect(err).NotTo(HaveOccurred(), "Failed to destroy application in namespace=%s", TargetedNs) + err = openebs.Client.DeleteVolume(openebs.PVCYaml, TargetedNs) + Expect(err).NotTo(HaveOccurred(), "Failed to delete volume for namespace=%s", TargetedNs) + }) + + It("Restore from non-scheduled backup to different Namespace", func() { + var status v1.RestorePhase + + By("Restoring from a non-scheduled backup to a different namespace") + status, err = velero.Client.CreateRestore(AppNs, TargetedNs, backupName) + if err != nil || status != v1.RestorePhaseCompleted { + dumpLogs() + } + + Expect(err).NotTo(HaveOccurred(), "Failed to create a restore from backup=%s", backupName) + Expect(status).To(Equal(v1.RestorePhaseCompleted), "Restore from backup=%s failed", backupName) + + By("Checking if restored PVC is bound or not") + phase, perr := k8s.Client.GetPVCPhase(app.PVCName, TargetedNs) + Expect(perr).NotTo(HaveOccurred(), "Failed to verify PVC=%s bound status for namespace=%s", app.PVCName, TargetedNs) + Expect(phase).To(Equal(corev1.ClaimBound), "PVC=%s not bound", app.PVCName) - By("Checking if restore has created Snapshot or not") + By("Checking if restored CVR are in error state") + ok := openebs.Client.CheckCVRStatus(app.PVCName, TargetedNs, v1alpha1.CVRStatusError) + Expect(ok).To(BeTrue(), "CVR for PVC=%s is not in errored state", app.PVCName) + }) + + It("Restore from scheduled backup to different Namespace", func() { + var status v1.RestorePhase + + By("Restoring from a scheduled backup to a different namespace") + status, err = velero.Client.CreateRestoreFromSchedule(AppNs, TargetedNs, scheduleName, 0) + if err != nil || status != v1.RestorePhaseCompleted { + dumpLogs() + } + Expect(err).NotTo(HaveOccurred(), "Failed to create a restore from schedule=%s", scheduleName) + Expect(status).To(Equal(v1.RestorePhaseCompleted), "Restore from schedule=%s failed", scheduleName) + + By("Checking if restored PVC is bound or not") + phase, err := k8s.Client.GetPVCPhase(app.PVCName, TargetedNs) + Expect(err).NotTo(HaveOccurred(), "Failed to verify PVC=%s bound status for namespace=%s", app.PVCName, TargetedNs) + Expect(phase).To(Equal(corev1.ClaimBound), "PVC=%s not bound", app.PVCName) + + By("Checking if restored CVR are in error state") + ok := openebs.Client.CheckCVRStatus(app.PVCName, TargetedNs, v1alpha1.CVRStatusError) + Expect(ok).To(BeTrue(), "CVR for PVC=%s is not in errored state", app.PVCName) + + By("Checking if restore has created snapshot or not") snapshotList, err := velero.Client.GetRestoredSnapshotFromSchedule(scheduleName) Expect(err).NotTo(HaveOccurred()) for snapshot := range snapshotList { - ok, err := openebs.Client.CheckSnapshot(app.PVCName, AppNs, snapshot) - Expect(err).NotTo(HaveOccurred()) - Expect(ok).Should(BeTrue()) + ok, err := openebs.Client.CheckSnapshot(app.PVCName, TargetedNs, snapshot) + if err != nil { + dumpLogs() + } + Expect(err).NotTo(HaveOccurred(), "Failed to verify restored snapshot from schedule=%s", scheduleName) + Expect(ok).Should(BeTrue(), "Snapshots are not restored from schedule=%s", scheduleName) } }) }) }) + +func dumpLogs() { + velero.Client.DumpLogs() + openebs.Client.DumpLogs() +} diff --git a/tests/velero/logs.go b/tests/velero/logs.go index cb1b026b..a933d04f 100644 --- a/tests/velero/logs.go +++ b/tests/velero/logs.go @@ -17,11 +17,14 @@ limitations under the License. package velero import ( + "fmt" "os" "time" + "github.com/openebs/velero-plugin/tests/k8s" v1 "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" log "github.com/vmware-tanzu/velero/pkg/cmd/util/downloadrequest" + corev1 "k8s.io/api/core/v1" ) // DumpBackupLogs dump logs of given backup on stdout @@ -33,3 +36,38 @@ func (c *ClientSet) DumpBackupLogs(backupName string) error { os.Stdout, time.Minute, false) } + +// DumpLogs dump logs of velero pod on stdout +func (c *ClientSet) DumpLogs() { + veleroPod := c.getPodName() + + for _, v := range veleroPod { + if err := k8s.Client.DumpLogs(VeleroNamespace, v[0], v[1]); err != nil { + fmt.Printf("Failed to dump velero logs, err=%s\n", err) + } + } +} + +// getPodName return velero pod name and container name +// {{"pod_1","container_1"},{"pod_2","container_2"},} +func (c *ClientSet) getPodName() [][]string { + podList, err := k8s.Client.GetPodList(VeleroNamespace, + "deploy=velero", + ) + if err != nil { + return [][]string{} + } + return getPodContainerList(podList) +} + +// returns {{"pod1","container1"},{"pod2","container2"},} +func getPodContainerList(podList *corev1.PodList) [][]string { + pod := make([][]string, 0) + + for _, p := range podList.Items { + for _, c := range p.Spec.Containers { + pod = append(pod, []string{p.Name, c.Name}) + } + } + return pod +} diff --git a/tests/velero/restore.go b/tests/velero/restore.go index faa6b7ed..bf74b50a 100644 --- a/tests/velero/restore.go +++ b/tests/velero/restore.go @@ -79,7 +79,7 @@ func (c *ClientSet) GetScheduledBackups(schedule string) ([]string, error) { } // CreateRestoreFromSchedule restore from given schedule's n'th backup for ns Namespace -func (c *ClientSet) CreateRestoreFromSchedule(ns, schedule string, n int) (v1.RestorePhase, error) { +func (c *ClientSet) CreateRestoreFromSchedule(ns, targetedNs, schedule string, n int) (v1.RestorePhase, error) { var status v1.RestorePhase var err error @@ -89,17 +89,22 @@ func (c *ClientSet) CreateRestoreFromSchedule(ns, schedule string, n int) (v1.Re } bkplist = bkplist[n:] for _, bkp := range bkplist { - if status, err = c.CreateRestore(ns, bkp); status != v1.RestorePhaseCompleted { + if status, err = c.CreateRestore(ns, targetedNs, bkp); status != v1.RestorePhaseCompleted { break } } return status, err } -// CreateRestore create restore from given backup for ns Namespace -func (c *ClientSet) CreateRestore(ns, backup string) (v1.RestorePhase, error) { +// CreateRestore create restore from given backup for ns Namespace to targetedNs +func (c *ClientSet) CreateRestore(ns, targetedNs, backup string) (v1.RestorePhase, error) { var status v1.RestorePhase snapVolume := true + nsMapping := make(map[string]string) + + if targetedNs != "" && ns != targetedNs { + nsMapping[ns] = targetedNs + } restoreName, err := c.generateRestoreName(backup) if err != nil { @@ -115,6 +120,7 @@ func (c *ClientSet) CreateRestore(ns, backup string) (v1.RestorePhase, error) { IncludedNamespaces: []string{ns}, RestorePVs: &snapVolume, BackupName: backup, + NamespaceMapping: nsMapping, }, } o, err := c.VeleroV1().