From 4aff46228d4c647af0798bb79c9cc4e36c93dea4 Mon Sep 17 00:00:00 2001 From: David Zaninovic Date: Thu, 27 Jul 2023 16:12:13 -0400 Subject: [PATCH] Add support for block volumes Signed-off-by: David Zaninovic --- changelogs/unreleased/6680-dzaninovic | 1 + .../CLI/PoC/overlays/plugins/node-agent.yaml | 6 + .../volume-snapshot-data-movement.md | 4 +- pkg/builder/persistent_volume_builder.go | 6 + pkg/cmd/cli/install/install.go | 3 + pkg/controller/data_upload_controller.go | 28 ++++- pkg/controller/data_upload_controller_test.go | 18 ++- pkg/datapath/file_system.go | 15 +-- pkg/exposer/csi_snapshot.go | 28 ++++- pkg/exposer/generic_restore.go | 26 ++++- pkg/exposer/host_path.go | 18 ++- pkg/exposer/host_path_test.go | 38 ++++-- pkg/exposer/types.go | 1 + pkg/install/daemonset.go | 17 ++- pkg/install/daemonset_test.go | 2 +- pkg/install/deployment.go | 7 ++ pkg/install/resources.go | 4 + pkg/uploader/kopia/block_backup.go | 52 +++++++++ pkg/uploader/kopia/block_restore.go | 110 ++++++++++++++++++ pkg/uploader/kopia/snapshot.go | 52 +++++---- pkg/uploader/kopia/snapshot_test.go | 80 ++++++++++++- pkg/uploader/provider/kopia.go | 11 +- pkg/uploader/provider/kopia_test.go | 20 ++-- pkg/uploader/provider/restic_test.go | 3 + pkg/util/kube/utils.go | 29 +++-- pkg/util/kube/utils_test.go | 9 +- .../docs/main/customize-installation.md | 4 + site/content/docs/main/file-system-backup.md | 4 +- tilt-resources/examples/node-agent.yaml | 6 + 29 files changed, 503 insertions(+), 99 deletions(-) create mode 100644 changelogs/unreleased/6680-dzaninovic create mode 100644 pkg/uploader/kopia/block_backup.go create mode 100644 pkg/uploader/kopia/block_restore.go diff --git a/changelogs/unreleased/6680-dzaninovic b/changelogs/unreleased/6680-dzaninovic new file mode 100644 index 00000000000..b4d735d1452 --- /dev/null +++ b/changelogs/unreleased/6680-dzaninovic @@ -0,0 +1 @@ +Add support for block volumes with Kopia \ No newline at end of file diff --git a/design/CLI/PoC/overlays/plugins/node-agent.yaml b/design/CLI/PoC/overlays/plugins/node-agent.yaml index dbb4ce18db6..0db26e2c54d 100644 --- a/design/CLI/PoC/overlays/plugins/node-agent.yaml +++ b/design/CLI/PoC/overlays/plugins/node-agent.yaml @@ -49,6 +49,9 @@ spec: - mountPath: /host_pods mountPropagation: HostToContainer name: host-pods + - mountPath: /var/lib/kubelet/plugins + mountPropagation: HostToContainer + name: host-plugins - mountPath: /scratch name: scratch - mountPath: /credentials @@ -60,6 +63,9 @@ spec: - hostPath: path: /var/lib/kubelet/pods name: host-pods + - hostPath: + path: /var/lib/kubelet/plugins + name: host-plugins - emptyDir: {} name: scratch - name: cloud-credentials diff --git a/design/volume-snapshot-data-movement/volume-snapshot-data-movement.md b/design/volume-snapshot-data-movement/volume-snapshot-data-movement.md index d006b29e21f..a9929c7714d 100644 --- a/design/volume-snapshot-data-movement/volume-snapshot-data-movement.md +++ b/design/volume-snapshot-data-movement/volume-snapshot-data-movement.md @@ -703,7 +703,7 @@ type Provider interface { In this case, we will extend the default kopia uploader to add the ability, when a given volume is for a block mode and is mapped as a device, we will use the [StreamingFile](https://pkg.go.dev/github.com/kopia/kopia@v0.13.0/fs#StreamingFile) to stream the device and backup to the kopia repository. ```go -func getLocalBlockEntry(kopiaEntry fs.Entry, log logrus.FieldLogger) (fs.Entry, error) { +func getLocalBlockEntry(kopiaEntry fs.Entry) (fs.Entry, error) { path := kopiaEntry.LocalFilesystemPath() fileInfo, err := os.Lstat(path) @@ -748,7 +748,7 @@ In the `pkg/uploader/kopia/snapshot.go` this is used in the Backup call like } if volMode == PersistentVolumeBlock { - sourceEntry, err = getLocalBlockEntry(sourceEntry, log) + sourceEntry, err = getLocalBlockEntry(sourceEntry) if err != nil { return nil, false, errors.Wrap(err, "Unable to get local block device entry") } diff --git a/pkg/builder/persistent_volume_builder.go b/pkg/builder/persistent_volume_builder.go index 5fee88c1964..4cf2e47f208 100644 --- a/pkg/builder/persistent_volume_builder.go +++ b/pkg/builder/persistent_volume_builder.go @@ -95,6 +95,12 @@ func (b *PersistentVolumeBuilder) StorageClass(name string) *PersistentVolumeBui return b } +// VolumeMode sets the PersistentVolume's volume mode. +func (b *PersistentVolumeBuilder) VolumeMode(volMode corev1api.PersistentVolumeMode) *PersistentVolumeBuilder { + b.object.Spec.VolumeMode = &volMode + return b +} + // NodeAffinityRequired sets the PersistentVolume's NodeAffinity Requirement. func (b *PersistentVolumeBuilder) NodeAffinityRequired(req *corev1api.NodeSelector) *PersistentVolumeBuilder { b.object.Spec.NodeAffinity = &corev1api.VolumeNodeAffinity{ diff --git a/pkg/cmd/cli/install/install.go b/pkg/cmd/cli/install/install.go index 68bb89ea29c..6f458f1ab85 100644 --- a/pkg/cmd/cli/install/install.go +++ b/pkg/cmd/cli/install/install.go @@ -66,6 +66,7 @@ type Options struct { BackupStorageConfig flag.Map VolumeSnapshotConfig flag.Map UseNodeAgent bool + PrivilegedAgent bool //TODO remove UseRestic when migration test out of using it UseRestic bool Wait bool @@ -110,6 +111,7 @@ func (o *Options) BindFlags(flags *pflag.FlagSet) { flags.BoolVar(&o.RestoreOnly, "restore-only", o.RestoreOnly, "Run the server in restore-only mode. Optional.") flags.BoolVar(&o.DryRun, "dry-run", o.DryRun, "Generate resources, but don't send them to the cluster. Use with -o. Optional.") flags.BoolVar(&o.UseNodeAgent, "use-node-agent", o.UseNodeAgent, "Create Velero node-agent daemonset. Optional. Velero node-agent hosts Velero modules that need to run in one or more nodes(i.e. Restic, Kopia).") + flags.BoolVar(&o.PrivilegedAgent, "privileged-agent", o.PrivilegedAgent, "Use privileged mode for the node agent. Optional. Required to backup block devices.") flags.BoolVar(&o.Wait, "wait", o.Wait, "Wait for Velero deployment to be ready. Optional.") flags.DurationVar(&o.DefaultRepoMaintenanceFrequency, "default-repo-maintain-frequency", o.DefaultRepoMaintenanceFrequency, "How often 'maintain' is run for backup repositories by default. Optional.") flags.DurationVar(&o.GarbageCollectionFrequency, "garbage-collection-frequency", o.GarbageCollectionFrequency, "How often the garbage collection runs for expired backups.(default 1h)") @@ -198,6 +200,7 @@ func (o *Options) AsVeleroOptions() (*install.VeleroOptions, error) { SecretData: secretData, RestoreOnly: o.RestoreOnly, UseNodeAgent: o.UseNodeAgent, + PrivilegedAgent: o.PrivilegedAgent, UseVolumeSnapshots: o.UseVolumeSnapshots, BSLConfig: o.BackupStorageConfig.Data(), VSLConfig: o.VolumeSnapshotConfig.Data(), diff --git a/pkg/controller/data_upload_controller.go b/pkg/controller/data_upload_controller.go index d16951075fc..54da751db8d 100644 --- a/pkg/controller/data_upload_controller.go +++ b/pkg/controller/data_upload_controller.go @@ -177,7 +177,10 @@ func (r *DataUploadReconciler) Reconcile(ctx context.Context, req ctrl.Request) return ctrl.Result{}, nil } - exposeParam := r.setupExposeParam(du) + exposeParam, err := r.setupExposeParam(du) + if err != nil { + return r.errorOut(ctx, du, err, "failed to set exposer parameters", log) + } // Expose() will trigger to create one pod whose volume is restored by a given volume snapshot, // but the pod maybe is not in the same node of the current controller, so we need to return it here. @@ -733,18 +736,33 @@ func (r *DataUploadReconciler) closeDataPath(ctx context.Context, duName string) r.dataPathMgr.RemoveAsyncBR(duName) } -func (r *DataUploadReconciler) setupExposeParam(du *velerov2alpha1api.DataUpload) interface{} { +func (r *DataUploadReconciler) setupExposeParam(du *velerov2alpha1api.DataUpload) (interface{}, error) { if du.Spec.SnapshotType == velerov2alpha1api.SnapshotTypeCSI { + pvc := &corev1.PersistentVolumeClaim{} + err := r.client.Get(context.Background(), types.NamespacedName{ + Namespace: du.Spec.SourceNamespace, + Name: du.Spec.SourcePVC, + }, pvc) + + if err != nil { + return nil, errors.Wrapf(err, "failed to get PVC %s/%s", du.Spec.SourceNamespace, du.Spec.SourcePVC) + } + + accessMode := exposer.AccessModeFileSystem + if pvc.Spec.VolumeMode != nil && *pvc.Spec.VolumeMode == corev1.PersistentVolumeBlock { + accessMode = exposer.AccessModeBlock + } + return &exposer.CSISnapshotExposeParam{ SnapshotName: du.Spec.CSISnapshot.VolumeSnapshot, SourceNamespace: du.Spec.SourceNamespace, StorageClass: du.Spec.CSISnapshot.StorageClass, HostingPodLabels: map[string]string{velerov1api.DataUploadLabel: du.Name}, - AccessMode: exposer.AccessModeFileSystem, + AccessMode: accessMode, Timeout: du.Spec.OperationTimeout.Duration, - } + }, nil } - return nil + return nil, nil } func (r *DataUploadReconciler) setupWaitExposePara(du *velerov2alpha1api.DataUpload) interface{} { diff --git a/pkg/controller/data_upload_controller_test.go b/pkg/controller/data_upload_controller_test.go index 270a084ad9f..34bc4a6aaed 100644 --- a/pkg/controller/data_upload_controller_test.go +++ b/pkg/controller/data_upload_controller_test.go @@ -306,6 +306,7 @@ func TestReconcile(t *testing.T) { name string du *velerov2alpha1api.DataUpload pod *corev1.Pod + pvc *corev1.PersistentVolumeClaim snapshotExposerList map[velerov2alpha1api.SnapshotType]exposer.SnapshotExposer dataMgr *datapath.Manager expectedProcessed bool @@ -345,11 +346,21 @@ func TestReconcile(t *testing.T) { }, { name: "Dataupload should be accepted", du: dataUploadBuilder().Result(), - pod: builder.ForPod(velerov1api.DefaultNamespace, dataUploadName).Volumes(&corev1.Volume{Name: "dataupload-1"}).Result(), + pod: builder.ForPod("fake-ns", dataUploadName).Volumes(&corev1.Volume{Name: "test-pvc"}).Result(), + pvc: builder.ForPersistentVolumeClaim("fake-ns", "test-pvc").Result(), expectedProcessed: false, expected: dataUploadBuilder().Phase(velerov2alpha1api.DataUploadPhaseAccepted).Result(), expectedRequeue: ctrl.Result{}, }, + { + name: "Dataupload should fail to get PVC information", + du: dataUploadBuilder().Result(), + pod: builder.ForPod("fake-ns", dataUploadName).Volumes(&corev1.Volume{Name: "wrong-pvc"}).Result(), + expectedProcessed: true, + expected: dataUploadBuilder().Phase(velerov2alpha1api.DataUploadPhaseFailed).Result(), + expectedRequeue: ctrl.Result{}, + expectedErrMsg: "failed to get PVC", + }, { name: "Dataupload should be prepared", du: dataUploadBuilder().SnapshotType(fakeSnapshotType).Result(), @@ -448,6 +459,11 @@ func TestReconcile(t *testing.T) { require.NoError(t, err) } + if test.pvc != nil { + err = r.client.Create(ctx, test.pvc) + require.NoError(t, err) + } + if test.dataMgr != nil { r.dataPathMgr = test.dataMgr } else { diff --git a/pkg/datapath/file_system.go b/pkg/datapath/file_system.go index 741f6ae086d..881f735face 100644 --- a/pkg/datapath/file_system.go +++ b/pkg/datapath/file_system.go @@ -133,10 +133,11 @@ func (fs *fileSystemBR) StartBackup(source AccessPoint, realSource string, paren if !fs.initialized { return errors.New("file system data path is not initialized") } - volMode := getPersistentVolumeMode(source) + + volMode, sourcePath := getPersistentVolumeMode(source) go func() { - snapshotID, emptySnapshot, err := fs.uploaderProv.RunBackup(fs.ctx, source.ByPath, realSource, tags, forceFull, parentSnapshot, volMode, fs) + snapshotID, emptySnapshot, err := fs.uploaderProv.RunBackup(fs.ctx, sourcePath, realSource, tags, forceFull, parentSnapshot, volMode, fs) if err == provider.ErrorCanceled { fs.callbacks.OnCancelled(context.Background(), fs.namespace, fs.jobName) @@ -155,10 +156,10 @@ func (fs *fileSystemBR) StartRestore(snapshotID string, target AccessPoint) erro return errors.New("file system data path is not initialized") } - volMode := getPersistentVolumeMode(target) + volMode, targetPath := getPersistentVolumeMode(target) go func() { - err := fs.uploaderProv.RunRestore(fs.ctx, snapshotID, target.ByPath, volMode, fs) + err := fs.uploaderProv.RunRestore(fs.ctx, snapshotID, targetPath, volMode, fs) if err == provider.ErrorCanceled { fs.callbacks.OnCancelled(context.Background(), fs.namespace, fs.jobName) @@ -172,11 +173,11 @@ func (fs *fileSystemBR) StartRestore(snapshotID string, target AccessPoint) erro return nil } -func getPersistentVolumeMode(source AccessPoint) uploader.PersistentVolumeMode { +func getPersistentVolumeMode(source AccessPoint) (uploader.PersistentVolumeMode, string) { if source.ByBlock != "" { - return uploader.PersistentVolumeBlock + return uploader.PersistentVolumeBlock, source.ByBlock } - return uploader.PersistentVolumeFilesystem + return uploader.PersistentVolumeFilesystem, source.ByPath } // UpdateProgress which implement ProgressUpdater interface to update progress status diff --git a/pkg/exposer/csi_snapshot.go b/pkg/exposer/csi_snapshot.go index df9b085461a..61701dfa580 100644 --- a/pkg/exposer/csi_snapshot.go +++ b/pkg/exposer/csi_snapshot.go @@ -233,9 +233,12 @@ func (e *csiSnapshotExposer) CleanUp(ctx context.Context, ownerObject corev1.Obj } func getVolumeModeByAccessMode(accessMode string) (corev1.PersistentVolumeMode, error) { - if accessMode == AccessModeFileSystem { + switch accessMode { + case AccessModeFileSystem: return corev1.PersistentVolumeFilesystem, nil - } else { + case AccessModeBlock: + return corev1.PersistentVolumeBlock, nil + default: return "", errors.Errorf("unsupported access mode %s", accessMode) } } @@ -355,6 +358,21 @@ func (e *csiSnapshotExposer) createBackupPod(ctx context.Context, ownerObject co var gracePeriod int64 = 0 + var volumeMounts []corev1.VolumeMount = nil + var volumeDevices []corev1.VolumeDevice = nil + + if backupPVC.Spec.VolumeMode != nil && *backupPVC.Spec.VolumeMode == corev1.PersistentVolumeBlock { + volumeDevices = []corev1.VolumeDevice{{ + Name: volumeName, + DevicePath: "/" + volumeName, + }} + } else { + volumeMounts = []corev1.VolumeMount{{ + Name: volumeName, + MountPath: "/" + volumeName, + }} + } + pod := &corev1.Pod{ ObjectMeta: metav1.ObjectMeta{ Name: podName, @@ -377,10 +395,8 @@ func (e *csiSnapshotExposer) createBackupPod(ctx context.Context, ownerObject co Image: podInfo.image, ImagePullPolicy: corev1.PullNever, Command: []string{"/velero-helper", "pause"}, - VolumeMounts: []corev1.VolumeMount{{ - Name: volumeName, - MountPath: "/" + volumeName, - }}, + VolumeMounts: volumeMounts, + VolumeDevices: volumeDevices, }, }, ServiceAccountName: podInfo.serviceAccount, diff --git a/pkg/exposer/generic_restore.go b/pkg/exposer/generic_restore.go index ca5cd68a3fa..370d128f299 100644 --- a/pkg/exposer/generic_restore.go +++ b/pkg/exposer/generic_restore.go @@ -82,7 +82,7 @@ func (e *genericRestoreExposer) Expose(ctx context.Context, ownerObject corev1.O return errors.Errorf("Target PVC %s/%s has already been bound, abort", sourceNamespace, targetPVCName) } - restorePod, err := e.createRestorePod(ctx, ownerObject, hostingPodLabels, selectedNode) + restorePod, err := e.createRestorePod(ctx, ownerObject, targetPVC, hostingPodLabels, selectedNode) if err != nil { return errors.Wrapf(err, "error to create restore pod") } @@ -247,7 +247,8 @@ func (e *genericRestoreExposer) RebindVolume(ctx context.Context, ownerObject co return nil } -func (e *genericRestoreExposer) createRestorePod(ctx context.Context, ownerObject corev1.ObjectReference, label map[string]string, selectedNode string) (*corev1.Pod, error) { +func (e *genericRestoreExposer) createRestorePod(ctx context.Context, ownerObject corev1.ObjectReference, targetPVC *corev1.PersistentVolumeClaim, + label map[string]string, selectedNode string) (*corev1.Pod, error) { restorePodName := ownerObject.Name restorePVCName := ownerObject.Name @@ -261,6 +262,21 @@ func (e *genericRestoreExposer) createRestorePod(ctx context.Context, ownerObjec var gracePeriod int64 = 0 + var volumeMounts []corev1.VolumeMount = nil + var volumeDevices []corev1.VolumeDevice = nil + + if targetPVC.Spec.VolumeMode != nil && *targetPVC.Spec.VolumeMode == corev1.PersistentVolumeBlock { + volumeDevices = []corev1.VolumeDevice{{ + Name: volumeName, + DevicePath: "/" + volumeName, + }} + } else { + volumeMounts = []corev1.VolumeMount{{ + Name: volumeName, + MountPath: "/" + volumeName, + }} + } + pod := &corev1.Pod{ ObjectMeta: metav1.ObjectMeta{ Name: restorePodName, @@ -283,10 +299,8 @@ func (e *genericRestoreExposer) createRestorePod(ctx context.Context, ownerObjec Image: podInfo.image, ImagePullPolicy: corev1.PullNever, Command: []string{"/velero-helper", "pause"}, - VolumeMounts: []corev1.VolumeMount{{ - Name: volumeName, - MountPath: "/" + volumeName, - }}, + VolumeMounts: volumeMounts, + VolumeDevices: volumeDevices, }, }, ServiceAccountName: podInfo.serviceAccount, diff --git a/pkg/exposer/host_path.go b/pkg/exposer/host_path.go index 458667d9233..dd902d90d4f 100644 --- a/pkg/exposer/host_path.go +++ b/pkg/exposer/host_path.go @@ -26,6 +26,7 @@ import ( ctrlclient "sigs.k8s.io/controller-runtime/pkg/client" "github.com/vmware-tanzu/velero/pkg/datapath" + "github.com/vmware-tanzu/velero/pkg/uploader" "github.com/vmware-tanzu/velero/pkg/util/filesystem" "github.com/vmware-tanzu/velero/pkg/util/kube" ) @@ -38,14 +39,19 @@ func GetPodVolumeHostPath(ctx context.Context, pod *corev1.Pod, volumeName strin cli ctrlclient.Client, fs filesystem.Interface, log logrus.FieldLogger) (datapath.AccessPoint, error) { logger := log.WithField("pod name", pod.Name).WithField("pod UID", pod.GetUID()).WithField("volume", volumeName) - volDir, err := getVolumeDirectory(ctx, logger, pod, volumeName, cli) + volDir, volMode, err := getVolumeDirectory(ctx, logger, pod, volumeName, cli) if err != nil { return datapath.AccessPoint{}, errors.Wrapf(err, "error getting volume directory name for volume %s in pod %s", volumeName, pod.Name) } logger.WithField("volDir", volDir).Info("Got volume dir") - pathGlob := fmt.Sprintf("/host_pods/%s/volumes/*/%s", string(pod.GetUID()), volDir) + volSubDir := "volumes" + if volMode == uploader.PersistentVolumeBlock { + volSubDir = "volumeDevices" + } + + pathGlob := fmt.Sprintf("/host_pods/%s/%s/*/%s", string(pod.GetUID()), volSubDir, volDir) logger.WithField("pathGlob", pathGlob).Debug("Looking for path matching glob") path, err := singlePathMatch(pathGlob, fs, logger) @@ -55,7 +61,9 @@ func GetPodVolumeHostPath(ctx context.Context, pod *corev1.Pod, volumeName strin logger.WithField("path", path).Info("Found path matching glob") - return datapath.AccessPoint{ - ByPath: path, - }, nil + if volMode == uploader.PersistentVolumeBlock { + return datapath.AccessPoint{ByBlock: path}, nil + } + + return datapath.AccessPoint{ByPath: path}, nil } diff --git a/pkg/exposer/host_path_test.go b/pkg/exposer/host_path_test.go index f71518d2d78..cfda7c474b3 100644 --- a/pkg/exposer/host_path_test.go +++ b/pkg/exposer/host_path_test.go @@ -29,22 +29,25 @@ import ( velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" "github.com/vmware-tanzu/velero/pkg/builder" velerotest "github.com/vmware-tanzu/velero/pkg/test" + "github.com/vmware-tanzu/velero/pkg/uploader" "github.com/vmware-tanzu/velero/pkg/util/filesystem" ) func TestGetPodVolumeHostPath(t *testing.T) { tests := []struct { name string - getVolumeDirFunc func(context.Context, logrus.FieldLogger, *corev1.Pod, string, ctrlclient.Client) (string, error) - pathMatchFunc func(string, filesystem.Interface, logrus.FieldLogger) (string, error) - pod *corev1.Pod - pvc string - err string + getVolumeDirFunc func(context.Context, logrus.FieldLogger, *corev1.Pod, string, ctrlclient.Client) ( + string, uploader.PersistentVolumeMode, error) + pathMatchFunc func(string, filesystem.Interface, logrus.FieldLogger) (string, error) + pod *corev1.Pod + pvc string + err string }{ { name: "get volume dir fail", - getVolumeDirFunc: func(context.Context, logrus.FieldLogger, *corev1.Pod, string, ctrlclient.Client) (string, error) { - return "", errors.New("fake-error-1") + getVolumeDirFunc: func(context.Context, logrus.FieldLogger, *corev1.Pod, string, ctrlclient.Client) ( + string, uploader.PersistentVolumeMode, error) { + return "", "", errors.New("fake-error-1") }, pod: builder.ForPod(velerov1api.DefaultNamespace, "fake-pod-1").Result(), pvc: "fake-pvc-1", @@ -52,8 +55,9 @@ func TestGetPodVolumeHostPath(t *testing.T) { }, { name: "single path match fail", - getVolumeDirFunc: func(context.Context, logrus.FieldLogger, *corev1.Pod, string, ctrlclient.Client) (string, error) { - return "", nil + getVolumeDirFunc: func(context.Context, logrus.FieldLogger, *corev1.Pod, string, ctrlclient.Client) ( + string, uploader.PersistentVolumeMode, error) { + return "", "", nil }, pathMatchFunc: func(string, filesystem.Interface, logrus.FieldLogger) (string, error) { return "", errors.New("fake-error-2") @@ -62,6 +66,18 @@ func TestGetPodVolumeHostPath(t *testing.T) { pvc: "fake-pvc-1", err: "error identifying unique volume path on host for volume fake-pvc-1 in pod fake-pod-2: fake-error-2", }, + { + name: "get block volume dir success", + getVolumeDirFunc: func(context.Context, logrus.FieldLogger, *corev1.Pod, string, ctrlclient.Client) ( + string, uploader.PersistentVolumeMode, error) { + return "fake-pvc-1", uploader.PersistentVolumeBlock, nil + }, + pathMatchFunc: func(string, filesystem.Interface, logrus.FieldLogger) (string, error) { + return "/host_pods/fake-pod-1-id/volumeDevices/kubernetes.io~csi/fake-pvc-1-id", nil + }, + pod: builder.ForPod(velerov1api.DefaultNamespace, "fake-pod-1").Result(), + pvc: "fake-pvc-1", + }, } for _, test := range tests { @@ -75,7 +91,9 @@ func TestGetPodVolumeHostPath(t *testing.T) { } _, err := GetPodVolumeHostPath(context.Background(), test.pod, test.pvc, nil, nil, velerotest.NewLogger()) - assert.EqualError(t, err, test.err) + if test.err != "" || err != nil { + assert.EqualError(t, err, test.err) + } }) } } diff --git a/pkg/exposer/types.go b/pkg/exposer/types.go index 253256eb97f..21c473366df 100644 --- a/pkg/exposer/types.go +++ b/pkg/exposer/types.go @@ -22,6 +22,7 @@ import ( const ( AccessModeFileSystem = "by-file-system" + AccessModeBlock = "by-block-device" ) // ExposeResult defines the result of expose. diff --git a/pkg/install/daemonset.go b/pkg/install/daemonset.go index b139f81242d..43f7731dda4 100644 --- a/pkg/install/daemonset.go +++ b/pkg/install/daemonset.go @@ -86,6 +86,14 @@ func DaemonSet(namespace string, opts ...podTemplateOption) *appsv1.DaemonSet { }, }, }, + { + Name: "host-plugins", + VolumeSource: corev1.VolumeSource{ + HostPath: &corev1.HostPathVolumeSource{ + Path: "/var/lib/kubelet/plugins", + }, + }, + }, { Name: "scratch", VolumeSource: corev1.VolumeSource{ @@ -102,13 +110,20 @@ func DaemonSet(namespace string, opts ...podTemplateOption) *appsv1.DaemonSet { "/velero", }, Args: daemonSetArgs, - + SecurityContext: &corev1.SecurityContext{ + Privileged: &c.privilegedAgent, + }, VolumeMounts: []corev1.VolumeMount{ { Name: "host-pods", MountPath: "/host_pods", MountPropagation: &mountPropagationMode, }, + { + Name: "host-plugins", + MountPath: "/var/lib/kubelet/plugins", + MountPropagation: &mountPropagationMode, + }, { Name: "scratch", MountPath: "/scratch", diff --git a/pkg/install/daemonset_test.go b/pkg/install/daemonset_test.go index 762e95b16b3..017d5004d22 100644 --- a/pkg/install/daemonset_test.go +++ b/pkg/install/daemonset_test.go @@ -35,7 +35,7 @@ func TestDaemonSet(t *testing.T) { ds = DaemonSet("velero", WithSecret(true)) assert.Equal(t, 7, len(ds.Spec.Template.Spec.Containers[0].Env)) - assert.Equal(t, 3, len(ds.Spec.Template.Spec.Volumes)) + assert.Equal(t, 4, len(ds.Spec.Template.Spec.Volumes)) ds = DaemonSet("velero", WithFeatures([]string{"foo,bar,baz"})) assert.Len(t, ds.Spec.Template.Spec.Containers[0].Args, 3) diff --git a/pkg/install/deployment.go b/pkg/install/deployment.go index 993d4b16f38..aaf1367a396 100644 --- a/pkg/install/deployment.go +++ b/pkg/install/deployment.go @@ -47,6 +47,7 @@ type podTemplateConfig struct { serviceAccountName string uploaderType string defaultSnapshotMoveData bool + privilegedAgent bool } func WithImage(image string) podTemplateOption { @@ -149,6 +150,12 @@ func WithServiceAccountName(sa string) podTemplateOption { } } +func WithPrivilegedAgent() podTemplateOption { + return func(c *podTemplateConfig) { + c.privilegedAgent = true + } +} + func Deployment(namespace string, opts ...podTemplateOption) *appsv1.Deployment { // TODO: Add support for server args c := &podTemplateConfig{ diff --git a/pkg/install/resources.go b/pkg/install/resources.go index a029ecc4a5a..cd858c7ba65 100644 --- a/pkg/install/resources.go +++ b/pkg/install/resources.go @@ -240,6 +240,7 @@ type VeleroOptions struct { SecretData []byte RestoreOnly bool UseNodeAgent bool + PrivilegedAgent bool UseVolumeSnapshots bool BSLConfig map[string]string VSLConfig map[string]string @@ -374,6 +375,9 @@ func AllResources(o *VeleroOptions) *unstructured.UnstructuredList { if len(o.Features) > 0 { dsOpts = append(dsOpts, WithFeatures(o.Features)) } + if o.PrivilegedAgent { + dsOpts = append(dsOpts, WithPrivilegedAgent()) + } ds := DaemonSet(o.Namespace, dsOpts...) if err := appendUnstructured(resources, ds); err != nil { fmt.Printf("error appending DaemonSet %s: %s\n", ds.GetName(), err.Error()) diff --git a/pkg/uploader/kopia/block_backup.go b/pkg/uploader/kopia/block_backup.go new file mode 100644 index 00000000000..9a8755a31c7 --- /dev/null +++ b/pkg/uploader/kopia/block_backup.go @@ -0,0 +1,52 @@ +/* +Copyright The Velero Contributors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package kopia + +import ( + "os" + "syscall" + + "github.com/kopia/kopia/fs" + "github.com/kopia/kopia/fs/virtualfs" + "github.com/pkg/errors" +) + +const ErrNotPermitted = "operation not permitted" + +func getLocalBlockEntry(kopiaEntry fs.Entry) (fs.Entry, error) { + path := kopiaEntry.LocalFilesystemPath() + + fileInfo, err := os.Lstat(path) + if err != nil { + return nil, errors.Wrapf(err, "unable to get the source device information %s", path) + } + + if (fileInfo.Sys().(*syscall.Stat_t).Mode & syscall.S_IFMT) != syscall.S_IFBLK { + return nil, errors.Errorf("source path %s is not a block device", path) + } + + device, err := os.Open(path) + if err != nil { + if os.IsPermission(err) || err.Error() == ErrNotPermitted { + return nil, errors.Wrapf(err, "no permission to open the source device %s, make sure that node agent is running in privileged mode", path) + } + return nil, errors.Wrapf(err, "unable to open the source device %s", path) + } + + sf := virtualfs.StreamingFileFromReader(kopiaEntry.Name(), device) + return virtualfs.NewStaticDirectory(kopiaEntry.Name(), []fs.Entry{sf}), nil +} diff --git a/pkg/uploader/kopia/block_restore.go b/pkg/uploader/kopia/block_restore.go new file mode 100644 index 00000000000..5a1f5cafe80 --- /dev/null +++ b/pkg/uploader/kopia/block_restore.go @@ -0,0 +1,110 @@ +/* +Copyright The Velero Contributors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package kopia + +import ( + "context" + "io" + "os" + "path/filepath" + "syscall" + + "github.com/kopia/kopia/fs" + "github.com/kopia/kopia/snapshot/restore" + "github.com/pkg/errors" +) + +type BlockOutput struct { + *restore.FilesystemOutput +} + +var _ restore.Output = &BlockOutput{} + +const bufferSize = 128 * 1024 + +func (o *BlockOutput) WriteFile(ctx context.Context, relativePath string, remoteFile fs.File) error { + targetFileName, err := filepath.EvalSymlinks(o.TargetPath) + if err != nil { + return errors.Wrapf(err, "unable to evaluate symlinks for %s", targetFileName) + } + + fileInfo, err := os.Lstat(targetFileName) + if err != nil { + return errors.Wrapf(err, "unable to get the target device information for %s", targetFileName) + } + + if (fileInfo.Sys().(*syscall.Stat_t).Mode & syscall.S_IFMT) != syscall.S_IFBLK { + return errors.Errorf("target file %s is not a block device", targetFileName) + } + + remoteReader, err := remoteFile.Open(ctx) + if err != nil { + return errors.Wrapf(err, "failed to open remote file %s", remoteFile.Name()) + } + defer remoteReader.Close() + + targetFile, err := os.Create(targetFileName) + if err != nil { + return errors.Wrapf(err, "failed to open file %s", targetFileName) + } + defer targetFile.Close() + + buffer := make([]byte, bufferSize) + + readData := true + for readData { + bytesToWrite, err := remoteReader.Read(buffer) + if err != nil { + if err != io.EOF { + return errors.Wrapf(err, "failed to read data from remote file %s", targetFileName) + } + readData = false + } + + if bytesToWrite > 0 { + offset := 0 + for bytesToWrite > 0 { + if bytesWritten, err := targetFile.Write(buffer[offset:bytesToWrite]); err == nil { + bytesToWrite -= bytesWritten + offset += bytesWritten + } else { + return errors.Wrapf(err, "failed to write data to file %s", targetFileName) + } + } + } + } + + return nil +} + +func (o *BlockOutput) BeginDirectory(ctx context.Context, relativePath string, e fs.Directory) error { + targetFileName, err := filepath.EvalSymlinks(o.TargetPath) + if err != nil { + return errors.Wrapf(err, "unable to evaluate symlinks for %s", targetFileName) + } + + fileInfo, err := os.Lstat(targetFileName) + if err != nil { + return errors.Wrapf(err, "unable to get the target device information for %s", o.TargetPath) + } + + if (fileInfo.Sys().(*syscall.Stat_t).Mode & syscall.S_IFMT) != syscall.S_IFBLK { + return errors.Errorf("target file %s is not a block device", o.TargetPath) + } + + return nil +} diff --git a/pkg/uploader/kopia/snapshot.go b/pkg/uploader/kopia/snapshot.go index acb4c00e80b..43e4dfdf988 100644 --- a/pkg/uploader/kopia/snapshot.go +++ b/pkg/uploader/kopia/snapshot.go @@ -121,25 +121,22 @@ func Backup(ctx context.Context, fsUploader SnapshotUploader, repoWriter repo.Re if fsUploader == nil { return nil, false, errors.New("get empty kopia uploader") } - - if volMode == uploader.PersistentVolumeBlock { - return nil, false, errors.New("unable to handle block storage") - } - - dir, err := filepath.Abs(sourcePath) + source, err := filepath.Abs(sourcePath) if err != nil { return nil, false, errors.Wrapf(err, "Invalid source path '%s'", sourcePath) } - // to be consistent with restic when backup empty dir returns one error for upper logic handle - dirs, err := os.ReadDir(dir) - if err != nil { - return nil, false, errors.Wrapf(err, "Unable to read dir in path %s", dir) - } else if len(dirs) == 0 { - return nil, true, nil + if volMode == uploader.PersistentVolumeFilesystem { + // to be consistent with restic when backup empty dir returns one error for upper logic handle + dirs, err := os.ReadDir(source) + if err != nil { + return nil, false, errors.Wrapf(err, "Unable to read dir in path %s", source) + } else if len(dirs) == 0 { + return nil, true, nil + } } - dir = filepath.Clean(dir) + source = filepath.Clean(source) sourceInfo := snapshot.SourceInfo{ UserName: udmrepo.GetRepoUser(), @@ -147,16 +144,23 @@ func Backup(ctx context.Context, fsUploader SnapshotUploader, repoWriter repo.Re Path: filepath.Clean(realSource), } if realSource == "" { - sourceInfo.Path = dir + sourceInfo.Path = source } - rootDir, err := getLocalFSEntry(dir) + sourceEntry, err := getLocalFSEntry(source) if err != nil { - return nil, false, errors.Wrap(err, "Unable to get local filesystem entry") + return nil, false, errors.Wrap(err, "unable to get local filesystem entry") + } + + if volMode == uploader.PersistentVolumeBlock { + sourceEntry, err = getLocalBlockEntry(sourceEntry) + if err != nil { + return nil, false, errors.Wrap(err, "unable to get local block device entry") + } } kopiaCtx := kopia.SetupKopiaLog(ctx, log) - snapID, snapshotSize, err := SnapshotSource(kopiaCtx, repoWriter, fsUploader, sourceInfo, rootDir, forceFull, parentSnapshot, tags, log, "Kopia Uploader") + snapID, snapshotSize, err := SnapshotSource(kopiaCtx, repoWriter, fsUploader, sourceInfo, sourceEntry, forceFull, parentSnapshot, tags, log, "Kopia Uploader") if err != nil { return nil, false, err } @@ -337,7 +341,8 @@ func findPreviousSnapshotManifest(ctx context.Context, rep repo.Repository, sour } // Restore restore specific sourcePath with given snapshotID and update progress -func Restore(ctx context.Context, rep repo.RepositoryWriter, progress *Progress, snapshotID, dest string, log logrus.FieldLogger, cancleCh chan struct{}) (int64, int32, error) { +func Restore(ctx context.Context, rep repo.RepositoryWriter, progress *Progress, snapshotID, dest string, volMode uploader.PersistentVolumeMode, + log logrus.FieldLogger, cancleCh chan struct{}) (int64, int32, error) { log.Info("Start to restore...") kopiaCtx := kopia.SetupKopiaLog(ctx, log) @@ -359,7 +364,7 @@ func Restore(ctx context.Context, rep repo.RepositoryWriter, progress *Progress, return 0, 0, errors.Wrapf(err, "Unable to resolve path %v", dest) } - output := &restore.FilesystemOutput{ + fsOutput := &restore.FilesystemOutput{ TargetPath: path, OverwriteDirectories: true, OverwriteFiles: true, @@ -367,11 +372,18 @@ func Restore(ctx context.Context, rep repo.RepositoryWriter, progress *Progress, IgnorePermissionErrors: true, } - err = output.Init(ctx) + err = fsOutput.Init(ctx) if err != nil { return 0, 0, errors.Wrap(err, "error to init output") } + var output restore.Output = fsOutput + if volMode == uploader.PersistentVolumeBlock { + output = &BlockOutput{ + fsOutput, + } + } + stat, err := restoreEntryFunc(kopiaCtx, rep, output, rootEntry, restore.Options{ Parallel: runtime.NumCPU(), RestoreDirEntryAtDepth: math.MaxInt32, diff --git a/pkg/uploader/kopia/snapshot_test.go b/pkg/uploader/kopia/snapshot_test.go index 232ea92c4d2..d4234868317 100644 --- a/pkg/uploader/kopia/snapshot_test.go +++ b/pkg/uploader/kopia/snapshot_test.go @@ -23,6 +23,7 @@ import ( "time" "github.com/kopia/kopia/fs" + "github.com/kopia/kopia/fs/virtualfs" "github.com/kopia/kopia/repo" "github.com/kopia/kopia/repo/manifest" "github.com/kopia/kopia/snapshot" @@ -594,11 +595,11 @@ func TestBackup(t *testing.T) { expectedError: errors.New("Unable to read dir"), }, { - name: "Unable to handle block mode", + name: "Source path is not a block device", sourcePath: "/", tags: nil, volMode: uploader.PersistentVolumeBlock, - expectedError: errors.New("unable to handle block storage"), + expectedError: errors.New("source path / is not a block device"), }, } @@ -660,6 +661,7 @@ func TestRestore(t *testing.T) { expectedBytes int64 expectedCount int32 expectedError error + volMode uploader.PersistentVolumeMode } // Define test cases @@ -697,6 +699,74 @@ func TestRestore(t *testing.T) { snapshotID: "snapshot-123", expectedError: nil, }, + { + name: "Expect block volume successful", + filesystemEntryFunc: func(ctx context.Context, rep repo.Repository, rootID string, consistentAttributes bool) (fs.Entry, error) { + return snapshotfs.EntryFromDirEntry(rep, &snapshot.DirEntry{Type: snapshot.EntryTypeFile}), nil + }, + restoreEntryFunc: func(ctx context.Context, rep repo.Repository, output restore.Output, rootEntry fs.Entry, options restore.Options) (restore.Stats, error) { + return restore.Stats{}, nil + }, + snapshotID: "snapshot-123", + expectedError: nil, + volMode: uploader.PersistentVolumeBlock, + }, + { + name: "Unable to evaluate symlinks for block volume", + filesystemEntryFunc: func(ctx context.Context, rep repo.Repository, rootID string, consistentAttributes bool) (fs.Entry, error) { + return snapshotfs.EntryFromDirEntry(rep, &snapshot.DirEntry{Type: snapshot.EntryTypeFile}), nil + }, + restoreEntryFunc: func(ctx context.Context, rep repo.Repository, output restore.Output, rootEntry fs.Entry, options restore.Options) (restore.Stats, error) { + err := output.BeginDirectory(ctx, "fake-dir", virtualfs.NewStaticDirectory("fake-dir", nil)) + return restore.Stats{}, err + }, + snapshotID: "snapshot-123", + expectedError: errors.New("unable to evaluate symlinks for"), + volMode: uploader.PersistentVolumeBlock, + dest: "/wrong-dest", + }, + { + name: "Target file is not a block device", + filesystemEntryFunc: func(ctx context.Context, rep repo.Repository, rootID string, consistentAttributes bool) (fs.Entry, error) { + return snapshotfs.EntryFromDirEntry(rep, &snapshot.DirEntry{Type: snapshot.EntryTypeFile}), nil + }, + restoreEntryFunc: func(ctx context.Context, rep repo.Repository, output restore.Output, rootEntry fs.Entry, options restore.Options) (restore.Stats, error) { + err := output.BeginDirectory(ctx, "fake-dir", virtualfs.NewStaticDirectory("fake-dir", nil)) + return restore.Stats{}, err + }, + snapshotID: "snapshot-123", + expectedError: errors.New("target file /tmp is not a block device"), + volMode: uploader.PersistentVolumeBlock, + dest: "/tmp", + }, + { + name: "Unable to evaluate symlinks for block volume in WriteFile()", + filesystemEntryFunc: func(ctx context.Context, rep repo.Repository, rootID string, consistentAttributes bool) (fs.Entry, error) { + return snapshotfs.EntryFromDirEntry(rep, &snapshot.DirEntry{Type: snapshot.EntryTypeFile}), nil + }, + restoreEntryFunc: func(ctx context.Context, rep repo.Repository, output restore.Output, rootEntry fs.Entry, options restore.Options) (restore.Stats, error) { + err := output.WriteFile(ctx, "wrong-file", nil) + return restore.Stats{}, err + }, + snapshotID: "snapshot-123", + expectedError: errors.New("unable to evaluate symlinks for"), + volMode: uploader.PersistentVolumeBlock, + dest: "/wrong-dest", + }, + { + name: "Target file is not a block device in WriteFile()", + filesystemEntryFunc: func(ctx context.Context, rep repo.Repository, rootID string, consistentAttributes bool) (fs.Entry, error) { + return snapshotfs.EntryFromDirEntry(rep, &snapshot.DirEntry{Type: snapshot.EntryTypeFile}), nil + }, + restoreEntryFunc: func(ctx context.Context, rep repo.Repository, output restore.Output, rootEntry fs.Entry, options restore.Options) (restore.Stats, error) { + err := output.WriteFile(ctx, "wrong-file", nil) + return restore.Stats{}, err + }, + snapshotID: "snapshot-123", + expectedError: errors.New("target file /tmp is not a block device"), + volMode: uploader.PersistentVolumeBlock, + dest: "/tmp", + }, } em := &manifest.EntryMetadata{ @@ -706,6 +776,10 @@ func TestRestore(t *testing.T) { for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { + if tc.volMode == "" { + tc.volMode = uploader.PersistentVolumeFilesystem + } + if tc.invalidManifestType { em.Labels[manifest.TypeLabelKey] = "" } else { @@ -725,7 +799,7 @@ func TestRestore(t *testing.T) { repoWriterMock.On("OpenObject", mock.Anything, mock.Anything).Return(em, nil) progress := new(Progress) - bytesRestored, fileCount, err := Restore(context.Background(), repoWriterMock, progress, tc.snapshotID, tc.dest, logrus.New(), nil) + bytesRestored, fileCount, err := Restore(context.Background(), repoWriterMock, progress, tc.snapshotID, tc.dest, tc.volMode, logrus.New(), nil) // Check if the returned error matches the expected error if tc.expectedError != nil { diff --git a/pkg/uploader/provider/kopia.go b/pkg/uploader/provider/kopia.go index 37882859ecb..e130e26da21 100644 --- a/pkg/uploader/provider/kopia.go +++ b/pkg/uploader/provider/kopia.go @@ -128,11 +128,6 @@ func (kp *kopiaProvider) RunBackup( return "", false, errors.New("path is empty") } - // For now, error on block mode - if volMode == uploader.PersistentVolumeBlock { - return "", false, errors.New("unable to currently support block mode") - } - log := kp.log.WithFields(logrus.Fields{ "path": path, "realSource": realSource, @@ -210,10 +205,6 @@ func (kp *kopiaProvider) RunRestore( "volumePath": volumePath, }) - if volMode == uploader.PersistentVolumeBlock { - return errors.New("unable to currently support block mode") - } - repoWriter := kopia.NewShimRepo(kp.bkRepo) progress := new(kopia.Progress) progress.InitThrottle(restoreProgressCheckInterval) @@ -231,7 +222,7 @@ func (kp *kopiaProvider) RunRestore( // We use the cancel channel to control the restore cancel, so don't pass a context with cancel to Kopia restore. // Otherwise, Kopia restore will not response to the cancel control but return an arbitrary error. // Kopia restore cancel is not designed as well as Kopia backup which uses the context to control backup cancel all the way. - size, fileCount, err := RestoreFunc(context.Background(), repoWriter, progress, snapshotID, volumePath, log, restoreCancel) + size, fileCount, err := RestoreFunc(context.Background(), repoWriter, progress, snapshotID, volumePath, volMode, log, restoreCancel) if err != nil { return errors.Wrapf(err, "Failed to run kopia restore") diff --git a/pkg/uploader/provider/kopia_test.go b/pkg/uploader/provider/kopia_test.go index 944cdbcceb8..c38d370ce35 100644 --- a/pkg/uploader/provider/kopia_test.go +++ b/pkg/uploader/provider/kopia_test.go @@ -94,12 +94,12 @@ func TestRunBackup(t *testing.T) { notError: false, }, { - name: "error on vol mode", + name: "success to backup block mode volume", hookBackupFunc: func(ctx context.Context, fsUploader kopia.SnapshotUploader, repoWriter repo.RepositoryWriter, sourcePath string, realSource string, forceFull bool, parentSnapshot string, volMode uploader.PersistentVolumeMode, tags map[string]string, log logrus.FieldLogger) (*uploader.SnapshotInfo, bool, error) { - return nil, true, nil + return &uploader.SnapshotInfo{}, false, nil }, volMode: uploader.PersistentVolumeBlock, - notError: false, + notError: true, }, } for _, tc := range testCases { @@ -125,31 +125,31 @@ func TestRunRestore(t *testing.T) { testCases := []struct { name string - hookRestoreFunc func(ctx context.Context, rep repo.RepositoryWriter, progress *kopia.Progress, snapshotID, dest string, log logrus.FieldLogger, cancleCh chan struct{}) (int64, int32, error) + hookRestoreFunc func(ctx context.Context, rep repo.RepositoryWriter, progress *kopia.Progress, snapshotID, dest string, volMode uploader.PersistentVolumeMode, log logrus.FieldLogger, cancleCh chan struct{}) (int64, int32, error) notError bool volMode uploader.PersistentVolumeMode }{ { name: "normal restore", - hookRestoreFunc: func(ctx context.Context, rep repo.RepositoryWriter, progress *kopia.Progress, snapshotID, dest string, log logrus.FieldLogger, cancleCh chan struct{}) (int64, int32, error) { + hookRestoreFunc: func(ctx context.Context, rep repo.RepositoryWriter, progress *kopia.Progress, snapshotID, dest string, volMode uploader.PersistentVolumeMode, log logrus.FieldLogger, cancleCh chan struct{}) (int64, int32, error) { return 0, 0, nil }, notError: true, }, { name: "failed to restore", - hookRestoreFunc: func(ctx context.Context, rep repo.RepositoryWriter, progress *kopia.Progress, snapshotID, dest string, log logrus.FieldLogger, cancleCh chan struct{}) (int64, int32, error) { + hookRestoreFunc: func(ctx context.Context, rep repo.RepositoryWriter, progress *kopia.Progress, snapshotID, dest string, volMode uploader.PersistentVolumeMode, log logrus.FieldLogger, cancleCh chan struct{}) (int64, int32, error) { return 0, 0, errors.New("failed to restore") }, notError: false, }, { - name: "failed to restore block mode", - hookRestoreFunc: func(ctx context.Context, rep repo.RepositoryWriter, progress *kopia.Progress, snapshotID, dest string, log logrus.FieldLogger, cancleCh chan struct{}) (int64, int32, error) { - return 0, 0, errors.New("failed to restore") + name: "normal block mode restore", + hookRestoreFunc: func(ctx context.Context, rep repo.RepositoryWriter, progress *kopia.Progress, snapshotID, dest string, volMode uploader.PersistentVolumeMode, log logrus.FieldLogger, cancleCh chan struct{}) (int64, int32, error) { + return 0, 0, nil }, volMode: uploader.PersistentVolumeBlock, - notError: false, + notError: true, }, } diff --git a/pkg/uploader/provider/restic_test.go b/pkg/uploader/provider/restic_test.go index f2203d7bdd8..62f44d04f3f 100644 --- a/pkg/uploader/provider/restic_test.go +++ b/pkg/uploader/provider/restic_test.go @@ -212,6 +212,9 @@ func TestResticRunRestore(t *testing.T) { for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { + if tc.volMode == "" { + tc.volMode = uploader.PersistentVolumeFilesystem + } resticRestoreCMDFunc = tc.hookResticRestoreFunc if tc.volMode == "" { tc.volMode = uploader.PersistentVolumeFilesystem diff --git a/pkg/util/kube/utils.go b/pkg/util/kube/utils.go index 3d8d4e3ef1c..057d221c619 100644 --- a/pkg/util/kube/utils.go +++ b/pkg/util/kube/utils.go @@ -35,6 +35,7 @@ import ( corev1client "k8s.io/client-go/kubernetes/typed/core/v1" "sigs.k8s.io/controller-runtime/pkg/client" + "github.com/vmware-tanzu/velero/pkg/uploader" "github.com/vmware-tanzu/velero/pkg/util/filesystem" ) @@ -121,7 +122,8 @@ func EnsureNamespaceExistsAndIsReady(namespace *corev1api.Namespace, client core // GetVolumeDirectory gets the name of the directory on the host, under /var/lib/kubelet/pods//volumes/, // where the specified volume lives. // For volumes with a CSIVolumeSource, append "/mount" to the directory name. -func GetVolumeDirectory(ctx context.Context, log logrus.FieldLogger, pod *corev1api.Pod, volumeName string, cli client.Client) (string, error) { +func GetVolumeDirectory(ctx context.Context, log logrus.FieldLogger, pod *corev1api.Pod, volumeName string, cli client.Client) ( + string, uploader.PersistentVolumeMode, error) { var volume *corev1api.Volume for i := range pod.Spec.Volumes { @@ -132,41 +134,50 @@ func GetVolumeDirectory(ctx context.Context, log logrus.FieldLogger, pod *corev1 } if volume == nil { - return "", errors.New("volume not found in pod") + return "", "", errors.New("volume not found in pod") } + volMode := uploader.PersistentVolumeFilesystem + // This case implies the administrator created the PV and attached it directly, without PVC. // Note that only one VolumeSource can be populated per Volume on a pod if volume.VolumeSource.PersistentVolumeClaim == nil { if volume.VolumeSource.CSI != nil { - return volume.Name + "/mount", nil + return volume.Name + "/mount", volMode, nil } - return volume.Name, nil + return volume.Name, volMode, nil } // Most common case is that we have a PVC VolumeSource, and we need to check the PV it points to for a CSI source. pvc := &corev1api.PersistentVolumeClaim{} err := cli.Get(ctx, client.ObjectKey{Namespace: pod.Namespace, Name: volume.VolumeSource.PersistentVolumeClaim.ClaimName}, pvc) if err != nil { - return "", errors.WithStack(err) + return "", "", errors.WithStack(err) } pv := &corev1api.PersistentVolume{} err = cli.Get(ctx, client.ObjectKey{Name: pvc.Spec.VolumeName}, pv) if err != nil { - return "", errors.WithStack(err) + return "", "", errors.WithStack(err) + } + + if pv.Spec.VolumeMode != nil && *pv.Spec.VolumeMode == corev1api.PersistentVolumeBlock { + volMode = uploader.PersistentVolumeBlock } // PV's been created with a CSI source. isProvisionedByCSI, err := isProvisionedByCSI(log, pv, cli) if err != nil { - return "", errors.WithStack(err) + return "", "", errors.WithStack(err) } if isProvisionedByCSI { - return pvc.Spec.VolumeName + "/mount", nil + if volMode == uploader.PersistentVolumeBlock { + return pvc.Spec.VolumeName, volMode, nil + } + return pvc.Spec.VolumeName + "/mount", volMode, nil } - return pvc.Spec.VolumeName, nil + return pvc.Spec.VolumeName, volMode, nil } // isProvisionedByCSI function checks whether this is a CSI PV by annotation. diff --git a/pkg/util/kube/utils_test.go b/pkg/util/kube/utils_test.go index 6edf0843513..cc298267fc4 100644 --- a/pkg/util/kube/utils_test.go +++ b/pkg/util/kube/utils_test.go @@ -164,6 +164,13 @@ func TestGetVolumeDirectorySuccess(t *testing.T) { pv: builder.ForPersistentVolume("a-pv").CSI("csi.test.com", "provider-volume-id").Result(), want: "a-pv/mount", }, + { + name: "Block CSI volume with a PVC/PV does not append '/mount' to the volume name", + pod: builder.ForPod("ns-1", "my-pod").Volumes(builder.ForVolume("my-vol").PersistentVolumeClaimSource("my-pvc").Result()).Result(), + pvc: builder.ForPersistentVolumeClaim("ns-1", "my-pvc").VolumeName("a-pv").Result(), + pv: builder.ForPersistentVolume("a-pv").CSI("csi.test.com", "provider-volume-id").VolumeMode(corev1.PersistentVolumeBlock).Result(), + want: "a-pv", + }, { name: "CSI volume mounted without a PVC appends '/mount' to the volume name", pod: builder.ForPod("ns-1", "my-pod").Volumes(builder.ForVolume("my-vol").CSISource("csi.test.com").Result()).Result(), @@ -204,7 +211,7 @@ func TestGetVolumeDirectorySuccess(t *testing.T) { } // Function under test - dir, err := GetVolumeDirectory(context.Background(), logrus.StandardLogger(), tc.pod, tc.pod.Spec.Volumes[0].Name, clientBuilder.Build()) + dir, _, err := GetVolumeDirectory(context.Background(), logrus.StandardLogger(), tc.pod, tc.pod.Spec.Volumes[0].Name, clientBuilder.Build()) require.NoError(t, err) assert.Equal(t, tc.want, dir) diff --git a/site/content/docs/main/customize-installation.md b/site/content/docs/main/customize-installation.md index 0d8621685d7..79613d60699 100644 --- a/site/content/docs/main/customize-installation.md +++ b/site/content/docs/main/customize-installation.md @@ -23,6 +23,10 @@ By default, `velero install` does not install Velero's [File System Backup][3]. If you've already run `velero install` without the `--use-node-agent` flag, you can run the same command again, including the `--use-node-agent` flag, to add the file system backup to your existing install. +## Block volume backup + +Block volume backup is supported for CSI volumes with Kopia uploader. To allow the Agent to access the block device, Agent must be installed in privileged mode. To enable it set `velero install` flags `--use-node-agent --privileged-agent --uploader-type "kopia" --features EnableCSI`. When running a backup set the `backup create` flag `--snapshot-move-data`. + ## Default Pod Volume backup to file system backup By default, `velero install` does not enable the use of File System Backup (FSB) to take backups of all pod volumes. You must apply an [annotation](file-system-backup.md/#using-opt-in-pod-volume-backup) to every pod which contains volumes for Velero to use FSB for the backup. diff --git a/site/content/docs/main/file-system-backup.md b/site/content/docs/main/file-system-backup.md index 3680498869e..0f60fec14ca 100644 --- a/site/content/docs/main/file-system-backup.md +++ b/site/content/docs/main/file-system-backup.md @@ -546,7 +546,7 @@ that it's backing up for the volumes to be backed up using FSB. 3. Velero then creates a `PodVolumeBackup` custom resource per volume listed in the pod annotation 4. The main Velero process now waits for the `PodVolumeBackup` resources to complete or fail 5. Meanwhile, each `PodVolumeBackup` is handled by the controller on the appropriate node, which: - - has a hostPath volume mount of `/var/lib/kubelet/pods` to access the pod volume data + - has a hostPath volume mount of `/var/lib/kubelet/pods` and `/var/lib/kubelet/plugins` to access the pod volume data - finds the pod volume's subdirectory within the above volume - based on the path selection, Velero invokes restic or kopia for backup - updates the status of the custom resource to `Completed` or `Failed` @@ -570,7 +570,7 @@ some reason (i.e. lack of cluster resources), the FSB restore will not be done. 5. Velero creates a `PodVolumeRestore` custom resource for each volume to be restored in the pod 6. The main Velero process now waits for each `PodVolumeRestore` resource to complete or fail 7. Meanwhile, each `PodVolumeRestore` is handled by the controller on the appropriate node, which: - - has a hostPath volume mount of `/var/lib/kubelet/pods` to access the pod volume data + - has a hostPath volume mount of `/var/lib/kubelet/pods` and `/var/lib/kubelet/plugins` to access the pod volume data - waits for the pod to be running the init container - finds the pod volume's subdirectory within the above volume - based on the path selection, Velero invokes restic or kopia for restore diff --git a/tilt-resources/examples/node-agent.yaml b/tilt-resources/examples/node-agent.yaml index d5c10fc47ee..835ba297ff1 100644 --- a/tilt-resources/examples/node-agent.yaml +++ b/tilt-resources/examples/node-agent.yaml @@ -49,6 +49,9 @@ spec: - mountPath: /host_pods mountPropagation: HostToContainer name: host-pods + - mountPath: /var/lib/kubelet/plugins + mountPropagation: HostToContainer + name: host-plugins - mountPath: /scratch name: scratch - mountPath: /credentials @@ -60,6 +63,9 @@ spec: - hostPath: path: /var/lib/kubelet/pods name: host-pods + - hostPath: + path: /var/lib/kubelet/plugins + name: host-plugins - emptyDir: {} name: scratch - name: cloud-credentials