diff --git a/pkg/application/inject/fuse/injector.go b/pkg/application/inject/fuse/injector.go index fa0bc5c2f82..dd1c5ea574a 100644 --- a/pkg/application/inject/fuse/injector.go +++ b/pkg/application/inject/fuse/injector.go @@ -153,13 +153,14 @@ func (s *Injector) inject(in runtime.Object, pvcName string, runtimeInfo base.Ru } // if it's not serverless enable or injection is done, skip - if !utils.ServerlessEnabled(metaObj.Labels) || utils.SidecarInjectDone(metaObj.Labels) { + if !utils.ServerlessEnabled(metaObj.Labels) || utils.InjectSidecarDone(metaObj.Labels) { continue } // 1. check if the pod spec has fluid volume claim injectFuseContainer := true - template, err := runtimeInfo.GetTemplateToInjectForFuse(pvcName) + enableCacheDir := utils.InjectCacheDirEnabled(metaObj.Labels) + template, err := runtimeInfo.GetTemplateToInjectForFuse(pvcName, enableCacheDir) if err != nil { return out, err } diff --git a/pkg/common/constants.go b/pkg/common/constants.go index 9d1ae67b615..45c694f3b97 100644 --- a/pkg/common/constants.go +++ b/pkg/common/constants.go @@ -117,6 +117,7 @@ const ( injectSidecar = ".sidecar" + inject InjectServerless = "serverless" + inject InjectFuseSidecar = "fuse" + injectSidecar + InjectCacheDir = "cachedir" + injectSidecar InjectWorkerSidecar = "worker" + injectSidecar InjectSidecarDone = "done" + injectSidecar ) diff --git a/pkg/ddc/base/runtime.go b/pkg/ddc/base/runtime.go index 23da796f1db..20e8ad07725 100644 --- a/pkg/ddc/base/runtime.go +++ b/pkg/ddc/base/runtime.go @@ -77,7 +77,7 @@ type RuntimeInfoInterface interface { IsDeprecatedPVName() bool - GetTemplateToInjectForFuse(pvcName string) (*common.FuseInjectionTemplate, error) + GetTemplateToInjectForFuse(pvcName string, enableCache bool) (*common.FuseInjectionTemplate, error) SetClient(client client.Client) } diff --git a/pkg/ddc/base/runtime_helper.go b/pkg/ddc/base/runtime_helper.go index f5e1186a729..310342174b3 100644 --- a/pkg/ddc/base/runtime_helper.go +++ b/pkg/ddc/base/runtime_helper.go @@ -31,8 +31,15 @@ import ( "k8s.io/apimachinery/pkg/types" ) +var ( + // datavolume- for JindoFS + // mem, ssd, hdd for Alluxio and GooseFS + // cache-dir for JuiceFS + cacheDirNames = []string{"datavolume-", "cache-dir", "mem", "ssd", "hdd"} +) + // GetTemplateToInjectForFuse gets template for fuse injection -func (info *RuntimeInfo) GetTemplateToInjectForFuse(pvcName string) (template *common.FuseInjectionTemplate, err error) { +func (info *RuntimeInfo) GetTemplateToInjectForFuse(pvcName string, enableCacheDir bool) (template *common.FuseInjectionTemplate, err error) { // TODO: create fuse container ds, err := info.getFuseDaemonset() if err != nil { @@ -51,15 +58,21 @@ func (info *RuntimeInfo) GetTemplateToInjectForFuse(pvcName string) (template *c UID: dataset.UID, } + // 0. remove the cache dir if required + if len(ds.Spec.Template.Spec.Containers) != 1 { + return template, fmt.Errorf("the length of containers of fuse %s in namespace %s is not 1", ds.Name, ds.Namespace) + } + if !enableCacheDir { + ds.Spec.Template.Spec.Containers[0].VolumeMounts = utils.TrimVolumeMounts(ds.Spec.Template.Spec.Containers[0].VolumeMounts, cacheDirNames) + ds.Spec.Template.Spec.Volumes = utils.TrimVolumes(ds.Spec.Template.Spec.Volumes, cacheDirNames) + } + // 1. set the pvc name template = &common.FuseInjectionTemplate{ PVCName: pvcName, } // 2. set the fuse container - if len(ds.Spec.Template.Spec.Containers) != 1 { - return template, fmt.Errorf("the length of containers of fuse %s in namespace %s is not 1", ds.Name, ds.Namespace) - } template.FuseContainer = ds.Spec.Template.Spec.Containers[0] template.FuseContainer.Name = common.FuseContainerName diff --git a/pkg/ddc/base/runtime_helper_test.go b/pkg/ddc/base/runtime_helper_test.go index ffc42f20e81..84a2abe8e70 100644 --- a/pkg/ddc/base/runtime_helper_test.go +++ b/pkg/ddc/base/runtime_helper_test.go @@ -495,7 +495,372 @@ func TestGetTemplateToInjectForFuse(t *testing.T) { t.Errorf("testcase %s failed due to error %v", testcase.name, err) } runtimeInfo.SetClient(fakeClient) - _, err = runtimeInfo.GetTemplateToInjectForFuse(testcase.pvcName) + _, err = runtimeInfo.GetTemplateToInjectForFuse(testcase.pvcName, true) + if (err == nil) == testcase.expectErr { + t.Errorf("testcase %s failed due to expecting want error: %v error %v", testcase.name, testcase.expectErr, err) + } + } +} + +func TestGetTemplateToInjectForFuseForCacheDir(t *testing.T) { + type runtimeInfo struct { + name string + namespace string + runtimeType string + } + type testCase struct { + name string + dataset *datav1alpha1.Dataset + pvcName string + enableCacheDir bool + info runtimeInfo + pv *corev1.PersistentVolume + pvc *corev1.PersistentVolumeClaim + fuse *appsv1.DaemonSet + expectErr bool + } + + hostPathCharDev := corev1.HostPathCharDev + hostPathDirectoryOrCreate := corev1.HostPathDirectoryOrCreate + bTrue := true + + testcases := []testCase{ + { + name: "jindo", + dataset: &datav1alpha1.Dataset{ + ObjectMeta: metav1.ObjectMeta{ + Name: "mydata", + Namespace: "big-data", + }, + }, + info: runtimeInfo{ + name: "mydata", + namespace: "big-data", + runtimeType: common.JindoRuntime, + }, + pvcName: "mydata", + pv: &corev1.PersistentVolume{ + ObjectMeta: metav1.ObjectMeta{ + Name: "big-data-mydata", + }, + Spec: corev1.PersistentVolumeSpec{ + PersistentVolumeSource: corev1.PersistentVolumeSource{ + CSI: &corev1.CSIPersistentVolumeSource{ + Driver: "fuse.csi.fluid.io", + VolumeAttributes: map[string]string{ + common.FluidPath: "/runtime-mnt/jindo/big-data/mydata/jindofs-fuse", + common.MountType: common.JindoRuntime, + }, + }, + }, + }, + }, + enableCacheDir: false, + expectErr: false, + pvc: &corev1.PersistentVolumeClaim{ + ObjectMeta: metav1.ObjectMeta{ + Name: "mydata", + Namespace: "big-data", + }, Spec: corev1.PersistentVolumeClaimSpec{ + VolumeName: "big-data-mydata", + }, + }, + fuse: &appsv1.DaemonSet{ + ObjectMeta: metav1.ObjectMeta{ + Name: "mydata-jindofs-fuse", + Namespace: "big-data", + }, + Spec: appsv1.DaemonSetSpec{ + Template: corev1.PodTemplateSpec{ + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + { + Name: "fuse", + Args: []string{ + "-oroot_ns=jindo", "-okernel_cache", "-oattr_timeout=9000", "-oentry_timeout=9000", + }, + Command: []string{"/entrypoint.sh"}, + Image: "mydata-pvc-name", + SecurityContext: &corev1.SecurityContext{ + Privileged: &bTrue, + }, VolumeMounts: []corev1.VolumeMount{ + { + Name: "datavolume-1", + MountPath: "/mnt/disk1", + }, { + Name: "fuse-device", + MountPath: "/dev/fuse", + }, { + Name: "jindofs-fuse-mount", + MountPath: "/jfs", + }, + }, + }, + }, + Volumes: []corev1.Volume{ + { + Name: "datavolume-1", + VolumeSource: corev1.VolumeSource{ + HostPath: &corev1.HostPathVolumeSource{ + Path: "/mnt/disk1", + Type: &hostPathDirectoryOrCreate, + }, + }}, + { + Name: "fuse-device", + VolumeSource: corev1.VolumeSource{ + HostPath: &corev1.HostPathVolumeSource{ + Path: "/dev/fuse", + Type: &hostPathCharDev, + }, + }, + }, + { + Name: "jindofs-fuse-mount", + VolumeSource: corev1.VolumeSource{ + HostPath: &corev1.HostPathVolumeSource{ + Path: "/runtime-mnt/jindo/big-data/mydata", + Type: &hostPathDirectoryOrCreate, + }, + }, + }, + }, + }, + }, + }, + }, + }, + { + name: "alluxio", + dataset: &datav1alpha1.Dataset{ + ObjectMeta: metav1.ObjectMeta{ + Name: "dataset1", + Namespace: "big-data", + }, + }, info: runtimeInfo{ + name: "dataset1", + namespace: "big-data", + runtimeType: common.ALLUXIO_RUNTIME, + }, + expectErr: false, + pvcName: "dataset1", + pv: &corev1.PersistentVolume{ + ObjectMeta: metav1.ObjectMeta{ + Name: "big-data-dataset1", + }, + Spec: corev1.PersistentVolumeSpec{ + PersistentVolumeSource: corev1.PersistentVolumeSource{ + CSI: &corev1.CSIPersistentVolumeSource{ + Driver: "fuse.csi.fluid.io", + VolumeAttributes: map[string]string{ + common.FluidPath: "/runtime-mnt/jindo/big-data/dataset1/jindofs-fuse", + common.MountType: common.JindoRuntime, + }, + }, + }, + }, + }, + pvc: &corev1.PersistentVolumeClaim{ + ObjectMeta: metav1.ObjectMeta{ + Name: "dataset1", + Namespace: "big-data", + }, Spec: corev1.PersistentVolumeClaimSpec{ + VolumeName: "big-data-dataset1", + }, + }, + fuse: &appsv1.DaemonSet{ + ObjectMeta: metav1.ObjectMeta{ + Name: "dataset1-fuse", + Namespace: "big-data", + }, + Spec: appsv1.DaemonSetSpec{ + Template: corev1.PodTemplateSpec{ + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + { + Name: "fuse", + Args: []string{ + "-oroot_ns=jindo", "-okernel_cache", "-oattr_timeout=9000", "-oentry_timeout=9000", + }, + Command: []string{"/entrypoint.sh"}, + Image: "test", + SecurityContext: &corev1.SecurityContext{ + Privileged: &bTrue, + }, + VolumeMounts: []corev1.VolumeMount{ + { + Name: "data", + MountPath: "/mnt/disk1", + }, { + Name: "fuse-device", + MountPath: "/dev/fuse", + }, { + Name: "jindofs-fuse-mount", + MountPath: "/jfs", + }, + }, + }, + }, + Volumes: []corev1.Volume{ + { + Name: "data", + VolumeSource: corev1.VolumeSource{ + HostPath: &corev1.HostPathVolumeSource{ + Path: "/runtime_mnt/dataset1", + }, + }}, + { + Name: "fuse-device", + VolumeSource: corev1.VolumeSource{ + HostPath: &corev1.HostPathVolumeSource{ + Path: "/dev/fuse", + Type: &hostPathCharDev, + }, + }, + }, + { + Name: "jindofs-fuse-mount", + VolumeSource: corev1.VolumeSource{ + HostPath: &corev1.HostPathVolumeSource{ + Path: "/runtime-mnt/jindo/big-data/dataset1", + Type: &hostPathDirectoryOrCreate, + }, + }, + }, + }, + }, + }, + }, + }, + }, { + name: "custome_envs", + dataset: &datav1alpha1.Dataset{ + ObjectMeta: metav1.ObjectMeta{ + Name: "customizedenv", + Namespace: "big-data", + }, + }, info: runtimeInfo{ + name: "customizedenv", + namespace: "big-data", + runtimeType: common.JindoRuntime, + }, + pvcName: "customizedenv", + pv: &corev1.PersistentVolume{ + ObjectMeta: metav1.ObjectMeta{ + Name: "big-data-customizedenv", + }, + Spec: corev1.PersistentVolumeSpec{ + PersistentVolumeSource: corev1.PersistentVolumeSource{ + CSI: &corev1.CSIPersistentVolumeSource{ + Driver: "fuse.csi.fluid.io", + VolumeAttributes: map[string]string{ + common.FluidPath: "/runtime-mnt/jindo/big-data/customizedenv/jindofs-fuse", + common.MountType: common.JindoRuntime, + }, + }, + }, + }, + }, + pvc: &corev1.PersistentVolumeClaim{ + ObjectMeta: metav1.ObjectMeta{ + Name: "customizedenv", + Namespace: "big-data", + }, Spec: corev1.PersistentVolumeClaimSpec{ + VolumeName: "big-data-customizedenv", + }, + }, + fuse: &appsv1.DaemonSet{ + ObjectMeta: metav1.ObjectMeta{ + Name: "customizedenv-jindofs-fuse", + Namespace: "big-data", + }, + Spec: appsv1.DaemonSetSpec{ + Template: corev1.PodTemplateSpec{ + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + { + Name: "fuse", + Args: []string{ + "-oroot_ns=jindo", "-okernel_cache", "-oattr_timeout=9000", "-oentry_timeout=9000", + }, + Command: []string{"/entrypoint.sh"}, + Image: "customizedenv-pvc-name", + Env: []corev1.EnvVar{ + { + Name: "FLUID_FUSE_MOUNTPOINT", + Value: "/jfs/jindofs-fuse", + }, + }, + SecurityContext: &corev1.SecurityContext{ + Privileged: &bTrue, + }, VolumeMounts: []corev1.VolumeMount{ + { + Name: "customizedenv", + MountPath: "/mnt/disk1", + }, { + Name: "fuse-device", + MountPath: "/dev/fuse", + }, { + Name: "jindofs-fuse-mount", + MountPath: "/jfs", + }, + }, + }, + }, + Volumes: []corev1.Volume{ + { + Name: "customizedenv", + VolumeSource: corev1.VolumeSource{ + HostPath: &corev1.HostPathVolumeSource{ + Path: "/mnt/disk1", + Type: &hostPathDirectoryOrCreate, + }, + }}, + { + Name: "fuse-device", + VolumeSource: corev1.VolumeSource{ + HostPath: &corev1.HostPathVolumeSource{ + Path: "/dev/fuse", + Type: &hostPathCharDev, + }, + }, + }, + { + Name: "jindofs-fuse-mount", + VolumeSource: corev1.VolumeSource{ + HostPath: &corev1.HostPathVolumeSource{ + Path: "/runtime-mnt/jindo/big-data/customizedenv", + Type: &hostPathDirectoryOrCreate, + }, + }, + }, + }, + }, + }, + }, + }, + }, + } + + objs := []runtime.Object{} + s := runtime.NewScheme() + _ = corev1.AddToScheme(s) + _ = datav1alpha1.AddToScheme(s) + _ = appsv1.AddToScheme(s) + for _, testcase := range testcases { + objs = append(objs, testcase.fuse, testcase.pv, testcase.pvc, testcase.dataset) + } + + fakeClient := fake.NewFakeClientWithScheme(s, objs...) + + for _, testcase := range testcases { + info := testcase.info + runtimeInfo, err := BuildRuntimeInfo(info.name, info.namespace, info.runtimeType, datav1alpha1.TieredStore{}) + if err != nil { + t.Errorf("testcase %s failed due to error %v", testcase.name, err) + } + runtimeInfo.SetClient(fakeClient) + _, err = runtimeInfo.GetTemplateToInjectForFuse(testcase.pvcName, testcase.enableCacheDir) if (err == nil) == testcase.expectErr { t.Errorf("testcase %s failed due to expecting want error: %v error %v", testcase.name, testcase.expectErr, err) } diff --git a/pkg/utils/annotations.go b/pkg/utils/annotations.go index 13bdb32324c..7d1f8392142 100644 --- a/pkg/utils/annotations.go +++ b/pkg/utils/annotations.go @@ -29,10 +29,14 @@ func WorkerSidecarEnabled(infos map[string]string) (match bool) { return enabled(infos, common.InjectWorkerSidecar) } -func SidecarInjectDone(infos map[string]string) (match bool) { +func InjectSidecarDone(infos map[string]string) (match bool) { return enabled(infos, common.InjectSidecarDone) } +func InjectCacheDirEnabled(infos map[string]string) (match bool) { + return enabled(infos, common.InjectCacheDir) +} + func enabled(infos map[string]string, name string) (match bool) { for key, value := range infos { if key == name && value == common.True { diff --git a/pkg/utils/annotations_test.go b/pkg/utils/annotations_test.go index f7f6c48e70f..862b2d107c0 100644 --- a/pkg/utils/annotations_test.go +++ b/pkg/utils/annotations_test.go @@ -202,7 +202,44 @@ func TestInjectionEnabled(t *testing.T) { } for _, testcase := range testcases { - got := SidecarInjectDone(testcase.annotations) + got := InjectSidecarDone(testcase.annotations) + if got != testcase.expect { + t.Errorf("The testcase %s's failed due to expect %v but got %v", testcase.name, testcase.expect, got) + } + } +} + +func TestCacheDirInjectionEnabled(t *testing.T) { + type testCase struct { + name string + annotations map[string]string + expect bool + } + + testcases := []testCase{ + { + name: "enable_Injection_done", + annotations: map[string]string{ + common.InjectCacheDir: "true", + }, + expect: true, + }, { + name: "disable_Injection_done", + annotations: map[string]string{ + common.InjectCacheDir: "false", + }, + expect: false, + }, { + name: "no_Injection", + annotations: map[string]string{ + "test": "false", + }, + expect: false, + }, + } + + for _, testcase := range testcases { + got := InjectCacheDirEnabled(testcase.annotations) if got != testcase.expect { t.Errorf("The testcase %s's failed due to expect %v but got %v", testcase.name, testcase.expect, got) } diff --git a/pkg/utils/volumes.go b/pkg/utils/volumes.go new file mode 100644 index 00000000000..47a5e26e35d --- /dev/null +++ b/pkg/utils/volumes.go @@ -0,0 +1,59 @@ +/* +Copyright 2021 The Fluid Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package utils + +import ( + "strings" + + corev1 "k8s.io/api/core/v1" +) + +// TrimVolumes trims the volumes +func TrimVolumes(inputs []corev1.Volume, excludeNames []string) (outputs []corev1.Volume) { + outputs = []corev1.Volume{} +outer: + for _, in := range inputs { + for _, excludeName := range excludeNames { + if strings.HasPrefix(in.Name, excludeName) { + log.V(1).Info("Skip the volume", "volume", in, "exludeName", excludeName) + continue outer + } + } + + outputs = append(outputs, in) + } + + return +} + +func TrimVolumeMounts(inputs []corev1.VolumeMount, excludeNames []string) (outputs []corev1.VolumeMount) { + outputs = []corev1.VolumeMount{} +outer: + for _, in := range inputs { + for _, excludeName := range excludeNames { + if strings.HasPrefix(in.Name, excludeName) { + log.V(1).Info("Skip the volumeMount", "volumeMount", in, "exludeName", excludeName) + continue outer + } + } + + outputs = append(outputs, in) + + } + + return +} diff --git a/pkg/utils/volumes_test.go b/pkg/utils/volumes_test.go new file mode 100644 index 00000000000..8df2ba0e099 --- /dev/null +++ b/pkg/utils/volumes_test.go @@ -0,0 +1,152 @@ +/* +Copyright 2021 The Fluid Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package utils + +import ( + "reflect" + "testing" + + corev1 "k8s.io/api/core/v1" +) + +func TestTrimVolumes(t *testing.T) { + testCases := map[string]struct { + volumes []corev1.Volume + names []string + wants []string + }{ + "no exclude": { + volumes: []corev1.Volume{ + { + Name: "test-1", + VolumeSource: corev1.VolumeSource{ + HostPath: &corev1.HostPathVolumeSource{ + Path: "/runtime_mnt/dataset1", + }, + }}, + { + Name: "fuse-device", + VolumeSource: corev1.VolumeSource{ + HostPath: &corev1.HostPathVolumeSource{ + Path: "/dev/fuse", + }, + }, + }, + { + Name: "jindofs-fuse-mount", + VolumeSource: corev1.VolumeSource{ + HostPath: &corev1.HostPathVolumeSource{ + Path: "/runtime-mnt/jindo/big-data/dataset1", + }, + }, + }, + }, + names: []string{"datavolume-", "cache-dir", "mem", "ssd", "hdd"}, + wants: []string{"test-1", "fuse-device", "jindofs-fuse-mount"}, + }, "exclude": { + volumes: []corev1.Volume{ + { + Name: "datavolume-1", + VolumeSource: corev1.VolumeSource{ + HostPath: &corev1.HostPathVolumeSource{ + Path: "/runtime_mnt/dataset1", + }, + }}, + { + Name: "fuse-device", + VolumeSource: corev1.VolumeSource{ + HostPath: &corev1.HostPathVolumeSource{ + Path: "/dev/fuse", + }, + }, + }, + { + Name: "jindofs-fuse-mount", + VolumeSource: corev1.VolumeSource{ + HostPath: &corev1.HostPathVolumeSource{ + Path: "/runtime-mnt/jindo/big-data/dataset1", + }, + }, + }, + }, + names: []string{"datavolume-", "cache-dir", "mem", "ssd", "hdd"}, + wants: []string{"fuse-device", "jindofs-fuse-mount"}, + }, + } + + for name, testCase := range testCases { + got := TrimVolumes(testCase.volumes, testCase.names) + gotNames := []string{} + for _, name := range got { + gotNames = append(gotNames, name.Name) + } + + if !reflect.DeepEqual(gotNames, testCase.wants) { + t.Errorf("%s check failure, want:%v, got:%v", name, testCase.names, gotNames) + } + } +} + +func TestTrimVolumeMounts(t *testing.T) { + testCases := map[string]struct { + volumeMounts []corev1.VolumeMount + names []string + wants []string + }{ + "no exclude": { + volumeMounts: []corev1.VolumeMount{ + { + Name: "test-1", + }, + { + Name: "fuse-device", + }, + { + Name: "jindofs-fuse-mount", + }, + }, + names: []string{"datavolumeMount-", "cache-dir", "mem", "ssd", "hdd"}, + wants: []string{"test-1", "fuse-device", "jindofs-fuse-mount"}, + }, "exclude": { + volumeMounts: []corev1.VolumeMount{ + { + Name: "datavolumeMount-1", + }, + { + Name: "fuse-device", + }, + { + Name: "jindofs-fuse-mount", + }, + }, + names: []string{"datavolumeMount-", "cache-dir", "mem", "ssd", "hdd"}, + wants: []string{"fuse-device", "jindofs-fuse-mount"}, + }, + } + + for name, testCase := range testCases { + got := TrimVolumeMounts(testCase.volumeMounts, testCase.names) + gotNames := []string{} + for _, name := range got { + gotNames = append(gotNames, name.Name) + } + + if !reflect.DeepEqual(gotNames, testCase.wants) { + t.Errorf("%s check failure, want:%v, got:%v", name, testCase.names, gotNames) + } + } +}