support juicefs cloud service (#1683)

* support juicefs cloud service Signed-off-by: zwwhdls <[email protected]> * fix golangci-lint Signed-off-by: zwwhdls <[email protected]> * add dataload for enterprise Signed-off-by: zwwhdls <[email protected]> * fix test Signed-off-by: zwwhdls <[email protected]>
fluid-cloudnative · Mar 31, 2022 · ed486d8 · ed486d8
1 parent bf3b453
commit ed486d8
Show file tree

Hide file tree

Showing 17 changed files with 729 additions and 94 deletions.
diff --git a/charts/fluid-dataloader/juicefs/templates/configmap.yaml b/charts/fluid-dataloader/juicefs/templates/configmap.yaml
@@ -27,7 +27,12 @@ data:
           local pod="${podNames[i]}"
 
           echo -e "juicefs warmup on $pod $targetPath starts"
-          /usr/local/bin/kubectl -n $ns exec -it $pod -- timeout $TIMEOUT juicefs warmup $targetPath
+          if [ $EDITION == 'community' ]
+          then
+          /usr/local/bin/kubectl -n $ns exec -it $pod -- timeout $TIMEOUT /usr/local/bin/juicefs warmup $targetPath
+          else
+          /usr/local/bin/kubectl -n $ns exec -it $pod -- timeout $TIMEOUT /usr/bin/juicefs warmup $targetPath
+          fi
           echo -e "juicefs warmup on $pod $targetPath ends"
         done
     }

diff --git a/charts/fluid-dataloader/juicefs/templates/dataloader.yaml b/charts/fluid-dataloader/juicefs/templates/dataloader.yaml
@@ -80,6 +80,10 @@ spec:
             - name: TIMEOUT
               value: {{ $val | quote }}
             {{- end }}
+            {{- if eq $key "edition" }}
+            - name: EDITION
+              value: {{ $val | quote }}
+            {{- end }}
             {{- end }}
             - name: DATA_PATH
               value: {{ $targetPaths | quote }}
@@ -107,12 +111,6 @@ spec:
             {{- end }}
             {{- end }}
               name: cachedir
-            {{- range $key, $val := .Values.dataloader.options }}
-            {{- if eq $key "mountpath" }}
-            - mountPath: {{ $val | quote }}
-            {{- end }}
-            {{- end }}
-              name: mountpath1
       volumes:
         - name: data-load-script
           configMap:
@@ -135,10 +133,3 @@ spec:
             path: {{ $val | quote }}
         {{- end }}
         {{- end }}
-        - name: mountpath1
-        {{- range $key, $val := .Values.dataloader.options }}
-        {{- if eq $key "mountpath" }}
-          hostPath:
-            path: {{ $val | quote }}
-        {{- end }}
-        {{- end }}
diff --git a/charts/juicefs/templates/fuse/daemonset.yaml b/charts/juicefs/templates/fuse/daemonset.yaml
@@ -85,6 +85,13 @@ spec:
                 name: {{ .Values.fuse.secretkeySecret }}
                 key: secret-key
           {{- end }}
+          {{- if .Values.fuse.tokenSecret }}
+          - name: TOKEN
+            valueFrom:
+              secretKeyRef:
+                name: {{ .Values.fuse.tokenSecret }}
+                key: token
+          {{- end }}
           readinessProbe:
             exec:
               command:
@@ -130,26 +137,25 @@ spec:
         {{- end }}
         - name: script
           configMap:
-            name: {{ template "juicefs.fullname" . }}-script
+            name: {{ template "juicefs.fullname" . }}-fuse-script
             defaultMode: 0755
 ---
 apiVersion: v1
 kind: ConfigMap
 metadata:
-  name: {{ template "juicefs.fullname" . }}-script
+  name: {{ template "juicefs.fullname" . }}-fuse-script
   labels:
     app: {{ template "juicefs.name" . }}
     chart: {{ template "juicefs.chart" . }}
     release: {{ .Release.Name }}
     heritage: {{ .Release.Service }}
+    role: juicefs-fuse
 data:
   script.sh: |
     #!/bin/bash
 
-    {{- if and .Values.fuse.storage .Values.fuse.bucket .Values.fuse.accesskeySecret .Values.fuse.secretkeySecret }}
-    /usr/local/bin/juicefs format --storage={{ .Values.fuse.storage }} --bucket={{ .Values.fuse.bucket }} --access-key=${ACCESS_KEY} --secret-key=${SECRET_KEY} ${METAURL} {{ .Values.fuse.name }}
-    {{- else }}
-    /usr/local/bin/juicefs format ${METAURL} {{ .Values.fuse.name }} --no-update
+    {{- if .Values.fuse.formatCmd }}
+    {{ .Values.fuse.formatCmd }}
     {{- end }}
 
     {{ .Values.fuse.command }}

diff --git a/charts/juicefs/templates/worker/statefuleset.yaml b/charts/juicefs/templates/worker/statefuleset.yaml
@@ -92,6 +92,13 @@ spec:
                 name: {{ .Values.fuse.secretkeySecret }}
                 key: secret-key
           {{- end }}
+          {{- if .Values.fuse.tokenSecret }}
+          - name: TOKEN
+            valueFrom:
+              secretKeyRef:
+                name: {{ .Values.fuse.tokenSecret }}
+                key: token
+          {{- end }}
           livenessProbe:
             exec:
               command:
@@ -124,5 +131,25 @@ spec:
         {{- end }}
         - name: script
           configMap:
-            name: {{ template "juicefs.fullname" . }}-script
+            name: {{ template "juicefs.fullname" . }}-worker-script
             defaultMode: 0755
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: {{ template "juicefs.fullname" . }}-worker-script
+  labels:
+    app: {{ template "juicefs.name" . }}
+    chart: {{ template "juicefs.chart" . }}
+    release: {{ .Release.Name }}
+    heritage: {{ .Release.Service }}
+    role: juicefs-worker
+data:
+  script.sh: |
+    #!/bin/bash
+
+    {{- if .Values.fuse.formatCmd }}
+    {{ .Values.fuse.formatCmd }}
+    {{- end }}
+
+    {{ .Values.worker.command }}
diff --git a/charts/juicefs/values.yaml b/charts/juicefs/values.yaml
@@ -1,5 +1,6 @@
 # This should not be modified in the usual case.
 fullnameOverride: juicefs
+edition: community
 
 ##
 ## Example:
@@ -29,6 +30,7 @@ worker:
   cacheDir: ""
   ports: []
   envs: []
+  command: ""
   resources:
     requests:
 #      cpu: "0.5"
@@ -48,6 +50,8 @@ fuse:
   bucket: ""
   metaurlSecret: ""
   storage: ""
+  tokenSecret: ""
+  formatCmd : ""
   subPath: ""
   criticalPod: false
   enabled: true

diff --git a/pkg/common/juicefs.go b/pkg/common/juicefs.go
@@ -32,7 +32,10 @@ const (
 
 	DefaultJuiceFSRuntimeImage = "registry.cn-hangzhou.aliyuncs.com/juicefs/juicefs-fuse:v1.0.0-beta2"
 
-	JuiceFSMountPath = "/bin/mount.juicefs"
+	JuiceFSCeMountPath = "/bin/mount.juicefs"
+	JuiceFSMountPath   = "/sbin/mount.juicefs"
+	JuiceCeCliPath     = "/usr/local/bin/juicefs"
+	JuiceCliPath       = "/usr/bin/juicefs"
 
 	JuiceFSFuseContainer = "juicefs-fuse"
 

diff --git a/pkg/ddc/juicefs/cacheinfo_parser.go b/pkg/ddc/juicefs/cacheinfo_parser.go
@@ -53,6 +53,7 @@ func parseCacheInfoFromConfigMap(configMap *v1.ConfigMap) (cacheinfo map[string]
 		}
 		configmapinfo[CacheDir] = value.Fuse.CacheDir
 		configmapinfo[MountPath] = value.Fuse.MountPath
+		configmapinfo[Edition] = value.Edition
 	}
 	return configmapinfo, nil
 }
diff --git a/pkg/ddc/juicefs/cacheinfo_parser_test.go b/pkg/ddc/juicefs/cacheinfo_parser_test.go
@@ -56,7 +56,7 @@ func TestGetCacheInfoFromConfigmap(t *testing.T) {
 	runtimeObjs = append(runtimeObjs, configMap)
 	runtimeObjs = append(runtimeObjs, dataSet.DeepCopy())
 	fakeClient := fake.NewFakeClientWithScheme(testScheme, runtimeObjs...)
-	wantCacheInfo := map[string]string{"cachedir": "/tmp/jfs-cache", "mountpath": "/runtime-mnt/juicefs/fluid/test-dataset/juicefs-fuse"}
+	wantCacheInfo := map[string]string{"cachedir": "/tmp/jfs-cache", "mountpath": "/runtime-mnt/juicefs/fluid/test-dataset/juicefs-fuse", "edition": "community"}
 	cacheinfo, err := GetCacheInfoFromConfigmap(fakeClient, dataSet.Name, dataSet.Namespace)
 	if err != nil {
 		t.Errorf("GetCacheInfoFromConfigmap failed.")
@@ -84,7 +84,7 @@ func Test_parseCacheInfoFromConfigMap(t *testing.T) {
 					"data": valuesConfigMapData,
 				},
 			}},
-			wantCacheInfo: map[string]string{"cachedir": "/tmp/jfs-cache", "mountpath": "/runtime-mnt/juicefs/fluid/test-dataset/juicefs-fuse"},
+			wantCacheInfo: map[string]string{"cachedir": "/tmp/jfs-cache", "mountpath": "/runtime-mnt/juicefs/fluid/test-dataset/juicefs-fuse", "edition": "community"},
 			wantErr:       false,
 		},
 		{

diff --git a/pkg/ddc/juicefs/const.go b/pkg/ddc/juicefs/const.go
@@ -38,8 +38,10 @@ const (
 	JuiceMetaUrl   = "metaurl"
 	JuiceAccessKey = "access-key"
 	JuiceSecretKey = "secret-key"
+	JuiceToken     = "token"
 
 	CacheDir               = "cachedir"
 	MountPath              = "mountpath"
+	Edition                = "edition"
 	DefaultDataLoadTimeout = "30m"
 )
diff --git a/pkg/ddc/juicefs/data_load.go b/pkg/ddc/juicefs/data_load.go
@@ -32,7 +32,7 @@ import (
 )
 
 // CreateDataLoadJob creates the job to load data
-func (e *JuiceFSEngine) CreateDataLoadJob(ctx cruntime.ReconcileRequestContext, targetDataload datav1alpha1.DataLoad) (err error) {
+func (j *JuiceFSEngine) CreateDataLoadJob(ctx cruntime.ReconcileRequestContext, targetDataload datav1alpha1.DataLoad) (err error) {
 	log := ctx.Log.WithName("createDataLoadJob")
 
 	// 1. Check if the helm release already exists
@@ -48,7 +48,7 @@ func (e *JuiceFSEngine) CreateDataLoadJob(ctx cruntime.ReconcileRequestContext,
 	// 2. install the helm chart if not exists
 	if !existed {
 		log.Info("DataLoad job helm chart not installed yet, will install")
-		valueFileName, err := e.generateDataLoadValueFile(ctx, targetDataload)
+		valueFileName, err := j.generateDataLoadValueFile(ctx, targetDataload)
 		if err != nil {
 			log.Error(err, "failed to generate dataload chart's value file")
 			return err
@@ -67,12 +67,12 @@ func (e *JuiceFSEngine) CreateDataLoadJob(ctx cruntime.ReconcileRequestContext,
 
 // generateDataLoadValueFile builds a DataLoadValue by extracted specifications from the given DataLoad, and
 // marshals the DataLoadValue to a temporary yaml file where stores values that'll be used by fluid dataloader helm chart
-func (e *JuiceFSEngine) generateDataLoadValueFile(r cruntime.ReconcileRequestContext, dataload datav1alpha1.DataLoad) (valueFileName string, err error) {
+func (j *JuiceFSEngine) generateDataLoadValueFile(r cruntime.ReconcileRequestContext, dataload datav1alpha1.DataLoad) (valueFileName string, err error) {
 	targetDataset, err := utils.GetDataset(r.Client, dataload.Spec.Dataset.Name, dataload.Spec.Dataset.Namespace)
 	if err != nil {
 		return "", err
 	}
-	e.Log.Info("target dataset", "dataset", targetDataset)
+	j.Log.Info("target dataset", "dataset", targetDataset)
 
 	imageName, imageTag := docker.GetWorkerImage(r.Client, dataload.Spec.Dataset.Name, "juicefs", dataload.Spec.Dataset.Namespace)
 
@@ -121,25 +121,30 @@ func (e *JuiceFSEngine) generateDataLoadValueFile(r cruntime.ReconcileRequestCon
 			options[key] = value
 		}
 	}
-	cacheinfo, err := GetCacheInfoFromConfigmap(e.Client, dataload.Spec.Dataset.Name, dataload.Spec.Dataset.Namespace)
+	cacheinfo, err := GetCacheInfoFromConfigmap(j.Client, dataload.Spec.Dataset.Name, dataload.Spec.Dataset.Namespace)
 	if err != nil {
 		return
 	}
 	for key, value := range cacheinfo {
 		options[key] = value
 	}
 
-	stsName := e.getWorkerName()
-	pods, err := e.GetRunningPodsOfStatefulSet(stsName, e.namespace)
+	stsName := j.getWorkerName()
+	pods, err := j.GetRunningPodsOfStatefulSet(stsName, j.namespace)
 	if err != nil || len(pods) == 0 {
 		return
 	}
 	podNames := []string{}
 	for _, pod := range pods {
 		podNames = append(podNames, pod.Name)
 	}
-	options["podNames"] = strings.Join(podNames, ":")
-	options["runtimeName"] = e.name
+	if cacheinfo[Edition] == "community" {
+		options["podNames"] = strings.Join(podNames, ":")
+	} else {
+		options["podNames"] = podNames[0]
+	}
+	options["edition"] = cacheinfo[Edition]
+	options["runtimeName"] = j.name
 	if _, ok := options["timeout"]; !ok {
 		options["timeout"] = DefaultDataLoadTimeout
 	}
@@ -154,7 +159,7 @@ func (e *JuiceFSEngine) generateDataLoadValueFile(r cruntime.ReconcileRequestCon
 	if err != nil {
 		return
 	}
-	e.Log.Info("dataload value", "value", string(data))
+	j.Log.Info("dataload value", "value", string(data))
 
 	valueFile, err := ioutil.TempFile(os.TempDir(), fmt.Sprintf("%s-%s-loader-values.yaml", dataload.Namespace, dataload.Name))
 	if err != nil {
@@ -167,18 +172,18 @@ func (e *JuiceFSEngine) generateDataLoadValueFile(r cruntime.ReconcileRequestCon
 	return valueFile.Name(), nil
 }
 
-func (e *JuiceFSEngine) CheckRuntimeReady() (ready bool) {
-	stsName := e.getWorkerName()
-	pods, err := e.GetRunningPodsOfStatefulSet(stsName, e.namespace)
+func (j *JuiceFSEngine) CheckRuntimeReady() (ready bool) {
+	stsName := j.getWorkerName()
+	pods, err := j.GetRunningPodsOfStatefulSet(stsName, j.namespace)
 	if err != nil || len(pods) == 0 {
 		return false
 	}
 	return true
 }
 
-func (e *JuiceFSEngine) CheckExistenceOfPath(targetDataload datav1alpha1.DataLoad) (notExist bool, err error) {
+func (j *JuiceFSEngine) CheckExistenceOfPath(targetDataload datav1alpha1.DataLoad) (notExist bool, err error) {
 	// get mount path
-	cacheinfo, err := GetCacheInfoFromConfigmap(e.Client, targetDataload.Spec.Dataset.Name, targetDataload.Spec.Dataset.Namespace)
+	cacheinfo, err := GetCacheInfoFromConfigmap(j.Client, targetDataload.Spec.Dataset.Name, targetDataload.Spec.Dataset.Namespace)
 	if err != nil {
 		return
 	}
@@ -188,15 +193,15 @@ func (e *JuiceFSEngine) CheckExistenceOfPath(targetDataload datav1alpha1.DataLoa
 	}
 
 	// get worker pod
-	stsName := e.getWorkerName()
-	pods, err := e.GetRunningPodsOfStatefulSet(stsName, e.namespace)
+	stsName := j.getWorkerName()
+	pods, err := j.GetRunningPodsOfStatefulSet(stsName, j.namespace)
 	if err != nil || len(pods) == 0 {
 		return true, err
 	}
 
 	// check path exist
 	pod := pods[0]
-	fileUtils := operations.NewJuiceFileUtils(pod.Name, common.JuiceFSWorkerContainer, e.namespace, e.Log)
+	fileUtils := operations.NewJuiceFileUtils(pod.Name, common.JuiceFSWorkerContainer, j.namespace, j.Log)
 	for _, target := range targetDataload.Spec.Target {
 		targetPath := filepath.Join(mountPath, target.Path)
 		isExist, err := fileUtils.IsExist(targetPath)

diff --git a/pkg/ddc/juicefs/data_load_test.go b/pkg/ddc/juicefs/data_load_test.go
@@ -25,6 +25,7 @@ import (
 
 var valuesConfigMapData = `
 fullnameOverride: test-dataset
+edition: community
 image: juicedata/juicefs-csi-driver
 imageTag: v0.11.0
 imagePullPolicy: IfNotPresent

diff --git a/pkg/ddc/juicefs/master_internal.go b/pkg/ddc/juicefs/master_internal.go
@@ -91,7 +91,7 @@ func (j *JuiceFSEngine) generateJuicefsValueFile(runtime *datav1alpha1.JuiceFSRu
 	}
 
 	valueFileName = valueFile.Name()
-	j.Log.V(1).Info("Save the values file", "valueFile", valueFileName)
+	j.Log.Info("Save the values file", "valueFile", valueFileName)
 
 	err = ioutil.WriteFile(valueFileName, data, 0400)
 	if err != nil {