Add juicefsruntime dataload function

charts/fluid-dataloader/juicefs/CHANGELOG.md

@@ -0,0 +1,4 @@
+### 0.1.0
+
+- Support parallel prefetch job
+- Support configurations by setting values

charts/fluid-dataloader/juicefs/Chart.yaml

@@ -0,0 +1,23 @@
+apiVersion: v2
+name: fluid-dataloader
+description: A Helm chart for Fluid to prefetch data
+
+# A chart can be either an 'application' or a 'library' chart.
+#
+# Application charts are a collection of templates that can be packaged into versioned archives
+# to be deployed.
+#
+# Library charts provide useful utilities or functions for the chart developer. They're included as
+# a dependency of application charts to inject those utilities and functions into the rendering
+# pipeline. Library charts do not define any templates and therefore cannot be deployed.
+type: application
+
+# This is the chart version. This version number should be incremented each time you make changes
+# to the chart and its templates, including the app version.
+# Versions are expected to follow Semantic Versioning (https://semver.org/)
+version: 0.1.0
+
+# This is the version number of the application being deployed. This version number should be
+# incremented each time you make changes to the application. Versions are not expected to
+# follow Semantic Versioning. They should reflect the version the application is using.
+appVersion: 0.1.0

charts/fluid-dataloader/juicefs/templates/configmap.yaml

@@ -0,0 +1,34 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: {{ printf "%s-data-load-script" .Release.Name }}
+  labels:
+    release: {{ .Release.Name }}
+    role: dataload-job
+data:
+  dataloader.distributedLoad: |
+    #!/usr/bin/env bash
+    set -xe
+
+    function main() {
+        paths="$DATA_PATH"
+        paths=(${paths//:/ })
+
+        targetPath=""
+        for((j=0;j<${#paths[@]};j++)) do
+          targetPath="$targetPath $MOUNTPATH${paths[j]}"
+        done
+
+        podNames="$POD_NAMES"
+        podNames=(${podNames//:/ })
+
+        ns="$POD_NAMESPACE"
+        for((i=0;i<${#podNames[@]};i++)) do
+          local pod="${podNames[i]}"
+
+          echo -e "juicefs warmup on $pod $targetPath starts"
+          /usr/local/bin/kubectl -n $ns exec -it $pod -- timeout $TIMEOUT juicefs warmup $targetPath
+          echo -e "juicefs warmup on $pod $targetPath ends"
+        done
+    }
+    main "$@"

charts/fluid-dataloader/juicefs/templates/dataloader.yaml

@@ -0,0 +1,144 @@
+# .Release.Name will be used to decide which dataset will be preload
+# .Release.Name should be like `<pvc-name>-load`(e.g. hbase-load for a PersistentVolumeClaim named `hbase`)
+# TODO: the length of .Release.Name won't exceed 53(limited by Helm), which means length of `<pvc-name>` can't exceed 48. This might be a problem.
+  {{/*  {{  $datasetName := "" -}}*/}}
+  {{/*  {{- $randomSuffix := "" -}}*/}}
+  {{/*  {{- if regexMatch "^[A-Za-z0-9._-]+-load-[A-Za-z0-9]{5}$" .Release.Name -}}*/}}
+  {{/*    {{- $arr := regexSplit "-load-" .Release.Name -1 -}}*/}}
+  {{/*    {{- $datasetName = first $arr -}}*/}}
+  {{/*    {{- $randomSuffix = last $arr -}}*/}}
+  {{/*  {{- else -}}*/}}
+  {{/*    {{- printf "Illegal release name. Should be like <dataset-name>-load-<suffix-length-5>. Current name: %s" .Release.Name | fail -}}*/}}
+  {{/*  {{- end }}*/}}
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: {{ printf "%s-job" .Release.Name }}
+  labels:
+    release: {{ .Release.Name }}
+    role: dataload-job
+    app: juicefs
+    targetDataset: {{ required "targetDataset should be set" .Values.dataloader.targetDataset }}
+spec:
+  backoffLimit: {{ .Values.dataloader.backoffLimit | default "3" }}
+  completions: 1
+  parallelism: 1
+  template:
+    metadata:
+      name: {{ printf "%s-loader" .Release.Name }}
+      labels:
+        release: {{ .Release.Name }}
+        role: dataload-pod
+        app: juicefs
+        targetDataset: {{ required "targetDataset should be set" .Values.dataloader.targetDataset }}
+    spec:
+      restartPolicy: OnFailure
+      {{- range $key, $val := .Values.dataloader.options }}
+      {{- if eq $key "runtimeName" }}
+      serviceAccountName: {{ printf "%s-loader" $val | quote }}
+      {{- end }}
+      {{- end }}
+      containers:
+        - name: dataloader
+          image: {{ required "Dataloader image should be set" .Values.dataloader.image }}
+          imagePullPolicy: IfNotPresent
+          command: ["/bin/sh", "-c"]
+          args: ["/scripts/juicefs_dataload.sh"]
+          {{- $targetPaths := "" }}
+          {{- range .Values.dataloader.targetPaths }}
+          {{- $targetPaths = cat $targetPaths (required "Path must be set" .path) ":" }}
+          {{- end }}
+          {{- $targetPaths = $targetPaths | nospace | trimSuffix ":" }}
+
+          {{- $pathReplicas := ""}}
+          {{- range .Values.dataloader.targetPaths }}
+          {{- $pathReplicas = cat $pathReplicas ( default 1 .replicas ) ":"}}
+          {{- end }}
+          {{- $pathReplicas = $pathReplicas | nospace | trimSuffix ":"}}
+
+          env:
+            - name: STORAGE_ADDRESS
+              valueFrom:
+                fieldRef:
+                  fieldPath: status.podIP
+            - name: CACHEDIR2
+              value: /test
+            {{- range $key, $val := .Values.dataloader.options }}
+            {{- if eq $key "cachedir" }}
+            - name: CACHEDIR
+              value: {{ $val | quote }}
+            {{- end }}
+            {{- if eq $key "mountpath" }}
+            - name: MOUNTPATH
+              value: {{ $val | quote }}
+            {{- end }}
+            {{- if eq $key "podNames" }}
+            - name: POD_NAMES
+              value: {{ $val | quote }}
+            {{- end }}
+            {{- if eq $key "timeout" }}
+            - name: TIMEOUT
+              value: {{ $val | quote }}
+            {{- end }}
+            {{- end }}
+            - name: DATA_PATH
+              value: {{ $targetPaths | quote }}
+            - name: PATH_REPLICAS
+              value: {{ $pathReplicas | quote }}
+            - name: POD_NAMESPACE
+              value: {{ .Release.Namespace | quote }}
+          envFrom:
+            - configMapRef:
+                name: {{ required "targetDataset should be set" .Values.dataloader.targetDataset }}-juicefs-values
+          securityContext:
+            privileged: true
+          volumeMounts:
+            - mountPath: /scripts
+              name: data-load-script
+            {{- range .Values.dataloader.targetPaths }}
+            {{- if .fluidNative }}
+            - mountPath: {{ .path | trimAll "/" | replace "/" "-" | printf "/data/%s"}}
+              name: {{ .path | trimAll "/" | replace "/" "-" | printf "native-%s"}}
+            {{- end }}
+            {{- end }}
+            {{- range $key, $val := .Values.dataloader.options }}
+            {{- if eq $key "cachedir" }}
+            - mountPath: {{ $val | quote }}
+            {{- end }}
+            {{- end }}
+              name: cachedir
+            {{- range $key, $val := .Values.dataloader.options }}
+            {{- if eq $key "mountpath" }}
+            - mountPath: {{ $val | quote }}
+            {{- end }}
+            {{- end }}
+              name: mountpath1
+      volumes:
+        - name: data-load-script
+          configMap:
+            name: {{ printf "%s-data-load-script" .Release.Name }}
+            items:
+              - key: dataloader.distributedLoad
+                path: juicefs_dataload.sh
+                mode: 365
+        {{- range .Values.dataloader.targetPaths }}
+        {{- if .fluidNative }}
+        - name: {{ .path | trimAll "/" | replace "/" "-" | printf "native-%s"}}
+          hostPath:
+            path: {{ .path }}
+        {{- end }}
+        {{- end }}
+        - name: cachedir
+        {{- range $key, $val := .Values.dataloader.options }}
+        {{- if eq $key "cachedir" }}
+          hostPath:
+            path: {{ $val | quote }}
+        {{- end }}
+        {{- end }}
+        - name: mountpath1
+        {{- range $key, $val := .Values.dataloader.options }}
+        {{- if eq $key "mountpath" }}
+          hostPath:
+            path: {{ $val | quote }}
+        {{- end }}
+        {{- end }}

charts/fluid-dataloader/juicefs/values.yaml

@@ -0,0 +1,34 @@
+# Default values for fluid-dataloader.
+# This is a YAML-formatted file.
+# Declare variables to be passed into your templates.
+
+dataloader:
+  # Optional
+  # Default: 3
+  # Description: how many times the prefetch job can fail, i.e. `Job.spec.backoffLimit`
+  backoffLimit: 3
+
+  # Required
+  # Description: the dataset that this DataLoad targets
+  targetDataset: #imagenet
+
+  # Optional
+  # Default: false
+  # Description: should load metadata from UFS when doing data load
+  loadMetadata: false
+
+  # Optional
+  # Default: (path: "/", replicas: 1, fluidNative: false)
+  # Description: which paths should the DataLoad load
+  targetPaths:
+    - path: "/"
+      replicas: 1
+      fluidNative: false
+
+  # Required
+  # Description: the image that the DataLoad job uses
+  image: #<juicefs-image>
+
+  # Optional
+  # Description: optional parameter DataLoad job uses
+  options:

charts/fluid/fluid/templates/role/dataset/rbac.yaml

@@ -46,12 +46,6 @@ rules:
       - juicefsruntimes/status
     verbs:
       - '*'
-  - apiGroups:
-      - ""
-    resources:
-      - events
-    verbs:
-      - '*'
   - apiGroups:
       - apps
     resources:

charts/fluid/fluid/templates/role/juicefs/rbac.yaml

@@ -29,6 +29,25 @@ rules:
       - datasets/status
     verbs:
       - '*'
+  - apiGroups:
+      - ""
+    resources:
+      - serviceaccounts
+    verbs:
+      - create
+      - list
+      - get
+      - delete
+  - apiGroups:
+      - rbac.authorization.k8s.io
+    resources:
+      - clusterroles
+      - clusterrolebindings
+    verbs:
+      - create
+      - list
+      - get
+      - delete
   - apiGroups:
       - apps
     resources:

charts/fluid/fluid/values.yaml

@@ -66,7 +66,7 @@ runtime:
     controller:
       image: fluidcloudnative/juicefsruntime-controller:v0.8.0-b491ba8
     fuse:
-      image: juicedata/juicefs-csi-driver:v0.11.0
+      image: registry.cn-hangzhou.aliyuncs.com/juicefs/juicefs-fuse:v1.0.0-beta2
 
 webhook:
   enabled: true

charts/juicefs/templates/fuse/daemonset.yaml

@@ -131,7 +131,7 @@ spec:
         - name: script
           configMap:
             name: {{ template "juicefs.fullname" . }}-script
-            defaultMode: 0777
+            defaultMode: 0755
 ---
 apiVersion: v1
 kind: ConfigMap

charts/juicefs/templates/role/rbac.yaml

@@ -0,0 +1,34 @@
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: {{ printf "%s-loader" .Release.Name }}
+rules:
+  - apiGroups:
+      - ""
+    resources:
+      - pods
+      - pods/exec
+    verbs:
+      - get
+      - create
+      - list
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: {{ printf "%s-loader" .Release.Name }}
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: {{ printf "%s-loader" .Release.Name }}
+subjects:
+  - kind: ServiceAccount
+    name: {{ printf "%s-loader" .Release.Name }}
+    namespace: {{ .Release.Namespace | quote }}
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: {{ printf "%s-loader" .Release.Name }}
+  namespace: {{ .Release.Namespace | quote }}