alluxio container network, sync master

Signed-off-by: mahao <[email protected]>

api/v1alpha1/jindoruntime_types.go

@@ -62,6 +62,10 @@ type JindoCompTemplateSpec struct {
 	// Any label already existed will be overriden
 	// +optional
 	Labels map[string]string `json:"labels,omitempty"`
+
+	// If disable JindoFS master or worker
+	// +optional
+	Disabled bool `json:"disabled,omitempty"`
 }
 
 // JindoFuseSpec is a description of the Jindo Fuse
@@ -120,6 +124,10 @@ type JindoFuseSpec struct {
 	// Defaults to OnRuntimeDeleted
 	// +optional
 	CleanPolicy FuseCleanPolicy `json:"cleanPolicy,omitempty"`
+
+	// If disable JindoFS fuse
+	// +optional
+	Disabled bool `json:"disabled,omitempty"`
 }
 
 // JindoRuntimeSpec defines the desired state of JindoRuntime

api/v1alpha1/juicefsruntime_types.go

@@ -50,6 +50,9 @@ type JuiceFSRuntimeSpec struct {
 	// Tiered storage used by JuiceFS
 	TieredStore TieredStore `json:"tieredstore,omitempty"`
 
+	// Configs of JuiceFS
+	Configs *[]string `json:"configs,omitempty"`
+
 	// The replicas of the worker, need to be specified
 	Replicas int32 `json:"replicas,omitempty"`
 
@@ -79,6 +82,9 @@ type JuiceFSCompTemplateSpec struct {
 	// +optional
 	Resources corev1.ResourceRequirements `json:"resources,omitempty"`
 
+	// Options
+	Options map[string]string `json:"options,omitempty"`
+
 	// Environment variables that will be used by JuiceFS component.
 	Env []corev1.EnvVar `json:"env,omitempty"`
 

charts/fluid-dataloader/jindofsx/CHANGELOG.md

@@ -0,0 +1,4 @@
+### 0.1.0
+
+- Support parallel prefetch job
+- Support configurations by setting values

charts/fluid-dataloader/jindofsx/Chart.yaml

@@ -0,0 +1,23 @@
+apiVersion: v2
+name: fluid-dataloader
+description: A Helm chart for Fluid to prefetch data
+
+# A chart can be either an 'application' or a 'library' chart.
+#
+# Application charts are a collection of templates that can be packaged into versioned archives
+# to be deployed.
+#
+# Library charts provide useful utilities or functions for the chart developer. They're included as
+# a dependency of application charts to inject those utilities and functions into the rendering
+# pipeline. Library charts do not define any templates and therefore cannot be deployed.
+type: application
+
+# This is the chart version. This version number should be incremented each time you make changes
+# to the chart and its templates, including the app version.
+# Versions are expected to follow Semantic Versioning (https://semver.org/)
+version: 0.1.0
+
+# This is the version number of the application being deployed. This version number should be
+# incremented each time you make changes to the application. Versions are not expected to
+# follow Semantic Versioning. They should reflect the version the application is using.
+appVersion: 0.1.0

charts/fluid-dataloader/jindofsx/templates/configmap.yaml

@@ -0,0 +1,139 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: {{ printf "%s-data-load-script" .Release.Name }}
+  labels:
+    release: {{ .Release.Name }}
+    role: dataload-job
+data:
+  dataloader.jindo.init: |
+    #!/usr/bin/env bash
+    set -xe
+    jindo_env_vars=(
+      STORAGE_ADDRESS
+    )
+    function public::jindo::init_conf() {
+      local IFS=$'\n' # split by line instead of space
+      for keyvaluepair in $(env); do
+        # split around the first "="
+        key=$(echo ${keyvaluepair} | cut -d= -f1)
+        value=$(echo ${keyvaluepair} | cut -d= -f2-)
+        if [[ "${jindo_env_vars[*]}" =~ "${key}" ]]; then
+          export ${key}=\"${value}\"
+        fi
+      done
+    }
+    main() {
+      public::jindo::init_conf
+    }
+    main
+  dataloader.distributedLoad: |
+    #!/usr/bin/env bash
+    set -xe
+
+    function distributedLoad() {
+        local path=$1
+        local replica=$2
+        local default=$3
+        local cmd="jindo fs -load"
+
+        if [[ $needLoadMetadata == 'true' ]]; then
+            echo -e "--- enable metaCache"
+            cmd="$cmd -meta"
+            echo -e "now cmd $cmd"
+        else
+            echo -e "--- disable metaCache"
+        fi
+
+        if [[ $loadMetadataOnly == 'true' ]]; then
+            echo -e "--- disable datacache"
+        else
+           echo -e "--- enable datacache"
+           cmd="$cmd -data"
+           echo -e "now cmd $cmd"
+        fi
+
+        if [[ $atomicCache == 'true' ]]; then
+            echo -e "--- enable atomicCache"
+            cmd="$cmd -atomic"
+            echo -e "now cmd $cmd"
+        else
+            echo -e "--- disable atomicCache"
+        fi
+
+        if [[ $loadMemorydata == 'true' ]]; then
+            echo -e "--- enable loadMemorydata"
+            cmd="$cmd -m"
+            echo -e "now cmd $cmd"
+        else
+            echo -e "--- disable loadMemorydata"
+        fi
+
+        if [[ $enbaleCacheListLocation == 'false' ]]; then
+            cmd="$cmd -s -R -replica $replica $default$path"
+            echo -e "execute cmd $cmd"
+            time $cmd
+        else
+            echo -e "--- begin download ossutil"
+            apt-get install wget -y
+            wget http://gosspublic.alicdn.com/ossutil/1.7.5/ossutil64
+            chmod 755 ossutil64
+            ./ossutil64 -e $cacheListEndpoint -i $cacheListAccessKeyId -k $cacheListAccessKeySecret cp $cacheListUrl /cachelist.txt
+            echo -e "--- enable File List"
+            cmd="$cmd -R -replica $cacheListReplica -cachelist /cachelist.txt -thread $cacheListThread $default/"
+            echo -e "execute cmd $cmd"
+            time $cmd
+        fi
+
+        #echo -e "distributedLoad and sleep start now"
+        #sleep 10m
+    }
+
+    function main() {
+        needLoadMetadata="$NEED_LOAD_METADATA"
+        loadMemorydata="$LOAD_MEMORY_DATA"
+        loadMetadataOnly="$LOAD_METADATA_ONLY"
+        atomicCache="$ENABLE_ATOMIC_CACHE"
+        cacheListReplica=$CACHE_LIST_REPLICA
+        cacheListThread=$CACHE_LIST_THREAD
+        enbaleCacheListLocation=$Enable_CACHE_LIST_LOCATION
+        cacheListAccessKeyId=$CACHE_LIST_ACCESSKEYID
+        cacheListAccessKeySecret=$CACHE_LIST_ACCESSKEYSECRET
+        cacheListEndpoint=$CACHE_LIST_ENDPOINT
+        cacheListUrl=$CACHE_LIST_URL
+        #judge whether to use locaion list
+        if [[ -z "$cacheListAccessKeyId" ]] || [[ -z "$cacheListAccessKeySecret" ]] || [[ -z "$cacheListEndpoint" ]] || [[ -z "$cacheListUrl" ]]; then
+            enbaleCacheListLocation=false
+        else
+            enbaleCacheListLocation=true
+        fi
+        if [[ -z "$cacheListReplica" ]]; then
+            cacheListReplica=1
+        else
+            echo -e "cacheListReplica $cacheListReplica"
+        fi
+        if [[ -z "$cacheListThread" ]]; then
+            cacheListThread=10
+        else
+            echo -e "cacheListThread $cacheListThread"
+        fi
+        dafault="jindo://"
+        paths="$DATA_PATH"
+        paths=(${paths//:/ })
+        replicas="$PATH_REPLICAS"
+        replicas=(${replicas//:/ })
+        for((i=0;i<${#paths[@]};i++)) do
+            local path="${paths[i]}"
+            local replica="${replicas[i]}"
+            echo -e "distributedLoad on $path starts"
+            distributedLoad ${paths[i]} ${replicas[i]} ${dafault}
+            #echo -e "distributedLoad on $path ends"
+        done
+    }
+
+    main "$@"
+
+
+
+
+