Jindo fuse recover support and upgrade jindofs version (fluid-cloudna…

…tive#1385) * JindoFS fuse recover Signed-off-by: frankleaf <[email protected]> * refine Signed-off-by: frankleaf <[email protected]> * refine Signed-off-by: frankleaf <[email protected]> * refine Signed-off-by: frankleaf <[email protected]> * refine Signed-off-by: frankleaf <[email protected]> * refine Signed-off-by: frankleaf <[email protected]> Signed-off-by: zwwhdls <[email protected]>
ldd91 · Mar 18, 2022 · a5611de · a5611de
1 parent d25d80b
commit a5611de
Show file tree

Hide file tree

Showing 15 changed files with 74 additions and 17 deletions.
diff --git a/api/v1alpha1/zz_generated.deepcopy.go b/api/v1alpha1/zz_generated.deepcopy.go
diff --git a/charts/fluid/fluid/values.yaml b/charts/fluid/fluid/values.yaml
@@ -40,9 +40,9 @@ runtime:
     portRange: 18000-19999
     enabled: false
     smartdata:
-      image: registry.cn-shanghai.aliyuncs.com/jindofs/smartdata:3.7.3
+      image: registry.cn-shanghai.aliyuncs.com/jindofs/smartdata:3.8.0
     fuse:
-      image: registry.cn-shanghai.aliyuncs.com/jindofs/jindo-fuse:3.7.3
+      image: registry.cn-shanghai.aliyuncs.com/jindofs/jindo-fuse:3.8.0
     controller:
       image: fluidcloudnative/jindoruntime-controller:v0.7.0-67ed8c2
     init:

diff --git a/charts/fluid/v0.6.0/values.yaml b/charts/fluid/v0.6.0/values.yaml
@@ -35,9 +35,9 @@ runtime:
     portRange: 18000-19999
     enabled: false
     smartdata:
-      image: registry.cn-shanghai.aliyuncs.com/jindofs/smartdata:3.7.3
+      image: registry.cn-shanghai.aliyuncs.com/jindofs/smartdata:3.8.0
     fuse:
-      image: registry.cn-shanghai.aliyuncs.com/jindofs/jindo-fuse:3.7.3
+      image: registry.cn-shanghai.aliyuncs.com/jindofs/jindo-fuse:3.8.0
     controller:
       image: fluidcloudnative/jindoruntime-controller:v0.6.0-48de610
   goosefs:

diff --git a/charts/jindofs/CHANGELOG.md b/charts/jindofs/CHANGELOG.md
@@ -1,7 +1,8 @@
 0.7.0
-Support logConfig
-Support fuse lazy start
-Support fuse critical pod
+
+Support logConfig <br/>
+Support fuse lazy start <br/>
+Support fuse critical pod <br/>
 
 0.8.0
 
@@ -22,5 +23,10 @@ Add owner Reference
 0.8.4
 
 Support more Posix
+
+0.8.5
+
+Add mountPropagation for registrar <br/>
+Add auto fuse recovery
 
 
diff --git a/charts/jindofs/Chart.yaml b/charts/jindofs/Chart.yaml
@@ -1,5 +1,5 @@
 apiVersion: v1
-appVersion: 3.7.3
+appVersion: 3.8.0
 version: 0.8.3
 description: FileSystem on the cloud based on Aliyun Object Storage aimed for data
   acceleration.

diff --git a/charts/jindofs/templates/fuse/daemonset.yaml b/charts/jindofs/templates/fuse/daemonset.yaml
@@ -106,6 +106,8 @@ spec:
             valueFrom:
               fieldRef:
                 fieldPath: status.hostIP
+          - name: FLUID_FUSE_MOUNTPOINT
+            value: {{ .Values.fuse.mountPath }}
           securityContext:
             privileged: true
             capabilities:

diff --git a/charts/jindofs/values.yaml b/charts/jindofs/values.yaml
@@ -4,7 +4,7 @@
 
 
 image: registry-vpc.__ACK_REGION_ID__.aliyuncs.com/jindo/smartdata
-imageTag: "3.7.3"
+imageTag: "3.8.0"
 imagePullPolicy: Always
 
 initPortCheck:
@@ -15,7 +15,7 @@ initPortCheck:
   portsToCheck: ""
 
 fuseImage: registry-vpc.__ACK_REGION_ID__.aliyuncs.com/jindo/jindo-fuse
-fuseImageTag: "3.7.3"
+fuseImageTag: "3.8.0"
 
 user: 0
 group: 0

diff --git a/pkg/common/jindo.go b/pkg/common/jindo.go
@@ -26,7 +26,7 @@ const (
 
 	JINDO_FUSE_IMAGE_ENV = "JINDO_FUSE_IMAGE_ENV"
 
-	DEFAULT_JINDO_RUNTIME_IMAGE = "registry.cn-shanghai.aliyuncs.com/jindofs/smartdata:3.7.3"
+	DEFAULT_JINDO_RUNTIME_IMAGE = "registry.cn-shanghai.aliyuncs.com/jindofs/smartdata:3.8.0"
 
 	JINDO_DNS_SERVER = "JINDO_DNS_SERVER_ENV"
 )
diff --git a/pkg/ddc/jindo/const.go b/pkg/ddc/jindo/const.go
@@ -31,4 +31,6 @@ const (
 	WOKRER_POD_ROLE = "jindo-worker"
 
 	runtimeFSType = "jindofs"
+
+	JINDO_FUSE_MONNTPATH = "/jfs/jindofs-fuse"
 )
diff --git a/pkg/ddc/jindo/transform.go b/pkg/ddc/jindo/transform.go
@@ -88,7 +88,7 @@ func (e *JindoEngine) transform(runtime *datav1alpha1.JindoRuntime) (value *Jind
 		},
 		Fuse: Fuse{
 			Args:     e.transformFuseArg(runtime, dataset),
-			HostPath: e.getMountPoint(),
+			HostPath: e.getHostMountPoint(),
 		},
 		Mounts: Mounts{
 			Master:            e.transformMasterMountPath(metaPath),
@@ -148,6 +148,7 @@ func (e *JindoEngine) transform(runtime *datav1alpha1.JindoRuntime) (value *Jind
 	e.transformLogConfig(runtime, value)
 	value.Master.DnsServer = dnsServer
 	value.Master.NameSpace = e.namespace
+	value.Fuse.MountPath = JINDO_FUSE_MONNTPATH
 	return value, err
 }
 
@@ -496,7 +497,7 @@ func (e *JindoEngine) transformFuseArg(runtime *datav1alpha1.JindoRuntime, datas
 func (e *JindoEngine) getSmartDataConfigs() (image, tag, dnsServer string) {
 	var (
 		defaultImage     = "registry.cn-shanghai.aliyuncs.com/jindofs/smartdata"
-		defaultTag       = "3.7.3"
+		defaultTag       = "3.8.0"
 		defaultDnsServer = "1.1.1.1"
 	)
 
@@ -520,7 +521,7 @@ func (e *JindoEngine) getSmartDataConfigs() (image, tag, dnsServer string) {
 func (e *JindoEngine) parseFuseImage() (image, tag string) {
 	var (
 		defaultImage = "registry.cn-shanghai.aliyuncs.com/jindofs/jindo-fuse"
-		defaultTag   = "3.7.3"
+		defaultTag   = "3.8.0"
 	)
 
 	image = docker.GetImageRepoFromEnv(common.JINDO_FUSE_IMAGE_ENV)

diff --git a/pkg/ddc/jindo/transform_fuse_test.go b/pkg/ddc/jindo/transform_fuse_test.go
@@ -160,7 +160,7 @@ func TestParseFuseImage(t *testing.T) {
 					Name:       "test",
 					Path:       "/",
 				}},
-			}}, &Jindo{}, "registry.cn-shanghai.aliyuncs.com/jindofs/jindo-fuse:3.7.3"},
+			}}, &Jindo{}, "registry.cn-shanghai.aliyuncs.com/jindofs/jindo-fuse:3.8.0"},
 	}
 	for _, test := range tests {
 		engine := &JindoEngine{Log: log.NullLogger{}}

diff --git a/pkg/ddc/jindo/transform_test.go b/pkg/ddc/jindo/transform_test.go
@@ -117,7 +117,7 @@ func TestParseSmartDataImage(t *testing.T) {
 					Name:       "test",
 					Path:       "/",
 				}},
-			}}, &Jindo{}, "registry.cn-shanghai.aliyuncs.com/jindofs/smartdata:3.7.3"},
+			}}, &Jindo{}, "registry.cn-shanghai.aliyuncs.com/jindofs/smartdata:3.8.0"},
 	}
 	for _, test := range tests {
 		engine := &JindoEngine{Log: log.NullLogger{}}

diff --git a/pkg/ddc/jindo/types.go b/pkg/ddc/jindo/types.go
@@ -78,6 +78,7 @@ type Fuse struct {
 	Labels         map[string]string `yaml:"labels,omitempty"`
 	CriticalPod    bool              `yaml:"criticalPod,omitempty"`
 	Resources      Resources         `yaml:"resources,omitempty"`
+	MountPath      string            `yaml:"mountPath,omitempty"`
 }
 
 type Mounts struct {

diff --git a/pkg/ddc/jindo/utils.go b/pkg/ddc/jindo/utils.go
@@ -27,6 +27,12 @@ func (e *JindoEngine) getMountPoint() (mountPath string) {
 	return fmt.Sprintf("%s/%s/%s/jindofs-fuse", mountRoot, e.namespace, e.name)
 }
 
+func (j *JindoEngine) getHostMountPoint() (mountPath string) {
+	mountRoot := getMountRoot()
+	j.Log.Info("mountRoot", "path", mountRoot)
+	return fmt.Sprintf("%s/%s/%s", mountRoot, j.namespace, j.name)
+}
+
 // getMountRoot returns the default path, if it's not set
 func getMountRoot() (path string) {
 	path, err := utils.GetMountRoot()

diff --git a/pkg/ddc/jindo/utils_test.go b/pkg/ddc/jindo/utils_test.go
@@ -16,7 +16,9 @@ limitations under the License.
 package jindo
 
 import (
+	"github.com/go-logr/logr"
 	"os"
+	"sigs.k8s.io/controller-runtime/pkg/log"
 	"testing"
 
 	"github.com/fluid-cloudnative/fluid/pkg/common"
@@ -79,3 +81,41 @@ func TestMountRootWithoutEnvSet(t *testing.T) {
 		}
 	}
 }
+
+func TestJindoFSEngine_getHostMountPoint(t *testing.T) {
+	type fields struct {
+		name      string
+		namespace string
+		Log       logr.Logger
+		MountRoot string
+	}
+	var tests = []struct {
+		name          string
+		fields        fields
+		wantMountPath string
+	}{
+		{
+			name: "test",
+			fields: fields{
+				name:      "jindofs",
+				namespace: "default",
+				Log:       log.NullLogger{},
+				MountRoot: "/tmp",
+			},
+			wantMountPath: "/tmp/jindo/default/jindofs",
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			j := &JindoEngine{
+				name:      tt.fields.name,
+				namespace: tt.fields.namespace,
+				Log:       tt.fields.Log,
+			}
+			os.Setenv("MOUNT_ROOT", tt.fields.MountRoot)
+			if gotMountPath := j.getHostMountPoint(); gotMountPath != tt.wantMountPath {
+				t.Errorf("getHostMountPoint() = %v, want %v", gotMountPath, tt.wantMountPath)
+			}
+		})
+	}
+}