From f45ee0b6f681f88ceaa21f33d4018a6f50870458 Mon Sep 17 00:00:00 2001 From: zhijian Date: Fri, 15 Mar 2024 17:27:34 +0800 Subject: [PATCH 01/14] recovry read only file handler --- pkg/vfs/handle.go | 20 +++++++++++++++-- pkg/vfs/internal.go | 2 +- pkg/vfs/vfs.go | 55 ++++++++++++++++++++++++++++++++++++--------- 3 files changed, 64 insertions(+), 13 deletions(-) diff --git a/pkg/vfs/handle.go b/pkg/vfs/handle.go index 4d7d34cd617c..eac59cfa4be5 100644 --- a/pkg/vfs/handle.go +++ b/pkg/vfs/handle.go @@ -160,9 +160,16 @@ func (h *handle) Close() { } } -func (v *VFS) newHandle(inode Ino) *handle { +func (v *VFS) newHandle(inode Ino, readOnly bool) *handle { v.hanleM.Lock() defer v.hanleM.Unlock() + var lowBits uint64 + if readOnly { + lowBits = 1 + } + for v.handleIno[v.nextfh] > 0 || v.nextfh&1 != lowBits { + v.nextfh++ // skip recovered fd + } fh := v.nextfh h := &handle{inode: inode, fh: fh} v.nextfh++ @@ -192,6 +199,15 @@ func (v *VFS) findHandle(inode Ino, fh uint64) *handle { return f } } + if fh&1 == 1 && inode != controlInode { + f := &handle{inode: inode, fh: fh} + f.cond = utils.NewCond(f) + v.handles[inode] = append(v.handles[inode], f) + if v.handleIno[fh] == 0 { + v.handleIno[fh] = inode + } + return f + } return nil } @@ -215,7 +231,7 @@ func (v *VFS) releaseHandle(inode Ino, fh uint64) { } func (v *VFS) newFileHandle(inode Ino, length uint64, flags uint32) uint64 { - h := v.newHandle(inode) + h := v.newHandle(inode, (flags&syscall.O_ACCMODE) == syscall.O_RDONLY) h.Lock() defer h.Unlock() h.flags = flags diff --git a/pkg/vfs/internal.go b/pkg/vfs/internal.go index 52ba913385b5..77d93c2069a1 100644 --- a/pkg/vfs/internal.go +++ b/pkg/vfs/internal.go @@ -52,7 +52,7 @@ func (v *VFS) getControlHandle(pid uint32) uint64 { defer controlMutex.Unlock() fh := controlHandlers[pid] if fh == 0 { - h := v.newHandle(controlInode) + h := v.newHandle(controlInode, false) fh = h.fh controlHandlers[pid] = fh } diff --git a/pkg/vfs/vfs.go b/pkg/vfs/vfs.go index be1d48cf189c..4bff277d1098 100644 --- a/pkg/vfs/vfs.go +++ b/pkg/vfs/vfs.go @@ -397,7 +397,7 @@ func (v *VFS) Opendir(ctx Context, ino Ino, flags uint32) (fh uint64, err syscal return } } - fh = v.newHandle(ino).fh + fh = v.newHandle(ino, true).fh return } @@ -519,7 +519,7 @@ func (v *VFS) Open(ctx Context, ino Ino, flags uint32) (entry *meta.Entry, fh ui err = syscall.EACCES return } - h := v.newHandle(ino) + h := v.newHandle(ino, true) fh = h.fh n := getInternalNode(ino) if n == nil { @@ -647,18 +647,32 @@ func (v *VFS) Release(ctx Context, ino Ino, fh uint64) { func (v *VFS) Read(ctx Context, ino Ino, buf []byte, off uint64, fh uint64) (n int, err syscall.Errno) { size := uint32(len(buf)) if IsSpecialNode(ino) { + if ino == controlInode && runtime.GOOS == "darwin" { + fh = v.getControlHandle(ctx.Pid()) + } + h := v.findHandle(ino, fh) + if h == nil { + err = syscall.EBADF + return + } + switch ino { + case logInode: + openAccessLog(fh) + case statsInode: + h.data = collectMetrics(v.registry) + case configInode: + v.Conf.Format = v.Meta.GetFormat() + if v.UpdateFormat != nil { + v.UpdateFormat(&v.Conf.Format) + } + v.Conf.Format.RemoveSecret() + h.data, _ = json.MarshalIndent(v.Conf, "", " ") + } + if ino == logInode { n = readAccessLog(fh, buf) } else { defer func() { logit(ctx, "read (%d,%d,%d,%d): %s (%d)", ino, size, off, fh, strerr(err), n) }() - if ino == controlInode && runtime.GOOS == "darwin" { - fh = v.getControlHandle(ctx.Pid()) - } - h := v.findHandle(ino, fh) - if h == nil { - err = syscall.EBADF - return - } h.Lock() defer h.Unlock() if off < h.off { @@ -687,10 +701,31 @@ func (v *VFS) Read(ctx Context, ino Ino, buf []byte, off uint64, fh uint64) (n i err = syscall.EBADF return } + if h.flags == 0 { + // recovered + var attr Attr + err = v.Meta.Open(ctx, ino, syscall.O_RDONLY, &attr) + if err != 0 { + v.releaseHandle(ino, fh) + err = syscall.EBADF + return + } + h.Lock() + v.UpdateLength(ino, &attr) + h.flags = syscall.O_RDONLY + h.reader = v.reader.Open(h.inode, attr.Length) + h.Unlock() + } + if off >= maxFileSize || off+uint64(size) >= maxFileSize { err = syscall.EFBIG return } + // there could be read operation for write-only if kernel writeback is enabled + if !v.Conf.FuseOpts.EnableWriteback && (h.flags&syscall.O_ACCMODE) != syscall.O_RDONLY { + err = syscall.EACCES + return + } if h.reader == nil { err = syscall.EBADF return From f727feb9073b4f1bad20f2a3826910d8c679f7ab Mon Sep 17 00:00:00 2001 From: zhijian Date: Fri, 15 Mar 2024 17:44:46 +0800 Subject: [PATCH 02/14] adjust --- pkg/vfs/handle.go | 2 +- pkg/vfs/vfs.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/vfs/handle.go b/pkg/vfs/handle.go index eac59cfa4be5..0f7ca29192aa 100644 --- a/pkg/vfs/handle.go +++ b/pkg/vfs/handle.go @@ -231,7 +231,7 @@ func (v *VFS) releaseHandle(inode Ino, fh uint64) { } func (v *VFS) newFileHandle(inode Ino, length uint64, flags uint32) uint64 { - h := v.newHandle(inode, (flags&syscall.O_ACCMODE) == syscall.O_RDONLY) + h := v.newHandle(inode, (flags&O_ACCMODE) == syscall.O_RDONLY) h.Lock() defer h.Unlock() h.flags = flags diff --git a/pkg/vfs/vfs.go b/pkg/vfs/vfs.go index 4bff277d1098..541dbbc8fb73 100644 --- a/pkg/vfs/vfs.go +++ b/pkg/vfs/vfs.go @@ -722,7 +722,7 @@ func (v *VFS) Read(ctx Context, ino Ino, buf []byte, off uint64, fh uint64) (n i return } // there could be read operation for write-only if kernel writeback is enabled - if !v.Conf.FuseOpts.EnableWriteback && (h.flags&syscall.O_ACCMODE) != syscall.O_RDONLY { + if !v.Conf.FuseOpts.EnableWriteback && (h.flags&O_ACCMODE) != syscall.O_RDONLY { err = syscall.EACCES return } From 191a632fb94c075709e4a49c529ffb9927102745 Mon Sep 17 00:00:00 2001 From: zhijian Date: Fri, 15 Mar 2024 19:21:26 +0800 Subject: [PATCH 03/14] test --- pkg/vfs/vfs.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/pkg/vfs/vfs.go b/pkg/vfs/vfs.go index 541dbbc8fb73..e63cc47f7c12 100644 --- a/pkg/vfs/vfs.go +++ b/pkg/vfs/vfs.go @@ -722,10 +722,10 @@ func (v *VFS) Read(ctx Context, ino Ino, buf []byte, off uint64, fh uint64) (n i return } // there could be read operation for write-only if kernel writeback is enabled - if !v.Conf.FuseOpts.EnableWriteback && (h.flags&O_ACCMODE) != syscall.O_RDONLY { - err = syscall.EACCES - return - } + //if !v.Conf.FuseOpts.EnableWriteback && (h.flags&O_ACCMODE) != syscall.O_RDONLY { + // err = syscall.EACCES + // return + //} if h.reader == nil { err = syscall.EBADF return From 11ce497e6df65f8f65300cfe497cc3b1ed337acf Mon Sep 17 00:00:00 2001 From: zhijian Date: Fri, 15 Mar 2024 19:42:22 +0800 Subject: [PATCH 04/14] fix test --- pkg/vfs/vfs_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/vfs/vfs_test.go b/pkg/vfs/vfs_test.go index 6c3ed62359e1..c8f8ccc11a8b 100644 --- a/pkg/vfs/vfs_test.go +++ b/pkg/vfs/vfs_test.go @@ -205,7 +205,7 @@ func TestVFSIO(t *testing.T) { t.Fatalf("write file: %s", e) } var attr meta.Attr - if e = v.Truncate(ctx, fe.Inode, (100<<20)+2, 1, &attr); e != 0 { + if e = v.Truncate(ctx, fe.Inode, (100<<20)+2, fh, &attr); e != 0 { t.Fatalf("truncate file: %s", e) } if n, e := v.CopyFileRange(ctx, fe.Inode, fh, 0, fe.Inode, fh, 10<<20, 10, 0); e != 0 || n != 10 { From 48fe898e2188737deab0c505e9aea7ad52280b6d Mon Sep 17 00:00:00 2001 From: zhijian Date: Fri, 15 Mar 2024 20:48:02 +0800 Subject: [PATCH 05/14] fix --- pkg/vfs/vfs.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/pkg/vfs/vfs.go b/pkg/vfs/vfs.go index e63cc47f7c12..6c5fad1f9461 100644 --- a/pkg/vfs/vfs.go +++ b/pkg/vfs/vfs.go @@ -656,8 +656,6 @@ func (v *VFS) Read(ctx Context, ino Ino, buf []byte, off uint64, fh uint64) (n i return } switch ino { - case logInode: - openAccessLog(fh) case statsInode: h.data = collectMetrics(v.registry) case configInode: From 35b1f9dba9ecd35364da1941ae40462504fc0a2b Mon Sep 17 00:00:00 2001 From: zhijian Date: Fri, 15 Mar 2024 21:02:36 +0800 Subject: [PATCH 06/14] fix --- pkg/vfs/handle.go | 6 +++++- pkg/vfs/vfs.go | 8 ++++---- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/pkg/vfs/handle.go b/pkg/vfs/handle.go index 0f7ca29192aa..f37c47e15f9e 100644 --- a/pkg/vfs/handle.go +++ b/pkg/vfs/handle.go @@ -230,8 +230,12 @@ func (v *VFS) releaseHandle(inode Ino, fh uint64) { } } +func hasReadPerm(flag uint32) bool { + return (flag & O_ACCMODE) != syscall.O_WRONLY +} + func (v *VFS) newFileHandle(inode Ino, length uint64, flags uint32) uint64 { - h := v.newHandle(inode, (flags&O_ACCMODE) == syscall.O_RDONLY) + h := v.newHandle(inode, hasReadPerm(flags)) h.Lock() defer h.Unlock() h.flags = flags diff --git a/pkg/vfs/vfs.go b/pkg/vfs/vfs.go index 6c5fad1f9461..8f863ef35655 100644 --- a/pkg/vfs/vfs.go +++ b/pkg/vfs/vfs.go @@ -720,10 +720,10 @@ func (v *VFS) Read(ctx Context, ino Ino, buf []byte, off uint64, fh uint64) (n i return } // there could be read operation for write-only if kernel writeback is enabled - //if !v.Conf.FuseOpts.EnableWriteback && (h.flags&O_ACCMODE) != syscall.O_RDONLY { - // err = syscall.EACCES - // return - //} + if !v.Conf.FuseOpts.EnableWriteback && !hasReadPerm(h.flags) { + err = syscall.EACCES + return + } if h.reader == nil { err = syscall.EBADF return From 0e7abbbdcd37411bfa6cfd61bc0e59b3e0e73c8c Mon Sep 17 00:00:00 2001 From: zhijian Date: Mon, 18 Mar 2024 10:06:40 +0800 Subject: [PATCH 07/14] adjust --- pkg/vfs/vfs.go | 28 +++++++++++++++++++++++++--- 1 file changed, 25 insertions(+), 3 deletions(-) diff --git a/pkg/vfs/vfs.go b/pkg/vfs/vfs.go index 8f863ef35655..d2aecc3ad81c 100644 --- a/pkg/vfs/vfs.go +++ b/pkg/vfs/vfs.go @@ -694,13 +694,35 @@ func (v *VFS) Read(ctx Context, ino Ino, buf []byte, off uint64, fh uint64) (n i readSizeHistogram.Observe(float64(n)) logit(ctx, "read (%d,%d,%d): %s (%d)", ino, size, off, strerr(err), n) }() - h := v.findHandle(ino, fh) + + var isRecovered bool + var findHandle = func(inode Ino, fh uint64) *handle { + v.hanleM.Lock() + defer v.hanleM.Unlock() + for _, f := range v.handles[inode] { + if f.fh == fh { + return f + } + } + if fh&1 == 1 && inode != controlInode { + isRecovered = true + f := &handle{inode: inode, fh: fh} + f.cond = utils.NewCond(f) + v.handles[inode] = append(v.handles[inode], f) + if v.handleIno[fh] == 0 { + v.handleIno[fh] = inode + } + return f + } + return nil + } + + h := findHandle(ino, fh) if h == nil { err = syscall.EBADF return } - if h.flags == 0 { - // recovered + if isRecovered { var attr Attr err = v.Meta.Open(ctx, ino, syscall.O_RDONLY, &attr) if err != 0 { From 4c521fe22dd61dc45a0d65cf3a0f3853d6ea8d66 Mon Sep 17 00:00:00 2001 From: zhijian Date: Mon, 18 Mar 2024 11:25:18 +0800 Subject: [PATCH 08/14] fix --- pkg/vfs/vfs.go | 9 +++++---- pkg/vfs/vfs_test.go | 1 + 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/pkg/vfs/vfs.go b/pkg/vfs/vfs.go index d2aecc3ad81c..c98df20d1099 100644 --- a/pkg/vfs/vfs.go +++ b/pkg/vfs/vfs.go @@ -741,15 +741,16 @@ func (v *VFS) Read(ctx Context, ino Ino, buf []byte, off uint64, fh uint64) (n i err = syscall.EFBIG return } + if h.reader == nil { + err = syscall.EBADF + return + } + // there could be read operation for write-only if kernel writeback is enabled if !v.Conf.FuseOpts.EnableWriteback && !hasReadPerm(h.flags) { err = syscall.EACCES return } - if h.reader == nil { - err = syscall.EBADF - return - } if !h.Rlock(ctx) { err = syscall.EINTR return diff --git a/pkg/vfs/vfs_test.go b/pkg/vfs/vfs_test.go index c8f8ccc11a8b..190538f38f03 100644 --- a/pkg/vfs/vfs_test.go +++ b/pkg/vfs/vfs_test.go @@ -67,6 +67,7 @@ func createTestVFS() (*VFS, object.ObjectStorage) { CacheSize: 10, CacheDir: "memory", }, + FuseOpts: &FuseOptions{}, } blob, _ := object.CreateStorage("mem", "", "", "", "") registry := prometheus.NewRegistry() // replace default so only JuiceFS metrics are exposed From b5fdf761bac56d94cf605393085ad690bcc11848 Mon Sep 17 00:00:00 2001 From: zhijian Date: Mon, 18 Mar 2024 11:55:55 +0800 Subject: [PATCH 09/14] fix test --- pkg/fuse/fuse_test.go | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/pkg/fuse/fuse_test.go b/pkg/fuse/fuse_test.go index 6d7cf33323f9..d7a53cf00e36 100644 --- a/pkg/fuse/fuse_test.go +++ b/pkg/fuse/fuse_test.go @@ -95,9 +95,10 @@ func mount(url, mp string) { })) conf := &vfs.Config{ - Meta: metaConf, - Format: *format, - Chunk: &chunkConf, + Meta: metaConf, + Format: *format, + Chunk: &chunkConf, + FuseOpts: &FuseOptions{}, } err = m.NewSession(true) From 179af719b67679792667359fc4d27e5e8a3bbda6 Mon Sep 17 00:00:00 2001 From: zhijian Date: Mon, 18 Mar 2024 12:04:31 +0800 Subject: [PATCH 10/14] fix test --- pkg/fuse/fuse_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/fuse/fuse_test.go b/pkg/fuse/fuse_test.go index d7a53cf00e36..7b980f9a5d02 100644 --- a/pkg/fuse/fuse_test.go +++ b/pkg/fuse/fuse_test.go @@ -98,7 +98,7 @@ func mount(url, mp string) { Meta: metaConf, Format: *format, Chunk: &chunkConf, - FuseOpts: &FuseOptions{}, + FuseOpts: &vfs.FuseOptions{}, } err = m.NewSession(true) From a7c82b3cd26b62f0ffb8013a48ab4fd3b075944e Mon Sep 17 00:00:00 2001 From: zhijian Date: Mon, 18 Mar 2024 14:39:20 +0800 Subject: [PATCH 11/14] Revert "adjust" This reverts commit 0e7abbbdcd37411bfa6cfd61bc0e59b3e0e73c8c. --- pkg/vfs/vfs.go | 28 +++------------------------- 1 file changed, 3 insertions(+), 25 deletions(-) diff --git a/pkg/vfs/vfs.go b/pkg/vfs/vfs.go index c98df20d1099..7ec928fa5613 100644 --- a/pkg/vfs/vfs.go +++ b/pkg/vfs/vfs.go @@ -694,35 +694,13 @@ func (v *VFS) Read(ctx Context, ino Ino, buf []byte, off uint64, fh uint64) (n i readSizeHistogram.Observe(float64(n)) logit(ctx, "read (%d,%d,%d): %s (%d)", ino, size, off, strerr(err), n) }() - - var isRecovered bool - var findHandle = func(inode Ino, fh uint64) *handle { - v.hanleM.Lock() - defer v.hanleM.Unlock() - for _, f := range v.handles[inode] { - if f.fh == fh { - return f - } - } - if fh&1 == 1 && inode != controlInode { - isRecovered = true - f := &handle{inode: inode, fh: fh} - f.cond = utils.NewCond(f) - v.handles[inode] = append(v.handles[inode], f) - if v.handleIno[fh] == 0 { - v.handleIno[fh] = inode - } - return f - } - return nil - } - - h := findHandle(ino, fh) + h := v.findHandle(ino, fh) if h == nil { err = syscall.EBADF return } - if isRecovered { + if h.flags == 0 { + // recovered var attr Attr err = v.Meta.Open(ctx, ino, syscall.O_RDONLY, &attr) if err != 0 { From 5508b10c6eca7df3e626ec0a85e5dba8aa42cd24 Mon Sep 17 00:00:00 2001 From: zhijian Date: Mon, 18 Mar 2024 14:47:29 +0800 Subject: [PATCH 12/14] use `O_RECOVERED = 1 << 31` flag --- pkg/vfs/handle.go | 4 +++- pkg/vfs/vfs.go | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/pkg/vfs/handle.go b/pkg/vfs/handle.go index f37c47e15f9e..8102a11a9113 100644 --- a/pkg/vfs/handle.go +++ b/pkg/vfs/handle.go @@ -191,6 +191,8 @@ func (v *VFS) findAllHandles(inode Ino) []*handle { return hs2 } +const O_RECOVERED = 1 << 31 // is recovered fd + func (v *VFS) findHandle(inode Ino, fh uint64) *handle { v.hanleM.Lock() defer v.hanleM.Unlock() @@ -200,7 +202,7 @@ func (v *VFS) findHandle(inode Ino, fh uint64) *handle { } } if fh&1 == 1 && inode != controlInode { - f := &handle{inode: inode, fh: fh} + f := &handle{inode: inode, fh: fh, flags: O_RECOVERED} f.cond = utils.NewCond(f) v.handles[inode] = append(v.handles[inode], f) if v.handleIno[fh] == 0 { diff --git a/pkg/vfs/vfs.go b/pkg/vfs/vfs.go index 7ec928fa5613..81e2465380a2 100644 --- a/pkg/vfs/vfs.go +++ b/pkg/vfs/vfs.go @@ -699,7 +699,7 @@ func (v *VFS) Read(ctx Context, ino Ino, buf []byte, off uint64, fh uint64) (n i err = syscall.EBADF return } - if h.flags == 0 { + if h.flags&O_RECOVERED != 0 { // recovered var attr Attr err = v.Meta.Open(ctx, ino, syscall.O_RDONLY, &attr) From eee389d88211d631ed24130064650714481c8df9 Mon Sep 17 00:00:00 2001 From: zhijian Date: Tue, 19 Mar 2024 11:01:20 +0800 Subject: [PATCH 13/14] fix flags --- pkg/vfs/handle.go | 6 +----- pkg/vfs/vfs.go | 4 ++++ 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/pkg/vfs/handle.go b/pkg/vfs/handle.go index 8102a11a9113..ef96c88130e1 100644 --- a/pkg/vfs/handle.go +++ b/pkg/vfs/handle.go @@ -232,12 +232,8 @@ func (v *VFS) releaseHandle(inode Ino, fh uint64) { } } -func hasReadPerm(flag uint32) bool { - return (flag & O_ACCMODE) != syscall.O_WRONLY -} - func (v *VFS) newFileHandle(inode Ino, length uint64, flags uint32) uint64 { - h := v.newHandle(inode, hasReadPerm(flags)) + h := v.newHandle(inode, (flags&O_ACCMODE) == syscall.O_RDONLY) h.Lock() defer h.Unlock() h.flags = flags diff --git a/pkg/vfs/vfs.go b/pkg/vfs/vfs.go index 81e2465380a2..23f5e37f0f6c 100644 --- a/pkg/vfs/vfs.go +++ b/pkg/vfs/vfs.go @@ -644,6 +644,10 @@ func (v *VFS) Release(ctx Context, ino Ino, fh uint64) { } } +func hasReadPerm(flag uint32) bool { + return (flag & O_ACCMODE) != syscall.O_WRONLY +} + func (v *VFS) Read(ctx Context, ino Ino, buf []byte, off uint64, fh uint64) (n int, err syscall.Errno) { size := uint32(len(buf)) if IsSpecialNode(ino) { From af58e471d75c22ce019047450cd2de41af9abbcd Mon Sep 17 00:00:00 2001 From: zhijian Date: Wed, 20 Mar 2024 10:42:06 +0800 Subject: [PATCH 14/14] fix --- pkg/vfs/vfs.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/vfs/vfs.go b/pkg/vfs/vfs.go index 23f5e37f0f6c..8c79bb76e175 100644 --- a/pkg/vfs/vfs.go +++ b/pkg/vfs/vfs.go @@ -730,7 +730,7 @@ func (v *VFS) Read(ctx Context, ino Ino, buf []byte, off uint64, fh uint64) (n i // there could be read operation for write-only if kernel writeback is enabled if !v.Conf.FuseOpts.EnableWriteback && !hasReadPerm(h.flags) { - err = syscall.EACCES + err = syscall.EBADF return } if !h.Rlock(ctx) {