From 72418a33f776549b47c684883240f4e943f2c4b9 Mon Sep 17 00:00:00 2001 From: Kohei Tokunaga Date: Wed, 29 May 2024 17:43:29 +0900 Subject: [PATCH] store: use Forgotten() for lifetime management with leveraging Initialized() Signed-off-by: Kohei Tokunaga --- cmd/go.mod | 2 + cmd/go.sum | 6 +-- go.mod | 2 + go.sum | 6 +-- store/fs.go | 108 ++++++++++++++++++++++------------------------------ 5 files changed, 54 insertions(+), 70 deletions(-) diff --git a/cmd/go.mod b/cmd/go.mod index cb3305743..8be8aab33 100644 --- a/cmd/go.mod +++ b/cmd/go.mod @@ -144,3 +144,5 @@ replace ( github.com/containerd/stargz-snapshotter/estargz => ../estargz github.com/containerd/stargz-snapshotter/ipfs => ../ipfs ) + +replace github.com/hanwen/go-fuse/v2 => github.com/ktock/go-fuse/v2 v2.1.1-0.20240611080711-f59a74eb644a diff --git a/cmd/go.sum b/cmd/go.sum index f994464f2..b3fd14ab5 100644 --- a/cmd/go.sum +++ b/cmd/go.sum @@ -153,8 +153,6 @@ github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1/go.mod h1:kpwsk12EmLe github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/hanwen/go-fuse/v2 v2.5.1 h1:OQBE8zVemSocRxA4OaFJbjJ5hlpCmIWbGr7r0M4uoQQ= -github.com/hanwen/go-fuse/v2 v2.5.1/go.mod h1:xKwi1cF7nXAOBCXujD5ie0ZKsxc8GGSA1rlMJc+8IJs= github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= github.com/hashicorp/errwrap v1.1.0 h1:OxrOeh75EUXMY8TBjag2fzXGZ40LB6IKw45YeGUDY2I= github.com/hashicorp/errwrap v1.1.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= @@ -197,6 +195,8 @@ github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/ktock/go-fuse/v2 v2.1.1-0.20240611080711-f59a74eb644a h1:Z2GoPZxNRB5/yOKsUWoBcZgKTwoPgOxg23cndMCt5sU= +github.com/ktock/go-fuse/v2 v2.1.1-0.20240611080711-f59a74eb644a/go.mod h1:ugNaD/iv5JYyS1Rcvi57Wz7/vrLQJo10mmketmoef48= github.com/kylelemons/godebug v0.0.0-20170820004349-d65d576e9348 h1:MtvEpTB6LX3vkb4ax0b5D2DHbNAUsen0Gx5wZoq3lV4= github.com/kylelemons/godebug v0.0.0-20170820004349-d65d576e9348/go.mod h1:B69LEHPfb2qLo0BaaOLcbitczOKLWTsrBG9LczfCD4k= github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0= @@ -210,7 +210,6 @@ github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= github.com/moby/locker v1.0.1 h1:fOXqR41zeveg4fFODix+1Ch4mj/gT0NE1XJbp/epuBg= github.com/moby/locker v1.0.1/go.mod h1:S7SDdo5zpBK84bzzVlKr2V0hz+7x9hWbYC/kq7oQppc= -github.com/moby/sys/mountinfo v0.6.2/go.mod h1:IJb6JQeOklcdMU9F5xQ8ZALD+CUr5VlGpwtX+VE0rpI= github.com/moby/sys/mountinfo v0.7.1 h1:/tTvQaSJRr2FshkhXiIpux6fQ2Zvc4j7tAhMTStAG2g= github.com/moby/sys/mountinfo v0.7.1/go.mod h1:IJb6JQeOklcdMU9F5xQ8ZALD+CUr5VlGpwtX+VE0rpI= github.com/moby/sys/sequential v0.5.0 h1:OPvI35Lzn9K04PBbCLW0g4LcFAJgHsvXsRyewg5lXtc= @@ -398,7 +397,6 @@ golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.7.0 h1:YsImfSBoP9QPYL0xyKJPq0gcaJdG3rInoqxTWbfQu9M= golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= diff --git a/go.mod b/go.mod index 8b884f85a..cb47fcea0 100644 --- a/go.mod +++ b/go.mod @@ -117,3 +117,5 @@ require ( // Import local package for estargz. replace github.com/containerd/stargz-snapshotter/estargz => ./estargz + +replace github.com/hanwen/go-fuse/v2 => github.com/ktock/go-fuse/v2 v2.1.1-0.20240611080711-f59a74eb644a diff --git a/go.sum b/go.sum index ffba27f8e..5fbc0d953 100644 --- a/go.sum +++ b/go.sum @@ -133,8 +133,6 @@ github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1/go.mod h1:kpwsk12EmLe github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/hanwen/go-fuse/v2 v2.5.1 h1:OQBE8zVemSocRxA4OaFJbjJ5hlpCmIWbGr7r0M4uoQQ= -github.com/hanwen/go-fuse/v2 v2.5.1/go.mod h1:xKwi1cF7nXAOBCXujD5ie0ZKsxc8GGSA1rlMJc+8IJs= github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= github.com/hashicorp/errwrap v1.1.0 h1:OxrOeh75EUXMY8TBjag2fzXGZ40LB6IKw45YeGUDY2I= github.com/hashicorp/errwrap v1.1.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= @@ -170,6 +168,8 @@ github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/ktock/go-fuse/v2 v2.1.1-0.20240611080711-f59a74eb644a h1:Z2GoPZxNRB5/yOKsUWoBcZgKTwoPgOxg23cndMCt5sU= +github.com/ktock/go-fuse/v2 v2.1.1-0.20240611080711-f59a74eb644a/go.mod h1:ugNaD/iv5JYyS1Rcvi57Wz7/vrLQJo10mmketmoef48= github.com/kylelemons/godebug v0.0.0-20170820004349-d65d576e9348 h1:MtvEpTB6LX3vkb4ax0b5D2DHbNAUsen0Gx5wZoq3lV4= github.com/kylelemons/godebug v0.0.0-20170820004349-d65d576e9348/go.mod h1:B69LEHPfb2qLo0BaaOLcbitczOKLWTsrBG9LczfCD4k= github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0= @@ -177,7 +177,6 @@ github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJ github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= github.com/moby/locker v1.0.1 h1:fOXqR41zeveg4fFODix+1Ch4mj/gT0NE1XJbp/epuBg= github.com/moby/locker v1.0.1/go.mod h1:S7SDdo5zpBK84bzzVlKr2V0hz+7x9hWbYC/kq7oQppc= -github.com/moby/sys/mountinfo v0.6.2/go.mod h1:IJb6JQeOklcdMU9F5xQ8ZALD+CUr5VlGpwtX+VE0rpI= github.com/moby/sys/mountinfo v0.7.1 h1:/tTvQaSJRr2FshkhXiIpux6fQ2Zvc4j7tAhMTStAG2g= github.com/moby/sys/mountinfo v0.7.1/go.mod h1:IJb6JQeOklcdMU9F5xQ8ZALD+CUr5VlGpwtX+VE0rpI= github.com/moby/sys/sequential v0.5.0 h1:OPvI35Lzn9K04PBbCLW0g4LcFAJgHsvXsRyewg5lXtc= @@ -329,7 +328,6 @@ golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.7.0 h1:YsImfSBoP9QPYL0xyKJPq0gcaJdG3rInoqxTWbfQu9M= golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= diff --git a/store/fs.go b/store/fs.go index 171e48378..2b91906c0 100644 --- a/store/fs.go +++ b/store/fs.go @@ -100,32 +100,18 @@ type fs struct { // inodes numbers of noeds inside each `diff` directory are prefixed by an unique uint32 // so that they don't conflict with nodes outside `diff` directories. layerMap *idMap - - knownNode map[string]map[string]*layerReleasable - knownNodeMu sync.Mutex } type layerReleasable struct { - n fusefs.InodeEmbedder - released bool - mu sync.Mutex + n fusefs.InodeEmbedder } func (lh *layerReleasable) releasable() bool { - lh.mu.Lock() - released := lh.released - lh.mu.Unlock() - return released && isForgotten(lh.n.EmbeddedInode()) -} - -func (lh *layerReleasable) release() { - lh.mu.Lock() - lh.released = true - lh.mu.Unlock() + return isForgotten(lh.n.EmbeddedInode()) } func isForgotten(n *fusefs.Inode) bool { - if !n.Forgotten() { + if !n.Initialized() || !n.Forgotten() { return false } for _, cn := range n.Children() { @@ -141,7 +127,7 @@ type inoReleasable struct { } func (r *inoReleasable) releasable() bool { - return r.n.EmbeddedInode().Forgotten() + return r.n.EmbeddedInode().Initialized() && r.n.EmbeddedInode().Forgotten() } func (fs *fs) newInodeWithID(ctx context.Context, p func(uint32) fusefs.InodeEmbedder) (*fusefs.Inode, syscall.Errno) { @@ -192,7 +178,7 @@ func (n *rootnode) Lookup(ctx context.Context, name string, out *fuse.EntryOut) out.Attr.Ino = uint64(ino) cn.Attr.Ino = uint64(ino) sAttr.Ino = uint64(ino) - return n.NewInode(ctx, cn, sAttr) + return n.NewPersistentInode(ctx, cn, sAttr) }) } refBytes, err := base64.StdEncoding.DecodeString(name) @@ -216,7 +202,7 @@ func (n *rootnode) Lookup(ctx context.Context, name string, out *fuse.EntryOut) out.Attr.Ino = uint64(ino) cn.attr.Ino = uint64(ino) sAttr.Ino = uint64(ino) - return n.NewInode(ctx, cn, sAttr) + return n.NewPersistentInode(ctx, cn, sAttr) }) } @@ -263,7 +249,7 @@ func (n *refnode) Lookup(ctx context.Context, name string, out *fuse.EntryOut) ( out.Attr.Ino = uint64(ino) cn.attr.Ino = uint64(ino) sAttr.Ino = uint64(ino) - return n.NewInode(ctx, cn, sAttr) + return n.NewPersistentInode(ctx, cn, sAttr) }) } @@ -284,19 +270,13 @@ func (n *refnode) Rmdir(ctx context.Context, name string) syscall.Errno { return syscall.EIO } if current == 0 { - n.fs.knownNodeMu.Lock() - lh, ok := n.fs.knownNode[n.ref.String()][targetDigest.String()] - if !ok { - n.fs.knownNodeMu.Unlock() - log.G(ctx).WithError(err).Warnf("node of layer %v/%v is not registered", n.ref, targetDigest) - return syscall.EIO + if cn := n.GetChild(name); cn != nil { + cn.RmAllChildren() + n.RmChild(name) } - lh.release() - delete(n.fs.knownNode[n.ref.String()], targetDigest.String()) - if len(n.fs.knownNode[n.ref.String()]) == 0 { - delete(n.fs.knownNode, n.ref.String()) + if len(n.Children()) == 0 { + n.EmbeddedInode().RmAllChildren() } - n.fs.knownNodeMu.Unlock() } log.G(ctx).WithField("refcounter", current).Infof("layer %v/%v is marked as RELEASE", n.ref, targetDigest) return syscall.ENOENT @@ -330,6 +310,37 @@ var _ = (fusefs.NodeLookuper)((*layernode)(nil)) // Lookup routes to the target file stored in the pool, based on the specified file name. func (n *layernode) Lookup(ctx context.Context, name string, out *fuse.EntryOut) (*fusefs.Inode, syscall.Errno) { + if cn := n.GetChild(name); cn != nil { + found := false + switch tn := cn.Operations().(type) { + case *fusefs.MemSymlink: + copyAttr(&out.Attr, &tn.Attr) + found = true + case *fusefs.MemRegularFile: + copyAttr(&out.Attr, &tn.Attr) + found = true + case *blobnode: + copyAttr(&out.Attr, &tn.attr) + found = true + default: + log.G(ctx).Debug("layernode.Lookup: uknown node type detected; trying NodeGetattrer provided by layer root node") + if na, ok := cn.Operations().(fusefs.NodeGetattrer); ok { + var ao fuse.AttrOut + errno := na.Getattr(ctx, nil, &ao) + if errno != 0 { + return nil, errno + } + copyAttr(&out.Attr, &ao.Attr) + found = true + } + } + if !found { + log.G(ctx).Warn("layernode.Lookup: uknown node type detected") + return nil, syscall.EIO + } + out.Attr.Ino = cn.StableAttr().Ino + return cn, 0 + } switch name { case layerInfoLink: info, err := n.fs.layerManager.getLayerInfo(ctx, n.refnode.ref, n.digest) @@ -350,28 +361,10 @@ func (n *layernode) Lookup(ctx context.Context, name string, out *fuse.EntryOut) out.Attr.Ino = uint64(ino) cn.Attr.Ino = uint64(ino) sAttr.Ino = uint64(ino) - return n.NewInode(ctx, cn, sAttr) + return n.NewPersistentInode(ctx, cn, sAttr) }) case layerLink, blobLink: - // Check if layer is already known - if name == layerLink { - n.fs.knownNodeMu.Lock() - if lh, ok := n.fs.knownNode[n.refnode.ref.String()][n.digest.String()]; ok { - var ao fuse.AttrOut - if errno := lh.n.(fusefs.NodeGetattrer).Getattr(ctx, nil, &ao); errno != 0 { - return nil, errno - } - copyAttr(&out.Attr, &ao.Attr) - n.fs.knownNodeMu.Unlock() - return n.NewInode(ctx, lh.n, fusefs.StableAttr{ - Mode: out.Attr.Mode, - Ino: out.Attr.Ino, - }), 0 - } - n.fs.knownNodeMu.Unlock() - } - // Resolve layer l, err := n.fs.layerManager.getLayer(ctx, n.refnode.ref, n.digest) if err != nil { @@ -398,7 +391,7 @@ func (n *layernode) Lookup(ctx context.Context, name string, out *fuse.EntryOut) out.Attr.Ino = uint64(ino) cn.attr.Ino = uint64(ino) sAttr.Ino = uint64(ino) - return n.NewInode(ctx, cn, sAttr) + return n.NewPersistentInode(ctx, cn, sAttr) }) } @@ -417,21 +410,12 @@ func (n *layernode) Lookup(ctx context.Context, name string, out *fuse.EntryOut) } copyAttr(&out.Attr, &ao.Attr) - cn = n.NewInode(ctx, root, fusefs.StableAttr{ + cn = n.NewPersistentInode(ctx, root, fusefs.StableAttr{ Mode: out.Attr.Mode, Ino: out.Attr.Ino, }) rr := &layerReleasable{n: root} - n.fs.knownNodeMu.Lock() - if n.fs.knownNode == nil { - n.fs.knownNode = make(map[string]map[string]*layerReleasable) - } - if n.fs.knownNode[n.refnode.ref.String()] == nil { - n.fs.knownNode[n.refnode.ref.String()] = make(map[string]*layerReleasable) - } - n.fs.knownNode[n.refnode.ref.String()][n.digest.String()] = rr - n.fs.knownNodeMu.Unlock() return rr, nil }) if err != nil || errno != 0 {