Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

store: use Forgotten() for lifetime management with leveraging Initialized() for readiness check #1705

Closed
wants to merge 1 commit into from
Closed

store: use Forgotten() for lifetime management with leveraging Initialized() for readiness check #1705

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions cmd/go.mod
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -144,3 +144,5 @@ replace (
github.com/containerd/stargz-snapshotter/estargz => ../estargz
github.com/containerd/stargz-snapshotter/ipfs => ../ipfs
)

replace github.com/hanwen/go-fuse/v2 => github.com/ktock/go-fuse/v2 v2.1.1-0.20240611080711-f59a74eb644a
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Do you have a PR

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yes, there is a PR for this: hanwen/go-fuse#520

6 changes: 2 additions & 4 deletions cmd/go.sum
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -153,8 +153,6 @@ github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1/go.mod h1:kpwsk12EmLe
github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
github.com/hanwen/go-fuse/v2 v2.5.1 h1:OQBE8zVemSocRxA4OaFJbjJ5hlpCmIWbGr7r0M4uoQQ=
github.com/hanwen/go-fuse/v2 v2.5.1/go.mod h1:xKwi1cF7nXAOBCXujD5ie0ZKsxc8GGSA1rlMJc+8IJs=
github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4=
github.com/hashicorp/errwrap v1.1.0 h1:OxrOeh75EUXMY8TBjag2fzXGZ40LB6IKw45YeGUDY2I=
github.com/hashicorp/errwrap v1.1.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4=
Expand Down Expand Up @@ -197,6 +195,8 @@ github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
github.com/ktock/go-fuse/v2 v2.1.1-0.20240611080711-f59a74eb644a h1:Z2GoPZxNRB5/yOKsUWoBcZgKTwoPgOxg23cndMCt5sU=
github.com/ktock/go-fuse/v2 v2.1.1-0.20240611080711-f59a74eb644a/go.mod h1:ugNaD/iv5JYyS1Rcvi57Wz7/vrLQJo10mmketmoef48=
github.com/kylelemons/godebug v0.0.0-20170820004349-d65d576e9348 h1:MtvEpTB6LX3vkb4ax0b5D2DHbNAUsen0Gx5wZoq3lV4=
github.com/kylelemons/godebug v0.0.0-20170820004349-d65d576e9348/go.mod h1:B69LEHPfb2qLo0BaaOLcbitczOKLWTsrBG9LczfCD4k=
github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0=
Expand All @@ -210,7 +210,6 @@ github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG
github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
github.com/moby/locker v1.0.1 h1:fOXqR41zeveg4fFODix+1Ch4mj/gT0NE1XJbp/epuBg=
github.com/moby/locker v1.0.1/go.mod h1:S7SDdo5zpBK84bzzVlKr2V0hz+7x9hWbYC/kq7oQppc=
github.com/moby/sys/mountinfo v0.6.2/go.mod h1:IJb6JQeOklcdMU9F5xQ8ZALD+CUr5VlGpwtX+VE0rpI=
github.com/moby/sys/mountinfo v0.7.1 h1:/tTvQaSJRr2FshkhXiIpux6fQ2Zvc4j7tAhMTStAG2g=
github.com/moby/sys/mountinfo v0.7.1/go.mod h1:IJb6JQeOklcdMU9F5xQ8ZALD+CUr5VlGpwtX+VE0rpI=
github.com/moby/sys/sequential v0.5.0 h1:OPvI35Lzn9K04PBbCLW0g4LcFAJgHsvXsRyewg5lXtc=
Expand Down Expand Up @@ -398,7 +397,6 @@ golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJ
golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.7.0 h1:YsImfSBoP9QPYL0xyKJPq0gcaJdG3rInoqxTWbfQu9M=
golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
Expand Down
2 changes: 2 additions & 0 deletions go.mod
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -117,3 +117,5 @@ require (

// Import local package for estargz.
replace github.com/containerd/stargz-snapshotter/estargz => ./estargz

replace github.com/hanwen/go-fuse/v2 => github.com/ktock/go-fuse/v2 v2.1.1-0.20240611080711-f59a74eb644a
6 changes: 2 additions & 4 deletions go.sum
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -133,8 +133,6 @@ github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1/go.mod h1:kpwsk12EmLe
github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
github.com/hanwen/go-fuse/v2 v2.5.1 h1:OQBE8zVemSocRxA4OaFJbjJ5hlpCmIWbGr7r0M4uoQQ=
github.com/hanwen/go-fuse/v2 v2.5.1/go.mod h1:xKwi1cF7nXAOBCXujD5ie0ZKsxc8GGSA1rlMJc+8IJs=
github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4=
github.com/hashicorp/errwrap v1.1.0 h1:OxrOeh75EUXMY8TBjag2fzXGZ40LB6IKw45YeGUDY2I=
github.com/hashicorp/errwrap v1.1.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4=
Expand Down Expand Up @@ -170,14 +168,15 @@ github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
github.com/ktock/go-fuse/v2 v2.1.1-0.20240611080711-f59a74eb644a h1:Z2GoPZxNRB5/yOKsUWoBcZgKTwoPgOxg23cndMCt5sU=
github.com/ktock/go-fuse/v2 v2.1.1-0.20240611080711-f59a74eb644a/go.mod h1:ugNaD/iv5JYyS1Rcvi57Wz7/vrLQJo10mmketmoef48=
github.com/kylelemons/godebug v0.0.0-20170820004349-d65d576e9348 h1:MtvEpTB6LX3vkb4ax0b5D2DHbNAUsen0Gx5wZoq3lV4=
github.com/kylelemons/godebug v0.0.0-20170820004349-d65d576e9348/go.mod h1:B69LEHPfb2qLo0BaaOLcbitczOKLWTsrBG9LczfCD4k=
github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0=
github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc=
github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=
github.com/moby/locker v1.0.1 h1:fOXqR41zeveg4fFODix+1Ch4mj/gT0NE1XJbp/epuBg=
github.com/moby/locker v1.0.1/go.mod h1:S7SDdo5zpBK84bzzVlKr2V0hz+7x9hWbYC/kq7oQppc=
github.com/moby/sys/mountinfo v0.6.2/go.mod h1:IJb6JQeOklcdMU9F5xQ8ZALD+CUr5VlGpwtX+VE0rpI=
github.com/moby/sys/mountinfo v0.7.1 h1:/tTvQaSJRr2FshkhXiIpux6fQ2Zvc4j7tAhMTStAG2g=
github.com/moby/sys/mountinfo v0.7.1/go.mod h1:IJb6JQeOklcdMU9F5xQ8ZALD+CUr5VlGpwtX+VE0rpI=
github.com/moby/sys/sequential v0.5.0 h1:OPvI35Lzn9K04PBbCLW0g4LcFAJgHsvXsRyewg5lXtc=
Expand Down Expand Up @@ -329,7 +328,6 @@ golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJ
golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.7.0 h1:YsImfSBoP9QPYL0xyKJPq0gcaJdG3rInoqxTWbfQu9M=
golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
Expand Down
108 changes: 46 additions & 62 deletions store/fs.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -100,32 +100,18 @@ type fs struct {
// inodes numbers of noeds inside each `diff` directory are prefixed by an unique uint32
// so that they don't conflict with nodes outside `diff` directories.
layerMap *idMap

knownNode map[string]map[string]*layerReleasable
knownNodeMu sync.Mutex
}

type layerReleasable struct {
n fusefs.InodeEmbedder
released bool
mu sync.Mutex
n fusefs.InodeEmbedder
}

func (lh *layerReleasable) releasable() bool {
lh.mu.Lock()
released := lh.released
lh.mu.Unlock()
return released && isForgotten(lh.n.EmbeddedInode())
}

func (lh *layerReleasable) release() {
lh.mu.Lock()
lh.released = true
lh.mu.Unlock()
return isForgotten(lh.n.EmbeddedInode())
}

func isForgotten(n *fusefs.Inode) bool {
if !n.Forgotten() {
if !n.Initialized() || !n.Forgotten() {
return false
}
for _, cn := range n.Children() {
Expand All @@ -141,7 +127,7 @@ type inoReleasable struct {
}

func (r *inoReleasable) releasable() bool {
return r.n.EmbeddedInode().Forgotten()
return r.n.EmbeddedInode().Initialized() && r.n.EmbeddedInode().Forgotten()
}

func (fs *fs) newInodeWithID(ctx context.Context, p func(uint32) fusefs.InodeEmbedder) (*fusefs.Inode, syscall.Errno) {
Expand Down Expand Up @@ -192,7 +178,7 @@ func (n *rootnode) Lookup(ctx context.Context, name string, out *fuse.EntryOut)
out.Attr.Ino = uint64(ino)
cn.Attr.Ino = uint64(ino)
sAttr.Ino = uint64(ino)
return n.NewInode(ctx, cn, sAttr)
return n.NewPersistentInode(ctx, cn, sAttr)
})
}
refBytes, err := base64.StdEncoding.DecodeString(name)
Expand All @@ -216,7 +202,7 @@ func (n *rootnode) Lookup(ctx context.Context, name string, out *fuse.EntryOut)
out.Attr.Ino = uint64(ino)
cn.attr.Ino = uint64(ino)
sAttr.Ino = uint64(ino)
return n.NewInode(ctx, cn, sAttr)
return n.NewPersistentInode(ctx, cn, sAttr)
})
}

Expand Down Expand Up @@ -263,7 +249,7 @@ func (n *refnode) Lookup(ctx context.Context, name string, out *fuse.EntryOut) (
out.Attr.Ino = uint64(ino)
cn.attr.Ino = uint64(ino)
sAttr.Ino = uint64(ino)
return n.NewInode(ctx, cn, sAttr)
return n.NewPersistentInode(ctx, cn, sAttr)
})
}

Expand All @@ -284,19 +270,13 @@ func (n *refnode) Rmdir(ctx context.Context, name string) syscall.Errno {
return syscall.EIO
}
if current == 0 {
n.fs.knownNodeMu.Lock()
lh, ok := n.fs.knownNode[n.ref.String()][targetDigest.String()]
if !ok {
n.fs.knownNodeMu.Unlock()
log.G(ctx).WithError(err).Warnf("node of layer %v/%v is not registered", n.ref, targetDigest)
return syscall.EIO
if cn := n.GetChild(name); cn != nil {
cn.RmAllChildren()
n.RmChild(name)
}
lh.release()
delete(n.fs.knownNode[n.ref.String()], targetDigest.String())
if len(n.fs.knownNode[n.ref.String()]) == 0 {
delete(n.fs.knownNode, n.ref.String())
if len(n.Children()) == 0 {
n.EmbeddedInode().RmAllChildren()
}
n.fs.knownNodeMu.Unlock()
}
log.G(ctx).WithField("refcounter", current).Infof("layer %v/%v is marked as RELEASE", n.ref, targetDigest)
return syscall.ENOENT
Expand Down Expand Up @@ -330,6 +310,37 @@ var _ = (fusefs.NodeLookuper)((*layernode)(nil))

// Lookup routes to the target file stored in the pool, based on the specified file name.
func (n *layernode) Lookup(ctx context.Context, name string, out *fuse.EntryOut) (*fusefs.Inode, syscall.Errno) {
if cn := n.GetChild(name); cn != nil {
found := false
switch tn := cn.Operations().(type) {
case *fusefs.MemSymlink:
copyAttr(&out.Attr, &tn.Attr)
found = true
case *fusefs.MemRegularFile:
copyAttr(&out.Attr, &tn.Attr)
found = true
case *blobnode:
copyAttr(&out.Attr, &tn.attr)
found = true
default:
log.G(ctx).Debug("layernode.Lookup: uknown node type detected; trying NodeGetattrer provided by layer root node")
if na, ok := cn.Operations().(fusefs.NodeGetattrer); ok {
var ao fuse.AttrOut
errno := na.Getattr(ctx, nil, &ao)
if errno != 0 {
return nil, errno
}
copyAttr(&out.Attr, &ao.Attr)
found = true
}
}
if !found {
log.G(ctx).Warn("layernode.Lookup: uknown node type detected")
return nil, syscall.EIO
}
out.Attr.Ino = cn.StableAttr().Ino
return cn, 0
}
switch name {
case layerInfoLink:
info, err := n.fs.layerManager.getLayerInfo(ctx, n.refnode.ref, n.digest)
Expand All @@ -350,28 +361,10 @@ func (n *layernode) Lookup(ctx context.Context, name string, out *fuse.EntryOut)
out.Attr.Ino = uint64(ino)
cn.Attr.Ino = uint64(ino)
sAttr.Ino = uint64(ino)
return n.NewInode(ctx, cn, sAttr)
return n.NewPersistentInode(ctx, cn, sAttr)
})
case layerLink, blobLink:

// Check if layer is already known
if name == layerLink {
n.fs.knownNodeMu.Lock()
if lh, ok := n.fs.knownNode[n.refnode.ref.String()][n.digest.String()]; ok {
var ao fuse.AttrOut
if errno := lh.n.(fusefs.NodeGetattrer).Getattr(ctx, nil, &ao); errno != 0 {
return nil, errno
}
copyAttr(&out.Attr, &ao.Attr)
n.fs.knownNodeMu.Unlock()
return n.NewInode(ctx, lh.n, fusefs.StableAttr{
Mode: out.Attr.Mode,
Ino: out.Attr.Ino,
}), 0
}
n.fs.knownNodeMu.Unlock()
}

// Resolve layer
l, err := n.fs.layerManager.getLayer(ctx, n.refnode.ref, n.digest)
if err != nil {
Expand All @@ -398,7 +391,7 @@ func (n *layernode) Lookup(ctx context.Context, name string, out *fuse.EntryOut)
out.Attr.Ino = uint64(ino)
cn.attr.Ino = uint64(ino)
sAttr.Ino = uint64(ino)
return n.NewInode(ctx, cn, sAttr)
return n.NewPersistentInode(ctx, cn, sAttr)
})
}

Expand All @@ -417,21 +410,12 @@ func (n *layernode) Lookup(ctx context.Context, name string, out *fuse.EntryOut)
}

copyAttr(&out.Attr, &ao.Attr)
cn = n.NewInode(ctx, root, fusefs.StableAttr{
cn = n.NewPersistentInode(ctx, root, fusefs.StableAttr{
Mode: out.Attr.Mode,
Ino: out.Attr.Ino,
})

rr := &layerReleasable{n: root}
n.fs.knownNodeMu.Lock()
if n.fs.knownNode == nil {
n.fs.knownNode = make(map[string]map[string]*layerReleasable)
}
if n.fs.knownNode[n.refnode.ref.String()] == nil {
n.fs.knownNode[n.refnode.ref.String()] = make(map[string]*layerReleasable)
}
n.fs.knownNode[n.refnode.ref.String()][n.digest.String()] = rr
n.fs.knownNodeMu.Unlock()
return rr, nil
})
if err != nil || errno != 0 {
Expand Down
Loading