Skip to content

Commit

Permalink
store: use Forgotten() for lifetime management with leveraging Initia…
Browse files Browse the repository at this point in the history 
…lized()

Signed-off-by: Kohei Tokunaga <[email protected]>
  • Loading branch information
@ktock
ktock committed Jun 11, 2024
1 parent 4c63858 commit 62e9b99
Show file tree
Hide file tree
Showing 5 changed files with 48 additions and 64 deletions.
2 changes: 2 additions & 0 deletions cmd/go.mod
View file
Original file line number Diff line number Diff line change
Expand Up @@ -144,3 +144,5 @@ replace (
github.com/containerd/stargz-snapshotter/estargz => ../estargz
github.com/containerd/stargz-snapshotter/ipfs => ../ipfs
)

replace github.com/hanwen/go-fuse/v2 => github.com/ktock/go-fuse/v2 v2.1.1-0.20240611080711-f59a74eb644a
6 changes: 2 additions & 4 deletions cmd/go.sum
View file
Original file line number Diff line number Diff line change
Expand Up @@ -153,8 +153,6 @@ github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1/go.mod h1:kpwsk12EmLe
github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
github.com/hanwen/go-fuse/v2 v2.5.1 h1:OQBE8zVemSocRxA4OaFJbjJ5hlpCmIWbGr7r0M4uoQQ=
github.com/hanwen/go-fuse/v2 v2.5.1/go.mod h1:xKwi1cF7nXAOBCXujD5ie0ZKsxc8GGSA1rlMJc+8IJs=
github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4=
github.com/hashicorp/errwrap v1.1.0 h1:OxrOeh75EUXMY8TBjag2fzXGZ40LB6IKw45YeGUDY2I=
github.com/hashicorp/errwrap v1.1.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4=
Expand Down Expand Up @@ -197,6 +195,8 @@ github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
github.com/ktock/go-fuse/v2 v2.1.1-0.20240611080711-f59a74eb644a h1:Z2GoPZxNRB5/yOKsUWoBcZgKTwoPgOxg23cndMCt5sU=
github.com/ktock/go-fuse/v2 v2.1.1-0.20240611080711-f59a74eb644a/go.mod h1:ugNaD/iv5JYyS1Rcvi57Wz7/vrLQJo10mmketmoef48=
github.com/kylelemons/godebug v0.0.0-20170820004349-d65d576e9348 h1:MtvEpTB6LX3vkb4ax0b5D2DHbNAUsen0Gx5wZoq3lV4=
github.com/kylelemons/godebug v0.0.0-20170820004349-d65d576e9348/go.mod h1:B69LEHPfb2qLo0BaaOLcbitczOKLWTsrBG9LczfCD4k=
github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0=
Expand All @@ -210,7 +210,6 @@ github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG
github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
github.com/moby/locker v1.0.1 h1:fOXqR41zeveg4fFODix+1Ch4mj/gT0NE1XJbp/epuBg=
github.com/moby/locker v1.0.1/go.mod h1:S7SDdo5zpBK84bzzVlKr2V0hz+7x9hWbYC/kq7oQppc=
github.com/moby/sys/mountinfo v0.6.2/go.mod h1:IJb6JQeOklcdMU9F5xQ8ZALD+CUr5VlGpwtX+VE0rpI=
github.com/moby/sys/mountinfo v0.7.1 h1:/tTvQaSJRr2FshkhXiIpux6fQ2Zvc4j7tAhMTStAG2g=
github.com/moby/sys/mountinfo v0.7.1/go.mod h1:IJb6JQeOklcdMU9F5xQ8ZALD+CUr5VlGpwtX+VE0rpI=
github.com/moby/sys/sequential v0.5.0 h1:OPvI35Lzn9K04PBbCLW0g4LcFAJgHsvXsRyewg5lXtc=
Expand Down Expand Up @@ -398,7 +397,6 @@ golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJ
golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.7.0 h1:YsImfSBoP9QPYL0xyKJPq0gcaJdG3rInoqxTWbfQu9M=
golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
Expand Down
2 changes: 2 additions & 0 deletions go.mod
View file
Original file line number Diff line number Diff line change
Expand Up @@ -117,3 +117,5 @@ require (

// Import local package for estargz.
replace github.com/containerd/stargz-snapshotter/estargz => ./estargz

replace github.com/hanwen/go-fuse/v2 => github.com/ktock/go-fuse/v2 v2.1.1-0.20240611080711-f59a74eb644a
6 changes: 2 additions & 4 deletions go.sum
View file
Original file line number Diff line number Diff line change
Expand Up @@ -133,8 +133,6 @@ github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1/go.mod h1:kpwsk12EmLe
github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
github.com/hanwen/go-fuse/v2 v2.5.1 h1:OQBE8zVemSocRxA4OaFJbjJ5hlpCmIWbGr7r0M4uoQQ=
github.com/hanwen/go-fuse/v2 v2.5.1/go.mod h1:xKwi1cF7nXAOBCXujD5ie0ZKsxc8GGSA1rlMJc+8IJs=
github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4=
github.com/hashicorp/errwrap v1.1.0 h1:OxrOeh75EUXMY8TBjag2fzXGZ40LB6IKw45YeGUDY2I=
github.com/hashicorp/errwrap v1.1.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4=
Expand Down Expand Up @@ -170,14 +168,15 @@ github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
github.com/ktock/go-fuse/v2 v2.1.1-0.20240611080711-f59a74eb644a h1:Z2GoPZxNRB5/yOKsUWoBcZgKTwoPgOxg23cndMCt5sU=
github.com/ktock/go-fuse/v2 v2.1.1-0.20240611080711-f59a74eb644a/go.mod h1:ugNaD/iv5JYyS1Rcvi57Wz7/vrLQJo10mmketmoef48=
github.com/kylelemons/godebug v0.0.0-20170820004349-d65d576e9348 h1:MtvEpTB6LX3vkb4ax0b5D2DHbNAUsen0Gx5wZoq3lV4=
github.com/kylelemons/godebug v0.0.0-20170820004349-d65d576e9348/go.mod h1:B69LEHPfb2qLo0BaaOLcbitczOKLWTsrBG9LczfCD4k=
github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0=
github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc=
github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=
github.com/moby/locker v1.0.1 h1:fOXqR41zeveg4fFODix+1Ch4mj/gT0NE1XJbp/epuBg=
github.com/moby/locker v1.0.1/go.mod h1:S7SDdo5zpBK84bzzVlKr2V0hz+7x9hWbYC/kq7oQppc=
github.com/moby/sys/mountinfo v0.6.2/go.mod h1:IJb6JQeOklcdMU9F5xQ8ZALD+CUr5VlGpwtX+VE0rpI=
github.com/moby/sys/mountinfo v0.7.1 h1:/tTvQaSJRr2FshkhXiIpux6fQ2Zvc4j7tAhMTStAG2g=
github.com/moby/sys/mountinfo v0.7.1/go.mod h1:IJb6JQeOklcdMU9F5xQ8ZALD+CUr5VlGpwtX+VE0rpI=
github.com/moby/sys/sequential v0.5.0 h1:OPvI35Lzn9K04PBbCLW0g4LcFAJgHsvXsRyewg5lXtc=
Expand Down Expand Up @@ -329,7 +328,6 @@ golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJ
golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.7.0 h1:YsImfSBoP9QPYL0xyKJPq0gcaJdG3rInoqxTWbfQu9M=
golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
Expand Down
96 changes: 40 additions & 56 deletions store/fs.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -100,32 +100,18 @@ type fs struct {
// inodes numbers of noeds inside each `diff` directory are prefixed by an unique uint32
// so that they don't conflict with nodes outside `diff` directories.
layerMap *idMap

knownNode map[string]map[string]*layerReleasable
knownNodeMu sync.Mutex
}

type layerReleasable struct {
n fusefs.InodeEmbedder
released bool
mu sync.Mutex
n fusefs.InodeEmbedder
}

func (lh *layerReleasable) releasable() bool {
lh.mu.Lock()
released := lh.released
lh.mu.Unlock()
return released && isForgotten(lh.n.EmbeddedInode())
}

func (lh *layerReleasable) release() {
lh.mu.Lock()
lh.released = true
lh.mu.Unlock()
return isForgotten(lh.n.EmbeddedInode())
}

func isForgotten(n *fusefs.Inode) bool {
if !n.Forgotten() {
if !n.Initialized() || !n.Forgotten() {
return false
}
for _, cn := range n.Children() {
Expand All @@ -141,7 +127,7 @@ type inoReleasable struct {
}

func (r *inoReleasable) releasable() bool {
return r.n.EmbeddedInode().Forgotten()
return r.n.EmbeddedInode().Initialized() && r.n.EmbeddedInode().Forgotten()
}

func (fs *fs) newInodeWithID(ctx context.Context, p func(uint32) fusefs.InodeEmbedder) (*fusefs.Inode, syscall.Errno) {
Expand Down Expand Up @@ -284,19 +270,13 @@ func (n *refnode) Rmdir(ctx context.Context, name string) syscall.Errno {
return syscall.EIO
}
if current == 0 {
n.fs.knownNodeMu.Lock()
lh, ok := n.fs.knownNode[n.ref.String()][targetDigest.String()]
if !ok {
n.fs.knownNodeMu.Unlock()
log.G(ctx).WithError(err).Warnf("node of layer %v/%v is not registered", n.ref, targetDigest)
return syscall.EIO
if cn := n.GetChild(name); cn != nil {
cn.RmAllChildren()
n.RmChild(name)
}
lh.release()
delete(n.fs.knownNode[n.ref.String()], targetDigest.String())
if len(n.fs.knownNode[n.ref.String()]) == 0 {
delete(n.fs.knownNode, n.ref.String())
if len(n.Children()) == 0 {
n.EmbeddedInode().RmAllChildren()
}
n.fs.knownNodeMu.Unlock()
}
log.G(ctx).WithField("refcounter", current).Infof("layer %v/%v is marked as RELEASE", n.ref, targetDigest)
return syscall.ENOENT
Expand Down Expand Up @@ -330,6 +310,37 @@ var _ = (fusefs.NodeLookuper)((*layernode)(nil))

// Lookup routes to the target file stored in the pool, based on the specified file name.
func (n *layernode) Lookup(ctx context.Context, name string, out *fuse.EntryOut) (*fusefs.Inode, syscall.Errno) {
if cn := n.GetChild(name); cn != nil {
found := false
switch tn := cn.Operations().(type) {
case *fusefs.MemSymlink:
copyAttr(&out.Attr, &tn.Attr)
found = true
case *fusefs.MemRegularFile:
copyAttr(&out.Attr, &tn.Attr)
found = true
case *blobnode:
copyAttr(&out.Attr, &tn.attr)
found = true
default:
log.G(ctx).Debug("layernode.Lookup: uknown node type detected; trying NodeGetattrer provided by layer root node")
if na, ok := cn.Operations().(fusefs.NodeGetattrer); ok {
var ao fuse.AttrOut
errno := na.Getattr(ctx, nil, &ao)
if errno != 0 {
return nil, errno
}
copyAttr(&out.Attr, &ao.Attr)
found = true
}
}
if !found {
log.G(ctx).Warn("layernode.Lookup: uknown node type detected")
return nil, syscall.EIO
}
out.Attr.Ino = cn.StableAttr().Ino
return cn, 0
}
switch name {
case layerInfoLink:
info, err := n.fs.layerManager.getLayerInfo(ctx, n.refnode.ref, n.digest)
Expand All @@ -354,24 +365,6 @@ func (n *layernode) Lookup(ctx context.Context, name string, out *fuse.EntryOut)
})
case layerLink, blobLink:

// Check if layer is already known
if name == layerLink {
n.fs.knownNodeMu.Lock()
if lh, ok := n.fs.knownNode[n.refnode.ref.String()][n.digest.String()]; ok {
var ao fuse.AttrOut
if errno := lh.n.(fusefs.NodeGetattrer).Getattr(ctx, nil, &ao); errno != 0 {
return nil, errno
}
copyAttr(&out.Attr, &ao.Attr)
n.fs.knownNodeMu.Unlock()
return n.NewInode(ctx, lh.n, fusefs.StableAttr{
Mode: out.Attr.Mode,
Ino: out.Attr.Ino,
}), 0
}
n.fs.knownNodeMu.Unlock()
}

// Resolve layer
l, err := n.fs.layerManager.getLayer(ctx, n.refnode.ref, n.digest)
if err != nil {
Expand Down Expand Up @@ -423,15 +416,6 @@ func (n *layernode) Lookup(ctx context.Context, name string, out *fuse.EntryOut)
})

rr := &layerReleasable{n: root}
n.fs.knownNodeMu.Lock()
if n.fs.knownNode == nil {
n.fs.knownNode = make(map[string]map[string]*layerReleasable)
}
if n.fs.knownNode[n.refnode.ref.String()] == nil {
n.fs.knownNode[n.refnode.ref.String()] = make(map[string]*layerReleasable)
}
n.fs.knownNode[n.refnode.ref.String()][n.digest.String()] = rr
n.fs.knownNodeMu.Unlock()
return rr, nil
})
if err != nil || errno != 0 {
Expand Down

0 comments on commit 62e9b99

Please sign in to comment.