cli: Collect storagegroup members with new V2 split

It was easier to implement it from scratch than to try to understand callbacks
and recursion.

Signed-off-by: Pavel Karpy <[email protected]>

cmd/neofs-cli/modules/storagegroup/put.go

@@ -12,6 +12,7 @@ import (
 	"github.com/nspcc-dev/neofs-node/cmd/neofs-cli/internal/key"
 	objectCli "github.com/nspcc-dev/neofs-node/cmd/neofs-cli/modules/object"
 	"github.com/nspcc-dev/neofs-node/pkg/services/object_manager/storagegroup"
+	"github.com/nspcc-dev/neofs-sdk-go/client"
 	cid "github.com/nspcc-dev/neofs-sdk-go/container/id"
 	"github.com/nspcc-dev/neofs-sdk-go/object"
 	oid "github.com/nspcc-dev/neofs-sdk-go/object/id"
@@ -80,6 +81,7 @@ func putSG(cmd *cobra.Command, _ []string) {
 		headPrm   internalclient.HeadObjectPrm
 		putPrm    internalclient.PutObjectPrm
 		getCnrPrm internalclient.GetContainerPrm
+		getPrm    internalclient.GetObjectPrm
 	)
 
 	cli := internalclient.GetSDKClientByFlag(ctx, cmd, commonflags.RPC)
@@ -96,12 +98,19 @@ func putSG(cmd *cobra.Command, _ []string) {
 	headPrm.SetClient(cli)
 	headPrm.SetPrivateKey(*pk)
 
+	headPrm.SetRawFlag(true)
+	getPrm.SetClient(cli)
+	getPrm.SetPrivateKey(*pk)
+	objectCli.Prepare(cmd, &getPrm)
+
 	sg, err := storagegroup.CollectMembers(sgHeadReceiver{
 		ctx:     ctx,
 		cmd:     cmd,
 		key:     pk,
 		ownerID: &ownerID,
-		prm:     headPrm,
+		prmHead: headPrm,
+		cli:     cli,
+		getPrm:  getPrm,
 	}, cnr, members, !resGetCnr.Container().IsHomomorphicHashingDisabled())
 	common.ExitOnErr(cmd, "could not collect storage group members: %w", err)
 
@@ -138,13 +147,40 @@ type sgHeadReceiver struct {
 	cmd     *cobra.Command
 	key     *ecdsa.PrivateKey
 	ownerID *user.ID
-	prm     internalclient.HeadObjectPrm
+	prmHead internalclient.HeadObjectPrm
+	cli     *client.Client
+	getPrm  internalclient.GetObjectPrm
+}
+
+type payloadWriter struct {
+	payload []byte
+}
+
+func (pw *payloadWriter) Write(p []byte) (n int, err error) {
+	pw.payload = append(pw.payload, p...)
+	return len(p), nil
+}
+
+func (c sgHeadReceiver) Get(addr oid.Address) (object.Object, error) {
+	pw := &payloadWriter{}
+	c.getPrm.SetPayloadWriter(pw)
+	c.getPrm.SetAddress(addr)
+
+	res, err := internalclient.GetObject(c.ctx, c.getPrm)
+	if err != nil {
+		return object.Object{}, fmt.Errorf("rpc error: %w", err)
+	}
+
+	obj := res.Header()
+	obj.SetPayload(pw.payload)
+
+	return *obj, nil
 }
 
 func (c sgHeadReceiver) Head(addr oid.Address) (any, error) {
-	c.prm.SetAddress(addr)
+	c.prmHead.SetAddress(addr)
 
-	res, err := internalclient.HeadObject(c.ctx, c.prm)
+	res, err := internalclient.HeadObject(c.ctx, c.prmHead)
 
 	var errSplitInfo *object.SplitInfoError
 

pkg/services/object/util/chain.go

@@ -4,17 +4,22 @@ import (
 	"errors"
 	"fmt"
 
+	cid "github.com/nspcc-dev/neofs-sdk-go/container/id"
 	"github.com/nspcc-dev/neofs-sdk-go/object"
 	oid "github.com/nspcc-dev/neofs-sdk-go/object/id"
 )
 
-// HeadReceiver is an interface of entity that can receive
-// object header or the information about the object relations.
-type HeadReceiver interface {
+// ObjectSource is an interface of entity that can receive
+// object header, the whole object or the information about
+// the object relations.
+type ObjectSource interface {
 	// Head must return one of:
 	// * object header (*object.Object);
 	// * structured information about split-chain (*object.SplitInfo).
 	Head(id oid.Address) (any, error)
+
+	// Get must return object by its address.
+	Get(address oid.Address) (object.Object, error)
 }
 
 // SplitMemberHandler is a handler of next split-chain element.
@@ -24,7 +29,7 @@ type HeadReceiver interface {
 type SplitMemberHandler func(member *object.Object, reverseDirection bool) (stop bool)
 
 // IterateAllSplitLeaves is an iterator over all object split-tree leaves in direct order.
-func IterateAllSplitLeaves(r HeadReceiver, addr oid.Address, h func(*object.Object)) error {
+func IterateAllSplitLeaves(r ObjectSource, addr oid.Address, h func(*object.Object)) error {
 	return IterateSplitLeaves(r, addr, func(leaf *object.Object) bool {
 		h(leaf)
 		return false
@@ -34,45 +39,178 @@ func IterateAllSplitLeaves(r HeadReceiver, addr oid.Address, h func(*object.Obje
 // IterateSplitLeaves is an iterator over object split-tree leaves in direct order.
 //
 // If member handler returns true, then the iterator aborts without error.
-func IterateSplitLeaves(r HeadReceiver, addr oid.Address, h func(*object.Object) bool) error {
-	var (
-		reverse bool
-		leaves  []*object.Object
-	)
-
-	if err := TraverseSplitChain(r, addr, func(member *object.Object, reverseDirection bool) (stop bool) {
-		reverse = reverseDirection
-
-		if reverse {
-			leaves = append(leaves, member)
-			return false
+func IterateSplitLeaves(r ObjectSource, addr oid.Address, h func(*object.Object) bool) error {
+	info, err := r.Head(addr)
+	if err != nil {
+		return fmt.Errorf("receiving information about the object: %w", err)
+	}
+
+	switch res := info.(type) {
+	default:
+		panic(fmt.Sprintf("unexpected result of %T: %T", r, info))
+	case *object.Object:
+		h(res)
+	case *object.SplitInfo:
+		if res.SplitID() == nil {
+			return iterateV2Split(r, res, addr.Container(), h)
+		} else {
+			return iterateV1Split(r, res, addr.Container(), h)
 		}
+	}
 
-		return h(member)
-	}); err != nil {
-		return err
+	return nil
+}
+
+func iterateV1Split(r ObjectSource, info *object.SplitInfo, cID cid.ID, handler func(*object.Object) bool) error {
+	var addr oid.Address
+	addr.SetContainer(cID)
+
+	linkID, ok := info.Link()
+	if ok {
+		addr.SetObject(linkID)
+
+		linkObj, err := headFromReceiver(r, addr)
+		if err != nil {
+			return err
+		}
+
+		for _, child := range linkObj.Children() {
+			addr.SetObject(child)
+
+			childHeader, err := headFromReceiver(r, addr)
+			if err != nil {
+				return err
+			}
+
+			if stop := handler(childHeader); stop {
+				return nil
+			}
+		}
+
+		handler(linkObj)
+
+		return nil
+	}
+
+	lastID, ok := info.LastPart()
+	if ok {
+		addr.SetObject(lastID)
+		return iterateFromLastObject(r, addr, handler)
 	}
 
-	for i := len(leaves) - 1; i >= 0; i-- {
-		if h(leaves[i]) {
+	return errors.New("neither link, nor last object ID is found")
+}
+
+func iterateV2Split(r ObjectSource, info *object.SplitInfo, cID cid.ID, handler func(*object.Object) bool) error {
+	var addr oid.Address
+	addr.SetContainer(cID)
+
+	linkID, ok := info.Link()
+	if ok {
+		addr.SetObject(linkID)
+
+		linkObjRaw, err := r.Get(addr)
+		if err != nil {
+			return fmt.Errorf("receiving link object %s: %w", addr, err)
+		}
+
+		if stop := handler(&linkObjRaw); stop {
+			return nil
+		}
+
+		var linkObj object.Link
+		err = linkObjRaw.ReadLink(&linkObj)
+		if err != nil {
+			return fmt.Errorf("decoding link object (%d): %w", addr, err)
+		}
+
+		for _, child := range linkObj.Objects() {
+			addr.SetObject(child.ObjectID())
+
+			childObj, err := headFromReceiver(r, addr)
+			if err != nil {
+				return fmt.Errorf("fetching child object (%s): %w", addr, err)
+			}
+
+			if stop := handler(childObj); stop {
+				return nil
+			}
+		}
+
+		return nil
+	}
+
+	lastID, ok := info.LastPart()
+	if ok {
+		addr.SetObject(lastID)
+		return iterateFromLastObject(r, addr, handler)
+	}
+
+	return errors.New("neither link, nor last object ID is found")
+}
+
+func iterateFromLastObject(r ObjectSource, lastAddr oid.Address, handler func(*object.Object) bool) error {
+	var idBuff []oid.ID
+	addr := lastAddr
+
+	for {
+		obj, err := headFromReceiver(r, addr)
+		if err != nil {
+			return err
+		}
+
+		oID, _ := obj.ID()
+		idBuff = append(idBuff, oID)
+
+		prevOID, set := obj.PreviousID()
+		if !set {
 			break
 		}
+
+		addr.SetObject(prevOID)
+	}
+
+	for i := len(idBuff) - 1; i >= 0; i-- {
+		addr.SetObject(idBuff[i])
+
+		childObj, err := headFromReceiver(r, addr)
+		if err != nil {
+			return err
+		}
+
+		if stop := handler(childObj); stop {
+			return nil
+		}
 	}
 
 	return nil
 }
 
+func headFromReceiver(r ObjectSource, addr oid.Address) (*object.Object, error) {
+	res, err := r.Head(addr)
+	if err != nil {
+		return nil, fmt.Errorf("fetching information about %s: %w", addr, err)
+	}
+
+	switch v := res.(type) {
+	case *object.Object:
+		return v, nil
+	default:
+		return nil, fmt.Errorf("unexpected information: %T", res)
+	}
+}
+
 // TraverseSplitChain is an iterator over object split-tree leaves.
 //
 // Traversal occurs in one of two directions, which depends on what pslit info was received:
 // * in direct order for link part;
 // * in reverse order for last part.
-func TraverseSplitChain(r HeadReceiver, addr oid.Address, h SplitMemberHandler) error {
+func TraverseSplitChain(r ObjectSource, addr oid.Address, h SplitMemberHandler) error {
 	_, err := traverseSplitChain(r, addr, h)
 	return err
 }
 
-func traverseSplitChain(r HeadReceiver, addr oid.Address, h SplitMemberHandler) (bool, error) {
+func traverseSplitChain(r ObjectSource, addr oid.Address, h SplitMemberHandler) (bool, error) {
 	v, err := r.Head(addr)
 	if err != nil {
 		return false, err

pkg/services/object_manager/storagegroup/collect.go

@@ -20,10 +20,10 @@ var (
 )
 
 // CollectMembers creates new storage group structure and fills it
-// with information about members collected via HeadReceiver.
+// with information about members collected via ObjectSource.
 //
 // Resulting storage group consists of physically stored objects only.
-func CollectMembers(r objutil.HeadReceiver, cnr cid.ID, members []oid.ID, calcHomoHash bool) (*storagegroup.StorageGroup, error) {
+func CollectMembers(r objutil.ObjectSource, cnr cid.ID, members []oid.ID, calcHomoHash bool) (*storagegroup.StorageGroup, error) {
 	var (
 		sumPhySize uint64
 		phyMembers []oid.ID

pkg/services/object_manager/storagegroup/collect_test.go

@@ -16,6 +16,10 @@ type mockedObjects struct {
 	hdr *object.Object
 }
 
+func (x *mockedObjects) Get(address oid.Address) (object.Object, error) {
+	return *x.hdr, nil
+}
+
 func (x *mockedObjects) Head(_ oid.Address) (any, error) {
 	return x.hdr, nil
 }