[WIP] Extract DAGService and friends

**DO NOT MERGE!** Refactor IPFS to use: ipfs/go-ipld-format#8 Context: IPLD isn't currently very usable from libp2p because interfaces like the DAGService are bundled into go-ipfs. This PR extracts these interfaces and some of the related helper functions. License: MIT Signed-off-by: Steven Allen <[email protected]>
ipfs · Jul 12, 2017 · 0ffa68e · 0ffa68e
1 parent 4316c97
commit 0ffa68e
Show file tree

Hide file tree

Showing 46 changed files with 183 additions and 357 deletions.
diff --git a/blockservice/blockservice.go b/blockservice/blockservice.go
@@ -258,6 +258,10 @@ type Session struct {
 	ses exchange.Fetcher
 }
 
+func (s *Session) Blockstore() blockstore.Blockstore {
+	return s.bs
+}
+
 // GetBlock gets a block in the context of a request session
 func (s *Session) GetBlock(ctx context.Context, c *cid.Cid) (blocks.Block, error) {
 	return getBlock(ctx, c, s.bs, s.ses)

diff --git a/core/commands/dht.go b/core/commands/dht.go
@@ -19,6 +19,7 @@ import (
 	b58 "gx/ipfs/QmT8rehPR3F6bmwL6zjUN8XpiDBFFpMP2myPdC6ApsWfJf/go-base58"
 	ipdht "gx/ipfs/QmTHyAbD9KzGrseLNzmEoNkVxA8F2h7LQG2iV6uhBqs6kX/go-libp2p-kad-dht"
 	cid "gx/ipfs/QmTprEaAA2A9bst5XH7exuyi5KzNMK3SEDNN8rBDnKWcUS/go-cid"
+	node "gx/ipfs/QmVHxZ8ovAuHiHTbJa68budGYAqmMUzb1bqDW1SVb6y5M9/go-ipld-format"
 	peer "gx/ipfs/QmXYjuNuxVzXKJCfWasQk1RqkhVLDM9jtUKhqc2WPQmFSB/go-libp2p-peer"
 )
 
@@ -374,7 +375,7 @@ func provideKeys(ctx context.Context, r routing.IpfsRouting, cids []*cid.Cid) er
 	return nil
 }
 
-func provideKeysRec(ctx context.Context, r routing.IpfsRouting, dserv dag.DAGService, cids []*cid.Cid) error {
+func provideKeysRec(ctx context.Context, r routing.IpfsRouting, dserv node.DAGService, cids []*cid.Cid) error {
 	provided := cid.NewSet()
 	for _, c := range cids {
 		kset := cid.NewSet()

diff --git a/core/commands/files/files.go b/core/commands/files/files.go
@@ -154,7 +154,7 @@ func statGetFormatOptions(req cmds.Request) (string, error) {
 	}
 }
 
-func statNode(ds dag.DAGService, fsn mfs.FSNode) (*Object, error) {
+func statNode(ds node.DAGService, fsn mfs.FSNode) (*Object, error) {
 	nd, err := fsn.GetNode()
 	if err != nil {
 		return nil, err

diff --git a/core/commands/pin.go b/core/commands/pin.go
@@ -17,6 +17,7 @@ import (
 	context "context"
 	u "gx/ipfs/QmSU6eubNdhXjFBJBSksTp8kv8YRub8mGAPv8tVJHmL2EU/go-ipfs-util"
 	cid "gx/ipfs/QmTprEaAA2A9bst5XH7exuyi5KzNMK3SEDNN8rBDnKWcUS/go-cid"
+	node "gx/ipfs/QmVHxZ8ovAuHiHTbJa68budGYAqmMUzb1bqDW1SVb6y5M9/go-ipld-format"
 )
 
 var PinCmd = &cmds.Command{
@@ -548,7 +549,7 @@ func pinLsAll(typeStr string, ctx context.Context, n *core.IpfsNode) (map[string
 	if typeStr == "indirect" || typeStr == "all" {
 		set := cid.NewSet()
 		for _, k := range n.Pinning.RecursiveKeys() {
-			err := dag.EnumerateChildren(n.Context(), n.DAG.GetLinks, k, set.Visit)
+			err := dag.EnumerateChildren(n.Context(), dag.GetLinksWithDAG(n.DAG), k, set.Visit)
 			if err != nil {
 				return nil, err
 			}
@@ -587,7 +588,7 @@ type pinVerifyOpts struct {
 
 func pinVerify(ctx context.Context, n *core.IpfsNode, opts pinVerifyOpts) <-chan interface{} {
 	visited := make(map[string]PinStatus)
-	getLinks := n.DAG.GetOfflineLinkService().GetLinks
+	ng := n.DAG.OfflineNodeGetter()
 	recPins := n.Pinning.RecursiveKeys()
 
 	var checkPin func(root *cid.Cid) PinStatus
@@ -597,7 +598,7 @@ func pinVerify(ctx context.Context, n *core.IpfsNode, opts pinVerifyOpts) <-chan
 			return status
 		}
 
-		links, err := getLinks(ctx, root)
+		links, err := node.GetLinks(ctx, ng, root)
 		if err != nil {
 			status := PinStatus{Ok: false}
 			if opts.explain {

diff --git a/core/commands/refs.go b/core/commands/refs.go
@@ -9,7 +9,6 @@ import (
 
 	cmds "github.com/ipfs/go-ipfs/commands"
 	"github.com/ipfs/go-ipfs/core"
-	dag "github.com/ipfs/go-ipfs/merkledag"
 	path "github.com/ipfs/go-ipfs/path"
 
 	u "gx/ipfs/QmSU6eubNdhXjFBJBSksTp8kv8YRub8mGAPv8tVJHmL2EU/go-ipfs-util"
@@ -220,7 +219,7 @@ type RefWrapper struct {
 
 type RefWriter struct {
 	out chan interface{}
-	DAG dag.DAGService
+	DAG node.DAGService
 	Ctx context.Context
 
 	Unique    bool
@@ -242,7 +241,7 @@ func (rw *RefWriter) writeRefsRecursive(n node.Node) (int, error) {
 	nc := n.Cid()
 
 	var count int
-	for i, ng := range dag.GetDAG(rw.Ctx, rw.DAG, n) {
+	for i, ng := range node.GetDAG(rw.Ctx, rw.DAG, n) {
 		lc := n.Links()[i].Cid
 		if rw.skip(lc) {
 			continue

diff --git a/core/core.go b/core/core.go
@@ -56,6 +56,7 @@ import (
 	b58 "gx/ipfs/QmT8rehPR3F6bmwL6zjUN8XpiDBFFpMP2myPdC6ApsWfJf/go-base58"
 	dht "gx/ipfs/QmTHyAbD9KzGrseLNzmEoNkVxA8F2h7LQG2iV6uhBqs6kX/go-libp2p-kad-dht"
 	cid "gx/ipfs/QmTprEaAA2A9bst5XH7exuyi5KzNMK3SEDNN8rBDnKWcUS/go-cid"
+	node "gx/ipfs/QmVHxZ8ovAuHiHTbJa68budGYAqmMUzb1bqDW1SVb6y5M9/go-ipld-format"
 	ds "gx/ipfs/QmVSase1JP7cq9QkPT46oNwdp9pT6kBkG3oqS14y3QcZjG/go-datastore"
 	metrics "gx/ipfs/QmVjRAPfRtResCMCE4eBqr4Beoa6A89P1YweG9wUS6RqUL/go-libp2p-metrics"
 	ma "gx/ipfs/QmXY77cVe7rVRQXZZQRioukUM7aRW3BTcAgJe12MCtb3Ji/go-multiaddr"
@@ -114,7 +115,7 @@ type IpfsNode struct {
 	BaseBlocks bstore.Blockstore    // the raw blockstore, no filestore wrapping
 	GCLocker   bstore.GCLocker      // the locker used to protect the blockstore during gc
 	Blocks     bserv.BlockService   // the block service, get/add blocks.
-	DAG        merkledag.DAGService // the merkle dag service, get/add objects.
+	DAG        node.DAGService      // the merkle dag service, get/add objects.
 	Resolver   *path.Resolver       // the path resolution system
 	Reporter   metrics.Reporter
 	Discovery  discovery.Service

diff --git a/core/coreunix/add.go b/core/coreunix/add.go
@@ -70,7 +70,7 @@ type AddedObject struct {
 	Bytes int64  `json:",omitempty"`
 }
 
-func NewAdder(ctx context.Context, p pin.Pinner, bs bstore.GCBlockstore, ds dag.DAGService) (*Adder, error) {
+func NewAdder(ctx context.Context, p pin.Pinner, bs bstore.GCBlockstore, ds node.DAGService) (*Adder, error) {
 	return &Adder{
 		ctx:        ctx,
 		pinning:    p,
@@ -90,7 +90,7 @@ type Adder struct {
 	ctx        context.Context
 	pinning    pin.Pinner
 	blockstore bstore.GCBlockstore
-	dagService dag.DAGService
+	dagService node.DAGService
 	Out        chan interface{}
 	Progress   bool
 	Hidden     bool
@@ -548,7 +548,7 @@ func outputDagnode(out chan interface{}, name string, dn node.Node) error {
 	return nil
 }
 
-func NewMemoryDagService() dag.DAGService {
+func NewMemoryDagService() node.DAGService {
 	// build mem-datastore for editor's intermediary nodes
 	bs := bstore.NewBlockstore(syncds.MutexWrap(ds.NewMapDatastore()))
 	bsrv := bserv.New(bs, offline.Exchange(bs))

diff --git a/core/coreunix/add_test.go b/core/coreunix/add_test.go
@@ -148,7 +148,7 @@ func TestAddGCLive(t *testing.T) {
 	defer cancel()
 
 	set := cid.NewSet()
-	err = dag.EnumerateChildren(ctx, node.DAG.GetLinks, last, set.Visit)
+	err = dag.EnumerateChildren(ctx, dag.GetLinksWithDAG(node.DAG), last, set.Visit)
 	if err != nil {
 		t.Fatal(err)
 	}

diff --git a/core/coreunix/metadata_test.go b/core/coreunix/metadata_test.go
@@ -18,11 +18,12 @@ import (
 	context "context"
 	u "gx/ipfs/QmSU6eubNdhXjFBJBSksTp8kv8YRub8mGAPv8tVJHmL2EU/go-ipfs-util"
 	cid "gx/ipfs/QmTprEaAA2A9bst5XH7exuyi5KzNMK3SEDNN8rBDnKWcUS/go-cid"
+	node "gx/ipfs/QmVHxZ8ovAuHiHTbJa68budGYAqmMUzb1bqDW1SVb6y5M9/go-ipld-format"
 	ds "gx/ipfs/QmVSase1JP7cq9QkPT46oNwdp9pT6kBkG3oqS14y3QcZjG/go-datastore"
 	dssync "gx/ipfs/QmVSase1JP7cq9QkPT46oNwdp9pT6kBkG3oqS14y3QcZjG/go-datastore/sync"
 )
 
-func getDagserv(t *testing.T) merkledag.DAGService {
+func getDagserv(t *testing.T) node.DAGService {
 	db := dssync.MutexWrap(ds.NewMapDatastore())
 	bs := bstore.NewBlockstore(db)
 	blockserv := bserv.New(bs, offline.Exchange(bs))

diff --git a/importer/balanced/balanced_test.go b/importer/balanced/balanced_test.go
@@ -16,11 +16,12 @@ import (
 
 	"context"
 	u "gx/ipfs/QmSU6eubNdhXjFBJBSksTp8kv8YRub8mGAPv8tVJHmL2EU/go-ipfs-util"
+	node "gx/ipfs/QmVHxZ8ovAuHiHTbJa68budGYAqmMUzb1bqDW1SVb6y5M9/go-ipld-format"
 )
 
 // TODO: extract these tests and more as a generic layout test suite
 
-func buildTestDag(ds dag.DAGService, spl chunk.Splitter) (*dag.ProtoNode, error) {
+func buildTestDag(ds node.DAGService, spl chunk.Splitter) (*dag.ProtoNode, error) {
 	dbp := h.DagBuilderParams{
 		Dagserv:  ds,
 		Maxlinks: h.DefaultLinksPerBlock,
@@ -34,7 +35,7 @@ func buildTestDag(ds dag.DAGService, spl chunk.Splitter) (*dag.ProtoNode, error)
 	return nd.(*dag.ProtoNode), nil
 }
 
-func getTestDag(t *testing.T, ds dag.DAGService, size int64, blksize int64) (*dag.ProtoNode, []byte) {
+func getTestDag(t *testing.T, ds node.DAGService, size int64, blksize int64) (*dag.ProtoNode, []byte) {
 	data := make([]byte, size)
 	u.NewTimeSeededRand().Read(data)
 	r := bytes.NewReader(data)

diff --git a/importer/helpers/dagbuilder.go b/importer/helpers/dagbuilder.go
@@ -16,13 +16,13 @@ import (
 // DagBuilderHelper wraps together a bunch of objects needed to
 // efficiently create unixfs dag trees
 type DagBuilderHelper struct {
-	dserv     dag.DAGService
+	dserv     node.DAGService
 	spl       chunk.Splitter
 	recvdErr  error
 	rawLeaves bool
 	nextData  []byte // the next item to return.
 	maxlinks  int
-	batch     *dag.Batch
+	batch     *node.Batch
 	fullPath  string
 	stat      os.FileInfo
 	prefix    *cid.Prefix
@@ -40,7 +40,7 @@ type DagBuilderParams struct {
 	Prefix *cid.Prefix
 
 	// DAGService to write blocks to (required)
-	Dagserv dag.DAGService
+	Dagserv node.DAGService
 
 	// NoCopy signals to the chunker that it should track fileinfo for
 	// filestore adds
@@ -56,7 +56,7 @@ func (dbp *DagBuilderParams) New(spl chunk.Splitter) *DagBuilderHelper {
 		rawLeaves: dbp.RawLeaves,
 		prefix:    dbp.Prefix,
 		maxlinks:  dbp.Maxlinks,
-		batch:     dbp.Dagserv.Batch(),
+		batch:     node.Batching(dbp.Dagserv),
 	}
 	if fi, ok := spl.Reader().(files.FileInfo); dbp.NoCopy && ok {
 		db.fullPath = fi.AbsPath()
@@ -106,7 +106,7 @@ func (db *DagBuilderHelper) Next() ([]byte, error) {
 }
 
 // GetDagServ returns the dagservice object this Helper is using
-func (db *DagBuilderHelper) GetDagServ() dag.DAGService {
+func (db *DagBuilderHelper) GetDagServ() node.DAGService {
 	return db.dserv
 }
 

diff --git a/importer/helpers/helpers.go b/importer/helpers/helpers.go
@@ -78,7 +78,7 @@ func (n *UnixfsNode) Set(other *UnixfsNode) {
 	}
 }
 
-func (n *UnixfsNode) GetChild(ctx context.Context, i int, ds dag.DAGService) (*UnixfsNode, error) {
+func (n *UnixfsNode) GetChild(ctx context.Context, i int, ds node.NodeGetter) (*UnixfsNode, error) {
 	nd, err := n.node.Links()[i].GetNode(ctx, ds)
 	if err != nil {
 		return nil, err

diff --git a/importer/importer.go b/importer/importer.go
@@ -11,14 +11,13 @@ import (
 	"github.com/ipfs/go-ipfs/importer/chunk"
 	h "github.com/ipfs/go-ipfs/importer/helpers"
 	trickle "github.com/ipfs/go-ipfs/importer/trickle"
-	dag "github.com/ipfs/go-ipfs/merkledag"
 
 	node "gx/ipfs/QmVHxZ8ovAuHiHTbJa68budGYAqmMUzb1bqDW1SVb6y5M9/go-ipld-format"
 )
 
 // Builds a DAG from the given file, writing created blocks to disk as they are
 // created
-func BuildDagFromFile(fpath string, ds dag.DAGService) (node.Node, error) {
+func BuildDagFromFile(fpath string, ds node.DAGService) (node.Node, error) {
 	stat, err := os.Lstat(fpath)
 	if err != nil {
 		return nil, err
@@ -37,7 +36,7 @@ func BuildDagFromFile(fpath string, ds dag.DAGService) (node.Node, error) {
 	return BuildDagFromReader(ds, chunk.NewSizeSplitter(f, chunk.DefaultBlockSize))
 }
 
-func BuildDagFromReader(ds dag.DAGService, spl chunk.Splitter) (node.Node, error) {
+func BuildDagFromReader(ds node.DAGService, spl chunk.Splitter) (node.Node, error) {
 	dbp := h.DagBuilderParams{
 		Dagserv:  ds,
 		Maxlinks: h.DefaultLinksPerBlock,
@@ -46,7 +45,7 @@ func BuildDagFromReader(ds dag.DAGService, spl chunk.Splitter) (node.Node, error
 	return bal.BalancedLayout(dbp.New(spl))
 }
 
-func BuildTrickleDagFromReader(ds dag.DAGService, spl chunk.Splitter) (node.Node, error) {
+func BuildTrickleDagFromReader(ds node.DAGService, spl chunk.Splitter) (node.Node, error) {
 	dbp := h.DagBuilderParams{
 		Dagserv:  ds,
 		Maxlinks: h.DefaultLinksPerBlock,

diff --git a/importer/importer_test.go b/importer/importer_test.go
@@ -8,15 +8,14 @@ import (
 	"testing"
 
 	chunk "github.com/ipfs/go-ipfs/importer/chunk"
-	dag "github.com/ipfs/go-ipfs/merkledag"
 	mdtest "github.com/ipfs/go-ipfs/merkledag/test"
 	uio "github.com/ipfs/go-ipfs/unixfs/io"
 
 	u "gx/ipfs/QmSU6eubNdhXjFBJBSksTp8kv8YRub8mGAPv8tVJHmL2EU/go-ipfs-util"
 	node "gx/ipfs/QmVHxZ8ovAuHiHTbJa68budGYAqmMUzb1bqDW1SVb6y5M9/go-ipld-format"
 )
 
-func getBalancedDag(t testing.TB, size int64, blksize int64) (node.Node, dag.DAGService) {
+func getBalancedDag(t testing.TB, size int64, blksize int64) (node.Node, node.DAGService) {
 	ds := mdtest.Mock()
 	r := io.LimitReader(u.NewTimeSeededRand(), size)
 	nd, err := BuildDagFromReader(ds, chunk.NewSizeSplitter(r, blksize))
@@ -26,7 +25,7 @@ func getBalancedDag(t testing.TB, size int64, blksize int64) (node.Node, dag.DAG
 	return nd, ds
 }
 
-func getTrickleDag(t testing.TB, size int64, blksize int64) (node.Node, dag.DAGService) {
+func getTrickleDag(t testing.TB, size int64, blksize int64) (node.Node, node.DAGService) {
 	ds := mdtest.Mock()
 	r := io.LimitReader(u.NewTimeSeededRand(), size)
 	nd, err := BuildTrickleDagFromReader(ds, chunk.NewSizeSplitter(r, blksize))
@@ -102,7 +101,7 @@ func BenchmarkTrickleReadFull(b *testing.B) {
 	runReadBench(b, nd, ds)
 }
 
-func runReadBench(b *testing.B, nd node.Node, ds dag.DAGService) {
+func runReadBench(b *testing.B, nd node.Node, ds node.DAGService) {
 	for i := 0; i < b.N; i++ {
 		ctx, cancel := context.WithCancel(context.Background())
 		read, err := uio.NewDagReader(ctx, nd, ds)

diff --git a/importer/trickle/trickle_test.go b/importer/trickle/trickle_test.go
@@ -17,9 +17,10 @@ import (
 	uio "github.com/ipfs/go-ipfs/unixfs/io"
 
 	u "gx/ipfs/QmSU6eubNdhXjFBJBSksTp8kv8YRub8mGAPv8tVJHmL2EU/go-ipfs-util"
+	node "gx/ipfs/QmVHxZ8ovAuHiHTbJa68budGYAqmMUzb1bqDW1SVb6y5M9/go-ipld-format"
 )
 
-func buildTestDag(ds merkledag.DAGService, spl chunk.Splitter) (*merkledag.ProtoNode, error) {
+func buildTestDag(ds node.DAGService, spl chunk.Splitter) (*merkledag.ProtoNode, error) {
 	dbp := h.DagBuilderParams{
 		Dagserv:  ds,
 		Maxlinks: h.DefaultLinksPerBlock,

diff --git a/importer/trickle/trickledag.go b/importer/trickle/trickledag.go
@@ -236,7 +236,7 @@ func trickleDepthInfo(node *h.UnixfsNode, maxlinks int) (int, int) {
 
 // VerifyTrickleDagStructure checks that the given dag matches exactly the trickle dag datastructure
 // layout
-func VerifyTrickleDagStructure(nd node.Node, ds dag.DAGService, direct int, layerRepeat int) error {
+func VerifyTrickleDagStructure(nd node.Node, ds node.NodeGetter, direct int, layerRepeat int) error {
 	pbnd, ok := nd.(*dag.ProtoNode)
 	if !ok {
 		return dag.ErrNotProtobuf
@@ -246,7 +246,7 @@ func VerifyTrickleDagStructure(nd node.Node, ds dag.DAGService, direct int, laye
 }
 
 // Recursive call for verifying the structure of a trickledag
-func verifyTDagRec(nd *dag.ProtoNode, depth, direct, layerRepeat int, ds dag.DAGService) error {
+func verifyTDagRec(nd *dag.ProtoNode, depth, direct, layerRepeat int, ds node.NodeGetter) error {
 	if depth == 0 {
 		// zero depth dag is raw data block
 		if len(nd.Links()) > 0 {