"filestore verify": add ability to specify hashes to verify

License: MIT
Signed-off-by: Kevin Atkinson <[email protected]>

core/commands/filestore.go

@@ -243,7 +243,10 @@ var verifyFileStore = &cmds.Command{
 	Helptext: cmds.HelpText{
 		Tagline: "Verify objects in filestore",
 		ShortDescription: `
-Verify nodes in the filestore.  The output is:
+Verify <hash> nodes in the filestore.  If no hashes are specified then
+verify everything in the filestore.
+
+The output is:
   <status> [<type> <filepath> <offset> <size> [<modtime>]]
 where <type>, <filepath>, <offset>, <size> and <modtime> are the same
 as in the "ls" command and <status> is one of
@@ -255,8 +258,8 @@ as in the "ls" command and <status> is one of
   incomplete: some of the blocks of the tree could not be read
 
   changed: the contents of the backing file have changed
-  no-file: the backing file can not be found
-  error:   the backing file can be found but could not be read
+  no-file: the backing file could not be found
+  error:   the backing file was found but could not be read
 
   ERROR:   the block could not be read due to an internal error
 
@@ -271,9 +274,8 @@ as in the "ls" command and <status> is one of
           the filestore
  
 If --basic is specified then just scan leaf nodes to verify that they
-are still valid.  Otherwise attempt to reconstruct the contents of of
-all nodes and also check for orphan nodes (unless --skip-orphans is
-also specified).
+are still valid.  Otherwise attempt to reconstruct the contents of
+all nodes and check for orphan nodes if applicable.
 
 The --level option specifies how thorough the checks should be.  A
 current meaning of the levels are:
@@ -284,11 +286,14 @@ current meaning of the levels are:
 
 The --verbose option specifies what to output.  The current values are:
   7-9: show everything
-  5-6: don't show child nodes with a status of: ok, <blank>, or complete
+  5-6: don't show child nodes unless there is a problem
   3-4: don't show child nodes
-  0-2: don't child nodes and don't show root nodes with of: ok or complete
+  0-2: don't show root nodes unless there is a problem
 `,
 	},
+	Arguments: []cmds.Argument{
+		cmds.StringArg("hash", false, true, "Hashs of nodes to verify."),
+	},
 	Options: []cmds.Option{
 		cmds.BoolOption("basic", "Perform a basic scan of leaf nodes only."),
 		cmds.IntOption("level", "l", "0-9, Verification level.").Default(6),
@@ -301,6 +306,11 @@ The --verbose option specifies what to output.  The current values are:
 			res.SetError(err, cmds.ErrNormal)
 			return
 		}
+		args := req.Arguments()
+		keys := make([]k.Key, 0)
+		for _, key := range args {
+			keys = append(keys, k.B58KeyDecode(key))
+		}
 		basic, _, err := req.Option("basic").Bool()
 		if err != nil {
 			res.SetError(err, cmds.ErrNormal)
@@ -325,12 +335,19 @@ The --verbose option specifies what to output.  The current values are:
 			res.SetError(err, cmds.ErrNormal)
 			return
 		}
-		if basic {
+
+		if basic && len(keys) == 0 {
 			ch, _ := fsutil.VerifyBasic(fs, level, verbose)
 			res.SetOutput(&chanWriter{ch: ch})
-		} else {
+		} else if basic {
+			ch, _ := fsutil.VerifyKeys(keys, node, fs, level)
+			res.SetOutput(&chanWriter{ch: ch})
+		} else if len(keys) == 0 {
 			ch, _ := fsutil.VerifyFull(node, fs, level, verbose, skipOrphans)
 			res.SetOutput(&chanWriter{ch: ch})
+		} else {
+			ch, _ := fsutil.VerifyKeysFull(keys, node, fs, level, verbose)
+			res.SetOutput(&chanWriter{ch: ch})
 		}
 	},
 	Marshalers: cmds.MarshalerMap{

filestore/util/verify.go

@@ -3,6 +3,7 @@ package filestore_util
 import (
 	"os"
 
+	b "github.com/ipfs/go-ipfs/blocks/blockstore"
 	k "github.com/ipfs/go-ipfs/blocks/key"
 	"github.com/ipfs/go-ipfs/core"
 	. "github.com/ipfs/go-ipfs/filestore"
@@ -34,6 +35,42 @@ func VerifyBasic(fs *Datastore, level int, verbose int) (<-chan ListRes, error)
 	return out, nil
 }
 
+func VerifyKeys(keys []k.Key, node *core.IpfsNode, fs *Datastore, level int) (<-chan ListRes, error) {
+	out := make(chan ListRes, 16)
+	verifyWhat := VerifyAlways
+	if level <= 6 {
+		verifyWhat = VerifyIfChanged
+	}
+	go func() {
+		defer close(out)
+		for _, key := range keys {
+			out <- verifyKey(key, fs, node.Blockstore, verifyWhat)
+		}
+	}()
+	return out, nil
+}
+
+func verifyKey(key k.Key, fs *Datastore, bs b.Blockstore, verifyWhat int) ListRes {
+	dsKey := key.DsKey()
+	dataObj, err := fs.GetDirect(dsKey)
+	if err == nil && dataObj.NoBlockData() {
+		res := ListRes{dsKey, dataObj, 0}
+		res.Status = verify(fs, dsKey, dataObj, verifyWhat)
+		return res
+	} else if err == nil {
+		return ListRes{dsKey, dataObj, StatusUnchecked}
+	}
+	found, _ := bs.Has(key)
+	if found {
+		return ListRes{dsKey, nil, StatusFound}
+	} else if err == ds.ErrNotFound && !found {
+		return ListRes{dsKey, nil, StatusKeyNotFound}
+	} else {
+		Logger.Errorf("%s: %v", key, err)
+		return ListRes{dsKey, nil, StatusError}
+	}
+}
+
 func VerifyFull(node *core.IpfsNode, fs *Datastore, level int, verbose int, skipOrphans bool) (<-chan ListRes, error) {
 	p := verifyParams{make(chan ListRes, 16), node, fs, level, verbose, skipOrphans, nil}
 	ch, err := ListKeys(p.fs)
@@ -47,6 +84,15 @@ func VerifyFull(node *core.IpfsNode, fs *Datastore, level int, verbose int, skip
 	return p.out, nil
 }
 
+func VerifyKeysFull(keys []k.Key, node *core.IpfsNode, fs *Datastore, level int, verbose int) (<-chan ListRes, error) {
+	p := verifyParams{make(chan ListRes, 16), node, fs, level, verbose, true, nil}
+	go func() {
+		defer close(p.out)
+		p.verifyKeys(keys)
+	}()
+	return p.out, nil
+}
+
 type verifyParams struct {
 	out  chan ListRes
 	node *core.IpfsNode
@@ -77,6 +123,25 @@ func (p *verifyParams) setStatus(dsKey ds.Key, status int) {
 	}
 }
 
+func (p *verifyParams) verifyKeys(keys []k.Key) {
+	p.skipOrphans = true 
+	for _, key := range keys {
+		dsKey := key.DsKey()
+		dagNode, dataObj, r := p.get(dsKey)
+		if dataObj == nil || AnError(r) {
+			/* nothing to do */
+		} else if dataObj.Internal() {
+			r = p.verifyNode(dagNode)
+		} else {
+			r = p.verifyLeaf(dsKey, dataObj)
+		}
+		res := ListRes{dsKey, dataObj, r}
+		res.Status = p.checkIfAppended(res)
+		p.out <- res
+		p.out <- EmptyListRes
+	}
+}
+
 func (p *verifyParams) verify(ch <-chan ListRes) {
 	p.seen = make(map[string]int)
 	unsafeToCont := false
@@ -89,15 +154,7 @@ func (p *verifyParams) verify(ch <-chan ListRes) {
 		if AnError(r) {
 			/* nothing to do */
 		} else if res.Internal() && res.WholeFile() {
-			if dagNode == nil {
-				// we expect a node, so even if the status is
-				// okay we should set it to an Error
-				if !AnError(r) {
-					r = StatusError
-				}
-			} else {
-				r = p.verifyNode(dagNode)
-			}
+			r = p.verifyNode(dagNode)
 		} else if res.WholeFile() {
 			r = p.verifyLeaf(res.Key, res.DataObj)
 		} else {
@@ -156,6 +213,9 @@ func (p *verifyParams) checkIfAppended(res ListRes) int {
 }
 
 func (p *verifyParams) verifyNode(n *node.Node) int {
+	if n == nil {
+		return StatusError
+	}
 	complete := true
 	for _, link := range n.Links {
 		key := k.Key(link.Hash).DsKey()