Multi: Support SPV in embedded wallet mode

.github/workflows/go.yml

@@ -52,7 +52,13 @@ jobs:
       fail-fast: false
       matrix:
         go: [1.16]
-        testsuite: ["unit-race", "itest-only", "itest-only walletimpl=remotewallet", "itest-only walletimpl=embeddedwallet_dcrw", "itest-only walletimpl=remotewallet backend=spv"]
+        testsuite:
+          - unit-race # unit tests
+          - itest-only # embedded wallet using dcrd for sync and chain ops
+          - itest-only walletimpl=embeddedwallet_dcrw # embedded wallet, dcrd sync but dcrw chain ops
+          - itest-only walletimpl=embeddedwallet_dcrw backend=spv # embedded wallet, spv sync and dcrw chain ops
+          - itest-only walletimpl=remotewallet # remote wallet dcrd sync
+          - itest-only walletimpl=remotewallet backend=spv # remote wallet spv sync
     steps:
       - name: Set up Go
         uses: actions/setup-go@v1

.gitignore

@@ -48,6 +48,7 @@ chainscan/csdrivers/*.log
 chainntnfs/*.log
 chainntnfs/.miner-logs*
 routing/chainview/*.log
+dcrlnd.log
 
 cmd/cmd
 *.key

build/log_default.go

@@ -1,4 +1,4 @@
-// +build !stdlog,!nolog
+// +build !stdlog,!nolog,!filelog
 
 package build
 

build/log_filelog.go

@@ -0,0 +1,23 @@
+// +build filelog
+
+package build
+
+import "os"
+
+var logf *os.File
+
+// LoggingType is a log type that writes to a file.
+const LoggingType = LogTypeStdOut
+
+// Write is a noop.
+func (w *LogWriter) Write(b []byte) (int, error) {
+	return logf.Write(b)
+}
+
+func init() {
+	var err error
+	logf, err = os.Create("dcrlnd.log")
+	if err != nil {
+		panic(err)
+	}
+}

chainregistry.go

@@ -6,6 +6,7 @@ import (
 	"fmt"
 	"io/ioutil"
 	"os"
+	"path/filepath"
 	"strings"
 	"sync"
 	"time"
@@ -212,17 +213,20 @@ func newChainControlFromConfig(cfg *Config, localDB, remoteDB *channeldb.DB,
 	// mode (using the wallet for chain operations).
 	if conn != nil && cfg.Node != "dcrw" {
 		return nil, fmt.Errorf("remote wallet mode only supports " +
-			"'dcrw' node config")
+			"'node=dcrw' config")
 	}
 
-	// When running in embedded wallet mode, we require an underlying dcrd
-	// instance whether running in dcrd or dcrw node modes, so we test the
-	// connection to the dcrd instance.
-	//
-	// Conversely, when running in remote wallet mode we only allow the
-	// dcrw node mode (per the above conditional check) so there's no need
-	// to check the connection to a dcrd instance.
-	if conn == nil {
+	// When running in embedded wallet mode with spv on, we only support
+	// running in dcrw mode.
+	if conn == nil && cfg.Dcrwallet.SPV && cfg.Node != "dcrw" {
+		return nil, fmt.Errorf("embedded wallet in SPV mode only " +
+			"supports 'node=dcrw' config")
+	}
+
+	// We only require a dcrd connection when running in embedded mode and
+	// not in SPV mode.
+	needsDcrd := conn == nil && !cfg.Dcrwallet.SPV
+	if needsDcrd {
 		// Load dcrd's TLS cert for the RPC connection.  If a raw cert
 		// was specified in the config, then we'll set that directly.
 		// Otherwise, we attempt to read the cert from the path
@@ -331,13 +335,22 @@ func newChainControlFromConfig(cfg *Config, localDB, remoteDB *channeldb.DB,
 		cc.chainIO = wc
 
 	default:
-		// Initialize an RPC syncer for this wallet and use it as
-		// blockchain IO source.
-		//
-		// We don't currently support running an embedded wallet in spv
-		// mode.
-		syncer, err := dcrwallet.NewRPCSyncer(*rpcConfig,
-			activeNetParams.Params)
+		// Initialize the appropriate syncer.
+		var syncer dcrwallet.WalletSyncer
+		switch cfg.Dcrwallet.SPV {
+		case false:
+			syncer, err = dcrwallet.NewRPCSyncer(*rpcConfig,
+				activeNetParams.Params)
+		case true:
+			spvCfg := &dcrwallet.SPVSyncerConfig{
+				Peers: cfg.Dcrwallet.SPVConnect,
+				Net:   activeNetParams.Params,
+				AppDataDir: filepath.Join(cfg.DataDir,
+					activeNetParams.Params.Name),
+			}
+			syncer, err = dcrwallet.NewSPVSyncer(spvCfg)
+		}
+
 		if err != nil {
 			return nil, err
 		}

go.mod

@@ -9,7 +9,8 @@ require (
 	github.com/Yawning/aez v0.0.0-20180408160647-ec7426b44926
 	github.com/btcsuite/btcwallet/walletdb v1.3.3
 	github.com/davecgh/go-spew v1.1.1
-	github.com/decred/dcrd v1.2.1-0.20210804150906-75b0f68a3583
+	github.com/decred/dcrd v1.2.1-0.20210816181553-5444fa50b93d
+	github.com/decred/dcrd/addrmgr/v2 v2.0.0-20210802141345-893802fc06b0
 	github.com/decred/dcrd/bech32 v1.1.1
 	github.com/decred/dcrd/blockchain/stake/v4 v4.0.0-20210802141345-893802fc06b0
 	github.com/decred/dcrd/blockchain/standalone/v2 v2.0.1-0.20210802141345-893802fc06b0

go.sum

@@ -87,9 +87,10 @@ github.com/dchest/siphash v1.2.2 h1:9DFz8tQwl9pTVt5iok/9zKyzA1Q6bRGiF3HPiEEVr9I=
 github.com/dchest/siphash v1.2.2/go.mod h1:q+IRvb2gOSrUnYoPqHiyHXS0FOBBOdl6tONBlVnOnt4=
 github.com/decred/base58 v1.0.3 h1:KGZuh8d1WEMIrK0leQRM47W85KqCAdl2N+uagbctdDI=
 github.com/decred/base58 v1.0.3/go.mod h1:pXP9cXCfM2sFLb2viz2FNIdeMWmZDBKG3ZBYbiSM78E=
-github.com/decred/dcrd v1.2.1-0.20210804150906-75b0f68a3583 h1:xNkbTlCNvRDpQzTwGUzJaH2iGOytZUWOmM2QSSGNopU=
-github.com/decred/dcrd v1.2.1-0.20210804150906-75b0f68a3583/go.mod h1:l5gFATzw76g1hCzI+7ARrUQ0E97EjZoWVHVtFVeK7xE=
+github.com/decred/dcrd v1.2.1-0.20210816181553-5444fa50b93d h1:ahvZdNexK6bKOg0+4mwBYQvu8QOCUoSsgYMMEcBXvaM=
+github.com/decred/dcrd v1.2.1-0.20210816181553-5444fa50b93d/go.mod h1:l5gFATzw76g1hCzI+7ARrUQ0E97EjZoWVHVtFVeK7xE=
 github.com/decred/dcrd/addrmgr/v2 v2.0.0-20210525214639-70483c835b7f/go.mod h1:VKxDZPQTcFkUhJjEsr+dcnExKlp8qdS4sUR5XUsonwg=
+github.com/decred/dcrd/addrmgr/v2 v2.0.0-20210802141345-893802fc06b0 h1:DqSiZEVMXiFCk7L70UPjmGSBvNHtxuS6BoPnXa+siJQ=
 github.com/decred/dcrd/addrmgr/v2 v2.0.0-20210802141345-893802fc06b0/go.mod h1:VKxDZPQTcFkUhJjEsr+dcnExKlp8qdS4sUR5XUsonwg=
 github.com/decred/dcrd/bech32 v1.1.1 h1:/+ab5lHgL99vUvxRzf8x7EMznKrv2stwpShB8dPHu7w=
 github.com/decred/dcrd/bech32 v1.1.1/go.mod h1:5Eng/MFsKR8KKDeSxGZdYpGs8CIKxiedcqYddVqQuj0=
@@ -121,6 +122,7 @@ github.com/decred/dcrd/chaincfg/v3 v3.0.1-0.20210802141345-893802fc06b0/go.mod h
 github.com/decred/dcrd/connmgr v1.1.0 h1:JtKI3XjHOlJktaoZupxz8FKEKj/dqGFYJOF+vD/4ydQ=
 github.com/decred/dcrd/connmgr v1.1.0/go.mod h1:LepSJ1qu+cnY6nmUdiVUsX/NTFcd73FNEegrY7wuEpU=
 github.com/decred/dcrd/connmgr/v3 v3.0.0/go.mod h1:cPI43Aggp1lOhrVG75eJ3c3BwuFx0NhT77FK34ky+ak=
+github.com/decred/dcrd/connmgr/v3 v3.0.1-0.20210525214639-70483c835b7f h1:BVwdQuI+koNW1Gac4Nh+A92xI/9s5WDC158Bo6Iqcbg=
 github.com/decred/dcrd/connmgr/v3 v3.0.1-0.20210525214639-70483c835b7f/go.mod h1:cPI43Aggp1lOhrVG75eJ3c3BwuFx0NhT77FK34ky+ak=
 github.com/decred/dcrd/container/apbf v1.0.0/go.mod h1:E2SwKtx92yZCll2avNBANNGvv9+MZ8mWI8hafwxOLmk=
 github.com/decred/dcrd/crypto/blake256 v1.0.0/go.mod h1:sQl2p6Y26YV+ZOcSTP6thNdn47hh8kt6rqSlvmrXFAc=

internal/testutils/remotewallet.go

@@ -55,15 +55,46 @@ func nextAvailablePort() int {
 	panic("no ports available for listening")
 }
 
-func consumeSyncMsgs(syncStream pb.WalletLoaderService_RpcSyncClient, onSyncedChan chan struct{}) {
+type rpcSyncer struct {
+	c pb.WalletLoaderService_RpcSyncClient
+}
+
+func (r *rpcSyncer) RecvSynced() (bool, error) {
+	msg, err := r.c.Recv()
+	if err != nil {
+		// All errors are final here.
+		return false, err
+	}
+	return msg.Synced, nil
+}
+
+type spvSyncer struct {
+	c pb.WalletLoaderService_SpvSyncClient
+}
+
+func (r *spvSyncer) RecvSynced() (bool, error) {
+	msg, err := r.c.Recv()
+	if err != nil {
+		// All errors are final here.
+		return false, err
+	}
+	return msg.Synced, nil
+}
+
+type syncer interface {
+	RecvSynced() (bool, error)
+}
+
+func consumeSyncMsgs(syncStream syncer, onSyncedChan chan struct{}) {
 	for {
-		msg, err := syncStream.Recv()
+		synced, err := syncStream.RecvSynced()
 		if err != nil {
 			// All errors are final here.
 			return
 		}
-		if msg.Synced {
+		if synced {
 			onSyncedChan <- struct{}{}
+			return
 		}
 	}
 }
@@ -81,13 +112,21 @@ func tlsCertFromFile(fname string) (*x509.CertPool, error) {
 	return cp, nil
 }
 
+type SPVConfig struct {
+	Address string
+}
+
+// NewCustomTestRemoteDcrwallet runs a dcrwallet instance for use during tests.
 func NewCustomTestRemoteDcrwallet(t TB, nodeName, dataDir string,
 	hdSeed, privatePass []byte,
-	dcrd *rpcclient.ConnConfig) (*grpc.ClientConn, func()) {
+	dcrd *rpcclient.ConnConfig, spv *SPVConfig) (*grpc.ClientConn, func()) {
 
-	// Save the dcrd ca file in the wallet dir.
-	cafile := path.Join(dataDir, "ca.cert")
-	ioutil.WriteFile(cafile, dcrd.Certificates, 0644)
+	if dcrd == nil && spv == nil {
+		t.Fatalf("either dcrd or spv config needs to be specified")
+	}
+	if dcrd != nil && spv != nil {
+		t.Fatalf("only one of dcrd or spv config needs to be specified")
+	}
 
 	tlsCertPath := path.Join(dataDir, "rpc.cert")
 	tlsKeyPath := path.Join(dataDir, "rpc.key")
@@ -99,10 +138,6 @@ func NewCustomTestRemoteDcrwallet(t TB, nodeName, dataDir string,
 	args := []string{
 		"--noinitialload",
 		"--debuglevel=debug",
-		"--rpcconnect=" + dcrd.Host,
-		"--username=" + dcrd.User,
-		"--password=" + dcrd.Pass,
-		"--cafile=" + cafile,
 		"--simnet",
 		"--nolegacyrpc",
 		"--grpclisten=" + addr,
@@ -202,17 +237,32 @@ func NewCustomTestRemoteDcrwallet(t TB, nodeName, dataDir string,
 		t.Fatalf("unable to create wallet: %v", err)
 	}
 
-	// Run the rpc syncer.
-	req := &pb.RpcSyncRequest{
-		NetworkAddress:    dcrd.Host,
-		Username:          dcrd.User,
-		Password:          []byte(dcrd.Pass),
-		Certificate:       dcrd.Certificates,
-		DiscoverAccounts:  true,
-		PrivatePassphrase: privatePass,
-	}
 	ctxSync, cancelSync := context.WithCancel(context.Background())
-	syncStream, err := loader.RpcSync(ctxSync, req)
+	var syncStream syncer
+	if dcrd != nil {
+		// Run the rpc syncer.
+		req := &pb.RpcSyncRequest{
+			NetworkAddress:    dcrd.Host,
+			Username:          dcrd.User,
+			Password:          []byte(dcrd.Pass),
+			Certificate:       dcrd.Certificates,
+			DiscoverAccounts:  true,
+			PrivatePassphrase: privatePass,
+		}
+		var res pb.WalletLoaderService_RpcSyncClient
+		res, err = loader.RpcSync(ctxSync, req)
+		syncStream = &rpcSyncer{c: res}
+	} else if spv != nil {
+		// Run the spv syncer.
+		req := &pb.SpvSyncRequest{
+			SpvConnect:        []string{spv.Address},
+			DiscoverAccounts:  true,
+			PrivatePassphrase: privatePass,
+		}
+		var res pb.WalletLoaderService_SpvSyncClient
+		res, err = loader.SpvSync(ctxSync, req)
+		syncStream = &spvSyncer{c: res}
+	}
 	if err != nil {
 		cancelSync()
 		t.Fatalf("error running rpc sync: %v", err)
@@ -290,7 +340,7 @@ func NewTestRemoteDcrwallet(t TB, dcrd *rpcclient.ConnConfig) (*grpc.ClientConn,
 
 	var seed [32]byte
 	c, tearDownWallet := NewCustomTestRemoteDcrwallet(t, "remotedcrw", tempDir,
-		seed[:], []byte("pass"), dcrd)
+		seed[:], []byte("pass"), dcrd, nil)
 	tearDown := func() {
 		tearDownWallet()
 

lncfg/dcrwallet.go

@@ -6,4 +6,7 @@ type DcrwalletConfig struct {
 	AccountNumber  int32  `long:"accountnumber" description:"The account number that dcrlnd should take control of for all onchain operations and offchain key derivation."`
 	ClientKeyPath  string `long:"clientkeypath" description:"The file containing a client private key to use when connecting to a remote wallet"`
 	ClientCertPath string `long:"clientcertpath" description:"The file containing the client certificate to use when connecting to a remote wallet"`
+
+	SPV        bool     `long:"spv" description:"Whether to use SPV mode when using an embedded wallet"`
+	SPVConnect []string `long:"spvconnect" description:"Addresses to connect to when using spv mode"`
 }

lntest/spv.go

@@ -35,7 +35,8 @@ type SpvBackendConfig struct {
 // using this node as a chain backend.
 func (b SpvBackendConfig) GenArgs() []string {
 	return []string{
-		"--node=dcrw",
+		"--dcrwallet.spv",
+		"--dcrwallet.spvconnect=" + b.harness.P2PAddress(),
 	}
 }
 

lnwallet/dcrwallet/driver.go

@@ -11,9 +11,7 @@ const (
 )
 
 var (
-	// We currently only support full chain sync (no spv), so hardcode
-	// the available backends.
-	availableBackends = []string{"dcrd"}
+	availableBackends = []string{"dcrd", "spv"}
 )
 
 // createNewWallet creates a new instance of DcrWallet given the proper list of

lnwallet/dcrwallet/log.go

@@ -2,6 +2,8 @@ package dcrwallet
 
 import (
 	"decred.org/dcrwallet/v2/chain"
+	"decred.org/dcrwallet/v2/p2p"
+	"decred.org/dcrwallet/v2/spv"
 	base "decred.org/dcrwallet/v2/wallet"
 	"decred.org/dcrwallet/v2/wallet/udb"
 	"github.com/decred/dcrlnd/build"
@@ -33,5 +35,7 @@ func UseLogger(logger slog.Logger) {
 	base.UseLogger(logger)
 	loader.UseLogger(logger)
 	chain.UseLogger(logger)
+	spv.UseLogger(logger)
+	p2p.UseLogger(logger)
 	udb.UseLogger(logger)
 }

lnwallet/dcrwallet/rpcsync.go

@@ -59,7 +59,7 @@ func (s *RPCSyncer) start(w *DcrWallet) error {
 
 			syncer := chain.NewSyncer(w.wallet, &chainRpcOpts)
 			syncer.SetCallbacks(&chain.Callbacks{
-				Synced: w.onRPCSyncerSynced,
+				Synced: w.onSyncerSynced,
 			})
 
 			dcrwLog.Debugf("Starting rpc syncer")