From f48a8511e37b6b99b839f7657aae479dc8b69973 Mon Sep 17 00:00:00 2001 From: Braydon Fuller Date: Thu, 16 Jan 2020 17:25:02 -0800 Subject: [PATCH 01/12] wallet: set tip after txs have been added --- lib/wallet/walletdb.js | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/lib/wallet/walletdb.js b/lib/wallet/walletdb.js index 746ff8679..05484eb60 100644 --- a/lib/wallet/walletdb.js +++ b/lib/wallet/walletdb.js @@ -2062,12 +2062,12 @@ class WalletDB extends EventEmitter { return 0; } - // Sync the state to the new tip. - await this.setTip(tip); - if (this.options.checkpoints && !this.state.marked) { - if (tip.height <= this.network.lastCheckpoint) + if (tip.height <= this.network.lastCheckpoint) { + // Sync the state to the new tip. + await this.setTip(tip); return 0; + } } let total = 0; @@ -2077,6 +2077,9 @@ class WalletDB extends EventEmitter { total += 1; } + // Sync the state to the new tip. + await this.setTip(tip); + if (total > 0) { this.logger.info('Connected WalletDB block %x (tx=%d).', tip.hash, total); From 6cdd72fa2d44cd37fe83bedf5437532d77c6b800 Mon Sep 17 00:00:00 2001 From: Braydon Fuller Date: Tue, 21 Jan 2020 15:14:24 -0800 Subject: [PATCH 02/12] test: add corruption test for wallet tip --- test/wallet-test.js | 62 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 62 insertions(+) diff --git a/test/wallet-test.js b/test/wallet-test.js index c9ca86e2d..0135837bb 100644 --- a/test/wallet-test.js +++ b/test/wallet-test.js @@ -1816,6 +1816,68 @@ describe('Wallet', function() { wallet.network.txStart = ACTUAL_TXSTART; wdb.height = ACTUAL_HEIGHT; } + + describe('Corruption', function() { + let workers = null; + let wdb = null; + + beforeEach(async () => { + workers = new WorkerPool({ + enabled: true, + size: 2 + }); + + wdb = new WalletDB({ workers }); + await workers.open(); + await wdb.open(); + }); + + afterEach(async () => { + await wdb.close(); + await workers.close(); + }); + + it('should not write tip with error in txs', async () => { + const alice = await wdb.create(); + const addr = await alice.receiveAddress(); + + const fund = new MTX(); + fund.addInput(dummyInput()); + fund.addOutput(addr, 5460 * 10); + + wdb._addTX = async () => { + throw new Error('Some assertion.'); + }; + + await assert.rejects(async () => { + await wdb.addBlock(nextBlock(wdb), [fund.toTX()]); + }, { + message: 'Some assertion.' + }); + + assert.equal(wdb.height, 0); + + const bal = await alice.getBalance(); + assert.equal(bal.confirmed, 0); + assert.equal(bal.unconfirmed, 0); + }); + + it('should write tip without error in txs', async () => { + const alice = await wdb.create(); + const addr = await alice.receiveAddress(); + + const fund = new MTX(); + fund.addInput(dummyInput()); + const amount = 5460 * 10; + fund.addOutput(addr, amount); + + await wdb.addBlock(nextBlock(wdb), [fund.toTX()]); + + assert.equal(wdb.height, 1); + + const bal = await alice.getBalance(); + assert.equal(bal.confirmed, amount); + assert.equal(bal.unconfirmed, amount); }); }); From 29b14cf3de436c12042f4e2b95f39c6877df1746 Mon Sep 17 00:00:00 2001 From: Braydon Fuller Date: Mon, 20 Jan 2020 12:02:44 -0800 Subject: [PATCH 03/12] wallet: prevent out-of-memory during sync with wallet as plugin --- lib/wallet/nodeclient.js | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/lib/wallet/nodeclient.js b/lib/wallet/nodeclient.js index c5a7ab115..a0684ebb0 100644 --- a/lib/wallet/nodeclient.js +++ b/lib/wallet/nodeclient.js @@ -36,18 +36,18 @@ class NodeClient extends AsyncEmitter { */ init() { - this.node.on('connect', (entry, block) => { + this.node.chain.on('connect', async (entry, block) => { if (!this.opened) return; - this.emit('block connect', entry, block.txs); + await this.emitAsync('block connect', entry, block.txs); }); - this.node.on('disconnect', (entry, block) => { + this.node.chain.on('disconnect', async (entry, block) => { if (!this.opened) return; - this.emit('block disconnect', entry); + await this.emitAsync('block disconnect', entry); }); this.node.on('tx', (tx) => { From 8526a593266779a9bfcb411de2163d3d67ff75fa Mon Sep 17 00:00:00 2001 From: Braydon Fuller Date: Wed, 22 Jan 2020 16:11:59 -0800 Subject: [PATCH 04/12] node: remove unused code for spvnode --- lib/node/spvnode.js | 87 ++------------------------------------------- 1 file changed, 2 insertions(+), 85 deletions(-) diff --git a/lib/node/spvnode.js b/lib/node/spvnode.js index 30a7112b6..d3c86013b 100644 --- a/lib/node/spvnode.js +++ b/lib/node/spvnode.js @@ -7,7 +7,6 @@ 'use strict'; const assert = require('bsert'); -const {Lock} = require('bmutex'); const NameState = require('../covenants/namestate'); const Chain = require('../blockchain/chain'); const Pool = require('../net/pool'); @@ -113,10 +112,6 @@ class SPVNode extends Node { noUnbound: this.config.bool('rs-no-unbound') }); - this.rescanJob = null; - this.scanLock = new Lock(); - this.watchLock = new Lock(); - this.init(); } @@ -134,9 +129,6 @@ class SPVNode extends Node { this.http.on('error', err => this.error(err)); this.pool.on('tx', (tx) => { - if (this.rescanJob) - return; - this.emit('tx', tx); }); @@ -145,15 +137,6 @@ class SPVNode extends Node { }); this.chain.on('connect', async (entry, block) => { - if (this.rescanJob) { - try { - await this.watchBlock(entry, block); - } catch (e) { - this.error(e); - } - return; - } - this.emit('connect', entry, block); }); @@ -221,64 +204,11 @@ class SPVNode extends Node { * Scan for any missed transactions. * Note that this will replay the blockchain sync. * @param {Number|Hash} start - Start block. - * @param {Bloom} filter - * @param {Function} iter - Iterator. - * @returns {Promise} - */ - - async scan(start, filter, iter) { - const unlock = await this.scanLock.lock(); - const height = this.chain.height; - - try { - await this.chain.replay(start); - - if (this.chain.height < height) { - // We need to somehow defer this. - // await this.connect(); - // this.startSync(); - // await this.watchUntil(height, iter); - } - } finally { - unlock(); - } - } - - /** - * Watch the blockchain until a certain height. - * @param {Number} height - * @param {Function} iter - * @returns {Promise} - */ - - watchUntil(height, iter) { - return new Promise((resolve, reject) => { - this.rescanJob = new RescanJob(resolve, reject, height, iter); - }); - } - - /** - * Handled watched block. - * @param {ChainEntry} entry - * @param {MerkleBlock} block * @returns {Promise} */ - async watchBlock(entry, block) { - const unlock = await this.watchLock.lock(); - try { - if (entry.height < this.rescanJob.height) { - await this.rescanJob.iter(entry, block.txs); - return; - } - this.rescanJob.resolve(); - this.rescanJob = null; - } catch (e) { - this.rescanJob.reject(e); - this.rescanJob = null; - } finally { - unlock(); - } + async scan(start) { + throw new Error('Not implemented.'); } /** @@ -428,19 +358,6 @@ class SPVNode extends Node { } } -/* - * Helpers - */ - -class RescanJob { - constructor(resolve, reject, height, iter) { - this.resolve = resolve; - this.reject = reject; - this.height = height; - this.iter = iter; - } -} - /* * Expose */ From c217297705a9e96d711228f731035811be7c0e7e Mon Sep 17 00:00:00 2001 From: Braydon Fuller Date: Thu, 23 Jan 2020 07:11:49 -0800 Subject: [PATCH 05/12] wallet: reset chain on rescan for spv --- lib/wallet/nodeclient.js | 3 +++ 1 file changed, 3 insertions(+) diff --git a/lib/wallet/nodeclient.js b/lib/wallet/nodeclient.js index a0684ebb0..bae7bd312 100644 --- a/lib/wallet/nodeclient.js +++ b/lib/wallet/nodeclient.js @@ -217,6 +217,9 @@ class NodeClient extends AsyncEmitter { */ async rescan(start) { + if (this.node.spv) + return this.node.chain.reset(start); + return this.node.chain.scan(start, this.filter, (entry, txs) => { return this.emitAsync('block rescan', entry, txs); }); From e7a32b95287489def9c5838fcd451c1103cb66ba Mon Sep 17 00:00:00 2001 From: Braydon Fuller Date: Thu, 23 Jan 2020 09:50:15 -0800 Subject: [PATCH 06/12] test: test for out-of-memory issue with wallet sync --- test/wallet-test.js | 61 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 61 insertions(+) diff --git a/test/wallet-test.js b/test/wallet-test.js index 0135837bb..bd9d65f55 100644 --- a/test/wallet-test.js +++ b/test/wallet-test.js @@ -10,6 +10,7 @@ const Network = require('../lib/protocol/network'); const util = require('../lib/utils/util'); const blake2b = require('bcrypto/lib/blake2b'); const random = require('bcrypto/lib/random'); +const FullNode = require('../lib/node/fullnode'); const WalletDB = require('../lib/wallet/walletdb'); const WorkerPool = require('../lib/workers/workerpool'); const Address = require('../lib/primitives/address'); @@ -22,6 +23,7 @@ const Script = require('../lib/script/script'); const policy = require('../lib/protocol/policy'); const HDPrivateKey = require('../lib/hd/private'); const Wallet = require('../lib/wallet/wallet'); +const {forValue} = require('./util/common'); const KEY1 = 'xprv9s21ZrQH143K3Aj6xQBymM31Zb4BVc7wxqfUhMZrzewdDVCt' + 'qUP9iWfcHgJofs25xbaUpCps9GDXj83NiWvQCAkWQhVj5J4CorfnpKX94AZ'; @@ -1816,6 +1818,8 @@ describe('Wallet', function() { wallet.network.txStart = ACTUAL_TXSTART; wdb.height = ACTUAL_HEIGHT; } + }); + }); describe('Corruption', function() { let workers = null; @@ -2174,4 +2178,61 @@ describe('Wallet', function() { assert.strictEqual(bal.clocked, value); }); }); + + describe('Node Integration', function() { + const ports = {p2p: 49331, node: 49332, wallet: 49333}; + let node, chain, miner, wdb = null; + + beforeEach(async () => { + node = new FullNode({ + memory: true, + network: 'regtest', + workers: true, + workersSize: 2, + plugins: [require('../lib/wallet/plugin')], + port: ports.p2p, + httpPort: ports.node, + env: { + 'HSD_WALLET_HTTP_PORT': ports.wallet.toString() + } + }); + + chain = node.chain; + miner = node.miner; + wdb = node.require('walletdb').wdb; + await node.open(); + }); + + afterEach(async () => { + await node.close(); + }); + + async function mineBlock(tip) { + const job = await miner.createJob(tip); + const block = await job.mineAsync(); + chain.add(block); + } + + it('should not stack in-memory block queue (oom)', async () => { + let height = 0; + + const addBlock = wdb.addBlock.bind(wdb); + wdb.addBlock = async (entry, txs) => { + await new Promise(resolve => setTimeout(resolve, 100)); + await addBlock(entry, txs); + }; + + async function raceForward() { + await mineBlock(); + + await forValue(node.chain, 'height', height + 1); + assert.equal(wdb.height, height); + + height += 1; + } + + for (let i = 0; i < 10; i++) + await raceForward(); + }); + }); }); From 9e2a14364d5d16feb5dc2338cf3814f73379f3cb Mon Sep 17 00:00:00 2001 From: Braydon Fuller Date: Wed, 22 Jan 2020 13:16:33 -0800 Subject: [PATCH 07/12] wallet: remove checkpoints that skip transactions --- lib/wallet/node.js | 1 - lib/wallet/plugin.js | 1 - lib/wallet/walletdb.js | 14 -------------- 3 files changed, 16 deletions(-) diff --git a/lib/wallet/node.js b/lib/wallet/node.js index 6665fc4a1..8c8450dc6 100644 --- a/lib/wallet/node.js +++ b/lib/wallet/node.js @@ -49,7 +49,6 @@ class WalletNode extends Node { memory: this.config.bool('memory'), maxFiles: this.config.uint('max-files'), cacheSize: this.config.mb('cache-size'), - checkpoints: this.config.bool('checkpoints'), wipeNoReally: this.config.bool('wipe-no-really'), spv: this.config.bool('spv') }); diff --git a/lib/wallet/plugin.js b/lib/wallet/plugin.js index 93f54f6c1..cd2e6e7f1 100644 --- a/lib/wallet/plugin.js +++ b/lib/wallet/plugin.js @@ -51,7 +51,6 @@ class Plugin extends EventEmitter { maxFiles: this.config.uint('max-files'), cacheSize: this.config.mb('cache-size'), witness: this.config.bool('witness'), - checkpoints: this.config.bool('checkpoints'), wipeNoReally: this.config.bool('wipe-no-really'), spv: node.spv }); diff --git a/lib/wallet/walletdb.js b/lib/wallet/walletdb.js index 05484eb60..b9495c232 100644 --- a/lib/wallet/walletdb.js +++ b/lib/wallet/walletdb.js @@ -2062,14 +2062,6 @@ class WalletDB extends EventEmitter { return 0; } - if (this.options.checkpoints && !this.state.marked) { - if (tip.height <= this.network.lastCheckpoint) { - // Sync the state to the new tip. - await this.setTip(tip); - return 0; - } - } - let total = 0; for (const tx of txs) { @@ -2302,7 +2294,6 @@ class WalletOptions { this.compression = true; this.spv = false; - this.checkpoints = false; this.wipeNoReally = false; if (options) @@ -2376,11 +2367,6 @@ class WalletOptions { this.spv = options.spv; } - if (options.checkpoints != null) { - assert(typeof options.checkpoints === 'boolean'); - this.checkpoints = options.checkpoints; - } - if (options.wipeNoReally != null) { assert(typeof options.wipeNoReally === 'boolean'); this.wipeNoReally = options.wipeNoReally; From b09d13fb1a1251af2c5c318fbb2613b74f2f39c0 Mon Sep 17 00:00:00 2001 From: Matthew Zipkin Date: Thu, 23 Jan 2020 16:14:00 -0500 Subject: [PATCH 08/12] wallet: remove unused `witness` option from plugin --- lib/wallet/plugin.js | 1 - 1 file changed, 1 deletion(-) diff --git a/lib/wallet/plugin.js b/lib/wallet/plugin.js index cd2e6e7f1..c0bdb585e 100644 --- a/lib/wallet/plugin.js +++ b/lib/wallet/plugin.js @@ -50,7 +50,6 @@ class Plugin extends EventEmitter { memory: this.config.bool('memory', node.memory), maxFiles: this.config.uint('max-files'), cacheSize: this.config.mb('cache-size'), - witness: this.config.bool('witness'), wipeNoReally: this.config.bool('wipe-no-really'), spv: node.spv }); From 5549c77b9fbe4b542be410303cf147adff0f44dc Mon Sep 17 00:00:00 2001 From: Braydon Fuller Date: Tue, 28 Jan 2020 14:57:24 -0800 Subject: [PATCH 09/12] test: check that confirmation count is correct --- test/wallet-test.js | 45 +++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 43 insertions(+), 2 deletions(-) diff --git a/test/wallet-test.js b/test/wallet-test.js index bd9d65f55..494083dda 100644 --- a/test/wallet-test.js +++ b/test/wallet-test.js @@ -5,6 +5,7 @@ 'use strict'; const assert = require('bsert'); +const {WalletClient} = require('hs-client'); const consensus = require('../lib/protocol/consensus'); const Network = require('../lib/protocol/network'); const util = require('../lib/utils/util'); @@ -2186,6 +2187,7 @@ describe('Wallet', function() { beforeEach(async () => { node = new FullNode({ memory: true, + apiKey: 'foo', network: 'regtest', workers: true, workersSize: 2, @@ -2210,7 +2212,7 @@ describe('Wallet', function() { async function mineBlock(tip) { const job = await miner.createJob(tip); const block = await job.mineAsync(); - chain.add(block); + return chain.add(block); } it('should not stack in-memory block queue (oom)', async () => { @@ -2226,7 +2228,7 @@ describe('Wallet', function() { await mineBlock(); await forValue(node.chain, 'height', height + 1); - assert.equal(wdb.height, height); + assert.equal(wdb.height, height + 1); height += 1; } @@ -2234,5 +2236,44 @@ describe('Wallet', function() { for (let i = 0; i < 10; i++) await raceForward(); }); + + it('should emit details with correct confirmation', async () => { + const wclient = new WalletClient({port: ports.wallet, apiKey: 'foo'}); + await wclient.open(); + + const info = await wclient.createWallet('test'); + const wallet = wclient.wallet('test', info.token); + await wallet.open(); + + const acct = await wallet.getAccount('default'); + const waddr = acct.receiveAddress; + + miner.addresses.length = 0; + miner.addAddress(waddr); + + let txCount = 0; + let confirmedCount = 0; + + wallet.on('tx', (details) => { + if (details.confirmations === 1) + txCount += 1; + }); + + wallet.on('confirmed', (details) => { + if (details.confirmations === 1) + confirmedCount += 1; + }); + + for (let i = 0; i < 101; i++) + await mineBlock(); + + await wallet.send({outputs: [{address: waddr, value: 1 * 1e8}]}); + await mineBlock(); + + await wclient.close(); + + assert.equal(txCount, 102); + assert.equal(confirmedCount, 1); + }); }); }); From 16322ee0d04b45700455d26df0ef66be9583d943 Mon Sep 17 00:00:00 2001 From: Braydon Fuller Date: Tue, 28 Jan 2020 14:58:17 -0800 Subject: [PATCH 10/12] wallet: use correct height when adding txs --- lib/wallet/http.js | 2 +- lib/wallet/walletdb.js | 38 ++++++++++++++++++++++++++++++++------ 2 files changed, 33 insertions(+), 7 deletions(-) diff --git a/lib/wallet/http.js b/lib/wallet/http.js index 32f1f26df..d9c7b57a8 100644 --- a/lib/wallet/http.js +++ b/lib/wallet/http.js @@ -1297,7 +1297,7 @@ class HTTP extends Server { if (!this.channel(name) && !this.channel('w:*')) return; - const json = details.getJSON(this.network, this.wdb.height); + const json = details.getJSON(this.network, this.wdb.liveHeight()); if (this.channel(name)) this.to(name, event, wallet.id, json); diff --git a/lib/wallet/walletdb.js b/lib/wallet/walletdb.js index b9495c232..e00ab1531 100644 --- a/lib/wallet/walletdb.js +++ b/lib/wallet/walletdb.js @@ -62,6 +62,7 @@ class WalletDB extends EventEmitter { this.primary = null; this.state = new ChainState(); + this.confirming = false; this.height = 0; this.wallets = new Map(); this.depth = 0; @@ -1657,6 +1658,22 @@ class WalletDB extends EventEmitter { this.height = state.height; } + /** + * Will return the current height and will increment + * to the current height of a block currently being + * added to the wallet. + * @returns {Number} + */ + + liveHeight() { + let height = this.height; + + if (this.confirming) + height += 1; + + return height; + } + /** * Mark current state. * @param {BlockMeta} block @@ -2064,13 +2081,22 @@ class WalletDB extends EventEmitter { let total = 0; - for (const tx of txs) { - if (await this._addTX(tx, tip)) - total += 1; - } + try { + // We set the state as confirming so that + // anything that uses the current height can + // increment by one until the block is fully + // added and the height is updated. + this.confirming = true; + for (const tx of txs) { + if (await this._addTX(tx, tip)) + total += 1; + } - // Sync the state to the new tip. - await this.setTip(tip); + // Sync the state to the new tip. + await this.setTip(tip); + } finally { + this.confirming = false; + } if (total > 0) { this.logger.info('Connected WalletDB block %x (tx=%d).', From c4609964a98a88c4fe669f228ac142f43b7b42ee Mon Sep 17 00:00:00 2001 From: Braydon Fuller Date: Tue, 4 Feb 2020 10:05:16 -0800 Subject: [PATCH 11/12] test: remove unnecessary api-key in test --- test/wallet-test.js | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/test/wallet-test.js b/test/wallet-test.js index 494083dda..d9fd3e88b 100644 --- a/test/wallet-test.js +++ b/test/wallet-test.js @@ -2187,7 +2187,6 @@ describe('Wallet', function() { beforeEach(async () => { node = new FullNode({ memory: true, - apiKey: 'foo', network: 'regtest', workers: true, workersSize: 2, @@ -2238,7 +2237,7 @@ describe('Wallet', function() { }); it('should emit details with correct confirmation', async () => { - const wclient = new WalletClient({port: ports.wallet, apiKey: 'foo'}); + const wclient = new WalletClient({port: ports.wallet}); await wclient.open(); const info = await wclient.createWallet('test'); From 3f653759cd1ed01dad9e1d9e759f770ec97fb5e9 Mon Sep 17 00:00:00 2001 From: Braydon Fuller Date: Tue, 4 Feb 2020 10:31:29 -0800 Subject: [PATCH 12/12] test: more assertions for tx and confirmed events --- test/wallet-test.js | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/test/wallet-test.js b/test/wallet-test.js index d9fd3e88b..f5c43b99b 100644 --- a/test/wallet-test.js +++ b/test/wallet-test.js @@ -2251,16 +2251,19 @@ describe('Wallet', function() { miner.addAddress(waddr); let txCount = 0; + let txConfirmedCount = 0; let confirmedCount = 0; wallet.on('tx', (details) => { if (details.confirmations === 1) + txConfirmedCount += 1; + else if (details.confirmations === 0) txCount += 1; }); wallet.on('confirmed', (details) => { - if (details.confirmations === 1) - confirmedCount += 1; + assert.equal(details.confirmations, 1); + confirmedCount += 1; }); for (let i = 0; i < 101; i++) @@ -2271,7 +2274,8 @@ describe('Wallet', function() { await wclient.close(); - assert.equal(txCount, 102); + assert.equal(txConfirmedCount, 102); + assert.equal(txCount, 1); assert.equal(confirmedCount, 1); }); });