Skip to content

Commit

Permalink
Merge pull request bitpay#1947 from micahriggan/feature/repair-script
Browse files Browse the repository at this point in the history 
feature(node): repair script
  • Loading branch information
@micahriggan
micahriggan authored Jan 25, 2019
2 parents 4b4ceaf + 88ed03a commit 0dd7303
Show file tree
Hide file tree
Showing 7 changed files with 1,759 additions and 1,716 deletions.
26 changes: 13 additions & 13 deletions package-lock.json
View file

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

7 changes: 4 additions & 3 deletions packages/bitcore-node/package.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -27,9 +27,10 @@
"test": "npm run tsc && node ./build/test/runners/unit.js && node ./build/test/runners/integration.js",
"test:ci": "npm run test",
"test:ci:integration": "npm run test:integration",
"repair:db": "npm run tsc && node ./build/test/verification/db-repair.js",
"verify:rpc": "npm run tsc && node ./build/test/verification/rpc-verify.js",
"verify:db": "npm run tsc && node ./build/test/verification/db-verify.js",
"resync":"npm run tsc && node ./build/test/verification/resync.js",
"resync": "npm run tsc && node ./build/test/verification/resync.js",
"test:unit": "npm run tsc && node ./build/test/runners/unit.js",
"test:integration": "npm run tsc && node ./build/test/runners/integration.js",
"benchmark": "npm run tsc && node build/test/benchmark/benchmark.js"
Expand Down Expand Up @@ -95,8 +96,8 @@
"socket.io-client": "^2.1.1",
"through2": "^2.0.3",
"underscore": "^1.8.3",
"web3": "^1.0.0-beta.34",
"web3-eth": "^1.0.0-beta.34",
"web3": "1.0.0-beta.34",
"web3-eth": "1.0.0-beta.34",
"winston": "^2.3.9"
}
}
6 changes: 5 additions & 1 deletion packages/bitcore-node/src/services/p2p.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -366,9 +366,13 @@ export class P2pWorker {
logger.info(`${chain}:${network} up to date.`);
break;
}
logger.info(`Re-Syncing ${headers.length} blocks for ${chain} ${network}`);
const headerCount = Math.min(headers.length, to - currentHeight);
logger.info(`Re-Syncing ${headerCount} blocks for ${chain} ${network}`);
let lastLog = Date.now();
for (let header of headers) {
if (currentHeight > to) {
break;
}
const block = await this.getBlock(header.hash);
await BlockStorage.processBlock({ chain, network, block, initialSyncComplete: true });
currentHeight++;
Expand Down
65 changes: 49 additions & 16 deletions packages/bitcore-node/test/verification/db-repair.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,16 +3,22 @@ import fs from 'fs';
import { Transform } from 'stream';
import { CoinStorage } from '../../src/models/coin';
import { Storage } from '../../src/services/storage';
import { P2pWorker } from '../../src/services/p2p';
import { Config } from '../../src/services/config';
import { BlockStorage } from '../../src/models/block';

(async () => {
const { CHAIN, NETWORK, FILE, DRYRUN } = process.env;
if (!CHAIN || !NETWORK || !FILE) {
console.log('CHAIN, NETWORK, and FILE env variable are required');
process.exit(1);
}

const chain = CHAIN;
const network = NETWORK;
const chain = CHAIN || '';
const network = NETWORK || '';
await Storage.start();
const chainConfig = Config.chainConfig({ chain, network });
const worker = new P2pWorker({ chain, network, chainConfig });
await worker.start();
const handleRepair = async data => {
switch (data.type) {
case 'DUPE_COIN':
Expand Down Expand Up @@ -46,6 +52,45 @@ import { Storage } from '../../src/services/storage';
});
}
break;
case 'MISSING_TX':
case 'MISSING_COIN_FOR_TXID':
case 'VALUE_MISMATCH':
case 'NEG_FEE':
const blockHeight = Number(data.payload.blockNum);
if (DRYRUN) {
console.log('WOULD RESYNC BLOCKS', blockHeight, 'to', blockHeight + 1);
console.log(data.payload);
} else {
console.log('Resyncing Blocks', blockHeight, 'to', blockHeight + 1);
await worker.resync(blockHeight - 1, blockHeight + 1);
}
break;
case 'DUPE_BLOCKHEIGHT':
case 'DUPE_BLOCKHASH':
const dupeBlock = await BlockStorage.collection
.find({ chain, network, height: data.payload.blockNum })
.toArray();

if (dupeBlock.length < 2) {
console.log('No action required.', dupeBlock.length, 'block');
return;
}

let toKeepBlock = dupeBlock[0];
const wouldBeDeletedBlock = dupeBlock.filter(c => c._id !== toKeepBlock._id);

if (DRYRUN) {
console.log('WOULD DELETE');
console.log(wouldBeDeletedBlock);
} else {
console.log('Deleting', wouldBeDeletedBlock.length, 'block');
await BlockStorage.collection.deleteMany({
chain,
network,
_id: { $in: wouldBeDeletedBlock.map(c => c._id) }
});
}
break;
default:
console.log('skipping');
}
Expand All @@ -64,9 +109,7 @@ import { Storage } from '../../src/services/storage';
console.log('Inspecting...');
console.log(dataStr);
await handleRepair(parsedData);
} catch (err) {
//console.log(err);
}
} catch (err) {}
}
}
}
Expand Down Expand Up @@ -94,14 +137,4 @@ import { Storage } from '../../src/services/storage';
};

getFileContents(FILE);

//type: 'DUPE_BLOCKHASH'
//type: 'NEG_FEE'
//type: 'DUPE_COIN'
//type: 'MISSING_COIN_FOR_TXID'
//type: 'MISSING_TX'
//type: 'VALUE_MISMATCH'
//type: 'DUPE_BLOCKHEIGHT'
//
// will need to handle each of those error types
})();
31 changes: 19 additions & 12 deletions packages/bitcore-node/test/verification/db-verify.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,7 @@ import { Storage } from '../../src/services/storage';
for (let tx of blockTxs) {
if (tx.fee < 0) {
allGood = false;
const error = { model: 'transaction', err: false, type: 'NEG_FEE', payload: tx };
const error = { model: 'transaction', err: false, type: 'NEG_FEE', payload: { tx, blockNum } };
console.log(JSON.stringify(error));
}
seenTxs[tx.txid] = tx;
Expand All @@ -48,7 +48,7 @@ import { Storage } from '../../src/services/storage';
const coins = seenCoins[txid];
if (!coins) {
allGood = false;
const error = { model: 'coin', err: false, type: 'MISSING_COIN_FOR_TXID', payload: txid };
const error = { model: 'coin', err: false, type: 'MISSING_COIN_FOR_TXID', payload: { txid, blockNum } };
console.log(JSON.stringify(error));
}
}
Expand All @@ -58,13 +58,18 @@ import { Storage } from '../../src/services/storage';
const coins = seenCoins[txid];
if (!tx) {
allGood = false;
const error = { model: 'transaction', err: false, type: 'MISSING_TX', payload: tx };
const error = { model: 'transaction', err: false, type: 'MISSING_TX', payload: { txid, blockNum } };
console.log(JSON.stringify(error));
} else {
const sum = Object.values(coins).reduce((prev, cur) => prev + cur.value, 0);
if (sum != tx.value) {
allGood = false;
const error = { model: 'coin+transactions', err: false, type: 'VALUE_MISMATCH', payload: { tx, coins } };
const error = {
model: 'coin+transactions',
err: false,
type: 'VALUE_MISMATCH',
payload: { tx, coins, blockNum }
};
console.log(JSON.stringify(error));
}
}
Expand All @@ -78,17 +83,19 @@ import { Storage } from '../../src/services/storage';
model: 'block',
err: false,
type: 'DUPE_BLOCKHEIGHT',
payload: { height: blockNum, blocksForHeight }
payload: { blockNum, blocksForHeight }
};
console.error(JSON.stringify(error));
console.log(JSON.stringify(error));
}
//blocks with same hash
const hashFromTx = blockTxs[0].blockHash;
const blocksForHash = await BlockStorage.collection.countDocuments({ chain, network, hash: hashFromTx });
if (blocksForHash !== 1) {
allGood = false;
const error = { model: 'block', err: false, type: 'DUPE_BLOCKHASH', payload: { hash: hashFromTx } };
console.error(JSON.stringify(error));
if (blockTxs.length > 0) {
const hashFromTx = blockTxs[0].blockHash;
const blocksForHash = await BlockStorage.collection.countDocuments({ chain, network, hash: hashFromTx });
if (blocksForHash !== 1) {
allGood = false;
const error = { model: 'block', err: false, type: 'DUPE_BLOCKHASH', payload: { hash: hashFromTx, blockNum } };
console.log(JSON.stringify(error));
}
}

return allGood;
Expand Down
Loading

0 comments on commit 0dd7303

Please sign in to comment.