diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md index 39cb993bf8..f14eaec343 100644 --- a/.github/pull_request_template.md +++ b/.github/pull_request_template.md @@ -15,6 +15,8 @@ Fixes Issue # . Insert **x** into the following checkboxes to confirm (eg. [x]): - [ ] Bug fix. - [ ] New feature. +- [ ] Enhancement. +- [ ] Unit test. - [ ] Breaking change (a fix or feature that causes existing functionality to not work as expected). - [ ] Requires documentation update. diff --git a/README.md b/README.md index 69e1910852..2fcfe82a1f 100644 --- a/README.md +++ b/README.md @@ -34,5 +34,7 @@ Please refer to the [wiki pages](https://github.com/aionnetwork/aion/wiki) for f Aion is released under the [GPL-V3 license](https://github.com/aionnetwork/aion/blob/dev/LICENSE) +### WE ARE HIRING! +If you are interested in being part of the Aion project, check out our available positions and apply [here](http://aion.humi.ca/job-board/it/697)! diff --git a/aion_api b/aion_api index 5a9c2e4468..75f2e8c6d5 160000 --- a/aion_api +++ b/aion_api @@ -1 +1 @@ -Subproject commit 5a9c2e44681d94354442fedb4f55e535af19312a +Subproject commit 75f2e8c6d530d6b9f14ecfc3dd9f7be75b9125ee diff --git a/aion_fastvm b/aion_fastvm index 4c442f63b5..9087a973f5 160000 --- a/aion_fastvm +++ b/aion_fastvm @@ -1 +1 @@ -Subproject commit 4c442f63b55241e057d3100dfecfa8c517597dac +Subproject commit 9087a973f595b7ae4c82c583cb80cbc46b4f4604 diff --git a/build.xml b/build.xml index 01b5a3a972..222a4681b0 100644 --- a/build.xml +++ b/build.xml @@ -279,6 +279,11 @@ + + + + + @@ -386,6 +391,9 @@ + + + diff --git a/modAion/build.xml b/modAion/build.xml index 8ca7563bef..2c8d6f244e 100644 --- a/modAion/build.xml +++ b/modAion/build.xml @@ -40,7 +40,7 @@ - @@ -51,7 +51,7 @@ - @@ -109,7 +109,7 @@ - @@ -130,31 +130,4 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/modAionBase/build.xml b/modAionBase/build.xml index 251c73bd31..6bf46710cc 100644 --- a/modAionBase/build.xml +++ b/modAionBase/build.xml @@ -37,7 +37,7 @@ debug="${compile.debug}" debuglevel="source,lines,vars" includeantruntime="false" - release="9" + release="10" srcdir="${main.src.dir}" destdir="${main.build.dir}" includes="**/*.java, module-info.java" @@ -72,7 +72,7 @@ debug="true" debuglevel="source,lines,vars" includeantruntime="false" - release="9" + release="10" srcdir="${test.src.dir}" destdir="${test.build.dir}" > @@ -86,7 +86,7 @@ debug="true" debuglevel="source,lines,vars" includeantruntime="false" - release="9" + release="10" srcdir="${test.src.dir}" destdir="${test.build.dir}" > @@ -106,19 +106,6 @@ /> - - - - - - - - - diff --git a/modAionBase/src/org/aion/base/db/IKeyValueStore.java b/modAionBase/src/org/aion/base/db/IKeyValueStore.java index 5f2f8c00ba..d480cd619d 100644 --- a/modAionBase/src/org/aion/base/db/IKeyValueStore.java +++ b/modAionBase/src/org/aion/base/db/IKeyValueStore.java @@ -1,4 +1,4 @@ -/******************************************************************************* +/* ****************************************************************************** * Copyright (c) 2017-2018 Aion foundation. * * This file is part of the aion network project. @@ -42,14 +42,11 @@ /** * Functionality for a key-value cache allowing itemized updates. * - * @param - * the data type of the keys - * @param - * the data type of the values + * @param the data type of the keys + * @param the data type of the values * @author Alexandra Roatis - * @implNote For the underlying DB connection, if [isClosed() == true], then all - * function calls which are documented to throw RuntimeException, will - * throw a RuntimeException. + * @implNote For the underlying DB connection, if [isClosed() == true], then all function calls + * which are documented to throw RuntimeException, will throw a RuntimeException. */ public interface IKeyValueStore extends AutoCloseable { @@ -60,8 +57,7 @@ public interface IKeyValueStore extends AutoCloseable { * Returns if the DB is empty or not. * * @return True if number of keys > 0, false otherwise - * @throws RuntimeException - * if the data store is closed + * @throws RuntimeException if the data store is closed */ boolean isEmpty(); @@ -69,23 +65,18 @@ public interface IKeyValueStore extends AutoCloseable { * Returns the set of keys for the database. * * @return Set of keys - * @throws RuntimeException - * if the data store is closed - * @apiNote Returns an empty set if the database keys could not be - * retrieved. + * @throws RuntimeException if the data store is closed + * @apiNote Returns an empty set if the database keys could not be retrieved. */ Set keys(); /** - * get retrieves a value from the database, returning an optional, it is - * fulfilled if a value was able to be retrieved from the DB, otherwise the - * optional is empty + * get retrieves a value from the database, returning an optional, it is fulfilled if a value + * was able to be retrieved from the DB, otherwise the optional is empty * * @param k - * @throws RuntimeException - * if the data store is closed - * @throws IllegalArgumentException - * if the key is null + * @throws RuntimeException if the data store is closed + * @throws IllegalArgumentException if the key is null */ Optional get(K k); @@ -93,44 +84,34 @@ public interface IKeyValueStore extends AutoCloseable { // ------------------------------------------------------------------------------------- /** - * Places or updates a value into the cache at the corresponding key. Makes - * no guarantees about when the value is actually inserted into the - * underlying data store. + * Places or updates a value into the cache at the corresponding key. Makes no guarantees about + * when the value is actually inserted into the underlying data store. * - * @param k - * the key for the new entry - * @param v - * the value for the new entry - * @throws RuntimeException - * if the underlying data store is closed - * @throws IllegalArgumentException - * if the key is null - * @implNote The choice of when to push the changes to the data store is - * left up to the implementation. + * @param k the key for the new entry + * @param v the value for the new entry + * @throws RuntimeException if the underlying data store is closed + * @throws IllegalArgumentException if the key is null + * @implNote The choice of when to push the changes to the data store is left up to the + * implementation. * @apiNote Put must have the following properties: - *
    - *
  1. Creates a new entry in the cache, if the key-value pair does - * not exist in the cache or underlying data store.
  2. - *
  3. Updates the entry in the cache when the key-value pair - * already exists. - *
  4. Deletes the entry when given a {@code null} value.
  5. - *
+ *
    + *
  1. Creates a new entry in the cache, if the key-value pair does not exist in the cache + * or underlying data store. + *
  2. Updates the entry in the cache when the key-value pair already exists. + *
  3. Deletes the entry when given a {@code null} value. + *
*/ void put(K k, V v); /** - * Delete an entry from the cache, marking it for deletion inside the data - * store. Makes no guarantees about when the value is actually deleted from - * the underlying data store. + * Delete an entry from the cache, marking it for deletion inside the data store. Makes no + * guarantees about when the value is actually deleted from the underlying data store. * - * @param k - * the key of the entry to be deleted - * @throws RuntimeException - * if the underlying data store is closed - * @throws IllegalArgumentException - * if the key is null - * @implNote The choice of when to push the changes to the data store is - * left up to the implementation. + * @param k the key of the entry to be deleted + * @throws RuntimeException if the underlying data store is closed + * @throws IllegalArgumentException if the key is null + * @implNote The choice of when to push the changes to the data store is left up to the + * implementation. */ void delete(K k); @@ -138,37 +119,41 @@ public interface IKeyValueStore extends AutoCloseable { // --------------------------------------------------------------------------------------------------- /** - * Puts or updates the data store with the given key-value pairs, as - * follows: + * Puts or updates the data store with the given key-value pairs, as follows: + * *
    - *
  • if the key is present in the data store, the stored - * value is overwritten
  • - *
  • if the key is not present in the data store, the new - * key-value pair is stored
  • - *
  • if the value is null, the matching stored key will be - * deleted from the data store, or
  • + *
  • if the key is present in the data store, the stored value is overwritten + *
  • if the key is not present in the data store, the new key-value pair is + * stored + *
  • if the value is null, the matching stored key will be deleted from the + * data store, or *
* - * @param inputMap - * a {@link Map} of key-value pairs to be updated in the database - * @throws RuntimeException - * if the data store is closed - * @throws IllegalArgumentException - * if the map contains a null key + * @param inputMap a {@link Map} of key-value pairs to be updated in the database + * @throws RuntimeException if the data store is closed + * @throws IllegalArgumentException if the map contains a null key */ void putBatch(Map inputMap); - void putToBatch(byte[] key, byte[] value); + void putToBatch(K key, V value); + void commitBatch(); /** * Similar to delete, except operates on a list of keys * * @param keys - * @throws RuntimeException - * if the data store is closed - * @throws IllegalArgumentException - * if the collection contains a null key + * @throws RuntimeException if the data store is closed + * @throws IllegalArgumentException if the collection contains a null key */ void deleteBatch(Collection keys); + + /** + * Checks that the data store connection is open. Throws a {@link RuntimeException} if the data + * store connection is closed. + * + * @implNote Always do this check after acquiring a lock on the class/data. Otherwise it might + * produce inconsistent results due to lack of synchronization. + */ + void check(); } diff --git a/modAionBase/src/org/aion/base/db/IPruneConfig.java b/modAionBase/src/org/aion/base/db/IPruneConfig.java new file mode 100644 index 0000000000..3b93b1fb50 --- /dev/null +++ b/modAionBase/src/org/aion/base/db/IPruneConfig.java @@ -0,0 +1,37 @@ +package org.aion.base.db; + +/** + * Interface for pruning configuration parameters. + * + * @author Alexandra Roatis + */ +public interface IPruneConfig { + + /** + * Indicates if pruning should be enabled or disabled. + * + * @return {@code true} when pruning enabled, {@code false} when pruning disabled. + */ + boolean isEnabled(); + + /** + * Indicates if archiving should be enabled or disabled. + * + * @return {@code true} when archiving enabled, {@code false} when archiving disabled. + */ + boolean isArchived(); + + /** + * @return the number of topmost blocks for which the full data should be maintained on disk. + */ + int getCurrentCount(); + + /** + * Gets the rate at which blocks should be archived (for which the full data should be + * maintained on disk). Blocks that are exact multiples of the returned value should be + * persisted on disk, regardless of other pruning. + * + * @return integer value representing the archive rate + */ + int getArchiveRate(); +} diff --git a/modAionBase/src/org/aion/base/db/IRepositoryConfig.java b/modAionBase/src/org/aion/base/db/IRepositoryConfig.java index 02671bd267..7a315bbfd9 100644 --- a/modAionBase/src/org/aion/base/db/IRepositoryConfig.java +++ b/modAionBase/src/org/aion/base/db/IRepositoryConfig.java @@ -1,4 +1,4 @@ -/******************************************************************************* +/* ****************************************************************************** * Copyright (c) 2017-2018 Aion foundation. * * This file is part of the aion network project. @@ -37,19 +37,17 @@ import java.util.Properties; /** - * Represents a configuration interface accepted that should be accepted by the - * repository to implement necessary configs + * Represents a configuration interface accepted that should be accepted by the repository to + * implement necessary configs * * @author yao */ public interface IRepositoryConfig { - /** - * @return absolute path to the DB folder containing files - */ + /** @return absolute path to the DB folder containing files */ String getDbPath(); - int getPrune(); + IPruneConfig getPruneConfig(); IContractDetails contractDetailsImpl(); diff --git a/modAionImpl/build.xml b/modAionImpl/build.xml index 382ec52054..c81b528ede 100644 --- a/modAionImpl/build.xml +++ b/modAionImpl/build.xml @@ -10,7 +10,7 @@ - + @@ -30,7 +30,6 @@ - @@ -42,6 +41,7 @@ +
@@ -56,7 +56,7 @@ - @@ -67,7 +67,7 @@ - @@ -139,7 +139,7 @@ - @@ -186,30 +186,4 @@
- - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/modAionImpl/module-info.java b/modAionImpl/module-info.java index 1c7aed72e9..bc189f890c 100644 --- a/modAionImpl/module-info.java +++ b/modAionImpl/module-info.java @@ -2,8 +2,6 @@ requires aion.base; requires aion.mcf; requires aion.log; - requires java.xml; - requires slf4j.api; requires aion.p2p; requires aion.p2p.impl; requires aion.rlp; @@ -15,6 +13,8 @@ requires aion.zero; requires aion.fastvm; requires jdk.management; + requires java.xml; + requires slf4j.api; exports org.aion.equihash; exports org.aion.zero.impl.blockchain; diff --git a/modAionImpl/src/org/aion/zero/impl/AionBlockchainImpl.java b/modAionImpl/src/org/aion/zero/impl/AionBlockchainImpl.java index 945253da13..de12445c31 100644 --- a/modAionImpl/src/org/aion/zero/impl/AionBlockchainImpl.java +++ b/modAionImpl/src/org/aion/zero/impl/AionBlockchainImpl.java @@ -1,4 +1,4 @@ -/******************************************************************************* +/* * Copyright (c) 2017-2018 Aion foundation. * * This file is part of the aion network project. @@ -19,8 +19,7 @@ * * Contributors: * Aion foundation. - ******************************************************************************/ - + */ package org.aion.zero.impl; import static java.lang.Math.max; @@ -122,7 +121,7 @@ public class AionBlockchainImpl implements IAionBlockchain { private A0BCConfig config; private long exitOn = Long.MAX_VALUE; - private IRepository repository; + private AionRepositoryImpl repository; private IRepositoryCache track; private TransactionStore transactionStore; private AionBlock bestBlock; @@ -139,8 +138,6 @@ public class AionBlockchainImpl implements IAionBlockchain { */ private volatile AionBlock pubBestBlock; - private volatile BigInteger pubBestTD = ZERO; - private volatile BigInteger totalDifficulty = ZERO; private ChainStatistics chainStats; @@ -221,7 +218,7 @@ private AionBlockchainImpl() { } protected AionBlockchainImpl(final A0BCConfig config, - final IRepository repository, + final AionRepositoryImpl repository, final ChainConfiguration chainConfig) { this.config = config; this.repository = repository; @@ -237,7 +234,7 @@ protected AionBlockchainImpl(final A0BCConfig config, this.parentHeaderValidator = this.chainConfiguration.createParentHeaderValidator(); this.blockHeaderValidator = this.chainConfiguration.createBlockHeaderValidator(); - this.transactionStore = ((AionRepositoryImpl) this.repository).getTransactionStore(); + this.transactionStore = this.repository.getTransactionStore(); this.minerCoinbase = this.config.getMinerCoinbase(); @@ -283,7 +280,7 @@ void setEventManager(IEventMgr eventManager) { } public AionBlockStore getBlockStore() { - return (AionBlockStore) repository.getBlockStore(); + return repository.getBlockStore(); } /** @@ -422,7 +419,7 @@ private static byte[] calcTxTrie(List transactions) { return txsState.getRootHash(); } - public IRepository getRepository() { + public AionRepositoryImpl getRepository() { return repository; } @@ -430,7 +427,7 @@ private State pushState(byte[] bestBlockHash) { State push = stateStack.push(new State()); this.bestBlock = getBlockStore().getBlockByHash(bestBlockHash); this.totalDifficulty = getBlockStore().getTotalDifficultyForHash(bestBlockHash); - this.repository = this.repository.getSnapshotTo(this.bestBlock.getStateRoot()); + this.repository = (AionRepositoryImpl) this.repository.getSnapshotTo(this.bestBlock.getStateRoot()); return push; } @@ -456,23 +453,16 @@ private AionBlockSummary tryConnectAndFork(final AionBlock block) { State savedState = pushState(block.getParentHash()); this.fork = true; - final AionBlockSummary summary; + AionBlockSummary summary = null; try { - // LOG.info("block " + block.toString()); - - // FIXME: adding block with no option for flush summary = add(block); - if (summary == null) { - return null; - } } catch (Throwable th) { LOG.error("Unexpected error: ", th); - return null; } finally { this.fork = false; } - if (isMoreThan(this.totalDifficulty, savedState.savedTD)) { + if (summary != null && isMoreThan(this.totalDifficulty, savedState.savedTD)) { if (LOG.isInfoEnabled()) LOG.info("branching: from = {}/{}, to = {}/{}", @@ -511,6 +501,17 @@ public boolean skipTryToConnect(long blockNumber) { return blockNumber > current + 32 || blockNumber < current - 32; } + /** + * Heuristic for skipping the call to tryToConnect with block number that was already pruned. + */ + public boolean isPruneRestricted(long blockNumber) { + // no restriction when not in TOP pruning mode + if (!repository.usesTopPruning()) { + return false; + } + return blockNumber < bestBlockNumber.get() - repository.getPruneBlockCount() + 1; + } + public synchronized ImportResult tryToConnect(final AionBlock block) { return tryToConnectInternal(block, System.currentTimeMillis() / THOUSAND_MS); } @@ -555,6 +556,7 @@ ImportResult tryToConnectInternal(final AionBlock block, long currTimeSeconds) { // to connect to the main chain final AionBlockSummary summary; if (bestBlock.isParentOf(block)) { + repository.syncToRoot(bestBlock.getStateRoot()); summary = add(block); ret = summary == null ? INVALID_BLOCK : IMPORTED_BEST; } else { @@ -572,7 +574,6 @@ ImportResult tryToConnectInternal(final AionBlock block, long currTimeSeconds) { // update best block reference if (ret == IMPORTED_BEST) { pubBestBlock = bestBlock; - pubBestTD = summary.getTotalDifficulty(); } // fire block events @@ -721,7 +722,7 @@ public synchronized AionBlockSummary add(AionBlock block) { List receipts = summary.getReceipts(); updateTotalDifficulty(block); - summary.setTotalDifficulty(getInternalTD()); + summary.setTotalDifficulty(block.getCumulativeDifficulty()); storeBlock(block, receipts); @@ -764,6 +765,7 @@ public synchronized AionBlockSummary add(AionBlock block, boolean rebuild) { LOG.warn("Block's given Receipt Hash doesn't match: {} != {}", receiptHash, receiptListHash); LOG.warn("Calculated receipts: " + receipts); } + track.rollback(); return null; } @@ -791,7 +793,7 @@ public synchronized AionBlockSummary add(AionBlock block, boolean rebuild) { track.rollback(); // block is bad so 'rollback' the state root to the original // state - ((AionRepositoryImpl) repository).setRoot(origRoot); + repository.setRoot(origRoot); } } @@ -804,7 +806,7 @@ public synchronized AionBlockSummary add(AionBlock block, boolean rebuild) { } transactionStore.flushBatch(); - ((AionRepositoryImpl) repository).commitBlock(block.getHeader()); + repository.commitBlock(block.getHeader()); if (LOG.isDebugEnabled()) LOG.debug("Block rebuilt: number: {}, hash: {}, TD: {}", block.getNumber(), block.getShortHash(), @@ -1099,7 +1101,7 @@ public synchronized void storeBlock(AionBlock block, List receipt } transactionStore.flushBatch(); - ((AionRepositoryImpl) repository).commitBlock(block.getHeader()); + repository.commitBlock(block.getHeader()); if (LOG.isDebugEnabled()) LOG.debug("Block saved: number: {}, hash: {}, TD: {}", block.getNumber(), block.getShortHash(), @@ -1139,7 +1141,7 @@ public synchronized void close() { @Override public BigInteger getTotalDifficulty() { - return pubBestTD; + return getBestBlock().getCumulativeDifficulty(); } // this method is for the testing purpose @@ -1153,6 +1155,7 @@ private BigInteger getInternalTD() { private void updateTotalDifficulty(AionBlock block) { totalDifficulty = totalDifficulty.add(block.getDifficultyBI()); + block.setCumulativeDifficulty(totalDifficulty); if (LOG.isDebugEnabled()) { LOG.debug("TD: updated to {}", totalDifficulty); } @@ -1160,11 +1163,10 @@ private void updateTotalDifficulty(AionBlock block) { @Override public void setTotalDifficulty(BigInteger totalDifficulty) { - this.pubBestTD = totalDifficulty; this.totalDifficulty = totalDifficulty; } - public void setRepository(IRepository repository) { + public void setRepository(AionRepositoryImpl repository) { this.repository = repository; } @@ -1353,7 +1355,7 @@ public List getListOfBodiesByHashes(List hashes) { private class State { - IRepository savedRepo = repository; + AionRepositoryImpl savedRepo = repository; AionBlock savedBest = bestBlock; BigInteger savedTD = totalDifficulty; } @@ -1384,11 +1386,14 @@ public synchronized boolean recoverWorldState(IRepository repository, AionBlock } long blockNumber = block.getNumber(); - LOG.info("Corrupt world state at block hash: {}, number: {}." + LOG.info("Pruned or corrupt world state at block hash: {}, number: {}." + " Looking for ancestor block with valid world state ...", block.getShortHash(), blockNumber); AionRepositoryImpl repo = (AionRepositoryImpl) repository; + // keeping track of the original root + byte[] originalRoot = repo.getRoot(); + Deque dirtyBlocks = new ArrayDeque<>(); // already known to be missing the state dirtyBlocks.push(block); @@ -1430,6 +1435,9 @@ public synchronized boolean recoverWorldState(IRepository repository, AionBlock // update the repository repo.flush(); + // setting the root back to its correct value + repo.syncToRoot(originalRoot); + // return a flag indicating if the recovery worked return repo.isValidRoot(block.getStateRoot()); } diff --git a/modAionImpl/src/org/aion/zero/impl/AionHub.java b/modAionImpl/src/org/aion/zero/impl/AionHub.java index 86c1530c06..fbdd702c48 100644 --- a/modAionImpl/src/org/aion/zero/impl/AionHub.java +++ b/modAionImpl/src/org/aion/zero/impl/AionHub.java @@ -1,4 +1,4 @@ -/******************************************************************************* +/* * Copyright (c) 2017-2018 Aion foundation. * * This file is part of the aion network project. @@ -19,14 +19,20 @@ * * Contributors: * Aion foundation. - * - ******************************************************************************/ - + */ package org.aion.zero.impl; +import static org.aion.crypto.HashUtil.EMPTY_TRIE_HASH; + +import java.io.File; +import java.math.BigInteger; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.Properties; +import java.util.ServiceLoader; +import java.util.concurrent.atomic.AtomicBoolean; import org.aion.base.db.IRepository; -import org.aion.base.db.IRepositoryCache; -import org.aion.base.type.Address; import org.aion.base.util.ByteUtil; import org.aion.evtmgr.EventMgrModule; import org.aion.evtmgr.IEvent; @@ -39,12 +45,10 @@ import org.aion.mcf.config.CfgNetP2p; import org.aion.mcf.db.IBlockStorePow; import org.aion.mcf.tx.ITransactionExecThread; -import org.aion.mcf.vm.types.DataWord; import org.aion.p2p.Handler; import org.aion.p2p.IP2pMgr; import org.aion.p2p.impl1.P2pMgr; import org.aion.utils.TaskDumpHeap; -import org.aion.vm.PrecompiledContracts; import org.aion.zero.impl.blockchain.AionPendingStateImpl; import org.aion.zero.impl.blockchain.ChainConfiguration; import org.aion.zero.impl.config.CfgAion; @@ -53,87 +57,86 @@ import org.aion.zero.impl.db.RecoveryUtils; import org.aion.zero.impl.pow.AionPoW; import org.aion.zero.impl.sync.SyncMgr; -import org.aion.zero.impl.sync.handler.*; +import org.aion.zero.impl.sync.handler.BlockPropagationHandler; +import org.aion.zero.impl.sync.handler.BroadcastNewBlockHandler; +import org.aion.zero.impl.sync.handler.BroadcastTxHandler; +import org.aion.zero.impl.sync.handler.ReqBlocksBodiesHandler; +import org.aion.zero.impl.sync.handler.ReqBlocksHeadersHandler; +import org.aion.zero.impl.sync.handler.ReqStatusHandler; +import org.aion.zero.impl.sync.handler.ResBlocksBodiesHandler; +import org.aion.zero.impl.sync.handler.ResBlocksHeadersHandler; +import org.aion.zero.impl.sync.handler.ResStatusHandler; import org.aion.zero.impl.tx.AionTransactionExecThread; import org.aion.zero.impl.types.AionBlock; import org.aion.zero.types.A0BlockHeader; import org.aion.zero.types.AionTransaction; import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.File; -import java.math.BigInteger; -import java.util.*; -import java.util.concurrent.atomic.AtomicBoolean; - -import static org.aion.crypto.HashUtil.EMPTY_TRIE_HASH; public class AionHub { - private static final Logger LOG = LoggerFactory.getLogger(LogEnum.GEN.name()); + private static final Logger genLOG = AionLoggerFactory.getLogger(LogEnum.GEN.name()); + + private static final Logger syncLOG = AionLoggerFactory.getLogger(LogEnum.SYNC.name()); - private static final Logger syncLog = AionLoggerFactory.getLogger(LogEnum.SYNC.name()); + private IP2pMgr p2pMgr; - private IP2pMgr p2pMgr; + private CfgAion cfg; - private CfgAion cfg; + private SyncMgr syncMgr; - private SyncMgr syncMgr; + private BlockPropagationHandler propHandler; - private BlockPropagationHandler propHandler; + private IPendingStateInternal mempool; - private IPendingStateInternal mempool; + private IAionBlockchain blockchain; - private IAionBlockchain blockchain; + // TODO: Refactor to interface later + private AionRepositoryImpl repository; - // TODO: Refactor to interface later - private AionRepositoryImpl repository; + private ITransactionExecThread txThread; - private ITransactionExecThread txThread; + private IEventMgr eventMgr; - private IEventMgr eventMgr; + private AionPoW pow; - private AionPoW pow; + private AtomicBoolean start = new AtomicBoolean(true); - private AtomicBoolean start = new AtomicBoolean(true); + /** + * A "cached" block that represents our local best block when the application is first booted. + */ + private volatile AionBlock startingBlock; - /** - * A "cached" block that represents our local best block when the application is - * first booted. - */ - private volatile AionBlock startingBlock; + /** + * Initialize as per the Initialization-on-demand + * holder pattern + */ + private static class Holder { - /** - * Initialize as per the Initialization-on-demand - * holder pattern - */ - private static class Holder { - static final AionHub INSTANCE = new AionHub(); - } + static final AionHub INSTANCE = new AionHub(); + } - public static AionHub inst() { - return Holder.INSTANCE; - } + public static AionHub inst() { + return Holder.INSTANCE; + } - public AionHub() { + public AionHub() { - this.cfg = CfgAion.inst(); + this.cfg = CfgAion.inst(); - // load event manager before init blockchain instance - loadEventMgr(); + // load event manager before init blockchain instance + loadEventMgr(); - AionBlockchainImpl blockchain = AionBlockchainImpl.inst(); - blockchain.setEventManager(this.eventMgr); - this.blockchain = blockchain; + AionBlockchainImpl blockchain = AionBlockchainImpl.inst(); + blockchain.setEventManager(this.eventMgr); + this.blockchain = blockchain; - this.repository = AionRepositoryImpl.inst(); + this.repository = AionRepositoryImpl.inst(); - this.mempool = AionPendingStateImpl.inst(); + this.mempool = AionPendingStateImpl.inst(); - this.txThread = AionTransactionExecThread.getInstance(); + this.txThread = AionTransactionExecThread.getInstance(); - loadBlockchain(); + loadBlockchain(); this.startingBlock = this.blockchain.getBestBlock(); if (!cfg.getConsensus().isSeed()) { @@ -143,60 +146,68 @@ public AionHub() { this.mempool.loadPendingTx(); } } else { - LOG.info("Seed node mode enabled!"); + genLOG.info("Seed node mode enabled!"); } - String reportsFolder = ""; - if (cfg.getReports().isEnabled()) { - File rpf = new File(cfg.getBasePath(), cfg.getReports().getPath()); - rpf.mkdirs(); - reportsFolder = rpf.getAbsolutePath(); - } + String reportsFolder = ""; + if (cfg.getReports().isEnabled()) { + File rpf = new File(cfg.getBasePath(), cfg.getReports().getPath()); + rpf.mkdirs(); + reportsFolder = rpf.getAbsolutePath(); + } - /* - * p2p hook up start sync mgr needs to be initialed after loadBlockchain() - * method - */ - CfgNetP2p cfgNetP2p = this.cfg.getNet().getP2p(); + /* + * p2p hook up start sync mgr needs to be initialed after loadBlockchain() + * method + */ + CfgNetP2p cfgNetP2p = this.cfg.getNet().getP2p(); - // there two p2p impletation , now just point to impl1. - this.p2pMgr = new P2pMgr(this.cfg.getNet().getId(), Version.KERNEL_VERSION, this.cfg.getId(), cfgNetP2p.getIp(), - cfgNetP2p.getPort(), this.cfg.getNet().getNodes(), cfgNetP2p.getDiscover(), cfgNetP2p.getMaxTempNodes(), - cfgNetP2p.getMaxActiveNodes(), cfgNetP2p.getShowStatus(), cfgNetP2p.getShowLog(), - cfgNetP2p.getBootlistSyncOnly(), cfgNetP2p.getErrorTolerance()); + // there are two p2p implementation , now just point to impl1. + this.p2pMgr = new P2pMgr(this.cfg.getNet().getId(), Version.KERNEL_VERSION, + this.cfg.getId(), cfgNetP2p.getIp(), + cfgNetP2p.getPort(), this.cfg.getNet().getNodes(), cfgNetP2p.getDiscover(), + cfgNetP2p.getMaxTempNodes(), + cfgNetP2p.getMaxActiveNodes(), + cfgNetP2p.getBootlistSyncOnly(), cfgNetP2p.getErrorTolerance()); - this.syncMgr = SyncMgr.inst(); - this.syncMgr.init(this.p2pMgr, this.eventMgr, this.cfg.getSync().getBlocksQueueMax(), - this.cfg.getSync().getShowStatus(), this.cfg.getReports().isEnabled(), reportsFolder); + this.syncMgr = SyncMgr.inst(); + this.syncMgr.init(this.p2pMgr, this.eventMgr, this.cfg.getSync().getBlocksQueueMax(), + this.cfg.getSync().getShowStatus(), this.cfg.getReports().isEnabled(), reportsFolder); - ChainConfiguration chainConfig = new ChainConfiguration(); - this.propHandler = new BlockPropagationHandler(1024, this.blockchain, this.p2pMgr, - chainConfig.createBlockHeaderValidator(), this.cfg.getNet().getP2p().isSyncOnlyNode()); + ChainConfiguration chainConfig = new ChainConfiguration(); + this.propHandler = new BlockPropagationHandler(1024, this.blockchain, this.p2pMgr, + chainConfig.createBlockHeaderValidator(), this.cfg.getNet().getP2p().isSyncOnlyNode()); - registerCallback(); - this.p2pMgr.run(); + registerCallback(); + this.p2pMgr.run(); - ((AionPendingStateImpl)this.mempool).setP2pMgr(this.p2pMgr); + ((AionPendingStateImpl) this.mempool).setP2pMgr(this.p2pMgr); - this.pow = new AionPoW(); - this.pow.init(blockchain, mempool, eventMgr); + this.pow = new AionPoW(); + this.pow.init(blockchain, mempool, eventMgr); if (cfg.getReports().isHeapDumpEnabled()) { - new Thread(new TaskDumpHeap(this.start, cfg.getReports().getHeapDumpInterval(), reportsFolder), "dump-heap") - .start(); + new Thread( + new TaskDumpHeap(this.start, cfg.getReports().getHeapDumpInterval(), reportsFolder), + "dump-heap") + .start(); } } private void registerCallback() { List cbs = new ArrayList<>(); - cbs.add(new ReqStatusHandler(syncLog, this.blockchain, this.p2pMgr, cfg.getGenesis().getHash())); - cbs.add(new ResStatusHandler(syncLog, this.p2pMgr, this.syncMgr)); - cbs.add(new ReqBlocksHeadersHandler(syncLog, this.blockchain, this.p2pMgr, this.cfg.getNet().getP2p().isSyncOnlyNode())); - cbs.add(new ResBlocksHeadersHandler(syncLog, this.syncMgr, this.p2pMgr)); - cbs.add(new ReqBlocksBodiesHandler(syncLog, this.blockchain, this.p2pMgr, this.cfg.getNet().getP2p().isSyncOnlyNode())); - cbs.add(new ResBlocksBodiesHandler(syncLog, this.syncMgr, this.p2pMgr)); - cbs.add(new BroadcastTxHandler(syncLog, this.mempool, this.p2pMgr, this.cfg.getNet().getP2p().isSyncOnlyNode())); - cbs.add(new BroadcastNewBlockHandler(syncLog, this.propHandler, this.p2pMgr)); + cbs.add(new ReqStatusHandler(syncLOG, this.blockchain, this.p2pMgr, + cfg.getGenesis().getHash())); + cbs.add(new ResStatusHandler(syncLOG, this.p2pMgr, this.syncMgr)); + cbs.add(new ReqBlocksHeadersHandler( + syncLOG, this.blockchain, this.p2pMgr, this.cfg.getNet().getP2p().isSyncOnlyNode())); + cbs.add(new ResBlocksHeadersHandler(syncLOG, this.syncMgr, this.p2pMgr)); + cbs.add(new ReqBlocksBodiesHandler( + syncLOG, this.blockchain, this.p2pMgr, this.cfg.getNet().getP2p().isSyncOnlyNode())); + cbs.add(new ResBlocksBodiesHandler(syncLOG, this.syncMgr, this.p2pMgr)); + cbs.add(new BroadcastTxHandler( + syncLOG, this.mempool, this.p2pMgr, this.cfg.getNet().getP2p().isSyncOnlyNode())); + cbs.add(new BroadcastNewBlockHandler(syncLOG, this.propHandler, this.p2pMgr)); this.p2pMgr.register(cbs); } @@ -207,7 +218,7 @@ private void loadEventMgr() { try { ServiceLoader.load(EventMgrModule.class); } catch (Exception e) { - LOG.error("load EventMgr service fail!" + e.toString()); + genLOG.error("load EventMgr service fail!" + e.toString()); throw e; } @@ -217,7 +228,7 @@ private void loadEventMgr() { try { this.eventMgr = EventMgrModule.getSingleton(prop).getEventMgr(); } catch (Throwable e) { - LOG.error("Can not load the Event Manager Module", e.getMessage()); + genLOG.error("Can not load the Event Manager Module", e.getMessage()); } if (eventMgr == null) { @@ -247,10 +258,6 @@ public IEventMgr getEventMgr() { return this.eventMgr; } - public ITransactionExecThread getTxThread() { - return this.txThread; - } - public BlockPropagationHandler getPropHandler() { return propHandler; } @@ -261,6 +268,11 @@ private void loadBlockchain() { this.repository.getBlockStore().load(); AionBlock bestBlock = this.repository.getBlockStore().getBestBlock(); + if (bestBlock != null) { + bestBlock + .setCumulativeDifficulty( + repository.getBlockStore().getTotalDifficultyForHash(bestBlock.getHash())); + } boolean recovered = true; boolean bestBlockShifted = true; @@ -268,46 +280,75 @@ private void loadBlockchain() { // fix the trie if necessary while (bestBlockShifted && // the best block was updated after recovery attempt - (countRecoveryAttempts < 5) && // allow 5 recovery attempts - bestBlock != null && // recover only for non-null blocks - !this.repository.isValidRoot(bestBlock.getStateRoot())) { + (countRecoveryAttempts < 5) && // allow 5 recovery attempts + bestBlock != null && // recover only for non-null blocks + !this.repository.isValidRoot(bestBlock.getStateRoot())) { + + genLOG.info( + "Recovery initiated due to corrupt world state at block " + bestBlock.getNumber() + + "."); long bestBlockNumber = bestBlock.getNumber(); byte[] bestBlockRoot = bestBlock.getStateRoot(); + // ensure that the genesis state exists before attempting recovery + AionGenesis genesis = cfg.getGenesis(); + if (!this.repository.isValidRoot(genesis.getStateRoot())) { + genLOG.info( + "Corrupt world state for genesis block hash: " + genesis.getShortHash() + + ", number: " + genesis + .getNumber() + "."); + + AionHubUtils.buildGenesis(genesis, repository); + + if (repository.isValidRoot(genesis.getStateRoot())) { + genLOG.info("Rebuilding genesis block SUCCEEDED."); + } else { + genLOG.info("Rebuilding genesis block FAILED."); + } + } + recovered = this.blockchain.recoverWorldState(this.repository, bestBlock); if (!this.repository.isValidRoot(bestBlock.getStateRoot())) { // reverting back one block - LOG.info("Rebuild state FAILED. Reverting to previous block."); + genLOG.info("Rebuild state FAILED. Reverting to previous block."); long blockNumber = bestBlock.getNumber() - 1; RecoveryUtils.Status status = RecoveryUtils.revertTo(this.blockchain, blockNumber); recovered = (status == RecoveryUtils.Status.SUCCESS) && this.repository - .isValidRoot(this.repository.getBlockStore().getChainBlockByNumber(blockNumber).getStateRoot()); + .isValidRoot(this.repository.getBlockStore().getChainBlockByNumber(blockNumber) + .getStateRoot()); } if (recovered) { bestBlock = this.repository.getBlockStore().getBestBlock(); + if (bestBlock != null) { + bestBlock.setCumulativeDifficulty(repository.getBlockStore() + .getTotalDifficultyForHash(bestBlock.getHash())); + } // checking is the best block has changed since attempting recovery if (bestBlock == null) { bestBlockShifted = true; } else { - bestBlockShifted = !(bestBlockNumber == bestBlock.getNumber()) || // block number changed - !(Arrays.equals(bestBlockRoot, bestBlock.getStateRoot())); // root hash changed + bestBlockShifted = + !(bestBlockNumber == bestBlock.getNumber()) || // block number changed + !(Arrays.equals(bestBlockRoot, + bestBlock.getStateRoot())); // root hash changed } if (bestBlockShifted) { - LOG.info("Rebuilding world state SUCCEEDED by REVERTING to a previous block."); + genLOG + .info("Rebuilding world state SUCCEEDED by REVERTING to a previous block."); } else { - LOG.info("Rebuilding world state SUCCEEDED."); + genLOG.info("Rebuilding world state SUCCEEDED."); } } else { - LOG.error("Rebuilding world state FAILED. " - + "Stop the kernel (Ctrl+C) and use the command line revert option to move back to a valid block. " - + "Check the Aion wiki for recommendations on choosing the block number."); + genLOG.error("Rebuilding world state FAILED. " + + "Stop the kernel (Ctrl+C) and use the command line revert option to move back to a valid block. " + + "Check the Aion wiki for recommendations on choosing the block number."); } countRecoveryAttempts++; @@ -316,33 +357,21 @@ private void loadBlockchain() { // rebuild from genesis if (1) no best block (2) recovery failed if (bestBlock == null || !recovered) { if (bestBlock == null) { - LOG.info("DB is empty - adding Genesis"); + genLOG.info("DB is empty - adding Genesis"); } else { - LOG.info("DB could not be recovered - adding Genesis"); + genLOG.info("DB could not be recovered - adding Genesis"); } AionGenesis genesis = cfg.getGenesis(); - // initialization section for network balance contract - IRepositoryCache track = repository.startTracking(); - - Address networkBalanceAddress = PrecompiledContracts.totalCurrencyAddress; - track.createAccount(networkBalanceAddress); - - for (Map.Entry addr : genesis.getNetworkBalances().entrySet()) { - track.addStorageRow(networkBalanceAddress, new DataWord(addr.getKey()), new DataWord(addr.getValue())); - } - - for (Address addr : genesis.getPremine().keySet()) { - track.createAccount(addr); - track.addBalance(addr, genesis.getPremine().get(addr).getBalance()); - } - track.flush(); + AionHubUtils.buildGenesis(genesis, repository); - repository.commitBlock(genesis.getHeader()); - this.repository.getBlockStore().saveBlock(genesis, genesis.getCumulativeDifficulty(), true); blockchain.setBestBlock(genesis); - blockchain.setTotalDifficulty(genesis.getCumulativeDifficulty()); + blockchain.setTotalDifficulty(genesis.getDifficultyBI()); + if (genesis.getCumulativeDifficulty().equals(BigInteger.ZERO)) { + // setting the object runtime value + genesis.setCumulativeDifficulty(genesis.getDifficultyBI()); + } if (this.eventMgr != null) { List evts = new ArrayList<>(); @@ -351,40 +380,49 @@ private void loadBlockchain() { this.eventMgr.registerEvent(evts); } else { - LOG.error("Event manager is null !!!"); + genLOG.error("Event manager is null !!!"); System.exit(-1); } - LOG.info("loaded genesis block ", 0, ByteUtil.toHexString(genesis.getStateRoot())); + genLOG.info("loaded genesis block ", 0, + ByteUtil.toHexString(genesis.getStateRoot())); } else { blockchain.setBestBlock(bestBlock); blockchain.setTotalDifficulty(this.repository.getBlockStore().getTotalDifficulty()); - LOG.info("loaded block ", blockchain.getBestBlock().getNumber(), + if (bestBlock.getCumulativeDifficulty().equals(BigInteger.ZERO)) { + // setting the object runtime value + bestBlock.setCumulativeDifficulty(this.repository.getBlockStore().getTotalDifficulty()); + } + + genLOG.info("loaded block ", blockchain.getBestBlock().getNumber(), LogUtil.toHexF8(blockchain.getBestBlock().getStateRoot())); } byte[] genesisHash = cfg.getGenesis().getHash(); - byte[] databaseGenHash = blockchain.getBlockByNumber(0) == null ? null : blockchain.getBlockByNumber(0).getHash(); + byte[] databaseGenHash = blockchain.getBlockByNumber(0) == null ? null + : blockchain.getBlockByNumber(0).getHash(); // this indicates that DB and genesis are inconsistent - if (genesisHash == null || databaseGenHash == null || (!Arrays.equals(genesisHash, databaseGenHash))) { + if (genesisHash == null || databaseGenHash == null || (!Arrays + .equals(genesisHash, databaseGenHash))) { if (genesisHash == null) { - LOG.error("failed to load genesis from config"); + genLOG.error("failed to load genesis from config"); } if (databaseGenHash == null) { - LOG.error("failed to load block 0 from database"); + genLOG.error("failed to load block 0 from database"); } - LOG.error("genesis json rootHash {} is inconsistent with database rootHash {}\n" + - "your configuration and genesis are incompatible, please do the following:\n" + - "\t1) Remove your database folder\n" + - "\t2) Verify that your genesis is correct by re-downloading the binary or checking online\n" + - "\t3) Reboot with correct genesis and empty database\n", - genesisHash == null ? "null" : ByteUtil.toHexString(genesisHash), - databaseGenHash == null ? "null" : ByteUtil.toHexString(databaseGenHash)); + genLOG.error("genesis json rootHash {} is inconsistent with database rootHash {}\n" + + "your configuration and genesis are incompatible, please do the following:\n" + + "\t1) Remove your database folder\n" + + "\t2) Verify that your genesis is correct by re-downloading the binary or checking online\n" + + + "\t3) Reboot with correct genesis and empty database\n", + genesisHash == null ? "null" : ByteUtil.toHexString(genesisHash), + databaseGenHash == null ? "null" : ByteUtil.toHexString(databaseGenHash)); System.exit(-1); } @@ -396,21 +434,21 @@ private void loadBlockchain() { } public void close() { - LOG.info(""); + genLOG.info(""); if (syncMgr != null) { syncMgr.shutdown(); - LOG.info(""); + genLOG.info(""); } if (p2pMgr != null) { p2pMgr.shutdown(); - LOG.info(""); + genLOG.info(""); } if (txThread != null) { txThread.shutdown(); - LOG.info(""); + genLOG.info(""); } if (eventMgr != null) { @@ -423,17 +461,17 @@ public void close() { if (getPendingState() != null) { getPendingState().shutDown(); - LOG.info(""); + genLOG.info(""); } - LOG.info("shutting down consensus..."); + genLOG.info("shutting down consensus..."); pow.shutdown(); - LOG.info("shutdown consensus... Done!"); + genLOG.info("shutdown consensus... Done!"); if (repository != null) { - LOG.info("shutting down DB..."); + genLOG.info("shutting down DB..."); repository.close(); - LOG.info("shutdown DB... Done!"); + genLOG.info("shutdown DB... Done!"); } this.start.set(false); diff --git a/modAionImpl/src/org/aion/zero/impl/AionHubUtils.java b/modAionImpl/src/org/aion/zero/impl/AionHubUtils.java new file mode 100644 index 0000000000..97fba0f1c7 --- /dev/null +++ b/modAionImpl/src/org/aion/zero/impl/AionHubUtils.java @@ -0,0 +1,59 @@ +/* + * Copyright (c) 2017-2018 Aion foundation. + * + * This file is part of the aion network project. + * + * The aion network project is free software: you can redistribute it + * and/or modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation, either version 3 of + * the License, or any later version. + * + * The aion network project is distributed in the hope that it will + * be useful, but WITHOUT ANY WARRANTY; without even the implied + * warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + * See the GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with the aion network project source files. + * If not, see . + * + * Contributors: + * Aion foundation. + */ +package org.aion.zero.impl; + +import java.math.BigInteger; +import java.util.Map; +import org.aion.base.db.IRepositoryCache; +import org.aion.base.type.Address; +import org.aion.mcf.vm.types.DataWord; +import org.aion.vm.PrecompiledContracts; +import org.aion.zero.impl.db.AionRepositoryImpl; + +/** {@link AionHub} functionality where a full instantiation of the class is not desirable. */ +public class AionHubUtils { + + public static void buildGenesis(AionGenesis genesis, AionRepositoryImpl repository) { + // initialization section for network balance contract + IRepositoryCache track = repository.startTracking(); + + Address networkBalanceAddress = PrecompiledContracts.totalCurrencyAddress; + track.createAccount(networkBalanceAddress); + + for (Map.Entry addr : genesis.getNetworkBalances().entrySet()) { + track.addStorageRow( + networkBalanceAddress, + new DataWord(addr.getKey()), + new DataWord(addr.getValue())); + } + + for (Address addr : genesis.getPremine().keySet()) { + track.createAccount(addr); + track.addBalance(addr, genesis.getPremine().get(addr).getBalance()); + } + track.flush(); + + repository.commitBlock(genesis.getHeader()); + repository.getBlockStore().saveBlock(genesis, genesis.getDifficultyBI(), true); + } +} diff --git a/modAionImpl/src/org/aion/zero/impl/StandaloneBlockchain.java b/modAionImpl/src/org/aion/zero/impl/StandaloneBlockchain.java index 83b5ef8f91..4b7e096b36 100644 --- a/modAionImpl/src/org/aion/zero/impl/StandaloneBlockchain.java +++ b/modAionImpl/src/org/aion/zero/impl/StandaloneBlockchain.java @@ -1,4 +1,4 @@ -/* ****************************************************************************** +/* * Copyright (c) 2017-2018 Aion foundation. * * This file is part of the aion network project. @@ -19,12 +19,13 @@ * * Contributors: * Aion foundation. - ******************************************************************************/ + */ package org.aion.zero.impl; import java.math.BigInteger; import java.util.*; import org.aion.base.db.IContractDetails; +import org.aion.base.db.IPruneConfig; import org.aion.base.db.IRepositoryCache; import org.aion.base.db.IRepositoryConfig; import org.aion.base.type.Address; @@ -33,6 +34,7 @@ import org.aion.crypto.ECKeyFac; import org.aion.db.impl.DBVendor; import org.aion.db.impl.DatabaseFactory; +import org.aion.mcf.config.CfgPrune; import org.aion.mcf.core.AccountState; import org.aion.mcf.core.ImportResult; import org.aion.mcf.valid.BlockHeaderValidator; @@ -42,7 +44,6 @@ import org.aion.zero.impl.blockchain.ChainConfiguration; import org.aion.zero.impl.core.energy.AbstractEnergyStrategyLimit; import org.aion.zero.impl.core.energy.TargetStrategy; -import org.aion.zero.impl.db.AionBlockStore; import org.aion.zero.impl.db.AionRepositoryImpl; import org.aion.zero.impl.db.ContractDetailsAion; import org.aion.zero.impl.types.AionBlock; @@ -68,8 +69,8 @@ public String getDbPath() { } @Override - public int getPrune() { - return -1; + public IPruneConfig getPruneConfig() { + return new CfgPrune(false); } @Override @@ -203,8 +204,8 @@ public String getDbPath() { } @Override - public int getPrune() { - return -1; + public IPruneConfig getPruneConfig() { + return new CfgPrune(false); } @Override @@ -333,13 +334,14 @@ public AbstractEnergyStrategyLimit getEnergyLimitStrategy() { } track.flush(); - // TODO: violates abstraction, consider adding to interface after - // stable - ((AionRepositoryImpl) bc.getRepository()).commitBlock(genesis.getHeader()); - ((AionBlockStore) bc.getRepository().getBlockStore()) - .saveBlock(genesis, genesis.getCumulativeDifficulty(), true); + bc.getRepository().commitBlock(genesis.getHeader()); + bc.getRepository().getBlockStore().saveBlock(genesis, genesis.getDifficultyBI(), true); bc.setBestBlock(genesis); - bc.setTotalDifficulty(genesis.getCumulativeDifficulty()); + bc.setTotalDifficulty(genesis.getDifficultyBI()); + if (genesis.getCumulativeDifficulty().equals(BigInteger.ZERO)) { + // setting the object runtime value + genesis.setCumulativeDifficulty(genesis.getDifficultyBI()); + } return new Bundle(this.defaultKeys, bc); } diff --git a/modAionImpl/src/org/aion/zero/impl/Version.java b/modAionImpl/src/org/aion/zero/impl/Version.java index 32e530e467..9a02465a25 100644 --- a/modAionImpl/src/org/aion/zero/impl/Version.java +++ b/modAionImpl/src/org/aion/zero/impl/Version.java @@ -1,7 +1,7 @@ package org.aion.zero.impl; public class Version { - public static final String KERNEL_VERSION = "0.2.7"; + public static final String KERNEL_VERSION = "0.2.8"; public static final String REPO_VERSION = "0.1.0"; public static final boolean FORK = true; } diff --git a/modAionImpl/src/org/aion/zero/impl/cli/Cli.java b/modAionImpl/src/org/aion/zero/impl/cli/Cli.java index c21424dfa9..4acfa1d171 100644 --- a/modAionImpl/src/org/aion/zero/impl/cli/Cli.java +++ b/modAionImpl/src/org/aion/zero/impl/cli/Cli.java @@ -132,11 +132,66 @@ public int call(final String[] args, final Cfg cfg) { } } break; + case "--state": { + String pruning_type = "full"; + if (args.length >= 2) { + pruning_type = args[1]; + } + try { + RecoveryUtils.pruneOrRecoverState(pruning_type); + } catch (Throwable t) { + System.out.println("Reorganizing the state storage FAILED due to:"); + t.printStackTrace(); + return 1; + } + break; + } + case "--dump-state-size": + long block_count = 2L; + + if (args.length < 2) { + System.out.println("Retrieving state size for top " + block_count + " blocks."); + RecoveryUtils.printStateTrieSize(block_count); + } else { + try { + block_count = Long.parseLong(args[1]); + } catch (NumberFormatException e) { + System.out.println("The given argument <" + args[1] + "> cannot be converted to a number."); + } + if (block_count < 1) { + System.out.println("The given argument <" + args[1] + "> is not valid."); + block_count = 2L; + } + + System.out.println("Retrieving state size for top " + block_count + " blocks."); + RecoveryUtils.printStateTrieSize(block_count); + } + break; + case "--dump-state": + long level = -1L; + + if (args.length < 2) { + System.out.println("Retrieving state for top main chain block..."); + RecoveryUtils.printStateTrieDump(level); + } else { + try { + level = Long.parseLong(args[1]); + } catch (NumberFormatException e) { + System.out.println("The given argument <" + args[1] + "> cannot be converted to a number."); + } + if (level == -1L) { + System.out.println("Retrieving state for top main chain block..."); + } else { + System.out.println("Retrieving state for main chain block at level " + level + "..."); + } + RecoveryUtils.printStateTrieDump(level); + } + break; case "--db-compact": RecoveryUtils.dbCompact(); break; case "--dump-blocks": - long count = 100L; + long count = 10L; if (args.length < 2) { System.out.println("Printing top " + count + " blocks from database."); @@ -147,10 +202,14 @@ public int call(final String[] args, final Cfg cfg) { } catch (NumberFormatException e) { System.out.println("The given argument <" + args[1] + "> cannot be converted to a number."); } + if (count < 1) { + System.out.println("The given argument <" + args[1] + "> is not valid."); + count = 10L; + } + System.out.println("Printing top " + count + " blocks from database."); RecoveryUtils.dumpBlocks(count); } - System.out.println("Finished printing blocks."); break; case "-v": System.out.println("\nVersion"); diff --git a/modAionImpl/src/org/aion/zero/impl/db/AionBlockStore.java b/modAionImpl/src/org/aion/zero/impl/db/AionBlockStore.java index 73e4132a5c..9ee514135b 100644 --- a/modAionImpl/src/org/aion/zero/impl/db/AionBlockStore.java +++ b/modAionImpl/src/org/aion/zero/impl/db/AionBlockStore.java @@ -1,4 +1,4 @@ -/******************************************************************************* +/* * Copyright (c) 2017-2018 Aion foundation. * * This file is part of the aion network project. @@ -19,8 +19,7 @@ * * Contributors: * Aion foundation. - ******************************************************************************/ - + */ package org.aion.zero.impl.db; import org.aion.base.db.IByteArrayKeyValueDatabase; @@ -635,7 +634,8 @@ public void pruneAndCorrect() { pruneSideChains(block); // bottom up repair of information - BigInteger parentTotalDifficulty = block.getCumulativeDifficulty(); + // initial TD set to genesis TD + BigInteger parentTotalDifficulty = block.getHeader().getDifficultyBI(); level = 1; while (level <= initialLevel) { parentTotalDifficulty = correctTotalDifficulty(level, parentTotalDifficulty); @@ -672,7 +672,7 @@ private void pruneSideChains(IAionBlock block) { blocks.delete(wrongBlock.getHash()); } - // set new block info without total difficulty + // set new block info with total difficulty = block difficulty blockInfo = new BlockInfo(); blockInfo.setCummDifficulty(block.getHeader().getDifficultyBI()); blockInfo.setHash(blockHash); @@ -697,6 +697,7 @@ private BigInteger correctTotalDifficulty(long level, BigInteger parentTotalDiff } else { // correct block info BlockInfo blockInfo = levelBlocks.remove(0); + // total difficulty previously set to block difficulty blockInfo.setCummDifficulty(blockInfo.getCummDifficulty().add(parentTotalDifficulty)); levelBlocks.add(blockInfo); setBlockInfoForLevel(level, levelBlocks); @@ -749,11 +750,14 @@ public BigInteger correctIndexEntry(AionBlock block, BigInteger parentTotalDiffi } } - public void dumpPastBlocks(long numberOfBlocks, String reportsFolder) throws IOException { + public String dumpPastBlocks(long numberOfBlocks, String reportsFolder) throws IOException { lock.readLock().lock(); try { long firstBlock = getMaxNumber(); + if (firstBlock < 0) { + return null; + } long lastBlock = firstBlock - numberOfBlocks; File file = new File(reportsFolder, System.currentTimeMillis() + "-blocks-report.out"); @@ -786,6 +790,7 @@ public void dumpPastBlocks(long numberOfBlocks, String reportsFolder) throws IOE } writer.close(); + return file.getName(); } finally { lock.readLock().unlock(); } diff --git a/modAionImpl/src/org/aion/zero/impl/db/AionRepositoryImpl.java b/modAionImpl/src/org/aion/zero/impl/db/AionRepositoryImpl.java index 58824e11f2..61ad71f1d8 100644 --- a/modAionImpl/src/org/aion/zero/impl/db/AionRepositoryImpl.java +++ b/modAionImpl/src/org/aion/zero/impl/db/AionRepositoryImpl.java @@ -1,4 +1,4 @@ -/******************************************************************************* +/* ****************************************************************************** * Copyright (c) 2017-2018 Aion foundation. * * This file is part of the aion network project. @@ -19,11 +19,15 @@ * * Contributors: * Aion foundation. - * ******************************************************************************/ - package org.aion.zero.impl.db; +import static org.aion.base.util.ByteUtil.EMPTY_BYTE_ARRAY; +import static org.aion.crypto.HashUtil.EMPTY_TRIE_HASH; + +import java.io.File; +import java.math.BigInteger; +import java.util.*; import org.aion.base.db.*; import org.aion.base.type.Address; import org.aion.base.util.Hex; @@ -42,28 +46,19 @@ import org.aion.zero.types.AionTransaction; import org.aion.zero.types.AionTxReceipt; -import java.io.File; -import java.math.BigInteger; -import java.util.*; - -import static org.aion.base.util.ByteUtil.EMPTY_BYTE_ARRAY; -import static org.aion.crypto.HashUtil.EMPTY_TRIE_HASH; - -/** - * Has direct database connection. - */ -public class AionRepositoryImpl extends AbstractRepository { +/** Has direct database connection. */ +public class AionRepositoryImpl + extends AbstractRepository { private TransactionStore transactionStore; /** * used by getSnapShotTo * - * @ATTENTION: when do snap shot, another instance will be created. Make - * sure it is used only by getSnapShotTo + *

@ATTENTION: when do snap shot, another instance will be created. Make sure it is used only + * by getSnapShotTo */ - protected AionRepositoryImpl() { - } + protected AionRepositoryImpl() {} protected AionRepositoryImpl(IRepositoryConfig repoConfig) { this.cfg = repoConfig; @@ -74,11 +69,13 @@ private static class AionRepositoryImplHolder { // configuration private static CfgAion config = CfgAion.inst(); // repository singleton instance - private final static AionRepositoryImpl inst = new AionRepositoryImpl( - new RepositoryConfig(new File(config.getBasePath(), config.getDb().getPath()).getAbsolutePath(), - -1, - ContractDetailsAion.getInstance(), - config.getDb())); + private static final AionRepositoryImpl inst = + new AionRepositoryImpl( + new RepositoryConfig( + new File(config.getBasePath(), config.getDb().getPath()) + .getAbsolutePath(), + ContractDetailsAion.getInstance(), + config.getDb())); } public static AionRepositoryImpl inst() { @@ -94,8 +91,9 @@ private void init() { initializeDatabasesAndCaches(); // Setup the cache for transaction data source. - this.transactionStore = new TransactionStore<>(transactionDatabase, - AionTransactionStoreSerializer.serializer); + this.transactionStore = + new TransactionStore<>( + transactionDatabase, AionTransactionStoreSerializer.serializer); // Setup block store. this.blockStore = new AionBlockStore(indexDatabase, blockDatabase, checkIntegrity); @@ -107,19 +105,18 @@ private void init() { } } - /** - * @implNote The transaction store is not locked within the repository implementation. - */ + /** @implNote The transaction store is not locked within the repository implementation. */ public TransactionStore getTransactionStore() { return this.transactionStore; } private Trie createStateTrie() { - return new SecureTrie(stateDatabase).withPruningEnabled(pruneBlockCount >= 0); + return new SecureTrie(stateDSPrune).withPruningEnabled(pruneEnabled); } @Override - public void updateBatch(Map stateCache, + public void updateBatch( + Map stateCache, Map> detailsCache) { rwLock.writeLock().lock(); @@ -148,17 +145,19 @@ public void updateBatch(Map stateCache, updateAccountState(address, accountState); if (LOG.isTraceEnabled()) { - LOG.trace("update: [{}],nonce: [{}] balance: [{}] [{}]", - Hex.toHexString(address.toBytes()), - accountState.getNonce(), - accountState.getBalance(), - Hex.toHexString(contractDetails.getStorageHash())); + LOG.trace( + "update: [{}],nonce: [{}] balance: [{}] [{}]", + Hex.toHexString(address.toBytes()), + accountState.getNonce(), + accountState.getBalance(), + Hex.toHexString(contractDetails.getStorageHash())); } } continue; } - ContractDetailsCacheImpl contractDetailsCache = (ContractDetailsCacheImpl) contractDetails; + ContractDetailsCacheImpl contractDetailsCache = + (ContractDetailsCacheImpl) contractDetails; if (contractDetailsCache.origContract == null) { contractDetailsCache.origContract = this.cfg.contractDetailsImpl(); @@ -166,7 +165,8 @@ public void updateBatch(Map stateCache, contractDetailsCache.origContract.setAddress(address); } catch (Exception e) { e.printStackTrace(); - LOG.error("contractDetailsCache setAddress exception [{}]", e.toString()); + LOG.error( + "contractDetailsCache setAddress exception [{}]", e.toString()); } contractDetailsCache.commit(); @@ -183,11 +183,12 @@ public void updateBatch(Map stateCache, updateAccountState(address, accountState); if (LOG.isTraceEnabled()) { - LOG.trace("update: [{}],nonce: [{}] balance: [{}] [{}]", - Hex.toHexString(address.toBytes()), - accountState.getNonce(), - accountState.getBalance(), - Hex.toHexString(contractDetails.getStorageHash())); + LOG.trace( + "update: [{}],nonce: [{}] balance: [{}] [{}]", + Hex.toHexString(address.toBytes()), + accountState.getNonce(), + accountState.getBalance(), + Hex.toHexString(contractDetails.getStorageHash())); } } } @@ -200,10 +201,9 @@ public void updateBatch(Map stateCache, } } - /** - * @implNote The method calling this method must handle the locking. - */ - private void updateContractDetails(final Address address, final IContractDetails contractDetails) { + /** @implNote The method calling this method must handle the locking. */ + private void updateContractDetails( + final Address address, final IContractDetails contractDetails) { // locked by calling method detailsDS.update(address, contractDetails); } @@ -364,9 +364,7 @@ public BigInteger getNonce(Address address) { return (account == null) ? BigInteger.ZERO : account.getNonce(); } - /** - * @implNote The method calling this method must handle the locking. - */ + /** @implNote The method calling this method must handle the locking. */ private void updateAccountState(Address address, AccountState accountState) { // locked by calling method worldState.update(address.toBytes(), accountState.getEncoded()); @@ -374,10 +372,10 @@ private void updateAccountState(Address address, AccountState accountState) { /** * @inheritDoc - * @implNote Any other method calling this can rely on the fact that - * the contract details returned is a newly created object by {@link IContractDetails#getSnapshotTo(byte[])}. - * Since this querying method it locked, the methods calling it - * may not need to be locked or synchronized, depending on the specific use case. + * @implNote Any other method calling this can rely on the fact that the contract details + * returned is a newly created object by {@link IContractDetails#getSnapshotTo(byte[])}. + * Since this querying method it locked, the methods calling it may not need to be locked + * or synchronized, depending on the specific use case. */ @Override public IContractDetails getContractDetails(Address address) { @@ -418,10 +416,9 @@ public boolean hasContractDetails(Address address) { /** * @inheritDoc - * @implNote Any other method calling this can rely on the fact that - * the account state returned is a newly created object. - * Since this querying method it locked, the methods calling it - * may not need to be locked or synchronized, depending on the specific use case. + * @implNote Any other method calling this can rely on the fact that the account state returned + * is a newly created object. Since this querying method it locked, the methods calling it + * may not need to be locked or synchronized, depending on the specific use case. */ @Override public AccountState getAccountState(Address address) { @@ -434,7 +431,8 @@ public AccountState getAccountState(Address address) { if (accountData.length != 0) { result = new AccountState(accountData); - LOG.debug("New AccountSate [{}], State [{}]", address.toString(), result.toString()); + LOG.debug( + "New AccountSate [{}], State [{}]", address.toString(), result.toString()); } return result; } finally { @@ -448,11 +446,13 @@ public boolean hasAccountState(Address address) { } /** - * @implNote The loaded objects are fresh copies of the original account - * state and contract details. + * @implNote The loaded objects are fresh copies of the original account state and contract + * details. */ @Override - public void loadAccountState(Address address, Map cacheAccounts, + public void loadAccountState( + Address address, + Map cacheAccounts, Map> cacheDetails) { AccountState account = getAccountState(address); @@ -485,13 +485,8 @@ public void setRoot(byte[] root) { } } - public void setPruneBlockCount(long pruneBlockCount) { - rwLock.writeLock().lock(); - try { - this.pruneBlockCount = pruneBlockCount; - } finally { - rwLock.writeLock().unlock(); - } + public long getPruneBlockCount() { + return this.pruneBlockCount; } public void commitBlock(A0BlockHeader blockHeader) { @@ -501,34 +496,46 @@ public void commitBlock(A0BlockHeader blockHeader) { worldState.sync(); detailsDS.syncLargeStorage(); - // temporarily removed since never used - /* if (pruneBlockCount >= 0) { - stateDSPrune.storeBlockChanges(blockHeader); - detailsDS.getStorageDSPrune().storeBlockChanges(blockHeader); - pruneBlocks(blockHeader); - } */ + if (pruneEnabled) { + if (blockHeader.getNumber() % archiveRate == 0 && stateDSPrune.isArchiveEnabled()) { + // archive block + worldState.saveDiffStateToDatabase( + blockHeader.getStateRoot(), stateDSPrune.getArchiveSource()); + } + stateDSPrune.storeBlockChanges(blockHeader.getHash(), blockHeader.getNumber()); + detailsDS + .getStorageDSPrune() + .storeBlockChanges(blockHeader.getHash(), blockHeader.getNumber()); + pruneBlocks(blockHeader); + } } finally { rwLock.writeLock().unlock(); } } - // TODO-AR: reenable state pruning - // temporarily removed since never used - /* private void pruneBlocks(A0BlockHeader curBlock) { - if (curBlock.getNumber() > bestBlockNumber) { // pruning only on - // increasing blocks + private void pruneBlocks(A0BlockHeader curBlock) { + if (curBlock.getNumber() > bestBlockNumber) { + // pruning only on increasing blocks long pruneBlockNumber = curBlock.getNumber() - pruneBlockCount; if (pruneBlockNumber >= 0) { byte[] pruneBlockHash = blockStore.getBlockHashByNumber(pruneBlockNumber); if (pruneBlockHash != null) { A0BlockHeader header = blockStore.getBlockByHash(pruneBlockHash).getHeader(); - // stateDSPrune.prune(header); - // detailsDS.getStorageDSPrune().prune(header); + stateDSPrune.prune(header.getHash(), header.getNumber()); + detailsDS.getStorageDSPrune().prune(header.getHash(), header.getNumber()); } } } bestBlockNumber = curBlock.getNumber(); - } */ + } + + /** + * @return {@code true} when pruning is enabled and archiving is disabled, {@code false} + * otherwise + */ + public boolean usesTopPruning() { + return pruneEnabled && !stateDSPrune.isArchiveEnabled(); + } public Trie getWorldState() { return worldState; @@ -543,8 +550,14 @@ public IRepository getSnapshotTo(byte[] root) { repo.blockStore = blockStore; repo.cfg = cfg; repo.stateDatabase = this.stateDatabase; - // repo.stateDSPrune = this.stateDSPrune; + repo.stateWithArchive = this.stateWithArchive; + repo.stateDSPrune = this.stateDSPrune; + + // pruning config + repo.pruneEnabled = this.pruneEnabled; repo.pruneBlockCount = this.pruneBlockCount; + repo.archiveRate = this.archiveRate; + repo.detailsDS = this.detailsDS; repo.isSnapshot = true; @@ -595,10 +608,7 @@ public void removeTxBatch(Set clearTxSet, boolean isPool) { } } - /** - * This function cannot for any reason fail, otherwise we may have dangling - * file IO locks - */ + /** This function cannot for any reason fail, otherwise we may have dangling file IO locks */ @Override public void close() { rwLock.writeLock().lock(); @@ -623,6 +633,16 @@ public void close() { LOGGEN.error("Exception occurred while closing the state database.", e); } + try { + if (stateArchiveDatabase != null) { + stateArchiveDatabase.close(); + LOGGEN.info("State archive database closed."); + stateArchiveDatabase = null; + } + } catch (Exception e) { + LOGGEN.error("Exception occurred while closing the state archive database.", e); + } + try { if (transactionDatabase != null) { transactionDatabase.close(); @@ -660,7 +680,8 @@ public void close() { pendingTxCacheDatabase = null; } } catch (Exception e) { - LOGGEN.error("Exception occurred while closing the pendingTxCacheDatabase store.", e); + LOGGEN.error( + "Exception occurred while closing the pendingTxCacheDatabase store.", e); } } finally { rwLock.writeLock().unlock(); @@ -668,12 +689,11 @@ public void close() { } /** - * Retrieves the underlying state database that sits below all caches. This - * is usually provided by {@link org.aion.db.impl.leveldb.LevelDB} or - * {@link org.aion.db.impl.leveldb.LevelDB}. - *

- * Note that referencing the state database directly is unsafe, and should - * only be used for debugging and testing purposes. + * Retrieves the underlying state database that sits below all caches. This is usually provided + * by {@link org.aion.db.impl.leveldb.LevelDB} or {@link org.aion.db.impl.leveldb.LevelDB}. + * + *

Note that referencing the state database directly is unsafe, and should only be used for + * debugging and testing purposes. * * @return */ @@ -681,13 +701,16 @@ public IByteArrayKeyValueDatabase getStateDatabase() { return this.stateDatabase; } + public IByteArrayKeyValueDatabase getStateArchiveDatabase() { + return this.stateArchiveDatabase; + } + /** - * Retrieves the underlying details database that sits below all caches. - * This is usually provided by {@link org.aion.db.impl.mockdb.MockDB} - * or {@link org.aion.db.impl.mockdb.MockDB}. - *

- * Note that referencing the state database directly is unsafe, and should - * only be used for debugging and testing purposes. + * Retrieves the underlying details database that sits below all caches. This is usually + * provided by {@link org.aion.db.impl.mockdb.MockDB} or {@link org.aion.db.impl.mockdb.MockDB}. + * + *

Note that referencing the state database directly is unsafe, and should only be used for + * debugging and testing purposes. * * @return */ @@ -695,17 +718,20 @@ public IByteArrayKeyValueDatabase getDetailsDatabase() { return this.detailsDatabase; } - /** - * For testing. - */ + /** For testing. */ public IByteArrayKeyValueDatabase getIndexDatabase() { return this.indexDatabase; } @Override public String toString() { - return "AionRepositoryImpl{ identityHashCode=" + System.identityHashCode(this) + ", " + // - "databaseGroupSize=" + (databaseGroup == null ? 0 : databaseGroup.size()) + '}'; + return "AionRepositoryImpl{ identityHashCode=" + + System.identityHashCode(this) + + ", " + + // + "databaseGroupSize=" + + (databaseGroup == null ? 0 : databaseGroup.size()) + + '}'; } @Override diff --git a/modAionImpl/src/org/aion/zero/impl/db/RecoveryUtils.java b/modAionImpl/src/org/aion/zero/impl/db/RecoveryUtils.java index f0f109a1f9..1f0a06fb4b 100644 --- a/modAionImpl/src/org/aion/zero/impl/db/RecoveryUtils.java +++ b/modAionImpl/src/org/aion/zero/impl/db/RecoveryUtils.java @@ -1,4 +1,4 @@ -/******************************************************************************* +/* ****************************************************************************** * Copyright (c) 2017-2018 Aion foundation. * * This file is part of the aion network project. @@ -22,27 +22,29 @@ ******************************************************************************/ package org.aion.zero.impl.db; +import java.io.IOException; +import java.util.HashMap; +import java.util.Map; import org.aion.base.type.IBlock; import org.aion.log.AionLoggerFactory; +import org.aion.mcf.config.CfgDb; import org.aion.mcf.db.IBlockStoreBase; import org.aion.zero.impl.AionBlockchainImpl; +import org.aion.zero.impl.AionGenesis; +import org.aion.zero.impl.AionHubUtils; import org.aion.zero.impl.config.CfgAion; import org.aion.zero.impl.core.IAionBlockchain; - -import java.io.File; -import java.io.IOException; -import java.util.HashMap; -import java.util.Map; +import org.aion.zero.impl.types.AionBlock; public class RecoveryUtils { public enum Status { - SUCCESS, FAILURE, ILLEGAL_ARGUMENT + SUCCESS, + FAILURE, + ILLEGAL_ARGUMENT } - /** - * Used by the CLI call. - */ + /** Used by the CLI call. */ public static Status revertTo(long nbBlock) { // ensure mining is disabled CfgAion cfg = CfgAion.inst(); @@ -66,9 +68,7 @@ public static Status revertTo(long nbBlock) { return status; } - /** - * Used by the CLI call. - */ + /** Used by the CLI call. */ public static void pruneAndCorrect() { // ensure mining is disabled CfgAion cfg = CfgAion.inst(); @@ -101,9 +101,7 @@ public static void pruneAndCorrect() { blockchain.getRepository().close(); } - /** - * Used by the CLI call. - */ + /** Used by the CLI call. */ public static void dbCompact() { // ensure mining is disabled CfgAion cfg = CfgAion.inst(); @@ -126,9 +124,7 @@ public static void dbCompact() { repository.close(); } - /** - * Used by the CLI call. - */ + /** Used by the CLI call. */ public static void dumpBlocks(long count) { // ensure mining is disabled CfgAion cfg = CfgAion.inst(); @@ -138,8 +134,8 @@ public static void dumpBlocks(long count) { cfg.getDb().setHeapCacheEnabled(false); Map cfgLog = new HashMap<>(); - cfgLog.put("DB", "INFO"); - cfgLog.put("GEN", "INFO"); + cfgLog.put("DB", "ERROR"); + cfgLog.put("GEN", "ERROR"); AionLoggerFactory.init(cfgLog); @@ -148,7 +144,12 @@ public static void dumpBlocks(long count) { AionBlockStore store = repository.getBlockStore(); try { - store.dumpPastBlocks(count, cfg.getBasePath()); + String file = store.dumpPastBlocks(count, cfg.getBasePath()); + if (file == null) { + System.out.println("The database is empty. Cannot print block information."); + } else { + System.out.println("Block information stored in " + file); + } } catch (IOException e) { e.printStackTrace(); } @@ -156,9 +157,7 @@ public static void dumpBlocks(long count) { repository.close(); } - /** - * Used by internal world state recovery method. - */ + /** Used by internal world state recovery method. */ public static Status revertTo(IAionBlockchain blockchain, long nbBlock) { IBlockStoreBase store = blockchain.getBlockStore(); @@ -170,12 +169,15 @@ public static Status revertTo(IAionBlockchain blockchain, long nbBlock) { long nbBestBlock = bestBlock.getNumber(); - System.out.println("Attempting to revert best block from " + nbBestBlock + " to " + nbBlock + " ..."); + System.out.println( + "Attempting to revert best block from " + nbBestBlock + " to " + nbBlock + " ..."); // exit with warning if the given block is larger or negative if (nbBlock < 0) { System.out.println( - "Negative values <" + nbBlock + "> cannot be interpreted as block numbers. Nothing to do."); + "Negative values <" + + nbBlock + + "> cannot be interpreted as block numbers. Nothing to do."); return Status.ILLEGAL_ARGUMENT; } if (nbBestBlock == 0) { @@ -184,13 +186,19 @@ public static Status revertTo(IAionBlockchain blockchain, long nbBlock) { } if (nbBlock == nbBestBlock) { System.out.println( - "The block " + nbBlock + " is the current best block stored in the database. Nothing to do."); + "The block " + + nbBlock + + " is the current best block stored in the database. Nothing to do."); return Status.ILLEGAL_ARGUMENT; } if (nbBlock > nbBestBlock) { - System.out.println("The block #" + nbBlock + " is greater than the current best block #" + nbBestBlock - + " stored in the database. " - + "Cannot move to that block without synchronizing with peers. Start Aion instance to sync."); + System.out.println( + "The block #" + + nbBlock + + " is greater than the current best block #" + + nbBestBlock + + " stored in the database. " + + "Cannot move to that block without synchronizing with peers. Start Aion instance to sync."); return Status.ILLEGAL_ARGUMENT; } @@ -203,4 +211,179 @@ public static Status revertTo(IAionBlockchain blockchain, long nbBlock) { // ok if we managed to get down to the expected block return (nbBestBlock == nbBlock) ? Status.SUCCESS : Status.FAILURE; } + + public static void printStateTrieSize(long blockNumber) { + // ensure mining is disabled + CfgAion cfg = CfgAion.inst(); + cfg.dbFromXML(); + cfg.getConsensus().setMining(false); + + Map cfgLog = new HashMap<>(); + cfgLog.put("DB", "ERROR"); + + AionLoggerFactory.init(cfgLog); + + // get the current blockchain + AionRepositoryImpl repository = AionRepositoryImpl.inst(); + AionBlockStore store = repository.getBlockStore(); + + long topBlock = store.getBestBlock().getNumber(); + if (topBlock < 0) { + System.out.println("The database is empty. Cannot print block information."); + return; + } + + long targetBlock = topBlock - blockNumber + 1; + if (targetBlock < 0) { + targetBlock = 0; + } + + AionBlock block; + byte[] stateRoot; + + while (targetBlock <= topBlock) { + block = store.getChainBlockByNumber(targetBlock); + if (block != null) { + stateRoot = block.getStateRoot(); + try { + System.out.println( + "Block hash: " + + block.getShortHash() + + ", number: " + + block.getNumber() + + ", tx count: " + + block.getTransactionsList().size() + + ", state trie kv count = " + + repository.getWorldState().getTrieSize(stateRoot)); + } catch (RuntimeException e) { + System.out.println( + "Block hash: " + + block.getShortHash() + + ", number: " + + block.getNumber() + + ", tx count: " + + block.getTransactionsList().size() + + ", state trie kv count threw exception: " + + e.getMessage()); + } + } else { + long count = store.getBlocksByNumber(targetBlock).size(); + System.out.println( + "Null block found at level " + + targetBlock + + ". There " + + (count == 1 ? "is 1 block" : "are " + count + " blocks") + + " at this level. No main chain block found."); + } + targetBlock++; + } + + repository.close(); + } + + public static void printStateTrieDump(long blockNumber) { + // ensure mining is disabled + CfgAion cfg = CfgAion.inst(); + cfg.dbFromXML(); + cfg.getConsensus().setMining(false); + + Map cfgLog = new HashMap<>(); + cfgLog.put("DB", "ERROR"); + + AionLoggerFactory.init(cfgLog); + + // get the current blockchain + AionRepositoryImpl repository = AionRepositoryImpl.inst(); + + AionBlockStore store = repository.getBlockStore(); + + AionBlock block; + + if (blockNumber == -1L) { + block = store.getBestBlock(); + if (block == null) { + System.out.println("The requested block does not exist in the database."); + return; + } + blockNumber = block.getNumber(); + } else { + block = store.getChainBlockByNumber(blockNumber); + if (block == null) { + System.out.println("The requested block does not exist in the database."); + return; + } + } + + byte[] stateRoot = block.getStateRoot(); + System.out.println( + "\nBlock hash: " + + block.getShortHash() + + ", number: " + + blockNumber + + ", tx count: " + + block.getTransactionsList().size() + + "\n\n" + + repository.getWorldState().getTrieDump(stateRoot)); + + repository.close(); + } + + public static void pruneOrRecoverState(String pruning_type) { + // ensure mining is disabled + CfgAion cfg = CfgAion.inst(); + cfg.fromXML(); + cfg.getConsensus().setMining(false); + + // setting pruning to the version requested + CfgDb.PruneOption option = CfgDb.PruneOption.fromValue(pruning_type); + cfg.getDb().setPrune(option.toString()); + + System.out.println("Reorganizing the state storage to " + option + " mode ..."); + + Map cfgLog = new HashMap<>(); + cfgLog.put("DB", "ERROR"); + cfgLog.put("CONS", "ERROR"); + + AionLoggerFactory.init(cfgLog); + + AionBlockchainImpl chain = AionBlockchainImpl.inst(); + AionRepositoryImpl repo = (AionRepositoryImpl) chain.getRepository(); + AionBlockStore store = repo.getBlockStore(); + + // dropping old state database + System.out.println("Deleting old data ..."); + repo.getStateDatabase().drop(); + if (pruning_type.equals("spread")) { + repo.getStateArchiveDatabase().drop(); + } + + // recover genesis + System.out.println("Rebuilding genesis block ..."); + AionGenesis genesis = cfg.getGenesis(); + AionHubUtils.buildGenesis(genesis, repo); + + // recover all blocks + AionBlock block = store.getBestBlock(); + System.out.println( + "Rebuilding the main chain " + + block.getNumber() + + " blocks (may take a while) ..."); + + long topBlockNumber = block.getNumber(); + long blockNumber = 1000; + + // recover in increments of 1k blocks + while (blockNumber < topBlockNumber) { + block = store.getChainBlockByNumber(blockNumber); + chain.recoverWorldState(repo, block); + System.out.println("Finished with blocks up to " + blockNumber + "."); + blockNumber += 1000; + } + + block = store.getBestBlock(); + chain.recoverWorldState(repo, block); + + repo.close(); + System.out.println("Reorganizing the state storage COMPLETE."); + } } diff --git a/modAionImpl/src/org/aion/zero/impl/db/RepositoryConfig.java b/modAionImpl/src/org/aion/zero/impl/db/RepositoryConfig.java index a2acc69722..c65456edcb 100644 --- a/modAionImpl/src/org/aion/zero/impl/db/RepositoryConfig.java +++ b/modAionImpl/src/org/aion/zero/impl/db/RepositoryConfig.java @@ -1,4 +1,4 @@ -/******************************************************************************* +/* ****************************************************************************** * Copyright (c) 2017-2018 Aion foundation. * * This file is part of the aion network project. @@ -19,23 +19,21 @@ * * Contributors: * Aion foundation. - * ******************************************************************************/ - package org.aion.zero.impl.db; +import java.util.Map; +import java.util.Properties; import org.aion.base.db.DetailsProvider; import org.aion.base.db.IContractDetails; +import org.aion.base.db.IPruneConfig; import org.aion.base.db.IRepositoryConfig; import org.aion.mcf.config.CfgDb; -import java.util.Map; -import java.util.Properties; - public class RepositoryConfig implements IRepositoryConfig { private final String dbPath; - private final int prune; + private final IPruneConfig cfgPrune; private final DetailsProvider detailsProvider; private final Map cfg; @@ -45,8 +43,8 @@ public String getDbPath() { } @Override - public int getPrune() { - return prune; + public IPruneConfig getPruneConfig() { + return cfgPrune; } @Override @@ -63,13 +61,11 @@ public Properties getDatabaseConfig(String db_name) { return new Properties(prop); } - public RepositoryConfig(final String dbPath, - final int prune, - final DetailsProvider detailsProvider, - final CfgDb cfgDb) { + public RepositoryConfig( + final String dbPath, final DetailsProvider detailsProvider, final CfgDb cfgDb) { this.dbPath = dbPath; - this.prune = prune; this.detailsProvider = detailsProvider; this.cfg = cfgDb.asProperties(); + this.cfgPrune = cfgDb.getPrune(); } } diff --git a/modAionImpl/src/org/aion/zero/impl/sync/PeerState.java b/modAionImpl/src/org/aion/zero/impl/sync/PeerState.java index 36e8020e15..7d0d52fabc 100644 --- a/modAionImpl/src/org/aion/zero/impl/sync/PeerState.java +++ b/modAionImpl/src/org/aion/zero/impl/sync/PeerState.java @@ -3,42 +3,28 @@ public class PeerState { public enum Mode { - /** - * The peer is in main-chain; use normal syncing strategy. - */ + /** The peer is in main-chain; use normal syncing strategy. */ NORMAL, - /** - * The peer is in side-chain; sync backward to find the fork point. - */ + /** The peer is in side-chain; sync backward to find the fork point. */ BACKWARD, - /** - * The peer is in side-chain; sync forward to catch up. - */ + /** The peer is in side-chain; sync forward to catch up. */ FORWARD } // TODO: enforce rules on this public enum State { - /** - * The initial state. - */ + /** The initial state. */ INITIAL, - /** - * Status request, waiting for response. - */ + /** Status request, waiting for response. */ STATUS_REQUESTED, - /** - * Block headers request, waiting for response. - */ + /** Block headers request, waiting for response. */ HEADERS_REQUESTED, - /** - * Block bodies request, waiting for response. - */ + /** Block bodies request, waiting for response. */ BODIES_REQUESTED, } @@ -46,6 +32,11 @@ public enum State { private Mode mode; private long base; + // used in FORWARD mode to prevent endlessly importing EXISTing blocks + // compute how many times to go forward without importing a new block + private int repeated; + private int maxRepeats; + // The syncing status private State state; private long lastHeaderRequest; @@ -98,4 +89,33 @@ public void setLastHeaderRequest(long lastStatusRequest) { public void resetLastHeaderRequest() { this.lastHeaderRequest = 0; } + + public int getRepeated() { + return repeated; + } + + public void resetRepeated() { + this.repeated = 0; + } + + public void incRepeated() { + this.repeated++; + } + + /** + * This number is set based on the BACKWARD step size and the size of each requested batch in + * FORWARD mode. Passing the number of repeats allowed means that we have entered in the + * previous BACKWARD step. If that step would have been viable, we never would have made another + * step back, so it effectively ends the FORWARD pass. + * + * @return The number of times that a node in FORWARD mode can import only blocks that already + * EXIST. + */ + public int getMaxRepeats() { + return maxRepeats; + } + + public void setMaxRepeats(int maxRepeats) { + this.maxRepeats = maxRepeats; + } } diff --git a/modAionImpl/src/org/aion/zero/impl/sync/SyncMgr.java b/modAionImpl/src/org/aion/zero/impl/sync/SyncMgr.java index 5d454d27aa..b13a9e13d9 100644 --- a/modAionImpl/src/org/aion/zero/impl/sync/SyncMgr.java +++ b/modAionImpl/src/org/aion/zero/impl/sync/SyncMgr.java @@ -88,7 +88,8 @@ public final class SyncMgr { private final BlockingQueue downloadedBlocks = new LinkedBlockingQueue<>(); // store the hashes of blocks which have been successfully imported - private final Map importedBlockHashes = Collections.synchronizedMap(new LRUMap<>(4096)); + private final Map importedBlockHashes = Collections + .synchronizedMap(new LRUMap<>(4096)); //private ExecutorService workers = Executors.newFixedThreadPool(5); private ExecutorService workers = Executors.newCachedThreadPool(new ThreadFactory() { @@ -109,6 +110,7 @@ public Thread newThread(Runnable r) { private BlockHeaderValidator blockHeaderValidator; private static final class AionSyncMgrHolder { + static final SyncMgr INSTANCE = new SyncMgr(); } @@ -117,14 +119,11 @@ public static SyncMgr inst() { } /** - * * @param _displayId String * @param _remoteBestBlockNumber long * @param _remoteBestBlockHash byte[] - * @param _remoteTotalDiff BigInteger - * null check for _remoteBestBlockHash && _remoteTotalDiff + * @param _remoteTotalDiff BigInteger null check for _remoteBestBlockHash && _remoteTotalDiff * implemented on ResStatusHandler before pass through - * */ public void updateNetworkStatus( String _displayId, @@ -136,28 +135,31 @@ public void updateNetworkStatus( BigInteger selfTd = this.chain.getTotalDifficulty(); // trigger send headers routine immediately - if(_remoteTotalDiff.compareTo(selfTd) > 0) { + if (_remoteTotalDiff.compareTo(selfTd) > 0) { this.getHeaders(selfTd); // update network best status - synchronized (this.networkStatus){ + synchronized (this.networkStatus) { BigInteger networkTd = this.networkStatus.getTargetTotalDiff(); - if(_remoteTotalDiff.compareTo(networkTd) > 0){ + if (_remoteTotalDiff.compareTo(networkTd) > 0) { String remoteBestBlockHash = Hex.toHexString(_remoteBestBlockHash); - log.debug( - "{} td={}->{} bn={}->{} bh={}->{}>", + if (log.isDebugEnabled()) { + log.debug( + "network-status-updated on-sync id={}->{} td={}->{} bn={}->{} bh={}->{}", this.networkStatus.getTargetDisplayId(), _displayId, - this.networkStatus.getTargetTotalDiff().toString(10), _remoteTotalDiff.toString(10), + this.networkStatus.getTargetTotalDiff().toString(10), + _remoteTotalDiff.toString(10), this.networkStatus.getTargetBestBlockNumber(), _remoteBestBlockNumber, this.networkStatus.getTargetBestBlockHash(), remoteBestBlockHash - ); + ); + } this.networkStatus.update( - _displayId, - _remoteTotalDiff, - _remoteBestBlockNumber, - remoteBestBlockHash + _displayId, + _remoteTotalDiff, + _remoteBestBlockNumber, + remoteBestBlockHash ); } } @@ -165,7 +167,7 @@ public void updateNetworkStatus( } public void init(final IP2pMgr _p2pMgr, final IEventMgr _evtMgr, final int _blocksQueueMax, - final boolean _showStatus, final boolean _printReport, final String _reportFolder) { + final boolean _showStatus, final boolean _printReport, final String _reportFolder) { this.p2pMgr = _p2pMgr; this.chain = AionBlockchainImpl.inst(); this.evtMgr = _evtMgr; @@ -177,15 +179,19 @@ public void init(final IP2pMgr _p2pMgr, final IEventMgr _evtMgr, final int _bloc long selfBest = this.chain.getBestBlock().getNumber(); SyncStatics statics = new SyncStatics(selfBest); - syncGb = new Thread(new TaskGetBodies(this.p2pMgr, this.start, this.downloadedHeaders, this.headersWithBodiesRequested, this.peerStates, log), "sync-gb"); + syncGb = new Thread(new TaskGetBodies(this.p2pMgr, this.start, this.downloadedHeaders, + this.headersWithBodiesRequested, this.peerStates, log), "sync-gb"); syncGb.start(); - syncIb = new Thread(new TaskImportBlocks(this.p2pMgr, this.chain, this.start, statics, this.downloadedBlocks, this.importedBlockHashes, this.peerStates, log), "sync-ib"); + syncIb = new Thread(new TaskImportBlocks(this.p2pMgr, this.chain, this.start, statics, + this.downloadedBlocks, this.importedBlockHashes, this.peerStates, log), "sync-ib"); syncIb.start(); syncGs = new Thread(new TaskGetStatus(this.start, this.p2pMgr, log), "sync-gs"); syncGs.start(); - if(_showStatus) { - syncSs = new Thread(new TaskShowStatus(this.start, INTERVAL_SHOW_STATUS, this.chain, this.networkStatus, statics, log, _printReport, _reportFolder), "sync-ss"); + if (_showStatus) { + syncSs = new Thread( + new TaskShowStatus(this.start, INTERVAL_SHOW_STATUS, this.chain, this.networkStatus, + statics, _printReport, _reportFolder, AionLoggerFactory.getLogger(LogEnum.P2P.name())), "sync-ss"); syncSs.start(); } @@ -200,45 +206,48 @@ private void setupEventHandler() { private AtomicBoolean queueFull = new AtomicBoolean(false); - private void getHeaders(BigInteger _selfTd){ + private void getHeaders(BigInteger _selfTd) { if (downloadedBlocks.size() > blocksQueueMax) { if (queueFull.compareAndSet(false, true)) { log.debug("Downloaded blocks queue is full. Stop requesting headers"); } } else { - workers.submit(new TaskGetHeaders(p2pMgr, chain.getBestBlock().getNumber(), _selfTd, peerStates, log)); + workers.submit( + new TaskGetHeaders(p2pMgr, chain.getBestBlock().getNumber(), _selfTd, peerStates, + log)); queueFull.set(false); } } /** - * * @param _nodeIdHashcode int * @param _displayId String * @param _headers List validate headers batch and add batch to imported headers */ - public void validateAndAddHeaders(int _nodeIdHashcode, String _displayId, List _headers) { + public void validateAndAddHeaders(int _nodeIdHashcode, String _displayId, + List _headers) { if (_headers == null || _headers.isEmpty()) { return; } if (log.isDebugEnabled()) { log.debug( - "", - _headers.get(0).getNumber(), - _headers.size(), - _displayId + "", + _headers.get(0).getNumber(), + _headers.size(), + _displayId ); } // filter imported block headers List filtered = new ArrayList<>(); A0BlockHeader prev = null; - for(A0BlockHeader current : _headers){ + for (A0BlockHeader current : _headers) { // ignore this batch if any invalidated header - if(!this.blockHeaderValidator.validate(current, log)) { - log.debug("", current.getNumber(), current.getHash()); + if (!this.blockHeaderValidator.validate(current, log)) { + log.debug("", current.getNumber(), + current.getHash()); // Print header to allow debugging log.debug("Invalid header: {}", current.toString()); @@ -247,41 +256,46 @@ public void validateAndAddHeaders(int _nodeIdHashcode, String _displayId, List", - _displayId, - current.getNumber(), - prev.getNumber() + 1, - ByteUtil.toHexString(current.getParentHash()), - ByteUtil.toHexString(prev.getHash())); + if (prev != null && (current.getNumber() != (prev.getNumber() + 1) || !Arrays + .equals(current.getParentHash(), prev.getHash()))) { + log.debug( + "", + _displayId, + current.getNumber(), + prev.getNumber() + 1, + ByteUtil.toHexString(current.getParentHash()), + ByteUtil.toHexString(prev.getHash())); return; } // add if not cached - if(!importedBlockHashes.containsKey(ByteArrayWrapper.wrap(current.getHash()))) + if (!importedBlockHashes.containsKey(ByteArrayWrapper.wrap(current.getHash()))) { filtered.add(current); + } prev = current; } // NOTE: the filtered headers is still continuous - if(!filtered.isEmpty()) + if (!filtered.isEmpty()) { downloadedHeaders.add(new HeadersWrapper(_nodeIdHashcode, _displayId, filtered)); + } } /** * @param _nodeIdHashcode int * @param _displayId String - * @param _bodies List - * Assemble and validate blocks batch and add batch - * to import queue from network response blocks bodies + * @param _bodies List Assemble and validate blocks batch and add batch to import queue + * from network response blocks bodies */ - public void validateAndAddBlocks(int _nodeIdHashcode, String _displayId, final List _bodies) { + public void validateAndAddBlocks(int _nodeIdHashcode, String _displayId, + final List _bodies) { HeadersWrapper hw = this.headersWithBodiesRequested.remove(_nodeIdHashcode); - if (hw == null || _bodies == null) + if (hw == null || _bodies == null) { return; + } // assemble batch List headers = hw.getHeaders(); @@ -293,27 +307,29 @@ public void validateAndAddBlocks(int _nodeIdHashcode, String _displayId, final L if (block == null) { log.error("", _displayId); break; - } else + } else { blocks.add(block); + } } int m = blocks.size(); - if (m == 0) + if (m == 0) { return; + } if (log.isDebugEnabled()) { log.debug("", - blocks.get(0).getNumber(), - blocks.size(), - _displayId); + blocks.get(0).getNumber(), + blocks.size(), + _displayId); } // add batch downloadedBlocks.add(new BlocksWrapper(_nodeIdHashcode, _displayId, blocks)); } - + public long getNetworkBestBlockNumber() { - synchronized (this.networkStatus){ + synchronized (this.networkStatus) { return this.networkStatus.getTargetBestBlockNumber(); } } diff --git a/modAionImpl/src/org/aion/zero/impl/sync/TaskGetHeaders.java b/modAionImpl/src/org/aion/zero/impl/sync/TaskGetHeaders.java index 7c8cf75f91..357bb20649 100644 --- a/modAionImpl/src/org/aion/zero/impl/sync/TaskGetHeaders.java +++ b/modAionImpl/src/org/aion/zero/impl/sync/TaskGetHeaders.java @@ -29,21 +29,20 @@ package org.aion.zero.impl.sync; -import org.aion.p2p.INode; -import org.aion.p2p.IP2pMgr; -import org.aion.zero.impl.sync.msg.ReqBlocksHeaders; -import org.slf4j.Logger; - import java.math.BigInteger; import java.util.Collection; import java.util.List; import java.util.Map; import java.util.Random; import java.util.stream.Collectors; +import org.aion.p2p.INode; +import org.aion.p2p.IP2pMgr; +import org.aion.zero.impl.sync.msg.ReqBlocksHeaders; +import org.slf4j.Logger; -/** - * @author chris - */ +import static org.aion.p2p.P2pConstant.BACKWARD_SYNC_STEP; + +/** @author chris */ final class TaskGetHeaders implements Runnable { private final IP2pMgr p2p; @@ -58,7 +57,12 @@ final class TaskGetHeaders implements Runnable { private final Random random = new Random(System.currentTimeMillis()); - TaskGetHeaders(IP2pMgr p2p, long selfNumber, BigInteger selfTd, Map peerStates, Logger log) { + TaskGetHeaders( + IP2pMgr p2p, + long selfNumber, + BigInteger selfTd, + Map peerStates, + Logger log) { this.p2p = p2p; this.selfNumber = selfNumber; this.selfTd = selfTd; @@ -78,8 +82,9 @@ public void run() { // higher td n.getTotalDifficulty() != null && n.getTotalDifficulty().compareTo(this.selfTd) >= 0 // not recently requested - && (now - 5000) > peerStates.computeIfAbsent(n.getIdHash(), k -> new PeerState(PeerState.Mode.NORMAL, selfNumber)).getLastHeaderRequest() - ) + && (now - 5000) > peerStates + .computeIfAbsent(n.getIdHash(), k -> new PeerState(PeerState.Mode.NORMAL, selfNumber)) + .getLastHeaderRequest()) .collect(Collectors.toList()); if (nodesFiltered.isEmpty()) { return; @@ -94,39 +99,51 @@ public void run() { // decide the start block number long from = 0; int size = 24; + + // depends on the number of blocks going BACKWARD + state.setMaxRepeats(BACKWARD_SYNC_STEP / size + 1); + switch (state.getMode()) { - case NORMAL: { - // update base block - state.setBase(selfNumber); - - // normal mode - long nodeNumber = node.getBestBlockNumber(); - if (nodeNumber >= selfNumber + 128) { - from = Math.max(1, selfNumber + 1 - 4); - } else if (nodeNumber >= selfNumber - 128) { - from = Math.max(1, selfNumber + 1 - 16); - } else { - // no need to request from this node. His TD is probably corrupted. - return; + case NORMAL: + { + // update base block + state.setBase(selfNumber); + + // normal mode + long nodeNumber = node.getBestBlockNumber(); + if (nodeNumber >= selfNumber + BACKWARD_SYNC_STEP) { + from = Math.max(1, selfNumber + 1 - 4); + } else if (nodeNumber >= selfNumber - BACKWARD_SYNC_STEP) { + from = Math.max(1, selfNumber + 1 - 16); + } else { + // no need to request from this node. His TD is probably corrupted. + return; + } + + break; + } + case BACKWARD: + { + // step back by 128 blocks + from = Math.max(1, state.getBase() - BACKWARD_SYNC_STEP); + break; + } + case FORWARD: + { + // start from base block + from = state.getBase() + 1; + break; } - - break; - } - case BACKWARD: { - // step back by 128 blocks - from = Math.max(1, state.getBase() - 128); - break; - } - case FORWARD: { - // start from base block - from = state.getBase() + 1; - break; - } } // send request if (log.isDebugEnabled()) { - log.debug("", state.getMode(), from, size, node.getIdShort()); + log.debug( + "", + state.getMode(), + from, + size, + node.getIdShort()); } ReqBlocksHeaders rbh = new ReqBlocksHeaders(from, size); this.p2p.send(node.getIdHash(), node.getIdShort(), rbh); diff --git a/modAionImpl/src/org/aion/zero/impl/sync/TaskImportBlocks.java b/modAionImpl/src/org/aion/zero/impl/sync/TaskImportBlocks.java index bb7577a2d2..3e579f5288 100644 --- a/modAionImpl/src/org/aion/zero/impl/sync/TaskImportBlocks.java +++ b/modAionImpl/src/org/aion/zero/impl/sync/TaskImportBlocks.java @@ -29,6 +29,11 @@ package org.aion.zero.impl.sync; +import java.util.List; +import java.util.Map; +import java.util.concurrent.BlockingQueue; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.stream.Collectors; import org.aion.base.util.ByteArrayWrapper; import org.aion.mcf.core.ImportResult; import org.aion.p2p.IP2pMgr; @@ -36,16 +41,12 @@ import org.aion.zero.impl.types.AionBlock; import org.slf4j.Logger; -import java.util.List; -import java.util.Map; -import java.util.concurrent.BlockingQueue; -import java.util.concurrent.atomic.AtomicBoolean; -import java.util.stream.Collectors; - /** - * @author chris * handle process of importing blocks to repo - * TODO: targeted send + * + *

TODO: targeted send + * + * @author chris */ final class TaskImportBlocks implements Runnable { @@ -73,8 +74,7 @@ final class TaskImportBlocks implements Runnable { final BlockingQueue downloadedBlocks, final Map importedBlockHashes, final Map peerStates, - final Logger log - ) { + final Logger log) { this.p2p = p2p; this.chain = _chain; this.start = _start; @@ -99,18 +99,56 @@ public void run() { List batch = bw.getBlocks().stream() .filter(b -> importedBlockHashes.get(ByteArrayWrapper.wrap(b.getHash())) == null) + .filter(b -> !chain.isPruneRestricted(b.getNumber())) .collect(Collectors.toList()); PeerState state = peerStates.get(bw.getNodeIdHash()); if (state == null) { - log.warn("This is not supposed to happen, but the peer is sending us blocks without ask"); + log.warn( + "This is not supposed to happen, but the peer is sending us blocks without ask"); + } + + ImportResult importResult = ImportResult.IMPORTED_NOT_BEST; + + // importing last block in batch to see if we can skip batch + if (state != null && state.getMode() == PeerState.Mode.FORWARD && !batch.isEmpty()) { + AionBlock b = batch.get(batch.size() - 1); + + try { + importResult = importBlock(b, bw.getDisplayId(), state); + } catch (Throwable e) { + log.error(" {}", e.toString()); + if (e.getMessage() != null && e.getMessage().contains("No space left on device")) { + log.error("Shutdown due to lack of disk space."); + System.exit(0); + } + continue; + } + + switch (importResult) { + case IMPORTED_BEST: + case IMPORTED_NOT_BEST: + case EXIST: + { + importedBlockHashes.put(ByteArrayWrapper.wrap(b.getHash()), true); + + long lastBlock = batch.get(batch.size() - 1).getNumber(); + + forwardModeUpdate(state, lastBlock, importResult, b.getNumber()); + + // since last import worked skipping the batch + batch.clear(); + log.info("Forward skip."); + break; + } + default: + break; + } } for (AionBlock b : batch) { - long t1 = System.currentTimeMillis(); - ImportResult importResult; try { - importResult = this.chain.tryToConnect(b); + importResult = importBlock(b, bw.getDisplayId(), state); } catch (Throwable e) { log.error(" {}", e.toString()); if (e.getMessage() != null && e.getMessage().contains("No space left on device")) { @@ -119,14 +157,7 @@ public void run() { } continue; } - long t2 = System.currentTimeMillis(); - log.info("", - bw.getDisplayId(), - b.getShortHash(), - b.getNumber(), - b.getTransactionsList().size(), - importResult, - t2 - t1); + switch (importResult) { case IMPORTED_BEST: case IMPORTED_NOT_BEST: @@ -154,14 +185,9 @@ public void run() { // we found the fork point state.setMode(PeerState.Mode.FORWARD); state.setBase(lastBlock); - + state.resetRepeated(); } else if (mode == PeerState.Mode.FORWARD) { - // continue - state.setBase(lastBlock); - // if the imported best block, switch back to normal mode - if (importResult == ImportResult.IMPORTED_BEST) { - state.setMode(PeerState.Mode.NORMAL); - } + forwardModeUpdate(state, lastBlock, importResult, b.getNumber()); } break; case NO_PARENT: @@ -176,6 +202,20 @@ public void run() { break; } } + + // if any block results in NO_PARENT, all subsequent blocks will too + if (importResult == ImportResult.NO_PARENT) { + log.debug("Stopped importing batch due to NO_PARENT result."); + break; + } + } + + if (state != null + && state.getMode() == PeerState.Mode.FORWARD + && importResult == ImportResult.EXIST) { + // increment the repeat count every time + // we finish a batch of imports with EXIST + state.incRepeated(); } if (state != null) { @@ -185,4 +225,46 @@ public void run() { this.statis.update(this.chain.getBestBlock().getNumber()); } } + + private ImportResult importBlock(AionBlock b, String displayId, PeerState state) { + ImportResult importResult; + long t1 = System.currentTimeMillis(); + importResult = this.chain.tryToConnect(b); + long t2 = System.currentTimeMillis(); + log.info( + "", + displayId, + (state != null ? state.getMode() : PeerState.Mode.NORMAL), + b.getShortHash(), + b.getNumber(), + b.getTransactionsList().size(), + importResult, + t2 - t1); + return importResult; + } + + private void forwardModeUpdate(PeerState state, long lastBlock, ImportResult importResult, long blockNumber) { + // continue + state.setBase(lastBlock); + // if the imported best block, switch back to normal mode + if (importResult == ImportResult.IMPORTED_BEST) { + state.setMode(PeerState.Mode.NORMAL); + // switch peers to NORMAL otherwise they may never switch back + for (PeerState peerState : peerStates.values()) { + if (peerState.getMode() != PeerState.Mode.NORMAL) { + peerState.setMode(PeerState.Mode.NORMAL); + peerState.setBase(blockNumber); + peerState.resetLastHeaderRequest(); + } + } + } + // if the maximum number of repeats is passed + // then the peer is stuck endlessly importing old blocks + // otherwise it would have found an IMPORTED block already + if (state.getRepeated() >= state.getMaxRepeats()) { + state.setMode(PeerState.Mode.NORMAL); + state.setBase(chain.getBestBlock().getNumber()); + state.resetLastHeaderRequest(); + } + } } diff --git a/modAionImpl/src/org/aion/zero/impl/sync/TaskShowStatus.java b/modAionImpl/src/org/aion/zero/impl/sync/TaskShowStatus.java index 01aae5e311..ea62e0b755 100644 --- a/modAionImpl/src/org/aion/zero/impl/sync/TaskShowStatus.java +++ b/modAionImpl/src/org/aion/zero/impl/sync/TaskShowStatus.java @@ -1,42 +1,35 @@ /* * Copyright (c) 2017-2018 Aion foundation. * - * This file is part of the aion network project. + * This file is part of the aion network project. * - * The aion network project is free software: you can redistribute it - * and/or modify it under the terms of the GNU General Public License - * as published by the Free Software Foundation, either version 3 of - * the License, or any later version. + * The aion network project is free software: you can redistribute it + * and/or modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation, either version 3 of + * the License, or any later version. * - * The aion network project is distributed in the hope that it will - * be useful, but WITHOUT ANY WARRANTY; without even the implied - * warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. - * See the GNU General Public License for more details. + * The aion network project is distributed in the hope that it will + * be useful, but WITHOUT ANY WARRANTY; without even the implied + * warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + * See the GNU General Public License for more details. * - * You should have received a copy of the GNU General Public License - * along with the aion network project source files. - * If not, see . + * You should have received a copy of the GNU General Public License + * along with the aion network project source files. + * If not, see . * - * The aion network project leverages useful source code from other - * open source projects. We greatly appreciate the effort that was - * invested in these projects and we thank the individual contributors - * for their work. For provenance information and contributors - * please see . - * - * Contributors to the aion source files in decreasing order of code volume: - * Aion foundation. + * Contributors: + * Aion foundation. */ package org.aion.zero.impl.sync; -import org.aion.base.util.Hex; -import org.aion.zero.impl.AionBlockchainImpl; -import org.aion.zero.impl.types.AionBlock; -import org.slf4j.Logger; - import java.io.IOException; import java.nio.file.Files; import java.nio.file.Paths; import java.util.concurrent.atomic.AtomicBoolean; +import org.aion.base.util.Hex; +import org.aion.zero.impl.AionBlockchainImpl; +import org.aion.zero.impl.types.AionBlock; +import org.slf4j.Logger; /** * The thread print out sync status @@ -55,22 +48,22 @@ final class TaskShowStatus implements Runnable { private final SyncStatics statics; - private final Logger log; - private final boolean printReport; private final String reportFolder; + private final Logger p2pLOG; + TaskShowStatus(final AtomicBoolean _start, int _interval, final AionBlockchainImpl _chain, - final NetworkStatus _networkStatus, final SyncStatics _statics, final Logger _log, - final boolean _printReport, final String _reportFolder) { + final NetworkStatus _networkStatus, final SyncStatics _statics, + final boolean _printReport, final String _reportFolder, final Logger _log) { this.start = _start; this.interval = _interval; this.chain = _chain; this.networkStatus = _networkStatus; this.statics = _statics; - this.log = _log; this.printReport = _printReport; this.reportFolder = _reportFolder; + this.p2pLOG = _log; } @Override @@ -78,24 +71,26 @@ public void run() { Thread.currentThread().setPriority(Thread.MIN_PRIORITY); while (this.start.get()) { AionBlock selfBest = this.chain.getBestBlock(); - String selfTd = this.chain.getTotalDifficulty().toString(10); + String selfTd = selfBest.getCumulativeDifficulty().toString(10); - String status = ""; + + "/" + this.networkStatus.getTargetBestBlockHash() + ""; - // print to std output - // thread to dump sync status enabled by sync mgr - System.out.println(status); + p2pLOG.info(status); // print to report file if (printReport) { try { - Files.write(Paths.get(reportFolder, System.currentTimeMillis() + "-sync-report.out"), - status.getBytes()); + Files.write( + Paths.get(reportFolder, System.currentTimeMillis() + "-sync-report.out"), + status.getBytes()); } catch (IOException e) { e.printStackTrace(); } @@ -104,10 +99,14 @@ public void run() { try { Thread.sleep(interval); } catch (InterruptedException e) { - if (log.isDebugEnabled()) { log.debug(""); } + if (p2pLOG.isDebugEnabled()) { + p2pLOG.debug("sync-ss shutdown"); + } return; } } - if (log.isDebugEnabled()) { log.debug(""); } + if (p2pLOG.isDebugEnabled()) { + p2pLOG.debug("sync-ss shutdown"); + } } } diff --git a/modAionImpl/src/org/aion/zero/impl/sync/handler/BlockPropagationHandler.java b/modAionImpl/src/org/aion/zero/impl/sync/handler/BlockPropagationHandler.java index 28f4e88ac4..dc169ce941 100644 --- a/modAionImpl/src/org/aion/zero/impl/sync/handler/BlockPropagationHandler.java +++ b/modAionImpl/src/org/aion/zero/impl/sync/handler/BlockPropagationHandler.java @@ -1,38 +1,37 @@ /* * Copyright (c) 2017-2018 Aion foundation. * - * This file is part of the aion network project. + * This file is part of the aion network project. * - * The aion network project is free software: you can redistribute it - * and/or modify it under the terms of the GNU General Public License - * as published by the Free Software Foundation, either version 3 of - * the License, or any later version. + * The aion network project is free software: you can redistribute it + * and/or modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation, either version 3 of + * the License, or any later version. * - * The aion network project is distributed in the hope that it will - * be useful, but WITHOUT ANY WARRANTY; without even the implied - * warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. - * See the GNU General Public License for more details. + * The aion network project is distributed in the hope that it will + * be useful, but WITHOUT ANY WARRANTY; without even the implied + * warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + * See the GNU General Public License for more details. * - * You should have received a copy of the GNU General Public License - * along with the aion network project source files. - * If not, see . + * You should have received a copy of the GNU General Public License + * along with the aion network project source files. + * If not, see . * - * The aion network project leverages useful source code from other - * open source projects. We greatly appreciate the effort that was - * invested in these projects and we thank the individual contributors - * for their work. For provenance information and contributors - * please see . + * The aion network project leverages useful source code from other + * open source projects. We greatly appreciate the effort that was + * invested in these projects and we thank the individual contributors + * for their work. For provenance information and contributors + * please see . * * Contributors to the aion source files in decreasing order of code volume: - * Aion foundation. - * team through the ethereumJ library. - * Ether.Camp Inc. (US) team through Ethereum Harmony. - * John Tromp through the Equihash solver. - * Samuel Neves through the BLAKE2 implementation. - * Zcash project team. - * Bitcoinj team. + * Aion foundation. + * team through the ethereumJ library. + * Ether.Camp Inc. (US) team through Ethereum Harmony. + * John Tromp through the Equihash solver. + * Samuel Neves through the BLAKE2 implementation. + * Zcash project team. + * Bitcoinj team. */ - package org.aion.zero.impl.sync.handler; import org.aion.base.util.ByteArrayWrapper; @@ -181,12 +180,13 @@ public PropStatus processIncomingBlock(final int nodeId, final String _displayId } // notify higher td peers in order to limit the rebroadcast on delay of res status updating - if(result.isBest()){ - BigInteger td = blockchain.getTotalDifficulty(); + if (result.isBest()) { + AionBlock bestBlock = blockchain.getBestBlock(); + BigInteger td = bestBlock.getCumulativeDifficulty(); ResStatus rs = new ResStatus( - blockchain.getBestBlock().getNumber(), + bestBlock.getNumber(), td.toByteArray(), - blockchain.getBestBlockHash(), + bestBlock.getHash(), genesis ); diff --git a/modAionImpl/src/org/aion/zero/impl/sync/handler/BroadcastTxHandler.java b/modAionImpl/src/org/aion/zero/impl/sync/handler/BroadcastTxHandler.java index 5675ef8bf3..6507fe8708 100644 --- a/modAionImpl/src/org/aion/zero/impl/sync/handler/BroadcastTxHandler.java +++ b/modAionImpl/src/org/aion/zero/impl/sync/handler/BroadcastTxHandler.java @@ -181,7 +181,7 @@ private List castRawTx(List broadCastTx) { @Override public void shutDown() { - log.info("BroadcastTxHandler shutdown!"); + log.info("BroadcastTxHandler shutting down!"); if (ex != null) { ex.shutdown(); } diff --git a/modAionImpl/src/org/aion/zero/impl/sync/handler/ReqStatusHandler.java b/modAionImpl/src/org/aion/zero/impl/sync/handler/ReqStatusHandler.java index 60967a8183..8c1e5b6f55 100644 --- a/modAionImpl/src/org/aion/zero/impl/sync/handler/ReqStatusHandler.java +++ b/modAionImpl/src/org/aion/zero/impl/sync/handler/ReqStatusHandler.java @@ -1,38 +1,37 @@ /* * Copyright (c) 2017-2018 Aion foundation. * - * This file is part of the aion network project. + * This file is part of the aion network project. * - * The aion network project is free software: you can redistribute it - * and/or modify it under the terms of the GNU General Public License - * as published by the Free Software Foundation, either version 3 of - * the License, or any later version. + * The aion network project is free software: you can redistribute it + * and/or modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation, either version 3 of + * the License, or any later version. * - * The aion network project is distributed in the hope that it will - * be useful, but WITHOUT ANY WARRANTY; without even the implied - * warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. - * See the GNU General Public License for more details. + * The aion network project is distributed in the hope that it will + * be useful, but WITHOUT ANY WARRANTY; without even the implied + * warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + * See the GNU General Public License for more details. * - * You should have received a copy of the GNU General Public License - * along with the aion network project source files. - * If not, see . + * You should have received a copy of the GNU General Public License + * along with the aion network project source files. + * If not, see . * - * The aion network project leverages useful source code from other - * open source projects. We greatly appreciate the effort that was - * invested in these projects and we thank the individual contributors - * for their work. For provenance information and contributors - * please see . + * The aion network project leverages useful source code from other + * open source projects. We greatly appreciate the effort that was + * invested in these projects and we thank the individual contributors + * for their work. For provenance information and contributors + * please see . * * Contributors to the aion source files in decreasing order of code volume: - * Aion foundation. - * team through the ethereumJ library. - * Ether.Camp Inc. (US) team through Ethereum Harmony. - * John Tromp through the Equihash solver. - * Samuel Neves through the BLAKE2 implementation. - * Zcash project team. - * Bitcoinj team. + * Aion foundation. + * team through the ethereumJ library. + * Ether.Camp Inc. (US) team through Ethereum Harmony. + * John Tromp through the Equihash solver. + * Samuel Neves through the BLAKE2 implementation. + * Zcash project team. + * Bitcoinj team. */ - package org.aion.zero.impl.sync.handler; import org.aion.p2p.Ctrl; @@ -42,22 +41,23 @@ import org.aion.zero.impl.core.IAionBlockchain; import org.aion.zero.impl.sync.Act; import org.aion.zero.impl.sync.msg.ResStatus; +import org.aion.zero.impl.types.AionBlock; import org.slf4j.Logger; /** * handler for status request from network - * + * * @author chris */ public final class ReqStatusHandler extends Handler { - private final Logger log; + private final Logger log; - private IAionBlockchain chain; + private IAionBlockchain chain; - private IP2pMgr mgr; + private IP2pMgr mgr; - private byte[] genesisHash; + private byte[] genesisHash; private final int UPDATE_INTERVAL = 500; @@ -65,25 +65,32 @@ public final class ReqStatusHandler extends Handler { private volatile long cacheTs = 0; - public ReqStatusHandler(final Logger _log, final IAionBlockchain _chain, final IP2pMgr _mgr, - final byte[] _genesisHash) { - super(Ver.V0, Ctrl.SYNC, Act.REQ_STATUS); - this.log = _log; - this.chain = _chain; - this.mgr = _mgr; - this.genesisHash = _genesisHash; - this.cache = new ResStatus(0, new byte[0], new byte[0], _genesisHash); - } + public ReqStatusHandler( + final Logger _log, + final IAionBlockchain _chain, + final IP2pMgr _mgr, + final byte[] _genesisHash) { + super(Ver.V0, Ctrl.SYNC, Act.REQ_STATUS); + this.log = _log; + this.chain = _chain; + this.mgr = _mgr; + this.genesisHash = _genesisHash; + this.cache = new ResStatus(0, new byte[0], new byte[0], _genesisHash); + } - @Override - public void receive(int _nodeIdHashcode, String _displayId, byte[] _msg) { - long now = System.currentTimeMillis(); + @Override + public void receive(int _nodeIdHashcode, String _displayId, byte[] _msg) { + long now = System.currentTimeMillis(); if ((now - cacheTs) > this.UPDATE_INTERVAL) { synchronized (cache) { try { - cache = new ResStatus(this.chain.getBestBlock().getNumber(), - this.chain.getTotalDifficulty().toByteArray(), this.chain.getBestBlockHash(), - this.genesisHash); + AionBlock bestBlock = chain.getBestBlock(); + cache = + new ResStatus( + bestBlock.getNumber(), + bestBlock.getCumulativeDifficulty().toByteArray(), + bestBlock.getHash(), + this.genesisHash); } catch (Exception e) { e.printStackTrace(); } @@ -92,9 +99,7 @@ public void receive(int _nodeIdHashcode, String _displayId, byte[] _msg) { } this.mgr.send(_nodeIdHashcode, _displayId, cache); - this.log.debug("", - _displayId, - cache.getBestBlockNumber() - ); - } + this.log.debug( + "", _displayId, cache.getBestBlockNumber()); + } } diff --git a/modAionImpl/src/org/aion/zero/impl/types/AionBlock.java b/modAionImpl/src/org/aion/zero/impl/types/AionBlock.java index 357a75472b..a4ef4a1d60 100644 --- a/modAionImpl/src/org/aion/zero/impl/types/AionBlock.java +++ b/modAionImpl/src/org/aion/zero/impl/types/AionBlock.java @@ -1,4 +1,4 @@ -/******************************************************************************* +/* * Copyright (c) 2017-2018 Aion foundation. * * This file is part of the aion network project. @@ -19,9 +19,7 @@ * * Contributors: * Aion foundation. - * - ******************************************************************************/ - + */ package org.aion.zero.impl.types; import org.aion.base.type.Address; @@ -59,6 +57,8 @@ public class AionBlock extends AbstractBlock imp private Trie txsState; + private BigInteger td = null; + /* Constructors */ private AionBlock() { } @@ -228,9 +228,11 @@ public BigInteger getDifficultyBI() { } public BigInteger getCumulativeDifficulty() { - // TODO: currently returning incorrect total difficulty - parseRLP(); - return new BigInteger(1, this.header.getDifficulty()); + if (td == null) { + return BigInteger.ZERO; + } else { + return td; + } } public long getTimestamp() { @@ -442,4 +444,8 @@ public static AionBlock createBlockFromNetwork(A0BlockHeader header, byte[] body return block; } + + public void setCumulativeDifficulty(BigInteger _td){ + td = _td; + } } diff --git a/modAionImpl/test/org/aion/db/AionContractDetailsTest.java b/modAionImpl/test/org/aion/db/AionContractDetailsTest.java index 4222d694fc..5c69f27978 100644 --- a/modAionImpl/test/org/aion/db/AionContractDetailsTest.java +++ b/modAionImpl/test/org/aion/db/AionContractDetailsTest.java @@ -1,4 +1,4 @@ -/******************************************************************************* +/* ****************************************************************************** * Copyright (c) 2017-2018 Aion foundation. * * This file is part of the aion network project. @@ -34,59 +34,61 @@ ******************************************************************************/ package org.aion.db; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +import java.util.HashMap; +import java.util.Map; +import java.util.Properties; import org.aion.base.db.IByteArrayKeyValueDatabase; import org.aion.base.db.IContractDetails; +import org.aion.base.db.IPruneConfig; import org.aion.base.db.IRepositoryConfig; import org.aion.base.type.Address; import org.aion.base.util.ByteUtil; import org.aion.db.impl.DBVendor; import org.aion.db.impl.DatabaseFactory; -import org.aion.db.impl.leveldb.LevelDBConstants; +import org.aion.mcf.config.CfgPrune; import org.aion.mcf.vm.types.DataWord; import org.aion.zero.db.AionContractDetailsImpl; import org.aion.zero.impl.db.AionRepositoryImpl; import org.aion.zero.impl.db.ContractDetailsAion; import org.apache.commons.lang3.RandomUtils; -import org.junit.Ignore; import org.junit.Test; -import java.util.HashMap; -import java.util.Map; -import java.util.Properties; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; - public class AionContractDetailsTest { - private static final int IN_MEMORY_STORAGE_LIMIT = 1000000; // CfgAion.inst().getDb().getDetailsInMemoryStorageLimit(); - - protected IRepositoryConfig repoConfig = new IRepositoryConfig() { - @Override - public String getDbPath() { - return ""; - } - - @Override - public int getPrune() { - return 0; - } - - @Override - public IContractDetails contractDetailsImpl() { - return ContractDetailsAion.createForTesting(0, 1000000).getDetails(); - } - - @Override - public Properties getDatabaseConfig(String db_name) { - Properties props = new Properties(); - props.setProperty(DatabaseFactory.Props.DB_TYPE, DBVendor.MOCKDB.toValue()); - props.setProperty(DatabaseFactory.Props.ENABLE_HEAP_CACHE, "false"); - return props; - } - }; - - private static IContractDetails deserialize(byte[] rlp, IByteArrayKeyValueDatabase externalStorage) { + private static final int IN_MEMORY_STORAGE_LIMIT = + 1000000; // CfgAion.inst().getDb().getDetailsInMemoryStorageLimit(); + + protected IRepositoryConfig repoConfig = + new IRepositoryConfig() { + @Override + public String getDbPath() { + return ""; + } + + @Override + public IPruneConfig getPruneConfig() { + return new CfgPrune(false); + } + + @Override + public IContractDetails contractDetailsImpl() { + return ContractDetailsAion.createForTesting(0, 1000000).getDetails(); + } + + @Override + public Properties getDatabaseConfig(String db_name) { + Properties props = new Properties(); + props.setProperty(DatabaseFactory.Props.DB_TYPE, DBVendor.MOCKDB.toValue()); + props.setProperty(DatabaseFactory.Props.ENABLE_HEAP_CACHE, "false"); + return props; + } + }; + + private static IContractDetails deserialize( + byte[] rlp, IByteArrayKeyValueDatabase externalStorage) { AionContractDetailsImpl result = new AionContractDetailsImpl(); result.setExternalStorageDataSource(externalStorage); result.decode(rlp); @@ -105,10 +107,11 @@ public void test_1() throws Exception { byte[] key_2 = ByteUtil.hexStringToBytes("222222"); byte[] val_2 = ByteUtil.hexStringToBytes("bbbbbb"); - AionContractDetailsImpl contractDetails = new AionContractDetailsImpl( - -1, //CfgAion.inst().getDb().getPrune(), - 1000000 //CfgAion.inst().getDb().getDetailsInMemoryStorageLimit() - ); + AionContractDetailsImpl contractDetails = + new AionContractDetailsImpl( + -1, // CfgAion.inst().getDb().getPrune(), + 1000000 // CfgAion.inst().getDb().getDetailsInMemoryStorageLimit() + ); contractDetails.setCode(code); contractDetails.put(new DataWord(key_1), new DataWord(val_1)); contractDetails.put(new DataWord(key_2), new DataWord(val_2)); @@ -117,20 +120,25 @@ public void test_1() throws Exception { AionContractDetailsImpl contractDetails_ = new AionContractDetailsImpl(data); - assertEquals(ByteUtil.toHexString(code), - ByteUtil.toHexString(contractDetails_.getCode())); + assertEquals(ByteUtil.toHexString(code), ByteUtil.toHexString(contractDetails_.getCode())); - assertEquals(ByteUtil.toHexString(val_1), - ByteUtil.toHexString(contractDetails_.get(new DataWord(key_1)).getNoLeadZeroesData())); + assertEquals( + ByteUtil.toHexString(val_1), + ByteUtil.toHexString( + contractDetails_.get(new DataWord(key_1)).getNoLeadZeroesData())); - assertEquals(ByteUtil.toHexString(val_2), - ByteUtil.toHexString(contractDetails_.get(new DataWord(key_2)).getNoLeadZeroesData())); + assertEquals( + ByteUtil.toHexString(val_2), + ByteUtil.toHexString( + contractDetails_.get(new DataWord(key_2)).getNoLeadZeroesData())); } @Test public void test_2() throws Exception { - byte[] code = ByteUtil.hexStringToBytes("7c0100000000000000000000000000000000000000000000000000000000600035046333d546748114610065578063430fe5f01461007c5780634d432c1d1461008d578063501385b2146100b857806357eb3b30146100e9578063dbc7df61146100fb57005b6100766004356024356044356102f0565b60006000f35b61008760043561039e565b60006000f35b610098600435610178565b8073ffffffffffffffffffffffffffffffffffffffff1660005260206000f35b6100c96004356024356044356101a0565b8073ffffffffffffffffffffffffffffffffffffffff1660005260206000f35b6100f1610171565b8060005260206000f35b610106600435610133565b8360005282602052816040528073ffffffffffffffffffffffffffffffffffffffff1660605260806000f35b5b60006020819052908152604090208054600182015460028301546003909301549192909173ffffffffffffffffffffffffffffffffffffffff1684565b5b60015481565b5b60026020526000908152604090205473ffffffffffffffffffffffffffffffffffffffff1681565b73ffffffffffffffffffffffffffffffffffffffff831660009081526020819052604081206002015481908302341080156101fe575073ffffffffffffffffffffffffffffffffffffffff8516600090815260208190526040812054145b8015610232575073ffffffffffffffffffffffffffffffffffffffff85166000908152602081905260409020600101548390105b61023b57610243565b3391506102e8565b6101966103ca60003973ffffffffffffffffffffffffffffffffffffffff3381166101965285166101b68190526000908152602081905260408120600201546101d6526101f68490526102169080f073ffffffffffffffffffffffffffffffffffffffff8616600090815260208190526040902060030180547fffffffffffffffffffffffff0000000000000000000000000000000000000000168217905591508190505b509392505050565b73ffffffffffffffffffffffffffffffffffffffff33166000908152602081905260408120548190821461032357610364565b60018054808201909155600090815260026020526040902080547fffffffffffffffffffffffff000000000000000000000000000000000000000016331790555b50503373ffffffffffffffffffffffffffffffffffffffff1660009081526020819052604090209081556001810192909255600290910155565b3373ffffffffffffffffffffffffffffffffffffffff166000908152602081905260409020600201555600608061019660043960048051602451604451606451600080547fffffffffffffffffffffffff0000000000000000000000000000000000000000908116909517815560018054909516909317909355600355915561013390819061006390396000f3007c0100000000000000000000000000000000000000000000000000000000600035046347810fe381146100445780637e4a1aa81461005557806383d2421b1461006957005b61004f6004356100ab565b60006000f35b6100636004356024356100fc565b60006000f35b61007460043561007a565b60006000f35b6001543373ffffffffffffffffffffffffffffffffffffffff9081169116146100a2576100a8565b60078190555b50565b73ffffffffffffffffffffffffffffffffffffffff8116600090815260026020526040902080547fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff0016600117905550565b6001543373ffffffffffffffffffffffffffffffffffffffff9081169116146101245761012f565b600582905560068190555b505056"); + byte[] code = + ByteUtil.hexStringToBytes( + "7c0100000000000000000000000000000000000000000000000000000000600035046333d546748114610065578063430fe5f01461007c5780634d432c1d1461008d578063501385b2146100b857806357eb3b30146100e9578063dbc7df61146100fb57005b6100766004356024356044356102f0565b60006000f35b61008760043561039e565b60006000f35b610098600435610178565b8073ffffffffffffffffffffffffffffffffffffffff1660005260206000f35b6100c96004356024356044356101a0565b8073ffffffffffffffffffffffffffffffffffffffff1660005260206000f35b6100f1610171565b8060005260206000f35b610106600435610133565b8360005282602052816040528073ffffffffffffffffffffffffffffffffffffffff1660605260806000f35b5b60006020819052908152604090208054600182015460028301546003909301549192909173ffffffffffffffffffffffffffffffffffffffff1684565b5b60015481565b5b60026020526000908152604090205473ffffffffffffffffffffffffffffffffffffffff1681565b73ffffffffffffffffffffffffffffffffffffffff831660009081526020819052604081206002015481908302341080156101fe575073ffffffffffffffffffffffffffffffffffffffff8516600090815260208190526040812054145b8015610232575073ffffffffffffffffffffffffffffffffffffffff85166000908152602081905260409020600101548390105b61023b57610243565b3391506102e8565b6101966103ca60003973ffffffffffffffffffffffffffffffffffffffff3381166101965285166101b68190526000908152602081905260408120600201546101d6526101f68490526102169080f073ffffffffffffffffffffffffffffffffffffffff8616600090815260208190526040902060030180547fffffffffffffffffffffffff0000000000000000000000000000000000000000168217905591508190505b509392505050565b73ffffffffffffffffffffffffffffffffffffffff33166000908152602081905260408120548190821461032357610364565b60018054808201909155600090815260026020526040902080547fffffffffffffffffffffffff000000000000000000000000000000000000000016331790555b50503373ffffffffffffffffffffffffffffffffffffffff1660009081526020819052604090209081556001810192909255600290910155565b3373ffffffffffffffffffffffffffffffffffffffff166000908152602081905260409020600201555600608061019660043960048051602451604451606451600080547fffffffffffffffffffffffff0000000000000000000000000000000000000000908116909517815560018054909516909317909355600355915561013390819061006390396000f3007c0100000000000000000000000000000000000000000000000000000000600035046347810fe381146100445780637e4a1aa81461005557806383d2421b1461006957005b61004f6004356100ab565b60006000f35b6100636004356024356100fc565b60006000f35b61007460043561007a565b60006000f35b6001543373ffffffffffffffffffffffffffffffffffffffff9081169116146100a2576100a8565b60078190555b50565b73ffffffffffffffffffffffffffffffffffffffff8116600090815260026020526040902080547fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff0016600117905550565b6001543373ffffffffffffffffffffffffffffffffffffffff9081169116146101245761012f565b600582905560068190555b505056"); Address address = Address.wrap(RandomUtils.nextBytes(Address.ADDRESS_LEN)); byte[] key_0 = ByteUtil.hexStringToBytes("18d63b70aa690ad37cb50908746c9a55"); @@ -197,48 +205,60 @@ public void test_2() throws Exception { AionContractDetailsImpl contractDetails_ = new AionContractDetailsImpl(data); - assertEquals(ByteUtil.toHexString(code), - ByteUtil.toHexString(contractDetails_.getCode())); + assertEquals(ByteUtil.toHexString(code), ByteUtil.toHexString(contractDetails_.getCode())); assertTrue(address.equals(contractDetails_.getAddress())); - assertEquals(ByteUtil.toHexString(val_1), + assertEquals( + ByteUtil.toHexString(val_1), ByteUtil.toHexString(contractDetails_.get(new DataWord(key_1)).getData())); - assertEquals(ByteUtil.toHexString(val_2), + assertEquals( + ByteUtil.toHexString(val_2), ByteUtil.toHexString(contractDetails_.get(new DataWord(key_2)).getData())); - assertEquals(ByteUtil.toHexString(val_3), + assertEquals( + ByteUtil.toHexString(val_3), ByteUtil.toHexString(contractDetails_.get(new DataWord(key_3)).getData())); - assertEquals(ByteUtil.toHexString(val_4), + assertEquals( + ByteUtil.toHexString(val_4), ByteUtil.toHexString(contractDetails_.get(new DataWord(key_4)).getData())); - assertEquals(ByteUtil.toHexString(val_5), + assertEquals( + ByteUtil.toHexString(val_5), ByteUtil.toHexString(contractDetails_.get(new DataWord(key_5)).getData())); - assertEquals(ByteUtil.toHexString(val_6), + assertEquals( + ByteUtil.toHexString(val_6), ByteUtil.toHexString(contractDetails_.get(new DataWord(key_6)).getData())); - assertEquals(ByteUtil.toHexString(val_7), + assertEquals( + ByteUtil.toHexString(val_7), ByteUtil.toHexString(contractDetails_.get(new DataWord(key_7)).getData())); - assertEquals(ByteUtil.toHexString(val_8), + assertEquals( + ByteUtil.toHexString(val_8), ByteUtil.toHexString(contractDetails_.get(new DataWord(key_8)).getData())); - assertEquals(ByteUtil.toHexString(val_9), + assertEquals( + ByteUtil.toHexString(val_9), ByteUtil.toHexString(contractDetails_.get(new DataWord(key_9)).getData())); - assertEquals(ByteUtil.toHexString(val_10), + assertEquals( + ByteUtil.toHexString(val_10), ByteUtil.toHexString(contractDetails_.get(new DataWord(key_10)).getData())); - assertEquals(ByteUtil.toHexString(val_11), + assertEquals( + ByteUtil.toHexString(val_11), ByteUtil.toHexString(contractDetails_.get(new DataWord(key_11)).getData())); - assertEquals(ByteUtil.toHexString(val_12), + assertEquals( + ByteUtil.toHexString(val_12), ByteUtil.toHexString(contractDetails_.get(new DataWord(key_12)).getData())); - assertEquals(ByteUtil.toHexString(val_13), + assertEquals( + ByteUtil.toHexString(val_13), ByteUtil.toHexString(contractDetails_.get(new DataWord(key_13)).getData())); } @@ -310,7 +330,6 @@ public void testExternalStorageTransition() { original.put(key, value); } - original.syncStorage(); assertTrue(!externalStorage.isEmpty()); diff --git a/modAionImpl/test/org/aion/zero/impl/BlockchainDataRecoveryTest.java b/modAionImpl/test/org/aion/zero/impl/BlockchainDataRecoveryTest.java index 993ba41f6a..86caf7f16c 100644 --- a/modAionImpl/test/org/aion/zero/impl/BlockchainDataRecoveryTest.java +++ b/modAionImpl/test/org/aion/zero/impl/BlockchainDataRecoveryTest.java @@ -1,4 +1,4 @@ -/******************************************************************************* +/* * Copyright (c) 2017-2018 Aion foundation. * * This file is part of the aion network project. @@ -17,21 +17,9 @@ * along with the aion network project source files. * If not, see . * - * The aion network project leverages useful source code from other - * open source projects. We greatly appreciate the effort that was - * invested in these projects and we thank the individual contributors - * for their work. For provenance information and contributors - * please see . - * - * Contributors to the aion source files in decreasing order of code volume: + * Contributors: * Aion foundation. - * team through the ethereumJ library. - * Ether.Camp Inc. (US) team through Ethereum Harmony. - * John Tromp through the Equihash solver. - * Samuel Neves through the BLAKE2 implementation. - * Zcash project team. - * Bitcoinj team. - ******************************************************************************/ + */ package org.aion.zero.impl; import org.aion.base.db.IByteArrayKeyValueDatabase; @@ -98,11 +86,12 @@ public void testRecoverWorldStateWithPartialWorldState() { AionBlock bestBlock = chain.getBestBlock(); assertThat(bestBlock.getNumber()).isEqualTo(NUMBER_OF_BLOCKS); - chain.getRepository().flush(); + AionRepositoryImpl repo = chain.getRepository(); + repo.flush(); // delete some world state root entries from the database - TrieImpl trie = (TrieImpl) ((AionRepositoryImpl) chain.getRepository()).getWorldState(); - IByteArrayKeyValueDatabase database = (IByteArrayKeyValueDatabase) trie.getCache().getDb(); + TrieImpl trie = (TrieImpl) repo.getWorldState(); + IByteArrayKeyValueDatabase database = repo.getStateDatabase(); for (byte[] key : statesToDelete) { database.delete(key); @@ -113,7 +102,7 @@ public void testRecoverWorldStateWithPartialWorldState() { assertThat(trie.isValidRoot(chain.getBestBlock().getStateRoot())).isFalse(); // call the recovery functionality - boolean worked = chain.recoverWorldState(chain.getRepository(), bestBlock); + boolean worked = chain.recoverWorldState(repo, bestBlock); // ensure that the blockchain is ok assertThat(chain.getBestBlockHash()).isEqualTo(bestBlock.getHash()); @@ -148,12 +137,13 @@ public void testRecoverWorldStateWithStartFromGenesis() { AionBlock bestBlock = chain.getBestBlock(); assertThat(bestBlock.getNumber()).isEqualTo(NUMBER_OF_BLOCKS); - chain.getRepository().flush(); + AionRepositoryImpl repo = chain.getRepository(); + repo.flush(); // System.out.println(Hex.toHexString(chain.getRepository().getRoot())); // delete some world state root entries from the database - TrieImpl trie = (TrieImpl) ((AionRepositoryImpl) chain.getRepository()).getWorldState(); - IByteArrayKeyValueDatabase database = (IByteArrayKeyValueDatabase) trie.getCache().getDb(); + TrieImpl trie = (TrieImpl) repo.getWorldState(); + IByteArrayKeyValueDatabase database = repo.getStateDatabase(); for (byte[] key : statesToDelete) { database.delete(key); @@ -165,7 +155,7 @@ public void testRecoverWorldStateWithStartFromGenesis() { assertThat(trie.isValidRoot(chain.getBestBlock().getStateRoot())).isFalse(); // call the recovery functionality - boolean worked = chain.recoverWorldState(chain.getRepository(), bestBlock); + boolean worked = chain.recoverWorldState(repo, bestBlock); // ensure that the blockchain is ok assertThat(chain.getBestBlockHash()).isEqualTo(bestBlock.getHash()); @@ -200,11 +190,12 @@ public void testRecoverWorldStateWithoutGenesis() { AionBlock bestBlock = chain.getBestBlock(); assertThat(bestBlock.getNumber()).isEqualTo(NUMBER_OF_BLOCKS); - chain.getRepository().flush(); + AionRepositoryImpl repo = chain.getRepository(); + repo.flush(); // delete some world state root entries from the database - TrieImpl trie = (TrieImpl) ((AionRepositoryImpl) chain.getRepository()).getWorldState(); - IByteArrayKeyValueDatabase database = (IByteArrayKeyValueDatabase) trie.getCache().getDb(); + TrieImpl trie = (TrieImpl) repo.getWorldState(); + IByteArrayKeyValueDatabase database = repo.getStateDatabase(); List statesToDelete = new ArrayList<>(); statesToDelete.addAll(database.keys()); @@ -218,7 +209,7 @@ public void testRecoverWorldStateWithoutGenesis() { assertThat(trie.isValidRoot(chain.getBestBlock().getStateRoot())).isFalse(); // call the recovery functionality - boolean worked = chain.recoverWorldState(chain.getRepository(), bestBlock); + boolean worked = chain.recoverWorldState(repo, bestBlock); // ensure that the blockchain is ok assertThat(chain.getBestBlockHash()).isEqualTo(bestBlock.getHash()); @@ -259,10 +250,10 @@ public void testRecoverIndexWithPartialIndex_MainChain() { AionBlock bestBlock = chain.getBestBlock(); assertThat(bestBlock.getNumber()).isEqualTo(NUMBER_OF_BLOCKS); - chain.getRepository().flush(); + AionRepositoryImpl repo = chain.getRepository(); + repo.flush(); // delete index entries from the database - AionRepositoryImpl repo = (AionRepositoryImpl) chain.getRepository(); IByteArrayKeyValueDatabase indexDatabase = repo.getIndexDatabase(); Map deletedInfo = new HashMap<>(); @@ -281,7 +272,7 @@ public void testRecoverIndexWithPartialIndex_MainChain() { assertThat(repo.isIndexed(bestBlock.getHash(), bestBlock.getNumber())).isFalse(); // call the recovery functionality - boolean worked = chain.recoverIndexEntry(chain.getRepository(), bestBlock); + boolean worked = chain.recoverIndexEntry(repo, bestBlock); // ensure that the blockchain is ok assertThat(chain.getBestBlockHash()).isEqualTo(bestBlock.getHash()); @@ -369,10 +360,10 @@ public void testRecoverIndexWithPartialIndex_ShorterSideChain() { assertThat(bestBlock.getNumber()).isEqualTo(NUMBER_OF_BLOCKS); assertThat(bestBlock.getHash()).isEqualTo(mainChainBlock.getHash()); - chain.getRepository().flush(); + AionRepositoryImpl repo = chain.getRepository(); + repo.flush(); // delete index entries from the database - AionRepositoryImpl repo = (AionRepositoryImpl) chain.getRepository(); IByteArrayKeyValueDatabase indexDatabase = repo.getIndexDatabase(); Map deletedInfo = new HashMap<>(); @@ -388,8 +379,7 @@ public void testRecoverIndexWithPartialIndex_ShorterSideChain() { } // call the recovery functionality for the main chain subsection - boolean worked = chain - .recoverIndexEntry(chain.getRepository(), chain.getBlockByHash(mainChainBlock.getParentHash())); + boolean worked = chain.recoverIndexEntry(repo, chain.getBlockByHash(mainChainBlock.getParentHash())); // ensure that the index was corrupted only for the side chain assertThat(repo.isIndexed(sideChainBlock.getHash(), sideChainBlock.getNumber())).isFalse(); @@ -397,7 +387,7 @@ public void testRecoverIndexWithPartialIndex_ShorterSideChain() { assertThat(worked).isTrue(); // call the recovery functionality - worked = chain.recoverIndexEntry(chain.getRepository(), sideChainBlock); + worked = chain.recoverIndexEntry(repo, sideChainBlock); // ensure that the blockchain is ok assertThat(chain.getBestBlockHash()).isEqualTo(bestBlock.getHash()); @@ -448,10 +438,10 @@ public void testRecoverIndexWithStartFromGenesis() { AionBlock bestBlock = chain.getBestBlock(); assertThat(bestBlock.getNumber()).isEqualTo(NUMBER_OF_BLOCKS); - chain.getRepository().flush(); + AionRepositoryImpl repo = chain.getRepository(); + repo.flush(); // delete index entries from the database - AionRepositoryImpl repo = (AionRepositoryImpl) chain.getRepository(); IByteArrayKeyValueDatabase indexDatabase = repo.getIndexDatabase(); Map deletedInfo = new HashMap<>(); @@ -470,7 +460,7 @@ public void testRecoverIndexWithStartFromGenesis() { assertThat(repo.isIndexed(bestBlock.getHash(), bestBlock.getNumber())).isFalse(); // call the recovery functionality - boolean worked = chain.recoverIndexEntry(chain.getRepository(), bestBlock); + boolean worked = chain.recoverIndexEntry(repo, bestBlock); // ensure that the blockchain is ok assertThat(chain.getBestBlockHash()).isEqualTo(bestBlock.getHash()); @@ -519,9 +509,9 @@ public void testRecoverIndexWithoutGenesis() { AionBlock bestBlock = chain.getBestBlock(); assertThat(bestBlock.getNumber()).isEqualTo(NUMBER_OF_BLOCKS); - chain.getRepository().flush(); + AionRepositoryImpl repo = chain.getRepository(); + repo.flush(); - AionRepositoryImpl repo = (AionRepositoryImpl) chain.getRepository(); IByteArrayKeyValueDatabase indexDatabase = repo.getIndexDatabase(); // deleting the entire index database @@ -531,7 +521,7 @@ public void testRecoverIndexWithoutGenesis() { assertThat(repo.isIndexed(bestBlock.getHash(), bestBlock.getNumber())).isFalse(); // call the recovery functionality - boolean worked = chain.recoverIndexEntry(chain.getRepository(), bestBlock); + boolean worked = chain.recoverIndexEntry(repo, bestBlock); // ensure that the best block is unchanged assertThat(chain.getBestBlockHash()).isEqualTo(bestBlock.getHash()); @@ -566,10 +556,10 @@ public void testRecoverIndexWithStartFromGenesisWithoutSize() { AionBlock bestBlock = chain.getBestBlock(); assertThat(bestBlock.getNumber()).isEqualTo(NUMBER_OF_BLOCKS); - chain.getRepository().flush(); + AionRepositoryImpl repo = chain.getRepository(); + repo.flush(); // delete index entries from the database - AionRepositoryImpl repo = (AionRepositoryImpl) chain.getRepository(); IByteArrayKeyValueDatabase indexDatabase = repo.getIndexDatabase(); Map deletedInfo = new HashMap<>(); @@ -592,7 +582,7 @@ public void testRecoverIndexWithStartFromGenesisWithoutSize() { assertThat(repo.isIndexed(bestBlock.getHash(), bestBlock.getNumber())).isFalse(); // call the recovery functionality - boolean worked = chain.recoverIndexEntry(chain.getRepository(), bestBlock); + boolean worked = chain.recoverIndexEntry(repo, bestBlock); // ensure that the blockchain is ok assertThat(chain.getBestBlockHash()).isEqualTo(bestBlock.getHash()); diff --git a/modAionImpl/test/org/aion/zero/impl/BlockchainForkingTest.java b/modAionImpl/test/org/aion/zero/impl/BlockchainForkingTest.java index d5e247c35e..0b82a9306b 100644 --- a/modAionImpl/test/org/aion/zero/impl/BlockchainForkingTest.java +++ b/modAionImpl/test/org/aion/zero/impl/BlockchainForkingTest.java @@ -39,12 +39,10 @@ import org.aion.crypto.ECKey; import org.aion.mcf.core.ImportResult; import org.aion.zero.impl.blockchain.ChainConfiguration; -import org.aion.zero.impl.db.AionRepositoryImpl; import org.aion.zero.impl.types.AionBlock; import org.junit.Test; import java.math.BigInteger; -import java.util.Collection; import java.util.Collections; import java.util.List; @@ -136,7 +134,7 @@ public void testInvalidFirstBlockDifficulty() { higherDifficultyBlock.getHeader().setDifficulty(difficulty.toByteArray()); System.out.println("before any processing: " + new ByteArrayWrapper(bc.getRepository().getRoot())); - System.out.println("trie: " + ((AionRepositoryImpl) bc.getRepository()).getWorldState().getTrieDump()); + System.out.println("trie: " + bc.getRepository().getWorldState().getTrieDump()); ImportResult result = bc.tryToConnect(standardBlock); assertThat(result).isEqualTo(ImportResult.IMPORTED_BEST); @@ -157,6 +155,12 @@ public void testInvalidFirstBlockDifficulty() { // the object reference here is intentional assertThat(bc.getBestBlock() == standardBlock).isTrue(); + + // check for correct state rollback + assertThat(bc.getRepository().getRoot()).isEqualTo(standardBlock.getStateRoot()); + assertThat(bc.getTotalDifficulty()) + .isEqualTo(bc.getRepository().getBlockStore().getTotalDifficultyForHash(standardBlock.getHash())); + } /*- @@ -226,7 +230,7 @@ public void testSecondBlockHigherDifficultyFork() { StandaloneBlockchain bc = bundle.bc; List accs = bundle.privateKeys; - // generate three blocks, on the third block we get flexiblity + // generate three blocks, on the third block we get flexibility // for what difficulties can occur BlockContext firstBlock = bc.createNewBlockInternal( @@ -274,4 +278,96 @@ public void testSecondBlockHigherDifficultyFork() { assertThat(bc.getBestBlock()).isEqualTo(fastBlockDescendant.block); } + + /** + * Test fork with exception. + */ + @Test + public void testSecondBlockHigherDifficultyFork_wExceptionOnFasterBlockAdd() { + StandaloneBlockchain.Builder builder = new StandaloneBlockchain.Builder(); + StandaloneBlockchain.Bundle bundle = builder.withValidatorConfiguration("simple").withDefaultAccounts().build(); + + long time = System.currentTimeMillis(); + + StandaloneBlockchain bc = bundle.bc; + + // generate three blocks, on the third block we get flexibility + // for what difficulties can occur + + BlockContext firstBlock = bc + .createNewBlockInternal(bc.getGenesis(), Collections.emptyList(), true, time / 1000L); + assertThat(bc.tryToConnectInternal(firstBlock.block, (time += 10))).isEqualTo(ImportResult.IMPORTED_BEST); + + // now connect the second block + BlockContext secondBlock = bc + .createNewBlockInternal(firstBlock.block, Collections.emptyList(), true, time / 1000L); + assertThat(bc.tryToConnectInternal(secondBlock.block, time += 10)).isEqualTo(ImportResult.IMPORTED_BEST); + + // now on the third block, we diverge with one block having higher TD than the other + BlockContext fasterSecondBlock = bc + .createNewBlockInternal(secondBlock.block, Collections.emptyList(), true, time / 1000L); + AionBlock slowerSecondBlock = new AionBlock(fasterSecondBlock.block); + + slowerSecondBlock.getHeader().setTimestamp(time / 1000L + 100); + + assertThat(bc.tryToConnectInternal(fasterSecondBlock.block, time + 100)).isEqualTo(ImportResult.IMPORTED_BEST); + assertThat(bc.tryToConnectInternal(slowerSecondBlock, time + 100)).isEqualTo(ImportResult.IMPORTED_NOT_BEST); + + time += 100; + + BlockContext fastBlockDescendant = bc + .createNewBlockInternal(fasterSecondBlock.block, Collections.emptyList(), true, time / 1000L); + BlockContext slowerBlockDescendant = bc + .createNewBlockInternal(slowerSecondBlock, Collections.emptyList(), true, time / 1000L + 100 + 1); + + // increment by another hundred (this is supposed to be when the slower block descendant is completed) + time += 100; + + assertThat(fastBlockDescendant.block.getDifficultyBI()) + .isGreaterThan(slowerBlockDescendant.block.getDifficultyBI()); + System.out.println("faster block descendant TD: " + fastBlockDescendant.block.getDifficultyBI()); + System.out.println("slower block descendant TD: " + slowerBlockDescendant.block.getDifficultyBI()); + + assertThat(bc.tryToConnectInternal(slowerBlockDescendant.block, time)).isEqualTo(ImportResult.IMPORTED_BEST); + + // corrupt the parent for the fast block descendant + bc.getRepository().getStateDatabase().delete(fasterSecondBlock.block.getStateRoot()); + assertThat(bc.getRepository().isValidRoot(fasterSecondBlock.block.getStateRoot())).isFalse(); + + // attempt adding the fastBlockDescendant + assertThat(bc.tryToConnectInternal(fastBlockDescendant.block, time)).isEqualTo(ImportResult.INVALID_BLOCK); + + // check for correct state rollback + assertThat(bc.getBestBlock()).isEqualTo(slowerBlockDescendant.block); + assertThat(bc.getRepository().getRoot()).isEqualTo(slowerBlockDescendant.block.getStateRoot()); + assertThat(bc.getTotalDifficulty()).isEqualTo(bc.getRepository().getBlockStore() + .getTotalDifficultyForHash(slowerBlockDescendant.block + .getHash())); + } + + @Test + public void testRollbackWithAddInvalidBlock() { + StandaloneBlockchain.Builder builder = new StandaloneBlockchain.Builder(); + StandaloneBlockchain.Bundle b = builder.withValidatorConfiguration("simple").build(); + + StandaloneBlockchain bc = b.bc; + AionBlock block = bc.createNewBlock(bc.getBestBlock(), Collections.emptyList(), true); + + assertThat(bc.tryToConnect(block)).isEqualTo(ImportResult.IMPORTED_BEST); + + // check that the returned block is the first block + assertThat(bc.getBestBlock() == block).isTrue(); + + AionBlock invalidBlock = bc.createNewBlock(bc.getBestBlock(), Collections.emptyList(), true); + invalidBlock.getHeader().setDifficulty(BigInteger.ONE.toByteArray()); + + // attempting to add invalid block + assertThat(bc.tryToConnect(invalidBlock)).isEqualTo(ImportResult.INVALID_BLOCK); + + // check for correct state rollback + assertThat(bc.getBestBlock()).isEqualTo(block); + assertThat(bc.getRepository().getRoot()).isEqualTo(block.getStateRoot()); + assertThat(bc.getTotalDifficulty()) + .isEqualTo(bc.getRepository().getBlockStore().getTotalDifficultyForHash(block.getHash())); + } } diff --git a/modAionImpl/test/org/aion/zero/impl/GenesisSpecificationTest.java b/modAionImpl/test/org/aion/zero/impl/GenesisSpecificationTest.java index 2082f12375..181c940843 100644 --- a/modAionImpl/test/org/aion/zero/impl/GenesisSpecificationTest.java +++ b/modAionImpl/test/org/aion/zero/impl/GenesisSpecificationTest.java @@ -1,4 +1,4 @@ -/******************************************************************************* +/* * Copyright (c) 2017-2018 Aion foundation. * * This file is part of the aion network project. @@ -31,7 +31,7 @@ * Samuel Neves through the BLAKE2 implementation. * Zcash project team. * Bitcoinj team. - ******************************************************************************/ + */ package org.aion.zero.impl; import static com.google.common.truth.Truth.assertThat; @@ -78,7 +78,7 @@ public void defaultGenesisBlockTest() throws HeaderStructureException { assertThat(genesis.getNrgLimit()).isEqualTo(AionGenesis.GENESIS_ENERGY_LIMIT); assertThat(genesis.getTxTrieRoot()).isEqualTo(HashUtil.EMPTY_TRIE_HASH); assertThat(genesis.getReceiptsRoot()).isEqualTo(HashUtil.EMPTY_TRIE_HASH); - assertThat(genesis.getCumulativeDifficulty()).isEqualTo(new BigInteger(1, AionGenesis.GENESIS_DIFFICULTY)); + assertThat(genesis.getDifficultyBI()).isEqualTo(new BigInteger(1, AionGenesis.GENESIS_DIFFICULTY)); assertThat(genesis.getTransactionsList().isEmpty()).isEqualTo(true); Map premined = genesis.getPremine(); @@ -128,7 +128,7 @@ public void overrideGenesisBlockTest() throws HeaderStructureException { assertThat(genesis.getNrgLimit()).isEqualTo(overrideValue.longValue()); assertThat(genesis.getTxTrieRoot()).isEqualTo(HashUtil.EMPTY_TRIE_HASH); assertThat(genesis.getReceiptsRoot()).isEqualTo(HashUtil.EMPTY_TRIE_HASH); - assertThat(genesis.getCumulativeDifficulty()).isEqualTo(overrideValue); + assertThat(genesis.getDifficultyBI()).isEqualTo(overrideValue); assertThat(genesis.getTransactionsList().isEmpty()).isEqualTo(true); assertThat(genesis.getPremine().keySet()).isEqualTo(accountStateSet); diff --git a/modAionImpl/test/org/aion/zero/impl/MockRepositoryConfig.java b/modAionImpl/test/org/aion/zero/impl/MockRepositoryConfig.java index 699260162a..15a29ea253 100644 --- a/modAionImpl/test/org/aion/zero/impl/MockRepositoryConfig.java +++ b/modAionImpl/test/org/aion/zero/impl/MockRepositoryConfig.java @@ -1,13 +1,14 @@ package org.aion.zero.impl; +import java.util.Properties; import org.aion.base.db.IContractDetails; +import org.aion.base.db.IPruneConfig; import org.aion.base.db.IRepositoryConfig; import org.aion.db.impl.DBVendor; import org.aion.db.impl.DatabaseFactory; +import org.aion.mcf.config.CfgPrune; import org.aion.zero.impl.db.ContractDetailsAion; -import java.util.Properties; - public class MockRepositoryConfig implements IRepositoryConfig { private DBVendor vendor = DBVendor.MOCKDB; @@ -17,8 +18,8 @@ public String getDbPath() { } @Override - public int getPrune() { - return 0; + public IPruneConfig getPruneConfig() { + return new CfgPrune(false); } @Override diff --git a/modAionImpl/test/org/aion/zero/impl/db/AionRepositoryImplTest.java b/modAionImpl/test/org/aion/zero/impl/db/AionRepositoryImplTest.java index 44f6aa4445..c05c59b414 100644 --- a/modAionImpl/test/org/aion/zero/impl/db/AionRepositoryImplTest.java +++ b/modAionImpl/test/org/aion/zero/impl/db/AionRepositoryImplTest.java @@ -1,4 +1,4 @@ -/******************************************************************************* +/* ****************************************************************************** * Copyright (c) 2017-2018 Aion foundation. * * This file is part of the aion network project. @@ -34,67 +34,68 @@ ******************************************************************************/ package org.aion.zero.impl.db; -import org.aion.base.db.IByteArrayKeyValueDatabase; -import org.aion.base.db.IContractDetails; -import org.aion.base.db.IRepositoryCache; -import org.aion.base.db.IRepositoryConfig; +import static com.google.common.truth.Truth.assertThat; + +import java.math.BigInteger; +import java.util.Optional; +import java.util.Properties; +import org.aion.base.db.*; import org.aion.base.type.Address; import org.aion.base.util.ByteUtil; -import org.aion.db.impl.DatabaseFactory; -import org.aion.db.impl.leveldb.LevelDBConstants; -import org.aion.mcf.core.AccountState; import org.aion.crypto.HashUtil; import org.aion.db.impl.DBVendor; +import org.aion.db.impl.DatabaseFactory; +import org.aion.mcf.config.CfgPrune; +import org.aion.mcf.core.AccountState; import org.aion.mcf.db.IBlockStoreBase; import org.aion.mcf.vm.types.DataWord; +import org.aion.zero.db.AionContractDetailsImpl; import org.junit.FixMethodOrder; import org.junit.Test; import org.junit.runners.MethodSorters; -import org.aion.zero.db.AionContractDetailsImpl; -import org.aion.zero.impl.db.AionRepositoryImpl; -import org.aion.zero.impl.db.ContractDetailsAion; - -import java.math.BigInteger; -import java.util.Optional; -import java.util.Properties; - -import static com.google.common.truth.Truth.assertThat; - @FixMethodOrder(MethodSorters.NAME_ASCENDING) public class AionRepositoryImplTest { - protected IRepositoryConfig repoConfig = new IRepositoryConfig() { - @Override - public String getDbPath() { - return ""; - } - - @Override - public int getPrune() { - return 0; - } - - @Override - public IContractDetails contractDetailsImpl() { - return ContractDetailsAion.createForTesting(0, 1000000).getDetails(); - } - - @Override - public Properties getDatabaseConfig(String db_name) { - Properties props = new Properties(); - props.setProperty(DatabaseFactory.Props.DB_TYPE, DBVendor.MOCKDB.toValue()); - props.setProperty(DatabaseFactory.Props.ENABLE_HEAP_CACHE, "false"); - return props; - } - }; + protected IRepositoryConfig repoConfig = + new IRepositoryConfig() { + @Override + public String getDbPath() { + return ""; + } + + @Override + public IPruneConfig getPruneConfig() { + return new CfgPrune(false); + } + + @Override + public IContractDetails contractDetailsImpl() { + return ContractDetailsAion.createForTesting(0, 1000000).getDetails(); + } + + @Override + public Properties getDatabaseConfig(String db_name) { + Properties props = new Properties(); + props.setProperty(DatabaseFactory.Props.DB_TYPE, DBVendor.MOCKDB.toValue()); + props.setProperty(DatabaseFactory.Props.ENABLE_HEAP_CACHE, "false"); + return props; + } + }; + + private static String value1 = + "CAF3CAF3CAF3CAF3CAF3CAF3CAF3CAF3CAF3CAF3CAF3CAF3CAF3CAF3CAF3CAF3"; + private static String value2 = + "CAFECAFECAFECAFECAFECAFECAFECAFECAFECAFECAFECAFECAFECAFECAFECAFE"; + private static String value3 = + "BEEFBEEFBEEFBEEFBEEFBEEFBEEFBEEFBEEFBEEFBEEFBEEFBEEFBEEFBEEFBEEF"; @Test public void testAccountStateUpdate() { AionRepositoryImpl repository = AionRepositoryImpl.createForTesting(repoConfig); byte[] originalRoot = repository.getRoot(); - Address defaultAccount = Address.wrap(ByteUtil.hexStringToBytes("CAF3CAF3CAF3CAF3CAF3CAF3CAF3CAF3CAF3CAF3CAF3CAF3CAF3CAF3CAF3CAF3")); + Address defaultAccount = Address.wrap(ByteUtil.hexStringToBytes(value1)); IRepositoryCache track = repository.startTracking(); track.addBalance(defaultAccount, BigInteger.valueOf(1)); @@ -112,7 +113,7 @@ public void testAccountAddCodeStorage() { AionRepositoryImpl repository = AionRepositoryImpl.createForTesting(repoConfig); IRepositoryCache track = repository.startTracking(); - Address defaultAccount = Address.wrap(ByteUtil.hexStringToBytes("CAF3CAF3CAF3CAF3CAF3CAF3CAF3CAF3CAF3CAF3CAF3CAF3CAF3CAF3CAF3CAF3")); + Address defaultAccount = Address.wrap(ByteUtil.hexStringToBytes(value1)); track.addBalance(defaultAccount, BigInteger.valueOf(1)); byte[] originalRoot = repository.getRoot(); @@ -133,7 +134,7 @@ public void testAccountStateUpdateStorageRow() { AionRepositoryImpl repository = AionRepositoryImpl.createForTesting(repoConfig); IRepositoryCache track = repository.startTracking(); - Address defaultAccount = Address.wrap(ByteUtil.hexStringToBytes("CAF3CAF3CAF3CAF3CAF3CAF3CAF3CAF3CAF3CAF3CAF3CAF3CAF3CAF3CAF3CAF3")); + Address defaultAccount = Address.wrap(ByteUtil.hexStringToBytes(value1)); track.addBalance(defaultAccount, BigInteger.valueOf(1)); // Consider the original root the one after an account has been added @@ -144,7 +145,8 @@ public void testAccountStateUpdateStorageRow() { track.flush(); - byte[] retrievedValue = repository.getStorageValue(defaultAccount, new DataWord(key)).getNoLeadZeroesData(); + byte[] retrievedValue = + repository.getStorageValue(defaultAccount, new DataWord(key)).getNoLeadZeroesData(); assertThat(retrievedValue).isEqualTo(value); byte[] newRoot = repository.getRoot(); @@ -158,7 +160,7 @@ public void testAccountStateUpdateStorageRowFlush() { AionRepositoryImpl repository = AionRepositoryImpl.createForTesting(repoConfig); IRepositoryCache track = repository.startTracking(); - Address defaultAccount = Address.wrap(ByteUtil.hexStringToBytes("CAF3CAF3CAF3CAF3CAF3CAF3CAF3CAF3CAF3CAF3CAF3CAF3CAF3CAF3CAF3CAF3")); + Address defaultAccount = Address.wrap(ByteUtil.hexStringToBytes(value1)); track.addBalance(defaultAccount, BigInteger.valueOf(1)); // Consider the original root the one after an account has been added @@ -172,9 +174,7 @@ public void testAccountStateUpdateStorageRowFlush() { repository.flush(); - /** - * Verify that the account has been flushed - */ + /** Verify that the account has been flushed */ IByteArrayKeyValueDatabase detailsDB = repository.getDetailsDatabase(); Optional serializedDetails = detailsDB.get(defaultAccount.toBytes()); @@ -185,20 +185,18 @@ public void testAccountStateUpdateStorageRowFlush() { assertThat(details.get(new DataWord(key))).isEqualTo(new DataWord(value)); } - /** - * Repo track test suite - */ - + /** Repo track test suite */ /** - * This test confirms that updates done on the repo track are successfully translated - * into the root repository. + * This test confirms that updates done on the repo track are successfully translated into the + * root repository. */ @Test public void testRepoTrackUpdateStorageRow() { final AionRepositoryImpl repository = AionRepositoryImpl.createForTesting(repoConfig); - final IRepositoryCache> repoTrack = repository.startTracking(); - final Address defaultAccount = Address.wrap(ByteUtil.hexStringToBytes("CAF3CAF3CAF3CAF3CAF3CAF3CAF3CAF3CAF3CAF3CAF3CAF3CAF3CAF3CAF3CAF3")); + final IRepositoryCache> repoTrack = + repository.startTracking(); + final Address defaultAccount = Address.wrap(ByteUtil.hexStringToBytes(value1)); final byte[] key = HashUtil.blake128("hello".getBytes()); final byte[] value = HashUtil.blake128("world".getBytes()); @@ -208,29 +206,24 @@ public void testRepoTrackUpdateStorageRow() { repoTrack.addStorageRow(defaultAccount, new DataWord(key), new DataWord(value)); - DataWord retrievedStorageValue = repoTrack.getStorageValue(defaultAccount, new DataWord(key)); + DataWord retrievedStorageValue = + repoTrack.getStorageValue(defaultAccount, new DataWord(key)); assertThat(retrievedStorageValue).isEqualTo(new DataWord(value)); // commit changes, then check that the root has updated repoTrack.flush(); - assertThat(repository.getStorageValue(defaultAccount, new DataWord(key))).isEqualTo(retrievedStorageValue); + assertThat(repository.getStorageValue(defaultAccount, new DataWord(key))) + .isEqualTo(retrievedStorageValue); final byte[] newRoot = repository.getRoot(); assertThat(newRoot).isNotEqualTo(originalRoot); } - /** - * Tests behaviour for trie when trying to revert to a previous root without - * first flushing. Note the behaviour here. Interestingly enough, it seems like - * the trie must first be flushed, so that the root node is in the caching/db layer. - * - * Otherwise the retrieval methods will not be able to find the temporal root value. - */ @Test public void testSyncToPreviousRootNoFlush() { - final Address FIRST_ACC = Address.wrap("CAFECAFECAFECAFECAFECAFECAFECAFECAFECAFECAFECAFECAFECAFECAFECAFE"); - final Address SECOND_ACC = Address.wrap("BEEFBEEFBEEFBEEFBEEFBEEFBEEFBEEFBEEFBEEFBEEFBEEFBEEFBEEFBEEFBEEF"); + final Address FIRST_ACC = Address.wrap(value2); + final Address SECOND_ACC = Address.wrap(value3); final AionRepositoryImpl repository = AionRepositoryImpl.createForTesting(repoConfig); byte[] originalRoot = repository.getRoot(); @@ -243,6 +236,10 @@ public void testSyncToPreviousRootNoFlush() { System.out.println("after first account added"); System.out.println(repository.getWorldState().getTrieDump()); + // check the update on the repo + BigInteger balance = repository.getBalance(FIRST_ACC); + assertThat(balance).isEqualTo(BigInteger.ONE); + byte[] firstRoot = repository.getRoot(); track = repository.startTracking(); @@ -257,18 +254,21 @@ public void testSyncToPreviousRootNoFlush() { assertThat(firstRoot).isNotEqualTo(originalRoot); assertThat(secondRoot).isNotEqualTo(firstRoot); + System.out.println("after sync to after first account added"); repository.syncToRoot(firstRoot); + assertThat(repository.isValidRoot(firstRoot)).isTrue(); + System.out.println(repository.getWorldState().getTrieDump()); assertThat(repository.getRoot()).isEqualTo(firstRoot); - BigInteger balance = repository.getBalance(FIRST_ACC); + balance = repository.getBalance(FIRST_ACC); // notice that the first blocks balance is also zero - assertThat(balance).isEqualTo(BigInteger.ZERO); + assertThat(balance).isEqualTo(BigInteger.ONE); } @Test public void testSyncToPreviousRootWithFlush() { - final Address FIRST_ACC = Address.wrap("CAFECAFECAFECAFECAFECAFECAFECAFECAFECAFECAFECAFECAFECAFECAFECAFE"); + final Address FIRST_ACC = Address.wrap(value2); AionRepositoryImpl repository = AionRepositoryImpl.createForTesting(repoConfig); byte[] originalRoot = repository.getRoot(); diff --git a/modAionImpl/test/org/aion/zero/impl/sync/BlockPropagationTest.java b/modAionImpl/test/org/aion/zero/impl/sync/BlockPropagationTest.java index 5b6fe0d228..96b9eb6e8c 100644 --- a/modAionImpl/test/org/aion/zero/impl/sync/BlockPropagationTest.java +++ b/modAionImpl/test/org/aion/zero/impl/sync/BlockPropagationTest.java @@ -1,21 +1,54 @@ +/* + * Copyright (c) 2017-2018 Aion foundation. + * + * This file is part of the aion network project. + * + * The aion network project is free software: you can redistribute it + * and/or modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation, either version 3 of + * the License, or any later version. + * + * The aion network project is distributed in the hope that it will + * be useful, but WITHOUT ANY WARRANTY; without even the implied + * warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + * See the GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with the aion network project source files. + * If not, see . + * + * Contributors: + * Aion foundation. + */ package org.aion.zero.impl.sync; import static com.google.common.truth.Truth.assertThat; import java.math.BigInteger; import java.nio.channels.SocketChannel; -import java.util.*; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; import java.util.concurrent.atomic.AtomicInteger; import org.aion.crypto.ECKey; import org.aion.crypto.ECKeyFac; import org.aion.crypto.HashUtil; -import org.aion.p2p.*; +import org.aion.p2p.Handler; +import org.aion.p2p.INode; +import org.aion.p2p.IP2pMgr; +import org.aion.p2p.IPeerMetric; +import org.aion.p2p.Msg; import org.aion.zero.impl.StandaloneBlockchain; import org.aion.zero.impl.sync.handler.BlockPropagationHandler; import org.aion.zero.impl.types.AionBlock; import org.junit.Test; -/** Unit tests for block propagation */ +/** + * Unit tests for block propagation + */ public class BlockPropagationTest { private static class NodeMock implements INode { @@ -50,7 +83,8 @@ public BigInteger getTotalDifficulty() { @Override public void updateStatus( - long _bestBlockNumber, byte[] _bestBlockHash, BigInteger _totalDifficulty) {} + long _bestBlockNumber, byte[] _bestBlockHash, BigInteger _totalDifficulty) { + } @Override public byte[] getIp() { @@ -81,6 +115,66 @@ public long getTimestamp() { public String getBinaryVersion() { return ""; } + + @Override + public void setPort(int _port) { + throw new IllegalStateException("not implemented"); + } + + @Override + public void setConnection(String _connection) { + throw new IllegalStateException("not implemented"); + } + + @Override + public IPeerMetric getPeerMetric() { + throw new IllegalStateException("not implemented"); + } + + @Override + public void refreshTimestamp() { + throw new IllegalStateException("not implemented"); + } + + @Override + public void setChannel(SocketChannel _channel) { + throw new IllegalStateException("not implemented"); + } + + @Override + public void setId(byte[] _id) { + throw new IllegalStateException("not implemented"); + } + + @Override + public void setBinaryVersion(String _revision) { + throw new IllegalStateException("not implemented"); + } + + @Override + public boolean getIfFromBootList() { + throw new IllegalStateException("not implemented"); + } + + @Override + public byte[] getBestBlockHash() { + throw new IllegalStateException("not implemented"); + } + + @Override + public String getConnection() { + throw new IllegalStateException("not implemented"); + } + + @Override + public SocketChannel getChannel() { + throw new IllegalStateException("not implemented"); + } + + @Override + public void setFromBootList(boolean _ifBoot) { + throw new IllegalStateException("not implemented"); + } } private static class P2pMock implements IP2pMgr { @@ -97,10 +191,12 @@ public Map getActiveNodes() { } @Override - public void shutdown() {} + public void shutdown() { + } @Override - public void run() {} + public void run() { + } @Override public List versions() { @@ -113,10 +209,12 @@ public int chainId() { } @Override - public void errCheck(int nodeIdHashcode, String _displayId) {} + public void errCheck(int nodeIdHashcode, String _displayId) { + } @Override - public void register(List _hs) {} + public void register(List _hs) { + } @Override public INode getRandom() { @@ -124,20 +222,51 @@ public INode getRandom() { } @Override - public void send(int _id, String s, Msg _msg) {} + public void send(int _id, String _displayId, Msg _msg) { + } @Override - public boolean isShowLog() { + public void closeSocket(SocketChannel _sc, String _reason) { + } + + @Override + public int getSelfIdHash() { + return 0; + } + + @Override + public void dropActive(int _nodeIdHash, String _reason) { + throw new IllegalStateException("not implemented."); + } + + @Override + public void configChannel(SocketChannel _channel) { + throw new IllegalStateException("not implemented."); + } + + @Override + public int getMaxActiveNodes() { + throw new IllegalStateException("not implemented."); + } + + @Override + public boolean isSyncSeedsOnly() { return false; } @Override - public void closeSocket(SocketChannel _sc, String _reason) {} + public int getMaxTempNodes() { throw new IllegalStateException("not implemented."); } @Override - public int getSelfIdHash() { - return 0; + public boolean validateNode(INode _node) { + throw new IllegalStateException("not implemented."); + } + + @Override + public int getSelfNetId() { + throw new IllegalStateException("not implemented."); } + } private static List generateDefaultAccounts() { @@ -148,19 +277,21 @@ private static List generateDefaultAccounts() { return accs; } - /** Test that we don't propagate back to the sender */ + /** + * Test that we don't propagate back to the sender + */ @Test public void testBlockPropagationReceiver() { List accounts = generateDefaultAccounts(); StandaloneBlockchain.Bundle bundle = - new StandaloneBlockchain.Builder() - .withValidatorConfiguration("simple") - .withDefaultAccounts(accounts) - .build(); + new StandaloneBlockchain.Builder() + .withValidatorConfiguration("simple") + .withDefaultAccounts(accounts) + .build(); AionBlock block = - bundle.bc.createNewBlock(bundle.bc.getGenesis(), Collections.EMPTY_LIST, true); + bundle.bc.createNewBlock(bundle.bc.getGenesis(), Collections.EMPTY_LIST, true); assertThat(block.getNumber()).isEqualTo(1); byte[] sender = HashUtil.h256("node1".getBytes()); @@ -170,29 +301,29 @@ public void testBlockPropagationReceiver() { node.put(1, senderMock); P2pMock p2pMock = - new P2pMock(node) { - @Override - public void send(int _nodeId, String s, Msg _msg) { - throw new RuntimeException("should not have called send"); - } - }; + new P2pMock(node) { + @Override + public void send(int _nodeId, String s, Msg _msg) { + throw new RuntimeException("should not have called send"); + } + }; StandaloneBlockchain.Bundle anotherBundle = - new StandaloneBlockchain.Builder() - .withValidatorConfiguration("simple") - .withDefaultAccounts(accounts) - .build(); + new StandaloneBlockchain.Builder() + .withValidatorConfiguration("simple") + .withDefaultAccounts(accounts) + .build(); BlockPropagationHandler handler = - new BlockPropagationHandler( - 1024, - anotherBundle.bc, // NOTE: not the same blockchain that generated the block - p2pMock, - anotherBundle.bc.getBlockHeaderValidator(), - false); + new BlockPropagationHandler( + 1024, + anotherBundle.bc, // NOTE: not the same blockchain that generated the block + p2pMock, + anotherBundle.bc.getBlockHeaderValidator(), + false); assertThat(handler.processIncomingBlock(senderMock.getIdHash(), "test", block)) - .isEqualTo(BlockPropagationHandler.PropStatus.CONNECTED); + .isEqualTo(BlockPropagationHandler.PropStatus.CONNECTED); } // given two peers, and one sends you a new block, propagate to the other @@ -201,13 +332,13 @@ public void testPropagateBlockToPeer() { List accounts = generateDefaultAccounts(); StandaloneBlockchain.Bundle bundle = - new StandaloneBlockchain.Builder() - .withValidatorConfiguration("simple") - .withDefaultAccounts(accounts) - .build(); + new StandaloneBlockchain.Builder() + .withValidatorConfiguration("simple") + .withDefaultAccounts(accounts) + .build(); AionBlock block = - bundle.bc.createNewBlock(bundle.bc.getGenesis(), Collections.EMPTY_LIST, true); + bundle.bc.createNewBlock(bundle.bc.getGenesis(), Collections.EMPTY_LIST, true); assertThat(block.getNumber()).isEqualTo(1); byte[] sender = HashUtil.h256("node1".getBytes()); @@ -222,20 +353,21 @@ public void testPropagateBlockToPeer() { AtomicInteger times = new AtomicInteger(); P2pMock p2pMock = - new P2pMock(node) { - @Override - public void send(int _nodeId, String s, Msg _msg) { - if (_nodeId != receiverMock.getIdHash()) - throw new RuntimeException("should only send to receiver"); - times.getAndIncrement(); + new P2pMock(node) { + @Override + public void send(int _nodeId, String s, Msg _msg) { + if (_nodeId != receiverMock.getIdHash()) { + throw new RuntimeException("should only send to receiver"); } - }; + times.getAndIncrement(); + } + }; StandaloneBlockchain.Bundle anotherBundle = - new StandaloneBlockchain.Builder() - .withValidatorConfiguration("simple") - .withDefaultAccounts(accounts) - .build(); + new StandaloneBlockchain.Builder() + .withValidatorConfiguration("simple") + .withDefaultAccounts(accounts) + .build(); assertThat(bundle.bc.genesis.getHash()).isEqualTo(anotherBundle.bc.genesis.getHash()); assertThat(block.getParentHash()).isEqualTo(bundle.bc.genesis.getHash()); @@ -245,16 +377,16 @@ public void send(int _nodeId, String s, Msg _msg) { assertThat(bestBlock.getHash()).isEqualTo(anotherBundle.bc.genesis.getHash()); BlockPropagationHandler handler = - new BlockPropagationHandler( - 1024, - anotherBundle.bc, // NOTE: not the same blockchain that generated the block - p2pMock, - anotherBundle.bc.getBlockHeaderValidator(), - false); + new BlockPropagationHandler( + 1024, + anotherBundle.bc, // NOTE: not the same blockchain that generated the block + p2pMock, + anotherBundle.bc.getBlockHeaderValidator(), + false); // block is processed assertThat(handler.processIncomingBlock(senderMock.getIdHash(), "test", block)) - .isEqualTo(BlockPropagationHandler.PropStatus.PROP_CONNECTED); + .isEqualTo(BlockPropagationHandler.PropStatus.PROP_CONNECTED); assertThat(times.get()).isEqualTo(1); } @@ -263,13 +395,13 @@ public void testIgnoreSameBlock() { List accounts = generateDefaultAccounts(); StandaloneBlockchain.Bundle bundle = - new StandaloneBlockchain.Builder() - .withValidatorConfiguration("simple") - .withDefaultAccounts(accounts) - .build(); + new StandaloneBlockchain.Builder() + .withValidatorConfiguration("simple") + .withDefaultAccounts(accounts) + .build(); AionBlock block = - bundle.bc.createNewBlock(bundle.bc.getGenesis(), Collections.EMPTY_LIST, true); + bundle.bc.createNewBlock(bundle.bc.getGenesis(), Collections.EMPTY_LIST, true); assertThat(block.getNumber()).isEqualTo(1); byte[] sender = HashUtil.h256("node1".getBytes()); @@ -284,36 +416,37 @@ public void testIgnoreSameBlock() { AtomicInteger times = new AtomicInteger(); P2pMock p2pMock = - new P2pMock(node) { - @Override - public void send(int _nodeId, String s, Msg _msg) { - if (_nodeId != receiverMock.getIdHash()) - throw new RuntimeException("should only send to receiver"); - times.getAndIncrement(); + new P2pMock(node) { + @Override + public void send(int _nodeId, String s, Msg _msg) { + if (_nodeId != receiverMock.getIdHash()) { + throw new RuntimeException("should only send to receiver"); } - }; + times.getAndIncrement(); + } + }; StandaloneBlockchain.Bundle anotherBundle = - new StandaloneBlockchain.Builder() - .withValidatorConfiguration("simple") - .withDefaultAccounts(accounts) - .build(); + new StandaloneBlockchain.Builder() + .withValidatorConfiguration("simple") + .withDefaultAccounts(accounts) + .build(); assertThat(bundle.bc.genesis.getHash()).isEqualTo(anotherBundle.bc.genesis.getHash()); BlockPropagationHandler handler = - new BlockPropagationHandler( - 1024, - anotherBundle.bc, // NOTE: not the same blockchain that generated the block - p2pMock, - anotherBundle.bc.getBlockHeaderValidator(), - false); + new BlockPropagationHandler( + 1024, + anotherBundle.bc, // NOTE: not the same blockchain that generated the block + p2pMock, + anotherBundle.bc.getBlockHeaderValidator(), + false); // block is processed assertThat(handler.processIncomingBlock(senderMock.getIdHash(), "test", block)) - .isEqualTo(BlockPropagationHandler.PropStatus.PROP_CONNECTED); + .isEqualTo(BlockPropagationHandler.PropStatus.PROP_CONNECTED); assertThat(handler.processIncomingBlock(senderMock.getIdHash(), "test", block)) - .isEqualTo(BlockPropagationHandler.PropStatus.DROPPED); + .isEqualTo(BlockPropagationHandler.PropStatus.DROPPED); assertThat(times.get()).isEqualTo(1); } @@ -323,13 +456,13 @@ public void testIgnoreSelfBlock() { List accounts = generateDefaultAccounts(); StandaloneBlockchain.Bundle bundle = - new StandaloneBlockchain.Builder() - .withValidatorConfiguration("simple") - .withDefaultAccounts(accounts) - .build(); + new StandaloneBlockchain.Builder() + .withValidatorConfiguration("simple") + .withDefaultAccounts(accounts) + .build(); AionBlock block = - bundle.bc.createNewBlock(bundle.bc.getGenesis(), Collections.EMPTY_LIST, true); + bundle.bc.createNewBlock(bundle.bc.getGenesis(), Collections.EMPTY_LIST, true); assertThat(block.getNumber()).isEqualTo(1); byte[] sender = HashUtil.h256("node1".getBytes()); @@ -340,26 +473,26 @@ public void testIgnoreSelfBlock() { AtomicInteger sendCount = new AtomicInteger(); P2pMock p2pMock = - new P2pMock(node) { - @Override - public void send(int _nodeId, String s, Msg _msg) { - sendCount.getAndIncrement(); - } - }; + new P2pMock(node) { + @Override + public void send(int _nodeId, String s, Msg _msg) { + sendCount.getAndIncrement(); + } + }; StandaloneBlockchain.Bundle anotherBundle = - new StandaloneBlockchain.Builder() - .withValidatorConfiguration("simple") - .withDefaultAccounts(accounts) - .build(); + new StandaloneBlockchain.Builder() + .withValidatorConfiguration("simple") + .withDefaultAccounts(accounts) + .build(); BlockPropagationHandler handler = - new BlockPropagationHandler( - 1024, - anotherBundle.bc, // NOTE: not the same blockchain that generated the block - p2pMock, - anotherBundle.bc.getBlockHeaderValidator(), - false); + new BlockPropagationHandler( + 1024, + anotherBundle.bc, // NOTE: not the same blockchain that generated the block + p2pMock, + anotherBundle.bc.getBlockHeaderValidator(), + false); // pretend that we propagate the new block handler.propagateNewBlock(block); // send counter incremented @@ -368,7 +501,7 @@ public void send(int _nodeId, String s, Msg _msg) { // so our blockchain should view this block as a new block // therefore if the filter fails, this block will actually be CONNECTED assertThat(handler.processIncomingBlock(senderMock.getIdHash(), "test", block)) - .isEqualTo(BlockPropagationHandler.PropStatus.DROPPED); + .isEqualTo(BlockPropagationHandler.PropStatus.DROPPED); // we expect the counter to be incremented once (on propagation) assertThat(sendCount.get()).isEqualTo(1); diff --git a/modApiServer/build.xml b/modApiServer/build.xml index 63423de0a7..dfb936a416 100644 --- a/modApiServer/build.xml +++ b/modApiServer/build.xml @@ -35,7 +35,7 @@ - @@ -63,7 +63,7 @@ - @@ -93,31 +93,4 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/modApiServer/src/org/aion/api/server/Api.java b/modApiServer/src/org/aion/api/server/Api.java index 743ba51e9e..2dc1402a3b 100644 --- a/modApiServer/src/org/aion/api/server/Api.java +++ b/modApiServer/src/org/aion/api/server/Api.java @@ -109,6 +109,14 @@ public Map contract_compileSolidity(final String _contrac Compiler.Result res = solc.compile(_contract.getBytes(), Compiler.Options.ABI, Compiler.Options.BIN); if (res.isFailed()) { LOG.info("contract compile error: [{}]", res.errors); + + /** + * Enhance performance by separating the log threads and kernel + * TODO: Implement a queue for strings + * TODO: Put every LOG message onto the queue + * TODO: Use a thread service to process these message + */ + CompiledContr ret = new CompiledContr(); ret.error = res.errors; compiledContracts.put("compile-error", ret); diff --git a/modApiServer/src/org/aion/api/server/ApiAion.java b/modApiServer/src/org/aion/api/server/ApiAion.java index 7f063b9716..4046facdc0 100644 --- a/modApiServer/src/org/aion/api/server/ApiAion.java +++ b/modApiServer/src/org/aion/api/server/ApiAion.java @@ -24,6 +24,21 @@ package org.aion.api.server; +import static org.aion.evtmgr.impl.evt.EventTx.STATE.GETSTATE; + +import java.math.BigInteger; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.atomic.AtomicLong; +import java.util.concurrent.locks.ReentrantLock; +import java.util.stream.Collectors; +import java.util.stream.Stream; import org.aion.api.server.nrgprice.NrgOracle; import org.aion.api.server.types.ArgTxCall; import org.aion.api.server.types.Fltr; @@ -42,13 +57,13 @@ import org.aion.evtmgr.impl.es.EventExecuteService; import org.aion.evtmgr.impl.evt.EventBlock; import org.aion.evtmgr.impl.evt.EventTx; -import org.aion.zero.impl.AionBlockchainImpl; import org.aion.zero.impl.AionGenesis; import org.aion.zero.impl.BlockContext; import org.aion.zero.impl.Version; import org.aion.zero.impl.blockchain.AionPendingStateImpl; import org.aion.zero.impl.blockchain.IAionChain; import org.aion.zero.impl.config.CfgAion; +import org.aion.zero.impl.core.IAionBlockchain; import org.aion.zero.impl.db.AionBlockStore; import org.aion.zero.impl.types.AionBlock; import org.aion.zero.impl.types.AionBlockSummary; @@ -56,16 +71,6 @@ import org.aion.zero.types.AionTransaction; import org.aion.zero.types.AionTxReceipt; -import java.math.BigInteger; -import java.util.*; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.atomic.AtomicLong; -import java.util.concurrent.locks.ReentrantLock; -import java.util.stream.Collectors; -import java.util.stream.Stream; - -import static org.aion.evtmgr.impl.evt.EventTx.STATE.GETSTATE; - public abstract class ApiAion extends Api { // these variables get accessed by the api worker threads. @@ -74,7 +79,7 @@ public abstract class ApiAion extends Api { // 2. underlying datastructure provides concurrency guarntees // delegate concurrency to underlying object - protected NrgOracle nrgOracle; + protected static NrgOracle NRG_ORACLE; protected IAionChain ac; // assumption: blockchainImpl et al. provide concurrency guarantee // using java.util.concurrent library objects @@ -658,16 +663,33 @@ protected boolean setReportedHashrate(String hashrate, String clientId) { return false; } + // Returns a fully initialized NrgOracle object. + protected void initNrgOracle(IAionChain _ac) { + if (NRG_ORACLE != null) { + return; + } + + IAionBlockchain bc = (IAionBlockchain)_ac.getBlockchain(); + long nrgPriceDefault = CfgAion.inst().getApi().getNrg().getNrgPriceDefault(); + long nrgPriceMax = CfgAion.inst().getApi().getNrg().getNrgPriceMax(); + + NrgOracle.Strategy oracleStrategy = NrgOracle.Strategy.SIMPLE; + if (CfgAion.inst().getApi().getNrg().isOracleEnabled()) + oracleStrategy = NrgOracle.Strategy.BLK_PRICE; + + NRG_ORACLE = new NrgOracle(bc, nrgPriceDefault, nrgPriceMax, oracleStrategy); + } + protected long getRecommendedNrgPrice() { - if (this.nrgOracle != null) - return this.nrgOracle.getNrgPrice(); + if (NRG_ORACLE != null) + return NRG_ORACLE.getNrgPrice(); else return CfgAion.inst().getApi().getNrg().getNrgPriceDefault(); } // leak the oracle instance. NrgOracle is threadsafe, so safe to do this, but bad design protected NrgOracle getNrgOracle() { - return this.nrgOracle; + return NRG_ORACLE; } protected long getDefaultNrgLimit() { diff --git a/modApiServer/src/org/aion/api/server/http/ApiWeb3Aion.java b/modApiServer/src/org/aion/api/server/http/ApiWeb3Aion.java index 1313fd7e7e..f6e07f4d26 100644 --- a/modApiServer/src/org/aion/api/server/http/ApiWeb3Aion.java +++ b/modApiServer/src/org/aion/api/server/http/ApiWeb3Aion.java @@ -1,4 +1,4 @@ -/******************************************************************************* +/* ****************************************************************************** * Copyright (c) 2017-2018 Aion foundation. * * This file is part of the aion network project. @@ -19,36 +19,79 @@ * * Contributors: * Aion foundation. - * + * ******************************************************************************/ package org.aion.api.server.http; +import static org.aion.base.util.ByteUtil.hexStringToBytes; +import static org.aion.base.util.ByteUtil.toHexString; + import com.google.common.cache.CacheBuilder; import com.google.common.cache.CacheLoader; import com.google.common.cache.LoadingCache; import com.google.common.util.concurrent.ListenableFuture; import com.google.common.util.concurrent.ListenableFutureTask; +import java.math.BigDecimal; +import java.math.BigInteger; +import java.math.RoundingMode; +import java.time.Instant; +import java.time.ZoneOffset; +import java.util.Collections; +import java.util.HashMap; +import java.util.LinkedList; +import java.util.List; +import java.util.ListIterator; +import java.util.Map; +import java.util.concurrent.ArrayBlockingQueue; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.ThreadFactory; +import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.locks.ReadWriteLock; +import java.util.concurrent.locks.ReentrantReadWriteLock; import org.aion.api.server.ApiAion; -import org.aion.api.server.nrgprice.NrgOracle; import org.aion.api.server.rpc.RpcError; import org.aion.api.server.rpc.RpcMsg; -import org.aion.api.server.types.*; +import org.aion.api.server.types.ArgFltr; +import org.aion.api.server.types.ArgTxCall; +import org.aion.api.server.types.Blk; +import org.aion.api.server.types.CompiledContr; +import org.aion.api.server.types.Evt; +import org.aion.api.server.types.Fltr; +import org.aion.api.server.types.FltrBlk; +import org.aion.api.server.types.FltrLg; +import org.aion.api.server.types.FltrTx; +import org.aion.api.server.types.NumericalValue; +import org.aion.api.server.types.SyncInfo; +import org.aion.api.server.types.Tx; +import org.aion.api.server.types.TxRecpt; import org.aion.base.db.IRepository; import org.aion.base.type.Address; import org.aion.base.type.Hash256; import org.aion.base.type.ITransaction; import org.aion.base.type.ITxReceipt; -import org.aion.base.util.*; +import org.aion.base.util.ByteArrayWrapper; +import org.aion.base.util.ByteUtil; +import org.aion.base.util.FastByteComparisons; +import org.aion.base.util.TypeConverter; +import org.aion.base.util.Utils; import org.aion.crypto.ECKey; import org.aion.crypto.HashUtil; import org.aion.evtmgr.IEventMgr; import org.aion.evtmgr.IHandler; import org.aion.evtmgr.impl.callback.EventCallback; import org.aion.evtmgr.impl.evt.EventTx; -import org.aion.mcf.config.*; import org.aion.mcf.account.Keystore; +import org.aion.mcf.config.CfgApi; +import org.aion.mcf.config.CfgApiNrg; +import org.aion.mcf.config.CfgApiRpc; +import org.aion.mcf.config.CfgApiZmq; +import org.aion.mcf.config.CfgNet; import org.aion.mcf.config.CfgNetP2p; +import org.aion.mcf.config.CfgSync; +import org.aion.mcf.config.CfgTx; import org.aion.mcf.core.AccountState; import org.aion.mcf.core.ImportResult; import org.aion.mcf.vm.types.DataWord; @@ -62,7 +105,6 @@ import org.aion.zero.impl.config.CfgAion; import org.aion.zero.impl.config.CfgConsensusPow; import org.aion.zero.impl.config.CfgEnergyStrategy; -import org.aion.zero.impl.core.IAionBlockchain; import org.aion.zero.impl.db.AionBlockStore; import org.aion.zero.impl.db.AionRepositoryImpl; import org.aion.zero.impl.sync.PeerState; @@ -76,21 +118,6 @@ import org.json.JSONArray; import org.json.JSONObject; -import java.math.BigDecimal; -import java.math.BigInteger; -import java.math.RoundingMode; -import java.time.Instant; -import java.time.ZoneOffset; -import java.util.*; -import java.util.HashMap; -import java.util.concurrent.*; -import java.util.concurrent.atomic.AtomicInteger; -import java.util.concurrent.locks.ReadWriteLock; -import java.util.concurrent.locks.ReentrantReadWriteLock; - -import static org.aion.base.util.ByteUtil.hexStringToBytes; -import static org.aion.base.util.ByteUtil.toHexString; - /** * @author chris lin, ali sharif * TODO: make implementation pass all spec tests: https://github.com/ethereum/rpc-tests @@ -172,17 +199,7 @@ public ApiWeb3Aion(final IAionChain _ac) { isFilterEnabled = CfgAion.inst().getApi().getRpc().isFiltersEnabled(); isSeedMode = CfgAion.inst().getConsensus().isSeed(); - - // instantiate nrg price oracle - IAionBlockchain bc = (IAionBlockchain)_ac.getBlockchain(); - long nrgPriceDefault = CfgAion.inst().getApi().getNrg().getNrgPriceDefault(); - long nrgPriceMax = CfgAion.inst().getApi().getNrg().getNrgPriceMax(); - - NrgOracle.Strategy oracleStrategy = NrgOracle.Strategy.SIMPLE; - if (CfgAion.inst().getApi().getNrg().isOracleEnabled()) - oracleStrategy = NrgOracle.Strategy.BLK_PRICE; - - this.nrgOracle = new NrgOracle(bc, nrgPriceDefault, nrgPriceMax, oracleStrategy); + initNrgOracle(_ac); if (isFilterEnabled) { evtMgr = this.ac.getAionHub().getEventMgr(); @@ -207,27 +224,24 @@ public ApiWeb3Aion(final IAionChain _ac) { .maximumSize(1) .refreshAfterWrite(OPS_RECENT_ENTITY_CACHE_TIME_SECONDS, TimeUnit.SECONDS) .build( - new CacheLoader() { - public ChainHeadView load(Integer key) { // no checked exception - ChainHeadView view = new ChainHeadView(OPS_RECENT_ENTITY_COUNT).update(); - return view; - } + new CacheLoader<>() { + public ChainHeadView load(Integer key) { // no checked exception + return new ChainHeadView(OPS_RECENT_ENTITY_COUNT).update(); + } - public ListenableFuture reload(final Integer key, ChainHeadView prev) { - try { - ListenableFutureTask task = ListenableFutureTask.create(new Callable() { - public ChainHeadView call() { - return new ChainHeadView(prev).update(); - } - }); - cacheUpdateExecutor.execute(task); - return task; - } catch (Throwable e) { - LOG.debug(" reload(final Integer key, + ChainHeadView prev) { + try { + ListenableFutureTask task = ListenableFutureTask + .create(() -> new ChainHeadView(prev).update()); + cacheUpdateExecutor.execute(task); + return task; + } catch (Throwable e) { + LOG.debug("(1), new CacheUpdateThreadFactory()); @@ -246,11 +260,8 @@ public MinerStatsView load(String key) { // no checked exception public ListenableFuture reload(final String key, MinerStatsView prev) { try { - ListenableFutureTask task = ListenableFutureTask.create(new Callable() { - public MinerStatsView call() { - return new MinerStatsView(prev).update(); - } - }); + ListenableFutureTask task = ListenableFutureTask.create( + () -> new MinerStatsView(prev).update()); MinerStatsExecutor.execute(task); return task; } catch (Throwable e) { @@ -271,7 +282,7 @@ public MinerStatsView call() { /* Return a reference to the AIONBlock without converting values to hex * Requied for the mining pool implementation */ - AionBlock getBlockRaw(int bn) { + private AionBlock getBlockRaw(int bn) { // long bn = this.parseBnOrId(_bnOrId); AionBlock nb = this.ac.getBlockchain().getBlockByNumber(bn); if (nb == null) { @@ -399,7 +410,7 @@ else if (_params instanceof JSONObject) { Address address = new Address(_address); String bnOrId = "latest"; - if (_bnOrId != null && !_bnOrId.equals(null)) + if (_bnOrId != null) bnOrId = _bnOrId + ""; if (!bnOrId.equalsIgnoreCase("latest")) { @@ -437,7 +448,7 @@ else if (_params instanceof JSONObject) { Address address = new Address(_address); String bnOrId = "latest"; - if (_bnOrId != null && !_bnOrId.equals(null)) + if (_bnOrId != null) bnOrId = _bnOrId + ""; DataWord key; @@ -486,7 +497,7 @@ else if (_params instanceof JSONObject) { Address address = new Address(_address); String bnOrId = "latest"; - if (_bnOrId != null && !_bnOrId.equals(null)) + if (_bnOrId != null) bnOrId = _bnOrId + ""; if (!bnOrId.equalsIgnoreCase("latest")) { @@ -573,7 +584,7 @@ else if (_params instanceof JSONObject) { Address address = new Address(_address); String bnOrId = "latest"; - if (_bnOrId != null && !_bnOrId.equals(null)) + if (_bnOrId != null) bnOrId = _bnOrId + ""; if (!bnOrId.equalsIgnoreCase("latest")) { @@ -684,7 +695,7 @@ else if (_params instanceof JSONObject) { ArgTxCall txParams = ArgTxCall.fromJSON(_tx, getNrgOracle(), getDefaultNrgLimit()); String bnOrId = "latest"; - if (_bnOrId != null && !_bnOrId.equals(null)) + if (_bnOrId != null) bnOrId = _bnOrId + ""; Long bn = parseBnOrId(bnOrId); @@ -1138,7 +1149,7 @@ else if (_params instanceof JSONObject) { int duration = 300; - if (_duration != null && !_duration.equals(null)) + if (_duration != null) duration = new BigInteger(_duration + "").intValueExact(); return new RpcMsg(unlockAccount(_account, _password, duration)); @@ -1517,8 +1528,6 @@ private static JSONObject configNet() { p2p.put("errorTolerance", configP2p.getErrorTolerance()); p2p.put("maxActiveNodes", configP2p.getMaxActiveNodes()); p2p.put("maxTempNodes", configP2p.getMaxTempNodes()); - p2p.put("showLog", configP2p.getShowLog()); - p2p.put("showStatus", configP2p.getShowStatus()); // end obj.put("p2p", p2p); @@ -1698,7 +1707,6 @@ public ChainHeadView(int _qSize) { } private JSONObject getJson(AionBlock _b) { - Map.Entry response; BigInteger totalDiff = ac.getAionHub().getBlockStore().getTotalDifficultyForHash(_b.getHash()); return Blk.AionBlockOnlyToJson(_b, totalDiff); } @@ -1824,7 +1832,7 @@ private JSONObject computeMetrics() { return metrics; } - public ChainHeadView update() { + ChainHeadView update() { // get the latest head AionBlock blk = getBestBlock(); @@ -1849,7 +1857,7 @@ public ChainHeadView update() { " blkHash: " + TypeConverter.toJsonHex(blk.getHash())); */ - while(FastByteComparisons.equal(hashQueue.peekFirst(), blk.getParentHash()) == false + while(!FastByteComparisons.equal(hashQueue.peekFirst(), blk.getParentHash()) && itr < qSize && blk.getNumber() > 2) { @@ -1900,11 +1908,11 @@ public ChainHeadView update() { return this; } - public JSONObject getResponse() { + JSONObject getResponse() { return response; } - public long getViewBestBlock() { + long getViewBestBlock() { return blkObjList.get(hashQueue.peekFirst()).getNumber(); } } @@ -2292,7 +2300,7 @@ public RpcMsg stratum_getHeaderByBlockNumber(Object _params) { JSONObject obj = new JSONObject(); - if (_blockNum != null && !_blockNum.equals(null)) { + if (_blockNum != null) { String bnStr = _blockNum + ""; try { int bnInt = Integer.decode(bnStr); @@ -2328,7 +2336,7 @@ private class MinerStatsView { private int qSize; private byte[] miner; - public MinerStatsView(MinerStatsView cv) { + MinerStatsView(MinerStatsView cv) { hashQueue = new LinkedList<>(cv.hashQueue); blocks = new HashMap<>(cv.blocks); response = new JSONObject(cv.response, JSONObject.getNames(cv.response)); @@ -2336,7 +2344,7 @@ public MinerStatsView(MinerStatsView cv) { miner = cv.miner; } - public MinerStatsView(int _qSize, byte[] _miner) { + MinerStatsView(int _qSize, byte[] _miner) { hashQueue = new LinkedList<>(); blocks = new HashMap<>(); response = new JSONObject(); @@ -2413,7 +2421,7 @@ private JSONObject buildResponse() { return o; } - public MinerStatsView update() { + MinerStatsView update() { // get the latest head AionBlock blk = getBestBlock(); @@ -2440,7 +2448,7 @@ public MinerStatsView update() { " blkHash: " + TypeConverter.toJsonHex(blk.getHash())); */ - while(FastByteComparisons.equal(hashQueue.peekFirst(), blk.getParentHash()) == false + while(!FastByteComparisons.equal(hashQueue.peekFirst(), blk.getParentHash()) && itr < qSize && blk.getNumber() > 2) { @@ -2485,7 +2493,7 @@ public MinerStatsView update() { return this; } - public JSONObject getResponse() { + JSONObject getResponse() { return response; } } diff --git a/modApiServer/src/org/aion/api/server/nrgprice/strategy/NrgBlockPriceAveraging.java b/modApiServer/src/org/aion/api/server/nrgprice/strategy/NrgBlockPriceAveraging.java index c6de9c1fb2..2f34be3ddb 100644 --- a/modApiServer/src/org/aion/api/server/nrgprice/strategy/NrgBlockPriceAveraging.java +++ b/modApiServer/src/org/aion/api/server/nrgprice/strategy/NrgBlockPriceAveraging.java @@ -27,6 +27,8 @@ * This class is NOT thread-safe * Policy: holder class (NrgOracle) should provide any concurrency guarantees it needs to * + * NOT TESTED. DON'T USE. + * * @author ali sharif */ public class NrgBlockPriceAveraging extends NrgPriceAdvisor { diff --git a/modApiServer/src/org/aion/api/server/pb/ApiAion0.java b/modApiServer/src/org/aion/api/server/pb/ApiAion0.java index 17bd085601..a2ff8f9b4c 100644 --- a/modApiServer/src/org/aion/api/server/pb/ApiAion0.java +++ b/modApiServer/src/org/aion/api/server/pb/ApiAion0.java @@ -19,7 +19,7 @@ * * Contributors: * Aion foundation. - * + * ******************************************************************************/ package org.aion.api.server.pb; @@ -50,6 +50,7 @@ import org.aion.api.server.ApiUtil; import org.aion.api.server.IApiAion; import org.aion.api.server.pb.Message.Retcode; +import org.aion.api.server.pb.Message.Servs; import org.aion.api.server.types.ArgTxCall; import org.aion.api.server.types.CompiledContr; import org.aion.api.server.types.EvtContract; @@ -96,16 +97,16 @@ @SuppressWarnings("Duplicates") public class ApiAion0 extends ApiAion implements IApiAion { - public final static byte JAVAAPI_VAR = 2; - private final static int JAVAAPI_REQHEADER_LEN = 4; - private final static int TX_HASH_LEN = 32; - private final static int ACCOUNT_CREATE_LIMIT = 100; + public static final byte JAVAAPI_VAR = 2; + private static final int JAVAAPI_REQHEADER_LEN = 4; + private static final int TX_HASH_LEN = 32; + private static final int ACCOUNT_CREATE_LIMIT = 100; private BlockingQueue pendingStatus; private BlockingQueue txWait; private Map> msgIdMapping; - static public boolean heartBeatMsg(byte[] msg) { + public static boolean heartBeatMsg(byte[] msg) { if (msg == null || msg.length != JAVAAPI_REQHEADER_LEN) { return false; } @@ -129,34 +130,66 @@ protected void onBlock(AionBlockSummary cbs) { } else { List txrs = cbs.getReceipts(); if (fltr.getType() == Fltr.Type.EVENT - && !Optional.ofNullable(txrs).orElse(Collections.emptyList()).isEmpty()) { + && !Optional.ofNullable(txrs).orElse(Collections.emptyList()).isEmpty()) { FltrCt _fltr = (FltrCt) fltr; for (AionTxReceipt txr : txrs) { AionTransaction tx = txr.getTransaction(); - Address contractAddress = Optional.ofNullable(tx.getTo()) - .orElse(tx.getContractAddress()); + Address contractAddress = + Optional.ofNullable(tx.getTo()).orElse(tx.getContractAddress()); Integer cnt = 0; - txr.getLogInfoList().forEach(bi -> bi.getTopics().forEach(lg -> { - if (_fltr.isFor(contractAddress, ByteUtil.toHexString(lg))) { - IBlock blk = (cbs).getBlock(); - List txList = blk.getTransactionsList(); - int insideCnt = 0; - for (AionTransaction t : txList) { - if (Arrays.equals(t.getHash(), tx.getHash())) { - break; - } - insideCnt++; - } - - EvtContract ec = new EvtContract(bi.getAddress().toBytes(), - bi.getData(), blk.getHash(), blk.getNumber(), cnt, - ByteUtil.toHexString(lg), false, insideCnt, tx.getHash()); - - _fltr.add(ec); - } - })); + txr.getLogInfoList() + .forEach( + bi -> + bi.getTopics() + .forEach( + lg -> { + if (_fltr.isFor( + contractAddress, + ByteUtil.toHexString( + lg))) { + IBlock + blk = + (cbs) + .getBlock(); + List + txList = + blk + .getTransactionsList(); + int insideCnt = 0; + for (AionTransaction t : + txList) { + if (Arrays.equals( + t.getHash(), + tx.getHash())) { + break; + } + insideCnt++; + } + + EvtContract ec = + new EvtContract( + bi.getAddress() + .toBytes(), + bi + .getData(), + blk + .getHash(), + blk + .getNumber(), + cnt, + ByteUtil + .toHexString( + lg), + false, + insideCnt, + tx + .getHash()); + + _fltr.add(ec); + } + })); } } } @@ -164,41 +197,52 @@ protected void onBlock(AionBlockSummary cbs) { } protected void pendingTxReceived(ITransaction _tx) { - installedFilters.values().forEach((f) -> { - if (f.getType() == Fltr.Type.TRANSACTION) { - f.add(new EvtTx(_tx)); - } - }); + installedFilters + .values() + .forEach( + (f) -> { + if (f.getType() == Fltr.Type.TRANSACTION) { + f.add(new EvtTx(_tx)); + } + }); } protected void pendingTxUpdate(ITxReceipt _txRcpt, EventTx.STATE _state) { - ByteArrayWrapper txHashW = ByteArrayWrapper.wrap( - ((AionTxReceipt) _txRcpt).getTransaction().getHash()); + ByteArrayWrapper txHashW = + ByteArrayWrapper.wrap(((AionTxReceipt) _txRcpt).getTransaction().getHash()); if (LOG.isTraceEnabled()) { - LOG.trace("ApiAion0.onPendingTransactionUpdate - txHash: [{}], state: [{}]", txHashW.toString(), _state.getValue()); + LOG.trace( + "ApiAion0.onPendingTransactionUpdate - txHash: [{}], state: [{}]", + txHashW.toString(), + _state.getValue()); } if (getMsgIdMapping().get(txHashW) != null) { if (pendingStatus.remainingCapacity() == 0) { pendingStatus.poll(); LOG.warn( - "ApiAion0.onPendingTransactionUpdate - txPend ingStatus queue full, drop the first message."); + "ApiAion0.onPendingTransactionUpdate - txPend ingStatus queue full, drop the first message."); } if (LOG.isTraceEnabled()) { - LOG.trace("ApiAion0.onPendingTransactionUpdate - the pending Tx state : [{}]", _state.getValue()); + LOG.trace( + "ApiAion0.onPendingTransactionUpdate - the pending Tx state : [{}]", + _state.getValue()); } - pendingStatus.add(new TxPendingStatus(txHashW, + pendingStatus.add( + new TxPendingStatus( + txHashW, getMsgIdMapping().get(txHashW).getValue(), getMsgIdMapping().get(txHashW).getKey(), _state.getValue(), - ByteArrayWrapper.wrap(((AionTxReceipt) _txRcpt).getExecutionResult() == null ? EMPTY_BYTE_ARRAY : ((AionTxReceipt) _txRcpt).getExecutionResult()), + ByteArrayWrapper.wrap( + ((AionTxReceipt) _txRcpt).getExecutionResult() == null + ? EMPTY_BYTE_ARRAY + : ((AionTxReceipt) _txRcpt).getExecutionResult()), ((AionTxReceipt) _txRcpt).getError())); - - if (_state.isPending()) { pendingReceipts.put(txHashW, ((AionTxReceipt) _txRcpt)); } else { @@ -209,15 +253,19 @@ protected void pendingTxUpdate(ITxReceipt _txRcpt, EventTx.STATE _state) { if (txWait.remainingCapacity() == 0) { txWait.poll(); if (LOG.isTraceEnabled()) { - LOG.trace("ApiAion0.onPendingTransactionUpdate - txWait queue full, drop the first message."); + LOG.trace( + "ApiAion0.onPendingTransactionUpdate - txWait queue full, drop the first message."); } } // waiting origin Api call status been callback try { - txWait.put(new TxWaitingMappingUpdate(txHashW, _state.getValue(), ((AionTxReceipt) _txRcpt))); + txWait.put( + new TxWaitingMappingUpdate( + txHashW, _state.getValue(), ((AionTxReceipt) _txRcpt))); } catch (InterruptedException e) { - LOG.error("ApiAion0.onPendingTransactionUpdate txWait.put exception", e.getMessage()); + LOG.error( + "ApiAion0.onPendingTransactionUpdate txWait.put exception", e.getMessage()); } } } @@ -236,15 +284,18 @@ private void cacheBlock(AionBlockSummary cbs) { private EventExecuteService eesBlkCache; private final class EpBlkCache implements Runnable { + boolean go = true; + @Override public void run() { while (go) { try { IEvent e = eesBlkCache.take(); - if (e.getEventType() == IHandler.TYPE.BLOCK0.getValue() && e.getCallbackType() == EventBlock.CALLBACK.ONBLOCK0.getValue()) { - cacheBlock((AionBlockSummary)e.getFuncArgs().get(0)); - } else if (e.getEventType() == IHandler.TYPE.POISONPILL.getValue()){ + if (e.getEventType() == IHandler.TYPE.BLOCK0.getValue() + && e.getCallbackType() == EventBlock.CALLBACK.ONBLOCK0.getValue()) { + cacheBlock((AionBlockSummary) e.getFuncArgs().get(0)); + } else if (e.getEventType() == IHandler.TYPE.POISONPILL.getValue()) { go = false; } } catch (Exception e) { @@ -264,21 +315,26 @@ public ApiAion0(IAionChain ac) { this.txWait = new LinkedBlockingQueue(MAP_SIZE); this.msgIdMapping = Collections.synchronizedMap(new LRUMap<>(MAP_SIZE, 100)); + initNrgOracle(ac); isFilterEnabled = CfgAion.inst().getApi().getZmq().isFiltersEnabled(); isBlkCacheEnabled = CfgAion.inst().getApi().getZmq().isBlockSummaryCacheEnabled(); if (isBlkCacheEnabled) { - explorerBlockCache = Collections.synchronizedMap(new LRUMap<>(20)); // use the default loadfactor - eesBlkCache = new EventExecuteService(100_000, "explorer-blk-cache", Thread.MIN_PRIORITY, LOG); + explorerBlockCache = + Collections.synchronizedMap(new LRUMap<>(20)); // use the default loadfactor + eesBlkCache = + new EventExecuteService( + 100_000, "explorer-blk-cache", Thread.MIN_PRIORITY, LOG); Set eventSN = new HashSet<>(); int sn = IHandler.TYPE.BLOCK0.getValue() << 8; eventSN.add(sn + EventBlock.CALLBACK.ONBLOCK0.getValue()); eesBlkCache.setFilter(eventSN); eesBlkCache.start(new EpBlkCache()); - IHandler hdrBlk = this.ac.getAionHub().getEventMgr().getHandler(IHandler.TYPE.BLOCK0.getValue()); + IHandler hdrBlk = + this.ac.getAionHub().getEventMgr().getHandler(IHandler.TYPE.BLOCK0.getValue()); if (hdrBlk != null) { hdrBlk.eventCallback(new EventCallback(eesBlkCache, LOG)); } @@ -287,13 +343,14 @@ public ApiAion0(IAionChain ac) { if (isFilterEnabled) { startES("EpApi"); - IHandler hdrTx = this.ac.getAionHub().getEventMgr().getHandler(IHandler.TYPE.TX0.getValue()); + IHandler hdrTx = + this.ac.getAionHub().getEventMgr().getHandler(IHandler.TYPE.TX0.getValue()); if (hdrTx != null) { hdrTx.eventCallback(new EventCallback(ees, LOG)); - } - IHandler hdrBlk = this.ac.getAionHub().getEventMgr().getHandler(IHandler.TYPE.BLOCK0.getValue()); + IHandler hdrBlk = + this.ac.getAionHub().getEventMgr().getHandler(IHandler.TYPE.BLOCK0.getValue()); if (hdrBlk != null) { hdrBlk.eventCallback(new EventCallback(ees, LOG)); } @@ -307,174 +364,237 @@ public byte[] process(byte[] request, byte[] socketId) { byte[] msgHash = ApiUtil.getApiMsgHash(request); if (request[0] < this.getApiVersion()) { - return msgHash == null ? ApiUtil.toReturnHeader(getApiVersion(), Retcode.r_fail_api_version_VALUE) - : ApiUtil.toReturnHeader(getApiVersion(), Retcode.r_fail_api_version_VALUE, msgHash); + return msgHash == null + ? ApiUtil.toReturnHeader(getApiVersion(), Retcode.r_fail_api_version_VALUE) + : ApiUtil.toReturnHeader( + getApiVersion(), Retcode.r_fail_api_version_VALUE, msgHash); } short service = (short) request[1]; switch ((short) request[2]) { - // General Module - case Message.Funcs.f_protocolVersion_VALUE: { - if (service != Message.Servs.s_net_VALUE) { - return ApiUtil.toReturnHeader(getApiVersion(), Retcode.r_fail_service_call_VALUE); - } - - // TODO : create query API for every module - Message.rsp_protocolVersion rsp = Message.rsp_protocolVersion.newBuilder() - .setApi(String.valueOf(this.getApiVersion())).setDb(AionHub.getRepoVersion()) - .setKernel(Version.KERNEL_VERSION).setMiner(EquihashMiner.VERSION) - .setNet(this.p2pProtocolVersion()) - .setTxpool(this.ac.getAionHub().getPendingState().getVersion()) - .setVm("0.1.0").build(); - - byte[] retHeader = ApiUtil.toReturnHeader(getApiVersion(), Retcode.r_success_VALUE); - return ApiUtil.combineRetMsg(retHeader, rsp.toByteArray()); - } - case Message.Funcs.f_minerAddress_VALUE: { + // General Module + case Message.Funcs.f_protocolVersion_VALUE: { + if (service != Message.Servs.s_net_VALUE) { + return ApiUtil.toReturnHeader( + getApiVersion(), Retcode.r_fail_service_call_VALUE); + } - if (service != Message.Servs.s_wallet_VALUE) { - return ApiUtil.toReturnHeader(getApiVersion(), Retcode.r_fail_service_call_VALUE); - } + // TODO : create query API for every module + Message.rsp_protocolVersion rsp = + Message.rsp_protocolVersion + .newBuilder() + .setApi(String.valueOf(this.getApiVersion())) + .setDb(AionHub.getRepoVersion()) + .setKernel(Version.KERNEL_VERSION) + .setMiner(EquihashMiner.VERSION) + .setNet(this.p2pProtocolVersion()) + .setTxpool(this.ac.getAionHub().getPendingState().getVersion()) + .setVm("0.1.0") + .build(); - String cb = this.getCoinbase(); - if (cb == null) { - LOG.debug("ApiAion0.process.coinbase - null coinbase"); - return ApiUtil.toReturnHeader(getApiVersion(), Retcode.r_wallet_nullcb_VALUE); + byte[] retHeader = + ApiUtil.toReturnHeader(getApiVersion(), Retcode.r_success_VALUE); + return ApiUtil.combineRetMsg(retHeader, rsp.toByteArray()); } + case Message.Funcs.f_minerAddress_VALUE: { + if (service != Message.Servs.s_wallet_VALUE) { + return ApiUtil.toReturnHeader( + getApiVersion(), Retcode.r_fail_service_call_VALUE); + } - Message.rsp_minerAddress rsp = Message.rsp_minerAddress.newBuilder() - .setMinerAddr(ByteString.copyFrom(TypeConverter.StringHexToByteArray(cb))).build(); - - byte[] retHeader = ApiUtil.toReturnHeader(getApiVersion(), Retcode.r_success_VALUE); - return ApiUtil.combineRetMsg(retHeader, rsp.toByteArray()); - } + String cb = this.getCoinbase(); + if (cb == null) { + LOG.debug("ApiAion0.process.coinbase - null coinbase"); + return ApiUtil.toReturnHeader( + getApiVersion(), Retcode.r_wallet_nullcb_VALUE); + } - case Message.Funcs.f_contractDeploy_VALUE: { + Message.rsp_minerAddress rsp = + Message.rsp_minerAddress + .newBuilder() + .setMinerAddr( + ByteString.copyFrom( + TypeConverter.StringHexToByteArray(cb))) + .build(); - if (service != Message.Servs.s_tx_VALUE) { - return ApiUtil.toReturnHeader(getApiVersion(), Retcode.r_fail_service_call_VALUE, msgHash); + byte[] retHeader = + ApiUtil.toReturnHeader(getApiVersion(), Retcode.r_success_VALUE); + return ApiUtil.combineRetMsg(retHeader, rsp.toByteArray()); } - Message.req_contractDeploy req; - byte[] data = parseMsgReq(request, msgHash); - ContractCreateResult result; - try { - req = Message.req_contractDeploy.parseFrom(data); - // TODO: the client api should send server binary code directly - // instead of str format like "0xhex".! - byte[] bytes = req.getData().toByteArray(); - if (bytes == null || bytes.length <= 4) { - return ApiUtil.toReturnHeader(getApiVersion(), Retcode.r_fail_ct_bytecode_VALUE, msgHash); + case Message.Funcs.f_contractDeploy_VALUE: { + if (service != Message.Servs.s_tx_VALUE) { + return ApiUtil.toReturnHeader( + getApiVersion(), Retcode.r_fail_service_call_VALUE, msgHash); } - ArgTxCall params = new ArgTxCall(Address.wrap(req.getFrom().toByteArray()), null, - Hex.decode(new String(bytes).substring(2)), BigInteger.ZERO, BigInteger.ZERO, req.getNrgLimit(), - req.getNrgPrice()); - - LOG.debug("ApiAion0.process.ContractDeploy - ArgsTxCall: [{}] ", params.toString()); - - result = this.createContract(params); + Message.req_contractDeploy req; + byte[] data = parseMsgReq(request, msgHash); + ContractCreateResult result; + try { + req = Message.req_contractDeploy.parseFrom(data); + // TODO: the client api should send server binary code directly + // instead of str format like "0xhex".! + byte[] bytes = req.getData().toByteArray(); + if (bytes == null || bytes.length <= 4) { + return ApiUtil.toReturnHeader( + getApiVersion(), Retcode.r_fail_ct_bytecode_VALUE, msgHash); + } - if (result != null) { - getMsgIdMapping().put(ByteArrayWrapper.wrap(result.transId), new AbstractMap.SimpleEntry<>( - ByteArrayWrapper.wrap(msgHash), ByteArrayWrapper.wrap(socketId))); - if (LOG.isDebugEnabled()) { - LOG.debug("ApiAion0.process.ContractDeploy - msgIdMapping.put: [{}] ", ByteArrayWrapper.wrap(result.transId).toString()); + ArgTxCall params = + new ArgTxCall( + Address.wrap(req.getFrom().toByteArray()), + null, + Hex.decode(new String(bytes).substring(2)), + BigInteger.ZERO, + BigInteger.ZERO, + req.getNrgLimit(), + req.getNrgPrice()); + + LOG.debug( + "ApiAion0.process.ContractDeploy - ArgsTxCall: [{}] ", + params.toString()); + + result = this.createContract(params); + + if (result != null) { + getMsgIdMapping() + .put( + ByteArrayWrapper.wrap(result.transId), + new AbstractMap.SimpleEntry<>( + ByteArrayWrapper.wrap(msgHash), + ByteArrayWrapper.wrap(socketId))); + if (LOG.isDebugEnabled()) { + LOG.debug( + "ApiAion0.process.ContractDeploy - msgIdMapping.put: [{}] ", + ByteArrayWrapper.wrap(result.transId).toString()); + } } + } catch (Exception e) { + LOG.error( + "ApiAion0.process.ContractDeploy exception [{}] ", e.getMessage()); + return ApiUtil.toReturnHeader( + getApiVersion(), Retcode.r_fail_function_exception_VALUE, msgHash); } - } catch (Exception e) { - LOG.error("ApiAion0.process.ContractDeploy exception [{}] ", e.getMessage()); - return ApiUtil.toReturnHeader(getApiVersion(), Retcode.r_fail_function_exception_VALUE, msgHash); - } - - Message.rsp_contractDeploy rsp = Message.rsp_contractDeploy.newBuilder() - .setContractAddress(ByteString.copyFrom(result != null ? result.address.toBytes() : EMPTY_BYTE_ARRAY)) - .setTxHash(ByteString.copyFrom(result != null ? result.transId : EMPTY_BYTE_ARRAY)).build(); - - byte[] retHeader = ApiUtil.toReturnHeader(getApiVersion(), Retcode.r_tx_Recved_VALUE, msgHash); - return ApiUtil.combineRetMsg(retHeader, rsp.toByteArray()); - } - // Authenication Module - case Message.Funcs.f_accounts_VALUE: { + Message.rsp_contractDeploy rsp = + Message.rsp_contractDeploy + .newBuilder() + .setContractAddress( + ByteString.copyFrom( + result != null + ? result.address.toBytes() + : EMPTY_BYTE_ARRAY)) + .setTxHash( + ByteString.copyFrom( + result != null + ? result.transId + : EMPTY_BYTE_ARRAY)) + .build(); - if (service != Message.Servs.s_wallet_VALUE) { - return ApiUtil.toReturnHeader(getApiVersion(), Retcode.r_fail_service_call_VALUE); + byte[] retHeader = + ApiUtil.toReturnHeader( + getApiVersion(), Retcode.r_tx_Recved_VALUE, msgHash); + return ApiUtil.combineRetMsg(retHeader, rsp.toByteArray()); } - // noinspection unchecked - List accounts = this.getAccounts(); - ArrayList al = new ArrayList<>(); - for (String s : accounts) { - al.add(ByteString.copyFrom(TypeConverter.StringHexToByteArray(s))); - } - Message.rsp_accounts rsp = Message.rsp_accounts.newBuilder().addAllAccout(al).build(); + // Authenication Module + case Message.Funcs.f_accounts_VALUE: { + if (service != Message.Servs.s_wallet_VALUE) { + return ApiUtil.toReturnHeader( + getApiVersion(), Retcode.r_fail_service_call_VALUE); + } - byte[] retHeader = ApiUtil.toReturnHeader(getApiVersion(), Retcode.r_success_VALUE); - return ApiUtil.combineRetMsg(retHeader, rsp.toByteArray()); - } - case Message.Funcs.f_blockNumber_VALUE: { + // noinspection unchecked + List accounts = this.getAccounts(); + ArrayList al = new ArrayList<>(); + for (String s : accounts) { + al.add(ByteString.copyFrom(TypeConverter.StringHexToByteArray(s))); + } + Message.rsp_accounts rsp = + Message.rsp_accounts.newBuilder().addAllAccout(al).build(); - if (service != Message.Servs.s_chain_VALUE) { - return ApiUtil.toReturnHeader(getApiVersion(), Retcode.r_fail_service_call_VALUE); + byte[] retHeader = + ApiUtil.toReturnHeader(getApiVersion(), Retcode.r_success_VALUE); + return ApiUtil.combineRetMsg(retHeader, rsp.toByteArray()); } + case Message.Funcs.f_blockNumber_VALUE: { + if (service != Message.Servs.s_chain_VALUE) { + return ApiUtil.toReturnHeader( + getApiVersion(), Retcode.r_fail_service_call_VALUE); + } - Message.rsp_blockNumber rsp = Message.rsp_blockNumber.newBuilder() - .setBlocknumber(this.getBestBlock().getNumber()).build(); - byte[] retHeader = ApiUtil.toReturnHeader(getApiVersion(), Retcode.r_success_VALUE); - return ApiUtil.combineRetMsg(retHeader, rsp.toByteArray()); - } - case Message.Funcs.f_unlockAccount_VALUE: { - - if (service != Message.Servs.s_wallet_VALUE) { - return ApiUtil.toReturnHeader(getApiVersion(), Retcode.r_fail_service_call_VALUE); + Message.rsp_blockNumber rsp = + Message.rsp_blockNumber + .newBuilder() + .setBlocknumber(this.getBestBlock().getNumber()) + .build(); + byte[] retHeader = + ApiUtil.toReturnHeader(getApiVersion(), Retcode.r_success_VALUE); + return ApiUtil.combineRetMsg(retHeader, rsp.toByteArray()); } + case Message.Funcs.f_unlockAccount_VALUE: { + if (service != Message.Servs.s_wallet_VALUE) { + return ApiUtil.toReturnHeader( + getApiVersion(), Retcode.r_fail_service_call_VALUE); + } - byte[] data = parseMsgReq(request, msgHash); - boolean result; - try { - Message.req_unlockAccount req = Message.req_unlockAccount.parseFrom(data); - result = this.unlockAccount(Address.wrap(req.getAccount().toByteArray()), req.getPassword(), - req.getDuration()); - } catch (InvalidProtocolBufferException e) { - LOG.error("ApiAion0.process.unlockAccount exception: [{}]", e.getMessage()); - return ApiUtil.toReturnHeader(getApiVersion(), Retcode.r_fail_function_exception_VALUE); + byte[] data = parseMsgReq(request, msgHash); + boolean result; + try { + Message.req_unlockAccount req = Message.req_unlockAccount.parseFrom(data); + result = + this.unlockAccount( + Address.wrap(req.getAccount().toByteArray()), + req.getPassword(), + req.getDuration()); + } catch (InvalidProtocolBufferException e) { + LOG.error("ApiAion0.process.unlockAccount exception: [{}]", e.getMessage()); + return ApiUtil.toReturnHeader( + getApiVersion(), Retcode.r_fail_function_exception_VALUE); + } + + byte[] retHeader = + ApiUtil.toReturnHeader(getApiVersion(), Retcode.r_success_VALUE); + return ApiUtil.combineRetMsg(retHeader, (byte) (result ? 0x01 : 0x00)); } - byte[] retHeader = ApiUtil.toReturnHeader(getApiVersion(), Retcode.r_success_VALUE); - return ApiUtil.combineRetMsg(retHeader, (byte) (result ? 0x01 : 0x00)); - } + // Transaction Module + case Message.Funcs.f_getBalance_VALUE: { + if (service != Message.Servs.s_chain_VALUE) { + return ApiUtil.toReturnHeader( + getApiVersion(), Retcode.r_fail_service_call_VALUE); + } - // Transaction Module - case Message.Funcs.f_getBalance_VALUE: { + byte[] data = parseMsgReq(request, msgHash); + BigInteger balance; + try { + Message.req_getBalance req = Message.req_getBalance.parseFrom(data); - if (service != Message.Servs.s_chain_VALUE) { - return ApiUtil.toReturnHeader(getApiVersion(), Retcode.r_fail_service_call_VALUE); - } + Address addr = Address.wrap(req.getAddress().toByteArray()); - byte[] data = parseMsgReq(request, msgHash); - BigInteger balance; - try { - Message.req_getBalance req = Message.req_getBalance.parseFrom(data); + balance = this.getBalance(addr); + } catch (InvalidProtocolBufferException e) { + LOG.error("ApiAion0.process.getbalance exception: [{}]", e.getMessage()); + return ApiUtil.toReturnHeader( + getApiVersion(), Retcode.r_fail_function_exception_VALUE); + } - Address addr = Address.wrap(req.getAddress().toByteArray()); + Message.rsp_getBalance rsp = + Message.rsp_getBalance + .newBuilder() + .setBalance(ByteString.copyFrom(balance.toByteArray())) + .build(); - balance = this.getBalance(addr); - } catch (InvalidProtocolBufferException e) { - LOG.error("ApiAion0.process.getbalance exception: [{}]", e.getMessage()); - return ApiUtil.toReturnHeader(getApiVersion(), Retcode.r_fail_function_exception_VALUE); + byte[] retHeader = + ApiUtil.toReturnHeader(getApiVersion(), Retcode.r_success_VALUE); + return ApiUtil.combineRetMsg(retHeader, rsp.toByteArray()); } - - Message.rsp_getBalance rsp = Message.rsp_getBalance.newBuilder() - .setBalance(ByteString.copyFrom(balance.toByteArray())).build(); - - byte[] retHeader = ApiUtil.toReturnHeader(getApiVersion(), Retcode.r_success_VALUE); - return ApiUtil.combineRetMsg(retHeader, rsp.toByteArray()); - } case Message.Funcs.f_getNonce_VALUE: { if (service != Message.Servs.s_chain_VALUE) { - return ApiUtil.toReturnHeader(getApiVersion(), Message.Retcode.r_fail_service_call_VALUE); + return ApiUtil.toReturnHeader( + getApiVersion(), Message.Retcode.r_fail_service_call_VALUE); } byte[] data = parseMsgReq(request, msgHash); @@ -487,1334 +607,1831 @@ public byte[] process(byte[] request, byte[] socketId) { nonce = this.getNonce(addr); } catch (InvalidProtocolBufferException e) { LOG.error("ApiAionA0.process.getNonce exception: [{}]", e.getMessage()); - return ApiUtil.toReturnHeader(getApiVersion(), Message.Retcode.r_fail_function_exception_VALUE); + return ApiUtil.toReturnHeader( + getApiVersion(), Message.Retcode.r_fail_function_exception_VALUE); } - Message.rsp_getNonce rsp = Message.rsp_getNonce.newBuilder() - .setNonce(ByteString.copyFrom(nonce.toByteArray())).build(); + Message.rsp_getNonce rsp = + Message.rsp_getNonce + .newBuilder() + .setNonce(ByteString.copyFrom(nonce.toByteArray())) + .build(); - byte[] retHeader = ApiUtil.toReturnHeader(getApiVersion(), Message.Retcode.r_success_VALUE); + byte[] retHeader = + ApiUtil.toReturnHeader( + getApiVersion(), Message.Retcode.r_success_VALUE); return ApiUtil.combineRetMsg(retHeader, rsp.toByteArray()); - - } - case Message.Funcs.f_compile_VALUE: { - - if (service != Message.Servs.s_tx_VALUE) { - return ApiUtil.toReturnHeader(getApiVersion(), Retcode.r_fail_service_call_VALUE); } + case Message.Funcs.f_getNrgPrice_VALUE: { + if (service != Servs.s_tx_VALUE) { + return ApiUtil.toReturnHeader( + getApiVersion(), Retcode.r_fail_service_call_VALUE); + } - byte[] data = parseMsgReq(request, msgHash); - try { - Message.req_compileSolidity req = Message.req_compileSolidity.parseFrom(data); - String source = req.getSource(); - if (source == null) { - return ApiUtil.toReturnHeader(getApiVersion(), Retcode.r_fail_null_compile_source_VALUE); - } - - @SuppressWarnings("unchecked") - Map _contrs = this.contract_compileSolidity(source); - if (_contrs != null && !_contrs.isEmpty()) { - Message.rsp_compile.Builder b = Message.rsp_compile.newBuilder(); - - for (Entry entry : _contrs.entrySet()) { - if (entry.getKey().contains("AionCompileError")) { - byte[] retHeader = ApiUtil.toReturnHeader(getApiVersion(), - Retcode.r_fail_compile_contract_VALUE); - Message.t_Contract tc = Message.t_Contract.newBuilder().setError(entry.getValue().error) - .build(); - return ApiUtil.combineRetMsg(retHeader, - b.putConstracts(entry.getKey(), tc).build().toByteArray()); - } - - CompiledContr _contr = entry.getValue(); - JSONArray abi = new JSONArray(); - for (Abi.Entry f : _contr.info.abiDefinition) { - abi.put(f.toJSON()); - } + long nrg = this.getRecommendedNrgPrice(); - Message.t_Contract tc = Message.t_Contract.newBuilder().setCode(_contr.code) - .setAbiDef(ByteString.copyFrom(abi.toString().getBytes())).setSource(_contr.info.source) - .build(); + try { + Message.rsp_getNrgPrice rsp = + Message.rsp_getNrgPrice.newBuilder().setNrgPrice(nrg).build(); - b.putConstracts(entry.getKey(), tc); - } - Message.rsp_compile rsp = b.build(); - byte[] retHeader = ApiUtil.toReturnHeader(getApiVersion(), Retcode.r_success_VALUE); + byte[] retHeader = + ApiUtil.toReturnHeader(getApiVersion(), Retcode.r_success_VALUE); return ApiUtil.combineRetMsg(retHeader, rsp.toByteArray()); - } else { - return ApiUtil.toReturnHeader(getApiVersion(), Retcode.r_fail_function_exception_VALUE); + } catch (Exception e) { + LOG.error("ApiAion0.process.getNrgPrice exception: [{}]", e.getMessage()); + return ApiUtil.toReturnHeader( + getApiVersion(), Retcode.r_fail_function_exception_VALUE); } - - } catch (InvalidProtocolBufferException e) { - LOG.error("ApiAion0.process.compile exception: [{}]", e.getMessage()); - return ApiUtil.toReturnHeader(getApiVersion(), Retcode.r_fail_function_exception_VALUE); - } - } - case Message.Funcs.f_sendTransaction_VALUE: { - - if (service != Message.Servs.s_tx_VALUE) { - return ApiUtil.toReturnHeader(getApiVersion(), Retcode.r_fail_service_call_VALUE, msgHash); - } - - byte[] data = parseMsgReq(request, msgHash); - Message.req_sendTransaction req; - byte[] result; - try { - req = Message.req_sendTransaction.parseFrom(data); - - ArgTxCall params = new ArgTxCall(Address.wrap(req.getFrom().toByteArray()), - Address.wrap(req.getTo().toByteArray()), req.getData().toByteArray(), - new BigInteger(req.getNonce().toByteArray()), new BigInteger(req.getValue().toByteArray()), - req.getNrg(), req.getNrgPrice()); - - result = this.sendTransaction(params); - } catch (InvalidProtocolBufferException e) { - LOG.error("ApiAion0.process.sendTransaction exception: [{}]", e.getMessage()); - return ApiUtil.toReturnHeader(getApiVersion(), Retcode.r_fail_function_exception_VALUE, msgHash); } + case Message.Funcs.f_compile_VALUE: { + if (service != Message.Servs.s_tx_VALUE) { + return ApiUtil.toReturnHeader( + getApiVersion(), Retcode.r_fail_service_call_VALUE); + } - if (result == null) { - return ApiUtil.toReturnHeader(getApiVersion(), Retcode.r_fail_sendTx_null_rep_VALUE, msgHash); - } + byte[] data = parseMsgReq(request, msgHash); + try { + Message.req_compileSolidity req = + Message.req_compileSolidity.parseFrom(data); + String source = req.getSource(); + if (source == null) { + return ApiUtil.toReturnHeader( + getApiVersion(), Retcode.r_fail_null_compile_source_VALUE); + } - getMsgIdMapping().put(ByteArrayWrapper.wrap(result), new AbstractMap.SimpleEntry<>( - ByteArrayWrapper.wrap(msgHash), ByteArrayWrapper.wrap(socketId))); - if (LOG.isDebugEnabled()) { - LOG.debug("ApiAion0.process.sendTransaction - msgIdMapping.put: [{}]", - ByteArrayWrapper.wrap(result).toString()); - } + @SuppressWarnings("unchecked") + Map _contrs = this.contract_compileSolidity(source); + if (_contrs != null && !_contrs.isEmpty()) { + Message.rsp_compile.Builder b = Message.rsp_compile.newBuilder(); + + for (Entry entry : _contrs.entrySet()) { + if (entry.getKey().contains("AionCompileError")) { + byte[] retHeader = + ApiUtil.toReturnHeader( + getApiVersion(), + Retcode.r_fail_compile_contract_VALUE); + Message.t_Contract tc = + Message.t_Contract + .newBuilder() + .setError(entry.getValue().error) + .build(); + return ApiUtil.combineRetMsg( + retHeader, + b.putConstracts(entry.getKey(), tc) + .build() + .toByteArray()); + } - Message.rsp_sendTransaction rsp = Message.rsp_sendTransaction.newBuilder() - .setTxHash(ByteString.copyFrom(result)).build(); + CompiledContr _contr = entry.getValue(); + JSONArray abi = new JSONArray(); + for (Abi.Entry f : _contr.info.abiDefinition) { + abi.put(f.toJSON()); + } - byte[] retHeader = ApiUtil.toReturnHeader(getApiVersion(), Retcode.r_tx_Recved_VALUE, msgHash); - return ApiUtil.combineRetMsg(retHeader, rsp.toByteArray()); - } - case Message.Funcs.f_getCode_VALUE: { - if (service != Message.Servs.s_tx_VALUE) { - return ApiUtil.toReturnHeader(getApiVersion(), Retcode.r_fail_service_call_VALUE); - } + Message.t_Contract tc = + Message.t_Contract + .newBuilder() + .setCode(_contr.code) + .setAbiDef( + ByteString.copyFrom( + abi.toString().getBytes())) + .setSource(_contr.info.source) + .build(); - byte[] data = parseMsgReq(request, msgHash); - Message.req_getCode req; - try { - req = Message.req_getCode.parseFrom(data); - Address to = Address.wrap(req.getAddress().toByteArray()); + b.putConstracts(entry.getKey(), tc); + } + Message.rsp_compile rsp = b.build(); + byte[] retHeader = + ApiUtil.toReturnHeader( + getApiVersion(), Retcode.r_success_VALUE); + return ApiUtil.combineRetMsg(retHeader, rsp.toByteArray()); + } else { + return ApiUtil.toReturnHeader( + getApiVersion(), Retcode.r_fail_function_exception_VALUE); + } - byte[] code = this.getCode(to); - if (code == null) { - return ApiUtil.toReturnHeader(getApiVersion(), Retcode.r_fail_null_rsp_VALUE); + } catch (InvalidProtocolBufferException e) { + LOG.error("ApiAion0.process.compile exception: [{}]", e.getMessage()); + return ApiUtil.toReturnHeader( + getApiVersion(), Retcode.r_fail_function_exception_VALUE); } - - Message.rsp_getCode rsp = Message.rsp_getCode.newBuilder().setCode(ByteString.copyFrom(code)).build(); - byte[] retHeader = ApiUtil.toReturnHeader(getApiVersion(), Retcode.r_success_VALUE); - return ApiUtil.combineRetMsg(retHeader, rsp.toByteArray()); - - } catch (Exception e) { - LOG.error("ApiAion0.process.getCode exception: [{}]", e.getMessage()); - return ApiUtil.toReturnHeader(getApiVersion(), Retcode.r_fail_function_exception_VALUE); - } - } - case Message.Funcs.f_getTransactionReceipt_VALUE: { - if (service != Message.Servs.s_tx_VALUE) { - return ApiUtil.toReturnHeader(getApiVersion(), Retcode.r_fail_service_call_VALUE); } + case Message.Funcs.f_sendTransaction_VALUE: { + if (service != Message.Servs.s_tx_VALUE) { + return ApiUtil.toReturnHeader( + getApiVersion(), Retcode.r_fail_service_call_VALUE, msgHash); + } - byte[] data = parseMsgReq(request, msgHash); - Message.req_getTransactionReceipt req; - try { - req = Message.req_getTransactionReceipt.parseFrom(data); + byte[] data = parseMsgReq(request, msgHash); + Message.req_sendTransaction req; + byte[] result; + try { + req = Message.req_sendTransaction.parseFrom(data); + + ArgTxCall params = + new ArgTxCall( + Address.wrap(req.getFrom().toByteArray()), + Address.wrap(req.getTo().toByteArray()), + req.getData().toByteArray(), + new BigInteger(req.getNonce().toByteArray()), + new BigInteger(req.getValue().toByteArray()), + req.getNrg(), + req.getNrgPrice()); + + result = this.sendTransaction(params); + } catch (InvalidProtocolBufferException e) { + LOG.error( + "ApiAion0.process.sendTransaction exception: [{}]", e.getMessage()); + return ApiUtil.toReturnHeader( + getApiVersion(), Retcode.r_fail_function_exception_VALUE, msgHash); + } - TxRecpt result = this.getTransactionReceipt(req.getTxHash().toByteArray()); if (result == null) { - return ApiUtil.toReturnHeader(getApiVersion(), Retcode.r_fail_getTxReceipt_null_recp_VALUE); + return ApiUtil.toReturnHeader( + getApiVersion(), Retcode.r_fail_sendTx_null_rep_VALUE, msgHash); } - List logs = new ArrayList<>(); - for (TxRecptLg log : result.logs) { - List al = new ArrayList<>(); - Collections.addAll(al, log.topics); - - Message.t_LgEle msgLog = Message.t_LgEle.newBuilder() - .setAddress(ByteString.copyFrom(Address.wrap(log.address).toBytes())) - .setData(ByteString.copyFrom(ByteUtil.hexStringToBytes(log.data))).addAllTopics(al).build(); - - logs.add(msgLog); + getMsgIdMapping() + .put( + ByteArrayWrapper.wrap(result), + new AbstractMap.SimpleEntry<>( + ByteArrayWrapper.wrap(msgHash), + ByteArrayWrapper.wrap(socketId))); + if (LOG.isDebugEnabled()) { + LOG.debug( + "ApiAion0.process.sendTransaction - msgIdMapping.put: [{}]", + ByteArrayWrapper.wrap(result).toString()); } - Message.rsp_getTransactionReceipt rsp = Message.rsp_getTransactionReceipt.newBuilder() - .setFrom(ByteString.copyFrom(result.fromAddr.toBytes())).setBlockNumber(result.blockNumber) - .setBlockHash(ByteString - .copyFrom(result.blockHash != null ? ByteUtil.hexStringToBytes(result.blockHash) - : EMPTY_BYTE_ARRAY)) - .setContractAddress(ByteString.copyFrom( - result.contractAddress != null ? ByteUtil.hexStringToBytes(result.contractAddress) - : EMPTY_BYTE_ARRAY)) - .setTxIndex(result.transactionIndex) - .setTxHash(ByteString.copyFrom( - result.transactionHash != null ? ByteUtil.hexStringToBytes(result.transactionHash) - : EMPTY_BYTE_ARRAY)) - .setTo(ByteString - .copyFrom(result.toAddr == null ? EMPTY_BYTE_ARRAY : result.toAddr.toBytes())) - .setNrgConsumed(result.nrgUsed).setCumulativeNrgUsed(result.cumulativeNrgUsed).addAllLogs(logs) + Message.rsp_sendTransaction rsp = + Message.rsp_sendTransaction + .newBuilder() + .setTxHash(ByteString.copyFrom(result)) .build(); - byte[] retHeader = ApiUtil.toReturnHeader(getApiVersion(), Retcode.r_success_VALUE); + byte[] retHeader = + ApiUtil.toReturnHeader( + getApiVersion(), Retcode.r_tx_Recved_VALUE, msgHash); return ApiUtil.combineRetMsg(retHeader, rsp.toByteArray()); - - } catch (Exception e) { - LOG.error("ApiAion0.process.getTransactionReceipt exception: [{}]", e.getMessage()); - return ApiUtil.toReturnHeader(getApiVersion(), Retcode.r_fail_function_exception_VALUE); - } - } - case Message.Funcs.f_call_VALUE: { - if (service != Message.Servs.s_tx_VALUE) { - return ApiUtil.toReturnHeader(getApiVersion(), Retcode.r_fail_service_call_VALUE); } + case Message.Funcs.f_getCode_VALUE: { + if (service != Message.Servs.s_tx_VALUE) { + return ApiUtil.toReturnHeader( + getApiVersion(), Retcode.r_fail_service_call_VALUE); + } - byte[] data = parseMsgReq(request, msgHash); - Message.req_call req; - - try { - req = Message.req_call.parseFrom(data); + byte[] data = parseMsgReq(request, msgHash); + Message.req_getCode req; + try { + req = Message.req_getCode.parseFrom(data); + Address to = Address.wrap(req.getAddress().toByteArray()); - Address from = Address.wrap(req.getFrom().toByteArray()); - Address to = Address.wrap(req.getTo().toByteArray()); + byte[] code = this.getCode(to); + if (code == null) { + return ApiUtil.toReturnHeader( + getApiVersion(), Retcode.r_fail_null_rsp_VALUE); + } - BigInteger value = new BigInteger(req.getValue().toByteArray()); - byte[] d = req.getData().toByteArray(); + Message.rsp_getCode rsp = + Message.rsp_getCode + .newBuilder() + .setCode(ByteString.copyFrom(code)) + .build(); + byte[] retHeader = + ApiUtil.toReturnHeader(getApiVersion(), Retcode.r_success_VALUE); + return ApiUtil.combineRetMsg(retHeader, rsp.toByteArray()); - ArgTxCall params = new ArgTxCall(from, to, d, BigInteger.ZERO, value, req.getNrg(), req.getNrgPrice()); - Message.rsp_call rsp = Message.rsp_call.newBuilder().setResult(ByteString.copyFrom(this.doCall(params))) - .build(); + } catch (Exception e) { + LOG.error("ApiAion0.process.getCode exception: [{}]", e.getMessage()); + return ApiUtil.toReturnHeader( + getApiVersion(), Retcode.r_fail_function_exception_VALUE); + } + } + case Message.Funcs.f_getTransactionReceipt_VALUE: { + if (service != Message.Servs.s_tx_VALUE) { + return ApiUtil.toReturnHeader( + getApiVersion(), Retcode.r_fail_service_call_VALUE); + } - byte[] retHeader = ApiUtil.toReturnHeader(getApiVersion(), Retcode.r_success_VALUE); - return ApiUtil.combineRetMsg(retHeader, rsp.toByteArray()); + byte[] data = parseMsgReq(request, msgHash); + Message.req_getTransactionReceipt req; + try { + req = Message.req_getTransactionReceipt.parseFrom(data); - } catch (Exception e) { - LOG.error("ApiAion0.process.call exception: [{}]", e.getMessage()); - return ApiUtil.toReturnHeader(getApiVersion(), Retcode.r_fail_function_exception_VALUE); - } - } - case Message.Funcs.f_getBlockByNumber_VALUE: { - if (service != Message.Servs.s_chain_VALUE) { - return ApiUtil.toReturnHeader(getApiVersion(), Retcode.r_fail_service_call_VALUE); - } + TxRecpt result = this.getTransactionReceipt(req.getTxHash().toByteArray()); + if (result == null) { + return ApiUtil.toReturnHeader( + getApiVersion(), Retcode.r_fail_getTxReceipt_null_recp_VALUE); + } - byte[] data = parseMsgReq(request, msgHash); - Message.req_getBlockByNumber req; + List logs = new ArrayList<>(); + for (TxRecptLg log : result.logs) { + List al = new ArrayList<>(); + Collections.addAll(al, log.topics); + + Message.t_LgEle msgLog = + Message.t_LgEle + .newBuilder() + .setAddress( + ByteString.copyFrom( + Address.wrap(log.address).toBytes())) + .setData( + ByteString.copyFrom( + ByteUtil.hexStringToBytes(log.data))) + .addAllTopics(al) + .build(); - try { - req = Message.req_getBlockByNumber.parseFrom(data); - long num = req.getBlockNumber(); - AionBlock blk = this.getBlock(num); - - return createBlockMsg(blk); - } catch (Exception e) { - LOG.error("ApiAion0.process.getBlockByNumber exception: [{}]", e.getMessage()); - return ApiUtil.toReturnHeader(getApiVersion(), Retcode.r_fail_function_exception_VALUE); - } - } - case Message.Funcs.f_getBlockByHash_VALUE: { - if (service != Message.Servs.s_chain_VALUE) { - return ApiUtil.toReturnHeader(getApiVersion(), Retcode.r_fail_service_call_VALUE); - } + logs.add(msgLog); + } - byte[] data = parseMsgReq(request, msgHash); - Message.req_getBlockByHash req; + Message.rsp_getTransactionReceipt rsp = + Message.rsp_getTransactionReceipt + .newBuilder() + .setFrom(ByteString.copyFrom(result.fromAddr.toBytes())) + .setBlockNumber(result.blockNumber) + .setBlockHash( + ByteString.copyFrom( + result.blockHash != null + ? ByteUtil.hexStringToBytes( + result.blockHash) + : EMPTY_BYTE_ARRAY)) + .setContractAddress( + ByteString.copyFrom( + result.contractAddress != null + ? ByteUtil.hexStringToBytes( + result.contractAddress) + : EMPTY_BYTE_ARRAY)) + .setTxIndex(result.transactionIndex) + .setTxHash( + ByteString.copyFrom( + result.transactionHash != null + ? ByteUtil.hexStringToBytes( + result.transactionHash) + : EMPTY_BYTE_ARRAY)) + .setTo( + ByteString.copyFrom( + result.toAddr == null + ? EMPTY_BYTE_ARRAY + : result.toAddr.toBytes())) + .setNrgConsumed(result.nrgUsed) + .setCumulativeNrgUsed(result.cumulativeNrgUsed) + .addAllLogs(logs) + .build(); - try { - req = Message.req_getBlockByHash.parseFrom(data); - byte[] hash = req.getBlockHash().toByteArray(); + byte[] retHeader = + ApiUtil.toReturnHeader(getApiVersion(), Retcode.r_success_VALUE); + return ApiUtil.combineRetMsg(retHeader, rsp.toByteArray()); - if (hash == null || hash.length != Hash256.BYTES) { - return ApiUtil.toReturnHeader(getApiVersion(), Retcode.r_fail_function_arguments_VALUE); + } catch (Exception e) { + LOG.error( + "ApiAion0.process.getTransactionReceipt exception: [{}]", + e.getMessage()); + return ApiUtil.toReturnHeader( + getApiVersion(), Retcode.r_fail_function_exception_VALUE); } - - AionBlock blk = this.getBlockByHash(hash); - return createBlockMsg(blk); - } catch (Exception e) { - LOG.error("ApiAion0.process.getBlockByHash exception: [{}]", e.getMessage()); - return ApiUtil.toReturnHeader(getApiVersion(), Retcode.r_fail_function_exception_VALUE); - } - } - case Message.Funcs.f_getTransactionByBlockHashAndIndex_VALUE: { - if (service != Message.Servs.s_chain_VALUE) { - return ApiUtil.toReturnHeader(getApiVersion(), Retcode.r_fail_service_call_VALUE); } + case Message.Funcs.f_call_VALUE: { + if (service != Message.Servs.s_tx_VALUE) { + return ApiUtil.toReturnHeader( + getApiVersion(), Retcode.r_fail_service_call_VALUE); + } - byte[] data = parseMsgReq(request, msgHash); - Message.req_getTransactionByBlockHashAndIndex req; + byte[] data = parseMsgReq(request, msgHash); + Message.req_call req; - try { - req = Message.req_getTransactionByBlockHashAndIndex.parseFrom(data); - long txIdx = req.getTxIndex(); - byte[] hash = req.getBlockHash().toByteArray(); + try { + req = Message.req_call.parseFrom(data); + + Address from = Address.wrap(req.getFrom().toByteArray()); + Address to = Address.wrap(req.getTo().toByteArray()); + + BigInteger value = new BigInteger(req.getValue().toByteArray()); + byte[] d = req.getData().toByteArray(); + + ArgTxCall params = + new ArgTxCall( + from, + to, + d, + BigInteger.ZERO, + value, + req.getNrg(), + req.getNrgPrice()); + Message.rsp_call rsp = + Message.rsp_call + .newBuilder() + .setResult(ByteString.copyFrom(this.doCall(params))) + .build(); - if (txIdx < -1 || hash == null || hash.length != Hash256.BYTES) { - return ApiUtil.toReturnHeader(getApiVersion(), Retcode.r_fail_function_arguments_VALUE); - } + byte[] retHeader = + ApiUtil.toReturnHeader(getApiVersion(), Retcode.r_success_VALUE); + return ApiUtil.combineRetMsg(retHeader, rsp.toByteArray()); - AionTransaction tx = this.getTransactionByBlockHashAndIndex(hash, txIdx); - if (tx == null) { - return ApiUtil.toReturnHeader(getApiVersion(), Retcode.r_fail_function_call_VALUE); + } catch (Exception e) { + LOG.error("ApiAion0.process.call exception: [{}]", e.getMessage()); + return ApiUtil.toReturnHeader( + getApiVersion(), Retcode.r_fail_function_exception_VALUE); } - - Message.rsp_getTransaction rsp = getRsp_getTransaction(tx); - - byte[] retHeader = ApiUtil.toReturnHeader(getApiVersion(), Retcode.r_success_VALUE); - return ApiUtil.combineRetMsg(retHeader, rsp.toByteArray()); - } catch (Exception e) { - LOG.error("ApiAion0.process.getTransactionByBlockHashAndIndex exception: [{}]", e.getMessage()); - return ApiUtil.toReturnHeader(getApiVersion(), Retcode.r_fail_function_exception_VALUE); - } - } - case Message.Funcs.f_getTransactionByBlockNumberAndIndex_VALUE: { - if (service != Message.Servs.s_chain_VALUE) { - return ApiUtil.toReturnHeader(getApiVersion(), Retcode.r_fail_service_call_VALUE); } + case Message.Funcs.f_getBlockByNumber_VALUE: { + if (service != Message.Servs.s_chain_VALUE) { + return ApiUtil.toReturnHeader( + getApiVersion(), Retcode.r_fail_service_call_VALUE); + } - byte[] data = parseMsgReq(request, msgHash); - Message.req_getTransactionByBlockNumberAndIndex req; + byte[] data = parseMsgReq(request, msgHash); + Message.req_getBlockByNumber req; - try { - req = Message.req_getTransactionByBlockNumberAndIndex.parseFrom(data); - long blkNr = req.getBlockNumber(); - long txIdx = req.getTxIndex(); + try { + req = Message.req_getBlockByNumber.parseFrom(data); + long num = req.getBlockNumber(); + AionBlock blk = this.getBlock(num); - if (blkNr < -1 || txIdx < -1) { - return ApiUtil.toReturnHeader(getApiVersion(), Retcode.r_fail_function_arguments_VALUE); + return createBlockMsg(blk); + } catch (Exception e) { + LOG.error( + "ApiAion0.process.getBlockByNumber exception: [{}]", + e.getMessage()); + return ApiUtil.toReturnHeader( + getApiVersion(), Retcode.r_fail_function_exception_VALUE); } - - AionTransaction tx = this.getTransactionByBlockNumberAndIndex(blkNr, txIdx); - if (tx == null) { - return ApiUtil.toReturnHeader(getApiVersion(), Retcode.r_fail_function_call_VALUE); + } + case Message.Funcs.f_getBlockByHash_VALUE: { + if (service != Message.Servs.s_chain_VALUE) { + return ApiUtil.toReturnHeader( + getApiVersion(), Retcode.r_fail_service_call_VALUE); } - Message.rsp_getTransaction rsp = getRsp_getTransaction(tx); - - byte[] retHeader = ApiUtil.toReturnHeader(getApiVersion(), Retcode.r_success_VALUE); - return ApiUtil.combineRetMsg(retHeader, rsp.toByteArray()); - } catch (Exception e) { - LOG.error("ApiAion0.process.getTransactionByBlockNumberAndIndex exception: [{}]", e.getMessage()); - return ApiUtil.toReturnHeader(getApiVersion(), Retcode.r_fail_function_exception_VALUE); - } - } - case Message.Funcs.f_getBlockTransactionCountByNumber_VALUE: { - if (service != Message.Servs.s_chain_VALUE) { - return ApiUtil.toReturnHeader(getApiVersion(), Retcode.r_fail_service_call_VALUE); - } + byte[] data = parseMsgReq(request, msgHash); + Message.req_getBlockByHash req; - byte[] data = parseMsgReq(request, msgHash); - Message.req_getBlockTransactionCountByNumber req; + try { + req = Message.req_getBlockByHash.parseFrom(data); + byte[] hash = req.getBlockHash().toByteArray(); - try { - req = Message.req_getBlockTransactionCountByNumber.parseFrom(data); - long blkNr = req.getBlockNumber(); + if (hash == null || hash.length != Hash256.BYTES) { + return ApiUtil.toReturnHeader( + getApiVersion(), Retcode.r_fail_function_arguments_VALUE); + } - if (blkNr < -1) { - return ApiUtil.toReturnHeader(getApiVersion(), Retcode.r_fail_function_arguments_VALUE); + AionBlock blk = this.getBlockByHash(hash); + return createBlockMsg(blk); + } catch (Exception e) { + LOG.error( + "ApiAion0.process.getBlockByHash exception: [{}]", e.getMessage()); + return ApiUtil.toReturnHeader( + getApiVersion(), Retcode.r_fail_function_exception_VALUE); } - - long cnt = this.getBlockTransactionCountByNumber(blkNr); - if (cnt == -1) { - return ApiUtil.toReturnHeader(getApiVersion(), Retcode.r_fail_function_call_VALUE); + } + case Message.Funcs.f_getTransactionByBlockHashAndIndex_VALUE: { + if (service != Message.Servs.s_chain_VALUE) { + return ApiUtil.toReturnHeader( + getApiVersion(), Retcode.r_fail_service_call_VALUE); } - Message.rsp_getBlockTransactionCount rsp = Message.rsp_getBlockTransactionCount.newBuilder() - .setTxCount((int) cnt).build(); + byte[] data = parseMsgReq(request, msgHash); + Message.req_getTransactionByBlockHashAndIndex req; - byte[] retHeader = ApiUtil.toReturnHeader(getApiVersion(), Retcode.r_success_VALUE); - return ApiUtil.combineRetMsg(retHeader, rsp.toByteArray()); - } catch (Exception e) { - LOG.error("ApiAion0.process.getBlockTransactionCountByNumber exception: [{}]", e.getMessage()); - return ApiUtil.toReturnHeader(getApiVersion(), Retcode.r_fail_function_exception_VALUE); - } - } - case Message.Funcs.f_getTransactionCount_VALUE: { - if (service != Message.Servs.s_chain_VALUE) { - return ApiUtil.toReturnHeader(getApiVersion(), Retcode.r_fail_service_call_VALUE); - } + try { + req = Message.req_getTransactionByBlockHashAndIndex.parseFrom(data); + long txIdx = req.getTxIndex(); + byte[] hash = req.getBlockHash().toByteArray(); - byte[] data = parseMsgReq(request, msgHash); - Message.req_getTransactionCount req; + if (txIdx < -1 || hash == null || hash.length != Hash256.BYTES) { + return ApiUtil.toReturnHeader( + getApiVersion(), Retcode.r_fail_function_arguments_VALUE); + } - try { - req = Message.req_getTransactionCount.parseFrom(data); - long blkNr = req.getBlocknumber(); - Address addr = Address.wrap(req.getAddress().toByteArray()); + AionTransaction tx = this.getTransactionByBlockHashAndIndex(hash, txIdx); + if (tx == null) { + return ApiUtil.toReturnHeader( + getApiVersion(), Retcode.r_fail_function_call_VALUE); + } - if (blkNr < -1) { - return ApiUtil.toReturnHeader(getApiVersion(), Retcode.r_fail_function_arguments_VALUE); - } + Message.rsp_getTransaction rsp = getRsp_getTransaction(tx); - long cnt = this.getTransactionCount(addr, blkNr); - if (cnt == -1) { - return ApiUtil.toReturnHeader(getApiVersion(), Retcode.r_fail_function_call_VALUE); + byte[] retHeader = + ApiUtil.toReturnHeader(getApiVersion(), Retcode.r_success_VALUE); + return ApiUtil.combineRetMsg(retHeader, rsp.toByteArray()); + } catch (Exception e) { + LOG.error( + "ApiAion0.process.getTransactionByBlockHashAndIndex exception: [{}]", + e.getMessage()); + return ApiUtil.toReturnHeader( + getApiVersion(), Retcode.r_fail_function_exception_VALUE); + } + } + case Message.Funcs.f_getTransactionByBlockNumberAndIndex_VALUE: { + if (service != Message.Servs.s_chain_VALUE) { + return ApiUtil.toReturnHeader( + getApiVersion(), Retcode.r_fail_service_call_VALUE); } - Message.rsp_getTransactionCount rsp = Message.rsp_getTransactionCount.newBuilder().setTxCount((int) cnt) - .build(); + byte[] data = parseMsgReq(request, msgHash); + Message.req_getTransactionByBlockNumberAndIndex req; - byte[] retHeader = ApiUtil.toReturnHeader(getApiVersion(), Retcode.r_success_VALUE); - return ApiUtil.combineRetMsg(retHeader, rsp.toByteArray()); + try { + req = Message.req_getTransactionByBlockNumberAndIndex.parseFrom(data); + long blkNr = req.getBlockNumber(); + long txIdx = req.getTxIndex(); - } catch (Exception e) { - LOG.error("ApiAion0.process.getTransactionCount exception: [{}]", e.getMessage()); - return ApiUtil.toReturnHeader(getApiVersion(), Retcode.r_fail_function_exception_VALUE); - } - } - case Message.Funcs.f_getBlockTransactionCountByHash_VALUE: { - if (service != Message.Servs.s_chain_VALUE) { - return ApiUtil.toReturnHeader(getApiVersion(), Retcode.r_fail_service_call_VALUE); - } + if (blkNr < -1 || txIdx < -1) { + return ApiUtil.toReturnHeader( + getApiVersion(), Retcode.r_fail_function_arguments_VALUE); + } - byte[] data = parseMsgReq(request, msgHash); - Message.req_getTransactionCountByHash req; + AionTransaction tx = this.getTransactionByBlockNumberAndIndex(blkNr, txIdx); + if (tx == null) { + return ApiUtil.toReturnHeader( + getApiVersion(), Retcode.r_fail_function_call_VALUE); + } - try { - req = Message.req_getTransactionCountByHash.parseFrom(data); - byte[] hash = req.getTxHash().toByteArray(); + Message.rsp_getTransaction rsp = getRsp_getTransaction(tx); - if (hash == null || hash.length != Hash256.BYTES) { - return ApiUtil.toReturnHeader(getApiVersion(), Retcode.r_fail_function_arguments_VALUE); + byte[] retHeader = + ApiUtil.toReturnHeader(getApiVersion(), Retcode.r_success_VALUE); + return ApiUtil.combineRetMsg(retHeader, rsp.toByteArray()); + } catch (Exception e) { + LOG.error( + "ApiAion0.process.getTransactionByBlockNumberAndIndex exception: [{}]", + e.getMessage()); + return ApiUtil.toReturnHeader( + getApiVersion(), Retcode.r_fail_function_exception_VALUE); } - - long cnt = this.getTransactionCountByHash(hash); - if (cnt == -1) { - return ApiUtil.toReturnHeader(getApiVersion(), Retcode.r_fail_function_call_VALUE); + } + case Message.Funcs.f_getBlockTransactionCountByNumber_VALUE: { + if (service != Message.Servs.s_chain_VALUE) { + return ApiUtil.toReturnHeader( + getApiVersion(), Retcode.r_fail_service_call_VALUE); } - Message.rsp_getTransactionCount rsp = Message.rsp_getTransactionCount.newBuilder().setTxCount((int) cnt) - .build(); + byte[] data = parseMsgReq(request, msgHash); + Message.req_getBlockTransactionCountByNumber req; - byte[] retHeader = ApiUtil.toReturnHeader(getApiVersion(), Retcode.r_success_VALUE); - return ApiUtil.combineRetMsg(retHeader, rsp.toByteArray()); + try { + req = Message.req_getBlockTransactionCountByNumber.parseFrom(data); + long blkNr = req.getBlockNumber(); - } catch (Exception e) { - LOG.error("ApiAion0.process.getTransactionCount exception: [{}]", e.getMessage()); - return ApiUtil.toReturnHeader(getApiVersion(), Retcode.r_fail_function_exception_VALUE); - } - } - case Message.Funcs.f_getTransactionByHash_VALUE: { - if (service != Message.Servs.s_chain_VALUE) { - return ApiUtil.toReturnHeader(getApiVersion(), Retcode.r_fail_service_call_VALUE); - } + if (blkNr < -1) { + return ApiUtil.toReturnHeader( + getApiVersion(), Retcode.r_fail_function_arguments_VALUE); + } - byte[] data = parseMsgReq(request, msgHash); - Message.req_getTransactionByHash req; + long cnt = this.getBlockTransactionCountByNumber(blkNr); + if (cnt == -1) { + return ApiUtil.toReturnHeader( + getApiVersion(), Retcode.r_fail_function_call_VALUE); + } - try { - req = Message.req_getTransactionByHash.parseFrom(data); - byte[] txHash = req.getTxHash().toByteArray(); + Message.rsp_getBlockTransactionCount rsp = + Message.rsp_getBlockTransactionCount + .newBuilder() + .setTxCount((int) cnt) + .build(); - if (txHash == null || txHash.length != this.getTxHashLen()) { - return ApiUtil.toReturnHeader(getApiVersion(), Retcode.r_fail_function_arguments_VALUE); + byte[] retHeader = + ApiUtil.toReturnHeader(getApiVersion(), Retcode.r_success_VALUE); + return ApiUtil.combineRetMsg(retHeader, rsp.toByteArray()); + } catch (Exception e) { + LOG.error( + "ApiAion0.process.getBlockTransactionCountByNumber exception: [{}]", + e.getMessage()); + return ApiUtil.toReturnHeader( + getApiVersion(), Retcode.r_fail_function_exception_VALUE); } - - AionTransaction tx = this.getTransactionByHash(txHash); - if (tx == null) { - return ApiUtil.toReturnHeader(getApiVersion(), Retcode.r_fail_function_call_VALUE); + } + case Message.Funcs.f_getTransactionCount_VALUE: { + if (service != Message.Servs.s_chain_VALUE) { + return ApiUtil.toReturnHeader( + getApiVersion(), Retcode.r_fail_service_call_VALUE); } - Message.rsp_getTransaction rsp = getRsp_getTransaction(tx); + byte[] data = parseMsgReq(request, msgHash); + Message.req_getTransactionCount req; - byte[] retHeader = ApiUtil.toReturnHeader(getApiVersion(), Retcode.r_success_VALUE); - return ApiUtil.combineRetMsg(retHeader, rsp.toByteArray()); + try { + req = Message.req_getTransactionCount.parseFrom(data); + long blkNr = req.getBlocknumber(); + Address addr = Address.wrap(req.getAddress().toByteArray()); - } catch (Exception e) { - LOG.error("ApiAion0.process.getTransactionCount exception: [{}]", e.getMessage()); - return ApiUtil.toReturnHeader(getApiVersion(), Retcode.r_fail_function_exception_VALUE); - } - } + if (blkNr < -1) { + return ApiUtil.toReturnHeader( + getApiVersion(), Retcode.r_fail_function_arguments_VALUE); + } - case Message.Funcs.f_getActiveNodes_VALUE: { - if (service != Message.Servs.s_net_VALUE) { - return ApiUtil.toReturnHeader(getApiVersion(), Retcode.r_fail_service_call_VALUE); - } + long cnt = this.getTransactionCount(addr, blkNr); + if (cnt == -1) { + return ApiUtil.toReturnHeader( + getApiVersion(), Retcode.r_fail_function_call_VALUE); + } - List nodes = new ArrayList<>(); - nodes.addAll(this.ac.getAionHub().getP2pMgr().getActiveNodes().values()); - List pl = new ArrayList<>(); - try { - for (INode n : nodes) { - Message.t_Node node = Message.t_Node.newBuilder().setBlockNumber(n.getBestBlockNumber()) - .setNodeId(ByteArrayWrapper.wrap(n.getId()).toString()) - .setRemoteP2PIp(ByteArrayWrapper.wrap(n.getIp()).toString()) - // TODO : api export latency, totalDiff and the - // blockHash - // .setLatency((float) n.get) + Message.rsp_getTransactionCount rsp = + Message.rsp_getTransactionCount + .newBuilder() + .setTxCount((int) cnt) .build(); - pl.add(node); - } + byte[] retHeader = + ApiUtil.toReturnHeader(getApiVersion(), Retcode.r_success_VALUE); + return ApiUtil.combineRetMsg(retHeader, rsp.toByteArray()); - Message.rsp_getActiveNodes rsp = Message.rsp_getActiveNodes.newBuilder().addAllNode(pl).build(); - byte[] retHeader = ApiUtil.toReturnHeader(getApiVersion(), Retcode.r_success_VALUE); - return ApiUtil.combineRetMsg(retHeader, rsp.toByteArray()); - } catch (Exception e) { - LOG.error("ApiAion0.process.getActiveNodes exception: [{}]", e.getMessage()); - return ApiUtil.toReturnHeader(getApiVersion(), Retcode.r_fail_function_exception_VALUE); + } catch (Exception e) { + LOG.error( + "ApiAion0.process.getTransactionCount exception: [{}]", + e.getMessage()); + return ApiUtil.toReturnHeader( + getApiVersion(), Retcode.r_fail_function_exception_VALUE); + } } - } + case Message.Funcs.f_getBlockTransactionCountByHash_VALUE: { + if (service != Message.Servs.s_chain_VALUE) { + return ApiUtil.toReturnHeader( + getApiVersion(), Retcode.r_fail_service_call_VALUE); + } - case Message.Funcs.f_getStaticNodes_VALUE: { - if (service != Message.Servs.s_net_VALUE) { - return ApiUtil.toReturnHeader(getApiVersion(), Retcode.r_fail_service_call_VALUE); - } - List al = Arrays.asList(this.getBootNodes()); - List nl = new ArrayList<>(); - for (String s : al) { - // TODO : get more node info - Message.t_Node n = Message.t_Node.newBuilder().setRemoteP2PIp(s).build(); - nl.add(n); - } + byte[] data = parseMsgReq(request, msgHash); + Message.req_getTransactionCountByHash req; - try { - Message.rsp_getStaticNodes rsp = Message.rsp_getStaticNodes.newBuilder().addAllNode(nl).build(); + try { + req = Message.req_getTransactionCountByHash.parseFrom(data); + byte[] hash = req.getTxHash().toByteArray(); - byte[] retHeader = ApiUtil.toReturnHeader(getApiVersion(), Retcode.r_success_VALUE); - return ApiUtil.combineRetMsg(retHeader, rsp.toByteArray()); - } catch (Exception e) { - LOG.error("ApiAion0.process.getStaticNodes exception: [{}]", e.getMessage()); - return ApiUtil.toReturnHeader(getApiVersion(), Retcode.r_fail_function_exception_VALUE); - } - } - case Message.Funcs.f_getSolcVersion_VALUE: { - if (service != Message.Servs.s_tx_VALUE) { - return ApiUtil.toReturnHeader(getApiVersion(), Retcode.r_fail_service_call_VALUE); - } + if (hash == null || hash.length != Hash256.BYTES) { + return ApiUtil.toReturnHeader( + getApiVersion(), Retcode.r_fail_function_arguments_VALUE); + } - String ver = this.solcVersion(); + long cnt = this.getTransactionCountByHash(hash); + if (cnt == -1) { + return ApiUtil.toReturnHeader( + getApiVersion(), Retcode.r_fail_function_call_VALUE); + } - try { - Message.rsp_getSolcVersion rsp = Message.rsp_getSolcVersion.newBuilder().setVer(ver).build(); + Message.rsp_getTransactionCount rsp = + Message.rsp_getTransactionCount + .newBuilder() + .setTxCount((int) cnt) + .build(); - byte[] retHeader = ApiUtil.toReturnHeader(getApiVersion(), Retcode.r_success_VALUE); - return ApiUtil.combineRetMsg(retHeader, rsp.toByteArray()); - } catch (Exception e) { - LOG.error("ApiAion0.process.getSolcVersion exception: [{}]", e.getMessage()); - return ApiUtil.toReturnHeader(getApiVersion(), Retcode.r_fail_function_exception_VALUE); - } - } - case Message.Funcs.f_isSyncing_VALUE: { - if (service != Message.Servs.s_net_VALUE) { - return ApiUtil.toReturnHeader(getApiVersion(), Retcode.r_fail_service_call_VALUE); + byte[] retHeader = + ApiUtil.toReturnHeader(getApiVersion(), Retcode.r_success_VALUE); + return ApiUtil.combineRetMsg(retHeader, rsp.toByteArray()); + + } catch (Exception e) { + LOG.error( + "ApiAion0.process.getTransactionCount exception: [{}]", + e.getMessage()); + return ApiUtil.toReturnHeader( + getApiVersion(), Retcode.r_fail_function_exception_VALUE); + } } + case Message.Funcs.f_getTransactionByHash_VALUE: { + if (service != Message.Servs.s_chain_VALUE) { + return ApiUtil.toReturnHeader( + getApiVersion(), Retcode.r_fail_service_call_VALUE); + } - try { - Message.rsp_isSyncing rsp = Message.rsp_isSyncing.newBuilder().setSyncing(!this.getSync().done).build(); + byte[] data = parseMsgReq(request, msgHash); + Message.req_getTransactionByHash req; - byte[] retHeader = ApiUtil.toReturnHeader(getApiVersion(), Retcode.r_success_VALUE); - return ApiUtil.combineRetMsg(retHeader, rsp.toByteArray()); - } catch (Exception e) { - LOG.error("ApiAion0.process.syncing exception: [{}]", e.getMessage()); - return ApiUtil.toReturnHeader(getApiVersion(), Retcode.r_fail_function_exception_VALUE); - } - } - case Message.Funcs.f_syncInfo_VALUE: { - if (service != Message.Servs.s_net_VALUE) { - return ApiUtil.toReturnHeader(getApiVersion(), Retcode.r_fail_service_call_VALUE); - } + try { + req = Message.req_getTransactionByHash.parseFrom(data); + byte[] txHash = req.getTxHash().toByteArray(); - SyncInfo sync = this.getSync(); + if (txHash == null || txHash.length != this.getTxHashLen()) { + return ApiUtil.toReturnHeader( + getApiVersion(), Retcode.r_fail_function_arguments_VALUE); + } - try { - Message.rsp_syncInfo rsp = Message.rsp_syncInfo.newBuilder().setChainBestBlock(sync.chainBestBlkNumber) - .setNetworkBestBlock(sync.networkBestBlkNumber).setSyncing(!sync.done) - .setMaxImportBlocks(24).build(); + AionTransaction tx = this.getTransactionByHash(txHash); + if (tx == null) { + return ApiUtil.toReturnHeader( + getApiVersion(), Retcode.r_fail_function_call_VALUE); + } - byte[] retHeader = ApiUtil.toReturnHeader(getApiVersion(), Retcode.r_success_VALUE); - return ApiUtil.combineRetMsg(retHeader, rsp.toByteArray()); - } catch (Exception e) { - LOG.error("ApiAion0.process.syncInfo exception: [{}]", e.getMessage()); - return ApiUtil.toReturnHeader(getApiVersion(), Retcode.r_fail_function_exception_VALUE); - } - } - case Message.Funcs.f_accountCreate_VALUE: { - if (service != Message.Servs.s_account_VALUE) { - return ApiUtil.toReturnHeader(getApiVersion(), Retcode.r_fail_service_call_VALUE); + Message.rsp_getTransaction rsp = getRsp_getTransaction(tx); + + byte[] retHeader = + ApiUtil.toReturnHeader(getApiVersion(), Retcode.r_success_VALUE); + return ApiUtil.combineRetMsg(retHeader, rsp.toByteArray()); + + } catch (Exception e) { + LOG.error( + "ApiAion0.process.getTransactionCount exception: [{}]", + e.getMessage()); + return ApiUtil.toReturnHeader( + getApiVersion(), Retcode.r_fail_function_exception_VALUE); + } } - byte[] data = parseMsgReq(request, msgHash); - Message.req_accountCreate req; + case Message.Funcs.f_getActiveNodes_VALUE: { + if (service != Message.Servs.s_net_VALUE) { + return ApiUtil.toReturnHeader( + getApiVersion(), Retcode.r_fail_service_call_VALUE); + } - try { - req = Message.req_accountCreate.parseFrom(data); - - List addressList = new ArrayList<>(); - List pKeyList = new ArrayList<>(); - for (int i = 0; i < req.getPasswordList().size() && i < ACCOUNT_CREATE_LIMIT; i++) { - - String addr = Keystore.create(req.getPassword(i)); - byte[] pKey; - if (req.getPrivateKey()) { - pKey = Keystore.getKey(addr, req.getPassword(i)).getPrivKeyBytes(); - if (pKey == null) { - pKey = ByteArrayWrapper.NULL_BYTE; - } + List nodes = new ArrayList<>( + this.ac.getAionHub().getP2pMgr().getActiveNodes().values()); + List pl = new ArrayList<>(); + try { + for (INode n : nodes) { + Message.t_Node node = + Message.t_Node + .newBuilder() + .setBlockNumber(n.getBestBlockNumber()) + .setNodeId(ByteArrayWrapper.wrap(n.getId()).toString()) + .setRemoteP2PIp( + ByteArrayWrapper.wrap(n.getIp()).toString()) + // TODO : api export latency, totalDiff and the + // blockHash + // .setLatency((float) n.get) + .build(); - pKeyList.add(ByteString.copyFrom(pKey)); + pl.add(node); } - addressList.add(ByteString.copyFrom(Address.wrap(addr).toBytes())); + Message.rsp_getActiveNodes rsp = + Message.rsp_getActiveNodes.newBuilder().addAllNode(pl).build(); + byte[] retHeader = + ApiUtil.toReturnHeader(getApiVersion(), Retcode.r_success_VALUE); + return ApiUtil.combineRetMsg(retHeader, rsp.toByteArray()); + } catch (Exception e) { + LOG.error( + "ApiAion0.process.getActiveNodes exception: [{}]", e.getMessage()); + return ApiUtil.toReturnHeader( + getApiVersion(), Retcode.r_fail_function_exception_VALUE); } + } - Message.rsp_accountCreate rsp = Message.rsp_accountCreate.newBuilder().addAllAddress(addressList) - .addAllPrivateKey(pKeyList).build(); + case Message.Funcs.f_getStaticNodes_VALUE: { + if (service != Message.Servs.s_net_VALUE) { + return ApiUtil.toReturnHeader( + getApiVersion(), Retcode.r_fail_service_call_VALUE); + } + String[] al = this.getBootNodes(); + List nl = new ArrayList<>(); + for (String s : al) { + // TODO : get more node info + Message.t_Node n = Message.t_Node.newBuilder().setRemoteP2PIp(s).build(); + nl.add(n); + } - byte[] retHeader = ApiUtil.toReturnHeader(getApiVersion(), Retcode.r_success_VALUE); - return ApiUtil.combineRetMsg(retHeader, rsp.toByteArray()); + try { + Message.rsp_getStaticNodes rsp = + Message.rsp_getStaticNodes.newBuilder().addAllNode(nl).build(); - } catch (Exception e) { - LOG.error("ApiAion0.process.accountCreate exception: [{}]", e.getMessage()); - return ApiUtil.toReturnHeader(getApiVersion(), Retcode.r_fail_function_exception_VALUE); + byte[] retHeader = + ApiUtil.toReturnHeader(getApiVersion(), Retcode.r_success_VALUE); + return ApiUtil.combineRetMsg(retHeader, rsp.toByteArray()); + } catch (Exception e) { + LOG.error( + "ApiAion0.process.getStaticNodes exception: [{}]", e.getMessage()); + return ApiUtil.toReturnHeader( + getApiVersion(), Retcode.r_fail_function_exception_VALUE); + } } - } + case Message.Funcs.f_getSolcVersion_VALUE: { + if (service != Message.Servs.s_tx_VALUE) { + return ApiUtil.toReturnHeader( + getApiVersion(), Retcode.r_fail_service_call_VALUE); + } - case Message.Funcs.f_accountLock_VALUE: { - if (service != Message.Servs.s_wallet_VALUE) { - return ApiUtil.toReturnHeader(getApiVersion(), Retcode.r_fail_service_call_VALUE); - } + String ver = this.solcVersion(); - byte[] data = parseMsgReq(request, msgHash); + try { + Message.rsp_getSolcVersion rsp = + Message.rsp_getSolcVersion.newBuilder().setVer(ver).build(); - if (data == null) { - return ApiUtil.toReturnHeader(getApiVersion(), Retcode.r_fail_function_arguments_VALUE); + byte[] retHeader = + ApiUtil.toReturnHeader(getApiVersion(), Retcode.r_success_VALUE); + return ApiUtil.combineRetMsg(retHeader, rsp.toByteArray()); + } catch (Exception e) { + LOG.error( + "ApiAion0.process.getSolcVersion exception: [{}]", e.getMessage()); + return ApiUtil.toReturnHeader( + getApiVersion(), Retcode.r_fail_function_exception_VALUE); + } } + case Message.Funcs.f_isSyncing_VALUE: { + if (service != Message.Servs.s_net_VALUE) { + return ApiUtil.toReturnHeader( + getApiVersion(), Retcode.r_fail_service_call_VALUE); + } - boolean result; - try { - Message.req_accountlock req = Message.req_accountlock.parseFrom(data); - result = this.lockAccount(Address.wrap(req.getAccount().toByteArray()), req.getPassword()); - } catch (InvalidProtocolBufferException e) { - LOG.error("ApiAion0.process.lockAccount exception: [{}]", e.getMessage()); - return ApiUtil.toReturnHeader(getApiVersion(), Retcode.r_fail_function_exception_VALUE); + try { + Message.rsp_isSyncing rsp = + Message.rsp_isSyncing + .newBuilder() + .setSyncing(!this.getSync().done) + .build(); + + byte[] retHeader = + ApiUtil.toReturnHeader(getApiVersion(), Retcode.r_success_VALUE); + return ApiUtil.combineRetMsg(retHeader, rsp.toByteArray()); + } catch (Exception e) { + LOG.error("ApiAion0.process.syncing exception: [{}]", e.getMessage()); + return ApiUtil.toReturnHeader( + getApiVersion(), Retcode.r_fail_function_exception_VALUE); + } } + case Message.Funcs.f_syncInfo_VALUE: { + if (service != Message.Servs.s_net_VALUE) { + return ApiUtil.toReturnHeader( + getApiVersion(), Retcode.r_fail_service_call_VALUE); + } - byte[] retHeader = ApiUtil.toReturnHeader(getApiVersion(), Retcode.r_success_VALUE); - return ApiUtil.combineRetMsg(retHeader, (byte) (result ? 0x01 : 0x00)); - } + SyncInfo sync = this.getSync(); - case Message.Funcs.f_userPrivilege_VALUE: { + try { + Message.rsp_syncInfo rsp = + Message.rsp_syncInfo + .newBuilder() + .setChainBestBlock(sync.chainBestBlkNumber) + .setNetworkBestBlock(sync.networkBestBlkNumber) + .setSyncing(!sync.done) + .setMaxImportBlocks(24) + .build(); - if (service != Message.Servs.s_privilege_VALUE) { - return ApiUtil.toReturnHeader(getApiVersion(), Retcode.r_fail_service_call_VALUE); + byte[] retHeader = + ApiUtil.toReturnHeader(getApiVersion(), Retcode.r_success_VALUE); + return ApiUtil.combineRetMsg(retHeader, rsp.toByteArray()); + } catch (Exception e) { + LOG.error("ApiAion0.process.syncInfo exception: [{}]", e.getMessage()); + return ApiUtil.toReturnHeader( + getApiVersion(), Retcode.r_fail_function_exception_VALUE); + } } + case Message.Funcs.f_accountCreate_VALUE: { + if (service != Message.Servs.s_account_VALUE) { + return ApiUtil.toReturnHeader( + getApiVersion(), Retcode.r_fail_service_call_VALUE); + } - return ApiUtil.toReturnHeader(getApiVersion(), Retcode.r_fail_unsupport_api_VALUE); - } - case Message.Funcs.f_mining_VALUE: { - if (service != Message.Servs.s_mine_VALUE) { - return ApiUtil.toReturnHeader(getApiVersion(), Retcode.r_fail_service_call_VALUE); - } + byte[] data = parseMsgReq(request, msgHash); + Message.req_accountCreate req; - Message.rsp_mining rsp = Message.rsp_mining.newBuilder().setMining(this.isMining()).build(); + try { + req = Message.req_accountCreate.parseFrom(data); + + List addressList = new ArrayList<>(); + List pKeyList = new ArrayList<>(); + for (int i = 0; + i < req.getPasswordList().size() && i < ACCOUNT_CREATE_LIMIT; + i++) { + + String addr = Keystore.create(req.getPassword(i)); + byte[] pKey; + if (req.getPrivateKey()) { + pKey = Keystore.getKey(addr, req.getPassword(i)).getPrivKeyBytes(); + if (pKey == null) { + pKey = ByteArrayWrapper.NULL_BYTE; + } - byte[] retHeader = ApiUtil.toReturnHeader(getApiVersion(), Retcode.r_success_VALUE); - return ApiUtil.combineRetMsg(retHeader, rsp.toByteArray()); - } + pKeyList.add(ByteString.copyFrom(pKey)); + } - case Message.Funcs.f_estimateNrg_VALUE: { - if (service != Message.Servs.s_tx_VALUE) { - return ApiUtil.toReturnHeader(getApiVersion(), Retcode.r_fail_service_call_VALUE, msgHash); - } + addressList.add(ByteString.copyFrom(Address.wrap(addr).toBytes())); + } - byte[] data = parseMsgReq(request, msgHash); - Message.req_estimateNrg req; - long result; - try { - req = Message.req_estimateNrg.parseFrom(data); + Message.rsp_accountCreate rsp = + Message.rsp_accountCreate + .newBuilder() + .addAllAddress(addressList) + .addAllPrivateKey(pKeyList) + .build(); - ArgTxCall params = new ArgTxCall(Address.wrap(req.getFrom().toByteArray()), - Address.wrap(req.getTo().toByteArray()), req.getData().toByteArray(), BigInteger.ZERO, - new BigInteger(req.getValue().toByteArray()), req.getNrg(), req.getNrgPrice()); + byte[] retHeader = + ApiUtil.toReturnHeader(getApiVersion(), Retcode.r_success_VALUE); + return ApiUtil.combineRetMsg(retHeader, rsp.toByteArray()); - result = this.estimateNrg(params); - } catch (InvalidProtocolBufferException e) { - LOG.error("ApiAion0.process.estimateNrg exception: [{}]", e.getMessage()); - return ApiUtil.toReturnHeader(getApiVersion(), Retcode.r_fail_function_exception_VALUE, msgHash); + } catch (Exception e) { + LOG.error("ApiAion0.process.accountCreate exception: [{}]", e.getMessage()); + return ApiUtil.toReturnHeader( + getApiVersion(), Retcode.r_fail_function_exception_VALUE); + } } - Message.rsp_estimateNrg rsp = Message.rsp_estimateNrg.newBuilder().setNrg(result).build(); + case Message.Funcs.f_accountLock_VALUE: { + if (service != Message.Servs.s_wallet_VALUE) { + return ApiUtil.toReturnHeader( + getApiVersion(), Retcode.r_fail_service_call_VALUE); + } - byte[] retHeader = ApiUtil.toReturnHeader(getApiVersion(), Retcode.r_success_VALUE); - return ApiUtil.combineRetMsg(retHeader, rsp.toByteArray()); - } + byte[] data = parseMsgReq(request, msgHash); - case Message.Funcs.f_exportAccounts_VALUE: - case Message.Funcs.f_backupAccounts_VALUE: { - if (service != Message.Servs.s_account_VALUE) { - return ApiUtil.toReturnHeader(getApiVersion(), Retcode.r_fail_service_call_VALUE); - } + if (data == null) { + return ApiUtil.toReturnHeader( + getApiVersion(), Retcode.r_fail_function_arguments_VALUE); + } - byte[] data = parseMsgReq(request, msgHash); - Message.req_exportAccounts req; + boolean result; + try { + Message.req_accountlock req = Message.req_accountlock.parseFrom(data); + result = + this.lockAccount( + Address.wrap(req.getAccount().toByteArray()), + req.getPassword()); + } catch (InvalidProtocolBufferException e) { + LOG.error("ApiAion0.process.lockAccount exception: [{}]", e.getMessage()); + return ApiUtil.toReturnHeader( + getApiVersion(), Retcode.r_fail_function_exception_VALUE); + } - try { - req = Message.req_exportAccounts.parseFrom(data); - } catch (Exception e) { - LOG.error("ApiAion0.process.exportAccounts exception: [{}]", e.getMessage()); - return ApiUtil.toReturnHeader(getApiVersion(), Retcode.r_fail_function_exception_VALUE); + byte[] retHeader = + ApiUtil.toReturnHeader(getApiVersion(), Retcode.r_success_VALUE); + return ApiUtil.combineRetMsg(retHeader, (byte) (result ? 0x01 : 0x00)); } - Map addrMap = new HashMap<>(); - for (int i = 0; i < req.getKeyFileList().size() && i < ACCOUNT_CREATE_LIMIT; i++) { - addrMap.put(Address.wrap(req.getKeyFile(i).getAddress().toByteArray()), - req.getKeyFile(i).getPassword()); + case Message.Funcs.f_userPrivilege_VALUE: { + if (service != Message.Servs.s_privilege_VALUE) { + return ApiUtil.toReturnHeader( + getApiVersion(), Retcode.r_fail_service_call_VALUE); + } + + return ApiUtil.toReturnHeader( + getApiVersion(), Retcode.r_fail_unsupport_api_VALUE); } + case Message.Funcs.f_mining_VALUE: { + if (service != Message.Servs.s_mine_VALUE) { + return ApiUtil.toReturnHeader( + getApiVersion(), Retcode.r_fail_service_call_VALUE); + } - Map res = ((short) request[2] == Message.Funcs.f_exportAccounts_VALUE) - ? Keystore.exportAccount(addrMap) - : Keystore.backupAccount(addrMap); + Message.rsp_mining rsp = + Message.rsp_mining.newBuilder().setMining(this.isMining()).build(); - List invalidKey = addrMap.keySet().parallelStream() - .filter(addr -> res.keySet().parallelStream().noneMatch(ad -> ad.equals(addr))) - .map(match -> ByteString.copyFrom(match.toBytes())).collect(Collectors.toList()); + byte[] retHeader = + ApiUtil.toReturnHeader(getApiVersion(), Retcode.r_success_VALUE); + return ApiUtil.combineRetMsg(retHeader, rsp.toByteArray()); + } - List keyBins = res.values().parallelStream().map(key -> ByteString.copyFrom(key.toBytes())) - .collect(Collectors.toList()); + case Message.Funcs.f_estimateNrg_VALUE: { + if (service != Message.Servs.s_tx_VALUE) { + return ApiUtil.toReturnHeader( + getApiVersion(), Retcode.r_fail_service_call_VALUE, msgHash); + } - Message.rsp_exportAccounts rsp = Message.rsp_exportAccounts.newBuilder().addAllKeyFile(keyBins) - .addAllFailedKey(invalidKey).build(); + byte[] data = parseMsgReq(request, msgHash); + Message.req_estimateNrg req; + long result; + try { + req = Message.req_estimateNrg.parseFrom(data); + + ArgTxCall params = + new ArgTxCall( + Address.wrap(req.getFrom().toByteArray()), + Address.wrap(req.getTo().toByteArray()), + req.getData().toByteArray(), + BigInteger.ZERO, + new BigInteger(req.getValue().toByteArray()), + req.getNrg(), + req.getNrgPrice()); + + result = this.estimateNrg(params); + } catch (InvalidProtocolBufferException e) { + LOG.error("ApiAion0.process.estimateNrg exception: [{}]", e.getMessage()); + return ApiUtil.toReturnHeader( + getApiVersion(), Retcode.r_fail_function_exception_VALUE, msgHash); + } - byte[] retHeader = ApiUtil.toReturnHeader(getApiVersion(), Retcode.r_success_VALUE); - return ApiUtil.combineRetMsg(retHeader, rsp.toByteArray()); + Message.rsp_estimateNrg rsp = + Message.rsp_estimateNrg.newBuilder().setNrg(result).build(); - } - case Message.Funcs.f_importAccounts_VALUE: { - if (service != Message.Servs.s_account_VALUE) { - return ApiUtil.toReturnHeader(getApiVersion(), Retcode.r_fail_service_call_VALUE); + byte[] retHeader = + ApiUtil.toReturnHeader(getApiVersion(), Retcode.r_success_VALUE); + return ApiUtil.combineRetMsg(retHeader, rsp.toByteArray()); } - byte[] data = parseMsgReq(request, msgHash); - Message.req_importAccounts req; + case Message.Funcs.f_exportAccounts_VALUE: + case Message.Funcs.f_backupAccounts_VALUE: { + if (service != Message.Servs.s_account_VALUE) { + return ApiUtil.toReturnHeader( + getApiVersion(), Retcode.r_fail_service_call_VALUE); + } - try { - req = Message.req_importAccounts.parseFrom(data); - } catch (Exception e) { - LOG.error("ApiAion0.process.importAccount exception: [{}]", e.getMessage()); - return ApiUtil.toReturnHeader(getApiVersion(), Retcode.r_fail_function_exception_VALUE); - } + byte[] data = parseMsgReq(request, msgHash); + Message.req_exportAccounts req; - Map importKey = req.getPrivateKeyList().parallelStream() - .collect(Collectors.toMap(Message.t_PrivateKey::getPrivateKey, Message.t_PrivateKey::getPassword)); + try { + req = Message.req_exportAccounts.parseFrom(data); + } catch (Exception e) { + LOG.error( + "ApiAion0.process.exportAccounts exception: [{}]", e.getMessage()); + return ApiUtil.toReturnHeader( + getApiVersion(), Retcode.r_fail_function_exception_VALUE); + } - if (importKey == null) { - LOG.error("ApiAion0.process.importAccount exception: [null importKey]"); - return ApiUtil.toReturnHeader(getApiVersion(), Retcode.r_fail_function_exception_VALUE); - } + Map addrMap = new HashMap<>(); + for (int i = 0; + i < req.getKeyFileList().size() && i < ACCOUNT_CREATE_LIMIT; + i++) { + addrMap.put( + Address.wrap(req.getKeyFile(i).getAddress().toByteArray()), + req.getKeyFile(i).getPassword()); + } - Set res = Keystore.importAccount(importKey); - if (res == null) { - throw new NullPointerException(); - } + Map res = + ((short) request[2] == Message.Funcs.f_exportAccounts_VALUE) + ? Keystore.exportAccount(addrMap) + : Keystore.backupAccount(addrMap); - Message.rsp_importAccounts rsp = Message.rsp_importAccounts.newBuilder().addAllInvalidKey(res).build(); + List invalidKey = + addrMap.keySet() + .parallelStream() + .filter( + addr -> + res.keySet() + .parallelStream() + .noneMatch(ad -> ad.equals(addr))) + .map(match -> ByteString.copyFrom(match.toBytes())) + .collect(Collectors.toList()); + + List keyBins = + res.values() + .parallelStream() + .map(key -> ByteString.copyFrom(key.toBytes())) + .collect(Collectors.toList()); + + Message.rsp_exportAccounts rsp = + Message.rsp_exportAccounts + .newBuilder() + .addAllKeyFile(keyBins) + .addAllFailedKey(invalidKey) + .build(); - byte[] retHeader = ApiUtil.toReturnHeader(getApiVersion(), Retcode.r_success_VALUE); - return ApiUtil.combineRetMsg(retHeader, rsp.toByteArray()); - } - case Message.Funcs.f_signedTransaction_VALUE: - case Message.Funcs.f_rawTransaction_VALUE: { - if (service != Message.Servs.s_tx_VALUE) { - return ApiUtil.toReturnHeader(getApiVersion(), Retcode.r_fail_service_call_VALUE, msgHash); + byte[] retHeader = + ApiUtil.toReturnHeader(getApiVersion(), Retcode.r_success_VALUE); + return ApiUtil.combineRetMsg(retHeader, rsp.toByteArray()); } - - byte[] data = parseMsgReq(request, msgHash); - Message.req_rawTransaction req; - byte[] result; - try { - req = Message.req_rawTransaction.parseFrom(data); - byte[] encodedTx = req.getEncodedTx().toByteArray(); - if (encodedTx == null) { - LOG.error("ApiAion0.process.rawTransaction exception: [null encodedTx]"); - return ApiUtil.toReturnHeader(getApiVersion(), Retcode.r_fail_function_arguments_VALUE); + case Message.Funcs.f_importAccounts_VALUE: { + if (service != Message.Servs.s_account_VALUE) { + return ApiUtil.toReturnHeader( + getApiVersion(), Retcode.r_fail_service_call_VALUE); } - result = this.sendTransaction(encodedTx); - } catch (Exception e) { - LOG.error("ApiAion0.process.rawTransaction exception: [{}]", e.getMessage()); - return ApiUtil.toReturnHeader(getApiVersion(), Retcode.r_fail_function_exception_VALUE, msgHash); - } + byte[] data = parseMsgReq(request, msgHash); + Message.req_importAccounts req; - if (result == null) { - return ApiUtil.toReturnHeader(getApiVersion(), Retcode.r_fail_sendTx_null_rep_VALUE, msgHash); - } + try { + req = Message.req_importAccounts.parseFrom(data); + } catch (Exception e) { + LOG.error("ApiAion0.process.importAccount exception: [{}]", e.getMessage()); + return ApiUtil.toReturnHeader( + getApiVersion(), Retcode.r_fail_function_exception_VALUE); + } - getMsgIdMapping().put(ByteArrayWrapper.wrap(result), new AbstractMap.SimpleEntry<>( - ByteArrayWrapper.wrap(msgHash), ByteArrayWrapper.wrap(socketId))); - if (LOG.isDebugEnabled()) { - LOG.debug("ApiAion0.process.rawTransaction - msgIdMapping.put: [{}]", ByteArrayWrapper.wrap(result).toString()); - } + Map importKey = + req.getPrivateKeyList() + .parallelStream() + .collect( + Collectors.toMap( + Message.t_PrivateKey::getPrivateKey, + Message.t_PrivateKey::getPassword)); + + if (importKey == null) { + LOG.error("ApiAion0.process.importAccount exception: [null importKey]"); + return ApiUtil.toReturnHeader( + getApiVersion(), Retcode.r_fail_function_exception_VALUE); + } - Message.rsp_sendTransaction rsp = Message.rsp_sendTransaction.newBuilder() - .setTxHash(ByteString.copyFrom(result)).build(); + Set res = Keystore.importAccount(importKey); + if (res == null) { + throw new NullPointerException(); + } - byte[] retHeader = ApiUtil.toReturnHeader(getApiVersion(), Retcode.r_tx_Recved_VALUE, msgHash); - return ApiUtil.combineRetMsg(retHeader, rsp.toByteArray()); - } - case Message.Funcs.f_eventRegister_VALUE: { - if (service != Message.Servs.s_tx_VALUE) { - return ApiUtil.toReturnHeader(getApiVersion(), Retcode.r_fail_service_call_VALUE); + Message.rsp_importAccounts rsp = + Message.rsp_importAccounts.newBuilder().addAllInvalidKey(res).build(); + + byte[] retHeader = + ApiUtil.toReturnHeader(getApiVersion(), Retcode.r_success_VALUE); + return ApiUtil.combineRetMsg(retHeader, rsp.toByteArray()); } + case Message.Funcs.f_signedTransaction_VALUE: + case Message.Funcs.f_rawTransaction_VALUE: { + if (service != Message.Servs.s_tx_VALUE) { + return ApiUtil.toReturnHeader( + getApiVersion(), Retcode.r_fail_service_call_VALUE, msgHash); + } - byte[] data = parseMsgReq(request, msgHash); - Message.req_eventRegister req; + byte[] data = parseMsgReq(request, msgHash); + Message.req_rawTransaction req; + byte[] result; + try { + req = Message.req_rawTransaction.parseFrom(data); + byte[] encodedTx = req.getEncodedTx().toByteArray(); + if (encodedTx == null) { + LOG.error( + "ApiAion0.process.rawTransaction exception: [null encodedTx]"); + return ApiUtil.toReturnHeader( + getApiVersion(), Retcode.r_fail_function_arguments_VALUE); + } - try { - req = Message.req_eventRegister.parseFrom(data); + result = this.sendTransaction(encodedTx); + } catch (Exception e) { + LOG.error( + "ApiAion0.process.rawTransaction exception: [{}]", e.getMessage()); + return ApiUtil.toReturnHeader( + getApiVersion(), Retcode.r_fail_function_exception_VALUE, msgHash); + } - List evtList = new ArrayList<>(req.getEventsList()); - if (evtList.isEmpty()) { - LOG.error("ApiNucoNcp.process.eventRegister : [{}]", "empty event list"); - return ApiUtil.toReturnHeader(getApiVersion(), Retcode.r_fail_function_arguments_VALUE); + if (result == null) { + return ApiUtil.toReturnHeader( + getApiVersion(), Retcode.r_fail_sendTx_null_rep_VALUE, msgHash); } - Message.t_FilterCt fltr = req.getFilter(); - List accList = new ArrayList<>(); - fltr.getAddressesList().forEach(a -> accList.add(a.toByteArray())); + getMsgIdMapping() + .put( + ByteArrayWrapper.wrap(result), + new AbstractMap.SimpleEntry<>( + ByteArrayWrapper.wrap(msgHash), + ByteArrayWrapper.wrap(socketId))); + if (LOG.isDebugEnabled()) { + LOG.debug( + "ApiAion0.process.rawTransaction - msgIdMapping.put: [{}]", + ByteArrayWrapper.wrap(result).toString()); + } - long lv = ByteUtil.byteArrayToLong(socketId); - FltrCt preFc = (FltrCt) installedFilters.get(lv); - if (Optional.ofNullable(preFc).isPresent()) { - preFc.getTopics().forEach(t -> { - if (!fltr.getTopicsList().contains(t)) { - evtList.add(t); - } - }); + Message.rsp_sendTransaction rsp = + Message.rsp_sendTransaction + .newBuilder() + .setTxHash(ByteString.copyFrom(result)) + .build(); + + byte[] retHeader = + ApiUtil.toReturnHeader( + getApiVersion(), Retcode.r_tx_Recved_VALUE, msgHash); + return ApiUtil.combineRetMsg(retHeader, rsp.toByteArray()); + } + case Message.Funcs.f_eventRegister_VALUE: { + if (service != Message.Servs.s_tx_VALUE) { + return ApiUtil.toReturnHeader( + getApiVersion(), Retcode.r_fail_service_call_VALUE); } - FltrCt fc = new FltrCt(fltr.getContractAddr().toByteArray(), fltr.getTo(), fltr.getFrom(), evtList, - accList, fltr.getExpireTime()); + byte[] data = parseMsgReq(request, msgHash); + Message.req_eventRegister req; + + try { + req = Message.req_eventRegister.parseFrom(data); + + List evtList = new ArrayList<>(req.getEventsList()); + if (evtList.isEmpty()) { + LOG.error( + "ApiNucoNcp.process.eventRegister : [{}]", "empty event list"); + return ApiUtil.toReturnHeader( + getApiVersion(), Retcode.r_fail_function_arguments_VALUE); + } - installedFilters.put(lv, fc); + Message.t_FilterCt fltr = req.getFilter(); + List accList = new ArrayList<>(); + fltr.getAddressesList().forEach(a -> accList.add(a.toByteArray())); + + long lv = ByteUtil.byteArrayToLong(socketId); + FltrCt preFc = (FltrCt) installedFilters.get(lv); + if (Optional.ofNullable(preFc).isPresent()) { + preFc.getTopics() + .forEach( + t -> { + if (!fltr.getTopicsList().contains(t)) { + evtList.add(t); + } + }); + } - Message.rsp_eventRegister rsp = Message.rsp_eventRegister.newBuilder().setResult(true).build(); + FltrCt fc = + new FltrCt( + fltr.getContractAddr().toByteArray(), + fltr.getTo(), + fltr.getFrom(), + evtList, + accList, + fltr.getExpireTime()); - byte[] retHeader = ApiUtil.toReturnHeader(getApiVersion(), Retcode.r_success_VALUE); - return ApiUtil.combineRetMsg(retHeader, rsp.toByteArray()); + installedFilters.put(lv, fc); - } catch (Exception e) { - LOG.error("ApiAion0.process.eventRegister exception: [{}]", e.getMessage()); - return ApiUtil.toReturnHeader(getApiVersion(), Retcode.r_fail_function_exception_VALUE); - } - } - case Message.Funcs.f_eventDeregister_VALUE: { - if (service != Message.Servs.s_tx_VALUE) { - return ApiUtil.toReturnHeader(getApiVersion(), Retcode.r_fail_service_call_VALUE); + Message.rsp_eventRegister rsp = + Message.rsp_eventRegister.newBuilder().setResult(true).build(); + + byte[] retHeader = + ApiUtil.toReturnHeader(getApiVersion(), Retcode.r_success_VALUE); + return ApiUtil.combineRetMsg(retHeader, rsp.toByteArray()); + + } catch (Exception e) { + LOG.error("ApiAion0.process.eventRegister exception: [{}]", e.getMessage()); + return ApiUtil.toReturnHeader( + getApiVersion(), Retcode.r_fail_function_exception_VALUE); + } } + case Message.Funcs.f_eventDeregister_VALUE: { + if (service != Message.Servs.s_tx_VALUE) { + return ApiUtil.toReturnHeader( + getApiVersion(), Retcode.r_fail_service_call_VALUE); + } - byte[] data = parseMsgReq(request, msgHash); - Message.req_eventDeregister req; + byte[] data = parseMsgReq(request, msgHash); + Message.req_eventDeregister req; - try { - req = Message.req_eventDeregister.parseFrom(data); + try { + req = Message.req_eventDeregister.parseFrom(data); - List evtList = new ArrayList<>(req.getEventsList()); + List evtList = new ArrayList<>(req.getEventsList()); - byte[] conrtactAddr; - if (req.getContractAddr() == null) { - conrtactAddr = null; - } else { - conrtactAddr = req.getContractAddr().toByteArray(); - } + byte[] conrtactAddr; + if (req.getContractAddr() == null) { + conrtactAddr = null; + } else { + conrtactAddr = req.getContractAddr().toByteArray(); + } - if (evtList.isEmpty()) { - LOG.error("ApiAion0.process.eventRegister : [{}]", "empty event list"); - return ApiUtil.toReturnHeader(getApiVersion(), Retcode.r_fail_function_arguments_VALUE); - } + if (evtList.isEmpty()) { + LOG.error("ApiAion0.process.eventRegister : [{}]", "empty event list"); + return ApiUtil.toReturnHeader( + getApiVersion(), Retcode.r_fail_function_arguments_VALUE); + } - long lv = ByteUtil.byteArrayToLong(socketId); - FltrCt preFc = (FltrCt) installedFilters.get(lv); + long lv = ByteUtil.byteArrayToLong(socketId); + FltrCt preFc = (FltrCt) installedFilters.get(lv); - boolean changed = false; - if (Optional.ofNullable(preFc).isPresent() && Arrays.equals(preFc.getContractAddr(), conrtactAddr)) { - evtList.forEach(ev -> { - if (preFc.getTopics().contains(ev)) { - preFc.getTopics().remove(ev); - } - }); + boolean changed = false; + if (Optional.ofNullable(preFc).isPresent() + && Arrays.equals(preFc.getContractAddr(), conrtactAddr)) { + evtList.forEach( + ev -> preFc.getTopics().remove(ev)); - installedFilters.put(lv, preFc); - changed = true; - } + installedFilters.put(lv, preFc); + changed = true; + } - Message.rsp_eventRegister rsp = Message.rsp_eventRegister.newBuilder().setResult(changed).build(); + Message.rsp_eventRegister rsp = + Message.rsp_eventRegister.newBuilder().setResult(changed).build(); - byte[] retHeader = ApiUtil.toReturnHeader(getApiVersion(), Retcode.r_success_VALUE); - return ApiUtil.combineRetMsg(retHeader, rsp.toByteArray()); + byte[] retHeader = + ApiUtil.toReturnHeader(getApiVersion(), Retcode.r_success_VALUE); + return ApiUtil.combineRetMsg(retHeader, rsp.toByteArray()); - } catch (Exception e) { - LOG.error("ApiAion0.process.eventDeregister exception: [{}]", e.getMessage()); - return ApiUtil.toReturnHeader(getApiVersion(), Retcode.r_fail_function_exception_VALUE); - } - } - case Message.Funcs.f_getBlockDetailsByNumber_VALUE: { - if (service != Message.Servs.s_admin_VALUE) { - return ApiUtil.toReturnHeader(getApiVersion(), Retcode.r_fail_service_call_VALUE); + } catch (Exception e) { + LOG.error( + "ApiAion0.process.eventDeregister exception: [{}]", e.getMessage()); + return ApiUtil.toReturnHeader( + getApiVersion(), Retcode.r_fail_function_exception_VALUE); + } } + case Message.Funcs.f_getBlockDetailsByNumber_VALUE: { + if (service != Message.Servs.s_admin_VALUE) { + return ApiUtil.toReturnHeader( + getApiVersion(), Retcode.r_fail_service_call_VALUE); + } - byte[] data = parseMsgReq(request, msgHash); - Message.req_getBlockDetailsByNumber req; + byte[] data = parseMsgReq(request, msgHash); + Message.req_getBlockDetailsByNumber req; - try { - req = Message.req_getBlockDetailsByNumber.parseFrom(data); - long latestBlkNum = this.getBestBlock().getNumber(); + try { + req = Message.req_getBlockDetailsByNumber.parseFrom(data); + long latestBlkNum = this.getBestBlock().getNumber(); - List blkNum = req.getBlkNumbersList() - .parallelStream() - .filter(n -> n <= latestBlkNum) - .collect(Collectors.toSet()) + List blkNum = + req.getBlkNumbersList() + .parallelStream() + .filter(n -> n <= latestBlkNum) + .collect(Collectors.toSet()) .parallelStream() .sorted() .collect(Collectors.toList()); - if (blkNum.size() > 1000) { - blkNum = blkNum.subList(0, 1000); - } - - List> blks = getBlkAndDifficultyForBlkNumList(blkNum); + if (blkNum.size() > 1000) { + blkNum = blkNum.subList(0, 1000); + } - if (blks == null) { - return ApiUtil.toReturnHeader(getApiVersion(), Retcode.r_fail_function_arguments_VALUE); - } else { + List> blks = + getBlkAndDifficultyForBlkNumList(blkNum); + if (blks == null) { + return ApiUtil.toReturnHeader( + getApiVersion(), Retcode.r_fail_function_arguments_VALUE); + } else { - List bds = getRsp_getBlockDetails(blks); - Message.rsp_getBlockDetailsByNumber rsp = Message.rsp_getBlockDetailsByNumber.newBuilder() - .addAllBlkDetails(bds) - .build(); + List bds = getRsp_getBlockDetails(blks); + Message.rsp_getBlockDetailsByNumber rsp = + Message.rsp_getBlockDetailsByNumber + .newBuilder() + .addAllBlkDetails(bds) + .build(); - byte[] retHeader = ApiUtil.toReturnHeader(getApiVersion(), Retcode.r_success_VALUE); - return ApiUtil.combineRetMsg(retHeader, rsp.toByteArray()); + byte[] retHeader = + ApiUtil.toReturnHeader( + getApiVersion(), Retcode.r_success_VALUE); + return ApiUtil.combineRetMsg(retHeader, rsp.toByteArray()); + } + } catch (Exception e) { + LOG.error( + "ApiAion0.process.getBlockDetailsByNumber exception: [{}]", + e.getMessage()); + return ApiUtil.toReturnHeader( + getApiVersion(), Retcode.r_fail_function_exception_VALUE); } - } catch (Exception e) { - LOG.error("ApiAion0.process.getBlockDetailsByNumber exception: [{}]", e.getMessage()); - return ApiUtil.toReturnHeader(getApiVersion(), Retcode.r_fail_function_exception_VALUE); - } - } - case Message.Funcs.f_getBlockSqlByRange_VALUE: { - if (service != Message.Servs.s_admin_VALUE) { - return ApiUtil.toReturnHeader(getApiVersion(), Retcode.r_fail_service_call_VALUE); } + case Message.Funcs.f_getBlockSqlByRange_VALUE: { + if (service != Message.Servs.s_admin_VALUE) { + return ApiUtil.toReturnHeader( + getApiVersion(), Retcode.r_fail_service_call_VALUE); + } - byte[] data = parseMsgReq(request, msgHash); - Message.req_getBlockSqlByRange req; - - try { - if (LOG.isDebugEnabled()) LOG.debug("BlockSqlByRange: start"); - - req = Message.req_getBlockSqlByRange.parseFrom(data); - long latestBlkNum = this.getBestBlock().getNumber(); + byte[] data = parseMsgReq(request, msgHash); + Message.req_getBlockSqlByRange req; - Long _blkStart = req.getBlkNumberStart(); - Long _blkEnd = req.getBlkNumberEnd(); + try { + if (LOG.isDebugEnabled()) { + LOG.debug("BlockSqlByRange: start"); + } - // no null check here + req = Message.req_getBlockSqlByRange.parseFrom(data); + long latestBlkNum = this.getBestBlock().getNumber(); - long blkStart = -1; - long blkEnd = -1; + Long _blkStart = req.getBlkNumberStart(); + Long _blkEnd = req.getBlkNumberEnd(); - if (_blkStart < 0) - blkStart = 0; - else - blkStart = _blkStart; + // no null check here - if (_blkEnd > latestBlkNum) - blkEnd = latestBlkNum; - else - blkEnd = _blkEnd; + long blkStart; + long blkEnd; - if (blkEnd < blkStart) - return ApiUtil.toReturnHeader(getApiVersion(), Retcode.r_fail_function_arguments_VALUE); + if (_blkStart < 0) { + blkStart = 0; + } else { + blkStart = _blkStart; + } - // truncate the thing - if (blkEnd - blkStart > 1000) { - blkStart = blkEnd - 1000 + 1; + if (_blkEnd > latestBlkNum) { + blkEnd = latestBlkNum; + } else { + blkEnd = _blkEnd; + } - if (blkStart < 0) blkStart = 0; - } + if (blkEnd < blkStart) { + return ApiUtil.toReturnHeader( + getApiVersion(), Retcode.r_fail_function_arguments_VALUE); + } - if (LOG.isDebugEnabled()) LOG.debug("BlockSqlByRange: range " + blkStart + "-" + blkEnd); + // truncate the thing + if (blkEnd - blkStart > 1000) { + blkStart = blkEnd - 1000 + 1; - Long lastBlockTimestamp = null; - long listLength = blkEnd - blkStart + 1; - List bds = new ArrayList<>(); + if (blkStart < 0) { + blkStart = 0; + } + } - for (int i = 0; i < listLength; i++) { - long blkNum = blkStart + i; - Map.Entry entry = getBlockWithTotalDifficulty(blkNum); - AionBlock b = entry.getKey(); - BigInteger td = entry.getValue(); - long blocktime = 0; - if (blkNum != 0 && lastBlockTimestamp == null) { - lastBlockTimestamp = getBlock(blkNum - 1).getTimestamp(); + if (LOG.isDebugEnabled()) { + LOG.debug("BlockSqlByRange: range " + blkStart + "-" + blkEnd); } - if (blkNum == 0) - blocktime = 0; - else - blocktime = b.getTimestamp() - lastBlockTimestamp; - lastBlockTimestamp = b.getTimestamp(); + Long lastBlockTimestamp = null; + long listLength = blkEnd - blkStart + 1; + List bds = new ArrayList<>(); + + for (int i = 0; i < listLength; i++) { + long blkNum = blkStart + i; + Map.Entry entry = + getBlockWithTotalDifficulty(blkNum); + AionBlock b = entry.getKey(); + BigInteger td = entry.getValue(); + long blocktime; + if (blkNum != 0 && lastBlockTimestamp == null) { + lastBlockTimestamp = getBlock(blkNum - 1).getTimestamp(); + } + if (blkNum == 0) { + blocktime = 0; + } else { + blocktime = b.getTimestamp() - lastBlockTimestamp; + } - String blockSql = generateBlockSqlStatement(b, td, blocktime); + lastBlockTimestamp = b.getTimestamp(); - List transactionSql = new ArrayList<>(); + String blockSql = generateBlockSqlStatement(b, td, blocktime); - AionBlockSummary bs = null; - if (explorerBlockCache != null) { - // remove from cache since after consumed, we're probably not gonna revisit it - bs = explorerBlockCache.remove(new ByteArrayWrapper(b.getHash())); - } + List transactionSql = new ArrayList<>(); - if (bs != null) { + AionBlockSummary bs = null; + if (explorerBlockCache != null) { + // remove from cache since after consumed, we're probably not gonna + // revisit it + bs = explorerBlockCache.remove(new ByteArrayWrapper(b.getHash())); + } - if (LOG.isDebugEnabled()) LOG.debug("BlockSqlByRange: cache HIT for #: " + b.getNumber()); + if (bs != null) { - Map receipts = new HashMap<>(); - for (AionTxReceipt r : bs.getReceipts()) { - receipts.put(new ByteArrayWrapper(r.getTransaction().getHash()), r); - } + if (LOG.isDebugEnabled()) { + LOG.debug("BlockSqlByRange: cache HIT for #: " + b.getNumber()); + } - List txns = b.getTransactionsList(); - for (int j = 0; j < txns.size(); j++) { - AionTransaction tx = txns.get(j); - AionTxReceipt r = receipts.get(new ByteArrayWrapper(tx.getHash())); - if (r == null) { - if (LOG.isDebugEnabled()) LOG.debug("BlockSqlByRange: transaction not in Block Summary: " + b.getNumber() + "." + j); - AionTxInfo ti = ((AionBlockchainImpl) this.ac.getAionHub().getBlockchain()) - .getTransactionInfoLite(tx.getHash(), b.getHash()); - r = ti.getReceipt(); + Map receipts = new HashMap<>(); + for (AionTxReceipt r : bs.getReceipts()) { + receipts.put( + new ByteArrayWrapper(r.getTransaction().getHash()), r); } - if (r == null) { - LOG.error("BlockSqlByRange: missing DB transaction: " + ByteUtil.toHexString(tx.getHash())); + + List txns = b.getTransactionsList(); + for (int j = 0; j < txns.size(); j++) { + AionTransaction tx = txns.get(j); + AionTxReceipt r = + receipts.get(new ByteArrayWrapper(tx.getHash())); + if (r == null) { + if (LOG.isDebugEnabled()) { + LOG.debug( + "BlockSqlByRange: transaction not in Block Summary: " + + b.getNumber() + + "." + + j); + } + AionTxInfo ti = + ((AionBlockchainImpl) + this.ac + .getAionHub() + .getBlockchain()) + .getTransactionInfoLite( + tx.getHash(), b.getHash()); + r = ti.getReceipt(); + } + if (r == null) { + LOG.error( + "BlockSqlByRange: missing DB transaction: " + + ByteUtil.toHexString(tx.getHash())); + } else { + transactionSql.add( + generateTransactionSqlStatement( + b, + tx, + r.getLogInfoList(), + j, + r.getEnergyUsed())); + } } - else { - transactionSql.add(generateTransactionSqlStatement(b, tx, r.getLogInfoList(), j, r.getEnergyUsed())); + } else { + if (LOG.isDebugEnabled()) { + LOG.debug( + "BlockSqlByRange: cache MISS for #: " + b.getNumber()); } + List txs = b.getTransactionsList(); + + transactionSql = + txs.parallelStream() + .filter(Objects::nonNull) + .map( + (AionTransaction tx) -> { + AionTxInfo ti = + ((AionBlockchainImpl) + this.ac + .getAionHub() + .getBlockchain()) + .getTransactionInfoLite( + tx.getHash(), + b.getHash()); + if (ti == null) { + LOG.error( + "BlockSqlByRange: missing DB transaction: " + + ByteUtil + .toHexString( + tx + .getHash())); + return null; + } else { + return generateTransactionSqlStatement( + b, + tx, + ti.getReceipt() + .getLogInfoList(), + ti.getIndex(), + ti.getReceipt() + .getEnergyUsed()); + } + }) + .filter(Objects::nonNull) + .collect(Collectors.toList()); } - } else { - if (LOG.isDebugEnabled()) LOG.debug("BlockSqlByRange: cache MISS for #: " + b.getNumber()); - List txs = b.getTransactionsList(); - - transactionSql = txs.parallelStream() - .filter(Objects::nonNull) - .map((AionTransaction tx) -> { - AionTxInfo ti = ((AionBlockchainImpl) this.ac.getAionHub().getBlockchain()) - .getTransactionInfoLite(tx.getHash(), b.getHash()); - if (ti == null) { - LOG.error("BlockSqlByRange: missing DB transaction: " + ByteUtil.toHexString(tx.getHash())); - return null; - } else { - return generateTransactionSqlStatement(b, tx, ti.getReceipt().getLogInfoList(), ti.getIndex(), ti.getReceipt().getEnergyUsed()); - } - }).filter(Objects::nonNull) - .collect(Collectors.toList()); - } - Message.t_BlockSql sqlObj = - Message.t_BlockSql.newBuilder() - .setBlockNumber(b.getNumber()) - .setBlockHash(ByteUtil.toHexString(b.getHash())) - .setParentHash(ByteUtil.toHexString(b.getParentHash())) - .setBlock(blockSql) - .addAllTx(transactionSql) - .build(); + Message.t_BlockSql sqlObj = + Message.t_BlockSql + .newBuilder() + .setBlockNumber(b.getNumber()) + .setBlockHash(ByteUtil.toHexString(b.getHash())) + .setParentHash(ByteUtil.toHexString(b.getParentHash())) + .setBlock(blockSql) + .addAllTx(transactionSql) + .build(); - bds.add(sqlObj); - } + bds.add(sqlObj); + } - Message.rsp_getBlockSqlByRange rsp = - Message.rsp_getBlockSqlByRange.newBuilder() - .addAllBlkSql(bds) - .build(); + Message.rsp_getBlockSqlByRange rsp = + Message.rsp_getBlockSqlByRange + .newBuilder() + .addAllBlkSql(bds) + .build(); - byte[] retHeader = ApiUtil.toReturnHeader(getApiVersion(), Retcode.r_success_VALUE); - return ApiUtil.combineRetMsg(retHeader, rsp.toByteArray()); - } catch (Exception e) { - LOG.error("ApiAion0.process.getBlockDetailsByNumber exception: [{}]", e.getMessage()); - return ApiUtil.toReturnHeader(getApiVersion(), Retcode.r_fail_function_exception_VALUE); - } - } - case Message.Funcs.f_getBlockDetailsByRange_VALUE: { - if (service != Message.Servs.s_admin_VALUE) { - return ApiUtil.toReturnHeader(getApiVersion(), Retcode.r_fail_service_call_VALUE); + byte[] retHeader = + ApiUtil.toReturnHeader(getApiVersion(), Retcode.r_success_VALUE); + return ApiUtil.combineRetMsg(retHeader, rsp.toByteArray()); + } catch (Exception e) { + LOG.error( + "ApiAion0.process.getBlockDetailsByNumber exception: [{}]", + e.getMessage()); + return ApiUtil.toReturnHeader( + getApiVersion(), Retcode.r_fail_function_exception_VALUE); + } } + case Message.Funcs.f_getBlockDetailsByRange_VALUE: { + if (service != Message.Servs.s_admin_VALUE) { + return ApiUtil.toReturnHeader( + getApiVersion(), Retcode.r_fail_service_call_VALUE); + } - byte[] data = parseMsgReq(request, msgHash); - Message.req_getBlockDetailsByRange req; - - try { - if (LOG.isDebugEnabled()) LOG.debug("getBlockDetailsByRange: start"); - - req = Message.req_getBlockDetailsByRange.parseFrom(data); - long latestBlkNum = this.getBestBlock().getNumber(); + byte[] data = parseMsgReq(request, msgHash); + Message.req_getBlockDetailsByRange req; - Long _blkStart = req.getBlkNumberStart(); - Long _blkEnd = req.getBlkNumberEnd(); + try { + if (LOG.isDebugEnabled()) { + LOG.debug("getBlockDetailsByRange: start"); + } - // no null check here + req = Message.req_getBlockDetailsByRange.parseFrom(data); + long latestBlkNum = this.getBestBlock().getNumber(); - long blkStart = -1; - long blkEnd = -1; + Long _blkStart = req.getBlkNumberStart(); + Long _blkEnd = req.getBlkNumberEnd(); - if (_blkStart < 0) - blkStart = 0; - else - blkStart = _blkStart; + // no null check here - // blocks requested in the future. return empty result - if (blkStart > latestBlkNum) { - Message.rsp_getBlockDetailsByRange rsp = - Message.rsp_getBlockDetailsByRange.newBuilder() - .addAllBlkDetails(new ArrayList<>()) - .build(); + long blkStart; + long blkEnd; - byte[] retHeader = ApiUtil.toReturnHeader(getApiVersion(), Retcode.r_success_VALUE); - return ApiUtil.combineRetMsg(retHeader, rsp.toByteArray()); - } + if (_blkStart < 0) { + blkStart = 0; + } else { + blkStart = _blkStart; + } - if (_blkEnd > latestBlkNum) - blkEnd = latestBlkNum; - else - blkEnd = _blkEnd; + // blocks requested in the future. return empty result + if (blkStart > latestBlkNum) { + Message.rsp_getBlockDetailsByRange rsp = + Message.rsp_getBlockDetailsByRange + .newBuilder() + .addAllBlkDetails(new ArrayList<>()) + .build(); - if (blkEnd < blkStart) - return ApiUtil.toReturnHeader(getApiVersion(), Retcode.r_fail_function_arguments_VALUE); + byte[] retHeader = + ApiUtil.toReturnHeader( + getApiVersion(), Retcode.r_success_VALUE); + return ApiUtil.combineRetMsg(retHeader, rsp.toByteArray()); + } - // truncate at the beginning of range - if (blkEnd - blkStart > 1000) { - blkStart = blkEnd - 1000 + 1; + if (_blkEnd > latestBlkNum) { + blkEnd = latestBlkNum; + } else { + blkEnd = _blkEnd; + } - if (blkStart < 0) blkStart = 0; - } + if (blkEnd < blkStart) { + return ApiUtil.toReturnHeader( + getApiVersion(), Retcode.r_fail_function_arguments_VALUE); + } - if (LOG.isDebugEnabled()) LOG.debug("getBlockDetailsByRange: range " + blkStart + "-" + blkEnd); + // truncate at the beginning of range + if (blkEnd - blkStart > 1000) { + blkStart = blkEnd - 1000 + 1; - Long lastBlockTimestamp = null; - long listLength = blkEnd - blkStart + 1; - List bds = new ArrayList<>(); + if (blkStart < 0) { + blkStart = 0; + } + } - for (int i = 0; i < listLength; i++) { - long blkNum = blkStart + i; - Map.Entry entry = getBlockWithTotalDifficulty(blkNum); - AionBlock b = entry.getKey(); - BigInteger td = entry.getValue(); - long blocktime = 0; - if (b.getNumber() > 0 && lastBlockTimestamp == null) - lastBlockTimestamp = getBlockByHash(b.getParentHash()).getTimestamp(); + if (LOG.isDebugEnabled()) { + LOG.debug("getBlockDetailsByRange: range " + blkStart + "-" + blkEnd); + } - if (lastBlockTimestamp != null) - blocktime = b.getTimestamp() - lastBlockTimestamp; + Long lastBlockTimestamp = null; + long listLength = blkEnd - blkStart + 1; + List bds = new ArrayList<>(); + + for (int i = 0; i < listLength; i++) { + long blkNum = blkStart + i; + Map.Entry entry = + getBlockWithTotalDifficulty(blkNum); + AionBlock b = entry.getKey(); + BigInteger td = entry.getValue(); + long blocktime = 0; + if (b.getNumber() > 0 && lastBlockTimestamp == null) { + lastBlockTimestamp = + getBlockByHash(b.getParentHash()).getTimestamp(); + } - lastBlockTimestamp = b.getTimestamp(); + if (lastBlockTimestamp != null) { + blocktime = b.getTimestamp() - lastBlockTimestamp; + } - Message.t_BlockDetail.Builder blockDetails = getBlockDetailsObj(b, td, blocktime); + lastBlockTimestamp = b.getTimestamp(); - List txDetails = new ArrayList<>(); + Message.t_BlockDetail.Builder blockDetails = + getBlockDetailsObj(b, td, blocktime); - AionBlockSummary bs = null; - if (explorerBlockCache != null) { - // remove from cache since after consumed, we're probably not gonna revisit it - bs = explorerBlockCache.remove(new ByteArrayWrapper(b.getHash())); - } + List txDetails = new ArrayList<>(); - if (bs != null) { + AionBlockSummary bs = null; + if (explorerBlockCache != null) { + // remove from cache since after consumed, we're probably not gonna + // revisit it + bs = explorerBlockCache.remove(new ByteArrayWrapper(b.getHash())); + } - if (LOG.isDebugEnabled()) LOG.debug("getBlockDetailsByRange: cache HIT for #: " + b.getNumber()); + if (bs != null) { - Map receipts = new HashMap<>(); - for (AionTxReceipt r : bs.getReceipts()) { - receipts.put(new ByteArrayWrapper(r.getTransaction().getHash()), r); - } + if (LOG.isDebugEnabled()) { + LOG.debug( + "getBlockDetailsByRange: cache HIT for #: " + + b.getNumber()); + } - List txns = b.getTransactionsList(); - for (int j = 0; j < txns.size(); j++) { - AionTransaction tx = txns.get(j); - AionTxReceipt r = receipts.get(new ByteArrayWrapper(tx.getHash())); - if (r == null) { - if (LOG.isDebugEnabled()) LOG.debug("getBlockDetailsByRange: transaction not in Block Summary: " + b.getNumber() + "." + j); - AionTxInfo ti = ((AionBlockchainImpl) this.ac.getAionHub().getBlockchain()) - .getTransactionInfoLite(tx.getHash(), b.getHash()); - r = ti.getReceipt(); + Map receipts = new HashMap<>(); + for (AionTxReceipt r : bs.getReceipts()) { + receipts.put( + new ByteArrayWrapper(r.getTransaction().getHash()), r); } - if (r == null) { - LOG.error("getBlockDetailsByRange: missing DB transaction: " + ByteUtil.toHexString(tx.getHash())); + + List txns = b.getTransactionsList(); + for (int j = 0; j < txns.size(); j++) { + AionTransaction tx = txns.get(j); + AionTxReceipt r = + receipts.get(new ByteArrayWrapper(tx.getHash())); + if (r == null) { + if (LOG.isDebugEnabled()) { + LOG.debug( + "getBlockDetailsByRange: transaction not in Block Summary: " + + b.getNumber() + + "." + + j); + } + AionTxInfo ti = + ((AionBlockchainImpl) + this.ac + .getAionHub() + .getBlockchain()) + .getTransactionInfoLite( + tx.getHash(), b.getHash()); + r = ti.getReceipt(); + } + if (r == null) { + LOG.error( + "getBlockDetailsByRange: missing DB transaction: " + + ByteUtil.toHexString(tx.getHash())); + } else { + txDetails.add( + getTxDetailsObj( + tx, + r.getLogInfoList(), + j, + r.getEnergyUsed(), + r.getError())); + } } - else { - txDetails.add(getTxDetailsObj(tx, r.getLogInfoList(), j, r.getEnergyUsed(), r.getError())); + } else { + if (LOG.isDebugEnabled()) { + LOG.debug( + "getBlockDetailsByRange: cache MISS for #: " + + b.getNumber()); } + List txs = b.getTransactionsList(); + + txDetails = + txs.parallelStream() + .filter(Objects::nonNull) + .map( + (AionTransaction tx) -> { + AionTxInfo ti = + ((AionBlockchainImpl) + this.ac + .getAionHub() + .getBlockchain()) + .getTransactionInfoLite( + tx.getHash(), + b.getHash()); + if (ti == null) { + LOG.error( + "getBlockDetailsByRange: missing DB transaction: " + + ByteUtil + .toHexString( + tx + .getHash())); + return null; + } else { + return getTxDetailsObj( + tx, + ti.getReceipt() + .getLogInfoList(), + ti.getIndex(), + ti.getReceipt() + .getEnergyUsed(), + ti.getReceipt().getError()); + } + }) + .filter(Objects::nonNull) + .collect(Collectors.toList()); } - } else { - if (LOG.isDebugEnabled()) LOG.debug("getBlockDetailsByRange: cache MISS for #: " + b.getNumber()); - List txs = b.getTransactionsList(); - - txDetails = txs.parallelStream() - .filter(Objects::nonNull) - .map((AionTransaction tx) -> { - AionTxInfo ti = ((AionBlockchainImpl) this.ac.getAionHub().getBlockchain()) - .getTransactionInfoLite(tx.getHash(), b.getHash()); - if (ti == null) { - LOG.error("getBlockDetailsByRange: missing DB transaction: " + ByteUtil.toHexString(tx.getHash())); - return null; - } else { - return getTxDetailsObj(tx, ti.getReceipt().getLogInfoList(), ti.getIndex(), ti.getReceipt().getEnergyUsed(), ti.getReceipt().getError()); - } - }).filter(Objects::nonNull) - .collect(Collectors.toList()); - } - bds.add(blockDetails.addAllTx(txDetails).build()); - } + bds.add(blockDetails.addAllTx(txDetails).build()); + } - Message.rsp_getBlockDetailsByRange rsp = - Message.rsp_getBlockDetailsByRange.newBuilder() - .addAllBlkDetails(bds) - .build(); + Message.rsp_getBlockDetailsByRange rsp = + Message.rsp_getBlockDetailsByRange + .newBuilder() + .addAllBlkDetails(bds) + .build(); - byte[] retHeader = ApiUtil.toReturnHeader(getApiVersion(), Retcode.r_success_VALUE); - return ApiUtil.combineRetMsg(retHeader, rsp.toByteArray()); - } catch (Exception e) { - LOG.error("ApiAion0.process.getBlockDetailsByNumber exception: [{}]", e.getMessage()); - return ApiUtil.toReturnHeader(getApiVersion(), Retcode.r_fail_function_exception_VALUE); - } - } - case Message.Funcs.f_getBlockDetailsByLatest_VALUE: { - if (service != Message.Servs.s_admin_VALUE) { - return ApiUtil.toReturnHeader(getApiVersion(), Retcode.r_fail_service_call_VALUE); + byte[] retHeader = + ApiUtil.toReturnHeader(getApiVersion(), Retcode.r_success_VALUE); + return ApiUtil.combineRetMsg(retHeader, rsp.toByteArray()); + } catch (Exception e) { + LOG.error( + "ApiAion0.process.getBlockDetailsByNumber exception: [{}]", + e.getMessage()); + return ApiUtil.toReturnHeader( + getApiVersion(), Retcode.r_fail_function_exception_VALUE); + } } + case Message.Funcs.f_getBlockDetailsByLatest_VALUE: { + if (service != Message.Servs.s_admin_VALUE) { + return ApiUtil.toReturnHeader( + getApiVersion(), Retcode.r_fail_service_call_VALUE); + } - byte[] data = parseMsgReq(request, msgHash); - Message.req_getBlockDetailsByLatest req; + byte[] data = parseMsgReq(request, msgHash); + Message.req_getBlockDetailsByLatest req; - try { - req = Message.req_getBlockDetailsByLatest.parseFrom(data); + try { + req = Message.req_getBlockDetailsByLatest.parseFrom(data); - // clip the requested count up to 1000 - Long count = req.getCount(); - if (count > 1000) { - count = 1000L; - } + // clip the requested count up to 1000 + Long count = req.getCount(); + if (count > 1000) { + count = 1000L; + } - // clip start block to 0 at the bottom - Long endBlock = this.getBestBlock().getNumber(); - Long startBlock = (endBlock - count + 1) >= 0 ? (endBlock - count + 1) : 0; + // clip start block to 0 at the bottom + Long endBlock = this.getBestBlock().getNumber(); + Long startBlock = (endBlock - count + 1) >= 0 ? (endBlock - count + 1) : 0; - List blkNum = LongStream.rangeClosed(startBlock, endBlock) - .boxed().collect(Collectors.toList()); + List blkNum = + LongStream.rangeClosed(startBlock, endBlock) + .boxed() + .collect(Collectors.toList()); - List> blks = getBlkAndDifficultyForBlkNumList(blkNum); + List> blks = + getBlkAndDifficultyForBlkNumList(blkNum); - if (blks == null) { - return ApiUtil.toReturnHeader(getApiVersion(), Retcode.r_fail_function_arguments_VALUE); - } else { - List bds = getRsp_getBlockDetails(blks); - Message.rsp_getBlockDetailsByLatest rsp = Message.rsp_getBlockDetailsByLatest.newBuilder() - .addAllBlkDetails(bds) - .build(); + if (blks == null) { + return ApiUtil.toReturnHeader( + getApiVersion(), Retcode.r_fail_function_arguments_VALUE); + } else { + List bds = getRsp_getBlockDetails(blks); + Message.rsp_getBlockDetailsByLatest rsp = + Message.rsp_getBlockDetailsByLatest + .newBuilder() + .addAllBlkDetails(bds) + .build(); - byte[] retHeader = ApiUtil.toReturnHeader(getApiVersion(), Retcode.r_success_VALUE); - return ApiUtil.combineRetMsg(retHeader, rsp.toByteArray()); + byte[] retHeader = + ApiUtil.toReturnHeader( + getApiVersion(), Retcode.r_success_VALUE); + return ApiUtil.combineRetMsg(retHeader, rsp.toByteArray()); + } + } catch (Exception e) { + LOG.error( + "ApiAion0.process.getBlockDetailsByLatest exception: [{}]", + e.getMessage()); + return ApiUtil.toReturnHeader( + getApiVersion(), Retcode.r_fail_function_exception_VALUE); } - } catch (Exception e) { - LOG.error("ApiAion0.process.getBlockDetailsByLatest exception: [{}]", e.getMessage()); - return ApiUtil.toReturnHeader(getApiVersion(), Retcode.r_fail_function_exception_VALUE); - } - } - case Message.Funcs.f_getBlocksByLatest_VALUE: { - if (service != Message.Servs.s_admin_VALUE) { - return ApiUtil.toReturnHeader(getApiVersion(), Retcode.r_fail_service_call_VALUE); } + case Message.Funcs.f_getBlocksByLatest_VALUE: { + if (service != Message.Servs.s_admin_VALUE) { + return ApiUtil.toReturnHeader( + getApiVersion(), Retcode.r_fail_service_call_VALUE); + } - byte[] data = parseMsgReq(request, msgHash); - Message.req_getBlocksByLatest req; + byte[] data = parseMsgReq(request, msgHash); + Message.req_getBlocksByLatest req; - try { - req = Message.req_getBlocksByLatest.parseFrom(data); + try { + req = Message.req_getBlocksByLatest.parseFrom(data); - // clip the requested count up to 1000 - Long count = req.getCount(); - if (count > 1000) { - count = 1000L; - } + // clip the requested count up to 1000 + Long count = req.getCount(); + if (count > 1000) { + count = 1000L; + } - // clip start block to 0 at the bottom - Long endBlock = this.getBestBlock().getNumber(); - Long startBlock = (endBlock - count + 1) >= 0 ? (endBlock - count + 1) : 0; + // clip start block to 0 at the bottom + Long endBlock = this.getBestBlock().getNumber(); + Long startBlock = (endBlock - count + 1) >= 0 ? (endBlock - count + 1) : 0; - List blkNum = LongStream.rangeClosed(startBlock, endBlock) - .boxed().collect(Collectors.toList()); + List blkNum = + LongStream.rangeClosed(startBlock, endBlock) + .boxed() + .collect(Collectors.toList()); - List> blks = getBlkAndDifficultyForBlkNumList(blkNum); + List> blks = + getBlkAndDifficultyForBlkNumList(blkNum); - if (blks == null) { - return ApiUtil.toReturnHeader(getApiVersion(), Retcode.r_fail_function_arguments_VALUE); - } else { - List bs = getRsp_getBlocks(blks); - Message.rsp_getBlocksByLatest rsp = Message.rsp_getBlocksByLatest.newBuilder() - .addAllBlks(bs) - .build(); + if (blks == null) { + return ApiUtil.toReturnHeader( + getApiVersion(), Retcode.r_fail_function_arguments_VALUE); + } else { + List bs = getRsp_getBlocks(blks); + Message.rsp_getBlocksByLatest rsp = + Message.rsp_getBlocksByLatest + .newBuilder() + .addAllBlks(bs) + .build(); - byte[] retHeader = ApiUtil.toReturnHeader(getApiVersion(), Retcode.r_success_VALUE); - return ApiUtil.combineRetMsg(retHeader, rsp.toByteArray()); + byte[] retHeader = + ApiUtil.toReturnHeader( + getApiVersion(), Retcode.r_success_VALUE); + return ApiUtil.combineRetMsg(retHeader, rsp.toByteArray()); + } + } catch (Exception e) { + LOG.error( + "ApiAion0.process.getBlocksByLatest exception: [{}]", + e.getMessage()); + return ApiUtil.toReturnHeader( + getApiVersion(), Retcode.r_fail_function_exception_VALUE); } - } catch (Exception e) { - LOG.error("ApiAion0.process.getBlocksByLatest exception: [{}]", e.getMessage()); - return ApiUtil.toReturnHeader(getApiVersion(), Retcode.r_fail_function_exception_VALUE); - } - } - case Message.Funcs.f_getAccountDetailsByAddressList_VALUE: { - if (service != Message.Servs.s_admin_VALUE) { - return ApiUtil.toReturnHeader(getApiVersion(), Retcode.r_fail_service_call_VALUE); } + case Message.Funcs.f_getAccountDetailsByAddressList_VALUE: { + if (service != Message.Servs.s_admin_VALUE) { + return ApiUtil.toReturnHeader( + getApiVersion(), Retcode.r_fail_service_call_VALUE); + } - byte[] data = parseMsgReq(request, msgHash); - Message.req_getAccountDetailsByAddressList req; - - try { - req = Message.req_getAccountDetailsByAddressList.parseFrom(data); - List num = req.getAddressesList(); + byte[] data = parseMsgReq(request, msgHash); + Message.req_getAccountDetailsByAddressList req; - if (num.size() > 1000) { - num = num.subList(0, 1000); - } + try { + req = Message.req_getAccountDetailsByAddressList.parseFrom(data); + List num = req.getAddressesList(); - List accounts = num.parallelStream() - .map(a -> { - BigInteger b = this.getBalance(Address.wrap(a.toByteArray())); + if (num.size() > 1000) { + num = num.subList(0, 1000); + } - Message.t_AccountDetail.Builder builder = Message.t_AccountDetail.newBuilder(); - if (b != null) - builder.setBalance(ByteString.copyFrom(b.toByteArray())); + List accounts = + num.parallelStream() + .map( + a -> { + BigInteger b = + this.getBalance( + Address.wrap(a.toByteArray())); + + Message.t_AccountDetail.Builder builder = + Message.t_AccountDetail.newBuilder(); + if (b != null) { + builder.setBalance( + ByteString.copyFrom( + b.toByteArray())); + } - builder.setAddress(a); + builder.setAddress(a); - return builder.build(); } - ).collect(Collectors.toList()); + return builder.build(); + }) + .collect(Collectors.toList()); - if (accounts == null) { - return ApiUtil.toReturnHeader(getApiVersion(), Retcode.r_fail_function_arguments_VALUE); - } + if (accounts == null) { + return ApiUtil.toReturnHeader( + getApiVersion(), Retcode.r_fail_function_arguments_VALUE); + } - Message.rsp_getAccountDetailsByAddressList rsp = Message.rsp_getAccountDetailsByAddressList.newBuilder() - .addAllAccounts(accounts) - .build(); + Message.rsp_getAccountDetailsByAddressList rsp = + Message.rsp_getAccountDetailsByAddressList + .newBuilder() + .addAllAccounts(accounts) + .build(); - byte[] retHeader = ApiUtil.toReturnHeader(getApiVersion(), Retcode.r_success_VALUE); - return ApiUtil.combineRetMsg(retHeader, rsp.toByteArray()); + byte[] retHeader = + ApiUtil.toReturnHeader(getApiVersion(), Retcode.r_success_VALUE); + return ApiUtil.combineRetMsg(retHeader, rsp.toByteArray()); - } catch (Exception e) { - LOG.error("ApiAion0.process.getBlockDetailsByNumber exception: [{}]", e.getMessage()); - return ApiUtil.toReturnHeader(getApiVersion(), Retcode.r_fail_function_exception_VALUE); + } catch (Exception e) { + LOG.error( + "ApiAion0.process.getBlockDetailsByNumber exception: [{}]", + e.getMessage()); + return ApiUtil.toReturnHeader( + getApiVersion(), Retcode.r_fail_function_exception_VALUE); + } } - } - // case Message.Funcs.f_eventQuery_VALUE: - // case Message.Funcs.f_submitWork_VALUE: - // case Message.Funcs.f_getWork_VALUE: - default: - return ApiUtil.toReturnHeader(getApiVersion(), Retcode.r_fail_function_call_VALUE); + // case Message.Funcs.f_eventQuery_VALUE: + // case Message.Funcs.f_submitWork_VALUE: + // case Message.Funcs.f_getWork_VALUE: + default: + return ApiUtil.toReturnHeader(getApiVersion(), Retcode.r_fail_function_call_VALUE); } } @@ -1828,11 +2445,14 @@ public void shutDown() { eesBlkCache.shutdown(); } } - @Override public Map> getMsgIdMapping() { + + @Override + public Map> getMsgIdMapping() { return this.msgIdMapping; } - @Override public TxWaitingMappingUpdate takeTxWait() throws Throwable { + @Override + public TxWaitingMappingUpdate takeTxWait() throws Throwable { return txWait.take(); } @@ -1846,7 +2466,8 @@ private byte[] createBlockMsg(AionBlock blk) { al.add(ByteString.copyFrom(tx.getHash())); } - BigInteger td = this.ac.getBlockchain().getTotalDifficultyByHash(Hash256.wrap(blk.getHash())); + BigInteger td = + this.ac.getBlockchain().getTotalDifficultyByHash(Hash256.wrap(blk.getHash())); Message.rsp_getBlock rsp = getRsp_getBlock(blk, al, td); byte[] retHeader = ApiUtil.toReturnHeader(getApiVersion(), Retcode.r_success_VALUE); @@ -1855,118 +2476,142 @@ private byte[] createBlockMsg(AionBlock blk) { } private Message.rsp_getTransaction getRsp_getTransaction(AionTransaction tx) { - return Message.rsp_getTransaction.newBuilder() - .setBlockhash(ByteString.copyFrom(tx.getBlockHash())) - .setBlocknumber(tx.getBlockNumber()) - .setFrom(ByteString.copyFrom(tx.getFrom().toBytes())) - .setNrgConsume(tx.getNrgConsume()) - .setNrgPrice(tx.getNrgPrice()) - .setTxHash(ByteString.copyFrom(tx.getHash())) - .setData(ByteString.copyFrom(tx.getData() == null ? EMPTY_BYTE_ARRAY : tx.getData())) - .setNonce(ByteString.copyFrom(tx.getNonce())) - .setTo(ByteString.copyFrom(tx.getTo() == null ? EMPTY_BYTE_ARRAY : tx.getTo().toBytes())) - .setValue(ByteString.copyFrom(tx.getValue())) - .setTxIndex((int)tx.getTxIndexInBlock()) - .setTimeStamp(ByteUtil.byteArrayToLong(tx.getTimeStamp())) - .build(); + return Message.rsp_getTransaction + .newBuilder() + .setBlockhash(ByteString.copyFrom(tx.getBlockHash())) + .setBlocknumber(tx.getBlockNumber()) + .setFrom(ByteString.copyFrom(tx.getFrom().toBytes())) + .setNrgConsume(tx.getNrgConsume()) + .setNrgPrice(tx.getNrgPrice()) + .setTxHash(ByteString.copyFrom(tx.getHash())) + .setData( + ByteString.copyFrom(tx.getData() == null ? EMPTY_BYTE_ARRAY : tx.getData())) + .setNonce(ByteString.copyFrom(tx.getNonce())) + .setTo( + ByteString.copyFrom( + tx.getTo() == null ? EMPTY_BYTE_ARRAY : tx.getTo().toBytes())) + .setValue(ByteString.copyFrom(tx.getValue())) + .setTxIndex((int) tx.getTxIndexInBlock()) + .setTimeStamp(ByteUtil.byteArrayToLong(tx.getTimeStamp())) + .build(); } - private Message.rsp_getBlock getRsp_getBlock(AionBlock blk, List al, BigInteger td) { - return Message.rsp_getBlock.newBuilder() - .setParentHash(ByteString.copyFrom(blk.getParentHash())) - .setMinerAddress(ByteString.copyFrom(blk.getCoinbase().toBytes())) - .setStateRoot(ByteString.copyFrom(blk.getStateRoot())) - .setTxTrieRoot(ByteString.copyFrom(blk.getTxTrieRoot())) - .setDifficulty(ByteString.copyFrom(blk.getDifficulty())) - .setExtraData(ByteString.copyFrom(blk.getExtraData())) - .setNrgConsumed(blk.getNrgConsumed()) - .setNrgLimit(blk.getNrgLimit()) - .setHash(ByteString.copyFrom(blk.getHash())) - .setLogsBloom(ByteString.copyFrom(blk.getLogBloom())) - .setNonce(ByteString.copyFrom(blk.getNonce())) - .setReceiptTrieRoot(ByteString.copyFrom(blk.getReceiptsRoot())) - .setTimestamp(blk.getTimestamp()).setBlockNumber(blk.getNumber()) - .setSolution(ByteString.copyFrom(blk.getHeader().getSolution())) - .addAllTxHash(al) - .setSize(blk.size()) - .setTotalDifficulty(ByteString.copyFrom(td.toByteArray())) - .build(); + private Message.rsp_getBlock getRsp_getBlock( + AionBlock blk, List al, BigInteger td) { + return Message.rsp_getBlock + .newBuilder() + .setParentHash(ByteString.copyFrom(blk.getParentHash())) + .setMinerAddress(ByteString.copyFrom(blk.getCoinbase().toBytes())) + .setStateRoot(ByteString.copyFrom(blk.getStateRoot())) + .setTxTrieRoot(ByteString.copyFrom(blk.getTxTrieRoot())) + .setDifficulty(ByteString.copyFrom(blk.getDifficulty())) + .setExtraData(ByteString.copyFrom(blk.getExtraData())) + .setNrgConsumed(blk.getNrgConsumed()) + .setNrgLimit(blk.getNrgLimit()) + .setHash(ByteString.copyFrom(blk.getHash())) + .setLogsBloom(ByteString.copyFrom(blk.getLogBloom())) + .setNonce(ByteString.copyFrom(blk.getNonce())) + .setReceiptTrieRoot(ByteString.copyFrom(blk.getReceiptsRoot())) + .setTimestamp(blk.getTimestamp()) + .setBlockNumber(blk.getNumber()) + .setSolution(ByteString.copyFrom(blk.getHeader().getSolution())) + .addAllTxHash(al) + .setSize(blk.size()) + .setTotalDifficulty(ByteString.copyFrom(td.toByteArray())) + .build(); } private List getRsp_getBlocks(List> blks) { - List bs = blks.parallelStream() - .filter(Objects::nonNull) - .map(blk -> { - AionBlock b = blk.getKey(); - return Message.t_Block.newBuilder() - .setBlockNumber(b.getNumber()) - .setDifficulty(ByteString.copyFrom(b.getDifficulty())) - .setExtraData(ByteString.copyFrom(b.getExtraData())) - .setHash(ByteString.copyFrom(b.getHash())) - .setLogsBloom(ByteString.copyFrom(b.getLogBloom())) - .setMinerAddress(ByteString.copyFrom(b.getCoinbase().toBytes())) - .setNonce(ByteString.copyFrom(b.getNonce())) - .setNrgConsumed(b.getNrgConsumed()) - .setNrgLimit(b.getNrgLimit()) - .setParentHash(ByteString.copyFrom(b.getParentHash())) - .setTimestamp(b.getTimestamp()) - .setTxTrieRoot(ByteString.copyFrom(b.getTxTrieRoot())) - .setReceiptTrieRoot(ByteString.copyFrom(b.getReceiptsRoot())) - .setStateRoot(ByteString.copyFrom(b.getStateRoot())) - .setSize(b.getEncoded().length) - .setSolution(ByteString.copyFrom(b.getHeader().getSolution())) - .setTotalDifficulty(ByteString.copyFrom(blk.getValue().toByteArray())) - .build(); - }).collect(Collectors.toList()); + return blks.parallelStream() + .filter(Objects::nonNull) + .map( + blk -> { + AionBlock b = blk.getKey(); - return bs; + return Message.t_Block + .newBuilder() + .setBlockNumber(b.getNumber()) + .setDifficulty(ByteString.copyFrom(b.getDifficulty())) + .setExtraData(ByteString.copyFrom(b.getExtraData())) + .setHash(ByteString.copyFrom(b.getHash())) + .setLogsBloom(ByteString.copyFrom(b.getLogBloom())) + .setMinerAddress( + ByteString.copyFrom(b.getCoinbase().toBytes())) + .setNonce(ByteString.copyFrom(b.getNonce())) + .setNrgConsumed(b.getNrgConsumed()) + .setNrgLimit(b.getNrgLimit()) + .setParentHash(ByteString.copyFrom(b.getParentHash())) + .setTimestamp(b.getTimestamp()) + .setTxTrieRoot(ByteString.copyFrom(b.getTxTrieRoot())) + .setReceiptTrieRoot( + ByteString.copyFrom(b.getReceiptsRoot())) + .setStateRoot(ByteString.copyFrom(b.getStateRoot())) + .setSize(b.getEncoded().length) + .setSolution( + ByteString.copyFrom( + b.getHeader().getSolution())) + .setTotalDifficulty( + ByteString.copyFrom( + blk.getValue().toByteArray())) + .build(); + }) + .collect(Collectors.toList()); } - private Message.t_BlockDetail.Builder getBlockDetailsObj(AionBlock b, BigInteger td, long blocktime) { - - Message.t_BlockDetail.Builder builder = Message.t_BlockDetail.newBuilder() - .setBlockNumber(b.getNumber()) - .setDifficulty(ByteString.copyFrom(b.getDifficulty())) - .setExtraData(ByteString.copyFrom(b.getExtraData())) - .setHash(ByteString.copyFrom(b.getHash())) - .setLogsBloom(ByteString.copyFrom(b.getLogBloom())) - .setMinerAddress(ByteString.copyFrom(b.getCoinbase().toBytes())) - .setNonce(ByteString.copyFrom(b.getNonce())) - .setNrgConsumed(b.getNrgConsumed()) - .setNrgLimit(b.getNrgLimit()) - .setParentHash(ByteString.copyFrom(b.getParentHash())) - .setTimestamp(b.getTimestamp()) - .setTxTrieRoot(ByteString.copyFrom(b.getTxTrieRoot())) - .setReceiptTrieRoot(ByteString.copyFrom(b.getReceiptsRoot())) - .setStateRoot(ByteString.copyFrom(b.getStateRoot())) - .setSize(b.getEncoded().length) - .setSolution(ByteString.copyFrom(b.getHeader().getSolution())) - .setTotalDifficulty(ByteString.copyFrom(td.toByteArray())) - .setBlockTime(blocktime); - - return builder; + private Message.t_BlockDetail.Builder getBlockDetailsObj( + AionBlock b, BigInteger td, long blocktime) { + + return Message.t_BlockDetail + .newBuilder() + .setBlockNumber(b.getNumber()) + .setDifficulty(ByteString.copyFrom(b.getDifficulty())) + .setExtraData(ByteString.copyFrom(b.getExtraData())) + .setHash(ByteString.copyFrom(b.getHash())) + .setLogsBloom(ByteString.copyFrom(b.getLogBloom())) + .setMinerAddress(ByteString.copyFrom(b.getCoinbase().toBytes())) + .setNonce(ByteString.copyFrom(b.getNonce())) + .setNrgConsumed(b.getNrgConsumed()) + .setNrgLimit(b.getNrgLimit()) + .setParentHash(ByteString.copyFrom(b.getParentHash())) + .setTimestamp(b.getTimestamp()) + .setTxTrieRoot(ByteString.copyFrom(b.getTxTrieRoot())) + .setReceiptTrieRoot(ByteString.copyFrom(b.getReceiptsRoot())) + .setStateRoot(ByteString.copyFrom(b.getStateRoot())) + .setSize(b.getEncoded().length) + .setSolution(ByteString.copyFrom(b.getHeader().getSolution())) + .setTotalDifficulty(ByteString.copyFrom(td.toByteArray())) + .setBlockTime(blocktime); } - private Message.t_TxDetail getTxDetailsObj(AionTransaction t, List _logs, int txIndex, long nrgConsumed, String error) { + private Message.t_TxDetail getTxDetailsObj( + AionTransaction t, List _logs, int txIndex, long nrgConsumed, String error) { - List tles = _logs.parallelStream() - .map(log -> { - List topics = new ArrayList<>(); - for (int i = 0; i < log.getTopics().size(); i++) { - topics.add(TypeConverter.toJsonHex(log.getTopics().get(i))); - } + List tles = + _logs.parallelStream() + .map( + log -> { + List topics = new ArrayList<>(); + for (int i = 0; i < log.getTopics().size(); i++) { + topics.add(TypeConverter.toJsonHex(log.getTopics().get(i))); + } - return Message.t_LgEle.newBuilder() + return Message.t_LgEle + .newBuilder() .setData(ByteString.copyFrom(log.getData())) - .setAddress(ByteString.copyFrom(log.getAddress().toBytes())) + .setAddress( + ByteString.copyFrom(log.getAddress().toBytes())) .addAllTopics(topics) .build(); - }).filter(Objects::nonNull).collect(Collectors.toList()); + }) + .filter(Objects::nonNull) + .collect(Collectors.toList()); Address contract = t.getContractAddress(); - Message.t_TxDetail.Builder tdBuilder = Message.t_TxDetail.newBuilder() + Message.t_TxDetail.Builder tdBuilder = + Message.t_TxDetail + .newBuilder() .setData(ByteString.copyFrom(t.getData())) .setTo(ByteString.copyFrom(t.getTo().toBytes())) .setFrom(ByteString.copyFrom(t.getFrom().toBytes())) @@ -1980,14 +2625,13 @@ private Message.t_TxDetail getTxDetailsObj(AionTransaction t, List _logs, i .setError(error) .addAllLogs(tles); - if (contract != null) + if (contract != null) { tdBuilder.setContract(ByteString.copyFrom(contract.toBytes())); + } return tdBuilder.build(); - } - private String generateBlockSqlStatement(AionBlock b, BigInteger td, long blocktime) { /* create table block_cache( @@ -2017,37 +2661,60 @@ block_timestamp bigint(64), num_transactions bigint(64), block_time bigint(64)); */ - String stmt = - b.getNumber()+","+ - - "'"+ByteUtil.toHexString(b.getHash())+"',"+ - "'"+ByteUtil.toHexString(b.getCoinbase().toBytes())+"',"+ - "'"+ByteUtil.toHexString(b.getParentHash())+"',"+ - "'"+ByteUtil.toHexString(b.getReceiptsRoot())+"',"+ - "'"+ByteUtil.toHexString(b.getStateRoot())+"',"+ - "'"+ByteUtil.toHexString(b.getTxTrieRoot())+"',"+ - - "'"+ByteUtil.toHexString(b.getExtraData())+"',"+ - "'"+ByteUtil.toHexString(b.getNonce())+"',"+ - "'"+ByteUtil.toHexString(b.getLogBloom())+"',"+ - "'"+ByteUtil.toHexString(b.getHeader().getSolution())+"',"+ - - "'"+ByteUtil.toHexString(b.getDifficulty())+"',"+ - "'"+ByteUtil.toHexString(td.toByteArray())+"',"+ - - b.getNrgConsumed()+","+ - b.getNrgLimit()+","+ - b.getEncoded().length+","+ - - b.getTimestamp()+","+ - b.getTransactionsList().size()+","+ - blocktime; - - return stmt; + return b.getNumber() + + "," + + "'" + + ByteUtil.toHexString(b.getHash()) + + "'," + + "'" + + ByteUtil.toHexString(b.getCoinbase().toBytes()) + + "'," + + "'" + + ByteUtil.toHexString(b.getParentHash()) + + "'," + + "'" + + ByteUtil.toHexString(b.getReceiptsRoot()) + + "'," + + "'" + + ByteUtil.toHexString(b.getStateRoot()) + + "'," + + "'" + + ByteUtil.toHexString(b.getTxTrieRoot()) + + "'," + + "'" + + ByteUtil.toHexString(b.getExtraData()) + + "'," + + "'" + + ByteUtil.toHexString(b.getNonce()) + + "'," + + "'" + + ByteUtil.toHexString(b.getLogBloom()) + + "'," + + "'" + + ByteUtil.toHexString(b.getHeader().getSolution()) + + "'," + + "'" + + ByteUtil.toHexString(b.getDifficulty()) + + "'," + + "'" + + ByteUtil.toHexString(td.toByteArray()) + + "'," + + b.getNrgConsumed() + + "," + + b.getNrgLimit() + + "," + + b.getEncoded().length + + "," + + b.getTimestamp() + + "," + + b.getTransactionsList().size() + + "," + + blocktime; } - private String generateTransactionSqlStatement(AionBlock b, AionTransaction t, List _logs, int txIndex, long nrgConsumed) { + private String generateTransactionSqlStatement( + AionBlock b, AionTransaction t, List _logs, int txIndex, long nrgConsumed) { JSONArray logs = new JSONArray(); for (Log l : _logs) { JSONArray log = new JSONArray(); @@ -2087,99 +2754,221 @@ block_timestamp bigint(64), primary key(block_number,transaction_index)); */ - String stmt = - "'"+ByteUtil.toHexString(t.getHash())+"',"+ - "'"+ByteUtil.toHexString(b.getHash())+"',"+ - - b.getNumber()+","+ - txIndex+","+ - - "'"+ByteUtil.toHexString(t.getFrom().toBytes())+"',"+ - "'"+ByteUtil.toHexString(t.getTo().toBytes())+"',"+ - - nrgConsumed+","+ - t.getNrgPrice()+","+ - - ByteUtil.byteArrayToLong(t.getTimeStamp())+","+ - b.getTimestamp()+","+ - - "'"+ByteUtil.toHexString(t.getValue())+"',"+ - "'"+logs.toString()+"',"+ - - "'"+ByteUtil.toHexString(t.getData())+"',"+ - "'"+ByteUtil.toHexString(t.getNonce())+"'"; - - return stmt; + return "'" + + ByteUtil.toHexString(t.getHash()) + + "'," + + "'" + + ByteUtil.toHexString(b.getHash()) + + "'," + + b.getNumber() + + "," + + txIndex + + "," + + "'" + + ByteUtil.toHexString(t.getFrom().toBytes()) + + "'," + + "'" + + ByteUtil.toHexString(t.getTo().toBytes()) + + "'," + + nrgConsumed + + "," + + t.getNrgPrice() + + "," + + ByteUtil.byteArrayToLong(t.getTimeStamp()) + + "," + + b.getTimestamp() + + "," + + "'" + + ByteUtil.toHexString(t.getValue()) + + "'," + + "'" + + logs.toString() + + "'," + + "'" + + ByteUtil.toHexString(t.getData()) + + "'," + + "'" + + ByteUtil.toHexString(t.getNonce()) + + "'"; } - private List getRsp_getBlockDetails(List> blks) { - List bds = blks.parallelStream().filter(Objects::nonNull).map(blk -> { - AionBlock b = blk.getKey(); - Message.t_BlockDetail.Builder builder = Message.t_BlockDetail.newBuilder() - .setBlockNumber(b.getNumber()) - .setDifficulty(ByteString.copyFrom(b.getDifficulty())) - .setExtraData(ByteString.copyFrom(b.getExtraData())) - .setHash(ByteString.copyFrom(b.getHash())) - .setLogsBloom(ByteString.copyFrom(b.getLogBloom())) - .setMinerAddress(ByteString.copyFrom(b.getCoinbase().toBytes())) - .setNonce(ByteString.copyFrom(b.getNonce())) - .setNrgConsumed(b.getNrgConsumed()) - .setNrgLimit(b.getNrgLimit()) - .setParentHash(ByteString.copyFrom(b.getParentHash())) - .setTimestamp(b.getTimestamp()) - .setTxTrieRoot(ByteString.copyFrom(b.getTxTrieRoot())) - .setReceiptTrieRoot(ByteString.copyFrom(b.getReceiptsRoot())) - .setStateRoot(ByteString.copyFrom(b.getStateRoot())) - .setSize(b.getEncoded().length) - .setSolution(ByteString.copyFrom(b.getHeader().getSolution())) - .setTotalDifficulty(ByteString.copyFrom(blk.getValue().toByteArray())); - - List txs = b.getTransactionsList(); - - List tds = txs.parallelStream().filter(Objects::nonNull).map((AionTransaction tx) -> { - AionTxInfo ti = ((AionBlockchainImpl) this.ac.getAionHub().getBlockchain()).getTransactionInfoLite(tx.getHash(), b.getHash()); - - List tles = ti.getReceipt().getLogInfoList().parallelStream() - .map(log -> { - List topics = new ArrayList<>(); - for (int i = 0; i < log.getTopics().size(); i++) { - topics.add(TypeConverter.toJsonHex(log.getTopics().get(i))); - } + private List getRsp_getBlockDetails( + List> blks) { - return Message.t_LgEle.newBuilder() - .setData(ByteString.copyFrom(log.getData())) - .setAddress(ByteString.copyFrom(log.getAddress().toBytes())) - .addAllTopics(topics) - .build(); - }).filter(Objects::nonNull).collect(Collectors.toList()); - - Message.t_TxDetail.Builder tdBuilder = Message.t_TxDetail.newBuilder() - .setData(ByteString.copyFrom(tx.getData())) - .setTo(ByteString.copyFrom(tx.getTo().toBytes())) - .setFrom(ByteString.copyFrom(tx.getFrom().toBytes())) - .setNonce(ByteString.copyFrom(tx.getNonce())) - .setValue(ByteString.copyFrom(tx.getValue())) - .setNrgConsumed(ti.getReceipt().getEnergyUsed()) - .setNrgPrice(tx.getNrgPrice()) - .setTxHash(ByteString.copyFrom(tx.getHash())) - .setTxIndex(ti.getIndex()) - .addAllLogs(tles); - - return tdBuilder.build(); - }).filter(Objects::nonNull).collect(Collectors.toList()); - - return builder.addAllTx(tds).build(); - }).filter(Objects::nonNull).collect(Collectors.toList()); - - return bds; + return blks.parallelStream() + .filter(Objects::nonNull) + .map( + blk -> { + AionBlock b = blk.getKey(); + Message.t_BlockDetail.Builder builder = + Message.t_BlockDetail + .newBuilder() + .setBlockNumber(b.getNumber()) + .setDifficulty( + ByteString.copyFrom(b.getDifficulty())) + .setExtraData( + ByteString.copyFrom(b.getExtraData())) + .setHash(ByteString.copyFrom(b.getHash())) + .setLogsBloom( + ByteString.copyFrom(b.getLogBloom())) + .setMinerAddress( + ByteString.copyFrom( + b.getCoinbase().toBytes())) + .setNonce(ByteString.copyFrom(b.getNonce())) + .setNrgConsumed(b.getNrgConsumed()) + .setNrgLimit(b.getNrgLimit()) + .setParentHash( + ByteString.copyFrom(b.getParentHash())) + .setTimestamp(b.getTimestamp()) + .setTxTrieRoot( + ByteString.copyFrom(b.getTxTrieRoot())) + .setReceiptTrieRoot( + ByteString.copyFrom( + b.getReceiptsRoot())) + .setStateRoot( + ByteString.copyFrom(b.getStateRoot())) + .setSize(b.getEncoded().length) + .setSolution( + ByteString.copyFrom( + b.getHeader().getSolution())) + .setTotalDifficulty( + ByteString.copyFrom( + blk.getValue().toByteArray())); + + List txs = b.getTransactionsList(); + + List tds = + txs.parallelStream() + .filter(Objects::nonNull) + .map( + (AionTransaction tx) -> { + AionTxInfo ti = + ((AionBlockchainImpl) + this.ac + .getAionHub() + .getBlockchain()) + .getTransactionInfoLite( + tx + .getHash(), + b + .getHash()); + + List tles = + ti.getReceipt() + .getLogInfoList() + .parallelStream() + .map( + log -> { + List< + String> + topics = + new ArrayList<>(); + for (int + i = + 0; + i + < log.getTopics() + .size(); + i++) { + topics + .add( + TypeConverter + .toJsonHex( + log.getTopics() + .get( + i))); + } + + return Message + .t_LgEle + .newBuilder() + .setData( + ByteString + .copyFrom( + log + .getData())) + .setAddress( + ByteString + .copyFrom( + log.getAddress() + .toBytes())) + .addAllTopics( + topics) + .build(); + }) + .filter( + Objects + ::nonNull) + .collect( + Collectors + .toList()); + + Message.t_TxDetail.Builder + tdBuilder = + Message.t_TxDetail + .newBuilder() + .setData( + ByteString + .copyFrom( + tx + .getData())) + .setTo( + ByteString + .copyFrom( + tx.getTo() + .toBytes())) + .setFrom( + ByteString + .copyFrom( + tx.getFrom() + .toBytes())) + .setNonce( + ByteString + .copyFrom( + tx + .getNonce())) + .setValue( + ByteString + .copyFrom( + tx + .getValue())) + .setNrgConsumed( + ti.getReceipt() + .getEnergyUsed()) + .setNrgPrice( + tx + .getNrgPrice()) + .setTxHash( + ByteString + .copyFrom( + tx + .getHash())) + .setTxIndex( + ti + .getIndex()) + .addAllLogs( + tles); + + return tdBuilder.build(); + }) + .filter(Objects::nonNull) + .collect(Collectors.toList()); + + return builder.addAllTx(tds).build(); + }) + .filter(Objects::nonNull) + .collect(Collectors.toList()); } - // all or nothing. if any block from list is not found, unchecked exception gets thrown by Map.entry() + // all or nothing. if any block from list is not found, unchecked exception gets thrown by + // Map.entry() // causes this function to return in Exception - private List> getBlkAndDifficultyForBlkNumList(List blkNum) { + private List> getBlkAndDifficultyForBlkNumList( + List blkNum) { return blkNum.parallelStream() - .map(n -> { return getBlockWithTotalDifficulty(n); }) - .collect(Collectors.toList()); + .map( + this::getBlockWithTotalDifficulty) + .collect(Collectors.toList()); } public byte getApiVersion() { @@ -2206,18 +2995,22 @@ public Map getPendingReceipts() { return this.pendingReceipts; } - @Override public BlockingQueue getPendingStatus() { + @Override + public BlockingQueue getPendingStatus() { return this.pendingStatus; } - @Override public BlockingQueue getTxWait() { + @Override + public BlockingQueue getTxWait() { return this.txWait; } @Override public byte[] parseMsgReq(byte[] request, byte[] msgHash) { - int headerLen = msgHash == null ? this.getApiHeaderLen() : this.getApiHeaderLen() + msgHash.length; - return ByteBuffer.allocate(request.length - headerLen).put(request, headerLen, request.length - headerLen) - .array(); + int headerLen = + msgHash == null ? this.getApiHeaderLen() : this.getApiHeaderLen() + msgHash.length; + return ByteBuffer.allocate(request.length - headerLen) + .put(request, headerLen, request.length - headerLen) + .array(); } } diff --git a/modApiServer/src/org/aion/api/server/pb/Message.java b/modApiServer/src/org/aion/api/server/pb/Message.java index 184c7cb2e2..1a83c8a259 100644 --- a/modApiServer/src/org/aion/api/server/pb/Message.java +++ b/modApiServer/src/org/aion/api/server/pb/Message.java @@ -417,6 +417,10 @@ public enum Funcs * f_getNonce = 56; */ f_getNonce(56), + /** + * f_getNrgPrice = 57; + */ + f_getNrgPrice(57), UNRECOGNIZED(-1), ; @@ -648,6 +652,10 @@ public enum Funcs * f_getNonce = 56; */ public static final int f_getNonce_VALUE = 56; + /** + * f_getNrgPrice = 57; + */ + public static final int f_getNrgPrice_VALUE = 57; public final int getNumber() { @@ -725,6 +733,7 @@ public static Funcs forNumber(int value) { case 54: return f_getBlockSqlByRange; case 55: return f_getBlockDetailsByRange; case 56: return f_getNonce; + case 57: return f_getNrgPrice; default: return null; } } @@ -70730,6 +70739,458 @@ public org.aion.api.server.pb.Message.rsp_getNonce getDefaultInstanceForType() { } + public interface rsp_getNrgPriceOrBuilder extends + // @@protoc_insertion_point(interface_extends:org.aion.api.server.pb.rsp_getNrgPrice) + com.google.protobuf.MessageOrBuilder { + + /** + * uint64 nrgPrice = 1; + */ + long getNrgPrice(); + } + /** + * Protobuf type {@code org.aion.api.server.pb.rsp_getNrgPrice} + */ + public static final class rsp_getNrgPrice extends + com.google.protobuf.GeneratedMessageV3 implements + // @@protoc_insertion_point(message_implements:org.aion.api.server.pb.rsp_getNrgPrice) + rsp_getNrgPriceOrBuilder { + private static final long serialVersionUID = 0L; + // Use rsp_getNrgPrice.newBuilder() to construct. + private rsp_getNrgPrice(com.google.protobuf.GeneratedMessageV3.Builder builder) { + super(builder); + } + private rsp_getNrgPrice() { + nrgPrice_ = 0L; + } + + @java.lang.Override + public final com.google.protobuf.UnknownFieldSet + getUnknownFields() { + return this.unknownFields; + } + private rsp_getNrgPrice( + com.google.protobuf.CodedInputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws com.google.protobuf.InvalidProtocolBufferException { + this(); + if (extensionRegistry == null) { + throw new java.lang.NullPointerException(); + } + int mutable_bitField0_ = 0; + com.google.protobuf.UnknownFieldSet.Builder unknownFields = + com.google.protobuf.UnknownFieldSet.newBuilder(); + try { + boolean done = false; + while (!done) { + int tag = input.readTag(); + switch (tag) { + case 0: + done = true; + break; + default: { + if (!parseUnknownFieldProto3( + input, unknownFields, extensionRegistry, tag)) { + done = true; + } + break; + } + case 8: { + + nrgPrice_ = input.readUInt64(); + break; + } + } + } + } catch (com.google.protobuf.InvalidProtocolBufferException e) { + throw e.setUnfinishedMessage(this); + } catch (java.io.IOException e) { + throw new com.google.protobuf.InvalidProtocolBufferException( + e).setUnfinishedMessage(this); + } finally { + this.unknownFields = unknownFields.build(); + makeExtensionsImmutable(); + } + } + public static final com.google.protobuf.Descriptors.Descriptor + getDescriptor() { + return org.aion.api.server.pb.Message.internal_static_org_aion_api_server_pb_rsp_getNrgPrice_descriptor; + } + + protected com.google.protobuf.GeneratedMessageV3.FieldAccessorTable + internalGetFieldAccessorTable() { + return org.aion.api.server.pb.Message.internal_static_org_aion_api_server_pb_rsp_getNrgPrice_fieldAccessorTable + .ensureFieldAccessorsInitialized( + org.aion.api.server.pb.Message.rsp_getNrgPrice.class, org.aion.api.server.pb.Message.rsp_getNrgPrice.Builder.class); + } + + public static final int NRGPRICE_FIELD_NUMBER = 1; + private long nrgPrice_; + /** + * uint64 nrgPrice = 1; + */ + public long getNrgPrice() { + return nrgPrice_; + } + + private byte memoizedIsInitialized = -1; + public final boolean isInitialized() { + byte isInitialized = memoizedIsInitialized; + if (isInitialized == 1) return true; + if (isInitialized == 0) return false; + + memoizedIsInitialized = 1; + return true; + } + + public void writeTo(com.google.protobuf.CodedOutputStream output) + throws java.io.IOException { + if (nrgPrice_ != 0L) { + output.writeUInt64(1, nrgPrice_); + } + unknownFields.writeTo(output); + } + + public int getSerializedSize() { + int size = memoizedSize; + if (size != -1) return size; + + size = 0; + if (nrgPrice_ != 0L) { + size += com.google.protobuf.CodedOutputStream + .computeUInt64Size(1, nrgPrice_); + } + size += unknownFields.getSerializedSize(); + memoizedSize = size; + return size; + } + + @java.lang.Override + public boolean equals(final java.lang.Object obj) { + if (obj == this) { + return true; + } + if (!(obj instanceof org.aion.api.server.pb.Message.rsp_getNrgPrice)) { + return super.equals(obj); + } + org.aion.api.server.pb.Message.rsp_getNrgPrice other = (org.aion.api.server.pb.Message.rsp_getNrgPrice) obj; + + boolean result = true; + result = result && (getNrgPrice() + == other.getNrgPrice()); + result = result && unknownFields.equals(other.unknownFields); + return result; + } + + @java.lang.Override + public int hashCode() { + if (memoizedHashCode != 0) { + return memoizedHashCode; + } + int hash = 41; + hash = (19 * hash) + getDescriptor().hashCode(); + hash = (37 * hash) + NRGPRICE_FIELD_NUMBER; + hash = (53 * hash) + com.google.protobuf.Internal.hashLong( + getNrgPrice()); + hash = (29 * hash) + unknownFields.hashCode(); + memoizedHashCode = hash; + return hash; + } + + public static org.aion.api.server.pb.Message.rsp_getNrgPrice parseFrom( + java.nio.ByteBuffer data) + throws com.google.protobuf.InvalidProtocolBufferException { + return PARSER.parseFrom(data); + } + public static org.aion.api.server.pb.Message.rsp_getNrgPrice parseFrom( + java.nio.ByteBuffer data, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws com.google.protobuf.InvalidProtocolBufferException { + return PARSER.parseFrom(data, extensionRegistry); + } + public static org.aion.api.server.pb.Message.rsp_getNrgPrice parseFrom( + com.google.protobuf.ByteString data) + throws com.google.protobuf.InvalidProtocolBufferException { + return PARSER.parseFrom(data); + } + public static org.aion.api.server.pb.Message.rsp_getNrgPrice parseFrom( + com.google.protobuf.ByteString data, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws com.google.protobuf.InvalidProtocolBufferException { + return PARSER.parseFrom(data, extensionRegistry); + } + public static org.aion.api.server.pb.Message.rsp_getNrgPrice parseFrom(byte[] data) + throws com.google.protobuf.InvalidProtocolBufferException { + return PARSER.parseFrom(data); + } + public static org.aion.api.server.pb.Message.rsp_getNrgPrice parseFrom( + byte[] data, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws com.google.protobuf.InvalidProtocolBufferException { + return PARSER.parseFrom(data, extensionRegistry); + } + public static org.aion.api.server.pb.Message.rsp_getNrgPrice parseFrom(java.io.InputStream input) + throws java.io.IOException { + return com.google.protobuf.GeneratedMessageV3 + .parseWithIOException(PARSER, input); + } + public static org.aion.api.server.pb.Message.rsp_getNrgPrice parseFrom( + java.io.InputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws java.io.IOException { + return com.google.protobuf.GeneratedMessageV3 + .parseWithIOException(PARSER, input, extensionRegistry); + } + public static org.aion.api.server.pb.Message.rsp_getNrgPrice parseDelimitedFrom(java.io.InputStream input) + throws java.io.IOException { + return com.google.protobuf.GeneratedMessageV3 + .parseDelimitedWithIOException(PARSER, input); + } + public static org.aion.api.server.pb.Message.rsp_getNrgPrice parseDelimitedFrom( + java.io.InputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws java.io.IOException { + return com.google.protobuf.GeneratedMessageV3 + .parseDelimitedWithIOException(PARSER, input, extensionRegistry); + } + public static org.aion.api.server.pb.Message.rsp_getNrgPrice parseFrom( + com.google.protobuf.CodedInputStream input) + throws java.io.IOException { + return com.google.protobuf.GeneratedMessageV3 + .parseWithIOException(PARSER, input); + } + public static org.aion.api.server.pb.Message.rsp_getNrgPrice parseFrom( + com.google.protobuf.CodedInputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws java.io.IOException { + return com.google.protobuf.GeneratedMessageV3 + .parseWithIOException(PARSER, input, extensionRegistry); + } + + public Builder newBuilderForType() { return newBuilder(); } + public static Builder newBuilder() { + return DEFAULT_INSTANCE.toBuilder(); + } + public static Builder newBuilder(org.aion.api.server.pb.Message.rsp_getNrgPrice prototype) { + return DEFAULT_INSTANCE.toBuilder().mergeFrom(prototype); + } + public Builder toBuilder() { + return this == DEFAULT_INSTANCE + ? new Builder() : new Builder().mergeFrom(this); + } + + @java.lang.Override + protected Builder newBuilderForType( + com.google.protobuf.GeneratedMessageV3.BuilderParent parent) { + Builder builder = new Builder(parent); + return builder; + } + /** + * Protobuf type {@code org.aion.api.server.pb.rsp_getNrgPrice} + */ + public static final class Builder extends + com.google.protobuf.GeneratedMessageV3.Builder implements + // @@protoc_insertion_point(builder_implements:org.aion.api.server.pb.rsp_getNrgPrice) + org.aion.api.server.pb.Message.rsp_getNrgPriceOrBuilder { + public static final com.google.protobuf.Descriptors.Descriptor + getDescriptor() { + return org.aion.api.server.pb.Message.internal_static_org_aion_api_server_pb_rsp_getNrgPrice_descriptor; + } + + protected com.google.protobuf.GeneratedMessageV3.FieldAccessorTable + internalGetFieldAccessorTable() { + return org.aion.api.server.pb.Message.internal_static_org_aion_api_server_pb_rsp_getNrgPrice_fieldAccessorTable + .ensureFieldAccessorsInitialized( + org.aion.api.server.pb.Message.rsp_getNrgPrice.class, org.aion.api.server.pb.Message.rsp_getNrgPrice.Builder.class); + } + + // Construct using org.aion.api.server.pb.Message.rsp_getNrgPrice.newBuilder() + private Builder() { + maybeForceBuilderInitialization(); + } + + private Builder( + com.google.protobuf.GeneratedMessageV3.BuilderParent parent) { + super(parent); + maybeForceBuilderInitialization(); + } + private void maybeForceBuilderInitialization() { + if (com.google.protobuf.GeneratedMessageV3 + .alwaysUseFieldBuilders) { + } + } + public Builder clear() { + super.clear(); + nrgPrice_ = 0L; + + return this; + } + + public com.google.protobuf.Descriptors.Descriptor + getDescriptorForType() { + return org.aion.api.server.pb.Message.internal_static_org_aion_api_server_pb_rsp_getNrgPrice_descriptor; + } + + public org.aion.api.server.pb.Message.rsp_getNrgPrice getDefaultInstanceForType() { + return org.aion.api.server.pb.Message.rsp_getNrgPrice.getDefaultInstance(); + } + + public org.aion.api.server.pb.Message.rsp_getNrgPrice build() { + org.aion.api.server.pb.Message.rsp_getNrgPrice result = buildPartial(); + if (!result.isInitialized()) { + throw newUninitializedMessageException(result); + } + return result; + } + + public org.aion.api.server.pb.Message.rsp_getNrgPrice buildPartial() { + org.aion.api.server.pb.Message.rsp_getNrgPrice result = new org.aion.api.server.pb.Message.rsp_getNrgPrice(this); + result.nrgPrice_ = nrgPrice_; + onBuilt(); + return result; + } + + public Builder clone() { + return (Builder) super.clone(); + } + public Builder setField( + com.google.protobuf.Descriptors.FieldDescriptor field, + java.lang.Object value) { + return (Builder) super.setField(field, value); + } + public Builder clearField( + com.google.protobuf.Descriptors.FieldDescriptor field) { + return (Builder) super.clearField(field); + } + public Builder clearOneof( + com.google.protobuf.Descriptors.OneofDescriptor oneof) { + return (Builder) super.clearOneof(oneof); + } + public Builder setRepeatedField( + com.google.protobuf.Descriptors.FieldDescriptor field, + int index, java.lang.Object value) { + return (Builder) super.setRepeatedField(field, index, value); + } + public Builder addRepeatedField( + com.google.protobuf.Descriptors.FieldDescriptor field, + java.lang.Object value) { + return (Builder) super.addRepeatedField(field, value); + } + public Builder mergeFrom(com.google.protobuf.Message other) { + if (other instanceof org.aion.api.server.pb.Message.rsp_getNrgPrice) { + return mergeFrom((org.aion.api.server.pb.Message.rsp_getNrgPrice)other); + } else { + super.mergeFrom(other); + return this; + } + } + + public Builder mergeFrom(org.aion.api.server.pb.Message.rsp_getNrgPrice other) { + if (other == org.aion.api.server.pb.Message.rsp_getNrgPrice.getDefaultInstance()) return this; + if (other.getNrgPrice() != 0L) { + setNrgPrice(other.getNrgPrice()); + } + this.mergeUnknownFields(other.unknownFields); + onChanged(); + return this; + } + + public final boolean isInitialized() { + return true; + } + + public Builder mergeFrom( + com.google.protobuf.CodedInputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws java.io.IOException { + org.aion.api.server.pb.Message.rsp_getNrgPrice parsedMessage = null; + try { + parsedMessage = PARSER.parsePartialFrom(input, extensionRegistry); + } catch (com.google.protobuf.InvalidProtocolBufferException e) { + parsedMessage = (org.aion.api.server.pb.Message.rsp_getNrgPrice) e.getUnfinishedMessage(); + throw e.unwrapIOException(); + } finally { + if (parsedMessage != null) { + mergeFrom(parsedMessage); + } + } + return this; + } + + private long nrgPrice_ ; + /** + * uint64 nrgPrice = 1; + */ + public long getNrgPrice() { + return nrgPrice_; + } + /** + * uint64 nrgPrice = 1; + */ + public Builder setNrgPrice(long value) { + + nrgPrice_ = value; + onChanged(); + return this; + } + /** + * uint64 nrgPrice = 1; + */ + public Builder clearNrgPrice() { + + nrgPrice_ = 0L; + onChanged(); + return this; + } + public final Builder setUnknownFields( + final com.google.protobuf.UnknownFieldSet unknownFields) { + return super.setUnknownFieldsProto3(unknownFields); + } + + public final Builder mergeUnknownFields( + final com.google.protobuf.UnknownFieldSet unknownFields) { + return super.mergeUnknownFields(unknownFields); + } + + + // @@protoc_insertion_point(builder_scope:org.aion.api.server.pb.rsp_getNrgPrice) + } + + // @@protoc_insertion_point(class_scope:org.aion.api.server.pb.rsp_getNrgPrice) + private static final org.aion.api.server.pb.Message.rsp_getNrgPrice DEFAULT_INSTANCE; + static { + DEFAULT_INSTANCE = new org.aion.api.server.pb.Message.rsp_getNrgPrice(); + } + + public static org.aion.api.server.pb.Message.rsp_getNrgPrice getDefaultInstance() { + return DEFAULT_INSTANCE; + } + + private static final com.google.protobuf.Parser + PARSER = new com.google.protobuf.AbstractParser() { + public rsp_getNrgPrice parsePartialFrom( + com.google.protobuf.CodedInputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws com.google.protobuf.InvalidProtocolBufferException { + return new rsp_getNrgPrice(input, extensionRegistry); + } + }; + + public static com.google.protobuf.Parser parser() { + return PARSER; + } + + @java.lang.Override + public com.google.protobuf.Parser getParserForType() { + return PARSER; + } + + public org.aion.api.server.pb.Message.rsp_getNrgPrice getDefaultInstanceForType() { + return DEFAULT_INSTANCE; + } + + } + private static final com.google.protobuf.Descriptors.Descriptor internal_static_org_aion_api_server_pb_t_Contract_descriptor; private static final @@ -71250,6 +71711,11 @@ public org.aion.api.server.pb.Message.rsp_getNonce getDefaultInstanceForType() { private static final com.google.protobuf.GeneratedMessageV3.FieldAccessorTable internal_static_org_aion_api_server_pb_rsp_getNonce_fieldAccessorTable; + private static final com.google.protobuf.Descriptors.Descriptor + internal_static_org_aion_api_server_pb_rsp_getNrgPrice_descriptor; + private static final + com.google.protobuf.GeneratedMessageV3.FieldAccessorTable + internal_static_org_aion_api_server_pb_rsp_getNrgPrice_fieldAccessorTable; public static com.google.protobuf.Descriptors.FileDescriptor getDescriptor() { @@ -71468,67 +71934,69 @@ public org.aion.api.server.pb.Message.rsp_getNonce getDefaultInstanceForType() { "tBlockDetailsByRange\0229\n\nblkDetails\030\001 \003(\013" + "2%.org.aion.api.server.pb.t_BlockDetail\"" + "\037\n\014req_getNonce\022\017\n\007address\030\001 \001(\014\"\035\n\014rsp_" + - "getNonce\022\r\n\005nonce\030\001 \001(\014*\204\001\n\005Servs\022\010\n\004s_h" + - "b\020\000\022\013\n\007s_admin\020\001\022\013\n\007s_chain\020\002\022\r\n\ts_accou" + - "nt\020\003\022\010\n\004s_tx\020\004\022\t\n\005s_net\020\005\022\n\n\006s_mine\020\006\022\017\n" + - "\013s_privilege\020\007\022\014\n\010s_wallet\020\010\022\010\n\004s_NA\020\t*\265" + - "\n\n\005Funcs\022\025\n\021f_protocolVersion\020\000\022\022\n\016f_min" + - "erAddress\020\001\022\016\n\nf_accounts\020\002\022\021\n\rf_blockNu" + - "mber\020\003\022\020\n\014f_getBalance\020\004\022\022\n\016f_getStorage" + - "At\020\005\022\023\n\017f_unlockAccount\020\006\022\025\n\021f_sendTrans" + - "action\020\007\022\032\n\026f_getTransactionByHash\020\010\022\r\n\t" + - "f_compile\020\t\022\024\n\020f_contractDeploy\020\n\022\031\n\025f_g" + - "etTransactionCount\020\013\022$\n f_getBlockTransa" + - "ctionCountByHash\020\014\022&\n\"f_getBlockTransact" + - "ionCountByNumber\020\r\022\r\n\tf_getCode\020\016\022\n\n\006f_c" + - "all\020\017\022\024\n\020f_getBlockByHash\020\020\022\026\n\022f_getBloc" + - "kByNumber\020\021\022\'\n#f_getTransactionByBlockHa" + - "shAndIndex\020\022\022)\n%f_getTransactionByBlockN" + - "umberAndIndex\020\023\022\033\n\027f_getTransactionRecei" + - "pt\020\024\022\022\n\016f_getCompilers\020\025\022\025\n\021f_compileSol" + - "idity\020\026\022\r\n\tf_getWork\020\027\022\020\n\014f_submitWork\020\030" + - "\022\035\n\031f_fetchQueuedTransactions\020\031\022\027\n\023f_sig" + - "nedTransaction\020\032\022\024\n\020f_rawTransaction\020\033\022\021" + - "\n\rf_estimateNrg\020\034\022\014\n\010f_mining\020\035\022\016\n\nf_has" + - "hrate\020\036\022\024\n\020f_getActiveNodes\020\037\022\024\n\020f_getSo" + - "lcVersion\020 \022\017\n\013f_isSyncing\020!\022\016\n\nf_syncIn" + - "fo\020\"\022\023\n\017f_getSystemInfo\020#\022\023\n\017f_eventRegi" + - "ster\020$\022\025\n\021f_eventDeregister\020%\022\023\n\017f_accou" + - "ntCreate\020&\022\021\n\rf_accountLock\020\'\022\023\n\017f_userP" + - "rivilege\020(\022\020\n\014f_eventQuery\020)\022\024\n\020f_import" + - "Accounts\020*\022\024\n\020f_exportAccounts\020+\022\034\n\030f_ge" + - "tBlockHeaderByNumber\020,\022\032\n\026f_getBlockHead" + - "erByHash\020-\022\037\n\033f_getCurrentTotalDifficult" + - "y\020.\022\024\n\020f_getStaticNodes\020/\022\035\n\031f_getBlockD" + - "etailsByNumber\0200\022\035\n\031f_getBlockDetailsByL" + - "atest\0201\022\027\n\023f_getBlocksByLatest\0202\022$\n f_ge" + - "tAccountDetailsByAddressList\0203\022\024\n\020f_back" + - "upAccounts\0204\022\010\n\004f_NA\0205\022\030\n\024f_getBlockSqlB" + - "yRange\0206\022\034\n\030f_getBlockDetailsByRange\0207\022\016" + - "\n\nf_getNonce\0208*\241\007\n\007Retcode\022\n\n\006r_fail\020\000\022\r" + - "\n\tr_success\020\001\022\023\n\017r_wallet_nullcb\020\002\022\025\n\021r_" + - "heartbeatReturn\020\003\022\025\n\021r_privilegeReturn\020\004" + - "\022\r\n\tr_tx_Init\020d\022\017\n\013r_tx_Recved\020e\022\020\n\014r_tx" + - "_Dropped\020f\022\023\n\017r_tx_NewPending\020g\022\020\n\014r_tx_" + - "Pending\020h\022\021\n\rr_tx_Included\020i\022\020\n\014r_tx_eve" + - "ntCb\020j\022\010\n\004r_NA\020k\022\036\n\021r_fail_header_len\020\377\377" + - "\377\377\377\377\377\377\377\001\022 \n\023r_fail_service_call\020\376\377\377\377\377\377\377\377" + - "\377\001\022!\n\024r_fail_function_call\020\375\377\377\377\377\377\377\377\377\001\022&\n" + - "\031r_fail_function_exception\020\374\377\377\377\377\377\377\377\377\001\022\037\n" + - "\022r_fail_api_version\020\373\377\377\377\377\377\377\377\377\001\022\037\n\022r_fail" + - "_ct_bytecode\020\372\377\377\377\377\377\377\377\377\001\022\034\n\017r_fail_null_r" + - "sp\020\371\377\377\377\377\377\377\377\377\001\022 \n\023r_fail_invalid_addr\020\370\377\377" + - "\377\377\377\377\377\377\001\022\'\n\032r_fail_null_compile_source\020\367\377" + - "\377\377\377\377\377\377\377\001\022$\n\027r_fail_compile_contract\020\366\377\377\377" + - "\377\377\377\377\377\001\022#\n\026r_fail_sendTx_null_rep\020\365\377\377\377\377\377\377" + - "\377\377\001\022\036\n\021r_fail_getcode_to\020\364\377\377\377\377\377\377\377\377\001\022*\n\035r" + - "_fail_getTxReceipt_null_recp\020\363\377\377\377\377\377\377\377\377\001\022" + - "(\n\033r_fail_zmqHandler_exception\020\362\377\377\377\377\377\377\377\377" + - "\001\022(\n\033r_fail_hit_pending_tx_limit\020\361\377\377\377\377\377\377" + - "\377\377\001\022%\n\030r_fail_txqueue_exception\020\360\377\377\377\377\377\377\377" + - "\377\001\022&\n\031r_fail_function_arguments\020\357\377\377\377\377\377\377\377" + - "\377\001\022!\n\024r_fail_unsupport_api\020\356\377\377\377\377\377\377\377\377\001\022\033\n" + - "\016r_fail_unknown\020\355\377\377\377\377\377\377\377\377\001b\006proto3" + "getNonce\022\r\n\005nonce\030\001 \001(\014\"#\n\017rsp_getNrgPri" + + "ce\022\020\n\010nrgPrice\030\001 \001(\004*\204\001\n\005Servs\022\010\n\004s_hb\020\000" + + "\022\013\n\007s_admin\020\001\022\013\n\007s_chain\020\002\022\r\n\ts_account\020" + + "\003\022\010\n\004s_tx\020\004\022\t\n\005s_net\020\005\022\n\n\006s_mine\020\006\022\017\n\013s_" + + "privilege\020\007\022\014\n\010s_wallet\020\010\022\010\n\004s_NA\020\t*\310\n\n\005" + + "Funcs\022\025\n\021f_protocolVersion\020\000\022\022\n\016f_minerA" + + "ddress\020\001\022\016\n\nf_accounts\020\002\022\021\n\rf_blockNumbe" + + "r\020\003\022\020\n\014f_getBalance\020\004\022\022\n\016f_getStorageAt\020" + + "\005\022\023\n\017f_unlockAccount\020\006\022\025\n\021f_sendTransact" + + "ion\020\007\022\032\n\026f_getTransactionByHash\020\010\022\r\n\tf_c" + + "ompile\020\t\022\024\n\020f_contractDeploy\020\n\022\031\n\025f_getT" + + "ransactionCount\020\013\022$\n f_getBlockTransacti" + + "onCountByHash\020\014\022&\n\"f_getBlockTransaction" + + "CountByNumber\020\r\022\r\n\tf_getCode\020\016\022\n\n\006f_call" + + "\020\017\022\024\n\020f_getBlockByHash\020\020\022\026\n\022f_getBlockBy" + + "Number\020\021\022\'\n#f_getTransactionByBlockHashA" + + "ndIndex\020\022\022)\n%f_getTransactionByBlockNumb" + + "erAndIndex\020\023\022\033\n\027f_getTransactionReceipt\020" + + "\024\022\022\n\016f_getCompilers\020\025\022\025\n\021f_compileSolidi" + + "ty\020\026\022\r\n\tf_getWork\020\027\022\020\n\014f_submitWork\020\030\022\035\n" + + "\031f_fetchQueuedTransactions\020\031\022\027\n\023f_signed" + + "Transaction\020\032\022\024\n\020f_rawTransaction\020\033\022\021\n\rf" + + "_estimateNrg\020\034\022\014\n\010f_mining\020\035\022\016\n\nf_hashra" + + "te\020\036\022\024\n\020f_getActiveNodes\020\037\022\024\n\020f_getSolcV" + + "ersion\020 \022\017\n\013f_isSyncing\020!\022\016\n\nf_syncInfo\020" + + "\"\022\023\n\017f_getSystemInfo\020#\022\023\n\017f_eventRegiste" + + "r\020$\022\025\n\021f_eventDeregister\020%\022\023\n\017f_accountC" + + "reate\020&\022\021\n\rf_accountLock\020\'\022\023\n\017f_userPriv" + + "ilege\020(\022\020\n\014f_eventQuery\020)\022\024\n\020f_importAcc" + + "ounts\020*\022\024\n\020f_exportAccounts\020+\022\034\n\030f_getBl" + + "ockHeaderByNumber\020,\022\032\n\026f_getBlockHeaderB" + + "yHash\020-\022\037\n\033f_getCurrentTotalDifficulty\020." + + "\022\024\n\020f_getStaticNodes\020/\022\035\n\031f_getBlockDeta" + + "ilsByNumber\0200\022\035\n\031f_getBlockDetailsByLate" + + "st\0201\022\027\n\023f_getBlocksByLatest\0202\022$\n f_getAc" + + "countDetailsByAddressList\0203\022\024\n\020f_backupA" + + "ccounts\0204\022\010\n\004f_NA\0205\022\030\n\024f_getBlockSqlByRa" + + "nge\0206\022\034\n\030f_getBlockDetailsByRange\0207\022\016\n\nf" + + "_getNonce\0208\022\021\n\rf_getNrgPrice\0209*\241\007\n\007Retco" + + "de\022\n\n\006r_fail\020\000\022\r\n\tr_success\020\001\022\023\n\017r_walle" + + "t_nullcb\020\002\022\025\n\021r_heartbeatReturn\020\003\022\025\n\021r_p" + + "rivilegeReturn\020\004\022\r\n\tr_tx_Init\020d\022\017\n\013r_tx_" + + "Recved\020e\022\020\n\014r_tx_Dropped\020f\022\023\n\017r_tx_NewPe" + + "nding\020g\022\020\n\014r_tx_Pending\020h\022\021\n\rr_tx_Includ" + + "ed\020i\022\020\n\014r_tx_eventCb\020j\022\010\n\004r_NA\020k\022\036\n\021r_fa" + + "il_header_len\020\377\377\377\377\377\377\377\377\377\001\022 \n\023r_fail_servi" + + "ce_call\020\376\377\377\377\377\377\377\377\377\001\022!\n\024r_fail_function_ca" + + "ll\020\375\377\377\377\377\377\377\377\377\001\022&\n\031r_fail_function_excepti" + + "on\020\374\377\377\377\377\377\377\377\377\001\022\037\n\022r_fail_api_version\020\373\377\377\377" + + "\377\377\377\377\377\001\022\037\n\022r_fail_ct_bytecode\020\372\377\377\377\377\377\377\377\377\001\022" + + "\034\n\017r_fail_null_rsp\020\371\377\377\377\377\377\377\377\377\001\022 \n\023r_fail_" + + "invalid_addr\020\370\377\377\377\377\377\377\377\377\001\022\'\n\032r_fail_null_c" + + "ompile_source\020\367\377\377\377\377\377\377\377\377\001\022$\n\027r_fail_compi" + + "le_contract\020\366\377\377\377\377\377\377\377\377\001\022#\n\026r_fail_sendTx_" + + "null_rep\020\365\377\377\377\377\377\377\377\377\001\022\036\n\021r_fail_getcode_to" + + "\020\364\377\377\377\377\377\377\377\377\001\022*\n\035r_fail_getTxReceipt_null_" + + "recp\020\363\377\377\377\377\377\377\377\377\001\022(\n\033r_fail_zmqHandler_exc" + + "eption\020\362\377\377\377\377\377\377\377\377\001\022(\n\033r_fail_hit_pending_" + + "tx_limit\020\361\377\377\377\377\377\377\377\377\001\022%\n\030r_fail_txqueue_ex" + + "ception\020\360\377\377\377\377\377\377\377\377\001\022&\n\031r_fail_function_ar" + + "guments\020\357\377\377\377\377\377\377\377\377\001\022!\n\024r_fail_unsupport_a" + + "pi\020\356\377\377\377\377\377\377\377\377\001\022\033\n\016r_fail_unknown\020\355\377\377\377\377\377\377\377" + + "\377\001b\006proto3" }; com.google.protobuf.Descriptors.FileDescriptor.InternalDescriptorAssigner assigner = new com.google.protobuf.Descriptors.FileDescriptor. InternalDescriptorAssigner() { @@ -72166,6 +72634,12 @@ public com.google.protobuf.ExtensionRegistry assignDescriptors( com.google.protobuf.GeneratedMessageV3.FieldAccessorTable( internal_static_org_aion_api_server_pb_rsp_getNonce_descriptor, new java.lang.String[] { "Nonce", }); + internal_static_org_aion_api_server_pb_rsp_getNrgPrice_descriptor = + getDescriptor().getMessageTypes().get(103); + internal_static_org_aion_api_server_pb_rsp_getNrgPrice_fieldAccessorTable = new + com.google.protobuf.GeneratedMessageV3.FieldAccessorTable( + internal_static_org_aion_api_server_pb_rsp_getNrgPrice_descriptor, + new java.lang.String[] { "NrgPrice", }); } // @@protoc_insertion_point(outer_class_scope) diff --git a/modApiServer/src/org/aion/api/server/zmq/ProtocolProcessor.java b/modApiServer/src/org/aion/api/server/zmq/ProtocolProcessor.java index 523a853cf8..4453dfd186 100644 --- a/modApiServer/src/org/aion/api/server/zmq/ProtocolProcessor.java +++ b/modApiServer/src/org/aion/api/server/zmq/ProtocolProcessor.java @@ -65,6 +65,7 @@ public class ProtocolProcessor implements Runnable { private static final long zmqHWM = 100_000; private static final int SOCKETID_LEN = 5; + private static final int SOCKET_RECV_TIMEOUT = 3000; public ProtocolProcessor(IHdlr _handler, final CfgApiZmq cfg) { this.handler = _handler; @@ -73,6 +74,8 @@ public ProtocolProcessor(IHdlr _handler, final CfgApiZmq cfg) { public void shutdown() throws InterruptedException { handler.shutDown(); + shutDown.set(true); + Thread.sleep(SOCKET_RECV_TIMEOUT); Proxy.shutdown(); } @@ -117,9 +120,8 @@ public void run() { } if (LOG.isInfoEnabled()) { - LOG.info("Shutting down Sockets..."); + LOG.info("Shutting down Zmq sockets..."); } - shutDown.set(true); // Shutdown HdlrZmq ((HdlrZmq) handler).shutdown(); // Shutdown ZmqSocket @@ -133,7 +135,7 @@ public void run() { ctx.close(); if (LOG.isInfoEnabled()) { - LOG.info("Shutdown Sockets... Done!"); + LOG.info("Shutdown Zmq sockets... Done!"); } } catch (Exception e) { @@ -172,7 +174,7 @@ private void eventRun(Context ctx) { try { byte[] socketId = ByteBuffer.allocate(5).put(ByteUtil.longToBytes(i), 3, 5).array(); sock.send(socketId, ZMQ.SNDMORE); - sock.send(rsp, ZMQ.PAIR); + sock.send(rsp, ZMQ.DONTWAIT); } catch (Exception e) { LOG.error("ProtocolProcessor.callbackRun sock.send exception: " + e.getMessage()); } @@ -187,8 +189,8 @@ private void eventRun(Context ctx) { } } sock.close(); - if (LOG.isInfoEnabled()) { - LOG.info("close eventRun Sockets..."); + if (LOG.isDebugEnabled()) { + LOG.debug("close eventRun sockets..."); } } @@ -228,7 +230,7 @@ private void callbackRun(Context ctx) { } try { sock.send(tps.getSocketId(), ZMQ.SNDMORE); - sock.send(rsp, ZMQ.PAIR); + sock.send(rsp, ZMQ.DONTWAIT); } catch (Exception e) { if (LOG.isErrorEnabled()) { LOG.error("ProtocolProcessor.callbackRun sock.send exception: " + e.getMessage()); @@ -236,14 +238,15 @@ private void callbackRun(Context ctx) { } } sock.close(); - if (LOG.isInfoEnabled()) { - LOG.info("close callbackRun Sockets..."); + if (LOG.isDebugEnabled()) { + LOG.debug("close callbackRun sockets..."); } } private void workerRun(ZMQ.Context ctx) { Socket sock = ctx.socket(ZMQ.DEALER); sock.connect(AION_ZMQ_WK_TH); + sock.setReceiveTimeOut(SOCKET_RECV_TIMEOUT); while (!shutDown.get()) { try { @@ -253,27 +256,27 @@ private void workerRun(ZMQ.Context ctx) { } if (socketId != null && socketId.length == SOCKETID_LEN) { byte[] req = sock.recv(0); - if (LOG.isTraceEnabled()) { - LOG.trace("ProtocolProcessor.workerRun reqMsg: [{}]", Hex.toHexString(req)); - } - byte[] rsp = ((HdlrZmq) this.handler).process(req, socketId); - if (LOG.isTraceEnabled()) { - LOG.trace("ProtocolProcessor.workerRun rspMsg: [{}]", Hex.toHexString(rsp)); - } + if (req != null) { + if (LOG.isTraceEnabled()) { + LOG.trace("ProtocolProcessor.workerRun reqMsg: [{}]", + Hex.toHexString(req)); + } + byte[] rsp = ((HdlrZmq) this.handler).process(req, socketId); + if (LOG.isTraceEnabled()) { + LOG.trace("ProtocolProcessor.workerRun rspMsg: [{}]", + Hex.toHexString(rsp)); + } - try { - sock.send(socketId, ZMQ.SNDMORE); - sock.send(rsp, ZMQ.PAIR); - } catch (Exception e) { - if (LOG.isErrorEnabled()) { - LOG.error("ProtocolProcessor.workerRun sock.send exception: " + e.getMessage()); + try { + sock.send(socketId, ZMQ.SNDMORE); + sock.send(rsp, ZMQ.DONTWAIT); + } catch (Exception e) { + if (LOG.isErrorEnabled()) { + LOG.error("ProtocolProcessor.workerRun sock.send exception: " + e + .getMessage()); + } } } - } else { - if (LOG.isErrorEnabled()) { - LOG.error("ProtocolProcessor.workerRun incorrect socketID [{}]", - socketId == null ? "null" : Hex.toHexString(socketId)); - } } } catch (Exception e) { if (LOG.isErrorEnabled()) { @@ -282,14 +285,15 @@ private void workerRun(ZMQ.Context ctx) { } } sock.close(); - if (LOG.isInfoEnabled()) { - LOG.info("close workerRun Sockets..."); + if (LOG.isDebugEnabled()) { + LOG.debug("close workerRun sockets..."); } } private void hbRun(ZMQ.Context ctx) { Socket sock = ctx.socket(ZMQ.DEALER); sock.connect(AION_ZMQ_HB_TH); + sock.setReceiveTimeOut(SOCKET_RECV_TIMEOUT); while (!shutDown.get()) { try { @@ -299,27 +303,26 @@ private void hbRun(ZMQ.Context ctx) { } if (socketId != null && socketId.length == SOCKETID_LEN) { byte[] req = sock.recv(0); - if (LOG.isTraceEnabled()) { - LOG.trace("ProtocolProcessor.hbRun reqMsg: [{}]", Hex.toHexString(req)); - } - byte[] rsp = ApiUtil.toReturnHeader(JAVAAPI_VAR, Message.Retcode.r_heartbeatReturn_VALUE); - if (LOG.isTraceEnabled()) { - LOG.trace("ProtocolProcessor.hbRun rspMsg: [{}]", Hex.toHexString(rsp)); - } + if (req != null) { + if (LOG.isTraceEnabled()) { + LOG.trace("ProtocolProcessor.hbRun reqMsg: [{}]", Hex.toHexString(req)); + } + byte[] rsp = ApiUtil + .toReturnHeader(JAVAAPI_VAR, Message.Retcode.r_heartbeatReturn_VALUE); + if (LOG.isTraceEnabled()) { + LOG.trace("ProtocolProcessor.hbRun rspMsg: [{}]", Hex.toHexString(rsp)); + } - try { - sock.send(socketId, ZMQ.SNDMORE); - sock.send(rsp, ZMQ.PAIR); - } catch (Exception e) { - if (LOG.isErrorEnabled()) { - LOG.error("ProtocolProcessor.hbRun sock.send exception: " + e.getMessage()); + try { + sock.send(socketId, ZMQ.SNDMORE); + sock.send(rsp, ZMQ.DONTWAIT); + } catch (Exception e) { + if (LOG.isErrorEnabled()) { + LOG.error("ProtocolProcessor.hbRun sock.send exception: " + e + .getMessage()); + } } } - } else { - if (LOG.isErrorEnabled()) { - LOG.error("ProtocolProcessor.hbRun incorrect socketID [{}]", - socketId == null ? "null" : Hex.toHexString(socketId)); - } } } catch (Exception e) { if (LOG.isErrorEnabled()) { @@ -328,8 +331,8 @@ private void hbRun(ZMQ.Context ctx) { } } sock.close(); - if (LOG.isInfoEnabled()) { - LOG.info("close hbRun Sockets..."); + if (LOG.isDebugEnabled()) { + LOG.debug("close hbRun sockets..."); } } } diff --git a/modApiServer/src/org/aion/api/server/zmq/Proxy.java b/modApiServer/src/org/aion/api/server/zmq/Proxy.java index d5c950ffdd..906c434ed7 100644 --- a/modApiServer/src/org/aion/api/server/zmq/Proxy.java +++ b/modApiServer/src/org/aion/api/server/zmq/Proxy.java @@ -105,7 +105,7 @@ static void proxy(Socket frontend, Socket backend, Socket callback, Socket event } } - LOG.info("zmq-proxy thread was interrupted."); + LOG.debug("zmq-proxy thread was interrupted."); } catch (Exception e) { LOG.error("aion.api.server.zmq.Proxy exception" + e.getMessage()); } @@ -173,10 +173,10 @@ private static boolean msgProcessSend(Socket receiver, Socket sender) { } public static void shutdown() throws InterruptedException { - LOG.info("zmq-proxy thread shuting down..."); + LOG.debug("zmq-proxy thread shutting down..."); shutDown.set(true); - LOG.info("waiting zmq-proxy thread shutdown"); + LOG.debug("waiting zmq-proxy thread shutdown"); Thread.sleep(3000); } } diff --git a/modBoot/build.xml b/modBoot/build.xml index 1af2ade438..1e298dc6b0 100644 --- a/modBoot/build.xml +++ b/modBoot/build.xml @@ -39,7 +39,7 @@ debug="on" debuglevel="source,lines,vars" includeantruntime="false" - release="9" + release="10" srcdir="./test" destdir="${dir.test}" includes="**/*Test.java" @@ -74,7 +74,7 @@ debug="${compile.debug}" debuglevel="source,lines,vars" includeantruntime="false" - release="9" + release="10" srcdir="./" destdir="${dir.dest}" includes="src/**/*.java,module-info.java" @@ -103,41 +103,4 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/modBoot/resource/config.xml b/modBoot/resource/config.xml index 06a3f2de99..fe0d84971a 100644 --- a/modBoot/resource/config.xml +++ b/modBoot/resource/config.xml @@ -36,8 +36,6 @@ 0.0.0.0 30303 false - false - false 128 @@ -64,6 +62,11 @@ database true + + + + + FULL leveldb @@ -71,11 +74,16 @@ false + + true + + log INFO ERROR INFO INFO ERROR INFO + INFO diff --git a/modBoot/src/org/aion/Aion.java b/modBoot/src/org/aion/Aion.java index ab9742bfbb..3931d5bc06 100644 --- a/modBoot/src/org/aion/Aion.java +++ b/modBoot/src/org/aion/Aion.java @@ -1,4 +1,4 @@ -/******************************************************************************* +/* * Copyright (c) 2017-2018 Aion foundation. * * This file is part of the aion network project. @@ -17,14 +17,16 @@ * along with the aion network project source files. * If not, see . * - * Contributors to the aion source files in decreasing order of code volume: - * + * Contributors: * Aion foundation. - * - ******************************************************************************/ - + */ package org.aion; +import static org.aion.crypto.ECKeyFac.ECKeyType.ED25519; +import static org.aion.crypto.HashUtil.H256Type.BLAKE2B_256; +import static org.aion.zero.impl.Version.KERNEL_VERSION; + +import java.util.ServiceLoader; import org.aion.api.server.http.NanoServer; import org.aion.api.server.pb.ApiAion0; import org.aion.api.server.pb.IHdlr; @@ -43,15 +45,9 @@ import org.aion.zero.impl.config.CfgAion; import org.slf4j.Logger; -import java.util.ServiceLoader; - -import static org.aion.crypto.ECKeyFac.ECKeyType.ED25519; -import static org.aion.crypto.HashUtil.H256Type.BLAKE2B_256; -import static org.aion.zero.impl.Version.KERNEL_VERSION; - public class Aion { - public static void main(String args[]) throws InterruptedException { + public static void main(String args[]) { /* * @ATTENTION: ECKey have two layer: tx layer is KeyFac optional, @@ -71,10 +67,10 @@ public static void main(String args[]) throws InterruptedException { * if in the config.xml id is set as default [NODE-ID-PLACEHOLDER] * return true which means should save back to xml config */ - if(cfg.fromXML()) - cfg.toXML(new String[]{ "--id=" + cfg.getId() }); + if (cfg.fromXML()) { + cfg.toXML(new String[]{"--id=" + cfg.getId()}); + } - try { ServiceLoader.load(AionLoggerFactory.class); } catch (Exception e) { @@ -82,23 +78,36 @@ public static void main(String args[]) throws InterruptedException { throw e; } + /* Outputs relevant logger configuration */ + if (!cfg.getLog().getLogFile()) { + System.out + .println("Logger disabled; to enable please check log settings in config.xml\n"); + } else if (!cfg.getLog().isValidPath() && cfg.getLog().getLogFile()) { + System.out.println("File path is invalid; please check log setting in config.xml\n"); + return; + } else if (cfg.getLog().isValidPath() && cfg.getLog().getLogFile()) { + System.out.println("Logger file path: '" + cfg.getLog().getLogPath() + "'\n"); + } + + /* + * Logger initialize with LOGFILE and LOGPATH (user config inputs) + */ + AionLoggerFactory + .init(cfg.getLog().getModules(), cfg.getLog().getLogFile(), cfg.getLog().getLogPath()); + Logger genLog = AionLoggerFactory.getLogger(LogEnum.GEN.name()); - // If commit this out, the config setting will be ignore. all log module been set to "INFO" Level - AionLoggerFactory.init(cfg.getLog().getModules()); - Logger LOG = AionLoggerFactory.getLogger(LogEnum.GEN.toString()); + String logo ="\n _____ \n" + + " .'. | .~ ~. |.. |\n" + + " .' `. | | | | ``.. |\n" + + " .''''''''`. | | | | ``.. |\n" + + ".' `. | `._____.' | ``|\n\n" + + " NETWORK v" + KERNEL_VERSION + + "\n\n"; - System.out.println( - " _____ \n" + - " .'. | .~ ~. |.. |\n" + - " .' `. | | | | ``.. |\n" + - " .''''''''`. | | | | ``.. |\n" + - ".' `. | `._____.' | ``|\n\n" + - " NETWORK v" + KERNEL_VERSION + - "\n\n" - ); + genLog.info(logo); IAionChain ac = AionFactory.create(); - + IMineRunner nm = null; if (!cfg.getConsensus().isSeed()) { @@ -117,22 +126,19 @@ public static void main(String args[]) throws InterruptedException { if (cfg.getApi().getZmq().getActive()) { IHdlr handler = new HdlrZmq(new ApiAion0(ac)); processor = new ProtocolProcessor(handler, cfg.getApi().getZmq()); - ProtocolProcessor finalProcessor = processor; - zmqThread = new Thread(() -> { - finalProcessor.run(); - }, "zmq-api"); + zmqThread = new Thread(processor, "zmq-api"); zmqThread.start(); } NanoServer rpcServer = null; - if(cfg.getApi().getRpc().getActive()) { - CfgApiRpc rpcCfg = cfg.getApi().getRpc(); + if (cfg.getApi().getRpc().getActive()) { + CfgApiRpc rpcCfg = cfg.getApi().getRpc(); rpcServer = new NanoServer( - rpcCfg.getIp(), - rpcCfg.getPort(), - rpcCfg.getCorsEnabled(), - rpcCfg.getEnabled(), - rpcCfg.getMaxthread()); + rpcCfg.getIp(), + rpcCfg.getPort(), + rpcCfg.getCorsEnabled(), + rpcCfg.getEnabled(), + rpcCfg.getMaxthread()); rpcServer.start(); } @@ -141,12 +147,14 @@ public static void main(String args[]) throws InterruptedException { * Shutdown hook for Ctrl+C */ class ShutdownThreadHolder { - final Thread zmqThread; - final IMineRunner miner; - final ProtocolProcessor pp; - final NanoServer rpc; - - private ShutdownThreadHolder(Thread zmqThread, IMineRunner nm, ProtocolProcessor pp, NanoServer rpc) { + + private final Thread zmqThread; + private final IMineRunner miner; + private final ProtocolProcessor pp; + private final NanoServer rpc; + + private ShutdownThreadHolder(Thread zmqThread, IMineRunner nm, ProtocolProcessor pp, + NanoServer rpc) { this.zmqThread = zmqThread; this.miner = nm; this.pp = pp; @@ -155,42 +163,53 @@ private ShutdownThreadHolder(Thread zmqThread, IMineRunner nm, ProtocolProcessor } ShutdownThreadHolder holder = new ShutdownThreadHolder(zmqThread, nm, processor, rpcServer); - + Runtime.getRuntime().addShutdownHook(new Thread(() -> { - LOG.info("Starting shutdown process..."); + genLog.info("Starting shutdown process..."); if (holder.rpc != null) { - LOG.info("Shutting down RpcServer"); + genLog.info("Shutting down RpcServer"); holder.rpc.shutdown(); - LOG.info("Shutdown RpcServer ... Done!"); + genLog.info("Shutdown RpcServer ... Done!"); } if (holder.pp != null) { - LOG.info("Shutting down zmq ProtocolProcessor"); + genLog.info("Shutting down zmq ProtocolProcessor"); try { holder.pp.shutdown(); - LOG.info("Shutdown zmq ProtocolProcessor... Done!"); + genLog.info("Shutdown zmq ProtocolProcessor... Done!"); } catch (InterruptedException e) { - LOG.info("Shutdown zmq ProtocolProcessor failed! {}", e.getMessage()); + genLog.info("Shutdown zmq ProtocolProcessor failed! {}", e.getMessage()); + Thread.currentThread().interrupt(); + } + } + + if (holder.zmqThread != null) { + genLog.info("Shutting down zmq thread"); + try { + holder.zmqThread.interrupt(); + genLog.info("Shutdown zmq thread... Done!"); + } catch (Exception e) { + genLog.info("Shutdown zmq thread failed! {}", e.getMessage()); Thread.currentThread().interrupt(); } } if (holder.miner != null) { - LOG.info("Shutting down sealer"); + genLog.info("Shutting down sealer"); holder.miner.stopMining(); holder.miner.shutdown(); - LOG.info("Shutdown sealer... Done!"); + genLog.info("Shutdown sealer... Done!"); } - LOG.info("Shutting down the AionHub..."); + genLog.info("Shutting down the AionHub..."); ac.getAionHub().close(); - LOG.info("---------------------------------------------"); - LOG.info("| Aion kernel graceful shutdown successful! |"); - LOG.info("---------------------------------------------"); + genLog.info("---------------------------------------------"); + genLog.info("| Aion kernel graceful shutdown successful! |"); + genLog.info("---------------------------------------------"); }, "shutdown")); } -} +} \ No newline at end of file diff --git a/modCrypto/build.xml b/modCrypto/build.xml index 79da0da40f..59f31348a8 100644 --- a/modCrypto/build.xml +++ b/modCrypto/build.xml @@ -19,13 +19,6 @@ - - - - - - - @@ -56,7 +49,7 @@ - + @@ -71,37 +64,11 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - + @@ -110,7 +77,7 @@ - + @@ -160,6 +127,4 @@ - - diff --git a/modDbImpl/build.xml b/modDbImpl/build.xml index 543e9d2fb6..c78257b5a8 100644 --- a/modDbImpl/build.xml +++ b/modDbImpl/build.xml @@ -44,7 +44,7 @@ - + @@ -54,7 +54,7 @@ - + @@ -98,7 +98,7 @@ - + diff --git a/modDbImpl/src/org/aion/db/generic/DatabaseWithCache.java b/modDbImpl/src/org/aion/db/generic/DatabaseWithCache.java index e2380e64ef..80636044ae 100644 --- a/modDbImpl/src/org/aion/db/generic/DatabaseWithCache.java +++ b/modDbImpl/src/org/aion/db/generic/DatabaseWithCache.java @@ -1,4 +1,4 @@ -/******************************************************************************* +/* ****************************************************************************** * Copyright (c) 2017-2018 Aion foundation. * * This file is part of the aion network project. @@ -163,14 +163,8 @@ public CacheStats getStats() { return this.loadingCache.stats(); } - /** - * Checks that the database connection is open. - * Throws a {@link RuntimeException} if the database connection is closed. - * - * @implNote Always do this check after acquiring a lock on the class/data. - * Otherwise it might produce inconsistent results due to lack of synchronization. - */ - private void check() { + @Override + public void check() { if (!database.isOpen()) { throw new RuntimeException("Database is not opened: " + this); } diff --git a/modDbImpl/src/org/aion/db/generic/LockedDatabase.java b/modDbImpl/src/org/aion/db/generic/LockedDatabase.java index ad7ed79c6d..9015430f35 100644 --- a/modDbImpl/src/org/aion/db/generic/LockedDatabase.java +++ b/modDbImpl/src/org/aion/db/generic/LockedDatabase.java @@ -366,6 +366,19 @@ public void deleteBatch(Collection keys) { } } + @Override + public void check() { + // acquire read lock + lock.readLock().lock(); + + try { + database.check(); + } finally { + // releasing read lock + lock.readLock().unlock(); + } + } + @Override public void drop() { // acquire write lock diff --git a/modDbImpl/src/org/aion/db/generic/TimedDatabase.java b/modDbImpl/src/org/aion/db/generic/TimedDatabase.java index 4a70fddc23..b101f6f697 100644 --- a/modDbImpl/src/org/aion/db/generic/TimedDatabase.java +++ b/modDbImpl/src/org/aion/db/generic/TimedDatabase.java @@ -1,4 +1,4 @@ -/******************************************************************************* +/* ****************************************************************************** * Copyright (c) 2017-2018 Aion foundation. * * This file is part of the aion network project. @@ -28,17 +28,16 @@ ******************************************************************************/ package org.aion.db.generic; +import java.util.Collection; +import java.util.Map; +import java.util.Optional; +import java.util.Set; import org.aion.base.db.IByteArrayKeyValueDatabase; import org.aion.base.util.Hex; import org.aion.log.AionLoggerFactory; import org.aion.log.LogEnum; import org.slf4j.Logger; -import java.util.Collection; -import java.util.Map; -import java.util.Optional; -import java.util.Set; - /** * Times different database operations and logs the time. * @@ -48,6 +47,7 @@ public class TimedDatabase implements IByteArrayKeyValueDatabase { /** Unlocked database. */ protected final IByteArrayKeyValueDatabase database; + protected static final Logger LOG = AionLoggerFactory.getLogger(LogEnum.DB.name()); public TimedDatabase(IByteArrayKeyValueDatabase _database) { @@ -59,7 +59,8 @@ public String toString() { return this.getClass().getSimpleName() + " over " + database.toString(); } - // IDatabase functionality ----------------------------------------------------------------------------------------- + // IDatabase functionality + // ----------------------------------------------------------------------------------------- @Override public boolean open() { @@ -164,7 +165,8 @@ public long approximateSize() { return result; } - // IKeyValueStore functionality ------------------------------------------------------------------------------------ + // IKeyValueStore functionality + // ------------------------------------------------------------------------------------ @Override public boolean isEmpty() { @@ -192,7 +194,13 @@ public Optional get(byte[] key) { Optional value = database.get(key); long t2 = System.nanoTime(); - LOG.debug(database.toString() + " get(key) in " + (t2 - t1) + " ns." + "\n\t\t\t\t\tkey = " + Hex.toHexString(key)); + LOG.debug( + database.toString() + + " get(key) in " + + (t2 - t1) + + " ns." + + "\n\t\t\t\t\tkey = " + + (key != null ? Hex.toHexString(key) : "null")); return value; } @@ -202,8 +210,15 @@ public void put(byte[] key, byte[] value) { database.put(key, value); long t2 = System.nanoTime(); - LOG.debug(database.toString() + " put(key,value) in " + (t2 - t1) + " ns." + "\n\t\t\t\t\tkey = " + Hex.toHexString(key) - + "\n\t\t\t\t\tvalue = " + Hex.toHexString(value)); + LOG.debug( + database.toString() + + " put(key,value) in " + + (t2 - t1) + + " ns." + + "\n\t\t\t\t\tkey = " + + (key != null ? Hex.toHexString(key) : "null") + + "\n\t\t\t\t\tvalue = " + + (value != null ? Hex.toHexString(value) : "null")); } @Override @@ -212,7 +227,13 @@ public void delete(byte[] key) { database.delete(key); long t2 = System.nanoTime(); - LOG.debug(database.toString() + " delete(key) in " + (t2 - t1) + " ns." + "\n\t\t\t\t\tkey = " + Hex.toHexString(key)); + LOG.debug( + database.toString() + + " delete(key) in " + + (t2 - t1) + + " ns." + + "\n\t\t\t\t\tkey = " + + (key != null ? Hex.toHexString(key) : "null")); } @Override @@ -221,7 +242,13 @@ public void putBatch(Map keyValuePairs) { database.putBatch(keyValuePairs); long t2 = System.nanoTime(); - LOG.debug(database.toString() + " putBatch(" + keyValuePairs.size() + ") in " + (t2 - t1) + " ns."); + LOG.debug( + database.toString() + + " putBatch(" + + (keyValuePairs != null ? keyValuePairs.size() : "null") + + ") in " + + (t2 - t1) + + " ns."); } @Override @@ -230,8 +257,15 @@ public void putToBatch(byte[] key, byte[] value) { database.putToBatch(key, value); long t2 = System.nanoTime(); - LOG.debug(database.toString() + " putToBatch(key,value) in " + (t2 - t1) + " ns." + "\n\t\t\t\t\tkey = " + Hex - .toHexString(key) + "\n\t\t\t\t\tvalue = " + Hex.toHexString(value)); + LOG.debug( + database.toString() + + " putToBatch(key,value) in " + + (t2 - t1) + + " ns." + + "\n\t\t\t\t\tkey = " + + Hex.toHexString(key) + + "\n\t\t\t\t\tvalue = " + + (value != null ? Hex.toHexString(value) : "null")); } @Override @@ -249,7 +283,22 @@ public void deleteBatch(Collection keys) { database.deleteBatch(keys); long t2 = System.nanoTime(); - LOG.debug(database.toString() + " deleteBatch(" + keys.size() + ") in " + (t2 - t1) + " ns."); + LOG.debug( + database.toString() + + " deleteBatch(" + + (keys != null ? keys.size() : "null") + + ") in " + + (t2 - t1) + + " ns."); + } + + @Override + public void check() { + long t1 = System.nanoTime(); + database.check(); + long t2 = System.nanoTime(); + + LOG.debug(database.toString() + " check() in " + (t2 - t1) + " ns."); } @Override diff --git a/modDbImpl/src/org/aion/db/impl/AbstractDB.java b/modDbImpl/src/org/aion/db/impl/AbstractDB.java index ee346f2ec8..9fa6243eed 100644 --- a/modDbImpl/src/org/aion/db/impl/AbstractDB.java +++ b/modDbImpl/src/org/aion/db/impl/AbstractDB.java @@ -1,4 +1,4 @@ -/******************************************************************************* +/* ****************************************************************************** * Copyright (c) 2017-2018 Aion foundation. * * This file is part of the aion network project. @@ -34,18 +34,16 @@ ******************************************************************************/ package org.aion.db.impl; -import org.aion.base.db.IByteArrayKeyValueDatabase; -import org.aion.base.util.ByteArrayWrapper; -import org.aion.log.AionLoggerFactory; -import org.aion.log.LogEnum; -import org.h2.store.fs.FileUtils; -import org.slf4j.Logger; - import java.io.File; import java.nio.file.Files; import java.nio.file.Path; import java.util.*; import java.util.stream.Stream; +import org.aion.base.db.IByteArrayKeyValueDatabase; +import org.aion.base.util.ByteArrayWrapper; +import org.aion.log.AionLoggerFactory; +import org.aion.log.LogEnum; +import org.slf4j.Logger; /** * Common functionality for database implementations. @@ -70,7 +68,8 @@ protected AbstractDB(String name) { this.name = name; } - protected AbstractDB(String name, String path, boolean enableDbCache, boolean enableDbCompression) { + protected AbstractDB( + String name, String path, boolean enableDbCache, boolean enableDbCompression) { this(name); Objects.requireNonNull(path, "The database path cannot be null."); @@ -81,14 +80,22 @@ protected AbstractDB(String name, String path, boolean enableDbCache, boolean en } protected String propertiesInfo() { - return ""; // + return ""; // } @Override public boolean commit() { - // not implemented since we always commit the changes to the database for this implementation - throw new UnsupportedOperationException("Only automatic commits are supported by " + this.toString()); + // not implemented since we always commit the changes to the database for this + // implementation + throw new UnsupportedOperationException( + "Only automatic commits are supported by " + this.toString()); } @Override @@ -119,22 +126,16 @@ public Optional getPath() { return Optional.ofNullable(this.path); } - /** - * Checks that the database connection is open. - * Throws a {@link RuntimeException} if the database connection is closed. - * - * @implNote Always do this check after acquiring a lock on the class/data. - * Otherwise it might produce inconsistent results due to lack of synchronization. - */ - protected void check() { + @Override + public void check() { if (!isOpen()) { throw new RuntimeException("Database is not opened: " + this); } } /** - * Checks that the given key is not null. - * Throws a {@link IllegalArgumentException} if the key is null. + * Checks that the given key is not null. Throws a {@link IllegalArgumentException} if the key + * is null. */ public static void check(byte[] k) { if (k == null) { @@ -143,8 +144,8 @@ public static void check(byte[] k) { } /** - * Checks that the given collection of keys does not contain null values. - * Throws a {@link IllegalArgumentException} if a null key is present. + * Checks that the given collection of keys does not contain null values. Throws a {@link + * IllegalArgumentException} if a null key is present. */ public static void check(Collection keys) { if (keys.contains(null)) { @@ -170,23 +171,21 @@ public boolean isPersistent() { } /** - * For testing the lock functionality of public methods. - * Helps ensure that locks are released after normal or exceptional execution. + * For testing the lock functionality of public methods. Helps ensure that locks are released + * after normal or exceptional execution. * - * @return {@code true} when the resource is locked, - * {@code false} otherwise + * @return {@code true} when the resource is locked, {@code false} otherwise */ @Override public boolean isLocked() { return false; } - /** - * Functionality for directly interacting with the heap cache. - */ + /** Functionality for directly interacting with the heap cache. */ public abstract boolean commitCache(Map cache); - // IKeyValueStore functionality ------------------------------------------------------------------------------------ + // IKeyValueStore functionality + // ------------------------------------------------------------------------------------ @Override public Optional get(byte[] k) { @@ -200,12 +199,11 @@ public Optional get(byte[] k) { } /** - * Database specific get functionality, without locking required. Locking is applied in {@link #get(byte[])}. + * Database specific get functionality, without locking required. Locking is applied in {@link + * #get(byte[])}. * - * @param k - * the key for which the method must return the associated value + * @param k the key for which the method must return the associated value * @return the value stored in the database for the give key. */ protected abstract byte[] getInternal(byte[] k); - -} \ No newline at end of file +} diff --git a/modDbImpl/src/org/aion/db/impl/h2/H2MVMap.java b/modDbImpl/src/org/aion/db/impl/h2/H2MVMap.java index fc08918bc2..9492420d81 100644 --- a/modDbImpl/src/org/aion/db/impl/h2/H2MVMap.java +++ b/modDbImpl/src/org/aion/db/impl/h2/H2MVMap.java @@ -1,4 +1,4 @@ -/******************************************************************************* +/* * Copyright (c) 2017-2018 Aion foundation. * * This file is part of the aion network project. @@ -31,7 +31,7 @@ * Samuel Neves through the BLAKE2 implementation. * Zcash project team. * Bitcoinj team. - ******************************************************************************/ + */ package org.aion.db.impl.h2; import org.aion.base.util.ByteArrayWrapper; @@ -148,7 +148,8 @@ public boolean open() { } catch (Exception e) { if (e instanceof NullPointerException) { LOG.error("Failed to open the database " + this.toString() - + ". A probable cause is that the H2 database cannot access the file path.", e); + + ". A probable cause is that the H2 database cannot access the file path. " + + "Check if you have two instances running on the same database.", e); } else { LOG.error("Failed to open the database " + this.toString() + " due to: ", e); } diff --git a/modDbImpl/src/org/aion/db/impl/leveldb/LevelDB.java b/modDbImpl/src/org/aion/db/impl/leveldb/LevelDB.java index db7b188433..31454d14a1 100644 --- a/modDbImpl/src/org/aion/db/impl/leveldb/LevelDB.java +++ b/modDbImpl/src/org/aion/db/impl/leveldb/LevelDB.java @@ -1,4 +1,4 @@ -/******************************************************************************* +/* * Copyright (c) 2017-2018 Aion foundation. * * This file is part of the aion network project. @@ -31,7 +31,7 @@ * Samuel Neves through the BLAKE2 implementation. * Zcash project team. * Bitcoinj team. - ******************************************************************************/ + */ package org.aion.db.impl.leveldb; import org.aion.base.util.ByteArrayWrapper; @@ -142,7 +142,14 @@ public boolean open() { try { db = JniDBFactory.factory.open(f, options); } catch (Exception e1) { - LOG.error("Failed to open the database " + this.toString() + " due to: ", e1); + if (e1.getMessage().contains("lock")) { + LOG.error("Failed to open the database " + this.toString() + + "\nCheck if you have two instances running on the same database." + + "\nFailure due to: ", e1); + } else { + LOG.error("Failed to open the database " + this.toString() + " due to: ", e1); + } + if (e1.getMessage() != null && e1.getMessage().contains("No space left on device")) { LOG.error("Shutdown due to lack of disk space."); System.exit(0); diff --git a/modDbImpl/src/org/aion/db/impl/rocksdb/RocksDBWrapper.java b/modDbImpl/src/org/aion/db/impl/rocksdb/RocksDBWrapper.java index 639bd798f3..48b447da40 100644 --- a/modDbImpl/src/org/aion/db/impl/rocksdb/RocksDBWrapper.java +++ b/modDbImpl/src/org/aion/db/impl/rocksdb/RocksDBWrapper.java @@ -1,3 +1,25 @@ +/* + * Copyright (c) 2017-2018 Aion foundation. + * + * This file is part of the aion network project. + * + * The aion network project is free software: you can redistribute it + * and/or modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation, either version 3 of + * the License, or any later version. + * + * The aion network project is distributed in the hope that it will + * be useful, but WITHOUT ANY WARRANTY; without even the implied + * warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + * See the GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with the aion network project source files. + * If not, see . + * + * Contributors: + * Aion foundation. + */ package org.aion.db.impl.rocksdb; import org.aion.base.util.ByteArrayWrapper; @@ -90,7 +112,13 @@ public boolean open() { try { db = RocksDB.open(options, f.getAbsolutePath()); } catch (RocksDBException e) { - LOG.error("Failed to open the database " + this.toString() + " due to: ", e); + if (e.getMessage().contains("lock")) { + LOG.error("Failed to open the database " + this.toString() + + "\nCheck if you have two instances running on the same database." + + "\nFailure due to: ", e); + } else { + LOG.error("Failed to open the database " + this.toString() + " due to: ", e); + } // close the connection and cleanup if needed close(); diff --git a/modEvtMgr/build.xml b/modEvtMgr/build.xml index ce16f070c9..e2fbe52e2a 100644 --- a/modEvtMgr/build.xml +++ b/modEvtMgr/build.xml @@ -17,7 +17,7 @@ - + diff --git a/modEvtMgrImpl/build.xml b/modEvtMgrImpl/build.xml index b587a7481a..ec2d130413 100644 --- a/modEvtMgrImpl/build.xml +++ b/modEvtMgrImpl/build.xml @@ -18,7 +18,7 @@ - + diff --git a/modLogger/build.xml b/modLogger/build.xml index c38ea44b37..49406f2a1e 100644 --- a/modLogger/build.xml +++ b/modLogger/build.xml @@ -22,7 +22,7 @@ debug="${compile.debug}" debuglevel="source,lines,vars" includeantruntime="false" - release="9" + release="10" srcdir="./" destdir="${dir.dest}" includes="src/**/*.java,module-info.java" diff --git a/modLogger/src/org/aion/log/AionLoggerFactory.java b/modLogger/src/org/aion/log/AionLoggerFactory.java index 5600bdbf29..e5cf758e91 100644 --- a/modLogger/src/org/aion/log/AionLoggerFactory.java +++ b/modLogger/src/org/aion/log/AionLoggerFactory.java @@ -1,4 +1,4 @@ -/******************************************************************************* +/* * Copyright (c) 2017-2018 Aion foundation. * * This file is part of the aion network project. @@ -19,49 +19,53 @@ * * Contributors: * Aion foundation. - * - ******************************************************************************/ - + */ package org.aion.log; +import ch.qos.logback.classic.Level; +import ch.qos.logback.classic.LoggerContext; +import ch.qos.logback.classic.encoder.PatternLayoutEncoder; +import ch.qos.logback.classic.spi.ILoggingEvent; +import ch.qos.logback.core.ConsoleAppender; +import ch.qos.logback.core.rolling.RollingFileAppender; +import ch.qos.logback.core.rolling.SizeAndTimeBasedRollingPolicy; +import ch.qos.logback.core.rolling.SizeBasedTriggeringPolicy; +import ch.qos.logback.core.rolling.helper.FileNamePattern; +import ch.qos.logback.core.util.FileSize; import java.util.HashMap; import java.util.Iterator; import java.util.Map; import java.util.Map.Entry; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; - -import ch.qos.logback.classic.encoder.PatternLayoutEncoder; -import ch.qos.logback.classic.spi.ILoggingEvent; -import ch.qos.logback.core.ConsoleAppender; -import ch.qos.logback.core.FileAppender; -import ch.qos.logback.core.rolling.*; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import ch.qos.logback.classic.Level; -import ch.qos.logback.classic.LoggerContext; - /** * Used to override SimpleLogger current log level - * - * final public int TRACE_INT = 00; final public int DEBUG_INT = 10; - * finalConcurrentHashMap public int INFO_INT = 20; final public int WARN_INT = - * 30; final public int ERROR_INT = 40; - * + * + * final public int TRACE_INT = 00; final public int DEBUG_INT = 10; finalConcurrentHashMap + * public int INFO_INT = 20; final public int WARN_INT = 30; final public int ERROR_INT = 40; + * * Default set to 50 which ignore output */ - public class AionLoggerFactory { /** - * Due to Cfg is abstract, use this static atribute to hold muti-chains - * config attribute List, which is chain neural. + * Due to Cfg is abstract, use this static attribute to hold muti-chains config attribute + * List, which is chain neural. */ private static Map logModules; + private static LoggerContext loggerContext; private static ConsoleAppender appender = new ConsoleAppender<>(); - private final static PatternLayoutEncoder encoder = new PatternLayoutEncoder(); + private static final PatternLayoutEncoder encoder = new PatternLayoutEncoder(); + + /** Static declaration of logFile */ + private static boolean logFile; + private static String logPath; + private static RollingFileAppender fileAppender; + static { logModules = new HashMap<>(); String level = LogLevels.INFO.name(); @@ -71,11 +75,66 @@ public class AionLoggerFactory { } public static void init(final Map _logModules) { + init(_logModules, false, "log"); + } + + public static void init(final Map _logModules, boolean _logToFile, String _logToPath) { logModules = _logModules; + logFile = _logToFile; + logPath = _logToPath; loggerContext = (LoggerContext) LoggerFactory.getILoggerFactory(); + /* Toggles file appending configurations */ + if (logFile) { + + /* Initialize Rolling-File-Appender */ + String fileName = logPath + "/aionCurrentLog.dat"; + fileAppender = new RollingFileAppender(); + fileAppender.setContext(loggerContext); + fileAppender.setName("aionlogger"); + fileAppender.setFile(fileName); + + /* Initialize Triggering-Policy (CONDITION) */ + SizeBasedTriggeringPolicy tp = new SizeBasedTriggeringPolicy(); + tp.setContext(loggerContext); + tp.start(); + + /* Initialize Rolling-Policy (BEHAVIOUR) */ + SizeAndTimeBasedRollingPolicy rp = new SizeAndTimeBasedRollingPolicy(); + rp.setContext(loggerContext); + + /* + * To modify period of each rollover; + * https://logback.qos.ch/manual/appenders.html#TimeBasedRollingPolicy + * (Currently set to PER DAY) + */ + FileNamePattern fnp = + new FileNamePattern( + logPath + "/%d{yyyy/MM, aux}/aion.%d{yyyy-MM-dd}.%i.log", loggerContext); + rp.setFileNamePattern(fnp.getPattern()); + + /* + * To modify size of each rollover file; + * https://logback.qos.ch/manual/appenders.html#SizeAndTimeBasedRollingPolicy + * (Currently set to 100MB) + */ + rp.setMaxFileSize(new FileSize(100 * 1000 * 1000)); + rp.setParent(fileAppender); + rp.start(); + + /* Sets TRIGGER & ROLLING policy */ + fileAppender.setTriggeringPolicy(tp); + fileAppender.setRollingPolicy(rp); + + /* Set fileAppender configurations */ + fileAppender.setContext(loggerContext); + fileAppender.setEncoder(encoder); + fileAppender.setAppend(true); + fileAppender.start(); + } + encoder.setContext(loggerContext); encoder.setPattern("%date{yy-MM-dd HH:mm:ss.SSS} %-5level %-4c [%thread]: %message%n"); encoder.start(); @@ -88,7 +147,8 @@ public static void init(final Map _logModules) { rootlogger.detachAndStopAllAppenders(); } - private static ConcurrentMap loggerMap = new ConcurrentHashMap(); + private static ConcurrentMap loggerMap = + new ConcurrentHashMap(); public static Logger getLogger(String label) { @@ -101,12 +161,18 @@ private static Logger newLogger(String label) { if (loggerContext == null) { // System.out.println("If you see this line, meaning you are under // the unit test!!! If you are not. should report an issue."); + // init(new HashMap<>(), false); init(new HashMap<>()); } ch.qos.logback.classic.Logger newlogger = loggerContext.getLogger(label); newlogger.addAppender(appender); + /* Toggles file appending */ + if (logFile) { + newlogger.addAppender(fileAppender); + } + boolean flag = false; Iterator> it = logModules.entrySet().iterator(); while (it.hasNext()) { @@ -114,27 +180,26 @@ private static Logger newLogger(String label) { if (logModule.getKey().equals(label)) { LogLevels logLevel = LogLevels.valueOf(logModule.getValue()); switch (logLevel) { - case TRACE: - newlogger.setLevel(Level.TRACE); - flag = true; - break; - case ERROR: - newlogger.setLevel(Level.ERROR); - flag = true; - break; - case INFO: - newlogger.setLevel(Level.INFO); - flag = true; - break; - case DEBUG: - newlogger.setLevel(Level.DEBUG); - flag = true; - break; + case TRACE: + newlogger.setLevel(Level.TRACE); + flag = true; + break; + case ERROR: + newlogger.setLevel(Level.ERROR); + flag = true; + break; + case INFO: + newlogger.setLevel(Level.INFO); + flag = true; + break; + case DEBUG: + newlogger.setLevel(Level.DEBUG); + flag = true; + break; } } - if (flag) - break; + if (flag) break; } if (!flag) { diff --git a/modLogger/src/org/aion/log/LogEnum.java b/modLogger/src/org/aion/log/LogEnum.java index 0013fefa0d..ba19b7361f 100644 --- a/modLogger/src/org/aion/log/LogEnum.java +++ b/modLogger/src/org/aion/log/LogEnum.java @@ -28,7 +28,7 @@ * */ public enum LogEnum { - GEN, CONS, SYNC, API, VM, NET, DB, EVTMGR, TXPOOL, TX; + GEN, CONS, SYNC, API, VM, NET, DB, EVTMGR, TXPOOL, TX, P2P; public static boolean contains(String _module) { for (LogEnum module : values()) diff --git a/modMcf/build.xml b/modMcf/build.xml index 1aa1732eb7..c4abab99bd 100644 --- a/modMcf/build.xml +++ b/modMcf/build.xml @@ -59,7 +59,7 @@ - + @@ -69,7 +69,7 @@ - + @@ -126,7 +126,7 @@ - + diff --git a/modMcf/src/org/aion/mcf/account/Keystore.java b/modMcf/src/org/aion/mcf/account/Keystore.java index f4b137a2fb..3627e238fd 100644 --- a/modMcf/src/org/aion/mcf/account/Keystore.java +++ b/modMcf/src/org/aion/mcf/account/Keystore.java @@ -64,13 +64,22 @@ public class Keystore { private static final Logger LOG = AionLoggerFactory.getLogger(LogEnum.API.name()); - private static final String KEYSTORE_PATH = System.getProperty("user.dir") + "/keystore"; - private static final Path PATH = Paths.get(KEYSTORE_PATH); private static final FileDateTimeComparator COMPARE = new FileDateTimeComparator(); private static final Pattern HEX_64 = Pattern.compile("^[\\p{XDigit}]{64}$"); private static final String ADDR_PREFIX = "0x"; private static final String AION_PREFIX = "a0"; private static final int IMPORT_LIMIT = 100; + private static final String KEYSTORE_PATH; + private static final Path PATH; + + static { + String storageDir = System.getProperty("local.storage.dir"); + if (storageDir == null || storageDir.equalsIgnoreCase("")) { + storageDir = System.getProperty("user.dir"); + } + KEYSTORE_PATH = storageDir + "/keystore"; + PATH = Paths.get(KEYSTORE_PATH); + } private static List getFiles() { File[] files = PATH.toFile().listFiles(); diff --git a/modMcf/src/org/aion/mcf/config/CfgDb.java b/modMcf/src/org/aion/mcf/config/CfgDb.java index 3917623956..ae79157455 100644 --- a/modMcf/src/org/aion/mcf/config/CfgDb.java +++ b/modMcf/src/org/aion/mcf/config/CfgDb.java @@ -1,4 +1,4 @@ -/******************************************************************************* +/* ****************************************************************************** * Copyright (c) 2017-2018 Aion foundation. * * This file is part of the aion network project. @@ -22,25 +22,22 @@ ******************************************************************************/ package org.aion.mcf.config; -import org.aion.base.util.Utils; -import org.aion.db.impl.DBVendor; +import static org.aion.db.impl.DatabaseFactory.Props; -import javax.xml.stream.XMLOutputFactory; -import javax.xml.stream.XMLStreamException; -import javax.xml.stream.XMLStreamReader; -import javax.xml.stream.XMLStreamWriter; import java.io.IOException; import java.io.StringWriter; import java.io.Writer; import java.util.HashMap; import java.util.Map; import java.util.Properties; +import javax.xml.stream.XMLOutputFactory; +import javax.xml.stream.XMLStreamException; +import javax.xml.stream.XMLStreamReader; +import javax.xml.stream.XMLStreamWriter; +import org.aion.base.util.Utils; +import org.aion.db.impl.DBVendor; -import static org.aion.db.impl.DatabaseFactory.Props; - -/** - * @author chris - */ +/** @author chris */ public class CfgDb { public static class Names { @@ -53,6 +50,7 @@ public static class Names { public static final String STORAGE = "storage"; public static final String STATE = "state"; + public static final String STATE_ARCHIVE = "stateArchive"; public static final String TRANSACTION = "transaction"; public static final String TX_CACHE = "pendingtxCache"; @@ -63,6 +61,8 @@ public static class Names { private String vendor; private boolean compression; private boolean check_integrity; + private CfgPrune prune; + private PruneOption prune_option; /** * Enabling expert mode allows more detailed database configurations. @@ -79,6 +79,8 @@ public CfgDb() { this.vendor = DBVendor.LEVELDB.toValue(); this.compression = false; this.check_integrity = true; + this.prune = new CfgPrune(false); + this.prune_option = PruneOption.FULL; if (expert) { this.specificConfig = new HashMap<>(); @@ -100,77 +102,89 @@ public void fromXML(final XMLStreamReader sr) throws XMLStreamException { case "check_integrity": this.check_integrity = Boolean.parseBoolean(Cfg.readValue(sr)); break; - // parameter considered only when expert==false + case "state-storage": + setPrune(Cfg.readValue(sr)); + break; + // parameter considered only when expert==false case "vendor": this.vendor = Cfg.readValue(sr); break; - // parameter considered only when expert==false + // parameter considered only when expert==false case Props.ENABLE_DB_COMPRESSION: this.compression = Boolean.parseBoolean(Cfg.readValue(sr)); break; - // parameter considered only when expert==true - case Names.DEFAULT: { - CfgDbDetails dbConfig = this.specificConfig.get(Names.DEFAULT); - dbConfig.fromXML(sr); - this.specificConfig.put(Names.DEFAULT, dbConfig); - break; - } - // parameter considered only when expert==true - case Names.BLOCK: { - CfgDbDetails dbConfig = new CfgDbDetails(); - dbConfig.fromXML(sr); - this.specificConfig.put(Names.BLOCK, dbConfig); - break; - } - // parameter considered only when expert==true - case Names.INDEX: { - CfgDbDetails dbConfig = new CfgDbDetails(); - dbConfig.fromXML(sr); - this.specificConfig.put(Names.INDEX, dbConfig); - break; - } - // parameter considered only when expert==true - case Names.DETAILS: { - CfgDbDetails dbConfig = new CfgDbDetails(); - dbConfig.fromXML(sr); - this.specificConfig.put(Names.DETAILS, dbConfig); - break; - } - // parameter considered only when expert==true - case Names.STORAGE: { - CfgDbDetails dbConfig = new CfgDbDetails(); - dbConfig.fromXML(sr); - this.specificConfig.put(Names.STORAGE, dbConfig); - break; - } - // parameter considered only when expert==true - case Names.STATE: { - CfgDbDetails dbConfig = new CfgDbDetails(); - dbConfig.fromXML(sr); - this.specificConfig.put(Names.STATE, dbConfig); - break; - } - // parameter considered only when expert==true - case Names.TRANSACTION: { - CfgDbDetails dbConfig = new CfgDbDetails(); - dbConfig.fromXML(sr); - this.specificConfig.put(Names.TRANSACTION, dbConfig); - break; - } - // parameter considered only when expert==true - case Names.TX_POOL: { - CfgDbDetails dbConfig = new CfgDbDetails(); - dbConfig.fromXML(sr); - this.specificConfig.put(Names.TX_POOL, dbConfig); - break; - } - // parameter considered only when expert==true - case Names.TX_CACHE: { - CfgDbDetails dbConfig = new CfgDbDetails(); - dbConfig.fromXML(sr); - this.specificConfig.put(Names.TX_CACHE, dbConfig); - break; - } + // parameter considered only when expert==true + case Names.DEFAULT: + { + CfgDbDetails dbConfig = this.specificConfig.get(Names.DEFAULT); + dbConfig.fromXML(sr); + this.specificConfig.put(Names.DEFAULT, dbConfig); + break; + } + // parameter considered only when expert==true + case Names.BLOCK: + { + CfgDbDetails dbConfig = new CfgDbDetails(); + dbConfig.fromXML(sr); + this.specificConfig.put(Names.BLOCK, dbConfig); + break; + } + // parameter considered only when expert==true + case Names.INDEX: + { + CfgDbDetails dbConfig = new CfgDbDetails(); + dbConfig.fromXML(sr); + this.specificConfig.put(Names.INDEX, dbConfig); + break; + } + // parameter considered only when expert==true + case Names.DETAILS: + { + CfgDbDetails dbConfig = new CfgDbDetails(); + dbConfig.fromXML(sr); + this.specificConfig.put(Names.DETAILS, dbConfig); + break; + } + // parameter considered only when expert==true + case Names.STORAGE: + { + CfgDbDetails dbConfig = new CfgDbDetails(); + dbConfig.fromXML(sr); + this.specificConfig.put(Names.STORAGE, dbConfig); + break; + } + // parameter considered only when expert==true + case Names.STATE: + { + CfgDbDetails dbConfig = new CfgDbDetails(); + dbConfig.fromXML(sr); + this.specificConfig.put(Names.STATE, dbConfig); + break; + } + // parameter considered only when expert==true + case Names.TRANSACTION: + { + CfgDbDetails dbConfig = new CfgDbDetails(); + dbConfig.fromXML(sr); + this.specificConfig.put(Names.TRANSACTION, dbConfig); + break; + } + // parameter considered only when expert==true + case Names.TX_POOL: + { + CfgDbDetails dbConfig = new CfgDbDetails(); + dbConfig.fromXML(sr); + this.specificConfig.put(Names.TX_POOL, dbConfig); + break; + } + // parameter considered only when expert==true + case Names.TX_CACHE: + { + CfgDbDetails dbConfig = new CfgDbDetails(); + dbConfig.fromXML(sr); + this.specificConfig.put(Names.TX_CACHE, dbConfig); + break; + } default: Cfg.skipElement(sr); break; @@ -200,18 +214,36 @@ public String toXML() { xmlWriter.writeEndElement(); xmlWriter.writeCharacters("\r\n\t\t"); - xmlWriter.writeComment("Boolean value. Enable/disable database integrity check run at startup."); + xmlWriter.writeComment( + "Boolean value. Enable/disable database integrity check run at startup."); xmlWriter.writeCharacters("\r\n\t\t"); xmlWriter.writeStartElement("check_integrity"); xmlWriter.writeCharacters(String.valueOf(this.check_integrity)); xmlWriter.writeEndElement(); + xmlWriter.writeCharacters("\r\n\t\t"); + xmlWriter.writeComment( + "Data pruning behavior for the state database. Options: FULL, TOP, SPREAD."); + xmlWriter.writeCharacters("\r\n\t\t"); + xmlWriter.writeComment("FULL: the state is not pruned"); + xmlWriter.writeCharacters("\r\n\t\t"); + xmlWriter.writeComment( + "TOP: the state is kept only for the top K blocks; limits sync to branching only within the stored blocks"); + xmlWriter.writeCharacters("\r\n\t\t"); + xmlWriter.writeComment( + "SPREAD: the state is kept for the top K blocks and at regular block intervals"); + xmlWriter.writeCharacters("\r\n\t\t"); + xmlWriter.writeStartElement("state-storage"); + xmlWriter.writeCharacters(this.prune_option.toString()); + xmlWriter.writeEndElement(); + if (!expert) { xmlWriter.writeCharacters("\r\n\t\t"); xmlWriter.writeComment( "Database implementation used to store data; supported options: leveldb, h2, rocksdb."); xmlWriter.writeCharacters("\r\n\t\t"); - xmlWriter.writeComment("Caution: changing implementation requires re-syncing from genesis!"); + xmlWriter.writeComment( + "Caution: changing implementation requires re-syncing from genesis!"); xmlWriter.writeCharacters("\r\n\t\t"); xmlWriter.writeStartElement("vendor"); xmlWriter.writeCharacters(this.vendor); @@ -248,6 +280,73 @@ public String getPath() { return this.path; } + public CfgPrune getPrune() { + return this.prune; + } + + /** + * Number of topmost blocks present in the database in TOP pruning mode. Information about these + * blocks is also kept in memory for later pruning. + */ + public static final int TOP_PRUNE_BLOCK_COUNT = 256; + /** + * Number of topmost blocks present in the database in SPREAD pruning mode. Information about + * these blocks is also kept in memory for later pruning. + */ + public static final int SPREAD_PRUNE_BLOCK_COUNT = 128; + /** At what frequency block states are being archived. */ + public static final int SPREAD_PRUNE_ARCHIVE_RATE = 10000; + + public enum PruneOption { + FULL, + TOP, + SPREAD; + + @Override + public String toString() { + return this.name(); + } + + public static PruneOption fromValue(String value) { + value = value.toUpperCase(); + + if (value != null) { + for (PruneOption color : values()) { + if (color.toString().equals(value)) { + return color; + } + } + } + + // return default value + return getDefault(); + } + + public static PruneOption getDefault() { + return FULL; + } + } + + public void setPrune(String _prune_option) { + this.prune_option = PruneOption.fromValue(_prune_option); + + switch (prune_option) { + case TOP: + // journal prune only + this.prune = new CfgPrune(TOP_PRUNE_BLOCK_COUNT); + break; + case SPREAD: + // journal prune with archived states + this.prune = new CfgPrune(SPREAD_PRUNE_BLOCK_COUNT, SPREAD_PRUNE_ARCHIVE_RATE); + break; + case FULL: + default: + // the default is no pruning + this.prune = new CfgPrune(false); + break; + } + } + public Map asProperties() { Map propSet = new HashMap<>(); @@ -291,4 +390,4 @@ public void setHeapCacheEnabled(boolean value) { } } } -} \ No newline at end of file +} diff --git a/modMcf/src/org/aion/mcf/config/CfgLog.java b/modMcf/src/org/aion/mcf/config/CfgLog.java index 0af4527b04..13419588bd 100644 --- a/modMcf/src/org/aion/mcf/config/CfgLog.java +++ b/modMcf/src/org/aion/mcf/config/CfgLog.java @@ -1,4 +1,4 @@ -/******************************************************************************* +/* * Copyright (c) 2017-2018 Aion foundation. * * This file is part of the aion network project. @@ -19,29 +19,26 @@ * * Contributors: * Aion foundation. - * - ******************************************************************************/ + */ package org.aion.mcf.config; import java.io.IOException; import java.io.StringWriter; import java.io.Writer; import java.util.*; - -import org.aion.log.LogEnum; -import org.aion.log.LogLevels; - import javax.xml.stream.XMLOutputFactory; import javax.xml.stream.XMLStreamException; import javax.xml.stream.XMLStreamReader; import javax.xml.stream.XMLStreamWriter; +import org.aion.log.LogEnum; +import org.aion.log.LogLevels; -/** - * @author chris - */ +/** @author chris */ public class CfgLog { private Map modules; + boolean logFile; + String logPath; public CfgLog() { modules = new HashMap<>(); @@ -51,8 +48,11 @@ public CfgLog() { modules.put(LogEnum.DB.name(), LogLevels.ERROR.name()); modules.put(LogEnum.SYNC.name(), LogLevels.INFO.name()); modules.put(LogEnum.API.name(), LogLevels.INFO.name()); + modules.put(LogEnum.P2P.name(), LogLevels.INFO.name()); modules.put(LogEnum.TX.name(), LogLevels.ERROR.name()); modules.put(LogEnum.TXPOOL.name(), LogLevels.ERROR.name()); + this.logFile = false; + this.logPath = "log"; } public void fromXML(final XMLStreamReader sr) throws XMLStreamException { @@ -61,16 +61,30 @@ public void fromXML(final XMLStreamReader sr) throws XMLStreamException { while (sr.hasNext()) { int eventType = sr.next(); switch (eventType) { - case XMLStreamReader.START_ELEMENT: - String elementName = sr.getLocalName().toUpperCase(); - if (LogEnum.contains(elementName)) - this.modules.put(elementName, Cfg.readValue(sr).toUpperCase()); - break; - case XMLStreamReader.END_ELEMENT: - break loop; - default: - //Cfg.skipElement(sr); - break; + case XMLStreamReader.START_ELEMENT: + + /* XML - Takes the input in config.xml and parse as T/F */ + String elementName = sr.getLocalName().toLowerCase(); + switch (elementName) { + case "log-file": + this.logFile = Boolean.parseBoolean(Cfg.readValue(sr)); + break; + case "log-path": + this.logPath = Cfg.readValue(sr); + break; + default: + break; + } + + elementName = sr.getLocalName().toUpperCase(); + if (LogEnum.contains(elementName)) + this.modules.put(elementName, Cfg.readValue(sr).toUpperCase()); + break; + case XMLStreamReader.END_ELEMENT: + break loop; + default: + // Cfg.skipElement(sr); + break; } } } @@ -85,6 +99,31 @@ public String toXML() { xmlWriter.writeCharacters("\r\n\t"); xmlWriter.writeStartElement("log"); xmlWriter.writeCharacters("\r\n"); + + /* + * XML - Displays tag/entry in the config.xml + * Boolean value to allow logger to be toggled ON and OFF + */ + xmlWriter.writeCharacters("\t\t"); + xmlWriter.writeComment("Enable/Disable logback service; if disabled, output will not be logged."); + xmlWriter.writeCharacters("\r\n\t\t"); + xmlWriter.writeStartElement("log-file"); + xmlWriter.writeCharacters(this.logFile + ""); + xmlWriter.writeEndElement(); + xmlWriter.writeCharacters("\r\n"); + + /* + * XML - Displays log-path in the config.xml + * String value to determine the folder path for log files + */ + xmlWriter.writeCharacters("\t\t"); + xmlWriter.writeComment("Sets the physical location on disk where log files will be stored."); + xmlWriter.writeCharacters("\r\n\t\t"); + xmlWriter.writeStartElement("log-path"); + xmlWriter.writeCharacters(this.logPath + ""); + xmlWriter.writeEndElement(); + xmlWriter.writeCharacters("\r\n"); + for (Map.Entry module : this.modules.entrySet()) { xmlWriter.writeCharacters("\t\t"); xmlWriter.writeStartElement(module.getKey().toUpperCase()); @@ -109,4 +148,18 @@ public Map getModules() { return this.modules; } -} \ No newline at end of file + /** Method checks whether logger is enabled/disabled */ + public boolean getLogFile() { + return this.logFile; + } + + /** Method returns user input folder path of logger */ + public String getLogPath() { + return this.logPath; + } + + /** Method checks folder path for illegal inputs */ + public boolean isValidPath() { + return logPath.length() > 0 && !logPath.matches(".*[-=+,.?;:'!@#$%^&*].*"); + } +} diff --git a/modMcf/src/org/aion/mcf/config/CfgNetP2p.java b/modMcf/src/org/aion/mcf/config/CfgNetP2p.java index a974d01d60..95b29d1581 100644 --- a/modMcf/src/org/aion/mcf/config/CfgNetP2p.java +++ b/modMcf/src/org/aion/mcf/config/CfgNetP2p.java @@ -38,8 +38,6 @@ public final class CfgNetP2p { this.ip = "127.0.0.1"; this.port = 30303; this.discover = false; - this.showStatus = false; - this.showLog = false; this.bootlistSyncOnly = false; this.maxTempNodes = 128; this.maxActiveNodes = 128; @@ -53,10 +51,6 @@ public final class CfgNetP2p { private boolean discover; - private boolean showStatus; - - private boolean showLog; - private boolean bootlistSyncOnly; private boolean isSyncOnlyNode; @@ -84,12 +78,6 @@ public void fromXML(final XMLStreamReader sr) throws XMLStreamException { case "discover": this.discover = Boolean.parseBoolean(Cfg.readValue(sr)); break; - case "show-status": - this.showStatus = Boolean.parseBoolean(Cfg.readValue(sr)); - break; - case "show-log": - this.showLog = Boolean.parseBoolean(Cfg.readValue(sr)); - break; case "bootlist-sync-only": this.bootlistSyncOnly = Boolean.parseBoolean(Cfg.readValue(sr)); break; @@ -141,11 +129,6 @@ String toXML() { xmlWriter.writeCharacters(this.discover + ""); xmlWriter.writeEndElement(); - xmlWriter.writeCharacters("\r\n\t\t\t"); - xmlWriter.writeStartElement("show-status"); - xmlWriter.writeCharacters(this.showStatus + ""); - xmlWriter.writeEndElement(); - xmlWriter.writeCharacters("\r\n\t\t\t"); xmlWriter.writeStartElement("max-temp-nodes"); xmlWriter.writeCharacters(this.maxTempNodes + ""); @@ -189,14 +172,6 @@ public boolean getDiscover() { return this.discover; } - public boolean getShowStatus() { - return this.showStatus; - } - - public boolean getShowLog() { - return this.showLog; - } - public boolean getBootlistSyncOnly() { return bootlistSyncOnly; } public int getMaxTempNodes() { diff --git a/modMcf/src/org/aion/mcf/config/CfgPrune.java b/modMcf/src/org/aion/mcf/config/CfgPrune.java new file mode 100644 index 0000000000..842c483fe2 --- /dev/null +++ b/modMcf/src/org/aion/mcf/config/CfgPrune.java @@ -0,0 +1,172 @@ +/* ****************************************************************************** + * Copyright (c) 2017-2018 Aion foundation. + * + * This file is part of the aion network project. + * + * The aion network project is free software: you can redistribute it + * and/or modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation, either version 3 of + * the License, or any later version. + * + * The aion network project is distributed in the hope that it will + * be useful, but WITHOUT ANY WARRANTY; without even the implied + * warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + * See the GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with the aion network project source files. + * If not, see . + * + * The aion network project leverages useful source code from other + * open source projects. We greatly appreciate the effort that was + * invested in these projects and we thank the individual contributors + * for their work. For provenance information and contributors + * please see . + * + * Contributors to the aion source files in decreasing order of code volume: + * Aion foundation. + ******************************************************************************/ +package org.aion.mcf.config; + +import javax.xml.stream.XMLStreamException; +import javax.xml.stream.XMLStreamReader; +import javax.xml.stream.XMLStreamWriter; +import org.aion.base.db.IPruneConfig; + +/** + * Configuration for data pruning behavior. + * + * @author Alexandra Roatis + */ +public class CfgPrune implements IPruneConfig { + + private boolean enabled; + private boolean archived; + private int current_count = MINIMUM_CURRENT_COUNT; + private int archive_rate = MINIMUM_ARCHIVE_RATE; + + private static final int MINIMUM_CURRENT_COUNT = 128; + private static final int MINIMUM_ARCHIVE_RATE = 1000; + + public CfgPrune(boolean _enabled) { + this.enabled = _enabled; + this.archived = _enabled; + } + + public CfgPrune(int _current_count) { + // enable journal pruning + this.enabled = true; + this.current_count = + _current_count > MINIMUM_CURRENT_COUNT ? _current_count : MINIMUM_CURRENT_COUNT; + // disable archiving + this.archived = false; + } + + public CfgPrune(int _current_count, int _archive_rate) { + // enable journal pruning + this.enabled = true; + this.current_count = + _current_count > MINIMUM_CURRENT_COUNT ? _current_count : MINIMUM_CURRENT_COUNT; + // enable archiving + this.archived = true; + this.archive_rate = + _archive_rate > MINIMUM_ARCHIVE_RATE ? _archive_rate : MINIMUM_ARCHIVE_RATE; + } + + public void fromXML(final XMLStreamReader sr) throws XMLStreamException { + loop: + while (sr.hasNext()) { + int eventType = sr.next(); + switch (eventType) { + case XMLStreamReader.START_ELEMENT: + String elementName = sr.getLocalName().toLowerCase(); + switch (elementName) { + case "enabled": + this.enabled = Boolean.parseBoolean(Cfg.readValue(sr)); + break; + case "archived": + this.archived = Boolean.parseBoolean(Cfg.readValue(sr)); + break; + case "current_count": + this.current_count = Integer.parseInt(Cfg.readValue(sr)); + // must be at least MINIMUM_CURRENT_COUNT + if (this.current_count < MINIMUM_CURRENT_COUNT) { + this.current_count = MINIMUM_CURRENT_COUNT; + } + break; + case "archive_rate": + this.archive_rate = Integer.parseInt(Cfg.readValue(sr)); + // must be at least MINIMUM_ARCHIVE_RATE + if (this.archive_rate < MINIMUM_ARCHIVE_RATE) { + this.archive_rate = MINIMUM_ARCHIVE_RATE; + } + break; + default: + Cfg.skipElement(sr); + break; + } + break; + case XMLStreamReader.END_ELEMENT: + break loop; + } + } + } + + public void toXML(XMLStreamWriter xmlWriter) throws XMLStreamException { + xmlWriter.writeCharacters("\r\n\t\t"); + xmlWriter.writeStartElement("prune"); + + xmlWriter.writeCharacters("\r\n\t\t\t"); + xmlWriter.writeComment("Boolean value. Enable/disable database pruning."); + xmlWriter.writeCharacters("\r\n\t\t\t"); + xmlWriter.writeStartElement("enabled"); + xmlWriter.writeCharacters(String.valueOf(this.enabled)); + xmlWriter.writeEndElement(); + + xmlWriter.writeCharacters("\r\n\t\t\t"); + xmlWriter.writeComment("Boolean value. Enable/disable database archiving."); + xmlWriter.writeCharacters("\r\n\t\t\t"); + xmlWriter.writeStartElement("archived"); + xmlWriter.writeCharacters(String.valueOf(this.archived)); + xmlWriter.writeEndElement(); + + xmlWriter.writeCharacters("\r\n\t\t\t"); + xmlWriter.writeComment( + "Integer value with minimum set to 128. Only blocks older than best block level minus this number are candidates for pruning."); + xmlWriter.writeCharacters("\r\n\t\t\t"); + xmlWriter.writeStartElement("current_count"); + xmlWriter.writeCharacters(String.valueOf(this.current_count)); + xmlWriter.writeEndElement(); + + xmlWriter.writeCharacters("\r\n\t\t\t"); + xmlWriter.writeComment( + "Integer value with minimum set to 1000. States for blocks that are exact multiples of this number will not be pruned."); + xmlWriter.writeCharacters("\r\n\t\t\t"); + xmlWriter.writeStartElement("archive_rate"); + xmlWriter.writeCharacters(String.valueOf(this.archive_rate)); + xmlWriter.writeEndElement(); + + xmlWriter.writeCharacters("\r\n\t\t"); + xmlWriter.writeEndElement(); + } + + @Override + public boolean isEnabled() { + return enabled; + } + + @Override + public boolean isArchived() { + return archived; + } + + @Override + public int getCurrentCount() { + return current_count; + } + + @Override + public int getArchiveRate() { + return archive_rate; + } +} diff --git a/modMcf/src/org/aion/mcf/db/AbstractRepository.java b/modMcf/src/org/aion/mcf/db/AbstractRepository.java index 9125791d61..60efd98d76 100644 --- a/modMcf/src/org/aion/mcf/db/AbstractRepository.java +++ b/modMcf/src/org/aion/mcf/db/AbstractRepository.java @@ -1,4 +1,4 @@ -/******************************************************************************* +/* ****************************************************************************** * * Copyright (c) 2017, 2018 Aion foundation. * @@ -18,9 +18,17 @@ * Contributors: * Aion foundation. *******************************************************************************/ - package org.aion.mcf.db; +import static org.aion.db.impl.DatabaseFactory.Props; + +import java.io.File; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Objects; +import java.util.Properties; +import java.util.concurrent.locks.ReadWriteLock; +import java.util.concurrent.locks.ReentrantReadWriteLock; import org.aion.base.db.IByteArrayKeyValueDatabase; import org.aion.base.db.IRepository; import org.aion.base.db.IRepositoryConfig; @@ -32,28 +40,20 @@ import org.aion.mcf.config.CfgDb; import org.aion.mcf.core.AccountState; import org.aion.mcf.db.exception.InvalidFilePathException; +import org.aion.mcf.ds.ArchivedDataSource; +import org.aion.mcf.trie.JournalPruneDataSource; import org.aion.mcf.trie.Trie; import org.aion.mcf.types.AbstractBlock; import org.aion.mcf.vm.types.DataWord; import org.slf4j.Logger; -import java.io.File; -import java.util.ArrayList; -import java.util.Collection; -import java.util.Objects; -import java.util.Properties; -import java.util.concurrent.locks.ReadWriteLock; -import java.util.concurrent.locks.ReentrantReadWriteLock; - -import static org.aion.db.impl.DatabaseFactory.Props; - -//import org.aion.dbmgr.exception.DriverManagerNoSuitableDriverRegisteredException; -// import org.aion.mcf.trie.JournalPruneDataSource; +// import org.aion.dbmgr.exception.DriverManagerNoSuitableDriverRegisteredException; -/** - * Abstract Repository class. - */ -public abstract class AbstractRepository, BH extends IBlockHeader, BSB extends IBlockStoreBase> +/** Abstract Repository class. */ +public abstract class AbstractRepository< + BLK extends AbstractBlock, + BH extends IBlockHeader, + BSB extends IBlockStoreBase> implements IRepository { // Logger @@ -63,14 +63,15 @@ public abstract class AbstractRepository databaseGroup; - // protected JournalPruneDataSource stateDSPrune; + protected ArchivedDataSource stateWithArchive; + protected JournalPruneDataSource stateDSPrune; protected DetailsDataStore detailsDS; // Read Write Lock @@ -103,6 +106,7 @@ public abstract class AbstractRepository vendorListString = new ArrayList<>(); -// for (String v : this.cfg.getVendorList()) { -// vendorListString.add("\"" + v + "\""); -// } -// throw new DriverManagerNoSuitableDriverRegisteredException( -// "Please check the vendor name field in /config/config.xml.\n" -// + "No suitable driver found with name \"" + this.cfg.getActiveVendor() -// + "\".\nPlease select a driver from the following vendor list: " + vendorListString); -// } + // } + // + // if (!Arrays.asList(this.cfg.getVendorList()).contains(this.cfg.getActiveVendor())) + // { + // + // ArrayList vendorListString = new ArrayList<>(); + // for (String v : this.cfg.getVendorList()) { + // vendorListString.add("\"" + v + "\""); + // } + // throw new DriverManagerNoSuitableDriverRegisteredException( + // "Please check the vendor name field in /config/config.xml.\n" + // + "No suitable driver found with name \"" + + // this.cfg.getActiveVendor() + // + "\".\nPlease select a driver from the following vendor list: + // " + vendorListString); + // } Properties sharedProps; @@ -171,13 +180,15 @@ protected void initializeDatabasesAndCaches() throws Exception { try { databaseGroup = new ArrayList<>(); - checkIntegrity = Boolean - .valueOf(cfg.getDatabaseConfig(CfgDb.Names.DEFAULT).getProperty(Props.CHECK_INTEGRITY)); + checkIntegrity = + Boolean.valueOf( + cfg.getDatabaseConfig(CfgDb.Names.DEFAULT) + .getProperty(Props.CHECK_INTEGRITY)); // getting state specific properties sharedProps = cfg.getDatabaseConfig(STATE_DB); - // locking enabled for state - sharedProps.setProperty(Props.ENABLE_LOCKING, "true"); + // locking enabled for state when JournalPrune not used + sharedProps.setProperty(Props.ENABLE_LOCKING, "false"); sharedProps.setProperty(Props.DB_PATH, cfg.getDbPath()); sharedProps.setProperty(Props.DB_NAME, STATE_DB); this.stateDatabase = connectAndOpen(sharedProps); @@ -241,10 +252,39 @@ protected void initializeDatabasesAndCaches() throws Exception { // Setup the cache for transaction data source. this.detailsDS = new DetailsDataStore<>(detailsDatabase, storageDatabase, this.cfg); - // disabling use of JournalPruneDataSource until functionality properly tested - // TODO-AR: enable pruning with the JournalPruneDataSource - // stateDSPrune = new JournalPruneDataSource<>(stateDatabase); - pruneBlockCount = pruneEnabled ? this.cfg.getPrune() : -1; + + // pruning config + pruneEnabled = this.cfg.getPruneConfig().isEnabled(); + pruneBlockCount = this.cfg.getPruneConfig().getCurrentCount(); + archiveRate = this.cfg.getPruneConfig().getArchiveRate(); + + if (pruneEnabled && this.cfg.getPruneConfig().isArchived()) { + // using state config for state_archive + sharedProps = cfg.getDatabaseConfig(STATE_DB); + sharedProps.setProperty(Props.ENABLE_LOCKING, "false"); + sharedProps.setProperty(Props.DB_PATH, cfg.getDbPath()); + sharedProps.setProperty(Props.DB_NAME, STATE_ARCHIVE_DB); + this.stateArchiveDatabase = connectAndOpen(sharedProps); + databaseGroup.add(stateArchiveDatabase); + + stateWithArchive = new ArchivedDataSource(stateDatabase, stateArchiveDatabase); + stateDSPrune = new JournalPruneDataSource(stateWithArchive); + + LOGGEN.info( + "Pruning and archiving ENABLED. Top block count set to {} and archive rate set to {}.", + pruneBlockCount, + archiveRate); + } else { + stateArchiveDatabase = null; + stateWithArchive = null; + stateDSPrune = new JournalPruneDataSource(stateDatabase); + + if (pruneEnabled) { + LOGGEN.info("Pruning ENABLED. Top block count set to {}.", pruneBlockCount); + } + } + + stateDSPrune.setPruneEnabled(pruneEnabled); } catch (Exception e) { // Setting up databases and caches went wrong. throw e; } @@ -269,16 +309,18 @@ private IByteArrayKeyValueDatabase connectAndOpen(Properties info) { // check object status if (db == null) { - LOG.error("Database <{}> connection could not be established for <{}>.", - info.getProperty(Props.DB_TYPE), - info.getProperty(Props.DB_NAME)); + LOG.error( + "Database <{}> connection could not be established for <{}>.", + info.getProperty(Props.DB_TYPE), + info.getProperty(Props.DB_NAME)); } // check persistence status if (!db.isCreatedOnDisk()) { - LOG.error("Database <{}> cannot be saved to disk for <{}>.", - info.getProperty(Props.DB_TYPE), - info.getProperty(Props.DB_NAME)); + LOG.error( + "Database <{}> cannot be saved to disk for <{}>.", + info.getProperty(Props.DB_TYPE), + info.getProperty(Props.DB_NAME)); } return db; diff --git a/modMcf/src/org/aion/mcf/db/DetailsDataStore.java b/modMcf/src/org/aion/mcf/db/DetailsDataStore.java index dd01eee23c..280cd38787 100644 --- a/modMcf/src/org/aion/mcf/db/DetailsDataStore.java +++ b/modMcf/src/org/aion/mcf/db/DetailsDataStore.java @@ -1,25 +1,42 @@ -/******************************************************************************* +/* ****************************************************************************** + * Copyright (c) 2017-2018 Aion foundation. * - * Copyright (c) 2017, 2018 Aion foundation. + * This file is part of the aion network project. * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. + * The aion network project is free software: you can redistribute it + * and/or modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation, either version 3 of + * the License, or any later version. * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. + * The aion network project is distributed in the hope that it will + * be useful, but WITHOUT ANY WARRANTY; without even the implied + * warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + * See the GNU General Public License for more details. * * You should have received a copy of the GNU General Public License - * along with this program. If not, see + * along with the aion network project source files. + * If not, see . * - * Contributors: + * The aion network project leverages useful source code from other + * open source projects. We greatly appreciate the effort that was + * invested in these projects and we thank the individual contributors + * for their work. For provenance information and contributors + * please see . + * + * Contributors to the aion source files in decreasing order of code volume: * Aion foundation. - *******************************************************************************/ + * team through the ethereumJ library. + * Ether.Camp Inc. (US) team through Ethereum Harmony. + * John Tromp through the Equihash solver. + * Samuel Neves through the BLAKE2 implementation. + * Zcash project team. + * Bitcoinj team. + ******************************************************************************/ package org.aion.mcf.db; +import static org.aion.base.util.ByteArrayWrapper.wrap; + +import java.util.*; import org.aion.base.db.IByteArrayKeyValueDatabase; import org.aion.base.db.IContractDetails; import org.aion.base.db.IRepositoryConfig; @@ -27,48 +44,42 @@ import org.aion.base.type.IBlockHeader; import org.aion.base.type.ITransaction; import org.aion.base.util.ByteArrayWrapper; +import org.aion.mcf.trie.JournalPruneDataSource; import org.aion.mcf.types.AbstractBlock; import org.aion.mcf.vm.types.DataWord; -import java.util.*; - -import static org.aion.base.util.ByteArrayWrapper.wrap; - -// import org.aion.mcf.trie.JournalPruneDataSource; - -/** - * Detail data storage , - */ -public class DetailsDataStore, BH extends IBlockHeader> { +/** Detail data storage , */ +public class DetailsDataStore< + BLK extends AbstractBlock, BH extends IBlockHeader> { - // private JournalPruneDataSource storageDSPrune; + private JournalPruneDataSource storageDSPrune; private IRepositoryConfig repoConfig; private IByteArrayKeyValueDatabase detailsSrc; private IByteArrayKeyValueDatabase storageSrc; private Set removes = new HashSet<>(); - public DetailsDataStore() { - } + public DetailsDataStore() {} - public DetailsDataStore(IByteArrayKeyValueDatabase detailsCache, IByteArrayKeyValueDatabase storageCache, + public DetailsDataStore( + IByteArrayKeyValueDatabase detailsCache, + IByteArrayKeyValueDatabase storageCache, IRepositoryConfig repoConfig) { this.repoConfig = repoConfig; withDb(detailsCache, storageCache); } - public DetailsDataStore withDb(IByteArrayKeyValueDatabase detailsSrc, - IByteArrayKeyValueDatabase storageSrc) { + public DetailsDataStore withDb( + IByteArrayKeyValueDatabase detailsSrc, IByteArrayKeyValueDatabase storageSrc) { this.detailsSrc = detailsSrc; this.storageSrc = storageSrc; - // this.storageDSPrune = new JournalPruneDataSource<>(storageSrc); + this.storageDSPrune = new JournalPruneDataSource(storageSrc); return this; } /** - * Fetches the ContractDetails from the cache, and if it doesn't exist, add - * to the remove set. + * Fetches the ContractDetails from the cache, and if it doesn't exist, add to the remove set. * * @param key * @return @@ -91,7 +102,7 @@ public synchronized IContractDetails get(byte[] key) { // Found something from cache or database, return it by decoding it. IContractDetails detailsImpl = repoConfig.contractDetailsImpl(); - detailsImpl.setDataSource(storageSrc); + detailsImpl.setDataSource(storageDSPrune); detailsImpl.decode(rawDetails.get()); // We can safely get as we checked // if it is present. @@ -111,7 +122,6 @@ public synchronized void update(Address key, IContractDetails contract // Remove from the remove set. removes.remove(wrappedKey); - } public synchronized void remove(byte[] key) { @@ -162,7 +172,7 @@ public void syncLargeStorage() { // Decode the details. IContractDetails detailsImpl = repoConfig.contractDetailsImpl(); - detailsImpl.setDataSource(storageSrc); + detailsImpl.setDataSource(storageDSPrune); detailsImpl.decode(rawDetails.get()); // We can safely get as we // checked if it is present. @@ -171,9 +181,9 @@ public void syncLargeStorage() { } } - /* public JournalPruneDataSource getStorageDSPrune() { + public JournalPruneDataSource getStorageDSPrune() { return storageDSPrune; - } */ + } public synchronized Set keys() { // TODO - @yao do we wanted a sorted set? diff --git a/modMcf/src/org/aion/mcf/ds/ArchivedDataSource.java b/modMcf/src/org/aion/mcf/ds/ArchivedDataSource.java new file mode 100644 index 0000000000..0f0f35d2fa --- /dev/null +++ b/modMcf/src/org/aion/mcf/ds/ArchivedDataSource.java @@ -0,0 +1,135 @@ +/* ****************************************************************************** + * Copyright (c) 2017-2018 Aion foundation. + * + * This file is part of the aion network project. + * + * The aion network project is free software: you can redistribute it + * and/or modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation, either version 3 of + * the License, or any later version. + * + * The aion network project is distributed in the hope that it will + * be useful, but WITHOUT ANY WARRANTY; without even the implied + * warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + * See the GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with the aion network project source files. + * If not, see . + * + * The aion network project leverages useful source code from other + * open source projects. We greatly appreciate the effort that was + * invested in these projects and we thank the individual contributors + * for their work. For provenance information and contributors + * please see . + * + * Contributors to the aion source files in decreasing order of code volume: + * Aion foundation. + ******************************************************************************/ +package org.aion.mcf.ds; + +import java.util.Collection; +import java.util.Map; +import java.util.Optional; +import java.util.Set; +import org.aion.base.db.IByteArrayKeyValueDatabase; +import org.aion.base.db.IByteArrayKeyValueStore; + +/** + * A data source with archived data that must no be deleted. + * + * @author Alexandra Roatis + */ +public class ArchivedDataSource implements IByteArrayKeyValueStore { + + IByteArrayKeyValueDatabase data, archive; + + public ArchivedDataSource(IByteArrayKeyValueDatabase _db, IByteArrayKeyValueDatabase _archive) { + this.data = _db; + this.archive = _archive; + } + + @Override + public boolean isEmpty() { + return data.isEmpty(); + } + + @Override + public Set keys() { + return data.keys(); + } + + @Override + public Optional get(byte[] key) { + return data.get(key); + } + + @Override + public void put(byte[] key, byte[] value) { + if (value != null) { + data.put(key, value); + } else { + // internal delete will check if archived + delete(key); + } + } + + @Override + public void delete(byte[] key) { + // delete key only if not archived + if (!archive.get(key).isPresent()) { + data.delete(key); + } + } + + @Override + public void putBatch(Map batch) { + for (Map.Entry entry : batch.entrySet()) { + // will check if archived + putToBatch(entry.getKey(), entry.getValue()); + } + commitBatch(); + } + + @Override + public void putToBatch(byte[] key, byte[] value) { + if (value != null) { + data.putToBatch(key, value); + } else { + // deleted key only if not archived + if (!archive.get(key).isPresent()) { + data.putToBatch(key, null); + } + } + } + + @Override + public void commitBatch() { + data.commitBatch(); + } + + @Override + public void deleteBatch(Collection keys) { + for (byte[] key : keys) { + // will check if archived + putToBatch(key, null); + } + commitBatch(); + } + + @Override + public void check() { + data.check(); + archive.check(); + } + + @Override + public void close() { + data.close(); + archive.close(); + } + + public IByteArrayKeyValueDatabase getArchiveDatabase() { + return archive; + } +} diff --git a/modMcf/src/org/aion/mcf/ds/XorDataSource.java b/modMcf/src/org/aion/mcf/ds/XorDataSource.java index e4f77cbe52..f71259481d 100644 --- a/modMcf/src/org/aion/mcf/ds/XorDataSource.java +++ b/modMcf/src/org/aion/mcf/ds/XorDataSource.java @@ -1,4 +1,4 @@ -/******************************************************************************* +/* ****************************************************************************** * Copyright (c) 2017-2018 Aion foundation. * * This file is part of the aion network project. @@ -34,13 +34,11 @@ ******************************************************************************/ package org.aion.mcf.ds; -import org.aion.base.db.IByteArrayKeyValueDatabase; +import java.util.*; import org.aion.base.db.IByteArrayKeyValueStore; import org.aion.base.util.ByteArrayWrapper; import org.aion.base.util.ByteUtil; -import java.util.*; - public class XorDataSource implements IByteArrayKeyValueStore { IByteArrayKeyValueStore source; byte[] subKey; @@ -89,8 +87,9 @@ public void putBatch(Map rows) { } public void updateBatch(Map rows, boolean erasure) { - //not supported - throw new UnsupportedOperationException("ByteArrayWrapper map not supported in XorDataSource.updateBatch yet"); + // not supported + throw new UnsupportedOperationException( + "ByteArrayWrapper map not supported in XorDataSource.updateBatch yet"); } @Override @@ -104,6 +103,11 @@ public void deleteBatch(Collection keys) { } + @Override + public void check() { + source.check(); + } + @Override public boolean isEmpty() { // TODO Auto-generated method stub diff --git a/modMcf/src/org/aion/mcf/trie/Cache.java b/modMcf/src/org/aion/mcf/trie/Cache.java index e0235d5df5..cc52f5f5fb 100644 --- a/modMcf/src/org/aion/mcf/trie/Cache.java +++ b/modMcf/src/org/aion/mcf/trie/Cache.java @@ -164,9 +164,9 @@ public synchronized void commit(boolean flushCache) { // batchMemorySize += length(key, value); } } - /* for (ByteArrayWrapper removedNode : removedNodes) { + for (ByteArrayWrapper removedNode : removedNodes) { batch.put(removedNode.getData(), null); - } */ + } this.dataSource.putBatch(batch); this.isDirty = false; diff --git a/modMcf/src/org/aion/mcf/trie/JournalPruneDataSource.java b/modMcf/src/org/aion/mcf/trie/JournalPruneDataSource.java index 753ba7614f..05aca77569 100644 --- a/modMcf/src/org/aion/mcf/trie/JournalPruneDataSource.java +++ b/modMcf/src/org/aion/mcf/trie/JournalPruneDataSource.java @@ -1,48 +1,66 @@ -/******************************************************************************* +/* ****************************************************************************** + * Copyright (c) 2017-2018 Aion foundation. * - * Copyright (c) 2017, 2018 Aion foundation. + * This file is part of the aion network project. * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. + * The aion network project is free software: you can redistribute it + * and/or modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation, either version 3 of + * the License, or any later version. * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. + * The aion network project is distributed in the hope that it will + * be useful, but WITHOUT ANY WARRANTY; without even the implied + * warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + * See the GNU General Public License for more details. * * You should have received a copy of the GNU General Public License - * along with this program. If not, see + * along with the aion network project source files. + * If not, see . * - * Contributors: + * The aion network project leverages useful source code from other + * open source projects. We greatly appreciate the effort that was + * invested in these projects and we thank the individual contributors + * for their work. For provenance information and contributors + * please see . + * + * Contributors to the aion source files in decreasing order of code volume: * Aion foundation. - *******************************************************************************/ + * team through the ethereumJ library. + * Ether.Camp Inc. (US) team through Ethereum Harmony. + * John Tromp through the Equihash solver. + * Samuel Neves through the BLAKE2 implementation. + * Zcash project team. + * Bitcoinj team. + ******************************************************************************/ package org.aion.mcf.trie; +import java.util.*; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.locks.ReadWriteLock; +import java.util.concurrent.locks.ReentrantReadWriteLock; import org.aion.base.db.IByteArrayKeyValueDatabase; import org.aion.base.db.IByteArrayKeyValueStore; -import org.aion.base.type.IBlock; -import org.aion.base.type.IBlockHeader; import org.aion.base.util.ByteArrayWrapper; - -import java.util.*; +import org.aion.log.AionLoggerFactory; +import org.aion.log.LogEnum; +import org.aion.mcf.ds.ArchivedDataSource; +import org.slf4j.Logger; /** - * The DataSource which doesn't immediately forward delete updates (unlike - * inserts) but collects them tied to the block where these changes were made - * (the changes are mapped to a block upon [storeBlockChanges] call). When the - * [prune] is called for a block the deletes for this block are submitted to the - * underlying DataSource with respect to following inserts. E.g. if the key was - * deleted at block N and then inserted at block N + 10 this delete is not - * passed. + * The DataSource which doesn't immediately forward delete updates (unlike inserts) but collects + * them tied to the block where these changes were made (the changes are mapped to a block upon + * [storeBlockChanges] call). When the [prune] is called for a block the deletes for this block are + * submitted to the underlying DataSource with respect to following inserts. E.g. if the key was + * deleted at block N and then inserted at block N + 10 this delete is not passed. */ -public class JournalPruneDataSource, BH extends IBlockHeader> - implements IByteArrayKeyValueStore { +public class JournalPruneDataSource implements IByteArrayKeyValueStore { - private class Updates { + private final ReadWriteLock lock = new ReentrantReadWriteLock(); + private static final Logger LOG = AionLoggerFactory.getLogger(LogEnum.DB.name()); - BH blockHeader; + private class Updates { + ByteArrayWrapper blockHeader; + long blockNumber; Set insertedKeys = new HashSet<>(); Set deletedKeys = new HashSet<>(); } @@ -59,84 +77,146 @@ public Ref(boolean dbRef) { public int getTotRefs() { return journalRefs + (dbRef ? 1 : 0); } + + @Override + public String toString() { + return "refs: " + String.valueOf(journalRefs) + " db: " + String.valueOf(dbRef); + } } Map refCount = new HashMap<>(); - private IByteArrayKeyValueDatabase src; + private IByteArrayKeyValueStore src; // block hash => updates private LinkedHashMap blockUpdates = new LinkedHashMap<>(); private Updates currentUpdates = new Updates(); - private boolean enabled = true; + private AtomicBoolean enabled = new AtomicBoolean(false); + private final boolean hasArchive; - public JournalPruneDataSource(IByteArrayKeyValueDatabase src) { + public JournalPruneDataSource(IByteArrayKeyValueStore src) { this.src = src; + this.hasArchive = src instanceof ArchivedDataSource; } - public void setPruneEnabled(boolean e) { - enabled = e; + public void setPruneEnabled(boolean _enabled) { + enabled.set(_enabled); } - /** - * ***** updates ****** - */ - public synchronized void put(byte[] key, byte[] value) { - ByteArrayWrapper keyW = new ByteArrayWrapper(key); + public boolean isArchiveEnabled() { + return hasArchive; + } - // Check to see the value exists. - if (value != null) { + public void put(byte[] key, byte[] value) { + checkNotNull(key); - // If it exists and pruning is enabled. - if (enabled) { - currentUpdates.insertedKeys.add(keyW); - incRef(keyW); - } + lock.writeLock().lock(); - // Insert into the database. - src.put(key, value); + try { + if (enabled.get()) { + // pruning enabled + ByteArrayWrapper keyW = ByteArrayWrapper.wrap(key); - } else { - // Value does not exist, so we delete from current updates - if (enabled) { - currentUpdates.deletedKeys.add(keyW); + // Check to see the value exists. + if (value != null) { + // If it exists and pruning is enabled. + currentUpdates.insertedKeys.add(keyW); + incRef(keyW); + + // put to source database. + src.put(key, value); + + } else { + check(); + + // Value does not exist, so we delete from current updates + currentUpdates.deletedKeys.add(keyW); + } + } else { + // pruning disabled + if (value != null) { + src.put(key, value); + } else { + check(); + } } - // TODO: Do we delete the key? + } catch (Exception e) { + if (e instanceof RuntimeException) { + throw e; + } else { + LOG.error("Could not put key-value pair due to ", e); + } + } finally { + lock.writeLock().unlock(); } } - public synchronized void delete(byte[] key) { - if (!enabled) { return; } - currentUpdates.deletedKeys.add(new ByteArrayWrapper(key)); - // delete is delayed + public void delete(byte[] key) { + checkNotNull(key); + if (!enabled.get()) { + check(); + return; + } + + lock.writeLock().lock(); + + try { + check(); + + currentUpdates.deletedKeys.add(ByteArrayWrapper.wrap(key)); + // delete is delayed + } catch (Exception e) { + if (e instanceof RuntimeException) { + throw e; + } else { + LOG.error("Could not delete key due to ", e); + } + } finally { + lock.writeLock().unlock(); + } } - public synchronized void updateBatch(Map rows) { - Map insertsOnly = new HashMap<>(); - for (Map.Entry entry : rows.entrySet()) { - ByteArrayWrapper keyW = new ByteArrayWrapper(entry.getKey()); - if (entry.getValue() != null) { - if (enabled) { - currentUpdates.insertedKeys.add(keyW); - incRef(keyW); + @Override + public void putBatch(Map inputMap) { + checkNotNull(inputMap.keySet()); + + lock.writeLock().lock(); + + try { + Map insertsOnly = new HashMap<>(); + if (enabled.get()) { + for (Map.Entry entry : inputMap.entrySet()) { + ByteArrayWrapper keyW = ByteArrayWrapper.wrap(entry.getKey()); + if (entry.getValue() != null) { + currentUpdates.insertedKeys.add(keyW); + incRef(keyW); + insertsOnly.put(entry.getKey(), entry.getValue()); + } else { + currentUpdates.deletedKeys.add(keyW); + } } - insertsOnly.put(entry.getKey(), entry.getValue()); } else { - if (enabled) { - currentUpdates.deletedKeys.add(keyW); + for (Map.Entry entry : inputMap.entrySet()) { + if (entry.getValue() != null) { + insertsOnly.put(entry.getKey(), entry.getValue()); + } } } + src.putBatch(insertsOnly); + } catch (Exception e) { + if (e instanceof RuntimeException) { + throw e; + } else { + LOG.error("Could not put batch due to ", e); + } + } finally { + lock.writeLock().unlock(); } - src.putBatch(insertsOnly); - } - - public synchronized void updateBatch(Map rows, boolean erasure) { - throw new UnsupportedOperationException(); } private void incRef(ByteArrayWrapper keyW) { Ref cnt = refCount.get(keyW); if (cnt == null) { - cnt = new Ref(src.get(keyW.getData()) != null); + cnt = new Ref(src.get(keyW.getData()).isPresent()); refCount.put(keyW, cnt); } cnt.journalRefs++; @@ -151,56 +231,75 @@ private Ref decRef(ByteArrayWrapper keyW) { return cnt; } - public synchronized void storeBlockChanges(BH header) { - if (!enabled) { return; } - currentUpdates.blockHeader = header; - blockUpdates.put(new ByteArrayWrapper(header.getHash()), currentUpdates); - currentUpdates = new Updates(); + public void storeBlockChanges(byte[] blockHash, long blockNumber) { + if (!enabled.get()) { + return; + } + + lock.writeLock().lock(); + + try { + ByteArrayWrapper hash = ByteArrayWrapper.wrap(blockHash); + currentUpdates.blockHeader = hash; + currentUpdates.blockNumber = blockNumber; + blockUpdates.put(hash, currentUpdates); + currentUpdates = new Updates(); + } finally { + lock.writeLock().unlock(); + } } - public synchronized void prune(BH header) { - if (!enabled) { return; } - ByteArrayWrapper blockHashW = new ByteArrayWrapper(header.getHash()); - Updates updates = blockUpdates.remove(blockHashW); - if (updates != null) { - for (ByteArrayWrapper insertedKey : updates.insertedKeys) { - decRef(insertedKey).dbRef = true; - } + public void prune(byte[] blockHash, long blockNumber) { + if (!enabled.get()) { + return; + } + + lock.writeLock().lock(); - Map batchRemove = new HashMap<>(); - for (ByteArrayWrapper key : updates.deletedKeys) { - Ref ref = refCount.get(key); - if (ref == null || ref.journalRefs == 0) { - batchRemove.put(key.getData(), null); - } else if (ref != null) { - ref.dbRef = false; + try { + ByteArrayWrapper blockHashW = ByteArrayWrapper.wrap(blockHash); + Updates updates = blockUpdates.remove(blockHashW); + if (updates != null) { + for (ByteArrayWrapper insertedKey : updates.insertedKeys) { + decRef(insertedKey).dbRef = true; } - } - src.putBatch(batchRemove); - rollbackForkBlocks(header.getNumber()); + List batchRemove = new ArrayList<>(); + for (ByteArrayWrapper key : updates.deletedKeys) { + Ref ref = refCount.get(key); + if (ref == null || ref.journalRefs == 0) { + batchRemove.add(key.getData()); + } else if (ref != null) { + ref.dbRef = false; + } + } + src.deleteBatch(batchRemove); + + rollbackForkBlocks(blockNumber); + } + } finally { + lock.writeLock().unlock(); } } private void rollbackForkBlocks(long blockNum) { for (Updates updates : new ArrayList<>(blockUpdates.values())) { - if (updates.blockHeader.getNumber() == blockNum) { + if (updates.blockNumber == blockNum) { rollback(updates.blockHeader); } } } - private synchronized void rollback(BH header) { - ByteArrayWrapper blockHashW = new ByteArrayWrapper(header.getHash()); + private void rollback(ByteArrayWrapper blockHashW) { Updates updates = blockUpdates.remove(blockHashW); - Map batchRemove = new HashMap<>(); + List batchRemove = new ArrayList<>(); for (ByteArrayWrapper insertedKey : updates.insertedKeys) { Ref ref = decRef(insertedKey); if (ref.getTotRefs() == 0) { - batchRemove.put(insertedKey.getData(), null); + batchRemove.add(insertedKey.getData()); } } - src.putBatch(batchRemove); + src.deleteBatch(batchRemove); } public Map getRefCount() { @@ -211,25 +310,59 @@ public LinkedHashMap getBlockUpdates() { return blockUpdates; } - /** - * *** other **** - */ + public int getDeletedKeysCount() { + lock.readLock().lock(); + try { + return currentUpdates.deletedKeys.size(); + } finally { + lock.readLock().unlock(); + } + } + + public int getInsertedKeysCount() { + lock.readLock().lock(); + try { + return currentUpdates.insertedKeys.size(); + } finally { + lock.readLock().unlock(); + } + } + public Optional get(byte[] key) { - return src.get(key); + lock.readLock().lock(); + try { + return src.get(key); + } catch (Exception e) { + LOG.error("Could not get key due to ", e); + throw e; + } finally { + lock.readLock().unlock(); + } } public Set keys() { - return src.keys(); + lock.readLock().lock(); + try { + return src.keys(); + } catch (Exception e) { + LOG.error("Could not get keys due to ", e); + throw e; + } finally { + lock.readLock().unlock(); + } } @Override public void close() { - src.close(); - } - - @Override - public void putBatch(Map inputMap) { - updateBatch(inputMap); + lock.writeLock().lock(); + + try { + src.close(); + } catch (Exception e) { + LOG.error("Could not close source due to ", e); + } finally { + lock.writeLock().unlock(); + } } @Override @@ -244,15 +377,84 @@ public void commitBatch() { @Override public void deleteBatch(Collection keys) { - throw new UnsupportedOperationException(); + checkNotNull(keys); + if (!enabled.get()) { + check(); + return; + } + + lock.writeLock().lock(); + + try { + check(); + + // deletes are delayed + keys.forEach(key -> currentUpdates.deletedKeys.add(ByteArrayWrapper.wrap(key))); + } catch (Exception e) { + if (e instanceof RuntimeException) { + throw e; + } else { + LOG.error("Could not delete batch due to ", e); + } + } finally { + lock.writeLock().unlock(); + } } @Override public boolean isEmpty() { - throw new UnsupportedOperationException(); + lock.readLock().lock(); + + try { + // the delayed deletes are not considered by this check until applied to the db + if (!currentUpdates.insertedKeys.isEmpty()) { + check(); + return false; + } else { + return src.isEmpty(); + } + } catch (Exception e) { + LOG.error("Could not check if empty due to ", e); + throw e; + } finally { + lock.readLock().unlock(); + } } - public IByteArrayKeyValueDatabase getSrc() { + public IByteArrayKeyValueStore getSrc() { return src; } + + public IByteArrayKeyValueDatabase getArchiveSource() { + if (!hasArchive) { + return null; + } else { + return ((ArchivedDataSource) src).getArchiveDatabase(); + } + } + + @Override + public void check() { + src.check(); + } + + /** + * Checks that the given key is not null. Throws a {@link IllegalArgumentException} if the key + * is null. + */ + public static void checkNotNull(byte[] k) { + if (k == null) { + throw new IllegalArgumentException("The data store does not accept null keys."); + } + } + + /** + * Checks that the given collection of keys does not contain null values. Throws a {@link + * IllegalArgumentException} if a null key is present. + */ + public static void checkNotNull(Collection keys) { + if (keys.contains(null)) { + throw new IllegalArgumentException("The data store does not accept null keys."); + } + } } diff --git a/modMcf/src/org/aion/mcf/trie/Trie.java b/modMcf/src/org/aion/mcf/trie/Trie.java index ca8a01742f..34b38e312b 100644 --- a/modMcf/src/org/aion/mcf/trie/Trie.java +++ b/modMcf/src/org/aion/mcf/trie/Trie.java @@ -1,36 +1,45 @@ -/******************************************************************************* +/* ****************************************************************************** + * Copyright (c) 2017-2018 Aion foundation. * - * Copyright (c) 2017, 2018 Aion foundation. + * This file is part of the aion network project. * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. + * The aion network project is free software: you can redistribute it + * and/or modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation, either version 3 of + * the License, or any later version. * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. + * The aion network project is distributed in the hope that it will + * be useful, but WITHOUT ANY WARRANTY; without even the implied + * warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + * See the GNU General Public License for more details. * * You should have received a copy of the GNU General Public License - * along with this program. If not, see + * along with the aion network project source files. + * If not, see . * - * Contributors: + * The aion network project leverages useful source code from other + * open source projects. We greatly appreciate the effort that was + * invested in these projects and we thank the individual contributors + * for their work. For provenance information and contributors + * please see . + * + * Contributors to the aion source files in decreasing order of code volume: * Aion foundation. *******************************************************************************/ package org.aion.mcf.trie; +import org.aion.base.db.IByteArrayKeyValueDatabase; + /** - * Trie interface for the main data structure in Ethereum - * which is used to store both the account state and storage of each account. + * Trie interface for the main data structure in Ethereum which is used to store both the account + * state and storage of each account. */ public interface Trie { /** * Gets a value from the trie for a given key * - * @param key - * - any length byte array + * @param key - any length byte array * @return an rlp encoded byte array of the stored object */ byte[] get(byte[] key); @@ -38,18 +47,15 @@ public interface Trie { /** * Insert or update a value in the trie for a specified key * - * @param key - * - any length byte array - * @param value - * rlp encoded byte array of the object to store + * @param key - any length byte array + * @param value rlp encoded byte array of the object to store */ void update(byte[] key, byte[] value); /** * Deletes a key/value from the trie for a given key * - * @param key - * - any length byte array + * @param key - any length byte array */ void delete(byte[] key); @@ -63,35 +69,36 @@ public interface Trie { /** * Set the top node of the trie * - * @param root - * - 32-byte SHA-3 hash of the root node + * @param root - 32-byte SHA-3 hash of the root node */ void setRoot(byte[] root); /** * Used to check for corruption in the database. * - * @param root - * a world state trie root + * @param root a world state trie root * @return {@code true} if the root is valid, {@code false} otherwise */ boolean isValidRoot(byte[] root); - /** - * Commit all the changes until now - */ + /** Commit all the changes until now */ void sync(); void sync(boolean flushCache); - /** - * Discard all the changes until now - */ + /** Discard all the changes until now */ @Deprecated void undo(); String getTrieDump(); + String getTrieDump(byte[] stateRoot); + + int getTrieSize(byte[] stateRoot); + boolean validate(); -} \ No newline at end of file + long saveFullStateToDatabase(byte[] stateRoot, IByteArrayKeyValueDatabase db); + + long saveDiffStateToDatabase(byte[] stateRoot, IByteArrayKeyValueDatabase db); +} diff --git a/modMcf/src/org/aion/mcf/trie/TrieImpl.java b/modMcf/src/org/aion/mcf/trie/TrieImpl.java index 51c4c75577..4b3ef2ff2b 100644 --- a/modMcf/src/org/aion/mcf/trie/TrieImpl.java +++ b/modMcf/src/org/aion/mcf/trie/TrieImpl.java @@ -1,77 +1,78 @@ -/******************************************************************************* +/* ****************************************************************************** + * Copyright (c) 2017-2018 Aion foundation. * - * Copyright (c) 2017, 2018 Aion foundation. + * This file is part of the aion network project. * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. + * The aion network project is free software: you can redistribute it + * and/or modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation, either version 3 of + * the License, or any later version. * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. + * The aion network project is distributed in the hope that it will + * be useful, but WITHOUT ANY WARRANTY; without even the implied + * warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + * See the GNU General Public License for more details. * * You should have received a copy of the GNU General Public License - * along with this program. If not, see + * along with the aion network project source files. + * If not, see . * - * Contributors: + * The aion network project leverages useful source code from other + * open source projects. We greatly appreciate the effort that was + * invested in these projects and we thank the individual contributors + * for their work. For provenance information and contributors + * please see . + * + * Contributors to the aion source files in decreasing order of code volume: * Aion foundation. *******************************************************************************/ package org.aion.mcf.trie; +import static java.util.Arrays.copyOfRange; +import static org.aion.base.util.ByteArrayWrapper.wrap; +import static org.aion.base.util.ByteUtil.EMPTY_BYTE_ARRAY; +import static org.aion.base.util.ByteUtil.matchingNibbleLength; +import static org.aion.crypto.HashUtil.EMPTY_TRIE_HASH; +import static org.aion.rlp.CompactEncoder.*; +import static org.aion.rlp.RLP.calcElementPrefixSize; +import static org.spongycastle.util.Arrays.concatenate; + +import java.io.*; +import java.util.*; +import org.aion.base.db.IByteArrayKeyValueDatabase; import org.aion.base.db.IByteArrayKeyValueStore; import org.aion.base.util.ByteArrayWrapper; import org.aion.base.util.FastByteComparisons; import org.aion.base.util.Hex; import org.aion.crypto.HashUtil; -import org.aion.mcf.trie.scan.CollectFullSetOfNodes; -import org.aion.mcf.trie.scan.CountNodes; -import org.aion.mcf.trie.scan.ScanAction; -import org.aion.mcf.trie.scan.TraceAllNodes; +import org.aion.mcf.trie.scan.*; import org.aion.rlp.RLP; import org.aion.rlp.RLPItem; import org.aion.rlp.RLPList; import org.aion.rlp.Value; -import java.io.*; -import java.util.*; - -import static java.util.Arrays.copyOfRange; -import static org.aion.base.util.ByteArrayWrapper.wrap; -import static org.aion.base.util.ByteUtil.EMPTY_BYTE_ARRAY; -import static org.aion.base.util.ByteUtil.matchingNibbleLength; -import static org.aion.crypto.HashUtil.EMPTY_TRIE_HASH; -import static org.aion.rlp.CompactEncoder.*; -import static org.aion.rlp.RLP.calcElementPrefixSize; -import static org.spongycastle.util.Arrays.concatenate; - /** - * The modified Merkle Patricia tree (trie) provides a persistent data structure - * to map between arbitrary-length binary data (byte arrays). It is defined in - * terms of a mutable data structure to map between 256-bit binary fragments and - * arbitrary-length binary data, typically implemented as a database. The core - * of the trie, and its sole requirement in terms of the protocol specification - * is to provide a single value that identifies a given set of key-value pairs, - * which may either a 32 byte sequence or the empty byte sequence. It is left as - * an implementation consideration to store and maintain the structure of the - * trie in a manner the allows effective and efficient realisation of the - * protocol. - *

- * The trie implements a caching mechanism and will use cached values if they - * are present. If a node is not present in the cache it will try to fetch it - * from the database and store the cached value. - *

- * Note: the data isn't persisted unless `sync` is explicitly called. - *

- * This Trie implementation supports node pruning (i.e. obsolete nodes are - * marked for removal in the Cache and actually removed from the underlying - * storage on [sync] call), but the algorithm is not suitable for the most - * general case. In general case a trie node might be referenced from several - * parent nodes and for correct pruning the reference counting algorithm needs - * to be implemented. As soon as the real life tree keys are hashes it is very - * unlikely the case so the pruning algorithm is simplified in this - * implementation. + * The modified Merkle Patricia tree (trie) provides a persistent data structure to map between + * arbitrary-length binary data (byte arrays). It is defined in terms of a mutable data structure to + * map between 256-bit binary fragments and arbitrary-length binary data, typically implemented as a + * database. The core of the trie, and its sole requirement in terms of the protocol specification + * is to provide a single value that identifies a given set of key-value pairs, which may either a + * 32 byte sequence or the empty byte sequence. It is left as an implementation consideration to + * store and maintain the structure of the trie in a manner the allows effective and efficient + * realisation of the protocol. + * + *

The trie implements a caching mechanism and will use cached values if they are present. If a + * node is not present in the cache it will try to fetch it from the database and store the cached + * value. + * + *

Note: the data isn't persisted unless `sync` is explicitly called. + * + *

This Trie implementation supports node pruning (i.e. obsolete nodes are marked for removal in + * the Cache and actually removed from the underlying storage on [sync] call), but the algorithm is + * not suitable for the most general case. In general case a trie node might be referenced from + * several parent nodes and for correct pruning the reference counting algorithm needs to be + * implemented. As soon as the real life tree keys are hashes it is very unlikely the case so the + * pruning algorithm is simplified in this implementation. * * @author Nick Savers * @since 20.05.2014 @@ -81,8 +82,7 @@ public class TrieImpl implements Trie { private static byte LIST_SIZE = 17; private static int MAX_SIZE = 20; - @Deprecated - private Object prevRoot; + @Deprecated private Object prevRoot; private Object root; private Cache cache; @@ -119,9 +119,7 @@ public Object getRoot() { return root; } - /** - * for testing TrieTest.testRollbackToRootScenarios - */ + /** for testing TrieTest.testRollbackToRootScenarios */ public void setRoot(Object root) { this.root = root; } @@ -152,9 +150,7 @@ public TrieImpl withPruningEnabled(boolean pruningEnabled) { return this; } - /** - * Retrieve a value from a key as String. - */ + /** Retrieve a value from a key as String. */ public byte[] get(String key) { return this.get(key.getBytes()); } @@ -169,9 +165,7 @@ public byte[] get(byte[] key) { } } - /** - * Insert key/value pair into trie. - */ + /** Insert key/value pair into trie. */ public void update(String key, String value) { this.update(key.getBytes(), value.getBytes()); } @@ -198,9 +192,7 @@ public synchronized boolean isValidRoot(byte[] root) { return !(this.getNode(root) == null); } - /** - * Delete a key/value pair from the trie. - */ + /** Delete a key/value pair from the trie. */ public void delete(String key) { this.update(key.getBytes(), EMPTY_BYTE_ARRAY); } @@ -215,8 +207,9 @@ public void delete(byte[] key) { @Override public byte[] getRootHash() { synchronized (cache) { - if (root == null || (root instanceof byte[] && ((byte[]) root).length == 0) || (root instanceof String && "" - .equals(root))) { + if (root == null + || (root instanceof byte[] && ((byte[]) root).length == 0) + || (root instanceof String && "".equals(root))) { return EMPTY_TRIE_HASH; } else if (root instanceof byte[]) { return (byte[]) this.getRoot(); @@ -241,8 +234,8 @@ private Object get(Object node, byte[] key) { byte[] k = unpackToNibbles(currentNode.get(0).asBytes()); Object v = currentNode.get(1).asObj(); - if (key.length - keypos >= k.length && Arrays - .equals(k, copyOfRange(key, keypos, k.length + keypos))) { + if (key.length - keypos >= k.length + && Arrays.equals(k, copyOfRange(key, keypos, k.length + keypos))) { node = v; keypos += k.length; } else { @@ -277,7 +270,7 @@ private Object insert(Object node, byte[] key, Object value) { } if (isEmptyNode(node)) { - Object[] newNode = new Object[] { packNibbles(key), value }; + Object[] newNode = new Object[] {packNibbles(key), value}; return this.putToCache(newNode); } @@ -295,7 +288,7 @@ private Object insert(Object node, byte[] key, Object value) { // Matching key pair (ie. there's already an object with this key) if (Arrays.equals(k, key)) { - Object[] newNode = new Object[] { packNibbles(key), value }; + Object[] newNode = new Object[] {packNibbles(key), value}; return this.putToCache(newNode); } @@ -311,7 +304,8 @@ private Object insert(Object node, byte[] key, Object value) { // Expand the 2 length slice to a 17 length slice // Create two nodes to putToCache into the new 17 length node Object oldNode = this.insert("", copyOfRange(k, matchingLength + 1, k.length), v); - Object newNode = this.insert("", copyOfRange(key, matchingLength + 1, key.length), value); + Object newNode = + this.insert("", copyOfRange(key, matchingLength + 1, key.length), value); // Create an expanded slice Object[] scaledSlice = emptyStringSlice(17); @@ -328,7 +322,8 @@ private Object insert(Object node, byte[] key, Object value) { // End of the chain, return return newHash; } else { - Object[] newNode = new Object[] { packNibbles(copyOfRange(key, 0, matchingLength)), newHash }; + Object[] newNode = + new Object[] {packNibbles(copyOfRange(key, 0, matchingLength)), newHash}; return this.putToCache(newNode); } } else { @@ -337,10 +332,15 @@ private Object insert(Object node, byte[] key, Object value) { Object[] newNode = copyNode(currentNode); // Replace the first nibble in the key - newNode[key[0]] = this.insert(currentNode.get(key[0]).asObj(), copyOfRange(key, 1, key.length), value); - - if (!FastByteComparisons - .equal(HashUtil.h256(getNode(newNode).encode()), HashUtil.h256(currentNode.encode()))) { + newNode[key[0]] = + this.insert( + currentNode.get(key[0]).asObj(), + copyOfRange(key, 1, key.length), + value); + + if (!FastByteComparisons.equal( + HashUtil.h256(getNode(newNode).encode()), + HashUtil.h256(currentNode.encode()))) { markRemoved(HashUtil.h256(currentNode.encode())); if (!isEmptyNode(currentNode.get(key[0]))) { markRemoved(currentNode.get(key[0]).asBytes()); @@ -379,9 +379,9 @@ private Object delete(Object node, byte[] key) { Object newNode; if (child.length() == PAIR_SIZE) { byte[] newKey = concatenate(k, unpackToNibbles(child.get(0).asBytes())); - newNode = new Object[] { packNibbles(newKey), child.get(1).asObj() }; + newNode = new Object[] {packNibbles(newKey), child.get(1).asObj()}; } else { - newNode = new Object[] { currentNode.get(0), hash }; + newNode = new Object[] {currentNode.get(0), hash}; } markRemoved(HashUtil.h256(currentNode.encode())); return this.putToCache(newNode); @@ -408,21 +408,22 @@ private Object delete(Object node, byte[] key) { Object[] newNode = null; if (amount == 16) { - newNode = new Object[] { packNibbles(new byte[] { 16 }), itemList[amount] }; + newNode = new Object[] {packNibbles(new byte[] {16}), itemList[amount]}; } else if (amount >= 0) { Value child = this.getNode(itemList[amount]); if (child.length() == PAIR_SIZE) { - key = concatenate(new byte[] { amount }, unpackToNibbles(child.get(0).asBytes())); - newNode = new Object[] { packNibbles(key), child.get(1).asObj() }; + key = concatenate(new byte[] {amount}, unpackToNibbles(child.get(0).asBytes())); + newNode = new Object[] {packNibbles(key), child.get(1).asObj()}; } else if (child.length() == LIST_SIZE) { - newNode = new Object[] { packNibbles(new byte[] { amount }), itemList[amount] }; + newNode = new Object[] {packNibbles(new byte[] {amount}), itemList[amount]}; } } else { newNode = itemList; } - if (!FastByteComparisons - .equal(HashUtil.h256(getNode(newNode).encode()), HashUtil.h256(currentNode.encode()))) { + if (!FastByteComparisons.equal( + HashUtil.h256(getNode(newNode).encode()), + HashUtil.h256(currentNode.encode()))) { markRemoved(HashUtil.h256(currentNode.encode())); } @@ -437,8 +438,8 @@ private void markRemoved(byte[] hash) { } /** - * Helper method to retrieve the actual node. If the node is not a list and - * length is > 32 bytes get the actual node from the db. + * Helper method to retrieve the actual node. If the node is not a list and length is > 32 bytes + * get the actual node from the db. */ private Value getNode(Object node) { @@ -465,7 +466,9 @@ private Object putToCache(Object node) { private boolean isEmptyNode(Object node) { Value n = new Value(node); - return (node == null || (n.isString() && (n.asString().isEmpty() || n.get(0).isNull())) || n.length() == 0); + return (node == null + || (n.isString() && (n.asString().isEmpty() || n.get(0).isNull())) + || n.length() == 0); } private Object[] copyNode(Value currentNode) { @@ -485,7 +488,8 @@ public boolean equals(Object trie) { if (this == trie) { return true; } - return trie instanceof Trie && Arrays.equals(this.getRootHash(), ((Trie) trie).getRootHash()); + return trie instanceof Trie + && Arrays.equals(this.getRootHash(), ((Trie) trie).getRootHash()); } @Override @@ -524,10 +528,7 @@ public TrieImpl copy() { } } - /** - * ****************************** Utility functions * - * ***************************** - */ + /** ****************************** Utility functions * ***************************** */ // Created an array of empty elements of required length private static Object[] emptyStringSlice(int l) { Object[] slice = new Object[l]; @@ -538,13 +539,12 @@ private static Object[] emptyStringSlice(int l) { } /** - * Insert/delete operations on a Trie structure leaves the old nodes in - * cache, this method scans the cache and removes them. The method is not - * thread safe, the tree should not be modified during the cleaning process. + * Insert/delete operations on a Trie structure leaves the old nodes in cache, this method scans + * the cache and removes them. The method is not thread safe, the tree should not be modified + * during the cleaning process. */ public void cleanCache() { synchronized (cache) { - CollectFullSetOfNodes collectAction = new CollectFullSetOfNodes(); this.scanTree(this.getRootHash(), collectAction); @@ -576,7 +576,6 @@ public void printFootPrint() { public void scanTree(byte[] hash, ScanAction scanAction) { synchronized (cache) { - Value node = this.getCache().get(hash); if (node == null) { throw new RuntimeException("Not found: " + Hex.toHexString(hash)); @@ -638,6 +637,57 @@ public void scanTreeLoop(byte[] hash, ScanAction scanAction) { } } + /** + * Scans the trie similar to {@link #scanTreeLoop(byte[], ScanAction)}, but stops once a state + * is found in the given database. + * + * @param hash state root + * @param scanAction action to perform on each node + * @param db database containing keys that need not be explored + */ + public void scanTreeDiffLoop( + byte[] hash, ScanAction scanAction, IByteArrayKeyValueDatabase db) { + + ArrayList hashes = new ArrayList<>(); + hashes.add(hash); + + while (!hashes.isEmpty()) { + synchronized (cache) { + byte[] myHash = hashes.remove(0); + Value node = this.getCache().get(myHash); + if (node == null) { + System.out.println("Skipped key. Not found: " + Hex.toHexString(myHash)); + } else { + if (node.isList()) { + List siblings = node.asList(); + if (siblings.size() == PAIR_SIZE) { + Value val = new Value(siblings.get(1)); + if (val.isHashCode() && !hasTerminator((byte[]) siblings.get(0))) { + // scanTree(val.asBytes(), scanAction); + byte[] valBytes = val.asBytes(); + if (!db.get(valBytes).isPresent()) { + hashes.add(valBytes); + } + } + } else { + for (int j = 0; j < LIST_SIZE; ++j) { + Value val = new Value(siblings.get(j)); + if (val.isHashCode()) { + // scanTree(val.asBytes(), scanAction); + byte[] valBytes = val.asBytes(); + if (!db.get(valBytes).isPresent()) { + hashes.add(valBytes); + } + } + } + } + scanAction.doOnNode(myHash, node); + } + } + } + } + } + public void deserialize(byte[] data) { synchronized (cache) { RLPList rlpList = (RLPList) RLP.decode2(data).get(0); @@ -695,21 +745,43 @@ public byte[] serialize() { byte[] keysHeader = RLP.encodeLongElementHeader(keysTotalSize); byte[] valsHeader = RLP.encodeListHeader(valsTotalSize); - byte[] listHeader = RLP.encodeListHeader( - keysTotalSize + keysHeader.length + valsTotalSize + valsHeader.length + root.length); - - byte[] rlpData = new byte[keysTotalSize + keysHeader.length + valsTotalSize + valsHeader.length - + listHeader.length + root.length]; + byte[] listHeader = + RLP.encodeListHeader( + keysTotalSize + + keysHeader.length + + valsTotalSize + + valsHeader.length + + root.length); + + byte[] rlpData = + new byte + [keysTotalSize + + keysHeader.length + + valsTotalSize + + valsHeader.length + + listHeader.length + + root.length]; // copy headers: // [ rlp_list_header, rlp_keys_header, rlp_keys, rlp_vals_header, // rlp_val] System.arraycopy(listHeader, 0, rlpData, 0, listHeader.length); System.arraycopy(keysHeader, 0, rlpData, listHeader.length, keysHeader.length); - System.arraycopy(valsHeader, 0, rlpData, (listHeader.length + keysHeader.length + keysTotalSize), + System.arraycopy( + valsHeader, + 0, + rlpData, + (listHeader.length + keysHeader.length + keysTotalSize), valsHeader.length); - System.arraycopy(root, 0, rlpData, - (listHeader.length + keysHeader.length + keysTotalSize + valsTotalSize + valsHeader.length), + System.arraycopy( + root, + 0, + rlpData, + (listHeader.length + + keysHeader.length + + keysTotalSize + + valsTotalSize + + valsHeader.length), root.length); int k_1 = 0; @@ -720,15 +792,26 @@ public byte[] serialize() { continue; } - System.arraycopy(key.getData(), 0, rlpData, (listHeader.length + keysHeader.length + k_1), + System.arraycopy( + key.getData(), + 0, + rlpData, + (listHeader.length + keysHeader.length + k_1), key.getData().length); k_1 += key.getData().length; byte[] valBytes = RLP.encodeElement(node.getValue().getData()); - System.arraycopy(valBytes, 0, rlpData, - listHeader.length + keysHeader.length + keysTotalSize + valsHeader.length + k_2, + System.arraycopy( + valBytes, + 0, + rlpData, + listHeader.length + + keysHeader.length + + keysTotalSize + + valsHeader.length + + k_2, valBytes.length); k_2 += valBytes.length; } @@ -758,6 +841,13 @@ public String getTrieDump() { } } + @Override + public String getTrieDump(byte[] stateRoot) { + TraceAllNodes traceAction = new TraceAllNodes(); + traceTrie(stateRoot, traceAction); + return "root: " + Hex.toHexString(stateRoot) + "\n" + traceAction.getOutput(); + } + public Set getTrieKeys(byte[] stateRoot) { CollectFullSetOfNodes traceAction = new CollectFullSetOfNodes(); traceTrie(stateRoot, traceAction); @@ -786,12 +876,14 @@ public boolean validate() { synchronized (cache) { final int[] cnt = new int[1]; try { - scanTree(getRootHash(), new ScanAction() { - @Override - public void doOnNode(byte[] hash, Value node) { - cnt[0]++; - } - }); + scanTree( + getRootHash(), + new ScanAction() { + @Override + public void doOnNode(byte[] hash, Value node) { + cnt[0]++; + } + }); } catch (Exception e) { return false; } @@ -799,4 +891,29 @@ public void doOnNode(byte[] hash, Value node) { } } + @Override + public long saveFullStateToDatabase(byte[] stateRoot, IByteArrayKeyValueDatabase db) { + ExtractToDatabase traceAction = new ExtractToDatabase(db); + traceTrie(stateRoot, traceAction); + return traceAction.count; + } + + private void traceDiffTrie(byte[] stateRoot, ScanAction action, IByteArrayKeyValueDatabase db) { + synchronized (cache) { + Value value = new Value(stateRoot); + + if (value.isHashCode() && !db.get(value.asBytes()).isPresent()) { + scanTreeDiffLoop(stateRoot, action, db); + } else { + action.doOnNode(stateRoot, value); + } + } + } + + @Override + public long saveDiffStateToDatabase(byte[] stateRoot, IByteArrayKeyValueDatabase db) { + ExtractToDatabase traceAction = new ExtractToDatabase(db); + traceDiffTrie(stateRoot, traceAction, db); + return traceAction.count; + } } diff --git a/modMcf/src/org/aion/mcf/trie/scan/ExtractToDatabase.java b/modMcf/src/org/aion/mcf/trie/scan/ExtractToDatabase.java new file mode 100644 index 0000000000..bb617f82f5 --- /dev/null +++ b/modMcf/src/org/aion/mcf/trie/scan/ExtractToDatabase.java @@ -0,0 +1,51 @@ +/* ****************************************************************************** + * Copyright (c) 2017-2018 Aion foundation. + * + * This file is part of the aion network project. + * + * The aion network project is free software: you can redistribute it + * and/or modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation, either version 3 of + * the License, or any later version. + * + * The aion network project is distributed in the hope that it will + * be useful, but WITHOUT ANY WARRANTY; without even the implied + * warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + * See the GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with the aion network project source files. + * If not, see . + * + * The aion network project leverages useful source code from other + * open source projects. We greatly appreciate the effort that was + * invested in these projects and we thank the individual contributors + * for their work. For provenance information and contributors + * please see . + * + * Contributors to the aion source files in decreasing order of code volume: + * Aion foundation. + ******************************************************************************/ +package org.aion.mcf.trie.scan; + +import org.aion.base.db.IByteArrayKeyValueDatabase; +import org.aion.rlp.Value; + +/** @author Alexandra Roatis */ +public class ExtractToDatabase implements ScanAction { + + // only the keys are relevant so the value will be this constant + byte[] dummy_value = new byte[] {0}; + IByteArrayKeyValueDatabase db; + public long count = 0; + + public ExtractToDatabase(IByteArrayKeyValueDatabase _db) { + this.db = _db; + } + + @Override + public void doOnNode(byte[] hash, Value node) { + db.put(hash, dummy_value); + count++; + } +} diff --git a/modMcf/test/org/aion/mcf/config/CfgLogTest.java b/modMcf/test/org/aion/mcf/config/CfgLogTest.java new file mode 100644 index 0000000000..c3eae3653e --- /dev/null +++ b/modMcf/test/org/aion/mcf/config/CfgLogTest.java @@ -0,0 +1,226 @@ +/* + * Copyright (c) 2017-2018 Aion foundation. + * + * This file is part of the aion network project. + * + * The aion network project is free software: you can redistribute it + * and/or modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation, either version 3 of + * the License, or any later version. + * + * The aion network project is distributed in the hope that it will + * be useful, but WITHOUT ANY WARRANTY; without even the implied + * warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + * See the GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with the aion network project source files. + * If not, see . + * + * Contributors: + * Aion foundation. + */ +package org.aion.mcf.config; + +import org.aion.log.AionLoggerFactory; + +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + +import java.io.File; +import java.io.IOException; +import java.nio.file.FileVisitResult; +import java.nio.file.Path; +import java.nio.file.SimpleFileVisitor; +import java.nio.file.attribute.BasicFileAttributes; +import java.util.Map; + +import static org.junit.Assert.*; + +/** + * Tests for CfgLog.java + */ +public class CfgLogTest extends CfgLog { + + private final static String[] toggle = { + "true", // valid entry + "maybe?", // invalid entry + "tRuE", // special entry + "" // null entry + }; + + private final static String[] path = { + "logger", // valid entry + "l!@#*g", // invalid entry + "log/logging/logger", // special entry + "" // null entry + }; + + private File testRoot; // Path: /home/joey/Desktop/IDE/aion/modMcf + private File generatedPath; + private String accumulatedPath; + private Map _logModules; + + @Before + public void setup() { + testRoot = new File("testLog"); + if (testRoot.exists()) { + deleteRecursively(testRoot); + testRoot.delete(); + } + testRoot.mkdirs(); + } + + @After + public void shutdown() { + if (testRoot.exists()) { + deleteRecursively(testRoot); + testRoot.delete(); + } + } + + /** + * Test for: + * - if parseBoolean() correctly parses user input + * - if getLogFile() returns the correct configuration + */ + @Test + public void testToggle() { + + // Test for default log configuration + CfgLog config = new CfgLog(); + assertFalse(config.getLogFile()); + + // Test for valid configuration + config.logFile = Boolean.parseBoolean(toggle[0]); + assertTrue(config.getLogFile()); + + // Test for capitalized entry + config.logFile = Boolean.parseBoolean(toggle[1]); + assertFalse(config.getLogFile()); + + // Test for invalid configuration + config.logFile = Boolean.parseBoolean(toggle[2]); + assertTrue(config.getLogFile()); + + // Test for null entry + config.logFile = Boolean.parseBoolean(toggle[3]); + assertFalse(config.getLogFile()); + + } + + /** + * Test for: + * - if isValidPath() validates user input log path + * - if getLogPath() returns the correct log path + */ + @Test + public void testPathInput() { + + // Test for default file path + CfgLog config = new CfgLog(); + assertTrue(config.isValidPath()); + assertEquals("log", config.getLogPath()); + + // Test for valid file path + config.logPath = path[0]; + assertTrue(config.isValidPath()); + assertEquals("logger", config.getLogPath()); + + // Test for invalid file path + config.logPath = path[1]; + assertFalse(config.isValidPath()); + + // Test for folder hierarchy path + config.logPath = path[2]; + assertTrue(config.isValidPath()); + assertEquals("log/logging/logger", config.getLogPath()); + + // Test for null path + config.logPath = path[3]; + assertFalse(config.isValidPath()); + + } + + /** + * Test for: + * - if archives are stored under correct file name + */ + @Test + public void testLoggedPath() { + + // Test Case Default + CfgLog config = new CfgLog(); + assertFalse(config.logFile && config.isValidPath()); + + if(config.logFile && config.isValidPath()) { + AionLoggerFactory.init(_logModules, config.logFile, "testLog/" + config.logPath); + generatedPath = testRoot.listFiles()[0]; + assertEquals("log", generatedPath.getName()); + reset(); + } + + // All Test Case + for(int a = 0; a < 4; a++){ + for(int b = 0; b < 4; b++){ + config.logFile = Boolean.parseBoolean(toggle[a]); + config.logPath = path[b]; + + if(config.logFile && config.isValidPath()) { + AionLoggerFactory.init(_logModules, config.logFile, "testLog/" + config.logPath); + generatedPath = testRoot.listFiles()[0]; + + accumulatedPath = generatedPath.getName(); + while(generatedPath.listFiles()[0].isDirectory()) { + generatedPath = generatedPath.listFiles()[0]; + accumulatedPath = accumulatedPath + "/" + generatedPath.getName(); + } + assertEquals(path[b], accumulatedPath); + reset(); + } + } + } + } + + public void reset() { + deleteRecursively(testRoot); + testRoot.mkdirs(); + } + + public static boolean deleteRecursively(File file) { + Path path = file.toPath(); + try { + java.nio.file.Files.walkFileTree(path, new SimpleFileVisitor() { + @Override + public FileVisitResult visitFile(final Path file, final BasicFileAttributes attrs) throws IOException { + java.nio.file.Files.delete(file); + return FileVisitResult.CONTINUE; + } + + @Override + public FileVisitResult visitFileFailed(final Path file, final IOException e) { + return handleException(e); + } + + private FileVisitResult handleException(final IOException e) { + // e.printStackTrace(); + return FileVisitResult.TERMINATE; + } + + @Override + public FileVisitResult postVisitDirectory(final Path dir, final IOException e) throws IOException { + if (e != null) + return handleException(e); + java.nio.file.Files.delete(dir); + return FileVisitResult.CONTINUE; + } + }); + } catch (IOException e) { + e.printStackTrace(); + return false; + } + + return true; + } +} diff --git a/modMcf/test/org/aion/mcf/trie/JournalPruneDataSourceTest.java b/modMcf/test/org/aion/mcf/trie/JournalPruneDataSourceTest.java new file mode 100644 index 0000000000..a32657474b --- /dev/null +++ b/modMcf/test/org/aion/mcf/trie/JournalPruneDataSourceTest.java @@ -0,0 +1,1561 @@ +/* ****************************************************************************** + * Copyright (c) 2017-2018 Aion foundation. + * + * This file is part of the aion network project. + * + * The aion network project is free software: you can redistribute it + * and/or modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation, either version 3 of + * the License, or any later version. + * + * The aion network project is distributed in the hope that it will + * be useful, but WITHOUT ANY WARRANTY; without even the implied + * warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + * See the GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with the aion network project source files. + * If not, see . + * + * The aion network project leverages useful source code from other + * open source projects. We greatly appreciate the effort that was + * invested in these projects and we thank the individual contributors + * for their work. For provenance information and contributors + * please see . + * + * Contributors to the aion source files in decreasing order of code volume: + * Aion foundation. + ******************************************************************************/ +package org.aion.mcf.trie; + +import static com.google.common.truth.Truth.assertThat; +import static org.junit.Assert.assertTrue; + +import java.util.*; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; +import org.aion.base.db.IByteArrayKeyValueDatabase; +import org.aion.db.impl.DatabaseFactory; +import org.aion.log.AionLoggerFactory; +import org.junit.After; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.Test; + +/** @author Alexandra Roatis */ +public class JournalPruneDataSourceTest { + + private static final String dbName = "TestDB"; + private static IByteArrayKeyValueDatabase source_db = DatabaseFactory.connect(dbName); + private static JournalPruneDataSource db; + + private static final byte[] k1 = "key1".getBytes(); + private static final byte[] v1 = "value1".getBytes(); + + private static final byte[] k2 = "key2".getBytes(); + private static final byte[] v2 = "value2".getBytes(); + + private static final byte[] k3 = "key3".getBytes(); + private static final byte[] v3 = "value3".getBytes(); + + @BeforeClass + public static void setup() { + // logging to see errors + Map cfg = new HashMap<>(); + cfg.put("DB", "INFO"); + + AionLoggerFactory.init(cfg); + } + + @Before + public void open() { + assertThat(source_db.open()).isTrue(); + db = new JournalPruneDataSource(source_db); + } + + @After + public void close() { + source_db.close(); + assertThat(source_db.isClosed()).isTrue(); + } + + // Pruning disabled tests ---------------------------------------------------- + + @Test + public void testPut_woPrune() { + db.setPruneEnabled(false); + + assertThat(db.get(k1).isPresent()).isFalse(); + db.put(k1, v1); + assertThat(db.get(k1).get()).isEqualTo(v1); + + // ensure no cached values + assertThat(db.getInsertedKeysCount()).isEqualTo(0); + assertThat(db.getDeletedKeysCount()).isEqualTo(0); + + // ensure the insert was propagated + assertThat(source_db.get(k1).get()).isEqualTo(v1); + } + + @Test + public void testPutBatch_woPrune() { + db.setPruneEnabled(false); + + assertThat(db.get(k1).isPresent()).isFalse(); + assertThat(db.get(k2).isPresent()).isFalse(); + + Map map = new HashMap<>(); + map.put(k1, v1); + map.put(k2, v2); + db.putBatch(map); + + assertThat(v1).isEqualTo(db.get(k1).get()); + assertThat(v2).isEqualTo(db.get(k2).get()); + + // ensure no cached values + assertThat(db.getInsertedKeysCount()).isEqualTo(0); + assertThat(db.getDeletedKeysCount()).isEqualTo(0); + + // ensure the inserts were propagated + assertThat(source_db.get(k1).get()).isEqualTo(v1); + assertThat(source_db.get(k2).get()).isEqualTo(v2); + } + + @Test + public void testUpdate_woPrune() { + db.setPruneEnabled(false); + + // insert + assertThat(db.get(k1).isPresent()).isFalse(); + db.put(k1, v1); + assertThat(db.get(k1).get()).isEqualTo(v1); + assertThat(source_db.get(k1).get()).isEqualTo(v1); + + // update + db.put(k1, v2); + assertThat(db.get(k1).get()).isEqualTo(v2); + assertThat(source_db.get(k1).get()).isEqualTo(v2); + + // indirect delete + db.put(k1, null); + assertThat(db.get(k1).isPresent()).isTrue(); + assertThat(source_db.get(k1).isPresent()).isTrue(); + + // direct delete + db.put(k2, v2); + assertThat(db.get(k2).get()).isEqualTo(v2); + assertThat(source_db.get(k2).get()).isEqualTo(v2); + db.delete(k2); + assertThat(db.get(k2).isPresent()).isTrue(); + assertThat(source_db.get(k2).isPresent()).isTrue(); + + // ensure no cached values + assertThat(db.getDeletedKeysCount()).isEqualTo(0); + assertThat(db.getInsertedKeysCount()).isEqualTo(0); + } + + @Test + public void testUpdateBatch_woPrune() { + db.setPruneEnabled(false); + + // ensure existence + assertThat(db.get(k1).isPresent()).isFalse(); + assertThat(db.get(k2).isPresent()).isFalse(); + assertThat(db.get(k3).isPresent()).isFalse(); + db.put(k1, v1); + db.put(k2, v2); + + assertThat(v1).isEqualTo(db.get(k1).get()); + assertThat(v2).isEqualTo(db.get(k2).get()); + + // check after update + Map ops = new HashMap<>(); + ops.put(k1, null); + ops.put(k2, v1); + ops.put(k3, v3); + db.putBatch(ops); + + assertThat(db.get(k1).isPresent()).isTrue(); + assertThat(db.get(k2).get()).isEqualTo(v1); + assertThat(db.get(k3).get()).isEqualTo(v3); + + assertThat(source_db.get(k1).isPresent()).isTrue(); + assertThat(source_db.get(k2).get()).isEqualTo(v1); + assertThat(source_db.get(k3).get()).isEqualTo(v3); + + // ensure no cached values + assertThat(db.getDeletedKeysCount()).isEqualTo(0); + assertThat(db.getInsertedKeysCount()).isEqualTo(0); + } + + @Test + public void testDelete_woPrune() { + db.setPruneEnabled(false); + + // ensure existence + db.put(k1, v1); + assertThat(db.get(k1).isPresent()).isTrue(); + + // delete not propagated + db.delete(k1); + assertThat(db.get(k1).isPresent()).isTrue(); + assertThat(source_db.get(k1).get()).isEqualTo(v1); + + // ensure no cached values + assertThat(db.getInsertedKeysCount()).isEqualTo(0); + assertThat(db.getDeletedKeysCount()).isEqualTo(0); + } + + @Test + public void testDeleteBatch_woPrune() { + db.setPruneEnabled(false); + + // ensure existence + Map map = new HashMap<>(); + map.put(k1, v1); + map.put(k2, v2); + map.put(k3, null); + db.putBatch(map); + + assertThat(db.get(k1).isPresent()).isTrue(); + assertThat(db.get(k2).isPresent()).isTrue(); + assertThat(db.get(k3).isPresent()).isFalse(); + + // deletes not propagated + db.deleteBatch(map.keySet()); + assertThat(db.get(k1).isPresent()).isTrue(); + assertThat(db.get(k2).isPresent()).isTrue(); + assertThat(db.get(k3).isPresent()).isFalse(); + + // ensure no cached values + assertThat(db.getInsertedKeysCount()).isEqualTo(0); + assertThat(db.getDeletedKeysCount()).isEqualTo(0); + } + + @Test + public void testKeys_woPrune() { + db.setPruneEnabled(false); + + // keys shouldn't be null even when empty + Set keys = db.keys(); + assertThat(db.isEmpty()).isTrue(); + assertThat(keys).isNotNull(); + assertThat(keys.size()).isEqualTo(0); + + // checking after put + db.put(k1, v1); + db.put(k2, v2); + assertThat(db.get(k1).get()).isEqualTo(v1); + assertThat(db.get(k2).get()).isEqualTo(v2); + + keys = db.keys(); + assertThat(keys.size()).isEqualTo(2); + + // checking after delete + db.delete(k2); + assertThat(db.get(k2).isPresent()).isTrue(); + + keys = db.keys(); + assertThat(keys.size()).isEqualTo(2); + + // checking after putBatch + Map ops = new HashMap<>(); + ops.put(k1, null); + ops.put(k2, v2); + ops.put(k3, v3); + db.putBatch(ops); + + keys = db.keys(); + assertThat(keys.size()).isEqualTo(3); + + // checking after deleteBatch + db.deleteBatch(ops.keySet()); + + keys = db.keys(); + assertThat(keys.size()).isEqualTo(3); + + // ensure no cached values + assertThat(db.getInsertedKeysCount()).isEqualTo(0); + assertThat(db.getDeletedKeysCount()).isEqualTo(0); + } + + @Test + public void testIsEmpty_woPrune() { + db.setPruneEnabled(false); + + assertThat(db.isEmpty()).isTrue(); + + // checking after put + db.put(k1, v1); + db.put(k2, v2); + assertThat(db.get(k1).get()).isEqualTo(v1); + assertThat(db.get(k2).get()).isEqualTo(v2); + + assertThat(db.isEmpty()).isFalse(); + + // checking after delete + db.delete(k2); + assertThat(db.get(k2).isPresent()).isTrue(); + assertThat(db.isEmpty()).isFalse(); + db.delete(k1); + assertThat(db.isEmpty()).isFalse(); + + // checking after putBatch + Map ops = new HashMap<>(); + ops.put(k1, null); + ops.put(k2, v2); + ops.put(k3, v3); + db.putBatch(ops); + assertThat(db.isEmpty()).isFalse(); + + // checking after deleteBatch + db.deleteBatch(ops.keySet()); + assertThat(db.isEmpty()).isFalse(); + + // ensure no cached values + assertThat(db.getInsertedKeysCount()).isEqualTo(0); + assertThat(db.getDeletedKeysCount()).isEqualTo(0); + } + + // Pruning enabled tests ---------------------------------------------------- + + private static final byte[] b0 = "block0".getBytes(); + + @Test + public void testPut_wPrune() { + db.setPruneEnabled(true); + + assertThat(db.get(k1).isPresent()).isFalse(); + db.put(k1, v1); + assertThat(db.get(k1).get()).isEqualTo(v1); + + // ensure cached values + assertThat(db.getInsertedKeysCount()).isEqualTo(1); + assertThat(db.getDeletedKeysCount()).isEqualTo(0); + + // ensure the insert was propagated + assertThat(source_db.get(k1).get()).isEqualTo(v1); + + // check store block + db.storeBlockChanges(b0, 0); + assertThat(db.getInsertedKeysCount()).isEqualTo(0); + assertThat(db.getDeletedKeysCount()).isEqualTo(0); + } + + @Test + public void testPutBatch_wPrune() { + db.setPruneEnabled(true); + + assertThat(db.get(k1).isPresent()).isFalse(); + assertThat(db.get(k2).isPresent()).isFalse(); + + Map map = new HashMap<>(); + map.put(k1, v1); + map.put(k2, v2); + db.putBatch(map); + + assertThat(v1).isEqualTo(db.get(k1).get()); + assertThat(v2).isEqualTo(db.get(k2).get()); + + // ensure cached values + assertThat(db.getInsertedKeysCount()).isEqualTo(2); + assertThat(db.getDeletedKeysCount()).isEqualTo(0); + + // ensure the inserts were propagated + assertThat(source_db.get(k1).get()).isEqualTo(v1); + assertThat(source_db.get(k2).get()).isEqualTo(v2); + + // check store block + db.storeBlockChanges(b0, 0); + assertThat(db.getInsertedKeysCount()).isEqualTo(0); + assertThat(db.getDeletedKeysCount()).isEqualTo(0); + } + + @Test + public void testUpdate_wPrune() { + db.setPruneEnabled(true); + + // insert + assertThat(db.get(k1).isPresent()).isFalse(); + db.put(k1, v1); + assertThat(db.get(k1).get()).isEqualTo(v1); + assertThat(source_db.get(k1).get()).isEqualTo(v1); + + // update + db.put(k1, v2); + assertThat(db.get(k1).get()).isEqualTo(v2); + assertThat(source_db.get(k1).get()).isEqualTo(v2); + + // indirect delete + db.put(k1, null); + assertThat(db.get(k1).isPresent()).isTrue(); + assertThat(source_db.get(k1).isPresent()).isTrue(); + + // direct delete + db.put(k2, v2); + assertThat(db.get(k2).get()).isEqualTo(v2); + assertThat(source_db.get(k2).get()).isEqualTo(v2); + db.delete(k2); + assertThat(db.get(k2).isPresent()).isTrue(); + assertThat(source_db.get(k2).isPresent()).isTrue(); + + // ensure cached values + assertThat(db.getDeletedKeysCount()).isEqualTo(2); + assertThat(db.getInsertedKeysCount()).isEqualTo(2); + + // check store block + db.storeBlockChanges(b0, 0); + assertThat(db.getInsertedKeysCount()).isEqualTo(0); + assertThat(db.getDeletedKeysCount()).isEqualTo(0); + } + + @Test + public void testUpdateBatch_wPrune() { + db.setPruneEnabled(true); + + // ensure existence + assertThat(db.get(k1).isPresent()).isFalse(); + assertThat(db.get(k2).isPresent()).isFalse(); + assertThat(db.get(k3).isPresent()).isFalse(); + db.put(k1, v1); + db.put(k2, v2); + + assertThat(v1).isEqualTo(db.get(k1).get()); + assertThat(v2).isEqualTo(db.get(k2).get()); + + // check after update + Map ops = new HashMap<>(); + ops.put(k1, null); + ops.put(k2, v1); + ops.put(k3, v3); + db.putBatch(ops); + + assertThat(db.get(k1).isPresent()).isTrue(); + assertThat(db.get(k2).get()).isEqualTo(v1); + assertThat(db.get(k3).get()).isEqualTo(v3); + + assertThat(source_db.get(k1).isPresent()).isTrue(); + assertThat(source_db.get(k2).get()).isEqualTo(v1); + assertThat(source_db.get(k3).get()).isEqualTo(v3); + + // ensure cached values + assertThat(db.getDeletedKeysCount()).isEqualTo(1); + assertThat(db.getInsertedKeysCount()).isEqualTo(3); + + // check store block + db.storeBlockChanges(b0, 0); + assertThat(db.getInsertedKeysCount()).isEqualTo(0); + assertThat(db.getDeletedKeysCount()).isEqualTo(0); + } + + @Test + public void testDelete_wPrune() { + db.setPruneEnabled(true); + + // ensure existence + db.put(k1, v1); + assertThat(db.get(k1).isPresent()).isTrue(); + + // delete not propagated + db.delete(k1); + assertThat(db.get(k1).isPresent()).isTrue(); + assertThat(source_db.get(k1).get()).isEqualTo(v1); + + // ensure cached values + assertThat(db.getDeletedKeysCount()).isEqualTo(1); + assertThat(db.getInsertedKeysCount()).isEqualTo(1); + + // check store block + db.storeBlockChanges(b0, 0); + assertThat(db.getInsertedKeysCount()).isEqualTo(0); + assertThat(db.getDeletedKeysCount()).isEqualTo(0); + } + + @Test + public void testDeleteBatch_wPrune() { + db.setPruneEnabled(true); + + // ensure existence + Map map = new HashMap<>(); + map.put(k1, v1); + map.put(k2, v2); + map.put(k3, null); + db.putBatch(map); + + assertThat(db.get(k1).isPresent()).isTrue(); + assertThat(db.get(k2).isPresent()).isTrue(); + assertThat(db.get(k3).isPresent()).isFalse(); + + // check presence after delete + db.deleteBatch(map.keySet()); + + // delete operations are delayed till pruning is called + assertThat(db.get(k1).isPresent()).isTrue(); + assertThat(db.get(k2).isPresent()).isTrue(); + assertThat(db.get(k3).isPresent()).isFalse(); + + // ensure cached values + assertThat(db.getDeletedKeysCount()).isEqualTo(3); + assertThat(db.getInsertedKeysCount()).isEqualTo(2); + + // check store block + db.storeBlockChanges(b0, 0); + assertThat(db.getInsertedKeysCount()).isEqualTo(0); + assertThat(db.getDeletedKeysCount()).isEqualTo(0); + } + + @Test + public void testKeys_wPrune() { + db.setPruneEnabled(true); + + // keys shouldn't be null even when empty + Set keys = db.keys(); + assertThat(db.isEmpty()).isTrue(); + assertThat(keys).isNotNull(); + assertThat(keys.size()).isEqualTo(0); + + // checking after put + db.put(k1, v1); + db.put(k2, v2); + assertThat(db.get(k1).get()).isEqualTo(v1); + assertThat(db.get(k2).get()).isEqualTo(v2); + + keys = db.keys(); + assertThat(keys.size()).isEqualTo(2); + + // checking after delete + db.delete(k2); + assertThat(db.get(k2).isPresent()).isTrue(); + + keys = db.keys(); + assertThat(keys.size()).isEqualTo(2); + + // checking after putBatch + Map ops = new HashMap<>(); + ops.put(k1, null); + ops.put(k2, v2); + ops.put(k3, v3); + db.putBatch(ops); + + keys = db.keys(); + assertThat(keys.size()).isEqualTo(3); + + // checking after deleteBatch + db.deleteBatch(ops.keySet()); + + keys = db.keys(); + assertThat(keys.size()).isEqualTo(3); + + // ensure no cached values + assertThat(db.getInsertedKeysCount()).isEqualTo(3); + assertThat(db.getDeletedKeysCount()).isEqualTo(3); + + // check store block + db.storeBlockChanges(b0, 0); + assertThat(db.getInsertedKeysCount()).isEqualTo(0); + assertThat(db.getDeletedKeysCount()).isEqualTo(0); + } + + @Test + public void testIsEmpty_wPrune() { + db.setPruneEnabled(true); + + assertThat(db.isEmpty()).isTrue(); + + // checking after put + db.put(k1, v1); + db.put(k2, v2); + assertThat(db.get(k1).get()).isEqualTo(v1); + assertThat(db.get(k2).get()).isEqualTo(v2); + + assertThat(db.isEmpty()).isFalse(); + + // checking after delete + db.delete(k2); + assertThat(db.get(k2).isPresent()).isTrue(); + assertThat(db.isEmpty()).isFalse(); + db.delete(k1); + assertThat(db.isEmpty()).isFalse(); + + // checking after putBatch + Map ops = new HashMap<>(); + ops.put(k1, null); + ops.put(k2, v2); + ops.put(k3, v3); + db.putBatch(ops); + assertThat(db.isEmpty()).isFalse(); + + // checking after deleteBatch + db.deleteBatch(ops.keySet()); + assertThat(db.isEmpty()).isFalse(); + + // ensure no cached values + assertThat(db.getInsertedKeysCount()).isEqualTo(3); + assertThat(db.getDeletedKeysCount()).isEqualTo(3); + + // check store block + db.storeBlockChanges(b0, 0); + assertThat(db.getInsertedKeysCount()).isEqualTo(0); + assertThat(db.getDeletedKeysCount()).isEqualTo(0); + } + + // Access with exception tests ---------------------------------------------------- + + @Test(expected = RuntimeException.class) + public void testIsEmpty_wClosedDatabase() { + source_db.close(); + assertThat(source_db.isOpen()).isFalse(); + + // attempt isEmpty on closed db + db.isEmpty(); + } + + @Test(expected = RuntimeException.class) + public void testIsEmpty_wClosedDatabase_wInsertedKeys() { + db.setPruneEnabled(true); + db.put(randomBytes(32), randomBytes(32)); + + source_db.close(); + assertThat(source_db.isOpen()).isFalse(); + assertThat(db.getInsertedKeysCount()).isEqualTo(1); + + // attempt isEmpty on closed db + db.isEmpty(); + } + + @Test(expected = RuntimeException.class) + public void testKeys_wClosedDatabase() { + source_db.close(); + assertThat(source_db.isOpen()).isFalse(); + + // attempt keys on closed db + db.keys(); + } + + @Test(expected = RuntimeException.class) + public void testGet_wClosedDatabase() { + source_db.close(); + assertThat(source_db.isOpen()).isFalse(); + + // attempt get on closed db + db.get(randomBytes(32)); + } + + @Test(expected = RuntimeException.class) + public void testPut_wClosedDatabase() { + source_db.close(); + assertThat(source_db.isOpen()).isFalse(); + + // attempt put on closed db + db.put(randomBytes(32), randomBytes(32)); + } + + @Test(expected = RuntimeException.class) + public void testPut_wClosedDatabase_wNullValue() { + source_db.close(); + assertThat(source_db.isOpen()).isFalse(); + + // attempt put on closed db + db.put(randomBytes(32), null); + } + + @Test(expected = RuntimeException.class) + public void testDelete_wClosedDatabase_wPrune() { + db.setPruneEnabled(true); + source_db.close(); + assertThat(source_db.isOpen()).isFalse(); + + // attempt delete on closed db + db.delete(randomBytes(32)); + } + + @Test(expected = RuntimeException.class) + public void testDelete_wClosedDatabase_woPrune() { + db.setPruneEnabled(false); + source_db.close(); + assertThat(source_db.isOpen()).isFalse(); + + // attempt delete on closed db + db.delete(randomBytes(32)); + } + + @Test(expected = RuntimeException.class) + public void testPutBatch_wClosedDatabase() { + source_db.close(); + assertThat(source_db.isOpen()).isFalse(); + + Map map = new HashMap<>(); + map.put(randomBytes(32), randomBytes(32)); + map.put(randomBytes(32), randomBytes(32)); + map.put(randomBytes(32), randomBytes(32)); + + // attempt putBatch on closed db + db.putBatch(map); + } + + @Test(expected = RuntimeException.class) + public void testDeleteBatch_wClosedDatabase_wPrune() { + db.setPruneEnabled(true); + source_db.close(); + assertThat(source_db.isOpen()).isFalse(); + + List list = new ArrayList<>(); + list.add(randomBytes(32)); + list.add(randomBytes(32)); + list.add(randomBytes(32)); + + // attempt deleteBatch on closed db + db.deleteBatch(list); + } + + @Test(expected = RuntimeException.class) + public void testDeleteBatch_wClosedDatabase_woPrune() { + db.setPruneEnabled(false); + source_db.close(); + assertThat(source_db.isOpen()).isFalse(); + + List list = new ArrayList<>(); + list.add(randomBytes(32)); + list.add(randomBytes(32)); + list.add(randomBytes(32)); + + // attempt deleteBatch on closed db + db.deleteBatch(list); + } + + @Test(expected = IllegalArgumentException.class) + public void testGet_wNullKey() { + assertThat(source_db.open()).isTrue(); + + // attempt get with null key + db.get(null); + } + + @Test(expected = IllegalArgumentException.class) + public void testPut_wNullKey() { + assertThat(source_db.open()).isTrue(); + + // attempt put with null key + db.put(null, randomBytes(32)); + } + + @Test(expected = IllegalArgumentException.class) + public void testDelete_wNullKey() { + assertThat(source_db.open()).isTrue(); + + // attempt delete with null key + db.delete(null); + } + + @Test(expected = IllegalArgumentException.class) + public void testPutBatch_wNullKey() { + assertThat(source_db.open()).isTrue(); + + Map map = new HashMap<>(); + map.put(randomBytes(32), randomBytes(32)); + map.put(randomBytes(32), randomBytes(32)); + map.put(null, randomBytes(32)); + + // attempt putBatch on closed db + db.putBatch(map); + } + + @Test(expected = IllegalArgumentException.class) + public void testDeleteBatch_wNullKey() { + assertThat(source_db.open()).isTrue(); + + List list = new ArrayList<>(); + list.add(randomBytes(32)); + list.add(randomBytes(32)); + list.add(null); + + // attempt deleteBatch on closed db + db.deleteBatch(list); + } + + public static byte[] randomBytes(int length) { + byte[] result = new byte[length]; + new Random().nextBytes(result); + return result; + } + + // Concurrent access tests ---------------------------------------------------- + + private static final int CONCURRENT_THREADS = 200; + private static final int TIME_OUT = 100; // in seconds + private static final boolean DISPLAY_MESSAGES = false; + + private void addThread_IsEmpty(List threads, JournalPruneDataSource db) { + threads.add( + () -> { + boolean check = db.isEmpty(); + if (DISPLAY_MESSAGES) { + System.out.println( + Thread.currentThread().getName() + + ": " + + (check ? "EMPTY" : "NOT EMPTY")); + } + }); + } + + private void addThread_Keys(List threads, JournalPruneDataSource db) { + threads.add( + () -> { + Set keys = db.keys(); + if (DISPLAY_MESSAGES) { + System.out.println( + Thread.currentThread().getName() + ": #keys = " + keys.size()); + } + }); + } + + private void addThread_Get(List threads, JournalPruneDataSource db, String key) { + threads.add( + () -> { + boolean hasValue = db.get(key.getBytes()).isPresent(); + if (DISPLAY_MESSAGES) { + System.out.println( + Thread.currentThread().getName() + + ": " + + key + + " " + + (hasValue ? "PRESENT" : "NOT PRESENT")); + } + }); + } + + private void addThread_Put(List threads, JournalPruneDataSource db, String key) { + threads.add( + () -> { + db.put(key.getBytes(), randomBytes(32)); + if (DISPLAY_MESSAGES) { + System.out.println( + Thread.currentThread().getName() + ": " + key + " ADDED"); + } + }); + } + + private void addThread_Delete(List threads, JournalPruneDataSource db, String key) { + threads.add( + () -> { + db.delete(key.getBytes()); + if (DISPLAY_MESSAGES) { + System.out.println( + Thread.currentThread().getName() + ": " + key + " DELETED"); + } + }); + } + + private void addThread_PutBatch(List threads, JournalPruneDataSource db, String key) { + threads.add( + () -> { + Map map = new HashMap<>(); + map.put((key + 1).getBytes(), randomBytes(32)); + map.put((key + 2).getBytes(), randomBytes(32)); + map.put((key + 3).getBytes(), randomBytes(32)); + db.putBatch(map); + if (DISPLAY_MESSAGES) { + System.out.println( + Thread.currentThread().getName() + + ": " + + (key + 1) + + ", " + + (key + 2) + + ", " + + (key + 3) + + " ADDED"); + } + }); + } + + private void addThread_DeleteBatch( + List threads, JournalPruneDataSource db, String key) { + threads.add( + () -> { + List list = new ArrayList<>(); + list.add((key + 1).getBytes()); + list.add((key + 2).getBytes()); + list.add((key + 3).getBytes()); + db.deleteBatch(list); + if (DISPLAY_MESSAGES) { + System.out.println( + Thread.currentThread().getName() + + ": " + + (key + 1) + + ", " + + (key + 2) + + ", " + + (key + 3) + + " DELETED"); + } + }); + } + + private void addThread_StoreBlockChanges( + List threads, JournalPruneDataSource db, String hash, long number) { + threads.add( + () -> { + db.storeBlockChanges(hash.getBytes(), number); + if (DISPLAY_MESSAGES) { + System.out.println( + Thread.currentThread().getName() + + ": block (" + + hash + + ", " + + number + + ") STORED"); + } + }); + } + + private void addThread_Prune( + List threads, JournalPruneDataSource db, String hash, long number) { + threads.add( + () -> { + db.prune(hash.getBytes(), number); + if (DISPLAY_MESSAGES) { + System.out.println( + Thread.currentThread().getName() + + ": block (" + + hash + + ", " + + number + + ") PRUNED"); + } + }); + } + + @Test + public void testConcurrentAccessOnOpenDatabase() throws InterruptedException { + assertThat(source_db.isOpen()).isTrue(); + db.setPruneEnabled(true); + + // create distinct threads with + List threads = new ArrayList<>(); + + int threadSetCount = CONCURRENT_THREADS / 8; + if (threadSetCount < 3) { + threadSetCount = 3; + } + + String keyStr, blockStr; + + for (int i = 0; i < threadSetCount; i++) { + // 1. thread that checks empty + addThread_IsEmpty(threads, db); + + // 2. thread that gets keys + addThread_Keys(threads, db); + + keyStr = "key-" + i + "."; + + // 3. thread that gets entry + addThread_Get(threads, db, keyStr); + + // 4. thread that puts entry + addThread_Put(threads, db, keyStr); + + // 5. thread that deletes entry + addThread_Delete(threads, db, keyStr); + + // 6. thread that puts entries + addThread_PutBatch(threads, db, keyStr); + + // 7. thread that deletes entry + addThread_DeleteBatch(threads, db, keyStr); + + blockStr = "block-" + i + "."; + + // 8. thread that stores block changes + addThread_StoreBlockChanges(threads, db, blockStr, i); + + // 9. thread that prunes + addThread_Prune(threads, db, blockStr, i); + } + + // run threads and check for exceptions + assertConcurrent("Testing concurrent access. ", threads, TIME_OUT); + + // ensuring close + db.close(); + assertThat(source_db.isClosed()).isTrue(); + } + + @Test + public void testConcurrentPut() throws InterruptedException { + assertThat(source_db.isOpen()).isTrue(); + db.setPruneEnabled(true); + + // create distinct threads with + List threads = new ArrayList<>(); + + for (int i = 0; i < CONCURRENT_THREADS; i++) { + addThread_Put(threads, db, "key-" + i); + } + + // run threads + assertConcurrent("Testing put(...) ", threads, TIME_OUT); + + // check that all values were added + assertThat(db.keys().size()).isEqualTo(CONCURRENT_THREADS); + + // ensuring close + db.close(); + assertThat(source_db.isClosed()).isTrue(); + } + + @Test + public void testConcurrentPutBatch() throws InterruptedException { + assertThat(source_db.isOpen()).isTrue(); + db.setPruneEnabled(true); + + // create distinct threads with + List threads = new ArrayList<>(); + + for (int i = 0; i < CONCURRENT_THREADS; i++) { + addThread_PutBatch(threads, db, "key-" + i); + } + + // run threads + assertConcurrent("Testing putBatch(...) ", threads, TIME_OUT); + + // check that all values were added + assertThat(db.keys().size()).isEqualTo(3 * CONCURRENT_THREADS); + + // ensuring close + db.close(); + assertThat(source_db.isClosed()).isTrue(); + } + + @Test + public void testConcurrentUpdate() throws InterruptedException { + assertThat(source_db.isOpen()).isTrue(); + db.setPruneEnabled(true); + + // create distinct threads with + List threads = new ArrayList<>(); + + int threadSetCount = CONCURRENT_THREADS / 4; + if (threadSetCount < 3) { + threadSetCount = 3; + } + + String keyStr, blockStr; + + for (int i = 0; i < threadSetCount; i++) { + keyStr = "key-" + i + "."; + + // 1. thread that puts entry + addThread_Put(threads, db, keyStr); + + // 2. thread that deletes entry + addThread_Delete(threads, db, keyStr); + + // 3. thread that puts entries + addThread_PutBatch(threads, db, keyStr); + + // 4. thread that deletes entry + addThread_DeleteBatch(threads, db, keyStr); + + blockStr = "block-" + i + "."; + + // 5. thread that stores block changes + addThread_StoreBlockChanges(threads, db, blockStr, i); + + // 6. thread that prunes + addThread_Prune(threads, db, blockStr, i); + } + + // run threads and check for exceptions + assertConcurrent("Testing concurrent updates. ", threads, TIME_OUT); + + // ensuring close + db.close(); + assertThat(source_db.isClosed()).isTrue(); + } + + /** + * From JUnit + * Wiki on multithreaded code and concurrency + */ + public static void assertConcurrent( + final String message, + final List runnables, + final int maxTimeoutSeconds) + throws InterruptedException { + final int numThreads = runnables.size(); + final List exceptions = Collections.synchronizedList(new ArrayList()); + final ExecutorService threadPool = Executors.newFixedThreadPool(numThreads); + try { + final CountDownLatch allExecutorThreadsReady = new CountDownLatch(numThreads); + final CountDownLatch afterInitBlocker = new CountDownLatch(1); + final CountDownLatch allDone = new CountDownLatch(numThreads); + for (final Runnable submittedTestRunnable : runnables) { + threadPool.submit( + () -> { + allExecutorThreadsReady.countDown(); + try { + afterInitBlocker.await(); + submittedTestRunnable.run(); + } catch (final Throwable e) { + exceptions.add(e); + } finally { + allDone.countDown(); + } + }); + } + // wait until all threads are ready + assertTrue( + "Timeout initializing threads! Perform long lasting initializations before passing runnables to assertConcurrent", + allExecutorThreadsReady.await(runnables.size() * 10, TimeUnit.MILLISECONDS)); + // start all test runners + afterInitBlocker.countDown(); + assertTrue( + message + " timeout! More than" + maxTimeoutSeconds + "seconds", + allDone.await(maxTimeoutSeconds, TimeUnit.SECONDS)); + } finally { + threadPool.shutdownNow(); + } + if (!exceptions.isEmpty()) { + for (Throwable e : exceptions) { + e.printStackTrace(); + } + } + assertTrue( + message + "failed with " + exceptions.size() + " exception(s):" + exceptions, + exceptions.isEmpty()); + } + + // Pruning tests ---------------------------------------------------- + + private static final byte[] b1 = "block1".getBytes(); + private static final byte[] b2 = "block2".getBytes(); + private static final byte[] b3 = "block3".getBytes(); + + private static final byte[] k4 = "key4".getBytes(); + private static final byte[] v4 = "value4".getBytes(); + + private static final byte[] k5 = "key5".getBytes(); + private static final byte[] v5 = "value5".getBytes(); + + private static final byte[] k6 = "key6".getBytes(); + private static final byte[] v6 = "value6".getBytes(); + + @Test + public void pruningTest() { + db.setPruneEnabled(true); + + // block 0 + db.put(k1, v1); + db.put(k2, v2); + db.put(k3, v3); + assertThat(db.getInsertedKeysCount()).isEqualTo(3); + assertThat(db.getDeletedKeysCount()).isEqualTo(0); + db.storeBlockChanges(b0, 0); + assertThat(db.getBlockUpdates().size()).isEqualTo(1); + + assertThat(source_db.get(k1).get()).isEqualTo(v1); + assertThat(source_db.get(k2).get()).isEqualTo(v2); + assertThat(source_db.get(k3).get()).isEqualTo(v3); + + // block 1 + db.put(k4, v4); + db.delete(k2); + assertThat(db.getInsertedKeysCount()).isEqualTo(1); + assertThat(db.getDeletedKeysCount()).isEqualTo(1); + db.storeBlockChanges(b1, 1); + assertThat(db.getBlockUpdates().size()).isEqualTo(2); + + assertThat(source_db.get(k2).isPresent()).isTrue(); + assertThat(source_db.get(k4).get()).isEqualTo(v4); + + // block 2 + db.put(k2, v3); + db.delete(k3); + assertThat(db.getInsertedKeysCount()).isEqualTo(1); + assertThat(db.getDeletedKeysCount()).isEqualTo(1); + db.storeBlockChanges(b2, 2); + assertThat(db.getBlockUpdates().size()).isEqualTo(3); + + assertThat(source_db.get(k2).get()).isEqualTo(v3); + assertThat(source_db.get(k3).isPresent()).isTrue(); + + // block 3 + db.put(k5, v5); + db.put(k6, v6); + db.delete(k2); + assertThat(db.getInsertedKeysCount()).isEqualTo(2); + assertThat(db.getDeletedKeysCount()).isEqualTo(1); + db.storeBlockChanges(b3, 3); + assertThat(db.getBlockUpdates().size()).isEqualTo(4); + + assertThat(source_db.get(k5).get()).isEqualTo(v5); + assertThat(source_db.get(k6).get()).isEqualTo(v6); + assertThat(source_db.get(k2).isPresent()).isTrue(); + + // prune block 0 + db.prune(b0, 0); + assertThat(db.getBlockUpdates().size()).isEqualTo(3); + + assertThat(source_db.get(k1).get()).isEqualTo(v1); + assertThat(source_db.get(k2).get()).isEqualTo(v3); + assertThat(source_db.get(k3).get()).isEqualTo(v3); + + // prune block 1 + db.prune(b1, 1); + assertThat(db.getBlockUpdates().size()).isEqualTo(2); + + assertThat(source_db.get(k4).get()).isEqualTo(v4); + // not deleted due to block 2 insert + assertThat(source_db.get(k2).get()).isEqualTo(v3); + + // prune block 2 + db.prune(b2, 2); + assertThat(db.getBlockUpdates().size()).isEqualTo(1); + + assertThat(source_db.get(k2).get()).isEqualTo(v3); + assertThat(source_db.get(k3).isPresent()).isFalse(); + + // prune block 3 + db.prune(b3, 3); + assertThat(db.getBlockUpdates().size()).isEqualTo(0); + + assertThat(source_db.get(k5).get()).isEqualTo(v5); + assertThat(source_db.get(k6).get()).isEqualTo(v6); + assertThat(source_db.get(k2).isPresent()).isFalse(); + } + + @Test + public void pruningTest_wBatch() { + db.setPruneEnabled(true); + + Map map = new HashMap<>(); + + // block 0 + map.put(k1, v1); + map.put(k2, v2); + map.put(k3, v3); + db.putBatch(map); + + assertThat(db.getInsertedKeysCount()).isEqualTo(3); + assertThat(db.getDeletedKeysCount()).isEqualTo(0); + db.storeBlockChanges(b0, 0); + assertThat(db.getBlockUpdates().size()).isEqualTo(1); + + assertThat(source_db.get(k1).get()).isEqualTo(v1); + assertThat(source_db.get(k2).get()).isEqualTo(v2); + assertThat(source_db.get(k3).get()).isEqualTo(v3); + + // block 1 + map.clear(); + map.put(k4, v4); + map.put(k2, null); + db.putBatch(map); + + assertThat(db.getInsertedKeysCount()).isEqualTo(1); + assertThat(db.getDeletedKeysCount()).isEqualTo(1); + db.storeBlockChanges(b1, 1); + assertThat(db.getBlockUpdates().size()).isEqualTo(2); + + assertThat(source_db.get(k2).isPresent()).isTrue(); + assertThat(source_db.get(k4).get()).isEqualTo(v4); + + // block 2 + map.clear(); + map.put(k2, v3); + map.put(k3, null); + db.putBatch(map); + + assertThat(db.getInsertedKeysCount()).isEqualTo(1); + assertThat(db.getDeletedKeysCount()).isEqualTo(1); + db.storeBlockChanges(b2, 2); + assertThat(db.getBlockUpdates().size()).isEqualTo(3); + + assertThat(source_db.get(k2).get()).isEqualTo(v3); + assertThat(source_db.get(k3).isPresent()).isTrue(); + + // block 3 + map.clear(); + map.put(k5, v5); + map.put(k6, v6); + map.put(k2, null); + db.putBatch(map); + + assertThat(db.getInsertedKeysCount()).isEqualTo(2); + assertThat(db.getDeletedKeysCount()).isEqualTo(1); + db.storeBlockChanges(b3, 3); + assertThat(db.getBlockUpdates().size()).isEqualTo(4); + + assertThat(source_db.get(k5).get()).isEqualTo(v5); + assertThat(source_db.get(k6).get()).isEqualTo(v6); + assertThat(source_db.get(k2).isPresent()).isTrue(); + + // prune block 0 + db.prune(b0, 0); + assertThat(db.getBlockUpdates().size()).isEqualTo(3); + + assertThat(source_db.get(k1).get()).isEqualTo(v1); + assertThat(source_db.get(k2).get()).isEqualTo(v3); + assertThat(source_db.get(k3).get()).isEqualTo(v3); + + // prune block 1 + db.prune(b1, 1); + assertThat(db.getBlockUpdates().size()).isEqualTo(2); + + assertThat(source_db.get(k4).get()).isEqualTo(v4); + // not deleted due to block 2 insert + assertThat(source_db.get(k2).get()).isEqualTo(v3); + + // prune block 2 + db.prune(b2, 2); + assertThat(db.getBlockUpdates().size()).isEqualTo(1); + + assertThat(source_db.get(k2).get()).isEqualTo(v3); + assertThat(source_db.get(k3).isPresent()).isFalse(); + + // prune block 3 + db.prune(b3, 3); + assertThat(db.getBlockUpdates().size()).isEqualTo(0); + + assertThat(source_db.get(k5).get()).isEqualTo(v5); + assertThat(source_db.get(k6).get()).isEqualTo(v6); + assertThat(source_db.get(k2).isPresent()).isFalse(); + } + + @Test + public void pruningTest_woStoredLevel() { + db.setPruneEnabled(true); + + source_db.put(k2, v2); + source_db.put(k3, v3); + + // block 2 + db.put(k2, v3); + db.delete(k3); + assertThat(db.getInsertedKeysCount()).isEqualTo(1); + assertThat(db.getDeletedKeysCount()).isEqualTo(1); + db.storeBlockChanges(b2, 2); + assertThat(db.getBlockUpdates().size()).isEqualTo(1); + + assertThat(source_db.get(k2).get()).isEqualTo(v3); + assertThat(source_db.get(k3).isPresent()).isTrue(); + + // block 3 + db.put(k5, v5); + db.put(k6, v6); + db.delete(k2); + assertThat(db.getInsertedKeysCount()).isEqualTo(2); + assertThat(db.getDeletedKeysCount()).isEqualTo(1); + db.storeBlockChanges(b3, 3); + assertThat(db.getBlockUpdates().size()).isEqualTo(2); + + assertThat(source_db.get(k5).get()).isEqualTo(v5); + assertThat(source_db.get(k6).get()).isEqualTo(v6); + assertThat(source_db.get(k2).isPresent()).isTrue(); + + // prune block 0 (not stored) + db.prune(b0, 0); + assertThat(db.getBlockUpdates().size()).isEqualTo(2); + + // prune block 1 (not stored) + db.prune(b1, 1); + assertThat(db.getBlockUpdates().size()).isEqualTo(2); + + assertThat(source_db.get(k2).get()).isEqualTo(v3); + assertThat(source_db.get(k3).get()).isEqualTo(v3); + assertThat(source_db.get(k5).get()).isEqualTo(v5); + assertThat(source_db.get(k6).get()).isEqualTo(v6); + + // prune block 2 + db.prune(b2, 2); + assertThat(db.getBlockUpdates().size()).isEqualTo(1); + + assertThat(source_db.get(k2).get()).isEqualTo(v3); + assertThat(source_db.get(k3).isPresent()).isFalse(); + + // prune block 3 + db.prune(b3, 3); + assertThat(db.getBlockUpdates().size()).isEqualTo(0); + + assertThat(source_db.get(k5).get()).isEqualTo(v5); + assertThat(source_db.get(k6).get()).isEqualTo(v6); + assertThat(source_db.get(k2).isPresent()).isFalse(); + } + + @Test + public void pruningTest_wFork_onCurrentLevel() { + db.setPruneEnabled(true); + + // block b0 + db.put(k1, v1); + db.put(k2, v2); + db.put(k3, v3); + db.storeBlockChanges(b0, 0); + assertThat(db.getBlockUpdates().size()).isEqualTo(1); + + assertThat(source_db.keys().size()).isEqualTo(3); + assertThat(source_db.get(k1).get()).isEqualTo(v1); + assertThat(source_db.get(k2).get()).isEqualTo(v2); + assertThat(source_db.get(k3).get()).isEqualTo(v3); + + // block b1 + db.put(k4, v4); + db.put(k1, v2); + db.delete(k2); + db.storeBlockChanges(b1, 1); + assertThat(db.getBlockUpdates().size()).isEqualTo(2); + + assertThat(source_db.keys().size()).isEqualTo(4); + assertThat(source_db.get(k1).get()).isEqualTo(v2); + assertThat(source_db.get(k2).get()).isEqualTo(v2); + assertThat(source_db.get(k3).get()).isEqualTo(v3); + assertThat(source_db.get(k4).get()).isEqualTo(v4); + + // block b2 + db.put(k5, v5); + db.delete(k3); + db.put(k2, v3); + db.put(k1, v4); + db.put(k4, v6); + db.storeBlockChanges(b2, 2); + assertThat(db.getBlockUpdates().size()).isEqualTo(3); + + assertThat(source_db.keys().size()).isEqualTo(5); + assertThat(source_db.get(k1).get()).isEqualTo(v4); + assertThat(source_db.get(k2).get()).isEqualTo(v3); + assertThat(source_db.get(k3).get()).isEqualTo(v3); + assertThat(source_db.get(k4).get()).isEqualTo(v6); + assertThat(source_db.get(k5).get()).isEqualTo(v5); + + // block b3 : note same level as block b2 + db.put(k6, v6); + db.delete(k4); + db.put(k2, v4); + db.put(k1, v3); + db.storeBlockChanges(b3, 2); + assertThat(db.getBlockUpdates().size()).isEqualTo(4); + + assertThat(source_db.keys().size()).isEqualTo(6); + assertThat(source_db.get(k1).get()).isEqualTo(v3); + assertThat(source_db.get(k2).get()).isEqualTo(v4); + assertThat(source_db.get(k3).get()).isEqualTo(v3); + assertThat(source_db.get(k4).get()).isEqualTo(v6); + assertThat(source_db.get(k5).get()).isEqualTo(v5); + assertThat(source_db.get(k6).get()).isEqualTo(v6); + + // prune block b0 + db.prune(b0, 0); + assertThat(db.getBlockUpdates().size()).isEqualTo(3); + assertThat(source_db.keys().size()).isEqualTo(6); + + // prune block b1 + db.prune(b1, 1); + assertThat(db.getBlockUpdates().size()).isEqualTo(2); + assertThat(source_db.keys().size()).isEqualTo(6); + + // prune block b3 at level 2 (should be called for main chain block) + db.prune(b3, 2); + // also removed the updates for block b2 + assertThat(db.getBlockUpdates().size()).isEqualTo(0); + + assertThat(source_db.keys().size()).isEqualTo(4); + assertThat(source_db.get(k1).get()).isEqualTo(v3); + assertThat(source_db.get(k2).get()).isEqualTo(v4); + assertThat(source_db.get(k3).get()).isEqualTo(v3); + assertThat(source_db.get(k4).isPresent()).isFalse(); + assertThat(source_db.get(k5).isPresent()).isFalse(); + assertThat(source_db.get(k6).get()).isEqualTo(v6); + } + + @Test + public void pruningTest_wFork_onPastLevel() { + db.setPruneEnabled(true); + + // block b0 + db.put(k1, v1); + db.put(k2, v2); + db.put(k3, v3); + db.storeBlockChanges(b0, 0); + assertThat(db.getBlockUpdates().size()).isEqualTo(1); + + assertThat(source_db.keys().size()).isEqualTo(3); + assertThat(source_db.get(k1).get()).isEqualTo(v1); + assertThat(source_db.get(k2).get()).isEqualTo(v2); + assertThat(source_db.get(k3).get()).isEqualTo(v3); + + // block b1 + db.put(k4, v4); + db.put(k1, v2); + db.delete(k2); + db.storeBlockChanges(b1, 1); + assertThat(db.getBlockUpdates().size()).isEqualTo(2); + + assertThat(source_db.keys().size()).isEqualTo(4); + assertThat(source_db.get(k1).get()).isEqualTo(v2); + assertThat(source_db.get(k2).get()).isEqualTo(v2); + assertThat(source_db.get(k3).get()).isEqualTo(v3); + assertThat(source_db.get(k4).get()).isEqualTo(v4); + + // block b2 : note same level as block b1 + db.put(k5, v5); + db.delete(k3); + db.put(k2, v3); + db.put(k1, v4); + db.storeBlockChanges(b2, 1); + assertThat(db.getBlockUpdates().size()).isEqualTo(3); + + assertThat(source_db.keys().size()).isEqualTo(5); + assertThat(source_db.get(k1).get()).isEqualTo(v4); + assertThat(source_db.get(k2).get()).isEqualTo(v3); + assertThat(source_db.get(k3).get()).isEqualTo(v3); + assertThat(source_db.get(k4).get()).isEqualTo(v4); + assertThat(source_db.get(k5).get()).isEqualTo(v5); + + // block b3 + db.put(k6, v6); + db.put(k2, v4); + db.put(k1, v3); + db.storeBlockChanges(b3, 2); + assertThat(db.getBlockUpdates().size()).isEqualTo(4); + + assertThat(source_db.keys().size()).isEqualTo(6); + assertThat(source_db.get(k1).get()).isEqualTo(v3); + assertThat(source_db.get(k2).get()).isEqualTo(v4); + assertThat(source_db.get(k3).get()).isEqualTo(v3); + assertThat(source_db.get(k4).get()).isEqualTo(v4); + assertThat(source_db.get(k5).get()).isEqualTo(v5); + assertThat(source_db.get(k6).get()).isEqualTo(v6); + + // prune block b0 + db.prune(b0, 0); + assertThat(db.getBlockUpdates().size()).isEqualTo(3); + assertThat(source_db.keys().size()).isEqualTo(6); + + // prune block b2 at level 1 : (should be called for main chain block) + db.prune(b2, 1); + assertThat(db.getBlockUpdates().size()).isEqualTo(1); + assertThat(source_db.keys().size()).isEqualTo(4); + assertThat(source_db.get(k1).get()).isEqualTo(v3); + assertThat(source_db.get(k2).get()).isEqualTo(v4); + assertThat(source_db.get(k3).isPresent()).isFalse(); + assertThat(source_db.get(k4).isPresent()).isFalse(); + assertThat(source_db.get(k5).get()).isEqualTo(v5); + assertThat(source_db.get(k6).get()).isEqualTo(v6); + + // prune block b3 + db.prune(b3, 2); + // also removed the updates for block b2 + assertThat(db.getBlockUpdates().size()).isEqualTo(0); + + assertThat(source_db.keys().size()).isEqualTo(4); + assertThat(source_db.get(k1).get()).isEqualTo(v3); + assertThat(source_db.get(k2).get()).isEqualTo(v4); + assertThat(source_db.get(k3).isPresent()).isFalse(); + assertThat(source_db.get(k4).isPresent()).isFalse(); + assertThat(source_db.get(k5).get()).isEqualTo(v5); + assertThat(source_db.get(k6).get()).isEqualTo(v6); + } +} diff --git a/modP2p/build.xml b/modP2p/build.xml index 46417ad275..8b325597e3 100644 --- a/modP2p/build.xml +++ b/modP2p/build.xml @@ -26,7 +26,7 @@ - + @@ -50,7 +50,7 @@ - + diff --git a/modP2p/module-info.java b/modP2p/module-info.java index fc1c1f4de8..14383816c4 100644 --- a/modP2p/module-info.java +++ b/modP2p/module-info.java @@ -1,4 +1,4 @@ module aion.p2p { requires aion.base; - exports org.aion.p2p; + exports org.aion.p2p; } diff --git a/modP2p/src/org/aion/p2p/INode.java b/modP2p/src/org/aion/p2p/INode.java index 1b1c7d3614..e9075e9992 100644 --- a/modP2p/src/org/aion/p2p/INode.java +++ b/modP2p/src/org/aion/p2p/INode.java @@ -1,31 +1,29 @@ /* * Copyright (c) 2017-2018 Aion foundation. * - * This file is part of the aion network project. + * This file is part of the aion network project. * - * The aion network project is free software: you can redistribute it - * and/or modify it under the terms of the GNU General Public License - * as published by the Free Software Foundation, either version 3 of - * the License, or any later version. + * The aion network project is free software: you can redistribute it + * and/or modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation, either version 3 of + * the License, or any later version. * - * The aion network project is distributed in the hope that it will - * be useful, but WITHOUT ANY WARRANTY; without even the implied - * warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. - * See the GNU General Public License for more details. + * The aion network project is distributed in the hope that it will + * be useful, but WITHOUT ANY WARRANTY; without even the implied + * warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + * See the GNU General Public License for more details. * - * You should have received a copy of the GNU General Public License - * along with the aion network project source files. - * If not, see . - * - * Contributors to the aion source files in decreasing order of code volume: - * - * Aion foundation. + * You should have received a copy of the GNU General Public License + * along with the aion network project source files. + * If not, see . * + * Contributors: + * Aion foundation. */ - package org.aion.p2p; import java.math.BigInteger; +import java.nio.channels.SocketChannel; /** * @@ -84,4 +82,29 @@ public interface INode { void updateStatus(long _bestBlockNumber, final byte[] _bestBlockHash, BigInteger _totalDifficulty); String getBinaryVersion(); + + boolean getIfFromBootList(); + + byte[] getBestBlockHash(); + + String getConnection(); + + SocketChannel getChannel(); + + void setFromBootList(boolean _ifBoot); + + void setConnection(String _connection); + + IPeerMetric getPeerMetric(); + + void refreshTimestamp(); + + void setChannel(SocketChannel _channel); + + void setId(byte[] _id); + + void setPort(int _port); + + void setBinaryVersion(String _revision); + } diff --git a/modP2p/src/org/aion/p2p/INodeMgr.java b/modP2p/src/org/aion/p2p/INodeMgr.java index ffc8e533ad..79576d3682 100644 --- a/modP2p/src/org/aion/p2p/INodeMgr.java +++ b/modP2p/src/org/aion/p2p/INodeMgr.java @@ -1,13 +1,77 @@ +/* + * Copyright (c) 2017-2018 Aion foundation. + * + * This file is part of the aion network project. + * + * The aion network project is free software: you can redistribute it + * and/or modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation, either version 3 of + * the License, or any later version. + * + * The aion network project is distributed in the hope that it will + * be useful, but WITHOUT ANY WARRANTY; without even the implied + * warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + * See the GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with the aion network project source files. + * If not, see . + * + * Contributors: + * Aion foundation. + */ package org.aion.p2p; +import java.util.List; +import java.util.Map; + public interface INodeMgr { - void timeoutActive(final IP2pMgr _p2pMgr); + void moveInboundToActive(int _channelHashCode); + + void moveOutboundToActive(int _nodeIdHash, String _shortId); + + void dropActive(int _nodeIdHash, String _reason); + + Map getOutboundNodes(); + + int activeNodesSize(); + + INode tempNodesTake() throws InterruptedException; + + boolean isSeedIp(String _ip); + + void addTempNode(INode _n); + + boolean notActiveNode(int k); + + void addOutboundNode(INode _n); + + void addInboundNode(INode _n); + + INode allocNode(String ip, int p0); + + INode getActiveNode(int k); + + List getActiveNodesList(); + + int tempNodesSize(); + + INode getInboundNode(int k); + + INode getOutboundNode(int k); + + String dumpNodeInfo(String selfShortId); + + void seedIpAdd(String _ip); + + void shutdown(); - void moveInboundToActive(int _channelHashCode, final IP2pMgr _p2pMgr); + void ban(int _nodeIdHash); - void moveOutboundToActive(int _nodeIdHash, String _shortId, final IP2pMgr _p2pMgr); + INode getRandom(); - void dropActive(int _nodeIdHash, final IP2pMgr _p2pMgr, String _reason); + Map getActiveNodesMap(); + void timeoutCheck(); } diff --git a/modP2p/src/org/aion/p2p/IP2pMgr.java b/modP2p/src/org/aion/p2p/IP2pMgr.java index 58cb082578..a23aae4b90 100644 --- a/modP2p/src/org/aion/p2p/IP2pMgr.java +++ b/modP2p/src/org/aion/p2p/IP2pMgr.java @@ -1,68 +1,54 @@ /* * Copyright (c) 2017-2018 Aion foundation. * - * This file is part of the aion network project. + * This file is part of the aion network project. * - * The aion network project is free software: you can redistribute it - * and/or modify it under the terms of the GNU General Public License - * as published by the Free Software Foundation, either version 3 of - * the License, or any later version. + * The aion network project is free software: you can redistribute it + * and/or modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation, either version 3 of + * the License, or any later version. * - * The aion network project is distributed in the hope that it will - * be useful, but WITHOUT ANY WARRANTY; without even the implied - * warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. - * See the GNU General Public License for more details. + * The aion network project is distributed in the hope that it will + * be useful, but WITHOUT ANY WARRANTY; without even the implied + * warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + * See the GNU General Public License for more details. * - * You should have received a copy of the GNU General Public License - * along with the aion network project source files. - * If not, see . - * - * Contributors to the aion source files in decreasing order of code volume: - * - * Aion foundation. + * You should have received a copy of the GNU General Public License + * along with the aion network project source files. + * If not, see . * + * Contributors: + * Aion foundation. */ - package org.aion.p2p; +import java.io.IOException; import java.nio.channels.SocketChannel; import java.util.List; import java.util.Map; -/** - * @author chris - */ +/** @author chris */ public interface IP2pMgr { - /** - * @return Map - */ + /** @return Map */ Map getActiveNodes(); - /** - * @param _hs List - */ + /** @param _hs List */ void register(final List _hs); - /** - * @return INode - */ + /** @return INode */ INode getRandom(); /** - * @param _id int + * @param _id int * @param _msg Msg */ void send(int _id, String _displayId, final Msg _msg); - /** - * Used to hook up with kernel to shutdown threads in network module - */ + /** Used to hook up with kernel to shutdown threads in network module. */ void shutdown(); - /** - * start all p2p process - */ + /** Starts all p2p processes. */ void run(); List versions(); @@ -73,7 +59,19 @@ public interface IP2pMgr { void closeSocket(final SocketChannel _sc, String _reason); - boolean isShowLog(); - void errCheck(int nodeIdHashcode, String _displayId); + + void dropActive(int _nodeIdHash, String _reason); + + void configChannel(SocketChannel _channel) throws IOException; + + int getMaxActiveNodes(); + + boolean isSyncSeedsOnly(); + + int getMaxTempNodes(); + + boolean validateNode(INode _node); + + int getSelfNetId(); } diff --git a/modP2p/src/org/aion/p2p/IPeerMetric.java b/modP2p/src/org/aion/p2p/IPeerMetric.java new file mode 100644 index 0000000000..c97c8f08ff --- /dev/null +++ b/modP2p/src/org/aion/p2p/IPeerMetric.java @@ -0,0 +1,56 @@ +/* + * Copyright (c) 2017-2018 Aion foundation. + * + * This file is part of the aion network project. + * + * The aion network project is free software: you can redistribute it + * and/or modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation, either version 3 of + * the License, or any later version. + * + * The aion network project is distributed in the hope that it will + * be useful, but WITHOUT ANY WARRANTY; without even the implied + * warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + * See the GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with the aion network project source files. + * If not, see . + * + * Contributors: + * Aion foundation. + */ +package org.aion.p2p; + +/** + * An interface for tracking peer connection and banning metrics. + */ +public interface IPeerMetric { + + /** + * Returns true only if we should not accept any more connections. + */ + boolean shouldNotConn(); + + /** + * Increments the failed connection counter. + */ + void incFailedCount(); + + /** + * Decrements the failed connection counter. + */ + void decFailedCount(); + + /** + * Sets the current time for tracking a banned connection. + */ + void ban(); + + /** + * Returns true only if the time between now and the last ban is greater than the banned + * connection retry interval. + */ + boolean notBan(); + +} diff --git a/modP2p/src/org/aion/p2p/P2pConstant.java b/modP2p/src/org/aion/p2p/P2pConstant.java index 00fa58e911..1584857f9c 100644 --- a/modP2p/src/org/aion/p2p/P2pConstant.java +++ b/modP2p/src/org/aion/p2p/P2pConstant.java @@ -23,5 +23,7 @@ public class P2pConstant { READ_MAX_RATE_TXBC = 20, // write queue timeout - WRITE_MSG_TIMEOUT = 5000; + WRITE_MSG_TIMEOUT = 5000, + + BACKWARD_SYNC_STEP = 128; } diff --git a/modP2pImpl/build.xml b/modP2pImpl/build.xml index 21cdf39b97..b0afed82de 100644 --- a/modP2pImpl/build.xml +++ b/modP2pImpl/build.xml @@ -13,13 +13,19 @@ + + + + + + @@ -36,7 +42,7 @@ - + @@ -77,7 +83,7 @@ - + diff --git a/modP2pImpl/module-info.java b/modP2pImpl/module-info.java index 06129eb24d..6c67322ad4 100644 --- a/modP2pImpl/module-info.java +++ b/modP2pImpl/module-info.java @@ -1,6 +1,7 @@ module aion.p2p.impl { requires aion.p2p; requires aion.base; + requires aion.log; requires miniupnpc.linux; exports org.aion.p2p.impl1; diff --git a/modP2pImpl/src/org/aion/p2p/impl/comm/Node.java b/modP2pImpl/src/org/aion/p2p/impl/comm/Node.java index 3f8eb4e788..761ae3354d 100644 --- a/modP2pImpl/src/org/aion/p2p/impl/comm/Node.java +++ b/modP2pImpl/src/org/aion/p2p/impl/comm/Node.java @@ -1,28 +1,25 @@ /* * Copyright (c) 2017-2018 Aion foundation. * - * This file is part of the aion network project. + * This file is part of the aion network project. * - * The aion network project is free software: you can redistribute it - * and/or modify it under the terms of the GNU General Public License - * as published by the Free Software Foundation, either version 3 of - * the License, or any later version. + * The aion network project is free software: you can redistribute it + * and/or modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation, either version 3 of + * the License, or any later version. * - * The aion network project is distributed in the hope that it will - * be useful, but WITHOUT ANY WARRANTY; without even the implied - * warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. - * See the GNU General Public License for more details. + * The aion network project is distributed in the hope that it will + * be useful, but WITHOUT ANY WARRANTY; without even the implied + * warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + * See the GNU General Public License for more details. * - * You should have received a copy of the GNU General Public License - * along with the aion network project source files. - * If not, see . - * - * Contributors to the aion source files in decreasing order of code volume: - * - * Aion foundation. + * You should have received a copy of the GNU General Public License + * along with the aion network project source files. + * If not, see . * + * Contributors: + * Aion foundation. */ - package org.aion.p2p.impl.comm; import java.math.BigInteger; @@ -31,8 +28,9 @@ import java.util.Arrays; import java.util.regex.Pattern; import org.aion.p2p.INode; +import org.aion.p2p.IPeerMetric; -/* +/** * * @author Chris * p2p://{node-id}@{ip}:{port} @@ -88,7 +86,12 @@ public final class Node implements INode { */ private String connection = ""; - public PeerMetric peerMetric = new PeerMetric(); + public IPeerMetric peerMetric = new PeerMetric(); + + @Override + public IPeerMetric getPeerMetric() { + return this.peerMetric; + } /** * constructor for initial stage of connections from network @@ -182,6 +185,7 @@ public static Node parseP2p(String _p2p) { return new Node(true, _id, _ip, _port); } + @Override public void setFromBootList(boolean _ifBoot) { this.fromBootList = _ifBoot; } @@ -190,6 +194,7 @@ public void setFromBootList(boolean _ifBoot) { * @param _id * byte[] */ + @Override public void setId(final byte[] _id) { this.id = _id; if (_id != null && _id.length == 36) { @@ -202,10 +207,12 @@ public void setId(final byte[] _id) { * @param _port * int */ + @Override public void setPort(final int _port) { this.port = _port; } + @Override public void setBinaryVersion(String _revision) { this.binaryVersion = _revision; } @@ -214,6 +221,7 @@ public void setBinaryVersion(String _revision) { * this method used to keep current node stage on either pending list or active * list */ + @Override public void refreshTimestamp() { this.timestamp = System.currentTimeMillis(); } @@ -222,6 +230,7 @@ public void refreshTimestamp() { * @param _channel * SocketChannel */ + @Override public void setChannel(final SocketChannel _channel) { this.channel = _channel; } @@ -230,13 +239,15 @@ public void setChannel(final SocketChannel _channel) { * @param _connection * String */ - void setConnection(String _connection) { + @Override + public void setConnection(String _connection) { this.connection = _connection; } /** * @return boolean */ + @Override public boolean getIfFromBootList() { return this.fromBootList; } @@ -270,6 +281,7 @@ public String getBinaryVersion() { /** * @return SocketChannel */ + @Override public SocketChannel getChannel() { return this.channel; } @@ -287,7 +299,8 @@ public int getIdHash() { /** * @return String */ - String getConnection() { + @Override + public String getConnection() { return this.connection; } @@ -301,7 +314,8 @@ public long getBestBlockNumber() { return this.bestBlockNumber; } - byte[] getBestBlockHash() { + @Override + public byte[] getBestBlockHash() { return this.bestBlockHash; } diff --git a/modP2pImpl/src/org/aion/p2p/impl/comm/NodeMgr.java b/modP2pImpl/src/org/aion/p2p/impl/comm/NodeMgr.java index abe281287d..78d7336f09 100644 --- a/modP2pImpl/src/org/aion/p2p/impl/comm/NodeMgr.java +++ b/modP2pImpl/src/org/aion/p2p/impl/comm/NodeMgr.java @@ -1,63 +1,74 @@ /* * Copyright (c) 2017-2018 Aion foundation. * - * This file is part of the aion network project. + * This file is part of the aion network project. * - * The aion network project is free software: you can redistribute it - * and/or modify it under the terms of the GNU General Public License - * as published by the Free Software Foundation, either version 3 of - * the License, or any later version. + * The aion network project is free software: you can redistribute it + * and/or modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation, either version 3 of + * the License, or any later version. * - * The aion network project is distributed in the hope that it will - * be useful, but WITHOUT ANY WARRANTY; without even the implied - * warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. - * See the GNU General Public License for more details. + * The aion network project is distributed in the hope that it will + * be useful, but WITHOUT ANY WARRANTY; without even the implied + * warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + * See the GNU General Public License for more details. * - * You should have received a copy of the GNU General Public License - * along with the aion network project source files. - * If not, see . - * - * Contributors to the aion source files in decreasing order of code volume: - * - * Aion foundation. + * You should have received a copy of the GNU General Public License + * along with the aion network project source files. + * If not, see . * + * Contributors: + * Aion foundation. */ - package org.aion.p2p.impl.comm; -import org.aion.p2p.INode; -import org.aion.p2p.INodeMgr; -import org.aion.p2p.IP2pMgr; - -import java.util.*; +import static org.aion.p2p.impl1.P2pMgr.p2pLOG; + +import java.security.SecureRandom; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.OptionalDouble; +import java.util.Random; +import java.util.Set; import java.util.concurrent.BlockingQueue; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.LinkedBlockingQueue; +import org.aion.p2p.INode; +import org.aion.p2p.INodeMgr; +import org.aion.p2p.IP2pMgr; public class NodeMgr implements INodeMgr { private final static int TIMEOUT_INBOUND_NODES = 10000; + private static final int TIMEOUT_OUTBOUND_NODES = 20000; + private final int maxActiveNodes; private final int maxTempNodes; + private static final Random random = new SecureRandom(); + private final Set seedIps = new HashSet<>(); - // private final Set multiActiveAllowIps = new HashSet<>(Arrays.asList( - // - // )); - private final BlockingQueue tempNodes = new LinkedBlockingQueue<>(); - private final Map outboundNodes = new ConcurrentHashMap<>(); - private final Map inboundNodes = new ConcurrentHashMap<>(); - private final Map activeNodes = new ConcurrentHashMap<>(); + private final IP2pMgr p2pMgr; - public NodeMgr(int _maxActiveNodes, int _maxTempNodes){ + private final BlockingQueue tempNodes = new LinkedBlockingQueue<>(); + private final Map outboundNodes = new ConcurrentHashMap<>(); + private final Map inboundNodes = new ConcurrentHashMap<>(); + private final Map activeNodes = new ConcurrentHashMap<>(); + + public NodeMgr(IP2pMgr _p2pMgr, int _maxActiveNodes, int _maxTempNodes) { this.maxActiveNodes = _maxActiveNodes; this.maxTempNodes = _maxTempNodes; + this.p2pMgr = _p2pMgr; } - public Map getOutboundNodes() { + public Map getOutboundNodes() { return outboundNodes; } @@ -76,16 +87,17 @@ private static String bytesToHex(byte[] bytes) { /** * @param selfShortId String */ + @Override public String dumpNodeInfo(String selfShortId) { StringBuilder sb = new StringBuilder(); sb.append("\n"); sb.append(String.format( - "======================================================================== p2p-status-%6s =========================================================================\n", - selfShortId)); + "======================================================================== p2p-status-%6s =========================================================================\n", + selfShortId)); sb.append(String.format( - "temp[%3d] inbound[%3d] outbound[%3d] active[%3d] s - seed node, td - total difficulty, # - block number, bv - binary version\n", - tempNodesSize(), inboundNodes.size(), outboundNodes.size(), activeNodes.size())); - List sorted = new ArrayList<>(activeNodes.values()); + "temp[%3d] inbound[%3d] outbound[%3d] active[%3d] s - seed node, td - total difficulty, # - block number, bv - binary version\n", + tempNodesSize(), inboundNodes.size(), outboundNodes.size(), activeNodes.size())); + List sorted = new ArrayList<>(activeNodes.values()); if (sorted.size() > 0) { sb.append("\n s"); // id & seed sb.append(" td"); @@ -97,30 +109,33 @@ public String dumpNodeInfo(String selfShortId) { sb.append(" bv"); sb.append(" ci\n"); sb.append( - "--------------------------------------------------------------------------------------------------------------------------------------------------------------------\n"); + "--------------------------------------------------------------------------------------------------------------------------------------------------------------------\n"); sorted.sort((n1, n2) -> { int tdCompare = n2.getTotalDifficulty().compareTo(n1.getTotalDifficulty()); if (tdCompare == 0) { Long n2Bn = n2.getBestBlockNumber(); Long n1Bn = n1.getBestBlockNumber(); return n2Bn.compareTo(n1Bn); - } else + } else { return tdCompare; + } }); - for (Node n : sorted) { + for (INode n : sorted) { try { sb.append(String.format("id:%6s %c %16s %10d %64s %15s %5d %8s %15s %12s\n", - n.getIdShort(), - n.getIfFromBootList() ? 'y' : ' ', n.getTotalDifficulty().toString(10), - n.getBestBlockNumber(), - n.getBestBlockHash() == null ? "" : bytesToHex(n.getBestBlockHash()), n.getIpStr(), - n.getPort(), - n.getConnection(), - n.getBinaryVersion(), - n.getChannel().hashCode()) + n.getIdShort(), + n.getIfFromBootList() ? 'y' : ' ', n.getTotalDifficulty().toString(10), + n.getBestBlockNumber(), + n.getBestBlockHash() == null ? "" : bytesToHex(n.getBestBlockHash()), + n.getIpStr(), + n.getPort(), + n.getConnection(), + n.getBinaryVersion(), + n.getChannel().hashCode()) ); } catch (Exception ex) { ex.printStackTrace(); + p2pLOG.error("NodeMgr dumpNodeInfo exception {}", ex.getMessage()); } } } @@ -130,10 +145,12 @@ public String dumpNodeInfo(String selfShortId) { /** * @param _ip String */ + @Override public void seedIpAdd(String _ip) { this.seedIps.add(_ip); } + @Override public boolean isSeedIp(String _ip) { return this.seedIps.contains(_ip); } @@ -141,89 +158,123 @@ public boolean isSeedIp(String _ip) { /** * @param _n Node */ - public synchronized void addTempNode(final Node _n) { - if(tempNodes.size() < maxTempNodes) + @Override + public synchronized void addTempNode(final INode _n) { + if (tempNodes.size() < maxTempNodes) { tempNodes.add(_n); + } } - public void addInboundNode(final Node _n) { + @Override + public void addInboundNode(final INode _n) { inboundNodes.put(_n.getChannel().hashCode(), _n); } - public void addOutboundNode(final Node _n) { + @Override + public void addOutboundNode(final INode _n) { outboundNodes.put(_n.getIdHash(), _n); } - public Node tempNodesTake() throws InterruptedException { + @Override + public INode tempNodesTake() throws InterruptedException { return tempNodes.take(); } + @Override public int tempNodesSize() { return tempNodes.size(); } + @Override public int activeNodesSize() { return activeNodes.size(); } - public boolean hasActiveNode(int k) { - return activeNodes.containsKey(k); + @Override + public boolean notActiveNode(int k) { + return !activeNodes.containsKey(k); } - public Node getActiveNode(int k) { + @Override + public INode getActiveNode(int k) { return activeNodes.get(k); } - public Node getInboundNode(int k) { + @Override + public INode getInboundNode(int k) { return inboundNodes.get(k); } - public Node getOutboundNode(int k) { + @Override + public INode getOutboundNode(int k) { return outboundNodes.get(k); } - public Node allocNode(String ip, int p0) { - Node n = new Node(ip, p0); - if (seedIps.contains(ip)) + @Override + public INode allocNode(String ip, int p0) { + INode n = new Node(ip, p0); + if (seedIps.contains(ip)) { n.setFromBootList(true); + } return n; } - public List getActiveNodesList() { - return new ArrayList(activeNodes.values()); + @Override + public List getActiveNodesList() { + return new ArrayList<>(activeNodes.values()); + } + + @Override + public HashMap getActiveNodesMap() { + synchronized (activeNodes) { + return new HashMap<>(activeNodes); + } } - public Map getActiveNodesMap() { - synchronized(activeNodes){ - return new HashMap(activeNodes); + @Override + public void timeoutCheck() { + timeoutInbound(); + timeoutOutBound(); + timeoutActive(); + } + + private void timeoutOutBound() { + Iterator outboundIt = getOutboundNodes().keySet().iterator(); + while (outboundIt.hasNext()) { + int outBound = outboundIt.next(); + INode node = getOutboundNodes().get(outBound); + if (System.currentTimeMillis() - node.getTimestamp() + > TIMEOUT_OUTBOUND_NODES) { + p2pMgr.closeSocket( + node.getChannel(), + "outbound-timeout node=" + node.getIdShort() + " ip=" + node.getIpStr()); + outboundIt.remove(); + } } } + @Override public INode getRandom() { - int nodesCount = activeNodes.size(); - if (nodesCount > 0) { - Random r = new Random(System.currentTimeMillis()); - List keysArr = new ArrayList<>(activeNodes.keySet()); + if (!activeNodes.isEmpty()) { + Object[] keysArr = activeNodes.keySet().toArray(); try { - int randomNodeKeyIndex = r.nextInt(keysArr.size()); - int randomNodeKey = keysArr.get(randomNodeKeyIndex); - return this.getActiveNode(randomNodeKey); + return this.getActiveNode((Integer) keysArr[random.nextInt(keysArr.length)]); } catch (IllegalArgumentException e) { System.out.println(""); return null; } - } else + } else { return null; + } } /** * @param _ip String * @return boolean - * @warning not thread safe - * helper function to check a specific ip a node associated with is - * is allowed to add to active list + * @warning not thread safe helper function to check a specific ip a node associated with is is + * allowed to add to active list */ - private boolean activeIpAllow(String _ip){ + private boolean activeIpAllow(String _ip) { return true; // enable this in case // if(multiActiveAllowIps.contains(_ip)) @@ -238,146 +289,157 @@ private boolean activeIpAllow(String _ip){ /** * @param _channelHashCode int - * @param _p2pMgr P2pMgr */ // Attention: move node from container need sync to avoid node not belong to // any container during transit. - public synchronized void moveInboundToActive(int _channelHashCode, final IP2pMgr _p2pMgr) { - Node node = inboundNodes.remove(_channelHashCode); + public synchronized void moveInboundToActive(int _channelHashCode) { + INode node = inboundNodes.remove(_channelHashCode); if (node != null) { - if(activeNodes.size() >= maxActiveNodes){ - _p2pMgr.closeSocket(node.getChannel(), "inbound -> active, active full"); + if (activeNodes.size() >= maxActiveNodes) { + p2pMgr.closeSocket(node.getChannel(), "inbound -> active, active full"); return; } - if(node.getIdHash() == _p2pMgr.getSelfIdHash()){ - _p2pMgr.closeSocket(node.getChannel(), "inbound -> active, self-connected"); + if (node.getIdHash() == p2pMgr.getSelfIdHash()) { + p2pMgr.closeSocket(node.getChannel(), "inbound -> active, self-connected"); return; } node.setConnection("inbound"); node.setFromBootList(seedIps.contains(node.getIpStr())); INode previous = activeNodes.putIfAbsent(node.getIdHash(), node); - if (previous != null) - _p2pMgr.closeSocket(node.getChannel(), "inbound -> active, node " + previous.getIdShort() + " exits"); - else if(!activeIpAllow(node.getIpStr())) - _p2pMgr.closeSocket(node.getChannel(), "inbound -> active, ip " + node.getIpStr() + " exits"); - else { - if (_p2pMgr.isShowLog()) - System.out.println(" active node-id=" + node.getIdShort() + " ip=" + node.getIpStr() + ">"); + if (previous != null) { + p2pMgr.closeSocket(node.getChannel(), + "inbound -> active, node " + previous.getIdShort() + " exits"); + } else if (!activeIpAllow(node.getIpStr())) { + p2pMgr.closeSocket(node.getChannel(), + "inbound -> active, ip " + node.getIpStr() + " exits"); + } else { + if (p2pLOG.isDebugEnabled()) { + p2pLOG.debug("inbound -> active node-id={} ip={}", node.getIdShort(), node + .getIpStr()); + } } } } /** * @param _nodeIdHash int - * @param _shortId String - * @param _p2pMgr P2pMgr + * @param _shortId String */ // Attention: move node from container need sync to avoid node not belong to // any container during transit. - public synchronized void moveOutboundToActive(int _nodeIdHash, String _shortId, final IP2pMgr _p2pMgr) { - Node node = outboundNodes.remove(_nodeIdHash); + public synchronized void moveOutboundToActive(int _nodeIdHash, String _shortId) { + INode node = outboundNodes.remove(_nodeIdHash); if (node != null) { - if(activeNodes.size() >= maxActiveNodes){ - _p2pMgr.closeSocket(node.getChannel(), "outbound -> active, active full"); + if (activeNodes.size() >= maxActiveNodes) { + p2pMgr.closeSocket(node.getChannel(), "outbound -> active, active full"); return; } - if(node.getIdHash() == _p2pMgr.getSelfIdHash()){ - _p2pMgr.closeSocket(node.getChannel(), "outbound -> active, self-connected"); + if (node.getIdHash() == p2pMgr.getSelfIdHash()) { + p2pMgr.closeSocket(node.getChannel(), "outbound -> active, self-connected"); return; } node.setConnection("outbound"); INode previous = activeNodes.putIfAbsent(_nodeIdHash, node); - if (previous != null) - _p2pMgr.closeSocket(node.getChannel(), "outbound -> active, node " + previous.getIdShort() + " exits"); - else { - if (_p2pMgr.isShowLog()) - System.out.println(" active node-id=" + _shortId + " ip=" + node.getIpStr() + ">"); + if (previous != null) { + p2pMgr.closeSocket(node.getChannel(), + "outbound -> active, node " + previous.getIdShort() + " exits"); + } else { + if (p2pLOG.isDebugEnabled()) { + p2pLOG.debug("outbound -> active node-id={} ip={}", _shortId, node.getIpStr()); + } } } } - public void timeoutInbound(final IP2pMgr _p2pMgr) { - Iterator inboundIt = inboundNodes.keySet().iterator(); + private void timeoutInbound() { + Iterator inboundIt = inboundNodes.keySet().iterator(); while (inboundIt.hasNext()) { - int key = (int) inboundIt.next(); - Node node = inboundNodes.get(key); + int key = inboundIt.next(); + INode node = inboundNodes.get(key); if (System.currentTimeMillis() - node.getTimestamp() > TIMEOUT_INBOUND_NODES) { - _p2pMgr.closeSocket(node.getChannel(), "inbound-timeout ip=" + node.getIpStr()); + p2pMgr.closeSocket(node.getChannel(), "inbound-timeout ip=" + node.getIpStr()); inboundIt.remove(); } } } - public void timeoutActive(IP2pMgr _p2pMgr) { + private void timeoutActive() { long now = System.currentTimeMillis(); - OptionalDouble average = activeNodes.values().stream().mapToLong(n -> now - n.getTimestamp()).average(); + OptionalDouble average = activeNodes.values().stream() + .mapToLong(n -> now - n.getTimestamp()).average(); double timeout = average.orElse(4000) * 5; timeout = Math.max(10000, Math.min(timeout, 60000)); - if (_p2pMgr.isShowLog()) { - System.out.printf("\n", average.orElse(0)); + if (p2pLOG.isDebugEnabled()) { + p2pLOG.debug("average-delay={}ms", (long)average.orElse(0)); } - Iterator activeIt = activeNodes.keySet().iterator(); + Iterator activeIt = activeNodes.keySet().iterator(); while (activeIt.hasNext()) { - int key = (int) activeIt.next(); - Node node = getActiveNode(key); + int key = activeIt.next(); + INode node = getActiveNode(key); if (now - node.getTimestamp() > timeout) { - _p2pMgr.closeSocket(node.getChannel(), "active-timeout node=" + node.getIdShort() + " ip=" + node.getIpStr()); + p2pMgr.closeSocket(node.getChannel(), + "active-timeout node=" + node.getIdShort() + " ip=" + node.getIpStr()); activeIt.remove(); } if (!node.getChannel().isConnected()) { - _p2pMgr.closeSocket(node.getChannel(), "channel-already-closed node=" + node.getIdShort() + " ip=" + node.getIpStr()); + p2pMgr.closeSocket(node.getChannel(), + "channel-already-closed node=" + node.getIdShort() + " ip=" + node.getIpStr()); activeIt.remove(); } } } - public void dropActive(int nodeIdHash, final IP2pMgr _p2pMgr, String _reason) { - Node node = activeNodes.remove(nodeIdHash); - if (node == null) + public void dropActive(int nodeIdHash, String _reason) { + INode node = activeNodes.remove(nodeIdHash); + if (node == null) { return; - _p2pMgr.closeSocket(node.getChannel(), _reason); + } + p2pMgr.closeSocket(node.getChannel(), _reason); } - /** - * @param _p2pMgr P2pMgr - */ - public void shutdown(final IP2pMgr _p2pMgr) { + @Override + public void shutdown() { try { - synchronized (outboundNodes){ - outboundNodes.forEach((k, n) -> _p2pMgr.closeSocket(n.getChannel(), "p2p-shutdown outbound node=" + n.getIdShort() + " ip=" + n.getIpStr())); + synchronized (outboundNodes) { + outboundNodes.forEach((k, n) -> p2pMgr.closeSocket(n.getChannel(), + "p2p-shutdown outbound node=" + n.getIdShort() + " ip=" + n.getIpStr())); outboundNodes.clear(); } - synchronized (inboundNodes){ - inboundNodes.forEach((k, n) -> _p2pMgr.closeSocket(n.getChannel(), "p2p-shutdown inbound ip=" + n.getIpStr())); + synchronized (inboundNodes) { + inboundNodes.forEach((k, n) -> p2pMgr + .closeSocket(n.getChannel(), "p2p-shutdown inbound ip=" + n.getIpStr())); inboundNodes.clear(); } - synchronized (activeNodes){ - activeNodes.forEach((k, n) -> _p2pMgr.closeSocket(n.getChannel(), "p2p-shutdown active node=" + n.getIdShort() + " ip=" + n.getIpStr())); + synchronized (activeNodes) { + activeNodes.forEach((k, n) -> p2pMgr.closeSocket(n.getChannel(), + "p2p-shutdown active node=" + n.getIdShort() + " ip=" + n.getIpStr())); activeNodes.clear(); } } catch (Exception e) { - + e.printStackTrace(); + p2pLOG.error("p2p-shutdown exception {}", e.getMessage()); } } + @Override public void ban(int _nodeIdHash) { - Node node = activeNodes.get(_nodeIdHash); + INode node = activeNodes.get(_nodeIdHash); if (node != null) { - node.peerMetric.ban(); + node.getPeerMetric().ban(); } } } diff --git a/modP2pImpl/src/org/aion/p2p/impl/comm/PeerMetric.java b/modP2pImpl/src/org/aion/p2p/impl/comm/PeerMetric.java index 227553216c..f80999b6a3 100644 --- a/modP2pImpl/src/org/aion/p2p/impl/comm/PeerMetric.java +++ b/modP2pImpl/src/org/aion/p2p/impl/comm/PeerMetric.java @@ -1,58 +1,77 @@ /* * Copyright (c) 2017-2018 Aion foundation. * - * This file is part of the aion network project. + * This file is part of the aion network project. * - * The aion network project is free software: you can redistribute it - * and/or modify it under the terms of the GNU General Public License - * as published by the Free Software Foundation, either version 3 of - * the License, or any later version. + * The aion network project is free software: you can redistribute it + * and/or modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation, either version 3 of + * the License, or any later version. * - * The aion network project is distributed in the hope that it will - * be useful, but WITHOUT ANY WARRANTY; without even the implied - * warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. - * See the GNU General Public License for more details. + * The aion network project is distributed in the hope that it will + * be useful, but WITHOUT ANY WARRANTY; without even the implied + * warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + * See the GNU General Public License for more details. * - * You should have received a copy of the GNU General Public License - * along with the aion network project source files. - * If not, see . - * - * Contributors to the aion source files in decreasing order of code volume: - * - * Aion foundation. + * You should have received a copy of the GNU General Public License + * along with the aion network project source files. + * If not, see . * + * Contributors: + * Aion foundation. */ - package org.aion.p2p.impl.comm; +import org.aion.p2p.IPeerMetric; import org.aion.p2p.P2pConstant; -public final class PeerMetric { +public final class PeerMetric implements IPeerMetric { private int metricFailedConn; private long metricFailedConnTs; private long metricBanConnTs; + /** + * Returns true only if we should not accept any more connections. + */ + @Override public boolean shouldNotConn() { return (metricFailedConn > P2pConstant.STOP_CONN_AFTER_FAILED_CONN && ((System.currentTimeMillis() - metricFailedConnTs) > P2pConstant.FAILED_CONN_RETRY_INTERVAL)) || ((System.currentTimeMillis() - metricBanConnTs) < P2pConstant.BAN_CONN_RETRY_INTERVAL); } + /** + * Increments the failed connection counter. + */ + @Override public void incFailedCount() { metricFailedConn++; metricFailedConnTs = System.currentTimeMillis(); } + /** + * Decrements the failed connection counter. + */ + @Override public void decFailedCount() { if (metricFailedConn > 0) metricFailedConn--; } - void ban() { + /** + * Sets the current time for tracking a banned connection. + */ + @Override + public void ban() { metricBanConnTs = System.currentTimeMillis(); } + /** + * Returns true only if the time between now and the last ban is greater than the banned + * connection retry interval. + */ + @Override public boolean notBan() { return ((System.currentTimeMillis() - metricBanConnTs) > P2pConstant.BAN_CONN_RETRY_INTERVAL); } diff --git a/modP2pImpl/src/org/aion/p2p/impl/zero/msg/ReqHandshake.java b/modP2pImpl/src/org/aion/p2p/impl/zero/msg/ReqHandshake.java index 701ff3852b..7c943ad27f 100644 --- a/modP2pImpl/src/org/aion/p2p/impl/zero/msg/ReqHandshake.java +++ b/modP2pImpl/src/org/aion/p2p/impl/zero/msg/ReqHandshake.java @@ -48,7 +48,7 @@ public class ReqHandshake extends Msg { public final static int LEN = 36 + 4 + 8 + 4; - public ReqHandshake(final byte[] _nodeId, int _netId, final byte[] _ip, int _port) { + ReqHandshake(final byte[] _nodeId, int _netId, final byte[] _ip, int _port) { super(Ver.V0, Ctrl.NET, Act.REQ_HANDSHAKE); this.nodeId = _nodeId; this.netId = _netId; diff --git a/modP2pImpl/src/org/aion/p2p/impl/zero/msg/ReqHandshake1.java b/modP2pImpl/src/org/aion/p2p/impl/zero/msg/ReqHandshake1.java index 339130d6ea..f653018706 100644 --- a/modP2pImpl/src/org/aion/p2p/impl/zero/msg/ReqHandshake1.java +++ b/modP2pImpl/src/org/aion/p2p/impl/zero/msg/ReqHandshake1.java @@ -25,6 +25,8 @@ package org.aion.p2p.impl.zero.msg; +import static org.aion.p2p.impl1.P2pMgr.p2pLOG; + import java.nio.ByteBuffer; import java.util.ArrayList; import java.util.List; @@ -50,15 +52,15 @@ public final class ReqHandshake1 extends ReqHandshake { private static final int MIN_LEN = LEN + 2; /** - * - * @param _nodeId byte[36] + * @param _nodeId byte[36] * @param _netId int * @param _ip byte[8] * @param _port int * @param _revision String * @param _versions List header contains 2 byte version */ - public ReqHandshake1(final byte[] _nodeId, int _netId, final byte[] _ip, int _port, final byte[] _revision, final List _versions) { + public ReqHandshake1(final byte[] _nodeId, int _netId, final byte[] _ip, int _port, + final byte[] _revision, final List _versions) { super(_nodeId, _netId, _ip, _port); this.revision = _revision; this.versions = _versions.subList(0, Math.min(MAX_VERSIONS_LEN, _versions.size())); @@ -77,7 +79,6 @@ public static ReqHandshake1 decode(final byte[] _bytes) { if (_bytes == null || _bytes.length < MIN_LEN) return null; else { - try{ ByteBuffer buf = ByteBuffer.wrap(_bytes); @@ -110,7 +111,9 @@ public static ReqHandshake1 decode(final byte[] _bytes) { return new ReqHandshake1(nodeId, netId, ip, port, revision, versions); } catch (Exception e) { - System.out.println(""); + if (p2pLOG.isDebugEnabled()) { + p2pLOG.debug("req-handshake-decode error={}", e.getMessage()); + } return null; } } diff --git a/modP2pImpl/src/org/aion/p2p/impl/zero/msg/ResActiveNodes.java b/modP2pImpl/src/org/aion/p2p/impl/zero/msg/ResActiveNodes.java index bd1b7e6ae8..e6e8534c44 100644 --- a/modP2pImpl/src/org/aion/p2p/impl/zero/msg/ResActiveNodes.java +++ b/modP2pImpl/src/org/aion/p2p/impl/zero/msg/ResActiveNodes.java @@ -1,28 +1,25 @@ /* * Copyright (c) 2017-2018 Aion foundation. * - * This file is part of the aion network project. + * This file is part of the aion network project. * - * The aion network project is free software: you can redistribute it - * and/or modify it under the terms of the GNU General Public License - * as published by the Free Software Foundation, either version 3 of - * the License, or any later version. + * The aion network project is free software: you can redistribute it + * and/or modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation, either version 3 of + * the License, or any later version. * - * The aion network project is distributed in the hope that it will - * be useful, but WITHOUT ANY WARRANTY; without even the implied - * warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. - * See the GNU General Public License for more details. + * The aion network project is distributed in the hope that it will + * be useful, but WITHOUT ANY WARRANTY; without even the implied + * warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + * See the GNU General Public License for more details. * - * You should have received a copy of the GNU General Public License - * along with the aion network project source files. - * If not, see . - * - * Contributors to the aion source files in decreasing order of code volume: - * - * Aion foundation. + * You should have received a copy of the GNU General Public License + * along with the aion network project source files. + * If not, see . * + * Contributors: + * Aion foundation. */ - package org.aion.p2p.impl.zero.msg; import java.nio.ByteBuffer; @@ -43,7 +40,7 @@ */ public final class ResActiveNodes extends Msg { - private final List nodes; + private final List nodes; private int count; @@ -58,19 +55,20 @@ public final class ResActiveNodes extends Msg { * @param _nodes * List */ - public ResActiveNodes(final List _nodes) { + public ResActiveNodes(final List _nodes) { super(Ver.V0, Ctrl.NET, Act.RES_ACTIVE_NODES); this.count = Math.min(MAX_NODES, _nodes.size()); - if (this.count > 0) + if (this.count > 0) { this.nodes = _nodes.subList(0, this.count); - else + } else { this.nodes = new ArrayList<>(); + } } /** * @return List */ - public List getNodes() { + public List getNodes() { return this.nodes; } @@ -80,9 +78,9 @@ public List getNodes() { * @return ResActiveNodes */ public static ResActiveNodes decode(final byte[] _bytes) { - if (_bytes == null || _bytes.length == 0 || (_bytes.length - 1) % NODE_BYTES_LENGTH != 0) + if (_bytes == null || _bytes.length == 0 || (_bytes.length - 1) % NODE_BYTES_LENGTH != 0) { return null; - else { + } else { try{ @@ -90,17 +88,18 @@ public static ResActiveNodes decode(final byte[] _bytes) { int count = buf.get(); // fix bug: https://github.com/aionnetwork/aion/issues/390 - if (_bytes.length != count * NODE_BYTES_LENGTH + 1) + if (_bytes.length != count * NODE_BYTES_LENGTH + 1) { return null; + } - ArrayList activeNodes = new ArrayList<>(); + ArrayList activeNodes = new ArrayList<>(); for (int i = 0; i < count; i++) { byte[] nodeIdBytes = new byte[36]; buf.get(nodeIdBytes); byte[] ipBytes = new byte[8]; buf.get(ipBytes); int port = buf.getInt(); - Node n = new Node(false, nodeIdBytes, ipBytes, port); + INode n = new Node(false, nodeIdBytes, ipBytes, port); activeNodes.add(n); } return new ResActiveNodes(activeNodes); diff --git a/modP2pImpl/src/org/aion/p2p/impl1/P2pMgr.java b/modP2pImpl/src/org/aion/p2p/impl1/P2pMgr.java index 8e7ef38e0b..d6bd041c03 100644 --- a/modP2pImpl/src/org/aion/p2p/impl1/P2pMgr.java +++ b/modP2pImpl/src/org/aion/p2p/impl1/P2pMgr.java @@ -1,563 +1,140 @@ /* * Copyright (c) 2017-2018 Aion foundation. * - * This file is part of the aion network project. + * This file is part of the aion network project. * - * The aion network project is free software: you can redistribute it - * and/or modify it under the terms of the GNU General Public License - * as published by the Free Software Foundation, either version 3 of - * the License, or any later version. + * The aion network project is free software: you can redistribute it + * and/or modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation, either version 3 of + * the License, or any later version. * - * The aion network project is distributed in the hope that it will - * be useful, but WITHOUT ANY WARRANTY; without even the implied - * warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. - * See the GNU General Public License for more details. + * The aion network project is distributed in the hope that it will + * be useful, but WITHOUT ANY WARRANTY; without even the implied + * warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + * See the GNU General Public License for more details. * - * You should have received a copy of the GNU General Public License - * along with the aion network project source files. - * If not, see . - * - * Contributors to the aion source files in decreasing order of code volume: - * - * Aion foundation. + * You should have received a copy of the GNU General Public License + * along with the aion network project source files. + * If not, see . * + * Contributors: + * Aion foundation. */ - package org.aion.p2p.impl1; +import java.io.IOException; +import java.net.InetSocketAddress; +import java.net.SocketException; +import java.nio.channels.SelectionKey; +import java.nio.channels.Selector; +import java.nio.channels.ServerSocketChannel; +import java.nio.channels.SocketChannel; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.BlockingQueue; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.LinkedBlockingQueue; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.ScheduledThreadPoolExecutor; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; +import org.aion.log.AionLoggerFactory; +import org.aion.log.LogEnum; +import org.aion.p2p.Ctrl; +import org.aion.p2p.Handler; +import org.aion.p2p.Header; +import org.aion.p2p.INode; +import org.aion.p2p.INodeMgr; +import org.aion.p2p.IP2pMgr; +import org.aion.p2p.Msg; import org.aion.p2p.P2pConstant; -import org.aion.p2p.*; +import org.aion.p2p.Ver; import org.aion.p2p.impl.TaskRequestActiveNodes; import org.aion.p2p.impl.TaskUPnPManager; -import org.aion.p2p.impl.comm.Act; import org.aion.p2p.impl.comm.Node; import org.aion.p2p.impl.comm.NodeMgr; -import org.aion.p2p.impl.zero.msg.*; +import org.aion.p2p.impl.zero.msg.ReqHandshake1; +import org.aion.p2p.impl.zero.msg.ResHandshake1; +import org.aion.p2p.impl1.tasks.MsgIn; +import org.aion.p2p.impl1.tasks.MsgOut; +import org.aion.p2p.impl1.tasks.TaskClear; +import org.aion.p2p.impl1.tasks.TaskConnectPeers; +import org.aion.p2p.impl1.tasks.TaskInbound; +import org.aion.p2p.impl1.tasks.TaskReceive; +import org.aion.p2p.impl1.tasks.TaskSend; +import org.aion.p2p.impl1.tasks.TaskStatus; import org.apache.commons.collections4.map.LRUMap; - -import java.io.IOException; -import java.io.UnsupportedEncodingException; -import java.net.InetSocketAddress; -import java.nio.ByteBuffer; -import java.nio.channels.*; -import java.util.*; -import java.util.concurrent.*; -import java.util.concurrent.atomic.AtomicBoolean; +import org.slf4j.Logger; /** * @author Chris p2p://{uuid}@{ip}:{port} */ public final class P2pMgr implements IP2pMgr { - private final static int PERIOD_SHOW_STATUS = 10000; - private final static int PERIOD_REQUEST_ACTIVE_NODES = 1000; - private final static int PERIOD_CONNECT_OUTBOUND = 1000; - private final static int PERIOD_CLEAR = 20000; - private final static int PERIOD_UPNP_PORT_MAPPING = 3600000; - - private final static int TIMEOUT_OUTBOUND_CONNECT = 10000; - private final static int TIMEOUT_OUTBOUND_NODES = 20000; - private final static int TIMEOUT_MSG_READ = 10000; + private static final int PERIOD_SHOW_STATUS = 10000; + private static final int PERIOD_REQUEST_ACTIVE_NODES = 1000; + private static final int PERIOD_UPNP_PORT_MAPPING = 3600000; + private static final int TIMEOUT_MSG_READ = 10000; - private final int maxTempNodes; - private final int maxActiveNodes; + // TODO: need refactor by passing the parameter in the later version to P2pMgr. + public static int txBroadCastRoute = + (Ctrl.SYNC << 8) + 6; // ((Ver.V0 << 16) + (Ctrl.SYNC << 8) + 6); - private final boolean syncSeedsOnly; - private final boolean showStatus; - private final boolean showLog; - private final int selfNetId; - private final String selfRevision; - private final byte[] selfNodeId; - private final int selfNodeIdHash; - private final String selfShortId; - private final byte[] selfIp; - private final int selfPort; - private final boolean upnpEnable; + public static final Logger p2pLOG = AionLoggerFactory.getLogger(LogEnum.P2P.name()); + private int maxTempNodes, maxActiveNodes, selfNetId, selfNodeIdHash, selfPort; + private boolean syncSeedsOnly, upnpEnable; + private String selfRevision, selfShortId; + private byte[] selfNodeId, selfIp; + private INodeMgr nodeMgr; private final Map> handlers = new ConcurrentHashMap<>(); private final Set versions = new HashSet<>(); - - private final NodeMgr nodeMgr; + private final Map errCnt = Collections.synchronizedMap(new LRUMap<>(128)); + private final AtomicBoolean start = new AtomicBoolean(true); private ServerSocketChannel tcpServer; private Selector selector; - - private ScheduledThreadPoolExecutor scheduledWorkers; - - private final Map errCnt = Collections.synchronizedMap(new LRUMap<>(128)); - + private ScheduledExecutorService scheduledWorkers; private int errTolerance; - - // TODO: need refactor by passing the parameter in the later version. - private final static int txBroadCastRoute = (Ctrl.SYNC << 8) + 6; // ((Ver.V0 << 16) + (Ctrl.SYNC << 8) + 6); - - enum Dest { - INBOUND, OUTBOUND, ACTIVE - } - - private static class MsgOut { - MsgOut(int _nodeId, String _displayId, Msg _msg, Dest _dest) { - nodeId = _nodeId; - displayId = _displayId; - msg = _msg; - dest = _dest; - timestamp = System.currentTimeMillis(); - } - - int nodeId; - String displayId; - Msg msg; - Dest dest; - long timestamp; - } - - private static class MsgIn { - MsgIn(int nid, String nsid, int route, byte[] msg) { - this.nid = nid; - this.nsid = nsid; - this.route = route; - this.msg = msg; - } - - int nid; - String nsid; - int route; - byte[] msg; - } - - private LinkedBlockingQueue sendMsgQue = new LinkedBlockingQueue<>(); - - private LinkedBlockingQueue receiveMsgQue = new LinkedBlockingQueue<>(); - - private AtomicBoolean start = new AtomicBoolean(true); + private BlockingQueue sendMsgQue = new LinkedBlockingQueue<>(); + private BlockingQueue receiveMsgQue = new LinkedBlockingQueue<>(); private static ReqHandshake1 cachedReqHandshake1; private static ResHandshake1 cachedResHandshake1; - private final class TaskInbound implements Runnable { - @Override - public void run() { - - // read buffer pre-alloc. @ max_body_size - ByteBuffer readBuf = ByteBuffer.allocate(P2pConstant.MAX_BODY_SIZE); - - while (start.get()) { - - try { - Thread.sleep(0, 1); - } catch (Exception e) { - } - - int num; - try { - num = selector.selectNow(); - } catch (IOException e) { - if (showLog) - System.out.println(""); - continue; - } - - if (num == 0) { - continue; - } - - Iterator keys = selector.selectedKeys().iterator(); - - while (keys.hasNext() && (num-- > 0)) { - - final SelectionKey sk = keys.next(); - keys.remove(); - - try{ - - if (!sk.isValid()) - continue; - - if (sk.isAcceptable()) - accept(); - - if (sk.isReadable()) { - - readBuf.rewind(); - - ChannelBuffer chanBuf = (ChannelBuffer) (sk.attachment()); - try { - - int ret; - int cnt = 0; - - while ((ret = ((SocketChannel) sk.channel()).read(readBuf)) > 0) { - cnt += ret; - } - - // read empty select key, continue. - if (cnt <= 0) { - continue; - } - - int prevCnt = cnt + chanBuf.buffRemain; - ByteBuffer forRead; - - if (chanBuf.buffRemain != 0) { - byte[] alreadyRead = new byte[cnt]; - - readBuf.position(0); - readBuf.get(alreadyRead); - forRead = ByteBuffer.allocate(prevCnt); - forRead.put(chanBuf.remainBuffer); - forRead.put(alreadyRead); - } else { - forRead = readBuf; - } - - do { - cnt = read(sk, forRead, prevCnt); - - if (prevCnt == cnt) { - break; - } else - prevCnt = cnt; - - } while (cnt > 0); - - // check if really read data. - if (cnt > prevCnt) { - chanBuf.buffRemain = 0; - throw new P2pException( - "IO read overflow! suppose read:" + prevCnt + " real left:" + cnt); - } - - chanBuf.buffRemain = cnt; - - if (cnt == 0) { - readBuf.rewind(); - } else { - // there are no perfect cycling buffer in jdk - // yet. - // simply just buff move for now. - // @TODO: looking for more efficient way. - - int currPos = forRead.position(); - chanBuf.remainBuffer = new byte[cnt]; - forRead.position(currPos - cnt); - forRead.get(chanBuf.remainBuffer); - readBuf.rewind(); - } - - } catch (NullPointerException e) { - closeSocket((SocketChannel) sk.channel(), chanBuf.displayId + "-read-msg-null-exception"); - chanBuf.isClosed.set(true); - } catch (P2pException e) { - closeSocket((SocketChannel) sk.channel(), chanBuf.displayId + "-read-msg-p2p-exception"); - chanBuf.isClosed.set(true); - - } catch (ClosedChannelException e) { - closeSocket((SocketChannel) sk.channel(), - chanBuf.displayId + "-read-msg-closed-channel-exception"); - - } catch (IOException e) { - closeSocket((SocketChannel) sk.channel(), - chanBuf.displayId + "-read-msg-io-exception: " + e.getMessage()); - chanBuf.isClosed.set(true); - - } catch (CancelledKeyException e) { - chanBuf.isClosed.set(true); - closeSocket((SocketChannel) sk.channel(), - chanBuf.displayId + "-read-msg-key-cancelled-exception"); - } catch (Exception e) { - if (showLog) - System.out.println(""); - } - } - } catch(Exception ex) { - if(showLog) { - System.out.println(""); - ex.printStackTrace(); - } - } - } - } - if (showLog) - System.out.println(""); - } - } - - // hash mapping channel id to write thread. - private int hash2Lane(int in) { - in ^= in >> (32 - 5); - in ^= in >> (32 - 10); - in ^= in >> (32 - 15); - in ^= in >> (32 - 20); - in ^= in >> (32 - 25); - return in & 0b11111; - } - - private final class TaskSend implements Runnable { - - static final int TOTAL_LANE = (1 << 5) - 1; - int lane; - - TaskSend(int _lane) { - this.lane = _lane; - } - - @Override - public void run() { - while (P2pMgr.this.start.get()) { - try { - MsgOut mo = sendMsgQue.take(); - // if timeout , throw away this msg. - long now = System.currentTimeMillis(); - if (now - mo.timestamp > P2pConstant.WRITE_MSG_TIMEOUT) { - if (showLog) - System.out.println(""); - continue; - } - - // if not belong to current lane, put it back. - int targetLane = hash2Lane(mo.nodeId); - if (targetLane != lane) { - sendMsgQue.offer(mo); - continue; - } - - Node node = null; - switch (mo.dest) { - case ACTIVE: - node = nodeMgr.getActiveNode(mo.nodeId); - break; - case INBOUND: - node = nodeMgr.getInboundNode(mo.nodeId); - break; - case OUTBOUND: - node = nodeMgr.getOutboundNode(mo.nodeId); - break; - } - - if (node != null) { - SelectionKey sk = node.getChannel().keyFor(selector); - if (sk != null) { - Object attachment = sk.attachment(); - if (attachment != null) { - TaskWrite tw = new TaskWrite(showLog, node.getIdShort(), node.getChannel(), mo.msg, - (ChannelBuffer) attachment, P2pMgr.this); - tw.run(); - } - } - } else { - if (showLog) - System.out.println("" + mo.displayId + " node-not-exit"); - } - } catch (InterruptedException e) { - if (showLog) - System.out.println(""); - return; - } catch (Exception e) { - if (showLog) - e.printStackTrace(); - } - } - } - } - - private final class TaskReceive implements Runnable { - @Override - public void run() { - - while (P2pMgr.this.start.get()) { - try { - MsgIn mi = receiveMsgQue.take(); - - List hs = handlers.get(mi.route); - if (hs == null) - continue; - for (Handler hlr : hs) { - if (hlr == null) - continue; - - try { - hlr.receive(mi.nid, mi.nsid, mi.msg); - } catch (Exception e) { - if (showLog) - e.printStackTrace(); - } - } - } catch (InterruptedException e) { - if (showLog) - System.out.println(""); - return; - } catch (Exception e) { - if (showLog) - e.printStackTrace(); - } - } - } - } - - private final class TaskStatus implements Runnable { - @Override - public void run() { - Thread.currentThread().setName("p2p-ts"); - String status = nodeMgr.dumpNodeInfo(selfShortId); - System.out.println(status); - System.out.println( - "--------------------------------------------------------------------------------------------------------------------------------------------------------------------"); - System.out.println("recv queue [" + receiveMsgQue.size() + "] send queue [" + sendMsgQue.size() + "]\n"); - } - } - - private final class TaskConnectPeers implements Runnable { - @Override - public void run() { - Thread.currentThread().setName("p2p-tcp"); - while (start.get()) { - try { - Thread.sleep(PERIOD_CONNECT_OUTBOUND); - } catch (InterruptedException e) { - if (showLog) - System.out.println(""); - } - - if (nodeMgr.activeNodesSize() >= maxActiveNodes) { - if (showLog) - System.out.println(""); - continue; - } - - Node node; - try { - node = nodeMgr.tempNodesTake(); - if (nodeMgr.isSeedIp(node.getIpStr())) - node.setFromBootList(true); - if (node.getIfFromBootList()) - nodeMgr.addTempNode(node); - // if (node.peerMetric.shouldNotConn()) { - // continue; - // } - } catch (InterruptedException e) { - if (showLog) - System.out.println(""); - return; - } catch (Exception e) { - if (showLog) - e.printStackTrace(); - continue; - } - int nodeIdHash = node.getIdHash(); - if (!nodeMgr.getOutboundNodes().containsKey(nodeIdHash) && !nodeMgr.hasActiveNode(nodeIdHash)) { - int _port = node.getPort(); - try { - SocketChannel channel = SocketChannel.open(); - - channel.socket().connect(new InetSocketAddress(node.getIpStr(), _port), - TIMEOUT_OUTBOUND_CONNECT); - configChannel(channel); - - if (channel.finishConnect() && channel.isConnected()) { - - if (showLog) - System.out.println(""); - - SelectionKey sk = channel.register(selector, SelectionKey.OP_READ); - ChannelBuffer rb = new ChannelBuffer(P2pMgr.this.showLog); - rb.displayId = node.getIdShort(); - rb.nodeIdHash = nodeIdHash; - sk.attach(rb); - - node.refreshTimestamp(); - node.setChannel(channel); - nodeMgr.addOutboundNode(node); - - if (showLog) - System.out.println(" id=" + node.getIdShort() + " ip=" - + node.getIpStr() + ">"); - sendMsgQue.offer(new MsgOut(node.getIdHash(), node.getIdShort(), cachedReqHandshake1, - Dest.OUTBOUND)); - // node.peerMetric.decFailedCount(); - - } else { - if (showLog) - System.out.println(""); - channel.close(); - // node.peerMetric.incFailedCount(); - } - } catch (IOException e) { - if (showLog) - System.out.println(""); - // node.peerMetric.incFailedCount(); - } catch (Exception e) { - if (showLog) - e.printStackTrace(); - } - } - } - } - } - - private final class TaskClear implements Runnable { - @Override - public void run() { - Thread.currentThread().setName("p2p-clr"); - while (start.get()) { - try { - Thread.sleep(PERIOD_CLEAR); - - nodeMgr.timeoutInbound(P2pMgr.this); - - Iterator outboundIt = nodeMgr.getOutboundNodes().keySet().iterator(); - while (outboundIt.hasNext()) { - - Object obj = outboundIt.next(); - - if (obj == null) - continue; - - int nodeIdHash = (int) obj; - Node node = nodeMgr.getOutboundNodes().get(nodeIdHash); - - if (node == null) - continue; - - if (System.currentTimeMillis() - node.getTimestamp() > TIMEOUT_OUTBOUND_NODES) { - closeSocket(node.getChannel(), "outbound-timeout node=" + node.getIdShort()); - outboundIt.remove(); - } - } - - nodeMgr.timeoutActive(P2pMgr.this); - - } catch (Exception e) { - } - } - } + public enum Dest { + INBOUND, + OUTBOUND, + ACTIVE } /** - * @param _nodeId - * byte[36] - * @param _ip - * String - * @param _port - * int - * @param _bootNodes - * String[] - * @param _upnpEnable - * boolean - * @param _maxTempNodes - * int - * @param _maxActiveNodes - * int - * @param _showStatus - * boolean - * @param _showLog - * boolean + * @param _nodeId byte[36] + * @param _ip String + * @param _port int + * @param _bootNodes String[] + * @param _upnpEnable boolean + * @param _maxTempNodes int + * @param _maxActiveNodes int */ - public P2pMgr(int _netId, String _revision, String _nodeId, String _ip, int _port, final String[] _bootNodes, - boolean _upnpEnable, int _maxTempNodes, int _maxActiveNodes, boolean _showStatus, boolean _showLog, - boolean _bootlistSyncOnly, int _errorTolerance) { + public P2pMgr( + final int _netId, + final String _revision, + final String _nodeId, + final String _ip, + final int _port, + final String[] _bootNodes, + final boolean _upnpEnable, + final int _maxTempNodes, + final int _maxActiveNodes, + final boolean _bootlistSyncOnly, + final int _errorTolerance) { + this.selfNetId = _netId; this.selfRevision = _revision; this.selfNodeId = _nodeId.getBytes(); @@ -568,16 +145,14 @@ public P2pMgr(int _netId, String _revision, String _nodeId, String _ip, int _por this.upnpEnable = _upnpEnable; this.maxTempNodes = _maxTempNodes; this.maxActiveNodes = _maxActiveNodes; - this.showStatus = _showStatus; - this.showLog = _showLog; this.syncSeedsOnly = _bootlistSyncOnly; this.errTolerance = _errorTolerance; - nodeMgr = new NodeMgr(_maxActiveNodes, _maxTempNodes); + nodeMgr = new NodeMgr(this, _maxActiveNodes, _maxTempNodes); for (String _bootNode : _bootNodes) { Node node = Node.parseP2p(_bootNode); - if (node != null && validateNode(node)) { + if (validateNode(node)) { nodeMgr.addTempNode(node); nodeMgr.seedIpAdd(node.getIpStr()); } @@ -588,397 +163,6 @@ public P2pMgr(int _netId, String _revision, String _nodeId, String _ip, int _por cachedResHandshake1 = new ResHandshake1(true, this.selfRevision); } - /** - * @param _node - * Node - * @return boolean - */ - private boolean validateNode(final Node _node) { - if (_node != null) { - boolean notSelfId = !Arrays.equals(_node.getId(), this.selfNodeId); - boolean notSameIpOrPort = !(Arrays.equals(selfIp, _node.getIp()) && selfPort == _node.getPort()); - boolean notActive = !nodeMgr.hasActiveNode(_node.getIdHash()); - boolean notOutbound = !nodeMgr.getOutboundNodes().containsKey(_node.getIdHash()); - return notSelfId && notSameIpOrPort && notActive && notOutbound; - } else - return false; - } - - /** - * @param _channel - * SocketChannel TODO: check option - */ - private void configChannel(final SocketChannel _channel) throws IOException { - _channel.configureBlocking(false); - _channel.socket().setSoTimeout(TIMEOUT_MSG_READ); - - // set buffer to 256k. - _channel.socket().setReceiveBufferSize(P2pConstant.RECV_BUFFER_SIZE); - _channel.socket().setSendBufferSize(P2pConstant.SEND_BUFFER_SIZE); - // _channel.setOption(StandardSocketOptions.SO_KEEPALIVE, true); - // _channel.setOption(StandardSocketOptions.TCP_NODELAY, true); - // _channel.setOption(StandardSocketOptions.SO_REUSEADDR, true); - } - - /** - * @param _sc - * SocketChannel - */ - public void closeSocket(final SocketChannel _sc, String _reason) { - if (showLog) - System.out.println(""); - - try { - SelectionKey sk = _sc.keyFor(selector); - _sc.close(); - if (sk != null) - sk.cancel(); - } catch (IOException e) { - if (showLog) - System.out.println(""); - } - } - - private void accept() { - SocketChannel channel; - try { - - if (nodeMgr.activeNodesSize() >= this.maxActiveNodes) - return; - - channel = tcpServer.accept(); - configChannel(channel); - - SelectionKey sk = channel.register(selector, SelectionKey.OP_READ); - sk.attach(new ChannelBuffer(this.showLog)); - - String ip = channel.socket().getInetAddress().getHostAddress(); - int port = channel.socket().getPort(); - - if (syncSeedsOnly && nodeMgr.isSeedIp(ip)) { - channel.close(); - return; - } - - Node node = nodeMgr.allocNode(ip, 0); - node.setChannel(channel); - nodeMgr.addInboundNode(node); - - if (showLog) - System.out.println(""); - - } catch (IOException e) { - if (showLog) - System.out.println(""); - } - } - - /** - * SocketChannel - * - * @throws IOException - * IOException - */ - private int readHeader(final ChannelBuffer _cb, ByteBuffer readBuffer, int cnt) throws IOException { - - if (cnt < Header.LEN) - return cnt; - - int origPos = readBuffer.position(); - - int startP = origPos - cnt; - - readBuffer.position(startP); - - _cb.readHead(readBuffer); - - readBuffer.position(origPos); - - return cnt - Header.LEN; - - } - - /** - * SocketChannel - * - * @throws IOException - * IOException - */ - private int readBody(final ChannelBuffer _cb, ByteBuffer readBuffer, int cnt) throws IOException { - - int bodyLen = _cb.header.getLen(); - - // some msg have nobody. - if (bodyLen == 0) { - _cb.body = new byte[0]; - return cnt; - } - - if (cnt < bodyLen) - return cnt; - - int origPos = readBuffer.position(); - int startP = origPos - cnt; - - readBuffer.position(startP); - - _cb.readBody(readBuffer); - - readBuffer.position(origPos); - - return cnt - bodyLen; - } - - /** - * @param _sk - * SelectionKey - * @throws IOException - * IOException - */ - private int read(final SelectionKey _sk, ByteBuffer _readBuffer, int _cnt) throws IOException { - - int currCnt = 0; - - if (_sk.attachment() == null) { - throw new P2pException("attachment is null"); - } - ChannelBuffer rb = (ChannelBuffer) _sk.attachment(); - - // read header - if (!rb.isHeaderCompleted()) { - currCnt = readHeader(rb, _readBuffer, _cnt); - } else { - currCnt = _cnt; - } - - // read body - if (rb.isHeaderCompleted() && !rb.isBodyCompleted()) { - currCnt = readBody(rb, _readBuffer, currCnt); - } - - if (!rb.isBodyCompleted()) - return currCnt; - - Header h = rb.header; - - byte[] bodyBytes = rb.body; - rb.refreshHeader(); - rb.refreshBody(); - - short ver = h.getVer(); - byte ctrl = h.getCtrl(); - byte act = h.getAction(); - int route = h.getRoute(); - - boolean underRC = rb.shouldRoute(route, - ((route == txBroadCastRoute) ? P2pConstant.READ_MAX_RATE_TXBC : P2pConstant.READ_MAX_RATE)); - - if (!underRC) { - if (showLog) - System.out.println(""); - return currCnt; - } - - switch (ver) { - case Ver.V0: - switch (ctrl) { - case Ctrl.NET: - try { - handleP2pMsg(_sk, act, bodyBytes); - } catch(Exception ex){ - if(showLog) - System.out.println(""); - } - break; - case Ctrl.SYNC: - - if (!handlers.containsKey(route)) { - if (showLog) - System.out.println(""); - return currCnt; - } - - handleKernelMsg(rb.nodeIdHash, route, bodyBytes); - break; - default: - if (showLog) - System.out.println( - ""); - break; - } - break; - default: - if (showLog) - System.out.println(""); - break; - } - - return currCnt; - - } - - /** - * @return boolean TODO: implementation - */ - private boolean handshakeRuleCheck(int netId) { - - // check net id - if (netId != selfNetId) - return false; - - // check supported protocol versions - return true; - } - - /** - * @param _buffer - * ChannelBuffer - * @param _channelHash - * int - * @param _nodeId - * byte[] - * @param _netId - * int - * @param _port - * int - * @param _revision - * byte[] - *

- * Construct node info after handshake request success - */ - private void handleReqHandshake(final ChannelBuffer _buffer, int _channelHash, final byte[] _nodeId, int _netId, - int _port, final byte[] _revision) { - Node node = nodeMgr.getInboundNode(_channelHash); - if (node != null && node.peerMetric.notBan()) { - if (handshakeRuleCheck(_netId)) { - _buffer.nodeIdHash = Arrays.hashCode(_nodeId); - _buffer.displayId = new String(Arrays.copyOfRange(_nodeId, 0, 6)); - node.setId(_nodeId); - node.setPort(_port); - - // handshake 1 - if (_revision != null) { - String binaryVersion; - try { - binaryVersion = new String(_revision, "UTF-8"); - } catch (UnsupportedEncodingException e) { - binaryVersion = "decode-fail"; - } - node.setBinaryVersion(binaryVersion); - nodeMgr.moveInboundToActive(_channelHash, this); - sendMsgQue.offer(new MsgOut(node.getIdHash(), node.getIdShort(), cachedResHandshake1, Dest.ACTIVE)); - } - - } else { - if (showLog) - System.out.println(""); - } - } - } - - private void handleResHandshake(int _nodeIdHash, String _binaryVersion) { - Node node = nodeMgr.getOutboundNodes().get(_nodeIdHash); - if (node != null && node.peerMetric.notBan()) { - node.refreshTimestamp(); - node.setBinaryVersion(_binaryVersion); - nodeMgr.moveOutboundToActive(node.getIdHash(), node.getIdShort(), this); - } - } - - /** - * @param _sk - * SelectionKey - * @param _act - * ACT - * @param _msgBytes - * byte[] - */ - private void handleP2pMsg(final SelectionKey _sk, byte _act, final byte[] _msgBytes) { - - - ChannelBuffer rb = (ChannelBuffer) _sk.attachment(); - - switch (_act) { - - case Act.REQ_HANDSHAKE: - if (_msgBytes.length > ReqHandshake.LEN) { - ReqHandshake1 reqHandshake1 = ReqHandshake1.decode(_msgBytes); - if (reqHandshake1 != null) { - handleReqHandshake(rb, _sk.channel().hashCode(), reqHandshake1.getNodeId(), - reqHandshake1.getNetId(), reqHandshake1.getPort(), reqHandshake1.getRevision()); - } - } - break; - - case Act.RES_HANDSHAKE: - if (rb.nodeIdHash == 0) - return; - - if (_msgBytes.length > ResHandshake.LEN) { - ResHandshake1 resHandshake1 = ResHandshake1.decode(_msgBytes); - if (resHandshake1 != null && resHandshake1.getSuccess()) - handleResHandshake(rb.nodeIdHash, resHandshake1.getBinaryVersion()); - - } - break; - - case Act.REQ_ACTIVE_NODES: - if (rb.nodeIdHash != 0) { - Node node = nodeMgr.getActiveNode(rb.nodeIdHash); - if (node != null) - sendMsgQue.offer(new MsgOut(node.getIdHash(), node.getIdShort(), - new ResActiveNodes(nodeMgr.getActiveNodesList()), Dest.ACTIVE)); - } - break; - - case Act.RES_ACTIVE_NODES: - if (syncSeedsOnly) - break; - - if (rb.nodeIdHash != 0) { - Node node = nodeMgr.getActiveNode(rb.nodeIdHash); - if (node != null) { - node.refreshTimestamp(); - ResActiveNodes resActiveNodes = ResActiveNodes.decode(_msgBytes); - if (resActiveNodes != null) { - List incomingNodes = resActiveNodes.getNodes(); - for (Node incomingNode : incomingNodes) { - if (nodeMgr.tempNodesSize() >= this.maxTempNodes) - return; - if (validateNode(incomingNode)) - nodeMgr.addTempNode(incomingNode); - } - } - } - } - break; - default: - if (showLog) - System.out.println(""); - break; - } - } - - /** - * @param _nodeIdHash - * int - * @param _route - * int - * @param _msgBytes - * byte[] - */ - private void handleKernelMsg(int _nodeIdHash, int _route, final byte[] _msgBytes) { - Node node = nodeMgr.getActiveNode(_nodeIdHash); - if (node != null) { - int nodeIdHash = node.getIdHash(); - String nodeDisplayId = node.getIdShort(); - node.refreshTimestamp(); - receiveMsgQue.offer(new MsgIn(nodeIdHash, nodeDisplayId, _route, _msgBytes)); - } - } - @Override public void run() { try { @@ -992,69 +176,72 @@ public void run() { tcpServer.socket().bind(new InetSocketAddress(Node.ipBytesToStr(selfIp), selfPort)); tcpServer.register(selector, SelectionKey.OP_ACCEPT); - Thread thrdIn = new Thread(new TaskInbound(), "p2p-in"); + Thread thrdIn = new Thread(getInboundInstance(), "p2p-in"); thrdIn.setPriority(Thread.NORM_PRIORITY); thrdIn.start(); - if (showLog) - this.handlers.forEach((route, callbacks) -> { - Handler handler = callbacks.get(0); - Header h = handler.getHeader(); - System.out.println(""); - }); + if (p2pLOG.isDebugEnabled()) { + this.handlers.forEach( + (route, callbacks) -> { + Handler handler = callbacks.get(0); + Header h = handler.getHeader(); + p2pLOG.debug("handler route={} v-c-a={}-{}-{} name={}", route, h.getVer(), + h.getCtrl(), h.getAction(), handler.getClass().getSimpleName()); + }); + } - for (int i = 0; i < TaskSend.TOTAL_LANE; i++) { - Thread thrdOut = new Thread(new TaskSend(i), "p2p-out-" + i); + int pNum = Runtime.getRuntime().availableProcessors(); + + for (int i = 0; i < (pNum << 1); i++) { + Thread thrdOut = new Thread(getSendInstance(i), "p2p-out-" + i); thrdOut.setPriority(Thread.NORM_PRIORITY); thrdOut.start(); } - for (int i = 0, m = Runtime.getRuntime().availableProcessors(); i < m; i++) { - Thread t = new Thread(new TaskReceive(), "p2p-worker-" + i); + for (int i = 0; i < pNum; i++) { + Thread t = new Thread(getReceiveInstance(), "p2p-worker-" + i); t.setPriority(Thread.NORM_PRIORITY); t.start(); } - if (upnpEnable) - scheduledWorkers.scheduleWithFixedDelay(new TaskUPnPManager(selfPort), 1, PERIOD_UPNP_PORT_MAPPING, - TimeUnit.MILLISECONDS); + if (upnpEnable) { + scheduledWorkers.scheduleWithFixedDelay( + new TaskUPnPManager(selfPort), + 1, + PERIOD_UPNP_PORT_MAPPING, + TimeUnit.MILLISECONDS); + } - if (showStatus) - scheduledWorkers.scheduleWithFixedDelay(new TaskStatus(), 2, PERIOD_SHOW_STATUS, TimeUnit.MILLISECONDS); + if (p2pLOG.isDebugEnabled()) { + scheduledWorkers.scheduleWithFixedDelay( + getStatusInstance(), + 2, + PERIOD_SHOW_STATUS, + TimeUnit.MILLISECONDS); + } - if (!syncSeedsOnly) - scheduledWorkers.scheduleWithFixedDelay(new TaskRequestActiveNodes(this), 5000, - PERIOD_REQUEST_ACTIVE_NODES, TimeUnit.MILLISECONDS); + if (!syncSeedsOnly) { + scheduledWorkers.scheduleWithFixedDelay( + new TaskRequestActiveNodes(this), + 5000, + PERIOD_REQUEST_ACTIVE_NODES, + TimeUnit.MILLISECONDS); + } - Thread thrdClear = new Thread(new TaskClear(), "p2p-clear"); + Thread thrdClear = new Thread(getClearInstance(), "p2p-clear"); thrdClear.setPriority(Thread.NORM_PRIORITY); thrdClear.start(); - Thread thrdConn = new Thread(new TaskConnectPeers(), "p2p-conn"); + Thread thrdConn = new Thread(getConnectPeersInstance(), "p2p-conn"); thrdConn.setPriority(Thread.NORM_PRIORITY); thrdConn.start(); - + } catch (SocketException e) { + p2pLOG.error("tcp-server-socket-exception {}", e.getMessage()); } catch (IOException e) { - if (showLog) - System.out.println(""); + p2pLOG.error("tcp-server-io-exception {}", e.getMessage()); } } - @Override - public INode getRandom() { - return nodeMgr.getRandom(); - } - - @Override - public Map getActiveNodes() { - return new HashMap<>(this.nodeMgr.getActiveNodesMap()); - } - - public int getTempNodesCount() { - return nodeMgr.tempNodesSize(); - } - @Override public void register(final List _cbs) { for (Handler _cb : _cbs) { @@ -1062,9 +249,7 @@ public void register(final List _cbs) { short ver = h.getVer(); byte ctrl = h.getCtrl(); if (Ver.filter(ver) != Ver.UNKNOWN && Ctrl.filter(ctrl) != Ctrl.UNKNOWN) { - if (!versions.contains(ver)) { - versions.add(ver); - } + versions.add(ver); int route = h.getRoute(); List routeHandlers = handlers.get(route); @@ -1079,8 +264,7 @@ public void register(final List _cbs) { } List supportedVersions = new ArrayList<>(versions); - cachedReqHandshake1 = new ReqHandshake1(selfNodeId, selfNetId, this.selfIp, this.selfPort, - this.selfRevision.getBytes(), supportedVersions); + cachedReqHandshake1 = getReqHandshake1Instance(supportedVersions); } @Override @@ -1096,8 +280,7 @@ public void shutdown() { for (List hdrs : handlers.values()) { hdrs.forEach(Handler::shutDown); } - - nodeMgr.shutdown(this); + nodeMgr.shutdown(); } @Override @@ -1106,48 +289,196 @@ public List versions() { } @Override - public int chainId() { - return selfNetId; + public void errCheck(int _nodeIdHash, String _displayId) { + int cnt = (errCnt.get(_nodeIdHash) == null ? 1 : (errCnt.get(_nodeIdHash) + 1)); + if (cnt > this.errTolerance) { + ban(_nodeIdHash); + errCnt.put(_nodeIdHash, 0); + + if (p2pLOG.isDebugEnabled()) { + p2pLOG.debug("ban node={} err-count={}", + (_displayId == null ? _nodeIdHash : _displayId), cnt); + } + } else { + errCnt.put(_nodeIdHash, cnt); + } } - @Override - public int getSelfIdHash() { - return this.selfNodeIdHash; + /** + * @param _sc SocketChannel + */ + public void closeSocket(final SocketChannel _sc, String _reason) { + + if (p2pLOG.isDebugEnabled()) { + p2pLOG.debug("close-socket reason={}", _reason); + } + + if (_sc != null) { + SelectionKey sk = _sc.keyFor(selector); + if (sk != null) { + sk.cancel(); + } + + try { + _sc.close(); + } catch (IOException e) { + p2pLOG.info("close-socket-io-exception, {}", e.getMessage()); + } + } } /** * Remove an active node if exists. * - * @param _nodeIdHash - * int - * @param _reason - * String + * @param _nodeIdHash int + * @param _reason String */ - void dropActive(int _nodeIdHash, String _reason) { - nodeMgr.dropActive(_nodeIdHash, this, _reason); - } - - public boolean isShowLog() { - return showLog; + @Override + public void dropActive(int _nodeIdHash, String _reason) { + nodeMgr.dropActive(_nodeIdHash, _reason); } + /** + * @param _node Node + * @return boolean + */ @Override - public void errCheck(int _nodeIdHash, String _displayId) { - int cnt = (errCnt.get(_nodeIdHash) == null ? 1 : (errCnt.get(_nodeIdHash) + 1)); - if (cnt > this.errTolerance) { - ban(_nodeIdHash); - errCnt.put(_nodeIdHash, 0); - if (showLog) { - System.out.println( - ""); - } + public boolean validateNode(final INode _node) { + if (_node != null) { + boolean notSelfId = !Arrays.equals(_node.getId(), this.selfNodeId); + boolean notSameIpOrPort = + !(Arrays.equals(selfIp, _node.getIp()) && selfPort == _node.getPort()); + boolean notActive = nodeMgr.notActiveNode(_node.getIdHash()); + boolean notOutbound = !nodeMgr.getOutboundNodes().containsKey(_node.getIdHash()); + return notSelfId && notSameIpOrPort && notActive && notOutbound; } else { - errCnt.put(_nodeIdHash, cnt); + return false; } } + /** + * @param _channel SocketChannel TODO: check option + */ + @Override + public void configChannel(final SocketChannel _channel) throws IOException { + _channel.configureBlocking(false); + _channel.socket().setSoTimeout(TIMEOUT_MSG_READ); + _channel.socket().setReceiveBufferSize(P2pConstant.RECV_BUFFER_SIZE); + _channel.socket().setSendBufferSize(P2pConstant.SEND_BUFFER_SIZE); + } + private void ban(int nodeIdHashcode) { nodeMgr.ban(nodeIdHashcode); - nodeMgr.dropActive(nodeIdHashcode, this, "ban"); + nodeMgr.dropActive(nodeIdHashcode, "ban"); + } + + // <------------------------ getter methods below ---------------------------> + + @Override + public INode getRandom() { + return this.nodeMgr.getRandom(); + } + + @Override + public Map getActiveNodes() { + return this.nodeMgr.getActiveNodesMap(); } -} \ No newline at end of file + + @Override + public int chainId() { + return this.selfNetId; + } + + @Override + public int getSelfIdHash() { + return this.selfNodeIdHash; + } + + public int getTempNodesCount() { + return this.nodeMgr.tempNodesSize(); + } + + @Override + public int getMaxActiveNodes() { + return this.maxActiveNodes; + } + + @Override + public int getMaxTempNodes() { + return this.maxTempNodes; + } + + @Override + public int getSelfNetId() { + return this.selfNetId; + } + + @Override + public boolean isSyncSeedsOnly() { + return this.syncSeedsOnly; + } + + + private TaskInbound getInboundInstance() { + return new TaskInbound( + this, + this.selector, + this.start, + this.nodeMgr, + this.tcpServer, + this.handlers, + this.sendMsgQue, + cachedResHandshake1, + this.receiveMsgQue); + } + + private TaskSend getSendInstance(int i) { + return new TaskSend( + this, + i, + this.sendMsgQue, + this.start, + this.nodeMgr, + this.selector); + } + + private TaskReceive getReceiveInstance() { + return new TaskReceive( + this.start, + this.receiveMsgQue, + this.handlers); + } + + private TaskStatus getStatusInstance() { + return new TaskStatus( + this.nodeMgr, + this.selfShortId, + this.sendMsgQue, + this.receiveMsgQue); + } + + private TaskClear getClearInstance() { + return new TaskClear(this, this.nodeMgr, this.start); + } + + private TaskConnectPeers getConnectPeersInstance() { + return new TaskConnectPeers( + this, + this.start, + this.nodeMgr, + this.maxActiveNodes, + this.selector, + this.sendMsgQue, + cachedReqHandshake1); + } + + private ReqHandshake1 getReqHandshake1Instance(List versions) { + return new ReqHandshake1( + selfNodeId, + selfNetId, + this.selfIp, + this.selfPort, + this.selfRevision.getBytes(), + versions); + } +} diff --git a/modP2pImpl/src/org/aion/p2p/impl1/TaskWrite.java b/modP2pImpl/src/org/aion/p2p/impl1/TaskWrite.java deleted file mode 100644 index 126ffdd557..0000000000 --- a/modP2pImpl/src/org/aion/p2p/impl1/TaskWrite.java +++ /dev/null @@ -1,122 +0,0 @@ -/* - * Copyright (c) 2017-2018 Aion foundation. - * - * This file is part of the aion network project. - * - * The aion network project is free software: you can redistribute it - * and/or modify it under the terms of the GNU General Public License - * as published by the Free Software Foundation, either version 3 of - * the License, or any later version. - * - * The aion network project is distributed in the hope that it will - * be useful, but WITHOUT ANY WARRANTY; without even the implied - * warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. - * See the GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with the aion network project source files. - * If not, see . - * - * Contributors to the aion source files in decreasing order of code volume: - * - * Aion foundation. - * - */ - -package org.aion.p2p.impl1; - -import java.io.IOException; -import java.nio.ByteBuffer; -import java.nio.channels.ClosedChannelException; -import java.nio.channels.SocketChannel; -import org.aion.p2p.Header; -import org.aion.p2p.Msg; - -/** @author chris */ -public class TaskWrite implements Runnable { - - private boolean showLog; - private String nodeShortId; - private SocketChannel sc; - private Msg msg; - private ChannelBuffer channelBuffer; - private P2pMgr p2pMgr; - - TaskWrite( - boolean _showLog, - String _nodeShortId, - final SocketChannel _sc, - final Msg _msg, - final ChannelBuffer _cb, - final P2pMgr _p2pMgr) { - this.showLog = _showLog; - this.nodeShortId = _nodeShortId; - this.sc = _sc; - this.msg = _msg; - this.channelBuffer = _cb; - this.p2pMgr = _p2pMgr; - } - - @Override - public void run() { - // reset allocated buffer and clear messages if the channel is closed - if (channelBuffer.isClosed.get()) { - channelBuffer.refreshHeader(); - channelBuffer.refreshBody(); - p2pMgr.dropActive(channelBuffer.nodeIdHash, "close-already"); - return; - } - - try { - channelBuffer.lock.lock(); - - /* - * @warning header set len (body len) before header encode - */ - byte[] bodyBytes = msg.encode(); - int bodyLen = bodyBytes == null ? 0 : bodyBytes.length; - Header h = msg.getHeader(); - h.setLen(bodyLen); - byte[] headerBytes = h.encode(); - - // print route - // System.out.println("write " + h.getVer() + "-" + h.getCtrl() + "-" + h.getAction()); - ByteBuffer buf = ByteBuffer.allocate(headerBytes.length + bodyLen); - buf.put(headerBytes); - if (bodyBytes != null) buf.put(bodyBytes); - buf.flip(); - - try { - while (buf.hasRemaining()) { - // @Attention: very important sleep , otherwise when NIO write buffer full, - // without sleep will hangup this thread. - Thread.sleep(0, 1); - sc.write(buf); - } - } catch (ClosedChannelException ex1) { - if (showLog) { - System.out.println( - ""); - } - channelBuffer.isClosed.set(true); - } catch (IOException ex2) { - String reason = ex2.getMessage(); - if (showLog) { - System.out.println( - ""); - } - if (reason.equals("Broken pipe")) { - channelBuffer.isClosed.set(true); - } - } - } catch (Exception e) { - e.printStackTrace(); - } finally { - channelBuffer.lock.unlock(); - } - } -} diff --git a/modP2pImpl/src/org/aion/p2p/impl1/ChannelBuffer.java b/modP2pImpl/src/org/aion/p2p/impl1/tasks/ChannelBuffer.java similarity index 51% rename from modP2pImpl/src/org/aion/p2p/impl1/ChannelBuffer.java rename to modP2pImpl/src/org/aion/p2p/impl1/tasks/ChannelBuffer.java index a3d8c54399..018f0132c4 100644 --- a/modP2pImpl/src/org/aion/p2p/impl1/ChannelBuffer.java +++ b/modP2pImpl/src/org/aion/p2p/impl1/tasks/ChannelBuffer.java @@ -1,63 +1,100 @@ /* * Copyright (c) 2017-2018 Aion foundation. * - * This file is part of the aion network project. + * This file is part of the aion network project. * - * The aion network project is free software: you can redistribute it - * and/or modify it under the terms of the GNU General Public License - * as published by the Free Software Foundation, either version 3 of - * the License, or any later version. + * The aion network project is free software: you can redistribute it + * and/or modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation, either version 3 of + * the License, or any later version. * - * The aion network project is distributed in the hope that it will - * be useful, but WITHOUT ANY WARRANTY; without even the implied - * warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. - * See the GNU General Public License for more details. + * The aion network project is distributed in the hope that it will + * be useful, but WITHOUT ANY WARRANTY; without even the implied + * warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + * See the GNU General Public License for more details. * - * You should have received a copy of the GNU General Public License - * along with the aion network project source files. - * If not, see . - * - * Contributors to the aion source files in decreasing order of code volume: - * - * Aion foundation. + * You should have received a copy of the GNU General Public License + * along with the aion network project source files. + * If not, see . * + * Contributors: + * Aion foundation. */ -package org.aion.p2p.impl1; +package org.aion.p2p.impl1.tasks; -import org.aion.p2p.Header; +import static org.aion.p2p.impl1.P2pMgr.p2pLOG; import java.nio.ByteBuffer; import java.util.HashMap; import java.util.Map; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.locks.Lock; +import org.aion.p2p.Header; /** * @author chris */ class ChannelBuffer { + // buffer for buffer remaining after NIO select read. + byte[] remainBuffer; + + int buffRemain = 0; + + private int nodeIdHash; + + private String displayId; + + Header header = null; + + private byte[] bsHead = new byte[Header.LEN]; + + byte[] body = null; + + Lock lock = new java.util.concurrent.locks.ReentrantLock(); + + /** + * Indicates whether this channel is closed. + */ + AtomicBoolean isClosed = new AtomicBoolean(false); + + private Map routes = new HashMap<>(); + + public String getDisplayId() { + return displayId; + } + + void setNodeIdHash(int nodeIdHash) { + this.nodeIdHash = nodeIdHash; + } + + void setDisplayId(String displayId) { + this.displayId = displayId; + } + + int getNodeIdHash() { + return nodeIdHash; + } + class RouteStatus { + long timestamp; int count; - RouteStatus(){ + + RouteStatus() { this.timestamp = System.currentTimeMillis(); count = 0; } } - private boolean showLog; - - private Map routes = new HashMap<>(); - ChannelBuffer(boolean _showLog){ - this.showLog = _showLog; + ChannelBuffer() { } /** - * @param _route int - * @param _maxReqsPerSec int requests within 1 s - * @return boolean flag if under route control + * @param _route int + * @param _maxReqsPerSec int requests within 1 s + * @return boolean flag if under route control */ synchronized boolean shouldRoute(int _route, int _maxReqsPerSec) { long now = System.currentTimeMillis(); @@ -69,48 +106,27 @@ synchronized boolean shouldRoute(int _route, int _maxReqsPerSec) { return true; } boolean shouldRoute = prev.count < _maxReqsPerSec; - if(shouldRoute) + if (shouldRoute) { prev.count++; - - if(showLog) { - if(!shouldRoute) - System.out.println(""); - // too many msgs - //else - // System.out.println(""); + } else { + if (p2pLOG.isDebugEnabled()) { + p2pLOG + .debug("route-cooldown={} node={} count={}", _route, this.getDisplayId(), + prev.count); + } } + return shouldRoute; - } else + } else { return true; + } } - RouteStatus getRouteCount(int _route){ + RouteStatus getRouteCount(int _route) { return routes.get(_route); } - // buffer for buffer remaining after NIO select read. - byte[] remainBuffer; - - int buffRemain = 0; - - int nodeIdHash = 0; - - String displayId = ""; - - Header header = null; - - private byte[] bsHead = new byte[Header.LEN]; - - byte[] body = null; - - Lock lock = new java.util.concurrent.locks.ReentrantLock(); - - /** - * Indicates whether this channel is closed. - */ - AtomicBoolean isClosed = new AtomicBoolean(false); - void readHead(ByteBuffer buf) { buf.get(bsHead); try { @@ -136,15 +152,15 @@ void refreshBody() { /** * @return boolean */ - boolean isHeaderCompleted() { - return header != null; + boolean isHeaderNotCompleted() { + return header == null; } /** * @return boolean */ - boolean isBodyCompleted() { - return this.header != null && this.body != null && body.length == header.getLen(); + boolean isBodyNotCompleted() { + return this.header == null || this.body == null || body.length != header.getLen(); } } diff --git a/modP2pImpl/src/org/aion/p2p/impl1/tasks/MsgIn.java b/modP2pImpl/src/org/aion/p2p/impl1/tasks/MsgIn.java new file mode 100644 index 0000000000..bff5fd7e23 --- /dev/null +++ b/modP2pImpl/src/org/aion/p2p/impl1/tasks/MsgIn.java @@ -0,0 +1,64 @@ +/* + * Copyright (c) 2017-2018 Aion foundation. + * + * This file is part of the aion network project. + * + * The aion network project is free software: you can redistribute it + * and/or modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation, either version 3 of + * the License, or any later version. + * + * The aion network project is distributed in the hope that it will + * be useful, but WITHOUT ANY WARRANTY; without even the implied + * warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + * See the GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with the aion network project source files. + * If not, see . + * + * Contributors: + * Aion foundation. + */ +package org.aion.p2p.impl1.tasks; + +/** + * An incoming message. + */ +public class MsgIn { + private final int nodeId; + private final String displayId; + private final int route; + private final byte[] msg; + + /** + * Constructs an incoming message. + * + * @param nodeId The node id. + * @param displayId The display id. + * @param route The route. + * @param msg The message. + */ + MsgIn(final int nodeId, final String displayId, final int route, final byte[] msg) { + this.nodeId = nodeId; + this.displayId = displayId; + this.route = route; + this.msg = msg; + } + + public int getNodeId() { + return this.nodeId; + } + + String getDisplayId() { + return this.displayId; + } + + int getRoute() { + return this.route; + } + + public byte[] getMsg() { + return this.msg; + } +} diff --git a/modP2pImpl/src/org/aion/p2p/impl1/tasks/MsgOut.java b/modP2pImpl/src/org/aion/p2p/impl1/tasks/MsgOut.java new file mode 100644 index 0000000000..1c6c3c0b0b --- /dev/null +++ b/modP2pImpl/src/org/aion/p2p/impl1/tasks/MsgOut.java @@ -0,0 +1,80 @@ +/* + * Copyright (c) 2017-2018 Aion foundation. + * + * This file is part of the aion network project. + * + * The aion network project is free software: you can redistribute it + * and/or modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation, either version 3 of + * the License, or any later version. + * + * The aion network project is distributed in the hope that it will + * be useful, but WITHOUT ANY WARRANTY; without even the implied + * warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + * See the GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with the aion network project source files. + * If not, see . + * + * Contributors: + * Aion foundation. + */ +package org.aion.p2p.impl1.tasks; + +import org.aion.p2p.Msg; +import org.aion.p2p.impl1.P2pMgr.Dest; + +/** + * An outgoing message. + */ +public class MsgOut { + + private final int nodeId; + private final String displayId; + private final int lane; + private final Msg msg; + private final Dest dest; + private final long timestamp; + + /** + * Constructs an outgoing message. + * + * @param nodeId The node id. + * @param displayId The display id. + * @param msg The message. + * @param dest The destination. + */ + public MsgOut(final int nodeId, final String displayId, final Msg msg, final Dest dest) { + this.nodeId = nodeId; + this.displayId = displayId; + this.msg = msg; + this.dest = dest; + this.lane = TaskSend.hash2Lane(nodeId); + this.timestamp = System.currentTimeMillis(); + } + + public int getNodeId() { + return this.nodeId; + } + + String getDisplayId() { + return this.displayId; + } + + public Msg getMsg() { + return this.msg; + } + + Dest getDest() { + return this.dest; + } + + public long getTimestamp() { + return this.timestamp; + } + + int getLane() { + return this.lane; + } +} diff --git a/modP2pImpl/src/org/aion/p2p/impl1/tasks/TaskClear.java b/modP2pImpl/src/org/aion/p2p/impl1/tasks/TaskClear.java new file mode 100644 index 0000000000..29834cd76d --- /dev/null +++ b/modP2pImpl/src/org/aion/p2p/impl1/tasks/TaskClear.java @@ -0,0 +1,57 @@ +/* + * Copyright (c) 2017-2018 Aion foundation. + * + * This file is part of the aion network project. + * + * The aion network project is free software: you can redistribute it + * and/or modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation, either version 3 of + * the License, or any later version. + * + * The aion network project is distributed in the hope that it will + * be useful, but WITHOUT ANY WARRANTY; without even the implied + * warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + * See the GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with the aion network project source files. + * If not, see . + * + * Contributors: + * Aion foundation. + */ +package org.aion.p2p.impl1.tasks; + +import static org.aion.p2p.impl1.P2pMgr.p2pLOG; + +import java.util.concurrent.atomic.AtomicBoolean; +import org.aion.p2p.INodeMgr; +import org.aion.p2p.IP2pMgr; + +public class TaskClear implements Runnable { + + private static final int PERIOD_CLEAR = 20000; + + private final IP2pMgr mgr; + private final INodeMgr nodeMgr; + private final AtomicBoolean start; + + public TaskClear(final IP2pMgr _mgr, final INodeMgr _nodeMgr, final AtomicBoolean _start) { + this.mgr = _mgr; + this.nodeMgr = _nodeMgr; + this.start = _start; + } + + @Override + public void run() { + while (start.get()) { + try { + Thread.sleep(PERIOD_CLEAR); + nodeMgr.timeoutCheck(); + } catch (Exception e) { + e.printStackTrace(); + p2pLOG.error("TaskClear exception {}", e.getMessage()); + } + } + } +} diff --git a/modP2pImpl/src/org/aion/p2p/impl1/tasks/TaskConnectPeers.java b/modP2pImpl/src/org/aion/p2p/impl1/tasks/TaskConnectPeers.java new file mode 100644 index 0000000000..883dfa1766 --- /dev/null +++ b/modP2pImpl/src/org/aion/p2p/impl1/tasks/TaskConnectPeers.java @@ -0,0 +1,174 @@ +/* + * Copyright (c) 2017-2018 Aion foundation. + * + * This file is part of the aion network project. + * + * The aion network project is free software: you can redistribute it + * and/or modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation, either version 3 of + * the License, or any later version. + * + * The aion network project is distributed in the hope that it will + * be useful, but WITHOUT ANY WARRANTY; without even the implied + * warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + * See the GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with the aion network project source files. + * If not, see . + * + * Contributors: + * Aion foundation. + */ +package org.aion.p2p.impl1.tasks; + +import static org.aion.p2p.impl1.P2pMgr.p2pLOG; + +import java.io.IOException; +import java.net.InetSocketAddress; +import java.nio.channels.SelectionKey; +import java.nio.channels.Selector; +import java.nio.channels.SocketChannel; +import java.util.concurrent.BlockingQueue; +import java.util.concurrent.atomic.AtomicBoolean; +import org.aion.p2p.INode; +import org.aion.p2p.INodeMgr; +import org.aion.p2p.IP2pMgr; +import org.aion.p2p.impl.zero.msg.ReqHandshake1; +import org.aion.p2p.impl1.P2pMgr.Dest; + +public class TaskConnectPeers implements Runnable { + + private static final int PERIOD_CONNECT_OUTBOUND = 1000; + private static final int TIMEOUT_OUTBOUND_CONNECT = 10000; + + private final INodeMgr nodeMgr; + private final int maxActiveNodes; + private final IP2pMgr mgr; + private final AtomicBoolean start; + private final BlockingQueue sendMsgQue; + private final Selector selector; + private final ReqHandshake1 cachedReqHS; + + public TaskConnectPeers( + final IP2pMgr _mgr, + final AtomicBoolean _start, + final INodeMgr _nodeMgr, + final int _maxActiveNodes, + final Selector _selector, + final BlockingQueue _sendMsgQue, + final ReqHandshake1 _cachedReqHS) { + + this.start = _start; + this.nodeMgr = _nodeMgr; + this.maxActiveNodes = _maxActiveNodes; + this.mgr = _mgr; + this.selector = _selector; + this.sendMsgQue = _sendMsgQue; + this.cachedReqHS = _cachedReqHS; + } + + @Override + public void run() { + Thread.currentThread().setName("p2p-tcp"); + while (this.start.get()) { + try { + Thread.sleep(PERIOD_CONNECT_OUTBOUND); + } catch (InterruptedException e) { + p2pLOG.warn("tcp-interrupted"); + } + + if (this.nodeMgr.activeNodesSize() >= this.maxActiveNodes) { + p2pLOG.warn("tcp-connect-peer pass max-active-nodes"); + continue; + } + + INode node; + try { + node = this.nodeMgr.tempNodesTake(); + if (this.nodeMgr.isSeedIp(node.getIpStr())) { + node.setFromBootList(true); + } + if (node.getIfFromBootList()) { + this.nodeMgr.addTempNode(node); + } + // if (node.peerMetric.shouldNotConn()) { + // continue; + // } + } catch (InterruptedException e) { + p2pLOG.error("tcp-interrupted"); + return; + } catch (Exception e) { + e.printStackTrace(); + p2pLOG.warn("tcp-Exception {}", e.getMessage()); + continue; + } + int nodeIdHash = node.getIdHash(); + if (!this.nodeMgr.getOutboundNodes().containsKey(nodeIdHash) + && this.nodeMgr.notActiveNode(nodeIdHash)) { + int _port = node.getPort(); + try { + SocketChannel channel = SocketChannel.open(); + + channel.socket() + .connect( + new InetSocketAddress(node.getIpStr(), _port), + TIMEOUT_OUTBOUND_CONNECT); + this.mgr.configChannel(channel); + + if (channel.isConnected()) { + + if (p2pLOG.isDebugEnabled()) { + p2pLOG.debug("success-connect node-id={} ip=", node.getIdShort(), + node.getIpStr()); + } + + SelectionKey sk = channel.register(this.selector, SelectionKey.OP_READ); + ChannelBuffer rb = new ChannelBuffer(); + rb.setDisplayId(node.getIdShort()); + rb.setNodeIdHash(nodeIdHash); + sk.attach(rb); + + node.refreshTimestamp(); + node.setChannel(channel); + this.nodeMgr.addOutboundNode(node); + + if (p2pLOG.isDebugEnabled()) { + p2pLOG.debug("prepare-request-handshake -> id={} ip={}", + node.getIdShort(), node.getIpStr()); + } + + this.sendMsgQue.offer( + new MsgOut( + node.getIdHash(), + node.getIdShort(), + this.cachedReqHS, + Dest.OUTBOUND)); + // node.peerMetric.decFailedCount(); + + } else { + if (p2pLOG.isDebugEnabled()) { + p2pLOG + .debug("fail-connect node-id -> id={} ip={}", node.getIdShort(), + node.getIpStr()); + } + + channel.close(); + // node.peerMetric.incFailedCount(); + } + } catch (IOException e) { + if (p2pLOG.isDebugEnabled()) { + p2pLOG.debug("connect-outbound io-exception addr={}:{} result={}", + node.getIpStr(), _port, e.getMessage()); + } + // node.peerMetric.incFailedCount(); + } catch (Exception e) { + e.printStackTrace(); + p2pLOG + .debug("connect-outbound exception -> id={} ip={}", node.getIdShort(), + node.getIpStr()); + } + } + } + } +} diff --git a/modP2pImpl/src/org/aion/p2p/impl1/tasks/TaskInbound.java b/modP2pImpl/src/org/aion/p2p/impl1/tasks/TaskInbound.java new file mode 100644 index 0000000000..b36f01304a --- /dev/null +++ b/modP2pImpl/src/org/aion/p2p/impl1/tasks/TaskInbound.java @@ -0,0 +1,585 @@ +/* + * Copyright (c) 2017-2018 Aion foundation. + * + * This file is part of the aion network project. + * + * The aion network project is free software: you can redistribute it + * and/or modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation, either version 3 of + * the License, or any later version. + * + * The aion network project is distributed in the hope that it will + * be useful, but WITHOUT ANY WARRANTY; without even the implied + * warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + * See the GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with the aion network project source files. + * If not, see . + * + * Contributors: + * Aion foundation. + */ +package org.aion.p2p.impl1.tasks; + +import static org.aion.p2p.impl1.P2pMgr.p2pLOG; +import static org.aion.p2p.impl1.P2pMgr.txBroadCastRoute; + +import java.io.IOException; +import java.io.UnsupportedEncodingException; +import java.nio.ByteBuffer; +import java.nio.channels.ClosedChannelException; +import java.nio.channels.ClosedSelectorException; +import java.nio.channels.SelectionKey; +import java.nio.channels.Selector; +import java.nio.channels.ServerSocketChannel; +import java.nio.channels.SocketChannel; +import java.util.Arrays; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.concurrent.BlockingQueue; +import java.util.concurrent.atomic.AtomicBoolean; +import org.aion.p2p.Ctrl; +import org.aion.p2p.Handler; +import org.aion.p2p.Header; +import org.aion.p2p.INode; +import org.aion.p2p.INodeMgr; +import org.aion.p2p.IP2pMgr; +import org.aion.p2p.P2pConstant; +import org.aion.p2p.Ver; +import org.aion.p2p.impl.comm.Act; +import org.aion.p2p.impl.zero.msg.ReqHandshake; +import org.aion.p2p.impl.zero.msg.ReqHandshake1; +import org.aion.p2p.impl.zero.msg.ResActiveNodes; +import org.aion.p2p.impl.zero.msg.ResHandshake; +import org.aion.p2p.impl.zero.msg.ResHandshake1; +import org.aion.p2p.impl1.P2pException; +import org.aion.p2p.impl1.P2pMgr.Dest; + +public class TaskInbound implements Runnable { + + private final IP2pMgr mgr; + private final Selector selector; + private final INodeMgr nodeMgr; + private final Map> handlers; + private final AtomicBoolean start; + private final ServerSocketChannel tcpServer; + private final BlockingQueue sendMsgQue; + private final ResHandshake1 cachedResHandshake1; + private final BlockingQueue receiveMsgQue; + + public TaskInbound( + final IP2pMgr _mgr, + final Selector _selector, + final AtomicBoolean _start, + final INodeMgr _nodeMgr, + final ServerSocketChannel _tcpServer, + final Map> _handlers, + final BlockingQueue _sendMsgQue, + final ResHandshake1 _cachedResHandshake1, + final BlockingQueue _receiveMsgQue) { + + this.mgr = _mgr; + this.selector = _selector; + this.start = _start; + this.nodeMgr = _nodeMgr; + this.tcpServer = _tcpServer; + this.handlers = _handlers; + this.sendMsgQue = _sendMsgQue; + this.cachedResHandshake1 = _cachedResHandshake1; + this.receiveMsgQue = _receiveMsgQue; + } + + @Override + public void run() { + + // readBuffer buffer pre-alloc. @ max_body_size + ByteBuffer readBuf = ByteBuffer.allocate(P2pConstant.MAX_BODY_SIZE); + + while (start.get()) { + try { + if (this.selector.selectNow() == 0) { + Thread.sleep(0, 1); + continue; + } + + } catch (IOException e) { + p2pLOG.warn("inbound-select-io-exception"); + continue; + } catch (ClosedSelectorException e) { + if (p2pLOG.isDebugEnabled()) { + p2pLOG.debug("inbound-select-close-exception"); + } + continue; + } catch (InterruptedException e) { + e.printStackTrace(); + p2pLOG.error("taskInbound exception {}", e.toString()); + return; + } + + Iterator keys = null; + try { + keys = this.selector.selectedKeys().iterator(); + while (keys.hasNext()) { + SelectionKey sk = null; + try { + sk = keys.next(); + if (!sk.isValid()) { + continue; + } + + if (sk.isAcceptable()) { + accept(); + } + if (sk.isReadable()) { + ChannelBuffer cb = (ChannelBuffer) sk.attachment(); + if (cb == null) { + throw new P2pException("attachment is null"); + } + try { + readBuffer(sk, cb, readBuf); + } catch (NullPointerException e) { + mgr.closeSocket((SocketChannel) sk.channel(), + cb.getDisplayId() + "-read-msg-null-exception"); + cb.isClosed.set(true); + } catch (P2pException e) { + mgr.closeSocket((SocketChannel) sk.channel(), + cb.getDisplayId() + "-read-msg-p2p-exception"); + cb.isClosed.set(true); + } catch (ClosedChannelException e) { + mgr.closeSocket((SocketChannel) sk.channel(), + cb.getDisplayId() + "-read-msg-closed-channel-exception"); + } + } + } catch (IOException e) { + this.mgr.closeSocket((SocketChannel) sk.channel(), + "inbound-io-exception=" + e.getMessage()); + } catch (Exception e) { + e.printStackTrace(); + } + } + + } catch (Exception ex) { + p2pLOG.error("inbound exception={}", ex.getMessage()); + } finally { + if (keys != null) { + keys.remove(); + } + } + } + + p2pLOG.info("p2p-pi shutdown"); + } + + private void accept() { + SocketChannel channel; + try { + + if (this.nodeMgr.activeNodesSize() >= this.mgr.getMaxActiveNodes()) { + return; + } + + channel = this.tcpServer.accept(); + if (channel != null) { + this.mgr.configChannel(channel); + + String ip = channel.socket().getInetAddress().getHostAddress(); + + if (this.mgr.isSyncSeedsOnly() && this.nodeMgr.isSeedIp(ip)) { + channel.close(); + return; + } + + int port = channel.socket().getPort(); + INode node = this.nodeMgr.allocNode(ip, port); + node.setChannel(channel); + + SelectionKey sk = channel.register(this.selector, SelectionKey.OP_READ); + sk.attach(new ChannelBuffer()); + this.nodeMgr.addInboundNode(node); + + p2pLOG.info("new-connection {}:{}", ip, port); + } + } catch (IOException e) { + if (p2pLOG.isDebugEnabled()) { + p2pLOG.debug("inbound-accept-io-exception"); + } + } catch (Exception e) { + e.printStackTrace(); + p2pLOG.error("TaskInbound exception {}", e.getMessage()); + } + } + + private int readHeader(final ChannelBuffer _cb, + final ByteBuffer _readBuf, int cnt) { + + if (cnt < Header.LEN) { + return cnt; + } + + int origPos = _readBuf.position(); + + int startP = origPos - cnt; + + _readBuf.position(startP); + + _cb.readHead(_readBuf); + + _readBuf.position(origPos); + + return cnt - Header.LEN; + } + + private int readBody(final ChannelBuffer _cb, ByteBuffer _readBuf, int _cnt) { + + int bodyLen = _cb.header.getLen(); + + // some msg have nobody. + if (bodyLen == 0) { + _cb.body = new byte[0]; + return _cnt; + } + + if (_cnt < bodyLen) { + return _cnt; + } + + int origPos = _readBuf.position(); + int startP = origPos - _cnt; + _readBuf.position(startP); + _cb.readBody(_readBuf); + _readBuf.position(origPos); + return _cnt - bodyLen; + } + + private void readBuffer(final SelectionKey _sk, final ChannelBuffer _cb, + final ByteBuffer _readBuf) throws Exception { + + _readBuf.rewind(); + + SocketChannel sc = (SocketChannel) _sk.channel(); + + int r; + int cnt = 0; + do { + r = sc.read(_readBuf); + cnt += r; + } while (r > 0); + + if (cnt < 1) { + return; + } + + int remainBufAll = _cb.buffRemain + cnt; + ByteBuffer bufferAll = calBuffer(_cb, _readBuf, cnt); + + do { + r = readMsg(_sk, bufferAll, remainBufAll); + if (remainBufAll == r) { + break; + } else { + remainBufAll = r; + } + } while (r > 0); + + _cb.buffRemain = r; + + if (r != 0) { + // there are no perfect cycling buffer in jdk + // yet. + // simply just buff move for now. + // @TODO: looking for more efficient way. + + int currPos = bufferAll.position(); + _cb.remainBuffer = new byte[r]; + bufferAll.position(currPos - r); + bufferAll.get(_cb.remainBuffer); + + } + + _readBuf.rewind(); + } + + private int readMsg(SelectionKey _sk, ByteBuffer _readBuf, int _cnt) throws IOException { + ChannelBuffer cb = (ChannelBuffer) _sk.attachment(); + if (cb == null) { + throw new P2pException("attachment is null"); + } + + int readCnt; + if (cb.isHeaderNotCompleted()) { + readCnt = readHeader(cb, _readBuf, _cnt); + } else { + readCnt = _cnt; + } + + if (cb.isBodyNotCompleted()) { + readCnt = readBody(cb, _readBuf, readCnt); + } + + if (cb.isBodyNotCompleted()) { + return readCnt; + } + + handleMsg(_sk, cb); + + return readCnt; + } + + private void handleMsg(SelectionKey _sk, ChannelBuffer _cb) { + + Header h = _cb.header; + byte[] bodyBytes = _cb.body; + + _cb.refreshHeader(); + _cb.refreshBody(); + + boolean underRC = _cb.shouldRoute(h.getRoute(), + ((h.getRoute() == txBroadCastRoute) ? P2pConstant.READ_MAX_RATE_TXBC + : P2pConstant.READ_MAX_RATE)); + + if (!underRC) { + if (p2pLOG.isDebugEnabled()) { + p2pLOG.debug("over-called-route={}-{}-{} calls={} node={}", h.getVer(), h.getCtrl(), h.getAction(), _cb.getRouteCount(h.getRoute()).count, _cb.getDisplayId()); + } + return; + } + + switch (h.getVer()) { + case Ver.V0: + switch (h.getCtrl()) { + case Ctrl.NET: + try { + handleP2pMsg(_sk, h.getAction(), bodyBytes); + } catch (Exception ex) { + if (p2pLOG.isDebugEnabled()) { + p2pLOG.debug("handle-p2p-msg error={}", ex.getMessage()); + } + } + break; + case Ctrl.SYNC: + if (!handlers.containsKey(h.getRoute())) { + if (p2pLOG.isDebugEnabled()) { + p2pLOG.debug("unregistered-route={}-{}-{} node={}", h.getVer(), h.getCtrl(), h.getAction(), _cb.getDisplayId()); + } + return; + } + + handleKernelMsg(_cb.getNodeIdHash(), h.getRoute(), bodyBytes); + break; + default: + if (p2pLOG.isDebugEnabled()) { + p2pLOG.debug("invalid-route={}-{}-{} node={}", h.getVer(), h.getCtrl(), h.getAction(), _cb.getDisplayId()); + } + break; + } + break; + default: + if (p2pLOG.isDebugEnabled()) { + p2pLOG.debug("unhandled-ver={} node={}", h.getVer(), _cb.getDisplayId()); + } + + break; + } + } + + private ByteBuffer calBuffer(ChannelBuffer _cb, ByteBuffer _readBuf, int _cnt) { + ByteBuffer r; + if (_cb.buffRemain != 0) { + byte[] alreadyRead = new byte[_cnt]; + _readBuf.position(0); + _readBuf.get(alreadyRead); + r = ByteBuffer.allocate(_cb.buffRemain + _cnt); + r.put(_cb.remainBuffer); + r.put(alreadyRead); + } else { + r = _readBuf; + } + + return r; + } + + /** + * @param _sk SelectionKey + * @param _act ACT + * @param _msgBytes byte[] + */ + private void handleP2pMsg(final SelectionKey _sk, byte _act, final byte[] _msgBytes) { + + ChannelBuffer rb = (ChannelBuffer) _sk.attachment(); + + switch (_act) { + case Act.REQ_HANDSHAKE: + if (_msgBytes.length > ReqHandshake.LEN) { + ReqHandshake1 reqHandshake1 = ReqHandshake1.decode(_msgBytes); + if (reqHandshake1 != null) { + handleReqHandshake( + rb, + _sk.channel().hashCode(), + reqHandshake1.getNodeId(), + reqHandshake1.getNetId(), + reqHandshake1.getPort(), + reqHandshake1.getRevision()); + } + } + break; + + case Act.RES_HANDSHAKE: + if (rb.getNodeIdHash() != 0) { + if (_msgBytes.length > ResHandshake.LEN) { + ResHandshake1 resHandshake1 = ResHandshake1.decode(_msgBytes); + if (resHandshake1 != null && resHandshake1.getSuccess()) { + handleResHandshake(rb.getNodeIdHash(), resHandshake1.getBinaryVersion()); + } + } + } + break; + + case Act.REQ_ACTIVE_NODES: + if (rb.getNodeIdHash() != 0) { + INode node = nodeMgr.getActiveNode(rb.getNodeIdHash()); + if (node != null) { + this.sendMsgQue.offer( + new MsgOut( + node.getIdHash(), + node.getIdShort(), + new ResActiveNodes(nodeMgr.getActiveNodesList()), + Dest.ACTIVE)); + } + } + break; + + case Act.RES_ACTIVE_NODES: + if (this.mgr.isSyncSeedsOnly() || rb.getNodeIdHash() == 0) { + break; + } + + INode node = nodeMgr.getActiveNode(rb.getNodeIdHash()); + if (node != null) { + node.refreshTimestamp(); + ResActiveNodes resActiveNodes = ResActiveNodes.decode(_msgBytes); + if (resActiveNodes != null) { + List incomingNodes = resActiveNodes.getNodes(); + for (INode incomingNode : incomingNodes) { + if (nodeMgr.tempNodesSize() >= this.mgr.getMaxTempNodes()) { + return; + } + + if (this.mgr.validateNode(incomingNode)) { + nodeMgr.addTempNode(incomingNode); + } + } + } + } + break; + default: + if (p2pLOG.isDebugEnabled()) { + p2pLOG.debug("unknown-route act={}", _act); + } + break; + } + } + + /** + * @param _buffer ChannelBuffer + * @param _channelHash int + * @param _nodeId byte[] + * @param _netId int + * @param _port int + * @param _revision byte[] + *

Construct node info after handshake request success + */ + private void handleReqHandshake( + final ChannelBuffer _buffer, + int _channelHash, + final byte[] _nodeId, + int _netId, + int _port, + final byte[] _revision) { + INode node = nodeMgr.getInboundNode(_channelHash); + if (node != null && node.getPeerMetric().notBan()) { + if (handshakeRuleCheck(_netId)) { + _buffer.setNodeIdHash(Arrays.hashCode(_nodeId)); + _buffer.setDisplayId(new String(Arrays.copyOfRange(_nodeId, 0, 6))); + node.setId(_nodeId); + node.setPort(_port); + + // handshake 1 + if (_revision != null) { + String binaryVersion; + try { + binaryVersion = new String(_revision, "UTF-8"); + } catch (UnsupportedEncodingException e) { + binaryVersion = "decode-fail"; + if (p2pLOG.isDebugEnabled()) { + p2pLOG.debug("handleReqHandshake decode-fail"); + } + } + node.setBinaryVersion(binaryVersion); + nodeMgr.moveInboundToActive(_channelHash); + this.sendMsgQue.offer( + new MsgOut( + node.getIdHash(), + node.getIdShort(), + this.cachedResHandshake1, + Dest.ACTIVE)); + } + + } else { + if (p2pLOG.isDebugEnabled()) { + p2pLOG.debug("handshake-rule-fail"); + } + } + } + } + + private void handleResHandshake(int _nodeIdHash, String _binaryVersion) { + INode node = nodeMgr.getOutboundNodes().get(_nodeIdHash); + if (node != null && node.getPeerMetric().notBan()) { + node.refreshTimestamp(); + node.setBinaryVersion(_binaryVersion); + nodeMgr.moveOutboundToActive(node.getIdHash(), node.getIdShort()); + } + } + + /** + * @param _nodeIdHash int + * @param _route int + * @param _msgBytes byte[] + */ + private void handleKernelMsg(int _nodeIdHash, int _route, final byte[] _msgBytes) { + INode node = nodeMgr.getActiveNode(_nodeIdHash); + if (node != null) { + int nodeIdHash = node.getIdHash(); + String nodeDisplayId = node.getIdShort(); + node.refreshTimestamp(); + this.receiveMsgQue.offer(new MsgIn(nodeIdHash, nodeDisplayId, _route, _msgBytes)); + } + } + + /** + * @return boolean TODO: implementation + */ + private boolean handshakeRuleCheck(int netId) { + // check net id + return netId == this.mgr.getSelfNetId(); + } + +// private String getReadOverflowMsg(int prevCnt, int cnt) { +// return "IO readBuffer overflow! suppose readBuffer:" + prevCnt + " real left:" + cnt; +// } +// +// private String getRouteMsg(short ver, byte ctrl, byte act, int count, String idStr) { +// return ""; +// } +// +// private String getUnregRouteMsg(short ver, byte ctrl, byte act, String idStr) { +// return ""; +// } +// +// private String getInvalRouteMsg(short ver, byte ctrl, byte act, String idStr) { +// return ""; +// } +} diff --git a/modP2pImpl/src/org/aion/p2p/impl1/tasks/TaskReceive.java b/modP2pImpl/src/org/aion/p2p/impl1/tasks/TaskReceive.java new file mode 100644 index 0000000000..7539828b05 --- /dev/null +++ b/modP2pImpl/src/org/aion/p2p/impl1/tasks/TaskReceive.java @@ -0,0 +1,83 @@ +/* + * Copyright (c) 2017-2018 Aion foundation. + * + * This file is part of the aion network project. + * + * The aion network project is free software: you can redistribute it + * and/or modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation, either version 3 of + * the License, or any later version. + * + * The aion network project is distributed in the hope that it will + * be useful, but WITHOUT ANY WARRANTY; without even the implied + * warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + * See the GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with the aion network project source files. + * If not, see . + * + * Contributors: + * Aion foundation. + */ +package org.aion.p2p.impl1.tasks; + +import static org.aion.p2p.impl1.P2pMgr.p2pLOG; + +import java.util.List; +import java.util.Map; +import java.util.concurrent.BlockingQueue; +import java.util.concurrent.atomic.AtomicBoolean; +import org.aion.p2p.Handler; + +public class TaskReceive implements Runnable { + + private final AtomicBoolean start; + private final BlockingQueue receiveMsgQue; + private final Map> handlers; + + public TaskReceive(final AtomicBoolean _start, + final BlockingQueue _receiveMsgQue, + final Map> _handlers) { + this.start = _start; + this.receiveMsgQue = _receiveMsgQue; + this.handlers = _handlers; + } + + @Override + public void run() { + while (this.start.get()) { + try { + MsgIn mi = this.receiveMsgQue.take(); + + List hs = this.handlers.get(mi.getRoute()); + if (hs == null) { + continue; + } + for (Handler hlr : hs) { + if (hlr == null) { + continue; + } + + try { + hlr.receive(mi.getNodeId(), mi.getDisplayId(), mi.getMsg()); + } catch (Exception e) { + if (p2pLOG.isDebugEnabled()) { + e.printStackTrace(); + p2pLOG.debug("TaskReceive exception {}", e.getMessage()); + } + } + } + } catch (InterruptedException e) { + p2pLOG.error("TaskReceive interrupted {}", e.getMessage()); + + return; + } catch (Exception e) { + if (p2pLOG.isDebugEnabled()) { + e.printStackTrace(); + p2pLOG.debug("TaskReceive exception {}", e.getMessage()); + } + } + } + } +} diff --git a/modP2pImpl/src/org/aion/p2p/impl1/tasks/TaskSend.java b/modP2pImpl/src/org/aion/p2p/impl1/tasks/TaskSend.java new file mode 100644 index 0000000000..325f776ec1 --- /dev/null +++ b/modP2pImpl/src/org/aion/p2p/impl1/tasks/TaskSend.java @@ -0,0 +1,156 @@ +/* + * Copyright (c) 2017-2018 Aion foundation. + * + * This file is part of the aion network project. + * + * The aion network project is free software: you can redistribute it + * and/or modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation, either version 3 of + * the License, or any later version. + * + * The aion network project is distributed in the hope that it will + * be useful, but WITHOUT ANY WARRANTY; without even the implied + * warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + * See the GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with the aion network project source files. + * If not, see . + * + * Contributors: + * Aion foundation. + */ +package org.aion.p2p.impl1.tasks; + +import static org.aion.p2p.impl1.P2pMgr.p2pLOG; + +import java.nio.channels.SelectionKey; +import java.nio.channels.Selector; +import java.util.concurrent.BlockingQueue; +import java.util.concurrent.Executors; +import java.util.concurrent.LinkedBlockingQueue; +import java.util.concurrent.RejectedExecutionException; +import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; +import org.aion.p2p.INode; +import org.aion.p2p.INodeMgr; +import org.aion.p2p.IP2pMgr; +import org.aion.p2p.P2pConstant; + +public class TaskSend implements Runnable { + + private static final int TOTAL_LANE = Math + .min(Runtime.getRuntime().availableProcessors() << 1, 32); + private static final int THREAD_Q_LIMIT = 20000; + + private final IP2pMgr mgr; + private final AtomicBoolean start; + private final BlockingQueue sendMsgQue; + private final INodeMgr nodeMgr; + private final Selector selector; + private final int lane; + + private static ThreadPoolExecutor tpe; + + public TaskSend( + final IP2pMgr _mgr, + final int _lane, + final BlockingQueue _sendMsgQue, + final AtomicBoolean _start, + final INodeMgr _nodeMgr, + final Selector _selector) { + + this.mgr = _mgr; + this.lane = _lane; + this.sendMsgQue = _sendMsgQue; + this.start = _start; + this.nodeMgr = _nodeMgr; + this.selector = _selector; + + if (tpe == null) { + tpe = new ThreadPoolExecutor(TOTAL_LANE + , TOTAL_LANE + , 0 + , TimeUnit.MILLISECONDS + , new LinkedBlockingQueue<>(THREAD_Q_LIMIT) + , Executors.defaultThreadFactory()); + } + } + + @Override + public void run() { + while (start.get()) { + try { + MsgOut mo = sendMsgQue.take(); + + // if timeout , throw away this msg. + long now = System.currentTimeMillis(); + if (now - mo.getTimestamp() > P2pConstant.WRITE_MSG_TIMEOUT) { + if (p2pLOG.isDebugEnabled()) { + p2pLOG.debug("timeout-msg to-node={} timestamp={}", mo.getDisplayId(), now); + } + continue; + } + + // if not belong to current lane, put it back. + if (mo.getLane() != lane) { + sendMsgQue.offer(mo); + continue; + } + + INode node = null; + switch (mo.getDest()) { + case ACTIVE: + node = nodeMgr.getActiveNode(mo.getNodeId()); + break; + case INBOUND: + node = nodeMgr.getInboundNode(mo.getNodeId()); + break; + case OUTBOUND: + node = nodeMgr.getOutboundNode(mo.getNodeId()); + break; + } + + if (node != null) { + SelectionKey sk = node.getChannel().keyFor(selector); + if (sk != null) { + Object attachment = sk.attachment(); + if (attachment != null) { + tpe.execute(new TaskWrite( + node.getIdShort(), + node.getChannel(), + mo.getMsg(), + (ChannelBuffer) attachment, + this.mgr)); + } + } + } else { + if (p2pLOG.isDebugEnabled()) { + p2pLOG.debug("msg-{} ->{} node-not-exit", mo.getDest().name(), mo.getDisplayId()); + } + } + } catch (InterruptedException e) { + p2pLOG.error("task-send-interrupted"); + return; + } catch (RejectedExecutionException e) { + p2pLOG.warn("task-send-reached thread queue limit"); + } catch (Exception e) { + e.printStackTrace(); + if (p2pLOG.isDebugEnabled()) { + p2pLOG.debug("TaskSend exception {}", e.getMessage()); + } + } + } + } + + // hash mapping channel id to write thread. + static int hash2Lane(int in) { + in ^= in >> (32 - 5); + in ^= in >> (32 - 10); + in ^= in >> (32 - 15); + in ^= in >> (32 - 20); + in ^= in >> (32 - 25); + return (in & 0b11111) * TOTAL_LANE / 32; + } +} diff --git a/modP2pImpl/src/org/aion/p2p/impl1/tasks/TaskStatus.java b/modP2pImpl/src/org/aion/p2p/impl1/tasks/TaskStatus.java new file mode 100644 index 0000000000..df03ec748d --- /dev/null +++ b/modP2pImpl/src/org/aion/p2p/impl1/tasks/TaskStatus.java @@ -0,0 +1,58 @@ +/* + * Copyright (c) 2017-2018 Aion foundation. + * + * This file is part of the aion network project. + * + * The aion network project is free software: you can redistribute it + * and/or modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation, either version 3 of + * the License, or any later version. + * + * The aion network project is distributed in the hope that it will + * be useful, but WITHOUT ANY WARRANTY; without even the implied + * warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + * See the GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with the aion network project source files. + * If not, see . + * + * Contributors: + * Aion foundation. + */ +package org.aion.p2p.impl1.tasks; + +import static org.aion.p2p.impl1.P2pMgr.p2pLOG; + +import java.util.concurrent.BlockingQueue; +import org.aion.p2p.INodeMgr; + +public class TaskStatus implements Runnable { + + private final INodeMgr nodeMgr; + private final String selfShortId; + private final BlockingQueue sendMsgQue; + private final BlockingQueue receiveMsgQue; + + public TaskStatus(final INodeMgr _nodeMgr, + final String _selfShortId, + final BlockingQueue _sendMsgQue, + final BlockingQueue _receiveMsgQue) { + this.nodeMgr = _nodeMgr; + this.selfShortId = _selfShortId; + this.sendMsgQue = _sendMsgQue; + this.receiveMsgQue = _receiveMsgQue; + } + + @Override + public void run() { + Thread.currentThread().setName("p2p-ts"); + String status = this.nodeMgr.dumpNodeInfo(this.selfShortId); + + if (p2pLOG.isDebugEnabled()) { + p2pLOG.debug(status); + p2pLOG.debug("recv queue[{}] send queue[{}]", this.receiveMsgQue.size(), + this.sendMsgQue.size()); + } + } +} diff --git a/modP2pImpl/src/org/aion/p2p/impl1/tasks/TaskWrite.java b/modP2pImpl/src/org/aion/p2p/impl1/tasks/TaskWrite.java new file mode 100644 index 0000000000..467dfb3888 --- /dev/null +++ b/modP2pImpl/src/org/aion/p2p/impl1/tasks/TaskWrite.java @@ -0,0 +1,122 @@ +/* + * Copyright (c) 2017-2018 Aion foundation. + * + * This file is part of the aion network project. + * + * The aion network project is free software: you can redistribute it + * and/or modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation, either version 3 of + * the License, or any later version. + * + * The aion network project is distributed in the hope that it will + * be useful, but WITHOUT ANY WARRANTY; without even the implied + * warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + * See the GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with the aion network project source files. + * If not, see . + * + * Contributors: + * Aion foundation. + */ +package org.aion.p2p.impl1.tasks; + +import static org.aion.p2p.impl1.P2pMgr.p2pLOG; + +import java.io.IOException; +import java.nio.ByteBuffer; +import java.nio.channels.ClosedChannelException; +import java.nio.channels.SocketChannel; +import org.aion.p2p.Header; +import org.aion.p2p.IP2pMgr; +import org.aion.p2p.Msg; + +/** + * @author chris + */ +public class TaskWrite implements Runnable { + + private final String nodeShortId; + private final SocketChannel sc; + private final Msg msg; + private final ChannelBuffer channelBuffer; + private final IP2pMgr p2pMgr; + + TaskWrite( + final String _nodeShortId, + final SocketChannel _sc, + final Msg _msg, + final ChannelBuffer _cb, + final IP2pMgr _p2pMgr) { + this.nodeShortId = _nodeShortId; + this.sc = _sc; + this.msg = _msg; + this.channelBuffer = _cb; + this.p2pMgr = _p2pMgr; + } + + @Override + public void run() { + // reset allocated buffer and clear messages if the channel is closed + if (channelBuffer.isClosed.get()) { + channelBuffer.refreshHeader(); + channelBuffer.refreshBody(); + p2pMgr.dropActive(channelBuffer.getNodeIdHash(), "close-already"); + return; + } + + try { + channelBuffer.lock.lock(); + + /* + * @warning header set len (body len) before header encode + */ + byte[] bodyBytes = msg.encode(); + int bodyLen = bodyBytes == null ? 0 : bodyBytes.length; + Header h = msg.getHeader(); + h.setLen(bodyLen); + byte[] headerBytes = h.encode(); + + if (p2pLOG.isTraceEnabled()) { + p2pLOG.trace("write {}-{}-{}", h.getVer(), h.getCtrl(), h.getAction()); + } + + ByteBuffer buf = ByteBuffer.allocate(headerBytes.length + bodyLen); + buf.put(headerBytes); + if (bodyBytes != null) { + buf.put(bodyBytes); + } + buf.flip(); + + try { + while (buf.hasRemaining()) { + // @Attention: very important sleep , otherwise when NIO write buffer full, + // without sleep will hangup this thread. + Thread.sleep(0, 1); + sc.write(buf); + } + } catch (ClosedChannelException ex1) { + if (p2pLOG.isDebugEnabled()) { + p2pLOG.debug("closed-channel-exception node={}", this.nodeShortId); + } + + channelBuffer.isClosed.set(true); + } catch (IOException ex2) { + if (p2pLOG.isDebugEnabled()) { + p2pLOG.debug("write-msg-io-exception node={} err={}", this.nodeShortId, + ex2.getMessage()); + } + + if (ex2.getMessage().equals("Broken pipe")) { + channelBuffer.isClosed.set(true); + } + } + } catch (Exception e) { + e.printStackTrace(); + p2pLOG.error("TaskWrite exception {}", e.getMessage()); + } finally { + channelBuffer.lock.unlock(); + } + } +} diff --git a/modP2pImpl/test/org/aion/p2p/impl/LastThousands.java b/modP2pImpl/test/org/aion/p2p/impl/LastThousands.java index 4449ca4e39..39a671b51b 100644 --- a/modP2pImpl/test/org/aion/p2p/impl/LastThousands.java +++ b/modP2pImpl/test/org/aion/p2p/impl/LastThousands.java @@ -1,27 +1,28 @@ package org.aion.p2p.impl; -import org.aion.p2p.impl1.P2pMgr; -import org.junit.Ignore; -import org.junit.Test; +import static junit.framework.TestCase.assertEquals; import java.io.IOException; import java.net.Socket; import java.util.ArrayList; import java.util.List; import java.util.UUID; - -import static junit.framework.TestCase.assertEquals; +import org.aion.p2p.impl1.P2pMgr; +import org.junit.Ignore; +import org.junit.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; public class LastThousands { + private static Logger p2pLOG = LoggerFactory.getLogger("P2P"); private boolean checkPort(String host, int port) { boolean result = true; try { (new Socket(host, port)).close(); result = false; - } - catch(IOException e) { + } catch (IOException e) { // Could not connect. } return result; @@ -35,60 +36,56 @@ public void test() throws InterruptedException { int port = 30303; int max = 1000; int maxPort = port + max; - String[] testerP2p = new String[] { "p2p://" + nodeId + "@" + ip + ":" + port }; + String[] testerP2p = new String[]{"p2p://" + nodeId + "@" + ip + ":" + port}; P2pMgr tester = new P2pMgr(0, - "", - nodeId, - ip, - port, - new String[]{}, - false, - max, - max, - false, - false, - true, - 50); + "", + nodeId, + ip, + port, + new String[]{}, + false, + max, + max, + false, + 50); List examiners = new ArrayList<>(); - for(int i = port + 1; i <= maxPort; i++){ - if(checkPort(ip, i)) { + for (int i = port + 1; i <= maxPort; i++) { + if (checkPort(ip, i)) { System.out.println("examiner " + i); P2pMgr examiner = new P2pMgr(0, - "", - UUID.randomUUID().toString(), - ip, - i, - testerP2p, - false, - max, - max, - false, - true, - true, - 50); + "", + UUID.randomUUID().toString(), + ip, + i, + testerP2p, + false, + max, + max, + false, + 50); examiners.add(examiner); } } System.out.println("examiners " + examiners.size()); tester.run(); - for(P2pMgr examiner : examiners){ + for (P2pMgr examiner : examiners) { examiner.run(); } Thread.sleep(3000); - for(P2pMgr examiner : examiners){ + for (P2pMgr examiner : examiners) { assertEquals(1, examiner.getActiveNodes().size()); } - for(P2pMgr examiner : examiners){ + for (P2pMgr examiner : examiners) { assertEquals(max, tester.getActiveNodes().size()); } tester.shutdown(); - for(P2pMgr examiner : examiners){ + for (P2pMgr examiner : examiners) { examiner.shutdown(); } } diff --git a/modP2pImpl/test/org/aion/p2p/impl/P2pMgrTest.java b/modP2pImpl/test/org/aion/p2p/impl/P2pMgrTest.java index f46b8ee8f8..64cb98c55e 100644 --- a/modP2pImpl/test/org/aion/p2p/impl/P2pMgrTest.java +++ b/modP2pImpl/test/org/aion/p2p/impl/P2pMgrTest.java @@ -25,13 +25,12 @@ package org.aion.p2p.impl; -import org.aion.p2p.impl1.P2pMgr; -import org.junit.Test; +import static org.junit.Assert.assertEquals; import java.util.Map; import java.util.UUID; - -import static org.junit.Assert.assertEquals; +import org.aion.p2p.impl1.P2pMgr; +import org.junit.Test; /** * @author chris @@ -55,44 +54,37 @@ public Map.Entry newTwoNodeSetup() { int port2 = 30304; // we want node 1 to connect to node 2 - String[] nodes = new String[] { - "p2p://" + id2 + "@" + ip + ":" + port2 + String[] nodes = new String[]{ + "p2p://" + id2 + "@" + ip + ":" + port2 }; // to guarantee they don't receive the same port - while (port2 == port1) { - port2 = TestUtilities.getFreePort(); - } System.out.println("connector on: " + TestUtilities.formatAddr(id1, ip, port1)); P2pMgr connector = new P2pMgr(0, - "", - id1, - ip, - port1, - nodes, - false, - 128, - 128, - false, - true, - false, - 50); + "", + id1, + ip, + port1, + nodes, + false, + 128, + 128, + false, + 50); System.out.println("receiver on: " + TestUtilities.formatAddr(id2, ip, port2)); P2pMgr receiver = new P2pMgr(0, - "", - id2, - ip, - port2, - new String[0], - false, - 128, - 128, - false, - true, - false, - 50); + "", + id2, + ip, + port2, + new String[0], + false, + 128, + 128, + false, + 50); return Map.entry(connector, receiver); } @@ -101,72 +93,67 @@ public Map.Entry newTwoNodeSetup() { public void testIgnoreSameNodeIdAsSelf() { String[] nodes = new String[]{ - "p2p://" + nodeId1 + "@" + ip2+ ":" + port2 + "p2p://" + nodeId1 + "@" + ip2 + ":" + port2 }; P2pMgr p2p = new P2pMgr(0, - "", - nodeId1, - ip1, - port1, - nodes, - false, - 128, - 128, - false, - false, - false, - 50); + "", + nodeId1, + ip1, + port1, + nodes, + false, + 128, + 128, + false, + + 50); assertEquals(p2p.getTempNodesCount(), 0); } @Test - public void testIgnoreSameIpAndPortAsSelf(){ + public void testIgnoreSameIpAndPortAsSelf() { String[] nodes = new String[]{ - "p2p://" + nodeId2 + "@" + ip1+ ":" + port1 + "p2p://" + nodeId2 + "@" + ip1 + ":" + port1 }; P2pMgr p2p = new P2pMgr(0, - "", - nodeId1, - ip1, - port1, - nodes, - false, - 128, - 128, - false, - false, - false, - 50); - assertEquals(0,p2p.getTempNodesCount()); + "", + nodeId1, + ip1, + port1, + nodes, + false, + 128, + 128, + false, + 50); + assertEquals(0, p2p.getTempNodesCount()); } @Test - public void testTempNodes(){ + public void testTempNodes() { String[] nodes = new String[]{ - "p2p://" + nodeId2 + "@" + ip1+ ":" + port2, - "p2p://" + nodeId2 + "@" + ip2+ ":" + port1, - "p2p://" + nodeId2 + "@" + ip2+ ":" + port2, + "p2p://" + nodeId2 + "@" + ip1 + ":" + port2, + "p2p://" + nodeId2 + "@" + ip2 + ":" + port1, + "p2p://" + nodeId2 + "@" + ip2 + ":" + port2, }; P2pMgr p2p = new P2pMgr(0, - "", - nodeId1, - ip1, - port1, - nodes, - false, - 128, - 128, - false, - false, - false, - 50); + "", + nodeId1, + ip1, + port1, + nodes, + false, + 128, + 128, + false, + 50); assertEquals(p2p.getTempNodesCount(), 3); } } diff --git a/modP2pImpl/test/org/aion/p2p/impl/Test.java b/modP2pImpl/test/org/aion/p2p/impl/Test.java index 8bedcba68b..293061c8d8 100644 --- a/modP2pImpl/test/org/aion/p2p/impl/Test.java +++ b/modP2pImpl/test/org/aion/p2p/impl/Test.java @@ -1,11 +1,11 @@ package org.aion.p2p.impl; -import java.util.HashSet; -import java.util.Set; - import static junit.framework.TestCase.assertTrue; import static org.junit.Assert.assertFalse; +import java.util.HashSet; +import java.util.Set; + public class Test { @org.junit.Test diff --git a/modP2pImpl/test/org/aion/p2p/impl/zero/msg/ReqHandshake1Test.java b/modP2pImpl/test/org/aion/p2p/impl/zero/msg/ReqHandshake1Test.java index f25963ac5e..14856a92ab 100644 --- a/modP2pImpl/test/org/aion/p2p/impl/zero/msg/ReqHandshake1Test.java +++ b/modP2pImpl/test/org/aion/p2p/impl/zero/msg/ReqHandshake1Test.java @@ -28,16 +28,18 @@ import static org.junit.Assert.*; import java.util.ArrayList; +import java.util.Arrays; import java.util.List; import java.util.UUID; import java.util.concurrent.ThreadLocalRandom; import org.aion.p2p.Ctrl; -import org.aion.p2p.Msg; import org.aion.p2p.Ver; import org.aion.p2p.impl.comm.Act; import org.aion.p2p.impl.comm.Node; import org.junit.Before; import org.junit.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; /** * @author chris @@ -52,10 +54,13 @@ public class ReqHandshake1Test { private int port = ThreadLocalRandom.current().nextInt(); - private String randomIp = ThreadLocalRandom.current().nextInt(0,256) + "." + - ThreadLocalRandom.current().nextInt(0,256) + "." + - ThreadLocalRandom.current().nextInt(0,256) + "." + - ThreadLocalRandom.current().nextInt(0,256); + private static Logger p2pLOG = LoggerFactory.getLogger("P2P"); + + + private String randomIp = ThreadLocalRandom.current().nextInt(0, 256) + "." + + ThreadLocalRandom.current().nextInt(0, 256) + "." + + ThreadLocalRandom.current().nextInt(0, 256) + "." + + ThreadLocalRandom.current().nextInt(0, 256); private byte[] randomRevision; @@ -68,16 +73,17 @@ public void ReqHandshake2Test() { randomRevision = new byte[Byte.MAX_VALUE]; ThreadLocalRandom.current().nextBytes(randomRevision); randomVersions = new ArrayList<>(); - for(byte i = 0; i < 127; i ++){ - randomVersions.add((short)ThreadLocalRandom.current().nextInt(Short.MAX_VALUE + 1)); + for (byte i = 0; i < 127; i++) { + randomVersions.add((short) ThreadLocalRandom.current().nextInt(Short.MAX_VALUE + 1)); } } @Test public void testRoute() { - System.out.println("randomRevision " + randomRevision); - ReqHandshake1 req = new ReqHandshake1(validNodeId, netId, Node.ipStrToBytes(randomIp), port, randomRevision, randomVersions); + System.out.println("randomRevision " + Arrays.toString(randomRevision)); + ReqHandshake1 req = new ReqHandshake1(validNodeId, netId, Node.ipStrToBytes(randomIp), port, + randomRevision, randomVersions); assertEquals(Ver.V0, req.getHeader().getVer()); assertEquals(Ctrl.NET, req.getHeader().getCtrl()); assertEquals(Act.REQ_HANDSHAKE, req.getHeader().getAction()); @@ -86,7 +92,8 @@ public void testRoute() { @Test public void testValidEncodeDecode() { - ReqHandshake1 req1 = new ReqHandshake1(validNodeId, netId, Node.ipStrToBytes(randomIp), port, randomRevision, randomVersions); + ReqHandshake1 req1 = new ReqHandshake1(validNodeId, netId, Node.ipStrToBytes(randomIp), + port, randomRevision, randomVersions); byte[] bytes = req1.encode(); ReqHandshake1 req2 = ReqHandshake1.decode(bytes); @@ -101,11 +108,9 @@ public void testValidEncodeDecode() { @Test public void testInvalidEncodeDecode() { - ReqHandshake1 req1 = new ReqHandshake1(invalidNodeId, netId, Node.ipStrToBytes(randomIp), port, randomRevision, randomVersions); + ReqHandshake1 req1 = new ReqHandshake1(invalidNodeId, netId, Node.ipStrToBytes(randomIp), + port, randomRevision, randomVersions); byte[] bytes = req1.encode(); assertNull(bytes); } - - - } diff --git a/modP2pImpl/test/org/aion/p2p/impl/zero/msg/ResActiveNodesTest.java b/modP2pImpl/test/org/aion/p2p/impl/zero/msg/ResActiveNodesTest.java index 2a7ec04b98..2ce184b0d3 100644 --- a/modP2pImpl/test/org/aion/p2p/impl/zero/msg/ResActiveNodesTest.java +++ b/modP2pImpl/test/org/aion/p2p/impl/zero/msg/ResActiveNodesTest.java @@ -1,6 +1,7 @@ package org.aion.p2p.impl.zero.msg; import org.aion.p2p.Ctrl; +import org.aion.p2p.INode; import org.aion.p2p.Ver; import org.aion.p2p.impl.comm.Act; import org.aion.p2p.impl.comm.Node; @@ -49,18 +50,18 @@ public void testRoute() { public void testEncodeDecode() { int m = ThreadLocalRandom.current().nextInt(0, 20); - List srcNodes = new ArrayList<>(); + List srcNodes = new ArrayList<>(); for(int i = 0; i < m; i++){ srcNodes.add(randomNode()); } ResActiveNodes res = ResActiveNodes.decode(new ResActiveNodes(srcNodes).encode()); assertEquals(res.getNodes().size(), m); - List tarNodes = res.getNodes(); + List tarNodes = res.getNodes(); for(int i = 0; i < m; i++){ - Node srcNode = srcNodes.get(i); - Node tarNode = tarNodes.get(i); + INode srcNode = srcNodes.get(i); + INode tarNode = tarNodes.get(i); Assert.assertArrayEquals(srcNode.getId(), tarNode.getId()); Assert.assertEquals(srcNode.getIdHash(), tarNode.getIdHash()); diff --git a/modRlp/build.xml b/modRlp/build.xml index c1cdf02b36..458e616a54 100644 --- a/modRlp/build.xml +++ b/modRlp/build.xml @@ -38,7 +38,7 @@ debug="on" debuglevel="source,lines,vars" includeantruntime="false" - release="9" + release="10" srcdir="./test" destdir="${dir.test}" includes="**/*Test.java" @@ -56,7 +56,7 @@ debug="on" debuglevel="source,lines,vars" includeantruntime="false" - release="9" + release="10" srcdir="./test" destdir="${dir.test}" includes="**/*Test.java" @@ -107,7 +107,7 @@ debug="${compile.debug}" debuglevel="source,lines,vars" includeantruntime="false" - release="9" + release="10" srcdir="./" destdir="${dir.dest}" includes="src/**/*.java,module-info.java" @@ -127,41 +127,4 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/modTxPool/build.xml b/modTxPool/build.xml index 488e0fdede..8154ab8573 100644 --- a/modTxPool/build.xml +++ b/modTxPool/build.xml @@ -18,7 +18,7 @@ - + diff --git a/modTxPoolImpl/build.xml b/modTxPoolImpl/build.xml index dd62644ab1..c84a12c23c 100644 --- a/modTxPoolImpl/build.xml +++ b/modTxPoolImpl/build.xml @@ -38,7 +38,7 @@ - + @@ -48,7 +48,7 @@ - + @@ -107,7 +107,7 @@ - + diff --git a/script/prepack.sh b/script/prepack.sh index aaa0582d09..fcdd3d9463 100755 --- a/script/prepack.sh +++ b/script/prepack.sh @@ -5,6 +5,7 @@ JDK_RT="${PACK_PATH}/rt" WEB3JS_PATH="${PACK_PATH}/web3" CONFIG_PATH="${PACK_PATH}/config" DOCS_PATH="${PACK_PATH}/docs" +API_PATH="${PACK_PATH}/clientAPI" if [ ! -d "$PACK_PATH" ]; then mkdir $PACK_PATH @@ -42,3 +43,11 @@ if [ ! -d "$DOCS_PATH" ]; then mkdir $DOCS_PATH cp -r ./docs/** $DOCS_PATH fi + +# copy the client API files if can't find the client API env +if [ ! -d "$API_PATH" ]; then + mkdir $API_PATH + cp aion_api/pack/libAionApi-*.tar.gz $API_PATH +fi + +