Midway merge 0.5.1

.gitattributes

@@ -4,3 +4,4 @@
 *.pcap binary
 *.blocks binary
 *.json filter=lfs diff=lfs merge=lfs -text
+

Jenkinsfile

@@ -52,7 +52,8 @@ try {
                 }
                 stage('Test') {
                     sh './gradlew --no-daemon --parallel test'
-                    sh './artemis/src/main/resources/artemisTestScript.sh'
+                    // Disable Artemis Runtime Tests During Upgrade
+                    // sh './artemis/src/main/resources/artemisTestScript.sh'
                 }
             } finally {
                 archiveArtifacts '**/build/reports/**'

artemis/build.gradle

@@ -26,7 +26,7 @@ dependencies {
   api 'com.google.guava:guava:27.0.1-jre'
   implementation 'org.apache.tuweni:tuweni-crypto'
   implementation 'org.apache.tuweni:tuweni-plumtree'
-  implementation 'org.apache.tuweni:tuweni-ssz'
+  implementation files("../lib/tuweni-ssz-0.8.1-SNAPSHOT.jar")
   implementation 'org.apache.tuweni:tuweni-toml'
   implementation 'org.apache.tuweni:tuweni-units'
   implementation 'info.picocli:picocli'

artemis/src/test/java/tech/pegasys/artemis/BeaconNodeIntegrationTest.java

@@ -79,7 +79,7 @@ void testTwoNodes() throws InterruptedException, JsonProcessingException, IOExce
     node1.start();
     node2.start();
 
-    Thread.sleep(20000);
+    Thread.sleep(10000);
 
     P2PNetwork net1 = node1.p2pNetwork();
     P2PNetwork net2 = node2.p2pNetwork();

build.gradle

@@ -9,6 +9,7 @@ plugins {
   id 'net.ltgt.errorprone' version '0.7' apply false
   id 'net.researchgate.release' version '2.7.0'
   id 'com.gradle.build-scan' version '2.1'
+
 }
 
 defaultTasks 'build','checkLicenses'
@@ -91,6 +92,7 @@ allprojects {
         exclude 'web3j'
         //Excluding tech.pegasys.artemis.datastructures due to preferred formatting overwritten by plugin in getConstantsAsString.
         exclude '**/src/main/java/tech/pegasys/artemis/Constants.java'
+        exclude '**/proto'
       }
 
 

config/config.toml

@@ -43,8 +43,8 @@ MAX_DEPOSIT_AMOUNT = 32000000000
 FORK_CHOICE_BALANCE_INCREMENT = 1000000000
 EJECTION_BALANCE = 16000000000
 GENESIS_FORK_VERSION = 0
-GENESIS_SLOT = 0
-GENESIS_EPOCH = 0
+GENESIS_SLOT = 4294967296
+GENESIS_EPOCH = 536870912
 GENESIS_START_SHARD = 0
 SECONDS_PER_SLOT = 6
 MIN_ATTESTATION_INCLUSION_DELAY = 1
@@ -55,7 +55,6 @@ EPOCHS_PER_ETH1_VOTING_PERIOD = 16
 SLOTS_PER_HISTORICAL_ROOT = 64
 MIN_VALIDATOR_WITHDRAWABILITY_DELAY = 256
 PERSISTENT_COMMITTEE_PERIOD = 2048
-LATEST_BLOCK_ROOTS_LENGTH = 64
 LATEST_RANDAO_MIXES_LENGTH = 64
 LATEST_ACTIVE_INDEX_ROOTS_LENGTH = 64
 LATEST_SLASHED_EXIT_LENGTH = 64
@@ -70,9 +69,9 @@ MAX_ATTESTATIONS = 128
 MAX_DEPOSITS = 16
 MAX_VOLUNTARY_EXITS = 16
 MAX_TRANSFERS = 16
-DOMAIN_DEPOSIT = 0
-DOMAIN_ATTESTATION = 1
-DOMAIN_PROPOSAL = 2
-DOMAIN_EXIT = 3
-DOMAIN_RANDAO = 4
+DOMAIN_BEACON_BLOCK = 0
+DOMAIN_RANDAO = 1
+DOMAIN_ATTESTATION = 2
+DOMAIN_DEPOSIT = 3
+DOMAIN_VOLUNTARY_EXIT = 4
 DOMAIN_TRANSFER = 5

data/adapter/build.gradle

@@ -14,7 +14,6 @@ dependencies {
 
   implementation 'org.apache.tuweni:tuweni-crypto'
   implementation 'org.apache.tuweni:tuweni-units'
-  implementation 'org.apache.tuweni:tuweni-ssz'
   implementation 'com.google.guava:guava'
   implementation 'com.google.code.gson:gson'
   implementation 'org.apache.logging.log4j:log4j-api'

data/adapter/src/main/java/tech/pegasys/artemis/data/adapter/TimeSeriesAdapter.java

@@ -13,6 +13,7 @@
 
 package tech.pegasys.artemis.data.adapter;
 
+import com.google.common.primitives.UnsignedLong;
 import java.util.ArrayList;
 import java.util.List;
 import java.util.stream.IntStream;
@@ -24,7 +25,6 @@
 import tech.pegasys.artemis.datastructures.state.BeaconState;
 import tech.pegasys.artemis.datastructures.util.BeaconStateUtil;
 import tech.pegasys.artemis.util.alogger.ALogger;
-import tech.pegasys.artemis.util.hashtree.HashTreeUtil;
 
 /** Transforms a data record into a time series record */
 public class TimeSeriesAdapter implements DataAdapter<TimeSeriesRecord> {
@@ -38,8 +38,8 @@ public TimeSeriesAdapter(RawRecord input) {
   @Override
   public TimeSeriesRecord transform() {
 
-    long slot = this.input.getHeadBlock().getSlot();
-    long epoch = BeaconStateUtil.slot_to_epoch(this.input.getHeadBlock().getSlot());
+    UnsignedLong slot = UnsignedLong.valueOf(this.input.getHeadBlock().getSlot());
+    UnsignedLong epoch = BeaconStateUtil.slot_to_epoch(slot);
     BeaconBlock headBlock = this.input.getHeadBlock();
     BeaconState headState = this.input.getHeadState();
     BeaconBlock justifiedBlock = this.input.getJustifiedBlock();
@@ -48,10 +48,11 @@ public TimeSeriesRecord transform() {
     BeaconState finalizedState = this.input.getFinalizedState();
     int numValidators = headState.getValidator_registry().size();
 
-    Bytes32 lastJustifiedBlockRoot = HashTreeUtil.hash_tree_root(justifiedBlock.toBytes());
-    Bytes32 lastJustifiedStateRoot = HashTreeUtil.hash_tree_root(justifiedState.toBytes());
-    Bytes32 lastFinalizedBlockRoot = HashTreeUtil.hash_tree_root(finalizedBlock.toBytes());
-    Bytes32 lastFinalizedStateRoot = HashTreeUtil.hash_tree_root(finalizedState.toBytes());
+    Bytes32 headBlockRoot = headBlock.signed_root("signature");
+    Bytes32 lastJustifiedBlockRoot = justifiedBlock.signed_root("signature");
+    Bytes32 lastJustifiedStateRoot = justifiedState.hash_tree_root();
+    Bytes32 lastFinalizedBlockRoot = finalizedBlock.signed_root("signature");
+    Bytes32 lastFinalizedStateRoot = finalizedState.hash_tree_root();
 
     List<ValidatorJoin> validators = new ArrayList<>();
 
@@ -62,17 +63,17 @@ public TimeSeriesRecord transform() {
                   validators.add(
                       new ValidatorJoin(
                           headState.getValidator_registry().get(i),
-                          headState.getValidator_balances().get(i))));
+                          headState.getValidator_balances().get(i).longValue())));
     }
 
     return new TimeSeriesRecord(
         this.input.getDate(),
         this.input.getIndex(),
-        slot,
-        epoch,
+        slot.longValue(),
+        epoch.longValue(),
         this.input.getHeadBlock().getState_root().toHexString(),
-        this.input.getHeadBlock().getParent_root().toHexString(),
-        HashTreeUtil.hash_tree_root(this.input.getHeadBlock().toBytes()).toHexString(),
+        this.input.getHeadBlock().getPrevious_block_root().toHexString(),
+        this.input.getHeadBlock().signed_root("signature").toHexString(),
         lastJustifiedBlockRoot.toHexString(),
         lastJustifiedStateRoot.toHexString(),
         lastFinalizedBlockRoot.toHexString(),

data/build.gradle

@@ -12,8 +12,8 @@ dependencies {
 
   implementation 'org.apache.tuweni:tuweni-crypto'
   implementation 'org.apache.tuweni:tuweni-units'
-  implementation 'org.apache.tuweni:tuweni-ssz'
-  implementation 'com.fasterxml.jackson.core:jackson-databind'  
+  implementation files("../lib/tuweni-ssz-0.8.1-SNAPSHOT.jar")
+  implementation 'com.fasterxml.jackson.core:jackson-databind'
   implementation 'com.google.guava:guava'
   implementation 'org.apache.logging.log4j:log4j-api'
   runtime 'org.apache.logging.log4j:log4j-core'

data/metrics/build.gradle

@@ -16,7 +16,6 @@ dependencies {
     implementation 'io.vertx:vertx-web'
     implementation 'org.apache.tuweni:tuweni-crypto'
     implementation 'org.apache.tuweni:tuweni-units'
-    implementation 'org.apache.tuweni:tuweni-ssz'
     implementation 'com.fasterxml.jackson.core:jackson-databind'
     implementation 'com.google.guava:guava'
     implementation 'io.prometheus:simpleclient'

data/provider/build.gradle

@@ -14,7 +14,7 @@ dependencies {
 
   implementation 'org.apache.tuweni:tuweni-crypto'
   implementation 'org.apache.tuweni:tuweni-units'
-  implementation 'org.apache.tuweni:tuweni-ssz'
+  implementation files("../../lib/tuweni-ssz-0.8.1-SNAPSHOT.jar")
   implementation 'com.fasterxml.jackson.core:jackson-databind'
   implementation 'com.google.guava:guava'
   implementation 'org.apache.logging.log4j:log4j-api'

eth-reference-tests/build.gradle

@@ -10,7 +10,7 @@ dependencies {
 
   compile group: 'com.fasterxml.jackson.dataformat', name: 'jackson-dataformat-yaml', version: '2.9.8'
   testCompile 'org.miracl.milagro.amcl:milagro-crypto-java:0.4.0'
-  testCompile 'org.apache.tuweni:tuweni-bytes'
+  implementation 'org.apache.tuweni:tuweni-bytes'
   testCompile 'com.fasterxml.jackson.core:jackson-databind'
   testCompile 'com.fasterxml.jackson.dataformat:jackson-dataformat-yaml'
   testCompile 'org.junit.jupiter:junit-jupiter-api'

eth-reference-tests/src/test/java/pegasys/artemis/reference/BLSTestSuite.java

@@ -51,7 +51,7 @@ class BLSTestSuite {
 
   // TODO: reinstate the official tests once they have been updated
   // private static String testFile = "**/bls/test_bls.yml";
-  private static final String testFile = "**/test_bls_tmp.yml";
+  private static String testFile = "**/test_bls_tmp.yml";
 
   @ParameterizedTest(name = "{index}. message hash to G2 uncompressed {0} -> {1}")
   @MethodSource("readMessageHashG2Uncompressed")

eth-reference-tests/src/test/java/pegasys/artemis/reference/PermutedIndexTestSuite.java

@@ -37,7 +37,7 @@
 public class PermutedIndexTestSuite {
 
   // TODO: point this to the official test file repo when it is available and correct
-  private static final String testFile = "**/test_vector_permutated_index_tmp.yml";
+  private static String testFile = "**/test_vector_permutated_index_tmp.yml";
 
   @ParameterizedTest(name = "{index}. Test permuted index {0}")
   @MethodSource("readPermutedIndexTestVectors")

ethereum/datastructures/build.gradle

@@ -11,7 +11,7 @@ dependencies {
 
   implementation 'org.apache.tuweni:tuweni-crypto'
   implementation 'org.apache.tuweni:tuweni-units'
-  implementation 'org.apache.tuweni:tuweni-ssz'
+  implementation files("../../lib/tuweni-ssz-0.8.1-SNAPSHOT.jar")
   implementation 'com.google.guava:guava'
   implementation 'com.google.code.gson:gson'
   implementation 'org.apache.logging.log4j:log4j-api'

ethereum/datastructures/src/main/java/tech/pegasys/artemis/datastructures/Constants.java

@@ -13,19 +13,21 @@
 
 package tech.pegasys.artemis.datastructures;
 
+import com.google.common.primitives.UnsignedLong;
 import org.apache.tuweni.bytes.Bytes;
 import org.apache.tuweni.bytes.Bytes32;
 import tech.pegasys.artemis.util.bls.BLSSignature;
 import tech.pegasys.artemis.util.config.ArtemisConfiguration;
 
 public class Constants {
-  // The constants below are correct as of spec v0.4
+  // TODO: Update config.toml setting of constants for 0.5, non-existing
+  //  getter functions are purposefully being used here, so that we would
+  //  need to create their actual getters before being able to run succesfully.
 
   // Misc
   public static int SHARD_COUNT = 1024; // 2^10 shards
   public static int TARGET_COMMITTEE_SIZE = 128; // 2^7 validators
   public static int MAX_BALANCE_CHURN_QUOTIENT = 32; // 2^5
-  public static long BEACON_CHAIN_SHARD_NUMBER = -1L; // 2^64 - 1
   public static int MAX_INDICES_PER_SLASHABLE_VOTE = 4096; // 2^12 votes
   public static int MAX_EXIT_DEQUEUES_PER_EPOCH = 4; // 2^2 withdrawals
   public static int SHUFFLE_ROUND_COUNT = 90;
@@ -47,20 +49,21 @@ public class Constants {
   public static int MIN_SEED_LOOKAHEAD = 1; // 2^0 epochs (6.4 minutes)
   public static int ACTIVATION_EXIT_DELAY = 4; // 2^2 epochs (25.6 minutes)
   public static int EPOCHS_PER_ETH1_VOTING_PERIOD = 16; // 2^4 epochs (~1.7 hours)
+  public static int SLOTS_PER_HISTORICAL_ROOT = 8192; // 2^13 slots (~13 hours)
   public static int MIN_VALIDATOR_WITHDRAWABILITY_DELAY = 256; // 2^8 epochs (~27 hours)
+  public static int PERSISTENT_COMMITTEE_PERIOD = 2048; // 2^11 epochs (~9 days)
 
   // Initial values
   public static int GENESIS_FORK_VERSION = 0;
-  public static long GENESIS_SLOT = 0; // 2^32
+  public static long GENESIS_SLOT = 4294967296L; // 2^32
   public static long GENESIS_EPOCH = slot_to_epoch(GENESIS_SLOT);
   public static long GENESIS_START_SHARD = 0;
-  public static long FAR_FUTURE_EPOCH = -1L;
+  public static UnsignedLong FAR_FUTURE_EPOCH = UnsignedLong.MAX_VALUE;
   public static Bytes32 ZERO_HASH = Bytes32.ZERO;
   public static BLSSignature EMPTY_SIGNATURE = BLSSignature.empty();
   public static Bytes BLS_WITHDRAWAL_PREFIX_BYTE = Bytes.EMPTY;
 
   // State list lengths
-  public static int LATEST_BLOCK_ROOTS_LENGTH = 8192; // 2^13 slots (~13 hours)
   public static int LATEST_RANDAO_MIXES_LENGTH = 8192; // 2^13 epochs (~36 days)
   public static int LATEST_ACTIVE_INDEX_ROOTS_LENGTH = 8192; // 2^13 epochs (~36 days)
   public static int LATEST_SLASHED_EXIT_LENGTH = 8192; // 2^13 epochs (~36 days)
@@ -81,11 +84,11 @@ public class Constants {
   public static int MAX_TRANSFERS = 16; // 2^4
 
   // Signature domains
-  public static int DOMAIN_DEPOSIT = 0;
-  public static int DOMAIN_ATTESTATION = 1;
-  public static int DOMAIN_PROPOSAL = 2;
-  public static int DOMAIN_EXIT = 3;
-  public static int DOMAIN_RANDAO = 4;
+  public static int DOMAIN_BEACON_BLOCK = 0;
+  public static int DOMAIN_RANDAO = 1;
+  public static int DOMAIN_ATTESTATION = 2;
+  public static int DOMAIN_DEPOSIT = 3;
+  public static int DOMAIN_VOLUNTARY_EXIT = 4;
   public static int DOMAIN_TRANSFER = 5;
 
   // Artemis specific
@@ -106,10 +109,6 @@ public static void init(ArtemisConfiguration config) {
         config.getMaxBalanceChurnQuotient() != Integer.MIN_VALUE
             ? config.getMaxBalanceChurnQuotient()
             : MAX_BALANCE_CHURN_QUOTIENT; // 2^5
-    BEACON_CHAIN_SHARD_NUMBER =
-        config.getBeaconChainShardNumber() != -1L
-            ? config.getBeaconChainShardNumber()
-            : BEACON_CHAIN_SHARD_NUMBER; // 2^64 - 1
     MAX_INDICES_PER_SLASHABLE_VOTE =
         config.getMaxIndicesPerSlashableVote() != Integer.MIN_VALUE
             ? config.getMaxIndicesPerSlashableVote()
@@ -176,6 +175,10 @@ public static void init(ArtemisConfiguration config) {
         config.getEpochsPerEth1VotingPeriod() != Integer.MIN_VALUE
             ? config.getEpochsPerEth1VotingPeriod()
             : EPOCHS_PER_ETH1_VOTING_PERIOD; // 2^4 epochs (~1.7 hours)
+    SLOTS_PER_HISTORICAL_ROOT =
+        config.getSlotsPerHistoricalRoot() != Integer.MIN_VALUE
+            ? config.getSlotsPerHistoricalRoot()
+            : SLOTS_PER_HISTORICAL_ROOT;
     MIN_VALIDATOR_WITHDRAWABILITY_DELAY =
         config.getMinValidatorWithdrawabilityDelay() != Integer.MIN_VALUE
             ? config.getMinValidatorWithdrawabilityDelay()
@@ -195,7 +198,9 @@ public static void init(ArtemisConfiguration config) {
             ? config.getGenesisStartShard()
             : GENESIS_START_SHARD;
     FAR_FUTURE_EPOCH =
-        config.getFarFutureEpoch() != -1L ? config.getFarFutureEpoch() : FAR_FUTURE_EPOCH;
+        config.getFarFutureEpoch() != Long.MAX_VALUE
+            ? UnsignedLong.valueOf(config.getFarFutureEpoch())
+            : FAR_FUTURE_EPOCH;
     ZERO_HASH =
         !config.getZeroHash().equals(Bytes32.ZERO)
             ? (Bytes32) config.getZeroHash()
@@ -210,10 +215,6 @@ public static void init(ArtemisConfiguration config) {
             : BLS_WITHDRAWAL_PREFIX_BYTE; // TODO Verify
 
     // State list lengths
-    LATEST_BLOCK_ROOTS_LENGTH =
-        config.getLatestBlockRootsLength() != Integer.MIN_VALUE
-            ? config.getLatestBlockRootsLength()
-            : LATEST_BLOCK_ROOTS_LENGTH; // 2^13 slots (~13 hours)
     LATEST_RANDAO_MIXES_LENGTH =
         config.getLatestRandaoMixesLength() != Integer.MIN_VALUE
             ? config.getLatestRandaoMixesLength()
@@ -276,20 +277,22 @@ public static void init(ArtemisConfiguration config) {
             : MAX_TRANSFERS; // 2^4
 
     // Signature domains
-    DOMAIN_DEPOSIT =
-        config.getDomainDeposit() != Integer.MIN_VALUE ? config.getDomainDeposit() : DOMAIN_DEPOSIT;
+    DOMAIN_BEACON_BLOCK =
+        config.getDomainBeaconBlock() != Integer.MIN_VALUE
+            ? config.getDomainBeaconBlock()
+            : DOMAIN_BEACON_BLOCK;
+    DOMAIN_RANDAO =
+        config.getDomainRandao() != Integer.MIN_VALUE ? config.getDomainRandao() : DOMAIN_RANDAO;
     DOMAIN_ATTESTATION =
         config.getDomainAttestation() != Integer.MIN_VALUE
             ? config.getDomainAttestation()
             : DOMAIN_ATTESTATION;
-    DOMAIN_PROPOSAL =
-        config.getDomainProposal() != Integer.MIN_VALUE
-            ? config.getDomainProposal()
-            : DOMAIN_PROPOSAL;
-    DOMAIN_EXIT =
-        config.getDomainExit() != Integer.MIN_VALUE ? config.getDomainExit() : DOMAIN_EXIT;
-    DOMAIN_RANDAO =
-        config.getDomainRandao() != Integer.MIN_VALUE ? config.getDomainRandao() : DOMAIN_RANDAO;
+    DOMAIN_DEPOSIT =
+        config.getDomainDeposit() != Integer.MIN_VALUE ? config.getDomainDeposit() : DOMAIN_DEPOSIT;
+    DOMAIN_VOLUNTARY_EXIT =
+        config.getDomainVoluntaryExit() != Integer.MIN_VALUE
+            ? config.getDomainVoluntaryExit()
+            : DOMAIN_VOLUNTARY_EXIT;
     DOMAIN_TRANSFER =
         config.getDomainTransfer() != Integer.MIN_VALUE
             ? config.getDomainTransfer()
@@ -304,6 +307,12 @@ public static void init(ArtemisConfiguration config) {
             : DEPOSIT_DATA_SIZE;
   }
 
+  /**
+   * Return the epoch number of the given ``slot``.
+   *
+   * @param slot
+   * @return
+   */
   public static long slot_to_epoch(long slot) {
     return slot / SLOTS_PER_EPOCH;
   }