synthetic-network for nightly-pipeline

.github/actions/run-e2e-test/action.yml

@@ -26,6 +26,18 @@ inputs:
   follow-up-finalization-check:
     description: 'Whether to run a follow-up finalization check.'
     required: false
+  image-path:
+    description: 'Custom path to docker image for aleph-node'
+    required: false
+    default: aleph-test-docker
+  node-image:
+    description: 'Custom name of aleph-node image'
+    required: false
+    default: aleph-node:latest
+  compose-file:
+    description: 'Custom docker-compose configuration'
+    required: false
+    default: ''
 
 runs:
   using: 'composite'
@@ -34,15 +46,15 @@ runs:
     - name: Download artifact with docker image
       uses: actions/download-artifact@v2
       with:
-        name: aleph-test-docker
+        name: ${{ inputs.image-path }}
 
     - name: Load node docker image
       shell: bash
       run: docker load -i aleph-node.tar
 
     - name: Run consensus party
       shell: bash
-      run: ./.github/scripts/run_consensus.sh -m ${{ inputs.min-validator-count }} -n ${{ inputs.node-count }}
+      run: NODE_IMAGE=${{ inputs.node-image }} DOCKER_COMPOSE=${{ inputs.compose-file }} ./.github/scripts/run_consensus.sh -m ${{ inputs.min-validator-count }} -n ${{ inputs.node-count }}
 
     - name: Sleep
       shell: bash

.github/scripts/run_consensus.sh

@@ -8,11 +8,12 @@ NODE_COUNT=5
 MIN_VALIDATOR_COUNT=4
 DOCKER_COMPOSE=${DOCKER_COMPOSE:-"docker/docker-compose.yml"}
 OVERRIDE_DOCKER_COMPOSE=${OVERRIDE_DOCKER_COMPOSE:-""}
+NODE_IMAGE=${NODE_IMAGE:-aleph-node:latest}
 
 # default minimum validator count
 MIN_VALIDATOR_COUNT=4
 
-export NODE_IMAGE=aleph-node:latest
+export NODE_IMAGE
 
 mkdir -p docker/data/
 

.github/workflows/nightly-pipeline.yaml

@@ -43,6 +43,37 @@ jobs:
           if-no-files-found: error
           retention-days: 7
 
+  build-synthetic-network-docker:
+    needs: [build-test-docker]
+    name: Build docker image with the test node artifact and support for synthetic-network
+    runs-on: ubuntu-20.04
+    steps:
+      - name: Checkout source code
+        uses: actions/checkout@v2
+
+      - name: Download artifact with docker image for aleph-node
+        uses: actions/download-artifact@v2
+        with:
+          name: aleph-test-docker
+
+      - name: Load node docker image
+        shell: bash
+        run: docker load -i aleph-node.tar
+
+      - name: Build test docker image
+        id: build-image
+        run: |
+          scripts/synthetic-network/build_synthetic-network.sh
+          docker save -o aleph-node.tar aleph-node:syntheticnet
+
+      - name: Upload test docker image
+        uses: actions/upload-artifact@v2
+        with:
+          name: aleph-test-synthetic-docker
+          path: aleph-node.tar
+          if-no-files-found: error
+          retention-days: 7
+
   check-determinism:
     needs: [build-new-node]
     name: Verify runtime build determinism
@@ -59,7 +90,7 @@ jobs:
       - name: Install Protoc
         uses: arduino/setup-protoc@v1
         with:
-          version: '3.6.1'        
+          version: '3.6.1'
 
       - name: Install WASM target
         run: rustup target add wasm32-unknown-unknown
@@ -96,7 +127,7 @@ jobs:
       - name: Install Protoc
         uses: arduino/setup-protoc@v1
         with:
-          version: '3.6.1'        
+          version: '3.6.1'
 
       - name: Restore cache
         uses: ./.github/actions/restore-cache
@@ -140,9 +171,45 @@ jobs:
           randomized: true
         timeout-minutes: 60
 
+  run-e2e-high-out-latency:
+    needs: [build-synthetic-network-docker, build-test-client]
+    name: Run high out-latency test
+    runs-on: ubuntu-20.04
+    steps:
+      - name: Checkout source code
+        uses: actions/checkout@v2
+
+      - name: Run e2e test
+        uses: ./.github/actions/run-e2e-test
+        with:
+          test-case: high_out_latency
+          image-path: aleph-test-synthetic-docker
+          node-image: aleph-node:syntheticnet
+          compose-file: docker/docker-compose.synthetic-network.yml
+        timeout-minutes: 30
+
+  no_quorum_without_high_out_latency:
+    needs: [build-synthetic-network-docker, build-test-client]
+    name: Run high out-latency for every quorum
+    runs-on: ubuntu-20.04
+    steps:
+      - name: Checkout source code
+        uses: actions/checkout@v2
+
+      - name: Run e2e test
+        uses: ./.github/actions/run-e2e-test
+        with:
+          test-case: no_quorum_without_high_out_latency
+          image-path: aleph-test-synthetic-docker
+          node-image: aleph-node:syntheticnet
+          compose-file: docker/docker-compose.synthetic-network.yml
+        timeout-minutes: 15
+
   check-e2e-test-suite-completion:
     needs: [
       run-e2e-authorities-are-staking,
+      run-e2e-high-out-latency,
+      no_quorum_without_high_out_latency,
     ]
     name: Check e2e test suite completion
     runs-on: ubuntu-20.04

.gitmodules

@@ -0,0 +1,3 @@
+[submodule "scripts/synthetic-network/vendor/synthetic-network"]
+	path = scripts/synthetic-network/vendor/synthetic-network
+	url = https://github.com/daily-co/synthetic-network.git

Cargo.toml

@@ -18,6 +18,7 @@ exclude = [
   "fork-off",
   "benches/payout-stakers",
   "bin/cliain",
+  "scripts/synthetic-network/synthetic-link",
 ]
 
 [profile.release]

aleph-client/src/connections.rs

@@ -1,4 +1,4 @@
-use std::{thread::sleep, time::Duration};
+use std::{convert::AsRef, thread::sleep, time::Duration};
 
 use anyhow::anyhow;
 use codec::Decode;
@@ -58,6 +58,12 @@ impl SudoCall for RootConnection {
     }
 }
 
+impl AsRef<Connection> for Connection {
+    fn as_ref(&self) -> &Connection {
+        self
+    }
+}
+
 impl Connection {
     const DEFAULT_RETRIES: u32 = 10;
     const RETRY_WAIT_SECS: u64 = 1;
@@ -161,6 +167,12 @@ impl SignedConnection {
     }
 }
 
+impl AsRef<Connection> for SignedConnection {
+    fn as_ref(&self) -> &Connection {
+        &self.connection
+    }
+}
+
 impl RootConnection {
     pub async fn new(address: String, root: KeyPair) -> anyhow::Result<Self> {
         RootConnection::try_from_connection(Connection::new(address).await, root).await

aleph-client/src/pallets/session.rs

@@ -22,29 +22,30 @@ pub trait SessionUserApi {
 }
 
 #[async_trait::async_trait]
-impl SessionApi for Connection {
+impl<C: AsRef<Connection> + Sync + ?Sized> SessionApi for C {
     async fn get_next_session_keys(
         &self,
         account: AccountId,
         at: Option<BlockHash>,
     ) -> Option<SessionKeys> {
         let addrs = api::storage().session().next_keys(account);
 
-        self.get_storage_entry_maybe(&addrs, at).await
+        self.as_ref().get_storage_entry_maybe(&addrs, at).await
     }
 
     async fn get_session(&self, at: Option<BlockHash>) -> SessionIndex {
         let addrs = api::storage().session().current_index();
 
-        self.get_storage_entry_maybe(&addrs, at)
+        self.as_ref()
+            .get_storage_entry_maybe(&addrs, at)
             .await
             .unwrap_or_default()
     }
 
     async fn get_validators(&self, at: Option<BlockHash>) -> Vec<AccountId> {
         let addrs = api::storage().session().validators();
 
-        self.get_storage_entry(&addrs, at).await
+        self.as_ref().get_storage_entry(&addrs, at).await
     }
 }
 

aleph-client/src/waiting.rs

@@ -34,11 +34,23 @@ pub trait WaitingExt {
 }
 
 #[async_trait::async_trait]
-impl AlephWaiting for Connection {
+impl<C: AsRef<Connection> + Sync> AlephWaiting for C {
     async fn wait_for_block<P: Fn(u32) -> bool + Send>(&self, predicate: P, status: BlockStatus) {
         let mut block_sub = match status {
-            BlockStatus::Best => self.client.blocks().subscribe_best().await.unwrap(),
-            BlockStatus::Finalized => self.client.blocks().subscribe_finalized().await.unwrap(),
+            BlockStatus::Best => self
+                .as_ref()
+                .client
+                .blocks()
+                .subscribe_best()
+                .await
+                .unwrap(),
+            BlockStatus::Finalized => self
+                .as_ref()
+                .client
+                .blocks()
+                .subscribe_finalized()
+                .await
+                .unwrap(),
         };
 
         while let Some(Ok(block)) = block_sub.next().await {
@@ -54,8 +66,20 @@ impl AlephWaiting for Connection {
         status: BlockStatus,
     ) -> T {
         let mut block_sub = match status {
-            BlockStatus::Best => self.client.blocks().subscribe_best().await.unwrap(),
-            BlockStatus::Finalized => self.client.blocks().subscribe_finalized().await.unwrap(),
+            BlockStatus::Best => self
+                .as_ref()
+                .client
+                .blocks()
+                .subscribe_best()
+                .await
+                .unwrap(),
+            BlockStatus::Finalized => self
+                .as_ref()
+                .client
+                .blocks()
+                .subscribe_finalized()
+                .await
+                .unwrap(),
         };
 
         info!(target: "aleph-client", "waiting for event {}.{}", T::PALLET, T::EVENT);
@@ -81,7 +105,7 @@ impl AlephWaiting for Connection {
 
     async fn wait_for_era(&self, era: EraIndex, status: BlockStatus) {
         let addrs = aleph_zero::api::constants().staking().sessions_per_era();
-        let sessions_per_era = self.client.constants().at(&addrs).unwrap();
+        let sessions_per_era = self.as_ref().client.constants().at(&addrs).unwrap();
         let first_session_in_era = era * sessions_per_era;
 
         self.wait_for_session(first_session_in_era, status).await;