synthetic-network for nightly-pipeline (#804)

* as_ref in aleph-client for Connection types * rust API for synthetic-network's json API * refactored synthetic-link * new e2e-tests: high out-latency * refactored synthetic-network bash scripts * added git submodule for synthetic-network * added nightly e2e test for synthetic-network * refactored synthetic-link library * added load_config for synthetic-network e2e test * more refactoring of synthetic-link * missing synthetic-link in Cargo.toml * e2e-tests/config.rs cleaned * Into -> From in synthetic-link * slightly cleaned code for synthetic-network * added RUST_SRC_PATH in shell.nix for better support for IDEs * slightly cleaned synthetic-link * refactored synthetic-link: more types for ranged parameters * refactored synthetic-link lib * added code-docs to synthetic-link * using try_from for PortRange in synthetic-lib for input validation * missing deps after merge for synthetic-network * REVERT ME testing nightly pipeline * fix after merge * say no to install-protoc in github pipelines * Revert "say no to install-protoc in github pipelines" This reverts commit b991f75. * GITHUB_TOKEN for "Install Protoc" action * reverted changed file permissions * reverted rust-toolchain file * typo in description for PortRange * e2e-tests: config.validator_names() uses validators_count * e2e-tests: added description for the `latency` tests * shell.nix: added description for new ENV var RUST_SRC_PATH - it's for rust-analyzer * BACKUP: e2e-tests/config.rs with iterators * new verion of the synthetic-network e2e-test that supports non-dockerized runs (urls) * fix for e2e-test `no_quorum_without_high_latency` - missing call for urls instead of names * extended README.md for the synthetic-network: how to run e2e-tests with synthetic-network locally * renamed e2e-tests for high-out-latency with synthetic-network * renamed one of the e2e-tests pipelines in nightly for high-latency (synthetic-network) * Revert "REVERT ME testing nightly pipeline" This reverts commit 8eb1410.
Cardinal-Cryptography · Jan 4, 2023 · 6943547 · 6943547
1 parent 917b76e
commit 6943547
Show file tree

Hide file tree

Showing 29 changed files with 2,350 additions and 74 deletions.
diff --git a/.github/actions/run-e2e-test/action.yml b/.github/actions/run-e2e-test/action.yml
@@ -30,6 +30,18 @@ inputs:
     description: 'Whether to deploy the adder sample contract to the node.'
     required: false
     default: 'false'
+  image-path:
+    description: 'Custom path to docker image for aleph-node'
+    required: false
+    default: aleph-test-docker
+  node-image:
+    description: 'Custom name of aleph-node image'
+    required: false
+    default: aleph-node:latest
+  compose-file:
+    description: 'Custom docker-compose configuration'
+    required: false
+    default: ''
 
 runs:
   using: 'composite'
@@ -38,15 +50,15 @@ runs:
     - name: Download artifact with docker image
       uses: actions/download-artifact@v2
       with:
-        name: aleph-test-docker
+        name: ${{ inputs.image-path }}
 
     - name: Load node docker image
       shell: bash
       run: docker load -i aleph-node.tar
 
     - name: Run consensus party
       shell: bash
-      run: ./.github/scripts/run_consensus.sh -m ${{ inputs.min-validator-count }} -n ${{ inputs.node-count }}
+      run: NODE_IMAGE=${{ inputs.node-image }} DOCKER_COMPOSE=${{ inputs.compose-file }} ./.github/scripts/run_consensus.sh -m ${{ inputs.min-validator-count }} -n ${{ inputs.node-count }}
 
     - name: Sleep
       shell: bash

diff --git a/.github/scripts/run_consensus.sh b/.github/scripts/run_consensus.sh
@@ -8,11 +8,12 @@ NODE_COUNT=5
 MIN_VALIDATOR_COUNT=4
 DOCKER_COMPOSE=${DOCKER_COMPOSE:-"docker/docker-compose.yml"}
 OVERRIDE_DOCKER_COMPOSE=${OVERRIDE_DOCKER_COMPOSE:-""}
+NODE_IMAGE=${NODE_IMAGE:-aleph-node:latest}
 
 # default minimum validator count
 MIN_VALIDATOR_COUNT=4
 
-export NODE_IMAGE=aleph-node:latest
+export NODE_IMAGE
 
 mkdir -p docker/data/
 

diff --git a/.github/scripts/run_e2e_test.sh b/.github/scripts/run_e2e_test.sh
@@ -116,6 +116,10 @@ if [[ -n "${ADDER:-}" ]]; then
     ARGS+=(-e "ADDER_METADATA=/contracts/adder/target/ink/metadata.json")
 fi
 
+if [[ -n "${OUT_LATENCY:-}" ]]; then
+    ARGS+=(-e OUT_LATENCY)
+fi
+
 docker run -v "$(pwd)/contracts:/contracts" -v "$(pwd)/docker/data:/data" "${ARGS[@]}" aleph-e2e-client:latest
 
 exit $?
diff --git a/.github/workflows/build-and-push-cliain.yaml b/.github/workflows/build-and-push-cliain.yaml
@@ -21,6 +21,7 @@ jobs:
         uses: arduino/setup-protoc@v1
         with:
           version: '3.6.1'
+          repo-token: ${{ secrets.GITHUB_TOKEN }}
 
       - name: Cargo | Build release binary
         run: |
@@ -29,18 +30,18 @@ jobs:
       - name: GIT | Get branch name and commit SHA
         id: get_branch
         uses: ./.github/actions/get-branch
-      
+
       - name: Login to ECR
         uses: docker/login-action@v1
         with:
           registry: public.ecr.aws
           username: ${{ secrets.AWS_MAINNET_ACCESS_KEY_ID }}
           password: ${{ secrets.AWS_MAINNET_SECRET_ACCESS_KEY }}
-      
+
       - name: Set up Docker Buildx
         id: buildx
         uses: docker/setup-buildx-action@master
-      
+
       - name: Build and push latest docker image
         id: build-image
         env:

diff --git a/.github/workflows/build-node-and-runtime.yml b/.github/workflows/build-node-and-runtime.yml
@@ -37,6 +37,7 @@ jobs:
         uses: arduino/setup-protoc@v1
         with:
           version: '3.6.1'
+          repo-token: ${{ secrets.GITHUB_TOKEN }}
 
       - name: Install WASM target
         run: rustup target add wasm32-unknown-unknown

diff --git a/.github/workflows/build-send-postsync-hook-runtime-image.yml b/.github/workflows/build-send-postsync-hook-runtime-image.yml
@@ -36,7 +36,8 @@ jobs:
       - name: Install Protoc
         uses: arduino/setup-protoc@v1
         with:
-          version: '3.6.1'        
+          version: '3.6.1'
+          repo-token: ${{ secrets.GITHUB_TOKEN }}
 
       - name: Build binary
         run: |
@@ -76,7 +77,7 @@ jobs:
 
           docker tag ${{ env.ECR_PUSH_IMAGE }} ${{ env.ECR_LATEST }}
           docker push ${{ env.ECR_LATEST }}
-      
+
       - name: GIT | Checkout aleph-apps repo
         uses: actions/checkout@master
         with:
@@ -89,8 +90,8 @@ jobs:
         with:
           kustomize-version: "3.8.6"
 
-      - name: Update postsync hook docker image                                                                     
-        env:                                                                                                          
+      - name: Update postsync hook docker image
+        env:
           RELEASE_IMAGE: public.ecr.aws/p6e8q1z1/runtime-update-hook:${{ steps.vars.outputs.sha_short }}
           REGIONS_AWS: 'eu-central-1'
         run: |

diff --git a/.github/workflows/check-excluded-packages.yml b/.github/workflows/check-excluded-packages.yml
@@ -31,6 +31,7 @@ jobs:
         uses: arduino/setup-protoc@v1
         with:
           version: '3.6.1'
+          repo-token: ${{ secrets.GITHUB_TOKEN }}
 
       - name: Read excluded packages from Cargo.toml
         id: read_excluded

diff --git a/.github/workflows/e2e-tests-main-devnet.yml b/.github/workflows/e2e-tests-main-devnet.yml
@@ -68,7 +68,8 @@ jobs:
       - name: Install Protoc
         uses: arduino/setup-protoc@v1
         with:
-          version: '3.6.1'        
+          version: '3.6.1'
+          repo-token: ${{ secrets.GITHUB_TOKEN }}
 
       - name: Restore cache
         uses: ./.github/actions/restore-cache
@@ -115,7 +116,8 @@ jobs:
       - name: Install Protoc
         uses: arduino/setup-protoc@v1
         with:
-          version: '3.6.1'        
+          version: '3.6.1'
+          repo-token: ${{ secrets.GITHUB_TOKEN }}
 
       - name: Install WASM target
         run: rustup target add wasm32-unknown-unknown
@@ -153,7 +155,8 @@ jobs:
       - name: Install Protoc
         uses: arduino/setup-protoc@v1
         with:
-          version: '3.6.1'        
+          version: '3.6.1'
+          repo-token: ${{ secrets.GITHUB_TOKEN }}
 
       - name: Restore cache
         uses: ./.github/actions/restore-cache
@@ -839,7 +842,8 @@ jobs:
       - name: Install Protoc
         uses: arduino/setup-protoc@v1
         with:
-          version: '3.6.1'        
+          version: '3.6.1'
+          repo-token: ${{ secrets.GITHUB_TOKEN }}
 
       - name: Install WASM target
         run: rustup target add wasm32-unknown-unknown

diff --git a/.github/workflows/nightly-pipeline.yaml b/.github/workflows/nightly-pipeline.yaml
@@ -43,6 +43,37 @@ jobs:
           if-no-files-found: error
           retention-days: 7
 
+  build-synthetic-network-docker:
+    needs: [build-test-docker]
+    name: Build docker image with the test node artifact and support for synthetic-network
+    runs-on: ubuntu-20.04
+    steps:
+      - name: Checkout source code
+        uses: actions/checkout@v2
+
+      - name: Download artifact with docker image for aleph-node
+        uses: actions/download-artifact@v2
+        with:
+          name: aleph-test-docker
+
+      - name: Load node docker image
+        shell: bash
+        run: docker load -i aleph-node.tar
+
+      - name: Build test docker image
+        id: build-image
+        run: |
+          scripts/synthetic-network/build_synthetic-network.sh
+          docker save -o aleph-node.tar aleph-node:syntheticnet
+
+      - name: Upload test docker image
+        uses: actions/upload-artifact@v2
+        with:
+          name: aleph-test-synthetic-docker
+          path: aleph-node.tar
+          if-no-files-found: error
+          retention-days: 7
+
   check-determinism:
     needs: [build-new-node]
     name: Verify runtime build determinism
@@ -59,7 +90,8 @@ jobs:
       - name: Install Protoc
         uses: arduino/setup-protoc@v1
         with:
-          version: '3.6.1'        
+          version: '3.6.1'
+          repo-token: ${{ secrets.GITHUB_TOKEN }}
 
       - name: Install WASM target
         run: rustup target add wasm32-unknown-unknown
@@ -96,7 +128,8 @@ jobs:
       - name: Install Protoc
         uses: arduino/setup-protoc@v1
         with:
-          version: '3.6.1'        
+          version: '3.6.1'
+          repo-token: ${{ secrets.GITHUB_TOKEN }}
 
       - name: Restore cache
         uses: ./.github/actions/restore-cache
@@ -140,9 +173,45 @@ jobs:
           randomized: true
         timeout-minutes: 60
 
+  run-e2e-high-out-latency:
+    needs: [build-synthetic-network-docker, build-test-client]
+    name: Run high out-latency test
+    runs-on: ubuntu-20.04
+    steps:
+      - name: Checkout source code
+        uses: actions/checkout@v2
+
+      - name: Run e2e test
+        uses: ./.github/actions/run-e2e-test
+        with:
+          test-case: high_out_latency_for_all
+          image-path: aleph-test-synthetic-docker
+          node-image: aleph-node:syntheticnet
+          compose-file: docker/docker-compose.synthetic-network.yml
+        timeout-minutes: 30
+
+  run-e2e-no-quorum-without-high-out-latency:
+    needs: [build-synthetic-network-docker, build-test-client]
+    name: Run high out-latency for every quorum
+    runs-on: ubuntu-20.04
+    steps:
+      - name: Checkout source code
+        uses: actions/checkout@v2
+
+      - name: Run e2e test
+        uses: ./.github/actions/run-e2e-test
+        with:
+          test-case: high_out_latency_for_each_quorum
+          image-path: aleph-test-synthetic-docker
+          node-image: aleph-node:syntheticnet
+          compose-file: docker/docker-compose.synthetic-network.yml
+        timeout-minutes: 15
+
   check-e2e-test-suite-completion:
     needs: [
       run-e2e-authorities-are-staking,
+      run-e2e-high-out-latency,
+      run-e2e-no-quorum-without-high-out-latency,
     ]
     name: Check e2e test suite completion
     runs-on: ubuntu-20.04

diff --git a/.github/workflows/unit_tests.yml b/.github/workflows/unit_tests.yml
@@ -30,7 +30,8 @@ jobs:
       - name: Install Protoc
         uses: arduino/setup-protoc@v1
         with:
-          version: '3.6.1'        
+          version: '3.6.1'
+          repo-token: ${{ secrets.GITHUB_TOKEN }}
 
       - name: Install clippy and fmt
         run: rustup component add clippy rustfmt

diff --git a/.gitmodules b/.gitmodules
@@ -0,0 +1,3 @@
+[submodule "scripts/synthetic-network/vendor/synthetic-network"]
+	path = scripts/synthetic-network/vendor/synthetic-network
+	url = https://github.com/daily-co/synthetic-network.git
diff --git a/Cargo.toml b/Cargo.toml
@@ -18,7 +18,8 @@ exclude = [
   "fork-off",
   "benches/payout-stakers",
   "bin/cliain",
-  "contracts/adder"
+  "contracts/adder",
+  "scripts/synthetic-network/synthetic-link",
 ]
 
 [profile.release]