Reduce use of self-hosted runner and use cache (#409)

* try to replace self-host with github runner * rectify the usage of cache * remove time limit * use upload/download * fix download path * missing checkout * use ul/dl version 3 * try to use yaml anchor * add feature branch trigger * revert anchor as its not supported * try to use buildx cache * add sccache and file-filters * missing checkout * fix permission * move concurrency to workflow level * update ts-tests dep to check the file filter * unuse docker/build-push-action * cargo update * refine the file filter * try cargo chef * reuse buildx action * fix recipe path * prepend cargo build with apt * fix order of copy * use self-hosted docker * remove buildx * update comment * retry buildx * try local cache * update comment to test cached builds * fix docker save * modify comment to trigger GHA * separate dev and prod Dockefile * more comment and TYPE * update .gitignore * more comment * update README * fix typo
litentry · Mar 24, 2022 · 9a5dac9 · 9a5dac9
1 parent 63ac4bf
commit 9a5dac9
Show file tree

Hide file tree

Showing 16 changed files with 350 additions and 127 deletions.
diff --git a/.dockerignore b/.dockerignore
@@ -1,4 +1,2 @@
-**/polkadot
-**/token-server
 **/target
 **/ts-tests
diff --git a/.github/workflows/benchmark_runtime_weights.yml b/.github/workflows/benchmark_runtime_weights.yml
@@ -29,7 +29,7 @@ jobs:
 
       - name: Build docker image
         run: |
-          ./scripts/build-docker.sh runtime-benchmarks --features=runtime-benchmarks
+          ./scripts/build-docker.sh prod runtime-benchmarks --features=runtime-benchmarks
 
       - name: Push docker image
         run: |

diff --git a/.github/workflows/build_and_run_test.yml b/.github/workflows/build_and_run_test.yml
@@ -15,7 +15,36 @@ on:
 env:
   CARGO_TERM_COLOR: always
 
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
 jobs:
+  check-file-change:
+    runs-on: ubuntu-latest
+    outputs:
+      src: ${{ steps.filter.outputs.src }}
+    steps:
+      - uses: actions/checkout@v2
+        with:
+          fetch-depth: 0
+
+      # Checks to see if any files in the PR/commit match one of the listed file types.
+      # We can use this filter to decide whether or not to build docker images
+      - uses: dorny/paths-filter@v2
+        id: filter
+        with:
+          filters: |
+            src:
+              - 'node/**'
+              - 'pallets/**'
+              - 'primitives/**'
+              - 'runtime/**'
+              - 'docker/Dockerfile.dev'
+              - '**/Cargo.lock'
+              - '**/Cargo.toml'
+              - '**/rust-toolchain.toml'
+
   check-cargo-fmt:
     runs-on: ubuntu-latest
     steps:
@@ -37,7 +66,8 @@ jobs:
 
   check-cargo-clippy:
     runs-on: ubuntu-latest
-    needs: check-cargo-fmt
+    needs: [check-cargo-fmt, check-file-change]
+    if: needs.check-file-change.outputs.src == 'true'
     steps:
       - uses: actions/checkout@v2
         with:
@@ -56,41 +86,89 @@ jobs:
         run: make clippy
 
   build-docker:
-    if: "!contains(github.event.commits[0].message, '[SKIP CI]')"
-    concurrency:
-      group: ${{ github.ref }}
-      cancel-in-progress: true
+    # run the docker build on our self-hosted runner, which takes
+    # - 25min without cache, or
+    # - 15 min with cache
+    # on a standard github runner it takes 1 hour+
+    # see also https://github.com/litentry/litentry-parachain/issues/259
     runs-on: self-hosted
-    needs: check-cargo-fmt
+    needs: [check-cargo-fmt, check-file-change]
     steps:
       - name: Checkout codes
         uses: actions/checkout@v2
         with:
           fetch-depth: 0
 
+      - name: Setup docker buildx
+        uses: docker/setup-buildx-action@v1
+
+      # Syncing GHA cache seems to be very slow, we use local cache for a self-hosted runner
       - name: Build docker image
-        timeout-minutes: 40
+        if: needs.check-file-change.outputs.src == 'true'
+        uses: docker/build-push-action@v2
+        with:
+          context: .
+          tags: litentry/litentry-parachain:latest
+          outputs: type=docker,dest=litentry-parachain.tar
+          push: false
+          file: docker/Dockerfile.dev
+          cache-from: type=local,src=/tmp/parachain-buildx-cache
+          cache-to: type=local,dest=/tmp/parachain-buildx-cache-new,mode=max
+
+      - name: Move cache
+        if: needs.check-file-change.outputs.src == 'true'
+        # Temp fix for constantly growing cache, see
+        # https://github.com/docker/build-push-action/issues/252
+        # https://github.com/moby/buildkit/issues/1896
+        run: |
+          rm -rf /tmp/parachain-buildx-cache
+          mv /tmp/parachain-buildx-cache-new /tmp/parachain-buildx-cache
+
+      - name: Pull docker image optinally
+        if: needs.check-file-change.outputs.src == 'false'
         run: |
-          ./scripts/build-docker.sh
-          echo "============================="
-          docker images
+          docker pull litentry/litentry-parachain:latest
+          docker save litentry/litentry-parachain:latest -o litentry-parachain.tar
+
+      - name: Upload docker image
+        uses: actions/upload-artifact@v3
+        with:
+          name: docker-artifact
+          path: litentry-parachain.tar
+
+      - name: Remove dangling docker images if any
+        run: |
+          [ -z "$(docker images --filter=dangling=true -q)" ] || docker rmi -f $(docker images --filter=dangling=true -q)
 
   run-ts-tests:
-    runs-on: self-hosted
+    runs-on: ubuntu-latest
     needs: build-docker
     strategy:
       matrix:
         chain:
           - litmus
           - litentry
     steps:
+      - name: Checkout codes
+        uses: actions/checkout@v2
+        with:
+          fetch-depth: 0
+
+      - uses: actions/download-artifact@v3
+        with:
+          name: docker-artifact
+
+      - name: Load docker image
+        run: |
+          docker load -i litentry-parachain.tar
+
       - name: Run ts tests for ${{ matrix.chain }}
         timeout-minutes: 20
         run: |
           make test-ts-docker-${{ matrix.chain }}
 
       - name: Archive logs if test fails
-        uses: actions/upload-artifact@v2
+        uses: actions/upload-artifact@v3
         if: ${{ failure() }}
         with:
           name: ${{ matrix.chain }}-ts-tests-artifacts
@@ -104,7 +182,8 @@ jobs:
 
   run-cargo-unit-tests:
     runs-on: ubuntu-latest
-    needs: check-cargo-fmt
+    needs: [check-cargo-fmt, check-file-change]
+    if: needs.check-file-change.outputs.src == 'true'
     steps:
       - uses: actions/checkout@v2
         with:
@@ -123,7 +202,14 @@ jobs:
 
   run-cargo-runtime-tests:
     runs-on: ubuntu-latest
-    needs: check-cargo-fmt
+    needs: [check-cargo-fmt, check-file-change]
+    if: needs.check-file-change.outputs.src == 'true'
+    env:
+      RUST_BACKTRACE: full
+      RUSTC_WRAPPER: sccache
+      SCCACHE_CACHE_SIZE: 10G
+      SCCACHE_DIR: /home/runner/.cache/sccache
+      CARGO_INCREMENTAL: 0
     strategy:
       matrix:
         chain:
@@ -142,13 +228,64 @@ jobs:
           target: wasm32-unknown-unknown
           default: true
 
+      # use sccache to accelerate binary compilation
+      # see https://www.infinyon.com/blog/2021/04/github-actions-best-practices/
+      - name: Install sccache
+        env:
+          LINK: https://github.com/mozilla/sccache/releases/download
+          SCCACHE_VERSION: v0.2.15
+        run: |
+          SCCACHE_FILE=sccache-$SCCACHE_VERSION-x86_64-unknown-linux-musl
+          mkdir -p $HOME/.local/bin
+          curl -L "$LINK/$SCCACHE_VERSION/$SCCACHE_FILE.tar.gz" | tar xz
+          mv -f $SCCACHE_FILE/sccache $HOME/.local/bin/sccache
+          chmod +x $HOME/.local/bin/sccache
+          echo "$HOME/.local/bin" >> $GITHUB_PATH
+
+      - name: Cache cargo registry
+        uses: actions/cache@v2
+        continue-on-error: false
+        with:
+          path: |
+            ~/.cargo/registry
+            ~/.cargo/git
+          key: cargo-${{ hashFiles('**/Cargo.lock') }}
+          restore-keys: |
+            cargo-
+
+      - name: Cache sccache
+        uses: actions/cache@v2
+        continue-on-error: false
+        with:
+          path: /home/runner/.cache/sccache
+          key: sccache-${{ hashFiles('**/Cargo.lock') }}
+          restore-keys: |
+            sccache-
+
       - name: Run runtime integration tests
         run: cargo test --release -p ${{ matrix.chain }}-parachain-runtime --lib
 
+      - name: Print sccache stats
+        run: sccache --show-stats
+
   push-docker-image:
-    runs-on: self-hosted
+    runs-on: ubuntu-latest
     needs: ["check-cargo-clippy", "run-cargo-unit-tests", "run-cargo-runtime-tests", "run-ts-tests"]
     steps:
+      - uses: actions/download-artifact@v3
+        with:
+          name: docker-artifact
+
+      - name: Load docker image
+        run: |
+          docker load -i litentry-parachain.tar
+
+      - name: Dockerhub login
+        uses: docker/login-action@v1
+        with:
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_PASSWORD }}
+
       - name: Push docker image on dev branch if test passes
         if: ${{ success() && (github.event_name == 'push') && (github.ref == 'refs/heads/dev') }}
         run:

diff --git a/.github/workflows/build_docker_with_args.yml b/.github/workflows/build_docker_with_args.yml
@@ -3,6 +3,12 @@ name: Build docker with args
 on:
   workflow_dispatch:
     inputs:
+      dockerfile_type:
+        type: choice
+        description: The type of Dockerfile to use
+        options:
+        - dev
+        - prod
       docker_tag:
         description: The tag for the built docker image
         required: true
@@ -24,7 +30,7 @@ jobs:
 
       - name: Build docker image
         run: |
-          ./scripts/build-docker.sh ${{ github.event.inputs.docker_tag }} "${{ github.event.inputs.args }}"
+          ./scripts/build-docker.sh ${{ github.event.inputs.dockerfile_type }} ${{ github.event.inputs.docker_tag }} "${{ github.event.inputs.args }}"
           echo "============================="
           docker images
 
@@ -41,7 +47,7 @@ jobs:
           docker cp $(docker create --rm litentry/litentry-parachain:${{ github.event.inputs.docker_tag }}):/usr/local/bin/litentry-collator .
 
       - name: Upload the client binary
-        uses: actions/upload-artifact@v2
+        uses: actions/upload-artifact@v3
         with:
           name: litentry-collator
           path: |

diff --git a/.github/workflows/build_wasm.yml b/.github/workflows/build_wasm.yml
@@ -43,7 +43,7 @@ jobs:
           cp ${{ steps.srtool_build.outputs.wasm_compressed }} ${{ github.event.inputs.chain }}-parachain-runtime.compact.compressed.wasm
 
       - name: Upload wasm artefacts
-        uses: actions/upload-artifact@v2
+        uses: actions/upload-artifact@v3
         with:
           name: ${{ github.event.inputs.chain }}-parachain-runtime
           path: |

diff --git a/.github/workflows/create_release_draft.yml b/.github/workflows/create_release_draft.yml
@@ -65,7 +65,7 @@ jobs:
           cp ${{ steps.srtool_build.outputs.wasm_compressed }} ${{ matrix.chain }}-parachain-runtime.compact.compressed.wasm
 
       - name: Upload wasm artefacts
-        uses: actions/upload-artifact@v2
+        uses: actions/upload-artifact@v3
         with:
           name: ${{ matrix.chain }}-parachain-runtime
           path: |
@@ -85,7 +85,7 @@ jobs:
 
       - name: Build docker image
         run: |
-          ./scripts/build-docker.sh ${{ env.RELEASE_TAG }}
+          ./scripts/build-docker.sh prod ${{ env.RELEASE_TAG }}
           echo "============================="
           docker images
 
@@ -108,7 +108,7 @@ jobs:
           docker cp $(docker create --rm litentry/litentry-parachain:${{ env.RELEASE_TAG }}):/usr/local/bin/litentry-collator .
 
       - name: Upload the client binary
-        uses: actions/upload-artifact@v2
+        uses: actions/upload-artifact@v3
         with:
           name: litentry-collator
           if-no-files-found: ignore
@@ -136,7 +136,7 @@ jobs:
           fetch-depth: 0
 
       - name: Download all artefacts
-        uses: actions/download-artifact@v2
+        uses: actions/download-artifact@v3
 
       - name: Generate release notes
         run: |

diff --git a/.gitignore b/.gitignore
@@ -5,7 +5,6 @@
 .cargo/config
 
 **/node_modules/
-**/para-*
 
 docker/yarn.lock
 docker/package.json
@@ -14,3 +13,5 @@ docker/generated-*/
 ts-tests/package-lock.json
 
 tags.lock
+
+recipe.json