diff --git a/.dockerignore b/.dockerignore index 9d8aa0ed0d..f88ea00741 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,4 +1,2 @@ -**/polkadot -**/token-server **/target **/ts-tests diff --git a/.github/workflows/benchmark_runtime_weights.yml b/.github/workflows/benchmark_runtime_weights.yml index 751d8186cb..12cafe5101 100644 --- a/.github/workflows/benchmark_runtime_weights.yml +++ b/.github/workflows/benchmark_runtime_weights.yml @@ -29,7 +29,7 @@ jobs: - name: Build docker image run: | - ./scripts/build-docker.sh runtime-benchmarks --features=runtime-benchmarks + ./scripts/build-docker.sh prod runtime-benchmarks --features=runtime-benchmarks - name: Push docker image run: | diff --git a/.github/workflows/build_and_run_test.yml b/.github/workflows/build_and_run_test.yml index 3fa6d18dd3..38d2111cc7 100644 --- a/.github/workflows/build_and_run_test.yml +++ b/.github/workflows/build_and_run_test.yml @@ -15,7 +15,36 @@ on: env: CARGO_TERM_COLOR: always +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + jobs: + check-file-change: + runs-on: ubuntu-latest + outputs: + src: ${{ steps.filter.outputs.src }} + steps: + - uses: actions/checkout@v2 + with: + fetch-depth: 0 + + # Checks to see if any files in the PR/commit match one of the listed file types. + # We can use this filter to decide whether or not to build docker images + - uses: dorny/paths-filter@v2 + id: filter + with: + filters: | + src: + - 'node/**' + - 'pallets/**' + - 'primitives/**' + - 'runtime/**' + - 'docker/Dockerfile.dev' + - '**/Cargo.lock' + - '**/Cargo.toml' + - '**/rust-toolchain.toml' + check-cargo-fmt: runs-on: ubuntu-latest steps: @@ -37,7 +66,8 @@ jobs: check-cargo-clippy: runs-on: ubuntu-latest - needs: check-cargo-fmt + needs: [check-cargo-fmt, check-file-change] + if: needs.check-file-change.outputs.src == 'true' steps: - uses: actions/checkout@v2 with: @@ -56,27 +86,62 @@ jobs: run: make clippy build-docker: - if: "!contains(github.event.commits[0].message, '[SKIP CI]')" - concurrency: - group: ${{ github.ref }} - cancel-in-progress: true + # run the docker build on our self-hosted runner, which takes + # - 25min without cache, or + # - 15 min with cache + # on a standard github runner it takes 1 hour+ + # see also https://github.com/litentry/litentry-parachain/issues/259 runs-on: self-hosted - needs: check-cargo-fmt + needs: [check-cargo-fmt, check-file-change] steps: - name: Checkout codes uses: actions/checkout@v2 with: fetch-depth: 0 + - name: Setup docker buildx + uses: docker/setup-buildx-action@v1 + + # Syncing GHA cache seems to be very slow, we use local cache for a self-hosted runner - name: Build docker image - timeout-minutes: 40 + if: needs.check-file-change.outputs.src == 'true' + uses: docker/build-push-action@v2 + with: + context: . + tags: litentry/litentry-parachain:latest + outputs: type=docker,dest=litentry-parachain.tar + push: false + file: docker/Dockerfile.dev + cache-from: type=local,src=/tmp/parachain-buildx-cache + cache-to: type=local,dest=/tmp/parachain-buildx-cache-new,mode=max + + - name: Move cache + if: needs.check-file-change.outputs.src == 'true' + # Temp fix for constantly growing cache, see + # https://github.com/docker/build-push-action/issues/252 + # https://github.com/moby/buildkit/issues/1896 + run: | + rm -rf /tmp/parachain-buildx-cache + mv /tmp/parachain-buildx-cache-new /tmp/parachain-buildx-cache + + - name: Pull docker image optinally + if: needs.check-file-change.outputs.src == 'false' run: | - ./scripts/build-docker.sh - echo "=============================" - docker images + docker pull litentry/litentry-parachain:latest + docker save litentry/litentry-parachain:latest -o litentry-parachain.tar + + - name: Upload docker image + uses: actions/upload-artifact@v3 + with: + name: docker-artifact + path: litentry-parachain.tar + + - name: Remove dangling docker images if any + run: | + [ -z "$(docker images --filter=dangling=true -q)" ] || docker rmi -f $(docker images --filter=dangling=true -q) run-ts-tests: - runs-on: self-hosted + runs-on: ubuntu-latest needs: build-docker strategy: matrix: @@ -84,13 +149,26 @@ jobs: - litmus - litentry steps: + - name: Checkout codes + uses: actions/checkout@v2 + with: + fetch-depth: 0 + + - uses: actions/download-artifact@v3 + with: + name: docker-artifact + + - name: Load docker image + run: | + docker load -i litentry-parachain.tar + - name: Run ts tests for ${{ matrix.chain }} timeout-minutes: 20 run: | make test-ts-docker-${{ matrix.chain }} - name: Archive logs if test fails - uses: actions/upload-artifact@v2 + uses: actions/upload-artifact@v3 if: ${{ failure() }} with: name: ${{ matrix.chain }}-ts-tests-artifacts @@ -104,7 +182,8 @@ jobs: run-cargo-unit-tests: runs-on: ubuntu-latest - needs: check-cargo-fmt + needs: [check-cargo-fmt, check-file-change] + if: needs.check-file-change.outputs.src == 'true' steps: - uses: actions/checkout@v2 with: @@ -123,7 +202,14 @@ jobs: run-cargo-runtime-tests: runs-on: ubuntu-latest - needs: check-cargo-fmt + needs: [check-cargo-fmt, check-file-change] + if: needs.check-file-change.outputs.src == 'true' + env: + RUST_BACKTRACE: full + RUSTC_WRAPPER: sccache + SCCACHE_CACHE_SIZE: 10G + SCCACHE_DIR: /home/runner/.cache/sccache + CARGO_INCREMENTAL: 0 strategy: matrix: chain: @@ -142,13 +228,64 @@ jobs: target: wasm32-unknown-unknown default: true + # use sccache to accelerate binary compilation + # see https://www.infinyon.com/blog/2021/04/github-actions-best-practices/ + - name: Install sccache + env: + LINK: https://github.com/mozilla/sccache/releases/download + SCCACHE_VERSION: v0.2.15 + run: | + SCCACHE_FILE=sccache-$SCCACHE_VERSION-x86_64-unknown-linux-musl + mkdir -p $HOME/.local/bin + curl -L "$LINK/$SCCACHE_VERSION/$SCCACHE_FILE.tar.gz" | tar xz + mv -f $SCCACHE_FILE/sccache $HOME/.local/bin/sccache + chmod +x $HOME/.local/bin/sccache + echo "$HOME/.local/bin" >> $GITHUB_PATH + + - name: Cache cargo registry + uses: actions/cache@v2 + continue-on-error: false + with: + path: | + ~/.cargo/registry + ~/.cargo/git + key: cargo-${{ hashFiles('**/Cargo.lock') }} + restore-keys: | + cargo- + + - name: Cache sccache + uses: actions/cache@v2 + continue-on-error: false + with: + path: /home/runner/.cache/sccache + key: sccache-${{ hashFiles('**/Cargo.lock') }} + restore-keys: | + sccache- + - name: Run runtime integration tests run: cargo test --release -p ${{ matrix.chain }}-parachain-runtime --lib + - name: Print sccache stats + run: sccache --show-stats + push-docker-image: - runs-on: self-hosted + runs-on: ubuntu-latest needs: ["check-cargo-clippy", "run-cargo-unit-tests", "run-cargo-runtime-tests", "run-ts-tests"] steps: + - uses: actions/download-artifact@v3 + with: + name: docker-artifact + + - name: Load docker image + run: | + docker load -i litentry-parachain.tar + + - name: Dockerhub login + uses: docker/login-action@v1 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_PASSWORD }} + - name: Push docker image on dev branch if test passes if: ${{ success() && (github.event_name == 'push') && (github.ref == 'refs/heads/dev') }} run: diff --git a/.github/workflows/build_docker_with_args.yml b/.github/workflows/build_docker_with_args.yml index a7a2d6450b..cc526cf4e5 100644 --- a/.github/workflows/build_docker_with_args.yml +++ b/.github/workflows/build_docker_with_args.yml @@ -3,6 +3,12 @@ name: Build docker with args on: workflow_dispatch: inputs: + dockerfile_type: + type: choice + description: The type of Dockerfile to use + options: + - dev + - prod docker_tag: description: The tag for the built docker image required: true @@ -24,7 +30,7 @@ jobs: - name: Build docker image run: | - ./scripts/build-docker.sh ${{ github.event.inputs.docker_tag }} "${{ github.event.inputs.args }}" + ./scripts/build-docker.sh ${{ github.event.inputs.dockerfile_type }} ${{ github.event.inputs.docker_tag }} "${{ github.event.inputs.args }}" echo "=============================" docker images @@ -41,7 +47,7 @@ jobs: docker cp $(docker create --rm litentry/litentry-parachain:${{ github.event.inputs.docker_tag }}):/usr/local/bin/litentry-collator . - name: Upload the client binary - uses: actions/upload-artifact@v2 + uses: actions/upload-artifact@v3 with: name: litentry-collator path: | diff --git a/.github/workflows/build_wasm.yml b/.github/workflows/build_wasm.yml index d4986ed27f..4591027e4b 100644 --- a/.github/workflows/build_wasm.yml +++ b/.github/workflows/build_wasm.yml @@ -43,7 +43,7 @@ jobs: cp ${{ steps.srtool_build.outputs.wasm_compressed }} ${{ github.event.inputs.chain }}-parachain-runtime.compact.compressed.wasm - name: Upload wasm artefacts - uses: actions/upload-artifact@v2 + uses: actions/upload-artifact@v3 with: name: ${{ github.event.inputs.chain }}-parachain-runtime path: | diff --git a/.github/workflows/create_release_draft.yml b/.github/workflows/create_release_draft.yml index 01974b2f70..c1abadf842 100644 --- a/.github/workflows/create_release_draft.yml +++ b/.github/workflows/create_release_draft.yml @@ -65,7 +65,7 @@ jobs: cp ${{ steps.srtool_build.outputs.wasm_compressed }} ${{ matrix.chain }}-parachain-runtime.compact.compressed.wasm - name: Upload wasm artefacts - uses: actions/upload-artifact@v2 + uses: actions/upload-artifact@v3 with: name: ${{ matrix.chain }}-parachain-runtime path: | @@ -85,7 +85,7 @@ jobs: - name: Build docker image run: | - ./scripts/build-docker.sh ${{ env.RELEASE_TAG }} + ./scripts/build-docker.sh prod ${{ env.RELEASE_TAG }} echo "=============================" docker images @@ -108,7 +108,7 @@ jobs: docker cp $(docker create --rm litentry/litentry-parachain:${{ env.RELEASE_TAG }}):/usr/local/bin/litentry-collator . - name: Upload the client binary - uses: actions/upload-artifact@v2 + uses: actions/upload-artifact@v3 with: name: litentry-collator if-no-files-found: ignore @@ -136,7 +136,7 @@ jobs: fetch-depth: 0 - name: Download all artefacts - uses: actions/download-artifact@v2 + uses: actions/download-artifact@v3 - name: Generate release notes run: | diff --git a/.gitignore b/.gitignore index c0a0e1735b..8a7534fbc2 100644 --- a/.gitignore +++ b/.gitignore @@ -5,7 +5,6 @@ .cargo/config **/node_modules/ -**/para-* docker/yarn.lock docker/package.json @@ -14,3 +13,5 @@ docker/generated-*/ ts-tests/package-lock.json tags.lock + +recipe.json diff --git a/Cargo.lock b/Cargo.lock index 31c6865dba..0ee3225e1b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -99,9 +99,9 @@ dependencies = [ [[package]] name = "anyhow" -version = "1.0.55" +version = "1.0.56" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "159bb86af3a200e19a068f4224eae4c8bb2d0fa054c7e5d1cacd5cef95e684cd" +checksum = "4361135be9122e0870de935d7c439aef945b9f9ddd4199a553b5270b49c82a27" [[package]] name = "approx" @@ -301,9 +301,9 @@ dependencies = [ [[package]] name = "async-task" -version = "4.1.0" +version = "4.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "677d306121baf53310a3fd342d88dc0824f6bbeace68347593658525565abee8" +checksum = "30696a84d817107fc028e049980e09d5e140e8da8f1caeb17e8e950658a3cea9" [[package]] name = "async-trait" @@ -645,9 +645,9 @@ checksum = "8d696c370c750c948ada61c69a0ee2cbbb9c50b1019ddb86d9317157a99c2cae" [[package]] name = "blocking" -version = "1.1.0" +version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "046e47d4b2d391b1f6f8b407b1deb8dee56c1852ccd868becf2710f601b5f427" +checksum = "c6ccb65d468978a086b69884437ded69a90faab3bbe6e67f242173ea728acccc" dependencies = [ "async-channel", "async-task", @@ -1234,9 +1234,9 @@ dependencies = [ [[package]] name = "crossbeam-channel" -version = "0.5.2" +version = "0.5.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e54ea8bc3fb1ee042f5aace6e3c6e025d3874866da222930f70ce62aceba0bfa" +checksum = "fdbfe11fe19ff083c48923cf179540e8cd0535903dc35e178a1fdeeb59aef51f" dependencies = [ "cfg-if 1.0.0", "crossbeam-utils", @@ -1255,10 +1255,11 @@ dependencies = [ [[package]] name = "crossbeam-epoch" -version = "0.9.7" +version = "0.9.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c00d6d2ea26e8b151d99093005cb442fb9a37aeaca582a03ec70946f49ab5ed9" +checksum = "1145cf131a2c6ba0615079ab6a638f7e1973ac9c2634fcbeaaad6114246efe8c" dependencies = [ + "autocfg", "cfg-if 1.0.0", "crossbeam-utils", "lazy_static", @@ -1268,9 +1269,9 @@ dependencies = [ [[package]] name = "crossbeam-utils" -version = "0.8.7" +version = "0.8.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b5e5bed1f1c269533fa816a0a5492b3545209a205ca1a54842be180eb63a16a6" +checksum = "0bf124c720b7686e3c2663cf54062ab0f68a88af2fb6a030e87e30bf721fcb38" dependencies = [ "cfg-if 1.0.0", "lazy_static", @@ -1354,7 +1355,7 @@ dependencies = [ [[package]] name = "cumulus-client-cli" version = "0.1.0" -source = "git+https://github.com/paritytech/cumulus?branch=polkadot-v0.9.17#76479e7fef3af7c8828a44647847b01afd5fefe5" +source = "git+https://github.com/paritytech/cumulus?branch=polkadot-v0.9.17#db11baacc325537be74ad34517fcb28ed9ded6c6" dependencies = [ "clap", "sc-cli", @@ -1364,7 +1365,7 @@ dependencies = [ [[package]] name = "cumulus-client-collator" version = "0.1.0" -source = "git+https://github.com/paritytech/cumulus?branch=polkadot-v0.9.17#76479e7fef3af7c8828a44647847b01afd5fefe5" +source = "git+https://github.com/paritytech/cumulus?branch=polkadot-v0.9.17#db11baacc325537be74ad34517fcb28ed9ded6c6" dependencies = [ "cumulus-client-consensus-common", "cumulus-client-network", @@ -1388,7 +1389,7 @@ dependencies = [ [[package]] name = "cumulus-client-consensus-aura" version = "0.1.0" -source = "git+https://github.com/paritytech/cumulus?branch=polkadot-v0.9.17#76479e7fef3af7c8828a44647847b01afd5fefe5" +source = "git+https://github.com/paritytech/cumulus?branch=polkadot-v0.9.17#db11baacc325537be74ad34517fcb28ed9ded6c6" dependencies = [ "async-trait", "cumulus-client-consensus-common", @@ -1417,7 +1418,7 @@ dependencies = [ [[package]] name = "cumulus-client-consensus-common" version = "0.1.0" -source = "git+https://github.com/paritytech/cumulus?branch=polkadot-v0.9.17#76479e7fef3af7c8828a44647847b01afd5fefe5" +source = "git+https://github.com/paritytech/cumulus?branch=polkadot-v0.9.17#db11baacc325537be74ad34517fcb28ed9ded6c6" dependencies = [ "async-trait", "cumulus-relay-chain-interface", @@ -1438,7 +1439,7 @@ dependencies = [ [[package]] name = "cumulus-client-consensus-relay-chain" version = "0.1.0" -source = "git+https://github.com/paritytech/cumulus?branch=polkadot-v0.9.17#76479e7fef3af7c8828a44647847b01afd5fefe5" +source = "git+https://github.com/paritytech/cumulus?branch=polkadot-v0.9.17#db11baacc325537be74ad34517fcb28ed9ded6c6" dependencies = [ "async-trait", "cumulus-client-consensus-common", @@ -1462,7 +1463,7 @@ dependencies = [ [[package]] name = "cumulus-client-network" version = "0.1.0" -source = "git+https://github.com/paritytech/cumulus?branch=polkadot-v0.9.17#76479e7fef3af7c8828a44647847b01afd5fefe5" +source = "git+https://github.com/paritytech/cumulus?branch=polkadot-v0.9.17#db11baacc325537be74ad34517fcb28ed9ded6c6" dependencies = [ "async-trait", "cumulus-relay-chain-interface", @@ -1487,7 +1488,7 @@ dependencies = [ [[package]] name = "cumulus-client-pov-recovery" version = "0.1.0" -source = "git+https://github.com/paritytech/cumulus?branch=polkadot-v0.9.17#76479e7fef3af7c8828a44647847b01afd5fefe5" +source = "git+https://github.com/paritytech/cumulus?branch=polkadot-v0.9.17#db11baacc325537be74ad34517fcb28ed9ded6c6" dependencies = [ "cumulus-primitives-core", "cumulus-relay-chain-interface", @@ -1511,7 +1512,7 @@ dependencies = [ [[package]] name = "cumulus-client-service" version = "0.1.0" -source = "git+https://github.com/paritytech/cumulus?branch=polkadot-v0.9.17#76479e7fef3af7c8828a44647847b01afd5fefe5" +source = "git+https://github.com/paritytech/cumulus?branch=polkadot-v0.9.17#db11baacc325537be74ad34517fcb28ed9ded6c6" dependencies = [ "cumulus-client-collator", "cumulus-client-consensus-common", @@ -1540,7 +1541,7 @@ dependencies = [ [[package]] name = "cumulus-pallet-aura-ext" version = "0.1.0" -source = "git+https://github.com/paritytech/cumulus?branch=polkadot-v0.9.17#76479e7fef3af7c8828a44647847b01afd5fefe5" +source = "git+https://github.com/paritytech/cumulus?branch=polkadot-v0.9.17#db11baacc325537be74ad34517fcb28ed9ded6c6" dependencies = [ "frame-executive", "frame-support", @@ -1558,7 +1559,7 @@ dependencies = [ [[package]] name = "cumulus-pallet-dmp-queue" version = "0.1.0" -source = "git+https://github.com/paritytech/cumulus?branch=polkadot-v0.9.17#76479e7fef3af7c8828a44647847b01afd5fefe5" +source = "git+https://github.com/paritytech/cumulus?branch=polkadot-v0.9.17#db11baacc325537be74ad34517fcb28ed9ded6c6" dependencies = [ "cumulus-primitives-core", "frame-support", @@ -1576,7 +1577,7 @@ dependencies = [ [[package]] name = "cumulus-pallet-parachain-system" version = "0.1.0" -source = "git+https://github.com/paritytech/cumulus?branch=polkadot-v0.9.17#76479e7fef3af7c8828a44647847b01afd5fefe5" +source = "git+https://github.com/paritytech/cumulus?branch=polkadot-v0.9.17#db11baacc325537be74ad34517fcb28ed9ded6c6" dependencies = [ "cumulus-pallet-parachain-system-proc-macro", "cumulus-primitives-core", @@ -1606,7 +1607,7 @@ dependencies = [ [[package]] name = "cumulus-pallet-parachain-system-proc-macro" version = "0.1.0" -source = "git+https://github.com/paritytech/cumulus?branch=polkadot-v0.9.17#76479e7fef3af7c8828a44647847b01afd5fefe5" +source = "git+https://github.com/paritytech/cumulus?branch=polkadot-v0.9.17#db11baacc325537be74ad34517fcb28ed9ded6c6" dependencies = [ "proc-macro-crate 1.1.3", "proc-macro2", @@ -1617,7 +1618,7 @@ dependencies = [ [[package]] name = "cumulus-pallet-session-benchmarking" version = "3.0.0" -source = "git+https://github.com/paritytech/cumulus?branch=polkadot-v0.9.17#76479e7fef3af7c8828a44647847b01afd5fefe5" +source = "git+https://github.com/paritytech/cumulus?branch=polkadot-v0.9.17#db11baacc325537be74ad34517fcb28ed9ded6c6" dependencies = [ "frame-benchmarking", "frame-support", @@ -1631,7 +1632,7 @@ dependencies = [ [[package]] name = "cumulus-pallet-xcm" version = "0.1.0" -source = "git+https://github.com/paritytech/cumulus?branch=polkadot-v0.9.17#76479e7fef3af7c8828a44647847b01afd5fefe5" +source = "git+https://github.com/paritytech/cumulus?branch=polkadot-v0.9.17#db11baacc325537be74ad34517fcb28ed9ded6c6" dependencies = [ "cumulus-primitives-core", "frame-support", @@ -1648,7 +1649,7 @@ dependencies = [ [[package]] name = "cumulus-pallet-xcmp-queue" version = "0.1.0" -source = "git+https://github.com/paritytech/cumulus?branch=polkadot-v0.9.17#76479e7fef3af7c8828a44647847b01afd5fefe5" +source = "git+https://github.com/paritytech/cumulus?branch=polkadot-v0.9.17#db11baacc325537be74ad34517fcb28ed9ded6c6" dependencies = [ "cumulus-primitives-core", "frame-support", @@ -1666,7 +1667,7 @@ dependencies = [ [[package]] name = "cumulus-primitives-core" version = "0.1.0" -source = "git+https://github.com/paritytech/cumulus?branch=polkadot-v0.9.17#76479e7fef3af7c8828a44647847b01afd5fefe5" +source = "git+https://github.com/paritytech/cumulus?branch=polkadot-v0.9.17#db11baacc325537be74ad34517fcb28ed9ded6c6" dependencies = [ "frame-support", "parity-scale-codec", @@ -1682,7 +1683,7 @@ dependencies = [ [[package]] name = "cumulus-primitives-parachain-inherent" version = "0.1.0" -source = "git+https://github.com/paritytech/cumulus?branch=polkadot-v0.9.17#76479e7fef3af7c8828a44647847b01afd5fefe5" +source = "git+https://github.com/paritytech/cumulus?branch=polkadot-v0.9.17#db11baacc325537be74ad34517fcb28ed9ded6c6" dependencies = [ "async-trait", "cumulus-primitives-core", @@ -1705,7 +1706,7 @@ dependencies = [ [[package]] name = "cumulus-primitives-timestamp" version = "0.1.0" -source = "git+https://github.com/paritytech/cumulus?branch=polkadot-v0.9.17#76479e7fef3af7c8828a44647847b01afd5fefe5" +source = "git+https://github.com/paritytech/cumulus?branch=polkadot-v0.9.17#db11baacc325537be74ad34517fcb28ed9ded6c6" dependencies = [ "cumulus-primitives-core", "sp-inherents", @@ -1716,7 +1717,7 @@ dependencies = [ [[package]] name = "cumulus-primitives-utility" version = "0.1.0" -source = "git+https://github.com/paritytech/cumulus?branch=polkadot-v0.9.17#76479e7fef3af7c8828a44647847b01afd5fefe5" +source = "git+https://github.com/paritytech/cumulus?branch=polkadot-v0.9.17#db11baacc325537be74ad34517fcb28ed9ded6c6" dependencies = [ "cumulus-primitives-core", "frame-support", @@ -1733,7 +1734,7 @@ dependencies = [ [[package]] name = "cumulus-relay-chain-interface" version = "0.1.0" -source = "git+https://github.com/paritytech/cumulus?branch=polkadot-v0.9.17#76479e7fef3af7c8828a44647847b01afd5fefe5" +source = "git+https://github.com/paritytech/cumulus?branch=polkadot-v0.9.17#db11baacc325537be74ad34517fcb28ed9ded6c6" dependencies = [ "async-trait", "cumulus-primitives-core", @@ -1754,7 +1755,7 @@ dependencies = [ [[package]] name = "cumulus-relay-chain-local" version = "0.1.0" -source = "git+https://github.com/paritytech/cumulus?branch=polkadot-v0.9.17#76479e7fef3af7c8828a44647847b01afd5fefe5" +source = "git+https://github.com/paritytech/cumulus?branch=polkadot-v0.9.17#db11baacc325537be74ad34517fcb28ed9ded6c6" dependencies = [ "async-trait", "cumulus-primitives-core", @@ -1782,7 +1783,7 @@ dependencies = [ [[package]] name = "cumulus-test-relay-sproof-builder" version = "0.1.0" -source = "git+https://github.com/paritytech/cumulus?branch=polkadot-v0.9.17#76479e7fef3af7c8828a44647847b01afd5fefe5" +source = "git+https://github.com/paritytech/cumulus?branch=polkadot-v0.9.17#db11baacc325537be74ad34517fcb28ed9ded6c6" dependencies = [ "cumulus-primitives-core", "parity-scale-codec", @@ -1982,9 +1983,9 @@ dependencies = [ [[package]] name = "dyn-clone" -version = "1.0.4" +version = "1.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ee2626afccd7561a06cf1367e2950c4718ea04565e20fb5029b6c7d8ad09abcf" +checksum = "21e50f3adc76d6a43f5ed73b698a87d0760ca74617f60f7c3b879003536fdd28" [[package]] name = "ed25519" @@ -2706,9 +2707,9 @@ dependencies = [ [[package]] name = "h2" -version = "0.3.11" +version = "0.3.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d9f1f717ddc7b2ba36df7e871fd88db79326551d3d6f1fc406fbfd28b582ff8e" +checksum = "62eeb471aa3e3c9197aa4bfeabfe02982f6dc96f750486c0bb0009ac58b26d2b" dependencies = [ "bytes 1.1.0", "fnv", @@ -2725,9 +2726,9 @@ dependencies = [ [[package]] name = "handlebars" -version = "4.2.1" +version = "4.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "25546a65e5cf1f471f3438796fc634650b31d7fcde01d444c309aeb28b92e3a8" +checksum = "99d6a30320f094710245150395bc763ad23128d6a1ebbad7594dc4164b62c56b" dependencies = [ "log", "pest", @@ -3096,9 +3097,9 @@ dependencies = [ [[package]] name = "ipnet" -version = "2.3.1" +version = "2.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "68f2d64f2edebec4ce84ad108148e67e1064789bee435edc5b60ad398714a3a9" +checksum = "35e70ee094dc02fd9c13fdad4940090f22dbd6ac7c9e7094a46cf0232a50bc7c" [[package]] name = "itertools" @@ -3607,9 +3608,9 @@ checksum = "830d08ce1d1d941e6b30645f1a0eb5643013d835ce3779a5fc208261dbe10f55" [[package]] name = "libc" -version = "0.2.119" +version = "0.2.120" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1bf2e165bb3457c8e098ea76f3e3bc9db55f87aa90d52d0e6be741470916aaa4" +checksum = "ad5c14e80759d0939d013e6ca49930e59fc53dd8e5009132f76240c179380c09" [[package]] name = "libloading" @@ -4172,9 +4173,9 @@ dependencies = [ [[package]] name = "libz-sys" -version = "1.1.3" +version = "1.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "de5435b8549c16d423ed0c03dbaafe57cf6c3344744f1242520d59c9d8ecec66" +checksum = "6f35facd4a5673cb5a48822be2be1d4236c1c99cb4113cab7061ac720d5bf859" dependencies = [ "cc", "pkg-config", @@ -4479,9 +4480,9 @@ dependencies = [ [[package]] name = "lz4" -version = "1.23.2" +version = "1.23.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aac20ed6991e01bf6a2e68cc73df2b389707403662a8ba89f68511fb340f724c" +checksum = "4edcb94251b1c375c459e5abe9fb0168c1c826c3370172684844f8f3f8d1a885" dependencies = [ "libc", "lz4-sys", @@ -4489,9 +4490,9 @@ dependencies = [ [[package]] name = "lz4-sys" -version = "1.9.2" +version = "1.9.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dca79aa95d8b3226213ad454d328369853be3a1382d89532a854f4d69640acae" +checksum = "d7be8908e2ed6f31c02db8a9fa962f03e36c53fbfde437363eae3306b85d7e17" dependencies = [ "cc", "libc", @@ -4627,9 +4628,9 @@ dependencies = [ [[package]] name = "mick-jaeger" -version = "0.1.7" +version = "0.1.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fd2c2cc134e57461f0898b0e921f0a7819b5e3f3a4335b9aa390ce81a5f36fb9" +checksum = "69672161530e8aeca1d1400fbf3f1a1747ff60ea604265a4e906c2442df20532" dependencies = [ "futures 0.3.21", "rand 0.8.5", @@ -4673,14 +4674,15 @@ dependencies = [ [[package]] name = "mio" -version = "0.8.0" +version = "0.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ba272f85fa0b41fc91872be579b3bbe0f56b792aa361a380eb669469f68dafb2" +checksum = "7ba42135c6a5917b9db9cd7b293e5409e1c6b041e6f9825e92e55a894c63b6f8" dependencies = [ "libc", "log", "miow 0.3.7", "ntapi", + "wasi 0.11.0+wasi-snapshot-preview1", "winapi 0.3.9", ] @@ -4879,13 +4881,12 @@ checksum = "2bf50223579dc7cdcfb3bfcacf7069ff68243f8c363f62ffa99cf000a6b9c451" [[package]] name = "nom" -version = "7.1.0" +version = "7.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1b1d11e1ef389c76fe5b81bcaf2ea32cf88b62bc494e19f493d0b30e7a930109" +checksum = "a8903e5a29a317527874d0402f867152a3d21c908bb0b933e416c65e301d4c36" dependencies = [ "memchr", "minimal-lexical", - "version_check", ] [[package]] @@ -4983,9 +4984,9 @@ dependencies = [ [[package]] name = "once_cell" -version = "1.9.0" +version = "1.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "da32515d9f6e6e489d7bc9d84c71b060db7247dc035bbe44eac88cf87486d8d5" +checksum = "87f3e037eac156d1775da914196f0f37741a274155e34a0b7e427c35d2a2ecb9" [[package]] name = "opaque-debug" @@ -5321,7 +5322,7 @@ dependencies = [ [[package]] name = "pallet-collator-selection" version = "3.0.0" -source = "git+https://github.com/paritytech/cumulus?branch=polkadot-v0.9.17#76479e7fef3af7c8828a44647847b01afd5fefe5" +source = "git+https://github.com/paritytech/cumulus?branch=polkadot-v0.9.17#db11baacc325537be74ad34517fcb28ed9ded6c6" dependencies = [ "frame-benchmarking", "frame-support", @@ -6010,7 +6011,7 @@ dependencies = [ [[package]] name = "parachain-info" version = "0.1.0" -source = "git+https://github.com/paritytech/cumulus?branch=polkadot-v0.9.17#76479e7fef3af7c8828a44647847b01afd5fefe5" +source = "git+https://github.com/paritytech/cumulus?branch=polkadot-v0.9.17#db11baacc325537be74ad34517fcb28ed9ded6c6" dependencies = [ "cumulus-primitives-core", "frame-support", @@ -6022,9 +6023,9 @@ dependencies = [ [[package]] name = "parity-db" -version = "0.3.7" +version = "0.3.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "09aa6c5bb8070cf0456d9fc228b3022e900aae9092c48c9c45facf97422efc1d" +checksum = "865edee5b792f537356d9e55cbc138e7f4718dc881a7ea45a18b37bf61c21e3d" dependencies = [ "blake2-rfc", "crc32fast", @@ -7910,9 +7911,9 @@ dependencies = [ [[package]] name = "regex" -version = "1.5.4" +version = "1.5.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d07a8629359eb56f1e2fb1652bb04212c072a87ba68546a04065d525673ac461" +checksum = "1a11647b6b25ff05a515cb92c365cec08801e83423a235b51e231e1808747286" dependencies = [ "aho-corasick", "memchr", @@ -10482,9 +10483,9 @@ checksum = "6bdef32e8150c2a081110b42772ffe7d7c9032b606bc226c8260fd97e0976601" [[package]] name = "syn" -version = "1.0.86" +version = "1.0.88" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8a65b3f4ffa0092e9887669db0eae07941f023991ab58ea44da8fe8e2d511c6b" +checksum = "ebd69e719f31e88618baa1eaa6ee2de5c9a1c004f1e9ecdb58e8352a13f20a01" dependencies = [ "proc-macro2", "quote", @@ -10531,9 +10532,9 @@ dependencies = [ [[package]] name = "termcolor" -version = "1.1.2" +version = "1.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2dfed899f0eb03f32ee8c6a0aabdb8a7949659e3466561fc0adf54e26d88c5f4" +checksum = "bab24d30b911b2376f3a13cc2cd443142f0c81dda04c118693e35b3835757755" dependencies = [ "winapi-util", ] @@ -10658,7 +10659,7 @@ dependencies = [ "bytes 1.1.0", "libc", "memchr", - "mio 0.8.0", + "mio 0.8.1", "num_cpus", "once_cell", "pin-project-lite 0.2.8", @@ -10744,9 +10745,9 @@ checksum = "360dfd1d6d30e05fda32ace2c8c70e9c0a9da713275777f5a4dbb8a1893930c6" [[package]] name = "tracing" -version = "0.1.31" +version = "0.1.32" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f6c650a8ef0cd2dd93736f033d21cbd1224c5a967aa0c258d00fcf7dafef9b9f" +checksum = "4a1bdf54a7c28a2bbf701e1d2233f6c77f473486b94bee4f9678da5a148dca7f" dependencies = [ "cfg-if 1.0.0", "pin-project-lite 0.2.8", @@ -10756,9 +10757,9 @@ dependencies = [ [[package]] name = "tracing-attributes" -version = "0.1.19" +version = "0.1.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8276d9a4a3a558d7b7ad5303ad50b53d58264641b82914b7ada36bd762e7a716" +checksum = "2e65ce065b4b5c53e73bb28912318cb8c9e9ad3921f1d669eb0e68b4c8143a2b" dependencies = [ "proc-macro2", "quote", @@ -10767,9 +10768,9 @@ dependencies = [ [[package]] name = "tracing-core" -version = "0.1.22" +version = "0.1.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "03cfcb51380632a72d3111cb8d3447a8d908e577d31beeac006f836383d29a23" +checksum = "aa31669fa42c09c34d94d8165dd2012e8ff3c66aca50f3bb226b68f216f2706c" dependencies = [ "lazy_static", "valuable", @@ -11144,6 +11145,12 @@ version = "0.10.0+wasi-snapshot-preview1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1a143597ca7c7793eff794def352d41792a93c481eb1042423ff7ff72ba2c31f" +[[package]] +name = "wasi" +version = "0.11.0+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" + [[package]] name = "wasm-bindgen" version = "0.2.79" diff --git a/Makefile b/Makefile index 5771432488..cfecb30026 100644 --- a/Makefile +++ b/Makefile @@ -49,9 +49,13 @@ srtool-build-wasm-litentry: srtool-build-wasm-litmus: @./scripts/build-wasm.sh litmus -.PHONY: build-docker ## Build docker image -build-docker: - @./scripts/build-docker.sh +.PHONY: build-docker-dev ## Build docker image using Dockerfile.dev +build-docker-dev: + @./scripts/build-docker.sh dev + +.PHONY: build-docker-prod ## Build docker image using Dockerfile.prod +build-docker-prod: + @./scripts/build-docker.sh prod .PHONY: build-node-benchmarks ## Build release node with `runtime-benchmarks` feature build-node-benchmarks: diff --git a/README.md b/README.md index e30bda9c72..6b0117215f 100644 --- a/README.md +++ b/README.md @@ -27,8 +27,13 @@ make build-node To build the `litentry/litentry-parachain` docker image locally: ``` -make build-docker +make build-docker-dev +or +make build-docker-prod ``` +depending on which Dockerfile should be used: [Dockerfile.dev](./docker/Dockerfile.dev) or [Dockerfile.prod](./docker/Dockerfile.prod). +The dev version has integrated [`cargo-chef`](https://github.com/LukeMathWalker/cargo-chef) for caching. +As of funtionalities they should be identical. To build the litentry-parachain runtime wasm manually: ``` diff --git a/docker/Dockerfile.dev b/docker/Dockerfile.dev new file mode 100644 index 0000000000..2d5d6ca941 --- /dev/null +++ b/docker/Dockerfile.dev @@ -0,0 +1,64 @@ +# =================================== +# stage 1: install cargo-chef +# https://hub.docker.com/_/rust +# our host is based on bullseye/sid => rust:latest +# =================================== +FROM rust:latest as chef +USER root +RUN cargo install cargo-chef +WORKDIR /app + +# =================================== +# stage 2: generate cargo-chef recipe +# =================================== +FROM chef AS cacher +COPY . . +RUN rustup default nightly && \ + rustup target add wasm32-unknown-unknown --toolchain nightly +RUN cargo chef prepare --recipe-path recipe.json + +# =================================== +# stage 3: build +# =================================== +FROM chef as builder +WORKDIR /litentry +RUN rustup default nightly && \ + rustup target add wasm32-unknown-unknown --toolchain nightly + +RUN apt-get update && \ + apt-get dist-upgrade -y -o Dpkg::Options::="--force-confold" && \ + apt-get install -y cmake pkg-config libssl-dev git clang libclang-dev + +# copy the recipe and toolchain file to make sure we are using consistent `cargo` +COPY --from=cacher /app/recipe.json /litentry/recipe.json +COPY rust-toolchain.toml /litentry/rust-toolchain.toml +RUN cargo chef cook --release --recipe-path recipe.json + +# copy and build the real src +COPY . /litentry +ARG BUILD_ARGS +RUN cargo build --release $BUILD_ARGS + +# =================================== +# stage 4: packaging +# =================================== +FROM ubuntu:20.04 +LABEL maintainer="litentry-dev" + +COPY --from=builder /litentry/target/release/litentry-collator /usr/local/bin + +RUN useradd -m -u 1000 -U -s /bin/sh -d /litentry litentry && \ + mkdir -p /data /litentry/.local/share && \ + chown -R litentry:litentry /data && \ + ln -s /data /litentry/.local/share/litentry-collator && \ +# unclutter and minimize the attack surface + rm -rf /usr/bin /usr/sbin && \ +# check if executable works in this container + /usr/local/bin/litentry-collator --version + +USER litentry +EXPOSE 30333 9933 9944 9615 +VOLUME ["/data"] + +ENTRYPOINT ["/usr/local/bin/litentry-collator"] +CMD ["--help"] diff --git a/docker/Dockerfile b/docker/Dockerfile.prod similarity index 95% rename from docker/Dockerfile rename to docker/Dockerfile.prod index 609dccd8b7..d4e3795374 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile.prod @@ -17,13 +17,12 @@ RUN apt-get update && \ apt-get install -y cmake pkg-config libssl-dev git clang libclang-dev ARG BUILD_ARGS - RUN cargo build --release $BUILD_ARGS +RUN strip target/release/litentry-collator # ========================== # stage 2: packaging # ========================== - FROM ubuntu:20.04 LABEL maintainer="litentry-dev" @@ -43,4 +42,4 @@ EXPOSE 30333 9933 9944 9615 VOLUME ["/data"] ENTRYPOINT ["/usr/local/bin/litentry-collator"] -CMD ["--help"] +CMD ["--help"] \ No newline at end of file diff --git a/node/src/command.rs b/node/src/command.rs index 3736d1e58c..a70f3c2cad 100644 --- a/node/src/command.rs +++ b/node/src/command.rs @@ -95,7 +95,7 @@ fn load_spec(id: &str) -> std::result::Result, St if chain_spec.is_litmus() { Box::new(chain_specs::litmus::ChainSpec::from_json_file(path.into())?) } else { - // By default litentry is used + // Fallback: use Litentry chain spec Box::new(chain_spec) } }, diff --git a/runtime/litentry/src/lib.rs b/runtime/litentry/src/lib.rs index cce98b3346..d853b127fa 100644 --- a/runtime/litentry/src/lib.rs +++ b/runtime/litentry/src/lib.rs @@ -132,14 +132,7 @@ impl_opaque_keys! { /// This runtime version. #[sp_version::runtime_version] pub const VERSION: RuntimeVersion = RuntimeVersion { - // Note: // It's important to match `litentry-parachain-runtime`, which is runtime pkg name - // otherwise no extrinsic can be submitted. - // In logs it's shown: - // Failed to submit extrinsic: Extrinsic verification error: Runtime error: Execution failed: - // Other("Wasm execution trapped: wasm trap: unreachable ... - // - // However our CI passes (TODO) spec_name: create_runtime_str!("litentry-parachain"), impl_name: create_runtime_str!("litentry-parachain"), authoring_version: 1, diff --git a/runtime/litmus/src/lib.rs b/runtime/litmus/src/lib.rs index 463003ed9a..fa0b601247 100644 --- a/runtime/litmus/src/lib.rs +++ b/runtime/litmus/src/lib.rs @@ -132,14 +132,7 @@ impl_opaque_keys! { /// This runtime version. #[sp_version::runtime_version] pub const VERSION: RuntimeVersion = RuntimeVersion { - // Note: // It's important to match `litmus-parachain-runtime`, which is runtime pkg name - // otherwise no extrinsic can be submitted. - // In logs it's shown: - // Failed to submit extrinsic: Extrinsic verification error: Runtime error: Execution failed: - // Other("Wasm execution trapped: wasm trap: unreachable ... - // - // However our CI passes (TODO) spec_name: create_runtime_str!("litmus-parachain"), impl_name: create_runtime_str!("litmus-parachain"), authoring_version: 1, diff --git a/scripts/build-docker.sh b/scripts/build-docker.sh index aad487219e..6eabf153f0 100755 --- a/scripts/build-docker.sh +++ b/scripts/build-docker.sh @@ -1,11 +1,26 @@ #!/usr/bin/env bash set -eo pipefail +function usage() { + echo "Usage: $0 dev|prod [docker-tag] [build-args]" +} + +[ $# -lt 1 ] && (usage; exit 1) + ROOTDIR=$(git rev-parse --show-toplevel) cd "$ROOTDIR" -TAG="$1" -ARGS="$2" +TYPE="$1" +TAG="$2" +ARGS="$3" + +NOCACHE_FLAG= + +case "$TYPE" in + dev) ;; + prod) NOCACHE_FLAG="--no-cache" ;; + *) usage; exit 1 ;; +esac if [ -z "$TAG" ]; then TAG_COMMIT=`git rev-list --tags --max-count=1` @@ -20,6 +35,7 @@ if [ -z "$TAG" ]; then fi fi +echo "TYPE: $TYPE" echo "TAG: $TAG" echo "ARGS: $ARGS" @@ -29,7 +45,7 @@ GITREPO=litentry-parachain # Build the image echo "------------------------------------------------------------" echo "Building ${GITUSER}/${GITREPO}:${TAG} docker image ..." -docker build --rm --no-cache --pull -f ./docker/Dockerfile \ +docker build --rm ${NOCACHE_FLAG} --pull -f ./docker/Dockerfile.${TYPE} \ --build-arg BUILD_ARGS="$ARGS" \ -t ${GITUSER}/${GITREPO}:${TAG} .