Add srtool GHA (#2755)

This PR introduces the `srtool` GHA which was used in the old `cumulus`
repo to build and check runtimes on the weekly basis schedule. The job
is triggered:
- every Monday at 2AM, automatically
- on each tag push or push to the release branch
- can be triggered manually as well

Addresses #1271

.github/scripts/common/lib.sh

@@ -307,3 +307,40 @@ function increment_rc_tag() {
   ((suffix++))
   echo $suffix
 }
+
+function relative_parent() {
+    echo "$1" | sed -E 's/(.*)\/(.*)\/\.\./\1/g'
+}
+
+# Find all the runtimes, it returns the result as JSON object, compatible to be
+# used as Github Workflow Matrix. This call is exposed by the `scan` command and can be used as:
+# podman run --rm -it -v /.../fellowship-runtimes:/build docker.io/chevdor/srtool:1.70.0-0.11.1 scan
+function find_runtimes() {
+    libs=($(git grep -I -r --cached --max-depth 20 --files-with-matches 'construct_runtime!' -- '*lib.rs'))
+    re=".*-runtime$"
+    JSON=$(jq --null-input '{ "include": [] }')
+
+    # EXCLUDED_RUNTIMES is a space separated list of runtime names (without the -runtime postfix)
+    # EXCLUDED_RUNTIMES=${EXCLUDED_RUNTIMES:-"substrate-test"}
+    IFS=' ' read -r -a exclusions <<< "$EXCLUDED_RUNTIMES"
+
+    for lib in "${libs[@]}"; do
+        crate_dir=$(dirname "$lib")
+        cargo_toml="$crate_dir/../Cargo.toml"
+
+        name=$(toml get -r $cargo_toml 'package.name')
+        chain=${name//-runtime/}
+
+        if [[ "$name" =~ $re ]] && ! [[ ${exclusions[@]} =~ $chain ]]; then
+            lib_dir=$(dirname "$lib")
+            runtime_dir=$(relative_parent "$lib_dir/..")
+            ITEM=$(jq --null-input \
+                --arg chain "$chain" \
+                --arg name "$name" \
+                --arg runtime_dir "$runtime_dir" \
+                '{ "chain": $chain, "crate": $name, "runtime_dir": $runtime_dir }')
+            JSON=$(echo $JSON | jq ".include += [$ITEM]")
+        fi
+    done
+    echo $JSON
+}

.github/workflows/srtool.yml

@@ -0,0 +1,135 @@
+name: Srtool build
+
+env:
+  SUBWASM_VERSION: 0.20.0
+  TOML_CLI_VERSION: 0.2.4
+
+on:
+  push:
+    tags:
+      - "*"
+    branches:
+      - release-v[0-9]+.[0-9]+.[0-9]+*
+      - release-cumulus-v[0-9]+*
+      - release-polkadot-v[0-9]+*
+
+  schedule:
+    - cron: "00 02 * * 1" # 2AM weekly on monday
+
+  workflow_dispatch:
+
+jobs:
+  find-runtimes:
+      name: Scan repo paritytech/polkadot-sdk
+      outputs:
+        runtime: ${{ steps.get_runtimes_list.outputs.runtime }}
+      runs-on: ubuntu-latest
+      steps:
+        - name: Checkout
+          uses: actions/checkout@3df4ab11eba7bda6032a0b82a6bb43b11571feac # v4.0.0
+          with:
+            fetch-depth: 0
+
+        - name: Install tooling
+          run: |
+            URL=https://github.com/chevdor/toml-cli/releases/download/v${{ env.TOML_CLI_VERSION }}/toml_linux_amd64_v${{ env.TOML_CLI_VERSION }}.deb
+            curl -L $URL --output toml.deb
+            sudo dpkg -i toml.deb
+            toml --version; jq --version
+
+        - name: Scan runtimes
+          env:
+            EXCLUDED_RUNTIMES: "substrate-test"
+          run: |
+            . ./.github/scripts/common/lib.sh
+
+            echo "Github workspace: ${{ github.workspace }}"
+            echo "Current folder: $(pwd)"; ls -al
+            ls -al
+
+            MATRIX=$(find_runtimes | tee runtimes_list.json)
+            echo $MATRIX
+
+        - name: Get runtimes list
+          id: get_runtimes_list
+          run: |
+            ls -al
+            MATRIX=$(cat runtimes_list.json)
+            echo $MATRIX
+            echo "runtime=$MATRIX" >> $GITHUB_OUTPUT
+
+  srtool:
+    runs-on: ubuntu-latest
+    needs:
+      - find-runtimes
+    strategy:
+      fail-fast: false
+      matrix: ${{ fromJSON(needs.find-runtimes.outputs.runtime) }}
+
+    steps:
+      - uses: actions/checkout@3df4ab11eba7bda6032a0b82a6bb43b11571feac # v4.0.0
+        with:
+          fetch-depth: 0
+
+      - name: Srtool build
+        id: srtool_build
+        uses: chevdor/[email protected]
+        with:
+          chain: ${{ matrix.chain }}
+          runtime_dir: ${{ matrix.runtime_dir }}
+
+      - name: Summary
+        run: |
+          echo '${{ steps.srtool_build.outputs.json }}' | jq > ${{ matrix.chain }}-srtool-digest.json
+          cat ${{ matrix.chain }}-srtool-digest.json
+          echo "Compact Runtime: ${{ steps.srtool_build.outputs.wasm }}"
+          echo "Compressed Runtime: ${{ steps.srtool_build.outputs.wasm_compressed }}"
+
+      # it takes a while to build the runtime, so let's save the artifact as soon as we have it
+      - name: Archive Artifacts for ${{ matrix.chain }}
+        uses: actions/upload-artifact@0b7f8abb1508181956e8e162db84b466c27e18ce # v3.1.2
+        with:
+          name: ${{ matrix.chain }}-runtime
+          path: |
+            ${{ steps.srtool_build.outputs.wasm }}
+            ${{ steps.srtool_build.outputs.wasm_compressed }}
+            ${{ matrix.chain }}-srtool-digest.json
+
+      # We now get extra information thanks to subwasm
+      - name: Install subwasm
+        run: |
+          wget https://github.com/chevdor/subwasm/releases/download/v${{ env.SUBWASM_VERSION }}/subwasm_linux_amd64_v${{ env.SUBWASM_VERSION }}.deb
+          sudo dpkg -i subwasm_linux_amd64_v${{ env.SUBWASM_VERSION }}.deb
+          subwasm --version
+
+      - name: Show Runtime information
+        shell: bash
+        run: |
+          subwasm info ${{ steps.srtool_build.outputs.wasm }}
+          subwasm info ${{ steps.srtool_build.outputs.wasm_compressed }}
+          subwasm --json info ${{ steps.srtool_build.outputs.wasm }} > ${{ matrix.chain }}-info.json
+          subwasm --json info ${{ steps.srtool_build.outputs.wasm_compressed }} > ${{ matrix.chain }}-compressed-info.json
+
+      - name: Extract the metadata
+        shell: bash
+        run: |
+          subwasm meta ${{ steps.srtool_build.outputs.wasm }}
+          subwasm --json meta ${{ steps.srtool_build.outputs.wasm }} > ${{ matrix.chain }}-metadata.json
+
+      - name: Check the metadata diff
+        shell: bash
+        # the following subwasm call will error for chains that are not known and/or live, that includes shell for instance
+        run: |
+          subwasm diff ${{ steps.srtool_build.outputs.wasm }} --chain-b ${{ matrix.chain }} || \
+            echo "Subwasm call failed, check the logs. This is likely because ${{ matrix.chain }} is not known by subwasm" | \
+            tee ${{ matrix.chain }}-diff.txt
+
+      - name: Archive Subwasm results
+        uses: actions/upload-artifact@0b7f8abb1508181956e8e162db84b466c27e18ce # v3.1.2
+        with:
+          name: ${{ matrix.chain }}-runtime
+          path: |
+            ${{ matrix.chain }}-info.json
+            ${{ matrix.chain }}-compressed-info.json
+            ${{ matrix.chain }}-metadata.json
+            ${{ matrix.chain }}-diff.txt