feat(ci): build each crate individually (#4640)

* feat(ci): build each crate individually * fix(ci): use valid names for each job * feat(ci): builds and checks with and without all features * refactor(ci): build job matrix dinamically * fix: use a JSON_CRATES variable with resulting values * test: check-matrix * fix(ci): use "crate" in singular for reference * imp(ci): use a matrix for feature build arguments * fix(ci): use correct naming and includes * fix(ci): implement most recommendations given in review * fix(ci): use simpler shell script * fix: typo * fix: add string to file, not cmd * fix: some shellchecks * fix(ci): remove warnings and errors from shellcheck * imp(ci): add patch file for `Build crates individually` workflow * Remove unused configs in patch job Co-authored-by: teor <[email protected]>
ZcashFoundation · Jun 21, 2022 · b6a59a9 · b6a59a9
1 parent 769d069
commit b6a59a9
Show file tree

Hide file tree

Showing 3 changed files with 200 additions and 9 deletions.
diff --git a/.github/workflows/build-crates-individually.patch.yml b/.github/workflows/build-crates-individually.patch.yml
@@ -0,0 +1,69 @@
+name: Build crates individually
+
+# We need to keep the `matrix` job in this workflow as-is, as we need the results
+# to actually match the same `build` job names from the original file.
+on:
+  pull_request:
+    paths-ignore:
+      # production code and test code
+      - '**/*.rs'
+      # dependencies
+      - '**/Cargo.toml'
+      - '**/Cargo.lock'
+      # workflow definitions
+      - '.github/workflows/build-crates-individually.yml'
+
+jobs:
+  matrix:
+    name: Crates matrix
+    runs-on: ubuntu-latest
+    outputs:
+      matrix: ${{ steps.set-matrix.outputs.matrix }}
+    steps:
+      - uses: actions/[email protected]
+
+      - uses: actions-rs/toolchain@v1
+        with:
+          toolchain: stable
+          profile: minimal
+          override: true
+
+      - uses: actions-rs/[email protected]
+      # This step is meant to dynamically create a JSON containing the values of each crate
+      # available in this repo in the root directory. We use `cargo tree` to accomplish this task.
+      #
+      # The result from `cargo tree` is then transform to JSON values between double quotes, 
+      # and separated by commas, then added to a `crates.txt` and assigned to a $JSON_CRATES variable.
+      #
+      # A JSON object is created and assigned to a $MATRIX variable, which is use to create an output
+      # named `matrix`, which is then used as the input in following steps, 
+      # using ` ${{ fromJson(needs.matrix.outputs.matrix) }}`
+      - id: set-matrix
+        name: Dynamically build crates JSON
+        run: |
+          TEMP_DIR=$(mktemp -d)
+          echo "$(cargo tree --depth 0 --edges no-normal,no-dev,no-build,no-proc-macro --prefix none | cut -d ' ' -f1 | sed '/^$/d' | awk '{ printf "\"%s\",\n", $0 }' | sed '$ s/.$//')" > $TEMP_DIR/crates.txt
+          MATRIX=$( (
+            echo '{ "crate" : ['
+            echo "$(cat $TEMP_DIR/crates.txt)"
+            echo " ]}"
+          ) | jq -c .)
+          echo $MATRIX
+          echo $MATRIX | jq .
+          echo "::set-output name=matrix::$MATRIX"
+
+  check-matrix:
+    runs-on: ubuntu-latest
+    needs: [ matrix ]
+    steps:
+      - run: 'echo "No job required"'
+
+  build:
+    name: Build ${{ matrix.crate }} crate
+    needs: [ matrix, check-matrix ]
+    runs-on: ubuntu-latest
+    strategy:
+      matrix: ${{ fromJson(needs.matrix.outputs.matrix) }}
+
+    steps:
+      - run: 'echo "No job required"'
diff --git a/.github/workflows/build-crates-individually.yml b/.github/workflows/build-crates-individually.yml
@@ -0,0 +1,125 @@
+name: Build crates individually
+
+on:
+  workflow_dispatch:
+  push:
+    branches:
+      - main
+    paths:
+      # production code and test code
+      - '**/*.rs'
+      # dependencies
+      - '**/Cargo.toml'
+      - '**/Cargo.lock'
+      # workflow definitions
+      - '.github/workflows/build-crates-individually.yml'
+  pull_request:
+    paths:
+      # production code and test code
+      - '**/*.rs'
+      # dependencies
+      - '**/Cargo.toml'
+      - '**/Cargo.lock'
+      # workflow definitions
+      - '.github/workflows/build-crates-individually.yml'
+
+env:
+  CARGO_INCREMENTAL: 0
+  RUST_BACKTRACE: full
+  RUST_LIB_BACKTRACE: full
+  COLORBT_SHOW_HIDDEN: '1'
+
+jobs:
+  matrix:
+    name: Crates matrix
+    runs-on: ubuntu-latest
+    outputs:
+      matrix: ${{ steps.set-matrix.outputs.matrix }}
+    steps:
+      - uses: actions/[email protected]
+
+      - uses: actions-rs/toolchain@v1
+        with:
+          toolchain: stable
+          profile: minimal
+          override: true
+
+      - uses: actions-rs/[email protected]
+      # This step is meant to dynamically create a JSON containing the values of each crate
+      # available in this repo in the root directory. We use `cargo tree` to accomplish this task.
+      #
+      # The result from `cargo tree` is then transform to JSON values between double quotes, 
+      # and separated by commas, then added to a `crates.txt` and assigned to a $JSON_CRATES variable.
+      #
+      # A JSON object is created and assigned to a $MATRIX variable, which is use to create an output
+      # named `matrix`, which is then used as the input in following steps, 
+      # using ` ${{ fromJson(needs.matrix.outputs.matrix) }}`
+      - id: set-matrix
+        name: Dynamically build crates JSON
+        run: |
+          TEMP_DIR=$(mktemp -d)
+          echo "$(cargo tree --depth 0 --edges no-normal,no-dev,no-build,no-proc-macro --prefix none | cut -d ' ' -f1 | sed '/^$/d' | awk '{ printf "\"%s\",\n", $0 }' | sed '$ s/.$//')" > $TEMP_DIR/crates.txt
+          MATRIX=$( (
+            echo '{ "crate" : ['
+            echo "$(cat $TEMP_DIR/crates.txt)"
+            echo " ]}"
+          ) | jq -c .)
+          echo $MATRIX
+          echo $MATRIX | jq .
+          echo "::set-output name=matrix::$MATRIX"
+
+  check-matrix:
+    runs-on: ubuntu-latest
+    needs: [ matrix ]
+    steps:
+      - name: Install json2yaml
+        run: |
+          sudo npm install -g json2yaml
+
+      - name: Check matrix definition
+        run: |
+          matrix='${{ needs.matrix.outputs.matrix }}'
+          echo $matrix
+          echo $matrix | jq .
+          echo $matrix | json2yaml
+
+  build:
+    name: Build ${{ matrix.crate }} crate
+    needs: [ matrix, check-matrix ]
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: true
+      matrix: ${{ fromJson(needs.matrix.outputs.matrix) }}
+
+    steps:
+      - uses: actions/[email protected]
+        with:
+          persist-credentials: false
+
+      - uses: actions-rs/toolchain@v1
+        with:
+          toolchain: stable
+          profile: minimal
+          override: true
+
+        # We could use `features: ['', '--all-features', '--no-default-features']` as a matrix argument,
+        # but it's faster to run these commands sequentially, so they can re-use the local cargo cache.
+        # 
+        # Some Zebra crates do not have any features, and most don't have any default features.
+      - name: Build ${{ matrix.crate }} crate with no default features
+        uses: actions-rs/[email protected]
+        with:
+          command: build
+          args: --package ${{ matrix.crate }} --no-default-features
+
+      - name: Build ${{ matrix.crate }} crate normally
+        uses: actions-rs/[email protected]
+        with:
+          command: build
+          args: --package ${{ matrix.crate }}
+
+      - name: Build ${{ matrix.crate }} crate with all features
+        uses: actions-rs/[email protected]
+        with:
+          command: build
+          args: --package ${{ matrix.crate }} --all-features
diff --git a/.github/workflows/continous-integration-os.yml b/.github/workflows/continous-integration-os.yml
@@ -267,13 +267,14 @@ jobs:
           args: --locked --all-features --all-targets
 
   cargo-deny:
-    name: Check deny.toml ${{ matrix.checks }}
+    name: Check deny.toml ${{ matrix.checks }} ${{ matrix.features }}
     runs-on: ubuntu-latest
     strategy:
       matrix:
         checks:
           - bans
           - sources
+        features: ['', '--all-features', '--no-default-features']
 
     # Prevent sudden announcement of a new advisory from failing ci:
     continue-on-error: ${{ matrix.checks == 'advisories' }}
@@ -283,14 +284,10 @@ jobs:
         with:
           persist-credentials: false
 
-      - uses: EmbarkStudios/cargo-deny-action@v1
-        with:
-          command: check ${{ matrix.checks }}
-          args: --all-features --workspace
-
-      # this check runs with optional features off
+      # this check also runs with optional features off
       # so we expect some warnings about "skip tree root was not found"
-      - uses: EmbarkStudios/cargo-deny-action@v1
+      - name: Check ${{ matrix.checks }} with features ${{ matrix.features }}
+        uses: EmbarkStudios/cargo-deny-action@v1
         with:
           command: check ${{ matrix.checks }}
-          args: --workspace
+          args: --workspace with features ${{ matrix.features }}