build-test-sources.yaml

# Workflow triggered by pushes to the main branch and PRs targeting it
# (as long as the commits contain code changes).
#
# Jobs:
# - rustfmt: Checks formatting of the Rust sources using the 'rustfmt'
#            tool. The job is skipped if the workflow was triggered by a
#            PR marked as a draft.
# - fourmolu: Checks formatting of the Haskell sources using the
#             'fourmolu' tool. The job is skipped if the workflow was
#             triggered by a PR marked as a draft.
# - build-test: Build and test both the Haskell and Rust sources.
#               The job is skipped if the workflow was triggered by a PR
#               marked as a draft.
#
# The steps in 'build-test' are ordered to fail as fast as possible
# and restore caches as late as possible.
# The dependencies between the steps are described in inline comments below
# along with a few suggestions for improving parallelization.

name: Check format, build and run tests for Haskell and Rust sources

on:
  pull_request:
    branches: main
    types: [opened, synchronize, reopened, ready_for_review]
    paths:
    - '.github/workflows/build-test-sources.yaml'
    - '**.hs'
    - 'fourmolu.yaml'
    - 'stack*.yaml'
    - 'package.yaml'
    - 'base.cabal'
    - '**.rs'
    - 'testdata/**'
  push:
    branches: main
    paths:
    - '.github/workflows/build-test-sources.yaml'
    - '**.hs'
    - 'fourmolu.yaml'
    - 'stack*.yaml'
    - 'package.yaml'
    - 'base.cabal'
    - '**.rs'
    - 'testdata/**'
  workflow_dispatch: # allow manual trigger

env:
  dummy: 1                 # change to force cache invalidation
  CARGO_TERM_COLOR: always # implicitly adds '--color=always' to all cargo commands

jobs:

  rustfmt:
    runs-on: ubuntu-latest
    if: ${{ !github.event.pull_request.draft }}

    strategy:
      matrix:
        plan:
        - rust: "nightly-2023-04-01-x86_64-unknown-linux-gnu"

    steps:
    - name: Checkout
      uses: actions/checkout@v2
      with:
        submodules: recursive

    - name: Install Rust
      uses: actions-rs/toolchain@v1
      with:
        profile: minimal
        toolchain: ${{ matrix.plan.rust }}
        override: true
        components: rustfmt

    - name: Run rustfmt
      run: |
        cargo fmt --manifest-path rust-src/Cargo.toml --all -- --check
        cargo fmt --manifest-path rust-bins/Cargo.toml -- --check
        cargo fmt --manifest-path idiss/Cargo.toml -- --check
        cargo fmt --manifest-path mobile_wallet/Cargo.toml -- --check
        cargo fmt --manifest-path identity-provider-service/Cargo.toml -- --check

  fourmolu:
    runs-on: ubuntu-latest
    if: ${{ !github.event.pull_request.draft }}

    steps:
    - name: Download fourmolu
      uses: supplypike/setup-bin@v1
      with:
        uri: 'https://github.com/fourmolu/fourmolu/releases/download/v0.9.0.0/fourmolu-0.9.0.0-linux-x86_64'
        name: 'fourmolu'
        version: '0.9.0.0'

    - name: Checkout project
      uses: actions/checkout@v2

    - name: Run fourmolu
      run: |
        fourmolu --color always --mode check $(git ls-files '*.hs')

  # This job exists to make sure that concordium-base (the crate) compiles with
  # the version of the compiler stated in its Cargo.toml manifest. We keep this
  # as old as reasonable since we publish it as a library.
  # If this job fails consider carefully whether the new language feature used
  # adds value. If it does bump the rust version here and in Cargo.toml in rust-src/concordium-base
  check-base-compiles:
    needs: [fourmolu, rustfmt]
    runs-on: ubuntu-latest
    if: ${{ !github.event.pull_request.draft }}

    strategy:
      matrix:
        plan:
        - rust: 1.65

    steps:
    - name: Checkout
      uses: actions/checkout@v2
      with:
        submodules: recursive
    # RUST #

    # Set up Rust and restore dependencies and targets from cache.
    # This must be done before checking the Rust sources.
    - name: Install Rust
      uses: actions-rs/toolchain@v1
      with:
        profile: minimal
        toolchain: ${{ matrix.plan.rust }}
        override: true
        components: clippy

    - name: Check that concordium base compiles with older Rust version.
      run: |
        cargo check --manifest-path rust-src/concordium_base/Cargo.toml

  build-test:
    needs: [fourmolu, rustfmt]
    runs-on: ubuntu-latest
    if: ${{ !github.event.pull_request.draft }}

    strategy:
      matrix:
        plan:
        - ghc: 9.2.7 # used as cache key only; stack uses the one specified in stack.yaml
          rust: 1.68

    steps:
    - name: Checkout
      uses: actions/checkout@v2
      with:
        submodules: recursive
    # RUST #

    # Set up Rust and restore dependencies and targets from cache.
    # This must be done before checking the Rust sources.
    - name: Cache cargo dependencies and targets
      uses: actions/cache@v2
      with:
        path: |
          ~/.cargo/registry
          ~/.cargo/git
          rust-src/target
          rust-bins/target
          identity-provider-service/target
          idiss/target
          mobile_wallet/target
          lib
        key: ${{ runner.os }}-{{ env.dummy }}-rust-deps-${{ matrix.plan.rust }}-${{ hashFiles('**/Cargo.toml', '**/Cargo.lock') }}
        restore-keys: |
          ${{ runner.os }}-{{ env.dummy }}-rust-deps-${{ matrix.plan.rust }}
    - name: Install Rust
      uses: actions-rs/toolchain@v1
      with:
        profile: minimal
        toolchain: ${{ matrix.plan.rust }}
        override: true
        components: clippy
        target: aarch64-linux-android
    # Lightweight check that Rust code compiles.
    # Catching failure here avoids restoring of Haskell caches.
    - name: Check that all Rust targets build
      run: |
        cargo check --locked --manifest-path rust-src/Cargo.toml --workspace
        cargo check --locked --manifest-path rust-bins/Cargo.toml --workspace --features=vendored-ssl
        cargo check --locked --manifest-path idiss/Cargo.toml --workspace
        cargo check --locked --manifest-path mobile_wallet/Cargo.toml --workspace
        cargo check --locked --manifest-path mobile_wallet/Cargo.toml --target aarch64-linux-android
        cargo check --locked --manifest-path identity-provider-service/Cargo.toml --workspace --features=vendored-ssl
    # Rust code linting.
    # Could be run in parallel in a separate job
    # (the step is probably too fast for that to make sense though).
    - name: Run cargo clippy on all targets
      run: |
        cargo clippy --manifest-path rust-src/Cargo.toml --workspace -- -Dclippy::all
        cargo clippy --manifest-path rust-src/Cargo.toml --features concordium_base/encryption --workspace -- -Dclippy::all
        cargo clippy --manifest-path rust-src/Cargo.toml --features concordium_base/ffi --workspace -- -Dclippy::all
        cargo clippy --manifest-path rust-bins/Cargo.toml --workspace --features=vendored-ssl -- -Dclippy::all
        cargo clippy --manifest-path idiss/Cargo.toml --all-features -- -Dclippy::all
        cargo clippy --manifest-path mobile_wallet/Cargo.toml -- -Dclippy::all
        cargo clippy --manifest-path identity-provider-service/Cargo.toml --workspace --features=vendored-ssl -- -Dclippy::all

    - name: Check concordium base documentation
      run: |
          RUSTDOCFLAGS="-D warnings" cargo doc --manifest-path rust-src/concordium_base/Cargo.toml --no-deps --all-features

    # HASKELL #

    # Set up Haskell by caching '~/.stack', '.stack-work', and '~/.local/bin' separately.
    # This must be done before compiling the Haskell sources
    # (which in turns compiles the Rust sources).
    # The cache entry keys depend on the GHC version and contents of 'package.yaml' and 'stack.yaml'
    # but will fall back to cache entries from different versions if no match is found.
    - name: Cache stack global package DB
      id: stack-global
      uses: actions/cache@v2
      with:
        path: ~/.stack
        key: ${{ runner.os }}-{{ env.dummy }}-stack-global-${{ matrix.plan.ghc }}-${{ hashFiles('**.yaml') }}
        restore-keys: |
          ${{ runner.os }}-{{ env.dummy }}-stack-global-${{ matrix.plan.ghc }}
    - name: Cache stack-installed programs in '~/.local/bin'
      id: stack-programs
      uses: actions/cache@v2
      with:
        path: ~/.local/bin
        key: ${{ runner.os }}-{{ env.dummy }}-stack-programs-${{ matrix.plan.ghc }}-${{ hashFiles('**.yaml') }}
        restore-keys: |
          ${{ runner.os }}-{{ env.dummy }}-stack-programs-${{ matrix.plan.ghc }}
    - name: Cache '.stack-work'
      uses: actions/cache@v2
      with:
        path: .stack-work
        key: ${{ runner.os }}-{{ env.dummy }}-stack-work-${{ matrix.plan.ghc }}-${{ hashFiles('**.yaml') }}
        restore-keys: |
          ${{ runner.os }}-{{ env.dummy }}-stack-work-${{ matrix.plan.ghc }}

    - name: Install GHC (unless it was cached)
      if: steps.stack-programs.outputs.cache-hit != 'true' || steps.stack-global.outputs.cache-hit != 'true'
      run: |
        stack setup --install-ghc

    - name: Install protoc
      run: |
        wget https://github.com/protocolbuffers/protobuf/releases/download/v3.15.3/protoc-3.15.3-linux-x86_64.zip
        unzip protoc-3.15.3-linux-x86_64.zip
        sudo mv ./bin/protoc /usr/bin/protoc

    - name: Build haskell dependencies (unless they were cached)
      if: steps.stack-programs.outputs.cache-hit != 'true' || steps.stack-global.outputs.cache-hit != 'true'
      run: |
        stack build --test --bench --only-dependencies

    # Compile Rust and Haskell sources.
    # Must be done before running any tests.
    - name: Build all Haskell components
      run: |
        stack build --test --bench --force-dirty --ghc-options=-split-sections --no-run-tests --no-run-benchmarks --ghc-options "-Werror"

    # Run Rust and Haskell tests. Could run in parallel in separate jobs.
    - name: Test haskell
      run: |
        stack build --test --bench --ghc-options=-split-sections --no-run-benchmarks

    # Test rust sources after to avoid rebuilding Rust sources in the haskell
    # test job.
    - name: Test Rust crates
      run: |
        cargo test --manifest-path rust-src/Cargo.toml --all --verbose --release
        cargo test --manifest-path identity-provider-service/Cargo.toml --features=vendored-ssl --all --verbose --release
        cargo bench --manifest-path rust-src/Cargo.toml --features concordium_base/internal-test-helpers --no-run