test.yml

name: Tests

on:
  push:
  pull_request:

jobs:
  changes:
    name: Check if back-end or front-end files changed
    # Don't run this for pushes generated by push.yml on weblate-localization
    # branch → prevents an infinite loop of actions triggering actions
    #
    # Also, "push" event is not triggered for forks, so need to listen for
    # pull_requests, but only for external ones, so as not to run the action
    # twice
    if: |
      github.event.head_commit.committer.name != 'Hosted Weblate' &&
      github.event.head_commit.committer.name != 'github-actions' &&
      (
        github.event_name != 'pull_request'  ||
        github.event.pull_request.head.repo.fork == true
      )
    runs-on: ubuntu-latest
    outputs:
      backend_changed: ${{ steps.back-end-check.outputs.changed }}
      frontend: ${{ steps.filter.outputs.frontend }}
      frontend_changes: ${{ steps.filter.outputs.frontend_files }}

    steps:
      - uses: actions/checkout@v4

      - name: Find all changed files
        uses: dorny/paths-filter@v3
        id: filter
        with:
          base: ${{ github.ref }}
          list-files: escape
          filters: |
            frontend:
              - 'specifyweb/frontend/**'
            backend:
              - '**'

      - name: Check if any non-front-end files changed
        id: back-end-check
        run: |
          changed_files=`echo "${{steps.filter.outputs.backend_files}}" | tr " " "\n" | grep -v 'specifyweb/frontend/' || echo ""`
          echo "Changed back-end files: ${changed_files}"
          echo "changed=$([[ -z "${changed_files}" ]] && echo "" || echo "1")" >> $GITHUB_OUTPUT

  test-back-end:
    name: Run back-end tests
    needs: changes
    if: needs.changes.outputs.backend_changed
    runs-on: ubuntu-latest

    services:
      mariadb:
        image: mariadb:latest
        ports:
          - 3306
        env:
          MYSQL_USER: MasterUser
          MYSQL_PASSWORD: MasterPassword
          MYSQL_DATABASE: test_SpecifyDB
          MYSQL_ROOT_PASSWORD: password
        options:
          --health-cmd="mariadb-admin ping" --health-interval=5s
          --health-timeout=2s --health-retries=3

    steps:
      - uses: actions/checkout@v4

      - name: Get Specify 6 from cache
        id: cache-specify6
        uses: actions/cache@v3
        with:
          path: Specify6
          key: specify6.8.02-cache

      - name: Install Specify 6
        if: steps.cache-specify6.outputs.cache-hit != 'true'
        run: |
          wget https://update.specifysoftware.org/6802/Specify_unix_64.sh
          sh Specify_unix_64.sh -q -dir ./Specify6

      - name:
          Patch Specify datamodel (Sam, you made the Attachment.origFilename too
          long)
        run:
          sed -i 's/name="origFilename"
          type="java.lang.String"/name="origFilename" type="text"/'
          ./Specify6/config/specify_datamodel.xml

      - name: Install ubuntu dependencies
        run: |
          sudo apt update
          sudo apt install -y libmysqlclient-dev libsasl2-dev libldap2-dev libssl-dev

      - name: Set up Python ${{ matrix.python-version }}
        uses: actions/setup-python@v4
        with:
          python-version: 3.8

      - name: Install python dependencies
        run: |
          python -m pip install --upgrade pip
          python -m venv ve
          ve/bin/pip install -r requirements.txt

      - name: Install testing python dependencies
        run: ve/bin/pip install -r requirements-testing.txt

      - name: Create settings file
        run: |
          echo "THICK_CLIENT_LOCATION = '${{ github.workspace }}/Specify6'" >> specifyweb/settings/local_specify_settings.py
          echo "DATABASE_HOST = '127.0.0.1'" >> specifyweb/settings/local_specify_settings.py
          echo "DATABASE_PORT = ${{ job.services.mariadb.ports[3306] }}" >> specifyweb/settings/local_specify_settings.py

      - name: Need these files to be present
        run:
          make specifyweb/settings/build_version.py
          specifyweb/settings/secret_key.py

      - name: Verify MariaDB connection
        env:
          PORT: ${{ job.services.mariadb.ports[3306] }}
        run: |
          while ! mysqladmin ping -h"127.0.0.1" -P"$PORT" --silent; do
            sleep 1
          done

      - name: Run Mypy type checker
        run: VIRTUAL_ENV=./ve make typecheck

      # - name: Delete existing test database (if any). Use if database is corrupted
      #   env:
      #     PORT: ${{ job.services.mariadb.ports[3306] }}
      #   run: |
      #     mysql -h 127.0.0.1 -P $PORT -u MasterUser -p'MasterPassword' -e 'DROP DATABASE IF EXISTS test_SpecifyDB;';
      #     mysql -h 127.0.0.1 -P $PORT -u MasterUser -p'MasterPassword' -e 'SHOW DATABASES;'
      - name: Run test suite
        run: ./ve/bin/python manage.py test --verbosity=3 --keepdb
        # run: ./ve/bin/python manage.py test --verbosity=3 --noinput

  test-front-end:
    name: Run front-end tests
    needs: changes
    if: ${{ needs.changes.outputs.frontend == 'true' }}
    runs-on: ubuntu-latest

    steps:
      - uses: actions/checkout@v4
        with:
          fetch-depth: 0
          # Using a Personal Access Token from an admin account to bypass branch
          # protection rules and allow direct pushes to production. Inspirited
          # by: https://github.com/community/community/discussions/13836
          token: ${{ secrets.TESTS_PUSH_TO_GITHUB }}

      - uses: actions/setup-node@v4
        with:
          node-version-file: specifyweb/frontend/js_src/package.json
          cache: 'npm'
          cache-dependency-path: specifyweb/frontend/js_src/package-lock.json

      - name: Build frontend
        run: make frontend

        # See https://jestjs.io/docs/troubleshooting#tests-are-extremely-slow-on-docker-andor-continuous-integration-ci-server
      - name: Get number of CPU cores
        id: cpu-cores
        uses: SimenB/github-actions-cpu-cores@v1

      - name: Run JS test suite
        working-directory: specifyweb/frontend/js_src
        run: |
          npm run typecheck && \
          npm run unitTests -- --maxWorkers ${{ steps.cpu-cores.outputs.count }}

      - name: Clone branch that stores localization strings
        # Listen for changes to production branch
        if:
          github.ref == format('refs/heads/{0}',
          github.event.repository.default_branch)
        id: weblate
        uses: actions/checkout@v3
        with:
          path: weblate-localization
          ref: weblate-localization
          fetch-depth: 0

      - name: Localization Tests
        working-directory: specifyweb/frontend/js_src
        run: |
          npm run localizationTests -- --emit ../../../weblate-localization/strings

      - name: Sync localization strings with Weblate (only on production branch)
        if:
          github.ref == format('refs/heads/{0}',
          github.event.repository.default_branch)
        working-directory: weblate-localization
        run: |
          git add strings

          if git diff-index --quiet HEAD --; then
            echo "Localization strings are unchanged"
          else
            git config --local user.name "github-actions"
            git config --local user.email "41898282+github-actions[bot]@users.noreply.github.com"
            # Set the committer to the name and email of the person who
            # triggered the action
            git commit \
              --author '${{ github.event.head_commit.author.name }} <${{ github.event.head_commit.author.email }}>' \
              -m 'Sync localization strings with Weblate' \
              -m "Triggered by ${{ github.sha }} on branch ${{ github.ref }}"
            git push
          fi

      # This step must be run after pushing to github or else it will fail in
      # cases when new components are to be created (Weblate requires files to
      # be pushed before component can be created)
      - name: Make sure Weblate settings are up to date
        # Only run this step if previous step was not skipped
        if: steps.weblate.outcome == 'success'
        working-directory: specifyweb/frontend/js_src
        env:
          WEBLATE_API_TOKEN: ${{ secrets.WEBLATE_API_TOKEN }}
        run: npm run validateWeblate

      - name: Cleanup localization outfiles
        working-directory: specifyweb/frontend/js_src
        run: rm -rf ./localization-strings

      - name: Get relative path of all changed files
        working-directory: specifyweb/frontend/js_src
        id: changed
        run: |
          # Strip specifyweb/frontend/js_src/ from every file name
          changed_files=`sed 's/specifyweb\/frontend\/js_src\///g' <<< "${{ needs.changes.outputs.frontend_changes }}"`

          # Convert to array
          IFS=' ' read -r -a split_files <<< "$changed_files"

          # Exclude files that were removed
          for file in "${split_files[@]}"; do
              if [ ! -e "$file" ]; then
                  echo "File was removed: $file"
                  split_files=("${split_files[@]/$file}")
              fi
          done

          # Convert back to string
          changed="${split_files[*]}"

          echo "Changed front-end files: ${changed}"
          echo "changed=${changed}" >> $GITHUB_OUTPUT

      - name: Run ESLint auto fixes
        # Don't run this for production branch. Reasons:
        #  - ESLint auto fixes may break code. Don't want it to break production
        #    code
        #  - This would have already run on the feature branch by the time code
        #    is pushed into production.
        #  - There shouldn't be many direct pushes to production anyway
        if:
          github.ref != format('refs/heads/{0}',
          github.event.repository.default_branch)
        working-directory: specifyweb/frontend/js_src
        run: |
          # TODO: Once most errors are fixed remove "set +e" to not ignore errors
          set +e
          npx eslint --fix --color `echo "${{steps.changed.outputs.changed}}" | tr " " "\n"`
          set -e

      - name: Reformat all code with Prettier
        if: steps.changed.outputs.changed
        working-directory: specifyweb/frontend/js_src
        run: |
          npx prettier --write --minify `echo "${{steps.changed.outputs.changed}}" | tr " " "\n"` || true

      - name: Commit linted files (if made any changes)
        working-directory: specifyweb/frontend/js_src
        # Creates a new commit. Amending an existing commit is a bad idea
        # because:
        # - Would require original committer to force pull. May cause merge
        #   conflicts
        # - Changes that require linting might have been made over several
        #   commits. If you amend the last commit, than the fixes for all
        #   commits would be in the last commit.
        run: |
          git add .

          if git diff-index --quiet HEAD --; then
            echo "Linters did not detect any issues. Good job!"
          else
            git config --local user.name "github-actions"
            git config --local user.email "41898282+github-actions[bot]@users.noreply.github.com"
            # Set the committer to the name and email of the person who
            # triggered the action
            git commit \
              --author '${{ github.event.head_commit.author.name }} <${{ github.event.head_commit.author.email }}>' \
              -m 'Lint code with ESLint and Prettier' \
              -m "Triggered by ${{ github.sha }} on branch ${{ github.ref }}"
            git push
          fi