python-package.yml

# This workflow will install Python dependencies, run tests and lint with a variety of Python versions
# For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions

name: MSTICPy CI build and check

on:
  push:
    branches: [main]
  pull_request:
    branches: [main, release/*]
  schedule:
    - cron: "0 0 * * 0,2,4"

jobs:
  build:
    runs-on: ubuntu-latest
    permissions: read-all
    strategy:
      matrix:
        python-version: ["3.8", "3.9", "3.10", "3.11"]
    steps:
      # Print out details about the run
      - name: Dump GitHub context
        env:
          GITHUB_CONTEXT: ${{ toJSON(github) }}
        run: echo "$GITHUB_CONTEXT"
      - name: Dump job context
        env:
          JOB_CONTEXT: ${{ toJSON(job) }}
        run: echo "$JOB_CONTEXT"
      # end print details
      - uses: actions/checkout@v4
      - name: Set up Python ${{ matrix.python-version }}
        uses: actions/setup-python@v5
        with:
          python-version: ${{ matrix.python-version }}
      - name: Cache pip
        uses: actions/cache@v4
        with:
          # This path is specific to Ubuntu
          path: ~/.cache/pip
          # Look to see if there is a cache hit for the corresponding requirements file
          key: ${{ runner.os }}-pip-${{ hashFiles('requirements-all.txt') }}
          restore-keys: |
            ${{ runner.os }}-pip-${{ hashFiles('requirements-all.txt') }}
            ${{ runner.os }}-pip
      - name: Install dependencies
        run: |
          python -m pip install --upgrade pip wheel setuptools
          if [ -f requirements-all.txt ]; then
            python -m pip install -r requirements-all.txt
          elif [ -f requirements.txt ]; then
            python -m pip install -r requirements.txt;
          fi
          python -m pip install -e .
      - name: Install test dependencies
        # ToDo - remove pip install xgboost when flaml is fixed
        run: |
          if [ -f requirements-dev.txt ]; then
            python -m pip install -r requirements-dev.txt
          else
            echo "Missing requirements-dev.txt. Installing minimal requirements for testing."
            python -m pip install pytest pytest-cov pytest-xdist pytest-check aiohttp nbconvert jupyter_contrib_nbextensions
            python -m pip install Pygments respx pytest-xdist markdown beautifulsoup4 Pillow async-cache lxml
          fi
          python -m pip install "pandas>=1.3.0" "pygeohash>=1.2.0"
          python -m pip install "xgboost"
      - name: Prepare test dummy data
        run: |
          mkdir ~/.msticpy
          mkdir ~/.msticpy/mordor
          cp ./tests/testdata/geolite/GeoLite2-City.mmdb ~/.msticpy
          touch ~/.msticpy/GeoLite2-City.mmdb
          cp -r ./tests/testdata/mordor/* ~/.msticpy/mordor
          touch ~/.msticpy/mordor/mitre_tact_cache.pkl
          touch ~/.msticpy/mordor/mitre_tech_cache.pkl
          touch ~/.msticpy/mordor/mordor_cache.pkl
      - name: Pytest
        env:
          MAXMIND_AUTH: ${{ secrets.MAXMIND_AUTH }}
          IPSTACK_AUTH: ${{ secrets.IPSTACK_AUTH }}
          MSTICPYCONFIG: ./tests/msticpyconfig-test.yaml
          MSTICPY_BUILD_SOURCE: fork
          JUPYTER_PLATFORM_DIRS: 1
        run: |
          jupyter --paths
          pytest tests -n auto --junitxml=junit/test-${{ matrix.python-version }}-results.xml --cov=msticpy --cov-report=xml
        if: ${{ always() }}
      - name: Upload pytest test results
        uses: actions/upload-artifact@v4
        with:
          name: pytest-results-${{ matrix.python-version }}
          path: junit/test-${{ matrix.python-version }}-results.xml
        # Use always() to always run this step to publish test results when there are test failures
        if: ${{ always() }}
  docs:
    runs-on: ubuntu-latest
    permissions: read-all
    strategy:
      matrix:
        python-version: ["3.11"]
    steps:
      - uses: actions/checkout@v4
      - name: Set up Python ${{ matrix.python-version }}
        uses: actions/setup-python@v5
        with:
          python-version: ${{ matrix.python-version }}
      - name: Cache pip
        uses: actions/cache@v4
        with:
          # This path is specific to Ubuntu
          path: ~/.cache/pip
          # Look to see if there is a cache hit for the corresponding requirements file
          key: ${{ runner.os }}-pip-docs-${{ hashFiles('doc/requirements-all.txt') }}
          restore-keys: |
            ${{ runner.os }}-pip-docs-${{ hashFiles('requirements-all.txt') }}
            ${{ runner.os }}-pip-docs
            ${{ runner.os }}-pip
      - name: Sphinx Read the Docs build
        working-directory: docs
        run: |
          echo "Building docs from ${{ github.workspace }}/docs"
          python -m pip install sphinx readthedocs-sphinx-ext>=2.1.4 sphinx-rtd-theme>=1.0.0
          python -m pip install -r ${{ github.workspace }}/docs/requirements.txt
          make html
        env:
          SPHINX_NOGEN: "true"
  lint:
    runs-on: ubuntu-latest
    permissions: read-all
    strategy:
      matrix:
        python-version: ["3.11"]
    steps:
      - uses: actions/checkout@v4
      - name: Set up Python ${{ matrix.python-version }}
        uses: actions/setup-python@v5
        with:
          python-version: ${{ matrix.python-version }}
      - name: Cache pip
        uses: actions/cache@v4
        with:
          # This path is specific to Ubuntu
          path: ~/.cache/pip
          # Look to see if there is a cache hit for the corresponding requirements file
          key: ${{ runner.os }}-pip-lint-${{ hashFiles('requirements-all.txt') }}
          restore-keys: |
            ${{ runner.os }}-pip-lint-${{ hashFiles('requirements-all.txt') }}
            ${{ runner.os }}-pip-lint
            ${{ runner.os }}-pip
      - name: Install dependencies
        run: |
          python -m pip install --upgrade pip wheel setuptools
          if [ -f requirements-all.txt ]; then
            python -m pip install -r requirements-all.txt
          elif [ -f requirements.txt ]; then
            python -m pip install -r requirements.txt;
          fi
          python -m pip install -e .
      - name: Install test dependencies
        run: |
          if [ -f requirements-dev.txt ]; then
            python -m pip install -r requirements-dev.txt
          else
            echo "Missing requirements-dev.txt. Installing minimal requirements for testing."
            python -m pip install flake8 black bandit mypy pylint types-attrs pydocstyle pyroma
          fi
      - name: black
        run: |
          black --diff --check --exclude venv msticpy
        if: ${{ always() }}
      - name: flake8
        run: |
          # stop the build if there are Python syntax errors or undefined names
          flake8 msticpy --count --select=E9,F63,F7,F82 --show-source --statistics
          # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide
          flake8 --max-line-length=90 --exclude=tests* . --ignore=E501,W503 --jobs=auto
        if: ${{ always() }}
      - name: pylint
        run: |
          pylint msticpy --disable=duplicate-code --disable=E1135,E1101,E1133,W0101
        if: ${{ always() }}
      - name: Cache/restore MyPy data
        id: cache-mypy
        uses: actions/cache@v4
        with:
          # MyPy cache files are stored in `~/.mypy_cache`
          path: .mypy_cache
          key: ${{ runner.os }}-build-mypy-${{ github.ref }}-${{ github.sha }}
          restore-keys: |
            ${{ runner.os }}-build-mypy-${{ github.ref }}-${{ github.sha }}
            ${{ runner.os }}-build-mypy-${{ github.ref }}
            ${{ runner.os }}-build-mypy
      - name: mypy
        run: |
          mypy --ignore-missing-imports --follow-imports=silent --show-column-numbers --show-error-end --show-error-context --disable-error-code annotation-unchecked --junit-xml junit/mypy-test-${{ matrix.python-version }}-results.xml msticpy
        if: ${{ always() }}
      - name: Upload mypy test results
        uses: actions/upload-artifact@v4
        with:
          name: Mypy results ${{ matrix.python-version }}
          path: junit/mypy-test-${{ matrix.python-version }}-results.xml
        # Use always() to always run this step to publish test results when there are test failures
        if: ${{ always() }}
      - name: bandit
        run: |
          bandit -x tests -r -s B303,B404,B603,B607,B608,B113 msticpy
        if: ${{ always() }}
      - name: flake8
        run: |
          flake8 --max-line-length=90 --exclude=tests* . --ignore=E501,W503 --jobs=auto
        if: ${{ always() }}
      - name: pydocstyle
        run: |
          pydocstyle --convention=numpy msticpy
        if: ${{ always() }}
      - name: pyroma
        run: |
          pyroma --min 10 .
        if: ${{ always() }}
  check_status:
    runs-on: ubuntu-latest
    permissions: read-all
    needs: [build, lint, docs]
    steps:
      - name: File build fail issue
        if: ${{ env.GITHUB_REF_NAME == 'main' && ( needs.build.result == 'failure' || needs.lint.result == 'failure' ) }}
        uses: dacbd/create-issue-action@v1
        with:
          token: ${{ github.token }}
          title: "Build failed for main branch"
          body: The build failed on branch ${{ github.ref }}. Please investigate
          labels: build_break, bug, high_severity