ci.yml

name: CI
on:
  push

jobs:
  lint:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v3

      - uses: actions/setup-python@v4
        with:
          python-version: '3.10'

      - name: Install flake8
        run: pip install flake8

      - name: Run flake8
        run: flake8 dags/

  integrity_test:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v3

      - uses: actions/setup-python@v4
        with:
          python-version: '3.10'

      - name: Install integrity test requirements
        run: pip install -r integrity_tests/requirements.txt

      - name: Initialize Airflow DB
        run: airflow db init

      - name: Run integrity tests
        run: coverage run -m pytest integrity_tests/*

#  mock_pipeline_test:
#    runs-on: ubuntu-latest
#    needs:
#      - lint
#      - integrity_test
#    steps:
#      - uses: actions/checkout@v3
#
#      - uses: actions/setup-python@v4
#        with:
#          python-version: '3.10'
#
#      - name: Install system dependencies
#        run: sudo apt-get install -y libsasl2-dev build-essential
#
#      - name: Install mock_pipeline test requirements
#        run: pip install -r mock_pipeline_requirements.txt
#
#      - name: Generate mock-pipeline data
#        run: spark-submit --name spark-data-generate dags/spark/generate_data.py --warehouse-path ./spark-warehouse
#
#      - name: Run dbt
#        working-directory: dags/dbt
#        run : dbt run --target mock_pipeline
#
#      - name: Run dbt tests
#        working-directory: dags/dbt
#        run : dbt test --target mock_pipeline