upload: be more cautious about grabbing files for upload (#305) #913
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: CI | |
| on: | |
| pull_request: | |
| paths-ignore: | |
| - 'docs/**' | |
| push: | |
| branches: | |
| - main | |
| paths-ignore: | |
| - 'docs/**' | |
| # The goal here is to cancel older workflows when a PR is updated (because it's pointless work) | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.head_ref || github.ref_name }} | |
| cancel-in-progress: true | |
| jobs: | |
| unittest: | |
| name: unit tests | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Set up Python | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: '3.11' | |
| cache: pip | |
| - name: Install dependencies | |
| run: | | |
| python -m pip install --upgrade pip | |
| pip install ".[test]" | |
| - name: Create mock AWS credentials | |
| run: | | |
| mkdir ~/.aws && touch ~/.aws/credentials | |
| echo -e "[test]\naws_access_key_id = test\naws_secret_access_key = test" > ~/.aws/credentials | |
| - name: Test with pytest | |
| run: | | |
| python -m pytest --cov-report xml --cov=cumulus_library tests | |
| - name: Log missing coverage | |
| run: | | |
| coverage report -m --skip-covered | |
| - name: Generate coverage report | |
| if: github.ref != 'refs/heads/main' | |
| uses: orgoro/[email protected] | |
| with: | |
| coverageFile: coverage.xml | |
| token: ${{ secrets.GITHUB_TOKEN }} | |
| thresholdAll: .9 | |
| thresholdNew: 1 | |
| thresholdModified: 1 | |
| lint: | |
| runs-on: ubuntu-22.04 | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Set up Python | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: '3.11' | |
| - name: Install linters | |
| run: | | |
| python -m pip install --upgrade pip | |
| pip install ".[dev]" | |
| - name: Run sqlfluff on jinja templates | |
| run: | | |
| sqlfluff lint | |
| - name: Run ruff | |
| if: success() || failure() # still run ruff if above checks fails | |
| run: | | |
| ruff check | |
| ruff format --check | |
| regression: | |
| runs-on: ubuntu-latest | |
| permissions: | |
| id-token: write # This is required for requesting the JWT | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Set up Python | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: '3.11' | |
| - name: Install dependencies | |
| run: | | |
| python -m pip install --upgrade pip | |
| pip install . | |
| # See https://github.com/aws-actions/configure-aws-credentials for configuring | |
| # the aws side of this - the below action is just a light wrapper | |
| - name: Configure AWS Credentials | |
| uses: mcblair/[email protected] | |
| with: | |
| role-arn: arn:aws:iam::${{secrets.AWS_ACCOUNT}}:role/cumulus-library-ci | |
| region: us-east-1 | |
| profile-name: cumulus-library-ci | |
| - name: Rebuild tables | |
| env: | |
| PROFILE: cumulus-library-ci | |
| WG: cumulus | |
| DB: cumulus_library_regression_db | |
| run: | | |
| cumulus-library build -t core --profile $PROFILE --workgroup $WG --database $DB | |
| cumulus-library export -t core ./tests/regression/data_export/ --profile $PROFILE --workgroup $WG --database $DB | |
| - name: Compare vs known data | |
| run: python ./tests/regression/run_regression.py | |