diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index cf80e4aa30..be5ced3110 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -12,7 +12,51 @@ env: GHA_CACHE_KEY_VERSION: "v1" jobs: + check_version: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - name: Check Version + run: | + make check-version + + changelog: + runs-on: ubuntu-latest + steps: + # need to checkout otherwise paths-filter will fail on merge-queue trigger + - uses: actions/checkout@v3 + - if: github.ref != 'refs/heads/main' + uses: dorny/paths-filter@v2 + id: changes + with: + filters: | + src: + - 'unstructured/**' + - if: steps.changes.outputs.src == 'true' && github.ref != 'refs/heads/main' + uses: dangoslen/changelog-enforcer@v3 + + skip_conditions: + runs-on: ubuntu-latest + outputs: + no_code_changes: ${{ steps.version_only_change.outputs.should_skip }} + steps: + - id: version_only_change + uses: fkirc/skip-duplicate-actions@v5 + with: + paths_ignore: '["unstructured/__version__.py", "CHANGELOG.md"]' + + shellcheck: + runs-on: ubuntu-latest + needs: skip_conditions + if: needs.skip_conditions.outputs.no_code_changes == 'false' + steps: + - uses: actions/checkout@v3 + - name: ShellCheck + uses: ludeeus/action-shellcheck@master + setup: + needs: skip_conditions + if: needs.skip_conditions.outputs.no_code_changes == 'false' strategy: matrix: python-version: ["3.8","3.9","3.10","3.11"] @@ -39,12 +83,13 @@ jobs: [ ! -d "$NLTK_DATA" ] && mkdir "$NLTK_DATA" make install-ci - check-deps: + check_deps: strategy: matrix: python-version: ["3.8","3.9","3.10","3.11"] runs-on: ubuntu-latest - needs: setup + needs: [setup, skip_conditions] + if: needs.skip_conditions.outputs.no_code_changes == 'false' steps: - uses: actions/checkout@v3 - uses: actions/cache/restore@v3 @@ -69,12 +114,24 @@ jobs: source .venv/bin/activate make check-deps + check_deps_result: # report the final result of the check_deps job, so that it can be properly skipped as needed + if: needs.skip_conditions.outputs.no_code_changes == 'false' && always() + runs-on: ubuntu-latest + needs: + - skip_conditions + - check_deps + steps: + - name: Mark result as failed + if: needs.check_deps.result != 'success' + run: exit 1 + lint: strategy: matrix: python-version: ["3.8","3.9","3.10","3.11"] runs-on: ubuntu-latest - needs: setup + needs: [setup, skip_conditions] + if: needs.skip_conditions.outputs.no_code_changes == 'false' steps: - uses: actions/checkout@v3 - uses: actions/cache/restore@v3 @@ -97,14 +154,19 @@ jobs: - name: Lint run: | source .venv/bin/activate - make check + make check-src + make check-tests - shellcheck: + lint_result: # report the final result of the lint job, so that it can be properly skipped as needed + if: needs.skip_conditions.outputs.no_code_changes == 'false' && always() runs-on: ubuntu-latest + needs: + - skip_conditions + - lint steps: - - uses: actions/checkout@v3 - - name: ShellCheck - uses: ludeeus/action-shellcheck@master + - name: Mark result as failed + if: needs.lint.result != 'success' + run: exit 1 test_unit: strategy: @@ -113,7 +175,8 @@ jobs: runs-on: ubuntu-latest env: NLTK_DATA: ${{ github.workspace }}/nltk_data - needs: [setup, lint] + needs: [setup, skip_conditions, lint] + if: needs.skip_conditions.outputs.no_code_changes == 'false' steps: - uses: actions/checkout@v3 - uses: actions/cache/restore@v3 @@ -150,6 +213,17 @@ jobs: make test CI=true UNSTRUCTURED_INCLUDE_DEBUG_METADATA=true make check-coverage + test_unit_result: # report the final result of the test_unit job, so that it can be properly skipped as needed + if: needs.skip_conditions.outputs.no_code_changes == 'false' && always() + runs-on: ubuntu-latest + needs: + - skip_conditions + - test_unit + steps: + - name: Mark result as failed + if: needs.test_unit.result != 'success' + run: exit 1 + test_unit_no_extras: strategy: matrix: @@ -157,7 +231,8 @@ jobs: runs-on: ubuntu-latest env: NLTK_DATA: ${{ github.workspace }}/nltk_data - needs: [setup, lint] + needs: [setup, skip_conditions, lint] + if: needs.skip_conditions.outputs.no_code_changes == 'false' steps: - uses: actions/checkout@v3 - name: Set up Python ${{ matrix.python-version }} @@ -185,6 +260,17 @@ jobs: source .venv-base/bin/activate make test-no-extras CI=true + test_unit_no_extras_result: # report the final result of the test_unit_no_extras job, so that it can be properly skipped as needed + if: needs.skip_conditions.outputs.no_code_changes == 'false' && always() + runs-on: ubuntu-latest + needs: + - skip_conditions + - test_unit_no_extras + steps: + - name: Mark result as failed + if: needs.test_unit_no_extras.result != 'success' + run: exit 1 + test_unit_dependency_extras: # NOTE(newelh) - Split extras into separate steps in the same pipeline (avoid using matrix) strategy: @@ -194,7 +280,8 @@ jobs: runs-on: ubuntu-latest env: NLTK_DATA: ${{ github.workspace }}/nltk_data - needs: [setup, lint, test_unit_no_extras] + needs: [setup, skip_conditions, lint, test_unit_no_extras] + if: needs.skip_conditions.outputs.no_code_changes == 'false' steps: - uses: actions/checkout@v3 - name: Set up Python ${{ matrix.python-version }} @@ -227,6 +314,17 @@ jobs: tesseract --version make test-extra-${{ matrix.extra }} CI=true + test_unit_dependency_extras_result: # report the final result of the test_unit_dependency_extras job, so that it can be properly skipped as needed + if: needs.skip_conditions.outputs.no_code_changes == 'false' && always() + runs-on: ubuntu-latest + needs: + - skip_conditions + - test_unit_dependency_extras + steps: + - name: Mark result as failed + if: needs.test_unit_dependency_extras.result != 'success' + run: exit 1 + test_ingest: strategy: matrix: @@ -234,7 +332,8 @@ jobs: runs-on: ubuntu-latest-m env: NLTK_DATA: ${{ github.workspace }}/nltk_data - needs: [setup, lint] + needs: [setup, skip_conditions, lint] + if: needs.skip_conditions.outputs.no_code_changes == 'false' steps: - uses: actions/checkout@v3 - uses: actions/cache/restore@v3 @@ -329,6 +428,17 @@ jobs: make install-ingest-delta-table ./test_unstructured_ingest/test-ingest.sh + test_ingest_result: # report the final result of the test_ingest job, so that it can be properly skipped as needed + if: needs.skip_conditions.outputs.no_code_changes == 'false' && always() + runs-on: ubuntu-latest + needs: + - skip_conditions + - test_ingest + steps: + - name: Mark result as failed + if: needs.test_ingest.result != 'success' + run: exit 1 + test_unstructured_api_unit: strategy: matrix: @@ -337,7 +447,8 @@ jobs: runs-on: ubuntu-latest env: NLTK_DATA: ${{ github.workspace }}/nltk_data - needs: [setup, lint] + needs: [setup, skip_conditions, lint] + if: needs.skip_conditions.outputs.no_code_changes == 'false' steps: - uses: actions/checkout@v3 - uses: actions/cache/restore@v3 @@ -378,27 +489,23 @@ jobs: make install-nltk-models make test-unstructured-api-unit - changelog: + test_unstructured_api_unit_result: # report the final result of the test_unstructured_api_unit job, so that it can be properly skipped as needed + if: needs.skip_conditions.outputs.no_code_changes == 'false' && always() runs-on: ubuntu-latest + needs: + - skip_conditions + - test_unstructured_api_unit steps: - # need to checkout otherwise paths-filter will fail on merge-queue trigger - - uses: actions/checkout@v3 - - if: github.ref != 'refs/heads/main' - uses: dorny/paths-filter@v2 - id: changes - with: - filters: | - src: - - 'unstructured/**' - - - if: steps.changes.outputs.src == 'true' && github.ref != 'refs/heads/main' - uses: dangoslen/changelog-enforcer@v3 + - name: Mark result as failed + if: needs.test_unstructured_api_unit.result != 'success' + run: exit 1 # TODO - figure out best practice for caching docker images # (Using the virtualenv to get pytest) test_dockerfile: runs-on: ubuntu-latest - needs: [ setup, lint ] + needs: [setup, skip_conditions, lint] + if: needs.skip_conditions.outputs.no_code_changes == 'false' steps: - uses: actions/checkout@v3 - uses: actions/cache/restore@v3 @@ -422,3 +529,14 @@ jobs: echo "UNS_API_KEY=${{ secrets.UNS_API_KEY }}" > uns_test_env_file make docker-build make docker-test CI=true UNSTRUCTURED_INCLUDE_DEBUG_METADATA=true + + test_dockerfile_result: # report the final result of the test_dockerfile job, so that it can be properly skipped as needed + if: needs.skip_conditions.outputs.no_code_changes == 'false' && always() + runs-on: ubuntu-latest + needs: + - skip_conditions + - test_dockerfile + steps: + - name: Mark result as failed + if: needs.test_dockerfile.result != 'success' + run: exit 1 diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index 917c194ce0..5c56172723 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -23,9 +23,21 @@ on: - cron: '21 21 * * 5' jobs: + skip_conditions: + runs-on: ubuntu-latest + outputs: + no_code_changes: ${{ steps.version_only_change.outputs.should_skip }} + steps: + - id: version_only_change + uses: fkirc/skip-duplicate-actions@v5 + with: + paths_ignore: '["unstructured/__version__.py", "CHANGELOG.md"]' + analyze: name: Analyze runs-on: ubuntu-latest + needs: skip_conditions + if: needs.skip_conditions.outputs.no_code_changes == 'false' permissions: actions: read contents: read