Merge remote-tracking branch 'origin/main' into feat/weighted-average…

…-table-metrics
Unstructured-IO · Nov 20, 2024 · a4c549e · a4c549e
2 parents e5415a2 + 85ecdab
commit a4c549e
Show file tree

Hide file tree

Showing 1,072 changed files with 56,230 additions and 84,054 deletions.
diff --git a/.github/actions/base-cache/action.yml b/.github/actions/base-cache/action.yml
@@ -22,22 +22,25 @@ runs:
         lookup-only: ${{ inputs.check-only }}
     - name: Set up Python ${{ inputs.python-version }}
       if: steps.virtualenv-cache-restore.outputs.cache-hit != 'true'
-      uses: actions/setup-python@v4
+      uses: actions/setup-python@v5
       with:
         python-version: ${{ inputs.python-version }}
     - name: Setup virtual environment (no cache hit)
       if: steps.virtualenv-cache-restore.outputs.cache-hit != 'true'
       shell: bash
       run: |
         python${{ inputs.python-version }} -m pip install --upgrade virtualenv
-        python${{ inputs.python-version }} -m venv .venv
+        if [ ! -d ".venv" ]; then
+          python${{ inputs.python-version }} -m venv .venv
+        fi
         source .venv/bin/activate
         [ ! -d "$NLTK_DATA" ] && mkdir "$NLTK_DATA"
         if [ "${{ inputs.python-version == '3.12' }}" == "true" ]; then
           python -m ensurepip --upgrade
           python -m pip install --upgrade setuptools
         fi
         make install-ci
+        make install-nltk-models
     - name: Save Cache
       if: steps.virtualenv-cache-restore.outputs.cache-hit != 'true'
       id: virtualenv-cache-save

diff --git a/.github/actions/base-ingest-cache/action.yml b/.github/actions/base-ingest-cache/action.yml
@@ -18,25 +18,30 @@ runs:
         path: |
           .venv
           nltk_data
-        key: unstructured-ingest-${{ runner.os }}-${{ inputs.python-version }}-${{ hashFiles('requirements/ingest/*.txt') }}
+        key: unstructured-ingest-${{ runner.os }}-${{ inputs.python-version }}-${{ hashFiles('requirements/ingest/*.txt', 'requirements/*.txt') }}
         lookup-only: ${{ inputs.check-only }}
-    - name: Restore base virtual environment
-      if: steps.ingest-virtualenv-cache-restore.outputs.cache-hit != 'true'
-      uses: ./.github/actions/base-cache
-      with:
-        python-version: ${{ inputs.python-version }}
     - name: Set up Python ${{ inputs.python-version }}
       if: steps.ingest-virtualenv-cache-restore.outputs.cache-hit != 'true'
-      uses: actions/setup-python@v4
+      uses: actions/setup-python@v5
       with:
         python-version: ${{ inputs.python-version }}
     - name: Setup virtual environment (no cache hit)
       if: steps.ingest-virtualenv-cache-restore.outputs.cache-hit != 'true'
       shell: bash
       run: |
-        python${{ inputs.python-version }} -m venv .venv
+        python${{ inputs.python-version }} -m pip install --upgrade virtualenv
+        if [ ! -d ".venv" ]; then
+          python${{ inputs.python-version }} -m venv .venv
+        fi
         source .venv/bin/activate
-        make install-all-ingest
+        if [ "${{ inputs.python-version == '3.12' }}" == "true" ]; then
+          python -m ensurepip --upgrade
+          python -m pip install --upgrade setuptools
+        fi
+        make install-ci
+        make install-nltk-models
+        make install-all-docs
+        make install-ingest
     - name: Save Ingest Cache
       if: steps.ingest-virtualenv-cache-restore.outputs.cache-hit != 'true'
       id: ingest-virtualenv-cache-save
@@ -45,5 +50,5 @@ runs:
         path: |
           .venv
           nltk_data
-        key: unstructured-ingest-${{ runner.os }}-${{ inputs.python-version }}-${{ hashFiles('requirements/ingest/*.txt') }}
+        key: unstructured-ingest-${{ runner.os }}-${{ inputs.python-version }}-${{ hashFiles('requirements/ingest/*.txt', 'requirements/*.txt') }}
 
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -12,14 +12,15 @@ permissions:
   id-token: write
   contents: read
 
+env:
+  NLTK_DATA: ${{ github.workspace }}/nltk_data
+
 jobs:
   setup:
     strategy:
       matrix:
         python-version: ["3.9","3.10","3.11", "3.12"]
     runs-on: ubuntu-latest
-    env:
-      NLTK_DATA: ${{ github.workspace }}/nltk_data
     steps:
     - uses: actions/checkout@v4
     - uses: ./.github/actions/base-cache
@@ -55,12 +56,29 @@ jobs:
       - name: Install all extras
         run: make check-extras
 
+  check-licenses:
+    strategy:
+      matrix:
+        python-version: [ "3.12" ]
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v5
+        with:
+          python-version: ${{ matrix.python-version }}
+      # NOTE(robinson) - dependencies are installed first because liccheck
+      # produces an error if there is a a mismatch between the dep version
+      # in the requirements file and the dep version in site packages
+      - name: Install all doc and test dependencies
+        run: |
+          make install-ci
+          make check-licenses
+
   lint:
     strategy:
       matrix:
         python-version: ["3.9","3.10","3.11"]
-    env:
-      NLTK_DATA: ${{ github.workspace }}/nltk_data
     runs-on: ubuntu-latest
     needs: [setup, changelog]
     steps:
@@ -76,6 +94,7 @@ jobs:
     - name: Lint
       run: |
         source .venv/bin/activate
+        make install-ci
         make check
 
   shellcheck:
@@ -100,8 +119,6 @@ jobs:
       matrix:
         python-version: ["3.9","3.10","3.11", "3.12"]
     runs-on: ubuntu-latest
-    env:
-      NLTK_DATA: ${{ github.workspace }}/nltk_data
     needs: [setup, lint]
     steps:
     - uses: actions/checkout@v4
@@ -135,46 +152,11 @@ jobs:
         make test CI=true UNSTRUCTURED_INCLUDE_DEBUG_METADATA=true
         make check-coverage
 
-  test_chipper:
-    strategy:
-      matrix:
-        python-version: ["3.10"]
-    runs-on: ubuntu-latest
-    env:
-      NLTK_DATA: ${{ github.workspace }}/nltk_data
-      UNSTRUCTURED_HF_TOKEN: ${{ secrets.HF_TOKEN }}
-    needs: [setup, lint]
-    steps:
-    - uses: actions/checkout@v4
-    - name: Set up Python ${{ matrix.python-version }}
-      uses: actions/setup-python@v5
-      with:
-        python-version: ${{ matrix.python-version }}
-    - name: Setup virtual environment
-      uses: ./.github/actions/base-cache
-      with:
-        python-version: ${{ matrix.python-version }}
-    - name: Test
-      env:
-        UNS_API_KEY: ${{ secrets.UNS_API_KEY }}
-      run: |
-        source .venv/bin/activate
-        sudo apt-get update
-        sudo apt-get install -y poppler-utils
-        make install-pandoc install-test
-        sudo add-apt-repository -y ppa:alex-p/tesseract-ocr5
-        sudo apt-get update
-        sudo apt-get install -y tesseract-ocr tesseract-ocr-kor
-        tesseract --version
-        make test-chipper CI=true UNSTRUCTURED_INCLUDE_DEBUG_METADATA=true
-
   test_unit_no_extras:
     strategy:
       matrix:
         python-version: ["3.10"]
     runs-on: ubuntu-latest
-    env:
-      NLTK_DATA: ${{ github.workspace }}/nltk_data
     needs: [setup, lint]
     steps:
     - uses: actions/checkout@v4
@@ -191,17 +173,17 @@ jobs:
         UNS_API_KEY: ${{ secrets.UNS_API_KEY }}
       run: |
         source .venv/bin/activate
+        make install-ci
+        make install-nltk-models
         make test-no-extras CI=true
 
   test_unit_dependency_extras:
     # NOTE(newelh) - Split extras into separate steps in the same pipeline (avoid using matrix)
     strategy:
       matrix:
         python-version: ["3.10"]
-        extra: ["csv", "docx", "odt", "markdown", "pypandoc", "msg", "pdf-image", "pptx", "xlsx"]
+        extra: ["csv", "docx", "odt", "markdown", "pypandoc", "pdf-image", "pptx", "xlsx"]
     runs-on: ubuntu-latest
-    env:
-      NLTK_DATA: ${{ github.workspace }}/nltk_data
     needs: [setup, lint, test_unit_no_extras]
     steps:
     - uses: actions/checkout@v4
@@ -235,57 +217,27 @@ jobs:
         sudo apt-get update
         sudo apt-get install -y tesseract-ocr tesseract-ocr-kor
         tesseract --version
+        make install-${{ matrix.extra }}
         make test-extra-${{ matrix.extra }} CI=true
 
   setup_ingest:
     strategy:
       matrix:
         python-version: [ "3.9","3.10" ]
     runs-on: ubuntu-latest
-    env:
-      NLTK_DATA: ${{ github.workspace }}/nltk_data
     needs: [setup]
     steps:
       - uses: actions/checkout@v4
       - uses: ./.github/actions/base-ingest-cache
         with:
           python-version: ${{ matrix.python-version }}
-
-  test_ingest_unit:
-    strategy:
-      matrix:
-        python-version: [ "3.9","3.10" ]
-    runs-on: ubuntu-latest
-    env:
-      NLTK_DATA: ${{ github.workspace }}/nltk_data
-    needs: [ setup_ingest, lint ]
-    steps:
-      # actions/checkout MUST come before auth
-      - uses: 'actions/checkout@v4'
-      - name: Set up Python ${{ matrix.python-version }}
-        uses: actions/setup-python@v5
-        with:
-          python-version: ${{ matrix.python-version }}
-      - name: Get full Python version
-        id: full-python-version
-        run: echo version=$(python -c "import sys; print('-'.join(str(v) for v in sys.version_info))") >> $GITHUB_OUTPUT
-      - name: Setup virtual environment
-        uses: ./.github/actions/base-ingest-cache
-        with:
-          python-version: ${{ matrix.python-version }}
-      - name: Test Ingest (unit)
-        run: |
-          source .venv/bin/activate
-          PYTHONPATH=. pytest test_unstructured_ingest/unit
-
+          check-only: 'true'
 
   test_ingest_src:
     strategy:
       matrix:
         python-version: ["3.9","3.10"]
     runs-on: ubuntu-latest-m
-    env:
-      NLTK_DATA: ${{ github.workspace }}/nltk_data
     needs: [setup_ingest, lint]
     steps:
     # actions/checkout MUST come before auth
@@ -348,9 +300,10 @@ jobs:
         PINECONE_API_KEY: ${{secrets.PINECONE_API_KEY}}
         ASTRA_DB_APPLICATION_TOKEN: ${{secrets.ASTRA_DB_TOKEN}}
         ASTRA_DB_API_ENDPOINT: ${{secrets.ASTRA_DB_ENDPOINT}}
-        TABLE_OCR: "tesseract"
+        MXBAI_API_KEY: ${{secrets.MXBAI_API_KEY}}
         OCR_AGENT: "unstructured.partition.utils.ocr_models.tesseract_ocr.OCRAgentTesseract"
         CI: "true"
+        PYTHON: python${{ matrix.python-version }}
       run: |
         source .venv/bin/activate
         sudo apt-get update
@@ -362,111 +315,17 @@ jobs:
         sudo apt-get install -y tesseract-ocr-kor
         sudo apt-get install diffstat
         tesseract --version
+        make install-all-docs
+        make install-ingest
         ./test_unstructured_ingest/test-ingest-src.sh
 
 
-  test_ingest_dest:
-    environment: ci
-    strategy:
-      matrix:
-        python-version: ["3.9","3.10"]
-    runs-on: ubuntu-latest-m
-    env:
-      NLTK_DATA: ${{ github.workspace }}/nltk_data
-    needs: [setup_ingest, lint]
-    steps:
-    # actions/checkout MUST come before auth
-    - uses: 'actions/checkout@v4'
-    - name: Set up Python ${{ matrix.python-version }}
-      uses: actions/setup-python@v5
-      with:
-        python-version: ${{ matrix.python-version }}
-    - name: Get full Python version
-      id: full-python-version
-      run: echo version=$(python -c "import sys; print('-'.join(str(v) for v in sys.version_info))") >> $GITHUB_OUTPUT
-    - name: Setup virtual environment
-      uses: ./.github/actions/base-ingest-cache
-      with:
-        python-version: ${{ matrix.python-version }}
-    - name: Setup docker-compose
-      uses: KengoTODA/actions-setup-docker-compose@v1
-      with:
-        version: '2.22.0'
-    - name: Test (end-to-end)
-      env:
-        AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
-        AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
-        S3_INGEST_TEST_ACCESS_KEY: ${{ secrets.S3_INGEST_TEST_ACCESS_KEY }}
-        S3_INGEST_TEST_SECRET_KEY: ${{ secrets.S3_INGEST_TEST_SECRET_KEY }}
-        AZURE_SEARCH_ENDPOINT: ${{ secrets.AZURE_SEARCH_ENDPOINT }}
-        AZURE_SEARCH_API_KEY: ${{ secrets.AZURE_SEARCH_API_KEY }}
-        BOX_APP_CONFIG: ${{ secrets.BOX_APP_CONFIG }}
-        DROPBOX_APP_KEY: ${{ secrets.DROPBOX_APP_KEY }}
-        DROPBOX_APP_SECRET: ${{ secrets.DROPBOX_APP_SECRET }}
-        DROPBOX_REFRESH_TOKEN: ${{ secrets.DROPBOX_REFRESH_TOKEN }}
-        GCP_INGEST_SERVICE_KEY: ${{ secrets.GCP_INGEST_SERVICE_KEY }}
-        OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
-        MONGODB_URI: ${{ secrets.MONGODB_URI }}
-        MONGODB_DATABASE_NAME: ${{ secrets.MONGODB_DATABASE_NAME }}
-        AZURE_DEST_CONNECTION_STR: ${{ secrets.AZURE_DEST_CONNECTION_STR }}
-        PINECONE_API_KEY: ${{secrets.PINECONE_API_KEY}}
-        VECTARA_OAUTH_CLIENT_ID: ${{secrets.VECTARA_OAUTH_CLIENT_ID}}
-        VECTARA_OAUTH_SECRET: ${{secrets.VECTARA_OAUTH_SECRET}}
-        VECTARA_CUSTOMER_ID: ${{secrets.VECTARA_CUSTOMER_ID}}
-        ASTRA_DB_APPLICATION_TOKEN: ${{secrets.ASTRA_DB_TOKEN}}
-        ASTRA_DB_API_ENDPOINT: ${{secrets.ASTRA_DB_ENDPOINT}}
-        CLARIFAI_API_KEY: ${{secrets.CLARIFAI_API_KEY}}
-        DATABRICKS_HOST: ${{secrets.DATABRICKS_HOST}}
-        DATABRICKS_USERNAME: ${{secrets.DATABRICKS_USERNAME}}
-        DATABRICKS_PASSWORD: ${{secrets.DATABRICKS_PASSWORD}}
-        DATABRICKS_CATALOG: ${{secrets.DATABRICKS_CATALOG}}
-        TABLE_OCR: "tesseract"
-        OCR_AGENT: "unstructured.partition.utils.ocr_models.tesseract_ocr.OCRAgentTesseract"
-        CI: "true"
-      run: |
-        source .venv/bin/activate
-        sudo apt-get update
-        sudo apt-get install -y libmagic-dev poppler-utils libreoffice
-        make install-pandoc
-        sudo add-apt-repository -y ppa:alex-p/tesseract-ocr5
-        sudo apt-get update
-        sudo apt-get install -y tesseract-ocr
-        sudo apt-get install -y tesseract-ocr-kor
-        sudo apt-get install diffstat
-        tesseract --version
-        ./test_unstructured_ingest/test-ingest-dest.sh
-
-  test_ingest_help:
-    environment: ci
-    strategy:
-      matrix:
-        python-version: ["3.9","3.10","3.11", "3.12"]
-    runs-on: ubuntu-latest
-    needs: [setup_ingest, lint]
-    steps:
-      - uses: 'actions/checkout@v4'
-      - name: Set up Python ${{ matrix.python-version }}
-        uses: actions/setup-python@v5
-        with:
-          python-version: ${{ matrix.python-version }}
-      - name: Setup virtual environment
-        uses: ./.github/actions/base-ingest-cache
-        with:
-          python-version: ${{ matrix.python-version }}
-      - name: Validate --help
-        run: |
-          source .venv/bin/activate
-          ./test_unstructured_ingest/test-help.sh
-
-
   test_unstructured_api_unit:
     strategy:
       matrix:
         # NOTE(yuming): Unstructured API only use Python 3.10
         python-version: ["3.10"]
     runs-on: ubuntu-latest
-    env:
-      NLTK_DATA: ${{ github.workspace }}/nltk_data
     needs: [setup, lint]
     steps:
     - uses: actions/checkout@v4