Roman/optimize ingest ci (#1799)

### Description Currently the CI caches the CI dependencies but uses the hash of all files in `requirements/`. This isn't completely accurate since the ingest dependencies are installed in a later step and don't affect the cached environment. As part of this PR: * ingest dependencies were isolated into their own folder in `requirements/ingest/` * A new cache setup was introduced in the CI to restore the base cache -> install ingest dependencies -> cache it with a new id * new make target created to install all ingest dependencies via `pip install -r ...` * updates to Dockerfile to use `find ...` to install all dependencies, avoiding the need to update this when new deps are added. * update to pip-compile script to run over all `*.in` files in `requirements/`
Unstructured-IO · Oct 24, 2023 · 4802332 · 4802332
1 parent 37e8413
commit 4802332
Show file tree

Hide file tree

Showing 82 changed files with 501 additions and 418 deletions.
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -229,14 +229,56 @@ jobs:
         tesseract --version
         make test-extra-${{ matrix.extra }} CI=true
 
+  setup_ingest:
+    strategy:
+      matrix:
+        python-version: [ "3.8","3.9","3.10","3.11" ]
+    runs-on: ubuntu-latest
+    env:
+      NLTK_DATA: ${{ github.workspace }}/nltk_data
+    needs: [setup]
+    steps:
+      - uses: actions/checkout@v3
+      - uses: actions/cache/restore@v3
+        id: base-virtualenv-cache
+        with:
+          path: |
+            .venv
+            nltk_data
+          key: unstructured-${{ runner.os }}-${{ matrix.python-version }}-${{ env.GHA_CACHE_KEY_VERSION }}-${{ hashFiles('requirements/*.txt') }}
+      # Due to the dependency on setup, the cache should exist before this is ran. Set failed if it wasn't found.
+      - name: Setup base virtual environment (no cache hit)
+        if: steps.base-virtualenv-cache.outputs.cache-hit != 'true'
+        uses: actions/github-script@v3
+        with:
+          script: |
+            core.setFailed('base cached environment couldn't be found')
+      - uses: actions/cache@v3
+        id: virtualenv-cache
+        with:
+          path: |
+            .venv
+            nltk_data
+          key: unstructured-ingest-${{ runner.os }}-${{ matrix.python-version }}-${{ env.GHA_CACHE_KEY_VERSION }}-${{ hashFiles('requirements/ingest/*.txt') }}
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v4
+        with:
+          python-version: ${{ matrix.python-version }}
+      - name: Setup virtual environment (no cache hit)
+        run: |
+          python${{ matrix.python-version }} -m venv .venv
+          source .venv/bin/activate
+          make install-all-ingest
+
+
   test_ingest:
     strategy:
       matrix:
         python-version: ["3.8","3.9","3.10","3.11"]
     runs-on: ubuntu-latest-m
     env:
       NLTK_DATA: ${{ github.workspace }}/nltk_data
-    needs: [setup, lint]
+    needs: [setup_ingest, lint]
     steps:
     - uses: actions/checkout@v3
     - name: Set up Python ${{ matrix.python-version }}
@@ -252,14 +294,14 @@ jobs:
         path: |
           .venv
           nltk_data
-        key: unstructured-${{ runner.os }}-${{ steps.full-python-version.outputs.version }}-${{ env.GHA_CACHE_KEY_VERSION }}-${{ hashFiles('requirements/*.txt') }}
-    - name: Setup virtual environment (no cache hit)
+        key: unstructured-ingest-${{ runner.os }}-${{ matrix.python-version }}-${{ env.GHA_CACHE_KEY_VERSION }}-${{ hashFiles('requirements/ingest/*.txt') }}
+    # Due to the dependency on setup_ingest, the cache should exist before this is ran. Set failed if it wasn't found.
+    - name: Setup base virtual environment (no cache hit)
       if: steps.virtualenv-cache.outputs.cache-hit != 'true'
-      run: |
-        python${{ matrix.python-version}} -m venv .venv
-        source .venv/bin/activate
-        mkdir "$NLTK_DATA"
-        make install-ci
+      uses: actions/github-script@v3
+      with:
+        script: |
+          core.setFailed('cached environment couldn't be found')
     - name: Setup docker-compose
       uses: KengoTODA/actions-setup-docker-compose@v1
       with:
@@ -316,27 +358,6 @@ jobs:
         sudo apt-get install -y tesseract-ocr-kor
         sudo apt-get install diffstat
         tesseract --version
-        make install-ingest-s3
-        make install-ingest-airtable
-        make install-ingest-azure
-        make install-ingest-box
-        make install-ingest-confluence
-        make install-ingest-discord
-        make install-ingest-elasticsearch
-        make install-ingest-dropbox
-        make install-ingest-gcs
-        make install-ingest-google-drive
-        make install-ingest-github
-        make install-ingest-gitlab
-        make install-ingest-jira
-        make install-ingest-onedrive
-        make install-ingest-outlook
-        make install-ingest-salesforce
-        make install-ingest-slack
-        make install-ingest-wikipedia
-        make install-ingest-notion
-        make install-ingest-delta-table
-        pip install "unstructured[openai]"
         ./test_unstructured_ingest/test-ingest.sh
 
   test_unstructured_api_unit:

diff --git a/.github/workflows/ingest-test-fixtures-update-pr.yml b/.github/workflows/ingest-test-fixtures-update-pr.yml
@@ -36,26 +36,68 @@ jobs:
           [ ! -d "$NLTK_DATA" ] && mkdir "$NLTK_DATA"
           make install-ci
 
-  update-fixtures-and-pr:
-    runs-on: ubuntu-latest-m
+  setup_ingest:
+    strategy:
+      matrix:
+        python-version: [ "3.8","3.9","3.10","3.11" ]
+    runs-on: ubuntu-latest
     env:
       NLTK_DATA: ${{ github.workspace }}/nltk_data
     needs: [setup]
     steps:
       - uses: actions/checkout@v3
       - uses: actions/cache/restore@v3
+        id: base-virtualenv-cache
+        with:
+          path: |
+            .venv
+            nltk_data
+          key: unstructured-${{ runner.os }}-${{ matrix.python-version }}-${{ env.GHA_CACHE_KEY_VERSION }}-${{ hashFiles('requirements/*.txt') }}
+      # Due to the dependency on setup, the cache should exist before this is ran. Set failed if it wasn't found.
+      - name: Setup base virtual environment (no cache hit)
+        if: steps.base-virtualenv-cache.outputs.cache-hit != 'true'
+        uses: actions/github-script@v3
+        with:
+          script: |
+            core.setFailed('base cached environment couldn't be found')
+      - uses: actions/cache@v3
         id: virtualenv-cache
         with:
           path: |
             .venv
             nltk_data
-          key: unstructured-${{ runner.os }}-${{ env.PYTHON_VERSION }}-${{ env.GHA_CACHE_KEY_VERSION }}-${{ hashFiles('requirements/*.txt') }}
+          key: unstructured-ingest-${{ runner.os }}-${{ matrix.python-version }}-${{ env.GHA_CACHE_KEY_VERSION }}-${{ hashFiles('requirements/ingest/*.txt') }}
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v4
+        with:
+          python-version: ${{ matrix.python-version }}
       - name: Setup virtual environment (no cache hit)
         run: |
-          python${{ env.PYTHON_VERSION }} -m venv .venv
+          python${{ matrix.python-version }} -m venv .venv
           source .venv/bin/activate
-          mkdir "$NLTK_DATA"
-          make install-ci
+          make install-all-ingest
+
+  update-fixtures-and-pr:
+    runs-on: ubuntu-latest-m
+    env:
+      NLTK_DATA: ${{ github.workspace }}/nltk_data
+    needs: [setup_ingest]
+    steps:
+      - uses: actions/checkout@v3
+      - uses: actions/cache/restore@v3
+        id: virtualenv-cache
+        with:
+          path: |
+            .venv
+            nltk_data
+          key: unstructured-ingest-${{ runner.os }}-${{ matrix.python-version }}-${{ env.GHA_CACHE_KEY_VERSION }}-${{ hashFiles('requirements/ingest/*.txt') }}
+      # Due to the dependency on setup_ingest, the cache should exist before this is ran. Set failed if it wasn't found.
+      - name: Setup base virtual environment (no cache hit)
+        if: steps.virtualenv-cache.outputs.cache-hit != 'true'
+        uses: actions/github-script@v3
+        with:
+          script: |
+            core.setFailed('cached environment couldn't be found')
       - name: Setup docker-compose
         uses: KengoTODA/actions-setup-docker-compose@v1
         with:
@@ -108,27 +150,6 @@ jobs:
           sudo apt-get install -y tesseract-ocr
           sudo apt-get install -y tesseract-ocr-kor
           tesseract --version
-          make install-ingest-s3
-          make install-ingest-azure
-          make install-ingest-airtable
-          make install-ingest-box
-          make install-ingest-confluence
-          make install-ingest-discord
-          make install-ingest-elasticsearch
-          make install-ingest-dropbox
-          make install-ingest-gcs
-          make install-ingest-google-drive
-          make install-ingest-github
-          make install-ingest-gitlab
-          make install-ingest-jira
-          make install-ingest-onedrive
-          make install-ingest-outlook
-          make install-ingest-salesforce
-          make install-ingest-slack
-          make install-ingest-wikipedia
-          make install-ingest-notion
-          make install-ingest-delta-table
-          pip install "unstructured[openai]"
           ./test_unstructured_ingest/test-ingest.sh
 
       - name: Save branch name to environment file

diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -1,4 +1,4 @@
-## 0.10.26-dev2
+## 0.10.26-dev3
 
 ### Enhancements
 

diff --git a/Dockerfile b/Dockerfile
@@ -24,36 +24,7 @@ COPY requirements requirements
 
 RUN python3.10 -m pip install pip==${PIP_VERSION} && \
   dnf -y groupinstall "Development Tools" && \
-  pip install --no-cache -r requirements/base.txt && \
-  pip install --no-cache -r requirements/test.txt && \
-  pip install --no-cache -r requirements/huggingface.txt && \
-  pip install --no-cache -r requirements/dev.txt && \
-  pip install --no-cache -r requirements/ingest-box.txt && \
-  pip install --no-cache -r requirements/ingest-confluence.txt && \
-  pip install --no-cache -r requirements/ingest-discord.txt && \
-  pip install --no-cache -r requirements/ingest-dropbox.txt && \
-  pip install --no-cache -r requirements/ingest-elasticsearch.txt && \
-  pip install --no-cache -r requirements/ingest-gcs.txt && \
-  pip install --no-cache -r requirements/ingest-github.txt && \
-  pip install --no-cache -r requirements/ingest-gitlab.txt && \
-  pip install --no-cache -r requirements/ingest-google-drive.txt && \
-  pip install --no-cache -r requirements/ingest-notion.txt && \
-  pip install --no-cache -r requirements/ingest-onedrive.txt && \
-  pip install --no-cache -r requirements/ingest-outlook.txt && \
-  pip install --no-cache -r requirements/ingest-reddit.txt && \
-  pip install --no-cache -r requirements/ingest-s3.txt && \
-  pip install --no-cache -r requirements/ingest-slack.txt && \
-  pip install --no-cache -r requirements/ingest-wikipedia.txt && \
-  pip install --no-cache -r requirements/extra-csv.txt && \
-  pip install --no-cache -r requirements/extra-docx.txt && \
-  pip install --no-cache -r requirements/extra-epub.txt && \
-  pip install --no-cache -r requirements/extra-markdown.txt && \
-  pip install --no-cache -r requirements/extra-msg.txt && \
-  pip install --no-cache -r requirements/extra-odt.txt && \
-  pip install --no-cache -r requirements/extra-pandoc.txt && \
-  pip install --no-cache -r requirements/extra-pdf-image.txt && \
-  pip install --no-cache -r requirements/extra-pptx.txt && \
-  pip install --no-cache -r requirements/extra-xlsx.txt && \
+  find requirements/ -type f -name "*.txt" -exec python3 -m pip install --no-cache -r '{}' ';' && \
   dnf -y groupremove "Development Tools" && \
   dnf clean all
 

diff --git a/Makefile b/Makefile
@@ -105,6 +105,11 @@ install-xlsx:
 .PHONY: install-all-docs
 install-all-docs: install-base install-csv install-docx install-epub install-odt install-pypandoc install-markdown install-msg install-pdf-image install-pptx install-xlsx
 
+.PHONY: install-all-ingest
+install-all-ingest:
+	find requirements/ingest -type f -name "*.txt" -exec python3 -m pip install -r '{}' ';'
+
+
 .PHONY: install-ingest-google-drive
 install-ingest-google-drive:
 	python3 -m pip install -r requirements/ingest-google-drive.txt

diff --git a/requirements/ingest-airtable.in b/requirements/ingest-airtable.in
diff --git a/requirements/ingest-azure-cognitive-search.in b/requirements/ingest-azure-cognitive-search.in
diff --git a/requirements/ingest-azure.in b/requirements/ingest-azure.in
diff --git a/requirements/ingest-bedrock.in b/requirements/ingest-bedrock.in
diff --git a/requirements/ingest-biomed.in b/requirements/ingest-biomed.in
diff --git a/requirements/ingest-box.in b/requirements/ingest-box.in
diff --git a/requirements/ingest-confluence.in b/requirements/ingest-confluence.in
diff --git a/requirements/ingest-delta-table.in b/requirements/ingest-delta-table.in
diff --git a/requirements/ingest-discord.in b/requirements/ingest-discord.in
diff --git a/requirements/ingest-dropbox.in b/requirements/ingest-dropbox.in
diff --git a/requirements/ingest-elasticsearch.in b/requirements/ingest-elasticsearch.in
diff --git a/requirements/ingest-gcs.in b/requirements/ingest-gcs.in
diff --git a/requirements/ingest-gitlab.in b/requirements/ingest-gitlab.in
diff --git a/requirements/ingest-google-drive.in b/requirements/ingest-google-drive.in
diff --git a/requirements/ingest-jira.in b/requirements/ingest-jira.in
diff --git a/requirements/ingest-notion.in b/requirements/ingest-notion.in
diff --git a/requirements/ingest-openai.in b/requirements/ingest-openai.in
diff --git a/requirements/ingest-reddit.in b/requirements/ingest-reddit.in
diff --git a/requirements/ingest-s3.in b/requirements/ingest-s3.in
diff --git a/requirements/ingest-salesforce.in b/requirements/ingest-salesforce.in
diff --git a/requirements/ingest-slack.in b/requirements/ingest-slack.in
diff --git a/requirements/ingest-wikipedia.in b/requirements/ingest-wikipedia.in
diff --git a/requirements/ingest/airtable.in b/requirements/ingest/airtable.in
@@ -0,0 +1,3 @@
+-c ../constraints.in
+-c ../base.txt
+pyairtable