feat: refactor ingest

CHANGELOG.md

@@ -1,8 +1,15 @@
-## 0.14.1-dev0
+## 0.14.1-dev1
 
 * **Add support for Python 3.12**. `unstructured` now works with Python 3.12!
 
 ### Features
+* **Large improvements to the ingest process:**
+  * Support for multiprocessing and async, with limits for both.
+  * Streamlined to process when mapping CLI invocations to the underlying code
+  * More granular steps introduced to give better control over process (i.e. dedicated step to uncompress files already in the local filesystem, new optional staging step before upload)
+  * Use the python client when calling the unstructured api for partitioning or chunking
+  * Saving the final content is now a dedicated destination connector (local) set as the default if none are provided. Avoids adding new files locally if uploading elsewhere.
+  * Leverage last modified date when deciding if new files should be downloaded and reprocessed.
 
 ### Fixes
 

examples/ingest/chroma/ingest.sh

@@ -16,7 +16,7 @@ PYTHONPATH=. ./unstructured/ingest/main.py \
   --input-path example-docs/book-war-and-peace-1p.txt \
   --output-dir local-to-chroma \
   --strategy fast \
-  --chunk-elements \
+  --chunking-strategy by_title \
   --embedding-provider "<an unstructured embedding provider, ie. langchain-huggingface>" \
   --num-processes 2 \
   --verbose \

examples/ingest/clarifai/ingest.sh

@@ -10,7 +10,7 @@ PYTHONPATH=. ./unstructured/ingest/main.py \
   --input-path example-docs/book-war-and-peace-1225p.txt \
   --output-dir local-output-to-clarifai \
   --strategy fast \
-  --chunk-elements \
+  --chunking-strategy by_title \
   --num-processes 2 \
   --verbose \
   clarifai \

examples/ingest/elasticsearch/destination.sh

@@ -15,7 +15,7 @@ PYTHONPATH=. ./unstructured/ingest/main.py \
   --input-path example-docs/book-war-and-peace-1225p.txt \
   --output-dir local-to-elasticsearch \
   --strategy fast \
-  --chunk-elements \
+  --chunking-strategy by_title \
   --embedding-provider "<an unstructured embedding provider, ie. langchain-huggingface>" \
   --num-processes 2 \
   --verbose \

examples/ingest/mongodb/destination.sh

@@ -15,7 +15,7 @@ PYTHONPATH=. ./unstructured/ingest/main.py \
   --input-path example-docs/book-war-and-peace-1225p.txt \
   --output-dir local-to-mongodb \
   --strategy fast \
-  --chunk-elements \
+  --chunking-strategy by_title \
   --embedding-provider "<an unstructured embedding provider, ie. langchain-huggingface>" \
   --num-processes 2 \
   --verbose \

examples/ingest/opensearch/destination.sh

@@ -15,7 +15,7 @@ PYTHONPATH=. ./unstructured/ingest/main.py \
   --input-path example-docs/book-war-and-peace-1225p.txt \
   --output-dir local-to-opensearch \
   --strategy fast \
-  --chunk-elements \
+  --chunking-strategy by_title \
   --embedding-provider "<an unstructured embedding provider, ie. langchain-huggingface>" \
   --num-processes 2 \
   --verbose \

examples/ingest/pinecone/ingest.sh

@@ -16,7 +16,7 @@ PYTHONPATH=. ./unstructured/ingest/main.py \
   --input-path example-docs/book-war-and-peace-1225p.txt \
   --output-dir local-to-pinecone \
   --strategy fast \
-  --chunk-elements \
+  --chunking-strategy by_title \
   --embedding-provider "<an unstructured embedding provider, ie. langchain-huggingface>" \
   --num-processes 2 \
   --verbose \

examples/ingest/qdrant/ingest.sh

@@ -12,7 +12,7 @@ unstructured-ingest \
   --input-path example-docs/book-war-and-peace-1225p.txt \
   --output-dir local-output-to-qdrant \
   --strategy fast \
-  --chunk-elements \
+  --chunking-strategy by_title \
   --embedding-provider "$EMBEDDING_PROVIDER" \
   --num-processes 2 \
   --verbose \

examples/ingest/sql/ingest.sh

@@ -12,7 +12,7 @@ PYTHONPATH=. ./unstructured/ingest/main.py \
   --input-path example-docs/book-war-and-peace-1225p.txt \
   --output-dir local-to-pinecone \
   --strategy fast \
-  --chunk-elements \
+  --chunking-strategy by_title \
   --embedding-provider "<an unstructured embedding provider, ie. langchain-huggingface>" \
   --num-processes 2 \
   --verbose \

examples/ingest/weaviate/ingest.sh

@@ -16,7 +16,7 @@ PYTHONPATH=. ./unstructured/ingest/main.py \
   --reprocess \
   --input-path example-docs/book-war-and-peace-1225p.txt \
   --work-dir weaviate-work-dir \
-  --chunk-elements \
+  --chunking-strategy by_title \
   --chunk-new-after-n-chars 2500 --chunk-multipage-sections \
   --embedding-provider "langchain-huggingface" \
   weaviate \

test_unstructured_ingest/dest/astra.sh

@@ -47,7 +47,7 @@ PYTHONPATH=. ./unstructured/ingest/main.py \
   --verbose \
   --input-path example-docs/book-war-and-peace-1p.txt \
   --work-dir "$WORK_DIR" \
-  --chunk-elements \
+  --chunking-strategy by_title \
   --chunk-max-characters 1500 \
   --chunk-multipage-sections \
   --embedding-provider "langchain-huggingface" \

test_unstructured_ingest/dest/chroma.sh

@@ -45,7 +45,7 @@ PYTHONPATH=. ./unstructured/ingest/main.py \
   --verbose \
   --input-path example-docs/book-war-and-peace-1p.txt \
   --work-dir "$WORK_DIR" \
-  --chunk-elements \
+  --chunking-strategy by_title \
   --chunk-max-characters 1500 \
   --chunk-multipage-sections \
   --embedding-provider "langchain-huggingface" \

test_unstructured_ingest/dest/clarifai.sh

@@ -70,7 +70,7 @@ PYTHONPATH=. ./unstructured/ingest/main.py \
   --input-path example-docs/book-war-and-peace-1p.txt \
   --output-dir "$OUTPUT_DIR" \
   --strategy fast \
-  --chunk-elements \
+  --chunking-strategy by_title \
   --num-processes "$max_processes" \
   --work-dir "$WORK_DIR" \
   --verbose \

test_unstructured_ingest/dest/elasticsearch.sh

@@ -45,7 +45,7 @@ PYTHONPATH=. ./unstructured/ingest/main.py \
   --reprocess \
   --input-path example-docs/book-war-and-peace-1225p.txt \
   --work-dir "$WORK_DIR" \
-  --chunk-elements \
+  --chunking-strategy by_title \
   --chunk-combine-text-under-n-chars 200 \
   --chunk-new-after-n-chars 2500 \
   --chunk-max-characters 38000 \

test_unstructured_ingest/dest/pinecone.sh

@@ -91,7 +91,7 @@ PYTHONPATH=. ./unstructured/ingest/main.py \
   --reprocess \
   --input-path example-docs/book-war-and-peace-1225p.txt \
   --work-dir "$WORK_DIR" \
-  --chunk-elements \
+  --chunking-strategy by_title \
   --chunk-combine-text-under-n-chars 200 --chunk-new-after-n-chars 2500 --chunk-max-characters 38000 --chunk-multipage-sections \
   --embedding-provider "langchain-huggingface" \
   pinecone \

test_unstructured_ingest/dest/qdrant.sh

@@ -60,7 +60,7 @@ PYTHONPATH=. ./unstructured/ingest/main.py \
   --reprocess \
   --input-path example-docs/book-war-and-peace-1225p.txt \
   --work-dir "$WORK_DIR" \
-  --chunk-elements \
+  --chunking-strategy by_title \
   --chunk-combine-text-under-n-chars 200 --chunk-new-after-n-chars 2500 --chunk-max-characters 38000 --chunk-multipage-sections \
   --embedding-provider "langchain-huggingface" \
   qdrant \

test_unstructured_ingest/dest/s3.sh

@@ -7,7 +7,6 @@ SCRIPT_DIR=$(dirname "$DEST_PATH")
 cd "$SCRIPT_DIR"/.. || exit 1
 OUTPUT_FOLDER_NAME=s3-dest
 OUTPUT_ROOT=${OUTPUT_ROOT:-$SCRIPT_DIR}
-OUTPUT_DIR=$OUTPUT_ROOT/structured-output/$OUTPUT_FOLDER_NAME
 WORK_DIR=$OUTPUT_ROOT/workdir/$OUTPUT_FOLDER_NAME
 max_processes=${MAX_PROCESSES:=$(python3 -c "import os; print(os.cpu_count())")}
 DESTINATION_S3="s3://utic-dev-tech-fixtures/utic-ingest-test-fixtures-output/$(uuidgen)/"
@@ -16,7 +15,6 @@ CI=${CI:-"false"}
 # shellcheck disable=SC1091
 source "$SCRIPT_DIR"/cleanup.sh
 function cleanup() {
-  cleanup_dir "$OUTPUT_DIR"
   cleanup_dir "$WORK_DIR"
 
   if aws s3 ls "$DESTINATION_S3" --region us-east-2; then
@@ -31,7 +29,6 @@ RUN_SCRIPT=${RUN_SCRIPT:-./unstructured/ingest/main.py}
 PYTHONPATH=${PYTHONPATH:-.} "$RUN_SCRIPT" \
   local \
   --num-processes "$max_processes" \
-  --output-dir "$OUTPUT_DIR" \
   --strategy fast \
   --verbose \
   --reprocess \

test_unstructured_ingest/dest/sharepoint-embed-cog-index.sh

@@ -98,7 +98,7 @@ PYTHONPATH=${PYTHONPATH:-.} "$RUN_SCRIPT" \
   --path "Shared Documents" \
   --recursive \
   --embedding-provider "langchain-huggingface" \
-  --chunk-elements \
+  --chunking-strategy by_title \
   --chunk-multipage-sections \
   --work-dir "$WORK_DIR" \
   azure-cognitive-search \