bugfix/correctly share session handler across ingest docs

CHANGELOG.md

@@ -1,4 +1,4 @@
-## 0.10.29-dev0
+## 0.10.29-dev1
 
 ### Enhancements
 
@@ -8,6 +8,8 @@
 
 ### Fixes
 
+* **Ingest session handler not being shared correctly** All ingest docs that leverage the session handler should only need to set it once per process. It was recreating it each time because the right values weren't being set nor available given how dataclasses work in python.
+
 ## 0.10.28
 
 ### Enhancements
@@ -1627,4 +1629,4 @@ of an email.
 
 ## 0.2.0
 
-* Initial release of unstructured
+* Initial release of unstructured

unstructured/__version__.py

@@ -1 +1 @@
-__version__ = "0.10.29-dev0"  # pragma: no cover
+__version__ = "0.10.29-dev1"  # pragma: no cover

unstructured/ingest/connector/google_drive.py

@@ -2,7 +2,7 @@
 import json
 import os
 import typing as t
-from dataclasses import dataclass
+from dataclasses import dataclass, field
 from datetime import datetime
 from mimetypes import guess_extension
 from pathlib import Path
@@ -85,7 +85,7 @@ class SimpleGoogleDriveConfig(ConfigSessionHandleMixin, BaseConnectorConfig):
     # Google Drive Specific Options
     drive_id: str
     service_account_key: str
-    extension: t.Optional[str]
+    extension: t.Optional[str] = None
     recursive: bool = False
 
     def __post_init__(self):
@@ -105,7 +105,7 @@ def create_session_handle(
 @dataclass
 class GoogleDriveIngestDoc(IngestDocSessionHandleMixin, IngestDocCleanupMixin, BaseIngestDoc):
     connector_config: SimpleGoogleDriveConfig
-    meta: t.Dict[str, str]
+    meta: t.Dict[str, str] = field(default_factory=dict)
     registry_name: str = "google_drive"
 
     @property

unstructured/ingest/connector/jira.py

@@ -250,7 +250,7 @@ class JiraIngestDoc(IngestDocSessionHandleMixin, IngestDocCleanupMixin, BaseInge
     """
 
     connector_config: SimpleJiraConfig
-    file_meta: JiraFileMeta
+    file_meta: t.Optional[JiraFileMeta] = None
     registry_name: str = "jira"
 
     @cached_property

unstructured/ingest/interfaces.py

@@ -655,6 +655,7 @@ def create_session_handle(self) -> BaseSessionHandle:
         session related resources across all document handling for a given subprocess."""
 
 
+@dataclass
 class IngestDocSessionHandleMixin:
     connector_config: ConfigSessionHandleMixin
     _session_handle: t.Optional[BaseSessionHandle] = None

unstructured/ingest/pipeline/source.py

@@ -16,23 +16,26 @@ def run(self, ingest_doc_dict: dict) -> t.Optional[str]:
         try:
             global session_handle
             doc = create_ingest_doc_from_dict(ingest_doc_dict)
-            filename = doc.filename
-            if not self.read_config.re_download and filename.is_file() and filename.stat().st_size:
-                logger.info(f"File exists: {filename}, skipping download")
-                return filename
             if isinstance(doc, IngestDocSessionHandleMixin):
                 if session_handle is None:
                     # create via doc.session_handle, which is a property that creates a
                     # session handle if one is not already defined
                     session_handle = doc.session_handle
                 else:
-                    doc.session_handle = session_handle
-            # does the work necessary to load file into filesystem
-            # in the future, get_file_handle() could also be supported
-            if self.retry_strategy:
-                self.retry_strategy(doc.get_file)
+                    doc._session_handle = session_handle
+            if (
+                not self.read_config.re_download
+                and doc.filename.is_file()
+                and doc.filename.stat().st_size
+            ):
+                logger.info(f"File exists: {doc.filename}, skipping download")
+                # Still need to fetch metadata if file exists locally
+                doc.update_source_metadata()
             else:
-                doc.get_file()
+                if self.retry_strategy:
+                    self.retry_strategy(doc.get_file)
+                else:
+                    doc.get_file()
             for k, v in doc.to_dict().items():
                 ingest_doc_dict[k] = v
             return doc.filename