bugfix/isolate ingest v2 dependencies

CHANGELOG.md

@@ -1,4 +1,4 @@
-## 0.14.10-dev4
+## 0.14.10-dev5
 
 ### Enhancements
 

unstructured/__version__.py

@@ -1 +1 @@
-__version__ = "0.14.10-dev4"  # pragma: no cover
+__version__ = "0.14.10-dev5"  # pragma: no cover

unstructured/ingest/v2/cli/base/dest.py

@@ -4,9 +4,9 @@
 
 import click
 
-from unstructured.ingest.cli.utils import conform_click_options
 from unstructured.ingest.v2.cli.base.cmd import BaseCmd
 from unstructured.ingest.v2.cli.interfaces import CliConfig
+from unstructured.ingest.v2.cli.utils import conform_click_options
 from unstructured.ingest.v2.logger import logger
 
 

unstructured/ingest/v2/cli/base/src.py

@@ -4,7 +4,6 @@
 
 import click
 
-from unstructured.ingest.cli.utils import Group, conform_click_options
 from unstructured.ingest.v2.cli.base.cmd import BaseCmd
 from unstructured.ingest.v2.cli.configs import (
     ChunkerCliConfig,
@@ -13,6 +12,7 @@
     ProcessorCliConfig,
 )
 from unstructured.ingest.v2.cli.interfaces import CliConfig
+from unstructured.ingest.v2.cli.utils import Group, conform_click_options
 from unstructured.ingest.v2.logger import logger
 
 

unstructured/ingest/v2/cli/cmds/chroma.py

@@ -2,9 +2,9 @@
 
 import click
 
-from unstructured.ingest.cli.interfaces import Dict
 from unstructured.ingest.v2.cli.base import DestCmd
 from unstructured.ingest.v2.cli.interfaces import CliConfig
+from unstructured.ingest.v2.cli.utils import Dict
 from unstructured.ingest.v2.processes.connectors.chroma import CONNECTOR_TYPE
 
 

unstructured/ingest/v2/cli/configs/partition.py

@@ -2,8 +2,8 @@
 
 import click
 
-from unstructured.ingest.cli.interfaces import DelimitedString, Dict
 from unstructured.ingest.v2.cli.interfaces import CliConfig
+from unstructured.ingest.v2.cli.utils import DelimitedString, Dict
 
 
 @dataclass

unstructured/ingest/v2/cli/utils.py

@@ -3,6 +3,7 @@
 import sys
 from dataclasses import fields, is_dataclass
 from gettext import gettext, ngettext
+from gettext import gettext as _
 from pathlib import Path
 from typing import Any, ForwardRef, Optional, Type, TypeVar, Union, get_args, get_origin
 
@@ -12,6 +13,13 @@
 from unstructured.ingest.v2.logger import logger
 
 
+def conform_click_options(options: dict):
+    # Click sets all multiple fields as tuple, this needs to be updated to list
+    for k, v in options.items():
+        if isinstance(v, tuple):
+            options[k] = list(v)
+
+
 class Dict(click.ParamType):
     name = "dict"
 
@@ -179,3 +187,54 @@ def is_subclass(instance, class_type) -> bool:
 
     adjusted_dict = conform_dict(inner_d=flat_data, inner_config=config)
     return config.from_dict(adjusted_dict, apply_name_overload=False)
+
+
+class Group(click.Group):
+    def parse_args(self, ctx, args):
+        """
+        This allows for subcommands to be called with the --help flag without breaking
+        if parent command is missing any of its required parameters
+        """
+
+        try:
+            return super().parse_args(ctx, args)
+        except click.MissingParameter:
+            if "--help" not in args:
+                raise
+
+            # remove the required params so that help can display
+            for param in self.params:
+                param.required = False
+            return super().parse_args(ctx, args)
+
+    def format_commands(self, ctx: click.Context, formatter: click.HelpFormatter) -> None:
+        """
+        Copy of the original click.Group format_commands() method but replacing
+        'Commands' -> 'Destinations'
+        """
+        commands = []
+        for subcommand in self.list_commands(ctx):
+            cmd = self.get_command(ctx, subcommand)
+            # What is this, the tool lied about a command.  Ignore it
+            if cmd is None:
+                continue
+            if cmd.hidden:
+                continue
+
+            commands.append((subcommand, cmd))
+
+        # allow for 3 times the default spacing
+        if len(commands):
+            if formatter.width:
+                limit = formatter.width - 6 - max(len(cmd[0]) for cmd in commands)
+            else:
+                limit = -6 - max(len(cmd[0]) for cmd in commands)
+
+            rows = []
+            for subcommand, cmd in commands:
+                help = cmd.get_short_help_str(limit)
+                rows.append((subcommand, help))
+
+            if rows:
+                with formatter.section(_("Destinations")):
+                    formatter.write_dl(rows)

unstructured/ingest/v2/processes/connectors/__init__.py

@@ -1 +1,47 @@
 from __future__ import annotations
+
+import unstructured.ingest.v2.processes.connectors.fsspec  # noqa: F401
+from unstructured.ingest.v2.processes.connector_registry import (
+    add_destination_entry,
+    add_source_entry,
+)
+
+from .astra import CONNECTOR_TYPE as ASTRA_CONNECTOR_TYPE
+from .astra import astra_destination_entry
+from .chroma import CONNECTOR_TYPE as CHROMA_CONNECTOR_TYPE
+from .chroma import chroma_destination_entry
+from .elasticsearch import CONNECTOR_TYPE as ELASTICSEARCH_CONNECTOR_TYPE
+from .elasticsearch import elasticsearch_destination_entry, elasticsearch_source_entry
+from .google_drive import CONNECTOR_TYPE as GOOGLE_DRIVE_CONNECTOR_TYPE
+from .google_drive import google_drive_source_entry
+from .local import CONNECTOR_TYPE as LOCAL_CONNECTOR_TYPE
+from .local import local_destination_entry, local_source_entry
+from .onedrive import CONNECTOR_TYPE as ONEDRIVE_CONNECTOR_TYPE
+from .onedrive import onedrive_source_entry
+from .opensearch import CONNECTOR_TYPE as OPENSEARCH_CONNECTOR_TYPE
+from .opensearch import opensearch_destination_entry, opensearch_source_entry
+from .weaviate import CONNECTOR_TYPE as WEAVIATE_CONNECTOR_TYPE
+from .weaviate import weaviate_destination_entry
+
+add_destination_entry(destination_type=ASTRA_CONNECTOR_TYPE, entry=astra_destination_entry)
+
+add_destination_entry(destination_type=CHROMA_CONNECTOR_TYPE, entry=chroma_destination_entry)
+
+add_source_entry(source_type=ELASTICSEARCH_CONNECTOR_TYPE, entry=elasticsearch_source_entry)
+add_destination_entry(
+    destination_type=ELASTICSEARCH_CONNECTOR_TYPE, entry=elasticsearch_destination_entry
+)
+
+add_source_entry(source_type=GOOGLE_DRIVE_CONNECTOR_TYPE, entry=google_drive_source_entry)
+
+add_source_entry(source_type=LOCAL_CONNECTOR_TYPE, entry=local_source_entry)
+add_destination_entry(destination_type=LOCAL_CONNECTOR_TYPE, entry=local_destination_entry)
+
+add_source_entry(source_type=ONEDRIVE_CONNECTOR_TYPE, entry=onedrive_source_entry)
+
+add_source_entry(source_type=OPENSEARCH_CONNECTOR_TYPE, entry=opensearch_source_entry)
+add_destination_entry(
+    destination_type=OPENSEARCH_CONNECTOR_TYPE, entry=opensearch_destination_entry
+)
+
+add_destination_entry(destination_type=WEAVIATE_CONNECTOR_TYPE, entry=weaviate_destination_entry)

unstructured/ingest/v2/processes/connectors/astra.py

@@ -20,7 +20,6 @@
 from unstructured.ingest.v2.logger import logger
 from unstructured.ingest.v2.processes.connector_registry import (
     DestinationRegistryEntry,
-    add_destination_entry,
 )
 from unstructured.utils import requires_dependencies
 
@@ -143,13 +142,10 @@ def run(self, contents: list[UploadContent], **kwargs: Any) -> None:
             collection.insert_many(chunk)
 
 
-add_destination_entry(
-    destination_type=CONNECTOR_TYPE,
-    entry=DestinationRegistryEntry(
-        connection_config=AstraConnectionConfig,
-        upload_stager_config=AstraUploadStagerConfig,
-        upload_stager=AstraUploadStager,
-        uploader_config=AstraUploaderConfig,
-        uploader=AstraUploader,
-    ),
+astra_destination_entry = DestinationRegistryEntry(
+    connection_config=AstraConnectionConfig,
+    upload_stager_config=AstraUploadStagerConfig,
+    upload_stager=AstraUploadStager,
+    uploader_config=AstraUploaderConfig,
+    uploader=AstraUploader,
 )

unstructured/ingest/v2/processes/connectors/chroma.py

@@ -23,7 +23,6 @@
 from unstructured.ingest.v2.logger import logger
 from unstructured.ingest.v2.processes.connector_registry import (
     DestinationRegistryEntry,
-    add_destination_entry,
 )
 from unstructured.staging.base import flatten_dict
 from unstructured.utils import requires_dependencies
@@ -197,13 +196,10 @@ def run(self, contents: list[UploadContent], **kwargs: Any) -> None:
             self.upsert_batch(collection, self.prepare_chroma_list(chunk))
 
 
-add_destination_entry(
-    destination_type=CONNECTOR_TYPE,
-    entry=DestinationRegistryEntry(
-        connection_config=ChromaConnectionConfig,
-        uploader=ChromaUploader,
-        uploader_config=ChromaUploaderConfig,
-        upload_stager=ChromaUploadStager,
-        upload_stager_config=ChromaUploadStagerConfig,
-    ),
+chroma_destination_entry = DestinationRegistryEntry(
+    connection_config=ChromaConnectionConfig,
+    uploader=ChromaUploader,
+    uploader_config=ChromaUploaderConfig,
+    upload_stager=ChromaUploadStager,
+    upload_stager_config=ChromaUploadStagerConfig,
 )

unstructured/ingest/v2/processes/connectors/elasticsearch.py

@@ -31,8 +31,6 @@
 from unstructured.ingest.v2.processes.connector_registry import (
     DestinationRegistryEntry,
     SourceRegistryEntry,
-    add_destination_entry,
-    add_source_entry,
 )
 from unstructured.staging.base import flatten_dict
 from unstructured.utils import requires_dependencies
@@ -386,24 +384,18 @@ def run(self, contents: list[UploadContent], **kwargs: Any) -> None:
                     )
 
 
-add_source_entry(
-    source_type=CONNECTOR_TYPE,
-    entry=SourceRegistryEntry(
-        connection_config=ElasticsearchConnectionConfig,
-        indexer=ElasticsearchIndexer,
-        indexer_config=ElasticsearchIndexerConfig,
-        downloader=ElasticsearchDownloader,
-        downloader_config=ElasticsearchDownloaderConfig,
-    ),
+elasticsearch_source_entry = SourceRegistryEntry(
+    connection_config=ElasticsearchConnectionConfig,
+    indexer=ElasticsearchIndexer,
+    indexer_config=ElasticsearchIndexerConfig,
+    downloader=ElasticsearchDownloader,
+    downloader_config=ElasticsearchDownloaderConfig,
 )
 
-add_destination_entry(
-    destination_type=CONNECTOR_TYPE,
-    entry=DestinationRegistryEntry(
-        connection_config=ElasticsearchConnectionConfig,
-        upload_stager_config=ElasticsearchUploadStagerConfig,
-        upload_stager=ElasticsearchUploadStager,
-        uploader_config=ElasticsearchUploaderConfig,
-        uploader=ElasticsearchUploader,
-    ),
+elasticsearch_destination_entry = DestinationRegistryEntry(
+    connection_config=ElasticsearchConnectionConfig,
+    upload_stager_config=ElasticsearchUploadStagerConfig,
+    upload_stager=ElasticsearchUploadStager,
+    uploader_config=ElasticsearchUploaderConfig,
+    uploader=ElasticsearchUploader,
 )

unstructured/ingest/v2/processes/connectors/fsspec/__init__.py

@@ -1 +1,37 @@
 from __future__ import annotations
+
+from unstructured.ingest.v2.processes.connector_registry import (
+    add_destination_entry,
+    add_source_entry,
+)
+
+from .azure import CONNECTOR_TYPE as AZURE_CONNECTOR_TYPE
+from .azure import azure_destination_entry, azure_source_entry
+from .box import CONNECTOR_TYPE as BOX_CONNECTOR_TYPE
+from .box import box_destination_entry, box_source_entry
+from .dropbox import CONNECTOR_TYPE as DROPBOX_CONNECTOR_TYPE
+from .dropbox import dropbox_destination_entry, dropbox_source_entry
+from .gcs import CONNECTOR_TYPE as GCS_CONNECTOR_TYPE
+from .gcs import gcs_destination_entry, gcs_source_entry
+from .s3 import CONNECTOR_TYPE as S3_CONNECTOR_TYPE
+from .s3 import s3_destination_entry, s3_source_entry
+from .sftp import CONNECTOR_TYPE as SFTP_CONNECTOR_TYPE
+from .sftp import sftp_destination_entry, sftp_source_entry
+
+add_source_entry(source_type=AZURE_CONNECTOR_TYPE, entry=azure_source_entry)
+add_destination_entry(destination_type=AZURE_CONNECTOR_TYPE, entry=azure_destination_entry)
+
+add_source_entry(source_type=BOX_CONNECTOR_TYPE, entry=box_source_entry)
+add_destination_entry(destination_type=BOX_CONNECTOR_TYPE, entry=box_destination_entry)
+
+add_source_entry(source_type=DROPBOX_CONNECTOR_TYPE, entry=dropbox_source_entry)
+add_destination_entry(destination_type=DROPBOX_CONNECTOR_TYPE, entry=dropbox_destination_entry)
+
+add_source_entry(source_type=GCS_CONNECTOR_TYPE, entry=gcs_source_entry)
+add_destination_entry(destination_type=GCS_CONNECTOR_TYPE, entry=gcs_destination_entry)
+
+add_source_entry(source_type=S3_CONNECTOR_TYPE, entry=s3_source_entry)
+add_destination_entry(destination_type=S3_CONNECTOR_TYPE, entry=s3_destination_entry)
+
+add_source_entry(source_type=SFTP_CONNECTOR_TYPE, entry=sftp_source_entry)
+add_destination_entry(destination_type=SFTP_CONNECTOR_TYPE, entry=sftp_destination_entry)

unstructured/ingest/v2/processes/connectors/fsspec/azure.py

@@ -9,8 +9,6 @@
 from unstructured.ingest.v2.processes.connector_registry import (
     DestinationRegistryEntry,
     SourceRegistryEntry,
-    add_destination_entry,
-    add_source_entry,
 )
 from unstructured.ingest.v2.processes.connectors.fsspec.fsspec import (
     FsspecAccessConfig,
@@ -131,22 +129,16 @@ async def run_async(self, path: Path, file_data: FileData, **kwargs: Any) -> Non
         return await super().run_async(path=path, file_data=file_data, **kwargs)
 
 
-add_source_entry(
-    source_type=CONNECTOR_TYPE,
-    entry=SourceRegistryEntry(
-        indexer=AzureIndexer,
-        indexer_config=AzureIndexerConfig,
-        downloader=AzureDownloader,
-        downloader_config=AzureDownloaderConfig,
-        connection_config=AzureConnectionConfig,
-    ),
+azure_source_entry = SourceRegistryEntry(
+    indexer=AzureIndexer,
+    indexer_config=AzureIndexerConfig,
+    downloader=AzureDownloader,
+    downloader_config=AzureDownloaderConfig,
+    connection_config=AzureConnectionConfig,
 )
 
-add_destination_entry(
-    destination_type=CONNECTOR_TYPE,
-    entry=DestinationRegistryEntry(
-        uploader=AzureUploader,
-        uploader_config=AzureUploaderConfig,
-        connection_config=AzureConnectionConfig,
-    ),
+azure_destination_entry = DestinationRegistryEntry(
+    uploader=AzureUploader,
+    uploader_config=AzureUploaderConfig,
+    connection_config=AzureConnectionConfig,
 )