lint fixes

unstructured/ingest/cli/cmds/airtable.py

@@ -82,7 +82,7 @@ def airtable_source(ctx: click.Context, **options):
     try:
         configs = extract_configs(options, validate=[AirtableCliConfig])
         runner = Airtable(
-            **configs,
+            **configs,  # type: ignore
         )
         runner.run(**options)
     except Exception as e:

unstructured/ingest/cli/cmds/azure.py

@@ -60,7 +60,7 @@ def azure_source(ctx: click.Context, **options):
     try:
         configs = extract_configs(options, validate=[AzureCliConfig])
         runner = Azure(
-            **configs,
+            **configs,  # type: ignore
         )
         runner.run(**options)
     except Exception as e:

unstructured/ingest/cli/cmds/biomed.py

@@ -81,7 +81,7 @@ def biomed_source(ctx: click.Context, **options):
     try:
         configs = extract_configs(options, validate=[BiomedCliConfig])
         runner = Biomed(
-            **configs,
+            **configs,  # type: ignore
         )
         runner.run(**options)
     except Exception as e:

unstructured/ingest/cli/cmds/box.py

@@ -47,7 +47,7 @@ def box_source(ctx: click.Context, **options):
     try:
         configs = extract_configs(options, validate=[BoxCliConfig])
         runner = Box(
-            **configs,
+            **configs,  # type: ignore
         )
         runner.run(**options)
     except Exception as e:

unstructured/ingest/cli/cmds/confluence.py

@@ -87,7 +87,7 @@ def confluence_source(ctx: click.Context, **options):
     try:
         configs = extract_configs(options, validate=[ConfluenceCliConfig])
         runner = Confluence(
-            **configs,
+            **configs,  # type: ignore
         )
         runner.run(**options)
     except Exception as e:

unstructured/ingest/cli/cmds/delta_table.py

@@ -67,7 +67,7 @@ def delta_table_source(ctx: click.Context, **options):
     try:
         configs = extract_configs(options, validate=[DeltaTableCliConfig])
         runner = DeltaTable(
-            **configs,
+            **configs,  # type: ignore
         )
         runner.run(**options)
     except Exception as e:

unstructured/ingest/cli/cmds/discord.py

@@ -61,7 +61,7 @@ def discord_source(ctx: click.Context, **options):
     try:
         configs = extract_configs(options, validate=[DiscordCliConfig])
         runner = Discord(
-            **configs,
+            **configs,  # type: ignore
         )
         runner.run(**options)
     except Exception as e:

unstructured/ingest/cli/cmds/dropbox.py

@@ -46,7 +46,7 @@ def dropbox_source(ctx: click.Context, **options):
     try:
         configs = extract_configs(options, validate=[DropboxCliConfig])
         runner = Dropbox(
-            **configs,
+            **configs,  # type: ignore
         )
         runner.run(**options)
     except Exception as e:

unstructured/ingest/cli/cmds/elasticsearch.py

@@ -63,7 +63,7 @@ def elasticsearch_source(ctx: click.Context, **options):
     try:
         configs = extract_configs(options, validate=[ElasticsearchCliConfig])
         runner = ElasticSearch(
-            **configs,
+            **configs,  # type: ignore
         )
         runner.run(**options)
     except Exception as e:

unstructured/ingest/cli/cmds/fsspec.py

@@ -27,7 +27,7 @@ def fsspec_source(ctx: click.Context, **options):
     try:
         configs = extract_configs(options)
         runner = Fsspec(
-            **configs,
+            **configs,  # type: ignore
         )
         runner.run(**options)
     except Exception as e:

unstructured/ingest/cli/cmds/gcs.py

@@ -49,7 +49,7 @@ def gcs_source(ctx: click.Context, **options):
     try:
         configs = extract_configs(options, validate=([GcsCliConfig]))
         runner = GCS(
-            **configs,
+            **configs,  # type: ignore
         )
         runner.run(**options)
     except Exception as e:

unstructured/ingest/cli/cmds/github.py

@@ -72,7 +72,7 @@ def github_source(ctx: click.Context, **options):
     try:
         configs = extract_configs(options, validate=([GithubCliConfig]))
         runner = Github(
-            **configs,
+            **configs,  # type: ignore
         )
         runner.run(**options)
     except Exception as e:

unstructured/ingest/cli/cmds/gitlab.py

@@ -72,7 +72,7 @@ def gitlab_source(ctx: click.Context, **options):
     try:
         configs = extract_configs(options, validate=([GitlabCliConfig]))
         runner = Gitlab(
-            **configs,
+            **configs,  # type: ignore
         )
         runner.run(**options)
     except Exception as e:

unstructured/ingest/cli/cmds/google_drive.py

@@ -61,7 +61,7 @@ def google_drive_source(ctx: click.Context, **options):
     try:
         configs = extract_configs(options, validate=([GoogleDriveCliConfig]))
         runner = GoogleDrive(
-            **configs,
+            **configs,  # type: ignore
         )
         runner.run(**options)
     except Exception as e:

unstructured/ingest/cli/cmds/jira.py

@@ -89,7 +89,7 @@ def jira_source(ctx: click.Context, **options):
     try:
         configs = extract_configs(options, validate=([JiraCliConfig]))
         runner = Jira(
-            **configs,
+            **configs,  # type: ignore
         )
         runner.run(**options)
     except Exception as e:

unstructured/ingest/cli/cmds/local.py

@@ -55,7 +55,7 @@ def local_source(ctx: click.Context, **options):
     try:
         configs = extract_configs(options, validate=([LocalCliConfig]))
         runner = Local(
-            **configs,
+            **configs,  # type: ignore
         )
         runner.run(**options)
     except Exception as e:

unstructured/ingest/cli/cmds/notion.py

@@ -62,7 +62,7 @@ def notion_source(ctx: click.Context, **options):
     try:
         configs = extract_configs(options, validate=([NotionCliConfig]))
         runner = Notion(
-            **configs,
+            **configs,  # type: ignore
         )
         runner.run(**options)
     except Exception as e:

unstructured/ingest/cli/cmds/onedrive.py

@@ -83,7 +83,7 @@ def onedrive_source(ctx: click.Context, **options):
     try:
         configs = extract_configs(options, validate=([OnedriveCliConfig]))
         runner = OneDrive(
-            **configs,
+            **configs,  # type: ignore
         )
         runner.run(**options)
     except Exception as e:

unstructured/ingest/cli/cmds/outlook.py

@@ -84,7 +84,7 @@ def outlook_source(ctx: click.Context, **options):
     try:
         configs = extract_configs(options, validate=([OutlookCliConfig]))
         runner = Outlook(
-            **configs,
+            **configs,  # type: ignore
         )
         runner.run(**options)
     except Exception as e:

unstructured/ingest/cli/cmds/reddit.py

@@ -84,7 +84,7 @@ def reddit_source(ctx: click.Context, **options):
     try:
         configs = extract_configs(options, validate=([RedditCliConfig]))
         runner = Reddit(
-            **configs,
+            **configs,  # type: ignore
         )
         runner.run(**options)
     except Exception as e:

unstructured/ingest/cli/cmds/s3.py

@@ -59,7 +59,7 @@ def s3_source(ctx: click.Context, **options):
     try:
         configs = extract_configs(options, validate=[S3CliConfig])
         s3_runner = S3(
-            **configs,
+            **configs,  # type: ignore
         )
         s3_runner.run(**options)
     except Exception as e:
@@ -79,7 +79,7 @@ def s3_dest(ctx: click.Context, **options):
     try:
         configs = extract_configs(options, validate=[S3CliConfig])
         s3_runner = S3(
-            **configs,
+            **configs,  # type: ignore
             writer_type="s3",
             writer_kwargs=options,
         )

unstructured/ingest/cli/cmds/salesforce.py

@@ -73,7 +73,7 @@ def salesforce_source(ctx: click.Context, **options):
     try:
         configs = extract_configs(options, validate=([SalesforceCliConfig]))
         runner = Salesforce(
-            **configs,
+            **configs,  # type: ignore
         )
         runner.run(**options)
     except Exception as e:

unstructured/ingest/cli/cmds/sharepoint.py

@@ -82,7 +82,7 @@ def sharepoint_source(ctx: click.Context, **options):
     try:
         configs = extract_configs(data=options, validate=[SharepointCliConfig])
         sharepoint_runner = SharePoint(
-            **configs,
+            **configs,  # type: ignore
         )
         sharepoint_runner.run(**options)
     except Exception as e:

unstructured/ingest/cli/cmds/slack.py

@@ -72,7 +72,7 @@ def slack_source(ctx: click.Context, **options):
     try:
         configs = extract_configs(data=options, validate=[SlackCliConfig])
         sharepoint_runner = Slack(
-            **configs,
+            **configs,  # type: ignore
         )
         sharepoint_runner.run(**options)
     except Exception as e:

unstructured/ingest/cli/cmds/wikipedia.py

@@ -53,7 +53,7 @@ def wikipedia_source(ctx: click.Context, **options):
     try:
         configs = extract_configs(data=options, validate=[WikipediaCliConfig])
         sharepoint_runner = Wikipedia(
-            **configs,
+            **configs,  # type: ignore
         )
         sharepoint_runner.run(**options)
     except Exception as e:

unstructured/ingest/cli/utils.py

@@ -23,7 +23,11 @@ def conform_click_options(options: dict):
             options[k] = list(v)
 
 
-def extract_configs(data: dict, validate: t.List[t.Type[BaseConfig]]) -> t.Dict[str, BaseConfig]:
+def extract_configs(
+    data: dict,
+    validate: t.Optional[t.List[t.Type[BaseConfig]]] = None,
+) -> t.Dict[str, BaseConfig]:
+    validate = validate if validate else []
     res = {
         "read_config": CliReadConfig.from_dict(data),
         "partition_config": CliPartitionConfig.from_dict(data),

unstructured/ingest/connector/biomed.py

@@ -164,7 +164,7 @@ def urls_to_metadata(urls):
                             download_filepath=(Path(self.read_config.download_dir) / local_path)
                             .resolve()
                             .as_posix(),
-                            output_filepath=(Path(self.partition_config.output_dir) / local_path)
+                            output_filepath=(Path(self.processor_config.output_dir) / local_path)
                             .resolve()
                             .as_posix(),
                         ),
@@ -246,7 +246,7 @@ def traverse(path, download_dir, output_dir):
                                 .resolve()
                                 .as_posix(),
                                 output_filepath=(
-                                    Path(self.partition_config.output_dir) / local_path
+                                    Path(self.processor_config.output_dir) / local_path
                                 )
                                 .resolve()
                                 .as_posix(),

unstructured/ingest/connector/fsspec.py

@@ -3,7 +3,7 @@
 import typing as t
 from contextlib import suppress
 from dataclasses import dataclass, field
-from pathlib import Path
+from pathlib import Path, PurePath
 
 from unstructured.ingest.error import SourceConnectionError
 from unstructured.ingest.interfaces import (
@@ -246,11 +246,7 @@ def write(self, docs: t.List[BaseIngestDoc]) -> None:
         for doc in docs:
             s3_file_path = doc.base_filename
             s3_folder = self.connector_config.path
-            if s3_folder[-1] != "/":
-                s3_folder = f"{s3_file_path}/"
-            if s3_file_path[0] == "/":
-                s3_file_path = s3_file_path[1:]
 
-            s3_output_path = s3_folder + s3_file_path
+            s3_output_path = str(PurePath(s3_folder, s3_file_path)) if s3_file_path else s3_folder
             logger.debug(f"Uploading {doc._output_filename} -> {s3_output_path}")
             fs.put_file(lpath=doc._output_filename, rpath=s3_output_path)

unstructured/ingest/pipeline/__init__.py

@@ -1,5 +1,5 @@
 from .doc_factory import DocFactory
-from .interfaces import PipelineContext
+from .interfaces import PipelineContext, ReformatNode
 from .partition import Partitioner
 from .pipeline import Pipeline
 from .reformat.chunking import Chunker
@@ -16,4 +16,5 @@
     "Pipeline",
     "Writer",
     "Chunker",
+    "ReformatNode",
 ]

unstructured/ingest/pipeline/doc_factory.py

@@ -9,7 +9,7 @@ class DocFactory(DocFactoryNode):
     def initialize(self):
         self.source_doc_connector.initialize()
 
-    def run(self) -> t.Iterable[str]:
+    def run(self, *args, **kwargs) -> t.Iterable[str]:
         docs = self.source_doc_connector.get_ingest_docs()
         json_docs = [doc.to_json() for doc in docs]
         return json_docs