Update local fork

.github/workflows/e2e.yml

@@ -69,7 +69,7 @@ jobs:
           make integration-test-embedders
 
   source_connectors_integration_test:
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-latest-m
     needs: [ setup ]
     steps:
     - uses: 'actions/checkout@v4'
@@ -84,16 +84,25 @@ jobs:
       uses: ./.github/actions/base-cache
       with:
         python-version: "3.10"
-    - name: Test (end-to-end)
-      run : |
-        source .venv/bin/activate
+    - name: Setup up docker
+      run: |
         sudo make install-docker-compose
         docker compose version
+    - name: Run Integration Tests
+      env:
+        DATABRICKS_HOST: ${{secrets.DATABRICKS_HOST}}
+        DATABRICKS_CATALOG: ${{secrets.DATABRICKS_CATALOG}}
+        DATABRICKS_CLIENT_ID: ${{secrets.DATABRICKS_CLIENT_ID}}
+        DATABRICKS_CLIENT_SECRET: ${{secrets.DATABRICKS_CLIENT_SECRET}}
+        CONFLUENCE_USER_EMAIL: ${{secrets.CONFLUENCE_USER_EMAIL}}
+        CONFLUENCE_API_TOKEN: ${{secrets.CONFLUENCE_API_TOKEN}}
+      run : |
+        source .venv/bin/activate
         make install-test
         make integration-test-connectors-src
 
   destination_connectors_integration_test:
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-latest-m
     needs: [ setup ]
     steps:
     - uses: 'actions/checkout@v4'
@@ -108,11 +117,25 @@ jobs:
       uses: ./.github/actions/base-cache
       with:
         python-version: "3.10"
-    - name: Test (end-to-end)
-      run : |
-        source .venv/bin/activate
+    - name: Setup up docker
+      run: |
         sudo make install-docker-compose
         docker compose version
+    - name: Run Integration Tests
+      env:
+        DATABRICKS_HOST: ${{secrets.DATABRICKS_HOST}}
+        DATABRICKS_CATALOG: ${{secrets.DATABRICKS_CATALOG}}
+        DATABRICKS_CLIENT_ID: ${{secrets.DATABRICKS_CLIENT_ID}}
+        DATABRICKS_CLIENT_SECRET: ${{secrets.DATABRICKS_CLIENT_SECRET}}
+        S3_INGEST_TEST_ACCESS_KEY: ${{ secrets.S3_INGEST_TEST_ACCESS_KEY }}
+        S3_INGEST_TEST_SECRET_KEY: ${{ secrets.S3_INGEST_TEST_SECRET_KEY }}
+        MS_CLIENT_CRED: ${{ secrets.MS_CLIENT_CRED }}
+        MS_CLIENT_ID: ${{ secrets.MS_CLIENT_ID }}
+        MS_TENANT_ID: ${{ secrets.MS_TENANT_ID }}
+        MS_USER_EMAIL: ${{ secrets.MS_USER_EMAIL }}
+        MS_USER_PNAME: ${{ secrets.MS_USER_PNAME }}
+      run : |
+        source .venv/bin/activate
         make install-test
         make integration-test-connectors-dest
 
@@ -140,8 +163,6 @@ jobs:
       env:
         AIRTABLE_PERSONAL_ACCESS_TOKEN: ${{ secrets.AIRTABLE_PERSONAL_ACCESS_TOKEN }}
         BOX_APP_CONFIG: ${{ secrets.BOX_APP_CONFIG }}
-        CONFLUENCE_API_TOKEN: ${{ secrets.CONFLUENCE_API_TOKEN }}
-        CONFLUENCE_USER_EMAIL: ${{ secrets.CONFLUENCE_USER_EMAIL }}
         DATABRICKS_HOST: ${{secrets.DATABRICKS_HOST}}
         DATABRICKS_CATALOG: ${{secrets.DATABRICKS_CATALOG}}
         DATABRICKS_CLIENT_ID: ${{secrets.DATABRICKS_CLIENT_ID}}
@@ -261,10 +282,6 @@ jobs:
         ASTRA_DB_APPLICATION_TOKEN: ${{secrets.ASTRA_DB_APPLICATION_TOKEN}}
         ASTRA_DB_API_ENDPOINT: ${{secrets.ASTRA_DB_ENDPOINT}}
         CLARIFAI_API_KEY: ${{secrets.CLARIFAI_API_KEY}}
-        DATABRICKS_HOST: ${{secrets.DATABRICKS_HOST}}
-        DATABRICKS_CATALOG: ${{secrets.DATABRICKS_CATALOG}}
-        DATABRICKS_CLIENT_ID: ${{secrets.DATABRICKS_CLIENT_ID}}
-        DATABRICKS_CLIENT_SECRET: ${{secrets.DATABRICKS_CLIENT_SECRET}}
         SHAREPOINT_CLIENT_ID: ${{secrets.SHAREPOINT_CLIENT_ID}}
         SHAREPOINT_CRED: ${{secrets.SHAREPOINT_CRED}}
         KDBAI_BEARER_TOKEN: ${{ secrets.KDBAI_BEARER_TOKEN }}

.gitignore

@@ -196,6 +196,7 @@ tags
 .ruff_cache/
 
 .ppm
+.vs
 
 example-docs/*_images
 examples/**/output/
@@ -207,3 +208,4 @@ metricsdiff.txt
 annotated/
 
 tmp_ingest/
+.vs

CHANGELOG.md

@@ -1,10 +1,62 @@
-## 0.0.26-dev4
+## 0.2.2
+
+### Enhancements
+
+* **Remove `overwrite` field** from fsspec and databricks connectors
+* **Added migration for GitLab Source V2**
+* **Added V2 confluence source connector**
+* **Added OneDrive destination connector**
+* **Qdrant destination to v2**
+* **Migrate Kafka Source Connector to V2**
+
+## 0.2.1
+
+### Enhancements
+
+* **File system based indexers return a record display name**
+* **Add singlestore source connector**
+* **Astra DB V2 Source Connector** Create a v2 version of the Astra DB Source Connector.
+* **Support native async requests from unstructured-client**
+* **Support filtering element types in partitioner step**
+
+### Fixes
+
+* **Fix Databricks Volumes file naming** Add .json to end of upload file.
+* **Fix SQL Type destination precheck** Change to context manager "with".
+
+## 0.2.0
+
+### Enhancements
+
+* **Add snowflake source and destination connectors**
+* **Migrate Slack Source Connector to V2**
+* **Migrate Slack Source Connector to V2**
+* **Add Delta Table destination to v2**
+* **Migrate Slack Source Connector to V2**
+
+## 0.1.1
+
+### Enhancements
+
+* **Update KDB.AI vectorstore integration to 1.4**
+* **Add sqlite and postgres source connectors**
+* **Add sampling functionality for indexers in fsspec connectors**
+
+### Fixes
+
+* **Fix Databricks Volumes destination** Fix for filenames to not be hashes.
+
+## 0.1.0
 
 ### Enhancements
 
 * **Move default API URL parameter value to serverless API**
 * **Add check that access config always wrapped in Secret**
 * **Add togetherai embedder support**
+* **Refactor sqlite and postgres to be distinct connectors to support better input validation**
+* **Added MongoDB source V2 connector**
+* **Support optional access configs on connection configs**
+* **Refactor databricks into distinct connectors based on auth type**
 
 ### Fixes
 

Makefile

@@ -141,6 +141,7 @@ integration-test-embedders:
 integration-test-connectors-src:
 	PYTHONPATH=. pytest --tags source -sv test/integration/connectors
 
+
 .PHONY: integration-test-connectors-dest
 integration-test-connectors-dest:
 	PYTHONPATH=. pytest --tags destination -sv test/integration/connectors

requirements/connectors/gitlab.txt

@@ -6,7 +6,7 @@ charset-normalizer==3.3.2
     # via requests
 idna==3.10
     # via requests
-python-gitlab==4.12.2
+python-gitlab==4.13.0
     # via -r ./connectors/gitlab.in
 requests==2.32.3
     # via

requirements/connectors/kdbai.in

@@ -1,3 +1,3 @@
 -c ../common/constraints.txt
 
-kdbai-client
+kdbai-client>=1.4.0

requirements/connectors/kdbai.txt

@@ -1,18 +1,20 @@
 # This file was autogenerated by uv via the following command:
-#    uv pip compile ./connectors/kdbai.in --output-file ./connectors/kdbai.txt --no-strip-extras --python-version 3.9
+#    uv pip compile kdbai.in --output-file kdbai.txt --no-strip-extras --python-version 3.9
 certifi==2024.8.30
     # via requests
-charset-normalizer==3.3.2
+charset-normalizer==3.4.0
     # via requests
 idna==3.10
     # via requests
-kdbai-client==1.3.0
-    # via -r ./connectors/kdbai.in
+kdbai-client==1.4.0
+    # via -r kdbai.in
 numpy==1.26.4
     # via
-    #   -c ./connectors/../common/constraints.txt
+    #   -c ../common/constraints.txt
     #   pandas
     #   pykx
+packaging==24.1
+    # via kdbai-client
 pandas==2.2.3
     # via
     #   kdbai-client
@@ -35,5 +37,5 @@ tzdata==2024.2
     # via pandas
 urllib3==1.26.20
     # via
-    #   -c ./connectors/../common/constraints.txt
+    #   -c ../common/constraints.txt
     #   requests

requirements/connectors/slack.in

@@ -1,3 +1,3 @@
 -c ../common/constraints.txt
 
-slack_sdk
+slack_sdk[optional]

requirements/connectors/slack.txt

@@ -1,4 +1,69 @@
 # This file was autogenerated by uv via the following command:
 #    uv pip compile ./connectors/slack.in --output-file ./connectors/slack.txt --no-strip-extras --python-version 3.9
-slack-sdk==3.33.1
+aiodns==3.2.0
+    # via slack-sdk
+aiohappyeyeballs==2.4.3
+    # via aiohttp
+aiohttp==3.10.10
+    # via slack-sdk
+aiosignal==1.3.1
+    # via aiohttp
+async-timeout==4.0.3
+    # via aiohttp
+attrs==24.2.0
+    # via aiohttp
+boto3==1.34.131
+    # via slack-sdk
+botocore==1.34.131
+    # via
+    #   -c ./connectors/../common/constraints.txt
+    #   boto3
+    #   s3transfer
+cffi==1.17.1
+    # via pycares
+frozenlist==1.4.1
+    # via
+    #   aiohttp
+    #   aiosignal
+greenlet==3.1.1
+    # via sqlalchemy
+idna==3.10
+    # via yarl
+jmespath==1.0.1
+    # via
+    #   boto3
+    #   botocore
+multidict==6.1.0
+    # via
+    #   aiohttp
+    #   yarl
+propcache==0.2.0
+    # via yarl
+pycares==4.4.0
+    # via aiodns
+pycparser==2.22
+    # via cffi
+python-dateutil==2.9.0.post0
+    # via botocore
+s3transfer==0.10.3
+    # via boto3
+six==1.16.0
+    # via python-dateutil
+slack-sdk[optional]==3.33.1
     # via -r ./connectors/slack.in
+sqlalchemy==2.0.36
+    # via slack-sdk
+typing-extensions==4.12.2
+    # via
+    #   multidict
+    #   sqlalchemy
+urllib3==1.26.20
+    # via
+    #   -c ./connectors/../common/constraints.txt
+    #   botocore
+websocket-client==1.8.0
+    # via slack-sdk
+websockets==13.1
+    # via slack-sdk
+yarl==1.15.5
+    # via aiohttp

requirements/connectors/snowflake.in

@@ -0,0 +1,3 @@
+-c ../common/constraints.txt
+
+snowflake-connector-python

requirements/connectors/snowflake.txt

@@ -0,0 +1,53 @@
+# This file was autogenerated by uv via the following command:
+#    uv pip compile snowflake.in --output-file snowflake.txt --no-strip-extras --python-version 3.9
+asn1crypto==1.5.1
+    # via snowflake-connector-python
+certifi==2024.8.30
+    # via
+    #   requests
+    #   snowflake-connector-python
+cffi==1.17.1
+    # via
+    #   cryptography
+    #   snowflake-connector-python
+charset-normalizer==3.4.0
+    # via
+    #   requests
+    #   snowflake-connector-python
+cryptography==43.0.3
+    # via
+    #   pyopenssl
+    #   snowflake-connector-python
+filelock==3.16.1
+    # via snowflake-connector-python
+idna==3.10
+    # via
+    #   requests
+    #   snowflake-connector-python
+packaging==24.1
+    # via snowflake-connector-python
+platformdirs==4.3.6
+    # via snowflake-connector-python
+pycparser==2.22
+    # via cffi
+pyjwt==2.9.0
+    # via snowflake-connector-python
+pyopenssl==24.2.1
+    # via snowflake-connector-python
+pytz==2024.2
+    # via snowflake-connector-python
+requests==2.32.3
+    # via snowflake-connector-python
+snowflake-connector-python==3.12.2
+    # via -r snowflake.in
+sortedcontainers==2.4.0
+    # via snowflake-connector-python
+tomlkit==0.13.2
+    # via snowflake-connector-python
+typing-extensions==4.12.2
+    # via snowflake-connector-python
+urllib3==1.26.20
+    # via
+    #   -c ../common/constraints.txt
+    #   requests
+    #   snowflake-connector-python

requirements/remote/client.in

@@ -1,3 +1,3 @@
 -c ../common/constraints.txt
 
-unstructured-client >= 0.25.8
+unstructured-client >= 0.26.1