feat: add file-based access permissions for SharePoint ingest

.github/workflows/ci.yml

@@ -288,6 +288,9 @@ jobs:
         SHAREPOINT_CLIENT_ID: ${{secrets.SHAREPOINT_CLIENT_ID}}
         SHAREPOINT_CRED: ${{secrets.SHAREPOINT_CRED}}
         SHAREPOINT_SITE: ${{secrets.SHAREPOINT_SITE}}
+        SHAREPOINT_PERMISSIONS_APP_ID: ${{secrets.SHAREPOINT_PERMISSIONS_APP_ID}}
+        SHAREPOINT_PERMISSIONS_APP_CRED: ${{secrets.SHAREPOINT_PERMISSIONS_APP_CRED}}
+        SHAREPOINT_PERMISSIONS_TENANT: ${{secrets.SHAREPOINT_PERMISSIONS_TENANT}}
         SLACK_TOKEN: ${{ secrets.SLACK_TOKEN }}
         UNS_API_KEY: ${{ secrets.UNS_API_KEY }}
         NOTION_API_KEY: ${{ secrets.NOTION_API_KEY }}

.github/workflows/ingest-test-fixtures-update-pr.yml

@@ -85,6 +85,9 @@ jobs:
           SHAREPOINT_CLIENT_ID: ${{secrets.SHAREPOINT_CLIENT_ID}}
           SHAREPOINT_CRED: ${{secrets.SHAREPOINT_CRED}}
           SHAREPOINT_SITE: ${{secrets.SHAREPOINT_SITE}}
+          SHAREPOINT_PERMISSIONS_APP_ID: ${{secrets.SHAREPOINT_PERMISSIONS_APP_ID}}
+          SHAREPOINT_PERMISSIONS_APP_CRED: ${{secrets.SHAREPOINT_PERMISSIONS_APP_CRED}}
+          SHAREPOINT_PERMISSIONS_TENANT: ${{secrets.SHAREPOINT_PERMISSIONS_TENANT}}
           SLACK_TOKEN: ${{ secrets.SLACK_TOKEN }}
           UNS_API_KEY: ${{ secrets.UNS_API_KEY }}
           NOTION_API_KEY: ${{ secrets.NOTION_API_KEY }}

CHANGELOG.md

@@ -1,14 +1,15 @@
-## 0.10.22-dev3
+## 0.10.22-dev4
 
 ### Enhancements
 
 ### Features
 
+* **Adds permissions(RBAC) data ingestion functionality for the Sharepoint connector.** Problem: Role based access control is an important component in many data storage systems. Users may need to pass permissions (RBAC) data to downstream systems when ingesting data. Feature: Added permissions data ingestion functionality to the Sharepoint connector.
+
 ### Fixes
 
 * **Fixes PDF list parsing creating duplicate list items** Previously a bug in PDF list item parsing caused removal of other elements and duplication of the list items
 
-
 ## 0.10.21
 
 * **Adds Scarf analytics**.

docs/source/source_connectors/sharepoint.rst

@@ -22,6 +22,9 @@ Run Locally
           --client-id "<Microsoft Sharepoint app client-id>" \
           --client-cred "<Microsoft Sharepoint app client-secret>" \
           --site "<e.g https://contoso.sharepoint.com or https://contoso.admin.sharepoint.com to process all sites within tenant>" \
+          --permissions-application-id "<Microsoft Graph API application id, to process permissions (rbac) data>" \
+          --permissions-client-cred "<Microsoft Graph API application credentials, to process permissions (rbac) data>" \
+          --permissions-tenant "<e.g https://contoso.onmicrosoft.com (tenant URL) to process permission (rbac) data>" \
           --files-only "Flag to process only files within the site(s)" \
           --output-dir sharepoint-ingest-output \
           --num-processes 2 \
@@ -46,6 +49,10 @@ Run Locally
                 client_id="<Microsoft Sharepoint app client-id>",
                 client_cred="<Microsoft Sharepoint app client-secret>",
                 site="<e.g https://contoso.sharepoint.com to process all sites within tenant>",
+                # Credentials to process data about permissions (rbac) within the tenant
+                permissions_application_id="<Microsoft Graph API application id>",
+                permissions_client_cred="<Microsoft Graph API application credentials>",
+                permissions_tenant="<e.g https://contoso.onmicrosoft.com to process permission info within tenant>",
                 # Flag to process only files within the site(s)
                 files_only=True,
                 path="Shared Documents",
@@ -68,6 +75,9 @@ You can also use upstream connectors with the ``unstructured`` API. For this you
           --client-id "<Microsoft Sharepoint app client-id>" \
           --client-cred "<Microsoft Sharepoint app client-secret>" \
           --site "<e.g https://contoso.sharepoint.com or https://contoso.admin.sharepoint.com to process all sites within tenant>" \
+          --permissions-application-id "<Microsoft Graph API application id, to process permissions (rbac) data>" \
+          --permissions-client-cred "<Microsoft Graph API application credentials, to process permissions (rbac) data>" \
+          --permissions-tenant "<e.g https://contoso.onmicrosoft.com (tenant URL) to process permission (rbac) data>" \
           --files-only "Flag to process only files within the site(s)" \
           --output-dir sharepoint-ingest-output \
           --num-processes 2 \
@@ -98,6 +108,10 @@ You can also use upstream connectors with the ``unstructured`` API. For this you
                 client_id="<Microsoft Sharepoint app client-id>",
                 client_cred="<Microsoft Sharepoint app client-secret>",
                 site="<e.g https://contoso.sharepoint.com to process all sites within tenant>",
+                # Credentials to process data about permissions (rbac) within the tenant
+                permissions_application_id="<Microsoft Graph API application id>",
+                permissions_client_cred="<Microsoft Graph API application credentials>",
+                permissions_tenant="<e.g https://contoso.onmicrosoft.com to process permission info within tenant>",
                 # Flag to process only files within the site(s)
                 files_only=True,
                 path="Shared Documents",

examples/ingest/sharepoint/ingest.sh

@@ -12,6 +12,8 @@
 # To get the credentials for your Sharepoint app, follow these steps:
 # https://github.com/vgrem/Office365-REST-Python-Client/wiki/How-to-connect-to-SharePoint-Online-and-and-SharePoint-2013-2016-2019-on-premises--with-app-principal
 
+# To setup your application and obtain permissions related variables (--permissions-application-id, --permissions-client-cred, --permissions-tenant), follow these steps:
+# https://tsmatz.wordpress.com/2016/10/07/application-permission-with-v2-endpoint-and-microsoft-graph
 
 
 SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
@@ -22,6 +24,9 @@ PYTHONPATH=. ./unstructured/ingest/main.py \
     --client-id "<Microsoft Sharepoint app client-id>" \
     --client-cred "<Microsoft Sharepoint app client-secret>" \
     --site "<e.g https://contoso.sharepoint.com or https://contoso.admin.sharepoint.com to process all sites within tenant>" \
+    --permissions-application-id "<Microsoft Graph API application id>" \
+    --permissions-client-cred "<Microsoft Graph API application credentials>" \
+    --permissions-tenant "<e.g https://contoso.onmicrosoft.com to process permission info within tenant>" \
     --files-only "Flag to process only files within the site(s)" \
     --output-dir sharepoint-ingest-output \
     --num-processes 2 \

test_unstructured_ingest/check-diff-expected-output.sh

@@ -46,6 +46,7 @@ if [ "$OVERWRITE_FIXTURES" != "false" ]; then
 elif ! diff -ru "$EXPECTED_OUTPUT_DIR" "$OUTPUT_DIR" ; then
     "$SCRIPT_DIR"/json-to-clean-text-folder.sh "$EXPECTED_OUTPUT_DIR" "$EXPECTED_OUTPUT_DIR_TEXT"
     "$SCRIPT_DIR"/json-to-clean-text-folder.sh "$OUTPUT_DIR" "$OUTPUT_DIR_TEXT"
+    "$SCRIPT_DIR"/clean-permissions-files.sh "$OUTPUT_DIR_TEXT"
     diff -ru "$EXPECTED_OUTPUT_DIR_TEXT" "$OUTPUT_DIR_TEXT"> outputdiff.txt
     cat outputdiff.txt
     diffstat -c outputdiff.txt

test_unstructured_ingest/clean-permissions-files.sh

@@ -0,0 +1,27 @@
+#!/usr/bin/env bash
+
+# Description: Delete (cleanup) permissions files in a folder, so that they are not included in
+#              text diff tests.
+#
+# Arguments:
+#   - $1: Name of the folder to do the cleanup operation in.
+
+set +e
+if [ "$#" -ne 1 ]; then
+    echo "Please provide a folder to clean the files in: $0 <folder_path>"
+    exit 1
+fi
+
+folder_path="$1"
+if [ ! -d "$folder_path" ]; then
+    echo "'$folder_path' is not a directory. Please provide a folder / directory."
+    exit 1
+fi
+
+for file in "$folder_path"/*_SEP_*; do
+    if [ -e "$file" ]; then
+        rm "$file"
+    fi
+done
+
+echo "Completed cleanup for permissions files"