feat(ingest): add new ingestion source PowerBI Report Server

docs/cli.md

@@ -102,6 +102,7 @@ We use a plugin architecture so that you can install only the dependencies you a
 | [starburst-trino-usage](./generated/ingestion/sources/trino.md)                 | `pip install 'acryl-datahub[starburst-trino-usage]'`       | Starburst Trino usage statistics source |
 | [nifi](./generated/ingestion/sources/nifi.md)                                   | `pip install 'acryl-datahub[nifi]'`                        | Nifi source                         |
 | [powerbi](./generated/ingestion/sources/powerbi.md)                             | `pip install 'acryl-datahub[powerbi]'`                     | Microsoft Power BI source           |
+| [powerbi-report-server](./generated/ingestion/sources/powerbi-report-server.md) | `pip install 'acryl-datahub[powerbi-report-server]'`       | Microsoft Power BI Report Server source           |
 
 ### Sinks
 

metadata-ingestion/docs/sources/powerbi-report-server/powerbi_report_server.md

@@ -0,0 +1,15 @@
+# PowerBI Report Server
+
+## Concept mapping 
+
+| Power BI Report Server    | Datahub             |                                                                                               
+| ------------------------- | ------------------- |
+| `Paginated Report`        | `Dashboard`         |
+| `Power BI Report`         | `Dashboard`         |
+| `Mobile Report`           | `Dashboard`         |
+| `Linked Report`           | `Dashboard`         |
+| `Dataset, Datasource`     | `N/A`               |
+
+### Note
+
+Current [PowerBI](https://powerbi.microsoft.com/en-us/what-is-power-bi/) module used only for work with [PowerBI Report Server](https://powerbi.microsoft.com/en-us/report-server/).

metadata-ingestion/docs/sources/powerbi-report-server/powerbi_report_server_recipe.yml

@@ -0,0 +1,27 @@
+source:
+  type: powerbireportserver.report_server.PowerBiReportServerDashboardSource
+  config:
+    # Your Power BI Report Server Windows username
+    username: username
+    # Your Power BI Report Server Windows password
+    password: password
+    # Your Workstation name
+    workstation_name: workstation_name
+    # Your Power BI Report Server host URL, example: localhost:80
+    host_port: host_port
+    # Your alias for Power BI Report Server host URL, example: local_powerbi_report_server
+    server_alias: server_alias
+    # Workspace's dataset environments, example: (PROD, DEV, QA, STAGE)
+    env: DEV
+    # Workspace's dataset environments, example: (PROD, DEV, QA, STAGE)
+    graphql_url: http://localhost:8080/api/graphql
+    # Your Power BI Report Server base virtual directory name for reports
+    report_virtual_directory_name: Reports
+    #  Your Power BI Report Server base virtual directory name for report server
+    report_server_virtual_directory_name: ReportServer
+    # dataset_type_mapping is fixed mapping of Power BI datasources type to equivalent Datahub "data platform" dataset
+    dataset_type_mapping:
+        PostgreSql: postgres
+        Oracle: oracle
+sink:
+  # sink configs

metadata-ingestion/examples/mce_files/data_platforms.json

@@ -656,6 +656,26 @@
     },
     "proposedDelta": null
   },
+  {
+    "auditHeader": null,
+    "proposedSnapshot": {
+      "com.linkedin.pegasus2avro.metadata.snapshot.DataPlatformSnapshot": {
+        "urn": "urn:li:dataPlatform:powerbi-report-server",
+        "aspects": [
+          {
+            "com.linkedin.pegasus2avro.dataplatform.DataPlatformInfo": {
+              "datasetNameDelimiter": ".",
+              "name": "powerbi-report-server",
+              "displayName": "Power BI Report Server",
+              "type": "OTHERS",
+              "logoUrl": "/assets/platforms/powerbireportserverlogo.png"
+            }
+          }
+        ]
+      }
+    },
+    "proposedDelta": null
+  },
   {
     "auditHeader": null,
     "proposedSnapshot": {

metadata-ingestion/setup.py

@@ -178,6 +178,8 @@ def get_long_description():
     "deltalake",
 }
 
+powerbi_report_server = {"requests", "requests_ntlm"}
+
 usage_common = {
     "sqlparse",
 }
@@ -282,6 +284,7 @@ def get_long_description():
     "starburst-trino-usage": sql_common | usage_common | trino,
     "nifi": {"requests", "packaging"},
     "powerbi": {"orderedset"} | microsoft_common,
+    "powerbi-report-server": {"orderedset"} | powerbi_report_server,
     "vertica": sql_common | {"sqlalchemy-vertica[vertica-python]==0.0.5"},
 }
 
@@ -366,6 +369,7 @@ def get_long_description():
             "hive",
             "starburst-trino-usage",
             "powerbi",
+            "powerbi-report-server",
             "vertica",
             "salesforce"
             # airflow is added below
@@ -507,6 +511,7 @@ def get_long_description():
         "starburst-trino-usage = datahub.ingestion.source.usage.starburst_trino_usage:TrinoUsageSource",
         "nifi = datahub.ingestion.source.nifi:NifiSource",
         "powerbi = datahub.ingestion.source.powerbi:PowerBiDashboardSource",
+        "powerbi-report-server = datahub.ingestion.source.powerbi_report_server:PowerBiReportServerDashboardSource",
         "iceberg = datahub.ingestion.source.iceberg.iceberg:IcebergSource",
         "vertica = datahub.ingestion.source.sql.vertica:VerticaSource",
         "presto-on-hive = datahub.ingestion.source.sql.presto_on_hive:PrestoOnHiveSource",

metadata-ingestion/src/datahub/ingestion/source/powerbi_report_server/__init__.py

@@ -0,0 +1,3 @@
+from datahub.ingestion.source.powerbi_report_server.report_server import (
+    PowerBiReportServerDashboardSource,
+)

metadata-ingestion/src/datahub/ingestion/source/powerbi_report_server/constants.py

@@ -0,0 +1,98 @@
+from enum import Enum
+
+
+class CreatedFrom(Enum):
+    REPORT = "Report"
+    DATASET = "Dataset"
+    VISUALIZATION = "Visualization"
+    UNKNOWN = "Unknown"
+
+
+class RelationshipDirection(Enum):
+    INCOMING = "INCOMING"
+    OUTGOING = "OUTGOING"
+
+
+class Constant:
+    """
+    keys used in powerbi plugin
+    """
+
+    DATASET = "DATASET"
+    REPORTS = "REPORTS"
+    REPORT = "REPORT"
+    REPORT_DATASOURCES = "REPORT_DATASOURCES"
+    TYPE_REPORT = "Report"
+    DATASOURCE = "DATASOURCE"
+    DATASET_DATASOURCES = "DATASET_DATASOURCES"
+    DatasetId = "DatasetId"
+    ReportId = "ReportId"
+    PowerBiReportId = "ReportId"
+    Dataset_URN = "DatasetURN"
+    DASHBOARD_ID = "powerbi.linkedin.com/dashboards/{}"
+    DASHBOARD = "dashboard"
+    DATASETS = "DATASETS"
+    DATASET_ID = "powerbi.linkedin.com/datasets/{}"
+    DATASET_PROPERTIES = "datasetProperties"
+    SUBSCRIPTION = "SUBSCRIPTION"
+    SYSTEM = "SYSTEM"
+    CATALOG_ITEM = "CATALOG_ITEM"
+    EXCEL_WORKBOOK = "EXCEL_WORKBOOK"
+    EXTENSIONS = "EXTENSIONS"
+    FAVORITE_ITEM = "FAVORITE_ITEM"
+    FOLDERS = "FOLDERS"
+    KPIS = "KPIS"
+    LINKED_REPORTS = "LINKED_REPORTS"
+    LINKED_REPORT = "LINKED_REPORT"
+    ME = "ME"
+    MOBILE_REPORTS = "MOBILE_REPORTS"
+    MOBILE_REPORT = "MOBILE_REPORT"
+    POWERBI_REPORTS = "POWERBI_REPORTS"
+    POWERBI_REPORT = "POWERBI_REPORT"
+    POWERBI_REPORT_DATASOURCES = "POWERBI_REPORT_DATASOURCES"
+    TYPE_POWERBI_REPORT = "PowerBIReport"
+    RESOURCE = "RESOURCE"
+    SESSION = "SESSION"
+    SYSTEM_POLICIES = "SYSTEM_POLICIES"
+    DATASET_KEY = "datasetKey"
+    BROWSERPATH = "browsePaths"
+    DATAPLATFORM_INSTANCE = "dataPlatformInstance"
+    STATUS = "status"
+    VALUE = "value"
+    ID = "ID"
+    DASHBOARD_INFO = "dashboardInfo"
+    DASHBOARD_KEY = "dashboardKey"
+    CORP_USER = "corpuser"
+    CORP_USER_INFO = "corpUserInfo"
+    OWNERSHIP = "ownership"
+    CORP_USER_KEY = "corpUserKey"
+
+
+API_ENDPOINTS = {
+    Constant.CATALOG_ITEM: "{PBIRS_BASE_URL}/CatalogItems({CATALOG_ID})",
+    Constant.DATASETS: "{PBIRS_BASE_URL}/Datasets",
+    Constant.DATASET: "{PBIRS_BASE_URL}/Datasets({DATASET_ID})",
+    Constant.DATASET_DATASOURCES: "{PBIRS_BASE_URL}/Datasets({DATASET_ID})/DataSources",
+    Constant.DATASOURCE: "{PBIRS_BASE_URL}/DataSources({DATASOURCE_ID})",
+    Constant.EXCEL_WORKBOOK: "{PBIRS_BASE_URL}/ExcelWorkbooks({EXCEL_WORKBOOK_ID})",
+    Constant.EXTENSIONS: "{PBIRS_BASE_URL}/Extensions",
+    Constant.FAVORITE_ITEM: "{PBIRS_BASE_URL}/FavoriteItems({FAVORITE_ITEM_ID})",
+    Constant.FOLDERS: "{PBIRS_BASE_URL}/Folders({FOLDER_ID})",
+    Constant.KPIS: "{PBIRS_BASE_URL}/Kpis({KPI_ID})",
+    Constant.LINKED_REPORTS: "{PBIRS_BASE_URL}/LinkedReports",
+    Constant.LINKED_REPORT: "{PBIRS_BASE_URL}/LinkedReports({LINKED_REPORT_ID})",
+    Constant.ME: "{PBIRS_BASE_URLL}/Me",
+    Constant.MOBILE_REPORTS: "{PBIRS_BASE_URL}/MobileReports",
+    Constant.MOBILE_REPORT: "{PBIRS_BASE_URL}/MobileReports({MOBILE_REPORT_ID})",
+    Constant.POWERBI_REPORTS: "{PBIRS_BASE_URL}/PowerBiReports",
+    Constant.POWERBI_REPORT: "{PBIRS_BASE_URL}/PowerBiReports({POWERBI_REPORT_ID})",
+    Constant.POWERBI_REPORT_DATASOURCES: "{PBIRS_BASE_URL}/PowerBiReports({ID})/DataSources",
+    Constant.REPORTS: "{PBIRS_BASE_URL}/Reports",
+    Constant.REPORT: "{PBIRS_BASE_URL}/Reports({REPORT_ID})",
+    Constant.REPORT_DATASOURCES: "{PBIRS_BASE_URL}/Reports({ID})/DataSources",
+    Constant.RESOURCE: "{PBIRS_BASE_URL}/Resources({RESOURCE_GET})",
+    Constant.SESSION: "{PBIRS_BASE_URL}/Session",
+    Constant.SUBSCRIPTION: "{PBIRS_BASE_URL}/Subscriptions({SUBSCRIPTION_ID})",
+    Constant.SYSTEM: "{PBIRS_BASE_URL}/System",
+    Constant.SYSTEM_POLICIES: "{PBIRS_BASE_URL}/System/Policies",
+}

metadata-ingestion/src/datahub/ingestion/source/powerbi_report_server/graphql_domain.py

@@ -0,0 +1,89 @@
+from datetime import datetime
+from typing import List, Optional
+
+from pydantic import BaseModel, Field
+
+from .constants import RelationshipDirection
+
+
+class CorpUserEditableInfo(BaseModel):
+    display_name: str = Field(alias="displayName")
+    title: str
+    about_me: Optional[str] = Field(alias="aboutMe")
+    teams: Optional[List[str]]
+    skills: Optional[List[str]]
+    picture_link: Optional[str] = Field(alias="pictureLink")
+
+
+class CorpUserEditableProperties(CorpUserEditableInfo):
+    slack: Optional[str]
+    phone: Optional[str]
+    email: str
+
+
+class CorpUserStatus(BaseModel):
+    active: bool
+
+
+class GlobalTags(BaseModel):
+    tags: List[str]
+
+
+class EntityRelationship(BaseModel):
+    type: str
+    direction: RelationshipDirection
+    entity: str
+    created: datetime
+
+
+class EntityRelationshipsResult(BaseModel):
+    start: int
+    count: int
+    total: int
+    relationships: Optional[EntityRelationship]
+
+
+class CorpUserProperties(BaseModel):
+    active: bool
+    display_name: str = Field(alias="displayName")
+    email: str
+    title: Optional[str]
+    manager: Optional["CorpUser"]
+    department_id: Optional[int] = Field(alias="departmentId")
+    department_name: Optional[str] = Field(alias="departmentName")
+    first_name: Optional[str] = Field(alias="firstName")
+    last_name: Optional[str] = Field(alias="lastName")
+    full_name: Optional[str] = Field(alias="fullName")
+    country_code: Optional[str] = Field(alias="countryCode")
+
+
+class CorpUser(BaseModel):
+    urn: str
+    type: str
+    username: str
+    properties: CorpUserProperties
+    editable_properties: Optional[CorpUserEditableProperties] = Field(
+        alias="editableProperties"
+    )
+    status: Optional[CorpUserStatus]
+    tags: Optional[GlobalTags]
+    relationships: Optional[EntityRelationshipsResult]
+    editableInfo: Optional[CorpUserEditableInfo] = Field(alias="editableInfo")
+    global_tags: Optional[GlobalTags] = Field(alias="globalTags")
+
+    def get_urn_part(self):
+        return "{}".format(self.username)
+
+    def __members(self):
+        return (self.username,)
+
+    def __eq__(self, instance):
+        return (
+            isinstance(instance, CorpUser) and self.__members() == instance.__members()
+        )
+
+    def __hash__(self):
+        return hash(self.__members())
+
+
+CorpUserProperties.update_forward_refs()