fix(ingestion/tableau): restructure the tableau graphql datasource query

metadata-ingestion/setup.py

@@ -721,7 +721,7 @@
         "snowflake-summary = datahub.ingestion.source.snowflake.snowflake_summary:SnowflakeSummarySource",
         "snowflake-queries = datahub.ingestion.source.snowflake.snowflake_queries:SnowflakeQueriesSource",
         "superset = datahub.ingestion.source.superset:SupersetSource",
-        "tableau = datahub.ingestion.source.tableau:TableauSource",
+        "tableau = datahub.ingestion.source.tableau.tableau:TableauSource",
         "openapi = datahub.ingestion.source.openapi:OpenApiSource",
         "metabase = datahub.ingestion.source.metabase:MetabaseSource",
         "teradata = datahub.ingestion.source.sql.teradata:TeradataSource",

metadata-ingestion/src/datahub/ingestion/source/tableau_common.py → metadata-ingestion/src/datahub/ingestion/source/tableau/tableau_common.py

@@ -10,7 +10,7 @@
 
 import datahub.emitter.mce_builder as builder
 from datahub.configuration.common import ConfigModel
-from datahub.ingestion.source import tableau_constant as c
+from datahub.ingestion.source.tableau import tableau_constant as c
 from datahub.metadata.com.linkedin.pegasus2avro.dataset import (
     DatasetLineageType,
     FineGrainedLineage,
@@ -223,19 +223,19 @@ class MetadataQueryException(Exception):
         description
         isHidden
         folderName
-        upstreamFields {
-            name
-            datasource {
-                id
-            }
-        }
-        upstreamColumns {
-            name
-            table {
-                __typename
-                id
-            }
-        }
+        # upstreamFields {
+        #     name
+        #     datasource {
+        #         id
+        #     }
+        # }
+        # upstreamColumns {
+        #     name
+        #     table {
+        #         __typename
+        #         id
+        #     }
+        # }
         ... on ColumnField {
             dataCategory
             role
@@ -336,6 +336,26 @@ class MetadataQueryException(Exception):
 }
 """
 
+
+datasource_upstream_fields_graphql_query = """
+{
+    id
+    upstreamFields {
+        name
+        datasource {
+            id
+        }
+    }
+    upstreamColumns {
+        name
+        table {
+            __typename
+            id
+        }
+    }
+}
+"""
+
 published_datasource_graphql_query = """
 {
     __typename
@@ -368,19 +388,19 @@ class MetadataQueryException(Exception):
         description
         isHidden
         folderName
-        upstreamFields {
-            name
-            datasource {
-                id
-            }
-        }
-        upstreamColumns {
-            name
-            table {
-                __typename
-                id
-            }
-        }
+        # upstreamFields {
+        #     name
+        #     datasource {
+        #         id
+        #     }
+        # }
+        # upstreamColumns {
+        #     name
+        #     table {
+        #         __typename
+        #         id
+        #     }
+        # }
         ... on ColumnField {
             dataCategory
             role
@@ -910,6 +930,40 @@ def make_filter(filter_dict: dict) -> str:
     return filter
 
 
+def query_metadata_cursor_based_pagination(
+    server: Server,
+    main_query: str,
+    connection_name: str,
+    first: int,
+    after: Optional[str],
+    qry_filter: str = "",
+) -> dict:
+    query = f"""
+        query GetItems(
+          $first: Int,
+          $after: String
+        ) {{
+            {connection_name} (  first: $first, after: $after, filter:{{ {qry_filter} }})
+            {{
+                nodes {main_query}
+                pageInfo {{
+                    hasNextPage
+                    endCursor
+                }}
+            }}
+    }}"""  # {{ is to escape { character of f-string
+
+    result = server.metadata.query(
+        query=query,
+        variables={
+            "first": first,
+            "after": after,
+        },
+    )
+
+    return result
+
+
 def query_metadata(
     server: Server,
     main_query: str,
@@ -940,7 +994,7 @@ def query_metadata(
 
 def get_filter_pages(query_filter: dict, page_size: int) -> List[dict]:
     filter_pages = [query_filter]
-    # If this is primary id filter so we can use divide this query list into
+    # If this is primary id filter, so we can use divide this query list into
     # multiple requests each with smaller filter list (of order page_size).
     # It is observed in the past that if list of primary ids grow beyond
     # a few ten thousands then tableau server responds with empty response

metadata-ingestion/src/datahub/ingestion/source/tableau_constant.py → metadata-ingestion/src/datahub/ingestion/source/tableau/tableau_constant.py

@@ -66,6 +66,7 @@
 EXTRACT_LAST_INCREMENTAL_UPDATE_TIME = "extractLastIncrementalUpdateTime"
 EXTRACT_LAST_UPDATE_TIME = "extractLastUpdateTime"
 PUBLISHED_DATA_SOURCES_CONNECTION = "publishedDatasourcesConnection"
+FIELDS_CONNECTION = "fieldsConnection"
 DATA_SOURCE_FIELDS = "datasourceFields"
 SHEETS_CONNECTION = "sheetsConnection"
 CREATED_AT = "createdAt"