implement cross tab data api (fix #25)

bertrandmartel · Jul 28, 2021 · 52f59e1 · 52f59e1
1 parent 071d60c
commit 52f59e1
Show file tree

Hide file tree

Showing 6 changed files with 209 additions and 0 deletions.
diff --git a/README.md b/README.md
@@ -209,6 +209,27 @@ The prefix values, I've encountered are: `vud` and `vudcsv`. The default is `vud
 
 [Try this on repl.it](https://replit.com/@bertrandmartel/TableauCovidWyomingCsv)
 
+#### Download Cross Tab data
+
+For Tableau URL that have the crosstab feature enabled, you can download the crosstab using:
+
+```python
+from tableauscraper import TableauScraper as TS
+
+url = "https://tableau.soa.org/t/soa-public/views/USPostLevelTermMortalityExperienceInteractiveTool/DataTable2"
+
+ts = TS()
+ts.loads(url)
+wb = ts.getWorkbook()
+
+wb.setParameter(inputName="Count or Amount", value="Amount")
+
+data = wb.getCrossTabData(
+    sheetName="Data Table 2 - Premium Jump & PLT Duration")
+
+print(data)
+```
+
 #### Go to sheet
 
 Get list of all sheets with subsheets visible or invisible, ability to send a go-to-sheet command (dashboar button) :

diff --git a/tableauscraper/TableauWorkbook.py b/tableauscraper/TableauWorkbook.py
@@ -165,6 +165,30 @@ def getCsvData(self, sheetName, prefix="vudcsv"):
             print("no viewIds found in json info")
         return None
 
+    def getCrossTabData(self, sheetName):
+        r = tableauscraper.api.exportCrosstabServerDialog(self._scraper)
+
+        sheets = [
+            t for t in r["vqlCmdResponse"]["layoutStatus"]["applicationPresModel"]["presentationLayerNotification"][
+                0]["presModelHolder"]["genExportCrosstabOptionsDialogPresModel"]["thumbnailSheetPickerItems"]
+            if t["sheetName"] == sheetName
+        ]
+        if len(sheets) == 0:
+            self._scraper.logger.warning(
+                f"sheet {sheetName} not found in API result")
+            return None
+
+        sheetId = sheets[0]["sheetdocId"]
+        r = tableauscraper.api.exportCrosstabToCsvServer(
+            self._scraper, sheetId)
+        resultKey = r[
+            "vqlCmdResponse"]["layoutStatus"]["applicationPresModel"]["presentationLayerNotification"][0]["presModelHolder"]["genExportFilePresModel"]["resultKey"]
+        r = tableauscraper.api.downloadCrossTabData(self._scraper, resultKey)
+        try:
+            return pd.read_csv(io.StringIO(r), sep='\t')
+        except (ParserError, EmptyDataError):
+            return None
+
     def getStoryPoints(self):
         return tableauscraper.utils.getStoryPointsFromInfo(self._originalInfo)
 

diff --git a/tableauscraper/api.py b/tableauscraper/api.py
@@ -195,6 +195,49 @@ def goToSheet(scraper, windowId):
     return r.json()
 
 
+def exportCrosstabServerDialog(scraper):
+    delayExecution(scraper)
+    payload = (
+        ("thumbnailUris", (None, json.dumps({}))),
+    )
+    r = scraper.session.post(
+        f'{scraper.host}{scraper.tableauData["vizql_root"]}/sessions/{scraper.tableauData["sessionid"]}/commands/tabsrv/export-crosstab-server-dialog',
+        files=payload,
+        verify=scraper.verify
+    )
+    scraper.lastActionTime = time.time()
+    return r.json()
+
+
+def exportCrosstabToCsvServer(scraper, sheetId):
+    delayExecution(scraper)
+    payload = (
+        ("sheetdocId", (None, sheetId)),
+        ("useTabs", (None, "true")),
+        ("sendNotifications", (None, "true")),
+    )
+    r = scraper.session.post(
+        f'{scraper.host}{scraper.tableauData["vizql_root"]}/sessions/{scraper.tableauData["sessionid"]}/commands/tabsrv/export-crosstab-to-csvserver',
+        files=payload,
+        verify=scraper.verify
+    )
+    scraper.lastActionTime = time.time()
+    return r.json()
+
+
+def downloadCrossTabData(scraper, resultKey):
+    r = scraper.session.get(
+        f'{scraper.host}{scraper.tableauData["vizql_root"]}/tempfile/sessions/{scraper.tableauData["sessionid"]}/',
+        params={
+            "key": resultKey,
+            "keepfile": "yes",
+            "attachment": "yes"
+        },
+        verify=scraper.verify)
+    scraper.lastActionTime = time.time()
+    return r.content.decode('utf-16')
+
+
 def setActiveStoryPoint(scraper, storyBoard, storyPointId):
     delayExecution(scraper)
     payload = (

diff --git a/tests/python/test_TableauWorkbook.py b/tests/python/test_TableauWorkbook.py
@@ -15,6 +15,10 @@
 from tests.python.test_common import storyPointsCmdResponse as storyPointsCmdResponse
 from tests.python.test_common import tableauDownloadableCsvData as tableauDownloadableCsvData
 from tests.python.test_common import tableauDataResponseWithStoryPointsNav
+from tests.python.test_common import tableauCrossTabData
+from tests.python.test_common import tableauExportCrosstabServerDialog
+from tests.python.test_common import tableauExportCrosstabToCsvServer
+import json
 
 
 def test_TableauWorkbook(mocker: MockerFixture) -> None:
@@ -278,3 +282,25 @@ def test_goToStoryPoint(mocker: MockerFixture) -> None:
     storyWb = wb.goToStoryPoint(storyPointId=1)
     assert type(storyWb) is TableauWorkbook
     assert len(storyWb.worksheets) == 1
+
+
+# def test_getCrossTabData(mocker: MockerFixture) -> None:
+#     mocker.patch(
+#         "tableauscraper.api.getTableauViz", return_value=tableauVizHtmlResponse
+#     )
+#     mocker.patch("tableauscraper.api.getTableauData",
+#                  return_value=tableauDataResponse)
+#     mocker.patch("tableauscraper.api.exportCrosstabServerDialog",
+#                  return_value=json.loads(tableauExportCrosstabServerDialog))
+#     mocker.patch("tableauscraper.api.exportCrosstabToCsvServer",
+#                  return_value=json.loads(tableauExportCrosstabToCsvServer))
+#     mocker.patch("tableauscraper.api.downloadCrossTabData",
+#                  return_value=tableauCrossTabData)
+
+#     ts = TS()
+#     ts.loads(fakeUri)
+#     wb = ts.getWorkbook()
+
+#     data = wb.getCrossTabData(sheetName="[WORKSHEET1]")
+#     assert data.shape[0] == 3
+#     assert data.shape[1] == 1
diff --git a/tests/python/test_api.py b/tests/python/test_api.py
@@ -14,6 +14,9 @@
 from tests.python.test_common import tableauDownloadableSummaryData
 from tests.python.test_common import tableauDownloadableUnderlyingData
 from tests.python.test_common import tableauDownloadableCsvData
+from tests.python.test_common import tableauExportCrosstabServerDialog
+from tests.python.test_common import tableauExportCrosstabToCsvServer
+from tests.python.test_common import tableauCrossTabData
 
 
 def test_getTableauViz(httpserver):
@@ -200,6 +203,48 @@ def test_levelDrill(httpserver, mocker: MockerFixture):
     assert result == vqlCmdResponse
 
 
+def test_export_crosstab_server_dialog(httpserver, mocker: MockerFixture):
+    mocker.patch(
+        "tableauscraper.api.getTableauViz", return_value=tableauVizHtmlResponse
+    )
+    mocker.patch("tableauscraper.api.getTableauData",
+                 return_value=tableauDataResponse)
+    ts = TS()
+    ts.loads(fakeUri)
+    httpserver.serve_content(json.dumps(tableauExportCrosstabServerDialog))
+    ts.host = httpserver.url + "/"
+    result = api.exportCrosstabServerDialog(scraper=ts)
+    assert result == tableauExportCrosstabServerDialog
+
+
+def test_tableau_export_crosstab_to_csv_server(httpserver, mocker: MockerFixture):
+    mocker.patch(
+        "tableauscraper.api.getTableauViz", return_value=tableauVizHtmlResponse
+    )
+    mocker.patch("tableauscraper.api.getTableauData",
+                 return_value=tableauDataResponse)
+    ts = TS()
+    ts.loads(fakeUri)
+    httpserver.serve_content(json.dumps(tableauExportCrosstabToCsvServer))
+    ts.host = httpserver.url + "/"
+    result = api.exportCrosstabToCsvServer(scraper=ts, sheetId="xxx")
+    assert result == tableauExportCrosstabToCsvServer
+
+
+def test_tableau_downloadable_csv_data(httpserver, mocker: MockerFixture):
+    mocker.patch(
+        "tableauscraper.api.getTableauViz", return_value=tableauVizHtmlResponse
+    )
+    mocker.patch("tableauscraper.api.getTableauData",
+                 return_value=tableauDataResponse)
+    ts = TS()
+    ts.loads(fakeUri)
+    httpserver.serve_content(tableauCrossTabData.encode("utf-16"))
+    ts.host = httpserver.url + "/"
+    result = api.downloadCrossTabData(scraper=ts, resultKey="xxx")
+    assert result == tableauCrossTabData
+
+
 def test_delayExcution():
     ts = TS()
     ts.lastActionTime = time.time()

diff --git a/tests/python/test_common.py b/tests/python/test_common.py
@@ -1422,6 +1422,56 @@
 Albany;5;2;11;Capital Region;Vrai;Vrai;18/06/2021;0;Capital Region;42.65337;-73.773834;Albany Medical Center Hospital;Albany Medical Center;;1;Capital District Regional Office;6/17/2021;Rest of State;249;1457
 """
 
+tableauExportCrosstabServerDialog = """
+{
+	"vqlCmdResponse": {
+		"layoutStatus": {
+			"applicationPresModel": {
+				"presentationLayerNotification": [{
+					"presModelHolder": {
+						"genExportCrosstabOptionsDialogPresModel": {
+							"thumbnailSheetPickerItems": [{
+								"thumbnailUri": "",
+								"sheetName": "[WORKSHEET1]",
+								"sheetdocId": "{XXXXX-XXXX-XXXX-XXXX-XXXXXXXXXX}"
+							}]
+						}
+					}
+				}],
+				"dashboardObjectsLibrary": []
+			},
+			"isWorldNew": false,
+			"guid": ""
+		}
+	}
+}
+"""
+
+tableauExportCrosstabToCsvServer = """
+{
+	"vqlCmdResponse": {
+		"layoutStatus": {
+			"applicationPresModel": {
+				"presentationLayerNotification": [{
+					"presModelHolder": {
+						"genExportFilePresModel": {
+							"resultKey": "3224154322"
+						}
+					}
+				}]
+			}
+		}
+	}
+}
+"""
+
+tableauCrossTabData = """"
+Header1 	Header2 	
+1	A 	
+2	B 	
+3	C 	
+"""
+
 tableauStoryPointsInfoNav = {
     'sheetName': '[WORKSHEET1]',
     'worldUpdate': {