Workaround pandas bug in datetimes with time zones (#3910)

A bug in to_dict(orient="records") in pandas/core/frame.py prevents
datetimes with time zones to be worked with. This works around the
issue in superset by re-implementing the logic of pandas in the
correct way. Until pandas fixes the issue this code should stay.

pandas-dev/pandas#18372

This closes #1929

superset/dataframe.py

@@ -14,6 +14,7 @@
 
 import numpy as np
 import pandas as pd
+from pandas.core.common import _maybe_box_datetimelike
 from pandas.core.dtypes.dtypes import ExtensionDtype
 from past.builtins import basestring
 
@@ -48,7 +49,10 @@ def size(self):
 
     @property
     def data(self):
-        return self.__df.to_dict(orient='records')
+        # work around for https://github.com/pandas-dev/pandas/issues/18372
+        return [dict((k, _maybe_box_datetimelike(v))
+                     for k, v in zip(self.__df.columns, np.atleast_1d(row)))
+                for row in self.__df.values]
 
     @classmethod
     def db_type(cls, dtype):

tests/core_tests.py

@@ -5,6 +5,7 @@
 from __future__ import unicode_literals
 
 import csv
+import datetime
 import doctest
 import io
 import json
@@ -13,9 +14,11 @@
 import unittest
 
 from flask import escape
+import pandas as pd
+import psycopg2
 import sqlalchemy as sqla
 
-from superset import appbuilder, db, jinja_context, sm, sql_lab, utils
+from superset import appbuilder, dataframe, db, jinja_context, sm, sql_lab, utils
 from superset.connectors.sqla.models import SqlaTable
 from superset.models import core as models
 from superset.models.sql_lab import Query
@@ -786,6 +789,22 @@ def test_viz_get_fillna_for_columns(self):
             {'name': ' NULL', 'sum__num': 0},
         )
 
+    def test_dataframe_timezone(self):
+        tz = psycopg2.tz.FixedOffsetTimezone(offset=60, name=None)
+        data = [(datetime.datetime(2017, 11, 18, 21, 53, 0, 219225, tzinfo=tz),),
+                (datetime.datetime(2017, 11, 18, 22, 6, 30, 61810, tzinfo=tz,),)]
+        df = dataframe.SupersetDataFrame(pd.DataFrame(data=list(data),
+                                                      columns=['data', ]))
+        data = df.data
+        self.assertDictEqual(
+            data[0],
+            {'data': pd.Timestamp('2017-11-18 21:53:00.219225+0100', tz=tz), },
+        )
+        self.assertDictEqual(
+            data[1],
+            {'data': pd.Timestamp('2017-11-18 22:06:30.061810+0100', tz=tz), },
+        )
+
 
 if __name__ == '__main__':
     unittest.main()