msgpack: support datetime extended type

Tarantool supports datetime type since version 2.10.0 [1]. This patch
introduced the support of Tarantool datetime type in msgpack decoders
and encoders.

Tarantool datetime objects are decoded to `tarantool.Datetime` type.
`tarantool.Datetime` and `pandas.Timestamp` may be encoded to Tarantool
datetime objects.

`tarantool.Datetime` is basically a `pandas.Timestamp` wrapper. You can
create `tarantool.Datetime` objects
- from `pandas.Timestamp` object,
- by using the same API as in `pandas.Timestamp()` [2],
- from another `tarantool.Datetime` object.

To work with datetime data as a `pandas.Timestamp`, convert
`tarantool.Datetime` object to a `pandas.Timestamp` with
`to_pd_timestamp()` method call. You can use this `pandas.Timestamp`
object to build a `tarantool.Datetime` object before sending data to
Tarantool.

To work with data as `numpy.datetime64` or `datetime.datetime`, convert
to a `pandas.Timestamp` and then use `to_datetime64()` or
`to_datetime()` converter.

pandas.Timestamp was chosen to store data because it could be used
to store both nanoseconds and timezone information. In-build Python
datetime.datetime supports microseconds at most, numpy.datetime64 do not
support timezones.

There are two reasons to use custom type instead of plain
pandas.Timestamp:
- tzindex may be lost on conversion to pandas.Timestamp
- Tarantool datetime interval type is planned to be stored in custom
  type tarantool.Interval and we'll need a way to support arithmetic
  between datetime and interval.

This patch does not yet introduce the support of timezones in datetime.

1. tarantool/tarantool#5941
2. https://pandas.pydata.org/docs/reference/api/pandas.Timestamp.html

Part of #204

CHANGELOG.md

@@ -9,6 +9,27 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 ### Added
 - Decimal type support (#203).
 - UUID type support (#202).
+- Datetime type support and tarantool.Datetime type (#204).
+
+  Tarantool datetime objects are decoded to `tarantool.Datetime`
+  type. `tarantool.Datetime` and `pandas.Timestamp` may be encoded
+  to Tarantool datetime objects.
+
+  `tarantool.Datetime` is basically a `pandas.Timestamp` wrapper.
+  You can create `tarantool.Datetime` objects
+  - from `pandas.Timestamp` object,
+  - by using the same API as in `pandas.Timestamp()`,
+  - from another `tarantool.Datetime` object.
+
+  To work with datetime data as a `pandas.Timestamp`, convert
+  `tarantool.Datetime` object to a `pandas.Timestamp` with
+  `to_pd_timestamp()` method call. You can use this
+  `pandas.Timestamp` object to build a `tarantool.Datetime`
+  object before sending data to Tarantool.
+
+  To work with data as `numpy.datetime64` or `datetime.datetime`,
+  convert to a `pandas.Timestamp` and then use `to_datetime64()`
+  or `to_datetime()` converter.
 
 ### Changed
 - Bump msgpack requirement to 1.0.4 (PR #223).

requirements.txt

@@ -1 +1,2 @@
 msgpack>=1.0.4
+pandas

tarantool/__init__.py

@@ -32,6 +32,10 @@
     ENCODING_DEFAULT,
 )
 
+from tarantool.msgpack_ext.types.datetime import (
+    Datetime,
+)
+
 __version__ = "0.9.0"
 
 
@@ -91,7 +95,7 @@ def connectmesh(addrs=({'host': 'localhost', 'port': 3301},), user=None,
 
 __all__ = ['connect', 'Connection', 'connectmesh', 'MeshConnection', 'Schema',
            'Error', 'DatabaseError', 'NetworkError', 'NetworkWarning',
-           'SchemaError', 'dbapi']
+           'SchemaError', 'dbapi', 'Datetime']
 
 # ConnectionPool is supported only for Python 3.7 or newer.
 if sys.version_info.major >= 3 and sys.version_info.minor >= 7:

tarantool/error.py

@@ -119,6 +119,16 @@ class MsgpackWarning(UserWarning):
     Warning with encoding or decoding of MP_EXT types
     '''
 
+class ExtTypeError(ValueError):
+    '''
+    Error related to tarantool.Datetime type
+    '''
+
+class ExtTypeWarning(UserWarning):
+    '''
+    Warning related to tarantool.Datetime type
+    '''
+
 __all__ = ("Warning", "Error", "InterfaceError", "DatabaseError", "DataError",
            "OperationalError", "IntegrityError", "InternalError",
            "ProgrammingError", "NotSupportedError", "MsgpackError",

tarantool/msgpack_ext/datetime.py

@@ -0,0 +1,12 @@
+from tarantool.msgpack_ext.types.datetime import Datetime
+
+EXT_ID = 4
+
+def encode(obj):
+    return obj.msgpack_encode()
+
+def encode_pd_timestamp(obj):
+    return Datetime(obj).msgpack_encode()
+
+def decode(data):
+    return Datetime(data)

tarantool/msgpack_ext/packer.py

@@ -1,17 +1,39 @@
 from decimal import Decimal
 from uuid import UUID
 from msgpack import ExtType
+import pandas
+
+from tarantool.msgpack_ext.types.datetime import Datetime
 
 import tarantool.msgpack_ext.decimal as ext_decimal
 import tarantool.msgpack_ext.uuid as ext_uuid
+import tarantool.msgpack_ext.datetime as ext_datetime
 
 encoders = [
-    {'type': Decimal, 'ext': ext_decimal},
-    {'type': UUID,    'ext': ext_uuid   },
+    {
+        'type': Decimal,
+        'ext_id': ext_decimal.EXT_ID,
+        'encoder': ext_decimal.encode,
+    },
+    {
+        'type': UUID,
+        'ext_id': ext_uuid.EXT_ID,
+        'encoder': ext_uuid.encode,
+    },
+    {
+        'type': Datetime,
+        'ext_id': ext_datetime.EXT_ID,
+        'encoder': ext_datetime.encode,
+    },
+    {
+        'type': pandas.Timestamp,
+        'ext_id': ext_datetime.EXT_ID,
+        'encoder': ext_datetime.encode_pd_timestamp,
+    },
 ]
 
 def default(obj):
     for encoder in encoders:
         if isinstance(obj, encoder['type']):
-            return ExtType(encoder['ext'].EXT_ID, encoder['ext'].encode(obj))
+            return ExtType(encoder['ext_id'], encoder['encoder'](obj))
     raise TypeError("Unknown type: %r" % (obj,))

tarantool/msgpack_ext/types/datetime.py

@@ -0,0 +1,121 @@
+from copy import deepcopy
+
+import pandas
+
+# https://www.tarantool.io/en/doc/latest/dev_guide/internals/msgpack_extensions/#the-datetime-type
+#
+# The datetime MessagePack representation looks like this:
+# +---------+----------------+==========+-----------------+
+# | MP_EXT  | MP_DATETIME    | seconds  | nsec; tzoffset; |
+# | = d7/d8 | = 4            |          | tzindex;        |
+# +---------+----------------+==========+-----------------+
+# MessagePack data contains:
+#
+# * Seconds (8 bytes) as an unencoded 64-bit signed integer stored in the
+#   little-endian order.
+# * The optional fields (8 bytes), if any of them have a non-zero value.
+#   The fields include nsec (4 bytes), tzoffset (2 bytes), and
+#   tzindex (2 bytes) packed in the little-endian order.
+#
+# seconds is seconds since Epoch, where the epoch is the point where the time
+# starts, and is platform dependent. For Unix, the epoch is January 1,
+# 1970, 00:00:00 (UTC). Tarantool uses a double type, see a structure
+# definition in src/lib/core/datetime.h and reasons in
+# https://github.com/tarantool/tarantool/wiki/Datetime-internals#intervals-in-c
+#
+# nsec is nanoseconds, fractional part of seconds. Tarantool uses int32_t, see
+# a definition in src/lib/core/datetime.h.
+#
+# tzoffset is timezone offset in minutes from UTC. Tarantool uses a int16_t type,
+# see a structure definition in src/lib/core/datetime.h.
+#
+# tzindex is Olson timezone id. Tarantool uses a int16_t type, see a structure
+# definition in src/lib/core/datetime.h. If both tzoffset and tzindex are
+# specified, tzindex has the preference and the tzoffset value is ignored.
+
+SECONDS_SIZE_BYTES  = 8
+NSEC_SIZE_BYTES     = 4
+TZOFFSET_SIZE_BYTES = 2
+TZINDEX_SIZE_BYTES  = 2
+
+BYTEORDER = 'little'
+
+NSEC_IN_SEC = 1000000000
+
+
+def get_bytes_as_int(data, cursor, size):
+    part = data[cursor:cursor + size]
+    return int.from_bytes(part, BYTEORDER, signed=True), cursor + size
+
+def get_int_as_bytes(data, size):
+    return data.to_bytes(size, byteorder=BYTEORDER, signed=True)
+
+def msgpack_decode(data):
+    cursor = 0
+    seconds, cursor = get_bytes_as_int(data, cursor, SECONDS_SIZE_BYTES)
+
+    if len(data) > SECONDS_SIZE_BYTES:
+        nsec, cursor     = get_bytes_as_int(data, cursor, NSEC_SIZE_BYTES)
+        tzoffset, cursor = get_bytes_as_int(data, cursor, TZOFFSET_SIZE_BYTES)
+        tzindex, cursor  = get_bytes_as_int(data, cursor, TZINDEX_SIZE_BYTES)
+    else:
+        nsec = 0
+        tzoffset = 0
+        tzindex = 0
+
+    if (tzoffset != 0) or (tzindex != 0):
+        raise NotImplementedError
+
+    total_nsec = seconds * NSEC_IN_SEC + nsec
+
+    timestamp = pandas.to_datetime(total_nsec, unit='ns')
+    return timestamp, tzoffset, tzindex
+
+class Datetime():
+    def __init__(self, *args, **kwargs):
+        if len(args) > 0:
+            data = args[0]
+            if isinstance(data, bytes):
+                timestamp, tzoffset, tzindex = msgpack_decode(data)
+            elif isinstance(data, pandas.Timestamp):
+                timestamp = deepcopy(data)
+            elif isinstance(data, Datetime):
+                timestamp = deepcopy(data._timestamp)
+        else:
+            timestamp = pandas.Timestamp(*args, **kwargs)
+
+        self._timestamp = timestamp
+
+    def __eq__(self, other):
+        if isinstance(other, Datetime):
+            return self._timestamp == other._timestamp
+        elif isinstance(other, pandas.Timestamp):
+            return self._timestamp == other
+        else:
+            return False
+
+    def to_pd_timestamp(self):
+        return deepcopy(self._timestamp)
+
+    def __str__(self):
+        return f'tarantool.Datetime(timestamp={self._timestamp})'
+
+    def __repr__(self):
+        return f'tarantool.Datetime(timestamp={self._timestamp})'
+
+    def msgpack_encode(self):
+        ts_value = self._timestamp.value
+
+        seconds = ts_value // NSEC_IN_SEC
+        nsec = ts_value % NSEC_IN_SEC
+        tzoffset = 0
+        tzindex = 0
+
+        buf = get_int_as_bytes(seconds, SECONDS_SIZE_BYTES)
+
+        if (nsec != 0) or (tzoffset != 0) or (tzindex != 0):
+            buf = buf + get_int_as_bytes(nsec, NSEC_SIZE_BYTES)
+            buf = buf + get_int_as_bytes(tzoffset, TZOFFSET_SIZE_BYTES)
+            buf = buf + get_int_as_bytes(tzindex, TZINDEX_SIZE_BYTES)
+
+        return buf

tarantool/msgpack_ext/unpacker.py

@@ -1,9 +1,11 @@
 import tarantool.msgpack_ext.decimal as ext_decimal
 import tarantool.msgpack_ext.uuid as ext_uuid
+import tarantool.msgpack_ext.datetime as ext_datetime
 
 decoders = {
-    ext_decimal.EXT_ID: ext_decimal.decode,
-    ext_uuid.EXT_ID   : ext_uuid.decode   ,
+    ext_decimal.EXT_ID : ext_decimal.decode ,
+    ext_uuid.EXT_ID    : ext_uuid.decode    ,
+    ext_datetime.EXT_ID: ext_datetime.decode,
 }
 
 def ext_hook(code, data):

test/suites/__init__.py

@@ -17,13 +17,14 @@
 from .test_ssl import TestSuite_Ssl
 from .test_decimal import TestSuite_Decimal
 from .test_uuid import TestSuite_UUID
+from .test_datetime import TestSuite_Datetime
 
 test_cases = (TestSuite_Schema_UnicodeConnection,
               TestSuite_Schema_BinaryConnection,
               TestSuite_Request, TestSuite_Protocol, TestSuite_Reconnect,
               TestSuite_Mesh, TestSuite_Execute, TestSuite_DBAPI,
               TestSuite_Encoding, TestSuite_Pool, TestSuite_Ssl,
-              TestSuite_Decimal, TestSuite_UUID)
+              TestSuite_Decimal, TestSuite_UUID, TestSuite_Datetime)
 
 def load_tests(loader, tests, pattern):
     suite = unittest.TestSuite()

test/suites/lib/skip.py

@@ -154,3 +154,14 @@ def skip_or_run_UUID_test(func):
 
     return skip_or_run_test_tarantool(func, '2.4.1',
                                       'does not support UUID type')
+
+def skip_or_run_datetime_test(func):
+    """Decorator to skip or run datetime-related tests depending on
+    the tarantool version.
+
+    Tarantool supports datetime type only since 2.10.0 version.
+    See https://github.com/tarantool/tarantool/issues/5941
+    """
+
+    return skip_or_run_test_pcall_require(func, 'datetime',
+                                      'does not support datetime type')