Skip to content

Commit

Permalink
Improve decoding fractional epoch timestamps
Browse files Browse the repository at this point in the history
Add `test_timestamp_seconds_float`:

  Publish single reading in JSON format to MQTT broker,
  using a timestamp as Unix Epoch in seconds, as float number.
  Proof that the timestamp is processed and stored correctly.
  • Loading branch information
amotl committed Nov 28, 2021
1 parent 7a4dc1a commit 9294369
Show file tree
Hide file tree
Showing 2 changed files with 49 additions and 5 deletions.
25 changes: 22 additions & 3 deletions kotori/daq/storage/influx.py
Original file line number Diff line number Diff line change
@@ -1,5 +1,7 @@
# -*- coding: utf-8 -*-
# (c) 2015-2021 Andreas Motl <[email protected]>
import math

import requests
from copy import deepcopy
from funcy import project
Expand Down Expand Up @@ -177,7 +179,7 @@ def format_chunk(self, meta, data):
# Decode timestamp.
chunk['time'] = data[time_field]
if is_number(chunk['time']):
chunk['time'] = int(float(chunk['time']))
chunk['time'] = float(chunk['time'])

# Remove timestamp from data payload.
del data[time_field]
Expand Down Expand Up @@ -209,18 +211,35 @@ def format_chunk(self, meta, data):
timestamp = chunk['time'] = parse_timestamp(chunk['time'])

# Heuristically compute timestamp precision
if isinstance(timestamp, int):
if isinstance(timestamp, (int, float)):
if timestamp >= 1e17 or timestamp <= -1e17:
time_precision = 'n'
elif timestamp >= 1e14 or timestamp <= -1e14:
time_precision = 'u'
elif timestamp >= 1e11 or timestamp <= -1e11:
time_precision = 'ms'

# FIXME: Is this a reasonable default?
# TODO: Is this a reasonable default?
else:
time_precision = 's'

# Support fractional epoch timestamps like `1637431069.6585083`.
if isinstance(timestamp, float):
fractional, whole = math.modf(timestamp)
fracdigits = len(str(fractional)) - 2
if fracdigits > 0:
if fracdigits <= 3:
exponent = 3
time_precision = "ms"
elif fracdigits <= 6:
exponent = 6
time_precision = "u"
else:
exponent = 9
time_precision = "n"
timestamp = timestamp * (10 ** exponent)

chunk['time'] = int(timestamp)
chunk['time_precision'] = time_precision

"""
Expand Down
29 changes: 27 additions & 2 deletions test/test_daq_timestamp.py
Original file line number Diff line number Diff line change
Expand Up @@ -35,10 +35,10 @@ def test_timestamp_rfc3339(machinery, create_influxdb, reset_influxdb):


@pytest_twisted.inlineCallbacks
def test_timestamp_seconds(machinery, create_influxdb, reset_influxdb):
def test_timestamp_seconds_integer(machinery, create_influxdb, reset_influxdb):
"""
Publish single reading in JSON format to MQTT broker,
using a timestamp as Unix Epoch in seconds.
using a timestamp as Unix Epoch in seconds, as integer number.
Proof that the timestamp is processed and stored correctly.
"""

Expand All @@ -59,6 +59,31 @@ def test_timestamp_seconds(machinery, create_influxdb, reset_influxdb):
yield record


@pytest_twisted.inlineCallbacks
def test_timestamp_seconds_float(machinery, create_influxdb, reset_influxdb):
"""
Publish single reading in JSON format to MQTT broker,
using a timestamp as Unix Epoch in seconds, as float number.
Proof that the timestamp is processed and stored correctly.
"""

# Submit a single measurement, with timestamp.
data = {
'temperature': 42.84,
'humidity': 83.1,
'timestamp': 1637431069.6585083
}
yield mqtt_json_sensor(settings.mqtt_topic_json, data)

# Wait for some time to process the message.
yield sleep(PROCESS_DELAY_MQTT)

# Proof that data arrived in InfluxDB.
record = influx_sensors.get_first_record()
assert record == {u'time': u'2021-11-20T17:57:49.658508Z', u'humidity': 83.1, u'temperature': 42.84}
yield record


@pytest_twisted.inlineCallbacks
def test_timestamp_milliseconds(machinery, create_influxdb, reset_influxdb):
"""
Expand Down

0 comments on commit 9294369

Please sign in to comment.