From f5da1482ccf282ec45295abb4854af1b84f66c5f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=BCrgen=20Hansmann?= Date: Tue, 7 Jan 2025 15:59:01 +0100 Subject: [PATCH 1/6] PB-1319: made format_time() more robust Currently during several steps of timestamp handling and conversions, the displayed departure times in the viewer are 1 hour off. The new format_time() should be more robust and also corresponsing tests are added. --- chsdi/lib/opentransapi/opentransapi.py | 34 +++++++++++--------------- tests/integration/test_opentransapi.py | 6 +++++ 2 files changed, 20 insertions(+), 20 deletions(-) diff --git a/chsdi/lib/opentransapi/opentransapi.py b/chsdi/lib/opentransapi/opentransapi.py index c18be6549d..ef81328ba1 100644 --- a/chsdi/lib/opentransapi/opentransapi.py +++ b/chsdi/lib/opentransapi/opentransapi.py @@ -4,29 +4,23 @@ import xml.etree.ElementTree as et from pytz import timezone from datetime import datetime -from dateutil import tz +from dateutil.parser import isoparse import re -def format_time(str_date_time, fmt="%Y-%m-%dT%H:%M:%SZ"): - from_zone = tz.tzutc() - to_zone = tz.gettz('Europe/Zurich') +def format_time(str_date_time): + # Though the documentation of the OJP 2.0 API is not too verbose on this point (see: + # https://opentransportdata.swiss/de/cookbook/ojpstopeventrequest/), it seems, the + # timestamps are always handeled in some form of ISO 8601 datetime format. + # Using isoparse() should be able to handle all the needed cases, e.g. + # - "Z" as timezone designator + # - timezone offsets, e.g. "+01:00" + # - sometimes the returned timestamps have an unexpected number of + # fractional seconds, e.g. 7 instead of 6, should be handled, too + date_time = isoparse(str_date_time) - try: - date_time = datetime.strptime(str_date_time, fmt) - except ValueError: - # sometimes the timestamp of the OJP 2.0 API's response has 7 digits for the - # milliseconds. 6 are expected and only 6 can be handled by Python. - # Hence we need to safely truncate everything between the last . and - # the +01:00 part of the timestamp, e.g.: - # 2024-11-01T15:39:45.5348804+01:00 - # Use regex to capture and truncate everything between the last '.' and - # the first '+' to 6 digits - truncated_date_time = re.sub(r'(\.\d{6})\d*(?=\+)', r'\1', str_date_time) - date_time = datetime.strptime(truncated_date_time, '%Y-%m-%dT%H:%M:%S.%f%z') - date_time_utc = date_time.replace(tzinfo=from_zone) - date_time_zurich = date_time_utc.astimezone(to_zone) - return date_time_zurich.strftime('%d/%m/%Y %H:%M') + # Return time in local time, as needed. + return date_time.strftime('%d/%m/%Y %H:%M') class OpenTrans: @@ -80,7 +74,7 @@ def xml_to_array(self, xml_data): results.append({ 'id': el_id, 'label': el_service_name, - 'currentDate': format_time(el_current_date, fmt="%Y-%m-%dT%H:%M:%S.%f%z"), + 'currentDate': format_time(el_current_date), 'departureDate': format_time(el_departure_date), 'estimatedDate': self._convert_estimated_date(el.find('.//ojp:ServiceDeparture/ojp:EstimatedTime', ns)), 'destinationName': el_destination_name, diff --git a/tests/integration/test_opentransapi.py b/tests/integration/test_opentransapi.py index e73d62f459..23479be70c 100644 --- a/tests/integration/test_opentransapi.py +++ b/tests/integration/test_opentransapi.py @@ -54,6 +54,12 @@ def test_stationboard(self, mock_requests): self.assertEqual(results[0]["destinationName"], "Hogwarts") self.assertEqual(results[0]["destinationId"], "ch:1:sloid:91178::3") + # assert, that several timestamp formats are correctly handled and transformed into the + # correct local time + self.assertEqual(format_time("2024-11-19T08:52:00Z"), "19/11/2024 08:52") + self.assertEqual(format_time("2024-11-19T08:52:00.1234567"), "19/11/2024 08:52") + self.assertEqual(format_time("2024-11-19T08:52:00+01:00"), "19/11/2024 08:52") + @requests_mock.Mocker() def test_stationboard_nonexisting_station(self, mock_requests): now = datetime.now(timezone('Europe/Zurich')).isoformat(timespec="microseconds") From a8bd607ab9458fe71e39feeb626670b5aa108a1b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=BCrgen=20Hansmann?= Date: Tue, 7 Jan 2025 16:03:44 +0100 Subject: [PATCH 2/6] PB-1319: small typo in comment --- chsdi/lib/opentransapi/opentransapi.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/chsdi/lib/opentransapi/opentransapi.py b/chsdi/lib/opentransapi/opentransapi.py index ef81328ba1..b953c83108 100644 --- a/chsdi/lib/opentransapi/opentransapi.py +++ b/chsdi/lib/opentransapi/opentransapi.py @@ -11,7 +11,7 @@ def format_time(str_date_time): # Though the documentation of the OJP 2.0 API is not too verbose on this point (see: # https://opentransportdata.swiss/de/cookbook/ojpstopeventrequest/), it seems, the - # timestamps are always handeled in some form of ISO 8601 datetime format. + # timestamps are always returned in some form of ISO 8601 datetime format. # Using isoparse() should be able to handle all the needed cases, e.g. # - "Z" as timezone designator # - timezone offsets, e.g. "+01:00" From 0c38e38f91f721b13397e6f666762e663aba12e8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=BCrgen=20Hansmann?= Date: Tue, 7 Jan 2025 19:52:17 +0100 Subject: [PATCH 3/6] PB-1319: correct format for timestamps used in OJPStopEventRequest Accodring to https://opentransportdata.swiss/de/cookbook/ojpstopeventrequest/ the timestamps used in OJPStopEventRequests should be in Zulu time, to prevent their code from trying to interpret the given times as local times. This is now implemented correctly and a few test for timestamp conversions are added. --- chsdi/lib/opentransapi/opentransapi.py | 19 ++++++++++++++++--- tests/integration/test_opentransapi.py | 6 ++++-- 2 files changed, 20 insertions(+), 5 deletions(-) diff --git a/chsdi/lib/opentransapi/opentransapi.py b/chsdi/lib/opentransapi/opentransapi.py index b953c83108..1443ba7d5d 100644 --- a/chsdi/lib/opentransapi/opentransapi.py +++ b/chsdi/lib/opentransapi/opentransapi.py @@ -3,6 +3,7 @@ import requests import xml.etree.ElementTree as et from pytz import timezone +from pytz import UTC from datetime import datetime from dateutil.parser import isoparse import re @@ -19,8 +20,12 @@ def format_time(str_date_time): # fractional seconds, e.g. 7 instead of 6, should be handled, too date_time = isoparse(str_date_time) + # Convert to local timezone explicitly + local_tz = timezone('Europe/Zurich') # Replace with your local timezone + local_date_time = date_time.astimezone(local_tz) + # Return time in local time, as needed. - return date_time.strftime('%d/%m/%Y %H:%M') + return local_date_time.strftime('%d/%m/%Y %H:%M') class OpenTrans: @@ -32,9 +37,13 @@ def __init__(self, open_trans_api_key, open_trans_url): def get_departures(self, station_id, number_results=5, request_dt_time=False): if not request_dt_time: - request_dt_time = datetime.now(timezone('Europe/Zurich')).strftime('%Y-%m-%dT%H:%M:%S') + # Note: according to https://opentransportdata.swiss/de/cookbook/ojpstopeventrequest/ + # the timestamps used in the OJPStopEventRequest should preferably be in Zulu time + # and MUST include the seconds, in order to prevent their code from trying to interpret + # the given times as a form of local time! + request_dt_time = datetime.now(UTC).strftime('%Y-%m-%dT%H:%M:%SZ') self.station_id = station_id - api_response_xml = self.send_post(station_id, request_dt_time, number_results) # request_dt_time in format 2017-12-11T14:26:18Z + api_response_xml = self.send_post(station_id, request_dt_time, number_results) # zulu times! results = self.xml_to_array(api_response_xml) return results @@ -86,6 +95,10 @@ def create_ojp_payload(self, station_id, request_dt_time, number_results=5): # ATTENTION: The value "swisstopo_Abfahrtsmonitor" for the RequestorRef # in the payload below is suggested by the OJP product owner. # Hence it MUST NOT be changed! + + # Further ATTENTION: + # Timestamps used for RequestTimestamp and DepArrTime MUST be in Zulu time, see: + # https://opentransportdata.swiss/de/cookbook/ojpstopeventrequest/ payload = f""" diff --git a/tests/integration/test_opentransapi.py b/tests/integration/test_opentransapi.py index 23479be70c..a233aafc0e 100644 --- a/tests/integration/test_opentransapi.py +++ b/tests/integration/test_opentransapi.py @@ -54,10 +54,12 @@ def test_stationboard(self, mock_requests): self.assertEqual(results[0]["destinationName"], "Hogwarts") self.assertEqual(results[0]["destinationId"], "ch:1:sloid:91178::3") + def test_format_time(self): # assert, that several timestamp formats are correctly handled and transformed into the # correct local time - self.assertEqual(format_time("2024-11-19T08:52:00Z"), "19/11/2024 08:52") - self.assertEqual(format_time("2024-11-19T08:52:00.1234567"), "19/11/2024 08:52") + self.assertEqual(format_time("2024-11-19T08:52:00Z"), "19/11/2024 09:52") + self.assertEqual(format_time("2024-11-19T09:52:00.123456789"), "19/11/2024 09:52") + self.assertEqual(format_time("2024-11-19T08:52:00.123456789Z"), "19/11/2024 09:52") self.assertEqual(format_time("2024-11-19T08:52:00+01:00"), "19/11/2024 08:52") @requests_mock.Mocker() From 7fabdc4693f99ff47a9bebce06971523c54f303f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=BCrgen=20Hansmann?= Date: Wed, 8 Jan 2025 09:20:37 +0100 Subject: [PATCH 4/6] PB-1319: using the term UTC instead of Zulu --- chsdi/lib/opentransapi/opentransapi.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/chsdi/lib/opentransapi/opentransapi.py b/chsdi/lib/opentransapi/opentransapi.py index 1443ba7d5d..24c396b67d 100644 --- a/chsdi/lib/opentransapi/opentransapi.py +++ b/chsdi/lib/opentransapi/opentransapi.py @@ -38,12 +38,12 @@ def __init__(self, open_trans_api_key, open_trans_url): def get_departures(self, station_id, number_results=5, request_dt_time=False): if not request_dt_time: # Note: according to https://opentransportdata.swiss/de/cookbook/ojpstopeventrequest/ - # the timestamps used in the OJPStopEventRequest should preferably be in Zulu time + # the timestamps used in the OJPStopEventRequest should preferably be in UTC # and MUST include the seconds, in order to prevent their code from trying to interpret # the given times as a form of local time! request_dt_time = datetime.now(UTC).strftime('%Y-%m-%dT%H:%M:%SZ') self.station_id = station_id - api_response_xml = self.send_post(station_id, request_dt_time, number_results) # zulu times! + api_response_xml = self.send_post(station_id, request_dt_time, number_results) # UTC! results = self.xml_to_array(api_response_xml) return results @@ -97,7 +97,7 @@ def create_ojp_payload(self, station_id, request_dt_time, number_results=5): # Hence it MUST NOT be changed! # Further ATTENTION: - # Timestamps used for RequestTimestamp and DepArrTime MUST be in Zulu time, see: + # Timestamps used for RequestTimestamp and DepArrTime MUST be in UTC, see: # https://opentransportdata.swiss/de/cookbook/ojpstopeventrequest/ payload = f""" From 81d15a2932a37bfd5a6838e9f77dff66f50b9973 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=BCrgen=20Hansmann?= Date: Wed, 8 Jan 2025 11:03:06 +0100 Subject: [PATCH 5/6] PB-1319: treatment of timezone-naive timestamps The format_time method now interprets timezone-naive timestamps as local times. Timezone-naive timestamps are not expected in the response, but if so, we enforce treatment as local times. --- chsdi/lib/opentransapi/opentransapi.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/chsdi/lib/opentransapi/opentransapi.py b/chsdi/lib/opentransapi/opentransapi.py index 24c396b67d..d77c9608e9 100644 --- a/chsdi/lib/opentransapi/opentransapi.py +++ b/chsdi/lib/opentransapi/opentransapi.py @@ -18,12 +18,16 @@ def format_time(str_date_time): # - timezone offsets, e.g. "+01:00" # - sometimes the returned timestamps have an unexpected number of # fractional seconds, e.g. 7 instead of 6, should be handled, too + local_tz = timezone('Europe/Zurich') date_time = isoparse(str_date_time) + # If for some reason we receive a timezone-naive timestamp, we assume it is a + # local time + if date_time.tzinfo is None: + date_time = date_time.replace(tzinfo=local_tz) + # Convert to local timezone explicitly - local_tz = timezone('Europe/Zurich') # Replace with your local timezone local_date_time = date_time.astimezone(local_tz) - # Return time in local time, as needed. return local_date_time.strftime('%d/%m/%Y %H:%M') From 1720a86af5bb11a6fec7e58d690f5848daea13ee Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=BCrgen=20Hansmann?= Date: Wed, 8 Jan 2025 17:17:40 +0100 Subject: [PATCH 6/6] PB-1319: removed obsolete if clause --- chsdi/lib/opentransapi/opentransapi.py | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/chsdi/lib/opentransapi/opentransapi.py b/chsdi/lib/opentransapi/opentransapi.py index d77c9608e9..7c3de775e1 100644 --- a/chsdi/lib/opentransapi/opentransapi.py +++ b/chsdi/lib/opentransapi/opentransapi.py @@ -39,13 +39,12 @@ def __init__(self, open_trans_api_key, open_trans_url): self.url = open_trans_url # URL of API self.station_id = None - def get_departures(self, station_id, number_results=5, request_dt_time=False): - if not request_dt_time: - # Note: according to https://opentransportdata.swiss/de/cookbook/ojpstopeventrequest/ - # the timestamps used in the OJPStopEventRequest should preferably be in UTC - # and MUST include the seconds, in order to prevent their code from trying to interpret - # the given times as a form of local time! - request_dt_time = datetime.now(UTC).strftime('%Y-%m-%dT%H:%M:%SZ') + def get_departures(self, station_id, number_results=5): + # Note: according to https://opentransportdata.swiss/de/cookbook/ojpstopeventrequest/ + # the timestamps used in the OJPStopEventRequest should preferably be in UTC + # and MUST include the seconds, in order to prevent their code from trying to interpret + # the given times as a form of local time! + request_dt_time = datetime.now(UTC).strftime('%Y-%m-%dT%H:%M:%SZ') self.station_id = station_id api_response_xml = self.send_post(station_id, request_dt_time, number_results) # UTC! results = self.xml_to_array(api_response_xml)