Merge pull request #4487 from geoadmin/feat_PB-974_migration_from_tri…

…as_to_ojp_for_transport_stop_requests

PB-974: Migration from trias to ojp for transport stop requests

.env.ci

@@ -23,3 +23,5 @@ SQLALCHEMY_ISOLATION_LEVEL=AUTOCOMMIT
 SQLALCHEMY_POOL_PRE_PING=True
 DISABLE_POLYFILL=false
 POLYFILL_URL="//cdnjs.cloudflare.com/polyfill/v3/polyfill.min.js?version=4.8.0&features=fetch,requestAnimationFrame,Element.prototype.classList,URL"
+OPENTRANS_URL=https://dummy.com/api
+OPENTRANS_API_KEY=dummy-key

.env.default

@@ -30,3 +30,5 @@ WSGI_THREADS=2
 LOGS_DIR=./logs
 DISABLE_POLYFILL=false
 POLYFILL_URL="//cdnjs.cloudflare.com/polyfill/v3/polyfill.min.js?version=4.8.0&features=fetch,requestAnimationFrame,Element.prototype.classList,URL"
+OPENTRANS_URL=https://dummy.com/api
+OPENTRANS_API_KEY=dummy-key

Pipfile

@@ -13,7 +13,8 @@ geojson = "~=2.5"
 lark_parser = "~=0.7.8"
 networkx = "~=2.8"
 papyrus = "~=2.4"
-SQLAlchemy = "~=1.4"
+SQLAlchemy = "~=1.4.48"
+GeoAlchemy2 = "~=0.13.0"
 Pillow = "~=9.2"
 polib = "~=1.1"
 psycopg2-binary = "~=2.9"
@@ -51,6 +52,7 @@ sphinx-rtd-theme = "~=1.0"
 nose2 = "*"
 coverage = "~=4.5"
 numpy = "*"
+requests-mock = "~=1.12.1"
 
 [requires]
 python_version = "3.9"

chsdi/config/base.ini.in

@@ -60,6 +60,7 @@ vector_bucket = ${VECTOR_BUCKET}
 datageoadminhost = ${DATAGEOADMINHOST}
 hist_maps_data_host = ${HIST_MAPS_DATA_HOST}
 opentrans_api_key = ${OPENTRANS_API_KEY}
+opentrans_url = ${OPENTRANS_URL}
 empty_geotables = ch.bav.sachplan-infrastruktur-schifffahrt_anhoerung,ch.bfe.sachplan-uebertragungsleitungen_anhoerung,ch.blw.emapis-bewaesserung,ch.blw.emapis-elektrizitaetsversorgung,ch.blw.emapis-milchleitung,ch.blw.emapis-seilbahnen,ch.vbs.sachplan-infrastruktur-militaer_anhoerung,ch.sem.sachplan-asyl_anhoerung,ch.astra.sachplan-infrastruktur-strasse_anhoerung,ch.bav.sachplan-infrastruktur-schiene_anhorung
 max_featureids_request = 20
 default_cache_control = ${CACHE_CONTROL}

chsdi/lib/opentransapi/opentransapi.py

@@ -5,13 +5,36 @@
 from pytz import timezone
 from datetime import datetime
 from dateutil import tz
+import re
+
+
+def format_time(str_date_time, fmt="%Y-%m-%dT%H:%M:%SZ"):
+    from_zone = tz.tzutc()
+    to_zone = tz.gettz('Europe/Zurich')
+
+    try:
+        date_time = datetime.strptime(str_date_time, fmt)
+    except ValueError:
+        # sometimes the timestamp of the OJP 2.0 API's response has 7 digits for the
+        # milliseconds. 6 are expected and only 6 can be handled by Python.
+        # Hence we need to safely truncate everything between the last . and
+        # the +01:00 part of the timestamp, e.g.:
+        # 2024-11-01T15:39:45.5348804+01:00
+        # Use regex to capture and truncate everything between the last '.' and
+        # the first '+' to 6 digits
+        truncated_date_time = re.sub(r'(\.\d{6})\d*(?=\+)', r'\1', str_date_time)
+        date_time = datetime.strptime(truncated_date_time, '%Y-%m-%dT%H:%M:%S.%f%z')
+    date_time_utc = date_time.replace(tzinfo=from_zone)
+    date_time_zurich = date_time_utc.astimezone(to_zone)
+    return date_time_zurich.strftime('%d/%m/%Y %H:%M')
 
 
 class OpenTrans:
 
-    def __init__(self, open_trans_api_key):
+    def __init__(self, open_trans_api_key, open_trans_url):
         self.open_trans_api_key = open_trans_api_key  # Get API key from config .ini
-        self.url = 'https://api.opentransportdata.swiss/trias2020'  # URL of API
+        self.url = open_trans_url  # URL of API
+        self.station_id = None
 
     def get_departures(self, station_id, number_results=5, request_dt_time=False):
         if not request_dt_time:
@@ -21,95 +44,102 @@ def get_departures(self, station_id, number_results=5, request_dt_time=False):
         results = self.xml_to_array(api_response_xml)
         return results
 
-    def _format_time(self, str_date_time):
-        from_zone = tz.tzutc()
-        to_zone = tz.gettz('Europe/Zurich')
-        date_time = datetime.strptime(str_date_time, '%Y-%m-%dT%H:%M:%SZ')
-        date_time_utc = date_time.replace(tzinfo=from_zone)
-        date_time_zurich = date_time_utc.astimezone(to_zone)
-        return date_time_zurich.strftime('%d/%m/%Y %H:%M')
-
     def _convert_estimated_date(self, el_estimated):
         # the field estimatedDate is not mandatory
         if el_estimated == None:
             return 'nodata'
-        return self._format_time(el_estimated.text)
+        return format_time(el_estimated.text)
 
     def _check_element(self, el_name, el):
         if el == None:
             raise OpenTransException("An xml node %s of the OpenTransportData API response is missing." % el_name)
         return el.text
 
     def xml_to_array(self, xml_data):
-        ns = {'trias': 'http://www.vdv.de/trias'}
-        root = et.fromstring(xml_data)
-        el_stop_events = root.findall('.//trias:StopEvent', ns)
-        if not el_stop_events:
+        # Define namespaces for OJP and SIRI
+        ns = {
+            'ojp': 'http://www.vdv.de/ojp',
+            'siri': 'http://www.siri.org.uk/siri'
+        }
+        root = et.fromstring(xml_data.decode('utf-8'))
+        el_stop_points = root.findall('.//ojp:StopEventResult/ojp:StopEvent', ns)
+
+        if not el_stop_points:
             raise OpenTransNoStationException("No data available for the station %s." % str(self.station_id))
 
         results = []
-        for el in el_stop_events:
-            el_id = self._check_element('StopPointRef', el.find('./trias:ThisCall/trias:CallAtStop/trias:StopPointRef', ns))
-            el_label = self._check_element('PublichedLineName', el.find('./trias:Service/trias:PublishedLineName/trias:Text', ns))
-            el_current_date = self._check_element('ResponseTimestamp', root.find('.//{http://www.siri.org.uk/siri}ResponseTimestamp'))
-            el_departure_date = self._check_element('TimetabledTime', el.find('./trias:ThisCall/trias:CallAtStop/trias:ServiceDeparture/trias:TimetabledTime', ns))
-            el_destination_name = self._check_element('DestinationText', el.find('./trias:Service/trias:DestinationText/trias:Text', ns))
-            el_destination_id = self._check_element('DestinationStopPointRef', el.find('./trias:Service/trias:DestinationStopPointRef', ns))
 
+        for el in el_stop_points:
+            el_id = self._check_element('StopPointRef', el.find('.//siri:StopPointRef', ns))
+            el_service_name = self._check_element('PublishedServiceName', el.find('.//ojp:Service/ojp:PublishedServiceName/ojp:Text', ns))
+            el_current_date = self._check_element('ResponseTimestamp', root.find('.//siri:ResponseTimestamp', ns))
+            el_departure_date = self._check_element('TimetabledTime', el.find('.//ojp:ServiceDeparture/ojp:TimetabledTime', ns))
+            el_destination_name = self._check_element('DestinationText', el.find('.//ojp:DestinationText/ojp:Text', ns))
+            el_destination_id = self._check_element('DestinationStopPointRef', el.find('.//ojp:DestinationStopPointRef', ns))
+            # Append the data to results
             results.append({
                 'id': el_id,
-                'label': el_label,
-                'currentDate': self._format_time(el_current_date),
-                'departureDate': self._format_time(el_departure_date),
-                'estimatedDate': self._convert_estimated_date(el.find('./trias:ThisCall/trias:CallAtStop/trias:ServiceDeparture/trias:EstimatedTime', ns)),
+                'label': el_service_name,
+                'currentDate': format_time(el_current_date, fmt="%Y-%m-%dT%H:%M:%S.%f%z"),
+                'departureDate': format_time(el_departure_date),
+                'estimatedDate': self._convert_estimated_date(el.find('.//ojp:ServiceDeparture/ojp:EstimatedTime', ns)),
                 'destinationName': el_destination_name,
-                'destinationId': el_destination_id})
-
+                'destinationId': el_destination_id
+            })
         return results
 
+    def create_ojp_payload(self, station_id, request_dt_time, number_results=5):
+        # ATTENTION: The value "swisstopo_Abfahrtsmonitor" for the RequestorRef
+        # in the payload below is suggested by the OJP product owner.
+        # Hence it MUST NOT be changed!
+        payload = f"""<?xml version="1.0" encoding="UTF-8"?>
+        <OJP xmlns='http://www.vdv.de/ojp' xmlns:siri='http://www.siri.org.uk/siri' version='2.0' xmlns:xsi='http://www.w3.org/2001/XMLSchema-instance' xsi:schemaLocation='http://www.vdv.de/ojp ../../../../OJP4/OJP.xsd'>
+            <OJPRequest>
+                <siri:ServiceRequest>
+                    <siri:RequestTimestamp>{request_dt_time}</siri:RequestTimestamp>
+                    <siri:RequestorRef>swisstopo_Abfahrtsmonitor</siri:RequestorRef>
+                    <OJPStopEventRequest>
+                        <siri:RequestTimestamp>{request_dt_time}</siri:RequestTimestamp>
+                        <siri:MessageIdentifier>SER</siri:MessageIdentifier>
+                        <Location>
+                            <PlaceRef>
+                                <siri:StopPointRef>{station_id}</siri:StopPointRef>
+                            </PlaceRef>
+                                <DepArrTime>{request_dt_time}</DepArrTime>
+                        </Location>
+                        <Params>
+                            <NumberOfResults>{number_results}</NumberOfResults>
+                            <StopEventType>departure</StopEventType>
+                            <IncludePreviousCalls>false</IncludePreviousCalls>
+                            <IncludeOnwardCalls>true</IncludeOnwardCalls>
+                            <IncludeRealtimeData>true</IncludeRealtimeData>
+                        </Params>
+                    </OJPStopEventRequest>
+                </siri:ServiceRequest>
+            </OJPRequest>
+        </OJP>
+    """
+        # strip any non needed whitespaces from the payload in order to keep the data traffic to
+        # the minimum necessary
+        return re.sub(r">\s+<", "><", payload.strip())
+
     def send_post(self, station_id, request_dt_time, number_results=5):
-        self.headers = {
+        headers = {
             'authorization': self.open_trans_api_key,
             'content-type': 'application/xml; charset=utf-8',
             'accept-charset': 'utf-8'
         }
-        self.xml_data = u"""
-                    <?xml version="1.0" encoding="UTF-8"?>
-                    <Trias version="1.1" xmlns="http://www.vdv.de/trias" xmlns:siri="http://www.siri.org.uk/siri" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance">
-                        <ServiceRequest>
-                            <siri:RequestTimestamp>%s</siri:RequestTimestamp>
-                            <siri:RequestorRef>EPSa</siri:RequestorRef>
-                            <RequestPayload>
-                                <StopEventRequest>
-                                    <Location>
-                                        <LocationRef>
-                                            <StopPointRef>%s</StopPointRef>
-                                        </LocationRef>
-                                        <DepArrTime>%s</DepArrTime>
-                                    </Location>
-                                    <Params>
-                                        <NumberOfResults>%s</NumberOfResults>
-                                        <StopEventType>departure</StopEventType>
-                                        <IncludePreviousCalls>false</IncludePreviousCalls>
-                                        <IncludeOnwardCalls>true</IncludeOnwardCalls>
-                                        <IncludeRealtimeData>true</IncludeRealtimeData>
-                                    </Params>
-                                </StopEventRequest>
-                            </RequestPayload>
-                        </ServiceRequest>
-                    </Trias>
-                    """ % (str(request_dt_time), str(station_id), str(request_dt_time), str(number_results))
-
-        self.r = requests.post(self.url, self.xml_data, headers=self.headers, timeout=5)
-
-        if (self.r.status_code == 429):
+        xml_data = self.create_ojp_payload(str(station_id), str(request_dt_time), str(number_results))
+        resp = requests.post(url=self.url, data=xml_data, headers=headers, timeout=5)
+
+        if (resp.status_code == 429):
             raise OpenTransRateLimitException("The rate limit of OpenTransportdata has exceeded")
 
-        if (self.r.status_code != requests.codes.ok):
-            self.r.raise_for_status()
+        if (resp.status_code != requests.codes.ok):
+            resp.raise_for_status()
 
-        self.r.encoding = 'utf-8'  # TODO better encoding solution
-        return self.r.text.encode('utf-8')
+        resp.encoding = 'utf-8'  # TODO better encoding solution
+        return resp.text.encode('utf-8')
 
 
 class OpenTransException(Exception):

chsdi/views/stationboard.py

@@ -17,9 +17,12 @@ class TransportView(object):
 
     def __init__(self, request):
         self.opentrans_api_key = get_current_registry().settings['opentrans_api_key']  # Get API key from config .ini
+        self.opentrans_url = get_current_registry().settings['opentrans_url']
         if self.opentrans_api_key == '':
-            raise HTTPInternalServerError('The opentrans_api_key has no value, is registeret in .ini')
-        self.ot_api = opentransapi.OpenTrans(self.opentrans_api_key)
+            raise HTTPInternalServerError('The opentrans_api_key has no value, is registered in .ini')
+        if self.opentrans_url == '':
+            raise HTTPInternalServerError('The opentrans_url has no value, is registered in .ini')
+        self.ot_api = opentransapi.OpenTrans(self.opentrans_api_key, self.opentrans_url)
         self.request = request
         if request.matched_route.name == 'stationboard':
             id = request.matchdict['id']

tests/integration/helpers.py

@@ -0,0 +1,66 @@
+def generate_mock_response(departures, now):
+    """Generate a dynamic mock OJP response with correct structure."""
+    departure_events = "".join(
+        f"""
+        <StopEventResult>
+            <StopEvent>
+                <ThisCall>
+                    <CallAtStop>
+                        <siri:StopPointRef>{dep['id']}</siri:StopPointRef>
+                        <StopPointName>
+                            <Text>{dep['label']}</Text>
+                        </StopPointName>
+                        <ServiceDeparture>
+                            <TimetabledTime>{dep['departureDate']}</TimetabledTime>
+                            <EstimatedTime>{dep.get('estimatedDate', dep['departureDate'])}</EstimatedTime>
+                        </ServiceDeparture>
+                    </CallAtStop>
+                </ThisCall>
+                <Service>
+                    <PublishedServiceName>
+                        <Text>{dep['label']}</Text>
+                    </PublishedServiceName>
+                    <DestinationText>
+                        <Text>{dep['destinationName']}</Text>
+                    </DestinationText>
+                    <DestinationStopPointRef>{dep['destinationId']}</DestinationStopPointRef>
+                </Service>
+            </StopEvent>
+        </StopEventResult>
+        """
+        for dep in departures
+    )
+
+    mock_response = f"""<?xml version="1.0" ?>
+    <OJP xmlns:siri="http://www.siri.org.uk/siri" xmlns="http://www.vdv.de/ojp" version="2.0">
+        <OJPResponse>
+            <siri:ServiceDelivery>
+                <siri:ResponseTimestamp>{now}</siri:ResponseTimestamp>
+                <OJPStopEventDelivery>
+                    <siri:ResponseTimestamp>{now}</siri:ResponseTimestamp>
+                    {departure_events}
+                </OJPStopEventDelivery>
+            </siri:ServiceDelivery>
+        </OJPResponse>
+    </OJP>"""
+
+    return mock_response
+
+
+def generate_mock_empty_response(now):
+    # This mocked response will contain just enough for parsing, but will
+    # lead to no detected station, i.e.: station not existing case
+    mock_response = f"""<?xml version="1.0" ?>
+    <OJP xmlns:siri="http://www.siri.org.uk/siri" xmlns="http://www.vdv.de/ojp" version="2.0">
+        <OJPResponse>
+            <siri:ServiceDelivery>
+                <siri:ResponseTimestamp>{now}</siri:ResponseTimestamp>
+                <OJPStopEventDelivery>
+                    <siri:ResponseTimestamp>{now}</siri:ResponseTimestamp>
+                    <!-- No StopEventResult entries -->
+                </OJPStopEventDelivery>
+            </siri:ServiceDelivery>
+        </OJPResponse>
+    </OJP>"""
+
+    return mock_response